diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8480bd9696..5cea4dc7f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -966,8 +966,16 @@ jobs: tests/integration/test_pg_clickhouse_live_ch.py tests/integration/test_cell_tenancy_pg.py tests/integration/test_router_directory_pg.py + tests/integration/test_pg_online_index.py + tests/integration/test_pg_owned_cleanup.py tests/integration/test_cell_unset_gate_pg.py tests/integration/test_cell_prepare.py + tests/integration/test_cell_attach.py + tests/integration/test_cell_attach_two_replicas.py + tests/integration/test_cell_attach_cleanup.py + tests/integration/test_cell_control_cleanup.py + tests/integration/test_cell_control_fence.py + tests/integration/test_directory_control_cleanup.py tests/integration/test_cell_org_triggers_pg.py tests/integration/test_source_access_store.py tests/integration/test_source_document_store.py @@ -986,6 +994,34 @@ jobs: tests/integration/test_marked_rollup_migration_native.py tests/integration/test_marked_durable_folds_native.py tests/integration/test_marked_sql_surface_native.py + - name: pytest online-index durability on a private physical standby + timeout-minutes: 8 + env: + LOOM_PG_IMAGE: postgres:${{ matrix.postgres }} + LOOM_PG_CONTAINER: loom-scale-index-drill-pg + LOOM_PG_HOST_PORT: "55432" + LOOM_PG_INDEX_DRILL: "1" + LOOM_PG_INDEX_DRILL_DSN: postgres://loom:loomdev@localhost:55432/loom + run: | + set -euo pipefail + for name in loom-scale-index-drill-pg loom-scale-index-drill-pg-standby; do + if docker container inspect "$name" >/dev/null 2>&1; then + echo "drill container already exists: $name" >&2 + exit 1 + fi + done + if docker network inspect loom-scale-index-drill-pg-net >/dev/null 2>&1; then + echo "drill network already exists" >&2 + exit 1 + fi + cleanup() { + docker rm -f -v loom-scale-index-drill-pg loom-scale-index-drill-pg-standby || true + docker network rm loom-scale-index-drill-pg-net || true + } + trap cleanup EXIT + bash dev/ci_start_postgres.sh + docker update --memory 1g --memory-swap 1g loom-scale-index-drill-pg loom-scale-index-drill-pg-standby + pytest -q tests/integration/test_pg_online_index_durability.py # The tenant-isolation gates again, against a second server that enforces auth: # the Postgres read paths (heads by key and by id, narrowed scopes) see real tokens. # They run whether or not the memory suites passed (`!cancelled()`). diff --git a/.github/workflows/docker-smoke.yml b/.github/workflows/docker-smoke.yml index 3088015c40..8a0a64defa 100644 --- a/.github/workflows/docker-smoke.yml +++ b/.github/workflows/docker-smoke.yml @@ -12,6 +12,10 @@ on: "server/**", "schemas/**", "scripts/gates/docker*", + "scripts/gates/cell_trigger_acceptance.py", + "scripts/gates/testsupport/cell_trigger/**", + "bench/cell_trigger_gate.py", + "tests/unit/test_cell_trigger_ci_creator.py", "frontend/**", "tests/fixtures/docker_provider_app.py", "scripts/install-code-toolchains.sh", @@ -33,6 +37,9 @@ on: "tests/unit/test_scip_checkout_fetch_ref.py", "install.sh", "pyproject.toml", + "uv.lock", + "dev/clickhouse-ci/log_queries_readonly.xml", + "dev/clickhouse-ci/custom_settings_prefixes.xml", ".github/workflows/docker-smoke.yml", ] pull_request: @@ -45,6 +52,10 @@ on: "server/**", "schemas/**", "scripts/gates/docker*", + "scripts/gates/cell_trigger_acceptance.py", + "scripts/gates/testsupport/cell_trigger/**", + "bench/cell_trigger_gate.py", + "tests/unit/test_cell_trigger_ci_creator.py", "frontend/**", "tests/fixtures/docker_provider_app.py", "scripts/install-code-toolchains.sh", @@ -66,6 +77,9 @@ on: "tests/unit/test_scip_checkout_fetch_ref.py", "install.sh", "pyproject.toml", + "uv.lock", + "dev/clickhouse-ci/log_queries_readonly.xml", + "dev/clickhouse-ci/custom_settings_prefixes.xml", ".github/workflows/docker-smoke.yml", ] # Weekly, so a critical CVE fixed upstream fails here before a release's scan. @@ -255,3 +269,49 @@ jobs: -v "$RUNNER_TEMP/pylib:/pylib:ro" -e PYTHONPATH=/pylib -e PYTHONDONTWRITEBYTECODE=1 \ -e LOOM_ENV_FILE= -e LOOM_TEST_REQUIRE_OWNER_CHECK=1 --entrypoint python loom:sandbox-ci \ -m pytest -q -p no:cacheprovider tests/unit/test_scip_checkout_fetch_ref.py + + cell-trigger-acceptance: + name: Cell trigger end-to-end acceptance + if: github.event_name == 'pull_request' || github.event_name == 'push' + needs: clickhouse + runs-on: ubuntu-24.04-arm + timeout-minutes: 150 + env: + PYTHONDONTWRITEBYTECODE: "1" + LOOM_ENV_FILE: "" + LOOM_SETTINGS_FILE: "" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + id: python + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + version: "0.12.19" + enable-cache: true + cache-dependency-glob: uv.lock + - name: Install the locked native dependencies + run: uv sync --locked --python "${{ steps.python.outputs.python-path }}" --extra dev --extra code + - name: Check native tools + run: | + command -v docker + command -v openssl + docker info >/dev/null + - name: Check triggers against real API writes and delivery passes + env: + CLICKHOUSE_IMAGE: ${{ needs.clickhouse.outputs.image }} + PYTHONPYCACHEPREFIX: ${{ runner.temp }}/trigger-bytecode + run: .venv/bin/python -B scripts/gates/cell_trigger_acceptance.py --clickhouse-image "$CLICKHOUSE_IMAGE" + - name: Remove only this attempt's created stores + if: always() + run: .venv/bin/python -B scripts/gates/cell_trigger_acceptance.py --cleanup-only + - name: Retain sanitized trigger acceptance evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: cell-trigger-acceptance + path: ${{ runner.temp }}/cell-trigger-artifacts + retention-days: 7 diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py new file mode 100644 index 0000000000..9cf9f5b206 --- /dev/null +++ b/bench/cell_trigger_gate.py @@ -0,0 +1,1754 @@ +"""Measure API p99 and real delivery throughput with cell org triggers on and off. + +Requires an independently owned, attached scratch fixture and its source/ownership +manifest. The outer fixture owner stops its processes and drops its exact databases. +This driver restores triggers but never deletes fixture rows or changes providers. +The namespace uses append_only so independent writes keep their exact heads. +""" + +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import ipaddress +import json +import math +import os +import re +import socket +import stat +import statistics +import struct +import subprocess +import sys +import time +import uuid +from collections.abc import Callable, Mapping, Sequence +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, TextIO, cast +from urllib.parse import unquote, urlsplit + +import asyncpg +import clickhouse_connect +import httpx + +from scripts.gates.testsupport.cell_trigger.fixture import FILES, native_failure_metadata +from server.core import cell_triggers +from server.core.hashing import hash_text, sha256_value + +CALLS, WARMUP, ROUNDS, BATCHES = 2000, 50, 5, 5 +VALUE_BYTES, KEY_CEILING = 4096, 100_000 +IO_S, SETTLE_S, RESTORE_S = 30.0, 60.0, 15.0 +SUBJECT = "process:benchmark" +_SHA = re.compile(r"[a-f0-9]{64}") +_DB = re.compile(r"[A-Za-z_][A-Za-z0-9_-]*") +_DEFAULT_DBS = { + "loom", + "loom_router", + "loom_telemetry", + "default", + "system", + "postgres", + "template0", + "template1", + "information_schema", +} + + +class Invalid(ValueError): + """The fixture or a sample did not meet the declared measurement contract.""" + + +def _require(ok: object, field: str) -> None: + if not ok: + raise Invalid(field) + + +def _planned_keys(batch: int) -> int: + _require(type(batch) is int and batch > 0, "delivery batch") + count = 2 * ROUNDS * (CALLS + WARMUP + BATCHES * batch) + _require(count <= KEY_CEILING, "100000 key ceiling") + return count + + +def _arm_order() -> tuple[bool, ...]: + return tuple( + on for pair in range(ROUNDS) for on in ((True, False) if pair % 2 == 0 else (False, True)) + ) + + +def _payload(key: str) -> str: + prefix = f"Cell trigger measurement {key}: " + _require(len(prefix) < VALUE_BYTES and prefix.isascii(), "payload key") + return prefix + ("abcdefghijklmnopqrstuvwxyz " * VALUE_BYTES)[: VALUE_BYTES - len(prefix)] + + +def _endpoint(value: str, *, pg: bool = False) -> tuple[str, str, int, str]: + """Validate the literal target without resolving names or accepting DSN overrides.""" + try: + u = urlsplit(value) + host, port = u.hostname, u.port + _require( + u.scheme in ({"postgres", "postgresql"} if pg else {"http", "https"}), "endpoint scheme" + ) + _require( + host and ipaddress.ip_address(host).is_loopback and port and 0 < port < 65536, + "numeric loopback endpoint", + ) + _require(not u.query and not u.fragment, "endpoint overrides") + if not pg: + _require(not u.username and not u.password and u.path in ("", "/"), "HTTP endpoint") + database = unquote(u.path.removeprefix("/")) if pg else "" + if pg: + _scratch(database) + return u.scheme, str(ipaddress.ip_address(str(host))), int(port or 0), database + except (ValueError, TypeError): + raise Invalid("invalid local endpoint") from None + + +def _scratch(value: str) -> None: + _require( + bool(_DB.fullmatch(value)) and value.lower() not in _DEFAULT_DBS, + "explicit scratch database", + ) + + +def _digest(value: Any) -> str: + return hashlib.sha256( + json.dumps(value, sort_keys=True, default=str, separators=(",", ":")).encode() + ).hexdigest() + + +@dataclass(frozen=True) +class Fixture: + """Nonsecret receipt from the owner that created and launched this exact fixture.""" + + schema: int + base_url: str + pg_host: str + pg_port: int + pg_database: str + ch_host: str + ch_port: int + ch_database: str + ch_telemetry_database: str + cell: str + org: str + namespace: str + deployment_id: str + epoch: int + instance_id: str + build_commit: str + source_tree: str + launch_manifest_sha256: str + ownership_receipt_sha256: str + api_configuration_sha256: str + cli_configuration_sha256: str + shared_configuration_sha256: str + background_workers: str + controller_excluded: bool + pg_internal_port: int | None + + @classmethod + def read(cls, path: str) -> Fixture: + try: + value, _ = _receipt(path, max_bytes=16_384) + _require(set(value) == set(cls.__dataclass_fields__), "fixture fields") + _require(type(value["schema"]) is int and value["schema"] == 2, "fixture schema") + return cls(**value) + except (TypeError, ValueError, OSError): + raise Invalid("fixture manifest") from None + + def validate(self, args: argparse.Namespace, config: Any) -> int: + _require(type(self.schema) is int and self.schema == 2, "fixture schema") + integers = {"schema", "pg_port", "ch_port", "epoch"} + _require( + all(type(getattr(self, key)) is int for key in integers) + and type(self.controller_excluded) is bool + and all( + type(value) is str + for key, value in asdict(self).items() + if key not in integers | {"controller_excluded", "pg_internal_port"} + ), + "fixture field types", + ) + supplied = _endpoint(args.pg_dsn, pg=True) + configured = _endpoint(config.pg_dsn, pg=True) + _require(supplied == configured, "configured PG endpoint") + _require( + supplied[1:] == (self.pg_host, self.pg_port, self.pg_database), "manifest PG endpoint" + ) + _require(_endpoint(args.base_url) == _endpoint(self.base_url), "manifest HTTP endpoint") + for host, port in ((self.ch_host, self.ch_port), (config.ch_host, config.ch_port)): + _endpoint(f"http://[{host}]:{port}" if ":" in host else f"http://{host}:{port}") + _require( + (self.ch_host, self.ch_port) == (config.ch_host, config.ch_port), "manifest CH endpoint" + ) + for name in (self.pg_database, self.ch_database, self.ch_telemetry_database): + _scratch(name) + _require( + (self.ch_database, self.ch_telemetry_database) + == (config.ch_database, config.ch_telemetry_database), + "manifest CH aliases", + ) + _require( + (self.org, self.namespace, self.cell) == (args.org, args.namespace, config.cell), + "manifest identity", + ) + _require( + all((self.org, self.namespace, self.cell, self.instance_id, self.deployment_id)), + "manifest identity fields", + ) + _require( + self.background_workers == config.background_workers == "off" + and self.controller_excluded is True, + "exclusive fixture", + ) + _require( + config.storage_authority == "postgres" + and config.embedding_provider == "fake" + and config.llm_provider == "fake", + "explicit fake providers", + ) + _require( + self.build_commit == config.build_commit + and bool(re.fullmatch(r"[a-f0-9]{40}", self.build_commit)), + "source build", + ) + _require(bool(re.fullmatch(r"[a-f0-9]{40}", self.source_tree)), "source tree") + for value in ( + self.launch_manifest_sha256, + self.ownership_receipt_sha256, + self.api_configuration_sha256, + self.cli_configuration_sha256, + self.shared_configuration_sha256, + ): + _require(type(value) is str and bool(_SHA.fullmatch(value)), "fixture receipt digest") + actual = _configuration(config) + _require( + actual["cell_router_token"] == actual["router_operator_token"] == "", + "empty native CLI credentials", + ) + _require( + self.api_configuration_sha256 != self.cli_configuration_sha256, + "distinct API and CLI configurations", + ) + _require(self.cli_configuration_sha256 == _digest(actual), "fixture CLI configuration") + _require( + self.shared_configuration_sha256 == _digest(_shared_configuration(config)), + "fixture shared configuration", + ) + _require(type(self.epoch) is int and self.epoch > 0, "authority epoch") + _require( + self.pg_internal_port is None + or type(self.pg_internal_port) is int + and 0 < self.pg_internal_port < 65536, + "internal PG port", + ) + return _planned_keys(config.pg_delivery_batch_rows) + + +def _configuration(config: Any) -> dict[str, Any]: + from server.settings import Settings + + values = config.model_dump(mode="json") + _require(type(values) is dict and set(values) == set(Settings.model_fields), "Settings fields") + return cast(dict[str, Any], values) + + +def _shared_configuration(config: Any) -> dict[str, Any]: + values = dict(_configuration(config)) + del values["cell_router_token"] + return values + + +def _source(fixture: Fixture) -> dict[str, Any]: + """Fixed read-only Git commands bind the launched build to this clean checkout.""" + root = Path(__file__).resolve().parents[1] + values = [] + for command in (("rev-parse", "HEAD"), ("rev-parse", "HEAD^{tree}"), ("status", "--porcelain")): + result = subprocess.run( + ["git", *command], cwd=root, capture_output=True, text=True, timeout=5, check=False + ) + _require(result.returncode == 0, "source Git read") + _require(len(result.stdout) <= 65_536, "source status size") + values.append(result.stdout.strip()) + head, tree, status = values + _require( + head == fixture.build_commit and tree == fixture.source_tree and not status, + "clean source head/tree", + ) + return {"head": head, "tree": tree, "clean": True} + + +_PUBLIC_WORKLOAD = frozenset( + { + "storage_authority", + "background_workers", + "embedding_provider", + "embedding_model", + "embedding_dim", + "embedding_send_dimensions", + "embedding_worker_enabled", + "embedding_cache_shared", + "embedding_cache_shared_max_chars", + "embedding_cache_size", + "embedding_cache_ttl_s", + "llm_provider", + "llm_model", + } +) + + +def _workload(config: Any) -> dict[str, Any]: + return { + key: value + for key, value in _configuration(config).items() + if (key.startswith("pg_") and key != "pg_dsn") + or key.startswith("memory_conflict") + or key in _PUBLIC_WORKLOAD + } + + +def _receipt_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + _require(key not in result, "duplicate receipt field") + result[key] = value + return result + + +def _receipt(path: str, *, max_bytes: int = 65_536) -> tuple[dict[str, Any], str]: + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, "rb") as source: + metadata = os.fstat(source.fileno()) + _require( + stat.S_ISREG(metadata.st_mode) and metadata.st_uid == os.geteuid(), + "owned regular receipt", + ) + raw = source.read(max_bytes + 1) + _require(len(raw) <= max_bytes, "receipt size") + value = json.loads(raw, object_pairs_hook=_receipt_object) + _require(isinstance(value, dict), "receipt object") + return value, hashlib.sha256(raw).hexdigest() + + +def _owned_server(server: dict[str, Any]) -> None: + from server.core.provider_process import _start_ticks + + _require( + set(server) + == {"pid", "start_ticks", "boot_id", "uid", "cwd", "health_instance", "instance_id"}, + "launch server fields", + ) + pid = server["pid"] + _require( + type(pid) is int + and pid > 1 + and type(server["uid"]) is int + and server["uid"] == os.geteuid(), + "launch process ownership", + ) + _require( + isinstance(server["start_ticks"], str) and server["start_ticks"].isdigit(), + "launch process birth", + ) + root = Path(__file__).resolve().parents[1] + _require( + server["cwd"] == str(root) and server["health_instance"] == f"{socket.gethostname()}:{pid}", + "launch process source/hostname/PID", + ) + proc = Path("/proc") / str(pid) + _require(proc.stat().st_uid == os.geteuid(), "actual process ownership") + birth = _start_ticks(pid) + _require(birth == server["start_ticks"], "actual process birth") + boot = Path("/proc/sys/kernel/random/boot_id").read_text().strip() + _require( + boot == server["boot_id"] and os.readlink(proc / "cwd") == str(root), + "actual process boot/source", + ) + _require(_start_ticks(pid) == birth, "stable process birth") + + +_CREDENTIAL_POSTURE = { + "api_cell_router_token_present": True, + "cli_cell_router_token_present": False, + "api_router_operator_token_present": False, + "cli_router_operator_token_present": False, +} + + +def _attested_configuration(configuration: dict[str, Any], fixture: Fixture, config: Any) -> None: + from server.core import deployment_config + + hashes = { + "api_configuration_sha256": fixture.api_configuration_sha256, + "cli_configuration_sha256": fixture.cli_configuration_sha256, + "shared_configuration_sha256": fixture.shared_configuration_sha256, + } + _require( + set(configuration) == {"schema", "workload", "fingerprints", "credential_posture", *hashes} + and type(configuration["schema"]) is int + and configuration["schema"] == 2, + "configuration artifact schema", + ) + _require(all(configuration[key] == value for key, value in hashes.items()), "launch digests") + _require( + fixture.cli_configuration_sha256 == _digest(_configuration(config)), + "launch CLI configuration", + ) + _require( + fixture.shared_configuration_sha256 == _digest(_shared_configuration(config)), + "launch shared configuration", + ) + _require( + type(configuration["credential_posture"]) is dict + and all(type(value) is bool for value in configuration["credential_posture"].values()) + and configuration["credential_posture"] == _CREDENTIAL_POSTURE, + "launch credential posture", + ) + _require( + _digest(configuration["workload"]) == _digest(_workload(config)), "launch workload settings" + ) + _require( + configuration["fingerprints"] == deployment_config.fingerprints(config), + "launch configuration fingerprints", + ) + + +def _launch(args: argparse.Namespace, fixture: Fixture, config: Any) -> dict[str, Any]: + from server.core import deployment_config + + receipt, digest = _receipt(args.launch_manifest) + _require(digest == fixture.launch_manifest_sha256, "launch receipt hash") + _require( + set(receipt) == {"schema", "server", "source", "configuration_artifact"} + and type(receipt["schema"]) is int + and receipt["schema"] == 2, + "launch receipt schema", + ) + server, source, artifact = ( + receipt[key] for key in ("server", "source", "configuration_artifact") + ) + _require( + isinstance(server, dict) and isinstance(source, dict) and isinstance(artifact, dict), + "launch receipt objects", + ) + _require( + source == {"head": fixture.build_commit, "tree": fixture.source_tree}, + "launch source identity", + ) + _require(server.get("instance_id") == fixture.instance_id, "launch server UUID") + _require( + set(artifact) == {"path", "sha256"} + and isinstance(artifact["path"], str) + and Path(artifact["path"]).is_absolute(), + "configuration artifact path", + ) + configuration, artifact_digest = _receipt(artifact["path"]) + _require(artifact_digest == artifact["sha256"], "configuration artifact hash") + _attested_configuration(configuration, fixture, config) + fingerprints = deployment_config.fingerprints(config) + _owned_server(server) + return { + "health_instance": server["health_instance"], + "fingerprints": fingerprints, + "launch_sha256": digest, + "configuration_artifact_sha256": artifact_digest, + "birth_sha256": _digest(server), + } + + +class Artifact: + def __init__(self, path: str): + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + self.file: TextIO = os.fdopen(fd, "w", encoding="utf-8") + self.records = 0 + + def emit(self, kind: str, **record: Any) -> None: + self.records += 1 + _require(self.records <= KEY_CEILING + 1000, "output record ceiling") + line = json.dumps({"kind": kind, **record}, sort_keys=True, allow_nan=False) + _require(len(line) <= 65_536, "output record size") + self.file.write(line + "\n") + + def close(self) -> None: + try: + self.file.flush() + os.fsync(self.file.fileno()) + except BaseException: # Retain the flush or sync failure through final close. + try: + self.file.close() + except BaseException: # noqa: BLE001 — close cannot replace failed artifact sync. + pass + raise + else: + self.file.close() + + +async def _close_pg(conn: Any, end: float | None = None) -> None: + from server.core import pg + + await pg.close_owned(conn, end=time.monotonic() + IO_S if end is None else end) + + +class ReadOnlyCH: + """Public ClickHouse client restricted at connection startup, before any query.""" + + def __init__(self, client: Any): + self.client = client + + @classmethod + async def open(cls) -> ReadOnlyCH: + from server.core.ch_names import NamedClient + from server.core.ch_target import connection_kwargs + + raw = await clickhouse_connect.get_async_client( + **connection_kwargs(), + settings={ + "readonly": 1, + "max_execution_time": int(IO_S), + "log_comment": "bench.cell_trigger_gate.preflight", + }, + connect_timeout=IO_S, + send_receive_timeout=IO_S, + ) + return cls(NamedClient(raw)) + + async def query_all( + self, sql: str, params: dict[str, Any] | None = None + ) -> list[dict[str, Any]]: + result = await self.client.query(sql, parameters=params or {}) + return list(result.named_results()) + + async def close(self) -> None: + async with asyncio.timeout(IO_S): + await self.client.close() + + +async def _health( + args: argparse.Namespace, fixture: Fixture, launch: Mapping[str, Any] | None = None +) -> None: + from server.settings import settings + + context = httpx.AsyncClient(timeout=IO_S, follow_redirects=False, trust_env=False) + client = await context.__aenter__() + try: + health = await client.get(f"{args.base_url.rstrip('/')}/v1/health") + deployment = await client.get(f"{args.base_url.rstrip('/')}/v1/health/deployment") + _require(health.status_code == deployment.status_code == 200, "health HTTP status") + h, d = health.json(), deployment.json() + _require( + h.get("build_commit") == fixture.build_commit + and d.get("deployment_id") == fixture.deployment_id, + "serving source/deployment", + ) + _require( + h.get("deployment_config", {}).get("instance_id") == fixture.instance_id, + "serving instance", + ) + _require( + h.get("ch_ok") is True and h.get("pg_ok") is True and h.get("status") == "ok", + "serving health", + ) + _require( + h.get("fake_embedding") is True + and h.get("fake_llm") is True + and h.get("pending_migrations") == [], + "serving provider/schema", + ) + _require(h.get("storage_authority") == "postgres", "serving authority") + for field in ("llm_provider", "llm_model", "embedding_model"): + _require(h.get(field) == getattr(settings, field), "serving " + field) + _require( + h.get("llm_provider") == "fake" + and type(h.get("embedding_dim_configured")) is int + and h["embedding_dim_configured"] == settings.embedding_dim + and type(h.get("embedding_dim_probed")) is int + and h["embedding_dim_probed"] == settings.embedding_dim, + "serving embedding dimensions/fake provider", + ) + launch = launch if launch is not None else _launch(args, fixture, settings) + _require(h.get("instance") == launch["health_instance"], "serving owned process") + except BaseException as error: + try: + await context.__aexit__(type(error), error, error.__traceback__) + except BaseException: # noqa: BLE001 — HTTP cleanup cannot replace work failure. + pass + raise + else: + await context.__aexit__(None, None, None) + + +def _object(value: Any) -> dict[str, Any]: + if isinstance(value, str): + value = json.loads(value) + _require(isinstance(value, dict), "metadata JSON object") + return cast(dict[str, Any], value) + + +async def _proof( + conn: Any, ch: Any, args: argparse.Namespace, fixture: Fixture, *, native: bool = False +) -> dict[str, Any]: + from server.core import cell, instances + from server.settings import settings + + fixture.validate(args, settings) + source = _source(fixture) + launch = _launch(args, fixture, settings) + actual = await conn.fetchrow( + "SELECT current_database() AS db, inet_server_port() AS port, " + "pg_is_in_recovery() AS recovery" + ) + _require( + actual and actual["db"] == fixture.pg_database and actual["recovery"] is False, + "actual PG identity/primary", + ) + if fixture.pg_internal_port is not None: + _require(actual["port"] == fixture.pg_internal_port, "PG internal port") + authority = await conn.fetchrow("SELECT authority, deployment_id, epoch FROM loom.authority") + _require( + authority + and (authority["authority"], authority["deployment_id"], authority["epoch"]) + == ("postgres", fixture.deployment_id, fixture.epoch), + "PG authority identity", + ) + first = await conn.fetchrow("SELECT is_first, cell FROM loom.cell_first") + mode = await conn.fetchval("SELECT mode FROM loom.tenancy_mode") + own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") + held = await conn.fetchval( + "SELECT EXISTS (SELECT 1 FROM loom.tenancy WHERE org = $1)", args.org + ) + _require( + first and first["cell"] == fixture.cell and mode == "enforced" and held is True, + "enforced held org", + ) + _require(own and own["next_fp"] is None, "stable accepted fingerprint") + current = _object(own["current_fp"]) + mine = cell.mine() + mine["deployment_id"] = fixture.deployment_id + _require(cell.matches(mine, current), "accepted local fingerprint") + db = await ch.query_all("SELECT currentDatabase() AS db") + _require(db == [{"db": fixture.ch_database}], "actual CH database") + records = await ch.query_all( + "SELECT authority, epoch, deployment_id FROM loom.storage_authority FINAL" + ) + _require(len(records) == 1 and records[0] == dict(authority), "paired CH authority") + namespace = await ch.query_all( + "SELECT default_conflict_policy FROM loom.namespaces FINAL WHERE " + "org={org:String} AND namespace={ns:String} LIMIT 2", + {"org": args.org, "ns": args.namespace}, + ) + _require(namespace == [{"default_conflict_policy": "append_only"}], "independent write policy") + for table in ("provider_configuration", "runtime_overrides"): + rows = await ch.query_all(f"SELECT count() AS n FROM loom.{table}") + _require(rows == [{"n": 0}], f"fresh {table}") + live = await instances.read_live(ch, all_deployments=True) + _require( + len(live) == 1 + and live[0]["instance_id"] == fixture.instance_id + and live[0]["build_commit"] == fixture.build_commit + and live[0].get("serving") == 1 + and live[0].get("host") == socket.gethostname() + and not live[0]["workers_lease"], + "exclusive serving source", + ) + _require( + all(live[0].get(key) == value for key, value in launch["fingerprints"].items()), + "serving configuration fingerprints", + ) + beats = await ch.query_all( + "SELECT instance_id, cell, tenancy_mode, fingerprint FROM " + "loom.cell_heartbeats WHERE instance_id = {id:String}", + {"id": fixture.instance_id}, + ) + _require( + len(beats) == 1 + and beats[0]["cell"] == fixture.cell + and beats[0]["tenancy_mode"] == "enforced", + "serving cell heartbeat", + ) + reported = _object(beats[0]["fingerprint"]) + _require( + reported + and cell.matches(mine, reported) + and reported.get("serves") is True + and reported.get("marked") is True, + "serving fingerprint", + ) + await _debt(conn, 0) + await _health(args, fixture, launch) + if native: + await cell.refresh() + _require(not cell.cli_gate_reason(), "native fresh cell acceptance") + return { + "deployment_id": fixture.deployment_id, + "epoch": fixture.epoch, + "instance_id": fixture.instance_id, + "fingerprint_sha256": _digest(current), + "api_configuration_sha256": fixture.api_configuration_sha256, + "cli_configuration_sha256": fixture.cli_configuration_sha256, + "shared_configuration_sha256": fixture.shared_configuration_sha256, + "shared_configuration_excludes": ["cell_router_token"], + "credential_posture": _CREDENTIAL_POSTURE, + "api_configuration_attested": True, + "source": source, + "launch": launch, + } + + +async def _preflight(args: argparse.Namespace, fixture: Fixture) -> None: + conn = ch = None + failure: BaseException | None = None + try: + conn = await asyncpg.connect( + args.pg_dsn, + timeout=IO_S, + command_timeout=IO_S, + server_settings={"default_transaction_read_only": "on"}, + ) + ch = await ReadOnlyCH.open() + async with asyncio.timeout(IO_S): + await _proof(conn, ch, args, fixture) + except BaseException as error: # noqa: BLE001 — cleanup must retain any preflight failure. + failure = error + try: + if ch is not None: + await ch.close() + except BaseException as error: # noqa: BLE001 — both metadata clients must be closed. + if failure is None: + failure = error + try: + if conn is not None: + await _close_pg(conn) + except BaseException as error: # noqa: BLE001 — preserve the first work or close failure. + if failure is None: + failure = error + if failure is not None: + raise failure + + +async def _debt(conn: Any, expected: int) -> None: + row = await conn.fetchrow( + "SELECT (SELECT count(*) FROM loom.memory_deliveries WHERE targets <> " + "0) AS pending, (SELECT count(*) FROM loom.audit_outbox) AS audits, " + "(SELECT count(*) FROM loom.memory_deliveries WHERE attempts <> 0 OR " + "last_error <> '') AS errors" + ) + _require( + row and row["pending"] == expected and row["audits"] == row["errors"] == 0, + "fixture delivery/audit debt", + ) + + +@dataclass +class Triggers: + tables: tuple[str, ...] + invariant: str + org: str + on: bool = True + + @staticmethod + async def snapshot(conn: Any) -> tuple[tuple[str, ...], str, str]: + tables = tuple(await cell_triggers.planned_tables(conn)) + rows = await conn.fetch( + "SELECT c.relname AS table_name, t.tgname AS name, t.tgenabled AS " + "enabled, pg_get_triggerdef(t.oid) AS definition FROM pg_trigger t " + "JOIN pg_class c ON c.oid = t.tgrelid JOIN pg_namespace n ON n.oid " + "= c.relnamespace WHERE n.nspname = 'loom' AND NOT t.tgisinternal " + "ORDER BY c.relname, t.tgname" + ) + functions = await conn.fetch( + "SELECT p.proname, pg_get_functiondef(p.oid) AS definition FROM " + "pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace WHERE " + "n.nspname = 'loom' AND p.prokind = 'f' ORDER BY p.proname, p.oid" + ) + org = [dict(r) for r in rows if r["name"] in cell_triggers.ORG_TRIGGERS] + other = [dict(r) for r in rows if r["name"] not in cell_triggers.ORG_TRIGGERS] + return ( + tables, + _digest({"functions": [dict(r) for r in functions], "triggers": other}), + _digest(org), + ) + + @classmethod + async def capture(cls, conn: Any) -> Triggers: + _require(not await cell_triggers.missing(conn), "canonical ON trigger baseline") + tables, invariant, org = await cls.snapshot(conn) + _require(tables, "guarded tables") + return cls(tables, invariant, org) + + async def verify(self, conn: Any) -> None: + tables, invariant, org = await self.snapshot(conn) + _require( + tables == self.tables and invariant == self.invariant, + "trigger table/function/tenancy drift", + ) + _require(org == (self.org if self.on else _digest([])), "prior trigger treatment") + + async def set(self, conn: Any, on: bool) -> None: + await self.verify(conn) + transaction = conn.transaction() + await transaction.start() + try: + for table in self.tables: + if table == "tenancy": + continue + _require(bool(_DB.fullmatch(table)), "guarded table identifier") + for name in cell_triggers.ORG_TRIGGERS: + await conn.execute(f'DROP TRIGGER IF EXISTS {name} ON loom."{table}"') + if on: + await conn.execute(cell_triggers.create_statement(table, name)) + except BaseException: + try: + await transaction.rollback() + except BaseException: # noqa: BLE001 — rollback cannot replace the DDL failure. + pass + raise + await transaction.commit() + self.on = on + await self.verify(conn) + + async def restore(self, conn: Any) -> None: + # Recreate the captured exact org guards even after a partial failed toggle. + transaction = conn.transaction() + await transaction.start() + try: + for table in self.tables: + if table == "tenancy": + continue + for name in cell_triggers.ORG_TRIGGERS: + await conn.execute(f'DROP TRIGGER IF EXISTS {name} ON loom."{table}"') + await conn.execute(cell_triggers.create_statement(table, name)) + except BaseException: + try: + await transaction.rollback() + except BaseException: # noqa: BLE001 — rollback cannot replace the DDL failure. + pass + raise + await transaction.commit() + self.on = True + await self.verify(conn) + _require(not await cell_triggers.missing(conn), "restored canonical triggers") + + +@dataclass +class Samples: + on: bool + api_ms: list[float] + delivery_rps: float + + +def _quantile(values: Sequence[float], fraction: float) -> float: + _require(values and all(math.isfinite(v) and v >= 0 for v in values), "finite latency samples") + return sorted(values)[math.ceil(fraction * len(values)) - 1] + + +def _statistics(samples: Sequence[Samples]) -> dict[str, Any]: + _require(len(samples) == 2 * ROUNDS, "complete arm samples") + arms: dict[bool, dict[str, Any]] = {} + for on in (True, False): + arm = [s for s in samples if s.on is on] + _require( + len(arm) == ROUNDS and all(len(s.api_ms) == CALLS for s in arm), "complete run samples" + ) + p99 = [_quantile(s.api_ms, 0.99) for s in arm] + rps = [s.delivery_rps for s in arm] + _require(all(math.isfinite(r) and r > 0 for r in rps), "positive finite throughput") + arms[on] = { + "p99_ms": p99, + "median_p99_ms": statistics.median(p99), + "min_p99_ms": min(p99), + "max_p99_ms": max(p99), + "rps": rps, + "median_rps": statistics.median(rps), + } + delta = arms[True]["median_p99_ms"] - arms[False]["median_p99_ms"] + loss = 1 - arms[True]["median_rps"] / arms[False]["median_rps"] + paired = [] + for i in range(0, len(samples), 2): + a, b = samples[i : i + 2] + _require(a.on is not b.on, "paired arm order") + paired.append((_quantile(a.api_ms, 0.99) - _quantile(b.api_ms, 0.99)) * (1 if a.on else -1)) + return { + "on": arms[True], + "off": arms[False], + "p99_delta_ms": delta, + "throughput_loss": loss, + "latency_within_limit": delta <= 10, + "throughput_within_limit": loss <= 0.10, + "paired_p99_differences_ms": paired, + } + + +@dataclass +class RequestRow: + key: str + memory_id: str + version: int + content_sha256: str + elapsed_ms: float + measured: bool + admission: str + + +async def _memory_set( + args: argparse.Namespace, *, arm: int = 0, emit: Callable[..., None] = lambda *_a, **_kw: None +) -> list[RequestRow]: + rows = [] + prefix = f"gate-{uuid.uuid4().hex}-{arm}" + context = httpx.AsyncClient( + timeout=IO_S, + headers={"Authorization": f"Bearer {args.token}"}, + follow_redirects=False, + trust_env=False, + ) + client = await context.__aenter__() + try: + for i in range(CALLS + WARMUP): + key = f"{prefix}-{i}" + value = _payload(key) + started = time.perf_counter_ns() + response = None + try: + response = await client.post( + f"{args.base_url.rstrip('/')}/v1/memory.set", + json={ + "org": args.org, + "namespace": args.namespace, + "agent": "bench", + "memory_key": key, + "memory_type": "semantic", + "value": value, + }, + ) + _require(response.status_code == 200, "memory.set HTTP status") + out = response.json() + _require( + isinstance(out, dict) + and out.get("status") == "active" + and not out.get("noop") + and not out.get("degraded") + and not out.get("duplicate_of") + and not out.get("merged_into_key"), + "memory.set outcome", + ) + memory_id = str(uuid.UUID(out["memory_id"])) + _require( + type(out.get("version")) is int and out["version"] > 0, "memory.set version" + ) + _require( + out.get("memory_key") == key + and out.get("namespace") == args.namespace + and out.get("content_sha256") == sha256_value(value), + "memory.set identity/hash", + ) + admission = "derived_ready" + elapsed = (time.perf_counter_ns() - started) / 1_000_000 + _require(math.isfinite(elapsed) and elapsed >= 0, "memory.set elapsed") + row = RequestRow( + key, + memory_id, + out["version"], + sha256_value(value), + elapsed, + i >= WARMUP, + admission, + ) + rows.append(row) + emit("request", arm=arm, index=i, status=200, **asdict(row)) + except BaseException as error: + elapsed = (time.perf_counter_ns() - started) / 1_000_000 + code = "" + if response is not None: + try: + envelope = response.json() + candidate = envelope.get("detail", {}).get("code", "") + if isinstance(candidate, str) and re.fullmatch( + r"[A-Z][A-Z0-9_]{0,63}", candidate + ): + code = candidate + except (ValueError, TypeError, AttributeError): + pass + try: + emit( + "request_error", + arm=arm, + index=i, + key=key, + measured=i >= WARMUP, + elapsed_ms=elapsed, + status=response.status_code if response is not None else None, + code=code, + error_type=type(error).__name__, + ) + except BaseException: # noqa: BLE001 — diagnostics cannot replace the request failure. + pass + raise + except BaseException as error: + try: + await context.__aexit__(type(error), error, error.__traceback__) + except BaseException: # noqa: BLE001 — HTTP cleanup cannot replace work failure. + pass + raise + else: + await context.__aexit__(None, None, None) + return rows + + +@dataclass +class Block: + rows: list[dict[str, Any]] + audits: list[dict[str, Any]] + documents: list[dict[str, Any]] + seqs: list[tuple[str, str, int]] = field(default_factory=list) + + +async def _build_block(args: argparse.Namespace, batch: int) -> Block: + from server.llm import customer + from server.memory.store import pg_memory, pg_publish + from server.memory.store.rows import ( + MemoryRow, + build_first_version_row, + build_memory_version_row, + next_version, + written_now, + ) + + _planned_keys(batch) + prefix = f"gate-delivery-{uuid.uuid4().hex}" + rows, audits, documents = [], [], [] + with customer.for_org(args.org, namespace=args.namespace, subject=SUBJECT): + for index in range(BATCHES * batch): + key = f"{prefix}-{index}" + value = _payload(key) + stored = json.dumps(value, sort_keys=True, separators=(",", ":")) + row = build_memory_version_row( + MemoryRow.from_snapshot( + build_first_version_row( + { + "org": args.org, + "namespace": args.namespace, + "agent": "bench", + "memory_key": key, + "memory_type": "semantic", + "value": stored, + "version": next_version(), + "content_sha256": sha256_value(value), + "actor": SUBJECT, + "source_class": "asserted", + }, + now=written_now(), + ) + ) + ) + row = pg_memory.decode_row(pg_memory.encode_row(row)) + document = await _expected_document(args, row, worker=True) + rows.append(row) + audits.append(pg_publish.audit_row_for(row, request_id="", writer=SUBJECT)) + documents.append(document) + return Block(rows, audits, documents) + + +async def _expected_document( + args: argparse.Namespace, row: dict[str, Any], *, worker: bool = False +) -> dict[str, Any]: + from server.embedding import get_embedder + from server.llm import customer + from server.memory.domain.embed_text import embedding_texts + from server.memory.store.rows import parse_memory_value, search_document_row + + embedder = get_embedder() + raw = str(row["value"]) + excerpt, text, key_text = embedding_texts( + row["memory_type"], parse_memory_value(raw), raw, row["memory_key"] + ) + wanted = sorted({t for t in (text, key_text) if t}) + with customer.for_org(args.org, namespace=args.namespace, subject=SUBJECT): + vectors = dict(zip(wanted, await embedder.embed_many(wanted), strict=True)) + from server.memory.store.conflict_resolver import OWED_MARK + from server.settings import settings + + return search_document_row( + row, + conflict_checked_at=OWED_MARK + if worker and settings.memory_conflict_resolver_enabled + else None, + embedding_model=embedder.model_name, + embedding=vectors.get(text, []), + key_embedding=vectors.get(key_text, [0.0] * embedder.dim), + key_text=key_text, + excerpt=excerpt, + ) + + +async def _stage(conn: Any, ch: Any, block: Block) -> None: + from server.memory.store import pg_memory + + await _debt(conn, 0) + for offset in range(0, len(block.rows), 100): + rows = block.rows[offset : offset + 100] + fresh = await conn.fetchval( + "SELECT count(*) FROM loom.memory_current WHERE org=$1 AND namespace=$2 " + "AND (memory_id=ANY($3::uuid[]) OR memory_key=ANY($4::text[]))", + rows[0]["org"], + rows[0]["namespace"], + [uuid.UUID(str(row["memory_id"])) for row in rows], + [row["memory_key"] for row in rows], + ) + _require(fresh == 0, "fresh absent authority identity") + await _absent(ch, block) + committed = await pg_memory.commit_versions( + block.rows, + block.audits, + targets=pg_memory.MEMORY_ROW | pg_memory.SEARCH_DOC | pg_memory.AUDIT_ROW, + publisher_lease=False, + ) + _require( + len(committed.seqs) == len(block.rows) and all(t == 7 for t in committed.targets), + "staged delivery count/targets", + ) + block.seqs = committed.seqs + # The authority may canonicalize the full persisted row; retain its exact result. + _require( + len(committed.rows) == len(block.rows) and len(committed.audits) == len(block.audits), + "staged row/audit count", + ) + for expected, actual in zip(block.rows, committed.rows, strict=True): + _compare(expected, actual, tuple(expected), "committed full row") + for expected, actual in zip(block.audits, committed.audits, strict=True): + _compare(expected, actual, tuple(expected), "committed full audit") + await _debt(conn, len(block.rows)) + due = await conn.fetchval( + "SELECT count(*) FROM loom.memory_deliveries " + "WHERE targets=7 AND lease_until<=clock_timestamp() AND NOT held AND attempts=0" + ) + _require(due == len(block.rows), "exact fresh due deliveries") + + +async def _delivery( + args: argparse.Namespace, batch: int, *, emit: Callable[..., None], arm: int +) -> float: + from server.llm import customer + from server.memory.workers import pg_delivery + from server.settings import settings + + results: list[dict[str, Any]] = [] + _index = 0 + failure: BaseException | None = None + with customer.for_org(args.org, namespace=args.namespace, subject=SUBJECT): + started = time.perf_counter_ns() + try: + for _index in range(BATCHES): + _require(settings.pg_delivery_batch_rows == batch, "unchanged delivery batch") + result = await pg_delivery.run_pass() + results.append(asdict(result)) + _require( + result.claimed == result.published == batch and result.failed == 0, + "real worker claimed/published/failure", + ) + except BaseException as error: + failure = error + try: + emit("worker_error", arm=arm, index=_index, error_type=type(error).__name__) + except BaseException: # noqa: BLE001 — diagnostics cannot replace the worker failure. + pass + raise + finally: + elapsed = (time.perf_counter_ns() - started) / 1_000_000_000 + for result_index, outcome in enumerate(results): + try: + emit("worker", arm=arm, index=result_index, **outcome) + except BaseException: # Prior results cannot replace failed work. + if failure is None: + raise + _require(math.isfinite(elapsed) and elapsed > 0, "positive worker block elapsed") + rps = BATCHES * batch / elapsed + emit("delivery_block", arm=arm, elapsed_seconds=elapsed, rows=BATCHES * batch, rps=rps) + return rps + + +def _normal(value: Any) -> Any: + from datetime import datetime + + if isinstance(value, (bytes, bytearray)): + return hash_text(value) + if isinstance(value, uuid.UUID): + return str(value) + if isinstance(value, str) and re.fullmatch(r"\d{4}-\d\d-\d\dT.*", value): + try: + value = datetime.fromisoformat(value) + except ValueError: + pass + if isinstance(value, datetime): + return value.replace(tzinfo=None).isoformat(timespec="milliseconds") + if isinstance(value, Mapping): + return {k: _normal(v) for k, v in value.items()} + if isinstance(value, (tuple, list)): + return [_normal(v) for v in value] + return value + + +def _stored_vector(value: Any) -> list[float]: + _require( + isinstance(value, list) + and all(isinstance(x, (int, float)) and math.isfinite(x) for x in value), + "finite vector", + ) + try: + # The insert driver stores the high 16 bits of each Float32 component. + return [struct.unpack(" None: + for field_name in fields: + left, right = expected.get(field_name), actual.get(field_name) + if field_name in ("embedding", "memory_key_embedding"): + left = _stored_vector(left) + _require( + isinstance(right, list) + and len(left) == len(right) + and all(isinstance(x, (int, float)) and math.isfinite(x) for x in right) + and left == right, + phase + " vector", + ) + elif field_name == "value_json": + try: + # PostgreSQL keeps JSON text; the native ClickHouse decoder returns its value. + left = json.loads(left) if isinstance(left, str) else left + right = json.loads(right) if isinstance(right, str) else right + same = sha256_value(left) == sha256_value(right) + except (ValueError, TypeError) as error: + raise Invalid(phase + " " + field_name) from error + _require(same, phase + " " + field_name) + else: + _require(_normal(left) == _normal(right), phase + " " + field_name) + + +async def _absent(ch: Any, block: Block) -> None: + for offset in range(0, len(block.rows), 100): + rows = block.rows[offset : offset + 100] + params = { + "org": rows[0]["org"], + "ns": rows[0]["namespace"], + "ids": [str(r["memory_id"]) for r in rows], + "keys": [r["memory_key"] for r in rows], + "hashes": [a["row_hash"] for a in block.audits[offset : offset + 100]], + } + for table, predicate in ( + ("memory", "memory_id IN {ids:Array(UUID)}"), + ("memory_embeddings", "memory_key IN {keys:Array(String)}"), + ("audit_log", "row_hash IN {hashes:Array(String)}"), + ): + final = "" if table == "audit_log" else " FINAL" + found = await ch.query_all( + f"SELECT count() AS n FROM loom.{table}{final} WHERE org = " + f"{{org:String}} AND namespace = {{ns:String}} AND {predicate}", + params, + ) + _require(found == [{"n": 0}], "fresh absent " + table) + + +async def _verify( + conn: Any, ch: Any, args: argparse.Namespace, identities: Sequence[RequestRow] | Block +) -> None: + block = identities if isinstance(identities, Block) else None + wanted = identities.rows if isinstance(identities, Block) else [asdict(r) for r in identities] + for offset in range(0, len(wanted), 100): + group = wanted[offset : offset + 100] + ids = [uuid.UUID(str(r["memory_id"])) for r in group] + stored = await conn.fetch( + "SELECT c.row::text AS head, r.row::text AS revision, " + "r.audit::text AS audit, r.seq, d.targets, d.attempts, " + "d.last_error FROM loom.memory_current c JOIN " + "loom.memory_revisions r ON " + "(r.org,r.namespace,r.seq)=(c.org,c.namespace,c.seq) LEFT JOIN " + "loom.memory_deliveries d ON " + "(d.org,d.namespace,d.seq)=(r.org,r.namespace,r.seq) WHERE " + "c.org=$1 AND c.namespace=$2 AND c.memory_id=ANY($3::uuid[])", + args.org, + args.namespace, + ids, + ) + _require(len(stored) == len(group), "exact authority head/revision count") + by_id = {json.loads(r["head"])["memory_id"]: r for r in stored} + params = { + "org": args.org, + "ns": args.namespace, + "ids": [str(i) for i in ids], + "keys": [r.get("memory_key", r.get("key")) for r in group], + } + projected = await ch.query_all( + "SELECT * FROM loom.memory FINAL WHERE org={org:String} AND " + "namespace={ns:String} AND memory_id IN {ids:Array(UUID)}", + params, + ) + docs = await ch.query_all( + "SELECT * FROM loom.memory_embeddings FINAL WHERE org={org:String} " + "AND namespace={ns:String} AND memory_key IN {keys:Array(String)}", + params, + ) + _require(len(projected) == len(docs) == len(group), "exact projection/vector counts") + projections = {str(r["memory_id"]): r for r in projected} + documents = {r["memory_key"]: r for r in docs} + audit_hashes = [] + authority_audits = {} + for index, expected in enumerate(group): + id = str(expected["memory_id"]) + _require(id in by_id and id in projections, "authority/projection identity") + record = by_id[id] + _require( + record["targets"] in (None, 0) + and record["attempts"] in (None, 0) + and record["last_error"] in (None, ""), + "delivery settled without failure", + ) + head, revision, audit = ( + json.loads(record[name]) for name in ("head", "revision", "audit") + ) + _compare(head, revision, tuple(head), "authority full row") + if block: + _compare(expected, head, tuple(expected), "authority staged row") + _compare( + block.audits[offset + index], audit, tuple(audit), "authority staged audit" + ) + _require( + (args.org, args.namespace, record["seq"]) == block.seqs[offset + index], + "authority staged sequence", + ) + else: + key = str(expected["key"]) + _require(head["memory_key"] == key, "authority API memory_key") + _require(json.loads(head["value"]) == _payload(key), "authority API value") + _require(head["version"] == expected["version"], "authority API version") + _require( + head["content_sha256"] == expected["content_sha256"], + "authority API content_sha256", + ) + _require(head["status"] == "active", "authority API status") + _compare(head, projections[id], tuple(head), "full memory projection") + key = str(head["memory_key"]) + _require(key in documents, "search document identity") + document = documents[key] + if block: + expected_document = block.documents[offset + index] + _compare( + expected_document, + document, + tuple(k for k in expected_document if k != "created_at"), + "full search document", + ) + else: + expected_document = await _expected_document(args, head) + _compare( + expected_document, + document, + tuple( + k + for k in expected_document + if k not in ("created_at", "conflict_checked_at") + ), + "full API search document", + ) + audit_hashes.append(audit["row_hash"]) + authority_audits[audit["row_hash"]] = audit + params["hashes"] = audit_hashes + audits = await ch.query_all( + "SELECT * FROM loom.audit_log WHERE org={org:String} AND " + "namespace={ns:String} AND row_hash IN {hashes:Array(String)}", + params, + ) + _require( + sorted(hash_text(r["row_hash"]) for r in audits) == sorted(audit_hashes), + "exact stored audit hashes", + ) + for audit in audits: + expected_audit = authority_audits[hash_text(audit["row_hash"])] + _compare(expected_audit, audit, tuple(expected_audit), "full stored audit") + await _debt(conn, 0) + + +async def _settle_api( + conn: Any, ch: Any, args: argparse.Namespace, rows: Sequence[RequestRow] +) -> None: + end = asyncio.get_running_loop().time() + SETTLE_S + while True: + pending = await conn.fetchval( + "SELECT count(*) FROM loom.memory_deliveries WHERE org=$1 AND " + "namespace=$2 AND targets<>0", + args.org, + args.namespace, + ) + if pending == 0: + break + _require(asyncio.get_running_loop().time() < end, "API publication settlement deadline") + await asyncio.sleep(0.1) + await _verify(conn, ch, args, rows) + + +async def _physical(conn: Any, ch: Any) -> dict[str, Any]: + from server.core.ch_names import database, telemetry_database + + pg = await conn.fetch( + "SELECT relname, n_live_tup AS estimated_rows, " + "pg_total_relation_size(relid) AS bytes FROM pg_stat_user_tables WHERE " + "schemaname='loom' ORDER BY relname" + ) + parts = await ch.query_all( + "SELECT database, table, sum(rows) AS rows, sum(bytes_on_disk) AS " + "bytes FROM system.parts WHERE active AND database IN " + "{dbs:Array(String)} GROUP BY database, table ORDER BY database, table", + {"dbs": [database(), telemetry_database()]}, + ) + exact = {} + for table in ("memory_current", "memory_revisions", "memory_deliveries", "audit_outbox"): + exact[table] = await conn.fetchval(f"SELECT count(*) FROM loom.{table}") + return { + "postgres_rows": exact, + "postgres_estimates": [dict(r) for r in pg], + "clickhouse_parts": parts, + } + + +async def _run( + args: argparse.Namespace, fixture: Fixture, artifact: Artifact, ch: Any +) -> dict[str, Any]: + from server.core.ch import set_query_tag + from server.settings import settings + + set_query_tag("bench.cell_trigger_gate") + batch = settings.pg_delivery_batch_rows + conn = await asyncpg.connect( + args.pg_dsn, + timeout=IO_S, + command_timeout=IO_S, + server_settings={"loom.tenancy_checked": "on"}, + ) + triggers = None + samples = [] + try: + async with asyncio.timeout(IO_S): + await _proof(conn, ch, args, fixture, native=True) + triggers = await Triggers.capture(conn) + for arm, on in enumerate(_arm_order()): + async with asyncio.timeout(IO_S): + proof = await _proof(conn, ch, args, fixture, native=True) + await triggers.set(conn, on) + artifact.emit( + "arm_proof", + arm=arm, + on=on, + **proof, + tables=triggers.tables, + invariant_sha256=triggers.invariant, + org_triggers_sha256=triggers.org, + ) + api = await _memory_set(args, arm=arm, emit=artifact.emit) + async with asyncio.timeout(SETTLE_S): + await _settle_api(conn, ch, args, api) + block = await _build_block(args, batch) + stage_started = time.perf_counter_ns() + try: + async with asyncio.timeout(SETTLE_S): + await _stage(conn, ch, block) + except BaseException as error: + await _stage_failure(conn, block, artifact, arm, stage_started, error) + raise + rps = await _delivery(args, batch, emit=artifact.emit, arm=arm) + async with asyncio.timeout(SETTLE_S): + await _verify(conn, ch, args, block) + artifact.emit( + "settled", + arm=arm, + api_rows=len(api), + delivery_rows=len(block.rows), + physical=await _physical(conn, ch), + ) + latencies = [r.elapsed_ms for r in api if r.measured] + samples.append(Samples(on, latencies, rps)) + artifact.emit( + "arm", + arm=arm, + on=on, + p50_ms=_quantile(latencies, 0.5), + p99_ms=_quantile(latencies, 0.99), + delivery_rps=rps, + ) + from server.core.background import drain_task_sets, flush_registered_buffers + + _require(await drain_task_sets(timeout_s=IO_S), "driver descendant drain") + _require(await flush_registered_buffers(), "driver buffer drain") + summary = _statistics(samples) + artifact.emit("provisional_statistics", **summary) + return summary + finally: + from server.cli import cell_attach_pg as apg + + prior = sys.exception() + cleanup = asyncio.create_task( + apg._work_result(lambda: _cleanup(triggers, conn, artifact)), + name="trigger-gate-cleanup", + ) + interrupted = False + while not cleanup.done(): + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError: + interrupted = True + except BaseException: # noqa: BLE001 — preserve an earlier operation failure + break + try: + if isinstance(result := cleanup.result(), apg._Interrupted): + raise result.error + except BaseException as error: + try: + artifact.emit("cleanup_error", error_type=type(error).__name__) + except BaseException: # noqa: BLE001 — diagnostics cannot replace work or cleanup failure. + pass + if prior is None: + raise + if interrupted and prior is None: + raise asyncio.CancelledError + + +def _stage_plan(value: Any) -> list[dict[str, Any]]: + if type(value) is str: + _require(len(value) <= 65_536, "bounded stage plan bytes") + value = json.loads(value) + _require(type(value) is list and len(value) == 1, "one stage plan") + pending = [(value[0]["Plan"], 0)] + nodes = [] + node_types = { + "Nested Loop", + "Hash Join", + "Merge Join", + "Seq Scan", + "Index Scan", + "Index Only Scan", + "Bitmap Heap Scan", + "Bitmap Index Scan", + "Sort", + "Incremental Sort", + "Function Scan", + "LockRows", + "Hash", + "Materialize", + "Memoize", + "Gather", + "Gather Merge", + "Result", + "Append", + "Merge Append", + "Unique", + "Aggregate", + "Limit", + } + join_types = {"Inner", "Left", "Right", "Full", "Semi", "Anti", "Right Anti"} + while pending: + node, depth = pending.pop() + _require(type(node) is dict and len(nodes) < 32 and depth <= 8, "bounded stage plan") + kind = node.get("Node Type") + out: dict[str, Any] = { + "depth": depth, + "node_type": kind if type(kind) is str and kind in node_types else "other", + } + if "Join Type" in node: + join = node["Join Type"] + out["join_type"] = join if type(join) is str and join in join_types else "other" + for source, target in ( + ("Plan Rows", "plan_rows"), + ("Startup Cost", "startup_cost"), + ("Total Cost", "total_cost"), + ): + number = node.get(source) + _require( + type(number) in {int, float} and math.isfinite(number) and 0 <= number <= 1e18, + "bounded stage plan estimate", + ) + out[target] = number + nodes.append(out) + children = node.get("Plans", []) + _require(type(children) is list and len(children) <= 32, "bounded stage plan children") + pending.extend((child, depth + 1) for child in reversed(children)) + return nodes + + +async def _stage_failure( + conn: Any, block: Block, artifact: Artifact, arm: int, started: int, error: BaseException +) -> None: + try: + artifact.emit( + "stage_error", + arm=arm, + key_count=len(block.rows), + elapsed_ms=max(0, time.perf_counter_ns() - started) / 1_000_000, + ) + except BaseException: # noqa: BLE001 — stage diagnostics cannot replace the work failure. + pass + try: + causes = native_failure_metadata(error).get("driver", {}).get("causes", []) + if not any(cause.get("sqlstate") == "57014" for cause in causes): + return + from server.memory.store import pg_memory + + async with asyncio.timeout(IO_S): + _require(not conn.is_in_transaction(), "idle stage plan observer") + keys = sorted({pg_memory.identity_key(row) for row in block.rows}) + plan = await conn.fetchval( + "EXPLAIN (FORMAT JSON) " + pg_memory._LOCK_HEADS, + *pg_memory._key_arrays(keys), + ) + # This independent session plans afresh after rollback; it cannot expose + # the failed pooled backend's cached plan or its former wait state. + artifact.emit( + "freshplan_after_failure", + arm=arm, + key_count=len(keys), + observer="independent_connection", + analyze=False, + nodes=_stage_plan(plan), + ) + except BaseException: # noqa: BLE001 — a failed observation preserves the stage's first error. + pass + + +async def _cleanup(triggers: Triggers | None, conn: Any, artifact: Artifact) -> None: + end = time.monotonic() + RESTORE_S + try: + if triggers is not None: + await _restore(triggers, conn, end) + artifact.emit("triggers_restored", verified=True) + finally: + prior = sys.exception() + try: + await _close_pg(conn, end) + except BaseException: + if prior is None: + raise + try: + artifact.emit("connection_cleanup_error", error_type=type(sys.exception()).__name__) + except BaseException: # noqa: BLE001 — diagnostics cannot replace restoration failure. + pass + + +async def _restore(triggers: Triggers, conn: Any, end: float) -> None: + from server.cli import cell_attach_pg as apg + + await apg.supervise(conn, triggers.restore(conn), end=end) + + +def main() -> int: + from server.cli import _ch + from server.settings import settings + from server.wiring import wire + + parser = argparse.ArgumentParser(description=__doc__) + for name in ( + "base-url", + "pg-dsn", + "token", + "org", + "namespace", + "fixture-manifest", + "launch-manifest", + "output", + ): + parser.add_argument("--" + name, required=True) + args = parser.parse_args() + artifact = None + verdict: dict[str, Any] = {"verdict": "INVALID"} + phase = "local_manifest" + try: + fixture = Fixture.read(args.fixture_manifest) + total = fixture.validate(args, settings) + source = _source(fixture) + launch = _launch(args, fixture, settings) + artifact = Artifact(args.output) + artifact.emit( + "configuration", + manifest=asdict(fixture), + shared_configuration_excludes=["cell_router_token"], + credential_posture=_CREDENTIAL_POSTURE, + api_configuration_attested=True, + source=source, + launch=launch, + manifest_sha256=_digest(asdict(fixture)), + calls=CALLS, + warmup=WARMUP, + rounds=ROUNDS, + batches=BATCHES, + batch_rows=settings.pg_delivery_batch_rows, + logical_keys=total, + value_bytes=VALUE_BYTES, + order=_arm_order(), + driver_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + ) + phase = "readonly_preflight" + asyncio.run(_preflight(args, fixture)) + phase = "native_driver" + wire() + + async def run(ch: Any) -> dict[str, Any]: + return await _run(args, fixture, artifact, ch) + + summary = _ch.run_connected(run, orgs=(args.org,)) + artifact.emit("driver_cleanup", native_returned=True, outer_database_cleanup_required=True) + verdict = { + "verdict": "PASS" + if summary["latency_within_limit"] and summary["throughput_within_limit"] + else "FAIL", + "p99_delta_ms": summary["p99_delta_ms"], + "throughput_loss": summary["throughput_loss"], + "outer_database_cleanup_required": True, + } + except BaseException as error: # noqa: BLE001 — every failed run must emit INVALID + # Never serialize exception text: clients may include credentials or values. + reason = str(error) if isinstance(error, Invalid) else "operation refused" + failure = None + try: + failure = native_failure_metadata( + error, files=(*FILES, "cell_trigger_gate.py", "pg.py", "pg_memory.py") + ) + except BaseException: # noqa: BLE001 — diagnostic failure must preserve INVALID + pass + verdict = { + "verdict": "INVALID", + "phase": phase, + "reason": reason, + "error_type": type(error).__name__, + **({"failure": failure} if failure is not None else {}), + } + if artifact is not None: + try: + artifact.emit( + "invalid", + phase=phase, + reason=reason, + error_type=type(error).__name__, + **({"failure": failure} if failure is not None else {}), + ) + except BaseException: # noqa: BLE001 — secondary output cannot replace INVALID. + pass + finally: + if artifact is not None: + try: + artifact.close() + except BaseException as error: # noqa: BLE001 — required close fails without losing INVALID. + if verdict["verdict"] != "INVALID": + verdict = { + "verdict": "INVALID", + "error_type": type(error).__name__, + "phase": "artifact_close", + } + print(json.dumps(verdict, sort_keys=True, allow_nan=False), flush=True) + return 0 if verdict["verdict"] == "PASS" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/deploy/helm/loom/templates/_helpers.tpl b/deploy/helm/loom/templates/_helpers.tpl index 9b2c1c258a..ddda25aca8 100644 --- a/deploy/helm/loom/templates/_helpers.tpl +++ b/deploy/helm/loom/templates/_helpers.tpl @@ -685,14 +685,11 @@ initContainers: {{/* Pod spec fields the Deployments, the hook Jobs and the analytics export share; `serviceAccount` names another account than the release's. */}} -{{/* `leads`: the pod may hold the workers lease, which alone reads the -Kubernetes API (retireLostPods). */}} {{- define "loom.podCommon" -}} -{{- $leads := .leads -}} {{- $sa := .serviceAccount -}} {{- with .root -}} serviceAccountName: {{ default (include "loom.serviceAccountName" .) $sa }} -automountServiceAccountToken: {{ or .Values.serviceAccount.automountToken (and .Values.retireLostPods $leads) }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountToken }} # Service links would inject LOOM_SERVICE_HOST and the like into Loom's env namespace. enableServiceLinks: false {{- if not .Values.sandbox.hostUsers }} diff --git a/deploy/helm/loom/templates/analytics-export.yaml b/deploy/helm/loom/templates/analytics-export.yaml index e6b0355650..403d7f0009 100644 --- a/deploy/helm/loom/templates/analytics-export.yaml +++ b/deploy/helm/loom/templates/analytics-export.yaml @@ -81,7 +81,7 @@ spec: app.kubernetes.io/component: analytics-export spec: restartPolicy: Never - {{- include "loom.podCommon" (dict "root" $ "leads" false "serviceAccount" $name) | nindent 10 }} + {{- include "loom.podCommon" (dict "root" $ "serviceAccount" $name) | nindent 10 }} {{- include "loom.waitForDevClickhouse" $ | nindent 10 }} containers: - name: analytics-export diff --git a/deploy/helm/loom/templates/deployment.yaml b/deploy/helm/loom/templates/deployment.yaml index fb5d91f8cf..5f94f19304 100644 --- a/deploy/helm/loom/templates/deployment.yaml +++ b/deploy/helm/loom/templates/deployment.yaml @@ -76,7 +76,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: - {{- include "loom.podCommon" (dict "root" . "leads" (not .Values.workers.enabled)) | nindent 6 }} + {{- include "loom.podCommon" (dict "root" .) | nindent 6 }} {{- if .Values.topologySpread.enabled }} topologySpreadConstraints: - maxSkew: {{ .Values.topologySpread.zoneSkew }} diff --git a/deploy/helm/loom/templates/hooks.yaml b/deploy/helm/loom/templates/hooks.yaml index cdd65cccc1..d55dbe9a64 100644 --- a/deploy/helm/loom/templates/hooks.yaml +++ b/deploy/helm/loom/templates/hooks.yaml @@ -65,7 +65,7 @@ spec: app.kubernetes.io/component: {{ .name }} spec: restartPolicy: Never - {{- include "loom.podCommon" (dict "root" $ "leads" false) | nindent 6 }} + {{- include "loom.podCommon" (dict "root" $) | nindent 6 }} containers: - name: {{ .name }} image: {{ include "loom.image" $ }} @@ -83,7 +83,7 @@ spec: value: {{ $v | toString | quote }} {{- end }} # Its participant proof names the pod, so a hook pod lost mid-run is - # retired like any other (provider_process.pod_gone). + # verified against the same exact process identity as every participant. - name: POD_NAME valueFrom: { fieldRef: { fieldPath: metadata.name } } - name: POD_NAMESPACE diff --git a/deploy/helm/loom/templates/networkpolicy.yaml b/deploy/helm/loom/templates/networkpolicy.yaml index 6a1ed604cb..14903323dc 100644 --- a/deploy/helm/loom/templates/networkpolicy.yaml +++ b/deploy/helm/loom/templates/networkpolicy.yaml @@ -87,10 +87,6 @@ spec: {{- include "loom.anywhereBut" . | nindent 8 }} ports: - port: 443 - {{- if .Values.retireLostPods }} - # The Kubernetes API (443 on EKS; 6443 on most other clusters). - - port: 6443 - {{- end }} {{- if and (not .Values.devClickhouse.enabled) (ne (int .Values.clickhouse.port) 443) }} - port: {{ .Values.clickhouse.port }} {{- end }} diff --git a/deploy/helm/loom/templates/rbac.yaml b/deploy/helm/loom/templates/rbac.yaml deleted file mode 100644 index 72a0ff9d90..0000000000 --- a/deploy/helm/loom/templates/rbac.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- if .Values.retireLostPods }} -{{- if not (or .Values.serviceAccount.create .Values.serviceAccount.name) }} -{{- fail "retireLostPods grants `get pods` to the release's service account: set serviceAccount.create or serviceAccount.name, not the namespace's default" }} -{{- end }} -{{- /* Only `get` on pods: the workers leader asks whether a lost participant's -pod still exists (server/core/provider_process.py `pod_gone`). */}} -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "loom.fullname" . }} - labels: - {{- include "loom.labels" . | nindent 4 }} -rules: - - apiGroups: [""] - resources: [pods] - verbs: [get] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "loom.fullname" . }} - labels: - {{- include "loom.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ include "loom.fullname" . }} -subjects: - - kind: ServiceAccount - name: {{ include "loom.serviceAccountName" . }} - namespace: {{ .Release.Namespace }} -{{- end }} diff --git a/deploy/helm/loom/templates/serviceaccount.yaml b/deploy/helm/loom/templates/serviceaccount.yaml index 4536a11c5c..0c5c212fd8 100644 --- a/deploy/helm/loom/templates/serviceaccount.yaml +++ b/deploy/helm/loom/templates/serviceaccount.yaml @@ -9,5 +9,5 @@ metadata: annotations: {{- toYaml . | nindent 4 }} {{- end }} -automountServiceAccountToken: {{ or .Values.serviceAccount.automountToken .Values.retireLostPods }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountToken }} {{- end }} diff --git a/deploy/helm/loom/templates/tests/health.yaml b/deploy/helm/loom/templates/tests/health.yaml index ecbf9feb1e..1eea0f7a09 100644 --- a/deploy/helm/loom/templates/tests/health.yaml +++ b/deploy/helm/loom/templates/tests/health.yaml @@ -10,7 +10,7 @@ metadata: helm.sh/hook-delete-policy: before-hook-creation spec: restartPolicy: Never - {{- include "loom.podCommon" (dict "root" . "leads" false) | nindent 2 }} + {{- include "loom.podCommon" (dict "root" .) | nindent 2 }} containers: - name: health image: {{ include "loom.image" . }} diff --git a/deploy/helm/loom/templates/workers.yaml b/deploy/helm/loom/templates/workers.yaml index 87208891a5..d515200cc0 100644 --- a/deploy/helm/loom/templates/workers.yaml +++ b/deploy/helm/loom/templates/workers.yaml @@ -45,7 +45,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: - {{- include "loom.podCommon" (dict "root" . "leads" true) | nindent 6 }} + {{- include "loom.podCommon" (dict "root" .) | nindent 6 }} {{- if .Values.topologySpread.enabled }} # Leader and standby on different nodes. topologySpreadConstraints: diff --git a/deploy/helm/loom/values.yaml b/deploy/helm/loom/values.yaml index 186a56f8b2..e9a6449804 100644 --- a/deploy/helm/loom/values.yaml +++ b/deploy/helm/loom/values.yaml @@ -222,15 +222,9 @@ serviceAccount: name: "" # IRSA (eks.amazonaws.com/role-arn) or other cloud identity annotations. annotations: {} - # Mount the service-account token (retireLostPods mounts it too). + # Mount the service-account token for workloads that explicitly need it. automountToken: false -# The workers leader retires the provider participant of a pod Kubernetes has -# deleted (lost with its node, evicted past its grace period), which would -# otherwise hold the next model change until `loom participant retire`. Grants -# the service account `get` on pods in this namespace and mounts its token. -retireLostPods: true - podAnnotations: {} podLabels: {} # amd64 by default: the C++ compiler tier's scip-clang ships for x86_64 only. The @@ -428,8 +422,8 @@ assistant: # Ingress to the Loom port from `ingressFrom` only. Egress to DNS, the # release's web pods, the extractor's pods (with extractor.enabled), the dev # ClickHouse (with devClickhouse.enabled), the EKS Pod Identity agent, and TCP -# 443, the ClickHouse port, postgres.port (with postgres.host) and 6443 (with -# retireLostPods) anywhere except `egressExcept` (providers, the IdP and git remotes +# 443, the ClickHouse port and postgres.port (with postgres.host) anywhere except +# `egressExcept` (providers, the IdP and git remotes # have no fixed range); a ClickHouse or Postgres inside the cluster needs an # extraEgress rule. Off, `guardMetadata` gives the Loom pods an egress-only policy # that allows every pod and every address but `egressExcept`. diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 78e1d4c689..45b319c1d8 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -18,6 +18,60 @@ that every transport enforces (`auth_actually_required()`, and `LOOM_MCP_AUTH_MO refuses to boot without them. [Storage authority](storage-authority.md) describes what a cell holds and how its two stores agree. +## Participant lifecycle baseline + +A participant advertising the ninth `host_proof` field `cell-retirement-v1` +enrolls before its first fresh primary acceptance read and uses permanent guarded +retirement. A serving cell heartbeat includes its `participant_id` beside the +fingerprint, allowing a replacement to be joined to the participant registry. +Workers children enroll independently and do not publish serving instance rows. +Native commands hold one admission scope through their complete org or principal +work and drain: claim lookup and release share that scope; token readiness, keyring +read, signing and audit share another. Connections and helper calls do not retire +a command that still has principal work to perform. Store pairing is verified +before enrollment and fresh acceptance. Org-free diagnostics do not install an +accepting view. The pre-schema `ch-setup` ownership probe discards its diagnostic +mode result and checks only store/cell identity before raw database user/profile +setup; its optional analyst-row work opens a separate verified, enrolled scope +and obtains fresh acceptance. That bootstrap exception admits no tenant or token +work. + +Local physical-death evidence requires the same host, kernel boot and PID namespace. +It accepts an absent PID, a different process birth, or the captured birth in a +single fresh kernel process record showing a zombie with exactly one thread. A +zombie leader with other threads is unproved. Known-zero tagged queries remain a +separate requirement before retirement; kernel process exit does not settle a +query still executing in a store. + +The capability identifies installed code; it does not prove that every older +process or unsafe retirement client has disappeared. + +Before online configuration finalization is supported, stop every unsafe manual +retirement and pruning writer and prevent its binary/image from restarting. +Inventory and positively terminate every accepting server, workers child, +standalone worker and native command that overlapped those writers. Include paused +boots, already retired/pruned identities and lifecycle-capable processes started +during the overlap: old maintenance code could have hidden their rows too. Only +then start replacement accepting and maintenance processes with this lifecycle. +A shared image tag, pod deletion, readiness or heartbeat expiry is insufficient. +Keep exact PID/boot/namespace/start or trusted runtime termination evidence with +the deployment's upgrade record. No CLI flag or stored acknowledgement substitutes +for that evidence, and the runtime census does not verify the external inventory. + +Census retained legacy rows while unsafe writers are stopped. Recertify a retained +identity only after exact verifiable physical death and known-zero cross-replica +queries. Keep empty or unknown proofs quarantined. If the legacy inventory is +incomplete, stop all accepting/native-command and retirement/pruning processes, +positively terminate or fence every runtime that could execute an old binary, and +prevent old restarts. Only after that physical quiescence may the operator request +an exact audited cleanup of selected unknown legacy participant identities. There +is no automatic age-based cleanup or affirmation bypass. Restart only guarded +lifecycle processes afterward. Unknown or partitioned runtimes remain blockers. + +This baseline is an operational prerequisite, not a release-cut requirement. +Configuration finalization also needs the directory's controller-fencing protocol; +the lifecycle certificate alone cannot authorize reservation release. + ## Why whole deployments A cell shares no store, key, lease or worker with another. Every guarantee a single @@ -389,7 +443,8 @@ open meanwhile, as on a deployment never prepared, since the cell's Postgres, wh holds its state, cannot tell the two apart (ClickHouse carries no cell state: adding it would make the write path read a second store for tenancy) and such a process never reads the directory; where every process has `LOOM_CELL` unset, `loom doctor`'s naming -of it (which reads the directory, and ships with attach) is the only signal. A +of it (which reads the directory, and ships with the router's read-only scope) is the only +signal. A `(true, …)` row with `recording` and the own row: the first cell recording. Both present with the own row otherwise: `enforced`, a later cell's mode reading as `enforced` whatever its row says (its triggers, mode cache and CLI treat it so, and @@ -485,11 +540,13 @@ with the same configuration, or `--finish`, first writes `next` wherever it is m It reruns attach's overlap refusals against every other cell's directory row, counting both values of each, writes the new fingerprint as `next` to the directory and then to `loom.cell_self`, so old and new processes both serve while the new configuration rolls -out. `loom cell update --finish` waits, at most `--wait` (default 10 minutes), until -every live process reports `next` and the longest request deadline has passed; when the -change turns SCIM on or changes the cell's OIDC client it then runs attach's SCIM -precondition (refusing, or `--revoke-unprovisioned`); only then does it make `next` the -only value, in `loom.cell_self` first and the directory last. A widened issuer or +out. Finalization first narrows the cell's accepted value durably while the directory +keeps both reservations. It then requires a complete post-cutoff participant census, +permanent guarded retirement or verified exact death for every captured identity, +and a serving target replacement outside that captured set. Heartbeat expiry and +request deadlines do not prove this. Only afterward may the directory release its +old reservation. Online finalization needs the [lifecycle baseline](#participant-lifecycle-baseline) +and controller fencing; SCIM-on changes remain gated on actual per-cell SCIM support. A widened issuer or pattern is reserved in the directory before any process accepts it, and a narrowed one is released only after none does. A refusal or a wait that runs out keeps both values and prints the processes not yet reporting `next`; rerunning resumes, and @@ -514,6 +571,8 @@ version and fields, and beside them keys that no fingerprint compares: they are needs no column for them (`server.core.instances.cell_row`). Their presence proves the build that reports them; there is no generation key. +- `participant_id`: this server's provider participant UUID, for joining its actual + process lifecycle to the serving instance row. A workers child publishes no instance row. - `serves`: `LOOM_SERVES_TRAFFIC`. - `marked`: whether this process's pooled Postgres sessions keep the tenancy marker (`loom.tenancy_checked`). The cell refresh reads it with the state rows, after `RESET ALL` @@ -676,20 +735,19 @@ inserts cannot hold the write off, a new-org request waits on attach for a few s at most, and an attach that stalls holding the write's lock releases it within seconds. Attach takes `LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE` (which waits for running inserts and queues later ones behind it) in short tries, in one transaction with -server-side timeouts, then writes the mode and commits; after any error in that -transaction it ends that session's backend (`pg_terminate_backend`, the backend named by -the pid and `backend_start` attach captured when it connected) and waits until it has -left `pg_stat_activity`, then reads the mode back before it retries or reports; a mode -read back as `enforced` is written again in a fresh transaction, whose commit waits for -the standby, before attach goes on, since a commit ended while it waited is on the -primary alone. The attach command fixes those timeouts, and its tests cover each of -these cases. The mode cache reads the mode with a plain `SELECT`, which takes no row -lock and never waits on attach. An insert that finds `enforced` (a process whose cache +server-side timeouts, then writes the mode and commits on its exact ATTACH backend. +Only recognized pre-COMMIT contention can retry, after acknowledged rollback and a check +that this backend still holds ATTACH. Unexpected failure or an uncertain COMMIT closes +the owned session and ends the run. Attach neither finds nor ends another backend. A +fresh run takes the lock and claims again; whether an earlier commit landed is decided +by a durable mode read whose COMMIT waits for the standby +([attaching a cell](#attaching-a-cell): step 3 and the durable read). +The mode cache reads the mode with a plain `SELECT`, which takes no row lock and never +waits on attach. An insert that finds `enforced` (a process whose cache still reads `recording`) writes nothing: the request goes on to the placement call as in -`enforced`, and a CLI command refuses as in `enforced`, naming the org. A process whose -cache still reads `enforced` after step 3 or step 4 returns the cell to `recording` places -through the router's gate until its cache expires. -`loom doctor` lists any tenancy row the directory does not place. +`enforced`, and a CLI command refuses as in `enforced`, naming the org. A first attach +never lowers the mode once it commits `enforced`. `loom doctor`, with the router's +read-only scope, lists any tenancy row the directory does not place. - **It runs after verification, before anything is written for the org.** The op gate (`gates.authorize`) checks the org the transport resolves (`gates.resolve_org`, with @@ -789,9 +847,9 @@ through the router's gate until its cache expires. serves past a table whose lock it cannot take, logged with the holders of its locks, or one the role cannot alter, logged as such (the third exception under Invariants). A first cell is `recording` until attach sets `enforced` while it runs, so no boot can - have created them: attach creates them itself with the same pass - (`cell_triggers.ensure`) once it has set `enforced`, and the cell goes `serving` only - once `cell_triggers.missing` names no defect. A later cell is `enforced` from prepare + have created them: attach creates them with `cell_triggers.ensure_table`, passing + `recording_first=True`, before step 2 and again before setting `enforced`. + The cell goes `serving` only once `cell_triggers.missing` names no defect. A later cell is `enforced` from prepare and gets them at its first migrating boot after prepare, which its attach requires the same way. `loom doctor` names a disabled or altered one in any state, a missing one on an `enforced` cell, and a partitioned table with an `org` column as unsupported, since @@ -882,14 +940,13 @@ through the router's gate until its cache expires. yet, a later cell's configured orgs at attach and a changed one at `loom cell update` included, passes the gate, a closed cell and `placements_max` included, so a later cell closed or full before attach refuses the attach, naming the org. -- **The first cell records until `enforced` as a single deployment does.** In - `recording`, which lasts until step 3 writes `enforced` to the cell's mode (and again - whenever step 3 or step 4 returns the cell to it), a verified request for a new org - writes a tenancy row with no placement call, so the router's rate, `placements_max` - and `loom cell limit --closed` do not apply: new orgs arrive as they do on a single - deployment, bounded by its own signup, issuer and JIT settings, and step 2 or step 4 - places them unrefused. `loom doctor` lists the recorded orgs that hold no data, so the - operator sees what attach will place. +- **The first cell records until `enforced` as a single deployment does.** In `recording`, + which lasts until step 3 writes `enforced` to the cell's mode (the mode only rises after + it), a verified request for a new org writes a tenancy row with no placement call, so the + router's rate, `placements_max` and `loom cell limit --closed` do not apply: new orgs + arrive as they do on a single deployment, bounded by its own signup, issuer and JIT + settings, and step 2 or step 4 places them unrefused. `loom doctor` lists the recorded + orgs that hold no data, so the operator sees what attach will place. - **Configured orgs are placed at attach.** The cell's `oidc_jit_org`, the `self_signup_org` of a cell with signup on, and its code orgs (`LOOM_CODE_ORG` and the org of each `LOOM_CODE_REPOS_JSON` entry the code config keeps) and tracker org are @@ -983,51 +1040,145 @@ refuses on any failure, naming it: revocation prefix, so a person the identity provider provisions later is admitted, since it will never deprovision a person it does not count as assigned. -The first attach turns the existing deployment into the first cell, in order: - -1. check the mode is `recording`, as prepare set it: every verified request has written - its org's tenancy row if absent while the directory answered that the cell is - `attaching`; -2. in one directory transaction, write `attach_started_at`, the cell's issuers, patterns - and fingerprint (both values of `loom.cell_self`), and the placements, then the - tenancy rows, of every org the cell holds, from one snapshot, straight into the - directory (no other cell exists yet): its configured orgs (of both values), the - distinct `org` of every table of both schemas (generated from the schema, as - erasure's table classification is), and the orgs of catalogued bucket objects; never - `''`, `'-'` or `'*'`; -3. set `enforced` as above, within `--wait`, which the wait below then shares, starting - no try that could commit the mode with less left than that wait's minimum (the mode - cache's lifetime plus the longest request deadline): attach then refuses, reporting - the mode it reads back (`recording`, and a rerun resumes from step 1), or the mode as - unknown when it cannot read it, and a rerun then resumes by the mode it reads; then - wait, within what remains of `--wait` (default 10 minutes, then refuse and set - `recording` back), until every live process reports `enforced` and then the mode - cache's lifetime plus the longest request deadline have passed; then run the SCIM - precondition again (`--revoke-unprovisioned` again if given), for accounts made - before `enforced` took hold; if it refuses, the cell stays `enforced` and - `attaching`, and running attach again resumes from the wait, as a resumed attach does - (below); -4. wait, within what remains of `--wait`, until no transaction that could have read - `recording` is still open, judged from a Postgres snapshot attach takes once it has - read `enforced` back in this run (not from a clock): a transaction on the cell's - database that started before it, from any client, background worker or logical - replication connection, read-only ones included, a writing transaction on another - database of the same server that did (it holds back every snapshot's horizon, and is - named as such), a prepared transaction (which refuses at once), or one whose state - attach cannot read counts as open, and Postgres's own maintenance processes, physical - replication connections (every standby's and a base backup's, which run no SQL) and - attach's own connections do not. Past `--wait` attach refuses, naming what it waited - on, and leaves the cell `enforced`, so a rerun resumes from step 3's wait. The attach - command fixes the exact query, and its tests cover each kind of transaction. Once the - wait passes, place the tenancy rows written since the snapshot and every org step 2's - sources yield when read again (the distinct `org` of every table of both schemas and - the orgs of catalogued bucket objects; a row or an object an unchecked writer, such - as an older CLI or a worker, wrote in `recording` after the snapshot is placed too), - check again that every live process reports `enforced` (refusing and returning to - `recording` as step 3 does if one does not), and mark the cell `serving`. The - directory records the highest mode attach has raised or tried to raise, which - `loom cell restore` writes back only upward: a cell whose step 3 was refused at the - lock is restored `enforced`, and its resumed attach then runs from step 3's wait. +The first attach (`loom cell attach`, its procedure on +[attaching the first cell](../operations/cell-attach.md)) turns the existing deployment +into the first cell. It aliases nothing (an `oauth_issuer_host` on another host refuses at +`attach/begin`), and refuses with SCIM on until per-cell SCIM ships. A run holds an +advisory lock, `cell.ATTACH_LOCK_KEY`, on a session of its own for its life (prepare +takes the same lock), after checking that the session is one direct +backend that keeps the tenancy marker: a lock left on a pooler's server connection would +refuse every later run, and attach's tenancy rows would be refused as an unchecked +writer's. A lost or unreadable lock session ends the run: it never reconnects or +reacquires the lock. Rerun attach to obtain a fresh directory claim and recompute the +remaining work. Every local acceptance write uses this exact backend, so losing the +lock also ends its authority to change acceptance. The lock serializes prepare, attach +and update, and keeps concurrent scans from holding the snapshot horizon that step 4 +waits on. + +It refuses, before any write, naming each: + +- stores other than the registered ones: the Postgres deployment id, ClickHouse's + `loom.storage_authority` and the shared host's `/v1/health/deployment` each name the + directory row's deployment id, `loom.cell_first` reads `(true, )`, the row is the + first cell's, `attaching`, with no other directory row; +- a database encoding other than UTF8 (prepare refuses the same), so a tenancy row + within the length limit is always an org a request can name; +- a synchronous standby not attached now, a durability profile that would not refuse + without one, or a role that may not write the durable read's logical message; a role + of attach's own lacking its grants (`pg_read_all_stats` among them) or a server with + `max_prepared_transactions` above 0; +- **the tenancy-privilege check**: a login role, other than attach's own (the role attach + runs as, which the operator runs it with; superusers; and a role that itself owns + `loom.tenancy`, `loom.tenancy_mode` or a guarded table) and those named with + `--pg-allow-role`, that holds or reaches through any membership a right to make an org + held or lower the mode: `INSERT` on any column of `loom.tenancy` or + `UPDATE` of its `org`, `INSERT` on `loom.tenancy_mode` or `UPDATE` of its `mode`, + `TRIGGER` on a `loom` table, `CREATE` on the database or a schema that is not a system + one (a function there can shadow one the servers call unqualified on their marked + session), `SET` or `ALTER SYSTEM` on `session_replication_role`, `ALTER SYSTEM` on + `loom.tenancy_checked`, membership in `pg_write_server_files` or + `pg_execute_server_program`, or ownership of a `loom` table, of the trigger functions or + of `loom.cell_first_immutable()`, the database or a schema; +- **the unmarked-session check**: a session that sends no marker reads one, or runs as + `replica`; a `pg_db_role_setting` entry of `loom.tenancy_checked` or + `session_replication_role` for this database or every one, matched case-insensitively; + any logical-replication subscription into this database, enabled or not (its apply + worker fires no origin trigger); +- the heartbeats (both tables, stopped rows included for the widened window, the live + window plus a stopping pod's drain and retirement): a live process with `LOOM_CELL` + unset or another cell, one without the heartbeat's keys, `marked` false, `router` false + where `serves` is true, a fingerprint on neither `loom.cell_self` value, several + `keys_fp` values, or none live; a booting process (no cell row yet, no mode, `marked` + or `router` null) is waited on within `--wait`; in `recording` a stopped process with + `LOOM_CELL` unset refuses too; +- the ClickHouse writer check and the scan's table list (`server/cli/cell_writers.py`, + `server/cli/cell_scan.py`), and a settings profile or quota that caps what attach's user + reads. + +Then, in order (the step numbers are the ones the rest of this page uses): + +- **`attach/begin`**, on every run, before any long step: the router probes the stores + attach wrote its nonces to, checks the upstream, and writes the fingerprints and + `keys_fp`; a directory restored to before it is written again. +- **Step 1, the mode**: `recording`, or `enforced` on a resume, which goes from the + triggers, through `attach/enforced` again, to the durable read. +- **The triggers**: created by attach itself, on every run, before step 2 and again just + before step 3 (so a table a migration added during the scan is guarded from the commit + on): one table at a time under the migration lock, each DDL waiting 100 ms for its lock + and retried a second apart within `--wait`; an autovacuum holding it gets one try just + past `deadlock_timeout`, so it yields, and an anti-wraparound vacuum is named. Then + `cell_triggers.missing`; any defect refuses. A refused attach in `recording` leaves the + triggers, which pass every row there. +- **Step 2, the scan**, in `recording`: one statement per table, no long transaction (step 4 + reads every source again): the Postgres tables (`loom.tenancy` among them) by skip scans + on an `org`-led btree index, then the ClickHouse tables, each flushed and synced first, + with the configured orgs of both fingerprints. Log tables (`cell_scan.LOG_TABLES`) are not + read: a log row keeps the org a caller typed before any check, so placing a log-only org + would let anyone pin an org name to the cell; `loom doctor --stray-orgs` names such orgs. + An org longer than `cell.ORG_MAX_BYTES` refuses, naming it, before any tenancy row is + written; a name no request can carry is skipped and named. Then the tenancy rows of the + orgs found, on a marked session, and no placement: unscoped callers are answered from the + tenancy rows, and placing now would move the point of no return earlier. +- **The pre-commit checks**: the heartbeats, the tenancy-privilege and unmarked-session + checks and the standbys again, and the scan's elapsed time within half of `--scan-budget`, + since step 4 reads the same tables again. Each refuses in `recording`, which can still be + backed out. +- **Step 3, the mode commit**: `attach/enforced`, then on the same ATTACH session an EXCLUSIVE + lock on `loom.tenancy_mode`, 2 s a try within `--wait`, the attach lock confirmed and the + cheap heartbeat refusals run before each try (a booting process skips the try). Each try, + including COMMIT, has a client deadline: the lock wait plus a 3 s statement margin and + a 5 s standby allowance. Each mode or trigger try reserves the smaller of one second + and one quarter of its original client bound for cleanup. Its independent owner + closes the session through the public driver API while query/transaction work is + still alive, then cancels and reaps that work within the same absolute deadline. + Borrowed checks and trigger DDL leave failed-session cleanup to that owner. Close + failure or exhausted cleanup time aborts the transport and preserves the original + failure; abort alone does not prove that server locks disappeared. A commit past + its deadline has an unknown outcome until the durable read. + Only acknowledged pre-COMMIT rollback with ATTACH still held permits another try on + that backend. Any other failure ends the run without terminating another backend; + rerun to take the lock and claim afresh. From an acknowledged commit on, the mode + only rises: every later refusal leaves the cell `enforced` and `attaching`, and a + rerun resumes from the durable read. +- **The durable read**, in every run that reads `enforced`: the mode and the `first` row + read in a transaction that emits a transactional logical message, whose commit waits for + the synchronous standbys, with the standbys checked attached just before and after. Then + `X`, the snapshot's xmax, which must exceed that message's xid. A READ COMMITTED writer + that read `recording` holds an xid below `X`, and a REPEATABLE READ one a snapshot older + than `X`: the trigger functions are `VOLATILE`, and the message's xid is assigned after + the mode commit. +- **The heartbeats** again: a process with `LOOM_CELL` unset or another cell, without the + keys, unmarked, or with no router where it serves, refuses at once. No process depends on + its cached mode (a request records a new org through `tenancy._RECORD`, which reads the + mode `FOR SHARE`), so neither a cache lifetime nor a request deadline is waited out. +- **Step 4**: wait, within `--wait`, while any backend of the cell's database but the + polling one has an xid older than `X`, or any but an autovacuum a snapshot older than `X` + (compared through `age()`, exactly `< X` and wraparound-safe); past `--wait` attach + refuses, naming each, and each backend of another database holding the horizon. A prepared + transaction, or a row of the cell's database whose state is hidden, refuses at once. Then + the cheap checks (the writer check, `missing`, the Postgres checks, the heartbeats with + every stopped process with `LOOM_CELL` unset aged out, the standbys); the rescan of every + source in full, Postgres before ClickHouse (the outbox and delivery tables move rows to + ClickHouse), and the queue and drain tables and the tables a view reads before the others + (`cell_scan.MOVE_SOURCES`); the tenancy rows; the placements, in `attach/place` batches; + the writer, trigger, Postgres and standby checks again; and the final checks, under a + floor of their own (the larger of + what `--wait` left and `instances.boot_allowance_s()`): the heartbeats, read through the + rescan every half of the time the leader keeps a stopped row (`instances.PRUNE_AFTER_S`), + refusing any process that ran with `LOOM_CELL` unset during it, `loom.cell_self` equal to + the directory's fingerprints, one `keys_fp` equal to the one `begin` wrote, the mode + `enforced`; then `attach/serving`. + +`--wait` (default 10 minutes) is the wall time the bounded waits share, each lock try +included and never longer than what is left, but for a floor of 100 ms: every try runs once +before the wait is judged spent, so `--wait 0` is one try of each wait with no retry; +`--scan-budget` (default 2 hours) bounds each full scan, which `--wait` does not count. +`--dry-run` runs the refusals and the scan, prints what would be placed and skipped and the +scan's time, and writes nothing; the router's checks at `attach/begin` do not run. On a +`serving` row attach first verifies that the connected authority, configured cell and +bucket match the directory registration. It then reads the cell's mode: `enforced` is +done (exit 0), and any other mode refuses: the directory and the cell's Postgres disagree, +as after a restore of one of them, and attach lowers and repairs neither. A later cell is attached only once the first is `serving`. It must hold no org data (attach refuses otherwise) and starts in `enforced`. Its attach runs, in order: the @@ -1046,13 +1197,18 @@ cell with SCIM on admits no one until its first provisioning cycle has run. A ce nearing its stores' limits is closed to new orgs: its placement calls answer 503, so its issuers' new orgs wait for capacity rather than spill elsewhere. -A resumed first attach, for a cell whose directory row has `attach_started_at`, reruns -every refusal of the list above that applies to the first cell. It then continues by the -cell's mode: in `recording` from step 1; in `enforced`, from step 3's wait (every live -process reports `enforced`, then the cache's lifetime and the longest request deadline -pass) and then its SCIM precondition, so the SCIM check that ends an attach stopped -after setting `enforced`, whatever stopped it, always runs after `enforced` has taken -hold. +A resumed first attach reruns every refusal and continues by the directory row and the +cell's mode: in `recording` from the scan, in `enforced` from the durable read; whenever +the cell reads `enforced` it posts `attach/enforced` again, so a directory restored below +the cell's mode is raised. + +The checks of the stores' writers are point-in-time: a grant, a role setting, a +subscription, a ClickHouse writer or object added after `serving` passes them, as does a +configuration file attach cannot read (`postgresql.conf`, `postgresql.auto.conf`, a +standby's own) or a role that runs code as the server's OS user. `loom doctor` runs the +tenancy-privilege and unmarked-session reads again on every run and warns on what it finds +(`cell_tenancy_privilege`, `cell_unchecked_writers`); the runbook has the operator confirm +the rest. ## What each cell runs @@ -1102,9 +1258,9 @@ closed: every cell keeps serving the orgs it holds, routing continues from each replica's signed snapshot of its last cache (written to shared storage on every directory change), and the placements, lookups of uncached orgs, `loom cell` commands that read or write the directory and the recording cells' state reads answer 503 or -refuse until the directory is back, while `loom doctor` reports its directory checks as -not run and, with `LOOM_CELL` set, exits non-zero; rebuilding a lost directory from the -cells is a design of its own. +refuse until the directory is back, while `loom doctor`, with the router's read-only +scope, reports its directory checks as not run and, with `LOOM_CELL` set, exits +non-zero; rebuilding a lost directory from the cells is a design of its own. Each cell keeps, in its own Postgres, a copy of its directory row's configuration fingerprint (`loom.cell_self`), which its processes compare their configuration with and @@ -1116,7 +1272,106 @@ one command, `loom cell limit --cell [--closed | --open] [--placements-max ( | none)]`, which writes the directory alone, with the router's operator credential and none of the cell's stores, so it works while the cell is down; the router judges a placement from -the directory row. +the directory row. An explicit `--open` refuses while the cell is `attaching`, including +an already-open row and a combined quota edit. This behavior break preserves the closed, +zero-placement boundary used by state cleanup. Close and quota-only edits still work; +an attaching cell can complete attach while closed, then reopen once `serving`. + +## Control mutation fencing + +A cell controller holds the session-level attach lock on one direct, marked, UTF8 +Postgres connection for its whole run. Every write to `loom.cell_self`, `loom.cell_first` +or `loom.tenancy_mode` checks that lock and its observed local state, then writes and +commits on that exact connection. Prepare passes its lock connection to its row writer; +attach commits mode on its lock connection. A lost connection ends the run. It cannot +reconnect and continue an earlier claim. Terminating the client does not prove its +server backend ended: another controller must actually acquire the attach lock before +it starts, and an unresponsive backend can keep that lock until it is positively ended. + +The directory stores one opaque UUID, `cells.mutation_generation`, for each cell. +`POST /cells/v1/operator/cells//claim` takes the directory advisory lock and cell +row lock, compares the full observed registration and exact current/next JSON pair, +generates a fresh UUID on the server, audits it and commits synchronously before +answering. Every execution generates another UUID, including a repeated HTTP request. +The caller never proposes a replacement UUID. Registration assigns a fresh default; +a registration rerun reads the existing value without resetting it. Removing and +registering the same name therefore cannot restore an earlier generation. + +Before its first local acceptance write, prepare, attach or a recovery controller +claims the generation while holding its local lock. Every subsequent directory +mutation carries `expected`: cell, first-cell flag, host, deployment id, bucket, prefix, +probe/upstream addresses, exact current/next pair and the claimed generation. The claim +carries the same fields without a generation. The directory compares them under its +locks before any mutation or no-op success. Canonical JSON comparison distinguishes +`false`, `0` and `0.0`. A stale request cannot release a reservation after its local +backend dies and another controller claims and widens acceptance, even if the directory +pair later returns to its earlier values. A delayed claim can invalidate another +controller's work; it cannot let a dead local session write acceptance state. + +A lost acknowledgement is an unknown outcome. Stop the attempt and rerun: obtain the +local lock, read current state and claim a fresh UUID before recomputing the action. +Do not copy a latest UUID onto an old action. A successful claim or control transaction +waits through synchronous COMMIT. A no-op transaction emits transactional WAL so it +also proves an observed earlier write durable. Acquisition, statements, COMMIT and +cleanup have explicit bounds; an interrupted or uncertain local transaction never +continues on another connection. No operation record, persistent census, phase or lease +is added. The UUID is coordination metadata and holds no memory content. + +One ephemeral owner supervises the command's raw session and child work. Opening keeps +its 75-second bound and each local control transaction keeps its 60-second bound, +reserving cleanup time inside that bound. On failure it sends public connection close +before cancelling child work, so active statements and rollback cannot prevent bounded +cleanup. An unexpected transaction error propagates immediately; it never waits for +rollback on a session the owner will discard. Uncapped command work retains the existing +one-second session-exit allowance, which cannot extend an expired bounded phase. A +transport abort remains an unknown outcome, not proof that the server backend ended. +Mode-lock contention retries only after an acknowledged pre-COMMIT rollback and a check +that the same backend still holds ATTACH. A COMMIT, rollback or ownership-check failure +never becomes a busy retry. After attempting the mode update, the diagnostic stays +unknown until rollback plus ownership or COMMIT is acknowledged. + +Directory control mutations supervise the complete callback transaction independently, +including acquisition and COMMIT. One 131-second bound includes public close, child +settlement and pool-context exit, with cleanup time reserved inside it. Both a failed +attempt and an acknowledged successful transaction close only their own acquired +connection before releasing the pool context. Unexpected errors never wait for rollback +on that discarded session. A late acquisition cannot start mutation work or obtain a +fresh cleanup allowance; acquisition without a known proxy remains an unknown outcome. +Ordinary live placement retains its separate transaction and connection handling. +Repeated interruption keeps the same close task and original end; a completed child +cancellation remains a terminal failure. Fallback abort errors cannot replace the first +operation or cleanup failure. + +The shared directory pool opens physical sessions only for foreground acquisitions; +pool object creation does not establish connectivity. Healthy released sessions are +cached under the existing size cap, recycling and idle expiry. A later cold acquisition +opens a new session. Fixed JSON codec initialization quarantines its never-borrowed raw +session on failure; native interruptions return to that foreground consumer after driver +cleanup, with no autonomous opening or replenishment task. If quarantine fails, the +original failure records that session closure is unknown. + +The source census for acceptance and operator directory writes is checked by +`tests/unit/test_cell_control_census.py`, including dynamic cleanup tables, guarded +entry points and the named exceptions: + +| Writer | Fence | +| --------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | +| `cell_prepare._write_rows` | Fresh claim, observed local rows, exact lock backend | +| `cell_attach_pg._mode_transaction` | Attach's claim, supervised mode try, recording/first/fingerprint comparison, exact lock backend | +| `cell_control.backout` | Fresh claim, fenced close and zero-placement read, observed local rows, exact lock backend | +| `directory.attach_begin`, `attach_place`, `attach_enforced`, `attach_serving`, `remove` | Claimed generation, full registration and exact pair, including retries/no-ops | +| `directory.set_limit`, `rotate_credential` | Read and compare the current generation/registration/pair; do not claim or rotate | +| `directory.register` | Bootstrap default or immutable rerun lookup; never replaces existing acceptance or generation | +| `directory.place` | Immutable placement only; keeps its org/cell locks and quota checks; never writes local acceptance | + +Read-only routes are exempt. Router migrations and an offline directory restore are +maintenance operations. Future restore/update acceptance writers must use the same +claim and lock-session helpers; a new directory mutation must use the common comparison +before its no-op branch as well. Direct privileged SQL is not an online control writer. +The [control upgrade and recovery procedure](../operations/cell-control.md) defines the +required fleet turnover and offline restore of generations. N-1 data-plane builds can +read and insert on the additive schema; old control clients and router handlers cannot +participate in this protocol. ## Operations @@ -1126,19 +1381,20 @@ the directory row. there); for an org with no placement it names the first cell, which the lookup does not tell apart. - `loom cell limit --cell --closed` closes a cell to new orgs, `--open` reopens - it, and `--placements-max ` sets its placement quota (`--placements-max none` + a serving cell, and `--placements-max ` sets its placement quota (`--placements-max none` clears it). -- `loom cell remove --cell ` removes the directory row of a cell that is `attaching` - with no placement (`--cell` is the cell's `LOOM_CELL`, unset by then). For the first cell - it reads the cell's Postgres alone, on one session from `LOOM_PG_DSN` with no ClickHouse, - and refuses unless it reads `open` and records the row's deployment id, which enforces the - back-out order: unset `LOOM_CELL` on every process (each serves as in `recording`), then - the full-state cleanup, then the removal +- `loom cell remove --cell ` removes the directory row of an `attaching` cell + with no placement. For both first and later cells, keep `LOOM_CELL=` and the + registered store settings on the control client. Removal takes the direct marked + ATTACH backend, verifies the paired deployment and registration, and claims a fresh + directory generation before deleting the row. The first cell must also read `open`, + enforcing the back-out order: stop or unset `LOOM_CELL` on the accepting processes, + complete full-state cleanup, then remove the directory row ([troubleshooting](../operations/troubleshooting.md#a-process-with-loom_cell-unset-is-gated)). Removing the row first answers every new org 503; clearing the state first gates every - process with `LOOM_CELL` set. A first cell with a placement goes forward, never back. A - removed cell's stores keep whatever state rows they hold, so the full-state cleanup runs - before they are prepared again. + accepting process with `LOOM_CELL` set. A first cell with a placement goes forward, + never back. Removal leaves the stores' state rows intact; full-state cleanup must + precede preparing those stores again. - The router's operator credential is `lco_` and 128 lowercase hex (a router with no `LOOM_ROUTER_OPERATOR_SHA256` answers the operator API 503 with `NO_OPERATOR_CREDENTIAL`), which `loom cell credential new --operator` makes; the @@ -1200,30 +1456,36 @@ the directory row. ## Failure modes -| Failure | What happens | -| ------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| The directory is down | requests by host, by issuer and to the first cell are routed; new orgs and uncached lookups answer 503 | -| The router's placement call fails | the request answers 503 and nothing is written; orgs the cell holds are served | -| A cell's issuer places orgs in a flood | from `enforced` on, its placement calls answer 503 with `Retry-After` past its rate, and its calls for new orgs answer 503 past its quota if one is set; other cells place as before | -| A SCIM token of another cell, or an application at the wrong cell's host | the process does not boot, or `/scim/v2` answers 401 and writes nothing | -| A cell's Postgres or ClickHouse | that cell fails as a single deployment does ([storage authority](storage-authority.md#failure)); other cells serve | -| A cell's own rows or tenancy mode are unreadable | with `LOOM_CELL` set a process keeps its last rows and mode within the stale bound, then is gated, a server process whose first read fails does not boot (gated and unready with Postgres migrations pending), and a workers child that never read them is gated; with it unset a server process keeps its verdict with no bound, and a failed first read fails its boot (gated until a read lands with Postgres migrations pending) | -| A `first` or mode row removed, or both beside a tenancy or own row | the org triggers, where a cell has them, act as in `enforced` and every process is gated; `loom doctor` names it, a prepare rerun writes them back | -| No state, tenancy or own row on a cell the directory registers | open; `loom doctor` names it; `loom cell restore` closes it after a restore, a prepare rerun after a stopped prepare | -| A cell's `first` value is not yet read | every request but health probes and the probe route answers 503 (SCIM with `Retry-After`); the process is unready and runs no worker (a workers child, or a server process with Postgres migrations pending; any other server process fails its boot) | -| A request for another cell's org | 409 `WRONG_CELL` after verification, nothing written | -| A platform, `*`-org or principal-less call for an unheld org | on a first cell, 503 while the directory answers `attaching` or cannot be read before a `serving` answer; 409 `WRONG_CELL` once `serving` and on a later cell | -| A pooler that strips the tenancy marker from a cell's sessions | the `cell_marker` readiness check fails in `enforced` and `partial` and warns in `recording`; the heartbeat reports `marked` false and attach refuses the process | -| A cell restored to an earlier time | `loom cell restore` writes placed orgs' rows back before it starts | -| A first cell restored to before its prepare, not yet attached | at processes with `LOOM_CELL` set, every request but health probes and the probe route answers 503 until prepare reruns | -| The directory lost | every cell serves the orgs it holds; placements and uncached lookups answer 503, directory commands refuse, until it is back | -| A cell in `recording`, the directory unreadable | a new org answers 503; the orgs it holds keep serving | -| A process stuck during first attach | the attach refuses after `--wait` and the cell returns to `recording` | -| A recording insert held open during first attach | attach stops trying once too little of `--wait` is left for the wait, reporting the mode it reads back; a rerun resumes | -| Attach stalls while it holds the mode table's lock | Postgres ends its session within seconds and releases the lock; new orgs answer within their deadline | -| A transaction open since before step 4's snapshot | attach refuses once `--wait` runs out (a prepared one at once), naming what it waited on, and leaves `enforced` | -| A CLI command with `LOOM_CELL` unset, started before attach's step 3 | it read the state while the first cell was `recording` and keeps writing ClickHouse after `serving`; the runbook sets `LOOM_CELL` on every CLI before attach | -| A CLI of a build before the state read | it is not refused on an `enforced` cell; every CLI and CI image runs a build with the read before attach ([upgrades](../operations/upgrades.md#upgrade-procedure)) | +| Failure | What happens | +| ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| The directory is down | requests by host, by issuer and to the first cell are routed; new orgs and uncached lookups answer 503 | +| The router's placement call fails | the request answers 503 and nothing is written; orgs the cell holds are served | +| A cell's issuer places orgs in a flood | from `enforced` on, its placement calls answer 503 with `Retry-After` past its rate, and its calls for new orgs answer 503 past its quota if one is set; other cells place as before | +| A SCIM token of another cell, or an application at the wrong cell's host | the process does not boot, or `/scim/v2` answers 401 and writes nothing | +| A cell's Postgres or ClickHouse | that cell fails as a single deployment does ([storage authority](storage-authority.md#failure)); other cells serve | +| A cell's own rows or tenancy mode are unreadable | with `LOOM_CELL` set a process keeps its last rows and mode within the stale bound, then is gated, a server process whose first read fails does not boot (gated and unready with Postgres migrations pending), and a workers child that never read them is gated; with it unset a server process keeps its verdict with no bound, and a failed first read fails its boot (gated until a read lands with Postgres migrations pending) | +| A `first` or mode row removed, or both beside a tenancy or own row | the org triggers, where a cell has them, act as in `enforced` and every process is gated; `loom doctor` names it, a prepare rerun writes them back | +| No state, tenancy or own row on a cell the directory registers | open; `loom doctor` names it; `loom cell restore` closes it after a restore, a prepare rerun after a stopped prepare | +| A cell's `first` value is not yet read | every request but health probes and the probe route answers 503 (SCIM with `Retry-After`); the process is unready and runs no worker (a workers child, or a server process with Postgres migrations pending; any other server process fails its boot) | +| A request for another cell's org | 409 `WRONG_CELL` after verification, nothing written | +| A platform, `*`-org or principal-less call for an unheld org | on a first cell, 503 while the directory answers `attaching` or cannot be read before a `serving` answer; 409 `WRONG_CELL` once `serving` and on a later cell | +| A pooler that strips the tenancy marker from a cell's sessions | the `cell_marker` readiness check fails in `enforced` and `partial` and warns in `recording`; the heartbeat reports `marked` false and attach refuses the process | +| A cell restored to an earlier time | `loom cell restore` writes placed orgs' rows back before it starts | +| A first cell restored to before its prepare, not yet attached | at processes with `LOOM_CELL` set, every request but health probes and the probe route answers 503 until prepare reruns | +| The directory lost | every cell serves the orgs it holds; placements and uncached lookups answer 503, directory commands refuse, until it is back | +| A cell in `recording`, the directory unreadable | a new org answers 503; the orgs it holds keep serving | +| A process stuck during first attach | before step 3, attach refuses after `--wait` and the cell stays `recording`; after it, attach refuses, leaving the cell `enforced` and `attaching`, and a rerun resumes | +| A recording insert held open during first attach | the mode commit's tries wait behind its `FOR SHARE` lock within `--wait`; past it attach refuses and the cell stays `recording`; a rerun resumes | +| Attach stalls while it holds the mode table's lock | Postgres ends its session within seconds and releases the lock; new orgs answer within their deadline | +| A transaction open since before step 4's snapshot | attach refuses once `--wait` runs out (a prepared one at once), naming what it waited on, and leaves `enforced` | +| A CLI command with `LOOM_CELL` unset, started before attach's step 3 | it read the state while the first cell was `recording` and keeps writing ClickHouse after `serving`; the runbook sets `LOOM_CELL` on every CLI before attach, and `loom doctor --stray-orgs` names the orgs it left without a tenancy row | +| A CLI of a build before the state read | it is not refused on an `enforced` cell; every CLI and CI image runs a build with the read before attach ([upgrades](../operations/upgrades.md#upgrade-procedure)) | +| An org found only in a log table of the first cell | never placed; once it is placed in another cell, its log rows here (`agent_events` identities among them) are outside its erasure there until they are removed by hand (the runbook's procedure); `loom doctor --stray-orgs` names it | +| An org longer than `cell.ORG_MAX_BYTES` written before the limit | the first attach refuses in its scan, naming it, until the runbook's manual procedure removes it; one first found in step 4 is skipped and named | +| A writer, grant, role setting or subscription added after `serving` | attach's checks ran once and pass it; `loom doctor` warns on the Postgres ones on every run | +| `LOOM_SERVES_TRAFFIC=false` on a pod a load balancer routes to | attach does not require its router answer; its new orgs answer 503 | +| A process that changes between attach's last check and the mode commit, or an HA manager that empties `synchronous_standby_names` between the checks around the durable read | not seen by attach: a window of seconds | +| A cell process with a longer `LOOM_INSTANCE_HEARTBEAT_SECONDS` than attach's | it can fall outside attach's live window while live; the runbook requires one value | ## Rolling upgrades @@ -1306,11 +1568,16 @@ from its cache ([upgrades](../operations/upgrades.md#upgrade-procedure)). `loom doctor` names), with their functions in `schemas/pg_migrations/011_tenancy_triggers.sql` and `012_tenancy_guard.sql`; `server/cli/doctor_cell.py`: `loom doctor`'s cell checks; -- `server/cli/cell_scan.py`: the first attach's org scan, a library no command calls: the - tables of both stores with an `org` column, judged by engine and against erasure's - classification, every host of a cluster listing the same, and the distinct orgs of each - table outside `LOG_TABLES`; `server/cli/cell_writers.py`: the ClickHouse writer check (the - users, user directories, objects and replicas outside the operator's allow lists); +- `server/cli/cell_attach.py`: `loom cell attach`; `server/cli/cell_attach_pg.py`: its + Postgres reads and writes (the session checks, the attach lock, the tenancy-privilege + and unmarked-session checks doctor shares, the trigger pass, the mode commit, the + durable read, step 4's wait); `server/cli/cell_attach_beats.py`: its judgment of the + heartbeats and the rescan's watch; +- `server/cli/cell_scan.py`: the first attach's org scan: the tables of both stores with + an `org` column, judged by engine and against erasure's classification, every host of a + cluster listing the same, and the distinct orgs of each table outside `LOG_TABLES`; + `server/cli/cell_writers.py`: the ClickHouse writer check (the users, user directories, + objects and replicas outside the operator's allow lists); `server/cli/cell_ch.py`: what both share (the cluster's hosts, the per-host reads); - `schemas/pg_migrations/008_cells.sql`, `schemas/pg_migrations/009_cell_probe.sql` and `schemas/migrations/094_cell_heartbeats.sql`: the cell's tables; @@ -1384,126 +1651,118 @@ An integration test with two cells and a router in CI's Postgres job: - every route, the batch entries of MCP and every CLI command included, refuses another cell's org and writes nothing in its Postgres or ClickHouse (the route, op-field and CLI censuses cover the rest); -- the first attach places every org with a row in any table, an org first seen during - the attach is placed in step 4, and so is one a direct Postgres insert wrote between - step 2's snapshot and `enforced`, a ClickHouse row an older CLI wrote then and a - bucket object catalogued then; a process past `LOOM_JWT_REVOCATION_MAX_STALE_SECONDS` - of failed reads of its own rows or mode is gated and serves again once a read lands; a - pod with `LOOM_CELL` unset is unready, naming the setting, answers 503 to a bearer - request, passes its health probe, answers the probe route and runs no worker on an - attached cell and on a later cell between prepare and attach, keeps serving on a first - cell in `recording`, and keeps the last mode it read through a failed read, - `recording` included after step 3 or step 4 returned the cell to it; a CLI command - declaring an org with it unset is refused on an `enforced` or `partial` cell, an - org-free one (a token revocation) runs, also with Postgres down on a deployment never - prepared, and a `loom cell` store command refuses; `loom cell limit --closed` succeeds - with the cell's Postgres down; after `loom cell credential --cell --next` both - credentials place and read state, `--finish` refuses the old credential, for a cell - finished before its pods hold the new one the router refuses its calls with 401, its - request answers 503 writing nothing, and it recovers by recording the old hash again, - `--revert` drops a pending `next`, which is then refused, and the old credential keeps - placing, and a second `--next` while one is pending and a hash another cell records as - current or next are refused; `loom cell limit` and `loom cell credential --next`, - `--finish` and `--revert` refuse while the directory is unavailable; concurrent - placements at one under `placements_max` across two router replicas write one; a CLI - command declaring an org with `LOOM_CELL_ROUTER_TOKEN` set refuses, naming it, on a - first cell once it is `enforced` (`attaching` included) and on a later cell, and on a - first cell in `recording` runs and records the org; attach with `placements_max` below - the orgs the first cell holds places them all, and a later cell's attach past its - quota or closed refuses, naming the org; a first cell with `--closed` and - `placements_max` set before step 2 attaches, and an org a request records between step - 2 and `enforced` is placed by step 4 with no 503; after step 3's write of the cell's - mode, a process whose cache still reads `recording` sends a new org's request to the - placement call, which a closed cell answers 503; a CLI command whose insert finds - `enforced` refuses, naming the org, and makes no placement call; after step 3's - refused wait or step 4's recheck returns the cell to `recording`, a new org is - recorded with no placement call, and one at a process whose cache still reads - `enforced` places through the gate until the cache expires; each ordering of attach's - write of the mode with a recording insert holds as stated above (running, starting - during the write, a stream of overlapping inserts each shorter than a try, with attach - setting `enforced` within a try or two, held across several tries, held until too - little of `--wait` is left, a stalled attach, a commit cut off by its timeout, a - commit whose connection is lost while it waits for the standby, read back only once - that backend has gone and written again durably if it reads `enforced`); a - `REPEATABLE READ` transaction whose first statement ran before `enforced` committed, - on attach's role or a second role, holds step 4 until it ends and its write of an - unheld org is then placed, one whose first statement ran after is refused by the - trigger, a transaction that starts after attach takes the lock but before its commit - is visible holds step 4, a transaction opened in `recording` that outlives one run's - `--wait` holds the rerun's step 4, after a refusal and after `loom cell restore` - raised the mode, an autovacuum worker on a loom table does not hold step 4 and a - background worker with a transaction open since `recording` does, a prepared - transaction (after a restart with `max_prepared_transactions` raised during the wait) - makes step 4 refuse at once, naming it, with the mode still `enforced` and a rerun - refused until it is resolved and the setting is back to 0, a write transaction open in - a second database on the same server holds step 4 and is named, a long read on the - synchronous standby, or on a second standby, with `hot_standby_feedback` on does not - hold it, nor does a base backup running across it, a write held open on a - `replication=database` connection holds step 4, and so does a `REPEATABLE READ` - transaction on one, opened in `recording` and writing nothing before `enforced`, a - long logical dump (a `REPEATABLE READ` read-only transaction) on the cell's database - past `--wait` makes step 4 refuse, naming it, with the cell left `enforced`, and the - first cell's attach without `pg_read_all_stats` or with `max_prepared_transactions` - above 0 refuses before step 1 with the mode still `recording`, while a later cell's - attach does not require either; a missing or disabled org trigger is refused by attach - and named by `loom doctor`, a write for an unheld org on an `enforced` cell whose mode - row was deleted is refused and the state named by `loom doctor`; on that cell a pod - with `LOOM_CELL` unset is unready and runs no worker, and a CLI command declaring an - org with it unset is refused, as a pod is; updating, deleting or truncating the - `first` row on an `enforced` cell is refused, also under - `session_replication_role = replica`, and a missing or disabled `first`-row trigger is - refused by attach and named by `loom doctor`; a running process with `LOOM_CELL` set - on a cell whose mode row is removed answers 503 to a bearer request and runs no worker - pass, and serves again after a rerun of `loom cell prepare` writes the row back while - it runs; a cell the directory registers with no state, tenancy or own row is named by - `loom doctor`, its processes with `LOOM_CELL` set stay unready, a process with it - unset serves as a single deployment, and `loom cell restore` writes both rows back, an - attached cell then serving again and an unattached one after a prepare rerun; a cell - with both rows absent beside a tenancy row, and one beside only its own row, has the - function refuse an unheld org (a boot creates no trigger there), every process gated - and `loom doctor` naming it, and a prepare rerun writes the rows back; a Postgres-only - restore of an attached first cell to before its prepare is named by `loom doctor` and - closed by `loom cell restore`; a prepare rerun during a pending `loom cell update` on - an attached cell, with processes on both values and `loom.cell_self` lost, writes both - values back and keeps both sets of processes serving, and one before attach refuses - the two configurations; a prepare rerun on an attached cell whose `current` is missing - and whose pods' configuration differs from the directory row refuses, and one before - attach whose heartbeats report another bucket prefix refuses; a prepare rerun given a - credential hash that is neither the row's `current` nor its `next` is refused; a - prepare rerun on a `serving` later cell whose prefix holds data writes a removed mode - row back; a cell with a mode row and no `first` row has the function refuse an unheld - org (a boot creates no trigger there), every process gated and `loom doctor` naming - it, and a prepare rerun writes the row back; a later cell whose mode row is set to - `recording` records nothing, its trigger refuses an unheld org, a process with - `LOOM_CELL` set there serves as in `enforced`, a pod with `LOOM_CELL` unset there is - unready and runs no worker, `loom cell prepare` names the row, a CLI command declaring - an org with `LOOM_CELL_ROUTER_TOKEN` set refuses, naming it, and the router refuses - its state read; a build N-1 process's `memory.set` and delivery pass on a - never-prepared deployment after N's migration succeed and write no tenancy row, and on - the first cell in `recording` a boot creates no org trigger, and one created by hand - reads the state and looks nothing up, a write under - `session_replication_role = replica` for an unheld org is skipped (origin triggers), - an `INSERT ... ON CONFLICT DO UPDATE` that inserts a row for an unheld org is refused - and one that hits an existing row of it is not, a marked session's every write passes, - and the triggers on an `enforced` cell cover every table erasure classifies with an - `org` column but the three exclusions; and an org-keyed write while attach holds the - mode table's lock returns at once, with new-org requests answering within their - deadline and mode-cache reads returning at once throughout; `loom cell restore` after - a refusal at step 3's lock leaves the cell `enforced`, and the resumed attach runs - from step 3's wait; a `LOOM_CELL_ROUTER_TOKEN` not of the `lcr_` form is refused at - boot and named by `loom doctor`; `loom cell credential new` prints a credential of - that form and its SHA-256, with the directory and the stores unreachable; prepare and - `--next` refuse an argument that is not 64 lowercase hex characters, the credential - and its bare 128-character secret included, storing nothing; orgs a first cell records - in `recording` with no data are listed by `loom doctor`; a placement flood with no - quota set is held to the rate limit at each router replica, `loom doctor` names the - cell as one with no quota, and a call refused past it at one replica answers 503 with - a `Retry-After` of at most 60 s, after which its retry at that replica, with no other - call of the cell admitted between, is admitted; a cell's first prepare, with no - directory row yet, passes the deployment-id check; a pod or CLI that lost - `LOOM_PG_DSN` or `LOOM_STORAGE_AUTHORITY` on a cell is refused by the authority check; - the first cell's existing tokens, refresh tokens and OAuth clients keep working - through the cutover, a logout through the shared host revokes its refresh token; +- the first attach places every org with a row in any table, an org first seen during the + attach is placed in step 4, and so is one a direct Postgres insert wrote between step 2's + scan and `enforced`, a ClickHouse row an older CLI wrote then and a bucket object + catalogued then; a process past `LOOM_JWT_REVOCATION_MAX_STALE_SECONDS` of failed reads of + its own rows or mode is gated and serves again once a read lands; a pod with `LOOM_CELL` + unset is unready, naming the setting, answers 503 to a bearer request, passes its health + probe, answers the probe route and runs no worker on an attached cell and on a later cell + between prepare and attach, keeps serving on a first cell in `recording`, and keeps the + last mode it read through a failed read, `recording` included; a CLI command declaring an + org with it unset is refused on an `enforced` or `partial` cell, an org-free one (a token + revocation) runs, also with Postgres down on a deployment never prepared, and a + `loom cell` store command refuses; `loom cell limit --closed` succeeds with the cell's + Postgres down; after `loom cell credential --cell --next` both credentials place + and read state, `--finish` refuses the old credential, for a cell finished before its pods + hold the new one the router refuses its calls with 401, its request answers 503 writing + nothing, and it recovers by recording the old hash again, `--revert` drops a pending + `next`, which is then refused, and the old credential keeps placing, and a second `--next` + while one is pending and a hash another cell records as current or next are refused; + `loom cell limit` and `loom cell credential --next`, `--finish` and `--revert` refuse + while the directory is unavailable; concurrent placements at one under `placements_max` + across two router replicas write one; a CLI command declaring an org with + `LOOM_CELL_ROUTER_TOKEN` set refuses, naming it, on a first cell once it is `enforced` + (`attaching` included) and on a later cell, and on a first cell in `recording` runs and + records the org; attach with `placements_max` below the orgs the first cell holds places + them all, and a later cell's attach past its quota or closed refuses, naming the org; a + first cell with `--closed` and `placements_max` set before step 2 attaches, and an org a + request records between step 2 and `enforced` is placed by step 4 with no 503; after step + 3's write of the cell's mode, a process whose cache still reads `recording` sends a new + org's request to the placement call, which a closed cell answers 503; a CLI command whose + insert finds `enforced` refuses, naming the org, and makes no placement call; a refusal + after step 3's write leaves the cell `enforced` and `attaching`, where a new org is placed + through the gate; each ordering of attach's write of the mode with a recording insert + holds as stated above (running, starting during the write, a stream of overlapping inserts + each shorter than a try, with attach setting `enforced` within a try or two, held across + several tries, held past `--wait` (attach refuses, the cell still `recording`), a stalled + attach, a commit cut off by its timeout, and a commit whose connection is lost while it + waits for the standby, which the durable read decides); a `REPEATABLE READ` transaction + whose first statement ran before `enforced` committed, on attach's role or a second role, + holds step 4 until it ends and its write of an unheld org is then placed, one whose first + statement ran after is refused by the trigger, a transaction that starts after attach + takes the lock but before its commit is visible holds step 4, a transaction opened in + `recording` that outlives one run's `--wait` holds the rerun's step 4 after a refusal, an + autovacuum worker on a loom table does not hold step 4 and a background worker with a + transaction open since `recording` does, a prepared transaction (after a restart with + `max_prepared_transactions` raised during the wait) makes step 4 refuse at once, naming + it, with the mode still `enforced` and a rerun refused until it is resolved and the + setting is back to 0, a write transaction open in a second database on the same server + holds step 4 and is named, a long read on the synchronous standby, or on a second standby, + with `hot_standby_feedback` on does not hold it, nor does a base backup running across it, + a write held open on a `replication=database` connection holds step 4, and so does a + `REPEATABLE READ` transaction on one, opened in `recording` and writing nothing before + `enforced`, a long logical dump (a `REPEATABLE READ` read-only transaction) on the cell's + database past `--wait` makes step 4 refuse, naming it, with the cell left `enforced`, and + the first cell's attach without `pg_read_all_stats` or with `max_prepared_transactions` + above 0 refuses before step 1 with the mode still `recording`, while a later cell's attach + does not require either; a missing or disabled org trigger is refused by attach and named + by `loom doctor`, a write for an unheld org on an `enforced` cell whose mode row was + deleted is refused and the state named by `loom doctor`; on that cell a pod with + `LOOM_CELL` unset is unready and runs no worker, and a CLI command declaring an org with + it unset is refused, as a pod is; updating, deleting or truncating the `first` row on an + `enforced` cell is refused, also under `session_replication_role = replica`, and a missing + or disabled `first`-row trigger is refused by attach and named by `loom doctor`; a running + process with `LOOM_CELL` set on a cell whose mode row is removed answers 503 to a bearer + request and runs no worker pass, and serves again after a rerun of `loom cell prepare` + writes the row back while it runs; a cell the directory registers with no state, tenancy + or own row is named by `loom doctor`, its processes with `LOOM_CELL` set stay unready, a + process with it unset serves as a single deployment, and `loom cell restore` writes both + rows back, an attached cell then serving again and an unattached one after a prepare + rerun; a cell with both rows absent beside a tenancy row, and one beside only its own row, + has the function refuse an unheld org (a boot creates no trigger there), every process + gated and `loom doctor` naming it, and a prepare rerun writes the rows back; a + Postgres-only restore of an attached first cell to before its prepare is named by + `loom doctor` and closed by `loom cell restore`; a prepare rerun during a pending + `loom cell update` on an attached cell, with processes on both values and `loom.cell_self` + lost, writes both values back and keeps both sets of processes serving, and one before + attach refuses the two configurations; a prepare rerun on an attached cell whose `current` + is missing and whose pods' configuration differs from the directory row refuses, and one + before attach whose heartbeats report another bucket prefix refuses; a prepare rerun given + a credential hash that is neither the row's `current` nor its `next` is refused; a prepare + rerun on a `serving` later cell whose prefix holds data writes a removed mode row back; a + cell with a mode row and no `first` row has the function refuse an unheld org (a boot + creates no trigger there), every process gated and `loom doctor` naming it, and a prepare + rerun writes the row back; a later cell whose mode row is set to `recording` records + nothing, its trigger refuses an unheld org, a process with `LOOM_CELL` set there serves as + in `enforced`, a pod with `LOOM_CELL` unset there is unready and runs no worker, + `loom cell prepare` names the row, a CLI command declaring an org with + `LOOM_CELL_ROUTER_TOKEN` set refuses, naming it, and the router refuses its state read; a + build N-1 process's `memory.set` and delivery pass on a never-prepared deployment after + N's migration succeed and write no tenancy row, and on the first cell in `recording` a + boot creates no org trigger, and one created by hand reads the state and looks nothing up, + a write under `session_replication_role = replica` for an unheld org is skipped (origin + triggers), an `INSERT ... ON CONFLICT DO UPDATE` that inserts a row for an unheld org is + refused and one that hits an existing row of it is not, a marked session's every write + passes, and the triggers on an `enforced` cell cover every table erasure classifies with + an `org` column but the three exclusions; and an org-keyed write while attach holds the + mode table's lock returns at once, with new-org requests answering within their deadline + and mode-cache reads returning at once throughout; a refusal at step 3's lock leaves the + cell `recording` with its directory row `enforced`, and the rerun runs step 2 again; a + `LOOM_CELL_ROUTER_TOKEN` not of the `lcr_` form is refused at boot and named by + `loom doctor`; `loom cell credential new` prints a credential of that form and its + SHA-256, with the directory and the stores unreachable; prepare and `--next` refuse an + argument that is not 64 lowercase hex characters, the credential and its bare + 128-character secret included, storing nothing; orgs a first cell records in `recording` + with no data are listed by `loom doctor`; a placement flood with no quota set is held to + the rate limit at each router replica, `loom doctor` names the cell as one with no quota, + and a call refused past it at one replica answers 503 with a `Retry-After` of at most 60 + s, after which its retry at that replica, with no other call of the cell admitted between, + is admitted; a cell's first prepare, with no directory row yet, passes the deployment-id + check; a pod or CLI that lost `LOOM_PG_DSN` or `LOOM_STORAGE_AUTHORITY` on a cell is + refused by the authority check; the first cell's existing tokens, refresh tokens and OAuth + clients keep working through the cutover, a logout through the shared host revokes its + refresh token; - attach refuses a shared issuer, root secret, Keeper path, nested bucket prefix or configured org, a second cell that holds data, and a second attach while one runs; - an OIDC sign-in, an MCP client's authorization and a device flow for an org in the @@ -1517,12 +1776,12 @@ An integration test with two cells and a router in CI's Postgres job: application ends the person there, a platform-admin bot still reaches a cell whose SCIM deactivated everyone, and attach refuses a cell with unprovisioned live subjects or sharing another's OIDC client; -- an attach stopped in step 3 after it set `enforced`, before the wait passed, resumes - with the wait before the SCIM precondition, and one refused by the SCIM precondition - resumes from the wait too, the mode still `enforced`; a later attach refused before - its directory transaction leaves `attach_started_at` empty, so a rerun checks a new - cell's refusals again, and a resumed first attach finds its configured orgs placed in - this cell and carries on; +- an attach stopped after step 3's commit landed, before the durable read, says the commit's + outcome is unknown, and its rerun resumes by the mode it reads; one refused after the + durable read resumes from it, the mode still `enforced`; a later attach refused before its + directory transaction leaves `attach_started_at` empty, so a rerun checks a new cell's + refusals again, and a resumed first attach finds its configured orgs placed in this cell + and carries on; - a cell restored to before the first attach's `enforced` gets its mode back from `loom cell restore`; one whose `loom cell restore` was skipped answers 503 for an org it does not hold, another cell's org included, writes no tenancy row for it, and is diff --git a/docs/architecture/failure-and-recovery.md b/docs/architecture/failure-and-recovery.md index 40f8e4b57e..d54fb025b8 100644 --- a/docs/architecture/failure-and-recovery.md +++ b/docs/architecture/failure-and-recovery.md @@ -186,17 +186,16 @@ node. Nothing else counts: an unreachable process is not a stopped one. TTL is 60 s, renewed every 20 s; a contender takes it over once it expired and the holder's participant counts as stopped. `loom provider-lock clear --holder ` frees one whose holder has no participant row, which never counts as stopped. -- **Killed mid-reconfiguration.** The transition waits on a participant whose heartbeat - expired, since a paused process may resume with admitted work, until a process on - its host proves it gone (its container restarted, or its pid exited) and retires it. - The workers leader - deletes the rows of participants retired a day ago and lists, once a minute, those a - transition would wait on as `provider_participants_stale` in its `/v1/health`. Retire - one by hand with `loom participant retire --participant --yes` once its process - is confirmed gone; the command refuses a participant whose heartbeat is fresh or whose - queries still run. -- **A boot that fails after enrolling** retires its row; a crash-looping container leaves - none behind. +- **Killed mid-reconfiguration.** An expired heartbeat remains a blocker until the + exact process dies or completes guarded shutdown. A trusted local process can + verify its PID, namespace, boot and start identity, then retire it after all + cross-replica queries settle. Use `loom participant retire --participant ` on + that trusted host. Pod deletion or replacement and namespace changes alone do + not prove death. The workers leader prunes only certified permanent retirements + older than a day; unknown legacy rows remain visible. +- **A boot that fails after enrolling** remains nonretired unless all its actual + producers and writes stop. A shutdown deadline does not turn cancellation into + a retirement certificate. A process frozen past its heartbeat (a stopped container, a VM pause) holds the transition until it resumes and drains, or an operator retires it. diff --git a/docs/architecture/provider-reconfiguration.md b/docs/architecture/provider-reconfiguration.md index 71fa716069..622fb99dc4 100644 --- a/docs/architecture/provider-reconfiguration.md +++ b/docs/architecture/provider-reconfiguration.md @@ -36,21 +36,38 @@ stopped only once it retired itself (a graceful shutdown), a later process on it host proved it gone, or an operator retired it. An expired heartbeat is no proof: a paused process can hold admitted work outside ClickHouse and write it with the old configuration once it resumes, so the transition waits for it to beat again and -drain. Each row carries its host name, kernel boot, container id (Docker's, or a -Kubernetes pod's and container name), pid namespace, pid and start time; every 10 s -each process retires the participants of its own host and boot whose pid no longer -runs that process, or that ran in its own container before the container restarted -into a new pid namespace, once no query of theirs runs. That covers a killed process -whose container or worker restarts in place; two containers that share a host name -(host networking, two containers of one pod) never retire each other. An operator who confirmed its -process is gone retires it with `loom participant retire --participant --yes`, -which refuses one whose heartbeat is fresh or whose queries still run. The workers +drain. Each row carries its host name, kernel boot, container identity, pid namespace, +pid and start time. A later process on the same trusted host and pid namespace can +prove that the exact PID exited or was reused. A changed namespace, a missing pod +object, another pod UID or a terminal API phase does not prove physical termination. +`loom participant retire --participant ` performs that exact local verification +and refuses when evidence is unavailable or any cross-replica query remains. +The workers leader's `/v1/health` lists every participant a transition would wait on as `provider_participants_stale` (`heartbeat_expired` or `queries_running`). `system.processes` is read over `clusterAllReplicas(LOOM_CH_CLUSTER)`; a server without that cluster answers for itself, and any other failure of the read counts the participant as running. +Permanent retirement uses a separate process gate that never reopens. The outer +HTTP/WebSocket scope covers request-tail settlement; an inner scope covers actual +application tasks across middleware child-task boundaries, including OAuth and +control handlers. Worker passes and native command bodies use the same counter. +Shutdown closes admission, stops producers, waits for actual scope and registered +descendant completion, flushes buffers and checks carried and server-side writes. +Canceling a coroutine or exhausting the shutdown budget leaves its row nonretired. +A failed startup likewise cannot certify unfinished producers. Model-transition +`drained` is reversible and does not certify permanent retirement. + +The ninth `host_proof` field, `cell-retirement-v1`, identifies this lifecycle. +Only the guarded shutdown or verified-death writer publishes certified retirement. +The generic heartbeat writer cannot retire or reopen a retired identity; a +replacement process draws a fresh UUID. Pruning checks the latest certified row +before deleting all versions. Uncertified legacy retired rows stay visible and +cannot disappear through an age-based prune. Old reapers fail closed on the ninth +field, but old manual/pruning binaries must be stopped before the +[cells lifecycle baseline](cells.md#participant-lifecycle-baseline) is eligible. + Each bounded rebuild step holds the shared configuration Keeper key, renewed every 20 s (TTL 60 s). A contender takes over an expired key only when its holder's participant counts as stopped; an elapsed TTL alone is no proof. `loom diff --git a/docs/benchmarks/results.md b/docs/benchmarks/results.md index 07367d3688..fd940bf8ab 100644 --- a/docs/benchmarks/results.md +++ b/docs/benchmarks/results.md @@ -563,7 +563,7 @@ diff <(jq -S . /tmp/summary.json) <(jq -S . bench/code_ask_cache_threshold/summa ``` Embedding APIs are not bit-stable across provider revisions: expect cosines to agree to a -few thousandths. A threshold is robust only when its margin exceeds that drift. +few thousandths. A threshold's margin must exceed that drift. **With the guard on** (`sweep_with_guard`: paraphrase hits of 42 and false hits of 65 different pairs): @@ -1220,8 +1220,56 @@ a row for the state read and the `loom.tenancy` lookup. Loom's own writes are ma this predicts the ship gate (a `memory.set` p99 rise of at most 10 ms, a delivery pass at most 10% slower) holds by a wide margin; the cost lands on scripts and older builds writing a cell without the check. This measures the triggers on the statements, not a -whole `memory.set` through a server on a cell: the gate itself is measured end to end -with attach. +whole `memory.set` through a server on a cell; the end-to-end gate follows. + +A small local diagnostic on an attached first cell records 400 sequential `memory.set` +calls per run and 1,000-row statement inserts from marked and unmarked sessions, three +rounds with org triggers on and off. Its fixture has two server processes, a router, +Postgres 16 with a synchronous standby and ClickHouse in Docker on a shared host under +other load. These are diagnostic figures, not the end-to-end acceptance workload: + +| Local, three rounds | Triggers on | Triggers off | +| ----------------------------------- | ---------------------- | ---------------------- | +| `memory.set` p50 | 154.4, 99.3, 98.4 ms | 91.9, 101.2, 101.6 ms | +| `memory.set` p99 | 494.0, 291.7, 257.6 ms | 279.2, 244.7, 233.3 ms | +| 1,000-row insert, marked (median) | 11.0, 9.2, 8.5 ms | 9.7, 11.1, 10.6 ms | +| 1,000-row insert, unmarked (median) | 22.1, 21.4, 21.0 ms | 10.9, 9.6, 11.0 ms | + +The marked statement medians average 9.6 ms on and 10.5 ms off. A statement insert omits +the delivery worker's claim, embedding, publication and settlement work, so these figures +establish no delivery-throughput gate. The API p99 differences within each round are ++214.8, +47.0 and +24.3 ms; another small diagnostic has differences of −71.7, −10.0 and +−42.4 ms. Their opposing signs on the loaded host leave the API gate unresolved. Neither +end-to-end gate is established by these measurements. + +`bench/cell_trigger_gate.py` measures the acceptance workload on an independently owned, +attached scratch fixture with explicit fake providers, enforced mode and an already held +org. Its namespace uses `append_only` for independent writes: sibling values do not +rewrite one another's heads. The fixture verifies the stored policy before every arm, +and the row proof requires each exact returned version and active status. This measures +trigger cost for independent writes, not semantic conflict resolution. Each API run has +50 unmeasured warmups and 2,000 sequential measured requests with +4,096-byte values, with five runs per arm in counterbalanced order. Each delivery block +times five complete `pg_delivery.run_pass` calls at the unchanged configured batch size, +with five blocks per arm. The latency gate is `median(p99_on) - median(p99_off) <= 10 ms`; +the throughput gate is `1 - median(rows_per_second_on) / median(rows_per_second_off) <= 0.10`. +The driver retains every run and raw outcome; it does not substitute statement timings. + +```bash +PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py \ + --base-url --pg-dsn --token \ + --org --namespace --fixture-manifest \ + --launch-manifest --output +``` + +The fixture owner supplies the exact source, launch, configuration and ownership receipts, +excludes competing controllers and workers, and stops its processes before deleting its +exact scratch databases. The output path must not exist. The driver verifies source, +launch and configuration identities and the live fixture before native setup and every +arm, restores the org triggers within its cleanup bound and emits a final verdict only +after cleanup and artifact sync. It records the ownership-receipt digest supplied by +the fixture owner. +Source tests and the diagnostic figures above are not evidence that this workload passes. ## Reference computations diff --git a/docs/development/migrations.md b/docs/development/migrations.md index bbea754e87..0fcc560a45 100644 --- a/docs/development/migrations.md +++ b/docs/development/migrations.md @@ -157,20 +157,56 @@ operator step ([release process](./release-process.md#release-notes)). edited once merged, and every `CREATE`, `ALTER` or `DROP` of a table names `loom.`, except a rename's new name, where Postgres takes no schema (`tests/unit/test_storage_layout_rows.py`). A boot applies them under a Postgres -advisory lock, each file in one transaction with its ledger row. A migration waits at +advisory lock, each ordinary file in one transaction with its ledger row. A migration waits at most 5 s for a table lock (`lock_timeout`, then retried), so an `ALTER` never queues every query of the table behind a long transaction. A build serves a ledger that holds a later build's files, with a warning naming them; a ledger file it does not carry numbered at or below its own last file, or one whose checksum changed, refuses the boot. -Every file runs in one transaction, so a migration builds no index on a large table: the -build would hold off the table's writes, every commit answering 503, until it ends. Such -an index needs `CREATE INDEX CONCURRENTLY` outside a transaction, and a boot waiting on -the advisory lock deadlocks with it (a waiting session holds a snapshot the build waits -out): a change that adds such a file also takes the lock by retrying -`pg_try_advisory_lock`. When the first release that writes a table carries the file, the -table is small wherever the file runs, and its index may be built in the file's -transaction (`010_revision_delivered_at.sql`). +Ordinary files run in one transaction, so they build no index on a table that +already ships: a transactional build blocks writes until it ends. An index on a +table introduced by the same release may be transactional because that table is +small wherever the file runs (`010_revision_delivered_at.sql`). + +For an existing table, the shared Postgres runner accepts only this online file +form, in either the cell or router journal: + +```sql +-- loom:online-index loom_router.placements_by_cell_org ON loom_router.placements (cell, org) +CREATE INDEX CONCURRENTLY placements_by_cell_org ON loom_router.placements +USING btree (cell COLLATE "C", org COLLATE "C"); +``` + +The declaration and statement must name the same schema, table, index and ordered +columns. Only ordinary identifiers, a non-unique btree and C collations are +accepted. Expressions, predicates, included columns and additional statements +are refused before the ledger changes. This is not a general escape from +transactional migrations. + +The runner takes session advisory locks by retrying `pg_try_advisory_lock`, with +no transaction or snapshot between attempts. It releases the ordinary migration +key before acquiring a separate schema-scoped online-index key and running +concurrent DDL. A shipped blocking migrator can therefore finish its unchanged +journal while the index waits for old snapshots. After the build finishes, the +runner reacquires the ordinary key, records the file's checksum in the existing +ledger and commits a WAL-message fence. Later files wait for this ledger entry. + +Catalog validation checks the table, ordered columns and collations, btree +method, absence of constraints and expressions, and valid/ready state. A +matching incomplete index is dropped concurrently and rebuilt; a different +same-named object refuses without being dropped. A recorded but absent or +invalid index refuses as corruption. Even when every file is recorded, a build +carrying an online file validates its index and acknowledges a fresh WAL fence: +a primary-visible ledger row after a cancelled synchronous COMMIT is insufficient. +An interrupted build, drop or ledger write is retried from catalogs and the +existing ledger, without a separate job table. Cancellation terminates the +migration session before asyncpg can wait indefinitely on rollback. + +The router's dedicated migration connection has no statement timeout; its +serving pool retains its statement limit. The runner preserves a borrowed +connection's statement budget, including a stopped restore command's deadline. +Concurrent DDL retains the five-second lock timeout and can be interrupted and +retried without changing the file checksum. ## Schema dump and check diff --git a/docs/guides/tokens-and-grants.md b/docs/guides/tokens-and-grants.md index 2d1200b37e..ee3cc8bb73 100644 --- a/docs/guides/tokens-and-grants.md +++ b/docs/guides/tokens-and-grants.md @@ -26,7 +26,12 @@ loom mint-token --subject my-bot --org acme --namespaces support --roles writer `loom` ships with the server package and the Docker image, not with the SDK; in a container run `docker exec loom mint-token ...`. Minting needs the signing key, -`LOOM_JWT_SECRET`. A fresh clone gets a development secret from `.env.example`. +`LOOM_JWT_SECRET`, and access to the deployment's migrated stores. The command verifies +the paired stores and enrolls before reading the accepting cell state or signing. +Offline native minting and minting before migrations refuse. If only the keyring read +fails after successful enrollment, an ordinary mint can still sign with the root key. +Platform-admin minting also requires a fresh readable fleet and a recorded audit event. A fresh clone gets a development +secret from `.env.example`. `loom mint-token` and `loom doctor` warn while it is in use, and a deployment with auth on refuses to start with it (except the `make dev` stack). diff --git a/docs/operations/backup-and-recovery.md b/docs/operations/backup-and-recovery.md index 02de0a5c91..b2dfa6a12c 100644 --- a/docs/operations/backup-and-recovery.md +++ b/docs/operations/backup-and-recovery.md @@ -504,11 +504,11 @@ Bounded exceptions: - The events stream (budget violations) is drained on `SIGTERM`. A `SIGKILL` or OOM loses at most one flush interval of that telemetry, never audit or memory rows. -- Provider reconfiguration waits on a dead process until a process on its host proves it - gone, or, on Kubernetes, until the workers leader sees its pod gone. Elsewhere, one - whose host is gone too (a move to a new host) is retired by hand with - `loom participant retire --participant --yes` - ([provider reconfiguration](../architecture/provider-reconfiguration.md)). +- Provider reconfiguration waits on a process until guarded retirement or exact + physical-death proof with settled queries. On its trusted host, run + `loom participant retire --participant `. Pod deletion and heartbeat expiry + are insufficient. If its identity is unknown, keep the row until the + [physically quiescent lifecycle recovery](../architecture/cells.md#participant-lifecycle-baseline). - Rate-limit bucket fill is per host. It refills in seconds, so a new host starts with full buckets. diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md new file mode 100644 index 0000000000..451de55f3e --- /dev/null +++ b/docs/operations/cell-attach.md @@ -0,0 +1,160 @@ +# Attaching the first cell + +`loom cell attach` turns a prepared first cell (`loom cell prepare`, mode `recording`, +directory row `attaching`) into a `serving` one: it places every org the cell holds in the +router's directory, raises the cell's mode to `enforced` and creates its org triggers. This +page is the operator's procedure; the design is +[cells](../architecture/cells.md#attaching-a-cell). + +## Before attach + +Run attach at low load, as one run at a time, and not during a rollout, a provider +transition, a logical backup, a long erasure or a migration: each holds a transaction or a +snapshot long enough for step 4's wait to run out of `--wait`. Steps 2 and 4 run index skip +scans on the Postgres primary (each statement holds vacuum's horizon while it runs) and +read the `org` column of every ClickHouse table, which competes with serving load. + +Every item below is a precondition attach checks, or one it cannot check and relies on: + +1. Every router replica runs this release; every process and every CLI and CI image of + the cell runs it too ([upgrades](upgrades.md#upgrade-procedure), steps 16 and 17). +2. `LOOM_CELL` is set on every process of the cell. Every CLI and CI configuration that + writes for an org has `LOOM_CELL` set without `LOOM_CELL_ROUTER_TOKEN`: in `recording` + it then refuses only new orgs, so nothing has to change at the mode commit. With the + token removed, revoke the `INSERT` on `loom.tenancy` and the `UPDATE (since)` on + `loom.tenancy_mode` the CI role held for recording new orgs. +3. The ClickHouse writer credential is rotated after that rollout + ([upgrades](upgrades.md#upgrade-procedure), step 18). +4. No fingerprinted setting has changed since prepare, and none changes until attach ends: + a change gates every process against `loom.cell_self`. +5. `LOOM_SERVES_TRAFFIC=false` is set only on pods no load balancer routes to: attach + exempts them from the router check. +6. The cell's processes and the attach CLI share one `LOOM_INSTANCE_HEARTBEAT_SECONDS`: + attach judges which processes are live by its own value. +7. `LOOM_PG_DSN` points straight at Postgres, with no pooler of any mode, for the servers + and for attach: attach refuses a session that moves between backends or loses the + tenancy marker, and refuses a server whose heartbeat reports `marked` false. +8. SCIM is off: attach refuses with it on. +9. On a ClickHouse cluster of several hosts, attach's user exists on every host, and the + cluster has an interserver `` or a `remote_servers` user equal to attach's. + One replica that cannot sync blocks attach (`SYSTEM SYNC REPLICA … ON CLUSTER`): an + availability cost, not a safety one. +10. Only Loom's own roles and superusers hold the rights attach's tenancy-privilege check + counts (inserting a tenancy row, changing the mode, creating in a schema, triggers on + `loom` tables, owning them or their trigger functions, `session_replication_role`, + `ALTER SYSTEM` of `loom.tenancy_checked`, `pg_write_server_files`, + `pg_execute_server_program`), and no role that holds one, or owns what it counts, owns + a `SECURITY DEFINER` function another role can execute that does any of that. Attach + checks the catalog; it cannot read a function's body for intent, or a role that runs + code as the server's OS user. +11. `postgresql.conf`, `postgresql.auto.conf` and any pending `ALTER SYSTEM` entry, on the + primary and on every standby, set neither `loom.tenancy_checked` nor + `session_replication_role`. Attach sees a live entry on the primary (its unmarked + session reads it), never a standby's files. +12. Postgres has a synchronous standby attached (`synchronous_standby_names` set) and + `max_prepared_transactions = 0`; the database encoding is UTF8. +13. No org longer than 2048 bytes is in the cell's stores (below). `--dry-run` lists any. + +Attach's Postgres role needs the privileges of `pg_read_all_stats` +without `SET ROLE` (grant its membership with `INHERIT TRUE`), `USAGE` on schema +`loom`, `SELECT` on `loom.cell_first`, `loom.tenancy_mode`, `loom.cell_self`, +`loom.authority` and every org table, `UPDATE` on `loom.tenancy_mode`, `INSERT` on +`loom.tenancy`, `SELECT`, `INSERT`, `UPDATE` and `DELETE` on `loom.cell_probe`, `TEMPORARY` +on the database, `EXECUTE` on `pg_logical_emit_message`, and effective privileges of the owner +role of `loom.tenancy` and every guarded table without `SET ROLE` (grant their memberships +with `INHERIT TRUE`). Its ClickHouse user needs the grants the writer +check names (`SELECT` on `system.*` and on the configured databases, `SHOW USERS`, +`SHOW ROLES`, `SHOW ROW POLICIES`, `SHOW TABLES`, `SHOW COLUMNS`, +`SYSTEM FLUSH ASYNC INSERT QUEUE`, and on a cluster `SYSTEM SYNC REPLICA`, `CLUSTER` and +`READ ON REMOTE`), no settings profile or quota capping the rows or bytes it reads, and the +insert into `loom._migration_lock` that writes the probe nonce. The CLI needs write access +to the bucket's probe object and `LOOM_ROUTER_OPERATOR_TOKEN`. + +## Run it + +```bash +loom cell attach --cell --dry-run \ + --ch-allow-user --pg-allow-role +loom cell attach --cell \ + --ch-allow-user --pg-allow-role +``` + +`--dry-run` runs the refusals and step 2's scan, prints the count and a sample of the orgs +per source table, the ones it would skip, and whether the scan fit in half of +`--scan-budget`, and writes nothing; the router's own checks at `attach/begin` (the probe, +the hosts, the upstream) do not run. Name every ClickHouse user, user directory, object and +replica that may write the configured databases with `--ch-allow-user`, +`--ch-allow-user-directory`, `--ch-allow-object` and `--ch-allow-replica`, and every +Postgres role besides attach's own that may make an org held with `--pg-allow-role`; each +list is audited by the router. `--wait` (default 600 s) is the wall time the waits share; +every lock try runs at least once, with a lock wait of at least 100 ms, so `--wait 0` runs +one try of each wait with no retry. The final heartbeat checks get at least two heartbeats +plus the router timeout, with one extension for a process first seen booting; this floor +is not charged to `--wait`. `--scan-budget` (default 7200 s) bounds each full scan, and +step 2 must take at most half of it, since step 4 reads the same tables again. +Each Postgres mode or trigger try reserves cleanup time inside its client deadline. +On interruption its owner closes the session before cancelling transaction work. +A close that cannot complete uses transport abort and reports failure; that abort +does not establish that server locks or an unacknowledged commit have disappeared. +Retry the command to resolve an unknown commit through its durable state checks. + +A refusal names what to fix. Rerun attach after fixing it: every step is idempotent, and a +rerun resumes by the directory row and the cell's mode. On a `serving` row it verifies +that the connected authority, configured cell and bucket match the directory registration, +then requires `enforced` mode before printing that it is done. + +## The point of no return + +Step 3's mode commit is one-way. Before it, a refused attach leaves the cell `recording`, +its tenancy rows and its triggers in place (they pass every row in `recording`), and the +cell can be backed out (the first-cell branch of +[troubleshooting](troubleshooting.md#a-process-with-loom_cell-unset-is-gated)): +`loom cell remove` serves only a first cell that never left `recording`. After it, scoped +principals place their orgs live within seconds, every refusal leaves the cell `enforced` +and `attaching`, and the cell goes forward: fix the cause and rerun attach. Unsetting +`LOOM_CELL` then gates every process, an outage, not a rollback. A row with `placed > 0` +always goes forward. + +From `enforced` on, native minting requires the org to be held by this cell, with or +without `--platform-admin`. Admin account routes cannot bring in a new org (`WRONG_CELL`). +An already issued ordinary token scoped to its org places that org on its first request. +A direct call of the router's operator place route, or a live +placement with the cell's credential, cannot read the cell's Postgres and is accepted once +the directory mode is `enforced`; the cell's credential is the servers' alone. + +## Orgs attach does not place + +- **An org longer than 2048 bytes** refuses step 2, naming it, before any tenancy row is + written. One first found in step 4 is skipped and named. Remove such an org by hand, in + this order: its bucket objects, from the + keys of its rows in `loom.documents`, `loom.document_blobs` and `loom.blob_uploads`, so no + memory content is left outside erasure; then its rows in every table with an `org` column + of both ClickHouse databases and of Postgres schema `loom`, the erasure ledger tables + included; then its `loom.tenancy` row, on a session with `loom.tenancy_checked` set. +- **A name no request can carry** (not UTF-8, a NUL) found in a data table is skipped and + named; its rows stay outside placement. +- **An org found only in a log table** (`agent_events` and the tables derived from it, the + code search trace) is never placed: a log row keeps the org an unauthenticated caller + typed. Once that org is placed in another cell, its log rows here are outside its + erasure there. + +`loom doctor --stray-orgs` names the orgs the cell's stores hold without a tenancy row, +smallest tables first within `--stray-orgs-budget` (default 60 s), every table the budget +left unchecked and every table it could not read. Remove a stray org's rows by the same +procedure as an over-long org's. + +## After attach + +`loom doctor` runs attach's point-in-time checks again on every run: it warns on a Postgres +role outside the owner roles, the superusers and its own `--pg-allow-role` that can make an +org held or lower the mode (`cell_tenancy_privilege`). Attach exempts the role it runs as, +the operator's; doctor does not exempt the role it runs as, so a CI or monitoring role +holding such a right is named. It warns too on a role or database setting or a subscription +that passes an unchecked writer (`cell_unchecked_writers`). A grant or a writer added after +`serving` passes attach's checks, which ran once; doctor is how it is found. + +## Related + +- [Cells](../architecture/cells.md) +- [Troubleshooting: cells](troubleshooting.md#cells) +- [Upgrades](upgrades.md) diff --git a/docs/operations/cell-control.md b/docs/operations/cell-control.md new file mode 100644 index 0000000000..30892abf7f --- /dev/null +++ b/docs/operations/cell-control.md @@ -0,0 +1,146 @@ +# Cell control upgrades and recovery + +Cell control mutations require generation-aware clients and router handlers, and direct +access to the cell's Postgres. This page covers the runtime prerequisite, unknown +outcomes and registered-cell state cleanup. The protocol is described in +[cells](../architecture/cells.md#control-mutation-fencing). + +## Upgrade the control fleet + +Apply the router migrations with `loom router migrate`. The additive UUID column has a +server default and remains readable by N-1 data-plane code. Its volatile default may +rewrite the small cells table and takes a table lock; allow a bounded maintenance window. +Schema compatibility does not let old control programs participate in fencing. + +Before using generation-aware acceptance mutation, inventory and positively terminate +all old router control handlers and prepare, attach, restore, update, remove and manual +cleanup clients, including paused one-offs and outstanding requests. Replace every +launcher/image so those binaries cannot restart against the stores. Use exact process +and start identities, trustworthy evidence that the exact container ended, or independent +node fencing. A heartbeat expiry, missing pod object or one successful request to a router +replica does not prove this inventory. If the inventory is incomplete, stop/fence every +machine that could still run an old control program before resuming control traffic. +Compatible cell data-plane builds may continue serving during this procedure. Every +old router replica that could execute an operator handler must be positively retired; +continuing its data-plane routing alone does not remove that control capability. + +The fencing changes and their dependent CLI changes may be implemented and reviewed as +stacked concerns, merged in dependency order and shipped together. They require no +separate release cut. Running a protected operation requires the capable runtime and +completed control turnover procedure. Missing-generation requests refuse with +`CONTROL_UPGRADE_REQUIRED`; an absent claim route or incomplete operator snapshot tells +the CLI to upgrade the router replica. There is no compatibility fallback or operator +flag that proves a deployment completed its external process inventory. + +## Unknown outcomes + +A timeout, connection loss, cancelled COMMIT or lost acknowledgement ends the current +attempt. Keep the exact cell and router configuration, inspect the reported state and +rerun the command. The rerun must acquire the local attach lock and make a fresh claim +before it changes acceptance. A duplicate claim creates a fresh UUID; it never restores +a caller's earlier token. Do not replay a saved mutation with a copied current token. + +A client connection ending does not prove its Postgres backend ended. If the attach lock +remains held, inspect its exact holder with the query in +[troubleshooting](troubleshooting.md#a-process-with-loom_cell-unset-is-gated). Positively +end that exact owned backend, or wait until it ends. Do not terminate whichever backend +holds a mode-table lock: it may belong to another controller. No control run reconnects +to continue an uncertain attempt. + +## Clear a registered unattached cell's state + +Use this only for the troubleshooting page's eligible unattached cell with no placements, +or a first cell being backed out while its local mode is still `recording`. A first cell +in local `enforced` mode or any cell with placements goes forward. Complete the control +fleet turnover above, retain the existing stop/unset-`LOOM_CELL` procedure, and prevent +accepting or maintenance processes from restarting with the cell setting during cleanup. +Use the table-owning migration role. Never clear a restored copy beside its live source. + +Run the following with this cell's settings, including `LOOM_CELL`, `LOOM_PG_DSN`, its +ClickHouse and bucket configuration, `LOOM_CELL_ROUTER_URL` and the operator credential. +This read-only store pairing check must succeed before the helper claims or clears state. +It does not write participant rows or run the data-plane command gate. + +```python +import argparse +import asyncio + +import httpx + +from server.cli import cell as cell_cli +from server.cli import cell_control +from server.core import pg, pg_profile +from server.core.ch import get_client +from server.settings import settings + + +async def main(): + ch = get_client() + await ch.connect() + try: + recorded = await pg_profile.recorded_authority(ch) + if not recorded or recorded["authority"] != "postgres": + raise RuntimeError("the configured stores do not record Postgres authority") + pg.expect_deployment((str(recorded["deployment_id"]), int(recorded["epoch"]))) + url = cell_cli._cell_router(argparse.Namespace(router=None, cell=settings.cell)) + if not isinstance(url, httpx.URL): + raise RuntimeError("the router URL or operator credential was refused") + + async def call(path, body): + return await cell_control.request(url, settings.cell, None, path, body) + + async def cleanup(owner): + row = await cell_control.request(url, settings.cell, None, "", method="GET") + await cell_control.backout(owner, call, row) + + await cell_control.run("state cleanup", cleanup) + finally: + await ch.disconnect() + + +asyncio.run(main()) +``` + +The helper claims first, closes the cell using the existing limit operation, and requires +that operation's durable placement count to remain zero before any local state cleanup. +A queued placement either wins first and refuses cleanup, or observes the closed row and +cannot create a placement. An explicit `loom cell limit --open` now refuses for every +attaching cell, even one already open; a combined open/quota request changes neither +field. This behavior break keeps concurrent limit clients from reopening the cleanup +boundary. Close and quota-only edits remain available. The helper then compares and +deletes only the existing +`cell_first`, `tenancy_mode`, `cell_self` and `tenancy` rows, restores the first-row +triggers and drops only the existing org guard triggers, on the same locked backend. +It takes the migration lock for each bounded local transaction. Even an already-open +retry proves durability before reporting success. + +After the close is acknowledged, an abort leaves the cell closed. It never reopens +automatically or deletes a placement. +If a placement won, resume attach while closed: its existing-org placement batches bypass +the ordinary close/quota gate. Complete attach to `serving` before explicitly reopening +with `loom cell limit --open`. Otherwise rerun the cleanup with a fresh claim, then run +`loom cell remove --cell ` with the same cell settings. Removal now holds that +cell's direct Postgres lock and takes a fresh claim for both first and later cells. It +also requires a first cell's durable local state to read `open` before deleting the row. +Only then may the deployment be prepared again. + +## Restore the directory offline + +A directory backup can contain an earlier generation. Stop and positively terminate or +fence all control clients and router processes, including outstanding control requests, +and prevent restarts before restoring it. While control traffic remains stopped, execute +this on the restored directory and wait for its synchronous commit: + +```sql +BEGIN; +SET LOCAL lock_timeout = '5s'; +SET LOCAL statement_timeout = '10s'; +UPDATE loom_router.cells SET mutation_generation = gen_random_uuid(); +COMMIT; +``` + +Bound the client's complete transaction/COMMIT wait too. An uncertain outcome does not +permit APIs to resume: establish the restored writer's actual completion, retry the +rotation while still offline and obtain a known durable commit. Rotate every restored +row before restarting capable router/control binaries. Never copy generations from a +cell backup or rely on an audit retention window to fence restored requests. diff --git a/docs/operations/kubernetes.md b/docs/operations/kubernetes.md index fcdb9f07e2..5d6387fa01 100644 --- a/docs/operations/kubernetes.md +++ b/docs/operations/kubernetes.md @@ -228,17 +228,19 @@ graceful stop, and within the 30 s lease TTL when a pod or node dies. Every pod labelled `deploy:k8s:`, so a new leader runs the workers whose last pass is overdue rather than waiting out their interval again. -A pod that dies without stopping (node loss, eviction past the grace period) leaves its -provider participant row, which would hold the next embedding or model change from the -dashboard. With `retireLostPods` (the default) the workers leader retires it once its -heartbeat expired, no query of it runs and Kubernetes says the pod is gone: its object -deleted or replaced, or the pod `Failed` (evicted). That is a minute or two after a -deletion or an eviction, and after a node loss once Kubernetes deletes the node's pods -(five minutes of `NotReady` by default). It needs `get` on pods in the namespace (a Role -the chart creates for the release's service account) and that account's token, which only -the pods that may hold the workers lease mount. Without it, `/v1/health` names the row; -once the pod is gone, retire it with -`loom participant retire --participant --yes` ([replicas](./replicas.md#stopping-a-replica)). +A pod that dies without stopping leaves its provider participant row. A missing +or replaced pod object, a Failed/Succeeded API phase and a completed rollout do +not prove that an old process on a partitioned node stopped. Such rows remain +blockers until guarded shutdown or verified exact physical death with settled +queries. Loom requests no pod-read permission or service-account token for +participant retirement. The explicit `serviceAccount.automountToken` value stays +available for workloads that need a token for another integration. + +Use `loom participant retire --participant ` on the trusted host where its +exact PID/namespace/start identity is verifiable. If that evidence is unavailable, +use the [quiescent lifecycle recovery](../architecture/cells.md#participant-lifecycle-baseline) +after positively terminating or fencing every relevant runtime. API object +cleanup alone cannot authorize retirement. There is no migration Job. Every pod migrates at boot under a ClickHouse lease, so a Job would only repeat that step, and a pre-upgrade hook would run the new schema while @@ -427,7 +429,7 @@ NetworkPolicy whose one rule allows every pod and every address but those (none release: Helm refuses to take over an object it does not own, so the upgrade fails. - An engine that keeps address ranges off node IPs (Cilium's default) blocks host-network endpoints and the nodes' own addresses: NodeLocal DNSCache (so DNS), the EKS Pod - Identity agent, and the API server (which `retireLostPods` calls) where the engine + Identity agent, where the engine gives it an identity of its own. Turn on the engine's node matching, or set `guardMetadata: false` and allow them in your own policy. diff --git a/docs/operations/postgres.md b/docs/operations/postgres.md index d087a158c4..895086c474 100644 --- a/docs/operations/postgres.md +++ b/docs/operations/postgres.md @@ -44,6 +44,10 @@ pooler refuses or drops (its org triggers then refuse Loom's own writes for an o cell holds no row for, and the `loom.tenancy` guard its tenancy row for each newly placed org), and `loom doctor` names a session that lost it. Scripts and bulk loaders never set `session_replication_role = replica` on a cell: it skips the org triggers and the guard. +No role is granted `SET` or `ALTER SYSTEM` on `session_replication_role`, `ALTER SYSTEM` +on `loom.tenancy_checked`, or membership in `pg_write_server_files` or +`pg_execute_server_program` on a cell: `loom cell attach` refuses such a role, and +`loom doctor` warns on one ([attaching the first cell](cell-attach.md)). Every replica of the deployment, and its worker process, sets the same values. The full list, with the admission and delivery settings, is in the diff --git a/docs/operations/replicas.md b/docs/operations/replicas.md index f4c8729ef4..e635c0a490 100644 --- a/docs/operations/replicas.md +++ b/docs/operations/replicas.md @@ -101,14 +101,22 @@ it. ## Stopping a replica -Stop it gracefully (`docker stop`, SIGTERM): it marks its heartbeat row stopped -and retires its provider participant row. A forced kill leaves both; the -heartbeat row stops counting after three intervals. A process that restarts in -the same container retires the dead one's participant row itself. If a provider -reconfiguration waits on a dead process's row from elsewhere, retire it with the -`participant_id` its `/v1/health` reported, once you have confirmed it is gone -(the command refuses one whose heartbeat is fresh or whose queries still run): +Stop it gracefully (`docker stop`, SIGTERM). It first closes admission permanently, +stops its producers, and waits for actual request, worker and registered descendant +completion. It retires only after buffered, carried and server-side writes settle. +A canceled task that keeps executing or an exhausted shutdown budget leaves the +participant nonretired. A stopped or expired instance heartbeat is diagnostic. + +A trusted process on the same host and pid namespace can prove the exact old PID +exited or was reused, then retire it after all cross-replica queries settle: ```bash -loom participant retire --participant --yes +loom participant retire --participant ``` + +The command verifies retained boot, PID namespace and process-start identity; it +refuses remote or unknown identity evidence and live/uncountable queries. A +changed container namespace or Kubernetes API object is insufficient. An operator +assertion does not override these checks. For an inaccessible host, positively +terminate or fence the exact runtime before the documented +[quiescent lifecycle recovery](../architecture/cells.md#participant-lifecycle-baseline). diff --git a/docs/operations/troubleshooting.md b/docs/operations/troubleshooting.md index 3ede5f4702..1ca3ce0a40 100644 --- a/docs/operations/troubleshooting.md +++ b/docs/operations/troubleshooting.md @@ -295,7 +295,12 @@ each field means is in [monitoring](./monitoring.md#health-checks). - **The deployment is not meant to be a cell** (no directory row, or no router), or **an unattached later cell** (its row reads `is_first = false`, `state = 'attaching'` and `placed = 0`) that you are turning back into a single - deployment: clear the state in one transaction, in a psql session on `LOOM_PG_DSN` + deployment: for a registered cell, use the + [fenced state cleanup](cell-control.md#clear-a-registered-unattached-cells-state). + Do not run the SQL below for a registered cell. For a deployment whose directory + has no row, first positively stop/fence every control client and outstanding router + control request and prevent restarts; with no online acceptance controller, clear + the state in one transaction, in a psql session on `LOOM_PG_DSN` (`\connect` back to it after the check) as the role that owns schema `loom`'s tables (the role that migrates; altering a trigger needs the table's owner). The deployment is then `open`. An unattached later cell's directory row stays live until @@ -353,8 +358,9 @@ each field means is in [monitoring](./monitoring.md#health-checks). `state = 'attaching'` and `placed = 0`, whatever its `mode`, since an attach refused at its step-3 lock leaves it `enforced`; and `SELECT mode FROM loom.tenancy_mode` on its Postgres reads `recording`): back it out in this order. Unset `LOOM_CELL` on every - process, each of which then serves as in `recording`; run the state transaction and the - drop step above; then run `loom cell remove --cell ` with the router's operator + process, each of which then serves as in `recording`; run the + [fenced state cleanup](cell-control.md#clear-a-registered-unattached-cells-state), + with `LOOM_CELL` still set on that control client; then run `loom cell remove --cell ` with the router's operator credential and the cell's `LOOM_PG_DSN`, which refuses until that Postgres reads `open` and records the row's deployment id. Removing the row first would answer every new org 503, and clearing the state first would gate every process with `LOOM_CELL` set. The diff --git a/docs/operations/upgrades.md b/docs/operations/upgrades.md index 8caf8c789d..02e45f6c80 100644 --- a/docs/operations/upgrades.md +++ b/docs/operations/upgrades.md @@ -306,9 +306,10 @@ checkout. refused as a permission error (SQLSTATE 42501) rather than `LT001`. 16. Before a first cell's attach, every CLI and CI image runs a build with that state read, - and every CLI that writes for an org has `LOOM_CELL` set. No CLI sends a heartbeat, so - attach cannot check either, and a CLI of an older build is not refused on an `enforced` - cell ([cells](../architecture/cells.md#failure-modes)). + and every CLI that writes for an org has `LOOM_CELL` set. A CLI publishes no instance + or cell heartbeat for attach to read, so attach cannot check either, and a CLI of an + older build is not refused on an `enforced` cell + ([cells](../architecture/cells.md#failure-modes)). 17. Moving to a build whose first-cell processes read their cell's state from the router on a timer: every router replica (`loom router serve`) runs this build before any cell's @@ -320,6 +321,15 @@ checkout. answers them from one read per replica per `LOOM_ROUTER_DIRECTORY_CACHE_S` ([cells](../architecture/cells.md#the-heartbeats-keys)). +18. Before a first cell's attach, once `LOOM_CELL` is set on every process and every CLI and + CI configuration of the cell (step 16), rotate the ClickHouse credential they write + with: a new user or password for the cell's servers and CLIs, the old one dropped. A + CLI publishes no instance or cell heartbeat for attach to read, so attach cannot see a + job still configured with `LOOM_CELL` unset; the rotation cuts off any the rollout + missed before it can write an org's rows + past attach's last scan. Name the new user with `--ch-allow-user` + ([attaching the first cell](./cell-attach.md)). + While a migration applies, the old build keeps serving: every migration is additive. The rules that let the two builds serve side by side are in [rolling upgrades](../architecture/rolling-upgrades.md). diff --git a/docs/reference/cli.md b/docs/reference/cli.md index ff52c7d0fe..d89804a6be 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -38,7 +38,10 @@ Commands fall into two kinds: whatever `--token` says, `loom init` refuses before it signs anything up, and `loom call` exchanges the workload token itself. -`loom mint-token` needs only `LOOM_JWT_SECRET`: it signs a token locally. +`loom mint-token` signs locally with `LOOM_JWT_SECRET` after verifying its paired +stores and enrolling as a participant. It refuses when those stores cannot be verified +or enrollment fails, including offline and before the deployment is migrated. Run it +against the deployment's migrated stores. **Session commands** (`login`, `logout`, `whoami`, `call`, `token`) are the Python SDK's own (`loom.cli`; `python -m loom …` with only the SDK installed), so `loom` @@ -266,7 +269,7 @@ python scripts/codegen/docs_reference.py | `loom down` | Stop the local stack while preserving data; --delete-data performs a confirmed reset. | `--root`, `--file`, `--env-file`, `--delete-data`, `-y` | | `loom ch-setup` | Prepare a self-managed ClickHouse or a ClickHouse Cloud service for Loom: create the loom database and the read-only reader user, or print the SQL for an admin. | `--print-sql`, `--print-analyst-sql`, `--apply-analyst-sql` | | `loom participant` | The Loom processes a provider change waits for. | | -| `loom participant retire` | Retire the provider participant of a process that died without shutting down (its id is `participant_id` in that process's /v1/health). | `--participant`, `--yes` | +| `loom participant retire` | Verify exact physical death on this trusted host and retire its participant after all queries settle (the id is participant_id in /v1/health). | `--participant` | | `loom deployment` | The configuration every replica checks itself against. | | | `loom deployment publish` | Publish this env's configuration as the deployment's, replacing its root secret or LOOM_EMBEDDING_MODEL/DIM on purpose. | `--yes`, `--check` | | `loom settings` | Every setting's value in force, and overrides of the live ones. | | @@ -304,7 +307,7 @@ python scripts/codegen/docs_reference.py | `loom import-recall` | Import an external vector-recall agent-memory SQLite export into Loom. | `--sqlite`, `--org`, `--namespace`, `--agent`, `--base-url`, `--token`, `--dry-run` | | `loom import-brain` | Import a markdown agent 'brain' directory into Loom: MEMORY.md (always-injected HOT layer) → profile.set sections; memory/contexts/*.md (read-on-demand WARM layer) → knowledge.add docs. | `--dir`, `--org`, `--namespace`, `--agent`, `--base-url`, `--token`, `--dry-run` | | `loom claim-release` | Release a self-signup claim: revoke its grants, clear the owner, deactivate it. | `--org`, `--namespace`, `-y` | -| `loom mint-token` | Mint a JWT locally (requires LOOM_JWT_SECRET). | `--subject`, `--org`, `--namespaces`, `--roles`, `--platform-admin`, `--namespace-role`, `--ttl-days`, `--save`, `--read-scope`, `--read-user-id`, `--traffic-class`, `--consumer`, `--service-account`, `--user-token`, `--json` | +| `loom mint-token` | Mint a JWT locally (requires LOOM_JWT_SECRET and verified stores). | `--subject`, `--org`, `--namespaces`, `--roles`, `--platform-admin`, `--namespace-role`, `--ttl-days`, `--save`, `--read-scope`, `--read-user-id`, `--traffic-class`, `--consumer`, `--service-account`, `--user-token`, `--json` | | `loom revoke-token` | Revoke one token by jti or subject (writes loom.revoked_tokens through the revocation module; needs the configured ClickHouse). | `--jti`, `--subject`, `--list`, `--issued-before`, `--scim`, `--reason`, `--expires-at`, `--json` | | `loom sso` | Carry password accounts over to single sign-on. | | | `loom sso adopt` | Let SSO sign-ins reach the password accounts whose email is at --domain: each person then signs in with the identity provider to their own account, its namespaces and data. | `--domain`, `--org`, `--skip`, `--keep-tokens`, `--apply`, `--dry-run` | @@ -315,7 +318,7 @@ python scripts/codegen/docs_reference.py | `loom bot create` | Register a bot: its namespaces, and a key (--generate-key or --public-key) and/or a CI trust (--issuer, --audience, --claim). | `--name`, `--org`, `--namespace-role`, `--generate-key`, `--public-key`, `--issuer`, `--audience`, `--claim`, `--jwks-uri`, `--max-token-age`, `--max-access-token-ttl`, `--max-agents`, `--agent-claim`, `--platform-admin`, `--replace` | | `loom bot list` | Every bot, active or disabled. | | | `loom bot disable` | Refuse its proofs and revoke the tokens it holds. | `--name` | -| `loom doctor` | Diagnose pre-boot misconfig (Python version, settings, JWT, CH reachability) and print the problem list /v1/health carries, each with its fix. | `--json`, `--client`, `--server-url`, `--token` | +| `loom doctor` | Diagnose pre-boot misconfig (Python version, settings, JWT, CH reachability) and print the problem list /v1/health carries, each with its fix. | `--json`, `--client`, `--server-url`, `--pg-allow-role`, `--stray-orgs`, `--stray-orgs-budget`, `--token` | | `loom report` | Print the bug form's Environment field: versions and the `loom doctor` checks. | `--open` | | `loom drops` | Print the memory ingest drop ledger (loom.memory_drops) for a namespace, by stage/reason — what the pipeline discarded, truncated or gave up on. | `--namespace`, `--org`, `--hours` | | `loom namespaces-gc` | List (default) or erase throwaway namespaces — finished bench runs under the bench prefixes that nobody registered and nobody has written to for --idle-hours — in one list-scoped erasure saga per… | `--org`, `--prefix`, `--namespace`, `--exclude-prefix`, `--idle-hours`, `--include-registered`, `--erase`, `--no-purge`, `--skip-external`, `--batch`, `--show`, `--requester`, `--worker-pass` | @@ -330,6 +333,7 @@ python scripts/codegen/docs_reference.py | `loom cell which` | Print the lookup's cell and host for an org (the first cell if not placed). | `--org`, `--router` | | `loom cell remove` | Remove the directory row of a cell attaching with no placement. | `--cell`, `--router` | | `loom cell prepare` | Register this cell in the router's directory and write its rows. | `--cell`, `--host`, `--credential-sha256`, `--probe-url`, `--wait`, `--router` | +| `loom cell attach` | Attach the prepared first cell: place its orgs, serve. | `--cell`, `--wait`, `--scan-budget`, `--dry-run`, `--ch-allow-user`, `--ch-allow-user-directory`, `--ch-allow-object`, `--ch-allow-replica`, `--pg-allow-role`, `--router` | | `loom tracker-probe` | Read-only first contact with the tracker source: counts and newest timestamps per upstream table per credential, breaker states, the configured tracker namespaces. | | | `loom tracker-contract-check` | Read-only: does each tracker source's schema meet the record contract (docs/features/tracker/record-contract.md)? | `--source`, `--namespace`, `--json` | | `loom tracker-dupcheck-fit` | Refit tracker.duplicate_check's calibrated confidence for one tracker namespace now: replays the duplicate gold at filing time, fits, stores the fit. | `--namespace`, `--org`, `--negatives`, `--dry-run`, `--out` | diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index fdf11b688b..a0ba0f4094 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -1766,7 +1766,9 @@ Process shape, deployment, event stream and cost budgets. | `LOOM_SUPERVISOR_LEASE_TTL_S` | `int` | `30` | | auto mode: seconds a lease holder that stops renewing keeps the workers before another process takes them over (failover time). At least 24: below that a renewal blocked for its 2 s timeout leaves no retry before the holder must give up (server/workers/leader.py has the timings). | | `LOOM_KEEPER_SWEEP_INTERVAL_SECONDS` | `float` | `3600.0` | | Seconds between passes of the always-on sweep that deletes the KeeperMap keys (leases, claims, graph slots) a stopped process left behind. | | `LOOM_WORKERS_PROCESS` | `'child' \| 'inline'` | `'child'` | | auto mode: where the lease holder runs the workers. child = a separate process it starts and stops with the lease, so the passes do not share the event loop that serves requests; inline = in the serving process. | +| `LOOM_WORKERS_CHILD_RETIREMENT_TIMEOUT_S` | `float` | `1.0` | | At most this many seconds to certify a reaped worker child; also bounded by the remaining stop or restart delay. Expiry leaves its participant unretired. | | `LOOM_SHUTDOWN_READY_DELAY_S` | `float` | `15.0` | | Seconds a process keeps serving after SIGTERM while /v1/health/ready answers 503, so the load balancer stops routing to it before its port closes. Size it to the LB's probe interval × failure threshold. 0 = stop at once. | +| `LOOM_NATIVE_COMMAND_DRAIN_TIMEOUT_S` | `float` | `15.0` | | Seconds a native CLI command allows its producers, deferred writes and participant retirement to finish. Expiry leaves the participant unretired. | | `LOOM_BOOT_WARMUP_TIMEOUT_S` | `float` | `30.0` | | Seconds boot waits for the first load of the tracker registry and the runtime overrides, and the first publish of the dashboard bundle, before it serves; readiness stays false until the first two load. | | `LOOM_READY_REQUIRES_MIRRORS` | `bool` | `False` | | True: /v1/health/ready also waits until the mirror warm-up has queued its clones and while a clone of this host's git mirrors is running or queued (a failed clone does not hold it). Needs LOOM_CODE_MIRROR_WARMUP: without the warm-up nothing is cloned at boot and readiness does not wait for mirrors. Off by default: history ops answer `no_git_mirror` meanwhile. | | `LOOM_RUNTIME_OVERRIDES_REFRESH_SECONDS` | `float` | `10.0` | | Seconds between re-reads of loom.runtime_overrides (the admin rate-limit, latency-budget and live-setting overrides) in every process. | diff --git a/docs/reference/errors.md b/docs/reference/errors.md index 05d9907be6..17d199fa9f 100644 --- a/docs/reference/errors.md +++ b/docs/reference/errors.md @@ -432,6 +432,10 @@ Raised in `server/memory/domain/record_values.py`. Raised in `server/tracker/app/sources.py`. +#### `CONTROL_UPGRADE_REQUIRED` + +Raised in `server/router/app.py`. + #### `CREDENTIAL_NOT_READONLY` Raised in `server/tracker/app/sources.py`. @@ -752,6 +756,10 @@ Raised in `server/router/app.py`. Category `capacity`, retryable: yes. Raised in `server/router/app.py`. Category `dependency`, retryable: yes. +#### `PROCESS_STOPPING` + +Raised in `server/transport/model_maintenance.py`, `server/transport/retirement.py`. + #### `PROJECTION_BEHIND` HTTP 503, raised as `server.memory.app.pg_admission.ProjectionBehindError`. 503 — earlier commits in this namespace are not yet in ClickHouse, so a read of ClickHouse that must see every write the namespace committed would miss them. Category `capacity`, retryable: yes for an idempotent op, no for another op once it ran. diff --git a/frontend/e2e/assistant-launch.spec.ts b/frontend/e2e/assistant-launch.spec.ts index f62187437d..ee9f2ad1df 100644 --- a/frontend/e2e/assistant-launch.spec.ts +++ b/frontend/e2e/assistant-launch.spec.ts @@ -369,7 +369,9 @@ test("on a phone the card's citation number keeps its glyph's width, and each so await page.getByRole("button", { name: "Search", exact: true }).tap(); const search = page.getByRole("combobox", { name: "Search Loom" }); await search.pressSequentially("how often are invoices sent?"); - await expect(page.getByRole("option", { name: 'Ask AI about "how often are invoices sent?"' })).toBeVisible(); + const ask = page.getByRole("option", { name: 'Ask AI about "how often are invoices sent?"' }); + await expect(ask).toBeVisible(); + await expect(ask).toHaveAttribute("aria-selected", "true"); await page.keyboard.press("Enter"); const card = page.getByTestId("search-answer"); await expect(card.getByTestId("assistant-support")).toBeVisible(); diff --git a/llms.txt b/llms.txt index ed0d423d8f..311b6df634 100644 --- a/llms.txt +++ b/llms.txt @@ -105,6 +105,8 @@ Start with [the docs map](docs/index.md). To install Loom, a coding agent reads - [Exporting usage to a data warehouse](docs/operations/analytics-export.md): `loom export-analytics` writes one UTC day of usage and a snapshot of the namespaces to an S3 bucket as Parquet files, one file per entity per day, for a data warehouse to load. - [Backup and recovery](docs/operations/backup-and-recovery.md): A Loom backup is a ClickHouse backup, plus the document bucket while any catalog row names an object in it, and under the Postgres authority a backup of Postgres too. +- [Attaching the first cell](docs/operations/cell-attach.md): `loom cell attach` turns a prepared first cell (`loom cell prepare`, mode `recording`, directory row `attaching`) into a `serving` one: it places every org the cell holds in the router's directory, raises the cell's mode to `enforced` and… +- [Cell control upgrades and recovery](docs/operations/cell-control.md): Cell control mutations require generation-aware clients and router handlers, and direct access to the cell's Postgres. - [ClickHouse Cloud and Managed Postgres](docs/operations/clickhouse-cloud.md): This guide runs Loom on Kubernetes over two managed services: ClickHouse Cloud for projections, vectors, history and analytics, and ClickHouse Managed Postgres as the memory authority. - [Code index operations](docs/operations/code-index-operations.md): This page covers how to enable the code index workers, configure repositories and keep the index current; the code index runs only when an operator enables its workers and configures repositories. - [Deployment](docs/operations/deployment.md): Loom is one server image plus ClickHouse (and Postgres, under `LOOM_STORAGE_AUTHORITY=postgres`), with the document extractor service as an image of its own (container images). diff --git a/loom.example.yaml b/loom.example.yaml index 9dac2796bf..f115ace024 100644 --- a/loom.example.yaml +++ b/loom.example.yaml @@ -4977,11 +4977,19 @@ settings: # serves requests; inline = in the serving process. [restart] # workers_process: "child" + # At most this many seconds to certify a reaped worker child; also bounded by the + # remaining stop or restart delay. Expiry leaves its participant unretired. [restart] + # workers_child_retirement_timeout_s: 1.0 + # Seconds a process keeps serving after SIGTERM while /v1/health/ready answers 503, so # the load balancer stops routing to it before its port closes. Size it to the LB's # probe interval × failure threshold. 0 = stop at once. [restart] # shutdown_ready_delay_s: 15.0 + # Seconds a native CLI command allows its producers, deferred writes and participant + # retirement to finish. Expiry leaves the participant unretired. [restart] + # native_command_drain_timeout_s: 15.0 + # Seconds boot waits for the first load of the tracker registry and the runtime # overrides, and the first publish of the dashboard bundle, before it serves; # readiness stays false until the first two load. [restart] diff --git a/schemas/loom-settings.schema.json b/schemas/loom-settings.schema.json index 18ecaf4122..3a83e0f228 100644 --- a/schemas/loom-settings.schema.json +++ b/schemas/loom-settings.schema.json @@ -5475,6 +5475,12 @@ "type": "number", "description": "loom.namespace_write_marks is an in-process max per (org, namespace, plane), flushed as one INSERT this often." }, + "native_command_drain_timeout_s": { + "default": 15.0, + "exclusiveMinimum": 0, + "type": "number", + "description": "Seconds a native CLI command allows its producers, deferred writes and participant retirement to finish. Expiry leaves the participant unretired." + }, "oauth_access_token_ttl_seconds": { "default": 3600, "maximum": 86400, @@ -7133,6 +7139,12 @@ "type": "number", "description": "A pass with a failed leg on a worker whose interval is longer than this retries after this long, doubling with each further such pass, up to the interval: a weekly pass must not leave one namespace's transient failure standing for a week." }, + "workers_child_retirement_timeout_s": { + "default": 1.0, + "exclusiveMinimum": 0, + "type": "number", + "description": "At most this many seconds to certify a reaped worker child; also bounded by the remaining stop or restart delay. Expiry leaves its participant unretired." + }, "workers_process": { "default": "child", "enum": [ diff --git a/schemas/router_migrations/004_placements_by_cell_org.sql b/schemas/router_migrations/004_placements_by_cell_org.sql new file mode 100644 index 0000000000..774efeb883 --- /dev/null +++ b/schemas/router_migrations/004_placements_by_cell_org.sql @@ -0,0 +1,3 @@ +-- loom:online-index loom_router.placements_by_cell_org ON loom_router.placements (cell, org) +CREATE INDEX CONCURRENTLY placements_by_cell_org ON loom_router.placements +USING btree (cell COLLATE "C", org COLLATE "C"); diff --git a/schemas/router_migrations/005_control_generation.sql b/schemas/router_migrations/005_control_generation.sql new file mode 100644 index 0000000000..82baab882f --- /dev/null +++ b/schemas/router_migrations/005_control_generation.sql @@ -0,0 +1,3 @@ +-- A cell control claim invalidates requests from every earlier local lock owner. +-- Fresh values on insertion and each claim also fence removal/re-registration. +ALTER TABLE loom_router.cells ADD COLUMN mutation_generation uuid NOT NULL DEFAULT gen_random_uuid(); diff --git a/scripts/gates/cell_trigger_acceptance.py b/scripts/gates/cell_trigger_acceptance.py new file mode 100644 index 0000000000..d43d81d4f0 --- /dev/null +++ b/scripts/gates/cell_trigger_acceptance.py @@ -0,0 +1,1018 @@ +"""Run the fixed trigger acceptance workload on fresh, job-owned stores.""" + +from __future__ import annotations + +import argparse +import datetime +import hashlib +import json +import os +import re +import shutil +import signal +import socket +import subprocess +import sys +import time +import uuid +from pathlib import Path + +GIB = 1024**3 +SUPPORT = Path(__file__).resolve().parent / "testsupport/cell_trigger" + +_CLICKHOUSE_READY = """import http.client +import sys + +connection = http.client.HTTPConnection("127.0.0.1", int(sys.argv[1]), timeout=2) +ready = False +try: + connection.request( + "POST", "/", body="SELECT 1", + headers={"X-ClickHouse-User": "loom", "X-ClickHouse-Key": "loomdev"}, + ) + response = connection.getresponse() + ready = response.status == 200 and response.read(3) == b"1\\n" +except (OSError, http.client.HTTPException): + pass +finally: + connection.close() +if ready: + print("1") +""" + + +def require(ok: object, message: str) -> None: + if not ok: + raise RuntimeError(message) + + +def sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def failure_site(error: BaseException | None) -> dict | None: + if error is None: + return None + site = {"file": Path(__file__).name, "line": 0, "stage": "unknown"} + trace = error.__traceback__ + while trace is not None: + if ( + Path(trace.tb_frame.f_code.co_filename).resolve() == Path(__file__).resolve() + and trace.tb_frame.f_code.co_name != "require" + ): + site.update(line=trace.tb_lineno, stage=trace.tb_frame.f_code.co_name[:64]) + trace = trace.tb_next + return site + + +def source_hashes(root: Path) -> dict: + paths = { + name: SUPPORT / name + for name in ( + "fixture.py", + "native.py", + "tls_proxy.py", + "capture.py", + "owned_children.py", + "protocol.json", + ) + } + paths.update( + driver=root / "bench/cell_trigger_gate.py", + entrypoint=Path(__file__).resolve(), + workflow=root / ".github/workflows/docker-smoke.yml", + ) + return {name: sha(path) for name, path in paths.items()} + + +def write(path: Path, value: object, *, replace: bool = False) -> None: + target = path.with_name(path.name + ".tmp") if replace else path + with target.open("x", encoding="utf-8") as stream: + os.chmod(target, 0o600) + json.dump(value, stream, indent=2) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + if replace: + os.replace(target, path) + + +def source(root: Path) -> dict: + def git(*args: str) -> str: + return subprocess.check_output(["git", *args], cwd=root, text=True, timeout=10).strip() + + require(not git("status", "--porcelain"), "Acceptance requires a clean checkout") + return {"head": git("rev-parse", "HEAD"), "tree": git("rev-parse", "HEAD^{tree}")} + + +def resources(path: Path, available_gib: int = 10) -> None: + available = next( + int(line.split()[1]) + for line in Path("/proc/meminfo").read_text().splitlines() + if line.startswith("MemAvailable:") + ) + require(available >= available_gib * 1024**2, "Acceptance available RAM floor") + require(shutil.disk_usage(path).free >= 16 * GIB, "Acceptance requires 16 GiB free disk") + + +def ports(count: int) -> list[int]: + sockets = [] + try: + for _ in range(count): + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + sockets.append(sock) + return [sock.getsockname()[1] for sock in sockets] + finally: + for sock in sockets: + sock.close() + + +def container_matches(actual: dict, expected: dict, label: str) -> bool: + return ( + actual.get("Id") == expected["id"] + and actual.get("Created") == expected["created"] + and actual.get("Config", {}).get("Labels", {}).get("loom.ci.trigger") == label + ) + + +def native_cleanup_complete(folder: Path, fixture_id: str) -> None: + fixture = folder / "fixture" + if not fixture.exists(): + return # Native validation never began; only the creator's stores exist. + receipt = fixture / "outer-result.json" + require( + not receipt.is_symlink() + and receipt.stat().st_uid == os.geteuid() + and receipt.stat().st_size <= 65536, + "Private native cleanup receipt", + ) + value = json.loads(receipt.read_text()) + require( + value.get("schema") == 2 + and value.get("fixture_id") == fixture_id + and value.get("native_cleanup_complete") is True, + "Native retirement, scratch cleanup or child reap remains incomplete", + ) + + +class Stores: + def __init__(self, root: Path, folder: Path, fixture_id: str, run: dict) -> None: + self.root, self.folder, self.fixture_id, self.run = root, folder, fixture_id, run + self.items: list[dict] = [] + self.network = None + self.prefix = "loom-trigger-" + fixture_id[:12] + self.cancelled = 0 + self.terminal = False + self.phase = "store_creation" + self.creation_deadline = time.monotonic() + 900 + self.env = { + "PATH": os.environ["PATH"], + "HOME": str(folder), + "LANG": "C.UTF-8", + "DOCKER_CONFIG": str(folder / "docker"), + } + + def save(self) -> None: + write( + self.folder / "stores.json", + { + "fixture_id": self.fixture_id, + "ci_run": self.run, + "items": self.items, + "network": self.network, + }, + replace=True, + ) + + def interrupted(self, signum: int, _frame: object) -> None: + self.cancelled = self.cancelled or signum + if self.terminal: + raise SystemExit(128 + self.cancelled) + + def command(self, *argv: str, timeout: int = 120, cleanup: bool = False) -> str: + if not cleanup: + require(not self.cancelled, "Acceptance creation cancelled") + resources(self.folder, 4) + remaining = self.creation_deadline - time.monotonic() + require(remaining > 0, "Owned store creation deadline") + timeout = min(timeout, max(1, int(remaining))) + result = subprocess.run(argv, env=self.env, capture_output=True, text=True, timeout=timeout) + require(len(result.stdout) + len(result.stderr) <= 8 * 1024**2, "Creator output bound") + require(result.returncode == 0, "Owned store command failed: " + argv[0]) + return result.stdout.strip() + + def inspect(self, identity: str, *, cleanup: bool = False) -> dict: + rows = json.loads(self.command("docker", "inspect", identity, cleanup=cleanup)) + require(len(rows) == 1, "Exactly one owned container") + return rows[0] + + def create(self, role: str, args: list[str]) -> dict: + name = self.prefix + "-" + role + cidfile = self.folder / (role + ".cid") + # The intent precedes Docker. A failed create is resolved by its unique label. + intent = {"role": role, "name": name, "id": None, "created": None} + self.items.append(intent) + self.save() + self.command( + "docker", + "create", + "--name", + name, + "--cidfile", + str(cidfile), + "--label", + "loom.ci.trigger=" + self.fixture_id, + "--label", + "loom.ci.run=" + self.run["run_id"], + *args, + ) + identity = cidfile.read_text().strip() + require(re.fullmatch(r"[a-f0-9]{64}", identity), "Actual created container ID") + actual = self.inspect(identity) + require( + actual["Name"] == "/" + name + and actual["Config"]["Labels"]["loom.ci.trigger"] == self.fixture_id, + "Created container identity", + ) + intent.update(id=identity, created=actual["Created"]) + self.save() + write( + self.folder / (role + "-creator.json"), + { + "task": "cell-trigger-acceptance", + "ci_run": self.run, + "container_id": identity, + "created_at": actual["Created"], + }, + ) + resources(self.folder) + self.command("docker", "start", identity) + return intent + + def wait(self, phase: str, probe, seconds: int = 120) -> None: + require( + phase in {"primary_tcp", "standby_sync", "clickhouse_query"}, "Known readiness phase" + ) + self.phase = phase + end = time.monotonic() + seconds + while True: + require(not self.cancelled, "Owned store readiness cancelled") + try: + if probe(): + return + except (RuntimeError, subprocess.SubprocessError, ValueError): + pass + require(time.monotonic() < end, "Owned store readiness deadline") + time.sleep(1) + + def _clickhouse_ready(self, port: int) -> bool: + return ( + self.command(sys.executable, "-I", "-S", "-c", _CLICKHOUSE_READY, str(port), timeout=2) + == "1" + ) + + def failure_diagnostics(self) -> dict: + observations = [] + statuses = {"created", "running", "paused", "restarting", "removing", "exited", "dead"} + for item in self.items: + row = {"role": item["role"], "available": False} + observations.append(row) + if item["id"] is None: + continue + try: + records = json.loads( + self.command("docker", "inspect", item["id"], timeout=5, cleanup=True) + ) + require( + len(records) == 1 and container_matches(records[0], item, self.fixture_id), + "Diagnostic container ownership changed", + ) + state = records[0]["State"] + row.update( + available=True, + status=state["Status"] if state["Status"] in statuses else "unknown", + exit_code=state["ExitCode"] if type(state["ExitCode"]) is int else None, + oom_killed=state["OOMKilled"] if type(state["OOMKilled"]) is bool else None, + ) + if item["role"] in {"pg-primary", "pg-standby"} and state["Running"] is True: + value = self.command( + "docker", + "exec", + item["id"], + "psql", + "-U", + "loom", + "-d", + "loom", + "-Atc", + "SELECT CASE WHEN pg_is_in_recovery() THEN 'standby' ELSE 'primary' END " + "|| ':' || COALESCE((SELECT sync_state FROM pg_stat_replication " + "WHERE application_name='loom_standby'), 'absent')", + timeout=5, + cleanup=True, + ) + recovery, sync = value.split(":") + if recovery in {"primary", "standby"} and sync in { + "sync", + "async", + "potential", + "quorum", + "absent", + }: + row["pg_state"] = {"recovery": recovery == "standby", "standby_sync": sync} + except ( + OSError, + RuntimeError, + subprocess.SubprocessError, + ValueError, + KeyError, + ) as error: + row["diagnostic_error_type"] = type(error).__name__ + return {"phase": self.phase, "containers": observations} + + def start(self, ch_image: str) -> tuple[dict, dict, dict, list[int]]: + selected = ports(8) + ch_port, native_port, pg_port, standby_port = selected[:4] + self.command("docker", "pull", ch_image, timeout=300) + self.command("docker", "pull", "postgres:16", timeout=300) + network_name = self.prefix + "-net" + self.network = {"name": network_name, "id": None} + self.save() + self.network["id"] = self.command( + "docker", + "network", + "create", + "--label", + "loom.ci.trigger=" + self.fixture_id, + network_name, + ) + self.save() + memory_config = self.folder / "memory.xml" + memory_config.write_text( + "4831838208\n" + ) + memory_config.chmod(0o644) + ch_data = self.folder / "clickhouse-data" + ch_data.mkdir(mode=0o777) + ch_data.chmod(0o777) + # Matching ownership avoids recursive chown through the read-only dictionary mount. + ch_owner = ch_data.stat() + mounts = [ + (ch_data, "/var/lib/clickhouse", False), + (self.root / "deploy/lemmagen3", "/var/lib/clickhouse/lemmagen3", True), + (self.folder / "memory.xml", "/etc/clickhouse-server/config.d/memory.xml", True), + ] + for filename, section in ( + ("lemmatizers.xml", "config.d"), + ("nlp_user_settings.xml", "users.d"), + ("keeper.xml", "config.d"), + ("access_control.xml", "config.d"), + ): + mounts.append( + ( + self.root / "deploy/clickhouse-config" / filename, + "/etc/clickhouse-server/" + section + "/" + filename, + True, + ) + ) + for filename, section in ( + ("log_queries_readonly.xml", "users.d"), + ("custom_settings_prefixes.xml", "config.d"), + ): + mounts.append( + ( + self.root / "dev/clickhouse-ci" / filename, + "/etc/clickhouse-server/" + section + "/" + filename, + True, + ) + ) + ch_args = [ + "--memory", + str(5 * GIB), + "--memory-swap", + str(5 * GIB), + "-p", + f"127.0.0.1:{ch_port}:8123", + "-p", + f"127.0.0.1:{native_port}:9000", + "-e", + "CLICKHOUSE_DB=loom", + "-e", + f"CLICKHOUSE_UID={ch_owner.st_uid}", + "-e", + f"CLICKHOUSE_GID={ch_owner.st_gid}", + "-e", + "CLICKHOUSE_USER=loom", + "-e", + "CLICKHOUSE_PASSWORD=loomdev", + "-e", + "CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1", + "--ulimit", + "nofile=262144:262144", + ] + for host, target, readonly in mounts: + ch_args += ["-v", str(host) + ":" + target + (":ro" if readonly else "")] + ch = self.create("clickhouse", [*ch_args, ch_image]) + pg_data = self.folder / "pg-data" + pg_data.mkdir() + pg = self.create( + "pg-primary", + [ + "--network", + network_name, + "--network-alias", + "primary", + "--memory", + str(GIB), + "--memory-swap", + str(GIB), + "-p", + f"127.0.0.1:{pg_port}:5432", + "-v", + str(pg_data) + ":/var/lib/postgresql/data", + "-e", + "POSTGRES_USER=loom", + "-e", + "POSTGRES_PASSWORD=loomdev", + "-e", + "POSTGRES_DB=loom", + "postgres:16", + "-c", + "wal_level=logical", + "-c", + "max_wal_senders=10", + "-c", + "max_replication_slots=10", + "-c", + "synchronous_commit=on", + "-c", + "shared_buffers=128MB", + ], + ) + psql = [ + "docker", + "exec", + pg["id"], + "psql", + "-U", + "loom", + "-d", + "loom", + "-Atc", + ] + self.wait( + "primary_tcp", + lambda: self.command( + "docker", + "exec", + pg["id"], + "pg_isready", + "-U", + "loom", + "-d", + "loom", + "-h", + "127.0.0.1", + ), + ) + self.phase = "replication_configuration" + self.command(*psql, "CREATE ROLE replicator WITH REPLICATION LOGIN PASSWORD 'replicator'") + self.command( + "docker", + "exec", + pg["id"], + "bash", + "-c", + "echo 'host replication replicator all scram-sha-256' >> \"$PGDATA/pg_hba.conf\"", + ) + self.command(*psql, "ALTER SYSTEM SET synchronous_standby_names = 'FIRST 1 (loom_standby)'") + self.command(*psql, "SELECT pg_reload_conf()") + standby_data = self.folder / "standby-data" + standby_data.mkdir(mode=0o777) + standby_data.chmod(0o777) + standby_script = self.folder / "standby.sh" + standby_script.write_text("""#!/usr/bin/env bash +set -euo pipefail +export PGDATA=/var/lib/postgresql/data +if [ "$(id -u)" = 0 ]; then + chown postgres:postgres "$PGDATA" + exec gosu postgres bash "$0" +fi +pg_basebackup -h primary -U replicator -D "$PGDATA" -R -X stream +connection="host=primary user=replicator password=replicator" +echo "primary_conninfo = '$connection application_name=loom_standby'" \ + >> "$PGDATA/postgresql.auto.conf" +chmod 700 "$PGDATA" +exec postgres -D "$PGDATA" -c shared_buffers=128MB +""") + standby_script.chmod(0o644) + standby = self.create( + "pg-standby", + [ + "--network", + network_name, + "--memory", + str(GIB), + "--memory-swap", + str(GIB), + "-p", + f"127.0.0.1:{standby_port}:5432", + "-v", + str(standby_data) + ":/var/lib/postgresql/data", + "-e", + "PGPASSWORD=replicator", + "-v", + str(standby_script) + ":/standby.sh:ro", + "--entrypoint", + "bash", + "postgres:16", + "/standby.sh", + ], + ) + self.wait( + "standby_sync", + lambda: ( + self.command( + *psql, + ( + "SELECT sync_state FROM pg_stat_replication WHERE " + "application_name='loom_standby'" + ), + ) + == "sync" + ), + ) + self.wait("clickhouse_query", lambda: self._clickhouse_ready(ch_port)) + return ch, pg, standby, selected + + def observation(self, item: dict, role: str) -> dict: + actual = self.inspect(item["id"]) + require( + container_matches(actual, item, self.fixture_id) and actual["State"]["Running"], + "Actual job-owned running store", + ) + pid = actual["State"]["Pid"] + fields = Path("/proc", str(pid), "stat").read_text().rsplit(")", 1)[1].split() + creator = self.folder / (item["role"] + "-creator.json") + mappings = [ + { + "host": value["HostIp"], + "host_port": int(value["HostPort"]), + "internal_port": int(key.split("/")[0]), + "protocol": key.split("/")[1], + } + for key, values in actual["NetworkSettings"]["Ports"].items() + for value in values or [] + ] + disk_path = ( + self.folder + / { + "clickhouse": "clickhouse-data", + "pg_primary": "pg-data", + "pg_standby": "standby-data", + }[role] + ) + return { + "role": role, + "container_id": item["id"], + "container_name": actual["Name"].lstrip("/"), + "created_at": actual["Created"], + "owner_evidence": {"path": str(creator), "sha256": sha(creator)}, + "host_pid": pid, + "start_ticks": fields[19], + "boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + "uid": Path("/proc", str(pid)).stat().st_uid, + "ports": mappings, + "memory_limit_bytes": actual["HostConfig"]["Memory"], + "server_limit_bytes": 9 * GIB // 2 if role == "clickhouse" else 0, + "shared_buffers_bytes": 128 * 1024**2 if role != "clickhouse" else 0, + "retained": False, + "disk_path": str(disk_path), + } + + def cleanup(self) -> list[str]: + errors = [] + for item in reversed(self.items): + try: + if item["id"] is not None and not self.command( + "docker", + "ps", + "-a", + "--no-trunc", + "--filter", + "id=" + item["id"], + "--format", + "{{.ID}}", + cleanup=True, + ): + item["removed"] = True + self.save() + continue + if item.get("removed"): + require( + not self.command( + "docker", + "ps", + "-a", + "--no-trunc", + "--filter", + "id=" + item["id"], + "--format", + "{{.ID}}", + cleanup=True, + ), + "Previously removed container reappeared", + ) + continue + # Unique name plus exact random label resolves a partial create only. + actual = self.inspect(item["id"] or item["name"], cleanup=True) + if item["id"] is None: + require( + actual["Name"] == "/" + item["name"] + and actual["Config"]["Labels"].get("loom.ci.trigger") == self.fixture_id, + "Partial create belongs to this job", + ) + item.update(id=actual["Id"], created=actual["Created"]) + require( + container_matches(actual, item, self.fixture_id), "Cleanup ownership changed" + ) + self.command("docker", "rm", "--force", "--volumes", item["id"], cleanup=True) + remaining = self.command( + "docker", + "ps", + "-a", + "--no-trunc", + "--filter", + "id=" + item["id"], + "--format", + "{{.ID}}", + cleanup=True, + ) + require(not remaining, "Owned container remains after cleanup") + item["removed"] = True + self.save() + except ( + OSError, + RuntimeError, + subprocess.SubprocessError, + ValueError, + KeyError, + ) as error: + errors.append(type(error).__name__ + ":" + item["role"]) + if self.network is not None: + try: + if self.network["id"] is not None and not self.command( + "docker", + "network", + "ls", + "--no-trunc", + "--filter", + "id=" + self.network["id"], + "--format", + "{{.ID}}", + cleanup=True, + ): + self.network["removed"] = True + self.save() + if self.network.get("removed"): + require( + not self.command( + "docker", + "network", + "ls", + "--no-trunc", + "--filter", + "id=" + self.network["id"], + "--format", + "{{.ID}}", + cleanup=True, + ), + "Previously removed network reappeared", + ) + else: + rows = json.loads( + self.command( + "docker", + "network", + "inspect", + self.network["id"] or self.network["name"], + cleanup=True, + ) + ) + require( + len(rows) == 1 + and rows[0]["Labels"].get("loom.ci.trigger") == self.fixture_id + and (self.network["id"] is None or rows[0]["Id"] == self.network["id"]), + "Network cleanup ownership changed", + ) + self.network["id"] = rows[0]["Id"] + self.command("docker", "network", "rm", rows[0]["Id"], cleanup=True) + self.network["removed"] = True + self.save() + except ( + OSError, + RuntimeError, + subprocess.SubprocessError, + ValueError, + KeyError, + ) as error: + errors.append(type(error).__name__ + ":network") + write( + self.folder / "store-cleanup.json", + { + "removed": not errors, + "errors": errors, + "containers": [{"id": item["id"], "role": item["role"]} for item in self.items], + }, + replace=True, + ) + return errors + + +def dependency_receipt(root: Path, output: Path) -> dict: + # The CI job uses the same ARM64 CPython ABI as the reviewed codec contract. + sys.path.insert(0, str(SUPPORT)) + from fixture import CODEC_FILES, CODEC_VERSIONS + + value = { + "schema": 2, + "root": str(root), + "python": str(root / ".venv/bin/python"), + "python_sha256": sha(root / ".venv/bin/python"), + "lockfile_sha256": sha(root / "uv.lock"), + "versions": CODEC_VERSIONS, + "codec_sources": { + name: {"path": str(root / ".venv" / name), "sha256": sha(root / ".venv" / name)} + for name in CODEC_FILES + }, + } + write(output, value) + return {"path": str(output), "sha256": sha(output)} + + +def publish( + folder: Path, + destination: Path, + error: BaseException | None, + *, + phase: str, + diagnostic_error_type: str | None, +) -> None: + destination.mkdir(mode=0o700) + # Raw native logs/settings contain bearer and bootstrap credentials: never upload them. + fixture = folder / "fixture" + for name in ( + "outer-result.json", + "driver.jsonl", + "provenance.json", + "final-retirement.json", + "teardown.json", + ): + path = fixture / name + if path.is_file(): + shutil.copyfile(path, destination / name) + for name in ("source-before.json", "source-after.json", "startup-diagnostics.json"): + if (folder / name).is_file(): + shutil.copyfile(folder / name, destination / name) + intent_output = destination / "delivery-intent" + for arm in range(10): + for suffix in (".jsonl", "-receipt.json"): + path = fixture / "delivery-intent" / (f"arm-{arm}" + suffix) + if path.is_file() and not path.is_symlink(): + intent_output.mkdir(mode=0o700, exist_ok=True) + shutil.copyfile(path, intent_output / path.name) + if (folder / "store-cleanup.json").exists(): + shutil.copyfile(folder / "store-cleanup.json", destination / "store-cleanup.json") + write( + destination / "result.json", + { + "accepted": error is None, + "error_type": type(error).__name__ if error is not None else "", + "failure": failure_site(error), + "phase": phase, + "diagnostic_error_type": diagnostic_error_type, + "utc": datetime.datetime.now(datetime.UTC).isoformat(), + }, + ) + + +def cleanup_only(root: Path, run: dict) -> int: + state = Path(os.environ["RUNNER_TEMP"]) / "cell-trigger-state.json" + if not state.exists(): + return 0 # The creator never published an intent and could not create a store. + require( + not state.is_symlink() + and state.stat().st_uid == os.geteuid() + and state.stat().st_size < 65536, + "Owned CI cleanup pointer", + ) + pointer = json.loads(state.read_text()) + fixture_id = pointer["fixture_id"] + require(str(uuid.UUID(fixture_id)) == fixture_id, "Cleanup fixture UUID") + folder = Path(os.environ["RUNNER_TEMP"]) / ("cell-trigger-" + fixture_id) + require( + folder.resolve() == folder + and pointer == {"fixture_id": fixture_id, "folder": str(folder), "ci_run": run}, + "Cleanup pointer belongs to this exact CI attempt", + ) + ledger = folder / "stores.json" + require( + not ledger.is_symlink() + and ledger.stat().st_uid == os.geteuid() + and ledger.stat().st_size < 65536, + "Owned creator ledger", + ) + data = json.loads(ledger.read_text()) + require(data["fixture_id"] == fixture_id and data["ci_run"] == run, "Current creator run") + stores = Stores(root, folder, fixture_id, run) + roles = {"clickhouse", "pg-primary", "pg-standby"} + require( + len(data["items"]) <= 3 + and len({item["role"] for item in data["items"]}) == len(data["items"]), + "Bounded exact creator roles", + ) + for item in data["items"]: + require( + item["role"] in roles + and item["name"] == stores.prefix + "-" + item["role"] + and (item["id"] is None or re.fullmatch(r"[a-f0-9]{64}", item["id"])), + "Exact recorded container identity", + ) + network = data["network"] + require(network is None or network["name"] == stores.prefix + "-net", "Exact recorded network") + stores.items, stores.network = data["items"], network + native_cleanup_complete(folder, fixture_id) + errors = stores.cleanup() + destination = Path(os.environ["RUNNER_TEMP"]) / "cell-trigger-artifacts" + destination.mkdir(mode=0o700, exist_ok=True) + shutil.copyfile(folder / "store-cleanup.json", destination / "store-cleanup.json") + return int(bool(errors)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--clickhouse-image") + parser.add_argument("--cleanup-only", action="store_true") + args = parser.parse_args() + require( + os.environ.get("GITHUB_ACTIONS") == os.environ.get("CI") == "true", "GitHub Actions only" + ) + root = Path(os.environ["GITHUB_WORKSPACE"]).resolve() + run = { + "run_id": os.environ["GITHUB_RUN_ID"], + "attempt": os.environ["GITHUB_RUN_ATTEMPT"], + "repository": os.environ["GITHUB_REPOSITORY"], + } + if args.cleanup_only: + return cleanup_only(root, run) + require(args.clickhouse_image, "Resolved ClickHouse release image") + require( + Path(sys.executable) == root / ".venv/bin/python" and sys.version_info[:2] == (3, 12), + "Use the locked checkout Python 3.12", + ) + before = source(root) + resources(Path(os.environ["RUNNER_TEMP"])) + fixture_id = str(uuid.uuid4()) + folder = Path(os.environ["RUNNER_TEMP"]) / ("cell-trigger-" + fixture_id) + folder.mkdir(mode=0o700) + os.umask(0o077) + hashes_before = source_hashes(root) + write(folder / "source-before.json", {**before, "source_hashes": hashes_before, "ci_run": run}) + stores = Stores(root, folder, fixture_id, run) + stores.save() + write( + Path(os.environ["RUNNER_TEMP"]) / "cell-trigger-state.json", + {"fixture_id": fixture_id, "folder": str(folder), "ci_run": run}, + ) + for signum in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP): + signal.signal(signum, stores.interrupted) + error = None + diagnostic_error_type = None + cleanup_complete = False + try: + ch, primary, standby, selected = stores.start(args.clickhouse_image) + stores.phase = "dependency_metadata" + resources(folder) + dependencies = dependency_receipt(root, folder / "dependencies.json") + from fixture import runtime + + spec = { + "schema": 2, + "fixture_id": fixture_id, + "root": str(root), + **before, + "python": str(root / ".venv/bin/python"), + "python_sha256": sha(root / ".venv/bin/python"), + "lockfile_sha256": sha(root / "uv.lock"), + "helper_path": str(SUPPORT / "owned_children.py"), + "source_hashes": hashes_before, + "workdir": str(folder / "fixture"), + "ports": dict(zip(("api", "router", "probe", "tls"), selected[4:], strict=True)), + "databases": { + name: "trigger_" + fixture_id.replace("-", "") + "_" + str(index) + for index, name in enumerate( + ("pg_cell", "pg_directory", "ch_schema", "ch_telemetry") + ) + }, + "stores": { + role: stores.observation(item, role) + for item, role in ( + (ch, "clickhouse"), + (primary, "pg_primary"), + (standby, "pg_standby"), + ) + }, + "pg_admin_dsn": f"postgresql://loom:loomdev@127.0.0.1:{selected[2]}/postgres", + "pg_standby_dsn": f"postgresql://loom:loomdev@127.0.0.1:{selected[3]}/postgres", + "ch": { + "host": "127.0.0.1", + "port": selected[0], + "username": "loom", + "password": "loomdev", + "keeper_prefix": "/keeper_map_tables", + }, + "dependency_receipt": dependencies, + "ci_run": run, + } + write(folder / "spec.json", spec) + require(not stores.cancelled, "Cancelled before native acceptance") + stores.phase = "native" + require(runtime(spec) == 0, "Trigger acceptance did not pass") + require(source(root) == before, "Acceptance source changed") + except BaseException as caught: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + error = caught + try: + write(folder / "startup-diagnostics.json", stores.failure_diagnostics()) + except BaseException as diagnostic_error: # noqa: BLE001 — diagnostic failure cannot replace the selected first error. + diagnostic_error_type = type(diagnostic_error).__name__ + finally: + # Native ownership closes first; exact created containers are removed even on failure. + for signum in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP): + signal.signal(signum, stores.interrupted) + try: + native_cleanup_complete(folder, fixture_id) + cleanup_errors = stores.cleanup() + cleanup_complete = not cleanup_errors + if cleanup_errors and error is None: + error = RuntimeError("Owned store cleanup incomplete") + except BaseException as caught: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + error = caught if error is None else error + if stores.cancelled and error is None: + error = InterruptedError("Acceptance interrupted") + try: + require(source(root) == before, "Final acceptance source changed") + hashes_after = source_hashes(root) + write( + folder / "source-after.json", + {**before, "source_hashes": hashes_after, "ci_run": run}, + ) + require( + hashes_after == hashes_before, "Final driver, support or workflow bytes changed" + ) + except BaseException as caught: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + error = caught if error is None else error + try: + publish( + folder, + Path(os.environ["RUNNER_TEMP"]) / "cell-trigger-artifacts", + error, + phase=stores.phase, + diagnostic_error_type=diagnostic_error_type, + ) + except BaseException as caught: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + error = caught if error is None else error + stores.terminal = cleanup_complete + if stores.cancelled and error is None: + error = InterruptedError("Acceptance interrupted during publication") + print( + json.dumps( + { + "accepted": error is None, + "error_type": type(error).__name__ if error is not None else "", + "failure": failure_site(error), + "phase": stores.phase, + "diagnostic_error_type": diagnostic_error_type, + } + ) + ) + return int(error is not None or bool(stores.cancelled)) + + +if __name__ == "__main__": + try: + code = main() + except BaseException as error: # noqa: BLE001 — preflight failures must emit only nonsecret diagnostics. + result = { + "accepted": False, + "error_type": type(error).__name__, + "failure": failure_site(error), + } + print(json.dumps(result), flush=True) + if os.environ.get("GITHUB_ACTIONS") == "true" and os.environ.get("RUNNER_TEMP"): + output = Path(os.environ["RUNNER_TEMP"]) / "cell-trigger-artifacts" + output.mkdir(mode=0o700, exist_ok=True) + if not (output / "result.json").exists(): + write(output / "result.json", result) + code = 1 + raise SystemExit(code) diff --git a/scripts/gates/docker_provider_data.py b/scripts/gates/docker_provider_data.py index 63b6097679..175b4eac3c 100644 --- a/scripts/gates/docker_provider_data.py +++ b/scripts/gates/docker_provider_data.py @@ -1,11 +1,417 @@ """Seed/check nonempty rebuild surfaces inside the disposable Docker gate.""" import asyncio +import json +import os +import re +import signal import sys +import time +from multiprocessing import spawn +from pathlib import Path +from typing import TYPE_CHECKING, Any +from uuid import UUID +from server.core import provider_process from server.core.ch import get_client from server.insights.app.provider_command import model_command from server.settings import settings +from server.workers.child import child_argv + +if TYPE_CHECKING: + from server.core.repo_base import _CHClient + +SUPERVISOR = [ + "-m", + "uvicorn", + "docker_provider_app:app", + "--host", + "0.0.0.0", + "--port", + "7777", + "--workers", + "2", + "--timeout-graceful-shutdown", + "10", +] + + +class _WorkersPending(AssertionError): + """A supervised replacement has not spawned or enrolled yet.""" + + +def _process(pid: int) -> dict: + root = Path(f"/proc/{pid}") + stat = (root / "stat").read_text().rsplit(")", 1)[1].split() + return { + "pid": pid, + "parent": int(stat[1]), + "start": stat[19], + "command": (root / "cmdline").read_bytes().decode().rstrip("\0").split("\0"), + "namespace": os.stat(root / "ns/pid").st_ino, + "executable": os.readlink(root / "exe"), + } + + +def _supervisor() -> dict: + parent, me = _process(1), _process(os.getpid()) + assert parent["command"][1:] == SUPERVISOR, "Unexpected fixture supervisor" + assert parent["executable"] == me["executable"], "Fixture supervisor executable mismatch" + assert parent["namespace"] == me["namespace"], "Fixture supervisor PID namespace mismatch" + return parent + + +def _worker_process(raw: str, parent: dict) -> dict: + proof = raw.split("|") + assert len(proof) == 9 and provider_process.has_certificate(raw), "Worker certificate invalid" + assert proof[:4] == provider_process.host_proof().split("|")[:4], "Worker proof scope mismatch" + process = _process(int(proof[4])) + pid = process["pid"] + assert pid > 1, "Worker identity names the supervisor" + assert process["parent"] == 1, f"Worker PID {pid} parent mismatch: {process['parent']} != 1" + assert process["start"] == proof[5], f"Worker PID {pid} process-start proof mismatch" + assert process["namespace"] == parent["namespace"], f"Worker PID {pid} namespace mismatch" + assert process["executable"] == parent["executable"], f"Worker PID {pid} executable mismatch" + _spawn_command(process["command"], pid) + return process + + +def _spawn_command(command: list[str], pid: int) -> None: + # The fixed supervisor and exec helper use the same executable and container + # startup flags. Let stdlib reproduce those flags, including inherited -B. + expected = spawn.get_command_line(tracker_fd=0, pipe_handle=0) + assert len(command) == len(expected) and command[1:-2] == expected[1:-2], ( + f"Worker PID {pid} interpreter flags or -c mismatch" + ) + assert command[-1] == "--multiprocessing-fork", f"Worker PID {pid} fork argument mismatch" + assert re.fullmatch( + r"from multiprocessing\.spawn import spawn_main; " + r"spawn_main\(tracker_fd=[0-9]+, pipe_handle=[0-9]+\)", + command[-2], + ), f"Worker PID {pid} spawn command mismatch" + + +def _kernel_workers() -> dict[int, dict]: + """The supervisor's actual spawn children, before consulting participant history.""" + found = {} + children = Path("/proc/1/task/1/children").read_text().split() + for value in children: + pid = int(value) + try: + process = _process(pid) + except FileNotFoundError: + # An exited child can remain in the supervisor's census until reaped. + try: + state = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()[0] + except FileNotFoundError: + continue + if state in {"Z", "X"}: + continue + raise + command = process["command"] + if not any( + value.startswith("from multiprocessing.spawn import spawn_main;") for value in command + ): + continue # The multiprocessing resource tracker is not an application worker. + _spawn_command(command, pid) + assert process["parent"] == 1, f"Kernel child PID {pid} parent changed during census" + assert pid > 1 and pid not in found, "Invalid supervisor child PID census" + found[pid] = process + if len(found) < 2: + raise _WorkersPending("Expected two live supervisor spawn children") + assert len(found) == 2, f"Expected two live supervisor spawn children; found {len(found)}" + return found + + +def _workers(rows: list[dict], parent: dict) -> list[dict]: + here = provider_process.host_proof().split("|") + children = _kernel_workers() + workers = [] + for pid, process in children.items(): + matches = [] + for row in rows: + proof = str(row["host_proof"]).split("|") + if ( + len(proof) >= 6 + and row["state"] != "retired" + and proof[:4] == here[:4] + and proof[4] == str(pid) + and proof[5] == process["start"] + ): + matches.append(row) + if not matches: + raise _WorkersPending(f"Worker PID {pid} exact registrations: 0; expected 1") + assert len(matches) == 1, ( + f"Worker PID {pid} exact registrations: {len(matches)}; expected 1" + ) + row = matches[0] + assert _worker_process(row["host_proof"], parent) == process, ( + f"Worker PID {pid} changed during proof join" + ) + workers.append({key: row[key] for key in ("participant_id", "host_proof", "version")}) + assert len({row["participant_id"] for row in workers}) == 2, "Worker participant IDs collide" + assert _kernel_workers() == children, "Supervisor worker identities changed during census" + return sorted(workers, key=lambda row: row["host_proof"]) + + +def _proof(process: dict[str, Any]) -> str: + parts = provider_process.host_proof().split("|") + assert len(parts) == 8 and all(parts[:4]), "Incomplete fixture process scope" + parts[4:6] = [str(process["pid"]), process["start"]] + return provider_process.certified("|".join(parts)) + + +async def _proof_rows(client: "_CHClient", proofs: list[str]) -> list[dict[str, Any]]: + assert 1 <= len(proofs) <= 4 and len(set(proofs)) == len(proofs) + rows = await client.query_all( + "SELECT toString(participant_id) AS participant_id, state, version, host_proof " + "FROM loom._provider_participants FINAL WHERE state != 'retired' " + "AND host_proof IN {proofs:Array(String)} LIMIT 5", + {"proofs": proofs}, + ) + assert len(rows) <= len(proofs), "Duplicate captured process registrations" + return rows + + +async def _worker_rows(client) -> list[dict]: + return await _proof_rows(client, [_proof(p) for p in _kernel_workers().values()]) + + +def _background_children( + roots: list[dict[str, Any]], parent: dict[str, Any] +) -> dict[str, list[dict[str, Any]]]: + found = {} + for root in roots: + web = _worker_process(root["host_proof"], parent) + children = [] + for value in Path(f"/proc/{web['pid']}/task/{web['pid']}/children").read_text().split(): + process = _process(int(value)) + command = process["command"] + if "server.worker_main" not in command: + continue + assert process["parent"] == web["pid"], "Background child parent changed" + assert process["namespace"] == web["namespace"], "Background child namespace changed" + assert process["executable"] == web["executable"], "Background child executable changed" + assert len(command) == 7, "Background child command mismatch" + fds = [int(command[4]), int(command[6])] + assert min(fds) >= 0 and command == child_argv(*fds), ( + "Background child command mismatch" + ) + children.append(process) + assert len(children) <= 1, "More than one background child per web worker" + assert _worker_process(root["host_proof"], parent) == web, ( + "Web worker changed during child census" + ) + found[root["participant_id"]] = children + return found + + +async def workers(client) -> list[dict]: + parent = _supervisor() + result = _workers(await _worker_rows(client), parent) + assert len(result) == 2, "Worker registration census is incomplete" + children = _background_children(result, parent) + proofs = [_proof(p) for values in children.values() for p in values] + rows = await _proof_rows(client, proofs) if proofs else [] + for root in result: + root["children"] = [] + for process in children[root["participant_id"]]: + matches = [row for row in rows if row["host_proof"] == _proof(process)] + if not matches: + raise _WorkersPending("Background child exact registration is pending") + assert len(matches) == 1, "Background child registration is ambiguous" + row = matches[0] + assert provider_process.has_certificate(row["host_proof"]), ( + "Background child certificate invalid" + ) + root["children"].append( + {key: row[key] for key in ("participant_id", "host_proof", "version")} + ) + assert _background_children(result, parent) == children, ( + "Background child identities changed during census" + ) + assert len({row["participant_id"] for row in _captured(result)}) == 2 + len(proofs) + assert _supervisor() == parent, "Fixture supervisor changed during registration census" + return result + + +def _captured(roots: list[dict[str, Any]]) -> list[dict[str, Any]]: + assert 1 <= len(roots) <= 2 + assert all(type(root["children"]) is list and len(root["children"]) <= 1 for root in roots) + return [row for root in roots for row in (root, *root["children"])] + + +def _kill_worker(worker: dict, parent: dict) -> None: + proof = worker["host_proof"].split("|") + pid = int(proof[4]) + assert pid > 1, "Refusing to signal the supervisor" + captured = _worker_process(worker["host_proof"], parent) + fd = os.pidfd_open(pid) + try: + # A reused PID or changed supervisor refuses before the process-bound signal. + assert _process(pid) == captured, f"Worker PID {pid} changed after pidfd open" + assert _supervisor() == parent, "Fixture supervisor changed before worker signal" + signal.pidfd_send_signal(fd, signal.SIGKILL) + finally: + os.close(fd) + + +async def _retirement_rows(client, captured: list[dict]) -> list[dict]: + assert 1 <= len(captured) <= 4 + rows = await client.query_all( + "SELECT participant_id, state, version, host_proof " + "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)} LIMIT 5", + {"ids": [row["participant_id"] for row in captured]}, + ) + return [{**row, "participant_id": str(row["participant_id"])} for row in rows] + + +async def _native_retirement_observation(client, captured: list[dict]) -> dict: + try: + rows = await client.query_all( + "SELECT participant_id " + "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)}", + {"ids": [row["participant_id"] for row in captured]}, + ) + return { + "reference_count": len(rows), + "reference_identity": {str(row["participant_id"]) for row in rows} + == {row["participant_id"] for row in captured}, + } + except Exception: # noqa: BLE001 — diagnostic failure cannot replace the unproved retirement + return {"reference_read": "query_error"} + + +async def retired(client, captured: list[dict], *, observation: dict | None = None) -> bool: + from server.insights.store.provider_participants import running_queries + + def result(reason: str) -> bool: + if observation is not None: + observation["retirement"] = reason + return reason == "proved" + + rows = await _retirement_rows(client, captured) + if len(rows) != len(captured): + return result("row_count") + expected = {row["participant_id"]: row for row in captured} + for row in rows: + old = expected[row["participant_id"]] + if row["state"] != "retired": + return result("state") + if int(row["version"]) <= int(old["version"]): + return result("version") + if row["host_proof"] != old["host_proof"]: + return result("host_proof") + if not provider_process.has_certificate(row["host_proof"]): + return result("certificate") + running = await running_queries(row["participant_id"], ch=client) + if running != 0: + return result("queries_unknown" if running is None else "queries_running") + return result("proved") + + +async def _subtree_diagnostic( + client: "_CHClient", roots: list[dict[str, Any]], *, proved: bool +) -> None: + """Print closed role counts, preserving the original retirement failure.""" + try: + roles = {"web": roots, "background": [c for r in roots for c in r["children"]]} + summary = {} + for role, captured in roles.items(): + counts = { + "captured": len(captured), + "retirement_proved": 0, + "death_proved": 0, + "query_zero_proved": 0, + "unproved": 0, + "kernel_zombie_single_thread": 0, + "kernel_zombie_multi_thread": 0, + "kernel_present_non_zombie": 0, + "kernel_missing": 0, + "kernel_uninspectable": 0, + } + for row in captured: + kernel = "kernel_uninspectable" + proof = row["host_proof"].split("|") + here = provider_process.host_proof().split("|") + if len(proof) in {8, 9} and proof[:4] == here[:4] and proof[4].isdecimal(): + try: + raw = await asyncio.to_thread(Path(f"/proc/{int(proof[4])}/stat").read_text) + stat = raw.rsplit(")", 1)[1].split() + if stat[19] == proof[5]: + threads = int(stat[17]) + if stat[0] == "Z" and threads >= 1: + kernel = ( + "kernel_zombie_single_thread" + if threads == 1 + else "kernel_zombie_multi_thread" + ) + elif threads >= 1: + kernel = "kernel_present_non_zombie" + except FileNotFoundError: + kernel = "kernel_missing" + except (OSError, IndexError, ValueError): + pass # An unreadable identity does not establish a kernel state. + counts[kernel] += 1 + counts["death_proved"] += int(provider_process.proves_stopped(row["host_proof"])) + accepted = proved or (bool(roles["background"]) and await retired(client, [row])) + counts["retirement_proved"] += int(accepted) + counts["query_zero_proved"] += int(accepted) + counts["unproved"] += int(not accepted) + summary[role] = counts + print("Provider subtree retirement diagnostic: " + json.dumps(summary, sort_keys=True)) + except BaseException: # noqa: BLE001 — diagnostics must preserve the original retirement failure + print("Provider subtree retirement diagnostic unavailable") + + +async def crash_worker(client) -> None: + parent = _supervisor() + before = await workers(client) + victim = before[0] + for worker in _captured(before): + rows = await _retirement_rows(client, [worker]) + identity_matches = len(rows) == 1 and rows[0]["participant_id"] == worker["participant_id"] + if not identity_matches: + readback = { + "reader_count": len(rows), + "reader_identity": identity_matches, + **await _native_retirement_observation(client, [worker]), + } + raise AssertionError( + "Captured worker readback failed: " + json.dumps(readback, sort_keys=True) + ) + _kill_worker(victim, parent) + deadline = time.monotonic() + 240 + observation: dict = {"death": "not_checked", "retirement": "not_checked"} + while time.monotonic() < deadline: + assert _supervisor() == parent + stopped = provider_process.proves_stopped(victim["host_proof"]) + observation = {"death": "proved" if stopped else "unproved", "retirement": "not_checked"} + if stopped and await retired(client, _captured([victim]), observation=observation): + try: + after = _workers(await _worker_rows(client), parent) + except _WorkersPending: + observation["replacement"] = "pending" + await asyncio.sleep(1) + continue + old = {row["participant_id"] for row in before} + new = {row["participant_id"] for row in after} + observation.update( + replacement_count=len(new), + survivors=len(old & new), + victim_absent=victim["participant_id"] not in new, + ) + if len(new) == 2 and len(old & new) == 1 and victim["participant_id"] not in new: + await _subtree_diagnostic(client, [victim], proved=True) + return + await asyncio.sleep(1) + if observation["retirement"] == "row_count": + observation.update(await _native_retirement_observation(client, [victim])) + await _subtree_diagnostic(client, [victim], proved=False) + raise AssertionError( + "Crashed worker retirement or supervised replacement was not proved: " + + json.dumps(observation, sort_keys=True) + ) async def seed() -> None: @@ -69,12 +475,87 @@ async def verify(model: str, dimensions: int) -> None: print(f"PASS provider data: {model}/{dimensions}, memory + code metadata, no retained copies") +async def diagnostic(client, transition: str) -> None: + """Read capped aggregate evidence; never acknowledge or retire a participant.""" + from server.insights.store.provider_participants import PARTICIPANTS_SQL + + scope = provider_process.host_proof().split("|")[:4] + if len(scope) != 4 or not all(scope): + raise ValueError("Unavailable provider diagnostic scope") + sql = ( + "WITH retained AS (" + PARTICIPANTS_SQL + "), " + "lease AS (SELECT holder, renewed_at + toIntervalSecond(ttl_s) < now64(3) AS expired " + "FROM loom._migration_lock WHERE name = {name:String} LIMIT 2) " + "SELECT count() AS retained, " + "countIf(p.state = 'running') AS running, " + "countIf(p.state = 'drained' AND p.transition_id = {transition:UUID}) AS matching_drained, " + "countIf(p.state = 'drained' AND p.transition_id != {transition:UUID}) AS other_drained, " + "countIf(p.state = 'retired') AS retained_retired, " + "countIf(p.expired) AS expired, " + "countIf(arraySlice(splitByChar('|', p.host_proof), 1, 4) = {scope:Array(String)}) " + "AS same_scope, " + "(SELECT count() FROM lease) AS lease_rows, " + "(SELECT countIf(expired) FROM lease) AS expired_leases, " + "countIf(l.holder != '') AS retained_holders, " + "countIf(l.holder != '' AND p.state = 'drained' " + "AND p.transition_id = {transition:UUID}) AS matching_drained_holders " + "FROM retained AS p ANY LEFT JOIN lease AS l " + "ON toString(p.participant_id) = arrayElement(splitByChar('/', l.holder), 1) " + "SETTINGS max_execution_time = 3, max_rows_to_read = 10000, " + "max_bytes_to_read = 10485760, read_overflow_mode = 'throw', " + "max_result_rows = 1, max_result_bytes = 4096, result_overflow_mode = 'throw', " + "max_memory_usage = 134217728, max_threads = 1, select_sequential_consistency = 1" + ) + row = await asyncio.wait_for( + client.query_one( + sql, + { + "name": "provider_configuration", + "transition": str(UUID(transition)), + "scope": scope, + }, + ), + timeout=5, + ) + keys = { + "retained", + "running", + "matching_drained", + "other_drained", + "retained_retired", + "expired", + "same_scope", + "lease_rows", + "expired_leases", + "retained_holders", + "matching_drained_holders", + } + if ( + row is None + or row.keys() != keys + or any(type(value) is not int or not 0 <= value <= 10000 for value in row.values()) + ): + raise ValueError("Invalid provider diagnostic aggregate") + print("Provider reset diagnostic: " + json.dumps(row, sort_keys=True)) + + async def main() -> None: client = get_client() await client.connect() try: - if sys.argv[1] == "seed": + if sys.argv[1] == "diagnostic": + await diagnostic(client, sys.argv[2]) + elif sys.argv[1] == "seed": await seed() + elif sys.argv[1] == "workers": + print(json.dumps(await workers(client))) + elif sys.argv[1] == "crash-worker": + await crash_worker(client) + elif sys.argv[1] == "retired": + captured = json.loads(sys.argv[2]) + accepted = await retired(client, _captured(captured)) + await _subtree_diagnostic(client, captured, proved=accepted) + assert accepted, "Recreated workers did not retire" else: await verify(sys.argv[1], int(sys.argv[2])) finally: diff --git a/scripts/gates/docker_provider_smoke.py b/scripts/gates/docker_provider_smoke.py index a71ec4cf9f..036da0dbc0 100644 --- a/scripts/gates/docker_provider_smoke.py +++ b/scripts/gates/docker_provider_smoke.py @@ -59,9 +59,29 @@ def control(compose: list[str], failed_model: str) -> None: run(*compose, "exec", "-T", "provider-smoke", "python", "-c", script) -def data_check(compose: list[str], *args: str) -> None: +def data_check(compose: list[str], *args: str) -> list[dict] | None: source = (ROOT / "scripts/gates/docker_provider_data.py").read_text() + if args[0] == "diagnostic": + subprocess.run( + [*compose, "exec", "-T", "loom", "python", "-", *args], + input=source, + text=True, + check=True, + timeout=10, + stderr=subprocess.DEVNULL, + ) + return None + if args[0] == "workers": + result = subprocess.run( + [*compose, "exec", "-T", "loom", "python", "-", *args], + input=source, + text=True, + check=True, + capture_output=True, + ) + return json.loads(result.stdout) run(*compose, "exec", "-T", "loom", "python", "-", *args, input=source, text=True) + return None def dashboard_action(base: str, action: str, *args: str) -> dict: @@ -105,6 +125,7 @@ def wait_admission(base: str) -> None: def smoke_providers(base: str, compose: list[str]) -> None: override = Path(compose[-1]) configuration = json.loads(override.read_text()) + # Match serve.sh's finite request grace so the lifespan fits Compose's stop grace. configuration["services"]["loom"]["entrypoint"] = [ "python", "-m", @@ -116,6 +137,8 @@ def smoke_providers(base: str, compose: list[str]) -> None: "7777", "--workers", "2", + "--timeout-graceful-shutdown", + "10", ] configuration["services"]["loom"]["volumes"] = [ f"{ROOT / 'tests/fixtures/docker_provider_app.py'}:/app/docker_provider_app.py:ro" @@ -171,24 +194,35 @@ def smoke_providers(base: str, compose: list[str]) -> None: control(compose, "") dashboard_action(base, "retry") - # SIGKILL exercises persistent process proofs and ambiguous-effect recovery, - # not just an orderly shutdown. Restart only this generated stack's Loom. - run(*compose, "kill", "-s", "SIGKILL", "loom") - run(*compose, "up", "-d", "--wait", "--wait-timeout", "180", "loom") + # Keep the supervisor's PID namespace: a different namespace cannot certify + # death. A live worker retires the exact crashed worker after its queries end. + data_check(compose, "crash-worker") status = wait_status(base, lambda row: row["pending"] is None) assert status["embedding_model"] == "docker-smoke-16" and status["embedding_dimensions"] == 16 wait_admission(base) data_check(compose, "docker-smoke-16", "16") + workers = data_check(compose, "workers") + assert workers is not None and len(workers) == 2 run(*compose, "up", "-d", "--force-recreate", "--wait", "--wait-timeout", "180", "loom") + # Recreation is graceful; prove the captured workers actually retired instead + # of treating Docker's container replacement as process-death evidence. + data_check(compose, "retired", json.dumps(workers)) status = request(base, "/v1/admin/models") assert status["source"] == "dashboard" and status["embedding_model"] == "docker-smoke-16" wait_admission(base) data_check(compose, "docker-smoke-16", "16") reset = dashboard_action(base, "reset") assert reset["state"] == "queued", reset - status = wait_status( - base, lambda row: row["pending"] is None and row["source"] == "environment" - ) + try: + status = wait_status( + base, lambda row: row["pending"] is None and row["source"] == "environment" + ) + except BaseException: + try: + data_check(compose, "diagnostic", reset["config_id"]) + except BaseException: # noqa: BLE001 — diagnostic failure must preserve activation failure + print("Provider reset diagnostic unavailable") + raise assert status["embedding_model"] == environment["embedding_model"], status assert status["llm_model"] == environment["llm_model"], status wait_admission(base) diff --git a/scripts/gates/testsupport/cell_trigger/__init__.py b/scripts/gates/testsupport/cell_trigger/__init__.py new file mode 100644 index 0000000000..e5dad64539 --- /dev/null +++ b/scripts/gates/testsupport/cell_trigger/__init__.py @@ -0,0 +1 @@ +"""Isolated native trigger acceptance support.""" diff --git a/scripts/gates/testsupport/cell_trigger/capture.py b/scripts/gates/testsupport/cell_trigger/capture.py new file mode 100644 index 0000000000..3f1c78429a --- /dev/null +++ b/scripts/gates/testsupport/cell_trigger/capture.py @@ -0,0 +1,1040 @@ +"""Capture unchanged delivery intent and independently verify persisted rows.""" + +from __future__ import annotations + +import argparse +import datetime +import hashlib +import importlib.util +import json +import math +import os +import re +import socket +import stat +import struct +import sys +import uuid +from pathlib import Path + +from fixture import ( + FILES, + HERE, + PROTOCOL, + PROTOCOL_SHA, + SPEC_FIELDS, + digest, + json_object, + need, + private_stream, + read, + ref, + sha, + source, + validate, + write, +) + +LINE = 65536 +ARM_BYTES = 32 * 1024 * 1024 +TOTAL_BYTES = 320 * 1024 * 1024 +ORDER = (True, False, False, True, True, False, False, True, True, False) +SUBJECT = "process:benchmark" + + +def protocol() -> dict: + descriptor = os.open(HERE / PROTOCOL, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(descriptor, "rb") as stream: + info = os.fstat(stream.fileno()) + need( + stat.S_ISREG(info.st_mode) + and info.st_uid == os.geteuid() + and info.st_mode & 0o022 == 0 + and info.st_size <= 65536, + "owned fixed protocol constant", + ) + raw = stream.read(65537) + need( + len(raw) <= 65536 and hashlib.sha256(raw).hexdigest() == PROTOCOL_SHA, + "fixed protocol bytes", + ) + return json_object(raw) + + +def integer(value, low=0, high=2**32 - 1): + need(type(value) is int and low <= value <= high, "typed bounded integer") + return value + + +def identifier(value): + if isinstance(value, uuid.UUID): + value = str(value) + need( + type(value) is str and str(uuid.UUID(value)) == value and uuid.UUID(value).int != 0, + "canonical UUID", + ) + return value + + +def equal(left, right): + need( + json.dumps(left, sort_keys=True, allow_nan=False, separators=(",", ":")) + == json.dumps(right, sort_keys=True, allow_nan=False, separators=(",", ":")), + "exact typed equality", + ) + + +def instant(value, *, audit=False): + if isinstance(value, str): + need( + re.fullmatch( + r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d{1,6})?(?:Z|[+-]\d\d:\d\d)?", value + ), + "exact ISO datetime", + ) + value = datetime.datetime.fromisoformat(value) + need(type(value) is datetime.datetime, "typed datetime") + value = ( + value.replace(tzinfo=datetime.UTC) + if value.tzinfo is None + else value.astimezone(datetime.UTC) + ) + if audit: + need(value.microsecond % 1000 == 0, "audit millisecond precision") + return value.isoformat(timespec="microseconds") + need(value.microsecond == 0 and 0 <= value.timestamp() < 2**32, "memory second precision/range") + return value.replace(tzinfo=None).isoformat(timespec="seconds") + + +def hex64(value, *, native=False): + if native and isinstance(value, (bytes, bytearray)): + value = bytes(value).decode("ascii") + need(type(value) is str and re.fullmatch(r"[0-9a-f]{64}", value), "typed hex64") + return value + + +def payload(key): + prefix = f"Cell trigger measurement {key}: " + need( + type(key) is str and key.isascii() and len(prefix) < 4096 and len(key) <= 128, "payload key" + ) + return prefix + ("abcdefghijklmnopqrstuvwxyz " * 4096)[: 4096 - len(prefix)] + + +def content_hash(key): + return hashlib.sha256( + json.dumps(payload(key), sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + + +class Codec: + def __init__(self, specification): + self.p = specification + need( + len(self.p["memory_fields"]) == 61 + and len(set(self.p["memory_fields"])) == 61 + and set(self.p["memory_types"]) == set(self.p["memory_fields"]) + and len(self.p["sparse_defaults"]) == 43 + and len(self.p["audit_fields"]) == 13, + "closed protocol", + ) + + def row(self, original, *, origin="capture", comparison=False): + need( + type(original) is dict and set(original) <= set(self.p["memory_fields"]), + "memory row fields", + ) + value = dict(original) + if origin == "pg": + for key, default in self.p["sparse_defaults"].items(): + if key not in value: + value[key] = list(default) if isinstance(default, list) else default + need(set(value) == set(self.p["memory_fields"]), "complete memory row") + result = {} + for key in self.p["memory_fields"]: + item = value[key] + kind = self.p["memory_types"][key] + if kind.startswith("nullable_") and item is None: + result[key] = None + continue + kind = kind.removeprefix("nullable_") + if kind == "string": + need(type(item) is str and len(item.encode()) <= 8192, "memory string") + elif kind == "json_object_field": + if origin == "ch": + need(type(item) is dict, "native CH JSON object") + parsed = json_object(json.dumps(item, allow_nan=False)) + else: + need(type(item) is str, "encoded object JSON") + parsed = json_object(item) + need(parsed == {}, "primitive workload JSON object") + if comparison: + item = parsed + elif kind == "utc_second": + item = instant(item) + elif kind == "uuid": + item = identifier(item) + elif kind in {"array_uuid", "array_string"}: + need(type(item) is list and len(item) <= 64, "typed array") + if kind == "array_uuid": + item = [identifier(member) for member in item] + else: + need( + all(type(member) is str and len(member.encode()) <= 512 for member in item), + "string array", + ) + elif kind == "hex64_fixedstring": + item = hex64(item, native=origin == "ch") + elif kind == "finite_exact_float32_not_bool": + need(type(item) in {int, float} and math.isfinite(item), "finite Float32") + rounded = struct.unpack("!f", struct.pack("!f", item))[0] + need(rounded == item, "exact stored Float32") + item = rounded + elif kind == "positive_int63_not_bool": + item = integer(item, 1, 2**63 - 1) + elif kind == "uint8_0_or_1_not_bool": + item = integer(item, 0, 1) + elif kind == "uint32_not_bool": + item = integer(item) + else: + need(False, "unknown protocol type") + result[key] = item + return result + + def audit(self, value, *, origin="capture"): + need( + type(value) is dict and set(value) == set(self.p["audit_fields"]), + "complete audit fields", + ) + result = {} + for key in self.p["audit_fields"]: + item = value[key] + if key == "ts": + item = instant(item, audit=True) + elif key == "row_hash": + item = hex64(item, native=origin == "ch") + else: + need(type(item) is str and len(item.encode()) <= 8192, "audit string") + result[key] = item + timestamp = ( + datetime.datetime.fromisoformat(result["ts"]) + .astimezone(datetime.UTC) + .replace(tzinfo=None) + .isoformat() + ) + canonical = { + key: result[key] + for key in ("event_type", "org", "namespace", "actor", "subject", "request_id") + } + canonical["ts"] = timestamp + actual = hashlib.sha256( + json.dumps(canonical, sort_keys=True, ensure_ascii=True).encode() + ).hexdigest() + need(actual == result["row_hash"], "canonical raw-subject audit hash") + return result + + def expected(self, row, audit, context, *, api, metadata, driver): + key = row["memory_key"] + equal( + [ + row[k] + for k in ( + "org", + "namespace", + "agent", + "user_id", + "session_id", + "actor", + "source_class", + ) + ], + [context["org"], context["namespace"], "bench", "-", "-", SUBJECT, "asserted"], + ) + need( + row["memory_type"] == "semantic" and row["status"] == "active" and row["pinned"] == 0, + "workload semantic/active", + ) + need(row["confidence"] == 1.0, "workload confidence") + for field, default in self.p["sparse_defaults"].items(): + equal(row[field], default) + need( + row["value"] == json.dumps(payload(key)) and row["content_sha256"] == content_hash(key), + "exact stored payload/hash", + ) + expected_subject = dict( + memory_key=key, + memory_id=row["memory_id"], + version=row["version"], + content_sha256=row["content_sha256"], + source_class="asserted", + status="active", + pinned=False, + tombstoned=False, + ) + equal(json_object(audit["subject"]), expected_subject) + need( + audit["subject"] == json.dumps(expected_subject, sort_keys=True), + "raw native subject spelling", + ) + need( + audit["event_type"] == "memory.revision" + and audit["org"] == context["org"] + and audit["namespace"] == context["namespace"] + and audit["actor"] == SUBJECT, + "audit event/scope/actor", + ) + api_launch = context["_api_server"] if api else None + instance = ( + api_launch["health_instance"] if api else driver["host"] + ":" + str(driver["pid"]) + ) + equal( + { + k: audit[k] + for k in ("build_commit", "instance", "instance_label", "consumer", "token_id") + }, + dict( + build_commit=context["head"], + instance=instance, + instance_label=("iso:trigger:" + context["fixture_id"][:12])[:64], + consumer="service-account" if api else "", + token_id=metadata["summary"]["jti"] if api else "", + ), + ) + need( + ( + type(audit["request_id"]) is str + and audit["request_id"].isascii() + and 0 < len(audit["request_id"]) <= 128 + ) + if api + else audit["request_id"] == "", + "audit request identity", + ) + + +def birth(): + pid = os.getpid() + return dict( + pid=pid, + start_ticks=Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()[19], + boot_id=Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + uid=os.geteuid(), + host=socket.gethostname(), + environment_sha256=digest(dict(os.environ)), + ) + + +def line(value): + raw = ( + json.dumps(value, sort_keys=True, allow_nan=False, separators=(",", ":")) + "\n" + ).encode() + need(len(raw) <= LINE, "JSONL line bound") + return raw + + +class Capture: + def __init__(self, context, directory, driver_birth, codec): + self.s = context + self.directory = directory + self.birth = driver_birth + self.codec = codec + self.calls = 0 + self.total = 0 + need( + directory == Path(context["workdir"]) / "delivery-intent" and not directory.exists(), + "new exact intent directory", + ) + directory.mkdir(mode=0o700) + + def header(self, arm): + w = Path(self.s["workdir"]) + return dict( + schema=1, + kind="header", + fixture_id=self.s["fixture_id"], + arm=arm, + on=ORDER[arm], + batch_rows=500, + expected_rows=2500, + head=self.s["head"], + tree=self.s["tree"], + driver_sha256=self.s["source_hashes"]["driver"], + capture_sha256=self.s["source_hashes"][FILES[3]], + protocol_sha256=PROTOCOL_SHA, + context_sha256=self.s["capture_context_sha256"], + fixture_sha256=sha(w / "fixture.json"), + launch_sha256=sha(w / "serving-cell/launch.json"), + mint_metadata_sha256=self.s["mint_metadata"]["sha256"], + birth=self.birth, + ) + + def delegate(self, original): + async def build(args, batch): + arm = self.calls + need(arm < 10 and type(batch) is int and batch == 500, "exact build count/batch") + self.calls += 1 + block = await original(args, batch) + need(len(block.rows) == len(block.audits) == 2500, "exact delivery intent count") + path = self.directory / f"arm-{arm}.jsonl" + full = hashlib.sha256() + prefix = hashlib.sha256() + size = 0 + header = self.header(arm) + with private_stream(path, output=True) as stream: + for index in range(-1, 2501): + if index == -1: + value = header + elif index == 2500: + value = dict( + schema=1, + kind="trailer", + arm=arm, + count=2500, + prefix_sha256=prefix.hexdigest(), + ) + else: + value = dict( + schema=1, + kind="delivery", + arm=arm, + index=index, + row=self.codec.row(block.rows[index]), + audit=self.codec.audit(block.audits[index]), + ) + raw = line(value) + size += len(raw) + self.total += len(raw) + need(size <= ARM_BYTES and self.total <= TOTAL_BYTES, "intent byte bounds") + need(stream.write(raw) == len(raw), "complete intent write") + full.update(raw) + if index < 2500: + prefix.update(raw) + write( + self.directory / f"arm-{arm}-receipt.json", + dict( + schema=1, + arm=arm, + bytes=size, + records=2502, + sha256=full.hexdigest(), + header=header, + ), + ) + return block + + return build + + +def mint_metadata(context, minted): + w = Path(context["workdir"]) + with private_stream(w / "mint-summary.stdout") as stream: + raw = stream.read(4097) + need(len(raw) <= 4096 and raw.endswith(b"\n") and raw.count(b"\n") == 1, "mint stdout shape") + summary = json_object(raw) + need(set(summary) == set(protocol()["mint_summary_fields"]), "mint summary fields") + need(type(summary["jti"]) is str and re.fullmatch(r"[a-f0-9]{32}", summary["jti"]), "mint jti") + equal( + {k: v for k, v in summary.items() if k != "jti"}, + dict( + subject="benchmark", + org=context["org"], + namespaces=[context["namespace"]], + roles=["writer"], + ttl_days=1, + is_service_account=True, + read_scope="namespace", + read_user_id="", + traffic_class="", + consumer="", + token_path=str(w / "writer.jwt"), + ), + ) + with private_stream(w / "writer.jwt") as stream: + token = stream.read(16385) + need( + 0 < len(token) <= 16384 and token.endswith(b"\n") and token.count(b"\n") == 1, + "private native bearer file", + ) + born = ref(minted["launch_receipt"]) + terminal = ref(minted["terminal_receipt"]) + need( + type(terminal["returncode"]) is int and terminal["returncode"] == 0, "mint terminal success" + ) + equal(minted["source_sha256"], digest(context["source_hashes"])) + equal(minted["cwd"], context["root"]) + expected = [ + context["python"], + "-m", + "server.cli", + "mint-token", + "--subject", + "benchmark", + "--org", + context["org"], + "--namespaces", + context["namespace"], + "--roles", + "writer", + "--service-account", + "--ttl-days", + "1", + "--save", + str(w / "writer.jwt"), + "--json", + ] + equal(minted["redacted_argv"], expected) + for key in ("pid", "start_ticks", "boot_id", "uid", "configuration_sha256"): + equal(minted[key], born[key]) + retirement = read(w / "mint-proof.json") + equal(retirement["birth"], born) + need(retirement["state"] == "retired", "mint retirement receipt") + return dict( + schema=1, + summary=summary, + stdout=dict( + path=str(w / "mint-summary.stdout"), + sha256=hashlib.sha256(raw).hexdigest(), + bytes=len(raw), + ), + token=dict( + path=str(w / "writer.jwt"), sha256=hashlib.sha256(token).hexdigest(), bytes=len(token) + ), + process=minted, + retirement=dict(path=str(w / "mint-proof.json"), sha256=sha(w / "mint-proof.json")), + ), token.decode().strip() + + +def verify_token(metadata): + expected = metadata["token"] + with private_stream(Path(expected["path"])) as stream: + raw = stream.read(16385) + need( + len(raw) == expected["bytes"] and hashlib.sha256(raw).hexdigest() == expected["sha256"], + "same actual native bearer bytes", + ) + return raw.decode().strip() + + +def closed(value, fields): + need(type(value) is dict and set(value) == set(fields.split()), "closed artifact record") + + +def record_stream(path, *, maximum): + full = hashlib.sha256() + size = 0 + count = 0 + with private_stream(path) as stream: + while True: + raw = stream.readline(LINE + 1) + if not raw: + break + count += 1 + size += len(raw) + need( + len(raw) <= LINE and raw.endswith(b"\n") and size <= maximum and count <= 101000, + "bounded complete JSONL", + ) + full.update(raw) + yield json_object(raw), raw + return dict(path=str(path), sha256=full.hexdigest(), bytes=size, records=count) + + +class Workload: + def __init__(self, context, codec): + self.s = context + self.w = Path(context["workdir"]) + self.codec = codec + self.metadata = ref(context["mint_metadata"]) + verified, _ = mint_metadata(context, self.metadata["process"]) + equal(self.metadata, verified) + verify_token(self.metadata) + self.driver = read(self.w / "driver-birth.json") + self.driver_birth = ref(self.driver["launch_receipt"]) + self.finished = read(self.w / "driver-finished.json") + terminal = ref(self.finished["terminal_receipt"]) + need( + type(terminal["returncode"]) is int and terminal["returncode"] in (0, 1), + "completed actual driver status", + ) + for key in ( + "pid", + "start_ticks", + "boot_id", + "uid", + "configuration_sha256", + "launch_receipt", + ): + equal(self.driver[key], self.finished[key]) + for key in ("pid", "start_ticks", "boot_id", "uid"): + equal(self.driver[key], self.driver_birth[key]) + equal(self.driver["source_sha256"], digest(context["source_hashes"])) + equal(self.driver["cwd"], context["root"]) + self.launch = read(self.w / "serving-cell/launch.json") + self.configuration = ref(self.launch["configuration_artifact"]) + fixture = read(self.w / "fixture.json") + ownership = read( + self.w / "ownership.json", expected_sha256=fixture["ownership_receipt_sha256"] + ) + candidates = [ + item for item in ownership["processes"] if item["pid"] == self.launch["server"]["pid"] + ] + need(len(candidates) == 1, "one owner-recorded actual API birth") + actual = ref(candidates[0]["launch_receipt"]) + for key in ("pid", "start_ticks", "boot_id", "uid"): + equal(self.launch["server"][key], actual[key]) + equal(self.launch["source"], dict(head=context["head"], tree=context["tree"])) + equal(self.launch["server"]["instance_id"], context["api_instance_id"]) + equal(self.launch["server"]["cwd"], context["root"]) + equal( + self.launch["server"]["health_instance"], + socket.gethostname() + ":" + str(self.launch["server"]["pid"]), + ) + self.ids = set() + self.keys = set() + self.total = 0 + self.counts = {"api": 0, "delivery": 0} + self.refs = [ + dict(path=str(self.w / name), sha256=sha(self.w / name)) + for name in ( + "fixture.json", + "attached-context.json", + "ownership.json", + "serving-cell/launch.json", + "driver-birth.json", + "driver-finished.json", + ) + ] + self.refs.extend( + [ + self.launch["configuration_artifact"], + context["mint_metadata"], + self.metadata["stdout"], + self.metadata["token"], + self.metadata["process"]["launch_receipt"], + self.metadata["process"]["terminal_receipt"], + self.metadata["retirement"], + ] + ) + self.driver_header = None + self.iterator = None + + def __enter__(self): + self.iterator = self.inputs() + return self + + def __exit__(self, kind, error, traceback): + try: + if self.iterator is not None: + self.iterator.close() + except BaseException: + if error is None: + raise + return False + + def unique(self, item): + identifier(item["memory_id"]) + key = item["key"] + need( + item["memory_id"] not in self.ids and key not in self.keys, + "unique independent workload identities", + ) + self.ids.add(item["memory_id"]) + self.keys.add(key) + need(len(self.ids) <= 45500 and len(self.keys) <= 45500, "workload set bound") + self.counts[item["kind"]] += 1 + return item + + def delivery(self, arm): + path = self.w / "delivery-intent" / f"arm-{arm}.jsonl" + receipt = read(path.parent / f"arm-{arm}-receipt.json") + closed(receipt, "schema arm bytes records sha256 header") + equal([receipt[key] for key in ("schema", "arm", "records")], [1, arm, 2502]) + integer(receipt["bytes"], 1, ARM_BYTES) + hex64(receipt["sha256"]) + prefix = hashlib.sha256() + full = hashlib.sha256() + size = 0 + records = record_stream(path, maximum=ARM_BYTES) + try: + header, raw = next(records) + full.update(raw) + prefix.update(raw) + size += len(raw) + closed( + header, + ( + "schema kind fixture_id arm on batch_rows expected_rows head " + "tree driver_sha256 capture_sha256 protocol_sha256 " + "context_sha256 fixture_sha256 launch_sha256 " + "mint_metadata_sha256 birth" + ), + ) + equal(header, receipt["header"]) + for key, expected in dict( + schema=1, + kind="header", + fixture_id=self.s["fixture_id"], + arm=arm, + on=ORDER[arm], + batch_rows=500, + expected_rows=2500, + head=self.s["head"], + tree=self.s["tree"], + driver_sha256=self.s["source_hashes"]["driver"], + capture_sha256=self.s["source_hashes"][FILES[3]], + protocol_sha256=PROTOCOL_SHA, + context_sha256=sha(self.w / "attached-context.json"), + fixture_sha256=sha(self.w / "fixture.json"), + launch_sha256=sha(self.w / "serving-cell/launch.json"), + mint_metadata_sha256=self.s["mint_metadata"]["sha256"], + ).items(): + equal(header[key], expected) + born = header["birth"] + closed(born, "pid start_ticks boot_id uid host environment_sha256") + for key in ("pid", "start_ticks", "boot_id", "uid"): + equal(born[key], self.driver[key]) + need( + born["host"] == socket.gethostname() + and born["environment_sha256"] == self.driver["configuration_sha256"], + "actual driver capture birth/config", + ) + if self.driver_header is None: + self.driver_header = born + else: + equal(born, self.driver_header) + pattern = None + for index in range(2500): + value, raw = next(records) + size += len(raw) + full.update(raw) + prefix.update(raw) + closed(value, "schema kind arm index row audit") + equal( + [value[k] for k in ("schema", "kind", "arm", "index")], + [1, "delivery", arm, index], + ) + row = self.codec.row(value["row"]) + audit = self.codec.audit(value["audit"]) + match = re.fullmatch(r"(gate-delivery-[a-f0-9]{32})-(\d+)", row["memory_key"]) + need(match is not None and match[2] == str(index), "delivery exact key counter") + if pattern is None: + pattern = match[1] + need(pattern == match[1], "one original delivery prefix") + yield self.unique( + dict( + kind="delivery", + key=row["memory_key"], + memory_id=row["memory_id"], + version=row["version"], + content_sha256=row["content_sha256"], + row=row, + audit=audit, + ) + ) + trailer, raw = next(records) + size += len(raw) + full.update(raw) + equal( + trailer, + dict( + schema=1, kind="trailer", arm=arm, count=2500, prefix_sha256=prefix.hexdigest() + ), + ) + try: + next(records) + except StopIteration: + pass + else: + need(False, "extra intent record") + need( + receipt["bytes"] == size and receipt["sha256"] == full.hexdigest(), + "complete intent bytes", + ) + self.total += size + need(self.total <= TOTAL_BYTES, "intent aggregate") + self.refs.append(dict(path=str(path), sha256=full.hexdigest(), bytes=size)) + self.refs.append( + dict( + path=str(path.parent / f"arm-{arm}-receipt.json"), + sha256=sha(path.parent / f"arm-{arm}-receipt.json"), + ) + ) + finally: + pending = sys.exception() + try: + records.close() + except BaseException: + if pending is None: + raise + + def inputs(self): + records = record_stream(self.w / "driver.jsonl", maximum=64 * 1024 * 1024) + + def take(kind, fields=None): + value, _ = next(records) + need(value.get("kind") == kind, "original artifact event order") + if fields is not None: + closed(value, fields) + return value + + try: + config = take( + "configuration", + ( + "kind manifest shared_configuration_excludes " + "credential_posture api_configuration_attested source launch " + "manifest_sha256 calls warmup rounds batches batch_rows " + "logical_keys value_bytes order driver_sha256" + ), + ) + equal(config["manifest"], read(self.w / "fixture.json")) + need(config["manifest_sha256"] == digest(config["manifest"]), "artifact fixture digest") + equal(config["source"], dict(head=self.s["head"], tree=self.s["tree"], clean=True)) + equal( + config["launch"], + dict( + health_instance=self.launch["server"]["health_instance"], + fingerprints=self.configuration["fingerprints"], + launch_sha256=sha(self.w / "serving-cell/launch.json"), + configuration_artifact_sha256=self.launch["configuration_artifact"]["sha256"], + birth_sha256=digest(self.launch["server"]), + ), + ) + equal(config["credential_posture"], self.configuration["credential_posture"]) + for key in ( + "api_configuration_sha256", + "cli_configuration_sha256", + "shared_configuration_sha256", + ): + equal(config["manifest"][key], self.configuration[key]) + for key, expected in dict( + calls=2000, + warmup=50, + rounds=5, + batches=5, + batch_rows=500, + logical_keys=45500, + value_bytes=4096, + order=list(ORDER), + driver_sha256=self.s["source_hashes"]["driver"], + shared_configuration_excludes=["cell_router_token"], + api_configuration_attested=True, + ).items(): + equal(config[key], expected) + for arm in range(10): + proof = take( + "arm_proof", + ( + "kind arm on deployment_id epoch instance_id " + "fingerprint_sha256 api_configuration_sha256 " + "cli_configuration_sha256 shared_configuration_sha256 " + "shared_configuration_excludes credential_posture " + "api_configuration_attested source launch tables " + "invariant_sha256 org_triggers_sha256" + ), + ) + equal([proof["arm"], proof["on"]], [arm, ORDER[arm]]) + for key in ( + "source", + "launch", + "credential_posture", + "shared_configuration_excludes", + "api_configuration_attested", + ): + equal(proof[key], config[key]) + for key in ( + "deployment_id", + "epoch", + "instance_id", + "api_configuration_sha256", + "cli_configuration_sha256", + "shared_configuration_sha256", + ): + equal(proof[key], config["manifest"][key]) + pattern = None + for index in range(2050): + value = take( + "request", + ( + "kind arm index status key memory_id version content_sha256 " + "elapsed_ms measured admission" + ), + ) + equal( + [value[k] for k in ("arm", "index", "status", "measured", "admission")], + [arm, index, 200, index >= 50, "derived_ready"], + ) + need( + type(value["elapsed_ms"]) in {int, float} + and math.isfinite(value["elapsed_ms"]) + and value["elapsed_ms"] >= 0, + "finite recorded latency", + ) + integer(value["version"], 1, 2**63 - 1) + match = re.fullmatch( + r"(gate-[a-f0-9]{32}-" + str(arm) + r")-(\d+)", value["key"] + ) + need(match is not None and match[2] == str(index), "API exact key counter") + if pattern is None: + pattern = match[1] + need( + match[1] == pattern + and value["content_sha256"] == content_hash(value["key"]), + "API original prefix/hash", + ) + yield self.unique( + dict( + kind="api", + **{ + key: value[key] + for key in ("key", "memory_id", "version", "content_sha256") + }, + ) + ) + for index in range(5): + value = take( + "worker", + ( + "kind arm index claimed published failed expired_keys " + "expired_evidence swept_settled published_audits" + ), + ) + equal( + [value[k] for k in ("arm", "index", "claimed", "published", "failed")], + [arm, index, 500, 500, 0], + ) + for key in ( + "expired_keys", + "expired_evidence", + "swept_settled", + "published_audits", + ): + integer(value[key]) + value = take("delivery_block", "kind arm elapsed_seconds rows rps") + equal([value["arm"], value["rows"]], [arm, 2500]) + need( + type(value["elapsed_seconds"]) in {int, float} + and math.isfinite(value["elapsed_seconds"]) + and value["elapsed_seconds"] > 0 + and type(value["rps"]) in {int, float} + and math.isfinite(value["rps"]) + and value["rps"] > 0, + "finite worker record", + ) + value = take("settled", "kind arm api_rows delivery_rows physical") + equal([value["arm"], value["api_rows"], value["delivery_rows"]], [arm, 2050, 2500]) + value = take("arm", "kind arm on p50_ms p99_ms delivery_rps") + equal([value["arm"], value["on"]], [arm, ORDER[arm]]) + yield from self.delivery(arm) + take( + "provisional_statistics", + ( + "kind on off p99_delta_ms throughput_loss " + "latency_within_limit throughput_within_limit " + "paired_p99_differences_ms" + ), + ) + equal( + take("triggers_restored", "kind verified"), + dict(kind="triggers_restored", verified=True), + ) + equal( + take("driver_cleanup", "kind native_returned outer_database_cleanup_required"), + dict( + kind="driver_cleanup", + native_returned=True, + outer_database_cleanup_required=True, + ), + ) + try: + next(records) + except StopIteration as end: + self.refs.append(end.value) + else: + need(False, "extra driver event") + equal(self.counts, dict(api=20500, delivery=25000)) + verify_token(self.metadata) + finally: + pending = sys.exception() + try: + records.close() + except BaseException: + if pending is None: + raise + + def pages(self): + page = [] + for value in self.iterator: + page.append(value) + if len(page) == 100: + yield page + page = [] + if page: + yield page + + +def entry(context, directory, argv): + need( + set(context) + == SPEC_FIELDS + | { + "cell", + "org", + "namespace", + "api_instance_id", + "mint_metadata", + "capture_context_sha256", + }, + "capture context fields", + ) + validate({key: context[key] for key in SPEC_FIELDS}) + source(context) + need( + len(argv) == 16 + and len(set(argv[::2])) == 8 + and set(argv[::2]) + == { + "--base-url", + "--pg-dsn", + "--token", + "--org", + "--namespace", + "--fixture-manifest", + "--launch-manifest", + "--output", + }, + "original driver arguments", + ) + # There are eight required original pairs; no private option reaches its parser. + driver_path = Path(context["root"]) / "bench/cell_trigger_gate.py" + name = "_fixture_measured_driver" + need(name not in sys.modules, "single actual driver module") + spec = importlib.util.spec_from_file_location(name, driver_path) + need(spec is not None and spec.loader is not None, "exact driver module") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + capture = Capture(context, directory, birth(), Codec(protocol())) + module._build_block = capture.delegate(module._build_block) + sys.argv = [str(driver_path), *argv] + result = module.main() + return 1 if result == 0 and capture.calls != 10 else result + + +def main(): + try: + divider = sys.argv.index("--") + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--context", type=Path, required=True) + parser.add_argument("--capture-directory", type=Path, required=True) + args = parser.parse_args(sys.argv[1:divider]) + context = read(args.context) + context["capture_context_sha256"] = sha(args.context) + return entry(context, args.capture_directory, sys.argv[divider + 1 :]) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + print(type(error).__name__, file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gates/testsupport/cell_trigger/fixture.py b/scripts/gates/testsupport/cell_trigger/fixture.py new file mode 100644 index 0000000000..93c7a6ff44 --- /dev/null +++ b/scripts/gates/testsupport/cell_trigger/fixture.py @@ -0,0 +1,1376 @@ +"""Own the isolated CI trigger acceptance fixture and its cleanup.""" + +from __future__ import annotations + +import argparse +import ast +import datetime +import fcntl +import hashlib +import json +import math +import os +import re +import shutil +import signal +import socket +import stat +import subprocess +import sys +import threading +import time +import uuid +from contextlib import contextmanager +from pathlib import Path +from urllib.parse import urlsplit, urlunsplit + +GIB = 2**30 + +POLICY = dict( + admit_mem_bytes=10 * GIB, + admit_disk_bytes=16 * GIB, + stop_mem_bytes=4 * GIB, + stop_disk_bytes=16 * GIB, + stop_process_rss_bytes=7 * GIB // 2, + sample_seconds=1, + wall_seconds=7200, + drain_reserve_seconds=180, +) +HERE = Path(__file__).resolve().parent +FILES = ("fixture.py", "native.py", "tls_proxy.py", "capture.py", "owned_children.py") +PROTOCOL = "protocol.json" +PROTOCOL_SHA = "690f1a06d0a523600a3ef4dcfa5cb58cc48052812ecad4e6e9e591c9e4582924" +SPEC_FIELDS = { + "schema", + "fixture_id", + "root", + "head", + "tree", + "python", + "python_sha256", + "lockfile_sha256", + "helper_path", + "source_hashes", + "workdir", + "ports", + "databases", + "stores", + "pg_admin_dsn", + "pg_standby_dsn", + "ch", + "dependency_receipt", + "ci_run", +} +CODEC_FILES = ( + "pyvenv.cfg", + "lib/python3.12/site-packages/clickhouse_connect/_version.py", + "lib/python3.12/site-packages/clickhouse_connect/datatypes/dynamic.py", + "lib/python3.12/site-packages/clickhouse_connect/datatypes/format.py", + "lib/python3.12/site-packages/asyncpg/_version.py", + "lib/python3.12/site-packages/asyncpg/connection.py", + "lib/python3.12/site-packages/asyncpg/protocol/protocol.cpython-312-aarch64-linux-gnu.so", + "lib/python3.12/site-packages/asyncpg/protocol/record.cpython-312-aarch64-linux-gnu.so", +) +CODEC_VERSIONS = {"clickhouse-connect": "1.9.0", "asyncpg": "0.32.0"} + + +class Refused(RuntimeError): + command: dict | None = None + native: dict | None = None + + +def need(ok: object, field: str) -> None: + if not ok: + raise Refused(field) + + +def digest(value: object) -> str: + return hashlib.sha256( + json.dumps(value, sort_keys=True, default=str, separators=(",", ":")).encode() + ).hexdigest() + + +def sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def utc() -> str: + return datetime.datetime.now(datetime.UTC).isoformat() + + +def pairs(items: list[tuple]) -> dict: + out = {} + for key, value in items: + need(key not in out, "duplicate JSON field") + out[key] = value + return out + + +def invalid_constant(_value: str): + raise Refused("nonfinite JSON number") + + +def finite_float(value: str) -> float: + result = float(value) + need(math.isfinite(result), "nonfinite JSON number") + return result + + +def json_object(raw: bytes | str) -> dict: + value = json.loads( + raw, object_pairs_hook=pairs, parse_constant=invalid_constant, parse_float=finite_float + ) + + def depth(item, level=0): + need(level <= 8, "JSON nesting") + if isinstance(item, dict): + for member in item.values(): + depth(member, level + 1) + elif isinstance(item, list): + for member in item: + depth(member, level + 1) + + depth(value) + need(isinstance(value, dict), "JSON object") + return value + + +@contextmanager +def private_stream(path: Path, *, output: bool = False): + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL if output else os.O_RDONLY | os.O_NONBLOCK + fd = os.open(path, flags | os.O_NOFOLLOW, 0o600) + stream = None + first = None + try: + info = os.fstat(fd) + need( + stat.S_ISREG(info.st_mode) + and info.st_uid == os.geteuid() + and info.st_mode & 0o077 == 0, + "owned private stream", + ) + stream = os.fdopen(fd, "wb" if output else "rb", closefd=False) + yield stream + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error + finally: + if stream is not None: + if output: + try: + stream.flush() + os.fsync(fd) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + try: + stream.close() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + try: + os.close(fd) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + if first is not None: + raise first + + +def read(path: Path, *, limit: int = 65536, expected_sha256: str | None = None) -> dict: + with private_stream(path) as stream: + raw = stream.read(limit + 1) + need(len(raw) <= limit, "input size") + if expected_sha256 is not None: + need(hashlib.sha256(raw).hexdigest() == expected_sha256, "receipt bytes") + return json_object(raw) + + +def write(path: Path, value: dict, *, limit: int = 65536) -> dict: + raw = (json.dumps(value, sort_keys=True, allow_nan=False) + "\n").encode() + need(len(raw) <= limit, "receipt size") + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + first = None + stream = None + try: + stream = os.fdopen(fd, "wb", closefd=False) + need(stream.write(raw) == len(raw), "complete receipt write") + stream.flush() + os.fsync(fd) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error + finally: + if stream is not None: + try: + stream.close() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + try: + os.close(fd) + except BaseException: + if first is None: + raise + if first is not None: + raise first + return {"path": str(path), "sha256": hashlib.sha256(raw).hexdigest()} + + +def ref(value: dict) -> dict: + need(set(value) == {"path", "sha256"}, "receipt reference fields") + path = Path(value["path"]) + need(path.is_absolute(), "absolute receipt") + need( + isinstance(value["sha256"], str) and re.fullmatch(r"[a-f0-9]{64}", value["sha256"]), + "receipt digest", + ) + return read(path, expected_sha256=value["sha256"]) + + +def source(spec: dict) -> None: + need( + sys.version_info[:2] == (3, 12) and Path(sys.executable) == Path(spec["python"]), + "actual admitted interpreter", + ) + dependencies(spec, verify=True) + root = Path(spec["root"]) + for command, expected in ( + (["rev-parse", "HEAD"], spec["head"]), + (["rev-parse", "HEAD^{tree}"], spec["tree"]), + (["status", "--porcelain"], ""), + ): + result = subprocess.run( + ["git", *command], cwd=root, capture_output=True, text=True, check=False, timeout=5 + ) + need(result.returncode == 0 and result.stdout.strip() == expected, "frozen source") + for name, expected in spec["source_hashes"].items(): + path = ( + root / "bench/cell_trigger_gate.py" + if name == "driver" + else root / "scripts/gates/cell_trigger_acceptance.py" + if name == "entrypoint" + else root / ".github/workflows/docker-smoke.yml" + if name == "workflow" + else HERE / name + ) + need(sha(path) == expected, "private source bytes") + need( + Path(spec["helper_path"]) == HERE / "owned_children.py" + and sha(Path(spec["helper_path"])) == spec["source_hashes"]["owned_children.py"], + "owned child implementation", + ) + need(sha(Path(spec["python"])) == spec["python_sha256"], "private interpreter") + need(sha(root / "uv.lock") == spec["lockfile_sha256"], "locked dependency source") + + +def dependencies(spec: dict, *, verify: bool = False) -> None: + value = ref(spec["dependency_receipt"]) + need( + set(value) + == { + "schema", + "root", + "python", + "python_sha256", + "lockfile_sha256", + "versions", + "codec_sources", + } + and type(value["schema"]) is int + and value["schema"] == 2, + "closed dependency receipt", + ) + for key in ("root", "python", "python_sha256", "lockfile_sha256"): + need(value[key] == spec[key], "dependency source identity") + environment = Path(spec["root"]) / ".venv" + need( + Path(spec["python"]) == environment / "bin/python" + and value["versions"] == CODEC_VERSIONS + and set(value["codec_sources"]) == set(CODEC_FILES), + "fixed interpreter/codec versions and file set", + ) + for name in CODEC_FILES: + item = value["codec_sources"][name] + path = environment / name + need( + set(item) == {"path", "sha256"} + and item["path"] == str(path) + and type(item["sha256"]) is str + and re.fullmatch(r"[a-f0-9]{64}", item["sha256"]), + "exact fixed codec path/hash", + ) + if verify: + info = path.lstat() + need( + stat.S_ISREG(info.st_mode) + and info.st_size <= 32 * 1024 * 1024 + and sha(path) == item["sha256"], + "regular frozen codec bytes", + ) + if verify: + cfg = dict( + line.split(" = ", 1) + for line in (environment / "pyvenv.cfg").read_text().splitlines() + if " = " in line + ) + need( + cfg.get("implementation") == "CPython" + and sys.version_info[:2] == (3, 12) + and cfg.get("version_info") in {"3.12", ".".join(map(str, sys.version_info[:3]))} + and cfg.get("include-system-site-packages") == "false", + "fixed isolated CPython environment", + ) + for package, field, name in ( + ("clickhouse-connect", "version", CODEC_FILES[1]), + ("asyncpg", "__version__", CODEC_FILES[4]), + ): + module = ast.parse((environment / name).read_bytes()) + assignments = [ + node.value + for node in module.body + if ( + isinstance(node, ast.Assign) + and any( + isinstance(target, ast.Name) and target.id == field + for target in node.targets + ) + ) + or ( + isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) + and node.target.id == field + ) + ] + need( + len(assignments) == 1 + and ast.literal_eval(assignments[0]) == CODEC_VERSIONS[package], + "frozen codec source version", + ) + + +def database(name: str) -> str: + need( + isinstance(name, str) and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,62}", name), + "database identifier", + ) + need( + name.lower() + not in { + "loom", + "loom_router", + "loom_telemetry", + "default", + "system", + "postgres", + "template0", + "template1", + "information_schema", + }, + "scratch database", + ) + return name + + +def pg_dsn(admin: str, db: str) -> str: + value = urlsplit(admin) + need( + value.scheme in {"postgres", "postgresql"} + and value.hostname == "127.0.0.1" + and value.port + and not value.query + and not value.fragment, + "numeric PG DSN", + ) + return urlunsplit((value.scheme, value.netloc, "/" + database(db), "", "")) + + +def validate(spec: dict) -> None: + fields = { + "schema", + "fixture_id", + "root", + "head", + "tree", + "python", + "python_sha256", + "lockfile_sha256", + "helper_path", + "source_hashes", + "workdir", + "ports", + "databases", + "stores", + "pg_admin_dsn", + "pg_standby_dsn", + "ch", + "dependency_receipt", + "ci_run", + } + need(set(spec) == fields and type(spec["schema"]) is int and spec["schema"] == 2, "spec fields") + need(str(uuid.UUID(spec["fixture_id"])) == spec["fixture_id"], "fixture UUID") + for field in ("root", "python", "helper_path", "workdir"): + need(Path(spec[field]).is_absolute(), "absolute fixture path") + need( + all( + isinstance(spec[field], str) and re.fullmatch(r"[a-f0-9]{40}", spec[field]) + for field in ("head", "tree") + ), + "full source identity", + ) + need( + set(spec["source_hashes"]) == set(FILES) | {"driver", "entrypoint", "workflow", PROTOCOL}, + "source hash set", + ) + need(spec["source_hashes"][PROTOCOL] == PROTOCOL_SHA, "fixed capture protocol") + need( + all( + isinstance(value, str) and re.fullmatch(r"[a-f0-9]{64}", value) + for value in spec["source_hashes"].values() + ), + "source digest values", + ) + need(set(spec["ports"]) == {"api", "router", "probe", "tls"}, "port roles") + need( + len(set(spec["ports"].values())) == 4 + and all(type(p) is int and 0 < p < 65536 for p in spec["ports"].values()), + "distinct ports", + ) + need( + set(spec["databases"]) == {"pg_cell", "pg_directory", "ch_schema", "ch_telemetry"}, + "DB roles", + ) + need(len(set(spec["databases"].values())) == 4, "distinct databases") + for name in spec["databases"].values(): + database(name) + pg_dsn(spec["pg_admin_dsn"], spec["databases"]["pg_cell"]) + need( + set(spec["ch"]) == {"host", "port", "username", "password", "keeper_prefix"}, + "CH input fields", + ) + need( + spec["ch"]["host"] == "127.0.0.1" + and type(spec["ch"]["port"]) is int + and 0 < spec["ch"]["port"] < 65536, + "numeric CH endpoint", + ) + need( + os.environ.get("GITHUB_ACTIONS") == "true" and os.environ.get("CI") == "true", + "GitHub CI only", + ) + need( + spec["ci_run"] + == { + "run_id": os.environ.get("GITHUB_RUN_ID"), + "attempt": os.environ.get("GITHUB_RUN_ATTEMPT"), + "repository": os.environ.get("GITHUB_REPOSITORY"), + } + and str(spec["ci_run"]["run_id"]).isdigit(), + "actual CI run binding", + ) + dependencies(spec) + need(set(spec["stores"]) == {"clickhouse", "pg_primary", "pg_standby"}, "three owned stores") + for role, expected in spec["stores"].items(): + store_fields = { + "role", + "container_id", + "container_name", + "created_at", + "owner_evidence", + "host_pid", + "start_ticks", + "boot_id", + "uid", + "ports", + "memory_limit_bytes", + "server_limit_bytes", + "shared_buffers_bytes", + "retained", + "disk_path", + } + need(set(expected) == store_fields, "store input fields") + need(expected["role"] == role and expected["retained"] is False, "job-created store") + need( + type(expected["host_pid"]) is int + and expected["host_pid"] > 1 + and str(expected["start_ticks"]).isdigit(), + "store birth input", + ) + for mapping in expected["ports"]: + need( + set(mapping) == {"host", "host_port", "internal_port", "protocol"} + and mapping["host"] == "127.0.0.1" + and mapping["protocol"] == "tcp", + "store loopback mapping", + ) + if role == "clickhouse": + port = spec["ch"]["port"] + need( + expected["memory_limit_bytes"] == 5 * GIB + and expected["server_limit_bytes"] == 9 * GIB // 2, + "fixed CH caps", + ) + else: + target = urlsplit( + spec["pg_admin_dsn"] if role == "pg_primary" else spec["pg_standby_dsn"] + ) + need( + target.scheme in {"postgres", "postgresql"} + and target.hostname == "127.0.0.1" + and target.port + and not target.query + and not target.fragment, + "numeric owned PG endpoint", + ) + port = target.port + need( + type(expected["memory_limit_bytes"]) is int + and expected["memory_limit_bytes"] > 0 + and type(expected["shared_buffers_bytes"]) is int + and expected["shared_buffers_bytes"] > 0, + "observed PG caps", + ) + need(any(m["host_port"] == port for m in expected["ports"]), "store endpoint binding") + proof = ref(expected["owner_evidence"]) + need( + proof.get("container_id") == expected["container_id"] + and proof.get("created_at") == expected["created_at"] + and proof.get("task") == "cell-trigger-acceptance" + and proof.get("ci_run") == spec["ci_run"], + "positive creator evidence", + ) + + +def child_env(spec: dict, settings: dict, *, ca: bool = False) -> dict: + env = { + "PATH": "/usr/local/bin:/usr/bin:/bin", + "LANG": "C.UTF-8", + "TZ": "UTC", + "PYTHONPATH": spec["root"], + "LOOM_ENV_FILE": "", + "LOOM_SETTINGS_FILE": "", + "HOME": str(Path(spec["workdir"]) / "home"), + "TMPDIR": str(Path(spec["workdir"]) / "tmp"), + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONPYCACHEPREFIX": str(Path(spec["workdir"]) / "bytecode"), + "GITHUB_ACTIONS": os.environ.get("GITHUB_ACTIONS", ""), + "CI": os.environ.get("CI", ""), + **{ + "GITHUB_" + k: os.environ.get("GITHUB_" + k, "") + for k in ("RUN_ID", "RUN_ATTEMPT", "REPOSITORY") + }, + "LOOM_INSTANCE_LABEL": "iso:trigger:" + spec["fixture_id"][:12], + } + for key, value in settings.items(): + need(re.fullmatch(r"[a-z][a-z0-9_]*", key), "setting name") + env["LOOM_" + key.upper()] = value if isinstance(value, str) else json.dumps(value) + if ca: + env["SSL_CERT_FILE"] = str(Path(spec["workdir"]) / "ca.crt") + return env + + +def load_owned(path: Path): + from owned_children import OwnedChildren + + return OwnedChildren() + + +def error_metadata( + error: BaseException, *, files: tuple[str, ...] = FILES, fallback: bool = True +) -> dict: + site = {"file": "unknown", "line": 0, "function": "unknown"} + frame = sys._getframe(1) if fallback else None + line = frame.f_lineno if frame is not None else 0 + trace = error.__traceback__ + for _ in range(64): + if trace is None: + break + candidate = trace.tb_frame + if Path(candidate.f_code.co_filename).name in files and candidate.f_code.co_name not in { + "need", + "closed", + "equal", + "integer", + "identifier", + "hex64", + }: + frame = candidate + line = trace.tb_lineno + trace = trace.tb_next + while frame is not None and frame.f_code.co_name in {"select", "block_teardown"}: + frame = frame.f_back + if frame is not None: + line = frame.f_lineno + if frame is not None and Path(frame.f_code.co_filename).name in files: + site = { + "file": Path(frame.f_code.co_filename).name, + "line": line, + "function": frame.f_code.co_name[:64], + } + return { + "error_type": type(error).__name__[:64], + "site": site, + "outcome": "refused" + if isinstance(error, Refused) + else "error" + if isinstance(error, Exception) + else "cancelled", + } + + +def native_failure_metadata(error: BaseException, *, files: tuple[str, ...] = FILES) -> dict: + metadata = error_metadata(error, files=files) + if not re.fullmatch(r"[A-Za-z_][A-Za-z_0-9]{0,63}", metadata["error_type"]): + metadata["error_type"] = "unknown" + code = getattr(error, "code", None) + code = code if type(code) is int and 0 <= code < 2**31 else None + causes = [] + seen = {id(error)} + cause = error.__cause__ if error.__cause__ is not None else error.__context__ + for _ in range(4): + if cause is None or id(cause) in seen: + break + seen.add(id(cause)) + name = type(cause).__name__ + number = getattr(cause, "errno", None) + causes.append( + { + "error_type": name + if re.fullmatch(r"[A-Za-z_][A-Za-z_0-9]{0,63}", name) + else "unknown", + "errno": number if type(number) is int and -(2**31) <= number < 2**31 else None, + } + ) + cause_site = error_metadata(cause, files=files, fallback=False)["site"] + if cause_site["file"] != "unknown": + causes[-1]["site"] = cause_site + sqlstate = getattr(cause, "sqlstate", None) + if type(sqlstate) is str and re.fullmatch(r"[0-9A-Z]{5}", sqlstate): + causes[-1]["sqlstate"] = sqlstate + if name == "QueryCanceledError" and sqlstate == "57014": + # Compare fixed server messages; exception text never enters the receipt. + message = getattr(cause, "message", None) + causes[-1]["cancel_reason"] = ( + { + "canceling statement due to statement timeout": "statement_timeout", + "canceling statement due to user request": "user_request", + }.get(message, "unknown") + if type(message) is str + else "unknown" + ) + cause = cause.__cause__ if cause.__cause__ is not None else cause.__context__ + driver = {"code": code, "causes": causes} + sqlstate = getattr(error, "sqlstate", None) + if type(sqlstate) is str and re.fullmatch(r"[0-9A-Z]{5}", sqlstate): + driver["sqlstate"] = sqlstate + extra = getattr(error, "extra", None) + if type(extra) is dict and type(extra.get("outcome_unknown")) is bool: + driver["outcome_unknown"] = extra["outcome_unknown"] + if code is not None or causes or len(driver) > 2: + metadata["driver"] = driver + return metadata + + +class Owner: + def __init__(self, spec: dict, owned, note) -> None: + self.spec, self.owned, self.note = spec, owned, note + self.leaders: list[dict] = [] + self.failures: list[list] = [] + self.phase = "owner" + self.failure_overflow = False + self.failure_attempted = 0 + self.failure_written = 0 + self.diagnostic_failures = 0 + self.interruption = Refused("interrupted") + self.stopping = False + self.terminal_reporting = False + self.blocked = False + self.started = time.monotonic() + self.serial = 0 + self.sampled = 0.0 + self.guard_stop = threading.Event() + self.guard = None + + def select(self, error: BaseException, *, phase: str | None = None) -> None: + # Select before formatting evidence. No IO in a signal handler. + if len(self.failures) >= 64: + self.failure_overflow = True + return + entry = [error, {"phase": phase or self.phase}] + self.failures.append(entry) + if len(self.failures) > 64: + self.failure_overflow = True + del self.failures[64:] + selected_phase = entry[1]["phase"] + unknown = {"file": "unknown", "line": 0, "function": "unknown"} + try: + entry[1] = {"phase": selected_phase, **error_metadata(error)} + if isinstance(error, Refused): + if error.command is not None: + entry[1]["command"] = error.command + if error.native is not None: + entry[1]["native"] = error.native + except BaseException as secondary: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + entry[1] = { + "phase": selected_phase, + "error_type": type(error).__name__[:64], + "site": unknown, + "outcome": "evidence-error", + } + if len(self.failures) < 64: + self.failures.append( + [ + secondary, + { + "phase": "diagnostic", + "error_type": type(secondary).__name__[:64], + "site": unknown, + "outcome": "error", + }, + ] + ) + else: + self.failure_overflow = True + if len(self.failures) > 64: + self.failure_overflow = True + del self.failures[64:] + + def failure_summary(self) -> dict: + records = [] + snapshot = self.failures[:64] + for index, (_, record) in enumerate(snapshot): + if "outcome" not in record: + break + records.append(dict(index=index, primary=index == 0, **record)) + return { + "records": records, + "pending_records": len(snapshot) - len(records), + "overflow": self.failure_overflow or len(self.failures) > 64, + "event_attempts": self.failure_attempted, + "event_records_written": self.failure_written, + "diagnostic_write_failures": self.diagnostic_failures, + } + + def flush_failures(self) -> None: + # Snapshot once: failed evidence writes cannot recursively create more writes here. + records = self.failure_summary()["records"] + for index in range(self.failure_attempted, len(records)): + self.failure_attempted = index + 1 + try: + self.note("failure", **records[index]) + self.failure_written += 1 + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.diagnostic_failures += 1 + self.select(error, phase="diagnostic") + + @property + def first(self) -> BaseException | None: + return self.failures[0][0] if self.failures else None + + def record(self, event: str, **fields) -> None: + try: + self.note(event, **fields) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error, phase="diagnostic") + + def interrupt(self, _signum, _frame) -> None: + if not self.stopping: + self.stopping = True + self.select(self.interruption, phase="interrupt") + if self.terminal_reporting: + # Closed ownership permits cancellation to escape the final return boundary. + first = self.first + raise self.interruption if first is None else first + + def sample(self, *, admit: bool = False) -> None: + try: + now = time.monotonic() + available = next( + int(line.split()[1]) * 1024 + for line in Path("/proc/meminfo").read_text().splitlines() + if line.startswith("MemAvailable:") + ) + need(available >= POLICY["admit_mem_bytes" if admit else "stop_mem_bytes"], "RAM floor") + for path in [ + Path(self.spec["workdir"]), + *[Path(s["disk_path"]) for s in self.spec["stores"].values()], + ]: + need(shutil.disk_usage(path).free >= 16 * GIB, "disk floor") + boot = Path("/proc/sys/kernel/random/boot_id").read_text().strip() + for store in self.spec["stores"].values(): + identity = self.owned.identity(store["host_pid"]) + need(identity is not None, "resource store birth changed") + need(identity[1] == store["start_ticks"], "resource store birth changed") + need(boot == store["boot_id"], "resource store birth changed") + need( + os.stat(f"/proc/{store['host_pid']}").st_uid == store["uid"], + "resource store birth changed", + ) + roots = [self.owned.pid] + [s["host_pid"] for s in self.spec["stores"].values()] + births = {pid: self.owned.identity(pid) for pid in roots} + need(all(births.values()), "resource root disappeared") + observed = dict(births) + disappeared = set() + + def complete_forest() -> bool: + pending = list(roots) + seen = set() + while pending: + need(time.monotonic() - now < 3, "resource census stale") + pid = pending.pop() + if pid in seen: + continue + seen.add(pid) + before = self.owned.identity(pid) + if before is None: + need(pid not in births, "resource root disappeared") + disappeared.add(pid) + return False + need( + pid not in disappeared + and (pid not in observed or observed[pid][:2] == before[:2]), + "resource process birth changed", + ) + observed[pid] = before + need( + self.owned.rss(pid) * 1024 <= POLICY["stop_process_rss_bytes"], + "process RSS", + ) + try: + children = self.owned.children(pid) + except (FileNotFoundError, ProcessLookupError): + need( + pid not in births and self.owned.identity(pid) is None, + "resource child census unknown", + ) + disappeared.add(pid) + return False + after = self.owned.identity(pid) + if after is None: + need(pid not in births, "resource root disappeared") + disappeared.add(pid) + return False + need(after[:2] == before[:2], "resource process birth changed") + pending.extend(children) + return True + + # Discard an incomplete traversal; only a fresh whole forest can renew the sample. + for _attempt in range(3): + need(time.monotonic() - now < 3, "resource census stale") + for pid, birth in births.items(): + current = self.owned.identity(pid) + need( + current is not None and current[:2] == birth[:2], + "resource root birth changed", + ) + need( + Path("/proc/sys/kernel/random/boot_id").read_text().strip() == boot + and os.stat(f"/proc/{self.owned.pid}").st_uid == os.geteuid(), + "resource owner birth changed", + ) + for store in self.spec["stores"].values(): + need( + births[store["host_pid"]][1] == store["start_ticks"] + and Path("/proc/sys/kernel/random/boot_id").read_text().strip() + == store["boot_id"] + and os.stat(f"/proc/{store['host_pid']}").st_uid == store["uid"], + "resource store birth changed", + ) + if complete_forest(): + need(time.monotonic() - now < 3, "resource census stale") + break + else: + raise Refused("resource process census remained incomplete") + for item in tuple(self.leaders): + need(item["log"].stat().st_size <= 16 * 1024 * 1024, "child log bound") + driver_artifact = Path(self.spec["workdir"]) / "driver.jsonl" + if driver_artifact.exists(): + need(driver_artifact.stat().st_size <= 64 * 1024 * 1024, "driver artifact bound") + summary = Path(self.spec["workdir"]) / "mint-summary.stdout" + if summary.exists(): + need(summary.stat().st_size <= 4096, "mint stdout bound") + total = 0 + for arm in range(10): + path = Path(self.spec["workdir"]) / "delivery-intent" / f"arm-{arm}.jsonl" + if path.exists(): + size = path.stat().st_size + need(size <= 32 * 1024 * 1024, "intent file bound") + total += size + need(total <= 320 * 1024 * 1024, "intent aggregate bound") + self.sampled = now + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error, phase="resource") + + def start_guard(self) -> None: + def watch() -> None: + try: + while not self.guard_stop.wait(1): + self.sample() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error) + + self.sample(admit=True) + if self.first is not None: + raise self.first + self.guard = threading.Thread(target=watch, name="trigger-resource-readonly", daemon=False) + self.guard.start() + + def monitor(self, *, admit: bool = False, cleanup: bool = False) -> None: + self.flush_failures() + if admit: + self.sample(admit=True) + if time.monotonic() - self.started >= 7200: + self.select(Refused("global wall overrun; cleanup retains exclusion")) + if not cleanup: + try: + need( + not self.stopping and time.monotonic() - self.started < 7020, + "global work deadline", + ) + need(time.monotonic() - self.sampled < 3, "resource sampler stale") + for item in self.leaders: + if item["persistent"]: + need(item["process"].poll() is None, "fixture leader ended") + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error) + if self.first is not None and not cleanup: + raise self.first + + def start( + self, + role: str, + argv: list[str], + env: dict, + *, + persistent: bool = False, + cleanup: bool = False, + ) -> dict: + self.monitor(cleanup=cleanup) + source(self.spec) + argv = list(argv) + executable = shutil.which(argv[0], path=env.get("PATH")) + need(executable is not None and Path(executable).is_absolute(), "known child executable") + argv[0] = executable + self.serial += 1 + label = role + "-" + str(self.serial) + log_path = Path(self.spec["workdir"]) / (label + ".log") + fd = os.open(log_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + item = None + stdout_fd = None + try: + if role == "mint-token": + stdout_fd = os.open( + Path(self.spec["workdir"]) / "mint-summary.stdout", + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, + ) + process = subprocess.Popen( + argv, + cwd=self.spec["root"], + env=env, + stdout=stdout_fd if stdout_fd is not None else fd, + stderr=fd if stdout_fd is not None else subprocess.STDOUT, + start_new_session=True, + ) + # Register Popen before any observation/callback can fail. + item = { + "role": label, + "process": process, + "pidfd": None, + "identity": None, + "persistent": persistent, + "log": log_path, + "terminal": None, + "argv": list(argv), + "configuration_sha256": digest(env), + "launch_receipt": None, + "terminal_receipt": None, + } + item["signals"] = [] + self.leaders.append(item) + identity = self.owned.identity(process.pid) + need(identity and identity[0] == self.owned.pid, "direct child birth") + need(os.stat(f"/proc/{process.pid}").st_uid == os.geteuid(), "child UID") + item["identity"] = identity + item["pidfd"] = os.pidfd_open(process.pid) + after = self.owned.identity(process.pid) + need(after is not None and after[:2] == identity[:2], "stable pinned child") + item["launch_receipt"] = write( + Path(self.spec["workdir"]) / (label + "-birth.json"), + { + "pid": process.pid, + "start_ticks": identity[1], + "ppid": identity[0], + "uid": os.geteuid(), + "boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + "cwd": self.spec["root"], + "configuration_sha256": digest(env), + "utc": utc(), + }, + ) + self.record("birth", role=label, pid=process.pid, start_ticks=identity[1]) + if self.first is not None and not cleanup: + raise self.first + return item + except BaseException as error: + self.select(error) + raise + finally: + if stdout_fd is not None: + try: + os.close(stdout_fd) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error) + try: + os.close(fd) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error) + + def poll(self, item: dict) -> int | None: + result = item["process"].poll() + if result is not None and item["terminal"] is None: + item["terminal"] = {"returncode": result, "utc": utc()} + item["persistent"] = False + try: + item["terminal_receipt"] = write( + Path(self.spec["workdir"]) / (item["role"] + "-terminal.json"), item["terminal"] + ) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error) + self.record("terminal", role=item["role"], returncode=result) + return result + + def command( + self, role: str, argv: list[str], env: dict, *, seconds: float = 120, cleanup: bool = False + ) -> dict: + item = self.start(role, argv, env, cleanup=cleanup) + end = ( + time.monotonic() + seconds + if cleanup + else min(self.started + 7020, time.monotonic() + seconds) + ) + while self.poll(item) is None: + self.monitor(cleanup=cleanup) + need(time.monotonic() < end, "serial command deadline") + time.sleep(0.05) + if item["process"].returncode != 0: + error = Refused("native command refused") + error.command = { + "role": role if role in {"inspect", "keeper-prefix", "create"} else "other", + "serial": self.serial, + "returncode": item["process"].returncode, + } + raise error + if self.first is not None and not cleanup: + raise self.first + return item + + def send(self, item: dict, sig: int) -> None: + if self.poll(item) is not None: + return + before = self.owned.identity(item["process"].pid) + if ( + before is None + or before[0] != self.owned.pid + or (item["identity"] is not None and before[:2] != item["identity"][:2]) + ): + self.blocked = True + raise Refused("child identity mismatch") + # No generic reaper runs while Popen owns this direct child, so an unreaped + # Popen leader cannot have its PID reused during an interrupted birth capture. + if item["identity"] is None: + item["identity"] = before + if item["pidfd"] is None: + item["pidfd"] = os.pidfd_open(item["process"].pid) + need(os.stat(f"/proc/{item['process'].pid}").st_uid == os.geteuid(), "signal UID") + after = self.owned.identity(item["process"].pid) + need(after is not None and after[:2] == before[:2], "signal birth recheck") + try: + signal.pidfd_send_signal(item["pidfd"], sig) + item.setdefault("signals", []).append(sig) + except ProcessLookupError: + need(self.poll(item) is not None, "lost pinned child") + + def stop_one(self, item: dict, *, driver: bool = False, api: bool = False) -> None: + if self.poll(item) is not None: + self.terminal_outcome(item, driver=driver, api=api) + return + phases = ( + [(signal.SIGINT, 60), (signal.SIGTERM, 15)] + if driver + else [(signal.SIGTERM, 90 if api else 15)] + ) + for sig, seconds in phases + [(signal.SIGKILL, 5)]: + if self.poll(item) is not None: + break + if sig == signal.SIGKILL or (driver and sig == signal.SIGTERM): + self.select(Refused("cleanup escalation")) + self.send(item, sig) + end = time.monotonic() + seconds + while self.poll(item) is None and time.monotonic() < end: + self.monitor(cleanup=True) + time.sleep(0.05) + need(self.poll(item) is not None, "child not reaped") + item["process"].wait() + self.terminal_outcome(item, driver=driver, api=api) + + def terminal_outcome(self, item: dict, *, driver: bool, api: bool) -> None: + role = item["role"].split("-", 1)[0] + result = item["process"].returncode + sent = item.get("signals", []) + if driver: + ok = result in (0, 1) and not sent + elif api or role in {"router", "proxy"}: + # Uvicorn re-raises its captured TERM after graceful shutdown; the + # private proxy consumes TERM. Only an owner-requested stop qualifies. + ok = signal.SIGTERM in sent and result in ( + (0, -signal.SIGTERM) if role != "proxy" else (0,) + ) + else: + return + if not ok: + self.select(Refused("unexpected fixture terminal outcome")) + + def cleanup(self) -> None: + # Each Popen owns its status until all leaders are reaped. + self.flush_failures() + if self.owned is None: + need(not self.leaders, "uninitialized ownership with registered children") + return + for prefix in ("driver", "api", "router", "proxy", ""): + for item in self.leaders: + if item.get("cleaned") or (prefix and not item["role"].startswith(prefix)): + continue + try: + self.stop_one( + item, + driver=item["role"].startswith("driver"), + api=item["role"].startswith("api"), + ) + item["cleaned"] = True + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error) + if self.blocked or any(self.poll(item) is None for item in self.leaders): + raise Refused("owned leader cleanup unresolved; exclusion retained") + try: + self.owned.stop(None, lambda *args, **kwargs: self.record("adopted-cleanup", **kwargs)) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error) + need(not self.owned.children(self.owned.pid), "kernel children remain") + for item in self.leaders: + if item["pidfd"] is not None: + try: + os.close(item["pidfd"]) + item["pidfd"] = None + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error) + + +def runtime(spec: dict) -> int: + """The separately admitted path; never called by inert controls.""" + from native import Fixture + + validate(spec) + source(spec) + for port in spec["ports"].values(): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as check: + check.bind(("127.0.0.1", port)) + workdir = Path(spec["workdir"]) + need(workdir.is_absolute() and not workdir.exists(), "exclusive work directory") + os.mkdir(workdir, 0o700) + for name in ("home", "tmp", "bytecode"): + (workdir / name).mkdir(mode=0o700) + events = os.open( + workdir / "events.jsonl", os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600 + ) + + def note(event: str, **fields) -> None: + raw = (json.dumps(dict(utc=utc(), event=event, **fields), allow_nan=False) + "\n").encode() + need( + len(raw) <= 65536 and os.lseek(events, 0, os.SEEK_CUR) + len(raw) <= 16 * 1024 * 1024, + "event bound", + ) + need(os.write(events, raw) == len(raw), "complete event write") + os.fsync(events) + + owner = None + lock = None + fixture = None + release = False + verdict = "INVALID" + first = None + events_closed = False + lock_closed = False + + def selected() -> BaseException | None: + retained = owner.first if owner is not None else None + return retained if retained is not None else first + + try: + owner = Owner(spec, None, note) + owner.phase = "ownership" + owner.owned = load_owned(Path(spec["helper_path"])) + for signum in (signal.SIGINT, signal.SIGTERM): + signal.signal(signum, owner.interrupt) + owner.start_guard() + owner.monitor(admit=True) + lock = os.open( + "/tmp/loom-full-unit-suite.lock", os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600 + ) + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + # This isolated job owns the canonical slot through final native cleanup. + fixture = Fixture(spec, owner) + owner.phase = "setup" + fixture.setup() + owner.monitor(admit=True) + owner.phase = "measure" + result = fixture.measure() + owner.phase = "provenance" + fixture.provenance() + verdict = result + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error + if owner is not None: + owner.select(error) + finally: + if owner is not None: + try: + owner.phase = "cleanup" + owner.cleanup() + if fixture is not None: + owner.phase = "teardown" + fixture.teardown() + owner.phase = "cleanup" + owner.cleanup() + release = True + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + owner.select(error) + else: + release = True + if not release: + # Never exit over unreaped children or uncertain task writes. + while True: + if owner is not None: + try: + owner.phase = "cleanup" + owner.cleanup() + if fixture is not None: + owner.phase = "teardown" + fixture.teardown() + owner.phase = "cleanup" + owner.cleanup() + release = True + break + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + owner.select(error) + try: + time.sleep(1) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + if owner is not None: + owner.select(error) + if owner is not None: + try: + owner.phase = "monitor-stop" + owner.monitor(cleanup=True) + owner.guard_stop.set() + if owner.guard is not None: + owner.guard.join() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + owner.select(error) + owner.flush_failures() + first = owner.first if owner.first is not None else first + try: + os.fsync(events) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + if owner is not None: + owner.select(error, phase="events-close") + first = error if first is None else first + finally: + try: + os.close(events) + events_closed = True + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + if owner is not None: + owner.select(error, phase="events-close") + first = error if first is None else first + first = selected() + if first is not None: + verdict = "INVALID" + try: + write( + workdir / "outer-result.json", + { + "schema": 2, + "fixture_id": spec["fixture_id"], + "candidate_verdict": verdict, + "native_cleanup_complete": release + and (fixture is None or not fixture.teardown_incomplete), + "error_type": type(first).__name__ if first is not None else "", + "failure_evidence": owner.failure_summary() if owner is not None else None, + "utc": utc(), + }, + ) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + if owner is not None: + owner.select(error, phase="result") + first = error if first is None else first + verdict = "INVALID" + if lock is not None and release: + try: + os.close(lock) + lock_closed = True + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + if owner is not None: + owner.select(error, phase="unlock") + first = error if first is None else first + verdict = "INVALID" + if owner is not None and release and events_closed and (lock is None or lock_closed): + try: + owner.terminal_reporting = True + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + owner.select(error, phase="reporting-mode") + try: + first = selected() + if first is not None: + verdict = "INVALID" + # Candidate bytes require successful terminal status; they grant no PASS alone. + print( + json.dumps( + { + "candidate_verdict": verdict, + "error_type": type(first).__name__ if first is not None else "", + "failure_evidence": owner.failure_summary() if owner is not None else None, + } + ), + flush=True, + ) + return 0 if verdict == "PASS" and selected() is None else 1 + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + if owner is not None: + owner.select(error, phase="stdout") + first = selected() + first = error if first is None else first + verdict = "INVALID" + try: + write( + workdir / "reporting-failure.json", + { + "schema": 2, + "verdict": "INVALID", + "error_type": type(first).__name__, + "failure_evidence": owner.failure_summary() if owner is not None else None, + }, + ) + except BaseException as secondary: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + if owner is not None: + owner.select(secondary, phase="reporting") + return 1 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--spec", required=True, type=Path) + args = parser.parse_args() + try: + return runtime(read(args.spec)) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + print(json.dumps({"verdict": "INVALID", "error_type": type(error).__name__}), flush=True) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gates/testsupport/cell_trigger/native.py b/scripts/gates/testsupport/cell_trigger/native.py new file mode 100644 index 0000000000..79e58d48a9 --- /dev/null +++ b/scripts/gates/testsupport/cell_trigger/native.py @@ -0,0 +1,2521 @@ +"""Native operations for the isolated trigger acceptance fixture.""" + +from __future__ import annotations + +import argparse +import asyncio +import datetime +import hashlib +import json +import os +import re +import socket +import sys +import time +import uuid +from contextlib import asynccontextmanager +from pathlib import Path + +from capture import ( + Codec, + Workload, + closed, + equal, + hex64, + identifier, + integer, + json_object, + mint_metadata, + protocol, + verify_token, +) +from fixture import ( + FILES, + GIB, + HERE, + POLICY, + PROTOCOL_SHA, + Refused, + child_env, + database, + digest, + native_failure_metadata, + need, + pg_dsn, + read, + ref, + sha, + utc, + write, +) + +COUNTS = ("head_global", "head_scope", "revision_global", "revision_scope") +DEBT = ( + "delivery_global", + "delivery_scope", + "audit_global", + "audit_scope", + "error_global", + "error_scope", +) +PG_FIELDS = ( + tuple( + "c_" + k + for k in ( + "org", + "namespace", + "agent", + "user_id", + "session_id", + "memory_key", + "memory_id", + "version", + "seq", + "content_sha256", + ) + ) + + ("head",) + + tuple( + "r_" + k + for k in ( + "org", + "namespace", + "seq", + "user_id", + "memory_id", + "version", + "content_sha256", + "actor", + "source_class", + "request_id", + ) + ) + + ("revision", "audit", "targets", "attempts", "last_error") +) + + +def pg_census_sql(): + # One statement snapshot supplies all ten fixed counts and server observations. + counts = [] + for label, table in ( + ("head", "memory_current"), + ("revision", "memory_revisions"), + ("delivery", "memory_deliveries"), + ("audit", "audit_outbox"), + ("error", "memory_deliveries"), + ): + predicate = {"delivery": "targets<>0", "error": "(attempts<>0 OR last_error<>'')"}.get( + label, "TRUE" + ) + limit = 45501 if label in {"head", "revision"} else 1 + for scope in ("global", "scope"): + where = predicate + (" AND org=$1 AND namespace=$2" if scope == "scope" else "") + counts.append( + f"(SELECT count(*) FROM (SELECT 1 FROM loom.{table} WHERE " + f"{where} LIMIT {limit}) bounded) AS {label}_{scope}" + ) + return ( + "SELECT " + + ",".join(counts) + + ( + ",CASE WHEN octet_length(pg_current_snapshot()::text)<=16384 " + "THEN pg_current_snapshot()::text ELSE NULL END AS " + "snapshot,current_setting('transaction_isolation') AS " + "isolation,current_setting('transaction_read_only') AS " + "read_only,current_database() AS database,pg_backend_pid() " + "AS backend_pid" + ) + ) + + +def pg_record(value, fields): + need( + value is not None and list(value.keys()) == list(fields), "closed ordered PG result columns" + ) + return dict(value) + + +async def begin_observation(conn, context, isolation, *, backend=None): + initial = conn.is_in_transaction() + need(initial is False, "idle PG snapshot boundary") + statement = "BEGIN ISOLATION LEVEL " + isolation.upper() + " READ ONLY" + status = await conn.execute(statement) + active = conn.is_in_transaction() + need(status == "BEGIN" and active is True, "PG read-only BEGIN acknowledged") + value = pg_record( + await conn.fetchrow(pg_census_sql(), context["org"], context["namespace"]), + ( + *COUNTS[:2], + *COUNTS[2:], + *DEBT[:2], + *DEBT[2:4], + *DEBT[4:], + "snapshot", + "isolation", + "read_only", + "database", + "backend_pid", + ), + ) + for key in COUNTS: + need(integer(value[key]) == 45500, "exact global/scoped PG census") + for key in DEBT: + need(integer(value[key], 0, 1) == 0, "settled global/scoped PG debt") + snapshot = value.pop("snapshot") + need( + type(snapshot) is str + and len(snapshot.encode()) <= 16384 + and re.fullmatch(r"[0-9]+:[0-9]+:(?:[0-9]+(?:,[0-9]+)*)?", snapshot), + "actual bounded PG snapshot", + ) + need( + value["isolation"] == isolation + and value["read_only"] == "on" + and value["database"] == context["databases"]["pg_cell"], + "actual PG snapshot settings/database", + ) + integer(value["backend_pid"], 1, 2**31 - 1) + need(backend is None or backend == value["backend_pid"], "same owned PG backend") + return dict( + snapshot=snapshot, + snapshot_bytes=len(snapshot.encode()), + snapshot_sha256=hashlib.sha256(snapshot.encode()).hexdigest(), + isolation=value["isolation"], + read_only=value["read_only"], + database=value["database"], + backend_pid=value["backend_pid"], + begin_status=status, + initial_transaction_state=initial, + active_transaction_state=active, + counts={key: value[key] for key in (*COUNTS, *DEBT)}, + ) + + +async def end_observation(conn, observation): + status = await conn.execute("COMMIT") + ended = conn.is_in_transaction() + need(status == "COMMIT" and ended is False, "PG snapshot end acknowledged") + observation.update(end_status=status, ended_transaction_state=ended) + + +async def ch_result(ch, query, parameters, fields, limit): + result = await ch.query( + query, + parameters=parameters, + settings={ + "max_result_rows": limit, + "max_result_bytes": 20 * 1024 * 1024, + "result_overflow_mode": "throw", + "max_execution_time": 30, + }, + ) + need(list(result.column_names) == list(fields), "closed ordered CH result columns") + data = result.result_rows + need( + type(data) in {list, tuple} + and len(data) <= limit + and all(type(row) in {list, tuple} and len(row) == len(fields) for row in data), + "bounded CH tuples before maps", + ) + return data + + +async def ch_census(ch, context): + db = database(context["databases"]["ch_schema"]) + observed = {} + for label, table in (("memory", "memory"), ("audit", "audit_log")): + for scope in ("global", "scope"): + where = "event_type='memory.revision'" if label == "audit" else "1" + if scope == "scope": + where += " AND org={org:String} AND namespace={ns:String}" + result = await ch_result( + ch, + f"SELECT count() AS n FROM (SELECT 1 FROM `{db}`.{table}" + + (" FINAL" if label == "memory" else "") + + f" WHERE {where} LIMIT 45501)", + {"org": context["org"], "ns": context["namespace"]}, + ("n",), + 1, + ) + need( + len(result) == 1 and integer(result[0][0]) == 45500, + "exact CH global/scoped workload census", + ) + observed[label + "_" + scope] = result[0][0] + return observed + + +def pg_page_sql(): + c = ( + "org", + "namespace", + "agent", + "user_id", + "session_id", + "memory_key", + "memory_id", + "version", + "seq", + "content_sha256", + ) + r = ( + "org", + "namespace", + "seq", + "user_id", + "memory_id", + "version", + "content_sha256", + "actor", + "source_class", + "request_id", + ) + fields = ( + [f"c.{k} AS c_{k}" for k in c] + + ["c.row::text AS head"] + + [f"r.{k} AS r_{k}" for k in r] + + [ + "r.row::text AS revision", + "r.audit::text AS audit", + "d.targets", + "d.attempts", + "d.last_error", + ] + ) + text = ["c." + k for k in c if k not in {"memory_id", "version", "seq"}] + [ + "r." + k for k in r if k not in {"memory_id", "version", "seq"} + ] + sizes = " AND ".join(f"octet_length({k})<=512" for k in text) + return ( + "SELECT " + + ",".join(fields) + + ( + " FROM loom.memory_current c JOIN loom.memory_revisions r ON " + "(r.org,r.namespace,r.seq)=(c.org,c.namespace,c.seq) LEFT " + "JOIN loom.memory_deliveries d ON " + "(d.org,d.namespace,d.seq)=(c.org,c.namespace,c.seq) WHERE " + "c.org=$1 AND c.namespace=$2 AND c.memory_id=ANY($3::uuid[]) " + "AND octet_length(c.row::text)<=65536 AND " + "octet_length(r.row::text)<=65536 AND " + "octet_length(r.audit::text)<=16384 AND " + ) + + sizes + + ( + " AND (d.last_error IS NULL OR " + "octet_length(d.last_error)<=512) ORDER BY " + "c.memory_id,c.agent,c.user_id,c.session_id,c.memory_key " + "LIMIT $4" + ) + ) + + +def complete_ids(values, expected): + # Inspect multiplicity BEFORE constructing lookup maps. + need( + len(values) == len(expected) + and len(set(values)) == len(values) + and set(values) == set(expected), + "complete unique result multiset before map", + ) + + +def typed_revision(stored, row, audit, item, seen): + for prefix, names in ( + ( + "c_", + ( + "org", + "namespace", + "agent", + "user_id", + "session_id", + "memory_key", + "memory_id", + "version", + "content_sha256", + ), + ), + ( + "r_", + ( + "org", + "namespace", + "user_id", + "memory_id", + "version", + "content_sha256", + "actor", + "source_class", + ), + ), + ): + for key in names: + value = stored[prefix + key] + if key == "memory_id": + value = identifier(value) + elif key == "version": + value = integer(value, 1, 2**63 - 1) + else: + need(type(value) is str and len(value.encode()) <= 512, "typed PG identity text") + equal(value, row[key]) + for key, expected in ( + ("memory_key", item["key"]), + ("memory_id", item["memory_id"]), + ("version", item["version"]), + ("content_sha256", item["content_sha256"]), + ): + equal(row[key], expected) + seq = integer(stored["c_seq"], 1, 2**63 - 1) + need( + integer(stored["r_seq"], 1, 2**63 - 1) == seq and seq not in seen, + "unique exact PG revision link", + ) + seen.add(seq) + need(len(seen) <= 45500, "PG seq bound") + equal(stored["r_request_id"], audit["request_id"]) + delivery = [stored[key] for key in ("targets", "attempts", "last_error")] + if delivery != [None, None, None]: + need( + integer(delivery[0], 0, 0) == 0 + and integer(delivery[1], 0, 0) == 0 + and type(delivery[2]) is str + and delivery[2] == "", + "settled typed PG delivery", + ) + return seq + + +def chain_update(chain, index, value): + raw = json.dumps( + dict(index=index, value=value), sort_keys=True, allow_nan=False, separators=(",", ":") + ).encode() + chain.update(len(raw).to_bytes(8, "big")) + chain.update(raw) + + +async def verify_pages(conn, ch, context, workload, codec, native_hash): + db = database(context["databases"]["ch_schema"]) + seen_seq = set() + seen_hash = set() + count = 0 + expected_chain = hashlib.sha256() + observed_chain = hashlib.sha256() + authority_chain = hashlib.sha256() + raw_keys = hashlib.sha256() + for page in workload.pages(): + need(0 < len(page) <= 100, "fixed proof page") + ids = [item["memory_id"] for item in page] + complete_ids(ids, ids) + fetched = await conn.fetch( + pg_page_sql(), + context["org"], + context["namespace"], + [uuid.UUID(value) for value in ids], + len(page) + 1, + ) + need(len(fetched) == len(page), "PG page exact cardinality/overflow") + values = [pg_record(row, PG_FIELDS) for row in fetched] + complete_ids([identifier(row["c_memory_id"]) for row in values], ids) + heads = {identifier(row["c_memory_id"]): row for row in values} + checked = {} + audits = {} + hashes = [] + for item in page: + stored = heads[item["memory_id"]] + head = json_object(stored["head"]) + revision = json_object(stored["revision"]) + row = codec.row(head, origin="pg", comparison=True) + other = codec.row(revision, origin="pg", comparison=True) + equal(row, other) + audit = codec.audit(json_object(stored["audit"]), origin="pg") + native = dict(audit, ts=datetime.datetime.fromisoformat(audit["ts"])) + need(native_hash(native), "actual decoded native audit hash") + seq = typed_revision(stored, row, audit, item, seen_seq) + codec.expected( + row, + audit, + context, + api=item["kind"] == "api", + metadata=workload.metadata, + driver=workload.driver_header + or dict(host=socket.gethostname(), pid=workload.driver["pid"]), + ) + if item["kind"] == "delivery": + equal(row, codec.row(item["row"], comparison=True)) + equal(audit, item["audit"]) + need(audit["row_hash"] not in seen_hash, "unique workload audit identity") + seen_hash.add(audit["row_hash"]) + hashes.append(audit["row_hash"]) + checked[item["memory_id"]] = row + audits[audit["row_hash"]] = audit + intent = { + key: item[key] for key in ("kind", "key", "memory_id", "version", "content_sha256") + } + observed = dict( + intent, + key=row["memory_key"], + memory_id=row["memory_id"], + version=row["version"], + content_sha256=row["content_sha256"], + ) + if item["kind"] == "delivery": + intent.update(row=codec.row(item["row"], comparison=True), audit=item["audit"]) + observed.update(row=row, audit=audit) + chain_update(expected_chain, count, intent) + chain_update(observed_chain, count, observed) + chain_update(authority_chain, count, dict(row=row, audit=audit, seq=seq)) + chain_update(raw_keys, count, dict(head=sorted(head), revision=sorted(revision))) + count += 1 + fields = codec.p["memory_fields"] + data = await ch_result( + ch, + "SELECT " + + ",".join("`" + key + "`" for key in fields) + + ( + f" FROM `{db}`.memory FINAL WHERE org={{org:String}} AND " + f"namespace={{ns:String}} AND memory_id IN " + f"{{ids:Array(UUID)}} ORDER BY " + f"memory_id,agent,user_id,session_id,memory_key LIMIT " + f"{len(page) + 1}" + ), + dict( + org=context["org"], ns=context["namespace"], ids=[uuid.UUID(value) for value in ids] + ), + fields, + len(page) + 1, + ) + complete_ids([identifier(row[fields.index("memory_id")]) for row in data], ids) + for value in data: + row = codec.row(dict(zip(fields, value, strict=True)), origin="ch", comparison=True) + equal(row, checked[row["memory_id"]]) + fields = codec.p["audit_fields"] + data = await ch_result( + ch, + "SELECT " + + ",".join("`" + key + "`" for key in fields) + + ( + f" FROM `{db}`.audit_log WHERE org={{org:String}} AND " + f"namespace={{ns:String}} AND event_type='memory.revision' " + f"AND row_hash IN {{hashes:Array(String)}} ORDER BY row_hash " + f"LIMIT {len(page) + 1}" + ), + dict(org=context["org"], ns=context["namespace"], hashes=hashes), + fields, + len(page) + 1, + ) + complete_ids([hex64(row[fields.index("row_hash")], native=True) for row in data], hashes) + for value in data: + audit = codec.audit(dict(zip(fields, value, strict=True)), origin="ch") + equal(audit, audits[audit["row_hash"]]) + need(count == len(seen_seq) == len(seen_hash) == 45500, "complete exact workload identities") + need( + expected_chain.hexdigest() == observed_chain.hexdigest(), + "streamed intended/observed proof chain", + ) + return dict( + rows=count, + api=workload.counts["api"], + delivery=workload.counts["delivery"], + expected_sha256=expected_chain.hexdigest(), + observed_sha256=observed_chain.hexdigest(), + authority_sha256=authority_chain.hexdigest(), + pg_raw_key_sets_sha256=raw_keys.hexdigest(), + ) + + +def codec_contract(): + from server.memory.store.pg_memory import DATETIME_COLS + from server.memory.store.rows import FIRST_VERSION_DEFAULTS, MEMORY_PERSISTED_COLS + + specification = protocol() + codec = Codec(specification) + equal(list(MEMORY_PERSISTED_COLS), specification["memory_fields"]) + equal(FIRST_VERSION_DEFAULTS, specification["sparse_defaults"]) + equal( + sorted(DATETIME_COLS), + sorted( + key + for key, value in specification["memory_types"].items() + if value.endswith("utc_second") + ), + ) + return codec + + +async def independent_provenance(context, output): + from server.observability.audit import row_matches_hash + + codec = codec_contract() + context = dict( + context, _api_server=read(Path(context["workdir"]) / "serving-cell/launch.json")["server"] + ) + first = None + summary = None + try: + with Workload(context, codec) as workload: + async with connections(context) as (conn, ch): + try: + page = await begin_observation(conn, context, "repeatable read") + initial_ch = await ch_census(ch, context) + checked = await verify_pages( + conn, ch, context, workload, codec, row_matches_hash + ) + await end_observation(conn, page) + final = await begin_observation( + conn, context, "read committed", backend=page["backend_pid"] + ) + await end_observation(conn, final) + need( + conn.is_in_transaction() is False, + "final CH checks outside both PG snapshots", + ) + final_ch = await ch_census(ch, context) + summary = dict( + schema=2, + fixture_id=context["fixture_id"], + verified=True, + provenance=True, + **checked, + pg_page_observation=page, + pg_final_observation=final, + ch_initial_census=initial_ch, + ch_final_census=final_ch, + inputs=workload.refs, + source=dict( + head=context["head"], + tree=context["tree"], + hashes=context["source_hashes"], + ), + driver_birth=workload.driver, + api_birth=context["_api_server"], + mint_metadata=context["mint_metadata"], + hash_and_independent_stamps=True, + utc=utc(), + ) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error + finally: + try: + if conn.is_in_transaction(): + await conn.execute("ROLLBACK") + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + if first is not None: + raise first + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + if first is not None: + raise first + need(summary is not None, "complete proof before closure") + write(output, summary) + + +def api(context: dict, output: Path) -> None: + import uvicorn + + from bench.cell_trigger_gate import _workload + from server.core import deployment_config, instances + from server.core.provider_process import _start_ticks + from server.settings import settings + + actual = settings.model_dump(mode="json") + need( + actual["router_operator_token"] == "" + and re.fullmatch(r"lcr_[0-9a-f]{128}", actual["cell_router_token"]), + "API credential posture", + ) + candidate = dict(actual, cell_router_token="") + common = {k: v for k, v in actual.items() if k != "cell_router_token"} + folder = output.parent + artifact = write( + folder / "configuration.json", + { + "schema": 2, + "api_configuration_sha256": digest(actual), + "cli_configuration_sha256": digest(candidate), + "shared_configuration_sha256": digest(common), + "workload": _workload(settings), + "fingerprints": deployment_config.fingerprints(settings), + "credential_posture": { + "api_cell_router_token_present": True, + "cli_cell_router_token_present": False, + "api_router_operator_token_present": False, + "cli_router_operator_token_present": False, + }, + }, + ) + write(folder / "private-settings.json", actual, limit=1024 * 1024) + write( + output, + { + "schema": 2, + "server": { + "pid": os.getpid(), + "start_ticks": _start_ticks(os.getpid()), + "boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + "uid": os.geteuid(), + "cwd": os.getcwd(), + "health_instance": f"{socket.gethostname()}:{os.getpid()}", + "instance_id": instances.instance_id(), + }, + "source": {"head": context["head"], "tree": context["tree"]}, + "configuration_artifact": artifact, + }, + ) + uvicorn.run( + "server.main:app", + app_dir=context["root"], + host="127.0.0.1", + port=context["ports"]["api"], + loop="asyncio", + workers=1, + reload=False, + ) + + +@asynccontextmanager +async def connections(context: dict, *, admin: bool = False): + import asyncpg + import clickhouse_connect + + from server.core import pg + + conn = ch = None + first = None + try: + dsn = ( + context["pg_admin_dsn"] + if admin + else pg_dsn(context["pg_admin_dsn"], context["databases"]["pg_cell"]) + ) + conn = await asyncpg.connect( + dsn, + timeout=10, + command_timeout=30, + server_settings={ + "application_name": "trigger-fixture:" + context["fixture_id"], + "statement_timeout": "30000", + "idle_in_transaction_session_timeout": "30000", + }, + ) + values = {k: context["ch"][k] for k in ("host", "port", "username", "password")} + ch = await clickhouse_connect.get_async_client( + **values, + database="", + query_retries=0, + connect_timeout=10, + send_receive_timeout=30, + settings={"log_comment": "fixture.private", "async_insert": 0}, + ) + yield conn, ch + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error + finally: + if ch is not None: + try: + async with asyncio.timeout(15): + await ch.close() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + if conn is not None: + try: + await pg.close_owned(conn, end=time.monotonic() + 15) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + if first is not None: + raise first + + +async def rows(ch, query: str, parameters: dict | None = None) -> list[dict]: + result = await ch.query(query, parameters=parameters or {}) + return list(result.named_results()) + + +async def stores(context: dict, role: str, output: Path, *, drop: bool = False) -> None: + spec = context + db = database(spec["databases"][role]) + pg_role = role.startswith("pg_") + async with connections(spec, admin=True) as (conn, ch): + if drop: + creation = read(Path(spec["workdir"]) / (role + "-created.json")) + need( + creation["database"] == db and creation["fixture_id"] == spec["fixture_id"], + "exact created database", + ) + expected_id = spec["stores"]["pg_primary" if pg_role else "clickhouse"]["container_id"] + need(creation["store_container_id"] == expected_id, "created store identity") + if pg_role: + need( + await conn.fetchval( + "SELECT count(*) FROM pg_stat_activity WHERE datname=$1", db + ) + == 0, + "scratch PG sessions", + ) + exists = await conn.fetchval( + "SELECT EXISTS(SELECT FROM pg_database WHERE datname=$1)", db + ) + if exists: + need( + str(await conn.fetchval("SELECT oid FROM pg_database WHERE datname=$1", db)) + == creation["database_id"], + "same created PG database", + ) + await conn.execute(f'DROP DATABASE "{db}"') + else: + # Refuse asynchronous or running writes; do not flush a global queue. + aliases = [spec["databases"]["ch_schema"], spec["databases"]["ch_telemetry"]] + need( + ( + await rows( + ch, + ( + "SELECT count() AS n FROM system.asynchronous_inserts WHERE " + "database IN {dbs:Array(String)}" + ), + {"dbs": aliases}, + ) + ) + == [{"n": 0}], + "CH async writers", + ) + live = await rows( + ch, + ( + "SELECT count() AS n FROM system.processes WHERE query_id != " + "currentQueryID() AND (current_database IN " + "{dbs:Array(String)} OR arrayExists(d -> position(query, d) " + "> 0, {dbs:Array(String)}))" + ), + {"dbs": aliases}, + ) + need(live == [{"n": 0}], "CH running writers") + current = await rows( + ch, + "SELECT toString(uuid) AS id FROM system.databases WHERE name={db:String}", + {"db": db}, + ) + need( + not current or current == [{"id": creation["database_id"]}], + "same created CH database", + ) + await ch.command(f"DROP DATABASE IF EXISTS `{db}` SYNC") + write( + output, + { + "schema": 2, + "fixture_id": spec["fixture_id"], + "database": db, + "dropped": True, + "utc": utc(), + }, + ) + return + need(await conn.fetchval("SELECT NOT pg_is_in_recovery()"), "PG primary") + need( + 160000 <= int(await conn.fetchval("SHOW server_version_num")) < 170000, + "owned PG16 primary", + ) + need(await conn.fetchval("SHOW max_prepared_transactions") == "0", "prepared transactions") + need(await conn.fetchval("SHOW synchronous_standby_names"), "synchronous standby names") + need( + await conn.fetchval( + "SELECT EXISTS(SELECT FROM pg_stat_replication WHERE " + "state='streaming' AND sync_state='sync' AND flush_lsn IS " + "NOT NULL)" + ), + "synchronous standby", + ) + need( + await conn.fetchval("SELECT pg_size_bytes(current_setting('shared_buffers'))") + == spec["stores"]["pg_primary"]["shared_buffers_bytes"], + "primary buffers", + ) + import asyncpg + + standby = await asyncpg.connect( + spec["pg_standby_dsn"], + timeout=10, + command_timeout=10, + server_settings={ + "default_transaction_read_only": "on", + "statement_timeout": "10000", + "application_name": "trigger-fixture:" + context["fixture_id"], + }, + ) + standby_error = None + try: + need(await standby.fetchval("SELECT pg_is_in_recovery()"), "standby recovery") + need( + 160000 <= int(await standby.fetchval("SHOW server_version_num")) < 170000, + "owned PG16 standby", + ) + need( + await standby.fetchval("SELECT pg_size_bytes(current_setting('shared_buffers'))") + == spec["stores"]["pg_standby"]["shared_buffers_bytes"], + "standby buffers", + ) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + standby_error = error + try: + from server.core import pg + + await pg.close_owned(standby, end=time.monotonic() + 15) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + standby_error = error if standby_error is None else standby_error + if standby_error is not None: + raise standby_error + limits = await rows( + ch, "SELECT value FROM system.server_settings WHERE name='max_server_memory_usage'" + ) + need( + len(limits) == 1 + and int(limits[0]["value"]) + == spec["stores"]["clickhouse"]["server_limit_bytes"] + == 9 * GIB // 2, + "CH server memory cap", + ) + if pg_role: + need( + not await conn.fetchval( + "SELECT EXISTS(SELECT FROM pg_database WHERE datname=$1)", db + ), + "new PG database", + ) + write( + Path(spec["workdir"]) / (role + "-intent.json"), + { + "schema": 2, + "fixture_id": spec["fixture_id"], + "database": db, + "absent": True, + "utc": utc(), + }, + ) + await conn.execute(f"CREATE DATABASE \"{db}\" ENCODING 'UTF8' TEMPLATE template0") + need( + await conn.fetchval( + "SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname=$1", db + ) + == "UTF8", + "created UTF8", + ) + else: + need( + await rows( + ch, + "SELECT count() AS n FROM system.databases WHERE name={db:String}", + {"db": db}, + ) + == [{"n": 0}], + "new CH database", + ) + existing = await rows( + ch, + "SELECT name FROM system.zookeeper WHERE path={path:String}", + {"path": spec["ch"]["keeper_prefix"]}, + ) + need( + spec["databases"]["ch_schema"] not in {r["name"] for r in existing}, + "new private Keeper root", + ) + write( + Path(spec["workdir"]) / (role + "-intent.json"), + { + "schema": 2, + "fixture_id": spec["fixture_id"], + "database": db, + "absent": True, + "keeper_root_absent": True, + "utc": utc(), + }, + ) + await ch.command(f"CREATE DATABASE `{db}` ENGINE=Atomic") + identity = ( + str(await conn.fetchval("SELECT oid FROM pg_database WHERE datname=$1", db)) + if pg_role + else ( + await rows( + ch, + "SELECT toString(uuid) AS id FROM system.databases WHERE name={db:String}", + {"db": db}, + ) + )[0]["id"] + ) + write( + output, + { + "schema": 2, + "fixture_id": spec["fixture_id"], + "role": role, + "store_container_id": spec["stores"]["pg_primary" if pg_role else "clickhouse"][ + "container_id" + ], + "database": db, + "database_id": identity, + "created_at": utc(), + "created": True, + }, + ) + + +async def resolve_creation(context: dict, role: str, output: Path) -> None: + name = database(context["databases"][role]) + async with connections(context, admin=True) as (conn, ch): + if role.startswith("pg_"): + present = bool( + await conn.fetchval("SELECT EXISTS(SELECT FROM pg_database WHERE datname=$1)", name) + ) + else: + present = ( + await rows( + ch, + "SELECT count() AS n FROM system.databases WHERE name={db:String}", + {"db": name}, + ) + ) != [{"n": 0}] + # Presence after an unacknowledged create is not adoption authority. + write( + output, + { + "schema": 2, + "fixture_id": context["fixture_id"], + "role": role, + "database": name, + "present": present, + "utc": utc(), + }, + ) + + +def mint_retirement(group: list[dict], birth: dict, certified) -> dict: + matching = [] + for row in group: + parts = str(row["host_proof"]).split("|") + if len(parts) == 9 and (parts[0], parts[1], parts[4], parts[5]) == ( + socket.gethostname(), + birth["boot_id"], + str(birth["pid"]), + str(birth["start_ticks"]), + ): + matching.append(row) + need( + len(matching) == 1 + and matching[0]["state"] == "retired" + and certified(matching[0]["host_proof"]), + "mint native positive retirement", + ) + return { + "participant_id": str(matching[0]["participant_id"]), + "state": "retired", + "version": int(matching[0]["version"]), + "birth": birth, + } + + +async def final_retirement(context: dict, output: Path) -> None: + from server.core.provider_process import has_certificate + + targets = read(Path(context["workdir"]) / "final-participant-births.json")["processes"] + need(0 < len(targets) <= 16, "final participant target bound") + db = database(context["databases"]["ch_schema"]) + async with connections(context) as (_, ch): + group = await rows( + ch, + ( + f"SELECT participant_id,state,version,host_proof FROM " + f"`{db}`._provider_participants FINAL LIMIT 65" + ), + ) + need(len(group) <= 64, "final participant census bound") + observed = [] + for target in targets: + birth = ref(target["birth"]) + terminal = ref(target["terminal"]) + need(terminal["returncode"] == target["returncode"], "final terminal receipt") + observed.append( + dict( + role=target["role"], + terminal=target["terminal"], + **mint_retirement(group, birth, has_certificate), + ) + ) + write( + output, + { + "schema": 2, + "fixture_id": context["fixture_id"], + "retired": observed, + "verified": True, + "utc": utc(), + }, + ) + + +async def mint_proof(context: dict, output: Path) -> None: + from server.core.provider_process import has_certificate + + birth = ref(read(Path(context["workdir"]) / "mint-process.json")["launch_receipt"]) + db = database(context["databases"]["ch_schema"]) + async with connections(context) as (_, ch): + group = await rows( + ch, + ( + f"SELECT participant_id,state,version,host_proof FROM " + f"`{db}`._provider_participants FINAL LIMIT 65" + ), + ) + need(len(group) <= 64, "bounded native participant census") + observed = mint_retirement(group, birth, has_certificate) + write(output, {"schema": 2, "fixture_id": context["fixture_id"], **observed, "utc": utc()}) + + +async def proof( + context: dict, + output: Path, + *, + bootstrap: bool = False, + quiescent: bool = False, + keeper: bool = False, +) -> None: + from server.cli.cell_attach_beats import widened_window_s + from server.core import cell, cell_triggers + from server.settings import settings + + db = database(context["databases"]["ch_schema"]) + org, ns = context["org"], context["namespace"] + async with connections(context, admin=quiescent or keeper) as (conn, ch): + if keeper: + original = read(Path(context["workdir"]) / "cleanup-keeper-paths.json") + prefix = context["ch"]["keeper_prefix"].rstrip("/") + for entry in original["keeper_paths"]: + need(entry["path"].startswith(prefix + "/"), "recorded Keeper prefix") + parent = prefix + absent = False + for segment in entry["path"][len(prefix) + 1 :].split("/"): + children = await rows( + ch, + "SELECT name FROM system.zookeeper WHERE path={path:String}", + {"path": parent}, + ) + if segment not in {r["name"] for r in children}: + absent = True + break + parent += "/" + segment + need(absent, "owned Keeper path remains") + write( + output, + { + "schema": 2, + "fixture_id": context["fixture_id"], + "keeper_paths_absent": True, + "utc": utc(), + }, + ) + return + if quiescent: + names = [context["databases"]["pg_cell"], context["databases"]["pg_directory"]] + need( + await conn.fetchval( + "SELECT count(*) FROM pg_stat_activity WHERE datname=ANY($1::text[])", names + ) + == 0, + "all scratch PG sessions closed", + ) + need( + await conn.fetchval( + ( + "SELECT count(*) FROM pg_stat_activity WHERE " + "pid<>pg_backend_pid() AND application_name=$1" + ), + "trigger-fixture:" + context["fixture_id"], + ) + == 0, + "cut-off fixture admin sessions", + ) + aliases = [db, context["databases"]["ch_telemetry"]] + need( + await rows( + ch, + ( + "SELECT count() AS n FROM system.asynchronous_inserts WHERE " + "database IN {dbs:Array(String)}" + ), + {"dbs": aliases}, + ) + == [{"n": 0}], + "quiescent CH queue", + ) + need( + await rows( + ch, + ( + "SELECT count() AS n FROM system.processes WHERE query_id != " + "currentQueryID() AND (current_database IN " + "{dbs:Array(String)} OR arrayExists(d -> position(query, d) " + "> 0, {dbs:Array(String)}))" + ), + {"dbs": aliases}, + ) + == [{"n": 0}], + "quiescent CH writers", + ) + keeper_paths = [] + for row in await rows( + ch, + ( + "SELECT database,name,create_table_query FROM system.tables " + "WHERE database IN {dbs:Array(String)} AND engine='KeeperMap'" + ), + {"dbs": aliases}, + ): + match = re.search(r"KeeperMap\('([^']+)'", row["create_table_query"]) + need(match and match[1].startswith("/" + db + "/"), "quiescent private Keeper path") + keeper_paths.append( + { + "database": row["database"], + "table": row["name"], + "path": context["ch"]["keeper_prefix"].rstrip("/") + match[1], + } + ) + need(len(keeper_paths) <= 64, "Keeper census bound") + write( + output, + { + "schema": 2, + "fixture_id": context["fixture_id"], + "pg_sessions": 0, + "ch_writers": 0, + "keeper_paths": keeper_paths, + "utc": utc(), + }, + ) + return + authority = dict( + await conn.fetchrow("SELECT authority,deployment_id,epoch FROM loom.authority") + ) + authority["deployment_id"] = str(authority["deployment_id"]) + if bootstrap: + need( + authority["authority"] == "postgres" and authority["epoch"] > 0, + "native boot authority", + ) + pair = await rows( + ch, f"SELECT authority,deployment_id,epoch FROM `{db}`.storage_authority FINAL" + ) + need( + len(pair) == 1 and all(str(pair[0][k]) == str(v) for k, v in authority.items()), + "native boot pair", + ) + params = {"org": org, "ns": ns} + need( + await rows( + ch, + ( + "SELECT owner_email,default_conflict_policy " + f"FROM `{db}`.namespaces FINAL WHERE " + f"org={{org:String}} AND namespace={{ns:String}}" + ), + params, + ) + == [{"owner_email": "benchmark", "default_conflict_policy": "append_only"}], + "native registered namespace", + ) + need( + await rows( + ch, + ( + f"SELECT subject,role FROM `{db}`.namespace_grants FINAL " + f"WHERE org={{org:String}} AND namespace={{ns:String}} AND " + f"subject='benchmark' AND revoked_at IS NULL" + ), + params, + ) + == [{"subject": "benchmark", "role": "admin"}], + "native registration grant", + ) + need( + await conn.fetchval("SELECT count(*) FROM loom.memory_current") == 0, + "no bootstrap memories", + ) + write( + output, + { + "schema": 2, + "fixture_id": context["fixture_id"], + **authority, + "namespace_registered": True, + "admin_grant": True, + "memory_rows": 0, + "utc": utc(), + }, + ) + return + first = await conn.fetchrow("SELECT is_first,cell FROM loom.cell_first") + need( + first and first["is_first"] is True and first["cell"] == context["cell"], + "first cell tuple", + ) + need(await conn.fetchval("SELECT mode FROM loom.tenancy_mode") == "enforced", "enforced") + need( + await conn.fetchval("SELECT EXISTS(SELECT FROM loom.tenancy WHERE org=$1)", org), + "held fixture org", + ) + own = await conn.fetchrow("SELECT current_fp,next_fp FROM loom.cell_self") + accepted = ( + json.loads(own["current_fp"]) + if isinstance(own["current_fp"], str) + else own["current_fp"] + ) + need( + own["next_fp"] is None + and cell.matches(cell.fingerprint(str(authority["deployment_id"])), accepted), + "accepted stable fingerprint", + ) + need(not await cell_triggers.missing(conn), "canonical triggers") + paired = await rows( + ch, f"SELECT authority,deployment_id,epoch FROM `{db}`.storage_authority FINAL" + ) + need( + len(paired) == 1 and all(str(paired[0][k]) == str(v) for k, v in authority.items()), + "paired authority", + ) + for table in ("provider_configuration", "runtime_overrides"): + need( + await rows(ch, f"SELECT count() AS n FROM `{db}`.{table} FINAL") == [{"n": 0}], + "empty saved configuration", + ) + need( + await conn.fetchval( + "SELECT (SELECT count(*) FROM loom.memory_deliveries WHERE " + "targets<>0)+(SELECT count(*) FROM loom.audit_outbox)" + ) + == 0, + "settled setup", + ) + instance = context["api_instance_id"] + from types import SimpleNamespace + + from server.core import deployment_config, instances + from server.core.ch_names import NamedClient + + named = NamedClient(ch) + adapter = SimpleNamespace(query_all=lambda sql, params=None: rows(named, sql, params)) + live = await instances.read_live(adapter, all_deployments=True) + need( + len(live) == 1 + and live[0]["instance_id"] == instance + and live[0]["serving"] == 1 + and live[0]["build_commit"] == context["head"] + and live[0]["host"] == socket.gethostname() + and not live[0]["workers_lease"], + "single serving API source", + ) + need( + all(live[0].get(k) == v for k, v in deployment_config.fingerprints(settings).items()), + "live API fingerprints", + ) + from server.core.provider_process import has_certificate + + launch = read(Path(context["workdir"]) / "serving-cell/launch.json")["server"] + participants = await rows( + ch, + ( + f"SELECT participant_id,state,host_proof FROM " + f"`{db}`._provider_participants FINAL LIMIT 65" + ), + ) + need(0 < len(participants) <= 64, "bounded native participant evidence") + serving = 0 + for participant in participants: + participant["participant_id"] = str(participant["participant_id"]) + proof_value = participant["host_proof"] + need(has_certificate(proof_value), "native participant capability") + parts = proof_value.split("|") + own = (parts[0], parts[1], parts[4], parts[5]) == ( + socket.gethostname(), + launch["boot_id"], + str(launch["pid"]), + str(launch["start_ticks"]), + ) + if own: + need(participant["state"] == "running", "serving native participant") + serving += 1 + else: + need( + participant["state"] == "retired", + "prior native participant not positively retired", + ) + need(serving == 1, "single native serving participant") + beat = await rows( + ch, + ( + f"SELECT fingerprint,cell,tenancy_mode FROM " + f"`{db}`.cell_heartbeats WHERE instance_id={{id:String}}" + ), + {"id": instance}, + ) + need(len(beat) == 1, "API heartbeat") + value = json.loads(beat[0]["fingerprint"]) + need( + value.get("serves") is True + and value.get("marked") is True + and value.get("router") is True, + "native API router state", + ) + namespace = await rows( + ch, + ( + f"SELECT owner_email FROM `{db}`.namespaces FINAL WHERE " + f"org={{org:String}} AND namespace={{ns:String}}" + ), + {"org": org, "ns": ns}, + ) + need(namespace == [{"owner_email": "benchmark"}], "native namespace owner") + keeper = [] + for row in await rows( + ch, + ( + "SELECT database,name,create_table_query FROM system.tables " + "WHERE database IN {dbs:Array(String)} AND engine='KeeperMap'" + ), + {"dbs": [db, context["databases"]["ch_telemetry"]]}, + ): + match = re.search(r"KeeperMap\('([^']+)'", row["create_table_query"]) + need(match and match[1].startswith("/" + db + "/"), "private actual Keeper path") + keeper.append( + { + "database": row["database"], + "table": row["name"], + "path": context["ch"]["keeper_prefix"].rstrip("/") + match[1], + } + ) + write( + output, + { + "schema": 2, + "fixture_id": context["fixture_id"], + **authority, + "first": [True, context["cell"]], + "mode": "enforced", + "held": True, + "keeper_paths": keeper, + "native_participants": participants, + "widened_window_s": widened_window_s(), + "utc": utc(), + }, + ) + + +class Fixture: + def __init__(self, spec: dict, owner) -> None: + self.s, self.o = dict(spec), owner + self.w = Path(spec["workdir"]) + self.receipts = [] + self.created = [] + self.teardown_incomplete = False + self.retirement_attempted = False + self.retirement_receipt = None + self.settings: dict = {} + self.api = None + self.cell = "gate-" + spec["fixture_id"][:12] + self.org, self.namespace = ( + "trigger-" + spec["fixture_id"], + "trigger-" + spec["fixture_id"][:12], + ) + self.s.update(cell=self.cell, org=self.org, namespace=self.namespace) + self.context = self.w / "context.json" + write(self.context, self.s) + + def env(self, settings: dict | None = None, *, ca: bool = False) -> dict: + return child_env(self.s, self.settings if settings is None else settings, ca=ca) + + def cli(self, *args: str, settings: dict | None = None, seconds: float = 120): + return self.o.command( + args[0], + [self.s["python"], "-m", "server.cli", *args], + self.env(settings, ca=True), + seconds=seconds, + ) + + def receipt(self, operation: str, value: dict) -> None: + reference = write( + self.w / (operation + "-receipt.json"), {"schema": 2, "operation": operation, **value} + ) + self.receipts.append({"operation": operation, "receipt": reference}) + + def command_receipt(self, operation: str, item: dict, **observations) -> None: + self.receipt( + operation, + { + "returncode": item["process"].returncode, + **observations, + "process": self.process_receipt(item), + "log": { + "path": str(item["log"]), + "sha256": sha(item["log"]), + "bytes": item["log"].stat().st_size, + }, + "utc": utc(), + }, + ) + + def action( + self, task: str, *, role: str = "", settings: dict | None = None, cleanup: bool = False + ) -> dict: + target = self.w / ( + role + "-created.json" + if task == "create" + else task + ("-" + role if role else "") + ".json" + ) + if task == "quiescent": + target = self.w / ("quiescent-" + str(self.o.serial + 1) + ".json") + argv = [ + self.s["python"], + str(HERE / FILES[1]), + "--task", + task, + "--context", + str(self.context), + "--output", + str(target), + ] + if role: + argv += ["--role", role] + try: + self.o.command( + task, + argv, + self.env(settings), + seconds=600 if task == "provenance" else 120, + cleanup=cleanup, + ) + except BaseException as error: + if isinstance(error, Refused): + try: + failure = read(target.with_name(target.name + ".failure.json"), limit=4096) + closed( + failure, + "error_type site outcome driver" + if "driver" in failure + else "error_type site outcome", + ) + if "driver" in failure: + driver = failure["driver"] + closed( + driver, + "code causes" + + (" sqlstate" if "sqlstate" in driver else "") + + (" outcome_unknown" if "outcome_unknown" in driver else ""), + ) + need( + ( + "sqlstate" not in driver + or type(driver["sqlstate"]) is str + and re.fullmatch(r"[0-9A-Z]{5}", driver["sqlstate"]) + ) + and ( + "outcome_unknown" not in driver + or type(driver["outcome_unknown"]) is bool + ), + "native Postgres failure metadata", + ) + need( + ( + driver["code"] is None + or type(driver["code"]) is int + and 0 <= driver["code"] < 2**31 + ) + and type(driver["causes"]) is list + and len(driver["causes"]) <= 4, + "native driver failure metadata", + ) + for cause in driver["causes"]: + closed( + cause, + "error_type errno" + + (" sqlstate" if "sqlstate" in cause else "") + + (" cancel_reason" if "cancel_reason" in cause else "") + + (" site" if "site" in cause else ""), + ) + need( + "sqlstate" not in cause + or type(cause["sqlstate"]) is str + and re.fullmatch(r"[0-9A-Z]{5}", cause["sqlstate"]), + "native Postgres failure cause", + ) + if "cancel_reason" in cause: + need( + cause["error_type"] == "QueryCanceledError" + and cause.get("sqlstate") == "57014" + and type(cause["cancel_reason"]) is str + and cause["cancel_reason"] + in {"statement_timeout", "user_request", "unknown"}, + "native Postgres cancellation reason", + ) + if "site" in cause: + cause_site = cause["site"] + closed(cause_site, "file line function") + need( + cause_site["file"] in (*FILES, "unknown") + and type(cause_site["line"]) is int + and 0 <= cause_site["line"] < 2**31 + and isinstance(cause_site["function"], str) + and re.fullmatch( + r"[A-Za-z_][A-Za-z_0-9]{0,63}", cause_site["function"] + ), + "native failure cause site", + ) + need( + isinstance(cause["error_type"], str) + and re.fullmatch( + r"[A-Za-z_][A-Za-z_0-9]{0,63}", cause["error_type"] + ) + and ( + cause["errno"] is None + or type(cause["errno"]) is int + and -(2**31) <= cause["errno"] < 2**31 + ), + "native driver failure cause", + ) + site = failure["site"] + closed(site, "file line function") + need( + isinstance(failure["error_type"], str) + and re.fullmatch(r"[A-Za-z_][A-Za-z_0-9]{0,63}", failure["error_type"]) + and failure["outcome"] in {"refused", "error", "cancelled"} + and site["file"] in (*FILES, "unknown") + and type(site["line"]) is int + and 0 <= site["line"] < 2**31 + and isinstance(site["function"], str) + and re.fullmatch(r"[A-Za-z_][A-Za-z_0-9]{0,63}", site["function"]), + "native failure metadata", + ) + error.native = failure + except BaseException: # noqa: BLE001 — diagnostics must not replace the original command failure. + pass + # A terminal cleanup refusal is recorded, never retried as another mutation. + if cleanup and self.o.leaders and self.o.leaders[-1]["terminal"] is not None: + self.o.select(error) + return {"incomplete": True, "task": task, "error_type": type(error).__name__} + raise + return read(target) + + def http( + self, path: str, *, body: dict | None = None, token: str = "", tls: bool = False + ) -> dict: + import ssl + + import httpx + + port = self.s["ports"]["tls" if tls else "api"] + verify = ssl.create_default_context(cafile=str(self.w / "ca.crt")) if tls else True + context = httpx.Client(timeout=10, follow_redirects=False, trust_env=False, verify=verify) + client = context.__enter__() + try: + response = client.request( + "POST" if body is not None else "GET", + f"{'https' if tls else 'http'}://127.0.0.1:{port}{path}", + json=body, + headers={"Authorization": "Bearer " + token} if token else {}, + ) + need(response.status_code == 200, "native HTTP refused") + value = response.json() + need(isinstance(value, dict), "native HTTP shape") + except BaseException as error: + try: + context.__exit__(type(error), error, error.__traceback__) + except BaseException as secondary: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.o.select(error) + self.o.select(secondary) + raise + else: + context.__exit__(None, None, None) + self.o.monitor() + return value + + def wait_api(self) -> None: + import httpx + + end = min(time.monotonic() + 120, self.o.started + 7020) + while True: + self.o.monitor() + try: + value = self.http("/v1/health") + ready = value.get("readiness", {}) + checks = ready.get("checks", {}) + required = { + "draining", + "clickhouse", + "migrations", + "revocation", + "keyring", + "tracker_registry", + "runtime_overrides", + "calibration", + "config_drift", + "cell", + "sandbox", + } + if self.settings["cell"]: + required.add("cell_marker") + need( + value.get("ch_ok") is True + and value.get("pg_ok") is True + and value.get("storage_authority") == "postgres" + and value.get("pending_migrations") == [] + and ready.get("ready") is True + and ready.get("failing") == [] + and required <= set(checks) + and all( + checks[name].get("required", True) is True + and checks[name].get("ok") is True + for name in required + ) + and all( + isinstance(check, dict) + and type(check.get("ok")) is bool + and (check.get("required", True) is False or check["ok"] is True) + for check in checks.values() + ), + "healthy ready API stores", + ) + return + except (OSError, httpx.TransportError, Refused): + need(time.monotonic() < end, "API boot deadline") + time.sleep(1) + + def start_api(self, stage: str) -> None: + folder = self.w / stage + folder.mkdir(mode=0o700) + self.api = self.o.start( + "api-" + stage, + [ + self.s["python"], + str(HERE / FILES[1]), + "--task", + "api", + "--context", + str(self.context), + "--output", + str(folder / "launch.json"), + ], + self.env(), + persistent=True, + ) + self.wait_api() + self.launch = read(folder / "launch.json") + self.launch_path = folder / "launch.json" + + def setup(self) -> None: + import secrets + + # docker inspect is read-only and the input names exact positive IDs. + self.inspect_stores() + for role in ("pg_cell", "pg_directory", "ch_schema", "ch_telemetry"): + output = self.w / (role + "-created.json") + self.action("create", role=role) + # Native action writes the authoritative creation receipt, not this observation. + need(read(output)["created"] is True, "actual database creation receipt") + self.created.append(role) + cell = self.cli("cell", "credential", "new", settings={}) + operator = self.cli("cell", "credential", "new", "--operator", settings={}) + + def credential(item, prefix): + text = item["log"].read_text() + match = re.search(r"^credential: (" + prefix + r"[0-9a-f]{128})$", text, re.M) + need(match, "native credential output") + return match[1] + + cell_token = credential(cell, "lcr_") + self.operator = credential(operator, "lco_") + p, db = self.s["ports"], self.s["databases"] + self.settings = dict( + ch_host="127.0.0.1", + ch_port=self.s["ch"]["port"], + ch_user=self.s["ch"]["username"], + ch_password=self.s["ch"]["password"], + ch_database=db["ch_schema"], + ch_telemetry_database=db["ch_telemetry"], + pg_dsn=pg_dsn(self.s["pg_admin_dsn"], db["pg_cell"]), + storage_authority="postgres", + pg_durability="synchronous", + embedding_provider="fake", + embedding_model="fake-ngram-1024", + embedding_dim=1024, + llm_provider="fake", + llm_model="fake-echo-1", + require_real_providers=False, + background_workers="off", + pg_delivery_batch_rows=500, + auth_required=True, + identity_provider="gateway", + self_signup="off", + mcp_auth_mode="required", + jwt_secret=secrets.token_hex(32), + scim_token="", + cell="", + cell_probe_port=p["probe"], + cell_router_url=f"http://127.0.0.1:{p['router']}", + cell_router_token=cell_token, + router_operator_token="", + public_base_url=f"https://127.0.0.1:{p['tls']}", + build_commit=self.s["head"], + bind_host="127.0.0.1", + document_s3_bucket="", + ) + router = dict( + self.settings, + cell_router_token="", + router_pg_dsn=pg_dsn(self.s["pg_admin_dsn"], db["pg_directory"]), + router_operator_sha256=__import__("hashlib").sha256(self.operator.encode()).hexdigest(), + router_shared_host=f"127.0.0.1:{p['tls']}", + router_upstream_url=f"http://127.0.0.1:{p['api']}", + router_upstream_probe_url=f"http://127.0.0.1:{p['probe']}", + ) + self.command_receipt("directory_migrate", self.cli("router", "migrate", settings=router)) + self.o.start( + "router", + [ + self.s["python"], + "-m", + "server.cli", + "router", + "serve", + "--host", + "127.0.0.1", + "--port", + str(p["router"]), + ], + self.env(router), + persistent=True, + ) + self.certificate() + self.start_api("before-cell") + self.o.start( + "proxy", + [ + self.s["python"], + str(HERE / FILES[2]), + "--listen-port", + str(p["tls"]), + "--upstream-port", + str(p["api"]), + "--certificate", + str(self.w / "server.crt"), + "--key", + str(self.w / "server.key"), + ], + self.env({}), + persistent=True, + ) + minted = self.cli( + "mint-token", + "--subject", + "benchmark", + "--org", + self.org, + "--namespaces", + self.namespace, + "--roles", + "writer", + "--service-account", + "--ttl-days", + "1", + "--save", + str(self.w / "writer.jwt"), + "--json", + ) + write(self.w / "mint-process.json", self.process_receipt(minted)) + retirement = self.action("mint-proof") + metadata, self.token = mint_metadata(self.s, self.process_receipt(minted)) + self.s["mint_metadata"] = write(self.w / "mint-metadata.json", metadata) + self.command_receipt( + "mint", minted, retirement=retirement, metadata=self.s["mint_metadata"] + ) + result = self.http( + "/v1/namespace.register", + token=self.token, + body={ + "org": self.org, + "namespace": self.namespace, + "owner_email": "benchmark", + "default_conflict_policy": "append_only", + }, + ) + need( + result.get("registered") is True and result.get("namespace") == self.namespace, + "native namespace registration", + ) + write(self.w / "namespace-register.json", result) + self.receipt("namespace_register", result) + self.receipt("authority_boot", self.action("bootstrap-proof")) + import httpx + + end = min(time.monotonic() + 30, self.o.started + 7020) + while True: + self.o.monitor() + try: + self.http("/v1/health/deployment", tls=True) + break + except httpx.TransportError: + need(time.monotonic() < end, "TLS listener startup") + time.sleep(0.1) + operator_settings = dict( + self.settings, cell=self.cell, router_operator_token=self.operator, cell_router_token="" + ) + self.command_receipt( + "prepare", + self.cli( + "cell", + "prepare", + "--cell", + self.cell, + "--host", + f"127.0.0.1:{p['tls']}", + "--credential-sha256", + __import__("hashlib").sha256(cell_token.encode()).hexdigest(), + settings=operator_settings, + seconds=600, + ), + ) + self.o.stop_one(self.api, api=True) + self.settings["cell"] = self.cell + self.start_api("serving-cell") + # Read the actual source's widened interval in a bounded metadata child. + interval = self.action("window")["seconds"] + end = time.monotonic() + interval + while time.monotonic() < end: + self.o.monitor() + time.sleep(1) + self.command_receipt( + "attach", + self.cli( + "cell", + "attach", + "--cell", + self.cell, + "--wait", + "600", + "--scan-budget", + "600", + "--ch-allow-user", + self.s["ch"]["username"], + settings=operator_settings, + seconds=650, + ), + ) + self.s["api_instance_id"] = self.launch["server"]["instance_id"] + self.context = self.w / "attached-context.json" + write(self.context, self.s) + self.cli_settings = dict(self.settings, cell_router_token="") + self.attached = self.action("proof", settings=self.cli_settings) + self.receipt("post_attach_proof", self.attached) + + def inspect_stores(self, *, cleanup: bool = False) -> None: + for expected in self.s["stores"].values(): + command = self.o.command( + "inspect", + ["docker", "inspect", expected["container_id"]], + self.env({}), + cleanup=cleanup, + ) + raw = json.loads(command["log"].read_text()) + need(len(raw) == 1, "one inspected container") + actual = raw[0] + need( + actual["Id"] == expected["container_id"] + and actual["Created"] == expected["created_at"] + and actual["State"]["Running"] is True + and actual["State"]["Pid"] == expected["host_pid"] + and actual["HostConfig"]["Memory"] == expected["memory_limit_bytes"], + "current owned container", + ) + observed = [] + for internal, bindings in actual["NetworkSettings"]["Ports"].items(): + for binding in bindings or []: + observed.append( + { + "host": binding["HostIp"], + "host_port": int(binding["HostPort"]), + "internal_port": int(internal.split("/")[0]), + "protocol": internal.split("/")[1], + } + ) + need( + sorted(observed, key=digest) == sorted(expected["ports"], key=digest), + "actual loopback bindings", + ) + need( + expected["disk_path"] in {m["Source"] for m in actual["Mounts"]}, + "owned store filesystem", + ) + identity = self.o.owned.identity(expected["host_pid"]) + need( + identity + and identity[1] == expected["start_ticks"] + and os.stat(f"/proc/{expected['host_pid']}").st_uid == expected["uid"] + and Path("/proc/sys/kernel/random/boot_id").read_text().strip() + == expected["boot_id"], + "store process identity", + ) + prefix = self.o.command( + "keeper-prefix", + [ + "docker", + "exec", + self.s["stores"]["clickhouse"]["container_id"], + "clickhouse", + "extract-from-config", + "--config-file", + "/etc/clickhouse-server/config.xml", + "--key", + "keeper_map_path_prefix", + ], + self.env({}), + cleanup=cleanup, + ) + need( + prefix["log"].read_text().strip() == self.s["ch"]["keeper_prefix"], + "actual Keeper prefix", + ) + + def certificate(self) -> None: + def openssl(*args): + self.o.command("openssl", ["openssl", *args], self.env({})) + + previous = os.umask(0o077) + try: + openssl( + "req", + "-x509", + "-newkey", + "rsa:2048", + "-noenc", + "-keyout", + str(self.w / "ca.key"), + "-out", + str(self.w / "ca.crt"), + "-days", + "1", + "-subj", + "/CN=local-trigger-ca", + ) + openssl( + "req", + "-newkey", + "rsa:2048", + "-noenc", + "-keyout", + str(self.w / "server.key"), + "-out", + str(self.w / "server.csr"), + "-subj", + "/CN=127.0.0.1", + ) + extension = self.w / "server.ext" + extension.write_text("subjectAltName=IP:127.0.0.1\nextendedKeyUsage=serverAuth\n") + openssl( + "x509", + "-req", + "-in", + str(self.w / "server.csr"), + "-CA", + str(self.w / "ca.crt"), + "-CAkey", + str(self.w / "ca.key"), + "-CAcreateserial", + "-out", + str(self.w / "server.crt"), + "-days", + "1", + "-extfile", + str(extension), + ) + finally: + os.umask(previous) + openssl("verify", "-CAfile", str(self.w / "ca.crt"), str(self.w / "server.crt")) + dates = self.o.command( + "certificate-dates", + ["openssl", "x509", "-in", str(self.w / "server.crt"), "-noout", "-dates"], + self.env({}), + ) + values = dict(line.split("=", 1) for line in dates["log"].read_text().splitlines()) + parsed = { + key: datetime.datetime.strptime(value, "%b %d %H:%M:%S %Y %Z") + .replace(tzinfo=datetime.UTC) + .isoformat() + for key, value in values.items() + } + private = write( + self.w / "tls-files.json", + { + "schema": 2, + "files": [ + { + "path": str(self.w / name), + "uid": (self.w / name).stat().st_uid, + "mode": (self.w / name).stat().st_mode & 0o777, + "bytes": (self.w / name).stat().st_size, + "sha256": sha(self.w / name), + } + for name in ("ca.key", "ca.crt", "server.key", "server.crt") + ], + }, + ) + self.tls = { + "ca_certificate_sha256": sha(self.w / "ca.crt"), + "server_certificate_sha256": sha(self.w / "server.crt"), + "server_ip_san": "127.0.0.1", + "not_before": parsed["notBefore"], + "not_after": parsed["notAfter"], + "listener": f"https://127.0.0.1:{self.s['ports']['tls']}", + "upstream": f"http://127.0.0.1:{self.s['ports']['api']}", + "private_files_receipt": private, + } + + def measure(self) -> str: + # Receipt construction is explicit; no old single-digest schema is accepted. + config = ref(self.launch["configuration_artifact"]) + ownership = self.ownership() + ownership_ref = write(self.w / "ownership.json", ownership) + p, db = self.s["ports"], self.s["databases"] + fixture = dict( + schema=2, + base_url=f"http://127.0.0.1:{p['api']}", + pg_host="127.0.0.1", + pg_port=url_port(self.s["pg_admin_dsn"]), + pg_database=db["pg_cell"], + pg_internal_port=5432, + ch_host="127.0.0.1", + ch_port=self.s["ch"]["port"], + ch_database=db["ch_schema"], + ch_telemetry_database=db["ch_telemetry"], + cell=self.cell, + org=self.org, + namespace=self.namespace, + deployment_id=str(self.attached["deployment_id"]), + epoch=self.attached["epoch"], + instance_id=self.launch["server"]["instance_id"], + build_commit=self.s["head"], + source_tree=self.s["tree"], + launch_manifest_sha256=sha(self.launch_path), + ownership_receipt_sha256=ownership_ref["sha256"], + background_workers="off", + controller_excluded=True, + **{ + k: config[k] + for k in ( + "api_configuration_sha256", + "cli_configuration_sha256", + "shared_configuration_sha256", + ) + }, + ) + write(self.w / "fixture.json", fixture) + self.action("resolve", settings=self.cli_settings) + metadata = ref(self.s["mint_metadata"]) + need(verify_token(metadata) == self.token, "native bearer unchanged before driver") + item = self.o.start( + "driver", + [ + self.s["python"], + str(HERE / FILES[3]), + "--context", + str(self.context), + "--capture-directory", + str(self.w / "delivery-intent"), + "--", + "--base-url", + fixture["base_url"], + "--pg-dsn", + self.cli_settings["pg_dsn"], + "--token", + self.token, + "--org", + self.org, + "--namespace", + self.namespace, + "--fixture-manifest", + str(self.w / "fixture.json"), + "--launch-manifest", + str(self.launch_path), + "--output", + str(self.w / "driver.jsonl"), + ], + self.env(self.cli_settings), + ) + write( + self.w / "driver-birth.json", + dict(self.process_receipt(item), schema=2, fixture_id=self.s["fixture_id"]), + ) + while self.o.poll(item) is None: + self.o.monitor() + time.sleep(0.05) + write( + self.w / "driver-finished.json", + dict(self.process_receipt(item), schema=2, fixture_id=self.s["fixture_id"]), + ) + need(verify_token(metadata) == self.token, "native bearer unchanged after driver") + lines = item["log"].read_bytes().splitlines() + need(lines and len(lines[-1]) <= 65536, "driver final result") + result = json.loads(lines[-1]) + need( + result.get("verdict") in {"PASS", "FAIL"} + and item["process"].returncode == (0 if result["verdict"] == "PASS" else 1), + "completed driver result", + ) + return result["verdict"] + + def process_receipt(self, item: dict) -> dict: + identity = item["identity"] + args = list(item["argv"]) + for index, value in enumerate(args[:-1]): + if value in {"--token", "--pg-dsn"}: + args[index + 1] = "" + return { + "role": item["role"], + "pid": item["process"].pid, + "start_ticks": identity[1], + "boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + "uid": os.geteuid(), + "cwd": self.s["root"], + "executable": item["argv"][0], + "executable_sha256": sha(Path(item["argv"][0])) + if Path(item["argv"][0]).is_absolute() + else sha(Path(__import__("shutil").which(item["argv"][0]))), + "source_sha256": digest(self.s["source_hashes"]), + "configuration_sha256": item["configuration_sha256"], + "redacted_argv": args, + "launch_receipt": item["launch_receipt"], + "terminal_receipt": item["terminal_receipt"], + } + + def ownership(self) -> dict: + # All entries are observations from completed setup, never a future driver PID. + need( + len(self.o.leaders) <= 64 and len(self.attached["keeper_paths"]) <= 64, + "ownership entry bounds", + ) + return { + "schema": 2, + "fixture_id": self.s["fixture_id"], + "source": { + "root": self.s["root"], + "head": self.s["head"], + "tree": self.s["tree"], + "source_hashes": self.s["source_hashes"], + "driver_sha256": self.s["source_hashes"]["driver"], + "launcher_sha256": self.s["source_hashes"][FILES[0]], + "proxy_sha256": self.s["source_hashes"][FILES[2]], + "owned_children_sha256": self.s["source_hashes"]["owned_children.py"], + "python_executable": self.s["python"], + "python_sha256": self.s["python_sha256"], + "lockfile_sha256": self.s["lockfile_sha256"], + "dependency_receipt": self.s["dependency_receipt"], + }, + "stores": [ + {k: v for k, v in s.items() if k != "disk_path"} for s in self.s["stores"].values() + ], + "created_databases": [ + { + "role": role, + "store_container_id": read(self.w / (role + "-created.json"))[ + "store_container_id" + ], + "database": self.s["databases"][role], + "created_at": read(self.w / (role + "-created.json"))["created_at"], + "creation_receipt": { + "path": str(self.w / (role + "-created.json")), + "sha256": sha(self.w / (role + "-created.json")), + }, + "drop_required": True, + } + for role in self.created + ], + "keeper_paths": [ + dict( + row, + store_container_id=self.s["stores"]["clickhouse"]["container_id"], + creation_receipt={ + "path": str(self.w / "ch_schema-created.json"), + "sha256": sha(self.w / "ch_schema-created.json"), + }, + drop_required=True, + ) + for row in self.attached["keeper_paths"] + ], + "processes": [self.process_receipt(item) for item in self.o.leaders], + "tls": self.tls, + "native_receipts": self.receipts, + "controller_excluded": True, + "resource_policy": POLICY, + } + + def provenance(self) -> None: + self.action("provenance", settings=self.cli_settings) + + def block_teardown(self, error: BaseException) -> None: + # Retain the mutation barrier even if the negative receipt cannot be created. + self.teardown_incomplete = True + self.o.select(error) + try: + if not (self.w / "cleanup-incomplete.json").exists(): + write( + self.w / "cleanup-incomplete.json", + { + "schema": 2, + "fixture_id": self.s["fixture_id"], + "reason": "required cleanup evidence incomplete; stores retained", + "utc": utc(), + }, + ) + except BaseException as secondary: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.o.select(secondary) + + def retirement_evidence(self, retired: dict, final: list[dict]) -> None: + closed(retired, "schema fixture_id retired verified utc") + need( + type(retired["schema"]) is int + and retired["schema"] == 2 + and retired["fixture_id"] == self.s["fixture_id"] + and retired["verified"] is True + and type(retired["retired"]) is list + and len(retired["retired"]) == len(final), + "acknowledged complete final retirement", + ) + for value, item in zip(retired["retired"], final, strict=True): + closed(value, "role terminal participant_id state version birth") + equal(value["role"], item["role"]) + equal(value["terminal"], item["terminal_receipt"]) + equal(value["birth"], ref(item["launch_receipt"])) + terminal = ref(item["terminal_receipt"]) + need( + type(terminal["returncode"]) is int + and terminal["returncode"] == item["process"].returncode, + "final retirement terminal binding", + ) + need(value["state"] == "retired", "positive native retirement state") + identifier(value["participant_id"]) + integer(value["version"], 1, 2**63 - 1) + + def teardown(self) -> None: + # Only creation receipts authorize deletion, even after partial setup. + if not any( + (self.w / (role + suffix)).exists() + for role in self.s["databases"] + for suffix in ("-intent.json", "-created.json") + ): + return + self.inspect_stores(cleanup=True) + for port in self.s["ports"].values(): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.settimeout(1) + need(probe.connect_ex(("127.0.0.1", port)) != 0, "fixture port remains open") + quiescent = self.action("quiescent", settings={}, cleanup=True) + need( + quiescent.get("ch_writers") == quiescent.get("pg_sessions") == 0, + "server writer settlement unknown", + ) + if self.teardown_incomplete or (self.w / "cleanup-incomplete.json").exists(): + self.teardown_incomplete = True + self.o.select(Refused("known incomplete teardown; no mutation retry")) + return + if not (self.w / "cleanup-keeper-paths.json").exists(): + write(self.w / "cleanup-keeper-paths.json", {"keeper_paths": quiescent["keeper_paths"]}) + final = [item for item in self.o.leaders if item["role"].startswith(("api-", "driver-"))] + if final: + try: + if self.retirement_receipt is None: + need( + not self.retirement_attempted + and not (self.w / "final-retirement.json").exists(), + "unacknowledged final retirement artifact; no retry", + ) + self.retirement_attempted = True + need( + all( + item["terminal_receipt"] is not None + and item["launch_receipt"] is not None + for item in final + ), + "final native process receipts", + ) + write( + self.w / "final-participant-births.json", + { + "processes": [ + { + "role": item["role"], + "birth": item["launch_receipt"], + "terminal": item["terminal_receipt"], + "returncode": item["process"].returncode, + } + for item in final + ] + }, + ) + retired = self.action("final-retirement", settings={}, cleanup=True) + self.retirement_evidence(retired, final) + receipt = { + "path": str(self.w / "final-retirement.json"), + "sha256": sha(self.w / "final-retirement.json"), + } + equal(ref(receipt), retired) + self.retirement_receipt = receipt + self.retirement_evidence(ref(self.retirement_receipt), final) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.block_teardown(error) + return + for role in self.s["databases"]: + if (self.w / (role + "-intent.json")).exists() and not ( + self.w / (role + "-created.json") + ).exists(): + state = self.action("resolve-creation", role=role, settings={}, cleanup=True) + if state.get("present") is not False: + self.block_teardown(Refused("uncertain database ownership; leave untouched")) + return + for role in ("ch_telemetry", "ch_schema", "pg_directory", "pg_cell"): + created = self.w / (role + "-created.json") + if created.exists() and not (self.w / ("drop-" + role + ".json")).exists(): + result = self.action("drop", role=role, settings={}, cleanup=True) + if result.get("dropped") is not True: + self.block_teardown(Refused("native scratch cleanup incomplete")) + # A failed DROP may still run on the server; prove settlement afresh. + settled = self.action("quiescent", settings={}, cleanup=True) + need( + settled.get("ch_writers") == settled.get("pg_sessions") == 0, + "drop outcome still running", + ) + return + if not (self.w / "keeper-proof.json").exists(): + result = self.action("keeper-proof", settings={}, cleanup=True) + if result.get("keeper_paths_absent") is not True: + self.block_teardown(Refused("owned Keeper cleanup incomplete")) + return + if not (self.w / "teardown.json").exists(): + write( + self.w / "teardown.json", + { + "schema": 2, + "fixture_id": self.s["fixture_id"], + "dropped": [ + role + for role in self.s["databases"] + if (self.w / ("drop-" + role + ".json")).exists() + ], + "owned_ports_closed": True, + "pg_sessions_closed": True, + "job_container_cleanup_required": True, + "utc": utc(), + }, + ) + + +def url_port(dsn: str) -> int: + from urllib.parse import urlsplit + + port = urlsplit(dsn).port + need(port is not None, "explicit PG port") + return port + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--task", + required=True, + choices=[ + "api", + "create", + "drop", + "resolve-creation", + "mint-proof", + "final-retirement", + "proof", + "provenance", + "bootstrap-proof", + "quiescent", + "keeper-proof", + "window", + "resolve", + ], + ) + parser.add_argument("--context", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--role", default="") + args = parser.parse_args() + try: + context = read(args.context) + sys.path.insert(0, context["root"]) + if args.task == "api": + api(context, args.output) + elif args.task in {"create", "drop"}: + need(args.role in context["databases"], "fixed database role") + asyncio.run(stores(context, args.role, args.output, drop=args.task == "drop")) + elif args.task == "resolve-creation": + need(args.role in context["databases"], "fixed database role") + asyncio.run(resolve_creation(context, args.role, args.output)) + elif args.task == "window": + from server.cli.cell_attach_beats import widened_window_s + + write(args.output, {"seconds": widened_window_s()}) + elif args.task == "mint-proof": + asyncio.run(mint_proof(context, args.output)) + elif args.task == "provenance": + asyncio.run(independent_provenance(context, args.output)) + elif args.task == "final-retirement": + asyncio.run(final_retirement(context, args.output)) + elif args.task == "resolve": + from server.settings import settings + + codec_contract() + config = ref( + read(args.output.parent / "serving-cell/launch.json")["configuration_artifact"] + ) + actual = settings.model_dump(mode="json") + need( + actual["cell_router_token"] == actual["router_operator_token"] == "", + "CLI credentials", + ) + need( + digest(actual) == config["cli_configuration_sha256"] + and digest({k: v for k, v in actual.items() if k != "cell_router_token"}) + == config["shared_configuration_sha256"], + "actual CLI configuration", + ) + write( + args.output, + { + "schema": 2, + "cli_configuration_sha256": digest(actual), + "codec_protocol_sha256": PROTOCOL_SHA, + "verified": True, + }, + ) + else: + asyncio.run( + proof( + context, + args.output, + bootstrap=args.task == "bootstrap-proof", + quiescent=args.task == "quiescent", + keeper=args.task == "keeper-proof", + ) + ) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + try: + write( + args.output.with_name(args.output.name + ".failure.json"), + native_failure_metadata(error), + limit=4096, + ) + except BaseException: # noqa: BLE001 — diagnostic output must not replace the native failure. + pass + print(type(error).__name__, file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gates/testsupport/cell_trigger/owned_children.py b/scripts/gates/testsupport/cell_trigger/owned_children.py new file mode 100644 index 0000000000..b0c5d950f8 --- /dev/null +++ b/scripts/gates/testsupport/cell_trigger/owned_children.py @@ -0,0 +1,145 @@ +"""Reap only exact direct and adopted children of the fixture subreaper.""" + +from __future__ import annotations + +import ctypes +import os +import signal +import subprocess +import time +from pathlib import Path + + +class OwnedChildren: + """Signal only unreaped kernel children of this private subreaper.""" + + def __init__(self) -> None: + if not hasattr(os, "pidfd_open") or not hasattr(signal, "pidfd_send_signal"): + raise RuntimeError("exact owned-process signaling needs Linux pidfds") + if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0: + error = ctypes.get_errno() + raise OSError(error, "PR_SET_CHILD_SUBREAPER: " + os.strerror(error)) + self.pid = os.getpid() + + @staticmethod + def identity(pid: int) -> tuple[int, str, str] | None: + try: + fields = Path("/proc", str(pid), "stat").read_text().rsplit(")", 1)[1].split() + return int(fields[1]), fields[19], fields[0] + except (FileNotFoundError, ProcessLookupError): + return None + + @staticmethod + def children(pid: int) -> list[int]: + return [ + int(value) + for value in Path("/proc", str(pid), "task", str(pid), "children").read_text().split() + ] + + @staticmethod + def rss(pid: int) -> int: + try: + status = Path("/proc", str(pid), "status").read_text() + except FileNotFoundError: + return 0 # Vanished sample, never proof that the process used no memory. + return next( + (int(line.split()[1]) for line in status.splitlines() if line.startswith("VmRSS:")), 0 + ) + + def stop(self, process: subprocess.Popen | None, note: object) -> None: + started = None + last_note = None + diagnostic_error = None + # Keep each verified pidfd pinned, including across numeric PID reuse. + pinned: dict[int, tuple[int, set[int]]] = {} + + def cleanup_note(**details: object) -> None: + nonlocal diagnostic_error + try: + note("cleanup-wait", **details) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + diagnostic_error = error + + try: + while True: + try: + children = self.children(self.pid) + for pid in children: + row = self.identity(pid) + if row and row[0] == self.pid and row[2] == "Z": + if ( + process is not None + and process.returncode is None + and pid == process.pid + ): + process.poll() # Preserve the real leader exit status. + else: + os.waitpid(pid, os.WNOHANG) + children = self.children(self.pid) + if not children: + break # Kernel list is empty AFTER zombie reaping. + if started is None: + started = last_note = time.monotonic() + elapsed = time.monotonic() - started + sig = ( + signal.SIGINT + if elapsed < 30 + else (signal.SIGTERM if elapsed < 40 else signal.SIGKILL) + ) + for pid in children: + row = self.identity(pid) + if row is None or row[0] != self.pid or row[2] == "Z": + continue + if pid in pinned: + descriptor, sent = pinned[pid] + try: + signal.pidfd_send_signal(descriptor, 0) + except ProcessLookupError: + os.close(descriptor) + del pinned[pid] + if pid not in pinned: + try: + descriptor = os.pidfd_open(pid) + except ProcessLookupError: + continue + try: + exact = self.identity(pid) + if exact is None or exact[0] != self.pid or exact[1] != row[1]: + os.close(descriptor) + continue + except BaseException: + os.close(descriptor) + raise + pinned[pid] = (descriptor, set()) + descriptor, sent = pinned[pid] + if int(sig) not in sent: + try: + signal.pidfd_send_signal(descriptor, sig) + sent.add(int(sig)) + except ProcessLookupError: + pass + if time.monotonic() - last_note >= 30: + cleanup_note(owned_direct_pids=children) + last_note = time.monotonic() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + # Unknown census/signal state retains exclusion until the kernel + # proves that every direct/adopted child has been reaped. + if diagnostic_error is None: + cleanup_note( + reason="owned-child cleanup failed", error=type(error).__name__ + ) + diagnostic_error = error + try: + time.sleep(0.1) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + # Even a sleep/clock failure cannot escape over an unreaped child. + diagnostic_error = error + if process is not None: + process.wait() + finally: + for descriptor, _sent in pinned.values(): + os.close(descriptor) + if diagnostic_error is not None: + raise RuntimeError( + "cleanup failed; kernel children are now empty and reaped" + ) from diagnostic_error diff --git a/scripts/gates/testsupport/cell_trigger/protocol.json b/scripts/gates/testsupport/cell_trigger/protocol.json new file mode 100644 index 0000000000..850a6e3a27 --- /dev/null +++ b/scripts/gates/testsupport/cell_trigger/protocol.json @@ -0,0 +1,235 @@ +{ + "schema": 1, + "status": "Runtime and entry implementation; native acceptance required", + "memory_fields": [ + "org", + "namespace", + "agent", + "user_id", + "session_id", + "memory_key", + "memory_type", + "value", + "value_json", + "confidence", + "source_trace", + "version", + "tombstoned_at", + "created_at", + "expires_at", + "pinned", + "run_id", + "entities", + "memory_id", + "redacted_at", + "redaction_reason", + "status", + "conflict_with", + "duplicate_of", + "evidence_count", + "recall_count", + "last_recalled_at", + "cross_promoted_from_memory_id", + "cross_promoted_from_namespace", + "tags", + "superseded_by", + "rejection_reason", + "temporal_anchor", + "temporal_confidence", + "updated_at", + "valid_at", + "valid_to", + "invalid_at", + "char_limit", + "reinforcement_count", + "last_reinforced_at", + "mentioned_at", + "source_resource_id", + "content_sha256", + "files", + "concepts", + "verified_by", + "verified_at", + "verified_sha256", + "grounding", + "evidence", + "sensitivity", + "actor", + "source_class", + "lock_mode", + "lock_reason", + "lock_until", + "locked_by", + "locked_at", + "on_expire", + "expire_reason" + ], + "memory_types": { + "org": "string", + "namespace": "string", + "agent": "string", + "user_id": "string", + "session_id": "string", + "memory_key": "string", + "memory_type": "string", + "value": "string", + "source_trace": "string", + "run_id": "string", + "redaction_reason": "string", + "status": "string", + "duplicate_of": "string", + "cross_promoted_from_namespace": "string", + "superseded_by": "string", + "rejection_reason": "string", + "source_resource_id": "string", + "verified_by": "string", + "verified_sha256": "string", + "grounding": "string", + "evidence": "string", + "sensitivity": "string", + "actor": "string", + "source_class": "string", + "lock_mode": "string", + "lock_reason": "string", + "locked_by": "string", + "on_expire": "string", + "expire_reason": "string", + "tombstoned_at": "nullable_utc_second", + "created_at": "utc_second", + "expires_at": "nullable_utc_second", + "redacted_at": "nullable_utc_second", + "last_recalled_at": "nullable_utc_second", + "temporal_anchor": "nullable_utc_second", + "updated_at": "utc_second", + "valid_at": "nullable_utc_second", + "valid_to": "nullable_utc_second", + "invalid_at": "nullable_utc_second", + "last_reinforced_at": "utc_second", + "mentioned_at": "nullable_utc_second", + "verified_at": "nullable_utc_second", + "lock_until": "nullable_utc_second", + "locked_at": "nullable_utc_second", + "entities": "array_string", + "tags": "array_string", + "files": "array_string", + "concepts": "array_string", + "evidence_count": "uint32_not_bool", + "recall_count": "uint32_not_bool", + "reinforcement_count": "uint32_not_bool", + "confidence": "finite_exact_float32_not_bool", + "temporal_confidence": "finite_exact_float32_not_bool", + "value_json": "json_object_field", + "memory_id": "uuid", + "cross_promoted_from_memory_id": "nullable_uuid", + "conflict_with": "array_uuid", + "version": "positive_int63_not_bool", + "pinned": "uint8_0_or_1_not_bool", + "char_limit": "nullable_uint32_not_bool", + "content_sha256": "hex64_fixedstring" + }, + "sparse_defaults": { + "status": "active", + "conflict_with": [], + "duplicate_of": "", + "superseded_by": "", + "rejection_reason": "", + "invalid_at": null, + "redacted_at": null, + "redaction_reason": "", + "grounding": "", + "evidence": "", + "temporal_anchor": null, + "temporal_confidence": 0.0, + "source_resource_id": "", + "verified_by": "", + "verified_at": null, + "verified_sha256": "", + "cross_promoted_from_memory_id": null, + "cross_promoted_from_namespace": "", + "evidence_count": 1, + "recall_count": 0, + "last_recalled_at": null, + "reinforcement_count": 1, + "mentioned_at": null, + "source_trace": "", + "tombstoned_at": null, + "expires_at": null, + "pinned": 0, + "run_id": "-", + "entities": [], + "tags": [], + "files": [], + "concepts": [], + "valid_at": null, + "valid_to": null, + "char_limit": null, + "sensitivity": "", + "lock_mode": "", + "lock_reason": "", + "lock_until": null, + "locked_by": "", + "locked_at": null, + "on_expire": "", + "expire_reason": "" + }, + "audit_fields": [ + "ts", + "event_type", + "org", + "namespace", + "actor", + "subject", + "request_id", + "row_hash", + "build_commit", + "instance", + "instance_label", + "consumer", + "token_id" + ], + "audit_types": { + "ts": "utc_millisecond", + "event_type": "string", + "org": "string", + "namespace": "string", + "actor": "string", + "subject": "raw_subject_string", + "request_id": "string", + "row_hash": "hex64_fixedstring", + "build_commit": "string", + "instance": "string", + "instance_label": "string", + "consumer": "string", + "token_id": "string" + }, + "hash_byte_fields": { + "memory": [ + "content_sha256" + ], + "audit": [ + "row_hash" + ] + }, + "mint_summary_fields": [ + "jti", + "subject", + "org", + "namespaces", + "roles", + "ttl_days", + "is_service_account", + "read_scope", + "read_user_id", + "traffic_class", + "consumer", + "token_path" + ], + "page_rows": 100, + "query_limit_extra_rows": 1, + "memory_row_bytes": 65536, + "audit_row_bytes": 16384, + "query_result_bytes": 20971520, + "total_expected_rows": 45500, + "api_expected_rows": 20500, + "delivery_expected_rows": 25000 +} diff --git a/scripts/gates/testsupport/cell_trigger/tls_proxy.py b/scripts/gates/testsupport/cell_trigger/tls_proxy.py new file mode 100644 index 0000000000..e579fa688d --- /dev/null +++ b/scripts/gates/testsupport/cell_trigger/tls_proxy.py @@ -0,0 +1,168 @@ +"""Private fixed-target TLS bridge; run only after separate runtime admission.""" + +from __future__ import annotations + +import argparse +import asyncio +import signal +import ssl +import sys + + +class Bridge: + def __init__(self, upstream_port: int) -> None: + self.port = upstream_port + self.tasks: set[asyncio.Task] = set() + self.stopping = asyncio.Event() + self.failure: BaseException | None = None + + def fail(self, error: BaseException) -> None: + if self.failure is None: + self.failure = error + self.stopping.set() + + async def copy(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + while data := await asyncio.wait_for(reader.read(65536), 30): + writer.write(data) + await asyncio.wait_for(writer.drain(), 30) + + async def finish(self, pumps: list[asyncio.Task], endpoints: tuple) -> BaseException | None: + first = None + for pump in pumps: + if not pump.done(): + pump.cancel() + try: + outcomes = await asyncio.gather(*pumps, return_exceptions=True) + for outcome in outcomes: + if isinstance(outcome, BaseException) and not isinstance( + outcome, asyncio.CancelledError + ): + first = outcome if first is None else first + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + for endpoint in endpoints: + if endpoint is None: + continue + try: + endpoint.close() + await asyncio.wait_for(endpoint.wait_closed(), 5) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + return first + + async def drain(self, pumps: list[asyncio.Task], endpoints: tuple, first=None): + closing = asyncio.create_task(self.finish(pumps, endpoints)) + while not closing.done(): + try: + await asyncio.shield(closing) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + try: + outcome = closing.result() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + outcome = error + return outcome if first is None else first + + async def connection(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + task = asyncio.current_task() + assert task is not None + if len(self.tasks) >= 8 or self.stopping.is_set(): + error = await self.drain([], (writer,)) + if error is not None: + self.fail(error) + return + self.tasks.add(task) + upstream = None + pumps: list[asyncio.Task] = [] + first = None + try: + async with asyncio.timeout(120): + source, upstream = await asyncio.wait_for( + asyncio.open_connection("127.0.0.1", self.port, limit=65536), 5 + ) + pumps = [ + asyncio.create_task(self.copy(reader, upstream)), + asyncio.create_task(self.copy(source, writer)), + ] + done, _ = await asyncio.wait(pumps, return_when=asyncio.FIRST_COMPLETED) + for pump in done: + pump.result() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error + finally: + first = await self.drain(pumps, (upstream, writer), first) + self.tasks.discard(task) + if first is not None and not ( + self.stopping.is_set() and isinstance(first, asyncio.CancelledError) + ): + self.fail(first) + + async def serve(self, port: int, certificate: str, key: str) -> None: + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.minimum_version = ssl.TLSVersion.TLSv1_2 + context.set_alpn_protocols(["http/1.1"]) + context.load_cert_chain(certificate, key) + loop = asyncio.get_running_loop() + for signum in (signal.SIGINT, signal.SIGTERM): + loop.add_signal_handler(signum, self.stopping.set) + server = await asyncio.start_server( + self.connection, + "127.0.0.1", + port, + ssl=context, + limit=65536, + ssl_handshake_timeout=5, + ssl_shutdown_timeout=5, + ) + try: + await self.stopping.wait() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.fail(error) + finally: + closing = asyncio.create_task(self.shutdown(server)) + while not closing.done(): + try: + await asyncio.shield(closing) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.fail(error) + error = closing.result() + if error is not None: + self.fail(error) + if self.failure is not None: + raise self.failure + + async def shutdown(self, server) -> BaseException | None: + first = None + try: + server.close() + await server.wait_closed() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error + for task in tuple(self.tasks): + task.cancel() + result = await self.finish(list(self.tasks), ()) + return result if first is None else first + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--listen-port", required=True, type=int) + parser.add_argument("--upstream-port", required=True, type=int) + parser.add_argument("--certificate", required=True) + parser.add_argument("--key", required=True) + args = parser.parse_args() + if any(not 0 < port < 65536 for port in (args.listen_port, args.upstream_port)): + return 2 + if args.listen_port == args.upstream_port: + return 2 + try: + asyncio.run(Bridge(args.upstream_port).serve(args.listen_port, args.certificate, args.key)) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + # Error text may contain request bytes or private file names. + print(type(error).__name__, file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/server/auth/keyring.py b/server/auth/keyring.py index d4624a4369..ff5c413d4b 100644 --- a/server/auth/keyring.py +++ b/server/auth/keyring.py @@ -162,8 +162,9 @@ def sign_with_root_only() -> None: def load_for_offline_mint(*, run: Callable[..., Any] | None = None) -> bool: """For a process that mints without serving (a CLI, a dev or bench script): read the keyring once, or sign as kid `default` when ClickHouse - cannot be read. `run` is the CLI's `run_connected`; without it the read - uses a client of its own. True when it was read. Raises ValueError when + cannot be read. `run` can supply a connected read; without it the read + uses a client of its own. Native CLI minting uses `load_for_connected_mint` + inside its enrolled command instead. True when it was read. Raises ValueError when the keyring was recorded under another root secret: no server would accept what this process signs; through `run`, raises AuthorityMismatch when the shell is configured for another deployment (`connect_checked`).""" @@ -191,6 +192,24 @@ async def _read() -> bool: raise # a shell configured for another deployment: refused, not signed by root except Exception: # noqa: BLE001 — an offline mint signs with the root secret read = False + return _offline_mint_result(read) + + +async def load_for_connected_mint(*, ch: Any) -> bool: + """Read signing keys inside an already verified, enrolled native command.""" + from server.core.pg import AuthorityMismatch + from server.core.retirement import ProcessStopping + + try: + read = await refresh(ch=ch) + except (AuthorityMismatch, ProcessStopping): + raise + except Exception: # noqa: BLE001 — an enrolled mint can sign with the root secret + read = False + return _offline_mint_result(read) + + +def _offline_mint_result(read: bool) -> bool: if not read: print(" · signing keyring not read; signing as kid 'default'", file=sys.stderr) sign_with_root_only() diff --git a/server/auth/tenancy.py b/server/auth/tenancy.py index 8c6b4c9437..5042cb22b5 100644 --- a/server/auth/tenancy.py +++ b/server/auth/tenancy.py @@ -262,7 +262,9 @@ async def _place_once(org: str) -> tuple[str, str]: request cancelled leaves the call running for the others.""" task = _placing.get(org) if task is None: - task = asyncio.create_task(_place(org)) + from server.core.background import create_deferred_task + + task = create_deferred_task(_place(org), name="tenancy-placement") _placing[org] = task task.add_done_callback(functools.partial(_placed, org)) return await asyncio.shield(task) diff --git a/server/boot.py b/server/boot.py index 231844de05..3246c4af37 100644 --- a/server/boot.py +++ b/server/boot.py @@ -478,6 +478,9 @@ def _spawn_boot_loop(coro: Coroutine[Any, Any, Any], *, name: str) -> asyncio.Ta """A task that runs for the process's life: cancelled at shutdown, never registered as deferred, since a model transition's drain waits for those.""" task = asyncio.get_running_loop().create_task(coro, name=name) + from server.core.retirement import get_gate + + get_gate().track(task) _BOOT_TASKS.add(task) task.add_done_callback(_BOOT_TASKS.discard) return task diff --git a/server/cli/_ch.py b/server/cli/_ch.py index 09b4e99bc7..7abac27424 100644 --- a/server/cli/_ch.py +++ b/server/cli/_ch.py @@ -23,6 +23,7 @@ import logging import sys from collections.abc import Awaitable, Callable, Iterable +from contextlib import suppress from dataclasses import dataclass from typing import TYPE_CHECKING, Literal @@ -193,7 +194,9 @@ async def check_setup_stores() -> None: gated connect (`loom ch-setup`), the gate a gated connect would run, before it writes: its ClickHouse, where it already records a deployment, must record its Postgres's, and the Postgres `first` row must not name another cell. A ClickHouse with no Loom - database or record yet (a service set up before the first boot) passes.""" + database or record yet (a service set up before the first boot) passes. This + diagnostic discards the mode result and never installs an accepting view; + subsequent Loom row work must verify, enroll and check acceptance afresh.""" from server.core import pg, pg_profile from server.core.ch import get_client from server.core.ch_errors import error_code, error_line @@ -225,7 +228,12 @@ async def check_setup_stores() -> None: pg.expect_deployment((str(recorded["deployment_id"]), int(recorded["epoch"]))) await check_org_free(writes("ClickHouse users and grants, no tenant rows")) finally: - await disconnect(ch) + first = sys.exception() + try: + await disconnect(ch) + except BaseException: # a close failure must not replace the store-check refusal + if first is None: + raise async def check_orgs(orgs: Iterable[str]) -> None: @@ -295,12 +303,10 @@ async def connect_declared( from server.core.pg_profile import check_configured await check_configured(ch) - if org_free is not None and not named: - await check_org_free(org_free) - else: - await check_orgs(named) + await check_acceptance(orgs=named, org_free=org_free) except BaseException: - await disconnect(ch) + with suppress(BaseException): + await disconnect(ch) raise @@ -315,6 +321,39 @@ async def connect_code_plane(ch: CHClient, args: argparse.Namespace) -> None: await connect_declared(ch, orgs=() if getattr(args, "dry_run", False) else (org,)) +async def check_acceptance( + *, orgs: Iterable[str | None] | None = None, org_free: OrgFree | None = None +) -> None: + """Fresh post-enrollment gate check on already verified, open paired stores.""" + named = _declared(orgs, org_free) + if org_free is not None and not named: + await check_org_free(org_free) + else: + await check_orgs(named) + + +async def connect_verified(ch: CHClient) -> None: + """Verify paired stores without obtaining an acceptance view or admitting work. + + Native commands enroll next, then repeat their accepting org/cell checks. + Operator-only commands may keep using the org-free checked composition. + """ + from server.core.pg_profile import check_configured + + if not ch.connected: + await ch.connect() + try: + await check_configured(ch) + except BaseException: # Keep the pairing failure while closing its clients. + with suppress(BaseException): + await disconnect(ch) + raise + from server import boot + from server.core.background import register_buffer_drainer + + register_buffer_drainer("boot-writes", boot.drain_writes) + + async def connect_checked( ch: CHClient, *, orgs: Iterable[str | None] | None = None, org_free: OrgFree | None = None ) -> None: @@ -342,7 +381,11 @@ async def disconnect(ch: CHClient) -> None: pg.expect_deployment(None) try: await pg.close_pool() - finally: + except BaseException: # Keep the pool close failure while closing ClickHouse. + with suppress(BaseException): + await ch.disconnect() + raise + else: await ch.disconnect() @@ -365,11 +408,17 @@ def run_connected[T]( async def _go() -> T: ch = get_client() - await connect_checked(ch, orgs=named, org_free=org_free) + await connect_verified(ch) try: async with model_command(): - return await run(ch) - finally: + await check_acceptance(orgs=named, org_free=org_free) + result = await run(ch) + except BaseException: # Keep the command failure while disconnecting its clients. + with suppress(BaseException): + await disconnect(ch) + raise + else: await disconnect(ch) + return result return asyncio.run(_go()) diff --git a/server/cli/access.py b/server/cli/access.py index 25cfc05667..482aea7b1a 100644 --- a/server/cli/access.py +++ b/server/cli/access.py @@ -6,6 +6,7 @@ from __future__ import annotations import argparse +import asyncio import json import sys from datetime import UTC @@ -39,31 +40,34 @@ async def _lookup_owner(ch: Any) -> str | None: return None return str(row.get("owner_email") or "") or "(unknown)" - owner = run_connected(_lookup_owner, org_free=reads("reads")) - if owner is None: - print( - f"no claim found for {args.org}/{args.namespace}", - file=sys.stderr, - ) - return 1 - if not args.yes: - print( - f"This will release the claim on {args.org}/{args.namespace}" - f" (currently owned by {owner!r})." - ) - print( - "Grants are revoked and the namespace deactivated. Its memories are " - "kept, and the name stays reserved: nobody can claim it again." - ) - try: - resp = input("Continue? [y/N] ").strip().lower() - except EOFError: # no terminal to answer (a pipe, a cron): the answer is no - resp = "" - if resp not in ("y", "yes"): - print("aborted") - return 1 - async def _go(ch: Any) -> int: + from server.cli._ch import check_acceptance + + owner = await _lookup_owner(ch) + if owner is None: + print( + f"no claim found for {args.org}/{args.namespace}", + file=sys.stderr, + ) + return 1 + if not args.yes: + print( + f"This will release the claim on {args.org}/{args.namespace}" + f" (currently owned by {owner!r})." + ) + print( + "Grants are revoked and the namespace deactivated. Its memories are " + "kept, and the name stays reserved: nobody can claim it again." + ) + try: + resp = (await asyncio.to_thread(input, "Continue? [y/N] ")).strip().lower() + except EOFError: # no terminal to answer (a pipe, a cron): the answer is no + resp = "" + if resp not in ("y", "yes"): + print("aborted") + return 1 + # Confirmation can wait while the cell changes; check again before mutation. + await check_acceptance(orgs=(args.org,)) out = await release_claim( org=args.org, namespace=args.namespace, @@ -110,8 +114,6 @@ def _invalid_namespace_names(names: list[str]) -> list[str]: def _cmd_mint_token(args: argparse.Namespace) -> int: from server.auth import keyring - from server.auth.authz import GRANT_ROLES - from server.auth.jwt import mint_token, new_jti from server.core.principals import RESERVED_SUBJECT, is_reserved_subject if is_reserved_subject(args.subject): @@ -122,19 +124,49 @@ def _cmd_mint_token(args: argparse.Namespace) -> int: if refused: print(f" ✗ {refused}", file=sys.stderr) return 2 - if args.platform_admin and not _platform_admin_rollout_ready(): + + async def _go(ch: Any) -> int: + from server.cli._ch import check_acceptance + + if args.platform_admin and not await _platform_admin_rollout_ready(ch): + print( + " ✗ platform-admin tokens require a readable fleet with the rollout complete", + file=sys.stderr, + ) + return 1 + try: + await keyring.load_for_connected_mint(ch=ch) + except ValueError as e: # another root: no server would accept the token + print(f" ✗ {e}", file=sys.stderr) + return 1 + # Keyring reads can wait; signing must use a current accepting view too. + await check_acceptance(orgs=(args.org,)) + minted = _mint_token(args) + if isinstance(minted, int): + return minted + token, summary = minted + if args.platform_admin and not await _audit_platform_admin_mint(args, summary): + print(" ✗ the mint could not be recorded in the audit log; no token", file=sys.stderr) + return 1 + return _emit_minted_token(args, token, summary) + + try: + return run_connected(_go, orgs=(args.org,)) + except AuthorityMismatch: + raise + except Exception as e: # noqa: BLE001 — a mint without verified enrollment is refused print( - " ✗ platform-admin tokens require a readable fleet with the rollout complete", + f" ✗ token mint did not complete in its enrolled scope: {type(e).__name__}", file=sys.stderr, ) return 1 - try: - keyring.load_for_offline_mint( - run=lambda work: run_connected(work, org_free=reads("reads the signing keyring")) - ) - except ValueError as e: # another root: no server would accept the token - print(f" ✗ {e}", file=sys.stderr) - return 1 + + +def _mint_token(args: argparse.Namespace) -> tuple[str, dict[str, Any]] | int: + """Build and sign only inside the command's enrolled admission scope.""" + from server.auth.authz import GRANT_ROLES + from server.auth.jwt import mint_token, new_jti + ttl_seconds = (args.ttl_days * 86400) if args.ttl_days else None # Every minted token carries a jti; generated here so # the summary can name it beside (or instead of) the token — it is the @@ -238,11 +270,7 @@ def _cmd_mint_token(args: argparse.Namespace) -> int: "consumer": args.consumer or "", } - if args.platform_admin and not _audit_platform_admin_mint(args, summary): - # Never handed out unrecorded: the role spans every org. - print(" ✗ the mint could not be recorded in the audit log; no token", file=sys.stderr) - return 1 - return _emit_minted_token(args, token, summary) + return token, summary # not a setting: the longest a minted platform-admin token may live; longer is a bot's job @@ -272,45 +300,40 @@ def _platform_admin_refusal(args: argparse.Namespace) -> str | None: return None -def _platform_admin_rollout_ready() -> bool: +async def _platform_admin_rollout_ready(ch: Any) -> bool: from server.auth import machines try: - return bool( - run_connected( - machines.platform_admin_tokens_ready, org_free=reads("reads the live instances") - ) - ) + return bool(await machines.platform_admin_tokens_ready(ch)) except AuthorityMismatch: - raise # a shell configured for another deployment: its own one-line refusal + raise except Exception as e: # noqa: BLE001 — an unread fleet refuses an admin token print(f" ✗ fleet state could not be read: {type(e).__name__}", file=sys.stderr) return False -def _audit_platform_admin_mint(args: argparse.Namespace, summary: dict[str, Any]) -> bool: +async def _audit_platform_admin_mint(args: argparse.Namespace, summary: dict[str, Any]) -> bool: """Record a platform-admin mint before it is handed out; False when the audit row could not be written.""" import getpass from server.observability import audit - async def _emit(ch: Any) -> str: - return await audit.emit( - "auth.platform_admin_minted", - org=args.org, - namespace="-", - actor=f"cli:{getpass.getuser()}", - subject={ - "subject": args.subject, - "jti": summary["jti"], - "namespaces": summary["namespaces"], - "ttl_days": args.ttl_days, - }, - ) - try: - return bool(run_connected(_emit, orgs=(args.org,))) + return bool( + await audit.emit( + "auth.platform_admin_minted", + org=args.org, + namespace="-", + actor=f"cli:{getpass.getuser()}", + subject={ + "subject": args.subject, + "jti": summary["jti"], + "namespaces": summary["namespaces"], + "ttl_days": args.ttl_days, + }, + ) + ) except Exception as e: # noqa: BLE001 — an unrecorded mint is refused, never handed out print(f" ✗ audit log not reached: {type(e).__name__}: {e}", file=sys.stderr) return False @@ -714,7 +737,9 @@ def _add_revoke_token_parser(sub: Any) -> None: def _add_mint_token_parser(sub: Any) -> None: """The `mint-token` subcommand.""" - p_mint = sub.add_parser("mint-token", help="Mint a JWT locally (requires LOOM_JWT_SECRET).") + p_mint = sub.add_parser( + "mint-token", help="Mint a JWT locally (requires LOOM_JWT_SECRET and verified stores)." + ) p_mint.add_argument("--subject", required=True) p_mint.add_argument("--org", required=True) p_mint.add_argument( diff --git a/server/cli/analytics_export.py b/server/cli/analytics_export.py index f69e3f9396..63bd9732dd 100644 --- a/server/cli/analytics_export.py +++ b/server/cli/analytics_export.py @@ -9,7 +9,7 @@ from datetime import date, timedelta from typing import Any -from server.cli._ch import connect_declared, disconnect, reads +from server.cli._ch import check_acceptance, connect_verified, disconnect, reads from server.core.datetimes import utc_now as _now # The first botocore whose put_object takes IfNoneMatch (the `aws` extra's floor). @@ -90,17 +90,27 @@ def new_client() -> Any: check_credentials(role_arn=args.role_arn, region=args.region) put = s3_put(new_client, args.bucket, expected_owner=args.expected_bucket_owner) - # No model calls and no writes to ClickHouse: a plain connection, outside - # the provider barrier that `run_connected` enters. It connects inside the - # deadline's scope, so the driver's read timeout is the deadline too. + # Analytics export consumes org data even without inference; its entire native + # body participates in permanent retirement without taking model admission. + from server.insights.app.provider_command import participant_command + async def _go() -> list[Exported]: with statement_timeout(STATEMENT_TIMEOUT_S): ch = get_client() - await connect_declared(ch, org_free=reads("reads; writes only the export bucket")) + await connect_verified(ch) try: - return await export_day(day, prefix=args.prefix, put=put, name_key=name_key, ch=ch) + async with participant_command(): + await check_acceptance(org_free=reads("reads; writes only the export bucket")) + return await export_day( + day, prefix=args.prefix, put=put, name_key=name_key, ch=ch + ) finally: - await disconnect(ch) + first = sys.exception() + try: + await disconnect(ch) + except BaseException: # a disconnect failure must not replace the command failure + if first is None: + raise for e in asyncio.run(_go()): # A kept object's size is the earlier run's, which this run never read. diff --git a/server/cli/cell.py b/server/cli/cell.py index 9cdec5c894..fb1b04db95 100644 --- a/server/cli/cell.py +++ b/server/cli/cell.py @@ -1,7 +1,8 @@ """`loom cell credential | limit | which | remove`: the operator's commands that touch the -router's directory (docs/architecture/cells.md, "Operations"). Only `remove` reads a cell's -stores, a first cell's Postgres; the others run while a cell is down. `loom cell prepare` is -`server/cli/cell_prepare.py`. +router's directory (docs/architecture/cells.md, "Operations"). Removal holds the cell's +Postgres lock and claims its directory generation; limits and credentials need only the +directory. `loom cell prepare` is +`server/cli/cell_prepare.py`, `loom cell attach` `server/cli/cell_attach.py`. - `credential new` makes a router credential offline and prints it with its SHA-256; `credential --cell --next | --finish | --revert` rotates a cell's. @@ -10,7 +11,8 @@ - `which --org ` prints the cell and host the router's lookup answers for an org (the first cell's for an org with no placement, which it cannot tell apart). - `remove --cell ` removes the directory row of a cell that is `attaching` with no - placement; for the first cell, only once its Postgres reads `open` and records the row's + placement, holding its configured Postgres lock; for the first cell, only once it reads + `open` and records the row's deployment id (the troubleshooting page's back-out runs first). `credential` (except `new`), `limit` and `remove` call the router's operator API with @@ -33,6 +35,7 @@ from server.auth.public_url import is_loopback_host from server.core.cell import NAME, PLACEMENTS_MAX, SHA256, TROUBLESHOOTING from server.router.directory import ( + CONTROL_MAX_S, OPERATOR_CHANGE_MAX_S, OPERATOR_READ_MAX_S, REMOVE_MAX_S, @@ -45,7 +48,7 @@ # A lookup makes up to two directory reads (the org, then the first cell), each a # slot-and-connection wait and a statement. A connect fails fast. _OPERATOR_TIMEOUT = httpx.Timeout( # not a setting: derived - max(OPERATOR_CHANGE_MAX_S, OPERATOR_READ_MAX_S, REMOVE_MAX_S) + 10.0, connect=5.0 + max(CONTROL_MAX_S, OPERATOR_CHANGE_MAX_S, OPERATOR_READ_MAX_S, REMOVE_MAX_S) + 10.0, connect=5.0 ) _LOOKUP_TIMEOUT = httpx.Timeout( # not a setting: derived 2 * (2 * STATEMENT_TIMEOUT_S + 1) + 10.0, connect=5.0 @@ -155,6 +158,18 @@ def _operator_call( if isinstance(found, int): return found router = found + if path in ("limit", "credential"): + from server.cli import cell_control + + row = _read_row(router, args.cell) + if isinstance(row, str): + print(f"loom cell: {row}", file=sys.stderr) + return 1 + try: + body = {**(body or {}), **cell_control.preconditions(row)} + except cell_control.Refused as e: + print(str(e), file=sys.stderr) + return 1 try: r = _send(router, method, args.cell, path, body) except (httpx.ConnectError, httpx.ConnectTimeout, httpx.ProxyError) as e: @@ -306,43 +321,6 @@ def _read_row(router: httpx.URL, cell: str) -> dict[str, Any] | str: return answer -# The first cell's check: one Postgres session, its statements and the whole read bounded. -_CHECK_STATEMENT_MS = 5000 # not a setting: a bound on one CLI read, not on serving -_CHECK_S = 10.0 # not a setting: a bound on one CLI read, not on serving - - -async def _read_first_cell() -> tuple[str, str]: - """The cell state and the deployment id the cell's Postgres records ('' when none), on - one session of its own: no ClickHouse and no pool.""" - from server.core import cell, pg - - async with pg.dedicated(statement_timeout_ms=_CHECK_STATEMENT_MS) as conn: - state, _first = await cell.read_state_if_present(conn) - recorded = await conn.fetchval("SELECT deployment_id FROM loom.authority") - return state, str(recorded or "") - - -async def _first_cell_refusal(deployment_id: str) -> str: - """Why the first cell's row may not be removed yet ('' when it may): its Postgres still - holds a cell state, or is another deployment's. It reads `LOOM_PG_DSN` alone.""" - if not settings.pg_dsn: - return "the first cell keeps its state in Postgres: set LOOM_PG_DSN to the cell's" - state, recorded = await asyncio.wait_for(_read_first_cell(), timeout=_CHECK_S) - if state != "open": - return ( - f"the first cell's Postgres reads `{state}`: unset LOOM_CELL on every process, run " - f"the full-state cleanup ({TROUBLESHOOTING}), then remove the row" - ) - if not recorded: - return "this Postgres records no deployment id: point LOOM_PG_DSN at the cell's own" - if recorded != deployment_id: - return ( - f"this Postgres records deployment id {recorded!r}, and the directory's row " - f"{deployment_id!r}: point LOOM_PG_DSN at the cell's own Postgres" - ) - return "" - - def _cmd_remove(args: argparse.Namespace) -> int: router = _cell_router(args) if isinstance(router, int): @@ -370,24 +348,48 @@ def _cmd_remove(args: argparse.Namespace) -> int: if reason := remove_refusal(args.cell, state, placed): print(f"loom cell remove: refusing: {reason}", file=sys.stderr) return 1 - if row["is_first"]: - try: - refusal = asyncio.run(_first_cell_refusal(row["deployment_id"])) - except Exception as e: # noqa: BLE001 — an unread state refuses the removal - reason = f"no answer within {_CHECK_S:g} s" if isinstance(e, TimeoutError) else str(e) - refusal = f"the first cell's Postgres could not be read ({type(e).__name__}: {reason})" - if refusal: - print(f"loom cell remove: refusing: {refusal}", file=sys.stderr) - return 1 - return _operator_call( - args, - "", - None, - "a retry refused as not in the directory (NO_SUCH_CELL) means the row was removed", - method="DELETE", - router=router, - coded_404=True, - ) + try: + answer = asyncio.run(_remove_locked(args, router)) + except Exception as e: # noqa: BLE001 — an unknown control outcome stops the run + print(f"loom cell remove: {type(e).__name__}: {e}; stop and rerun", file=sys.stderr) + return 1 + print(json.dumps(answer, indent=2, sort_keys=True)) + return 0 + + +async def _remove_locked( + args: argparse.Namespace, + router: httpx.URL, + transport: httpx.AsyncBaseTransport | None = None, +) -> dict[str, Any]: + from server.cli import cell_control + from server.core import cell + + async def remove(owner: cell_control.Owner) -> dict[str, Any]: + conn = owner.conn + + async def call(path: str, body: dict[str, Any]) -> dict[str, Any]: + return await cell_control.request(router, args.cell, transport, path, body) + + # Re-read under the local lock; the earlier read only checked eligibility. + row = await cell_control.request(router, args.cell, transport, "", method="GET") + if reason := remove_refusal(args.cell, row["state"], row["placed_orgs"]): + raise cell_control._refused(reason) + await cell_control.paired(conn, row) + if row["is_first"]: + state, _ = await owner.transaction(lambda: cell.read_state_if_present(conn)) + if state != "open": + raise cell_control._refused( + f"the first cell's Postgres reads {state}: run the full-state cleanup " + f"({TROUBLESHOOTING}) before removing its directory row" + ) + row = await cell_control.claim(conn, call, row) + await cell_control.check_lock(conn) + return await cell_control.request( + router, args.cell, transport, "", cell_control.preconditions(row), method="DELETE" + ) + + return await cell_control.run("remove", remove) def _cmd_which(args: argparse.Namespace) -> int: @@ -438,7 +440,7 @@ def _add_cell_parser(sub: Any) -> None: limit.add_argument("--cell", required=True, help="The cell to limit") state = limit.add_mutually_exclusive_group() state.add_argument("--closed", action="store_true", help="Refuse its new orgs") - state.add_argument("--open", action="store_true", help="Take new orgs again") + state.add_argument("--open", action="store_true", help="Take new orgs again once serving") limit.add_argument( "--placements-max", metavar="N|none", help="Its quota of orgs (none: no quota)" ) @@ -459,6 +461,8 @@ def _add_cell_parser(sub: Any) -> None: remove.add_argument("--router", help="The router's URL (default: LOOM_CELL_ROUTER_URL)") remove.set_defaults(func=_cmd_remove) + from server.cli.cell_attach import add_attach_parser from server.cli.cell_prepare import add_prepare_parser add_prepare_parser(csub) + add_attach_parser(csub) diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py new file mode 100644 index 0000000000..ea20827a35 --- /dev/null +++ b/server/cli/cell_attach.py @@ -0,0 +1,745 @@ +"""`loom cell attach`: turn a prepared first cell into a `serving` one +(docs/architecture/cells.md, "Attaching a cell"; the runbook is +docs/operations/cell-attach.md). + +A run holds the attach lock on a direct session of the cell's Postgres for its life, and +goes, each step idempotent, so a rerun resumes by the directory row and the cell's mode. +Steps 1 to 4 are the design's, as cells.md numbers them: + +- the directory row: on `serving` it is done (exit 0) once the connected authority and + configured cell and bucket match the registration and the cell reads `enforced`; +- the refusals, before any write: the stores are the registered ones, the session is + direct and marked, the cell's processes (their heartbeats), the roles, settings and + subscriptions that would let an unchecked writer through, the ClickHouse writers, the + table list, durability, attach's own grants; +- `attach/begin` (the router probes the stores and writes the fingerprints); +- step 1, the mode: `recording`, or `enforced` on a resume; +- the triggers (`cell_triggers.ensure_table`), under the migration lock one try at a time; +- in `recording`: step 2, the scan, refusing an org past `cell.ORG_MAX_BYTES`, and the + tenancy rows of the orgs found; the triggers again; the pre-commit checks; step 3, + `attach/enforced` and the mode commit, from which on the mode only rises, so every + later refusal leaves the cell `enforced` and `attaching`; +- the durable read of `enforced`, the heartbeats, and step 4: the wait for every + transaction that could have read `recording`, the rescan, the tenancy rows and their + placements (`attach/place`), the final checks and `attach/serving`. + +`--wait` bounds the waits in wall time, `--scan-budget` each full scan. `--dry-run` runs +the refusals and the scan and prints what would be placed, writing nothing. +""" + +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import json +import sys +import time +from collections.abc import Callable, Iterable +from typing import Any + +import httpx + +from server.cli import cell as cell_cli +from server.cli import cell_attach_beats as beats +from server.cli import cell_attach_pg as apg +from server.cli import cell_control, cell_scan +from server.cli._ch import writes +from server.cli.cell_ch import Refusal, Refused, read_topology +from server.core import cell, cell_triggers +from server.settings import settings + +_WAIT_S = 600 # not a setting: a flag's default +_SCAN_BUDGET_S = 7200 # not a setting: a flag's default +_LOCK_STATEMENT_MS = 60_000 # not a setting: a bound on the lock session's catalog reads +_HEALTH_TIMEOUT = httpx.Timeout(10.0, connect=5.0) # not a setting: one health read + + +def _args_problem(args: argparse.Namespace) -> str: + if not cell.NAME.fullmatch(args.cell or ""): + return f"{args.cell!r} is no cell name (1 to 32 of a-z, 0-9, -)" + if args.cell != settings.cell: + return f"LOOM_CELL is {settings.cell!r}: attach runs with LOOM_CELL={args.cell}" + if settings.storage_authority != "postgres" or not settings.pg_dsn: + return ( + "a cell keeps its state in Postgres: set LOOM_STORAGE_AUTHORITY=postgres and " + "LOOM_PG_DSN" + ) + if args.wait < 0 or args.scan_budget <= 0: + return "--wait is 0 or more seconds, --scan-budget more than 0" + return "" + + +class Router: + """The router's operator API for one cell.""" + + def __init__( + self, url: httpx.URL, name: str, transport: httpx.AsyncBaseTransport | None + ) -> None: + self.url, self.name, self.transport = url, name, transport + + async def call(self, path: str, body: dict[str, Any] | None = None) -> dict[str, Any]: + return await cell_control.request( + self.url, + self.name, + self.transport, + path, + body, + method="GET" if body is None else "POST", + ) + + async def health(self, host: str) -> str: + """The deployment id `https:///v1/health/deployment` answers ('' when none).""" + try: + manager = httpx.AsyncClient(timeout=_HEALTH_TIMEOUT, transport=self.transport) + client = await manager.__aenter__() + try: + r = await client.get(f"https://{host}/v1/health/deployment") + except BaseException: # Client exit preserves the request's first failure. + with contextlib.suppress(BaseException): + await manager.__aexit__(*sys.exc_info()) + raise + except (httpx.HTTPError, ValueError): + return "" + # A required close failure after a successful read must reach the owner. + await manager.__aexit__(None, None, None) + try: + answer = r.json() + except (httpx.HTTPError, ValueError): + return "" + return str(answer.get("deployment_id") or "") if isinstance(answer, dict) else "" + + +def _too_long(found: cell_scan.ScanResult) -> list[Refusal]: + """Step 2's refusal of every org past `cell.ORG_MAX_BYTES`, before any tenancy row.""" + return [ + Refusal( + f"org of {len(org.encode())} bytes ({org[:64]}…)", + f"found in {', '.join(sorted(sources))}: longer than {cell.ORG_MAX_BYTES} bytes; " + "remove it first (docs/operations/cell-attach.md, the over-long org procedure)", + ) + for org, sources in sorted(found.too_long.items()) + ] + + +def _refused(subject: str, reason: str) -> Refused: + return Refused([Refusal(subject, reason)]) + + +def _encoded_size(body: Any) -> int: + return len(json.dumps(body, ensure_ascii=True, separators=(",", ":")).encode()) + + +def batches(orgs: Iterable[str], deployment_id: str) -> list[list[str]]: + """`orgs` in `attach/place` batches: at most ATTACH_BATCH each, and an encoded body + within the router's body bound.""" + from server.router.limits import ATTACH_BATCH, MAX_BODY_BYTES + + out: list[list[str]] = [] + batch: list[str] = [] + for org in sorted(orgs): + if batch and ( + len(batch) >= ATTACH_BATCH + or _encoded_size({"deployment_id": deployment_id, "orgs": [*batch, org]}) + > MAX_BODY_BYTES + ): + out.append(batch) + batch = [] + batch.append(org) + if batch: + out.append(batch) + return out + + +class Run: + """One attach run: its sessions, what it read, and what it prints.""" + + def __init__( + self, + ch: Any, + args: argparse.Namespace, + router: Router, + log: Callable[[str], None], + ) -> None: + self.ch, self.args, self.router, self.log = ch, args, router, log + self.name = args.cell + self.wait = apg.Wait(args.wait) + self.lock_conn: Any = None + self.owner: cell_control.Owner + self.row: dict[str, Any] = {} + self.mode = "" + self.own_fp: tuple[dict[str, Any] | None, dict[str, Any] | None] = (None, None) + self.keys_fp = "" # the live processes' one root-secret fingerprint, last read + self.begun_keys_fp = "" # the one this run's `attach/begin` wrote + self.report: dict[str, Any] = {"cell": self.name} + + # -- the lock session -------------------------------------------------------------- + + async def confirm_lock(self) -> None: + """A lost backend ends the owner; a new run must claim and start again.""" + await cell_control.check_lock(self.lock_conn) + + async def mutate(self, path: str, body: dict[str, Any]) -> dict[str, Any]: + answer = await cell_control.mutate(self.lock_conn, self.router.call, self.row, path, body) + if "mutation_generation" in answer: + self.row = {**self.row, **answer} + return answer + + # -- the refusals ------------------------------------------------------------------ + + async def read_own_row(self) -> None: + own = await self.lock_conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") + if own is None: + raise _refused("postgres loom.cell_self", "absent: run loom cell prepare first") + self.own_fp = (cell.json_object(own["current_fp"]), cell.json_object(own["next_fp"])) + + def judge_beats(self, stopped_unset: str) -> Callable[[list[beats.Beat]], beats.Verdict]: + def _judge(found: list[beats.Beat]) -> beats.Verdict: + if stopped_unset == "ignore": + found = [b for b in found if b.live or b.cell == self.name] + return beats.judge( + found, self.name, self.own_fp, "wait" if stopped_unset == "wait" else "refuse" + ) + + return _judge + + async def check_beats(self, stopped_unset: str) -> beats.Verdict: + verdict = await beats.settle( + lambda: beats.read_beats(self.ch), self.judge_beats(stopped_unset), self.wait + ) + (self.keys_fp,) = verdict.keys_fps + return verdict + + async def stores_refusals(self) -> list[Refusal]: + from server.core import pg_profile + + out = [] + row = self.row + if not row.get("is_first") or row.get("state") != "attaching": + out.append( + Refusal( + "directory row", + f"is {'the first' if row.get('is_first') else 'a later'} cell, " + f"{row.get('state')}: this attach is the first cell's, while attaching", + ) + ) + if row.get("other"): + out.append( + Refusal( + "directory row", + f"another cell, {row['other']}, is registered: remove it first " + "(loom cell remove)", + ) + ) + expected = str(row.get("deployment_id") or "") + recorded_pg = await self.lock_conn.fetchval("SELECT deployment_id FROM loom.authority") + if recorded_pg != expected: + out.append( + Refusal( + "postgres", f"records deployment id {recorded_pg!r}, the directory {expected!r}" + ) + ) + recorded = await pg_profile.recorded_authority(self.ch) + if recorded is None or str(recorded["deployment_id"]) != expected: + out.append(Refusal("clickhouse", f"records another deployment id than {expected!r}")) + state, first = await apg.read_mode(self.lock_conn) + if state in cell.MODES: + self.mode = state # what a refusal from here on leaves the cell at + if first != (True, self.name): + out.append( + Refusal("postgres loom.cell_first", f"reads {first}, not (true, {self.name})") + ) + if state not in ("recording", "enforced"): + out.append(Refusal("postgres cell state", f"reads {state}")) + answered = await self.router.health(str(row.get("host") or "")) + if answered != expected: + out.append( + Refusal( + f"https://{row.get('host')}/v1/health/deployment", + f"answers deployment id {answered!r}, not {expected!r}", + ) + ) + return out + + async def durability_refusals(self) -> list[Refusal]: + from server.core import pg, pg_profile + + out = [] + verdict = pg_profile.durability_verdict("synchronous", *await pg_profile.standby_state()) + if verdict.refusal: + out.append(Refusal("postgres durability", verdict.refusal)) + try: + await pg.check_durable_message() + except Exception as e: # noqa: BLE001 — a role that may not write it refuses + out.append(Refusal("postgres durability", f"pg_logical_emit_message refused: {e}")) + if (now := await pg_profile.standby_now()) != "synchronous": + out.append( + Refusal("postgres standbys", f"commits are acknowledged as {now}, not synchronous") + ) + return out + + async def ch_limit_refusals(self) -> list[Refusal]: + """A profile or quota that caps the rows or bytes attach's user reads would end a + scan early.""" + row = await self.ch.query_one( + "SELECT getSetting('max_rows_to_read') AS r, getSetting('max_bytes_to_read') AS b" + ) + out = [] + if row and (int(row["r"]) or int(row["b"])): + out.append( + Refusal("clickhouse user", "a settings profile caps the rows or bytes it reads") + ) + quota = await self.ch.query_all( + "SELECT quota_name FROM system.quota_usage " + "WHERE max_read_rows > 0 OR max_read_bytes > 0" + ) + out += [ + Refusal(f"clickhouse quota {q['quota_name']}", "caps the rows or bytes attach reads") + for q in quota + ] + return out + + async def writer_refusals(self) -> tuple[list[Refusal], Any, Any]: + from server.cli import cell_writers + + topology = await read_topology(self.ch) + if refusals := topology.refusals(): + return refusals, topology, {} + names = await cell_writers.host_names(self.ch, topology) if topology.multi else {} + allowed = cell_writers.Allowed( + users=frozenset(self.args.ch_allow_user), + user_directories=frozenset(self.args.ch_allow_user_directory), + objects=frozenset(self.args.ch_allow_object), + replicas=frozenset(self.args.ch_allow_replica), + ) + return await cell_writers.check_writers(self.ch, topology, names, allowed), topology, names + + async def catalog_refusals(self) -> list[Refusal]: + """The Postgres checks that run again before the mode commit and in step 4.""" + holders = await apg.privilege_holders(self.lock_conn, exempt_self=True) + self.report["own_roles"] = holders.own + out = apg.privilege_refusals(holders, self.args.pg_allow_role) + out += await apg.unmarked_refusals(self.lock_conn, await apg.session_passes()) + return out + + async def refusals(self) -> tuple[Any, Any, cell_scan.TableList]: + """Every refusal before any write, named together.""" + out = await self.stores_refusals() + out += await apg.encoding_refusals(self.lock_conn) + out += await self.catalog_refusals() + out += await self.durability_refusals() + rows = await self.lock_conn.fetch(cell_triggers.ORG_TABLES) + out += await apg.own_role_refusals(self.lock_conn, list(rows)) + out += await self.ch_limit_refusals() + out += beats.prune_window_refusals() + if out: + raise Refused(out) + state, _ = await apg.read_mode(self.lock_conn) + # A stopped process with LOOM_CELL unset refuses while the cell can still be + # backed out; on a resume in `enforced` step 4 waits for it to age out. + await self.check_beats("refuse" if state == "recording" else "ignore") + writers, topology, names = await self.writer_refusals() + out += writers + listed = await cell_scan.list_tables(self.ch, self.lock_conn, topology, names) + out += listed.refusals + if out: + raise Refused(out) + return topology, names, listed + + # -- begin ------------------------------------------------------------------------- + + async def begin(self) -> None: + from server.cli.cell_prepare import _delete_nonces, _new_nonces, _write_nonces + from server.router.limits import ATTACH_BEGIN_MAX_BYTES + + current, nxt = self.own_fp + nonces = _new_nonces() + body = { + "deployment_id": self.row["deployment_id"], + "current_fp": current, + "next_fp": nxt, + "keys_fp": self.keys_fp, + "ch_allow_users": sorted(self.args.ch_allow_user), + "ch_allow_user_directories": sorted(self.args.ch_allow_user_directory), + "ch_allow_objects": sorted(self.args.ch_allow_object), + "ch_allow_replicas": sorted(self.args.ch_allow_replica), + "pg_allow_roles": sorted(self.args.pg_allow_role), + "nonces": nonces, + } + body.update(cell_control.preconditions(self.row)) + if (size := _encoded_size(body)) > ATTACH_BEGIN_MAX_BYTES: + raise _refused( + "attach/begin", + f"its body is {size} bytes, past the router's {ATTACH_BEGIN_MAX_BYTES}: the " + "fingerprints or the allow lists are too large", + ) + try: + await _write_nonces(self.ch, nonces) + self.row = {**self.row, **await self.mutate("/attach/begin", body)} + self.begun_keys_fp = self.keys_fp + except BaseException: # Nonce cleanup preserves the first failure or cancellation. + with contextlib.suppress(BaseException): + await _delete_nonces(self.ch, nonces) + raise + else: + await _delete_nonces(self.ch, nonces) + + # -- the scans --------------------------------------------------------------------- + + async def scan(self, topology: Any, listed: cell_scan.TableList) -> cell_scan.ScanResult: + budget = cell_scan.Budget(self.args.scan_budget) + current, nxt = self.own_fp + return await cell_scan.scan( + self.ch, listed, topology, budget, cell_scan.configured_orgs(current, nxt) + ) + + def _named(self, found: cell_scan.ScanResult) -> dict[str, Any]: + return { + "malformed": {k: v[:20] for k, v in sorted(found.malformed.items())}, + "too_long": { + f"{o[:64]}… ({len(o.encode())} bytes)": sorted(s) for o, s in found.too_long.items() + }, + } + + async def triggers(self) -> None: + if refusals := await apg.ensure_triggers(self.owner, self.wait, self.log): + raise Refused(refusals) + + # -- step 3 ------------------------------------------------------------------------ + + async def ready_for_commit(self) -> list[str]: + """The cheap heartbeat refusals right before a mode-commit try: a refusal raises, + a booting process skips the try.""" + verdict = self.judge_beats("refuse")(await beats.read_beats(self.ch)) + if verdict.refusals: + raise Refused(verdict.refusals) + return verdict.booting + + async def commit_mode(self) -> None: + await self.mutate("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) + + def mark(mode: str) -> None: + self.mode = mode + + await apg.commit_enforced( + self.owner, + self.confirm_lock, + self.ready_for_commit, + self.wait, + self.log, + mark, + expected_cell=self.name, + expected_fp=self.own_fp, + ) + + async def durable(self) -> int: + """The bounded durable read of `enforced`; then X, the snapshot's + xmax, above the read's own xid.""" + from server.core import pg_profile + + with self.wait.counting(): + if await pg_profile.standby_now() != "synchronous": + raise _refused("postgres standbys", "not synchronous before the durable read") + read = await apg.durable_read(self.owner) + if await pg_profile.standby_now() != "synchronous": + raise _refused("postgres standbys", "not synchronous after the durable read") + if read.mode in cell.MODES: + self.mode = read.mode + if read.first != (True, self.name) or read.mode != "enforced": + raise _refused( + "postgres cell state", + f"the durable read saw mode {read.mode!r} and first row {read.first}: the mode " + "commit did not land; rerun attach", + ) + self.mode = "enforced" + xmax = await apg.snapshot_xmax(self.lock_conn) + if xmax <= read.xid: + raise _refused("postgres snapshot", f"its xmax {xmax} is not above {read.xid}") + return xmax + + # -- step 4 ------------------------------------------------------------------------ + + async def nonheartbeat_checks(self) -> None: + from server.core import pg_profile + + out, _, _ = await self.writer_refusals() + out += [ + Refusal("postgres triggers", d.text) + for d in await cell_triggers.missing(self.lock_conn) + ] + out += await self.catalog_refusals() + if out: + raise Refused(out) + if (now := await pg_profile.standby_now()) != "synchronous": + raise _refused("postgres standbys", f"commits are acknowledged as {now}") + + async def cheap_checks(self, stopped_unset: str) -> None: + await self.nonheartbeat_checks() + await self.check_beats(stopped_unset) + + async def final_checks(self, watch: beats.Watch) -> None: + from server.core import instances + + floor = apg.Wait(max(self.wait.left(), instances.boot_allowance_s())) + + async def _read() -> list[beats.Beat]: + found = await watch.read_once() + if refusals := watch.refusals(): + raise Refused(refusals) + # A cell-correct process whose cell row alone was swept. + return [b for b in found if not (b.cell is None and b.instance_id in watch.cell_ids)] + + verdict = await beats.settle( + _read, self.judge_beats("refuse"), floor, extend_s=instances.boot_allowance_s() + ) + if verdict.keys_fps != {self.begun_keys_fp}: + raise _refused( + "processes", + "the root-secret fingerprint changed during attach (a root-secret rotation): " + "rerun attach", + ) + + async def step4(self, xmax: int, topology: Any, names: Any) -> None: + from server.core import pg + + async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: + await apg.wait_transactions(conn, xmax, self.wait) + await self.cheap_checks("wait") + since = str((await self.ch.query_one("SELECT toString(now64(3)) AS t"))["t"]) + watch = beats.Watch(self.ch, since, self.name) + task = asyncio.create_task(watch.run()) + try: + listed = await cell_scan.list_tables(self.ch, self.lock_conn, topology, names) + found = await self.scan(topology, listed) + orgs = set(found.orgs) + self.report["tenancy_rows_written"] = await apg.write_tenancy(orgs) + placed = 0 + for batch in batches(orgs, self.row["deployment_id"]): + await self.confirm_lock() + answer = await self.mutate( + "/attach/place", {"deployment_id": self.row["deployment_id"], "orgs": batch} + ) + placed += int(answer.get("placed") or 0) + self.report.update(orgs_found=len(orgs), placed_now=placed, skipped=self._named(found)) + await self.nonheartbeat_checks() + finally: + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + await self.final_checks(watch) + row = await self.router.call("") + current, nxt = self.own_fp + if json.dumps((row.get("current_fp"), row.get("next_fp")), sort_keys=True) != json.dumps( + (current, nxt), sort_keys=True + ): + raise _refused( + "postgres loom.cell_self", + "differs from the directory's fingerprints (a fingerprinted setting changed " + "during attach?)", + ) + state, _ = await apg.read_mode(self.lock_conn) + if state != "enforced": + raise _refused("postgres cell state", f"reads {state}, not enforced") + + async def serving(self) -> dict[str, Any]: + from server.cli.cell_prepare import _delete_nonces, _new_nonces, _write_nonces + + nonces = _new_nonces() + try: + await _write_nonces(self.ch, nonces) + answer = await self.mutate( + "/attach/serving", {"deployment_id": self.row["deployment_id"], "nonces": nonces} + ) + except BaseException: # Nonce cleanup preserves the first failure or cancellation. + with contextlib.suppress(BaseException): + await _delete_nonces(self.ch, nonces) + raise + else: + await _delete_nonces(self.ch, nonces) + return answer + + # -- the run ----------------------------------------------------------------------- + + async def done(self) -> bool: + """Require matching Postgres and cell settings, then `enforced` mode.""" + from server.core import pg + + async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: + await cell_control.paired(conn, self.row) + state, _ = await apg.read_mode(conn) + if state != "enforced": + raise _refused( + "postgres cell state", + f"the directory says serving and the cell reads {state}: the directory and the " + "cell's Postgres disagree, as after a restore of one of them; attach lowers " + "and repairs neither", + ) + return True + + async def run(self) -> int: + self.row = await self.router.call("") + cell_control.preconditions(self.row) + if self.row.get("state") == "serving": + await self.done() + self.report.update(state="serving", done=True) + return 0 + return await cell_control.run("attach", self._owned) + + async def _owned(self, owner: cell_control.Owner) -> int: + self.owner, self.lock_conn = owner, owner.conn + await self.read_own_row() + topology, names, listed = await self.refusals() + if self.args.dry_run: + return await self.dry_run(topology, listed) + self.row = await cell_control.claim(self.lock_conn, self.router.call, self.row) + await self.begin() + state, _ = await apg.read_mode(self.lock_conn) + self.mode = state + await self.triggers() + if state == "recording": + found = await self.scan(topology, listed) + if found.too_long: + raise Refused(_too_long(found)) + written = await apg.write_tenancy(found.orgs) + self.log(f"step 2: {len(found.orgs)} orgs found, {written} tenancy rows written") + await self.triggers() + await self.precommit(found) + await self.commit_mode() + else: + await self.mutate("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) + xmax = await self.durable() + await self.check_beats("ignore") + await self.step4(xmax, topology, names) + answer = await self.serving() + self.report.update(state=answer.get("state"), placed_orgs=answer.get("placed_orgs")) + return 0 + + async def precommit(self, found: cell_scan.ScanResult) -> None: + from server.core import pg_profile + + out = await self.catalog_refusals() + if (now := await pg_profile.standby_now()) != "synchronous": + out.append(Refusal("postgres standbys", f"commits are acknowledged as {now}")) + if found.elapsed_s > self.args.scan_budget / 2: + out.append( + Refusal( + "--scan-budget", + f"step 2 took {found.elapsed_s:.0f} s, more than half of " + f"{self.args.scan_budget} s, and step 4 reads the same tables again: " + "raise it", + ) + ) + if out: + raise Refused(out) + await self.check_beats("refuse") + + async def dry_run(self, topology: Any, listed: cell_scan.TableList) -> int: + found = await self.scan(topology, listed) + per_source: dict[str, list[str]] = {} + for org, sources in found.orgs.items(): + for s in sources: + per_source.setdefault(s, []).append(org) + self.report.update( + dry_run=True, + scan_seconds=round(found.elapsed_s, 1), + half_budget_ok=found.elapsed_s <= self.args.scan_budget / 2, + orgs=len(found.orgs), + per_source={ + s: {"count": len(o), "sample": sorted(o)[:5]} for s, o in sorted(per_source.items()) + }, + skipped=self._named(found), + ) + return 0 + + +async def attach( + ch: Any, + args: argparse.Namespace, + url: httpx.URL, + transport: httpx.AsyncBaseTransport | None = None, +) -> int: + def log(text: str) -> None: + print(f"loom cell attach: {text}", file=sys.stderr) + + run = Run(ch, args, Router(url, args.cell, transport), log) + started = time.monotonic() + try: + code = await run.run() + except Refused as e: + for r in e.refusals: + log(f"refusing: {r}") + log(mode_line(run.mode)) + return 1 + except Exception as e: # noqa: BLE001 — a store that fails names itself; a rerun resumes + log(f"stopped ({type(e).__name__}: {e}); run it again") + log(mode_line(run.mode)) + return 1 + except BaseException: + # An interrupt: the mode line, then the interrupt as it came. + log(mode_line(run.mode)) + raise + run.report["seconds"] = round(time.monotonic() - started, 1) + print(json.dumps(run.report, indent=2, sort_keys=True, default=str)) + return code + + +def mode_line(mode: str) -> str: + """What a stopped run left the cell's mode at, by what it last knew.""" + if mode == "enforced": + return "the cell stays enforced and attaching: rerun attach to resume" + if mode == "unknown": + return ( + "the mode commit's outcome is unknown: rerun attach, which resumes by the mode it reads" + ) + if mode == "recording": + return "the cell stays recording" + return "the cell's state is unchanged" + + +def cmd_attach(args: argparse.Namespace) -> int: + if problem := _args_problem(args): + print(f"loom cell attach: {problem}", file=sys.stderr) + return 2 + url = cell_cli.operator_router(args) + if isinstance(url, str): + print(f"loom cell attach: {url}", file=sys.stderr) + return 2 + from server.cli._ch import run_connected + from server.core.ch import statement_timeout + + async def _run(ch: Any) -> int: + return await attach(ch, args, url) + + # Every ClickHouse statement, the connect included, may run as long as a scan may. + with statement_timeout(args.scan_budget): + return run_connected(_run, org_free=writes("attaches the cell; places the orgs it holds")) + + +def add_attach_parser(csub: Any) -> None: + p = csub.add_parser("attach", help="Attach the prepared first cell: place its orgs, serve.") + p.add_argument("--cell", required=True, help="This cell's name (LOOM_CELL)") + p.add_argument( + "--wait", + type=int, + default=_WAIT_S, + help="Seconds of wall time the waits share; 0 runs one try of each, with no retry", + ) + p.add_argument( + "--scan-budget", type=int, default=_SCAN_BUDGET_S, help="Seconds one full org scan may take" + ) + p.add_argument( + "--dry-run", + action="store_true", + help="Run the refusals and the scan and print what would be placed; write nothing " + "(the router's own begin refusals do not run)", + ) + for flag, what in ( + ("--ch-allow-user", "A ClickHouse user that may write the configured databases"), + ("--ch-allow-user-directory", "A ClickHouse user directory the check cannot read"), + ("--ch-allow-object", "A ClickHouse object (db.name) that writes the databases"), + ("--ch-allow-replica", "A replica outside the cluster (db.table:replica)"), + ("--pg-allow-role", "A Postgres role that may make an org held (the servers' role)"), + ): + p.add_argument(flag, action="append", default=[], help=what) + p.add_argument("--router", help="The router's URL (default: LOOM_CELL_ROUTER_URL)") + p.set_defaults(func=cmd_attach) diff --git a/server/cli/cell_attach_beats.py b/server/cli/cell_attach_beats.py new file mode 100644 index 0000000000..f49819a1e0 --- /dev/null +++ b/server/cli/cell_attach_beats.py @@ -0,0 +1,347 @@ +"""The first attach's judgment of the cell's processes, from their heartbeats +(docs/architecture/cells.md, "Attaching a cell" and "The heartbeat's keys"). + +Every check reads both heartbeat tables across every deployment sharing the ClickHouse, as +prepare does, stopped rows included (`instances.read_recent`), and judges in one order: + +- `cell` first: a live process whose `LOOM_CELL` is unset or another cell refuses at once; + a stopped one, or a row with no cell row that is no longer live, may still be writing + for the widened window after its stop beat, and is refused, or waited on until it ages + out, as the caller's `stopped_unset` says; +- then the keys a current build reports (`serves`, `marked`, `router`): a cell row + without them refuses, even while booting; +- then booting: a live instance with no cell row yet, no tenancy mode yet, `marked` null, + or `router` null where `serves` is true, is waited on; `marked` false, or `router` false + where `serves` is true, refuses, naming the process and the router's cause; +- and the fingerprint: each live process's matches `loom.cell_self`'s current or next. +""" + +from __future__ import annotations + +import asyncio +import json +import time +from collections.abc import Awaitable, Callable, Iterable, Mapping +from dataclasses import dataclass, field +from typing import Any, Literal + +from server.cli.cell_attach_pg import Wait +from server.cli.cell_ch import Refusal, Refused +from server.core import cell, instances + +StoppedUnset = Literal["refuse", "wait"] + + +def widened_window_s() -> float: + """The live window plus what a stopping process still does after its stop beat: it + drains its ClickHouse writes, then settles cut-off ones. A process whose last beat, + stopped or not, is younger than this may still write.""" + from server.boot import SHUTDOWN_DRAIN_BUDGET_S + from server.insights.app.provider_barrier import SHUTDOWN_RETIRE_S + + return instances.live_window_s() + SHUTDOWN_DRAIN_BUDGET_S + SHUTDOWN_RETIRE_S + + +@dataclass(frozen=True) +class Beat: + """One process: its instance row (`live`: inside the live window and not stopped) and + its cell row (`cell` None when it wrote none), the fingerprint JSON decoded.""" + + instance_id: str + live: bool + stopped: bool + keys_fp: str + last_beat: str + cell: str | None = None + mode: str = "" + value: dict[str, Any] | None = None + + +async def read_beats(ch: Any, *, since: str | None = None) -> list[Beat]: + rows = await instances.read_recent(ch, window_s=widened_window_s(), since=since) + ids = [str(r["instance_id"]) for r in rows] + cells = await ch.query_all( + f"SELECT instance_id, cell, tenancy_mode, fingerprint FROM {instances.CELL_TABLE} " + "WHERE has({ids:Array(String)}, instance_id)", + {"ids": ids}, + ) + by_id = {str(r["instance_id"]): r for r in cells} + out = [] + for r in rows: + found = by_id.get(str(r["instance_id"])) + beat = Beat( + instance_id=str(r["instance_id"]), + live=bool(r["live"]), + stopped=bool(r["stopped"]), + keys_fp=str(r["keys_fp"]), + last_beat=str(r["last_beat"]), + cell=None if found is None else str(found["cell"]), + mode="" if found is None else str(found["tenancy_mode"]), + value=None if found is None else _decoded(str(found["fingerprint"])), + ) + out.append(beat) + return out + + +def _decoded(text: str) -> dict[str, Any] | None: + try: + value = json.loads(text) + except ValueError: + return None + return value if isinstance(value, dict) else None + + +@dataclass +class Verdict: + """What one read found: the refusals, the processes waited on (booting, or a stopped + process with `LOOM_CELL` unset aging out), the live processes judged, their `keys_fp` + values, and whether one reports SCIM on.""" + + refusals: list[Refusal] = field(default_factory=list) + booting: list[str] = field(default_factory=list) + aging: list[str] = field(default_factory=list) + live: int = 0 + keys_fps: set[str] = field(default_factory=set) + scim: bool = False + + +def judge( + beats: Iterable[Beat], + name: str, + accepted: Iterable[Mapping[str, Any] | None], + stopped_unset: StoppedUnset, +) -> Verdict: + """The verdict on `beats` for the cell `name`, whose processes run `accepted` (its + `loom.cell_self` current and next).""" + values = [v for v in accepted if v is not None] + out = Verdict() + for b in beats: + who = f"process {b.instance_id}" + if b.cell is None: + if b.live: + out.booting.append(b.instance_id) + else: + _stopped(out, b, "stopped before reporting its LOOM_CELL", stopped_unset) + continue + if b.cell != name: + runs = f"runs with LOOM_CELL={b.cell}" if b.cell else "runs with LOOM_CELL unset" + if b.live: + out.refusals.append(Refusal(who, f"{runs}: set LOOM_CELL={name} on it")) + else: + _stopped(out, b, f"{runs} and stopped", stopped_unset) + continue + if not b.live: + continue + value = b.value or {} + if not all(k in value for k in ("serves", "marked", "router")): + out.refusals.append( + Refusal(who, "reports no serves, marked or router key: upgrade it to this release") + ) + continue + if value["marked"] is False: + out.refusals.append( + Refusal( + who, + "its Postgres sessions lose the tenancy marker (a pooler?): point " + "LOOM_PG_DSN straight at Postgres", + ) + ) + continue + serves = value["serves"] is True + if serves and value["router"] is False: + why = value.get("router_why") or "no cause reported" + out.refusals.append(Refusal(who, f"cannot read its router: {why}")) + continue + if value["marked"] is None or (serves and value["router"] is None) or not b.mode: + out.booting.append(b.instance_id) + continue + fields = value.get("fields") + if not isinstance(fields, dict) or not any(cell.matches(fields, v) for v in values): + out.refusals.append( + Refusal(who, "runs neither loom.cell_self's current nor its next configuration") + ) + continue + out.live += 1 + out.keys_fps.add(b.keys_fp) + out.scim = out.scim or bool(fields.get("scim")) + return out + + +def _stopped(out: Verdict, b: Beat, what: str, stopped_unset: StoppedUnset) -> None: + if stopped_unset == "wait": + out.aging.append(b.instance_id) + return + out.refusals.append( + Refusal( + f"process {b.instance_id}", + f"{what} at {b.last_beat}, and may still be writing (its drain and retirement " + "after its stop beat)", + ) + ) + + +def final_refusals(verdict: Verdict) -> list[Refusal]: + out = [] + if not verdict.live and not verdict.booting: + out.append(Refusal("processes", "no live process of this cell reports a heartbeat")) + if len(verdict.keys_fps) > 1: + out.append( + Refusal( + "processes", + "the live processes report different root-secret fingerprints (keys_fp): " + "finish the root-secret rollout first", + ) + ) + if verdict.scim: + out.append( + Refusal("processes", "SCIM is on: attach with SCIM on needs the per-cell SCIM release") + ) + return out + + +async def settle( + read: Callable[[], Awaitable[list[Beat]]], + judge_beats: Callable[[list[Beat]], Verdict], + wait: Wait, + *, + extend_s: float = 0.0, +) -> Verdict: + """Read and judge until nothing is waited on: Refused at the first refusal, and once + `wait` runs out naming what was waited on. With `extend_s`, the deadline is extended + once, to `extend_s` past the first sight of the last process first seen booting.""" + with wait.counting(): + return await _settle(read, judge_beats, wait, extend_s) + + +async def _settle( + read: Callable[[], Awaitable[list[Beat]]], + judge_beats: Callable[[list[Beat]], Verdict], + wait: Wait, + extend_s: float, +) -> Verdict: + first_seen: dict[str, float] = {} + extended = False + while True: + verdict = judge_beats(await read()) + if verdict.refusals: + raise Refused(verdict.refusals) + if not verdict.booting and not verdict.aging: + if refusals := final_refusals(verdict): + raise Refused(refusals) + return verdict + for i in verdict.booting: + first_seen.setdefault(i, wait.used()) + if await wait.pause(): + continue + if extend_s and not extended and verdict.booting: + extended = True + wait.seconds = max(wait.seconds, max(first_seen[i] for i in verdict.booting) + extend_s) + if await wait.pause(): + continue + raise Refused( + [Refusal(f"process {i}", "still booting at the end of --wait") for i in verdict.booting] + + [ + Refusal(f"process {i}", "stopped with LOOM_CELL unset, not aged out within --wait") + for i in verdict.aging + ] + ) + + +# -- the rescan's watch ---------------------------------------------------------------- + +_READ_TIMEOUT_S = 30.0 # not a setting: one heartbeat read's bound +_SKEW_S = 10.0 # not a setting: a margin for clock skew between ClickHouse hosts + + +def watch_period_s() -> float: + """`T'`: how long a row whose window reaches past the rescan's start is kept before the + leader's sweep may drop it, less a margin for clock skew.""" + return instances.PRUNE_AFTER_S - widened_window_s() - _SKEW_S + + +def prune_window_refusals() -> list[Refusal]: + """A heartbeat interval so long that the leader may sweep a stopped process's rows + before step 4's watch reads them refuses before any write.""" + if watch_period_s() < 2 * _READ_TIMEOUT_S: + return [ + Refusal( + "LOOM_INSTANCE_HEARTBEAT_SECONDS", + "so long that a stopped process's rows can be swept before attach reads " + "them: lower it", + ) + ] + return [] + + +class Watch: + """The processes seen during step 4's rescan, read every `T' / 2` from the rescan's + start (`since`), so a process with `LOOM_CELL` unset that ran during it is refused at + the end even once the leader swept its rows.""" + + def __init__( + self, ch: Any, since: str, name: str, clock: Callable[[], float] = time.monotonic + ) -> None: + self.ch, self.since, self.name = ch, since, name + self._clock = clock + self.cell_ids: set[str] = set() + self.unset: dict[str, str] = {} + self.no_cell: dict[str, str] = {} + self.gap = False + self._good = clock() + + async def read_once(self) -> list[Beat]: + try: + async with asyncio.timeout(_READ_TIMEOUT_S): + beats = await read_beats(self.ch, since=self.since) + except Exception: # A failed read still checks whether the heartbeat watch has a gap. + self._check_gap() + raise + self._check_gap() + self._good = self._clock() + for b in beats: + if b.cell == self.name: + self.cell_ids.add(b.instance_id) + elif b.cell is not None: + self.unset[b.instance_id] = b.last_beat + elif b.stopped: + self.no_cell[b.instance_id] = b.last_beat + return beats + + def _check_gap(self) -> None: + if self._clock() - self._good > watch_period_s(): + self.gap = True + + async def run(self) -> None: + """Read until cancelled; a failed read is retried at the next period.""" + while True: + await asyncio.sleep(watch_period_s() / 2) + try: + await self.read_once() + except Exception: # noqa: BLE001 — a failed read is retried; a gap refuses + continue + + def refusals(self) -> list[Refusal]: + out = [ + Refusal( + f"process {i}", + f"ran with LOOM_CELL unset or another cell during the rescan (last beat {at})", + ) + for i, at in sorted(self.unset.items()) + ] + out += [ + Refusal( + f"process {i}", + f"stopped during the rescan before reporting its LOOM_CELL (last beat {at})", + ) + for i, at in sorted(self.no_cell.items()) + if i not in self.cell_ids + ] + if self.gap: + out.append( + Refusal( + "heartbeats", + "not read for longer than the leader keeps a stopped process's rows: " + "rerun attach", + ) + ) + return out diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py new file mode 100644 index 0000000000..b2b751d8af --- /dev/null +++ b/server/cli/cell_attach_pg.py @@ -0,0 +1,874 @@ +"""The first attach's reads and writes on the cell's Postgres (docs/architecture/cells.md, +"Attaching a cell"): the session checks, the one-run lock, the catalog checks attach and +`loom doctor` share (the roles that can make an org held or lower the mode, the settings +and subscriptions that pass an unchecked writer), the trigger pass, the mode commit, its +durable read and the wait for the transactions that could have read `recording`. + +Every check returns its refusals as `cell_ch.Refusal` (subject, reason), as the org scan +and the ClickHouse writer check do, so attach and doctor print the same names.""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import time +from collections.abc import Awaitable, Callable, Coroutine, Iterable, Iterator +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from server.cli.cell_ch import Refusal, Refused +from server.core import cell, cell_triggers, pg +from server.core.cell import ATTACH_LOCK_KEY + +if TYPE_CHECKING: + from server.cli.cell_control import Owner + +_sleep = asyncio.sleep # the seam the wait tests replace + +# The pause between polls of a wait. +POLL_S = 1.0 # not a setting: a poll interval +# The least lock wait a try gets, however little of --wait is left. +TRY_FLOOR_MS = 100 # not a setting: one try's floor +# The migration lock's wait a trigger try (a boot holds it for its migrations), in ms; a +# try never waits longer than --wait has left. +_MIGRATION_LOCK_MS = 10_000 # not a setting: one try's bound +# A trigger try yields quickly to active writers. +_TRIGGER_LOCK_MS = 100 # not a setting: one try's bound +_TRIGGER_TRY_MARGIN_S = 8.0 # not a setting: catalog, DDL and standby allowance +# The mode commit's lock wait a try, in ms, and its statement bound's margin above it. +_MODE_LOCK_MS = 2_000 # not a setting: one try's bound +_MODE_STATEMENT_MARGIN_MS = 3_000 # not a setting: a margin over the lock wait +# COMMIT's standby wait is outside Postgres's statement timeout. +_MODE_STANDBY_S = 5.0 # not a setting: one try's standby allowance +# The server ends the lock session soon after its client is gone, releasing the lock. +_KEEPALIVE = {"tcp_keepalives_idle": "30", "tcp_keepalives_interval": "10"} + + +class Wait: + """`--wait`: the wall-clock seconds attach's bounded waits share. Each wait runs inside + `counting()`, which charges its elapsed time, lock tries included; time outside every + wait (a scan) is never charged.""" + + def __init__(self, seconds: float, clock: Callable[[], float] = time.monotonic) -> None: + self.seconds = float(seconds) + self.spent = 0.0 + self._clock = clock + self._since: float | None = None + + def used(self) -> float: + """The seconds charged so far, the wait running now included.""" + running = 0.0 if self._since is None else self._clock() - self._since + return self.spent + running + + def left(self) -> float: + return max(0.0, self.seconds - self.used()) + + @contextlib.contextmanager + def counting(self) -> Iterator[None]: + """Charge the block's wall time (a block inside another is charged once).""" + if self._since is not None: + yield + return + self._since = self._clock() + try: + yield + finally: + self.spent += self._clock() - self._since + self._since = None + + def bound_ms(self, most_ms: int) -> int: + """A lock timeout for one try: `most_ms`, or what is left when that is less, and + never under `TRY_FLOOR_MS`: every try runs once, so `--wait 0` is one try of each + wait with no retry, and only after a try is the wait judged spent.""" + return max(TRY_FLOOR_MS, min(most_ms, int(self.left() * 1000))) + + async def pause(self, seconds: float = POLL_S) -> bool: + """Wait `seconds` (at most what is left); False once none is left.""" + step = min(seconds, self.left()) + if step <= 0: + return False + with self.counting(): + await _sleep(step) + return True + + +@dataclass +class _Interrupted: + error: BaseException + + +async def _work_result[T](work: Callable[[], Awaitable[T]]) -> T | _Interrupted: + try: + return await work() + except (KeyboardInterrupt, SystemExit) as error: + # These exceptions otherwise stop asyncio's loop before the owner can close. + return _Interrupted(error) + + +async def supervise[T](conn: Any, work: Coroutine[Any, Any, T], *, end: float) -> T: + """Bound owned work, reserving cleanup inside its original phase deadline. + + A failed borrowed transaction must remain alive until public close sends its + cancellation. Closing the transport first can leave its backend locks held. + """ + task = asyncio.create_task(_work_result(lambda: work)) + reserve = min(1.0, max(0.0, end - time.monotonic()) / 4) + try: + done, _ = await asyncio.wait({task}, timeout=max(0.0, end - reserve - time.monotonic())) + if not done: + raise TimeoutError("attach Postgres phase deadline expired") + result = task.result() + if isinstance(result, _Interrupted): + raise result.error + return result + except BaseException: + try: + await pg.close_owned(conn, end=end) + except BaseException: # noqa: BLE001 — preserve the work failure or cancellation. + pass + if not task.done(): + task.cancel() + task.add_done_callback(pg._retrieve_task) + try: + await asyncio.wait({task}, timeout=max(0.0, end - time.monotonic())) + except BaseException: # noqa: BLE001 — repeated cancellation preserves the first failure. + try: + pg._terminate(conn) + except BaseException: # noqa: BLE001 — abort cannot replace the work failure. + pass + raise + + +def _split(key: int) -> str: + """The `pg_locks` columns that hold a bigint advisory key, in this database.""" + return ( + f"l.locktype = 'advisory' AND l.classid = ({key} >> 32)::oid " + f"AND l.objid = ({key} & 4294967295)::oid AND l.objsubid = 1 " + "AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database())" + ) + + +# Who holds an advisory lock (`where` from `_split`), never a query's text. +_ADVISORY_HOLDER = """ +SELECT l.pid, a.application_name, a.backend_start +FROM pg_locks l LEFT JOIN pg_stat_activity a ON a.pid = l.pid +WHERE {where} AND l.granted +""" + +_LOCK_HOLDER = f""" +SELECT l.pid, a.application_name, a.backend_start, a.client_addr::text AS client +FROM pg_locks l LEFT JOIN pg_stat_activity a ON a.pid = l.pid +WHERE {_split(ATTACH_LOCK_KEY)} AND l.granted +""" + +_IDENTITY = ( + "SELECT pg_backend_pid() AS pid, backend_start FROM pg_stat_activity " + "WHERE pid = pg_backend_pid()" +) + + +async def direct_refusals(conn: Any, what: str) -> list[Refusal]: + """Whether `conn` is one direct, marked session: the same backend across two + transactions, and the tenancy marker kept through `RESET ALL` (a pooler that strips + startup parameters drops it). Behind a transaction pooler a session-level lock would + outlive the CLI on the pooler's server connection, and attach's tenancy rows would be + refused as an unchecked writer's.""" + identities = [] + for _ in range(2): + tx = conn.transaction() + await tx.start() + identities.append(await conn.fetchrow(_IDENTITY)) + await tx.commit() + first, second = identities + out = [] + if first is None or second is None or tuple(first) != tuple(second): + out.append( + Refusal( + "postgres session", + f"{what}'s session moved between backends: point LOOM_PG_DSN straight at " + "Postgres, never through a pooler", + ) + ) + await conn.execute("RESET ALL") # a startup value survives it, a SET does not + if await conn.fetchval("SELECT current_setting($1, true)", pg.TENANCY_MARKER) != "on": + out.append( + Refusal( + "postgres session", + f"{what}'s session loses {pg.TENANCY_MARKER}: run it with LOOM_CELL set, " + "LOOM_PG_DSN straight at Postgres", + ) + ) + return out + + +_HELD = f""" +SELECT EXISTS (SELECT 1 FROM pg_locks l + WHERE {_split(ATTACH_LOCK_KEY)} AND l.granted AND l.pid = pg_backend_pid()) +""" + + +async def lock_holder(conn: Any) -> str: + """Who holds the attach lock ('' when no one), never a query's text.""" + rows = await conn.fetch(_LOCK_HOLDER) + return "; ".join( + f"pid {r['pid']} ({r['application_name'] or 'no name'}, from {r['client'] or 'local'}, " + f"since {r['backend_start']})" + for r in rows + ) + + +async def take_lock(conn: Any, what: str) -> None: + """Take the attach lock on `conn` for the session's life, confirmed held by this + backend; Refused naming its holder when another run holds it.""" + for name, value in _KEEPALIVE.items(): + await conn.execute(f"SET {name} = {value}") + if not await conn.fetchval("SELECT pg_try_advisory_lock($1)", ATTACH_LOCK_KEY): + holder = await lock_holder(conn) or "a session that just ended" + raise Refused( + [ + Refusal( + "attach lock", + f"held by {holder}: an attach or a prepare is running; {what} " + "runs one at a time", + ) + ] + ) + if not await lock_held(conn): + raise Refused( + [ + Refusal( + "attach lock", + "taken but not held by this session's backend: a pooler between " + "this command and Postgres; point LOOM_PG_DSN straight at it", + ) + ] + ) + + +async def lock_held(conn: Any) -> bool: + """Whether `conn`'s backend holds the attach lock.""" + return bool(await conn.fetchval(_HELD)) + + +async def encoding_refusals(conn: Any) -> list[Refusal]: + """A cell's database is UTF8: its `text` then holds no NUL or invalid UTF-8, so every + tenancy row within the length limit is an org a request can name.""" + encoding = await conn.fetchval( + "SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname = current_database()" + ) + if encoding == "UTF8": + return [] + return [ + Refusal( + "postgres database", + f"its encoding is {encoding}, not UTF8: a cell's database is UTF8 " + "(recreate it with ENCODING 'UTF8' and restore into it)", + ) + ] + + +# -- who can make an org held, or lower the mode -------------------------------------- + +_SYSTEM_SCHEMA = ( + "n.nspname NOT IN ('pg_catalog', 'information_schema') " + "AND n.nspname NOT LIKE 'pg\\_toast%' AND n.nspname NOT LIKE 'pg\\_temp\\_%'" +) + +# Every right a login role may hold that lets it make an org held, lower the mode, or run +# its own SQL as the servers on their marked sessions: each row a role and why. +# Membership can supply a right through SET ROLE or an ADMIN regrant without INHERIT. +_RIGHTS = f""" +WITH owned AS ( + SELECT c.relowner AS owner, 'owns loom.' || c.relname AS why + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'loom' AND c.relkind IN ('r', 'p') + UNION ALL + SELECT p.proowner, 'owns loom.' || p.proname || '()' FROM pg_proc p + WHERE p.oid IN (to_regprocedure('loom.tenancy_refuse_unheld()'), + to_regprocedure('loom.tenancy_refuse_unmarked()'), + to_regprocedure('loom.cell_first_immutable()')) + UNION ALL + SELECT d.datdba, 'owns the database' FROM pg_database d + WHERE d.datname = current_database() + UNION ALL + SELECT n.nspowner, 'owns schema ' || n.nspname FROM pg_namespace n WHERE {_SYSTEM_SCHEMA} +), +login AS (SELECT oid, rolname FROM pg_roles WHERE rolcanlogin) +SELECT DISTINCT l.rolname AS role, w.why FROM login l +JOIN pg_roles r ON pg_has_role(l.oid, r.oid, 'MEMBER'), LATERAL ( + SELECT 'may insert into loom.tenancy' + WHERE has_any_column_privilege(r.oid, 'loom.tenancy', 'INSERT') + UNION ALL SELECT 'may update loom.tenancy (org)' + WHERE has_column_privilege(r.oid, 'loom.tenancy', 'org', 'UPDATE') + UNION ALL SELECT 'may update loom.tenancy_mode (mode)' + WHERE has_column_privilege(r.oid, 'loom.tenancy_mode', 'mode', 'UPDATE') + UNION ALL SELECT 'may insert into loom.tenancy_mode' + WHERE has_any_column_privilege(r.oid, 'loom.tenancy_mode', 'INSERT') + UNION ALL SELECT 'holds TRIGGER on loom.' || c.relname + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'loom' AND c.relkind IN ('r', 'p') + AND has_table_privilege(r.oid, c.oid, 'TRIGGER') + UNION ALL SELECT 'may create in the database' + WHERE has_database_privilege(r.oid, current_database(), 'CREATE') + UNION ALL SELECT 'may create in schema ' || n.nspname FROM pg_namespace n + WHERE {_SYSTEM_SCHEMA} AND has_schema_privilege(r.oid, n.oid, 'CREATE') + UNION ALL SELECT 'may set session_replication_role' + WHERE has_parameter_privilege(r.oid, 'session_replication_role', 'SET') + OR has_parameter_privilege(r.oid, 'session_replication_role', 'ALTER SYSTEM') + UNION ALL SELECT 'may ALTER SYSTEM SET {pg.TENANCY_MARKER}' + WHERE has_parameter_privilege(r.oid, '{pg.TENANCY_MARKER}', 'ALTER SYSTEM') + UNION ALL SELECT 'is a member of ' || g.rolname FROM pg_roles g + WHERE g.rolname IN ('pg_write_server_files', 'pg_execute_server_program') + AND pg_has_role(r.oid, g.oid, 'MEMBER') + UNION ALL SELECT o.why || CASE WHEN o.owner = l.oid THEN '' + ELSE ' (as a member of ' || pg_get_userbyid(o.owner) || ')' + END + FROM owned o WHERE pg_has_role(l.oid, o.owner, 'MEMBER') +) w(why) +ORDER BY 1, 2 +""" + +# The roles attach counts as its own: superusers, a login role that itself owns +# loom.tenancy, loom.tenancy_mode or a guarded table (it can drop or disable the +# triggers, or lower the mode, whatever else it holds), and, for attach only, the role it +# runs as ($2): the operator runs attach, while doctor may run as a CI or monitoring role. +_OWN_ROLES = """ +SELECT rolname FROM pg_roles WHERE rolcanlogin AND (rolsuper OR oid IN ( + SELECT c.relowner FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'loom' AND c.relkind IN ('r', 'p') + AND (c.relname IN ('tenancy', 'tenancy_mode') OR c.relname = ANY($1::text[])) +)) OR ($2 AND rolname = current_user) +ORDER BY 1 +""" + + +@dataclass(frozen=True) +class Holders: + """The login roles holding a right `_RIGHTS` counts: the ones attach counts as its own + (`own`, named in its output) and every other with its reasons.""" + + own: list[str] + others: dict[str, list[str]] + + +async def privilege_holders(conn: Any, *, exempt_self: bool) -> Holders: + """The holders of a right `_RIGHTS` counts; `exempt_self` counts the role this session + runs as among the own roles (attach's, not doctor's).""" + guarded = [t for t in await cell_triggers.planned_tables(conn) if t != "tenancy"] + own = [str(r["rolname"]) for r in await conn.fetch(_OWN_ROLES, guarded, exempt_self)] + others: dict[str, list[str]] = {} + for r in await conn.fetch(_RIGHTS): + if r["role"] not in own: + others.setdefault(str(r["role"]), []).append(str(r["why"])) + return Holders(own, others) + + +def privilege_refusals(holders: Holders, allowed: Iterable[str]) -> list[Refusal]: + allow = set(allowed) + return [ + Refusal( + f"postgres role {role}", + f"{'; '.join(why)}: it can make an org held or lower the mode outside attach; " + "revoke it, or name the role with --pg-allow-role", + ) + for role, why in sorted(holders.others.items()) + if role not in allow + ] + + +# -- what passes an unchecked writer through the triggers ----------------------------- + +_ROLE_SETTINGS = f""" +SELECT coalesce(r.rolname, 'every role') AS role, coalesce(d.datname, 'every database') AS db, + cfg +FROM pg_db_role_setting s +LEFT JOIN pg_roles r ON r.oid = s.setrole +LEFT JOIN pg_database d ON d.oid = s.setdatabase, +LATERAL unnest(s.setconfig) cfg +WHERE s.setdatabase IN (0, (SELECT oid FROM pg_database WHERE datname = current_database())) + AND lower(split_part(cfg, '=', 1)) IN ('{pg.TENANCY_MARKER}', 'session_replication_role') +ORDER BY 1, 2, 3 +""" + +_SUBSCRIPTIONS = """ +SELECT subname, subenabled FROM pg_subscription +WHERE subdbid = (SELECT oid FROM pg_database WHERE datname = current_database()) +ORDER BY 1 +""" + + +async def unmarked_refusals(conn: Any, session_passes: bool) -> list[Refusal]: + """What lets a session that sends no marker pass the triggers: the session read + (`session_passes`, doctor's unmarked session), an `ALTER ROLE … SET` or + `ALTER DATABASE … SET` of the marker or `session_replication_role` for this database or + every one, and any logical-replication subscription into this database (its apply + worker runs as `replica`, firing no origin trigger).""" + out = [] + if session_passes: + out.append( + Refusal( + "postgres unmarked session", + "a session that sends no marker is marked, or runs as replica (a role, " + "database, ALTER SYSTEM or options setting): every unchecked writer passes", + ) + ) + for r in await conn.fetch(_ROLE_SETTINGS): + out.append( + Refusal( + f"postgres setting for {r['role']} in {r['db']}", + f"{r['cfg']}: an unchecked writer under it passes the triggers; reset it", + ) + ) + for r in await conn.fetch(_SUBSCRIPTIONS): + state = "enabled" if r["subenabled"] else "disabled" + out.append( + Refusal( + f"postgres subscription {r['subname']}", + f"{state}: its apply worker fires no origin trigger; drop it", + ) + ) + return out + + +async def session_passes() -> bool: + """Whether a session that sends no marker passes the triggers all the same: marked, + or in the replica role that fires no origin trigger.""" + async with pg.dedicated(statement_timeout_ms=5000, marked=False) as conn: + value = await conn.fetchval("SELECT current_setting($1, true)", pg.TENANCY_MARKER) + role = await conn.fetchval("SELECT current_setting('session_replication_role')") + return bool(value == "on" or role == "replica") + + +# -- attach's own role ------------------------------------------------------------------ + +_OWN_GRANTS = """ +SELECT need, held FROM (VALUES + ('effective privileges of pg_read_all_stats', + pg_has_role(current_user, 'pg_read_all_stats', 'USAGE')), + ('USAGE on schema loom', has_schema_privilege('loom', 'USAGE')), + ('SELECT on loom.cell_first', has_table_privilege('loom.cell_first', 'SELECT')), + ('SELECT on loom.tenancy_mode', has_table_privilege('loom.tenancy_mode', 'SELECT')), + ('UPDATE on loom.tenancy_mode', has_table_privilege('loom.tenancy_mode', 'UPDATE')), + ('SELECT on loom.cell_self', has_table_privilege('loom.cell_self', 'SELECT')), + ('SELECT on loom.authority', has_table_privilege('loom.authority', 'SELECT')), + ('INSERT on loom.tenancy', has_table_privilege('loom.tenancy', 'INSERT')), + ('SELECT, INSERT, UPDATE and DELETE on loom.cell_probe', + has_table_privilege('loom.cell_probe', 'SELECT') + AND has_table_privilege('loom.cell_probe', 'INSERT') + AND has_table_privilege('loom.cell_probe', 'UPDATE') + AND has_table_privilege('loom.cell_probe', 'DELETE')), + ('TEMPORARY on the database', has_database_privilege(current_database(), 'TEMPORARY')) +) v(need, held) +WHERE NOT held +""" + + +async def own_role_refusals(conn: Any, tables: list[Any]) -> list[Refusal]: + """What attach's own role lacks, `tables` being the rows `cell_triggers.ORG_TABLES` + read: its grants, `SELECT` on every org table, and effective owner-role privileges for + every table it gives triggers (or the trigger pass could not alter it); and a server that + allows prepared transactions.""" + role = await conn.fetchval("SELECT current_user") + lacks = [str(r["need"]) for r in await conn.fetch(_OWN_GRANTS)] + for r in tables: + if not await conn.fetchval( + "SELECT has_table_privilege($1, 'SELECT')", f"loom.{r['relname']}" + ): + lacks.append(f"SELECT on loom.{r['relname']}") + planned = set(cell_triggers.planned_names(tables)) + for r in tables: + if r["relname"] in planned and not r["owned"]: + lacks.append(f"effective privileges of the owner role of loom.{r['relname']}") + out = [Refusal(f"postgres role {role}", f"attach's role lacks {n}") for n in lacks] + out += await prepared_refusals(conn) + return out + + +async def prepared_refusals(conn: Any) -> list[Refusal]: + setting = await conn.fetchval("SELECT current_setting('max_prepared_transactions')::int") + if setting: + return [ + Refusal( + "postgres max_prepared_transactions", + f"is {setting}: a prepared transaction outlives every session, so step 4 " + "could not wait for it; set it to 0", + ) + ] + return [] + + +# -- the triggers ----------------------------------------------------------------------- + +_MIGRATION_LOCK = "SELECT pg_advisory_lock($1)" +_MIGRATION_UNLOCK = "SELECT pg_advisory_unlock($1)" + + +async def ensure_triggers(owner: Owner, wait: Wait, log: Callable[[str], None]) -> list[Refusal]: + """Give every planned table its triggers (`cell_triggers.ensure_table` with + `recording_first`), one try at a time under the migration lock, retrying a table whose + lock is busy a second apart within `wait`; one autovacuum holding it is asked to yield + by one try just past `deadlock_timeout`. A table this role cannot alter refuses at once. + Then `missing`: any defect refuses.""" + conn = owner.conn + deadlock_ms = int( + await conn.fetchval( + "SELECT setting::int * CASE unit WHEN 's' THEN 1000 ELSE 1 END " + "FROM pg_settings WHERE name = 'deadlock_timeout'" + ) + ) + with wait.counting(): + for table in await cell_triggers.planned_tables(conn): + if refusal := await _ensure_table(owner, table, deadlock_ms, wait, log): + return [refusal] + return [Refusal("postgres triggers", d.text) for d in await cell_triggers.missing(conn)] + + +async def _ensure_table( + owner: Owner, table: str, deadlock_ms: int, wait: Wait, log: Callable[[str], None] +) -> Refusal | None: + conn = owner.conn + yielded = False + said = "" + while True: + most_ms = _TRIGGER_LOCK_MS + holders = await cell_triggers.holders(conn, table) + if not yielded and any(h["backend_type"] == "autovacuum worker" for h in holders): + most_ms, yielded = deadlock_ms + 500, True + outcome = await _ensure_one(owner, table, most_ms, wait) + if outcome.answer == "applied": + return None + if outcome.answer == "denied": + return Refusal( + f"postgres loom.{table}", + "attach's role cannot alter it: grant inherited privileges of its owner role " + "without SET ROLE", + ) + if not await wait.pause(): + if outcome.holders.startswith(_MIGRATION_HELD): + return Refusal( + f"postgres loom.{table}", + f"its triggers were not created within --wait: {outcome.holders}", + ) + return _busy_refusal(table, await cell_triggers.holders(conn, table)) + # Said again only when the holders change: a retry a second is otherwise silent. + if outcome.holders != said: + said = outcome.holders + log(f"loom.{table}: its lock is busy ({said or 'no holder named'}); retrying") + + +_MIGRATION_HELD = "the migration lock" + + +async def _migration_holder(conn: Any, key: int) -> str: + """`the migration lock (held by …)`, naming the session that holds it.""" + try: + rows = await conn.fetch(_ADVISORY_HOLDER.format(where=_split(key))) + except Exception as e: # noqa: BLE001 — naming the holder is best effort + return f"{_MIGRATION_HELD} (its holder not read: {type(e).__name__})" + held = "; ".join( + f"pid {r['pid']}, {r['application_name'] or 'no name'}, since {r['backend_start']}" + for r in rows + ) + return f"{_MIGRATION_HELD} (held by {held or 'a session that just ended'})" + + +async def _ensure_one(owner: Owner, table: str, most_ms: int, wait: Wait) -> cell_triggers.Outcome: + """One try under the migration lock: the migration lock's wait, and then the DDL's lock + wait (at most `most_ms`), each bounded by what `wait` has left when it starts, with the + try floor. One client deadline covers the lock, DDL and cleanup, including COMMIT or + rollback. A busy migration lock answers `busy` naming that lock's holder.""" + from server.core.schema.pg_migrate import _LOCK_KEY + + bound_s = ( + wait.bound_ms(_MIGRATION_LOCK_MS) + wait.bound_ms(most_ms) + ) / 1000 + _TRIGGER_TRY_MARGIN_S + return await owner.call( + lambda: _trigger_try(owner.conn, table, most_ms, wait, _LOCK_KEY), total_s=bound_s + ) + + +async def _trigger_try( + conn: Any, table: str, most_ms: int, wait: Wait, key: int +) -> cell_triggers.Outcome: + await conn.execute(f"SET lock_timeout = '{wait.bound_ms(_MIGRATION_LOCK_MS)}ms'") + try: + await conn.execute(_MIGRATION_LOCK, key) + except Exception as e: + if getattr(e, "sqlstate", None) != "55P03": + raise + await conn.execute("RESET lock_timeout") + return cell_triggers.Outcome("busy", await _migration_holder(conn, key)) + await conn.execute("RESET lock_timeout") + outcome = await cell_triggers.ensure_table( + conn, + table, + recording_first=True, + lock_timeout=f"{wait.bound_ms(most_ms)}ms", + attempts=1, + ) + await conn.execute(_MIGRATION_UNLOCK, key) + return outcome + + +def _busy_refusal(table: str, holders: list[Any]) -> Refusal: + named = "; ".join( + f"pid {h['pid']} ({'holds' if h['granted'] else 'waiting'}, {h['backend_type']}, " + f"{h['application_name'] or 'no name'}, transaction since {h['xact_start']}" + + (", an anti-wraparound vacuum, which never yields" if h["wraparound"] else "") + + ")" + for h in holders + ) + return Refusal( + f"postgres loom.{table}", + f"its triggers' lock was not taken within --wait; held by {named or 'no one now'}", + ) + + +# -- step 3: the mode commit ------------------------------------------------------------ + +# Only acknowledged pre-COMMIT contention permits another try on the same session. +_TIMED_OUT = frozenset({"55P03", "57014"}) + + +async def commit_enforced( + owner: Owner, + confirm_lock: Callable[[], Any], + ready: Callable[[], Awaitable[list[str]]], + wait: Wait, + log: Callable[[str], None], + mark: Callable[[str], None], + *, + expected_cell: str, + expected_fp: tuple[dict[str, Any] | None, dict[str, Any] | None], +) -> None: + """Raise mode on the exact ATTACH-lock backend; an unknown COMMIT ends this run.""" + with wait.counting(): + while True: + await confirm_lock() + booting = await ready() + if not booting: + done = await _mode_try( + owner, wait.bound_ms(_MODE_LOCK_MS), mark, expected_cell, expected_fp + ) + if done: + return + log("the mode's lock is busy; retrying") + if not await wait.pause(): + if booting: + raise Refused( + [ + Refusal(f"process {p}", "still booting at the end of --wait") + for p in booting + ] + ) + raise Refused( + [ + Refusal( + "postgres loom.tenancy_mode", + "its lock was not taken within --wait: a long transaction reads " + "it FOR SHARE; rerun attach", + ) + ] + ) + + +async def _mode_try( + owner: Owner, + lock_ms: int, + mark: Callable[[str], None], + expected_cell: str, + expected_fp: tuple[dict[str, Any] | None, dict[str, Any] | None], +) -> bool: + """Keep work, acknowledged contention rollback and cleanup in one try budget.""" + bound_s = (lock_ms + _MODE_STATEMENT_MARGIN_MS) / 1000 + _MODE_STANDBY_S + return await owner.call( + lambda: _mode_transaction(owner.conn, lock_ms, mark, expected_cell, expected_fp), + total_s=bound_s, + ) + + +async def _mode_transaction( + conn: Any, + lock_ms: int, + mark: Callable[[str], None], + expected_cell: str, + expected_fp: tuple[dict[str, Any] | None, dict[str, Any] | None], +) -> bool: + from server.cli import cell_control + + tx = conn.transaction() + await tx.start() + updated = False + try: + await conn.execute(f"SET LOCAL lock_timeout = '{lock_ms}ms'") + await conn.execute( + f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" + ) + await cell_control.check_lock(conn) + await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") + state, first = await read_mode(conn) + own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") + observed = None if own is None else (own["current_fp"], own["next_fp"]) + if ( + state != "recording" + or first != (True, expected_cell) + or json.dumps(observed, sort_keys=True) != json.dumps(expected_fp, sort_keys=True) + ): + raise Refused([Refusal("postgres acceptance rows", "changed: stop and rerun attach")]) + mark("unknown") + updated = True + status = await conn.execute("UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()") + if status != "UPDATE 1": + raise Refused([Refusal("postgres loom.tenancy_mode", f"the mode update read {status}")]) + except Exception as error: + if getattr(error, "sqlstate", None) not in _TIMED_OUT: + raise + await tx.rollback() + await cell_control.check_lock(conn) + if updated: + mark("recording") + return False + await tx.commit() + mark("enforced") + return True + + +# -- the durable read ------------------------------------------------------------------- + +_DURABLE_TRY_S = 10.0 # not a setting: the state/WAL/COMMIT phase includes its cleanup + + +@dataclass(frozen=True) +class Durable: + """What the durable read saw: the mode, the `first` row, and the xid its logical + message took.""" + + mode: str + first: tuple[bool, str] | None + xid: int + + +async def durable_read(owner: Owner) -> Durable: + """Read and synchronously fence state on the owner's exact ATTACH backend.""" + from server.cli import cell_control + + conn = owner.conn + + async def read() -> Durable: + await conn.execute("SELECT pg_logical_emit_message(true, 'loom.attach', '')") + xid = await conn.fetchval("SELECT pg_current_xact_id_if_assigned()::text") + mode = await conn.fetchval("SELECT mode FROM loom.tenancy_mode") + first = await conn.fetchrow("SELECT is_first, cell FROM loom.cell_first") + if xid is None: + raise Refused([Refusal("postgres durable read", "its logical message took no xid")]) + return Durable( + mode=str(mode or ""), + first=None if first is None else (bool(first["is_first"]), str(first["cell"])), + xid=int(xid), + ) + + return await owner.call(lambda: cell_control.transaction(conn, read), total_s=_DURABLE_TRY_S) + + +async def snapshot_xmax(conn: Any) -> int: + return int(await conn.fetchval("SELECT pg_snapshot_xmax(pg_current_snapshot())::text")) + + +# -- step 4: the transactions that could have read `recording` -------------------------- + +# Every backend but this statement's own: of the cell's database, one whose xid or (but +# an autovacuum's) snapshot is older than X; of another database, one whose xid is older +# (it holds every snapshot's horizon). Both compared through age() against one reference, +# which is exactly `< X`, wraparound-safe, and takes no xid. +_OPEN = """ +WITH x AS (SELECT age($1::text::xid8::xid) AS a) +SELECT s.pid, s.backend_type, s.application_name, s.xact_start::text AS xact_start, + s.datid = (SELECT oid FROM pg_database WHERE datname = current_database()) AS ours, + coalesce(s.backend_xid IS NOT NULL AND age(s.backend_xid) > x.a, false) AS xid_old, + coalesce(s.backend_xmin IS NOT NULL AND age(s.backend_xmin) > x.a + AND s.backend_type <> 'autovacuum worker', false) AS xmin_old, + coalesce(s.query = '', false) AS hidden +FROM pg_stat_activity s, x +WHERE s.pid <> pg_backend_pid() +""" + + +def _named(r: Any) -> str: + return ( + f"pid {r['pid']} ({r['backend_type']}, {r['application_name'] or 'no name'}, " + f"transaction since {r['xact_start']})" + ) + + +async def wait_transactions(conn: Any, xmax: int, wait: Wait) -> None: + """Wait until no backend of the cell's database holds an xid or a snapshot older than + `xmax`; Refused at once on a prepared transaction or an unreadable row, and past + `wait` naming each backend waited on and each other database's horizon holder.""" + with wait.counting(): + await _wait_transactions(conn, xmax, wait) + + +async def _wait_transactions(conn: Any, xmax: int, wait: Wait) -> None: + while True: + if refusals := await prepared_refusals(conn): + raise Refused(refusals) + if await conn.fetchval("SELECT count(*) FROM pg_prepared_xacts"): + raise Refused( + [Refusal("postgres prepared transaction", "one is open: commit or roll it back")] + ) + rows = await conn.fetch(_OPEN, str(xmax)) + hidden = [r for r in rows if r["ours"] and r["hidden"]] + if hidden: + raise Refused( + [ + Refusal( + f"postgres backend {r['pid']}", + "its transaction state is hidden from attach's role: grant it " + "pg_read_all_stats", + ) + for r in hidden + ] + ) + waited = [r for r in rows if r["ours"] and (r["xid_old"] or r["xmin_old"])] + if not waited: + return + if not await wait.pause(): + horizon = [r for r in rows if not r["ours"] and r["xid_old"]] + raise Refused( + [ + Refusal(f"postgres {_named(r)}", "may have read `recording`: still open") + for r in waited + ] + + [ + Refusal( + f"postgres {_named(r)}", + "another database's transaction, holding the snapshot horizon", + ) + for r in horizon + ] + ) + + +# -- the tenancy rows ------------------------------------------------------------------- + + +async def write_tenancy(orgs: Iterable[str]) -> int: + """The tenancy rows of `orgs`, on a marked session, as one statement per thousand; the + rows written.""" + batch = sorted(orgs) + written = 0 + for i in range(0, len(batch), 1000): + status = await pg.execute( + "INSERT INTO loom.tenancy (org) SELECT unnest($1::text[]) ON CONFLICT DO NOTHING", + batch[i : i + 1000], + ) + written += int(status.split()[-1]) + return written + + +async def read_mode(conn: Any) -> tuple[str, tuple[bool, str] | None]: + """The cell's state (`cell.read_state`) and its `first` row.""" + state = await cell.read_state(conn) + first = await conn.fetchrow("SELECT is_first, cell FROM loom.cell_first") + return state, None if first is None else (bool(first["is_first"]), str(first["cell"])) diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py new file mode 100644 index 0000000000..93cc7e2e6b --- /dev/null +++ b/server/cli/cell_control.py @@ -0,0 +1,404 @@ +"""Fenced cell control: one directory claim and the exact local lock-owning backend.""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import sys +import time +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Any, cast +from uuid import UUID + +from server.cli import cell_attach_pg as apg +from server.cli.cell_ch import Refusal +from server.cli.cell_ch import Refused as Refused +from server.core import pg +from server.router.directory import CONTROL_FIELDS, CONTROL_MAX_S +from server.settings import settings + +# A control transaction has few metadata statements; COMMIT's standby wait is bounded too. +TRANSACTION_S = 60.0 # not a setting: a single control transaction's deadline +OPEN_S = 75.0 # not a setting: connection initialization and direct-session checks +REQUEST_S = CONTROL_MAX_S + 30.0 # not a setting: directory bound plus probes/connect/close +Call = Callable[[str, dict[str, Any]], Awaitable[dict[str, Any]]] + + +def _refused(reason: str) -> Refused: + return Refused([Refusal("cell control", reason)]) + + +def preconditions(row: dict[str, Any], *, claim: bool = False) -> dict[str, Any]: + """Carry the whole observed registration and exact fingerprint pair.""" + if any(name not in row for name in (*CONTROL_FIELDS, "mutation_generation")): + raise _refused("a router replica is not upgraded for cell control fencing") + try: + generation = str(UUID(row["mutation_generation"])) + except (ValueError, TypeError, AttributeError): + raise _refused( + "a router replica returned no readable cell control generation: upgrade it" + ) from None + values = {name: row[name] for name in CONTROL_FIELDS} + if not claim: + values["mutation_generation"] = generation + return {"expected": values} + + +async def check_lock(conn: Any) -> None: + try: + held = await apg.lock_held(conn) + except Exception as e: # noqa: BLE001 — an unreadable lock ends this ownership + raise _refused( + f"lock session lost ({type(e).__name__}): stop and rerun with a fresh claim" + ) from None + if not held: + raise _refused("attach lock lost: stop and rerun with a fresh claim") + + +async def paired(conn: Any, row: dict[str, Any]) -> None: + """The connected authority and configured bucket name the observed registration.""" + recorded = await conn.fetchval("SELECT deployment_id FROM loom.authority") + prefix = settings.document_s3_prefix if settings.document_s3_bucket else "" + if ( + not recorded + or recorded != row.get("deployment_id") + or settings.cell != row.get("cell") + or (settings.document_s3_bucket, prefix) != (row.get("bucket"), row.get("bucket_prefix")) + ): + raise _refused( + "this cell configuration and connected Postgres do not match the directory registration" + ) + + +async def claim(conn: Any, call: Call, row: dict[str, Any]) -> dict[str, Any]: + """Claim once; a lost acknowledgement ends the attempt, never restoring an old token.""" + await check_lock(conn) + await paired(conn, row) + answer = await call("/claim", preconditions(row, claim=True)) + await check_lock(conn) + preconditions(answer) + expected = {name: row[name] for name in CONTROL_FIELDS} + if json.dumps(expected, sort_keys=True) != json.dumps( + {name: answer[name] for name in CONTROL_FIELDS}, sort_keys=True + ): + raise _refused( + "claim returned a different registration or fingerprint pair: stop and rerun" + ) + return answer + + +async def mutate( + conn: Any, call: Call, row: dict[str, Any], path: str, body: dict[str, Any] +) -> dict[str, Any]: + """An unknown response stops the caller; a fresh run must claim and recompute.""" + await check_lock(conn) + answer = await call(path, {**body, **preconditions(row)}) + await check_lock(conn) + return answer + + +@dataclass(frozen=True) +class _Fatal: + error: SystemExit | KeyboardInterrupt + + +class _ForwardFatal(BaseException): + def __init__(self, fatal: _Fatal) -> None: + self.fatal = fatal + + +async def _captured[T](work: Callable[[], Awaitable[T]]) -> T | _Fatal: + try: + return await work() + except (SystemExit, KeyboardInterrupt) as error: + return _Fatal(error) + except _ForwardFatal as error: + return error.fatal + + +class Owner: + """One raw backend and one cleanup owner for a native control command.""" + + def __init__(self, *, exit_s: float = pg._RELEASE_TIMEOUT_S) -> None: + self.exit_s = exit_s + self.conn: Any = None + self.phase_end: float | None = None + self.close_end: float | None = None + self.closed = False + self._close_task: asyncio.Task[Any] | None = None + self._tasks: set[asyncio.Task[Any]] = set() + + def _start[T](self, work: Callable[[], Awaitable[T]]) -> asyncio.Task[T | _Fatal]: + task = asyncio.create_task(_captured(work)) + self._tasks.add(task) + task.add_done_callback(self._tasks.discard) + task.add_done_callback(pg._retrieve_task) + return task + + async def _open(self, what: str) -> None: + self.conn = await pg._connect_dedicated(statement_timeout_ms=60_000) + if self.closed: + # A connector returning after cancellation cannot publish a usable session. + await self.close() + raise _refused("connection arrived after the control attempt ended") + try: + await pg._check_deployment(self.conn) + await pg.init_connection(self.conn) + except Exception as error: # Classify unavailable openings; keep other failures. + pg._raise_if_unavailable(error) + raise + refused = await apg.direct_refusals(self.conn, what) + refused += await apg.encoding_refusals(self.conn) + if refused: + raise Refused(refused) + await apg.take_lock(self.conn, what) + + async def close(self) -> None: + if not self.closed: + self.closed = True + self.close_end = self.phase_end or (time.monotonic() + self.exit_s) + if self.conn is None: + return + if self._close_task is None: + self._close_task = asyncio.create_task( + _captured(lambda: pg.close_owned(self.conn, end=cast(float, self.close_end))) + ) + self._close_task.add_done_callback(pg._retrieve_task) + interrupted: asyncio.CancelledError | None = None + try: + while not self._close_task.done(): + remaining = cast(float, self.close_end) - time.monotonic() + if remaining <= 0: + raise TimeoutError("cell control cleanup deadline expired; outcome unknown") + try: + await asyncio.wait({self._close_task}, timeout=remaining) + except asyncio.CancelledError as error: + # Signals cannot abort the public cancellation already in flight. + interrupted = interrupted or error + result = self._close_task.result() + if isinstance(result, _Fatal): + raise _ForwardFatal(result) + except BaseException: + try: + pg._terminate(self.conn) + except BaseException: # noqa: BLE001 — abort cannot replace the first close failure. + pass + finally: + if not self._close_task.done(): + self._close_task.cancel() + if interrupted is not None: + raise interrupted from None + raise + if interrupted is not None: + raise interrupted + + async def _stop(self, tasks: set[asyncio.Task[Any]]) -> None: + for task in tasks: + if not task.done(): + task.cancel() + try: + if tasks: + await asyncio.wait(tasks, timeout=max(0, (self.close_end or 0) - time.monotonic())) + except BaseException: # noqa: BLE001 — cleanup cannot replace the original failure. + if self.conn is not None: + try: + pg._terminate(self.conn) + except BaseException: # noqa: BLE001 — failed abort preserves the caller's failure. + pass + + async def call[T](self, work: Callable[[], Awaitable[T]], *, total_s: float) -> T: + if self.closed or self.phase_end is not None: + raise _refused("a closed or active control phase cannot start another phase") + self.phase_end = time.monotonic() + total_s + task = self._start(work) + try: + done, _ = await asyncio.wait( + {task}, + timeout=max(0, self.phase_end - min(1.0, total_s / 4) - time.monotonic()), + ) + if not done: + raise TimeoutError( + "cell control phase deadline expired; outcome unknown; stop and rerun" + ) + result = task.result() + if isinstance(result, _Fatal): + raise _ForwardFatal(result) + except BaseException: + try: + await self.close() + except BaseException: # noqa: BLE001 — keep the original phase failure/cancellation. + pass + await self._stop({task}) + raise + self.phase_end = None + return result + + async def transaction[T](self, work: Callable[[], Awaitable[T]]) -> T: + return await self.call(lambda: transaction(self.conn, work), total_s=TRANSACTION_S) + + async def run[T]( + self, + opening: Callable[[], Awaitable[None]], + work: Callable[[], Awaitable[T]], + *, + opening_s: float, + ) -> T: + """Own one command; fatal guard failures leave only after bounded cleanup.""" + if self.conn is not None or self.phase_end is not None or self._tasks or self.closed: + raise _refused("a command needs a fresh control owner") + try: + await self.call(opening, total_s=opening_s) + result = await asyncio.shield(self._start(work)) + if isinstance(result, _Fatal): + raise _ForwardFatal(result) + except BaseException as original: + try: + await self.close() + except BaseException: # noqa: BLE001 — preserve the command's original failure. + pass + await self._stop(set(self._tasks)) + if isinstance(original, _ForwardFatal): + raise original.fatal.error from None + raise + try: + await self.close() + except _ForwardFatal as interrupted: + raise interrupted.fatal.error from None + finally: + await self._stop(set(self._tasks)) + return result + + +async def run[T](what: str, work: Callable[[Owner], Awaitable[T]]) -> T: + """Run native control with its existing opening, transaction and exit allowances.""" + owner = Owner() + return await owner.run(lambda: owner._open(what), lambda: work(owner), opening_s=OPEN_S) + + +async def transaction[T](conn: Any, work: Callable[[], Awaitable[T]]) -> T: + """Borrow the owner's backend; every failure discards it without a rollback wait.""" + tx = conn.transaction() + await tx.start() + await conn.execute("SET LOCAL lock_timeout = '5s'") + await conn.execute("SET LOCAL statement_timeout = '10s'") + await check_lock(conn) + await conn.execute("SELECT pg_logical_emit_message(true, 'loom.control', '')") + result = await work() + await tx.commit() + return result + + +async def request( + url: Any, + name: str, + transport: Any, + path: str, + body: dict[str, Any] | None = None, + *, + method: str = "POST", +) -> dict[str, Any]: + """Read or mutate one operator row; gateways and lost replies are unknown outcomes.""" + import httpx + + from server.cli import cell as cli + + endpoint = cli.at(url, f"/cells/v1/operator/cells/{name}" + path) + try: + async with asyncio.timeout(REQUEST_S): + manager = httpx.AsyncClient( + timeout=cli._OPERATOR_TIMEOUT, + trust_env=url.scheme == "https", + transport=transport, + headers={"Authorization": f"Bearer {settings.router_operator_token}"}, + ) + client = await manager.__aenter__() + try: + response = await client.request(method, endpoint, json=body) + except BaseException: # Client exit preserves the request's first failure. + with contextlib.suppress(BaseException): + await manager.__aexit__(*sys.exc_info()) + raise + else: + await manager.__aexit__(None, None, None) + except (httpx.HTTPError, TimeoutError) as e: + raise _refused(f"router reply unknown ({type(e).__name__}): stop and rerun") from None + answer = cli.router_answer(response, {"cell": name}, cli.OPERATOR_REFUSALS) + if answer is None: + raise _refused(cli.not_the_router(response)) + if response.status_code == 404 and answer.get("code") != "NO_SUCH_CELL": + raise _refused("a router replica is not upgraded for cell control fencing") + if response.status_code != 200: + raise _refused( + f"router refused {path or 'the read'} ({response.status_code}): {answer['error']}" + ) + return answer + + +async def _acceptance(conn: Any) -> dict[str, Any]: + return { + "first": dict(row) + if (row := await conn.fetchrow("SELECT is_first, cell FROM loom.cell_first")) + else None, + "mode": await conn.fetchval("SELECT mode FROM loom.tenancy_mode"), + "own": dict(row) + if (row := await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self")) + else None, + } + + +async def backout(owner: Owner, call: Call, row: dict[str, Any]) -> None: + """The runbook's exact state cleanup, fenced against queued controllers/placements.""" + from server.core.schema import pg_migrate + + conn = owner.conn + before = await owner.transaction(lambda: _acceptance(conn)) + first = before["first"] + if first is not None and (first["cell"], first["is_first"]) != (row["cell"], row["is_first"]): + raise _refused("the local first row names another registration") + already_open = not any(before.values()) and not await conn.fetchval( + "SELECT EXISTS(SELECT 1 FROM loom.tenancy)" + ) + if row["is_first"] and before["mode"] != "recording" and not already_open: + raise _refused("first-cell backout requires local recording mode; enforced goes forward") + row = await claim(conn, call, row) + if row.get("state") != "attaching": + raise _refused("a serving cell goes forward; state cleanup refused") + # Closing and reading placed under one directory row lock excludes an ordinary + # placement queued before the claim. A later failure deliberately leaves it closed. + limited = await mutate(conn, call, row, "/limit", {"closed": True}) + if type(limited.get("placed_orgs")) is not int or limited["placed_orgs"] != 0: + raise _refused("cell holds placements: cleanup refused; the cell remains closed") + + async def clear_state() -> None: + await paired(conn, row) + if json.dumps(await _acceptance(conn), sort_keys=True) != json.dumps( + before, sort_keys=True + ): + raise _refused("local acceptance changed: cleanup refused; the cell remains closed") + await conn.execute("SELECT pg_advisory_xact_lock($1)", pg_migrate._LOCK_KEY) + await conn.execute("ALTER TABLE loom.cell_first DISABLE TRIGGER cell_first_immutable") + for table in ("cell_first", "tenancy_mode", "cell_self", "tenancy"): + await conn.execute(f"DELETE FROM loom.{table}") + await conn.execute("ALTER TABLE loom.cell_first ENABLE ALWAYS TRIGGER cell_first_immutable") + await conn.execute( + "ALTER TABLE loom.cell_first ENABLE ALWAYS TRIGGER cell_first_no_truncate" + ) + + await owner.transaction(clear_state) + + # Once state is open, a migrating boot cannot recreate the org triggers. The same + # session still holds ATTACH; a retry repeats the durable state cleanup first. + async def clear_triggers() -> None: + await conn.execute("SELECT pg_advisory_xact_lock($1)", pg_migrate._LOCK_KEY) + triggers = await conn.fetch( + "SELECT t.tgname, c.relname FROM pg_trigger t JOIN pg_class c ON c.oid = t.tgrelid " + "JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'loom' " + "AND t.tgname IN ('tenancy_insert', 'tenancy_update', 'tenancy_record')" + ) + for trigger in triggers: + name, table = (str(trigger[k]).replace('"', '""') for k in ("tgname", "relname")) + await conn.execute(f'DROP TRIGGER "{name}" ON loom."{table}"') + + await owner.transaction(clear_triggers) diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index 8ad83098a9..517a461840 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -1,7 +1,10 @@ """`loom cell prepare`: register this cell in the router's directory and write the rows its processes read (docs/architecture/cells.md, "Preparing a cell" and "The tenancy check"). -It runs where the cell's stores are configured (`LOOM_CELL` set to the cell it names): +It runs where the cell's stores are configured (`LOOM_CELL` set to the cell it names), +holding the attach lock (`cell.ATTACH_LOCK_KEY`) on a direct session of the cell's Postgres, +after that session's checks (one backend, the tenancy marker kept, a UTF8 database), so it +never runs beside an attach or another prepare: 1. It reads the cell's Postgres (deployment id, `first`, mode and own rows, tenancy rows) and the live processes' heartbeats, and refuses a cell whose rows name another cell @@ -23,6 +26,7 @@ import argparse import asyncio +import contextlib import hashlib import json import secrets @@ -34,6 +38,7 @@ import httpx from server.cli import cell as cell_cli +from server.cli import cell_control from server.cli._ch import writes from server.core import cell from server.settings import settings @@ -101,14 +106,12 @@ def _args_problem(args: argparse.Namespace) -> str: return "" -async def _read_rows() -> _CellRows: - from server.core import pg, pg_profile - - deployment_id = await pg_profile.deployment_id() - first = await pg.fetchrow("SELECT is_first, cell FROM loom.cell_first") - mode = await pg.fetchval("SELECT mode FROM loom.tenancy_mode") - own = await pg.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") - tenancy_rows = await pg.fetchval("SELECT count(*) FROM loom.tenancy") +async def _read_rows(conn: Any, *, count_tenancy: bool = True) -> _CellRows: + deployment_id = str(await conn.fetchval("SELECT deployment_id FROM loom.authority") or "") + first = await conn.fetchrow("SELECT is_first, cell FROM loom.cell_first") + mode = await conn.fetchval("SELECT mode FROM loom.tenancy_mode") + own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") + tenancy_rows = await conn.fetchval("SELECT count(*) FROM loom.tenancy") if count_tenancy else 0 return _CellRows( deployment_id=deployment_id, first=None if first is None else (bool(first["is_first"]), str(first["cell"])), @@ -278,7 +281,7 @@ async def _bucket() -> None: try: await step() except Exception as e: # noqa: BLE001 — every nonce is tried; a stale one holds no data - print(f"loom cell prepare: the probe nonce in {where} stays: {e}", file=sys.stderr) + print(f"loom cell: the probe nonce in {where} stays: {e}", file=sys.stderr) def _register_timeout() -> httpx.Timeout: @@ -299,9 +302,11 @@ async def _register( token = settings.router_operator_token url = cell_cli.at(router, f"/cells/v1/operator/cells/{args.cell}/register") deadline = time.monotonic() + args.wait - async with httpx.AsyncClient( + manager = httpx.AsyncClient( timeout=_register_timeout(), trust_env=router.scheme == "https", transport=transport - ) as client: + ) + client = await manager.__aenter__() + try: while True: try: r = await client.post(url, json=body, headers={"Authorization": f"Bearer {token}"}) @@ -316,7 +321,7 @@ async def _register( f"{cell_cli.not_the_router(r)}; run prepare again once the router answers" ) if r.status_code == 200: - return answer + break code = answer.get("code") if code == "BACKEND_UNAVAILABLE": raise Refused( @@ -338,6 +343,14 @@ async def _register( print(f"loom cell prepare: {answer['error']}; retrying", file=sys.stderr) await _sleep(pause) + except BaseException: # Client exit preserves the first request or refusal. + with contextlib.suppress(BaseException): + await manager.__aexit__(*sys.exc_info()) + raise + else: + await manager.__aexit__(None, None, None) + return answer + def _own_rows( rows: _CellRows, answer: dict[str, Any], every: list[dict[str, Any]] @@ -374,15 +387,24 @@ def _own_rows( async def _write_rows( - rows: _CellRows, answer: dict[str, Any], current: dict[str, Any], nxt: dict[str, Any] | None + conn: Any, + rows: _CellRows, + answer: dict[str, Any], + current: dict[str, Any], + nxt: dict[str, Any] | None, ) -> list[str]: """Write the `first` and mode rows that are missing and the own row where it differs from `current` and `nxt`, in one transaction; the rows written.""" - from server.core import pg - is_first = bool(answer.get("is_first")) written = [] - async with pg.transaction() as conn: + + async def write() -> list[str]: + # A state CAS does not need to count every tenancy row again. + fresh = await _read_rows(conn, count_tenancy=False) + before = (rows.deployment_id, rows.first, rows.mode, rows.current, rows.next) + after = (fresh.deployment_id, fresh.first, fresh.mode, fresh.current, fresh.next) + if json.dumps(before, sort_keys=True) != json.dumps(after, sort_keys=True): + raise Refused("local acceptance rows changed: stop and rerun with a fresh claim") if rows.first is None: await conn.execute( "INSERT INTO loom.cell_first (is_first, cell) VALUES ($1, $2)", @@ -395,7 +417,9 @@ async def _write_rows( "INSERT INTO loom.tenancy_mode (mode) VALUES ($1)", str(answer["mode"]) ) written.append("mode") - if (rows.current, rows.next) != (current, nxt): + if json.dumps((rows.current, rows.next), sort_keys=True) != json.dumps( + (current, nxt), sort_keys=True + ): await conn.execute( "INSERT INTO loom.cell_self (current_fp, next_fp) VALUES ($1, $2) " "ON CONFLICT (id) DO UPDATE SET current_fp = EXCLUDED.current_fp, " @@ -404,7 +428,9 @@ async def _write_rows( nxt, ) written.append("own") - return written + return written + + return await cell_control.transaction(conn, write) def _answer_problem(rows: _CellRows, answer: dict[str, Any], beats: list[_Beat]) -> str: @@ -443,7 +469,24 @@ async def prepare( router: httpx.URL, transport: httpx.AsyncBaseTransport | None = None, ) -> int: - rows = await _read_rows() + """Prepare on one independently supervised ATTACH-lock backend.""" + try: + return await cell_control.run( + "prepare", lambda owner: _prepare_locked(ch, args, router, transport, owner) + ) + except cell_control.Refused as error: + raise Refused(str(error)) from None + + +async def _prepare_locked( + ch: Any, + args: argparse.Namespace, + router: httpx.URL, + transport: httpx.AsyncBaseTransport | None, + owner: cell_control.Owner, +) -> int: + conn = owner.conn + rows = await owner.transaction(lambda: _read_rows(conn)) if not rows.deployment_id: raise Refused("this cell's Postgres records no deployment id: start the cell once first") if rows.first is not None and rows.first[1] != settings.cell: @@ -470,7 +513,11 @@ async def prepare( try: await _write_nonces(ch, body["nonces"]) answer = await _register(args, router, body, transport) - finally: + except BaseException: # Nonce cleanup preserves the first failure or cancellation. + with contextlib.suppress(BaseException): + await _delete_nonces(ch, body["nonces"]) + raise + else: await _delete_nonces(ch, body["nonces"]) # The processes may have changed while the probe was retried: judge the ones live now. beats = await _read_beats(ch) @@ -478,7 +525,22 @@ async def prepare( if problem := _answer_problem(rows, answer, beats): raise Refused(problem) current, nxt = _own_rows(rows, answer, every) - written = await _write_rows(rows, answer, current, nxt) + + async def call(path: str, body: dict[str, Any]) -> dict[str, Any]: + return await cell_control.request(router, args.cell, transport, path, body) + + snapshot = await cell_control.request(router, args.cell, transport, "", method="GET") + snapshot = await cell_control.claim(conn, call, snapshot) + # A queued earlier controller may have committed before this claim. Its returned + # row, including the highest mode, supersedes the registration response. + answer = {**answer, **snapshot} + if problem := _answer_problem(rows, snapshot, beats): + raise Refused(problem) + current, nxt = _own_rows(rows, snapshot, every) + written = await owner.call( + lambda: _write_rows(conn, rows, answer, current, nxt), + total_s=cell_control.TRANSACTION_S, + ) print( json.dumps( { diff --git a/server/cli/cell_scan.py b/server/cli/cell_scan.py index 3bb6d8f9cd..f66435373f 100644 --- a/server/cli/cell_scan.py +++ b/server/cli/cell_scan.py @@ -68,6 +68,19 @@ } ) +# Tables a worker drains into another table, deleting what it moved: each is scanned +# before every table that is not one, so a row moved mid-scan is read where it was or where +# it went. A table a materialized view reads is ordered the same way, from +# `system.tables` dependencies (`order_sources`). +MOVE_SOURCES = frozenset( + { + "loom.learnings_queue", + "loom.memory_code_link_queue", + "loom.memory_extraction_queue", + "loom.memory_webhook_outbox", + } +) + # A refreshable view's temporary table while a refresh exchanges its target. _EXCHANGE = re.compile(r"^\.tmp\.inner(_id)?\.") # A provider transition's copies (server/erasure/store/provider_copies.py). @@ -339,6 +352,32 @@ async def list_postgres(conn: Any) -> tuple[list[str], list[Refusal]]: return names, refusals +def order_sources(tables: list[Table], view_sources: Iterable[str]) -> list[Table]: + """`tables` with the move sources (`MOVE_SOURCES`, and every table a view reads, + `view_sources`, canonical names) first, each group by name.""" + first = MOVE_SOURCES | set(view_sources) + return sorted(tables, key=lambda t: (t.canonical not in first, t.canonical)) + + +async def view_sources( + ch: Any, topology: Topology, names: Mapping[str, str], dbs: list[str] +) -> set[str]: + """The tables of `dbs` a view depends on, as canonical names.""" + rows = await per_host( + ch, + topology, + names, + "SELECT hostName() AS h, database, name FROM {source:tables} " + "WHERE database IN {dbs:Array(String)} AND notEmpty(dependencies_table)", + {"dbs": dbs}, + ) + return { + f"{ch_names.logical(str(r['database']))}.{r['name']}" + for found in rows.values() + for r in found + } + + async def list_tables( ch: Any, pg_conn: Any, topology: Topology, names: Mapping[str, str] ) -> TableList: @@ -371,7 +410,8 @@ async def _list_tables( if copies := provider_copies(tables): out.refusals += copies return out - out.clickhouse, problems = judge_clickhouse(tables, classified, topology) + scanned, problems = judge_clickhouse(tables, classified, topology) + out.clickhouse = order_sources(scanned, await view_sources(ch, topology, names, databases())) out.refusals += problems out.refusals += await reverse_clickhouse(ch, tables, classified) out.postgres, problems = await list_postgres(pg_conn) @@ -461,7 +501,7 @@ def pg_skip_scan_sql(table: str) -> str: return _PG_SKIP_SCAN.format(table=f"{pg_quote(schema)}.{pg_quote(name)}") -async def _pg_orgs(table: str, budget: Budget) -> list[str]: +async def pg_orgs(table: str, budget: Budget) -> list[str]: subject = f"postgres {table}" while True: left = budget.require(subject) @@ -616,7 +656,7 @@ async def _read_scan( return _body_rows(b"".join(chunks).decode("utf-8", "replace").splitlines(), "k") -async def _ch_orgs(ch: Any, table: Table, topology: Topology, budget: Budget) -> list[bytes | None]: +async def ch_orgs(ch: Any, table: Table, topology: Topology, budget: Budget) -> list[bytes | None]: subject = f"clickhouse {table.canonical}" while True: budget.require(subject) @@ -698,10 +738,10 @@ async def scan( result.add("configured", org) with query_tag(TAG): for name in tables.postgres: - for org in await _pg_orgs(name, budget): + for org in await pg_orgs(name, budget): result.add(f"postgres {name}", org) for table in tables.clickhouse: - for raw in await _ch_orgs(ch, table, topology, budget): + for raw in await ch_orgs(ch, table, topology, budget): result.add(f"clickhouse {table.canonical}", raw) result.elapsed_s = budget.elapsed() return result diff --git a/server/cli/code_run.py b/server/cli/code_run.py index 5479c3d851..5c11a94f60 100644 --- a/server/cli/code_run.py +++ b/server/cli/code_run.py @@ -117,10 +117,11 @@ async def _then_disconnect(coro: asyncio.Future[int] | Coroutine[Any, Any, int]) try: # A repository added from the Repositories page lives in the registry, not in # LOOM_CODE_REPOS_JSON: read it before the subcommand resolves its namespace. + from server.cli._ch import connect_verified from server.code.app.index.repo_registry import REGISTRY_UNREAD_MESSAGE, refresh_db_repos from server.code.domain.index.repos import RegistryUnread - await connect_declared(get_client(), org_free=reads("reads the repository registry")) + await connect_verified(get_client()) async with model_command(): # Never raises; while it has never succeeded, `configured_repos` refuses. await refresh_db_repos(force=True) @@ -133,13 +134,23 @@ async def _then_disconnect(coro: asyncio.Future[int] | Coroutine[Any, Any, int]) # Some native commands close their own client. Reopen it for # buffered-write drain/retirement before leaving enrollment. # The subcommand's own connect gated its writes; a drain checks nothing new. - await connect_declared( - get_client(), org_free=reads("drains the writes its own connect gated") - ) + first = sys.exception() + try: + await connect_declared( + get_client(), org_free=reads("drains the writes its own connect gated") + ) + except BaseException: # keep the subcommand failure if drain reconnect also fails + if first is None: + raise finally: + first = sys.exception() if not started and asyncio.iscoroutine(coro): coro.close() - await disconnect(get_client()) + try: + await disconnect(get_client()) + except BaseException: # a disconnect failure must not replace the subcommand failure + if first is None: + raise def _run(name: str, coro: asyncio.Future[int] | Coroutine[Any, Any, int]) -> int: diff --git a/server/cli/doctor.py b/server/cli/doctor.py index d1295ffb32..b4814b7978 100644 --- a/server/cli/doctor.py +++ b/server/cli/doctor.py @@ -482,7 +482,12 @@ async def read() -> tuple[Any, ...] | None: await tier_health.refresh_failing() return tier_health.problem_rows() finally: - await disconnect(client) + first = sys.exception() + try: + await disconnect(client) + except BaseException: # keep the diagnostic failure when close also fails + if first is None: + raise _read_code_registry() # the self-service entries too; a no-op once a read succeeded try: @@ -509,7 +514,12 @@ async def read() -> None: await connect_declared(client, org_free=reads("reads")) await repo_registry.refresh_db_repos(force=True) finally: - await disconnect(client) + first = sys.exception() + try: + await disconnect(client) + except BaseException: # keep the diagnostic failure when close also fails + if first is None: + raise try: asyncio.run(asyncio.wait_for(read(), timeout=5)) @@ -566,10 +576,15 @@ def grade(p: Any) -> str: return [(f"problem:{p.code}", grade(p), f"{p.message} Fix: {p.fix}") for p in problems] -def _run_doctor_checks() -> list[tuple[str, str, str]]: +def _run_doctor_checks(allowed_roles: tuple[str, ...] = ()) -> list[tuple[str, str, str]]: """The live checks (Python, settings, JWT, ClickHouse, LLM) followed - by the shared problem list, as (name, grade, detail).""" - return _run_live_checks() + _code_mirror_checks() + _problem_checks(_doctor_problems()) + by the shared problem list, as (name, grade, detail). `allowed_roles` is + `--pg-allow-role`, for a cell's checks.""" + return ( + _run_live_checks(allowed_roles) + + _code_mirror_checks() + + _problem_checks(_doctor_problems()) + ) def _code_mirror_checks() -> list[tuple[str, str, str]]: @@ -582,7 +597,7 @@ def _code_mirror_checks() -> list[tuple[str, str, str]]: return [("code_mirrors", WARN, f"not checked: {type(e).__name__}: {e}")] -def _run_live_checks() -> list[tuple[str, str, str]]: +def _run_live_checks(allowed_roles: tuple[str, ...] = ()) -> list[tuple[str, str, str]]: """Run the doctor's live checks and return them as (name, grade, detail) with grade one of PASS / WARN / FAIL. @@ -664,7 +679,7 @@ def _run_live_checks() -> list[tuple[str, str, str]]: if settings_ok and ch_ok and settings.cell: from server.cli import doctor_cell - checks += doctor_cell.cell_checks() + checks += doctor_cell.cell_checks(allowed_roles) # 5. LLM provider resolution: what extraction runs on and # what decided it; WARN when the provider was inferred (or named) but its @@ -677,6 +692,17 @@ def _run_live_checks() -> list[tuple[str, str, str]]: return checks +def _stray_org_checks(budget_s: float) -> list[tuple[str, str, str]]: + """`--stray-orgs`, on a cell only (LOOM_CELL set).""" + from server.settings import settings + + if not settings.cell: + return [("cell_stray_orgs", WARN, "not read: LOOM_CELL is unset (no cell)")] + from server.cli import doctor_cell + + return doctor_cell.stray_org_checks(budget_s) + + def _client_checks(args: argparse.Namespace) -> list[tuple[str, str, str]]: """`loom doctor --client`: the checks from the machine a coding agent runs on, with the CLI's own HTTP helpers. A crash in one of @@ -722,7 +748,9 @@ def _cmd_doctor(_args: argparse.Namespace) -> int: operator sees the full picture. Exit 0 = nothing failed (warnings are listed but do not fail); 1 = at least one FAIL. """ - checks = _run_doctor_checks() + checks = _run_doctor_checks(tuple(getattr(_args, "pg_allow_role", None) or ())) + if getattr(_args, "stray_orgs", False): + checks += _stray_org_checks(_args.stray_orgs_budget) # The client's side of the same picture, when asked. client = _client_checks(_args) if getattr(_args, "client", False) else [] checks = _client_machine_grades(checks, client) + client @@ -841,4 +869,21 @@ def _add_doctor_flags(p_doc: argparse.ArgumentParser) -> None: ), ) p_doc.add_argument("--server-url", default=None, help="Override the client's server URL.") + p_doc.add_argument( + "--pg-allow-role", + action="append", + default=[], + help="On a cell: a Postgres role that may make an org held (the servers' role).", + ) + p_doc.add_argument( + "--stray-orgs", + action="store_true", + help="On a cell: also read the orgs its stores hold without a tenancy row.", + ) + p_doc.add_argument( + "--stray-orgs-budget", + type=float, + default=60.0, + help="Seconds --stray-orgs may read for, smallest tables first (default 60).", + ) p_doc.add_argument("--token", default=None, help="Override the client's token.") diff --git a/server/cli/doctor_cell.py b/server/cli/doctor_cell.py index f2d6f80367..79fbebb5af 100644 --- a/server/cli/doctor_cell.py +++ b/server/cli/doctor_cell.py @@ -1,7 +1,11 @@ """`loom doctor`'s cell checks, run only with LOOM_CELL set (docs/architecture/cells.md): the deployment check, a partial cell state, the tenancy marker on this process's -sessions and on every session, and the defects of the org triggers and the `loom.tenancy` -guard (`cell_triggers.missing`). +sessions and on every session, the defects of the org triggers and the `loom.tenancy` +guard (`cell_triggers.missing`), and the point-in-time reads attach makes, run again on +every doctor run: the login roles that can make an org held or lower the mode, and the +role and database settings and subscriptions that pass an unchecked writer +(`cell_attach_pg`). `--stray-orgs` also reads the orgs the cell's stores hold without a +tenancy row (`stray_org_checks`). A refusal (ClickHouse access denied, no LOOM_PG_DSN, another recorded authority or a Postgres of another deployment) fails the doctor's exit and reads no cell state; @@ -11,6 +15,7 @@ from __future__ import annotations import asyncio +import sys from typing import Any from server.cli._ch import reads @@ -78,14 +83,34 @@ async def _unmarked_session() -> bool: """Whether a session that sends no marker passes the triggers all the same (a role, database, ALTER SYSTEM or options setting): marked, or in the replica role that fires no origin trigger.""" + from server.cli import cell_attach_pg + + return await cell_attach_pg.session_passes() + + +async def _writer_reads(allowed_roles: tuple[str, ...]) -> list[tuple[str, str, str]]: + """Attach's point-in-time Postgres reads, again: the login roles outside the owner + roles, the superusers and `--pg-allow-role` that can make an org held or lower the + mode, and the settings and subscriptions that pass an unchecked writer.""" + from server.cli import cell_attach_pg from server.core import pg - async with pg.dedicated(statement_timeout_ms=_STATEMENT_MS, marked=False) as conn: - value: str | None = await conn.fetchval( - "SELECT current_setting($1, true)", pg.TENANCY_MARKER - ) - role: str = await conn.fetchval("SELECT current_setting('session_replication_role')") - return value == "on" or role == "replica" + async with pg.dedicated(statement_timeout_ms=_STATEMENT_MS) as conn: + holders = await cell_attach_pg.privilege_holders(conn, exempt_self=False) + unchecked = await cell_attach_pg.unmarked_refusals(conn, False) + roles = cell_attach_pg.privilege_refusals(holders, allowed_roles) + return [ + ( + "cell_tenancy_privilege", + WARN if roles else PASS, + "; ".join(str(r) for r in roles) or "no role outside the owner roles can", + ), + ( + "cell_unchecked_writers", + WARN if unchecked else PASS, + "; ".join(str(r) for r in unchecked) or "no setting or subscription passes one", + ), + ] def _grades( @@ -149,7 +174,7 @@ def _grades( return out -async def _checks(ch: Any) -> list[tuple[str, str, str]]: +async def _checks(ch: Any, allowed_roles: tuple[str, ...] = ()) -> list[tuple[str, str, str]]: try: if before := await asyncio.wait_for(_deployment_check(ch), timeout=_SESSION_S): return [("cell_state", PASS, f"cell checks not run: {before}")] @@ -159,7 +184,14 @@ async def _checks(ch: Any) -> list[tuple[str, str, str]]: return [("cell_deployment", FAIL, f"deployment check refused: {e}")] except Exception as e: # noqa: BLE001 — an unreadable cell is named, not a failed run return [("cell_state", WARN, f"not read: {_reason(e)}")] - return _grades(state, marked, marks_all, defects) + try: + reads: list[tuple[str, str, str]] = await _bounded(_writer_reads(allowed_roles)) + except Exception as e: # noqa: BLE001 — an unread catalog is named, not a failed run + reads = [ + (name, WARN, f"not read: {_reason(e)}") + for name in ("cell_tenancy_privilege", "cell_unchecked_writers") + ] + return _grades(state, marked, marks_all, defects) + reads def _reason(e: BaseException) -> str: @@ -167,9 +199,10 @@ def _reason(e: BaseException) -> str: return f"{type(e).__name__}: {str(e) or 'no answer within the bound'}" -def cell_checks() -> list[tuple[str, str, str]]: +def cell_checks(allowed_roles: tuple[str, ...] = ()) -> list[tuple[str, str, str]]: """The cell checks, on the CLI's ClickHouse client connected for them alone (no CLI - deployment check: a deployment not booted yet is named, not refused).""" + deployment check: a deployment not booted yet is named, not refused). `allowed_roles` + is doctor's `--pg-allow-role`.""" from server.cli._ch import connect_declared, disconnect from server.core.ch import get_client @@ -180,10 +213,146 @@ async def _go() -> list[tuple[str, str, str]]: connect_declared(ch, org_free=reads("loom doctor reads a cell's state")), timeout=_SESSION_S, ) - return await _checks(ch) + return await _checks(ch, allowed_roles) except Exception as e: # noqa: BLE001 — a ClickHouse that does not answer is named return [("cell_state", WARN, f"not read: {_reason(e)}")] finally: - await disconnect(ch) + first = sys.exception() + try: + await disconnect(ch) + except BaseException: # keep the diagnostic failure when close also fails + if first is None: + raise return asyncio.run(_go()) + + +# -- --stray-orgs -------------------------------------------------------------------------- + + +async def _table_sizes(ch: Any, tables: list[Any]) -> dict[str, int]: + """Each ClickHouse table's active row count, from `system.parts` (KeeperMap: 0).""" + from server.core import ch_names + + rows = await ch.query_all( + "SELECT database, table, sum(rows) AS n FROM system.parts WHERE active " + "AND database IN {dbs:Array(String)} GROUP BY database, table", + {"dbs": sorted({t.database for t in tables})}, + ) + return {f"{ch_names.logical(str(r['database']))}.{r['table']}": int(r["n"]) for r in rows} + + +async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: + from server.cli import cell_scan + from server.cli.cell_ch import Refused, databases, read_topology + from server.core import pg + from server.core.cell_triggers import ORG_TABLES + + topology = await read_topology(ch) + if topology.multi: + from server.cli.cell_writers import host_names + + names = await host_names(ch, topology) + else: + names = {} + listed, problems = await cell_scan.list_clickhouse(ch, topology, names, databases()) + ch_tables = [t for t in listed if t.engine.endswith("MergeTree") or t.engine == "KeeperMap"] + sizes = await _table_sizes(ch, ch_tables) + async with pg.dedicated(statement_timeout_ms=_STATEMENT_MS) as conn: + held = {str(r["org"]) for r in await conn.fetch("SELECT org FROM loom.tenancy")} + pg_rows = await conn.fetch( + "SELECT 'loom.' || c.relname AS name, greatest(c.reltuples, 0)::bigint AS n " + "FROM pg_class c WHERE c.oid IN (SELECT to_regclass('loom.' || t.relname) FROM (" + + ORG_TABLES + + ") t)" + ) + # Smallest first: a budget that runs out leaves the largest unchecked. + work: list[tuple[int, str, Any]] = [(int(r["n"]), "postgres", str(r["name"])) for r in pg_rows] + work += [(sizes.get(t.canonical, 0), "clickhouse", t) for t in ch_tables] + work.sort(key=lambda w: (w[0], str(w[2]))) + budget = cell_scan.Budget(budget_s) + stray: dict[str, list[str]] = {} + unchecked: list[str] = [] + refused: list[str] = [] + for i, (_n, store, table) in enumerate(work): + found = cell_scan.ScanResult() + name = table if store == "postgres" else table.canonical + try: + if store == "postgres": + for org in await cell_scan.pg_orgs(table, budget): + found.add(name, org) + else: + for raw in await cell_scan.ch_orgs(ch, table, topology, budget): + found.add(name, raw) + except Refused as e: + # The budget spent leaves this table and the larger ones unread; any other + # refusal (a table with no org-led index, a replica that cannot sync) names + # this table alone, and the read goes on. + if budget.left() <= 1: + unchecked += [w[2] if w[1] == "postgres" else w[2].canonical for w in work[i:]] + break + refused += [str(r) for r in e.refusals] + continue + orgs = sorted(set(found.orgs) - held) + sorted(found.too_long) + bad = [m for ms in found.malformed.values() for m in ms] + if orgs or bad: + log = " (a log table: never placed)" if name in cell_scan.LOG_TABLES else "" + stray[f"{store} {name}{log}"] = orgs[:5] + bad[:5] + out = [ + ( + "cell_stray_orgs", + WARN if stray else PASS, + "; ".join(f"{t}: {', '.join(o)}" for t, o in sorted(stray.items())) + or "every org found has a tenancy row", + ) + ] + if unchecked: + out.append( + ( + "cell_stray_orgs_unchecked", + WARN, + f"not read within --stray-orgs-budget ({budget_s:g} s): " + ", ".join(unchecked), + ) + ) + if refused or problems: + out.append( + ( + "cell_stray_orgs_unread", + WARN, + "not read: " + "; ".join(refused + [str(p) for p in problems]), + ) + ) + return out + + +def stray_org_checks(budget_s: float) -> list[tuple[str, str, str]]: + """`loom doctor --stray-orgs`: the orgs the cell's stores hold without a tenancy row + (log-only orgs, names no request can carry, rows an unchecked writer left), smallest + tables first within one budget, naming every table left unchecked and every table it + could not read. Never a FAIL: the operator removes what it names by hand + (docs/operations/cell-attach.md).""" + from server.cli._ch import connect_declared, disconnect + from server.core.ch import get_client, statement_timeout + + async def _go() -> list[tuple[str, str, str]]: + ch = get_client() + try: + await asyncio.wait_for( + connect_declared(ch, org_free=reads("loom doctor reads a cell's orgs")), + timeout=_SESSION_S, + ) + if before := await asyncio.wait_for(_deployment_check(ch), timeout=_SESSION_S): + return [("cell_stray_orgs", PASS, f"not read: {before}")] + return await _stray_reads(ch, budget_s) + except Exception as e: # noqa: BLE001 — an unread store is named, not a failed run + return [("cell_stray_orgs", WARN, f"not read: {_reason(e)}")] + finally: + first = sys.exception() + try: + await disconnect(ch) + except BaseException: # keep the diagnostic failure when close also fails + if first is None: + raise + + with statement_timeout(budget_s): + return asyncio.run(_go()) diff --git a/server/cli/import_brain.py b/server/cli/import_brain.py index 8247dcf699..10bead6dba 100644 --- a/server/cli/import_brain.py +++ b/server/cli/import_brain.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import contextlib import logging from typing import Any @@ -157,7 +158,11 @@ async def _run() -> int: except Exception as e: # noqa: BLE001 — one document must not sink the rest fail += 1 log.warning("import-brain: knowledge.add %s failed: %s", uri, e) - finally: + except BaseException: # an importer failure must survive its writer close + with contextlib.suppress(BaseException): + await closer() + raise + else: await closer() print(f"import-brain: wrote {ok} item(s), {fail} failed") return 1 if fail and ok == 0 else 0 diff --git a/server/cli/import_clickgap.py b/server/cli/import_clickgap.py index 60f3fdc6fc..cc44181142 100644 --- a/server/cli/import_clickgap.py +++ b/server/cli/import_clickgap.py @@ -716,15 +716,16 @@ async def _clickgap_write_category_links(w: _ClickgapWrite) -> None: async def _clickgap_enroll(stack: Any, org: str) -> Any: """Connect and enroll before selecting the importer's effective embedder.""" from server import boot - from server.cli._ch import connect_checked + from server.cli._ch import check_acceptance, connect_verified from server.core.background import register_buffer_drainer from server.core.ch import get_client from server.embedding import get_embedder from server.insights.app.provider_command import model_command - await connect_checked(get_client(), orgs=(org,)) + await connect_verified(get_client()) register_buffer_drainer("boot-writes", boot.drain_writes) await stack.enter_async_context(model_command()) + await check_acceptance(orgs=(org,)) return get_embedder() @@ -900,7 +901,12 @@ async def _run_then_disconnect() -> int: async with command_stack: return await _run() finally: - await disconnect(get_client()) + first = sys.exception() + try: + await disconnect(get_client()) + except BaseException: # a disconnect failure must not replace the command failure + if first is None: + raise from contextlib import AsyncExitStack diff --git a/server/cli/import_recall.py b/server/cli/import_recall.py index fd0925accb..6e49c95b84 100644 --- a/server/cli/import_recall.py +++ b/server/cli/import_recall.py @@ -731,7 +731,11 @@ async def _run() -> int: f" ! write failed ({w['op']} {ident}): {e}", file=sys.stderr, ) - finally: + except BaseException: # an importer failure must survive its writer close + with contextlib.suppress(BaseException): + await closer() + raise + else: await closer() print(f"✓ import complete: {ok} writes ok, {failed} failed") @@ -755,6 +759,27 @@ def _publication_pending(r: Any) -> bool: return isinstance(detail, dict) and detail.get("code") == PublicationPending.code +def _make_http_writer( + args: argparse.Namespace, base_url: str, *, timeout_s: float +) -> tuple[Any, Any]: + import httpx + + token, source = resolve_token(args.token) + if refused_for_agent(source): + raise SystemExit(2) + headers = {"Authorization": f"Bearer {token}"} if token else {} + client = httpx.AsyncClient(timeout=timeout_s, headers=headers) + base = base_url.rstrip("/") + + async def _http_writer(op: str, payload: dict[str, Any]) -> None: + r = await client.post(f"{base}/v1/{op}", json=_recall_jsonable(payload)) + if _publication_pending(r): + return # committed; the server's delivery worker publishes it + r.raise_for_status() + + return _http_writer, client.aclose + + async def _make_writer( args: argparse.Namespace, inproc_ops: Callable[[], _OpHandlers], *, timeout_s: float ) -> tuple[Any, Any]: @@ -764,39 +789,31 @@ async def _make_writer( its handler) on a connected ClickHouse client.""" base_url = getattr(args, "base_url", None) if base_url: - import httpx - - token, source = resolve_token(args.token) - if refused_for_agent(source): - raise SystemExit(2) - headers = {"Authorization": f"Bearer {token}"} if token else {} - client = httpx.AsyncClient(timeout=timeout_s, headers=headers) - base = base_url.rstrip("/") - - async def _http_writer(op: str, payload: dict[str, Any]) -> None: - r = await client.post(f"{base}/v1/{op}", json=_recall_jsonable(payload)) - if _publication_pending(r): - return # committed; the server's delivery worker publishes it - r.raise_for_status() + return _make_http_writer(args, base_url, timeout_s=timeout_s) - return _http_writer, client.aclose - - from server.cli._ch import connect_checked, disconnect + from server.cli._ch import check_acceptance, connect_verified, disconnect from server.core.ch import get_client handlers = inproc_ops() ch = get_client() - await connect_checked(ch, orgs=(args.org,)) + await connect_verified(ch) from server import boot from server.core.background import register_buffer_drainer from server.insights.app.provider_command import model_command register_buffer_drainer("boot-writes", boot.drain_writes) command = model_command() + entered = False try: await command.__aenter__() - except BaseException: - await disconnect(ch) + entered = True + await check_acceptance(orgs=(args.org,)) + except BaseException: # entry failure must survive participant and connection cleanup + if entered: + with contextlib.suppress(BaseException): + await command.__aexit__(None, None, None) + with contextlib.suppress(BaseException): + await disconnect(ch) raise async def _inproc_writer(op: str, payload: dict[str, Any]) -> None: @@ -809,7 +826,11 @@ async def _inproc_writer(op: str, payload: dict[str, Any]) -> None: async def close() -> None: try: await command.__aexit__(None, None, None) - finally: + except BaseException: # participant exit failure must survive connection cleanup + with contextlib.suppress(BaseException): + await disconnect(ch) + raise + else: await disconnect(ch) return _inproc_writer, close diff --git a/server/cli/maintenance.py b/server/cli/maintenance.py index 53cb8e4004..9e9a3bbd0a 100644 --- a/server/cli/maintenance.py +++ b/server/cli/maintenance.py @@ -207,8 +207,9 @@ def _cmd_reextract(args: argparse.Namespace) -> int: the stored transcripts, the source untouched.""" import asyncio - from server.cli._ch import connect_declared, disconnect + from server.cli._ch import check_acceptance, connect_verified, disconnect from server.core.ch import get_client + from server.insights.app.provider_command import participant_command from server.memory.app.reextract import load_sessions, render_report, replay if args.into == args.namespace: @@ -225,33 +226,42 @@ def _cmd_reextract(args: argparse.Namespace) -> int: async def _go() -> int: ch = get_client() - await connect_declared( - ch, org_free=reads("reads; the replay writes through the server at --base-url") - ) + await connect_verified(ch) try: - sessions = await load_sessions(ch, org=args.org, namespace=args.namespace) + async with participant_command(): + await check_acceptance( + org_free=reads("reads; replay writes through the HTTP server") + ) + sessions = await load_sessions(ch, org=args.org, namespace=args.namespace) + if args.limit_sessions: + sessions = sessions[: args.limit_sessions] + if not sessions: + print( + f" no set_from_messages evidence in {args.org}/{args.namespace}", + file=sys.stderr, + ) + return 1 + rep = await replay( + sessions, + base_url=args.base_url, + token=token, + org=args.org, + into=args.into, + dates=args.dates, + concurrency=args.concurrency, + dry_run=args.dry_run, + ) + print( + render_report(rep, source=args.namespace, into=args.into, dry_run=args.dry_run) + ) + return 1 if rep.failed and rep.failed == rep.sessions else 0 finally: - await disconnect(ch) - if args.limit_sessions: - sessions = sessions[: args.limit_sessions] - if not sessions: - print( - f" no set_from_messages evidence in {args.org}/{args.namespace}", - file=sys.stderr, - ) - return 1 - rep = await replay( - sessions, - base_url=args.base_url, - token=token, - org=args.org, - into=args.into, - dates=args.dates, - concurrency=args.concurrency, - dry_run=args.dry_run, - ) - print(render_report(rep, source=args.namespace, into=args.into, dry_run=args.dry_run)) - return 1 if rep.failed and rep.failed == rep.sessions else 0 + first = sys.exception() + try: + await disconnect(ch) + except BaseException: # a disconnect failure must not replace the command failure + if first is None: + raise return asyncio.run(_go()) diff --git a/server/cli/participant.py b/server/cli/participant.py index e42832b6af..9154bf4be4 100644 --- a/server/cli/participant.py +++ b/server/cli/participant.py @@ -1,10 +1,9 @@ """`loom participant retire`: the manual recovery for a Loom process that died without shutting down. -Every Loom process registers a provider participant row, and a provider -reconfiguration waits until each live participant acknowledges the drain. A -participant that stops beating expires by itself; this retires one at once, -once an operator has confirmed its process is gone. +A participant retires only after permanent guarded shutdown or verifiable exact +physical death on a trusted host, with its cross-replica queries settled. +Heartbeat age and an operator affirmation are not evidence of death. """ from __future__ import annotations @@ -17,8 +16,7 @@ def _cmd_retire(args: argparse.Namespace) -> int: - """Retire a provider participant row whose process died without shutting - down, so a provider reconfiguration need not wait for it to expire.""" + """Verify exact process death and retire the participant after its queries settle.""" from uuid import UUID from server.cli._ch import run_connected @@ -28,13 +26,6 @@ def _cmd_retire(args: argparse.Namespace) -> int: except ValueError: print(f" ✗ {args.participant!r} is not a participant id", file=sys.stderr) return 2 - if not args.yes: - print( - " ✗ retiring a participant that is still running lets a reconfiguration proceed " - "under its writes; confirm its process is stopped, then pass --yes", - file=sys.stderr, - ) - return 2 async def _go(ch: Any) -> str: from server.insights.app.provider_barrier import retire_participant @@ -47,8 +38,9 @@ async def _go(ch: Any) -> str: return 1 if outcome == "live": print( - f" ✗ participant {participant} is still running: its heartbeat is fresh or " - "a query of its own still runs", + f" ✗ participant {participant} has no verifiable exact physical-death proof " + "or its cross-replica queries have not settled; run on its trusted host after " + "the exact process stops. Expiry and Kubernetes object deletion are not proof", file=sys.stderr, ) return 1 @@ -61,9 +53,8 @@ def register_participant_cli(sub: Any) -> None: ssub = p.add_subparsers(dest="participant_cmd", required=True) r = ssub.add_parser( "retire", - help="Retire the provider participant of a process that died without shutting " - "down (its id is `participant_id` in that process's /v1/health).", + help="Verify exact physical death on this trusted host and retire its participant " + "after all queries settle (the id is participant_id in /v1/health).", ) r.add_argument("--participant", required=True, help="The participant id to retire") - r.add_argument("--yes", action="store_true", help="Its process is confirmed stopped") r.set_defaults(func=_cmd_retire) diff --git a/server/cli/sql_access.py b/server/cli/sql_access.py index 8499eed5cd..c34dc37f25 100644 --- a/server/cli/sql_access.py +++ b/server/cli/sql_access.py @@ -70,16 +70,24 @@ def _connected( ) -> Any: """Run `work(ch)` on one plain connection: the command calls no model. `orgs` or `org_free` as `server.cli._ch.connect_declared` takes them.""" - from server.cli._ch import connect_declared, disconnect + from server.cli._ch import check_acceptance, connect_verified, disconnect from server.core.ch import get_client + from server.insights.app.provider_command import participant_command async def _go() -> Any: ch = get_client() - await connect_declared(ch, orgs=orgs, org_free=org_free) + await connect_verified(ch) try: - return await work(ch) + async with participant_command(): + await check_acceptance(orgs=orgs, org_free=org_free) + return await work(ch) finally: - await disconnect(ch) + first = sys.exception() + try: + await disconnect(ch) + except BaseException: # a disconnect failure must not replace the command failure + if first is None: + raise return asyncio.run(_go()) diff --git a/server/core/cell.py b/server/core/cell.py index 2aaea0f529..ea9b6ac0bb 100644 --- a/server/core/cell.py +++ b/server/core/cell.py @@ -62,9 +62,9 @@ # Postgres's btree entry limit (about 2.7 KB), so every org a cell records has a # placement the directory can index. ORG_MAX_BYTES = 2048 # not a setting: cells and the router judge one limit -# The advisory lock on a cell's Postgres that attach will hold, and prepare and update once -# they take it, and that the troubleshooting page's full-state cleanup takes before the -# migration lock. Positive: the page's holder query splits it into two `oid` halves. +# The advisory lock on a cell's Postgres that attach and prepare hold for their run, and that +# the troubleshooting page's full-state cleanup takes before the migration lock. Positive: +# the page's holder query splits it into two `oid` halves. ATTACH_LOCK_KEY = 0x6C6F6F6D_61746368 # "loomatch" @@ -272,6 +272,12 @@ def state(self) -> str: _awaiting_first_read = False +def discard_inherited_view() -> None: + """A child enrolls and reads primary state before using acceptance state.""" + global _view, _memo, _awaiting_first_read + _view, _memo, _awaiting_first_read = _View(), None, True + + def expect_first_read() -> None: global _awaiting_first_read _awaiting_first_read = True diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 19cf492b0e..784351d215 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -21,7 +21,7 @@ import dataclasses import logging import re -from typing import Any +from typing import Any, Literal, NamedTuple from server.core import cell, pg, readiness @@ -36,7 +36,9 @@ CELL_FIRST_TRIGGERS = ("cell_first_immutable", "cell_first_no_truncate") # Org-keyed tables the org triggers skip: the cell's own tenancy rows (`tenancy_record` # guards them instead), and the idempotency rows keyed by the caller's token org (a platform -# admin's replay row for its home org is neither refused nor placed). +# admin's replay row for its home org is not refused). Outside the guard is not outside the +# first attach: its org scan (`server/cli/cell_scan.py`) reads every table with an `org` +# column, these included, and places the orgs it finds. EXCLUDED = frozenset({"tenancy", "idempotency", "idempotency_fences"}) # A table's DDL waits this long for its lock a try, LOCK_ATTEMPTS tries LOCK_RETRY_S # apart: a boot never queues behind a long transaction (and every write of the table @@ -77,16 +79,30 @@ # What a trigger's DDL (SHARE ROW EXCLUSIVE) waited behind on a table, read after the # wait failed (pg_blocking_pids is empty by then): the locks held in a mode it conflicts # with (every mode but the two readers take), and the requests queued ahead of it in -# one, whose own blocker heads the queue. +# one, whose own blocker heads the queue. An anti-wraparound autovacuum, which never yields +# its lock, says so in its activity line (compared here, never returned). _HOLDERS = """ SELECT DISTINCT l.pid, l.granted, a.backend_type, a.state, a.application_name, - a.xact_start, a.wait_event + a.xact_start, a.wait_event, + coalesce(a.backend_type = 'autovacuum worker' + AND a.query LIKE '%(to prevent wraparound)', false) AS wraparound FROM pg_locks l JOIN pg_stat_activity a ON a.pid = l.pid WHERE l.relation = $1::regclass AND l.pid <> pg_backend_pid() AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database()) AND l.mode NOT IN ('AccessShareLock', 'RowShareLock') """ +# What a table's DDL came to (`ensure_table`, `_run_ddl`). +Applied = Literal["applied", "busy", "denied"] + + +class Outcome(NamedTuple): + """A table's DDL's answer, and with `busy` the sessions it waited behind.""" + + answer: Applied + holders: str = "" + + # pg_trigger.tgtype bits (src/include/catalog/pg_trigger.h). _ROW, _BEFORE, _INSERT, _UPDATE, _INSTEAD = 1, 2, 4, 16, 64 # The events each trigger must fire on. @@ -168,6 +184,11 @@ def _function_sql(function: str) -> re.Pattern[str]: ) +def create_statement(table: str, name: str) -> str: + """The CREATE TRIGGER for `name` on `loom.
`, as `ensure` writes it.""" + return _create(f"loom.{_ident(table)}", name) + + def _create(qualified: str, name: str) -> str: """The CREATE TRIGGER for `name` on the table `qualified` names.""" if name == TENANCY_TRIGGER: @@ -208,79 +229,160 @@ async def ensure(conn: Any) -> None: tries; a table whose lock it cannot take is logged with the holders of its locks, one the role cannot alter is logged as such, and either is skipped (the boot serves). A failed read of the state or the catalog raises. Runs on the migration session, which - holds the migration lock.""" + holds the migration lock. `ensure_table` runs one table's part alone.""" state = await cell.read_state(conn) if state == "open": return - # A table's statements, in one transaction: one lock wait, and every statement or none. - statements: dict[str, list[str]] = {} first = await _triggers(conn, CELL_FIRST_TRIGGERS) - for name in CELL_FIRST_TRIGGERS: - row = first.get(("cell_first", name)) - if row is not None and row["enabled"] != "A": - statements.setdefault("cell_first", []).append( - f"ALTER TABLE loom.cell_first ENABLE ALWAYS TRIGGER {name}" + always = [ + f"ALTER TABLE loom.cell_first ENABLE ALWAYS TRIGGER {name}" + for name in CELL_FIRST_TRIGGERS + if (row := first.get(("cell_first", name))) is not None and row["enabled"] != "A" + ] + if always: + await _run_ddl(conn, "cell_first", always) + if state != "enforced": + return + tables = await conn.fetch(ORG_TABLES) + for r in tables: + if r["relkind"] == "p": + # No trigger guards it: named here as missing() names it. + log.warning( + "cell triggers: loom.%s is partitioned: unsupported, unguarded", r["relname"] ) - if state == "enforced": - present = await _triggers(conn, (*ORG_TRIGGERS, TENANCY_TRIGGER)) - tables = await conn.fetch(ORG_TABLES) - for r in tables: - if r["relkind"] == "p": - # No trigger guards it: named here as missing() names it. - log.warning( - "cell triggers: loom.%s is partitioned: unsupported, unguarded", r["relname"] - ) - for r, names in _planned(tables): - table = r["relname"] - for name in names: - row = present.get((table, name)) - if row is None: - statements.setdefault(table, []).append(_create(f"loom.{_ident(table)}", name)) - elif row["enabled"] != "O": - statements.setdefault(table, []).append( - f"ALTER TABLE loom.{_ident(table)} ENABLE TRIGGER {name}" - ) - for table, sqls in statements.items(): - await _run_ddl(conn, table, sqls) + present = await _triggers(conn, (*ORG_TRIGGERS, TENANCY_TRIGGER)) + for r, names in _planned(tables): + if sqls := _table_statements(r["relname"], names, present): + await _run_ddl(conn, r["relname"], sqls) + + +async def planned_tables(conn: Any) -> list[str]: + """The tables `ensure_table` gives triggers: the guarded org tables and `loom.tenancy`.""" + return planned_names(await conn.fetch(ORG_TABLES)) + + +def planned_names(rows: list[Any]) -> list[str]: + """`planned_tables` from rows `ORG_TABLES` read.""" + return [r["relname"] for r, _ in _planned(rows)] + + +async def ensure_table( + conn: Any, + table: str, + *, + recording_first: bool = False, + lock_timeout: str = LOCK_TIMEOUT, + attempts: int = LOCK_ATTEMPTS, +) -> Outcome: + """`ensure` for one table of the planned set (`loom.tenancy` its guard, any other its + org triggers), so one table's DDL can run alone: in `enforced`, or with + `recording_first` also in a first cell's `recording`, where the org triggers' function + passes every row and a boot never creates them. `applied` when the DDL ran or nothing + was needed, `busy` when its lock was not taken in `attempts` tries of `lock_timeout`, + `denied` when this role cannot alter the table; a `busy` answer carries the sessions it + waited behind and logs nothing, so the caller says it as often as it retries. Raises + for a table outside the planned set, or in another state.""" + state = await cell.read_state(conn) + if state != "enforced" and not (recording_first and state == "recording"): + raise RuntimeError(f"cell triggers: not created in state {state!r}") + planned = [ + (r, names) for r, names in _planned(await conn.fetch(ORG_TABLES)) if r["relname"] == table + ] + if not planned: + raise ValueError(f"loom.{table} carries no cell triggers") + ((_, names),) = planned + sqls = _table_statements(table, names, await _triggers(conn, names)) + if not sqls: + return Outcome("applied") + return await _run_ddl(conn, table, sqls, lock_timeout, attempts, warn=False) + + +def _table_statements( + table: str, names: tuple[str, ...], present: dict[tuple[str, str], Any] +) -> list[str]: + """The DDL that gives `table` its named triggers, each present and enabled.""" + out = [] + for name in names: + row = present.get((table, name)) + if row is None: + out.append(_create(f"loom.{_ident(table)}", name)) + elif row["enabled"] != "O": + out.append(f"ALTER TABLE loom.{_ident(table)} ENABLE TRIGGER {name}") + return out -async def _run_ddl(conn: Any, table: str, sqls: list[str]) -> None: - """A table's DDL in one transaction, LOCK_ATTEMPTS short tries. The boot serves past - the two causes cells.md names (its lock not taken, a table this role cannot alter); - any other error fails it, as a failed read of the state does.""" +async def _run_ddl( + conn: Any, + table: str, + sqls: list[str], + lock_timeout: str = LOCK_TIMEOUT, + attempts: int = LOCK_ATTEMPTS, + *, + warn: bool = True, +) -> Outcome: + """A table's DDL in one transaction, `attempts` short tries: `applied`, or which of the + two causes cells.md names kept it from applying (`busy`: its lock not taken, with the + sessions it waited behind; `denied`: a table this role cannot alter), which a boot + serves past; any other error fails it, as a failed read of the state does. `warn` logs + a busy table once, after its last try, with its DDL and holders (a boot's `ensure`). + The borrowing caller owns failed-session cleanup, including cancellation/rollback.""" import asyncpg - for attempt in range(LOCK_ATTEMPTS): + for attempt in range(attempts): + rolled_back = False try: - async with conn.transaction(): - await conn.execute(f"SET LOCAL lock_timeout = '{LOCK_TIMEOUT}'") + tx = conn.transaction() + await tx.start() + try: + await conn.execute(f"SET LOCAL lock_timeout = '{lock_timeout}'") for sql in sqls: await conn.execute(sql) - return + except ( + asyncpg.exceptions.InsufficientPrivilegeError, + asyncpg.exceptions.LockNotAvailableError, + ): + await tx.rollback() + rolled_back = True + raise + await tx.commit() + return Outcome("applied") except asyncpg.exceptions.InsufficientPrivilegeError as e: + if not rolled_back: + raise log.warning( "cell triggers: not applied on loom.%s (%s): this role cannot alter it", table, e ) - return + return Outcome("denied") except asyncpg.exceptions.LockNotAvailableError as e: + if not rolled_back: + raise # Short tries outlast a short transaction; an autovacuum is left to finish. - if attempt < LOCK_ATTEMPTS - 1: + if attempt < attempts - 1: await asyncio.sleep(LOCK_RETRY_S) continue - log.warning( - "cell triggers: not applied on loom.%s (%s): %s; %s", - table, - e, - "; ".join(sqls), - await _holders(conn, table), - ) + held = await _holders(conn, table) + if warn: + log.warning( + "cell triggers: not applied on loom.%s (%s): %s; %s", + table, + e, + "; ".join(sqls), + held, + ) + return Outcome("busy", held) + return Outcome("busy") + + +async def holders(conn: Any, table: str) -> list[Any]: + """The sessions a blocked DDL on `table` waits behind (`_HOLDERS`).""" + return list(await conn.fetch(_HOLDERS, f"loom.{_ident(table)}")) async def _holders(conn: Any, table: str) -> str: """The sessions a blocked DDL waited behind, never their query text (a literal in it may be memory content).""" try: - rows = await conn.fetch(_HOLDERS, f"loom.{_ident(table)}") + rows = await holders(conn, table) except Exception as e: # noqa: BLE001 — naming the holders is best effort log.warning("cell triggers: the lock holders of loom.%s not read: %s", table, e) return "lock holders not read" diff --git a/server/core/instances.py b/server/core/instances.py index 27a35d8ec6..4f5263482d 100644 --- a/server/core/instances.py +++ b/server/core/instances.py @@ -53,6 +53,10 @@ _LIVE_INTERVALS = 3 # How often the leader drops the KeeperMap keys of processes gone an hour. _PRUNE_EVERY_S = 600.0 +# How old a row's last beat is before the leader drops its key. A contract across builds, +# not a tunable: attach reads stopped rows back that far (`read_recent`) while the leader +# that sweeps them may run another build, so a change is a design change of its own. +PRUNE_AFTER_S = 3600 # not a setting: a contract across builds # (pid, instance id, start time) of this process. _process: tuple[int, str, datetime] | None = None @@ -130,6 +134,12 @@ def live_window_s() -> float: return _LIVE_INTERVALS * float(settings.instance_heartbeat_seconds) +def boot_allowance_s() -> float: + """How long a booting process takes to report its first cell row and its first router + answer: two beats and one router read.""" + return 2 * float(settings.instance_heartbeat_seconds) + float(settings.cell_router_timeout_s) + + async def beat(ch: Any, *, stopped: bool = False) -> None: """Write this process's row. A KeeperMap insert replaces the key.""" await ch.insert(TABLE, [row(stopped=stopped)]) @@ -139,12 +149,16 @@ def cell_row() -> dict[str, Any]: """What this process reports about its cell (`loom.cell_heartbeats`). Beside the fingerprint's version and fields, its JSON carries keys no fingerprint compares (docs/architecture/cells.md, "The heartbeat's keys"): `serves` (LOOM_SERVES_TRAFFIC), - `marked` (whether this process's pooled sessions keep the tenancy marker, null until a + `participant_id` (this process's registered provider identity), `marked` + (whether this process's pooled sessions keep the tenancy marker, null until a cell-state read lands), and with LOOM_CELL set, until the `first` row is read and then on a first cell only, `router` and `router_why` (`cell_router.router_key`).""" + from server.core.provider_process import process_identity + reported = { **cell.as_value(cell.mine()), "serves": bool(settings.serves_traffic), + "participant_id": str(process_identity()), "marked": cell.marked(), **cell_router.router_key(), } @@ -197,6 +211,36 @@ async def read_live(ch: Any, *, all_deployments: bool = False) -> list[dict[str, return out +async def read_recent( + ch: Any, *, window_s: float, since: str | None = None +) -> list[dict[str, Any]]: + """Every process of every deployment sharing this ClickHouse whose last beat plus + `window_s` reaches past `since` (a ClickHouse `DateTime64(3)` as text), or past now + without one: stopped rows included, which `read_live` drops. `stopped` is its + newest row's flag and `live` whether that row is inside the live window and not + stopped, both judged in ClickHouse.""" + rows = await ch.query_all( + f""" + SELECT instance_id, + argMax(keys_fp, heartbeat_at) AS keys_fp, + argMax(stopped, heartbeat_at) AS stopped, + toString(max(heartbeat_at)) AS last_beat, + max(heartbeat_at) > now64(3) - toIntervalMillisecond({{live:UInt32}}) + AND stopped = 0 AS live + FROM {TABLE} + GROUP BY instance_id + HAVING max(heartbeat_at) + toIntervalMillisecond({{wide:UInt32}}) + > if({{since:String}} = '', now64(3), toDateTime64({{since:String}}, 3)) + """, + params={ + "live": int(live_window_s() * 1000), + "wide": int(window_s * 1000), + "since": since or "", + }, + ) + return [dict(r) for r in rows] + + def live_rows() -> list[dict[str, Any]]: """The live rows as of the last read; none before the first read or once that read is older than the live window (ClickHouse unreachable).""" @@ -224,14 +268,15 @@ async def _prune(ch: Any) -> None: # Each table's sweep stands alone: one failing leaves the other to run. try: await ch.execute( - "ALTER TABLE loom.instances DELETE WHERE heartbeat_at < now64(3) - toIntervalHour(1)" + "ALTER TABLE loom.instances DELETE " + f"WHERE heartbeat_at < now64(3) - toIntervalSecond({PRUNE_AFTER_S})" ) except Exception as e: # noqa: BLE001 — a later sweep retries log.warning("instance row sweep failed: %s", e) try: await ch.execute( "ALTER TABLE loom.cell_heartbeats DELETE " - "WHERE heartbeat_at < now64(3) - toIntervalHour(1)" + f"WHERE heartbeat_at < now64(3) - toIntervalSecond({PRUNE_AFTER_S})" ) except Exception as e: # noqa: BLE001 — a later sweep retries log.warning("cell heartbeat sweep failed: %s", e) @@ -331,8 +376,10 @@ def _reset_for_tests() -> None: __all__ = [ "CELL_TABLE", + "PRUNE_AFTER_S", "TABLE", "beat", + "boot_allowance_s", "cell_beat", "cell_row", "instance_id", @@ -341,6 +388,7 @@ def _reset_for_tests() -> None: "live_window_s", "publish_config", "read_live", + "read_recent", "schema_version", "start", "stop", diff --git a/server/core/pg.py b/server/core/pg.py index 576b1b1e4b..f672c9edde 100644 --- a/server/core/pg.py +++ b/server/core/pg.py @@ -21,6 +21,7 @@ import asyncio import json import logging +import time import weakref from collections.abc import AsyncIterator, Mapping from contextlib import asynccontextmanager, suppress @@ -379,6 +380,65 @@ def _terminate(conn: Any) -> None: conn.terminate() +def _retrieve_task(task: asyncio.Task[Any]) -> None: + if not task.cancelled(): + task.exception() + + +async def close_owned(conn: Any, *, end: float) -> None: + """Close an owned session by an absolute time.monotonic deadline. + + Public close sends cancellation before transport cleanup can discard it. Abort + is a bounded fallback, not evidence that the backend has released its locks. + The owner preserves an earlier operation failure when this cleanup fails. + """ + + async def close() -> SystemExit | KeyboardInterrupt | None: + try: + await conn.close(timeout=max(0, end - time.monotonic())) + except (SystemExit, KeyboardInterrupt) as error: + # A native fatal in a child Task stops the loop before its owner can clean up. + return error + return None + + task: asyncio.Task[Any] | None = None + interrupted: asyncio.CancelledError | None = None + try: + remaining = end - time.monotonic() + if remaining <= 0: + raise TimeoutError("Postgres owned-session cleanup deadline expired") + task = asyncio.create_task(close()) + while True: + remaining = end - time.monotonic() + if remaining <= 0: + raise TimeoutError("Postgres owned-session cleanup deadline expired") + try: + done, _ = await asyncio.wait({task}, timeout=remaining) + except asyncio.CancelledError as error: + # Keep cancellation delivery alive; another signal cannot reset its end. + interrupted = interrupted or error + continue + if not done: + raise TimeoutError("Postgres owned-session cleanup deadline expired") + break + if isinstance(result := task.result(), (SystemExit, KeyboardInterrupt)): + raise result + except BaseException: + try: + _terminate(conn) + except BaseException: # noqa: BLE001 — abort cannot replace the first cleanup failure. + pass + if task is not None: + if not task.done(): + task.cancel() + task.add_done_callback(_retrieve_task) + if interrupted is not None: + raise interrupted from None + raise + if interrupted is not None: + raise interrupted + + async def _release(pool: Any, conn: Any) -> None: """Back to the pool within a bound: a statement still in flight (a client timeout, a cancelled request) has its cancel sent and acknowledged and the connection reset; @@ -430,20 +490,13 @@ def may_have_committed(e: BaseException) -> bool: return isinstance(e, PostgresUnavailableError) and bool(e.extra.get("outcome_unknown")) -@asynccontextmanager -async def dedicated(*, statement_timeout_ms: int = 0, marked: bool = True) -> AsyncIterator[Any]: - """A connection of its own, outside the pool, with no client timeout and no server - statement timeout unless `statement_timeout_ms` asks for one: for work that runs as - long as it takes (migrations, an erasure's deletes). - A CLI process's first one checks the deployment (`expect_deployment`). - `statement_timeout_ms` bounds its statements as a startup value (a RESET ALL keeps - it); `marked` False leaves the tenancy marker out (`loom doctor` reads a session - without it).""" +async def _connect_dedicated(*, statement_timeout_ms: int = 0, marked: bool = True) -> Any: + """Return the raw dedicated session before its owner initializes it.""" import asyncpg _require_dsn() try: - conn = await asyncpg.connect( + return await asyncpg.connect( settings.pg_dsn, command_timeout=None, server_settings={ @@ -460,6 +513,20 @@ async def dedicated(*, statement_timeout_ms: int = 0, marked: bool = True) -> As except Exception as e: _raise_if_unavailable(e) raise + + +@asynccontextmanager +async def dedicated(*, statement_timeout_ms: int = 0, marked: bool = True) -> AsyncIterator[Any]: + """A connection of its own, outside the pool, with no client timeout and no server + statement timeout unless `statement_timeout_ms` asks for one: for work that runs as + long as it takes (migrations, an erasure's deletes). + A CLI process's first one checks the deployment (`expect_deployment`). + `statement_timeout_ms` bounds its statements as a startup value (a RESET ALL keeps + it); `marked` False leaves the tenancy marker out (`loom doctor` reads a session + without it).""" + import asyncpg + + conn = await _connect_dedicated(statement_timeout_ms=statement_timeout_ms, marked=marked) # Opening includes the deployment check and the codecs (jsonb reads back as a dict, as # the pool's): an outage there is the 503; the block's own errors pass through as # they are (an erasure's deadlock reaches its retry). diff --git a/server/core/provider_process.py b/server/core/provider_process.py index 6ef0d6af5a..aa88ffe6fc 100644 --- a/server/core/provider_process.py +++ b/server/core/provider_process.py @@ -5,11 +5,9 @@ still has a query running. """ -import asyncio import os import re import socket -from pathlib import Path from uuid import UUID, uuid4 _identity: tuple[int, UUID] | None = None @@ -98,12 +96,15 @@ def host_proof() -> str: def proves_stopped(proof: str) -> bool: """Whether the process `proof` names has stopped, judged from this one: same host - name and kernel boot, and either its pid is free or runs another process than - the one that started then (same pid namespace), or it ran in this container - before the container restarted into a new pid namespace (which kills every - process of the old one). Two containers that share a host name never prove - each other stopped; a process that cannot be inspected is not proven stopped.""" + name and kernel boot, and its pid is free, runs a different birth, or is a + single-thread zombie of that birth (same pid namespace). Another namespace or API + object proves nothing. Sharing a host name or container identity alone does + not prove death; an uninspectable process is not proven stopped.""" parts = proof.split("|") + if len(parts) == 9: + if parts[8] != LIFECYCLE: + return False + parts = parts[:8] if len(parts) not in (6, 7, 8): return False host, boot, container, pidns, pid, start = parts[:6] @@ -114,8 +115,8 @@ def proves_stopped(proof: str) -> bool: if host != socket.gethostname() or boot != my_boot: return False if pidns != my_pidns: - return bool(container) and container == _container() - # Only positive evidence: no such process, or another one under its pid. + return False # namespace identity alone says nothing about old processes + # Only positive evidence: absence, a different birth, or a fully exited group. # A pid this process may not inspect (hidepid, a permission error) proves nothing. try: os.kill(int(pid), 0) @@ -124,50 +125,47 @@ def proves_stopped(proof: str) -> bool: except (OSError, ValueError): return False try: - return _start_ticks(pid) != start - except (OSError, IndexError): + if _start_ticks(pid) != start: + return True + with open(f"/proc/{pid}/stat", encoding="ascii") as f: + stat = f.read().rsplit(")", 1)[1].split() + # A zombie leader can still have live siblings. Bind all three fields to + # one fresh record: only a single-thread zombie cannot resume this birth. + return stat[19] == start and stat[0] == "Z" and stat[17] == "1" + except (OSError, IndexError, ValueError): return False -_SA = "/var/run/secrets/kubernetes.io/serviceaccount" -_POD_CONTAINER = re.compile(r"^([0-9a-f-]{36})/") +LIFECYCLE = "cell-retirement-v1" -async def pod_gone(proof: str, *, timeout_s: float = 5.0) -> bool: - """Whether Kubernetes says the pod `proof` names is gone: no pod of that name - in its namespace, one with another UID, or this one in phase Failed or - Succeeded (an evicted pod keeps its object; a pod that restarts its - containers reaches those phases only once every one stopped for good). The - kubelet removes a pod's object once its containers stopped (or an operator - force-deleted it), so each is positive evidence a lost pod's process ended. - Asked only of this pod's own namespace, with its service-account token; - False wherever the API or the proof does not say.""" +def certified(proof: str) -> str: + """Preserve the exact captured identity and stamp the installed protocol.""" parts = proof.split("|") - found = _POD_CONTAINER.match(parts[2]) if len(parts) == 8 else None - namespace = os.environ.get("POD_NAMESPACE", "") - if not found or not namespace or parts[6] != namespace or not parts[7]: + if len(parts) == 9 and parts[8] == LIFECYCLE: + parts = parts[:8] + if len(parts) not in (6, 7, 8): + raise ValueError("Unrecognized participant identity proof") + parts += [""] * (8 - len(parts)) + if not all(parts[i] for i in (0, 1, 3, 4, 5)): + raise ValueError("Incomplete participant identity proof") + if not parts[4].isdecimal() or int(parts[4]) <= 0 or not parts[5].isdecimal(): + raise ValueError("Invalid participant process identity") + return "|".join((*parts, LIFECYCLE)) + + +def has_certificate(proof: str) -> bool: + if len(proof.split("|")) != 9: return False - host = os.environ.get("KUBERNETES_SERVICE_HOST", "") - port = os.environ.get("KUBERNETES_SERVICE_PORT", "443") try: - token = (await asyncio.to_thread(Path(f"{_SA}/token").read_text, "ascii")).strip() - except OSError: - return False - if not host or not token: + return certified(proof) == proof + except ValueError: return False - import httpx - url = f"https://{host}:{port}/api/v1/namespaces/{namespace}/pods/{parts[7]}" - try: - async with httpx.AsyncClient(verify=f"{_SA}/ca.crt", timeout=timeout_s) as client: - r = await client.get(url, headers={"Authorization": f"Bearer {token}"}) - except (httpx.HTTPError, OSError): - return False - if r.status_code == 404: - return True - if r.status_code != 200: - return False - pod = r.json() - if str(pod.get("metadata", {}).get("uid", "")) != found.group(1): - return True - return str(pod.get("status", {}).get("phase", "")) in ("Failed", "Succeeded") + +def participant_proof() -> str: + """Advertise capability only in a composition with installed lifecycle guards.""" + from server.core.retirement import get_gate + + proof = host_proof() + return certified(proof) if proof and get_gate().role else proof diff --git a/server/core/retirement.py b/server/core/retirement.py new file mode 100644 index 0000000000..f3dd7fc865 --- /dev/null +++ b/server/core/retirement.py @@ -0,0 +1,124 @@ +"""Permanent process admission and the evidence passed to the retirement writer. + +Model maintenance can reopen. This gate never does: the participant identity is +finished once shutdown begins, even when its shutdown budget expires. +""" + +from __future__ import annotations + +import asyncio +import os +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from dataclasses import dataclass +from typing import Any +from uuid import UUID + +from server.core import provider_process + + +class ProcessStopping(RuntimeError): + """This participant can never accept another unit of work.""" + + +class RetirementGate: + """One event loop; checking admission and incrementing never await.""" + + def __init__(self) -> None: + self.active = 0 + self.closed = False + self.role = "" + self.enrolled = False + self.producers_stopped = False + self.producers: set[asyncio.Task[Any]] = set() + # Native completion needs the first failed producer after its Task is discarded. + # Fresh gates, including fork reset, discard this one command's outcome. + self.producer_failure: BaseException | None = None + + def install(self, role: str) -> None: + if self.closed or self.role not in ("", role): + raise ProcessStopping("A retired participant cannot enroll again") + self.role = role + + def close(self) -> None: + self.closed = True + + def track[T](self, task: asyncio.Task[T]) -> asyncio.Task[T]: + """A process-lifetime producer must actually end, not merely be canceled.""" + self.producers.add(task) + task.add_done_callback(self._producer_done) + if self.closed: + task.cancel() + raise ProcessStopping("Cannot start a producer during shutdown") + return task + + def _producer_done(self, task: asyncio.Task[Any]) -> None: + self.producers.discard(task) + if self.role == "command" and not task.cancelled(): + error = task.exception() + if self.producer_failure is None: + self.producer_failure = error + + def stopped(self) -> None: + """Composition calls this after stopping every role-specific producer.""" + self.close() + self.producers_stopped = True + + @property + def quiescent(self) -> bool: + return ( + self.closed + and bool(self.role) + and self.producers_stopped + and self.active == 0 + and all(task.done() for task in self.producers) + ) + + @asynccontextmanager + async def admit(self) -> AsyncIterator[None]: + if self.closed or (self.role and not self.enrolled): + raise ProcessStopping("This process is stopping or has not enrolled") + self.active += 1 + try: + yield + finally: + self.active -= 1 + + +_pid = os.getpid() +_gate = RetirementGate() + + +def get_gate() -> RetirementGate: + """A fork cannot inherit an accepting view or its parent's retirement state.""" + global _pid, _gate + if _pid != os.getpid(): + role = _gate.role + _pid, _gate = os.getpid(), RetirementGate() + _gate.role = role + return _gate + + +@dataclass(frozen=True) +class RetirementProof: + """Internal evidence; only guarded shutdown or verified death constructs it.""" + + participant: UUID + host_proof: str + after: int + version: int | None = None + + +def guarded_proof(participant: UUID, *, version: int) -> RetirementProof: + gate = get_gate() + proof = provider_process.host_proof() + if not gate.quiescent or not gate.enrolled or not proof: + raise ProcessStopping("Permanent guarded quiescence is not established") + return RetirementProof(participant, provider_process.certified(proof), version - 1, version) + + +def death_proof(participant: UUID, proof: str, *, after: int) -> RetirementProof | None: + """Called only after the complete cross-replica running-query census is zero.""" + if not provider_process.proves_stopped(proof): + return None + return RetirementProof(participant, provider_process.certified(proof), after) diff --git a/server/core/schema/pg_migrate.py b/server/core/schema/pg_migrate.py index 7876c9f178..4a0fbdf6bf 100644 --- a/server/core/schema/pg_migrate.py +++ b/server/core/schema/pg_migrate.py @@ -1,6 +1,7 @@ """Postgres migrations: numbered SQL files in `schemas/pg_migrations/`, applied in -order at boot under a session advisory lock, each file in one transaction with -its ledger row (`loom.pg_migrations`, the file's SHA-256). The router's directory +order at boot under a session advisory lock, each ordinary file in a transaction +with its ledger row (`loom.pg_migrations`, the file's SHA-256). Declared online indexes +run concurrently outside transactions under a separate session key. The router's directory (`schemas/router_migrations/`, `server/router/directory.py`) migrates the same way, on its own database, ledger schema and lock. For schema `loom`, `migrate` also marks its session as one that ran the tenancy check and, after the files and before the lock @@ -8,23 +9,27 @@ The rules are the ClickHouse journal's (docs/development/migrations.md): additive only, never edit a merged file. A ledger row whose checksum no longer matches its -file refuses the boot, as does a ledger row for a file this build does not carry -when that file's number is above the build's last one (a newer build's schema is -served, as ClickHouse's is; an unknown lower number means the journal was edited). +file refuses the boot, as does an unknown file numbered at or below this build's +last migration. A build serves a newer schema, warning about unknown files numbered +above its last migration. """ from __future__ import annotations +import asyncio import hashlib import logging +import time from asyncio import sleep -from collections.abc import Callable +from collections.abc import Awaitable, Callable from contextlib import AbstractAsyncContextManager from pathlib import Path from typing import Any from server.core import pg +from server.core.ch import set_query_tag from server.core.paths import PG_MIGRATIONS_DIR +from server.core.schema import pg_online_index from server.core.schema.journal import migration_number, sorted_migrations log = logging.getLogger(__name__) @@ -104,33 +109,201 @@ async def migrate( bounded (`lock_timeout`), so an `ALTER` queued behind a long transaction never holds every read and write of the table behind it; the file is retried. For schema `loom` it marks the session and runs `cell_triggers.ensure` before the lock goes.""" + files, indexes = _validated(mig_dir, schema) + cm = (connect or pg.dedicated)() + conn = await cm.__aenter__() + task = asyncio.create_task( + _owned_result(lambda: _migrate(conn, files, indexes, schema, lock_key)) + ) + try: + result = await asyncio.shield(task) + if isinstance(result, (SystemExit, KeyboardInterrupt)): + raise result + except BaseException as error: # Keep the migration failure through owned cleanup. + end = time.monotonic() + pg._RELEASE_TIMEOUT_S + await _failed_migration(cm, conn, task, error, end) + raise + await cm.__aexit__(None, None, None) + return result + + +async def _owned_result[T](work: Callable[[], Awaitable[T]]) -> T | SystemExit | KeyboardInterrupt: + try: + return await work() + except (SystemExit, KeyboardInterrupt) as error: + # Transport native failures through child Tasks without stopping their supervisor. + return error + + +async def _failed_migration( + cm: AbstractAsyncContextManager[Any], + conn: Any, + task: asyncio.Task[Any], + error: BaseException, + end: float, +) -> None: + try: + await pg.close_owned(conn, end=end) + except BaseException: # noqa: BLE001 — preserve the migration failure/cancellation. + pass + if not task.done(): + task.cancel() + task.add_done_callback(pg._retrieve_task) + try: + await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) + except BaseException: # noqa: BLE001 — preserve the first failure on repeated cancellation. + try: + pg._terminate(conn) + except BaseException: # noqa: BLE001 — fallback abort cannot replace the migration failure. + pass + release = asyncio.create_task( + _owned_result(lambda: cm.__aexit__(type(error), error, error.__traceback__)) + ) + try: + done, _ = await asyncio.wait({release}, timeout=max(0, end - time.monotonic())) + if not done: + raise TimeoutError("migration context cleanup deadline expired") + release.result() + except BaseException: # noqa: BLE001 — a context exit cannot replace the migration failure. + try: + pg._terminate(conn) + except BaseException: # noqa: BLE001 — fallback abort cannot replace the migration failure. + pass + finally: + if not release.done(): + release.cancel() + release.add_done_callback(pg._retrieve_task) + + +def _validated( + mig_dir: Path, schema: str +) -> tuple[list[Path], dict[str, pg_online_index.OnlineIndex | None]]: files = migration_files(mig_dir) - done: list[str] = [] - # Closing the connection ends the session, and with it the advisory lock. - async with (connect or pg.dedicated)() as conn: - # Another boot migrating holds the lock for as long as its files take. - await conn.execute("SELECT pg_advisory_lock($1)", lock_key) - if schema == "loom": - # A migration's data steps only copy or rewrite rows already stored here: the - # org triggers pass them (server/core/cell_triggers.py). - await conn.execute(f"SET {pg.TENANCY_MARKER} = 'on'") - await conn.execute("SET lock_timeout = '5s'") - await conn.execute(_ledger(schema)) - rows = await conn.fetch(f"SELECT filename, checksum FROM {schema}.pg_migrations") - for name, sql, digest in _read(plan({r["filename"]: r["checksum"] for r in rows}, files)): - await _apply(conn, name, sql, digest, schema=schema) - log.info("applied Postgres migration %s", name) - done.append(name) - if schema == "loom": - # On every migrating boot, after the pending files and before the lock goes. - from server.core import cell_triggers - - await cell_triggers.ensure(conn) - return done + # Validate every online declaration before even creating the ledger. + indexes = {f.name: pg_online_index.parse(f.read_text()) for f in files} + if any(index is not None and index.schema != schema for index in indexes.values()): + raise ValueError("online-index target differs from its ledger schema") + return files, indexes + +async def migrate_borrowed( + conn: Any, + mig_dir: Path = PG_MIGRATIONS_DIR, + *, + schema: str = "loom", + lock_key: int = _LOCK_KEY, +) -> list[str]: + """Apply on the owner's session; its supervisor owns all failure cleanup. -def _read(files: list[Path]) -> list[tuple[str, str, str]]: - return [(f.name, f.read_text(), checksum(f)) for f in files] + The caller retains its operation deadline and must close the session before + cancelling this work. Normal completion releases only the migration key. + """ + files, indexes = _validated(mig_dir, schema) + return await _migrate(conn, files, indexes, schema, lock_key) + + +async def _lock(conn: Any, key: int) -> None: + while True: + set_query_tag("migrate.postgres.lock") + if await conn.fetchval("SELECT pg_try_advisory_lock($1)", key): + return + # Each try ends its statement/snapshot before another attempt. + await sleep(0.1) + + +async def _unlock(conn: Any, key: int) -> None: + if not await conn.fetchval("SELECT pg_advisory_unlock($1)", key): + raise RuntimeError("Postgres migration advisory lock was lost") + + +def _online_key(schema: str, lock_key: int) -> int: + value = hashlib.sha256(f"loom.online-index:{schema}:{lock_key}".encode()).digest() + key = int.from_bytes(value[:8], "big") & (2**63 - 1) + if key == lock_key: + raise ValueError("Postgres migration lock keys collide") + return key + + +async def _applied(conn: Any, schema: str, files: list[Path]) -> dict[str, str]: + rows = await conn.fetch(f"SELECT filename, checksum FROM {schema}.pg_migrations") + applied = {r["filename"]: r["checksum"] for r in rows} + plan(applied, files) + return applied + + +async def _record_online(conn: Any, path: Path, schema: str, *, pending: bool) -> None: + async with conn.transaction(): + if pending: + await conn.execute( + f"INSERT INTO {schema}.pg_migrations (filename, checksum) VALUES ($1, $2)", + path.name, + checksum(path), + ) + # A visible ledger from a cancelled synchronous commit needs a fresh flush. + await conn.execute(pg._DURABLE_MESSAGE) + + +async def _online( + conn: Any, + path: Path, + index: pg_online_index.OnlineIndex, + schema: str, + lock_key: int, + files: list[Path], +) -> bool: + await _unlock(conn, lock_key) + online_key = _online_key(schema, lock_key) + await _lock(conn, online_key) + applied = await _applied(conn, schema, files) + await pg_online_index.build(conn, index, recorded=path.name in applied) + # A shipped blocking migrator may wait here; the concurrent build is over. + await _lock(conn, lock_key) + applied = await _applied(conn, schema, files) + pending = path.name not in applied + await _record_online(conn, path, schema, pending=pending) + await _unlock(conn, online_key) + return pending + + +async def _migrate( + conn: Any, + files: list[Path], + indexes: dict[str, pg_online_index.OnlineIndex | None], + schema: str, + lock_key: int, +) -> list[str]: + done: list[str] = [] + await _lock(conn, lock_key) + if schema == "loom": + await conn.execute(f"SET {pg.TENANCY_MARKER} = 'on'") + await conn.execute("SET lock_timeout = '5s'") + await conn.execute("SET synchronous_commit = 'on'") + await conn.execute(_ledger(schema)) + applied = await _applied(conn, schema, files) + for path in files: + set_query_tag("migrate.postgres.apply") + index = indexes[path.name] + if index is not None: + changed = await _online(conn, path, index, schema, lock_key, files) + applied = await _applied(conn, schema, files) + else: + changed = path.name not in applied + if changed: + await _apply(conn, path.name, path.read_text(), checksum(path), schema=schema) + if changed: + log.info("applied Postgres migration %s", path.name) + done.append(path.name) + for index in indexes.values(): + set_query_tag("migrate.postgres.verify") + if index is not None and await pg_online_index.ready(conn, index) is not True: + raise ValueError(f"recorded online-index {index.qualified} is absent or invalid") + if schema == "loom": + from server.core import cell_triggers + + await cell_triggers.ensure(conn) + # A caller may lend its marked session and retain unrelated session keys. + await _unlock(conn, lock_key) + return done async def _apply(conn: Any, name: str, sql: str, digest: str, *, schema: str = "loom") -> None: diff --git a/server/core/schema/pg_online_index.py b/server/core/schema/pg_online_index.py new file mode 100644 index 0000000000..47c2b025a5 --- /dev/null +++ b/server/core/schema/pg_online_index.py @@ -0,0 +1,125 @@ +"""The one nontransactional Postgres migration form: a declared concurrent index.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any + +_ID = r"[a-z_][a-z0-9_]{0,62}" +_DECLARATION = re.compile( + rf"-- loom:online-index (?P{_ID})\.(?P{_ID}) ON " + rf"(?P{_ID})\.(?P
{_ID}) \((?P{_ID}(?:, {_ID})*)\)\n" +) +_CREATE = re.compile( + rf"CREATE\s+INDEX\s+CONCURRENTLY\s+(?P{_ID})\s+ON\s+" + rf"(?P{_ID})\.(?P
{_ID})\s+USING\s+btree\s*" + rf'\((?P{_ID}\s+COLLATE\s+"C"(?:\s*,\s*{_ID}\s+COLLATE\s+"C")*)\)\s*;\s*' +) + + +@dataclass(frozen=True) +class OnlineIndex: + schema: str + name: str + table: str + columns: tuple[str, ...] + sql: str + + @property + def qualified(self) -> str: + return f'"{self.schema}"."{self.name}"' + + +def parse(sql: str) -> OnlineIndex | None: + """Refuse malformed directives before a connection or ledger write is made.""" + if "loom:online-index" not in sql: + return None + declaration = _DECLARATION.match(sql) + create = _CREATE.fullmatch(sql[declaration.end() :]) if declaration else None + if declaration is None or create is None: + raise ValueError("invalid online-index migration declaration or SQL") + columns = tuple(declaration["columns"].split(", ")) + actual = tuple(part.split()[0] for part in create["columns"].split(",")) + if ( + declaration["schema"] != declaration["table_schema"] + or any(declaration[key] != create[key] for key in ("schema", "name", "table")) + or columns != actual + or len(set(columns)) != len(columns) + ): + raise ValueError("online-index SQL does not match its declaration") + return OnlineIndex( + declaration["schema"], + declaration["name"], + declaration["table"], + columns, + sql[declaration.end() :], + ) + + +_CATALOG = """ +SELECT c.relkind::text AS relkind, tn.nspname AS table_schema, t.relname AS table_name, + am.amname, i.indisvalid, i.indisready, i.indislive, + NOT i.indisunique AND NOT i.indisexclusion AND i.indpred IS NULL + AND i.indexprs IS NULL AND i.indnkeyatts = i.indnatts + AND NOT i.indisreplident AND c.reloptions IS NULL AS plain, + ARRAY(SELECT a.attname::text FROM unnest(i.indkey) WITH ORDINALITY k(attnum, n) + JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = k.attnum + ORDER BY k.n) AS columns, + ARRAY(SELECT co.collname::text FROM unnest(i.indcollation) WITH ORDINALITY k(oid, n) + JOIN pg_collation co ON co.oid = k.oid + JOIN pg_namespace ns ON ns.oid = co.collnamespace AND ns.nspname = 'pg_catalog' + ORDER BY k.n) AS collations, + ARRAY(SELECT v FROM unnest(i.indoption) v) AS options, + ARRAY(SELECT op.opcdefault FROM unnest(i.indclass) WITH ORDINALITY k(oid, n) + JOIN pg_opclass op ON op.oid = k.oid ORDER BY k.n) AS default_opclasses, + EXISTS(SELECT 1 FROM pg_constraint WHERE conindid = c.oid) + OR EXISTS(SELECT 1 FROM pg_depend WHERE classid = 'pg_class'::regclass + AND objid = c.oid AND deptype IN ('e', 'x')) + OR EXISTS(SELECT 1 FROM pg_inherits WHERE inhrelid = c.oid) AS dependent +FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace +LEFT JOIN pg_index i ON i.indexrelid = c.oid +LEFT JOIN pg_class t ON t.oid = i.indrelid +LEFT JOIN pg_namespace tn ON tn.oid = t.relnamespace +LEFT JOIN pg_am am ON am.oid = c.relam +WHERE n.nspname = $1 AND c.relname = $2 +""" + + +async def ready(conn: Any, index: OnlineIndex) -> bool | None: + """Absent, matching incomplete, or matching ready; foreign objects always refuse.""" + row = await conn.fetchrow(_CATALOG, index.schema, index.name) + if row is None: + return None + expected = { + "relkind": "i", + "table_schema": index.schema, + "table_name": index.table, + "amname": "btree", + "plain": True, + "columns": list(index.columns), + "collations": ["C"] * len(index.columns), + "options": [0] * len(index.columns), + "default_opclasses": [True] * len(index.columns), + "dependent": False, + } + differences = {key: row[key] for key, value in expected.items() if row[key] != value} + if differences: + raise ValueError( + f"online-index {index.qualified} has a different definition or dependency: " + f"{differences}" + ) + return bool(row["indisvalid"] and row["indisready"] and row["indislive"]) + + +async def build(conn: Any, index: OnlineIndex, *, recorded: bool) -> None: + state = await ready(conn, index) + if recorded and state is not True: + raise ValueError(f"recorded online-index {index.qualified} is absent or invalid") + if state is True: + return + if state is False: + await conn.execute(f"DROP INDEX CONCURRENTLY {index.qualified}") + await conn.execute(index.sql) + if await ready(conn, index) is not True: + raise ValueError(f"online-index {index.qualified} did not become valid and ready") diff --git a/server/insights/app/provider_barrier.py b/server/insights/app/provider_barrier.py index 503cd021e2..8399587f0c 100644 --- a/server/insights/app/provider_barrier.py +++ b/server/insights/app/provider_barrier.py @@ -11,6 +11,7 @@ import asyncio import contextlib import logging +import os import threading from collections.abc import AsyncIterator, Callable from contextlib import AbstractAsyncContextManager, asynccontextmanager @@ -18,10 +19,11 @@ from typing import Any from uuid import UUID +from server.core import retirement from server.core.background import drain_task_sets, flush_registered_buffers from server.core.ch import query_tag from server.core.model_work_gate import get_model_work_gate -from server.core.provider_process import host_proof, pod_gone, process_identity, proves_stopped +from server.core.provider_process import participant_proof, process_identity, proves_stopped from server.insights.store.provider_participants import ( OWN_VERSION_SQL, SEQUENTIAL, @@ -30,6 +32,7 @@ participants, prune_retired, record_participant, + record_retirement, running_queries, ) @@ -62,6 +65,8 @@ def stamp( ) -> tuple[UUID, str, int]: with self.lock: if transition is not None and state is not None: + if self.state == "retired" and state != "retired": + raise retirement.ProcessStopping("A retired UUID cannot re-enroll") self.transition, self.state = transition, state self.version = max(time_ns(), self.version + 1, floor + 1) return self.transition, self.state, self.version @@ -109,7 +114,7 @@ def beat(self) -> None: transition, state, version = _OWN.stamp(floor=seen) if state == "retired": return - row = [me, transition, state, version, STALE_AFTER_S, host_proof()] + row = [me, transition, state, version, STALE_AFTER_S, participant_proof()] self.ch.insert(TABLE, [row], column_names=_COLUMNS, settings=_TAG) def _run(self) -> None: @@ -146,6 +151,7 @@ def signal_stop(self) -> None: _HEARTBEAT = _Heartbeat() _registered = False +_process_pid = os.getpid() async def register_process() -> None: @@ -155,11 +161,20 @@ async def register_process() -> None: startup loads the pending gate or the committed replacement before admission. A second call (boot enrolls before its migrations) keeps the state. """ - global _registered + global _registered, _process_pid, _OWN, _HEARTBEAT + if _process_pid != os.getpid(): + _process_pid = os.getpid() + _registered, _OWN, _HEARTBEAT = False, _Own(), _Heartbeat() + from server.core import cell + + cell.discard_inherited_view() + if retirement.get_gate().closed: + raise retirement.ProcessStopping("A retired participant cannot re-enroll") if not _registered: _OWN.stamp(UUID(int=0), "running") await asyncio.to_thread(_HEARTBEAT.start) _registered = True + retirement.get_gate().enrolled = True async def enroll_before_migrations(ch: Any) -> None: @@ -216,18 +231,24 @@ async def distributed_drain(transition: UUID) -> bool: async def retire_process() -> bool: - """Retire only when no admitted or registered deferred writer remains, and no - write of this process can still land on the server.""" - global _registered - if get_model_work_gate().active or not await drain_task_sets(timeout_s=0): + """Permanently close admission; certify only actually completed role work.""" + gate = retirement.get_gate() + gate.close() + _HEARTBEAT.signal_stop() + if not gate.quiescent or get_model_work_gate().active: + return False + if not await drain_task_sets(timeout_s=0): + return False + if not await flush_registered_buffers() or not await drain_task_sets(timeout_s=0): return False if not await _server_side_settled(): return False - _HEARTBEAT.signal_stop() + if _HEARTBEAT.running or await running_queries(str(process_identity())) != 0: + return False _, _, version = _OWN.stamp(UUID(int=0), "retired") + proof = retirement.guarded_proof(process_identity(), version=version) with query_tag(_TAG["log_comment"]): - await record_participant(process_identity(), UUID(int=0), "retired", version=version) - _registered = False + await record_retirement(proof) return True @@ -243,20 +264,13 @@ async def retire_at_shutdown(ch: Any, *, budget_s: float = SHUTDOWN_RETIRE_S) -> Every attempt, settle and pause ends by `budget_s` from now.""" from server.core import rmt_carry - deadline = monotonic() + budget_s try: - async with asyncio.timeout(budget_s): - while True: - if await retire_process(): - return True - with contextlib.suppress(Exception): - await rmt_carry.settle(ch) - await asyncio.sleep(min(1.0, max(0.0, deadline - monotonic()))) + return await _retire_until(ch, monotonic() + budget_s) except TimeoutError: log.warning( "provider participant not retired at shutdown: %d carried write(s) unsettled " "or an insert of this process still running; retire it with " - "`loom participant retire --participant %s --yes` once it stopped", + "`loom participant retire --participant %s` on its trusted host once it stopped", rmt_carry.unsettled(), process_identity(), ) @@ -265,10 +279,23 @@ async def retire_at_shutdown(ch: Any, *, budget_s: float = SHUTDOWN_RETIRE_S) -> return False +async def _retire_until(ch: Any, deadline: float) -> bool: + """Retry the existing settlement checks under the caller's absolute deadline.""" + from server.core import rmt_carry + + async with asyncio.timeout_at(deadline): + while True: + if await retire_process(): + return True + with contextlib.suppress(Exception): + await rmt_carry.settle(ch) + await asyncio.sleep(min(1.0, max(0.0, deadline - monotonic()))) + + def participant_lifespan[T]( lifespan: Callable[[T], AbstractAsyncContextManager[None]], ) -> Callable[[T], AbstractAsyncContextManager[None]]: - """A boot that fails after enrolling retires its row instead of leaving one per attempt.""" + """A failed boot stops known producers but never certifies an incomplete shutdown.""" @asynccontextmanager async def wrapped(app: T) -> AsyncIterator[None]: @@ -279,6 +306,10 @@ async def wrapped(app: T) -> AsyncIterator[None]: yield except BaseException: if not started and _registered: + gate = retirement.get_gate() + gate.close() + for producer in tuple(gate.producers): + producer.cancel() try: await retire_process() except Exception as e: # noqa: BLE001 — the row expires by heartbeat instead @@ -288,22 +319,45 @@ async def wrapped(app: T) -> AsyncIterator[None]: return wrapped -async def retire_participant(participant: UUID, *, ch: Any = None) -> str: - """`retired`, `absent` (no live row), or `live`: its heartbeat is fresh or a - query tagged with its id still runs.""" - row = next( - (r for r in await participants(ch=ch) if str(r["participant_id"]) == str(participant)), - None, +async def retire_reaped_process(host_proof: str) -> None: + """Retire only the one recorded process matching its owner's captured birth.""" + rows = await participants(host_proof=host_proof) + if len(rows) != 1 or rows[0].get("host_proof") != host_proof: + return + await retire_participant(UUID(str(rows[0]["participant_id"])), expected_proof=host_proof) + + +async def retire_participant( + participant: UUID, *, ch: Any = None, expected_proof: str | None = None +) -> str: + """`retired`, `absent` (no row), or `live` (not proved stopped). + + Retirement needs the recorded identity proved absent in the same host, boot + and PID namespace, with tagged queries known zero. Unknown evidence blocks; + heartbeat expiry alone proves nothing. + """ + rows = ( + await participants(ch=ch) + if expected_proof is None + else await participants(ch=ch, host_proof=expected_proof) ) + row = next((r for r in rows if str(r["participant_id"]) == str(participant)), None) if row is None: return "absent" + if expected_proof is not None and (len(rows) != 1 or row.get("host_proof") != expected_proof): + return "live" if str(participant) == str(process_identity()): return "live" - if not row.get("expired"): + if not proves_stopped(str(row.get("host_proof") or "")): return "live" if await running_queries(participant, ch=ch) != 0: return "live" - await record_participant(participant, UUID(int=0), "retired", after=int(row["version"]), ch=ch) + proof = retirement.death_proof( + participant, str(row.get("host_proof") or ""), after=int(row["version"]) + ) + if proof is None: + return "live" + await record_retirement(proof, ch=ch) return "retired" @@ -312,9 +366,10 @@ async def retire_participant(participant: UUID, *, ch: Any = None) -> str: async def retire_dead_here(*, now: float | None = None) -> int: - """Retire the participants this host proves stopped (`proves_stopped`: a - restarted container or a respawned worker) with no query of theirs left - running. Every process's job, every `HERE_S`; how many it retired.""" + """Retire recorded process-start identities proved absent in this host, + boot and PID namespace, with tagged queries known zero. Unknown evidence + blocks. Run at most every `HERE_S`; return how many were retired. + """ global _last_here t = monotonic() if now is None else now if t - _last_here < HERE_S: @@ -328,9 +383,12 @@ async def retire_dead_here(*, now: float | None = None) -> int: continue if await running_queries(participant) != 0: continue - await record_participant( - UUID(participant), UUID(int=0), "retired", after=int(row["version"]) + proof = retirement.death_proof( + UUID(participant), str(row.get("host_proof") or ""), after=int(row["version"]) ) + if proof is None: + continue + await record_retirement(proof) log.info("provider participant %s retired: its process on this host is gone", participant) retired += 1 return retired @@ -348,10 +406,7 @@ def stale_participants() -> tuple[dict[str, str], ...] | None: async def sweep_stopped(*, now: float | None = None) -> None: - """Retire the expired participants with no query left whose Kubernetes pod is - gone (`pod_gone`), and remember the rest that block a reconfiguration (only - their process, Kubernetes or an operator can say they stopped). The workers - leader's job.""" + """Remember stale blockers; only exact verified death authorizes retirement.""" global _STALE, _last_sweep, _last_prune t = monotonic() if now is None else now if _STALE is not None and t - _last_sweep < SWEEP_S: @@ -369,11 +424,15 @@ async def sweep_stopped(*, now: float | None = None) -> None: entry = {"participant_id": participant, "heartbeat_at": str(row.get("heartbeat_at", ""))} if row.get("expired"): running = await running_queries(participant) != 0 - if not running and await pod_gone(str(row.get("host_proof") or "")): - await record_participant( - UUID(participant), UUID(int=0), "retired", after=int(row["version"]) + proof = ( + None + if running + else retirement.death_proof( + UUID(participant), str(row.get("host_proof") or ""), after=int(row["version"]) ) - log.info("provider participant %s retired: Kubernetes deleted its pod", participant) + ) + if proof is not None: + await record_retirement(proof) continue stale.append({**entry, "reason": "queries_running" if running else "heartbeat_expired"}) _STALE = tuple(stale) diff --git a/server/insights/app/provider_command.py b/server/insights/app/provider_command.py index 46ec2bb9a5..6c91b93f0a 100644 --- a/server/insights/app/provider_command.py +++ b/server/insights/app/provider_command.py @@ -3,14 +3,15 @@ import asyncio import contextlib import logging +import sys from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import asynccontextmanager -from server.core import settings_overrides +from server.core import retirement, settings_overrides from server.core.background import drain_task_sets, flush_registered_buffers -from server.core.ch import set_query_tag +from server.core.ch import get_client, set_query_tag from server.core.model_work_gate import get_model_work_gate -from server.insights.app.provider_barrier import register_process, retire_process +from server.insights.app.provider_barrier import _retire_until, register_process from server.insights.app.provider_runtime import load_active_configuration from server.settings import settings @@ -32,31 +33,125 @@ async def _adopt_overrides(*, failing: bool = False) -> bool: async def _keep_overrides( loaded: bool, sleep: Callable[[float], Awaitable[None]] = asyncio.sleep -) -> None: +) -> SystemExit | KeyboardInterrupt | None: """A long command (a rechunk, a re-embed) follows an override set while it runs.""" set_query_tag("cli.settings_overrides") - while True: - await sleep(max(1.0, float(settings.runtime_overrides_refresh_seconds))) - loaded = await _adopt_overrides(failing=not loaded) + try: + while True: + await sleep(max(1.0, float(settings.runtime_overrides_refresh_seconds))) + loaded = await _adopt_overrides(failing=not loaded) + except (SystemExit, KeyboardInterrupt) as error: + # A task's fatal exception must reach its owner before the loop stops. + return error + + +async def _drain_participant_until(end: float) -> None: + loop = asyncio.get_running_loop() + async with asyncio.timeout_at(end): + if not await drain_task_sets(timeout_s=max(0.0, end - loop.time())): + raise RuntimeError("Native command deferred writes did not drain") + if not await flush_registered_buffers(): + raise RuntimeError("Native command buffers did not flush") + if await _retire_until(get_client(), end) is not True: + raise RuntimeError("Native command retirement was not certified") + + +async def _finish_participant() -> None: + gate = retirement.get_gate() + gate.close() + from server.memory.app import embedder_calibration + + loop = asyncio.get_running_loop() + end = loop.time() + settings.native_command_drain_timeout_s + first = gate.producer_failure + stopped = False + try: + embedder_calibration.stop_refresh_loop() + stopped = True + except BaseException as error: # noqa: BLE001 — finish safe cleanup, then raise the first failure + if first is None: + first = error + producers = tuple(gate.producers) + for producer in producers: + producer.cancel() + try: + if producers: + _, pending = await asyncio.wait(producers, timeout=max(0.0, end - loop.time())) + if pending: + raise RuntimeError("Native command producers did not stop") + except BaseException as error: # noqa: BLE001 — finish safe cleanup, then raise the first failure + if first is None: + first = error + stopped = False + if first is None: + first = gate.producer_failure + if stopped: + gate.stopped() + owner = asyncio.current_task() + while True: + cancelling = owner.cancelling() if owner is not None else 0 + try: + if loop.time() >= end: + raise TimeoutError("Native command cleanup deadline expired") + await _drain_participant_until(end) + except asyncio.CancelledError as error: + if first is None: + first = error + if owner is not None and owner.cancelling() > cancelling: + continue # a new owner cancellation still allows cleanup within the same end + except BaseException as error: # noqa: BLE001 — report the first required cleanup failure + if first is None: + first = error + break + if first is not None: + raise first @asynccontextmanager -async def model_command() -> AsyncIterator[None]: - """A running command blocks a rebuild; a newborn loads the gate first. The - deployment's live-setting overrides apply before the work starts, as they do - in every server process, so a native command and the HTTP op agree.""" +async def participant_command() -> AsyncIterator[None]: + """Register all native org/principal work, including commands without models.""" + retirement.get_gate().install("command") await register_process() - keeper: asyncio.Task[None] | None = None try: - await load_active_configuration() - loaded = await _adopt_overrides() - keeper = asyncio.create_task(_keep_overrides(loaded), name="loom-cli-settings-overrides") - async with get_model_work_gate().admit(): + async with retirement.get_gate().admit(): yield - finally: - if keeper is not None: - keeper.cancel() - with contextlib.suppress(asyncio.CancelledError): - await keeper - if await drain_task_sets(timeout_s=15) and await flush_registered_buffers(): - await retire_process() + except BaseException: # Keep the body failure while attempting required retirement cleanup. + with contextlib.suppress(BaseException): + await _finish_participant() + raise + else: + await _finish_participant() + + +@asynccontextmanager +async def model_command() -> AsyncIterator[None]: + """The permanent lifecycle also holds the reversible model-work admission.""" + async with participant_command(): + keeper: asyncio.Task[SystemExit | KeyboardInterrupt | None] | None = None + try: + await load_active_configuration() + loaded = await _adopt_overrides() + keeper = asyncio.create_task( + _keep_overrides(loaded), name="loom-cli-settings-overrides" + ) + async with get_model_work_gate().admit(): + yield + finally: + first = sys.exception() + if keeper is not None: + owner = asyncio.current_task() + assert owner is not None + cancelling = owner.cancelling() + requested = keeper.cancel() + try: + try: + fatal = await keeper + except asyncio.CancelledError: + if not requested or owner.cancelling() != cancelling: + raise + else: + if fatal is not None: + raise fatal + except BaseException: # Keep the body's first failure through keeper cleanup. + if first is None: + raise diff --git a/server/insights/store/provider_participants.py b/server/insights/store/provider_participants.py index 256a40babb..e03c5b22bf 100644 --- a/server/insights/store/provider_participants.py +++ b/server/insights/store/provider_participants.py @@ -6,20 +6,38 @@ from server.core.ch import sequential_reads from server.core.ch_errors import error_code -from server.core.provider_process import host_proof, participant_tag, process_identity +from server.core.provider_process import ( + has_certificate, + participant_proof, + participant_tag, + process_identity, +) from server.core.repo_base import _CHClient, _default_client +from server.core.retirement import RetirementProof, get_gate from server.settings import settings TABLE = "loom._provider_participants" -# A participant whose last heartbeat is older than this may count as stopped. +# Heartbeat age is diagnostic; it never establishes permanent retirement. STALE_AFTER_S = 60 +# This conservative certificate subset never hides unknown retirement evidence. +# Bounded ASCII PIDs also exclude proofs the Python integer parser must refuse. +_CERTIFIED_RETIRED = ( + "state = 'retired' AND isValidUTF8(host_proof) " + "AND length(splitByChar('|', host_proof)) = 9 " + "AND arrayElement(splitByChar('|', host_proof), 9) = 'cell-retirement-v1' " + "AND arrayAll(i -> notEmpty(arrayElement(splitByChar('|', host_proof), i)), [1,2,4,5,6]) " + "AND length(arrayElement(splitByChar('|', host_proof), 5)) <= 20 " + "AND match(arrayElement(splitByChar('|', host_proof), 5), '^[1-9][0-9]*$') " + "AND match(arrayElement(splitByChar('|', host_proof), 6), '^[0-9]+$')" +) + # `expired` is decided on the server's clock, like the leases. PARTICIPANTS_SQL = ( "SELECT participant_id, transition_id, state, version, heartbeat_at, " "host_proof, " "heartbeat_at + toIntervalSecond(stale_after_s) < now64(3) AS expired " - f"FROM {TABLE} FINAL WHERE state != 'retired'" + f"FROM {TABLE} FINAL WHERE NOT ({_CERTIFIED_RETIRED})" ) ONE_SQL = ( "SELECT state, version, " @@ -58,15 +76,29 @@ def running_sql(*, inserts: bool = False) -> str: # Retired participants whose last row is older than this are deleted. PRUNE_AFTER_H = 24 _PRUNE_IDS_SQL = ( - f"SELECT participant_id FROM {TABLE} FINAL WHERE state = 'retired' " + f"SELECT participant_id, state, version, host_proof FROM {TABLE} FINAL " + f"WHERE {_CERTIFIED_RETIRED} " "AND heartbeat_at < now64(3) - toIntervalHour({hours:UInt32}) LIMIT 1000" ) _PRUNE_SQL = f"ALTER TABLE {TABLE} DELETE WHERE participant_id IN {{ids:Array(UUID)}}" -async def participants(*, ch: _CHClient | None = None) -> list[dict[str, Any]]: +async def participants( + *, ch: _CHClient | None = None, host_proof: str | None = None +) -> list[dict[str, Any]]: with sequential_reads(): - return await (ch or _default_client()).query_all(PARTICIPANTS_SQL) + if host_proof is None: + rows = await (ch or _default_client()).query_all(PARTICIPANTS_SQL) + else: + rows = await (ch or _default_client()).query_all( + PARTICIPANTS_SQL + " AND host_proof = {proof:String} LIMIT 2", + {"proof": host_proof}, + ) + return [ + r + for r in rows + if r["state"] != "retired" or not has_certificate(str(r.get("host_proof") or "")) + ] async def record_participant( @@ -80,9 +112,13 @@ async def record_participant( ) -> None: """`heartbeat_at` takes the server's clock (its column default); this process's own row carries its `host_proof`.""" - if state not in ("running", "drained", "retired"): + if state not in ("running", "drained"): raise ValueError("Unknown participant state") own = str(identity) == str(process_identity()) + if not own: + raise ValueError("A process writes only its own running/drained registration") + if get_gate().closed: + raise ValueError("A permanently closed participant cannot re-enroll") await (ch or _default_client()).insert( TABLE, [ @@ -91,24 +127,59 @@ async def record_participant( "transition_id": transition, "state": state, "version": version if version is not None else max(time_ns(), after + 1), - # A retired row is stopped at once, so every build's `provider-lock clear` - # frees its holder without waiting out the heartbeat window. - "stale_after_s": 0 if state == "retired" else STALE_AFTER_S, - "host_proof": host_proof() if own else "", + "stale_after_s": STALE_AFTER_S, + "host_proof": participant_proof(), + } + ], + ) + + +async def record_retirement(proof: RetirementProof, *, ch: _CHClient | None = None) -> None: + """The sole retired-row writer, passed evidence from the guarded lifecycle.""" + if not isinstance(proof, RetirementProof) or not has_certificate(proof.host_proof): + raise ValueError("Retirement requires verified process evidence") + await (ch or _default_client()).insert( + TABLE, + [ + { + "participant_id": proof.participant, + "transition_id": UUID(int=0), + "state": "retired", + "version": max(time_ns(), proof.after + 1, proof.version or 0), + "stale_after_s": 0, + "host_proof": proof.host_proof, } ], ) async def prune_retired(*, ch: _CHClient | None = None) -> int: - """Delete every row of the participants retired `PRUNE_AFTER_H` ago (each - process start leaves one); a process that was only paused enrolls again - with its next beat. Returns how many.""" + """Age selects already certified permanent retirements, never unknown rows. + + Retired UUIDs cannot re-enroll. An earlier in-flight beat is below the final + version; removing every version cannot resurrect that beat. + """ client = ch or _default_client() - rows = await client.query_all(_PRUNE_IDS_SQL, {"hours": PRUNE_AFTER_H}) - ids = [r["participant_id"] for r in rows] - if ids: - await client.execute(_PRUNE_SQL, {"ids": ids}) + with sequential_reads(): + rows = await client.query_all(_PRUNE_IDS_SQL, {"hours": PRUNE_AFTER_H}) + ids = [r["participant_id"] for r in rows if has_certificate(str(r.get("host_proof") or ""))] + if not ids: + return 0 + latest = await client.query_all( + f"SELECT participant_id, state, version, host_proof FROM {TABLE} FINAL " + "WHERE participant_id IN {ids:Array(UUID)}", + {"ids": ids}, + ) + expected = {str(r["participant_id"]): int(r["version"]) for r in rows} + ids = [ + r["participant_id"] + for r in latest + if r["state"] == "retired" + and int(r["version"]) == expected.get(str(r["participant_id"])) + and has_certificate(str(r.get("host_proof") or "")) + ] + if ids: + await client.execute(_PRUNE_SQL, {"ids": ids}) return len(ids) diff --git a/server/main.py b/server/main.py index ed588430a3..5099c26667 100644 --- a/server/main.py +++ b/server/main.py @@ -29,6 +29,7 @@ ) from server.code.app.index.repo_modes import repo_umask from server.core import readiness as _readiness +from server.core import retirement from server.core.ch import get_client, probe_reader_user from server.core.errors import classified from server.core.logging_config import configure_logging @@ -47,6 +48,22 @@ wire() +def verify_retirement_topology() -> None: + """Capability requires both sides of the request-tail and child-task boundaries.""" + from server.core.request_tail import RequestTailMiddleware + from server.transport.model_maintenance import ModelMaintenanceMiddleware + from server.transport.retirement import RetirementMiddleware + + classes: list[object] = [middleware.cls for middleware in app.user_middleware] + if not ( + bool(classes) + and all(cls in classes for cls in (RequestTailMiddleware, ModelMaintenanceMiddleware)) + and classes[0] is RetirementMiddleware + and classes.index(RequestTailMiddleware) < classes.index(ModelMaintenanceMiddleware) + ): + raise RuntimeError("Retirement middleware must wrap the request tail and application") + + async def _shut_down(ch: Any, lease: Any, supervisor: Any, emitter: Any) -> None: """The bounded part of a shutdown: the instance row's stop write and the drains share one budget, and the final cleanup gets what they left unused, so the whole @@ -57,24 +74,35 @@ async def _shut_down(ch: Any, lease: Any, supervisor: Any, emitter: Any) -> None started = time.monotonic() await _instances.stop(ch) left = max(0.0, boot.SHUTDOWN_DRAIN_BUDGET_S - (time.monotonic() - started)) - try: - await asyncio.wait_for(_shutdown_drains(lease, supervisor, emitter), timeout=left) - except TimeoutError: + from server.core.background import create_deferred_task + + drains = create_deferred_task( + _shutdown_drains(lease, supervisor, emitter), name="shutdown-drains" + ) + _, pending = await asyncio.wait({drains}, timeout=left) + if pending: + drains.cancel() + logging.getLogger(__name__).warning( - "shutdown drains exceeded %.0f s; retiring anyway", boot.SHUTDOWN_DRAIN_BUDGET_S + "shutdown drains exceeded %.0f s; permanent retirement remains unproved", + boot.SHUTDOWN_DRAIN_BUDGET_S, ) + else: + await drains unused = max(0.0, boot.SHUTDOWN_DRAIN_BUDGET_S - (time.monotonic() - started)) await boot.leave_deployment(ch, budget_s=SHUTDOWN_RETIRE_S + unused) async def _shutdown_drains(lease: Any, supervisor: Any, emitter: Any) -> None: """The lifespan's shutdown drains, in order; it bounds them all together.""" - from server.workers import leader as _leader - # An agent claim's last cutoff, which ends what an older replica's sign-in # with the code issued while the claim was landing: started first, so it # runs beside the drains below. - claims, started = asyncio.create_task(_agent_claim.drain()), time.monotonic() + from server.core.background import create_deferred_task + from server.workers import leader as _leader + + claims = create_deferred_task(_agent_claim.drain(), name="shutdown-agent-claims") + started = time.monotonic() try: await _leader.shutdown(lease, supervisor) # The service-side writes still in flight land while what they use is @@ -102,10 +130,29 @@ async def _shutdown_drains(lease: Any, supervisor: Any, emitter: Any) -> None: finally: claims.cancel() # a no-op once it is done await emitter.stop() + producers = tuple(retirement.get_gate().producers) + if producers: + await asyncio.gather(*producers, return_exceptions=True) _otel.shutdown() from server.clients import _http as _http_clients await _http_clients.aclose() + retirement.get_gate().stopped() + + +async def _stop_background(probe_listener: Any) -> None: + """Stop admission and signal every producer; retirement still waits for completion.""" + from server.code.app.index import mirrors + + retirement.get_gate().close() + _readiness.begin_drain() + for producer in tuple(retirement.get_gate().producers): + producer.cancel() + if probe_listener is not None: + await probe_listener.stop() + mirrors.stop_warmup() + embedder_calibration.stop_refresh_loop() + boot.cancel_boot_tasks() @asynccontextmanager @@ -145,6 +192,9 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: tls.require_served() + verify_retirement_topology() + retirement.get_gate().install("server") + track = retirement.get_gate().track from server.core.object_store import require_client require_client() @@ -179,7 +229,11 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: await boot.require_session_protection(ch) # The deployment's cell state, before the first heartbeat reports this process # serving and before the lease is decided on it. - cell_loops = await boot.start_cell_gate() + from server.insights.app.provider_barrier import register_process, retire_reaped_process + + await register_process() + for task in await boot.start_cell_gate(): + track(task) await boot.warn_newer_schema() await boot.warn_collection_named_namespaces() # Nothing here writes a column a migration just added while another process @@ -192,21 +246,21 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: from server.core import instances as _instances from server.workers import leader as _leader - instances_loop = await _instances.start( - ch, - _leader.leads_deployment, - label=_instance_mod.instance_label() or "", - workers_lease=_leader.advertised_lease, - on_beat=boot.refresh_instance_state, + track( + await _instances.start( + ch, + _leader.leads_deployment, + label=_instance_mod.instance_label() or "", + workers_lease=_leader.advertised_lease, + on_beat=boot.refresh_instance_state, + ) ) from server.core.background import register_buffer_drainer - from server.insights.app.provider_barrier import register_process register_buffer_drainer("boot-writes", boot.drain_writes) from server.insights.app.provider_runtime import load_active_configuration, refresh_loop - await register_process() await load_active_configuration() emitter = get_emitter() @@ -220,7 +274,7 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: # the background workers, the holder of the ClickHouse lease, in a child # process of its own unless LOOM_WORKERS_PROCESS=inline. lease = await _leader.decide_supervisor(settings.workers_mode()) - supervisor = _workers_child.worker_runner(lease) + supervisor = _workers_child.worker_runner(lease, retire_child=retire_reaped_process) _leader.set_current(lease) # Apply per-table retention overrides from LOOM_RETENTION_*_DAYS, if set — # from the deployment's leader only, and only where the TTL differs. @@ -239,24 +293,29 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: boot.publish_deployment_config(ch) # This process verifies tokens: it loads the revoked set and the signing # keyring now and keeps both fresh. - auth_loops = await boot.start_token_state() + for task in await boot.start_token_state(): + track(task) probe_listener = boot.start_probe_listener() - overrides_loop = boot.start_runtime_overrides() + track(boot.start_runtime_overrides()) await probe_reader_user(ch) from server.insights.app import analysis as _analysis # Provisioned and probed in the background: the lease can wait for minutes. - analysis_loop = _analysis.start_probe_loop() + track(_analysis.start_probe_loop()) from server.assistant.app import service as _assistant assistant_loop = _assistant.start_setup_loop() - tracker_sources_loop = boot.start_tracker_sources_refresher(ch) + if assistant_loop is not None: + track(assistant_loop) + track(boot.start_tracker_sources_refresher(ch)) from server.code.app.index import repo_registry as _repo_registry - code_repos_loop = await _repo_registry.start_refresher(ch) + track(await _repo_registry.start_refresher(ch)) from server.code.app.index import mirrors as _mirrors - _mirrors.start_warmup() # this host's git mirrors, in the background + mirror_loop = _mirrors.start_warmup() + if mirror_loop is not None: + retirement.get_gate().track(mirror_loop) boot.publish_frontend_assets(ch) boot.mark_process_start(ch) await boot.probe_sandbox() @@ -274,37 +333,22 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: "WorkerSupervisor: not started in pid %s — %s", os.getpid(), lease.reason ) lease_loop = boot.start_lease_loop(lease, supervisor, ch) + if lease_loop is not None: + track(lease_loop) if not get_model_work_gate().closed: await boot.provider_report() await boot.probe_vector_capabilities(ch) await boot.calibrate_similarity_gates() - provider_refresh = asyncio.create_task(refresh_loop(), name="provider-configuration-refresh") + track(asyncio.create_task(refresh_loop(), name="provider-configuration-refresh")) # SIGTERM: not ready first, the port closes LOOM_SHUTDOWN_READY_DELAY_S later. boot.install_drain_on_sigterm() try: yield finally: - _readiness.begin_drain() - provider_refresh.cancel() - await asyncio.gather(provider_refresh, return_exceptions=True) - if lease_loop is not None: - lease_loop.cancel() - for loop in (*cell_loops, *auth_loops, overrides_loop): - loop.cancel() - if probe_listener is not None: - await probe_listener.stop() - tracker_sources_loop.cancel() - code_repos_loop.cancel() - analysis_loop.cancel() - if assistant_loop is not None: - assistant_loop.cancel() - _mirrors.stop_warmup() - embedder_calibration.stop_refresh_loop() - instances_loop.cancel() - boot.cancel_boot_tasks() + await _stop_background(probe_listener) await _shut_down(ch, lease, supervisor, emitter) await boot.close_storage_authority() @@ -501,6 +545,10 @@ def attach_browser_middleware(target: FastAPI) -> None: http_transport.attach(app) mcp_transport.attach(app) +from server.transport.retirement import RetirementMiddleware # noqa: E402 + +app.add_middleware(RetirementMiddleware) + from server.auth import login as _login # noqa: E402 from server.auth import oauth as _oauth # noqa: E402 from server.auth import scim_routes as _scim_routes # noqa: E402 diff --git a/server/memory/app/embedder_calibration.py b/server/memory/app/embedder_calibration.py index cc9569ba6b..8ddf1f4eb5 100644 --- a/server/memory/app/embedder_calibration.py +++ b/server/memory/app/embedder_calibration.py @@ -592,7 +592,11 @@ async def _loop() -> None: if not get_model_work_gate().closed: await refresh_shared() - _refresh_task = asyncio.get_running_loop().create_task(_loop(), name="loom-calibration-refresh") + from server.core.retirement import get_gate + + _refresh_task = get_gate().track( + asyncio.get_running_loop().create_task(_loop(), name="loom-calibration-refresh") + ) return _refresh_task diff --git a/server/observability/events.py b/server/observability/events.py index e18d4d22ec..296bd9e491 100644 --- a/server/observability/events.py +++ b/server/observability/events.py @@ -197,8 +197,9 @@ async def start(self) -> None: # Process-lifetime task: it must not inherit the tag, request id or # auth of whichever request first started it. from server.core.background import detached_task + from server.core.retirement import get_gate - self._task = detached_task(self._drain_loop(), name="loom-events-drain") + self._task = get_gate().track(detached_task(self._drain_loop(), name="loom-events-drain")) async def stop(self) -> None: self._running = False diff --git a/server/observability/problems.py b/server/observability/problems.py index 1911c1e84e..4ecaa68e25 100644 --- a/server/observability/problems.py +++ b/server/observability/problems.py @@ -935,8 +935,9 @@ def _participant_problems(state: HealthState) -> list[Problem]: f"A provider reconfiguration would wait on these participants: {ids}. " "`heartbeat_expired` ones stopped heartbeating (they may be paused), " "and `queries_running` ones did so with a query still running.", - "Once a participant's process is confirmed stopped, retire it with " - "`loom participant retire --participant --yes`.", + "On its trusted host, verify exact physical death and settled queries with " + "`loom participant retire --participant `. Expiry or a deleted pod " + "object does not prove death.", source="runtime", ) ] diff --git a/server/operations/reach_manifest.py b/server/operations/reach_manifest.py index d1eafc7121..14f76a4693 100644 --- a/server/operations/reach_manifest.py +++ b/server/operations/reach_manifest.py @@ -584,6 +584,7 @@ "namespace.mark_seen": ("dashboard",), "namespace.register": ( "sdk_method", + "sdk_example", "dashboard", ), "namespace.register_category": ( diff --git a/server/router/app.py b/server/router/app.py index 4e24a7c922..52c9d3d157 100644 --- a/server/router/app.py +++ b/server/router/app.py @@ -51,6 +51,7 @@ from contextlib import asynccontextmanager from typing import Any, NamedTuple, Protocol from urllib.parse import parse_qs +from uuid import UUID import httpx from starlette.applications import Starlette @@ -542,7 +543,12 @@ async def set_limit(self, request: Request) -> JSONResponse: return refusal if refusal := _malformed_cell_refusal(request): return refusal - body = await _body(request) + body = _parsed(await _bounded(request, ATTACH_BEGIN_MAX_BYTES)) + if not isinstance(body, dict): + return _refused(400, "this call's body is a JSON object") + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected known = {"closed", "placements_max"} if not isinstance(body, dict) or not body or set(body) - known: return _refused(400, "a limit is a JSON object of closed and/or placements_max") @@ -556,7 +562,7 @@ async def set_limit(self, request: Request) -> JSONResponse: return _refused(400, "placements_max is a whole number, 0 or more, or null") return await _operator_answer( lambda: directory.set_limit( - request.path_params["cell"], closed=closed, placements_max=quota + request.path_params["cell"], closed=closed, placements_max=quota, expected=expected ) ) @@ -567,7 +573,12 @@ async def rotate_credential(self, request: Request) -> JSONResponse: return refusal if refusal := _malformed_cell_refusal(request): return refusal - body = await _body(request) + body = _parsed(await _bounded(request, ATTACH_BEGIN_MAX_BYTES)) + if not isinstance(body, dict): + return _refused(400, "this call's body is a JSON object") + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected if not isinstance(body, dict) or set(body) - {"step", "sha256"}: return _refused(400, "a rotation is a JSON object of step and sha256") step, sha = body.get("step"), body.get("sha256", "") @@ -578,7 +589,9 @@ async def rotate_credential(self, request: Request) -> JSONResponse: if step == "next" and _is_operator_hash(sha): return _refused(400, _OPERATOR_HASH_REFUSAL) response = await _operator_answer( - lambda: directory.rotate_credential(request.path_params["cell"], step, sha) + lambda: directory.rotate_credential( + request.path_params["cell"], step, sha, expected=expected + ) ) # This replica reads the hashes again on its next call. A 503 is included: its # change may have committed (BACKEND_UNAVAILABLE). @@ -632,7 +645,7 @@ async def register(self, request: Request) -> JSONResponse: ) async def _operator_body( - self, request: Request, limit: int = MAX_BODY_BYTES + self, request: Request, limit: int = ATTACH_BEGIN_MAX_BYTES ) -> dict[str, Any] | JSONResponse: """The operator call's JSON object, or its refusal: the credential, then the cell's name, then the body (413 past `limit`, 400 when it is no JSON object).""" @@ -660,6 +673,19 @@ async def operator_cell(self, request: Request) -> JSONResponse: lambda found: {**_attach_answer(found), "other": found.other}, ) + async def claim(self, request: Request) -> JSONResponse: + body = await self._operator_body(request, ATTACH_BEGIN_MAX_BYTES) + if isinstance(body, JSONResponse): + return body + expected = _control(body, claim=True) + if isinstance(expected, JSONResponse): + return expected + if body: + return _refused(400, "a control claim contains only expected") + return await _answered( + lambda: directory.claim(request.path_params["cell"], expected), _attach_answer + ) + async def attach_begin(self, request: Request) -> JSONResponse: """`attach/begin`: once the probe listener echoes attach's nonces from the stores the row names and the upstream reports the row's deployment id, write when attach @@ -668,6 +694,9 @@ async def attach_begin(self, request: Request) -> JSONResponse: body = await self._operator_body(request, ATTACH_BEGIN_MAX_BYTES) if isinstance(body, JSONResponse): return body + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected name = request.path_params["cell"] parsed = _begin(body) if isinstance(parsed, JSONResponse): @@ -676,17 +705,32 @@ async def attach_begin(self, request: Request) -> JSONResponse: found = await _read_attaching(name) if isinstance(found, JSONResponse): return found - if reason := directory.begin_refusal(found, begin): + if reason := directory.control_refusal(found, expected) or directory.begin_refusal( + found, begin + ): return _refused(409, reason) if refusal := await self._stores_check(found, nonces): return refusal - return await _answered(lambda: directory.attach_begin(name, begin), _attach_answer) + return await _answered( + lambda: directory.attach_begin(name, begin, expected=expected), _attach_answer + ) async def attach_place(self, request: Request) -> JSONResponse: """`attach/place {deployment_id, orgs}`: a batch of at most ATTACH_BATCH orgs.""" body = await self._operator_body(request) if isinstance(body, JSONResponse): return body + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected + if ( + len(json.dumps(body, ensure_ascii=True, separators=(",", ":")).encode()) + > MAX_BODY_BYTES + ): + return _refused( + 413, + f"a placement batch without its control envelope is at most {MAX_BODY_BYTES} bytes", + ) orgs = body.get("orgs") deployment_id = _text(body, "deployment_id", needed=True) if ( @@ -704,7 +748,7 @@ async def attach_place(self, request: Request) -> JSONResponse: name = request.path_params["cell"] batch = [o for o in named if o is not None] return await _answered( - lambda: directory.attach_place(name, deployment_id, batch), + lambda: directory.attach_place(name, deployment_id, batch, expected=expected), lambda counts: {"cell": name, "placed": counts[0], "placed_orgs": counts[1]}, ) @@ -713,12 +757,16 @@ async def attach_enforced(self, request: Request) -> JSONResponse: body = await self._operator_body(request) if isinstance(body, JSONResponse): return body + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected deployment_id = _text(body, "deployment_id", needed=True) if set(body) != {"deployment_id"} or deployment_id is None: return _refused(400, "attach/enforced is a JSON object of deployment_id") name = request.path_params["cell"] return await _answered( - lambda: directory.attach_enforced(name, deployment_id), _attach_answer + lambda: directory.attach_enforced(name, deployment_id, expected=expected), + _attach_answer, ) async def attach_serving(self, request: Request) -> JSONResponse: @@ -728,6 +776,9 @@ async def attach_serving(self, request: Request) -> JSONResponse: body = await self._operator_body(request) if isinstance(body, JSONResponse): return body + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected deployment_id = _text(body, "deployment_id", needed=True) nonces: Any = body.get("nonces") if set(body) != {"deployment_id", "nonces"} or deployment_id is None: @@ -738,13 +789,15 @@ async def attach_serving(self, request: Request) -> JSONResponse: found = await _read_attaching(name) if isinstance(found, JSONResponse): return found + if reason := directory.control_refusal(found, expected): + return _refused(409, reason) if not found.served(deployment_id): if reason := directory.attach_refusal(found, deployment_id, enforced=True): return _refused(409, reason) if refusal := await self._stores_check(found, nonces): return refusal response = await _answered( - lambda: directory.attach_serving(name, deployment_id), _attach_answer + lambda: directory.attach_serving(name, deployment_id, expected=expected), _attach_answer ) # A 503's change may have committed: this replica reads the cell's issuers and state # on its next request either way. @@ -760,13 +813,18 @@ async def remove(self, request: Request) -> JSONResponse: a cell `attaching` with no placement (`directory.remove`). The router cannot read the cell's Postgres: `loom cell remove` checks that a first cell's reads `open`, and a direct call skips that check.""" - if refusal := self._operator(request): - return refusal - if refusal := _malformed_cell_refusal(request): - return refusal + body = await self._operator_body(request, ATTACH_BEGIN_MAX_BYTES) + if isinstance(body, JSONResponse): + return body + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected + if body: + return _refused(400, "a removal contains only expected") name = request.path_params["cell"] response = await _answered( - lambda: directory.remove(name), lambda row: {"cell": row.cell, "removed": True} + lambda: directory.remove(name, expected=expected), + lambda row: {"cell": row.cell, "removed": True}, ) # A 503 is included: the removal may have committed (BACKEND_UNAVAILABLE). if response.status_code in (200, 503): @@ -1315,6 +1373,41 @@ async def _answered[T]( return JSONResponse(render(result)) +def _control(body: dict[str, Any], *, claim: bool = False) -> directory.Control | JSONResponse: + """Remove and validate the exact observed row; no caller may select a new token.""" + value = body.pop("expected", None) + if not isinstance(value, dict): + return _refused( + 409, + "cell control fencing requires upgraded control clients and every router replica", + code="CONTROL_UPGRADE_REQUIRED", + ) + keys = set(directory.CONTROL_FIELDS) | (set() if claim else {"mutation_generation"}) + if set(value) != keys: + return _refused( + 400, + "expected contains the complete registration, fingerprint pair and claimed generation", + ) + if type(value["is_first"]) is not bool or any( + not isinstance(value[name], str) or "\x00" in value[name] + for name in directory.CONTROL_IDENTITY + if name != "is_first" + ): + return _refused(400, "expected registration has invalid field types") + if any( + value[name] is not None and not isinstance(value[name], dict) + for name in ("current_fp", "next_fp") + ) or _holds_nul(value): + return _refused(400, "expected fingerprint pair contains JSON objects or null") + generation = None + if not claim: + try: + generation = UUID(value["mutation_generation"]) + except (TypeError, ValueError, AttributeError): + return _refused(400, "expected mutation_generation is a UUID") + return directory.Control({name: value[name] for name in directory.CONTROL_FIELDS}, generation) + + def _limits(row: directory.Cell) -> dict[str, Any]: return { "cell": row.cell, @@ -1344,6 +1437,8 @@ def _attach_answer(found: directory.Attaching) -> dict[str, Any]: "attach_started_at": started.isoformat() if started else None, "current_fp": row.current_fp, "next_fp": row.next_fp, + "mutation_generation": str(row.mutation_generation), + **{name: getattr(found, name) for name in directory._REGISTERED}, } @@ -1374,6 +1469,7 @@ async def _lifespan(_app: Starlette) -> AsyncIterator[None]: methods=["POST"], ), Route("/cells/v1/operator/cells/{cell}", router.operator_cell, methods=["GET"]), + Route("/cells/v1/operator/cells/{cell}/claim", router.claim, methods=["POST"]), Route("/cells/v1/operator/cells/{cell}", router.remove, methods=["DELETE"]), Route( "/cells/v1/operator/cells/{cell}/attach/begin", diff --git a/server/router/directory.py b/server/router/directory.py index a92967ed86..3e438cf6d9 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -20,13 +20,16 @@ import asyncio import hmac import json +import time from collections.abc import AsyncIterator, Awaitable, Callable, Iterable, Mapping from contextlib import AsyncExitStack, asynccontextmanager from dataclasses import dataclass from datetime import datetime from typing import Any +from uuid import UUID from server.core import cell as cell_state +from server.core import pg from server.core.paths import ROUTER_MIGRATIONS_DIR from server.router.limits import MAX_BODY_BYTES from server.settings import settings @@ -59,9 +62,9 @@ def _bound_s(statements: int) -> int: return STATEMENT_TIMEOUT_S + (statements + 1) * (STATEMENT_TIMEOUT_S + 1) -# The longest a limit or credential change can take: eight statements (begin, lock, row, -# check, update, audit, read-back, commit). Registration has its own bound, below. -OPERATOR_CHANGE_MAX_S = _bound_s(8) # not a setting: derived from the bound above +# The longest limit/credential change has ten statements: begin, directory lock, row, +# other row, WAL, credential check, update, audit, read-back and commit. +OPERATOR_CHANGE_MAX_S = _bound_s(10) # not a setting: derived from the bound above # The longest a registration's directory write can take: thirteen statements (begin, lock, # row, five refusal reads, three inserts, read-back, commit); and its read before the probe: # six statements (the row, the first cell, then the credential, deployment id, host and @@ -72,11 +75,11 @@ def _bound_s(statements: int) -> int: # The attach calls' statements, which tests/unit/test_router_attach.py counts against the # code: the operator's read of a cell's row (the row, any other row, audit); the read before # `begin`'s and `serving`'s probe (the row, any other row); `begin`, `enforced` and -# `serving` (begin, lock, row, other row, update, audit, read-back, commit); a batch (begin, -# lock, row, other row, insert, placements elsewhere, count, audit, commit); and the removal -# of an unattached cell (begin, lock, row, delete hosts, delete row, audit, commit). +# `serving` (begin, lock, row, other row, WAL, update, audit, read-back, commit); a batch +# (begin, lock, row, other row, WAL, insert, placements elsewhere, count, audit, commit); +# removal adds other-row and WAL reads before deleting hosts/row and auditing. OPERATOR_READ_STATEMENTS, ATTACH_READ_STATEMENTS = 3, 2 -ATTACH_CHANGE_STATEMENTS, ATTACH_PLACE_STATEMENTS, REMOVE_STATEMENTS = 8, 9, 7 +ATTACH_CHANGE_STATEMENTS, ATTACH_PLACE_STATEMENTS, REMOVE_STATEMENTS = 9, 10, 9 OPERATOR_READ_MAX_S = _bound_s(OPERATOR_READ_STATEMENTS) # not a setting: derived REMOVE_MAX_S = _bound_s(REMOVE_STATEMENTS) # not a setting: derived # Every directory connection commits through the synchronous standby and reads what @@ -107,6 +110,7 @@ class Cell: next_fp: dict[str, Any] | None # The new credential's hash during a rotation (`loom cell credential --next`). credential_next_sha256: str | None + mutation_generation: UUID def holds(self, presented: str) -> bool: """Whether `presented` (a credential's SHA-256) is this cell's, current or next.""" @@ -148,13 +152,37 @@ def _require_dsn() -> str: return settings.router_pg_dsn +class _PoolInitFailure(Exception): + """Carry native control intent through the driver's ordinary-error cleanup.""" + + def __init__(self, error: SystemExit | KeyboardInterrupt) -> None: + super().__init__("directory codec initialization interrupted") + self.error = error + + async def _init(conn: Any) -> None: - for typ in ("json", "jsonb"): - await conn.set_type_codec(typ, encoder=json.dumps, decoder=json.loads, schema="pg_catalog") + try: + for typ in ("json", "jsonb"): + await conn.set_type_codec( + typ, encoder=json.dumps, decoder=json.loads, schema="pg_catalog" + ) + except BaseException as original: + # Only builtin codecs run here: this raw session has never been borrowed or + # held command locks. Quarantine before the driver can lose its ownership. + try: + conn.terminate() + except BaseException: # noqa: BLE001 — disposal must preserve the first init failure. + try: + original.add_note("directory initialization session closure is unknown") + except BaseException: # noqa: BLE001 — diagnostics must preserve the first failure. + pass + if isinstance(original, (SystemExit, KeyboardInterrupt)): + raise _PoolInitFailure(original) from None + raise async def pool() -> Any: - """The directory's pool, opened on first use.""" + """The directory's pool; physical sessions open only for foreground consumers.""" global _pool if _pool is not None: return _pool @@ -162,19 +190,22 @@ async def pool() -> Any: async with _opening: if _pool is None: - _pool = await asyncpg.create_pool( - _require_dsn(), - min_size=1, - max_size=POOL_MAX, - init=_init, - server_settings={ - **_SESSION, - "statement_timeout": str(STATEMENT_TIMEOUT_S * 1000), - "idle_in_transaction_session_timeout": str(STATEMENT_TIMEOUT_S * 2000), - }, - command_timeout=STATEMENT_TIMEOUT_S + 1, - timeout=STATEMENT_TIMEOUT_S, - ) + try: + _pool = await asyncpg.create_pool( + _require_dsn(), + min_size=0, + max_size=POOL_MAX, + init=_init, + server_settings={ + **_SESSION, + "statement_timeout": str(STATEMENT_TIMEOUT_S * 1000), + "idle_in_transaction_session_timeout": str(STATEMENT_TIMEOUT_S * 2000), + }, + command_timeout=STATEMENT_TIMEOUT_S + 1, + timeout=STATEMENT_TIMEOUT_S, + ) + except _PoolInitFailure as failed: + raise failed.error from None return _pool @@ -191,13 +222,17 @@ async def _connection(*, lookup: bool = False) -> AsyncIterator[Any]: slot, the pool's opening and the connection); a lookup's also takes one of LOOKUP_SLOTS. A wait past the bound raises TimeoutError, an OSError, which `_reading` answers as the directory unavailable.""" - async with AsyncExitStack() as stack: - async with asyncio.timeout(STATEMENT_TIMEOUT_S): - if lookup: - await _lookups.acquire() - stack.callback(_lookups.release) - conn = await stack.enter_async_context((await pool()).acquire()) - yield conn + try: + async with AsyncExitStack() as stack: + async with asyncio.timeout(STATEMENT_TIMEOUT_S): + if lookup: + await _lookups.acquire() + stack.callback(_lookups.release) + conn = await stack.enter_async_context((await pool()).acquire()) + yield conn + except _PoolInitFailure as failed: + # Driver acquisition and context cleanup finish before control intent returns. + raise failed.error from None @asynccontextmanager @@ -231,7 +266,7 @@ async def migrate() -> list[str]: # The columns a Cell holds (read by name: `_cell`). _CELL_COLUMNS = ( "cell, is_first, host, state, mode, closed, placements_max, placed, credential_sha256, " - "credential_next_sha256, deployment_id, current_fp, next_fp" + "credential_next_sha256, deployment_id, current_fp, next_fp, mutation_generation" ) _CELL = f"SELECT {_CELL_COLUMNS} FROM loom_router.cells" @@ -302,6 +337,7 @@ async def set_limit( *, closed: bool | _Unset = UNSET, placements_max: int | None | _Unset = UNSET, + expected: Control, ) -> Cell: """`loom cell limit`: close or reopen a cell to new orgs and set or clear its placement quota, on its row alone, audited; the cell's stores are not touched. A @@ -311,6 +347,10 @@ async def set_limit( raise ValueError("set_limit sets closed, placements_max or both") async def change(conn: Any, row: Cell) -> dict[str, Any]: + if closed is False and row.state == "attaching": + raise OperatorRefused( + f"cell {cell_name!r} is attaching: it can reopen after attach reaches serving" + ) detail: dict[str, Any] = {} if closed is not UNSET: detail["closed"] = closed @@ -324,10 +364,12 @@ async def change(conn: Any, row: Cell) -> dict[str, Any]: ) return detail - return await _operator_change(cell_name, "limit", change) + return await _operator_change(cell_name, "limit", change, expected) -async def rotate_credential(cell_name: str, step: str, sha256: str = "") -> Cell: +async def rotate_credential( + cell_name: str, step: str, sha256: str = "", *, expected: Control +) -> Cell: """`loom cell credential`: `next` records a new credential's hash beside the current one, refused when any cell holds it as current or next and while one is pending; `finish` makes `next` current and drops the old; `revert` drops `next`. @@ -365,25 +407,22 @@ async def change(conn: Any, row: Cell) -> dict[str, Any]: ) return {"step": step, **({"sha256": sha256} if sha256 else {})} - return await _operator_change(cell_name, "credential", change) + return await _operator_change(cell_name, "credential", change, expected) async def _operator_change( - cell_name: str, action: str, change: Callable[[Any, Cell], Awaitable[dict[str, Any]]] + cell_name: str, + action: str, + change: Callable[[Any, Cell], Awaitable[dict[str, Any]]], + expected: Control, ) -> Cell: - async with _reading(), _connection() as conn, conn.transaction(): - await conn.execute("SELECT pg_advisory_xact_lock($1)", _DIRECTORY_LOCK) - locked = await conn.fetchrow(f"{_CELL} WHERE cell = $1 FOR UPDATE", cell_name) - if locked is None: - raise NoSuchCell(f"cell {cell_name!r} is not in the directory") - detail = await change(conn, _cell(locked)) - await conn.execute( - "INSERT INTO loom_router.audit (actor, action, detail) VALUES ('operator', $1, $2)", - action, - {"cell": cell_name, **detail}, - ) + async def change_locked(conn: Any, found: Attaching) -> Cell: + detail = await change(conn, found.row) + await _audit(conn, action, {"cell": cell_name, **detail}) row = await conn.fetchrow(f"{_CELL} WHERE cell = $1", cell_name) - return _cell(row) + return _cell(row) + + return await _attach_change(cell_name, expected, change_locked) @dataclass(frozen=True) @@ -866,17 +905,219 @@ async def attach_read(cell_name: str, *, audited: bool = False) -> Attaching: return _attaching(found, other) -@asynccontextmanager -async def _attach_change(cell_name: str) -> AsyncIterator[tuple[Any, Attaching]]: - """A directory transaction holding the directory lock and then the cell's row lock, with - the row and any other row's name.""" - async with _reading(), _connection() as conn, conn.transaction(): +# Every acceptance controller compares the full registration and exact JSON pair. +CONTROL_IDENTITY = ("cell", "is_first", "host", "deployment_id", *_REGISTERED) +CONTROL_FIELDS = (*CONTROL_IDENTITY, "current_fp", "next_fp") +CONTROL_MAX_S = _bound_s(10) # not a setting: transaction, commit and pool release bound + + +@dataclass(frozen=True) +class Control: + """The operator's observed registration and pair, with its claimed generation.""" + + values: dict[str, Any] + generation: UUID | None + + +def control_refusal(found: Attaching, expected: Control, *, claim: bool = False) -> str: + if not claim and expected.generation != found.row.mutation_generation: + return "cell control generation changed: stop and rerun the command with a fresh claim" + actual = { + name: getattr(found.row, name) if hasattr(found.row, name) else getattr(found, name) + for name in CONTROL_FIELDS + } + if _as_json(actual) != _as_json(expected.values): + return ( + "cell registration or fingerprints changed: stop and rerun from a fresh directory read" + ) + return "" + + +@dataclass(frozen=True) +class _ControlFatal: + error: SystemExit | KeyboardInterrupt + + +async def _control_result[T](work: Callable[[], Awaitable[T]]) -> T | _ControlFatal: + try: + return await work() + except (SystemExit, KeyboardInterrupt) as error: + return _ControlFatal(error) + except _ControlInterrupted as error: + return error.fatal + + +class _ControlOwner: + """Own only this operator attempt's proxy, through close and pool release.""" + + def __init__(self, seconds: float) -> None: + self.end = time.monotonic() + seconds + self.work_end = self.end - min(1.0, seconds / 4) + self.context = _connection() + self.conn: Any = None + self.entered = False + self.closing = False + self.close_task: asyncio.Task[Any] | None = None + self.exit_task: asyncio.Task[Any] | None = None + + def start[T](self, work: Callable[[], Awaitable[T]]) -> asyncio.Task[T | _ControlFatal]: + task = asyncio.create_task(_control_result(work)) + task.add_done_callback(pg._retrieve_task) + return task + + async def wait[T](self, task: asyncio.Task[T | _ControlFatal], end: float) -> T: + done, _ = await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) + if not done: + raise TimeoutError("directory control deadline expired; outcome unknown") + result = task.result() + if isinstance(result, _ControlFatal): + # Keep fatal exceptions inert in nested acquisition/cleanup children. + raise _ControlInterrupted(result) + return result + + async def close(self) -> None: + if self.conn is None: + return + if self.close_task is None: + self.close_task = self.start(lambda: pg.close_owned(self.conn, end=self.end)) + interrupted: asyncio.CancelledError | None = None + try: + while not self.close_task.done(): + remaining = self.end - time.monotonic() + if remaining <= 0: + raise TimeoutError("directory control deadline expired; outcome unknown") + try: + await asyncio.wait({self.close_task}, timeout=remaining) + except asyncio.CancelledError as error: + # Another signal cannot abandon the public cancellation in flight. + interrupted = interrupted or error + result = self.close_task.result() + if isinstance(result, _ControlFatal): + raise _ControlInterrupted(result) + except BaseException: + try: + pg._terminate(self.conn) + except BaseException: # noqa: BLE001 — abort must preserve the first cleanup failure. + pass + finally: + if not self.close_task.done(): + self.close_task.cancel() + if interrupted is not None: + raise interrupted from None + raise + if interrupted is not None: + raise interrupted + + async def release(self) -> None: + if self.entered: + if self.exit_task is None: + self.exit_task = self.start(lambda: self.context.__aexit__(None, None, None)) + try: + await self.wait(self.exit_task, self.end) + except BaseException: # An interrupted pool exit must not continue without its owner. + self.exit_task.cancel() + raise + + async def work[T](self, change: Callable[[Any], Awaitable[T]]) -> T: + self.conn = await self.context.__aenter__() + self.entered = True + if self.closing: + # A cancellation-resistant acquire cannot hand a late proxy to a mutation. + try: + await self.close() + finally: + await self.release() + raise TimeoutError("directory connection arrived after its attempt ended") + tx = self.conn.transaction() + await tx.start() + result = await change(self.conn) + await tx.commit() + return result + + async def finish(self, task: asyncio.Task[Any]) -> None: + self.closing = True + failure: BaseException | None = None + try: + await self.close() + except BaseException as error: # noqa: BLE001 — release must preserve the first failure. + failure = error + if not task.done(): + task.cancel() + try: + await asyncio.wait({task}, timeout=max(0, self.end - time.monotonic())) + except BaseException as error: # noqa: BLE001 — cleanup keeps the first failure. + failure = failure or error + try: + await self.release() + except BaseException as error: # noqa: BLE001 — cleanup keeps the first failure. + failure = failure or error + if failure is not None: + raise failure + + +class _ControlInterrupted(BaseException): + def __init__(self, fatal: _ControlFatal) -> None: + self.fatal = fatal + + +async def _control_transaction[T](change: Callable[[Any], Awaitable[T]], *, seconds: float) -> T: + """Bound the whole callback, close first, then settle work and pool ownership.""" + async with _reading(): + owner = _ControlOwner(seconds) + task = owner.start(lambda: owner.work(change)) + try: + result = await owner.wait(task, owner.work_end) + except BaseException as original: + try: + await owner.finish(task) + except BaseException: # noqa: BLE001 — preserve the original failure/cancellation. + pass + if isinstance(original, _ControlInterrupted): + raise original.fatal.error from None + raise + try: + await owner.finish(task) + except _ControlInterrupted as interrupted: + raise interrupted.fatal.error from None + return result + + +async def _attach_change[T]( + cell_name: str, + expected: Control, + change: Callable[[Any, Attaching], Awaitable[T]], + *, + claim: bool = False, +) -> T: + async def locked(conn: Any) -> T: await conn.execute("SELECT pg_advisory_xact_lock($1)", _DIRECTORY_LOCK) found = await conn.fetchrow(f"{_CELL_ATTACH} WHERE cell = $1 FOR UPDATE", cell_name) if found is None: raise NoSuchCell(f"cell {cell_name!r} is not in the directory") other = await conn.fetchval(_OTHER, cell_name) - yield conn, _attaching(found, other) + attaching = _attaching(found, other) + if reason := control_refusal(attaching, expected, claim=claim): + raise OperatorRefused(reason) + # Even a no-op needs synchronous WAL acknowledgement. + await conn.execute("SELECT pg_logical_emit_message(true, 'loom.control', '')") + return await change(conn, attaching) + + return await _control_transaction(locked, seconds=CONTROL_MAX_S) + + +async def claim(cell_name: str, expected: Control) -> Attaching: + """Rotate on EVERY execution, including an identical HTTP replay.""" + + async def change_locked(conn: Any, _found: Attaching) -> Attaching: + await conn.execute( + "UPDATE loom_router.cells SET mutation_generation = gen_random_uuid() WHERE cell = $1", + cell_name, + ) + await _audit(conn, "control_claim", {"cell": cell_name}) + written = await conn.fetchrow(f"{_CELL_ATTACH} WHERE cell = $1", cell_name) + return _attaching(written, _found.other) + + return await _attach_change(cell_name, expected, change_locked, claim=True) async def _audit(conn: Any, action: str, detail: dict[str, Any]) -> None: @@ -887,11 +1128,12 @@ async def _audit(conn: Any, action: str, detail: dict[str, Any]) -> None: ) -async def attach_begin(cell_name: str, begin: Begin) -> Attaching: +async def attach_begin(cell_name: str, begin: Begin, *, expected: Control) -> Attaching: """`attach/begin`: write `attach_started_at` once, the fingerprints while the row has none, and the root-secret fingerprint on every call, audited with the allow lists. Idempotent: a rerun posting the same fingerprints writes only `keys_fp`.""" - async with _attach_change(cell_name) as (conn, found): + + async def change_locked(conn: Any, found: Attaching) -> Attaching: if reason := begin_refusal(found, begin): raise OperatorRefused(reason) await conn.execute( @@ -914,10 +1156,14 @@ async def attach_begin(cell_name: str, begin: Begin) -> Attaching: }, ) written = await conn.fetchrow(f"{_CELL_ATTACH} WHERE cell = $1", cell_name) - return _attaching(written, None) + return _attaching(written, None) + return await _attach_change(cell_name, expected, change_locked) -async def attach_place(cell_name: str, deployment_id: str, orgs: list[str]) -> tuple[int, int]: + +async def attach_place( + cell_name: str, deployment_id: str, orgs: list[str], *, expected: Control +) -> tuple[int, int]: """`attach/place`: place a batch of the first cell's orgs, once its directory mode is `enforced`, past its closed mark and quota (the orgs are the cell's already); the count it placed now and the cell's count. An org already placed in the cell is skipped, so a @@ -927,7 +1173,8 @@ async def attach_place(cell_name: str, deployment_id: str, orgs: list[str]) -> t reads `enforced`, judges that.""" _refuse_long(orgs) batch = sorted(set(orgs)) - async with _attach_change(cell_name) as (conn, found): + + async def change_locked(conn: Any, found: Attaching) -> tuple[int, int]: if reason := attach_refusal(found, deployment_id, enforced=True): raise OperatorRefused(reason) new = await conn.fetch( @@ -966,13 +1213,16 @@ async def attach_place(cell_name: str, deployment_id: str, orgs: list[str]) -> t len(batch), len(new), ) - return len(new), int(placed) + return len(new), int(placed) + + return await _attach_change(cell_name, expected, change_locked) -async def attach_enforced(cell_name: str, deployment_id: str) -> Attaching: +async def attach_enforced(cell_name: str, deployment_id: str, *, expected: Control) -> Attaching: """`attach/enforced`: record directory mode `enforced` (registration wrote `recording`), after which the cell places new orgs live; idempotent.""" - async with _attach_change(cell_name) as (conn, found): + + async def change_locked(conn: Any, found: Attaching) -> Attaching: if reason := attach_refusal(found, deployment_id): raise OperatorRefused(reason) await conn.execute( @@ -986,7 +1236,9 @@ async def attach_enforced(cell_name: str, deployment_id: str) -> Attaching: {"cell": cell_name, "changed": found.row.mode != "enforced"}, ) written = await conn.fetchrow(f"{_CELL_ATTACH} WHERE cell = $1", cell_name) - return _attaching(written, None) + return _attaching(written, None) + + return await _attach_change(cell_name, expected, change_locked) def issuers(row: Cell) -> list[str]: @@ -1001,10 +1253,11 @@ def issuers(row: Cell) -> list[str]: return sorted(found) -async def attach_serving(cell_name: str, deployment_id: str) -> Attaching: +async def attach_serving(cell_name: str, deployment_id: str, *, expected: Control) -> Attaching: """`attach/serving`: mark the first cell `serving`, with the issuers its fingerprints accept computed from the row as locked. A `serving` row is answered as it is.""" - async with _attach_change(cell_name) as (conn, found): + + async def change_locked(conn: Any, found: Attaching) -> Attaching: row = found.row if found.served(deployment_id): await _audit(conn, "attach_serving", {"cell": cell_name, "serving_already": True}) @@ -1020,7 +1273,9 @@ async def attach_serving(cell_name: str, deployment_id: str) -> Attaching: ) await _audit(conn, "attach_serving", {"cell": cell_name, "issuers": accepted}) written = await conn.fetchrow(f"{_CELL_ATTACH} WHERE cell = $1", cell_name) - return _attaching(written, None) + return _attaching(written, None) + + return await _attach_change(cell_name, expected, change_locked) def remove_refusal(cell_name: str, state: str, placed: int) -> str: @@ -1035,7 +1290,7 @@ def remove_refusal(cell_name: str, state: str, placed: int) -> str: return "" -async def remove(cell_name: str) -> Cell: +async def remove(cell_name: str, *, expected: Control) -> Cell: """`DELETE /cells/v1/operator/cells/` (`loom cell remove`): remove a cell's row and its host rows, audited, while it is `attaching` with no placement, whatever its directory mode. A placement in flight either commits first (and the removal refuses) or @@ -1043,12 +1298,9 @@ async def remove(cell_name: str) -> Cell: first cell's Postgres reads `open` first, and a direct call skips that check, leaving a first cell still prepared answering its new orgs 503 and unable to register again until the full back-out has run.""" - async with _reading(), _connection() as conn, conn.transaction(): - await conn.execute("SELECT pg_advisory_xact_lock($1)", _DIRECTORY_LOCK) - found = await conn.fetchrow(f"{_CELL} WHERE cell = $1 FOR UPDATE", cell_name) - if found is None: - raise NoSuchCell(f"cell {cell_name!r} is not in the directory") - row = _cell(found) + + async def change_locked(conn: Any, found: Attaching) -> Cell: + row = found.row if reason := remove_refusal(cell_name, row.state, row.placed): raise OperatorRefused(reason) await conn.execute("DELETE FROM loom_router.hosts WHERE cell = $1", cell_name) @@ -1064,7 +1316,9 @@ async def remove(cell_name: str) -> Cell: "mode": row.mode, }, ) - return row + return row + + return await _attach_change(cell_name, expected, change_locked) async def lookup(org: str) -> tuple[str, str] | None: diff --git a/server/settings/catalog.py b/server/settings/catalog.py index 4f832043d6..e99497b18d 100644 --- a/server/settings/catalog.py +++ b/server/settings/catalog.py @@ -5224,10 +5224,16 @@ class RuntimeSettings(BaseModel): # process it starts and stops with the lease, so the passes do not share # the event loop that serves requests; inline = in the serving process. workers_process: Literal["child", "inline"] = "child" + # At most this many seconds to certify a reaped worker child; also bounded by + # the remaining stop or restart delay. Expiry leaves its participant unretired. + workers_child_retirement_timeout_s: float = Field(default=1.0, gt=0, allow_inf_nan=False) # Seconds a process keeps serving after SIGTERM while /v1/health/ready # answers 503, so the load balancer stops routing to it before its port # closes. Size it to the LB's probe interval × failure threshold. 0 = stop at once. shutdown_ready_delay_s: float = Field(default=15.0, ge=0) + # Seconds a native CLI command allows its producers, deferred writes and + # participant retirement to finish. Expiry leaves the participant unretired. + native_command_drain_timeout_s: float = Field(default=15.0, gt=0, allow_inf_nan=False) # Seconds boot waits for the first load of the tracker registry and the # runtime overrides, and the first publish of the dashboard bundle, before # it serves; readiness stays false until the first two load. diff --git a/server/transport/model_maintenance.py b/server/transport/model_maintenance.py index 7f59ba2e2c..f27092672d 100644 --- a/server/transport/model_maintenance.py +++ b/server/transport/model_maintenance.py @@ -6,7 +6,7 @@ from starlette.responses import JSONResponse from starlette.types import ASGIApp, Receive, Scope, Send -from server.core import cell, deployment_config +from server.core import cell, deployment_config, retirement from server.core.errors import classified from server.core.model_work_gate import ModelChangeInProgress, ModelWorkGate, get_model_work_gate from server.settings import settings @@ -111,6 +111,20 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: if scope["type"] not in ("http", "websocket"): await self.app(scope, receive, send) return + try: + async with retirement.get_gate().admit(): + await self._dispatch(scope, receive, send) + except retirement.ProcessStopping: + await _refuse( + scope, + receive, + send, + code="PROCESS_STOPPING", + message="This server is stopping. Retry another server.", + retry_after="1", + ) + + async def _dispatch(self, scope: Scope, receive: Receive, send: Send) -> None: if (reason := cell.gate_reason()) and not _health_probe(scope): await _refuse_cell(scope, receive, send, reason) return diff --git a/server/transport/retirement.py b/server/transport/retirement.py new file mode 100644 index 0000000000..396a304554 --- /dev/null +++ b/server/transport/retirement.py @@ -0,0 +1,28 @@ +"""Outer admission includes request-tail settlement after inner dispatch ends.""" + +from starlette.types import ASGIApp, Receive, Scope, Send + +from server.core import retirement +from server.transport.model_maintenance import _refuse + + +class RetirementMiddleware: + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] not in ("http", "websocket"): + await self.app(scope, receive, send) + return + try: + async with retirement.get_gate().admit(): + await self.app(scope, receive, send) + except retirement.ProcessStopping: + await _refuse( + scope, + receive, + send, + code="PROCESS_STOPPING", + message="This server is stopping. Retry another server.", + retry_after="1", + ) diff --git a/server/worker_main.py b/server/worker_main.py index 6854be4082..4a606a4291 100644 --- a/server/worker_main.py +++ b/server/worker_main.py @@ -20,6 +20,7 @@ from typing import Any from server import boot +from server.core import retirement from server.settings import settings from server.workers import child, leader from server.workers.supervisor import get_supervisor @@ -46,18 +47,20 @@ async def _trackers() -> None: # `with`, not set_query_tag: the supervisor and the shutdown run in this context after. with query_tag("boot.workers_child"): register_buffer_drainer("boot-writes", boot.drain_writes) + retirement.get_gate().install("worker") await register_process() await load_active_configuration() await get_emitter().start() await registry_sources.refresh_db_sources(force=True, ch=ch) if not get_model_work_gate().closed: await boot.calibrate_similarity_gates() + track = retirement.get_gate().track tasks = [ - asyncio.create_task(refresh_loop(), name="provider-configuration-refresh"), - asyncio.create_task(_trackers(), name="loom-tracker-sources-refresh"), - await repo_registry.start_refresher(ch), + track(asyncio.create_task(refresh_loop(), name="provider-configuration-refresh")), + track(asyncio.create_task(_trackers(), name="loom-tracker-sources-refresh")), + track(await repo_registry.start_refresher(ch)), # The workers read LIVE settings too: their passes see the deployment's overrides. - boot.start_runtime_overrides(), + track(boot.start_runtime_overrides()), ] await boot.await_warm_up(settings.boot_warmup_timeout_s, ("runtime_overrides",)) return tasks @@ -113,13 +116,18 @@ async def _serve(fd: int, alive_fd: int) -> None: supervisor.start_all() await stop.wait() log.info("workers child %s stopping", os.getpid()) + retirement.get_gate().close() embedder_calibration.stop_refresh_loop() for task in services: task.cancel() await asyncio.gather(*services, return_exceptions=True) await supervisor.stop_all(drain_timeout_s=leader.drain_timeout_s()) await _drain() + retirement.get_gate().stopped() finally: + retirement.get_gate().close() + for producer in tuple(retirement.get_gate().producers): + producer.cancel() beat.cancel() await boot.leave_deployment(ch, budget_s=SHUTDOWN_RETIRE_S) await boot.close_storage_authority() diff --git a/server/workers/child.py b/server/workers/child.py index 0d782ca1c4..6a24b8510f 100644 --- a/server/workers/child.py +++ b/server/workers/child.py @@ -27,7 +27,8 @@ import signal import sys import time -from collections.abc import Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping +from pathlib import Path from server.core.errors import error_kind from server.settings import settings @@ -58,9 +59,14 @@ class ChildWorkers: """The supervisor's start/stop/status surface, backed by a child process.""" - def __init__(self, lease: leader.SupervisorLease) -> None: + def __init__( + self, lease: leader.SupervisorLease, *, retire_child: Callable[[str], Awaitable[None]] + ) -> None: self._lease = lease + self._retire_callback = retire_child self._proc: asyncio.subprocess.Process | None = None + self._child_proof: str | None = None + self._child_parent_proof: str | None = None self._watch: asyncio.Task[None] | None = None self._publish: asyncio.Task[None] | None = None self._write_fd: int | None = None @@ -120,10 +126,13 @@ async def _keep_running(self) -> None: # The outcome reaches `/v1/health` (no token) as the lease's reason. log.warning("workers child could not start: %s: %s", type(e).__name__, e) outcome = f"could not start ({error_kind(e)})" + restart_at = time.monotonic() + RESTART_DELAY_S else: assert self._proc is not None self.last_exit = await self._proc.wait() + restart_at = time.monotonic() + RESTART_DELAY_S self._close_pipe() + await self._retire_child(self._proc, restart_at) outcome = f"exited (code {self.last_exit})" if not self._lease.fresh(): return # the lease loop stops the workers, and starts them once it is @@ -139,7 +148,7 @@ async def _keep_running(self) -> None: return log.error("workers child %s; restarting in %.0f s", outcome, RESTART_DELAY_S) self.restarts += 1 - await asyncio.sleep(RESTART_DELAY_S) + await asyncio.sleep(max(0.0, restart_at - time.monotonic())) async def _spawn(self) -> None: """Start the child. On an OSError the child's pipe ends are closed here and @@ -163,12 +172,69 @@ async def _spawn(self) -> None: finally: for fd in child_fds: os.close(fd) + self._capture_child_proof() asyncio.get_running_loop().add_reader(self._alive_fd, self._read_beats) log.info("workers child started: pid %s", self._proc.pid) self._publish = asyncio.get_running_loop().create_task( self._publish_loop(), name="loom-workers-child-lease" ) + def _capture_child_proof(self) -> None: + """Capture the owned child's birth while its PID still names our direct child.""" + from server.core import provider_process + + self._child_proof = None + self._child_parent_proof = None + proc = self._proc + if proc is None or proc.returncode is not None or proc.pid <= 1 or proc.pid == os.getpid(): + return + try: + parent_proof = provider_process.host_proof() + parent = parent_proof.split("|") + root = Path(f"/proc/{proc.pid}") + stat = (root / "stat").read_text().rsplit(")", 1)[1].split() + if ( + proc.returncode is not None + or len(parent) != 8 + or parent[4] != str(os.getpid()) + or int(stat[1]) != os.getpid() + or str(os.stat(root / "ns/pid").st_ino) != parent[3] + ): + return + parent[4:6] = [str(proc.pid), stat[19]] + self._child_proof = provider_process.certified("|".join(parent)) + self._child_parent_proof = parent_proof + except (OSError, ValueError, IndexError): + return # An unknown birth never authorizes post-reap retirement. + + async def _retire_child(self, proc: asyncio.subprocess.Process, end: float) -> None: + from server.core import provider_process + + proof = self._child_proof + if proc is not self._proc or proc.returncode is None or proof is None: + return + captured = proof.split("|") + parent_proof = provider_process.host_proof() + here = parent_proof.split("|") + if ( + parent_proof != self._child_parent_proof + or len(captured) != 9 + or not provider_process.has_certificate(proof) + or len(here) != 8 + or captured[:4] != here[:4] + or captured[4] != str(proc.pid) + or proc.pid == os.getpid() + or end <= time.monotonic() + ): + return + try: + async with asyncio.timeout( + min(settings.workers_child_retirement_timeout_s, max(0.0, end - time.monotonic())) + ): + await self._retire_callback(proof) + except Exception: # noqa: BLE001 — unproved retirement must not replace child cleanup + log.warning("reaped workers child retirement remains unproved") + def _send_deadline(self) -> None: """Send the lease deadline; it moves only while the lease is fresh.""" if self._lease.fresh(): @@ -225,6 +291,9 @@ async def _reap(self, kill_at: float) -> None: with contextlib.suppress(ProcessLookupError): proc.kill() await proc.wait() + if proc is not None: + await proc.wait() + await self._retire_child(proc, kill_at) finally: if proc is not None and proc.returncode is None: with contextlib.suppress(ProcessLookupError): @@ -258,12 +327,14 @@ def child_env(parent: Mapping[str, str]) -> dict[str, str]: _RUNNER: WorkerSupervisor | ChildWorkers | None = None -def worker_runner(lease: leader.SupervisorLease) -> WorkerSupervisor | ChildWorkers: +def worker_runner( + lease: leader.SupervisorLease, *, retire_child: Callable[[str], Awaitable[None]] +) -> WorkerSupervisor | ChildWorkers: """What runs the workers here: a child process for an `auto` lease holder (`LOOM_WORKERS_PROCESS=child`), else the in-process supervisor.""" global _RUNNER if lease.mode == "auto" and settings.workers_process == "child": - _RUNNER = ChildWorkers(lease) + _RUNNER = ChildWorkers(lease, retire_child=retire_child) else: _RUNNER = get_supervisor() return _RUNNER diff --git a/server/workers/supervisor.py b/server/workers/supervisor.py index 921bb0ddae..d873424fcd 100644 --- a/server/workers/supervisor.py +++ b/server/workers/supervisor.py @@ -30,6 +30,7 @@ from typing import Any from server.core.model_work_gate import get_model_work_gate +from server.core.retirement import ProcessStopping, get_gate from server.observability.traffic_class import bind_background from server.settings import settings from server.workers import leader, pass_context, status_store @@ -162,6 +163,7 @@ def _spawn(self, spec: WorkerSpec) -> None: _publish_streak(spec.name, 0) _publish_partial(spec.name, 0) task = asyncio.create_task(self._supervise_as_background(spec), name=_task_name(spec)) + get_gate().track(task) self._tasks.append(task) def _restart(self, spec: WorkerSpec, _ended: asyncio.Task[None]) -> None: @@ -280,7 +282,7 @@ async def _supervise(self, spec: WorkerSpec) -> None: try: # A pass past its deadline is cancelled and counted as a # failure (TimeoutError). - async with get_model_work_gate().admit(wait=True): + async with get_gate().admit(), get_model_work_gate().admit(wait=True): # Maintenance waiting consumes neither the pass deadline # nor its duration/accounting. An admitted pass finishes # normally; changing models never cancels it mid-write. @@ -305,6 +307,8 @@ async def _supervise(self, spec: WorkerSpec) -> None: streak=retry_streak, ) ) + except ProcessStopping: + return except pass_context.PassStopped: # The pass saw a stop or a stale lease between batches. Recorded # like a cancel; the loop top waits until the lease is fresh. diff --git a/tests/fixtures/provider_process_before_retirement.py b/tests/fixtures/provider_process_before_retirement.py new file mode 100644 index 0000000000..6ef0d6af5a --- /dev/null +++ b/tests/fixtures/provider_process_before_retirement.py @@ -0,0 +1,173 @@ +"""This process's provider-participant id and the tag its ClickHouse queries carry. + +Every ClickHouse client the process opens names the id in its HTTP User-Agent, +so `system.processes` says whether a participant that stopped heartbeating +still has a query running. +""" + +import asyncio +import os +import re +import socket +from pathlib import Path +from uuid import UUID, uuid4 + +_identity: tuple[int, UUID] | None = None + + +def process_identity() -> UUID: + """One id per process; a forked child draws its own.""" + global _identity + if _identity is None or _identity[0] != os.getpid(): + _identity = (os.getpid(), uuid4()) + return _identity[1] + + +def participant_tag(identity: UUID | str) -> str: + """What `system.processes.http_user_agent` contains for that participant's queries.""" + return f"loom-participant/{UUID(str(identity)).hex}" + + +def client_name() -> str: + """The `client_name` every ClickHouse client of this process is opened with.""" + return participant_tag(process_identity()) + + +# host name | kernel boot id | container | pid namespace | pid | process start +# (clock ticks after boot) | Kubernetes namespace and pod name (POD_NAMESPACE, +# POD_NAME: the host name is the pod name cut to 63 characters; '' elsewhere) +_proof: tuple[int, str] | None = None +# Docker mounts /etc/hostname from its container's directory; a Kubernetes container +# mounts its termination log from the pod's, under the container's name. +_CONTAINER = ( + re.compile(r"/containers/([0-9a-f]{64})/hostname "), + re.compile(r"/pods/([0-9a-f-]{36})/containers/([^/ ]+)/"), +) + + +def _start_ticks(pid: int | str) -> str: + with open(f"/proc/{pid}/stat", encoding="ascii") as f: + return f.read().rsplit(")", 1)[1].split()[19] + + +def _boot_and_pidns() -> tuple[str, str]: + with open("/proc/sys/kernel/random/boot_id", encoding="ascii") as f: + boot = f.read().strip() + return boot, str(os.stat("/proc/self/ns/pid").st_ino) + + +def _container() -> str: + """This container's id, stable across its restarts; '' outside one, or where + the mounts do not say.""" + try: + with open("/proc/self/mountinfo", encoding="utf-8") as f: + mounts = f.read() + except OSError: + return "" + for pattern in _CONTAINER: + found = pattern.search(mounts) + if found: + return "/".join(found.groups()) + return "" + + +def host_proof() -> str: + """What lets a later process on this host prove this one stopped (`proves_stopped`); + '' where /proc does not say.""" + global _proof + if _proof is None or _proof[0] != os.getpid(): + try: + boot, pidns = _boot_and_pidns() + value = "|".join( + ( + socket.gethostname(), + boot, + _container(), + pidns, + str(os.getpid()), + _start_ticks("self"), + os.environ.get("POD_NAMESPACE", ""), + os.environ.get("POD_NAME", ""), + ) + ) + except (OSError, IndexError): + value = "" + _proof = (os.getpid(), value) + return _proof[1] + + +def proves_stopped(proof: str) -> bool: + """Whether the process `proof` names has stopped, judged from this one: same host + name and kernel boot, and either its pid is free or runs another process than + the one that started then (same pid namespace), or it ran in this container + before the container restarted into a new pid namespace (which kills every + process of the old one). Two containers that share a host name never prove + each other stopped; a process that cannot be inspected is not proven stopped.""" + parts = proof.split("|") + if len(parts) not in (6, 7, 8): + return False + host, boot, container, pidns, pid, start = parts[:6] + try: + my_boot, my_pidns = _boot_and_pidns() + except OSError: + return False + if host != socket.gethostname() or boot != my_boot: + return False + if pidns != my_pidns: + return bool(container) and container == _container() + # Only positive evidence: no such process, or another one under its pid. + # A pid this process may not inspect (hidepid, a permission error) proves nothing. + try: + os.kill(int(pid), 0) + except ProcessLookupError: + return True + except (OSError, ValueError): + return False + try: + return _start_ticks(pid) != start + except (OSError, IndexError): + return False + + +_SA = "/var/run/secrets/kubernetes.io/serviceaccount" +_POD_CONTAINER = re.compile(r"^([0-9a-f-]{36})/") + + +async def pod_gone(proof: str, *, timeout_s: float = 5.0) -> bool: + """Whether Kubernetes says the pod `proof` names is gone: no pod of that name + in its namespace, one with another UID, or this one in phase Failed or + Succeeded (an evicted pod keeps its object; a pod that restarts its + containers reaches those phases only once every one stopped for good). The + kubelet removes a pod's object once its containers stopped (or an operator + force-deleted it), so each is positive evidence a lost pod's process ended. + Asked only of this pod's own namespace, with its service-account token; + False wherever the API or the proof does not say.""" + parts = proof.split("|") + found = _POD_CONTAINER.match(parts[2]) if len(parts) == 8 else None + namespace = os.environ.get("POD_NAMESPACE", "") + if not found or not namespace or parts[6] != namespace or not parts[7]: + return False + host = os.environ.get("KUBERNETES_SERVICE_HOST", "") + port = os.environ.get("KUBERNETES_SERVICE_PORT", "443") + try: + token = (await asyncio.to_thread(Path(f"{_SA}/token").read_text, "ascii")).strip() + except OSError: + return False + if not host or not token: + return False + import httpx + + url = f"https://{host}:{port}/api/v1/namespaces/{namespace}/pods/{parts[7]}" + try: + async with httpx.AsyncClient(verify=f"{_SA}/ca.crt", timeout=timeout_s) as client: + r = await client.get(url, headers={"Authorization": f"Bearer {token}"}) + except (httpx.HTTPError, OSError): + return False + if r.status_code == 404: + return True + if r.status_code != 200: + return False + pod = r.json() + if str(pod.get("metadata", {}).get("uid", "")) != found.group(1): + return True + return str(pod.get("status", {}).get("phase", "")) in ("Failed", "Succeeded") diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index e84ef1b5a1..acf7a1b199 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -250,6 +250,16 @@ async def asend(self: httpx.AsyncClient, request: httpx.Request, **kw: Any) -> h mp.undo() +@pytest.fixture(autouse=True) +def isolated_retirement_gate() -> Iterator[None]: + """Each test owns a fresh process gate; shutdown stays permanent within it.""" + from server.core import retirement + + with pytest.MonkeyPatch.context() as patch: + patch.setattr(retirement, "_gate", retirement.RetirementGate()) + yield + + @pytest.fixture(autouse=True) def _code_registry_read() -> Iterator[None]: """In-process code ops answer only once the self-service registry has been read diff --git a/tests/integration/fixtures/pg_migrate_n_minus_one.py b/tests/integration/fixtures/pg_migrate_n_minus_one.py new file mode 100644 index 0000000000..7876c9f178 --- /dev/null +++ b/tests/integration/fixtures/pg_migrate_n_minus_one.py @@ -0,0 +1,151 @@ +"""Postgres migrations: numbered SQL files in `schemas/pg_migrations/`, applied in +order at boot under a session advisory lock, each file in one transaction with +its ledger row (`loom.pg_migrations`, the file's SHA-256). The router's directory +(`schemas/router_migrations/`, `server/router/directory.py`) migrates the same way, +on its own database, ledger schema and lock. For schema `loom`, `migrate` also marks +its session as one that ran the tenancy check and, after the files and before the lock +goes, runs `cell_triggers.ensure` (docs/architecture/cells.md). + +The rules are the ClickHouse journal's (docs/development/migrations.md): additive +only, never edit a merged file. A ledger row whose checksum no longer matches its +file refuses the boot, as does a ledger row for a file this build does not carry +when that file's number is above the build's last one (a newer build's schema is +served, as ClickHouse's is; an unknown lower number means the journal was edited). +""" + +from __future__ import annotations + +import hashlib +import logging +from asyncio import sleep +from collections.abc import Callable +from contextlib import AbstractAsyncContextManager +from pathlib import Path +from typing import Any + +from server.core import pg +from server.core.paths import PG_MIGRATIONS_DIR +from server.core.schema.journal import migration_number, sorted_migrations + +log = logging.getLogger(__name__) + +# A fixed key for pg_advisory_lock: every Loom process migrates under it. +_LOCK_KEY = 0x6C6F6F6D_6D696772 # "loommigr" + + +def _ledger(schema: str) -> str: + # `schema` is one of this build's constants (`loom`, `loom_router`), never input. + return f""" +CREATE SCHEMA IF NOT EXISTS {schema}; +CREATE TABLE IF NOT EXISTS {schema}.pg_migrations ( + filename text PRIMARY KEY, + checksum text NOT NULL, + applied_at timestamptz NOT NULL DEFAULT now() +)""" + + +class MigrationDrift(RuntimeError): + """An applied file changed, or the ledger holds a file the journal lost.""" + + +def migration_files(mig_dir: Path = PG_MIGRATIONS_DIR) -> list[Path]: + return sorted_migrations(mig_dir.glob("*.sql")) + + +def checksum(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def plan(applied: dict[str, str], files: list[Path]) -> list[Path]: + """The files still to apply; raises MigrationDrift on a changed or lost file.""" + known = {f.name: f for f in files} + last = max((migration_number(f.name) or 0 for f in files), default=0) + for name, digest in applied.items(): + f = known.get(name) + if f is None: + if (migration_number(name) or 0) <= last: + raise MigrationDrift(f"applied Postgres migration {name} is not in this build") + continue + if checksum(f) != digest: + raise MigrationDrift(f"applied Postgres migration {name} was edited") + newer = sorted(n for n in applied if n not in known) + if newer: + log.warning("Postgres schema has migrations this build does not carry: %s", newer) + return [f for f in files if f.name not in applied] + + +async def pending(mig_dir: Path = PG_MIGRATIONS_DIR) -> list[str]: + """The files `migrate` would apply, read-only: every file when the ledger is not + there yet. For a process that applies none (`LOOM_BOOT_MIGRATE=0`).""" + import asyncpg + + try: + rows = await pg.fetch("SELECT filename, checksum FROM loom.pg_migrations") + except asyncpg.exceptions.UndefinedTableError: + rows = [] + applied = {r["filename"]: r["checksum"] for r in rows} + return [f.name for f in plan(applied, migration_files(mig_dir))] + + +async def migrate( + mig_dir: Path = PG_MIGRATIONS_DIR, + *, + connect: Callable[[], AbstractAsyncContextManager[Any]] | None = None, + schema: str = "loom", + lock_key: int = _LOCK_KEY, +) -> list[str]: + """Apply every pending file; returns their names. Concurrent boots serialize on + the advisory lock and the later ones find nothing pending. `connect`, `schema` + and `lock_key` name another database's connection, ledger and lock (the router's + directory); the defaults are the memory authority's. + + On a connection of its own, with no client command timeout and no statement + timeout: DDL on a large table runs as long as it takes. A lock it waits for is + bounded (`lock_timeout`), so an `ALTER` queued behind a long transaction never + holds every read and write of the table behind it; the file is retried. For schema + `loom` it marks the session and runs `cell_triggers.ensure` before the lock goes.""" + files = migration_files(mig_dir) + done: list[str] = [] + # Closing the connection ends the session, and with it the advisory lock. + async with (connect or pg.dedicated)() as conn: + # Another boot migrating holds the lock for as long as its files take. + await conn.execute("SELECT pg_advisory_lock($1)", lock_key) + if schema == "loom": + # A migration's data steps only copy or rewrite rows already stored here: the + # org triggers pass them (server/core/cell_triggers.py). + await conn.execute(f"SET {pg.TENANCY_MARKER} = 'on'") + await conn.execute("SET lock_timeout = '5s'") + await conn.execute(_ledger(schema)) + rows = await conn.fetch(f"SELECT filename, checksum FROM {schema}.pg_migrations") + for name, sql, digest in _read(plan({r["filename"]: r["checksum"] for r in rows}, files)): + await _apply(conn, name, sql, digest, schema=schema) + log.info("applied Postgres migration %s", name) + done.append(name) + if schema == "loom": + # On every migrating boot, after the pending files and before the lock goes. + from server.core import cell_triggers + + await cell_triggers.ensure(conn) + return done + + +def _read(files: list[Path]) -> list[tuple[str, str, str]]: + return [(f.name, f.read_text(), checksum(f)) for f in files] + + +async def _apply(conn: Any, name: str, sql: str, digest: str, *, schema: str = "loom") -> None: + import asyncpg + + record = f"INSERT INTO {schema}.pg_migrations (filename, checksum) VALUES ($1, $2)" + attempts = 5 # tries before the boot fails on a table lock held past `lock_timeout` + for attempt in range(1, attempts + 1): + try: + async with conn.transaction(): + await conn.execute(sql) + await conn.execute(record, name, digest) + return + except asyncpg.exceptions.LockNotAvailableError: + if attempt == attempts: + raise + log.warning("Postgres migration %s waited too long for a lock; retrying", name) + await sleep(attempt) diff --git a/tests/integration/fixtures/router_placement_n_minus_one.py b/tests/integration/fixtures/router_placement_n_minus_one.py new file mode 100644 index 0000000000..dea9c92c22 --- /dev/null +++ b/tests/integration/fixtures/router_placement_n_minus_one.py @@ -0,0 +1,141 @@ +"""Frozen pre-generation router placement/query code for schema coexistence tests. + +The serving statements and row model are copied from the checkpoint before the +control generation column. Connection acquisition/error handling are injected from +the test directory; no current placement projection or writer is imported. +""" + +from __future__ import annotations + +import hmac +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from typing import Any + +from server.core import cell as cell_state +from server.router.directory import OrgTooLong, PlacementRefused, _connection, _reading + +_PLACE_LOCK = 0x6C726F70 # "lrop" + + +@dataclass(frozen=True) +class Cell: + """What the router's APIs read of a cell's row.""" + + cell: str + is_first: bool + host: str + state: str + mode: str + closed: bool + placements_max: int | None + placed: int + credential_sha256: str + deployment_id: str + current_fp: dict[str, Any] | None + next_fp: dict[str, Any] | None + # The new credential's hash during a rotation (`loom cell credential --next`). + credential_next_sha256: str | None + + def holds(self, presented: str) -> bool: + """Whether `presented` (a credential's SHA-256) is this cell's, current or next.""" + hashes = (self.credential_sha256, self.credential_next_sha256) + return holds(presented, [h for h in hashes if h]) + + +def holds(presented: str, hashes: Iterable[str]) -> bool: + """Whether `presented` (a credential's SHA-256) is one of `hashes` (each set), in + constant time each.""" + return any(hmac.compare_digest(presented, h) for h in hashes) + + +_CELL_COLUMNS = ( + "cell, is_first, host, state, mode, closed, placements_max, placed, credential_sha256, " + "credential_next_sha256, deployment_id, current_fp, next_fp" +) + +_CELL = f"SELECT {_CELL_COLUMNS} FROM loom_router.cells" + +_WINNER = ( + "SELECT p.cell, c.deployment_id FROM loom_router.placements AS p " + "JOIN loom_router.cells AS c USING (cell) WHERE p.org = $1" +) + + +def _cell(row: Any) -> Cell: + return Cell(**dict(row)) + + +async def place(org: str, cell_name: str, refusal: Callable[[Cell], str]) -> tuple[str, str]: + """Place `org` in `cell_name` unless it is placed already; the (cell, deployment id) + that holds it. An org placed already answers its cell, whatever the cell's state. + Otherwise the cell's row is locked, the winner read again (an attach batch holds the row + lock, not the org's), and the row judged (`refusal`, then its quota), so a change to it + waits for the placement and the quota holds across router replicas; a refused placement + raises PlacementRefused and writes nothing, and a new org name longer than + `cell.ORG_MAX_BYTES` raises OrgTooLong (one placed before the limit answers its cell, as + any placed org does).""" + async with _reading(), _connection() as conn, conn.transaction(): + # Placements of one org run one at a time, whichever cell asks: the later one + # reads the earlier one's winner. + await conn.execute("SELECT pg_advisory_xact_lock($1, hashtext($2))", _PLACE_LOCK, org) + row = await conn.fetchrow(_WINNER, org) + if row is None: + _refuse_long([org]) + locked = await conn.fetchrow(f"{_CELL} WHERE cell = $1 FOR UPDATE", cell_name) + if locked is None: + raise PlacementRefused(f"cell {cell_name!r} is not in the directory") + # An attach batch of this cell, which holds the row lock and not the org's, may + # have placed the org since the first read: it is answered as placed. + row = await conn.fetchrow(_WINNER, org) + if row is None: + await _place_one(conn, org, cell_name, _cell(locked), refusal) + row = await conn.fetchrow(_WINNER, org) + return str(row["cell"]), str(row["deployment_id"]) + + +async def _place_one( + conn: Any, org: str, cell_name: str, candidate: Cell, refusal: Callable[[Cell], str] +) -> None: + """Judge the locked row (`refusal`, then its quota) and insert the placement; its count + and audit row only when the insert wrote it.""" + if reason := refusal(candidate): + raise PlacementRefused(reason) + quota = candidate.placements_max + if quota is not None and candidate.placed >= quota: + raise PlacementRefused(f"cell {cell_name!r} holds its quota of orgs") + inserted = await conn.fetchval( + "INSERT INTO loom_router.placements (org, cell) VALUES ($1, $2) " + "ON CONFLICT DO NOTHING RETURNING org", + org, + cell_name, + ) + if inserted is None: + return + await conn.execute( + "UPDATE loom_router.cells SET placed = placed + 1 WHERE cell = $1", cell_name + ) + await conn.execute( + "INSERT INTO loom_router.audit (actor, action, detail) VALUES ($1, 'place', $2)", + f"cell:{cell_name}", + {"org": org, "cell": cell_name}, + ) + + +def _refuse_long(orgs: Iterable[str]) -> None: + """OrgTooLong for the first org longer than `cell.ORG_MAX_BYTES`: the one check every + route that inserts a placement runs, so no route places such an org.""" + for org in orgs: + if reason := cell_state.org_name_refusal(org): + raise OrgTooLong(reason) + + +async def lookup(org: str) -> tuple[str, str] | None: + """The (cell, host) an org is placed in, or None.""" + async with _reading(), _connection(lookup=True) as conn: + row = await conn.fetchrow( + "SELECT c.cell, c.host FROM loom_router.placements AS p " + "JOIN loom_router.cells AS c USING (cell) WHERE p.org = $1", + org, + ) + return (str(row["cell"]), str(row["host"])) if row else None diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py new file mode 100644 index 0000000000..72f6604578 --- /dev/null +++ b/tests/integration/test_cell_attach.py @@ -0,0 +1,1124 @@ +"""`loom cell attach` (server/cli/cell_attach.py) against a real Postgres with a synchronous +standby, a real ClickHouse and the router served in process: a prepared first cell in +`recording` with orgs in both stores attaches to `serving` with every org placed and the +triggers created; a rerun on `serving` exits 0; a refusal after step 3 leaves `enforced` and +a rerun resumes from there; a second run is refused at the attach lock; the Postgres reads +attach makes refuse what they name (an unchecked writer's setting, a subscription, a role +that can make an org held, a transaction that may have read `recording`). + +The cell's Postgres and the directory are databases of the test's own; the configured +ClickHouse databases are scratch ones; the cell's processes are heartbeat rows the test +writes. Needs LOOM_PG_DSN (a server with a synchronous standby), LOOM_ROUTER_PG_DSN and +LOOM_CH_HOST.""" + +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import hashlib +import json +import os +import sys +import uuid +from collections.abc import AsyncIterator, Iterator +from typing import Any + +import httpx +import pytest + +from server.cli import cell as cell_cli +from server.cli import cell_attach, cell_attach_pg, cell_prepare +from server.cli.cell_ch import Refused +from server.core import cell, cell_probe, cell_triggers, instances, pg +from server.core import ch as ch_core +from server.core.schema import pg_migrate +from server.router import app as router_app +from server.router import directory +from server.settings import settings +from tests.integration._pg_database import create_database, drop_database +from tests.integration._resources import ch_client_from_env +from tests.unit.cell_control_owner import owner as _owner + +OPERATOR = "lco_" + "2" * 128 +CREDENTIAL_SHA = hashlib.sha256(b"lcr_first").hexdigest() +ROUTER = "http://127.0.0.1:7901" +HOST = "loom.example" + + +class _Network(httpx.AsyncBaseTransport): + """What attach and the router reach: the router at ROUTER, the shared host's health + route, the cell's upstream main port (:7777) and its probe listener (any other).""" + + def __init__(self) -> None: + self.listener = httpx.ASGITransport(app=cell_probe.listener_app()) + self.router = httpx.ASGITransport( + app=router_app.build_app(router_app.Router(probe_transport=self)) + ) + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + if request.url.host == "127.0.0.1" and request.url.port == 7901: + return await self.router.handle_async_request(request) + if request.url.host == HOST or request.url.port == 7777: + return httpx.Response(200, json=cell_probe.answer()) + return await self.listener.handle_async_request(request) + + +async def _admin(sql: str, *args: Any) -> Any: + import asyncpg + + conn = await asyncpg.connect(os.environ["LOOM_PG_DSN"]) + try: + return await conn.execute(sql, *args) + finally: + await conn.close() + + +@pytest.fixture(scope="module") +def cell_clickhouse_databases() -> Iterator[tuple[str, str]]: + """Migrate one owned pair; no authority write can replace the suite server's row.""" + from server.core.schema import migrate + + for name in ("LOOM_PG_DSN", "LOOM_ROUTER_PG_DSN", "LOOM_CH_HOST"): + os.environ.get(name) or pytest.skip(f"{name} not set") + suffix = uuid.uuid4().hex + names = (f"loom_it_attach_{suffix}", f"loom_it_attacht_{suffix}") + created: list[str] = [] + admin = ch_client_from_env(database="") + error: BaseException | None = None + try: + for name in names: + admin.command(f"CREATE DATABASE {name}") + created.append(name) + with pytest.MonkeyPatch.context() as patch: + patch.setattr(settings, "ch_database", names[0]) + patch.setattr(settings, "ch_telemetry_database", names[1]) + normal_client = migrate.client + bootstrap = normal_client() + patch.setattr( + migrate, "client", lambda **kwargs: normal_client(**kwargs) if kwargs else bootstrap + ) + try: + assert migrate.main([]) == 0 + finally: + if sys.exception() is None: + bootstrap.close() + else: + with contextlib.suppress(Exception): + bootstrap.close() + admin.close() + admin = None + yield names + except BaseException as caught: # noqa: BLE001 — exact owned cleanup retains the first failure. + error = caught + finally: + if admin is None: + try: + admin = ch_client_from_env(database="") + except BaseException as caught: # noqa: BLE001 — cleanup cannot replace the first failure. + error = error or caught + if admin is not None: + for name in reversed(created): + try: + admin.command(f"DROP DATABASE {name} SYNC") + except BaseException as caught: # noqa: BLE001 — attempt every exact owned cleanup. + error = error or caught + try: + admin.close() + except BaseException as caught: # noqa: BLE001 — close cannot replace the first failure. + error = error or caught + if error is not None: + raise error + + +@pytest.fixture +def genuine_cell_processes() -> bool: + """Synthetic cases own their rows; the live-replica case uses the real readers.""" + return False + + +@pytest.fixture +async def cell_stores( + monkeypatch: pytest.MonkeyPatch, + cell_clickhouse_databases: tuple[str, str], + genuine_cell_processes: bool, +) -> AsyncIterator[dict[str, Any]]: + cell_dsn = os.environ.get("LOOM_PG_DSN") or pytest.skip("LOOM_PG_DSN not set") + router_dsn = os.environ.get("LOOM_ROUTER_PG_DSN") or pytest.skip("LOOM_ROUTER_PG_DSN not set") + os.environ.get("LOOM_CH_HOST") or pytest.skip("LOOM_CH_HOST not set") + cell_db, own_cell = await create_database(cell_dsn) + try: + router_db, own_router = await create_database(router_dsn) + except BaseException as first: # Release only the database already created by this fixture. + try: + await drop_database(cell_dsn, cell_db) + except BaseException as error: # noqa: BLE001 — cleanup retains the creation failure. + first.add_note(f"cell database cleanup also failed: {type(error).__name__}") + raise + deployment = f"dep-{uuid.uuid4().hex[:8]}" + tag = f"it-{uuid.uuid4().hex[:8]}-" + for name, value in { + "cell": "first", + "storage_authority": "postgres", + "pg_dsn": own_cell, + "document_s3_bucket": "", + "router_pg_dsn": own_router, + "router_shared_host": HOST, + "router_upstream_probe_url": "http://listener.internal:7790", + "router_upstream_url": "http://upstream.internal:7777", + "router_operator_sha256": hashlib.sha256(OPERATOR.encode()).hexdigest(), + "router_operator_token": OPERATOR, + "ch_cluster": "", + "ch_database": cell_clickhouse_databases[0], + "ch_telemetry_database": cell_clickhouse_databases[1], + "public_base_url": f"https://{HOST}", + "oauth_issuer": "", + }.items(): + monkeypatch.setattr(settings, name, value) + monkeypatch.setattr(directory, "_pool", None) + # A step's poll interval and the waits stay short. + monkeypatch.setattr(cell_attach_pg, "POLL_S", 0.2) + ch = ch_core.CHClient() + monkeypatch.setattr(ch_core, "_ch", ch) + ids: list[str] = [] + real_live, real_recent = instances.read_live, instances.read_recent + + async def read_live(ch: Any, *, all_deployments: bool = False) -> list[dict[str, Any]]: + rows = await real_live(ch, all_deployments=all_deployments) + return [r for r in rows if r["instance_id"] in ids] + + async def read_recent( + ch: Any, *, window_s: float, since: str | None = None + ) -> list[dict[str, Any]]: + rows = await real_recent(ch, window_s=window_s, since=since) + return [r for r in rows if r["instance_id"] in ids] + + # These assertions inspect only the heartbeat identities created by this fixture. + if not genuine_cell_processes: + monkeypatch.setattr(instances, "read_live", read_live) + monkeypatch.setattr(instances, "read_recent", read_recent) + try: + await pg.close_pool() + await ch.connect() + await pg_migrate.migrate() + await pg.execute( + "INSERT INTO loom.authority (authority, epoch, deployment_id) " + "VALUES ('postgres', 1, $1)", + deployment, + ) + await directory.migrate() + await ch.insert( + "loom.storage_authority", + [ + { + "authority": "postgres", + "epoch": 1 if genuine_cell_processes else 10**12, + "deployment_id": deployment, + } + ], + ) + cell.set_deployment_id(deployment) + yield {"ch": ch, "ids": ids, "tag": tag, "deployment": deployment, "db": cell_db} + finally: + cell.set_deployment_id("") + await _close_cell_stores( + ch, deployment, ids, tag, (cell_dsn, cell_db), (router_dsn, router_db), sys.exception() + ) + + +async def _close_cell_stores( + ch: Any, + deployment: str, + ids: list[str], + tag: str, + cell_database: tuple[str, str], + router_database: tuple[str, str], + first: BaseException | None, +) -> None: + # Every operation owns an exact database, deployment, heartbeat id or org prefix. + cleanup = [ + ch.execute( + "ALTER TABLE loom.storage_authority DELETE WHERE deployment_id = {d:String} " + "SETTINGS mutations_sync = 2", + {"d": deployment}, + ), + *( + ch.execute( + f"ALTER TABLE {table} DELETE WHERE has({{ids:Array(String)}}, instance_id)", + {"ids": ids}, + ) + for table in (instances.TABLE, instances.CELL_TABLE) + ), + ch.execute("DELETE FROM loom.namespaces WHERE startsWith(org, {p:String})", {"p": tag}), + ch.disconnect(), + pg.close_pool(), + directory.close(), + drop_database(*cell_database), + drop_database(*router_database), + ] + for operation in cleanup: + try: + await operation + except BaseException as error: # noqa: BLE001 — attempt every owned cleanup; keep first failure. + if first is None: + first = error + else: + first.add_note(f"store cleanup also failed: {type(error).__name__}") + if first is not None: + raise first + + +def _value(**keys: Any) -> str: + reported = { + **cell.as_value(cell.mine()), + "serves": True, + "marked": True, + "router": True, + "router_why": "", + **keys, + } + return json.dumps(reported, sort_keys=True) + + +async def _beat( + stores: dict[str, Any], + instance_id: str, + *, + cell_name: str | None = "first", + mode: str = "recording", + stopped: bool = False, + **keys: Any, +) -> None: + """One process's heartbeat rows, as `instances.tick` writes them.""" + ch = stores["ch"] + if instance_id not in stores["ids"]: + stores["ids"].append(instance_id) + await ch.insert( + instances.TABLE, [{**instances.row(stopped=stopped), "instance_id": instance_id}] + ) + if cell_name is not None: + await ch.insert( + instances.CELL_TABLE, + [ + { + "instance_id": instance_id, + "cell": cell_name, + "tenancy_mode": mode, + "fingerprint": _value(**keys), + } + ], + ) + + +async def _prepare(stores: dict[str, Any], network: _Network) -> None: + pid = f"it-{uuid.uuid4().hex[:8]}" + await _beat(stores, pid, cell_name="first", mode="open") + args = argparse.Namespace( + cell="first", + host=HOST, + credential_sha256=CREDENTIAL_SHA, + probe_url="", + wait=0, + router=ROUTER, + ) + router = cell_cli.operator_router(args) + assert isinstance(router, httpx.URL) + assert await cell_prepare.prepare(stores["ch"], args, router, network) == 0 + stores["pid"] = pid + await _beat(stores, pid, mode="recording") + + +def _args(**over: Any) -> argparse.Namespace: + fields: dict[str, Any] = { + "cell": "first", + "wait": 20, + "scan_budget": 600, + "dry_run": False, + "ch_allow_user": [settings.ch_user or "default"], + "ch_allow_user_directory": [], + "ch_allow_object": [], + "ch_allow_replica": [], + "pg_allow_role": [], + "router": ROUTER, + } + return argparse.Namespace(**{**fields, **over}) + + +async def _attach(stores: dict[str, Any], network: _Network, **over: Any) -> int: + args = _args(**over) + url = cell_cli.operator_router(args) + assert isinstance(url, httpx.URL) + return await cell_attach.attach(stores["ch"], args, url, network) + + +async def _mode() -> str: + return str(await pg.fetchval("SELECT mode FROM loom.tenancy_mode")) + + +async def _placed() -> set[str]: + import asyncpg + + conn = await asyncpg.connect(settings.router_pg_dsn) + try: + rows = await conn.fetch("SELECT org FROM loom_router.placements WHERE cell = 'first'") + finally: + await conn.close() + return {r["org"] for r in rows} + + +async def _row() -> Any: + return await directory.first_cell_named("first") + + +@pytest.mark.asyncio +async def test_a_recording_first_cell_attaches_to_serving(cell_stores, capsys) -> None: + stores, network = cell_stores, _Network() + tag = stores["tag"] + await _prepare(stores, network) + capsys.readouterr() + # Orgs a recording cell holds: a tenancy row alone, a Postgres data row, a ClickHouse + # row, and one name no request can carry (skipped and named). + await pg.execute("INSERT INTO loom.tenancy (org) VALUES ($1)", tag + "signed-in") + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", tag + "pg") + await stores["ch"].insert( + "loom.namespaces", + [ + {"org": tag + "ch", "namespace": "n", "owner_email": ""}, + {"org": tag + "bad\x00org", "namespace": "n", "owner_email": ""}, + ], + ) + assert await _attach(stores, network) == 0 + report = json.loads(capsys.readouterr().out) + mine = {tag + "signed-in", tag + "pg", tag + "ch"} + assert report["state"] == "serving" + assert await _mode() == "enforced" + placed = await _placed() + assert {o for o in placed if o.startswith(tag)} == mine + held = {r["org"] for r in await pg.fetch("SELECT org FROM loom.tenancy")} + assert held == placed + assert any("clickhouse loom.namespaces" in k for k in report["skipped"]["malformed"]) + row = await _row() + assert (row.state, row.mode, row.placed) == ("serving", "enforced", len(placed)) + async with pg.dedicated() as conn: + assert await cell_triggers.missing(conn) == [] + # A rerun on `serving` is done. + assert await _attach(stores, network) == 0 + assert json.loads(capsys.readouterr().out)["done"] is True + + +@pytest.mark.asyncio +async def test_a_refusal_after_step_3_leaves_enforced_and_a_rerun_resumes( + cell_stores, monkeypatch, capsys +) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + org = stores["tag"] + "a" + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", org) + # A process with LOOM_CELL unset appears once the mode is committed: refused at once. + real = cell_attach.Run.durable + + async def durable_then_unset(self: Any) -> int: + xmax = await real(self) + await _beat(stores, f"it-unset-{uuid.uuid4().hex[:6]}", cell_name="") + return xmax + + monkeypatch.setattr(cell_attach.Run, "durable", durable_then_unset) + assert await _attach(stores, network) == 1 + assert "runs with LOOM_CELL unset" in capsys.readouterr().err + assert await _mode() == "enforced" + row = await _row() + assert (row.state, row.mode) == ("attaching", "enforced") + # Placed live once enforced; the rerun resumes from the durable read. + monkeypatch.setattr(cell_attach.Run, "durable", real) + for i in list(stores["ids"]): + if i.startswith("it-unset-"): + await _beat(stores, i, cell_name="first", mode="enforced") + await _beat(stores, stores["pid"], mode="enforced") + assert await _attach(stores, network) == 0 + assert org in await _placed() + assert (await _row()).state == "serving" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("defect", ["trigger", "catalog"]) +async def test_a_defect_during_the_rescan_keeps_the_cell_attaching( + cell_stores, monkeypatch, capsys, defect: str +) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + role = f"it_rescan_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + real_scan = cell_attach.Run.scan + + async def scan_then_defect(self: Any, *args: Any) -> Any: + found = await real_scan(self, *args) + if self.mode == "enforced": + if defect == "trigger": + await pg.execute( + f"ALTER TABLE loom.scope_generations DISABLE TRIGGER {cell_triggers.INSERT_TRIGGER}" + ) + else: + await pg.execute(f"GRANT INSERT (org) ON loom.tenancy TO {role}") + return found + + monkeypatch.setattr(cell_attach.Run, "scan", scan_then_defect) + try: + assert await _attach(stores, network) == 1 + refused = capsys.readouterr().err + assert ("postgres triggers" if defect == "trigger" else f"postgres role {role}") in refused + assert await _mode() == "enforced" + assert (await _row()).state == "attaching" + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_a_second_run_is_refused_at_the_lock(cell_stores, capsys) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + async with pg.dedicated() as holder: + await cell_attach_pg.take_lock(holder, "the test") + assert await _attach(stores, network) == 1 + assert "attach lock: held by pid" in capsys.readouterr().err + assert await _mode() == "recording" + + +@pytest.mark.asyncio +async def test_a_dry_run_writes_nothing(cell_stores, capsys) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + capsys.readouterr() + org = stores["tag"] + "dry" + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", org) + assert await _attach(stores, network, dry_run=True) == 0 + report = json.loads(capsys.readouterr().out) + assert report["per_source"]["postgres loom.scope_generations"] == { + "count": 1, + "sample": [org], + } + assert await _mode() == "recording" + assert await _placed() == set() + assert await pg.fetchval("SELECT count(*) FROM loom.tenancy") == 0 + async with pg.dedicated() as conn: + assert any(d.kind == "absent" for d in await cell_triggers.missing(conn)) + + +@pytest.mark.asyncio +async def test_an_org_past_the_limit_refuses_before_any_tenancy_row(cell_stores, capsys) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + long = "o" * (cell.ORG_MAX_BYTES + 1) + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", long) + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ('org-ok', '*')") + assert await _attach(stores, network) == 1 + assert f"longer than {cell.ORG_MAX_BYTES} bytes" in capsys.readouterr().err + assert await _mode() == "recording" + assert await pg.fetchval("SELECT count(*) FROM loom.tenancy") == 0 + assert (await _row()).placed == 0 + + +@pytest.mark.asyncio +async def test_settings_and_subscriptions_that_pass_unchecked_writers_refuse( + cell_stores, capsys +) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + db = stores["db"] + role = f"it_role_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + try: + # Stored as first spelled: a fresh session that never set the placeholder. + await _admin(f'ALTER ROLE {role} IN DATABASE "{db}" SET "Loom.Tenancy_Checked" = \'on\'') + async with pg.dedicated() as conn: + setting = await conn.fetchval( + "SELECT setconfig::text FROM pg_db_role_setting s JOIN pg_roles r " + "ON r.oid = s.setrole WHERE r.rolname = $1", + role, + ) + assert "Loom.Tenancy_Checked" in setting + refusals = await cell_attach_pg.unmarked_refusals(conn, False) + assert [r.subject for r in refusals] == [f"postgres setting for {role} in {db}"] + # Another database's entry does not refuse. + await _admin(f'ALTER ROLE {role} IN DATABASE "{db}" RESET ALL') + await _admin( + f"ALTER ROLE {role} IN DATABASE postgres SET session_replication_role = 'replica'" + ) + async with pg.dedicated() as conn: + assert await cell_attach_pg.unmarked_refusals(conn, False) == [] + # A role that can insert a tenancy row, a column grant alone, refuses attach. + await pg.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + await pg.execute(f"GRANT INSERT (org) ON loom.tenancy TO {role}") + assert await _attach(stores, network) == 1 + err = capsys.readouterr().err + assert f"postgres role {role}: may insert into loom.tenancy" in err + assert await _mode() == "recording" + assert await _attach(stores, network, pg_allow_role=[role], dry_run=True) == 0 + finally: + await pg.execute(f"REVOKE ALL ON loom.tenancy FROM {role}") + await pg.execute(f"REVOKE ALL ON SCHEMA loom FROM {role}") + await _admin(f"ALTER ROLE {role} IN DATABASE postgres RESET ALL") + await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_a_disabled_subscription_refuses(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + name = f"it_sub_{uuid.uuid4().hex[:6]}" + await pg.execute( + f"CREATE SUBSCRIPTION {name} CONNECTION 'host=127.0.0.1 dbname=none' " + "PUBLICATION none WITH (connect = false, enabled = false)" + ) + try: + async with pg.dedicated() as conn: + refusals = await cell_attach_pg.unmarked_refusals(conn, False) + assert [str(r) for r in refusals] == [ + f"postgres subscription {name}: disabled: its apply worker fires no origin " + "trigger; drop it" + ] + finally: + await pg.execute(f"ALTER SUBSCRIPTION {name} SET (slot_name = NONE)") + await pg.execute(f"DROP SUBSCRIPTION {name}") + + +@pytest.mark.asyncio +async def test_step_4_waits_for_a_transaction_that_may_have_read_recording(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + async with pg.dedicated() as writer, pg.dedicated() as poller: + tx = writer.transaction() + await tx.start() + await writer.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ('w', '*')") + # X is read after a later transaction completed, as after the durable read. + await poller.execute("BEGIN; SELECT pg_current_xact_id(); COMMIT") + xmax = await cell_attach_pg.snapshot_xmax(poller) + with pytest.raises(Refused, match="may have read `recording`"): + await cell_attach_pg.wait_transactions(poller, xmax, cell_attach_pg.Wait(0.5)) + await tx.rollback() + # A session idle outside a transaction holds nothing: the wait passes. + await cell_attach_pg.wait_transactions(poller, xmax, cell_attach_pg.Wait(0.5)) + + +@pytest.mark.asyncio +async def test_the_session_checks_and_the_encoding(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + async with pg.dedicated() as conn: + assert await cell_attach_pg.direct_refusals(conn, "attach") == [] + assert await cell_attach_pg.encoding_refusals(conn) == [] + async with pg.dedicated(marked=False) as conn: + refusals = await cell_attach_pg.direct_refusals(conn, "attach") + assert [r.reason.split(":")[0] for r in refusals] == [ + f"attach's session loses {pg.TENANCY_MARKER}" + ] + name = f"it_latin_{uuid.uuid4().hex[:6]}" + await _admin( + f"CREATE DATABASE {name} ENCODING 'LATIN1' LC_COLLATE 'C' LC_CTYPE 'C' TEMPLATE template0" + ) + try: + import asyncpg + + from tests.integration._pg_database import with_database + + conn = await asyncpg.connect(with_database(os.environ["LOOM_PG_DSN"], name)) + try: + (refusal,) = await cell_attach_pg.encoding_refusals(conn) + assert "LATIN1, not UTF8" in refusal.reason + finally: + await conn.close() + finally: + await _admin(f"DROP DATABASE {name}") + + +@pytest.mark.asyncio +async def test_the_mode_commit_waits_out_a_for_share_holder(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + logged: list[str] = [] + + async def ok() -> list[str]: + return [] + + async with pg.dedicated() as holder, pg.dedicated() as conn: + tx = holder.transaction() + await tx.start() + await holder.execute("SELECT mode FROM loom.tenancy_mode FOR SHARE") + + async def release() -> None: + await asyncio.sleep(3) + await tx.rollback() + + task = asyncio.create_task(release()) + try: + await cell_attach_pg.take_lock(conn, "mode test") + own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") + await cell_attach_pg.commit_enforced( + _owner(conn), + ok, + ok, + cell_attach_pg.Wait(20), + logged.append, + lambda _m: None, + expected_cell="first", + expected_fp=(own["current_fp"], own["next_fp"]), + ) + await task + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + assert await _mode() == "enforced" + assert logged == ["the mode's lock is busy; retrying"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("membership", ["set", "admin"]) +async def test_a_login_can_lower_mode_through_a_noninherited_group( + cell_stores, membership: str +) -> None: + await _prepare(cell_stores, _Network()) + group = f"it_group_{uuid.uuid4().hex[:6]}" + login = f"it_login_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {group} NOLOGIN") + try: + await _admin(f"CREATE ROLE {login} LOGIN") + async with pg.dedicated() as conn: + await conn.execute(f"GRANT USAGE ON SCHEMA loom TO {group}") + await conn.execute(f"GRANT UPDATE (mode) ON loom.tenancy_mode TO {group}") + flags = ( + "ADMIN FALSE, INHERIT FALSE, SET TRUE" + if membership == "set" + else "ADMIN TRUE, INHERIT FALSE, SET FALSE" + ) + await conn.execute(f"GRANT {group} TO {login} WITH {flags}") + assert not await conn.fetchval( + "SELECT has_column_privilege($1, 'loom.tenancy_mode', 'mode', 'UPDATE')", login + ) + holders = await cell_attach_pg.privilege_holders(conn, exempt_self=True) + await conn.execute(f"SET SESSION AUTHORIZATION {login}") + if membership == "admin": + await conn.execute(f"GRANT {group} TO {login} WITH SET TRUE") + await conn.execute(f"SET ROLE {group}") + assert ( + await conn.execute("UPDATE loom.tenancy_mode SET mode = 'enforced'") == "UPDATE 1" + ) + assert ( + await conn.execute("UPDATE loom.tenancy_mode SET mode = 'recording'") == "UPDATE 1" + ) + await conn.execute("RESET SESSION AUTHORIZATION") + assert "may update loom.tenancy_mode (mode)" in holders.others.get(login, []) + assert login not in holders.own + assert any( + r.subject == f"postgres role {login}" + for r in cell_attach_pg.privilege_refusals(holders, []) + ) + assert not any( + r.subject == f"postgres role {login}" + for r in cell_attach_pg.privilege_refusals(holders, [login]) + ) + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {login}, {group}") + await _admin(f"DROP ROLE IF EXISTS {login}") + await _admin(f"DROP ROLE {group}") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "grant,why", + [ + ("GRANT INSERT (org) ON loom.tenancy TO {r}", "may insert into loom.tenancy"), + ("GRANT UPDATE (org) ON loom.tenancy TO {r}", "may update loom.tenancy (org)"), + ("GRANT UPDATE (mode) ON loom.tenancy_mode TO {r}", "may update loom.tenancy_mode (mode)"), + ("GRANT INSERT (mode) ON loom.tenancy_mode TO {r}", "may insert into loom.tenancy_mode"), + ( + "GRANT INSERT, TRUNCATE ON loom.tenancy_mode TO {r}", + "may insert into loom.tenancy_mode", + ), + ( + "GRANT TRIGGER ON loom.scope_generations TO {r}", + "holds TRIGGER on loom.scope_generations", + ), + ("GRANT CREATE ON SCHEMA public TO {r}", "may create in schema public"), + ("GRANT CREATE ON SCHEMA loom TO {r}", "may create in schema loom"), + ( + "GRANT SET ON PARAMETER session_replication_role TO {r}", + "may set session_replication_role", + ), + ( + "SET loom.tenancy_checked = 'off'; " + "GRANT ALTER SYSTEM ON PARAMETER loom.tenancy_checked TO {r}", + "may ALTER SYSTEM SET loom.tenancy_checked", + ), + ("GRANT pg_write_server_files TO {r}", "is a member of pg_write_server_files"), + # A member of the owner role, ADMIN only: it can grant itself the rest. + ("GRANT {owner} TO {r} WITH ADMIN TRUE, INHERIT FALSE, SET FALSE", "owns loom.tenancy"), + ], +) +async def test_each_right_that_makes_an_org_held_is_named( + cell_stores, grant: str, why: str +) -> None: + await _prepare(cell_stores, _Network()) + role = f"it_r_{uuid.uuid4().hex[:6]}" + owner = await pg.fetchval("SELECT current_user") + await _admin(f"CREATE ROLE {role} LOGIN") + try: + async with pg.dedicated() as conn: + await conn.execute("GRANT USAGE ON SCHEMA loom TO " + role) + for sql in grant.format(r=role, owner=owner).split("; "): + await conn.execute(sql) + holders = await cell_attach_pg.privilege_holders(conn, exempt_self=True) + assert owner in holders.own + assert any(why in w for w in holders.others.get(role, [])), holders.others + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_a_function_owner_counts_though_it_owns_no_table(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + role = f"it_f_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + try: + async with pg.dedicated() as conn: + await conn.execute(f"ALTER FUNCTION loom.tenancy_refuse_unheld() OWNER TO {role}") + holders = await cell_attach_pg.privilege_holders(conn, exempt_self=True) + await conn.execute("ALTER FUNCTION loom.tenancy_refuse_unheld() OWNER TO CURRENT_USER") + assert holders.others[role] == ["owns loom.tenancy_refuse_unheld()"] + finally: + await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_a_repeatable_read_snapshot_from_before_x_holds_step_4(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + async with pg.dedicated() as reader, pg.dedicated() as poller: + tx = reader.transaction(isolation="repeatable_read") + await tx.start() + await reader.fetchval("SELECT mode FROM loom.tenancy_mode") # its snapshot, no xid + await poller.execute("BEGIN; SELECT pg_current_xact_id(); COMMIT") + xmax = await cell_attach_pg.snapshot_xmax(poller) + with pytest.raises(Refused, match="may have read `recording`"): + await cell_attach_pg.wait_transactions(poller, xmax, cell_attach_pg.Wait(0.5)) + await tx.rollback() + + +@pytest.mark.asyncio +async def test_prepare_is_refused_while_attach_holds_the_lock(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + args = argparse.Namespace( + cell="first", + host=HOST, + credential_sha256=CREDENTIAL_SHA, + probe_url="", + wait=0, + router=ROUTER, + ) + router = cell_cli.operator_router(args) + assert isinstance(router, httpx.URL) + async with pg.dedicated() as holder: + await cell_attach_pg.take_lock(holder, "attach") + with pytest.raises(cell_prepare.Refused, match="attach lock: held by pid"): + await cell_prepare.prepare(cell_stores["ch"], args, router, _Network()) + + +@pytest.mark.asyncio +async def test_doctor_names_the_same_holders_settings_and_subscriptions(cell_stores) -> None: + from server.cli import doctor_cell + + await _prepare(cell_stores, _Network()) + role = f"it_d_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + try: + await pg.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + await pg.execute(f"GRANT INSERT ON loom.tenancy TO {role}") + await _admin(f"ALTER ROLE {role} SET session_replication_role = 'replica'") + lines = {n: (g, d) for n, g, d in await doctor_cell._writer_reads(())} + assert lines["cell_tenancy_privilege"][0] == "WARN" + assert f"postgres role {role}" in lines["cell_tenancy_privilege"][1] + assert lines["cell_unchecked_writers"][0] == "WARN" + allowed = {n: g for n, g, _d in await doctor_cell._writer_reads((role,))} + assert allowed["cell_tenancy_privilege"] == "PASS" + finally: + await pg.execute(f"REVOKE ALL ON loom.tenancy FROM {role}") + await pg.execute(f"REVOKE ALL ON SCHEMA loom FROM {role}") + await _admin(f"ALTER ROLE {role} RESET ALL") + await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_doctor_names_stray_orgs_smallest_tables_first(cell_stores) -> None: + from server.cli import doctor_cell + + stores = cell_stores + await _prepare(stores, _Network()) + tag = stores["tag"] + await pg.execute("INSERT INTO loom.tenancy (org) VALUES ($1)", tag + "held") + await pg.execute( + "INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", tag + "held" + ) + await pg.execute( + "INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", tag + "stray" + ) + lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 300)} + grade, detail = lines["cell_stray_orgs"] + assert grade == "WARN" and tag + "stray" in detail and tag + "held" not in detail + + +@pytest.mark.asyncio +async def test_doctor_reads_smallest_first_and_leaves_the_largest_unchecked( + cell_stores, monkeypatch +) -> None: + from server.cli import cell_scan, doctor_cell + + stores = cell_stores + await _prepare(stores, _Network()) + tag = stores["tag"] + await pg.execute("CREATE TABLE loom.it_small (org text PRIMARY KEY)") + await pg.execute("CREATE TABLE loom.it_large (org text PRIMARY KEY)") + await pg.execute("INSERT INTO loom.it_small VALUES ($1)", tag + "small") + await pg.execute( + "INSERT INTO loom.it_large SELECT $1 || g FROM generate_series(1, 20000) g", tag + "l" + ) + await pg.execute("ANALYZE loom.it_small, loom.it_large") + # This ordering check owns two Postgres tables; shared ClickHouse logs keep growing. + monkeypatch.setattr( + cell_triggers, + "ORG_TABLES", + "SELECT relname FROM pg_class WHERE oid IN " + "('loom.it_small'::regclass, 'loom.it_large'::regclass)", + ) + + async def clickhouse_inventory(*_args: Any) -> tuple[list[Any], list[Any]]: + return [], [] + + monkeypatch.setattr(cell_scan, "list_clickhouse", clickhouse_inventory) + clock = [0.0] + real_budget = cell_scan.Budget + monkeypatch.setattr( + cell_scan, "Budget", lambda seconds: real_budget(seconds, clock=lambda: clock[0]) + ) + # The budget runs out on the largest table, as it does on a real cell when that table's + # read is what is too slow: every smaller one was read first. + real, read = cell_scan.pg_orgs, [] + + async def pg_orgs(table: str, budget: Any) -> list[str]: + read.append(table) + if table == "loom.it_large": + clock[0] += budget.left() + 0.1 + budget.require(table) + return await real(table, budget) + + monkeypatch.setattr(cell_scan, "pg_orgs", pg_orgs) + lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 3)} + assert read == ["loom.it_small", "loom.it_large"] + assert tag + "small" in lines["cell_stray_orgs"][1] + grade, detail = lines["cell_stray_orgs_unchecked"] + assert grade == "WARN" and detail.endswith(": loom.it_large") + + +@pytest.mark.asyncio +async def test_signaling_another_role_requires_an_effective_signal_grant(cell_stores) -> None: + import asyncpg + + caller = f"it_signal_{uuid.uuid4().hex[:6]}" + target = f"it_target_{uuid.uuid4().hex[:6]}" + password = uuid.uuid4().hex + await _admin(f"CREATE ROLE {caller} LOGIN") + try: + await _admin(f"CREATE ROLE {target} LOGIN PASSWORD '{password}'") + victim = await asyncpg.connect(settings.pg_dsn, user=target, password=password) + try: + pid = await victim.fetchval("SELECT pg_backend_pid()") + async with pg.dedicated() as admin, pg.dedicated() as signal: + await admin.execute(f"GRANT pg_signal_backend TO {caller} WITH INHERIT FALSE") + await signal.execute(f"SET SESSION AUTHORIZATION {caller}") + with pytest.raises(asyncpg.InsufficientPrivilegeError): + await signal.fetchval("SELECT pg_terminate_backend($1)", pid) + await admin.execute(f"GRANT pg_signal_backend TO {caller} WITH INHERIT TRUE") + assert await signal.fetchval("SELECT pg_terminate_backend($1)", pid) + finally: + await victim.close() + finally: + await _admin(f"DROP ROLE {caller}") + await _admin(f"DROP ROLE IF EXISTS {target}") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("capability", ["pg_read_all_stats"]) +async def test_attach_needs_its_predefined_capabilities_without_set_role( + cell_stores, monkeypatch, capsys, capability: str +) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + role = f"it_caps_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + real_dedicated = pg.dedicated + real_connect = pg._connect_dedicated + + async def as_attacher(**kwargs: Any) -> Any: + conn = await real_connect(**kwargs) + await conn.execute(f"SET SESSION AUTHORIZATION {role}") + return conn + + try: + async with real_dedicated() as conn: + owner = await conn.fetchval("SELECT current_user") + await conn.execute(f"GRANT {owner} TO {role} WITH INHERIT TRUE, SET FALSE") + await conn.execute(f"GRANT pg_read_all_stats TO {role}") + await conn.execute(f"GRANT {capability} TO {role} WITH INHERIT FALSE") + assert await conn.fetchval("SELECT pg_has_role($1, $2, 'MEMBER')", role, capability) + assert not await conn.fetchval("SELECT pg_has_role($1, $2, 'USAGE')", role, capability) + before = await _row() + with monkeypatch.context() as own: + own.setattr(pg, "_connect_dedicated", as_attacher) + assert await _attach(stores, network) == 1 + assert ( + f"attach's role lacks effective privileges of {capability}" in capsys.readouterr().err + ) + after = await _row() + assert (after.state, after.mode, after.placed) == (before.state, before.mode, before.placed) + assert await _mode() == "recording" + assert await pg.fetchval("SELECT count(*) FROM loom.tenancy") == 0 + async with real_dedicated() as conn: + await conn.execute(f"GRANT {capability} TO {role} WITH INHERIT TRUE") + with monkeypatch.context() as own: + own.setattr(pg, "_connect_dedicated", as_attacher) + assert await _attach(stores, network) == 0, capsys.readouterr().err + assert (await _row()).state == "serving" + finally: + async with real_dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_attach_requires_every_probe_privilege(cell_stores) -> None: + role = f"it_probe_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + privileges = ("SELECT", "INSERT", "UPDATE", "DELETE") + grants = [ + ("SELECT",), + *(tuple(p for p in privileges if p != missing) for missing in privileges), + privileges, + ] + observed: dict[tuple[str, ...], list[str]] = {} + try: + async with pg.dedicated() as conn: + await conn.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + for held in grants: + await conn.execute(f"REVOKE ALL ON loom.cell_probe FROM {role}") + await conn.execute(f"GRANT {', '.join(held)} ON loom.cell_probe TO {role}") + await conn.execute(f"SET ROLE {role}") + refusals = await cell_attach_pg.own_role_refusals(conn, []) + await conn.execute("RESET ROLE") + observed[held] = [r.reason for r in refusals if "loom.cell_probe" in r.reason] + assert observed == { + held: ( + [] + if held == privileges + else ["attach's role lacks SELECT, INSERT, UPDATE and DELETE on loom.cell_probe"] + ) + for held in grants + } + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_a_role_with_the_grants_but_not_the_owner_role_is_refused_before_any_write( + cell_stores, +) -> None: + from server.core import cell_triggers + + await _prepare(cell_stores, _Network()) + role = f"it_o_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + try: + async with pg.dedicated() as conn: + await conn.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + await conn.execute( + f"GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA loom TO {role}" + ) + await conn.execute(f"SET ROLE {role}") + rows = list(await conn.fetch(cell_triggers.ORG_TABLES)) + refusals = await cell_attach_pg.own_role_refusals(conn, rows) + lacking = [r.reason for r in refusals if "owner role" in r.reason] + assert ( + "attach's role lacks effective privileges of the owner role of loom.tenancy" + in lacking + ) + assert any("loom.scope_generations" in r for r in lacking) + # The trigger pass tells the cause apart: denied, not a busy lock. + outcome = await cell_triggers.ensure_table( + conn, "scope_generations", recording_first=True + ) + assert outcome.answer == "denied" + await conn.execute("RESET ROLE") + assert await _mode() == "recording" + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_doctor_names_a_table_it_cannot_read_and_reads_the_rest(cell_stores) -> None: + from server.cli import doctor_cell + + stores = cell_stores + await _prepare(stores, _Network()) + tag = stores["tag"] + # An org table with no org-led index: its skip scan refuses, the read goes on. + await pg.execute("CREATE TABLE loom.it_no_org_index (id int PRIMARY KEY, org text)") + await pg.execute("INSERT INTO loom.it_no_org_index VALUES (1, $1)", tag + "unindexed") + await pg.execute( + "INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", tag + "stray" + ) + lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 300)} + assert tag + "stray" in lines["cell_stray_orgs"][1] + grade, detail = lines["cell_stray_orgs_unread"] + assert grade == "WARN" and "postgres loom.it_no_org_index" in detail + assert "cell_stray_orgs_unchecked" not in lines + + +@pytest.mark.asyncio +async def test_attach_exempts_the_role_it_runs_as_and_doctor_does_not(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + role = f"it_ci_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + try: + async with pg.dedicated() as conn: + await conn.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + await conn.execute(f"GRANT SELECT ON ALL TABLES IN SCHEMA loom TO {role}") + await conn.execute(f"GRANT INSERT ON loom.tenancy TO {role}") + await conn.execute(f"SET ROLE {role}") + as_attach = await cell_attach_pg.privilege_holders(conn, exempt_self=True) + as_doctor = await cell_attach_pg.privilege_holders(conn, exempt_self=False) + await conn.execute("RESET ROLE") + # Attach runs as the operator's role; a CI role doctor runs as is warned about. + assert role in as_attach.own and role not in as_attach.others + assert role not in as_doctor.own + assert "may insert into loom.tenancy" in as_doctor.others[role] + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_wait_0_attaches_an_idle_cell(cell_stores, capsys) -> None: + """--wait 0 is one try of each wait, no retry: an idle cell still reaches serving.""" + stores, network = cell_stores, _Network() + await _prepare(stores, network) + org = stores["tag"] + "w0" + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", org) + capsys.readouterr() + assert await _attach(stores, network, wait=0) == 0, capsys.readouterr().err + assert json.loads(capsys.readouterr().out)["state"] == "serving" + assert org in await _placed() diff --git a/tests/integration/test_cell_attach_cleanup.py b/tests/integration/test_cell_attach_cleanup.py new file mode 100644 index 0000000000..77b3e297ce --- /dev/null +++ b/tests/integration/test_cell_attach_cleanup.py @@ -0,0 +1,154 @@ +"""Attach owners close active sessions before cancelling transaction work.""" + +import asyncio +import time + +import pytest + +from server.cli import cell_attach_pg as apg +from server.cli import cell_control +from server.core import cell_triggers, pg +from tests.integration.test_pg_owned_cleanup import database as database +from tests.integration.test_pg_owned_cleanup import released, sleeping +from tests.unit.cell_control_owner import owner as _owner + +pytestmark = pytest.mark.asyncio + + +@pytest.mark.parametrize( + "owner,phase", + [ + ("trigger", "query"), + ("trigger", "rollback"), + ("mode", "query"), + ("mode", "rollback"), + ("direct", "query"), + ], +) +@pytest.mark.parametrize("interruption", ["cancel", "deadline", "second-cancel"]) +async def test_attach_owner_releases_backend_locks_before_return( + database, monkeypatch, owner, phase, interruption +): + import asyncpg + + conn, observer = database + await conn.execute( + "CREATE SCHEMA loom; CREATE TABLE loom.tenancy_mode(mode text, since timestamptz); INSERT INTO loom.tenancy_mode VALUES('recording', now())" + ) + pid = conn.get_server_pid() + execute = asyncpg.Connection.execute + closing = asyncio.Event() + close = asyncpg.Connection.close + + async def intercepted(self, query, *args, **kwargs): + if self is conn: + if query.startswith("SET LOCAL statement_timeout"): + return await execute(self, "SET LOCAL statement_timeout = '10000ms'") + if query == "RAISE Python error": + raise asyncpg.LockNotAvailableError("trigger statement busy") + if phase == "rollback" and query.startswith("ROLLBACK"): + await execute(self, "SELECT pg_sleep(10)") + if owner == "mode" and query.startswith("LOCK TABLE"): + await execute(self, "SELECT * FROM probe FOR UPDATE") + if phase == "rollback": + raise asyncpg.LockNotAvailableError("mode statement busy") + return await execute(self, "SELECT pg_sleep(10)") + if owner == "direct" and query.startswith("BEGIN"): + await execute(self, query) + return await execute(self, "SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10)") + return await execute(self, query, *args, **kwargs) + + async def closing_session(self, *args, **kwargs): + was_open = not self.is_closed() + await close(self, *args, **kwargs) + if self is conn and was_open and interruption == "second-cancel": + closing.set() + await asyncio.Future() + + async def ensure(conn, table, **kwargs): + statements = ["SELECT * FROM probe FOR UPDATE"] + statements += ["RAISE Python error"] if phase == "rollback" else ["SELECT pg_sleep(10)"] + return await cell_triggers._run_ddl( + conn, table, statements, kwargs["lock_timeout"], 1, warn=False + ) + + async def connect(**_): + return conn + + monkeypatch.setattr(asyncpg.Connection, "execute", intercepted) + monkeypatch.setattr(asyncpg.Connection, "close", closing_session) + monkeypatch.setattr(cell_triggers, "ensure_table", ensure) + monkeypatch.setattr(apg, "_TRIGGER_TRY_MARGIN_S", 0.4) + monkeypatch.setattr(apg, "_MODE_STANDBY_S", 0.5) + monkeypatch.setattr(apg, "_MODE_STATEMENT_MARGIN_MS", 0) + monkeypatch.setattr(cell_control, "OPEN_S", 0.6) + monkeypatch.setattr(pg, "_connect_dedicated", connect) + await apg.take_lock(conn, "cleanup test") + if owner == "trigger": + work = apg._ensure_one(_owner(conn), "probe", 100, apg.Wait(0)) + elif owner == "mode": + work = apg._mode_try(_owner(conn), 100, lambda _: None, "first", (None, None)) + else: + + async def body(_owner): + raise AssertionError("direct checks unexpectedly completed") + + work = cell_control.run("test", body) + task = asyncio.create_task(work) + try: + await sleeping(observer, pid) + started = time.monotonic() + if interruption != "deadline": + task.cancel() + if interruption == "second-cancel": + await asyncio.wait_for(closing.wait(), 1) + task.cancel() + with pytest.raises(TimeoutError if interruption == "deadline" else asyncio.CancelledError): + await asyncio.wait_for(asyncio.shield(task), 1.2) + assert time.monotonic() - started < 0.7 + await released(observer, pid) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_unexpected_trigger_error_reaches_owner_before_rollback(database, monkeypatch): + import asyncpg + + from server.core import cell + + conn, observer = database + pid = conn.get_server_pid() + await conn.execute("SELECT pg_advisory_lock($1)", cell.ATTACH_LOCK_KEY) + execute = asyncpg.Connection.execute + original = ValueError("unexpected trigger operation failed") + + async def intercepted(self, query, *args, **kwargs): + if self is conn: + if query == "RAISE unexpected Python error": + assert await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1 AND locktype='advisory')", + pid, + ) + raise original + if query.startswith("ROLLBACK"): + await execute(self, "SELECT pg_sleep(10)") + return await execute(self, query, *args, **kwargs) + + monkeypatch.setattr(asyncpg.Connection, "execute", intercepted) + started = time.monotonic() + with pytest.raises(ValueError) as error: + await _owner(conn).call( + lambda: cell_triggers._run_ddl( + conn, + "probe", + ["SELECT * FROM probe FOR UPDATE", "RAISE unexpected Python error"], + attempts=1, + warn=False, + ), + total_s=0.5, + ) + assert error.value is original + assert time.monotonic() - started < 0.5 + await released(observer, pid) diff --git a/tests/integration/test_cell_attach_two_replicas.py b/tests/integration/test_cell_attach_two_replicas.py new file mode 100644 index 0000000000..d9cb647db4 --- /dev/null +++ b/tests/integration/test_cell_attach_two_replicas.py @@ -0,0 +1,492 @@ +"""Attach a first cell while two real API processes share its private stores. + +The router uses real HTTP upstreams and a test-owned, verified TLS listener. No +heartbeat row or health response is supplied by the test. Requires the same +synchronous Postgres and ClickHouse services as test_cell_attach. +""" + +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import hashlib +import ipaddress +import json +import os +import socket +import ssl +import subprocess +import sys +from collections.abc import AsyncIterator, Generator +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any +from uuid import UUID + +import httpx +import pytest +import uvicorn +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID + +from server.cli import cell_attach, cell_prepare +from server.core import cell_triggers, instances, pg, provider_process +from server.core.ch import query_tag +from server.insights.store import provider_participants +from server.router import app as router_app +from server.settings import settings +from tests.integration import test_cell_attach as attach_fixtures +from tests.integration._ports import free_port +from tests.integration._redact import redact +from tests.integration._resources import unique_org +from tests.integration._spawn import ROOT, loom_server +from tests.integration.test_cell_attach import ( + _args, + _mode, + _placed, + _row, +) + +_CREDENTIAL = "lcr_" + "3" * 128 +_WAIT_S = 45 + +cell_clickhouse_databases = attach_fixtures.cell_clickhouse_databases +cell_stores = attach_fixtures.cell_stores + + +@pytest.fixture +def genuine_cell_processes() -> bool: + return True + + +def _tls_files(root: Path) -> tuple[Path, Path, Path]: + """An ephemeral CA and IP-address leaf, trusted only by this test's clients.""" + now = datetime.now(UTC) + ca_key = ec.generate_private_key(ec.SECP256R1()) + ca_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "attach test CA")]) + ca = ( + x509.CertificateBuilder() + .subject_name(ca_name) + .issuer_name(ca_name) + .public_key(ca_key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - timedelta(minutes=1)) + .not_valid_after(now + timedelta(days=1)) + .add_extension(x509.BasicConstraints(ca=True, path_length=0), critical=True) + .sign(ca_key, hashes.SHA256()) + ) + key = ec.generate_private_key(ec.SECP256R1()) + leaf = ( + x509.CertificateBuilder() + .subject_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "localhost")])) + .issuer_name(ca_name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - timedelta(minutes=1)) + .not_valid_after(now + timedelta(days=1)) + .add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True) + .add_extension( + x509.SubjectAlternativeName([x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]), + critical=False, + ) + .add_extension(x509.ExtendedKeyUsage([ExtendedKeyUsageOID.SERVER_AUTH]), critical=False) + .sign(ca_key, hashes.SHA256()) + ) + ca_path, cert_path, key_path = (root / name for name in ("ca.pem", "server.pem", "key.pem")) + ca_path.write_bytes(ca.public_bytes(serialization.Encoding.PEM)) + cert_path.write_bytes(leaf.public_bytes(serialization.Encoding.PEM)) + key_path.write_bytes( + key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ) + ) + key_path.chmod(0o600) + return ca_path, cert_path, key_path + + +class _RouterServer(uvicorn.Server): + @contextlib.contextmanager + def capture_signals(self) -> Generator[None]: + # pytest owns the main thread's signals. + yield + + +@contextlib.asynccontextmanager +async def _router(port: int, cert: Path, key: Path) -> AsyncIterator[None]: + router = router_app.Router() + server = _RouterServer( + uvicorn.Config( + router_app.build_app(router), + host="127.0.0.1", + port=port, + ssl_certfile=str(cert), + ssl_keyfile=str(key), + log_level="warning", + timeout_graceful_shutdown=5, + ) + ) + task = asyncio.create_task(server.serve(), name="attach-test-router") + try: + async with asyncio.timeout(10): + while not server.started: + if task.done(): + await task + raise AssertionError("router stopped before listening") + await asyncio.sleep(0.05) + yield + finally: + first = sys.exception() + server.should_exit = True + try: + await asyncio.wait_for(task, timeout=10) + except BaseException as error: # noqa: BLE001 — finish all owned cleanup; keep first failure. + first = first or error + finally: + # A timed-out uvicorn task must not leave its listener or requests running. + for listener in getattr(server, "servers", ()): + try: + listener.close() + await listener.wait_closed() + except BaseException as error: # noqa: BLE001 — attempt remaining owned cleanup. + first = first or error + requests = tuple(server.server_state.tasks) + for request in requests: + request.cancel() + for operation in ( + asyncio.wait_for(asyncio.gather(*requests, return_exceptions=True), timeout=5), + router.close(), + ): + try: + await operation + except BaseException as error: # noqa: BLE001 — keep first; finish router cleanup. + first = first or error + if first is not None: + raise first + + +@contextlib.asynccontextmanager +async def _api(name: str, env: dict[str, str], port: int) -> AsyncIterator[str]: + """Keep the router loop available and retain each worker until its context settles.""" + owned = loom_server(name, {**env, "LOOM_INSTANCE_LABEL": name}, port=port) + first: BaseException | None = None + + async def finish[T](task: asyncio.Task[T]) -> T: + nonlocal first + while not task.done(): + try: + await asyncio.wait({task}) + except asyncio.CancelledError as error: + if first is None: + first = error + return task.result() + + entry = asyncio.create_task(asyncio.to_thread(owned.__enter__)) + try: + yield await asyncio.shield(entry) + except BaseException as error: # noqa: BLE001 — cleanup completes before the first error escapes. + first = error + finally: + try: + await finish(entry) + except BaseException as error: # noqa: BLE001 — a failed entry already ran its own cleanup. + first = first or error + else: + try: + await finish( + asyncio.create_task(asyncio.to_thread(owned.__exit__, None, None, None)) + ) + except BaseException as error: # noqa: BLE001 — retain the body error after owned cleanup. + if first is None: + first = error + else: + first.add_note(f"{name} process cleanup also failed: {type(error).__name__}") + if first is not None: + raise first + + +def _configure(monkeypatch: pytest.MonkeyPatch, host: str, api: int, probe: int) -> dict[str, str]: + values = { + "cell": "first", + "storage_authority": "postgres", + "pg_durability": "synchronous", + "bind_host": "127.0.0.1", + "serves_traffic": True, + "pg_dsn": settings.pg_dsn, + "router_pg_dsn": settings.router_pg_dsn, + "ch_database": settings.ch_database, + "ch_telemetry_database": settings.ch_telemetry_database, + "ch_cluster": "", + "document_s3_bucket": "", + "public_base_url": f"https://{host}", + "oauth_issuer": "", + "cell_router_url": f"https://{host}", + "cell_router_token": _CREDENTIAL, + "cell_probe_port": probe, + "router_shared_host": host, + "router_upstream_url": f"http://127.0.0.1:{api}", + "router_upstream_probe_url": f"http://127.0.0.1:{probe}", + "instance_heartbeat_seconds": 1, + "tenancy_mode_cache_s": 1, + "router_directory_cache_s": 1, + "background_workers": "off", + "embedding_provider": "fake", + "embedding_model": "fake-ngram-3072", + "embedding_dim": 3072, + "llm_provider": "fake", + "llm_model": "fake-echo-1", + "require_real_providers": False, + "boot_migrate": False, + "auth_required": True, + "identity_provider": "gateway", + "self_signup": "off", + "mcp_auth_mode": "required", + } + for key, value in values.items(): + monkeypatch.setattr(settings, key, value) + env = { + f"LOOM_{key.upper()}": str(value).lower() if isinstance(value, bool) else str(value) + for key, value in values.items() + } + # API processes get the cell credential, never directory/operator authority. + env.update(LOOM_ROUTER_OPERATOR_TOKEN="", LOOM_ROUTER_OPERATOR_SHA256="", LOOM_ROUTER_PG_DSN="") + return env + + +async def _identities(client: httpx.AsyncClient, bases: list[str]) -> list[dict[str, Any]]: + identities = [] + for base in bases: + response = await client.get(f"{base}/v1/health") + assert response.status_code == 200, response.text + health = response.json() + host, pid = health["instance"].rsplit(":", 1) + assert host == socket.gethostname() + text = await asyncio.to_thread(Path(f"/proc/{int(pid)}/stat").read_text) + stat = text.rsplit(")", 1)[1].split() + assert int(stat[1]) == os.getpid(), "the API must be this test's child" + assert stat[0] not in {"Z", "X"} + identities.append( + { + "pid": int(pid), + "start": stat[19], + "participant": health["participant_id"], + "instance": health["deployment_config"]["instance_id"], + "label": health["instance_label"], + } + ) + assert len({i["pid"] for i in identities}) == 2 + assert len({i["participant"] for i in identities}) == 2 + assert len({i["instance"] for i in identities}) == 2 + assert {i["label"] for i in identities} == {"attach-two-a", "attach-two-b"} + return identities + + +async def _live(ch: Any, identities: list[dict[str, Any]], mode: str) -> None: + """Match unfiltered live and cell rows to the exact HTTP and kernel witnesses.""" + expected = {i["instance"] for i in identities} + async with asyncio.timeout(_WAIT_S): + while True: + rows = await instances.read_live(ch, all_deployments=True) + # KeeperMap PRIMARY KEY instance_id: one current row, no merge ordering. + cells = await ch.query_all(f"SELECT * FROM {instances.CELL_TABLE}") + by_id = {r["instance_id"]: r for r in cells} + if {r["instance_id"] for r in rows} == expected and all( + i["instance"] in by_id + and by_id[i["instance"]]["tenancy_mode"] == mode + and ( + mode != "enforced" + or next(r for r in rows if r["instance_id"] == i["instance"])["serving"] + ) + for i in identities + ): + break + await asyncio.sleep(0.2) + processes = await ch.query_all( + f"SELECT instance_id, pid FROM {instances.TABLE} " + "WHERE instance_id IN {ids:Array(String)}", + {"ids": sorted(expected)}, + ) + pids = {r["instance_id"]: int(r["pid"]) for r in processes} + boot = await asyncio.to_thread(Path("/proc/sys/kernel/random/boot_id").read_text) + participants = await provider_participants.participants(ch=ch) + by_participant = {str(r["participant_id"]): r for r in participants} + for identity in identities: + assert pids[identity["instance"]] == identity["pid"] + raw = by_id[identity["instance"]] + fingerprint = json.loads(raw["fingerprint"]) + assert raw["cell"] == "first" + assert fingerprint["participant_id"] == identity["participant"] + participant = by_participant[identity["participant"]] + assert participant["state"] == "running" + proof = participant["host_proof"] + assert provider_process.has_certificate(proof) + parts = proof.split("|") + assert parts[0] == socket.gethostname() + assert parts[1] == boot.strip() + assert parts[3] == str(os.stat("/proc/self/ns/pid").st_ino) + assert (parts[4], parts[5]) == (str(identity["pid"]), identity["start"]) + assert not provider_process.proves_stopped(proof) + if mode == "enforced": + assert fingerprint["router"] is True + assert fingerprint["marked"] is True + assert next(r for r in rows if r["instance_id"] == identity["instance"])["serving"] + identity["proof"] = proof + + +async def _retired(ch: Any, identities: list[dict[str, Any]]) -> None: + """A helper's TERM/KILL/reap alone does not establish guarded retirement.""" + rows = await ch.query_all( + f"SELECT participant_id, state, host_proof FROM {provider_participants.TABLE} FINAL " + "WHERE participant_id IN {ids:Array(UUID)}", + {"ids": [UUID(i["participant"]) for i in identities]}, + ) + by_id = {str(row["participant_id"]): row for row in rows} + for identity in identities: + assert provider_process.proves_stopped(identity["proof"]) + row = by_id[identity["participant"]] + assert row["state"] == "retired", "stopping or killing is not retirement proof" + assert row["host_proof"] == identity["proof"] + assert provider_process.has_certificate(row["host_proof"]) + + +async def _mint(env: dict[str, str], org: str) -> str: + # A fresh native command reads this cell's keyring, not pytest's signing cache. + result = await asyncio.to_thread( + subprocess.run, + [ + sys.executable, + "-m", + "server.cli", + "mint-token", + "--subject", + "attach-two", + "--org", + org, + "--namespaces", + "n", + "--roles", + "writer", + "--ttl-days", + "1", + "--json", + ], + cwd=ROOT, + env={**os.environ, **env, "LOOM_CELL_ROUTER_TOKEN": ""}, + capture_output=True, + text=True, + timeout=_WAIT_S, + check=False, + ) + assert result.returncode == 0, redact(result.stderr) + token = json.loads(result.stdout)["token"] + assert isinstance(token, str) and token + return token + + +async def _requests(client: httpx.AsyncClient, bases: list[str], org: str, token: str) -> None: + headers = {"Authorization": f"Bearer {token}"} + key = {"org": org, "namespace": "n", "agent": "attach-test", "memory_key": "shared"} + for index, base in enumerate(bases): + unauthenticated = await client.post(f"{base}/v1/memory.get", json=key) + assert unauthenticated.status_code == 401, unauthenticated.text + value = {"writer": index} + written = await client.post( + f"{base}/v1/memory.set", + headers=headers, + json={**key, "memory_type": "semantic", "value": value}, + ) + assert written.status_code == 200, written.text + for reader in bases: + read = await client.post(f"{reader}/v1/memory.get", headers=headers, json=key) + assert read.status_code == 200, read.text + assert read.json()["value"] == value + + +@pytest.mark.asyncio +async def test_attach_with_two_live_api_replicas( + cell_stores: dict[str, Any], + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + stores = cell_stores + api_a, api_b, probe, router_port = (free_port() for _ in range(4)) + host = f"127.0.0.1:{router_port}" + env = _configure(monkeypatch, host, api_a, probe) + ca, certificate, key = _tls_files(tmp_path) + env["SSL_CERT_FILE"] = str(ca) + trust = ssl.create_default_context(cafile=str(ca)) + router = httpx.URL(f"https://{host}") + org = unique_org("attach-two") + identities: list[dict[str, Any]] = [] + with query_tag("test.cell_attach_two_replicas"): + async with _router(router_port, certificate, key): + try: + async with ( + _api("attach-two-a", env, api_a) as a, + _api("attach-two-b", env, api_b) as b, + httpx.AsyncClient(verify=trust, timeout=30) as client, + ): + identities = await _identities(client, [a, b]) + stores["ids"].extend(i["instance"] for i in identities) + await _live(stores["ch"], identities, "open") + args = argparse.Namespace( + cell="first", + host=host, + credential_sha256=hashlib.sha256(_CREDENTIAL.encode()).hexdigest(), + probe_url="", + wait=20, + router=str(router), + ) + assert ( + await cell_prepare.prepare( + stores["ch"], args, router, httpx.AsyncHTTPTransport(verify=trust) + ) + == 0 + ) + await _live(stores["ch"], identities, "recording") + await pg.execute("INSERT INTO loom.tenancy (org) VALUES ($1)", org) + await stores["ch"].insert( + "loom.namespaces", [{"org": org, "namespace": "n", "owner_email": ""}] + ) + capsys.readouterr() + assert ( + await cell_attach.attach( + stores["ch"], + _args(router=str(router), wait=_WAIT_S), + router, + httpx.AsyncHTTPTransport(verify=trust), + ) + == 0 + ) + assert json.loads(capsys.readouterr().out)["state"] == "serving" + assert await _mode() == "enforced" + row = await _row() + assert (row.state, row.mode) == ("serving", "enforced") + assert org in await _placed() + async with pg.dedicated() as conn: + assert await cell_triggers.missing(conn) == [] + await _live(stores["ch"], identities, "enforced") + token = await _mint(env, org) + await _requests(client, [a, b, str(router)], org, token) + assert await _identities(client, [a, b]) == [ + {k: v for k, v in identity.items() if k != "proof"} + for identity in identities + ] + finally: + first = sys.exception() + if identities: + try: + await _retired(stores["ch"], identities) + except ( + BaseException + ) as error: # Retirement failure cannot hide the test failure. + if first is None: + raise + first.add_note( + f"retirement proof also failed: {type(error).__name__}: {error}" + ) diff --git a/tests/integration/test_cell_control_cleanup.py b/tests/integration/test_cell_control_cleanup.py new file mode 100644 index 0000000000..8f68939b62 --- /dev/null +++ b/tests/integration/test_cell_control_cleanup.py @@ -0,0 +1,424 @@ +"""Native cell-control interruption closes its actual Postgres backend and locks.""" + +from __future__ import annotations + +import argparse +import asyncio +import time +from collections.abc import AsyncIterator +from typing import Any + +import httpx +import pytest + +from server.cli import cell_control, cell_prepare +from server.core import pg +from server.settings import settings +from tests.integration._pg_database import own_database +from tests.integration.test_pg_owned_cleanup import released, sleeping + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +async def native_database(monkeypatch) -> AsyncIterator[Any]: + import asyncpg + + async with own_database() as dsn: + monkeypatch.setattr(settings, "pg_dsn", dsn) + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(pg, "_expected", None) + observer = await asyncpg.connect(dsn) + await observer.execute( + "CREATE TABLE probe(id int PRIMARY KEY); INSERT INTO probe VALUES(1)" + ) + try: + yield observer + finally: + await observer.close(timeout=1) + + +@pytest.mark.parametrize("interrupt", ["cancel", "deadline"]) +async def test_native_prepare_interrupt_releases_backend_and_locks( + native_database, monkeypatch, interrupt +): + observed = asyncio.Event() + backend: dict[str, Any] = {} + + async def read(conn, **_): + backend["pid"] = conn.get_server_pid() + observed.set() + await conn.execute("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10)") + + monkeypatch.setattr(cell_prepare, "_read_rows", read) + if interrupt == "deadline": + monkeypatch.setattr(cell_control, "TRANSACTION_S", 0.4) + task = asyncio.create_task( + cell_prepare.prepare(None, argparse.Namespace(), httpx.URL("http://router")) + ) + await asyncio.wait_for(observed.wait(), 3) + await sleeping(native_database, backend["pid"]) + began = time.monotonic() + if interrupt == "cancel": + task.cancel("native prepare cancelled") + with pytest.raises(asyncio.CancelledError if interrupt == "cancel" else TimeoutError): + await asyncio.wait_for(task, 1.5) + assert time.monotonic() - began < 1.5 + await released(native_database, backend["pid"]) + + +async def test_native_prepare_body_failure_is_not_hidden_by_rollback(native_database, monkeypatch): + import asyncpg + + execute = asyncpg.Connection.execute + original = ValueError("original native transaction failure") + backend = {} + + async def read(conn, **_): + backend["pid"] = conn.get_server_pid() + await conn.execute("SELECT * FROM probe FOR UPDATE") + raise original + + async def stalled_rollback(self, query, *args, **kwargs): + if query.startswith("ROLLBACK"): + await execute(self, "SELECT pg_sleep(10)") + return await execute(self, query, *args, **kwargs) + + monkeypatch.setattr(asyncpg.Connection, "execute", stalled_rollback) + monkeypatch.setattr(cell_prepare, "_read_rows", read) + monkeypatch.setattr(cell_control, "TRANSACTION_S", 0.4) + began = time.monotonic() + with pytest.raises(ValueError, match="original native transaction failure") as captured: + await asyncio.wait_for( + cell_prepare.prepare(None, argparse.Namespace(), httpx.URL("http://router")), 1.5 + ) + assert captured.value is original + assert time.monotonic() - began < 0.4 + await released(native_database, backend["pid"]) + + +async def test_native_prepare_second_cancel_preserves_first_and_releases_locks( + native_database, monkeypatch +): + import asyncpg + + real_close = asyncpg.Connection.close + observed, close_started, close_return = asyncio.Event(), asyncio.Event(), asyncio.Event() + backend = {} + close_interrupted = asyncio.Event() + monkeypatch.setattr(cell_control, "TRANSACTION_S", 0.4) + + async def read(conn, **_): + backend["conn"], backend["pid"] = conn, conn.get_server_pid() + observed.set() + await conn.execute("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10)") + + async def delayed_close(self, **kwargs): + await real_close(self, **kwargs) + if self is backend.get("conn"): + close_started.set() + try: + await close_return.wait() + except asyncio.CancelledError: + close_interrupted.set() + raise + + monkeypatch.setattr(cell_prepare, "_read_rows", read) + monkeypatch.setattr(asyncpg.Connection, "close", delayed_close) + task = asyncio.create_task( + cell_prepare.prepare(None, argparse.Namespace(), httpx.URL("http://router")) + ) + try: + await observed.wait() + await sleeping(native_database, backend["pid"]) + task.cancel("first native cancellation") + await asyncio.wait_for(close_started.wait(), 1) + task.cancel("second cancellation") + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not close_interrupted.is_set(), "second signal cannot interrupt public close" + assert not task.done() + with pytest.raises(asyncio.CancelledError, match="first native cancellation"): + await asyncio.wait_for(task, 0.5) + await released(native_database, backend["pid"]) + finally: + close_return.set() + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("stage", ["open", "phase", "body"]) +@pytest.mark.parametrize("kind", ["org", "keyboard"]) +async def test_native_fatal_guard_is_inert_until_backend_closed(native_database, stage, kind): + import os + import sys + + # A separate interpreter keeps a regression in child fatal-exception transport + # from stopping pytest's event loop. Proof runs before the native command exits. + from tests.integration._pg_database import with_database + + base = os.environ["LOOM_PG_DSN"] + dsn = with_database(base, await native_database.fetchval("SELECT current_database()")) + script = """ +import argparse, asyncio, json, os, sys +import asyncpg, httpx +from server.cli import cell_control, cell_prepare +from server.cli._ch import OrgNotHeld +from server.core import pg +from server.settings import settings +stage, kind = sys.argv[1:] +settings.pg_dsn = os.environ['LOOM_NATIVE_CLEANUP_DSN'] +settings.cell = 'first' +pg._expected = None +original = OrgNotHeld('org refused') if kind == 'org' else KeyboardInterrupt('guard interrupted') +backend = {} +real_init = pg.init_connection +async def initialize(conn): + await real_init(conn) + backend['pid'] = conn.get_server_pid() + if stage == 'open': + raise original +async def read(conn, **kwargs): + backend['pid'] = conn.get_server_pid() + await conn.execute('SELECT * FROM probe FOR UPDATE') + if stage == 'phase': + raise original + return cell_prepare._CellRows('dep-first', None, '', None, None, 0) +async def beats(ch): + raise original +pg.init_connection = initialize +cell_prepare._read_rows = read +cell_prepare._read_beats = beats +async def main(): + observer = await asyncpg.connect(settings.pg_dsn) + try: + try: + await cell_prepare.prepare(None, argparse.Namespace(), httpx.URL('http://router')) + except BaseException as found: + assert found is original + assert not await observer.fetchval('SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1)', backend['pid']) + assert not await observer.fetchval('SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)', backend['pid']) + print(json.dumps({'original': True, 'backend_absent': True})) + else: + raise AssertionError('fatal guard became success') + finally: + await observer.close(timeout=1) +asyncio.run(main()) +""" + child = await asyncio.create_subprocess_exec( + sys.executable, + "-c", + script, + stage, + kind, + env={**os.environ, "LOOM_NATIVE_CLEANUP_DSN": dsn}, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, _stderr = await asyncio.wait_for(child.communicate(), 10) + assert child.returncode == 0, "native fatal exception interrupted its supervisor" + assert b'"backend_absent": true' in stdout + finally: + if child.returncode is None: + child.kill() + await child.wait() + + +@pytest.mark.parametrize("sqlstate", ["55P03", "57014"]) +async def test_mode_contention_retries_only_after_real_rollback_on_same_owned_backend( + native_database, monkeypatch, sqlstate +): + import asyncpg + + from server.cli import cell_attach_pg as apg + from tests.unit.cell_control_owner import owner + + conn = await pg._connect_dedicated(statement_timeout_ms=10_000) + await pg.init_connection(conn) + await apg.take_lock(conn, "real mode contention") + await conn.execute("INSERT INTO loom.cell_first(is_first,cell) VALUES(true,'first')") + await conn.execute("INSERT INTO loom.tenancy_mode(mode) VALUES('recording')") + fp = {"v": 1} + await conn.execute("INSERT INTO loom.cell_self(current_fp) VALUES($1)", fp) + pid, marks, injected = conn.get_server_pid(), [], False + execute = asyncpg.Connection.execute + + async def contend(self, query, *args, **kwargs): + nonlocal injected + if self is conn and query.startswith("UPDATE loom.tenancy_mode") and not injected: + injected = True + await execute( + self, + "DO $$ BEGIN RAISE EXCEPTION USING ERRCODE='" + + sqlstate + + "', MESSAGE='test contention'; END $$", + ) + return await execute(self, query, *args, **kwargs) + + monkeypatch.setattr(asyncpg.Connection, "execute", contend) + owned = owner(conn) + try: + assert not await apg._mode_try(owned, 100, marks.append, "first", (fp, None)) + assert conn.get_server_pid() == pid and await apg.lock_held(conn) + assert await conn.fetchval("SELECT mode FROM loom.tenancy_mode") == "recording" + assert marks == ["unknown", "recording"] and not owned.closed + assert await apg._mode_try(owned, 100, marks.append, "first", (fp, None)) + assert await native_database.fetchval("SELECT mode FROM loom.tenancy_mode") == "enforced" + assert marks == ["unknown", "recording", "unknown", "enforced"] + assert conn.get_server_pid() == pid and await apg.lock_held(conn) + finally: + await owned.close() + await released(native_database, pid) + + +async def test_native_attach_run_body_uses_sole_owner_on_cancellation(native_database, monkeypatch): + from uuid import UUID + + from server.cli import cell_attach + from server.router.directory import CONTROL_FIELDS + + observed, backend = asyncio.Event(), {} + + class Router: + async def call(self, _path): + return { + **dict.fromkeys(CONTROL_FIELDS, ""), + "state": "attaching", + "mutation_generation": str(UUID(int=1)), + } + + async def read(self): + backend["pid"] = self.lock_conn.get_server_pid() + observed.set() + await self.lock_conn.execute("BEGIN; SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10)") + + monkeypatch.setattr(cell_attach.Run, "read_own_row", read) + run = cell_attach.Run(None, argparse.Namespace(cell="first", wait=0), Router(), lambda _: None) + task = asyncio.create_task(run.run()) + await observed.wait() + await sleeping(native_database, backend["pid"]) + task.cancel("native attach cancellation") + with pytest.raises(asyncio.CancelledError, match="native attach cancellation"): + await asyncio.wait_for(task, 1.3) + await released(native_database, backend["pid"]) + + +async def test_attach_role_needs_stats_but_not_backend_signal_privilege(native_database): + import uuid + + from server.cli import cell_attach_pg as apg + + role = f"it_attach_grants_{uuid.uuid4().hex[:10]}" + conn = native_database + database = await conn.fetchval("SELECT current_database()") + await conn.execute(f"CREATE ROLE {role}") + try: + await conn.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + await conn.execute( + f"GRANT SELECT ON loom.cell_first, loom.tenancy_mode, loom.cell_self, loom.authority TO {role}" + ) + await conn.execute(f"GRANT UPDATE ON loom.tenancy_mode TO {role}") + await conn.execute(f"GRANT INSERT ON loom.tenancy TO {role}") + await conn.execute(f"GRANT SELECT, INSERT, UPDATE, DELETE ON loom.cell_probe TO {role}") + await conn.execute(f'GRANT TEMPORARY ON DATABASE "{database}" TO {role}') + await conn.execute(f"GRANT pg_read_all_stats TO {role} WITH INHERIT FALSE") + await conn.execute(f"SET SESSION AUTHORIZATION {role}") + assert not await conn.fetchval( + "SELECT pg_has_role(current_user, 'pg_signal_backend', 'USAGE')" + ) + refused = await apg.own_role_refusals(conn, []) + assert [r.reason for r in refused] == [ + "attach's role lacks effective privileges of pg_read_all_stats" + ] + await conn.execute("RESET SESSION AUTHORIZATION") + await conn.execute(f"GRANT pg_read_all_stats TO {role} WITH INHERIT TRUE") + await conn.execute(f"SET SESSION AUTHORIZATION {role}") + assert not await conn.fetchval( + "SELECT pg_has_role(current_user, 'pg_signal_backend', 'USAGE')" + ) + assert await apg.own_role_refusals(conn, []) == [] + finally: + await conn.execute("RESET SESSION AUTHORIZATION") + await conn.execute(f"DROP OWNED BY {role}") + await conn.execute(f"DROP ROLE {role}") + + +@pytest.mark.parametrize("late", [False, True]) +@pytest.mark.parametrize("kind", ["exit", "keyboard"]) +async def test_native_public_close_fatal_stays_inside_owner(native_database, late, kind): + import os + import sys + + from tests.integration._pg_database import with_database + + dsn = with_database( + os.environ["LOOM_PG_DSN"], await native_database.fetchval("SELECT current_database()") + ) + script = r""" +import asyncio, os, sys +import asyncpg +from server.cli import cell_control +from server.core import pg +from server.settings import settings +late, kind = sys.argv[1:] +settings.pg_dsn = os.environ['LOOM_NATIVE_CLEANUP_DSN'] +settings.cell = 'first' +pg._expected = None +original = SystemExit(17) if kind == 'exit' else KeyboardInterrupt('close interrupted') +real_connect, real_close = pg._connect_dedicated, asyncpg.Connection.close +backend = {} +async def connect(**kwargs): + conn = await real_connect(**kwargs) + backend['pid'] = conn.get_server_pid() + if late == 'True': + try: await asyncio.Future() + except asyncio.CancelledError: pass + return conn +async def close(self, **kwargs): + ours = self.get_server_pid() == backend.get('pid') + await real_close(self, **kwargs) + if ours: + backend['closes'] = backend.get('closes', 0) + 1 + raise original +async def body(owner): + await owner.conn.execute('SELECT * FROM probe FOR UPDATE') + return 19 +pg._connect_dedicated = connect +asyncpg.Connection.close = close +if late == 'True': cell_control.OPEN_S = .5 +async def main(): + observer = await asyncpg.connect(settings.pg_dsn) + try: + try: await cell_control.run('native close', body) + except BaseException as caught: + if late == 'True': assert isinstance(caught, TimeoutError), type(caught) + else: assert caught is original, type(caught) + assert backend['closes'] == 1 + assert not await observer.fetchval('SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1)', backend['pid']) + assert not await observer.fetchval('SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)', backend['pid']) + print('LOOP_STILL_RUNNING') + else: raise AssertionError('required close fatal became success') + finally: + asyncpg.Connection.close = real_close + await observer.close(timeout=1) +asyncio.run(main()) +""" + child = await asyncio.create_subprocess_exec( + sys.executable, + "-c", + script, + str(late), + kind, + env={**os.environ, "LOOM_NATIVE_CLEANUP_DSN": dsn}, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, stderr = await asyncio.wait_for(child.communicate(), 10) + assert child.returncode == 0 and b"LOOP_STILL_RUNNING" in stdout, stderr.decode() + finally: + if child.returncode is None: + child.kill() + await child.wait() diff --git a/tests/integration/test_cell_control_fence.py b/tests/integration/test_cell_control_fence.py new file mode 100644 index 0000000000..2e45540fb4 --- /dev/null +++ b/tests/integration/test_cell_control_fence.py @@ -0,0 +1,809 @@ +"""Control requests remain fenced after their cell's lock backend dies.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import os +import uuid +from collections.abc import AsyncIterator +from typing import Any +from urllib.parse import urlsplit, urlunsplit + +import httpx +import pytest + +from server.core import cell +from server.router import app, directory +from server.settings import settings +from tests.unit.cell_control_owner import owner as _owner + +TOKEN = "lco_fence-test" + + +@pytest.fixture +async def control_stores(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[Any]: + import asyncpg + + dsn = os.environ.get("LOOM_ROUTER_PG_DSN") or pytest.skip("LOOM_ROUTER_PG_DSN not set") + admin = await asyncpg.connect(dsn) + name = f"loom_fence_{uuid.uuid4().hex[:12]}" + await admin.execute(f'CREATE DATABASE "{name}"') + own = urlunsplit(urlsplit(dsn)._replace(path=f"/{name}")) + monkeypatch.setattr(settings, "router_pg_dsn", own) + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(TOKEN.encode()).hexdigest() + ) + monkeypatch.setattr(directory, "_pool", None) + conn = await asyncpg.connect(own) + try: + await directory.migrate() + await conn.execute( + "INSERT INTO loom_router.cells (cell, is_first, host, deployment_id, credential_sha256, state) " + "VALUES ('first', true, 'first.example.test', 'dep-first', $1, 'attaching')", + "a" * 64, + ) + transport = httpx.ASGITransport(app=app.build_app()) + async with httpx.AsyncClient( + transport=transport, + base_url="http://test", + headers={"Authorization": f"Bearer {TOKEN}"}, + ) as client: + yield client, conn, own + finally: + await conn.close() + await directory.close() + await admin.execute(f'DROP DATABASE "{name}" WITH (FORCE)') + await admin.close() + + +def expected(row: dict[str, Any], *, generation: bool = True) -> dict[str, Any]: + out = { + k: row[k] + for k in ( + "cell", + "is_first", + "host", + "deployment_id", + "bucket", + "bucket_prefix", + "probe_url", + "upstream_url", + "current_fp", + "next_fp", + ) + } + if generation: + out["mutation_generation"] = row["mutation_generation"] + return {"expected": out} + + +async def read(client: httpx.AsyncClient) -> dict[str, Any]: + response = await client.get("/cells/v1/operator/cells/first") + assert response.status_code == 200, response.text + return response.json() + + +async def claim(client: httpx.AsyncClient, row: dict[str, Any]) -> dict[str, Any]: + response = await client.post( + "/cells/v1/operator/cells/first/claim", json=expected(row, generation=False) + ) + assert response.status_code == 200, response.text + return response.json() + + +@pytest.mark.asyncio +async def test_stale_http_after_backend_death_and_pair_aba(control_stores, monkeypatch) -> None: + import asyncpg + + client, conn, own = control_stores + a = await asyncpg.connect(own) + b = await asyncpg.connect(own) + release = asyncio.Event() + entered = asyncio.Event() + task = None + try: + await a.execute("SELECT pg_advisory_lock($1)", cell.ATTACH_LOCK_KEY) + snapshot = await read(client) + old = await claim(client, snapshot) + real = directory.set_limit + + async def delayed(*args, **kwargs): + entered.set() + await release.wait() + return await real(*args, **kwargs) + + monkeypatch.setattr(directory, "set_limit", delayed) + task = asyncio.create_task( + client.post( + "/cells/v1/operator/cells/first/limit", json={**expected(old), "closed": True} + ) + ) + await asyncio.wait_for(entered.wait(), 3) + await conn.execute("SELECT pg_terminate_backend($1)", a.get_server_pid()) + assert await b.fetchval("SELECT pg_try_advisory_lock($1)", cell.ATTACH_LOCK_KEY) + newer = await claim(client, await read(client)) + # A complete pair ABA cannot reauthorize A: only the current generation can write. + await conn.execute( + "UPDATE loom_router.cells SET current_fp = '{\"v\":1}', next_fp = '{\"v\":1}'" + ) + await conn.execute("UPDATE loom_router.cells SET current_fp = NULL, next_fp = NULL") + assert newer["mutation_generation"] != old["mutation_generation"] + release.set() + response = await task + assert response.status_code == 409, response.text + assert not await conn.fetchval("SELECT closed FROM loom_router.cells") + finally: + release.set() + if task is not None: + await task + await a.close() + await b.close() + + +@pytest.mark.asyncio +async def test_replayed_claim_lost_ack_and_noop_wrong_generation(control_stores) -> None: + client, conn, _ = control_stores + await conn.execute("UPDATE loom_router.cells SET state = 'serving'") + snapshot = await read(client) + lost = await claim(client, snapshot) # its caller lost this acknowledgement + current = await claim(client, snapshot) # duplicate HTTP execution must rotate again + assert lost["mutation_generation"] != current["mutation_generation"] + response = await client.post( + "/cells/v1/operator/cells/first/limit", json={**expected(lost), "closed": False} + ) + assert response.status_code == 409, response.text + response = await client.post( + "/cells/v1/operator/cells/first/limit", json={**expected(current), "closed": False} + ) + assert response.status_code == 200, response.text + assert ( + str(await conn.fetchval("SELECT mutation_generation FROM loom_router.cells")) + == current["mutation_generation"] + ) + + +@pytest.fixture +async def local_control(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[Any]: + import asyncpg + + from server.core import pg + from tests.integration._pg_database import own_database + + async with own_database() as dsn: + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "pg_dsn", dsn) + monkeypatch.setattr(settings, "document_s3_bucket", "") + conn = await asyncpg.connect(dsn, server_settings={pg.TENANCY_MARKER: "on"}) + await directory._init(conn) + await conn.execute( + "INSERT INTO loom.authority (authority, epoch, deployment_id) VALUES ('postgres', 1, 'dep-first')" + ) + try: + yield conn, dsn + finally: + await conn.close() + await pg.close_pool() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("writer", ["prepare", "attach"]) +async def test_dead_local_lock_backend_cannot_write_acceptance(local_control, writer) -> None: + import asyncpg + + from server.cli import cell_attach_pg as apg + from server.cli import cell_prepare + + admin, dsn = local_control + dead = await asyncpg.connect(dsn) + await directory._init(dead) + await dead.execute("SELECT pg_advisory_lock($1)", cell.ATTACH_LOCK_KEY) + rows = cell_prepare._CellRows("dep-first", None, "", None, None, 0) + fp = cell.as_value({"deployment_id": "dep-first"}) + await admin.execute("SELECT pg_terminate_backend($1)", dead.get_server_pid()) + successor = await asyncpg.connect(dsn) + try: + assert await successor.fetchval("SELECT pg_try_advisory_lock($1)", cell.ATTACH_LOCK_KEY) + if writer == "prepare": + with pytest.raises( + (asyncpg.InterfaceError, asyncpg.PostgresError, asyncpg.InternalClientError) + ): + await cell_prepare._write_rows( + dead, rows, {"is_first": True, "mode": "recording"}, fp, None + ) + assert not await admin.fetchval("SELECT EXISTS(SELECT 1 FROM loom.cell_self)") + else: + # A dead controller cannot switch to another session to complete its mode write. + await admin.execute( + "INSERT INTO loom.cell_first(is_first, cell) VALUES (true, 'first')" + ) + await admin.execute("INSERT INTO loom.tenancy_mode(mode) VALUES ('recording')") + await admin.execute("INSERT INTO loom.cell_self(current_fp) VALUES ($1)", fp) + with pytest.raises( + (asyncpg.InterfaceError, asyncpg.PostgresError, asyncpg.InternalClientError) + ): + await apg._mode_try(_owner(dead), 100, lambda _m: None, "first", (fp, None)) + assert await admin.fetchval("SELECT mode FROM loom.tenancy_mode") == "recording" + finally: + await successor.close() + await dead.close() + + +@pytest.mark.asyncio +async def test_same_session_prepare_and_mode_commit(local_control) -> None: + from server.cli import cell_attach_pg as apg + from server.cli import cell_prepare + + conn, _ = local_control + await apg.take_lock(conn, "test") + fp = cell.as_value({"deployment_id": "dep-first"}) + rows = await cell_prepare._read_rows(conn) + written = await cell_prepare._write_rows( + conn, rows, {"is_first": True, "mode": "recording"}, fp, None + ) + assert written == ["first", "mode", "own"] + assert await apg.lock_held(conn) + assert await apg._mode_try(_owner(conn), 100, lambda _m: None, "first", (fp, None)) + result = await apg.durable_read(_owner(conn)) + assert result.mode == "enforced" and result.first == (True, "first") + assert await apg.lock_held(conn) + # No-op prepare remains a same-backend synchronous durability boundary. + rows = await cell_prepare._read_rows(conn) + assert ( + await cell_prepare._write_rows(conn, rows, {"is_first": True, "mode": "enforced"}, fp, None) + == [] + ) + assert ( + await _owner(conn).transaction(lambda: conn.fetchval("SELECT pg_backend_pid()")) + == conn.get_server_pid() + ) + + +@pytest.mark.asyncio +async def test_recreated_row_fences_previous_generation(control_stores) -> None: + client, conn, _ = control_stores + previous = await claim(client, await read(client)) + response = await client.request( + "DELETE", "/cells/v1/operator/cells/first", json=expected(previous) + ) + assert response.status_code == 200, response.text + await conn.execute( + "INSERT INTO loom_router.cells(cell, is_first, host, state, deployment_id, credential_sha256) VALUES ('first', true, 'first.example.test', 'attaching', 'dep-first', $1)", + "a" * 64, + ) + recreated = await read(client) + assert recreated["mutation_generation"] != previous["mutation_generation"] + response = await client.post( + "/cells/v1/operator/cells/first/limit", json={**expected(previous), "closed": True} + ) + assert response.status_code == 409 + assert not await conn.fetchval("SELECT closed FROM loom_router.cells") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "field,value", + [("current_fp", {"v": 1, "fields": {"x": False}}), ("host", "other.example.test")], +) +async def test_claim_checks_exact_pair_and_registration(control_stores, field, value) -> None: + client, conn, _ = control_stores + original = await read(client) + if field == "current_fp": + await conn.execute('UPDATE loom_router.cells SET current_fp = \'{"v":1,"fields":{"x":0}}\'') + original[field] = value + response = await client.post( + "/cells/v1/operator/cells/first/claim", json=expected(original, generation=False) + ) + assert response.status_code == 409 + + +@pytest.mark.asyncio +async def test_missing_generation_requires_upgrade_after_auth(control_stores) -> None: + client, conn, _ = control_stores + for path in ( + "limit", + "credential", + "attach/begin", + "attach/place", + "attach/enforced", + "attach/serving", + ): + response = await client.post(f"/cells/v1/operator/cells/first/{path}", json={}) + assert (response.status_code, response.json()["code"]) == (409, "CONTROL_UPGRADE_REQUIRED") + response = await client.post( + f"/cells/v1/operator/cells/first/{path}", + content=b"broken", + headers={"Authorization": "Bearer lco_wrong"}, + ) + assert response.status_code == 401 + assert await conn.fetchval("SELECT count(*) FROM loom_router.audit") == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("aba", [False, True]) +async def test_inflight_finalizer_cannot_release_after_new_owner_rewidens( + control_stores, + local_control, + aba, +) -> None: + """Use the production mutation guard with a test-only narrowing HTTP action. + + The update command is a later concern. This exercises its directory boundary without + introducing an update route or implementation in this foundation. + """ + import asyncpg + from starlette.responses import JSONResponse + from starlette.routing import Route + + from server.cli import cell_attach_pg as apg + + client, directory_admin, _ = control_stores + local_admin, local_dsn = local_control + current = {"v": 1, "fields": {"deployment_id": "dep-first", "bearer_issuers": ["old"]}} + nxt = {"v": 1, "fields": {"deployment_id": "dep-first", "bearer_issuers": ["new"]}} + await directory_admin.execute( + "UPDATE loom_router.cells SET current_fp = $1::jsonb, next_fp = $2::jsonb", + json.dumps(current), + json.dumps(nxt), + ) + await local_admin.execute("INSERT INTO loom.cell_first(is_first, cell) VALUES (true, 'first')") + await local_admin.execute("INSERT INTO loom.tenancy_mode(mode) VALUES ('recording')") + await local_admin.execute( + "INSERT INTO loom.cell_self(current_fp, next_fp) VALUES ($1, $2)", nxt, None + ) + a = await asyncpg.connect(local_dsn) + b = await asyncpg.connect(local_dsn) + await directory._init(b) + started, resume = asyncio.Event(), asyncio.Event() + task = None + router = app.Router() + + async def finalizer(request): + body = await router._operator_body(request) + if isinstance(body, JSONResponse): + return body + compare = app._control(body) + if isinstance(compare, JSONResponse): + return compare + started.set() + await resume.wait() # request already sent; caller can no longer retract it + + async def change(): + async def mutate(conn, found): + await conn.execute( + "UPDATE loom_router.cells SET current_fp = next_fp, next_fp = NULL, issuers = ARRAY['new'] WHERE cell = 'first'" + ) + return found + + return await directory._attach_change("first", compare, mutate) + + return await app._answered(change, app._attach_answer) + + client._transport.app.routes.insert( + 0, Route("/test-finalize/{cell}", finalizer, methods=["POST"]) + ) + try: + await apg.take_lock(a, "old controller") + old = await claim(client, await read(client)) + task = asyncio.create_task(client.post("/test-finalize/first", json=expected(old))) + await asyncio.wait_for(started.wait(), 3) + await local_admin.execute("SELECT pg_terminate_backend($1)", a.get_server_pid()) + await apg.take_lock(b, "new controller") + newer = await claim(client, await read(client)) + + # New owner claims BEFORE widening local acceptance, on the owned backend. + async def widen(): + await b.execute("UPDATE loom.cell_self SET current_fp = $1, next_fp = $2", current, nxt) + + if aba: + await directory_admin.execute( + "UPDATE loom_router.cells SET current_fp = next_fp, next_fp = NULL" + ) + await directory_admin.execute( + "UPDATE loom_router.cells SET current_fp = $1::jsonb, next_fp = $2::jsonb", + json.dumps(current), + json.dumps(nxt), + ) + await _owner(b).transaction(widen) + assert newer["mutation_generation"] != old["mutation_generation"] + resume.set() + response = await task + assert response.status_code == 409, response.text + assert (await read(client))["next_fp"] == nxt + assert await local_admin.fetchval("SELECT next_fp FROM loom.cell_self") == nxt + finally: + resume.set() + if task is not None: + await task + await a.close() + await b.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("placement_first", [False, True]) +async def test_backout_closure_serializes_queued_placement( + control_stores, + local_control, + placement_first, +) -> None: + import asyncpg + + from server.cli import cell_attach_pg as apg + from server.cli import cell_control + + client, directory_admin, directory_dsn = control_stores + local_admin, _ = local_control + await directory_admin.execute("UPDATE loom_router.cells SET mode = 'enforced'") + await local_admin.execute("INSERT INTO loom.cell_first(is_first, cell) VALUES (true, 'first')") + await local_admin.execute("INSERT INTO loom.tenancy_mode(mode) VALUES ('recording')") + await local_admin.execute("INSERT INTO loom.cell_self(current_fp) VALUES ('{\"v\":1}')") + await apg.take_lock(local_admin, "backout test") + original = await read(client) + blocker = await asyncpg.connect(directory_dsn) + placement = cleanup = None + + async def call(path, body): + response = await client.post("/cells/v1/operator/cells/first" + path, json=body) + if response.status_code != 200: + raise cell_control._refused(response.json()["error"]) + return response.json() + + async def wait_for_lock(key): + for _ in range(100): + count = await directory_admin.fetchval( + "SELECT count(*) FROM pg_locks WHERE locktype = 'advisory' AND classid::bigint = $1 AND NOT granted", + key, + ) + if count: + return + await asyncio.sleep(0.01) + pytest.fail("owned test request never reached its lock") + + tx = blocker.transaction() + await tx.start() + try: + if placement_first: + await blocker.fetchrow( + "SELECT cell FROM loom_router.cells WHERE cell = 'first' FOR UPDATE" + ) + placement = asyncio.create_task( + directory.place("queued", "first", app.placement_refusal) + ) + # The placement has its org lock and is queued at the cell row before claim. + for _ in range(100): + if await directory_admin.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_locks WHERE locktype='advisory' AND classid::bigint=$1 AND granted)", + directory._PLACE_LOCK, + ): + break + await asyncio.sleep(0.01) + else: + pytest.fail("placement did not acquire its org lock") + cleanup = asyncio.create_task(cell_control.backout(_owner(local_admin), call, original)) + for _ in range(100): + waiting = await directory_admin.fetchval( + "SELECT count(*) FROM pg_stat_activity WHERE datname = current_database() " + "AND application_name = 'loom-router' AND wait_event_type = 'Lock'" + ) + if waiting >= 2: + break + await asyncio.sleep(0.01) + else: + pytest.fail("placement and claim did not both queue behind the cell row") + await tx.commit() + assert await placement == ("first", "dep-first") + with pytest.raises(cell_control.Refused, match="holds placements"): + await cleanup + assert await local_admin.fetchval("SELECT EXISTS(SELECT 1 FROM loom.cell_self)") + assert await local_admin.fetchval("SELECT mode FROM loom.tenancy_mode") == "recording" + else: + await blocker.execute( + "SELECT pg_advisory_xact_lock($1, hashtext($2))", directory._PLACE_LOCK, "queued" + ) + placement = asyncio.create_task( + directory.place("queued", "first", app.placement_refusal) + ) + await wait_for_lock(directory._PLACE_LOCK) + await cell_control.backout(_owner(local_admin), call, original) + assert not await local_admin.fetchval("SELECT EXISTS(SELECT 1 FROM loom.cell_self)") + await tx.commit() + with pytest.raises(directory.PlacementRefused, match="closed"): + await placement + assert await directory_admin.fetchval("SELECT closed FROM loom_router.cells") + assert (await read(client))["placed_orgs"] == int(placement_first) + finally: + if blocker.is_in_transaction(): + await tx.rollback() + for task in (placement, cleanup): + if task is not None and not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + except Exception: # noqa: BLE001 — preserve the test failure while reaping owned tasks + pass + await blocker.close() + + +@pytest.mark.asyncio +async def test_registration_rerun_keeps_claim_and_prior_column_lists_still_work(control_stores): + client, conn, _ = control_stores + held = await claim(client, await read(client)) + reg = directory.Registration( + cell="first", + is_first=True, + host="first.example.test", + deployment_id="dep-first", + credential_sha256="a" * 64, + bucket="", + bucket_prefix="", + probe_url="", + tenancy_rows=0, + state_rows=0, + prefix_objects=0, + processes_without_cell=0, + ) + row, written = await directory.register(reg) + assert not written and str(row.mutation_generation) == held["mutation_generation"] + # The prior data plane uses explicit lists and does not interpret the added column. + legacy = await conn.fetchrow( + "SELECT cell, is_first, host, state, mode, closed, placements_max, placed, credential_sha256, credential_next_sha256, deployment_id, current_fp, next_fp FROM loom_router.cells" + ) + assert legacy["cell"] == "first" and "mutation_generation" not in legacy + await conn.execute( + "INSERT INTO loom_router.cells(cell,host,state,credential_sha256,deployment_id) VALUES ('later','later.example.test','attaching',$1,'dep-later')", + "b" * 64, + ) + generations = await conn.fetch("SELECT mutation_generation FROM loom_router.cells") + assert len({r["mutation_generation"] for r in generations}) == 2 + + +@pytest.mark.asyncio +async def test_attach_lock_dies_between_confirmation_and_mode_write(local_control): + """The ready callback is a real scheduling gap after the lock check.""" + import argparse + + import asyncpg + + from server.cli import cell_attach + from server.cli import cell_attach_pg as apg + + admin, dsn = local_control + fp = cell.as_value({"deployment_id": "dep-first"}) + await admin.execute("INSERT INTO loom.cell_first(is_first,cell) VALUES(true,'first')") + await admin.execute("INSERT INTO loom.tenancy_mode(mode) VALUES('recording')") + await admin.execute("INSERT INTO loom.cell_self(current_fp) VALUES($1)", fp) + a, b = await asyncpg.connect(dsn), await asyncpg.connect(dsn) + await apg.take_lock(a, "old attach") + + class Router: + async def call(self, path, body): + assert path == "/attach/enforced" + return {"cell": "first"} + + run = cell_attach.Run( + None, argparse.Namespace(cell="first", wait=0), Router(), lambda _text: None + ) + run.lock_conn = a + run.owner = _owner(a) + run.mode = "recording" + run.own_fp = (fp, None) + run.row = { + "cell": "first", + "is_first": True, + "host": "first.example.test", + "deployment_id": "dep-first", + "bucket": "", + "bucket_prefix": "", + "probe_url": "", + "upstream_url": "", + "current_fp": fp, + "next_fp": None, + "mutation_generation": str(uuid.UUID(int=1)), + } + + async def ready(): + await admin.execute("SELECT pg_terminate_backend($1)", a.get_server_pid()) + await apg.take_lock(b, "successor") + return [] + + run.ready_for_commit = ready + try: + with pytest.raises( + (asyncpg.InterfaceError, asyncpg.PostgresError, asyncpg.InternalClientError) + ): + await run.commit_mode() + assert await apg.lock_held(b) + assert await admin.fetchval("SELECT mode FROM loom.tenancy_mode") == "recording" + finally: + await a.close() + await b.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fresh_open", [False, True]) +async def test_backout_close_cannot_be_reopened_before_same_session_cleanup( + control_stores, local_control, fresh_open +): + from server.cli import cell_attach_pg as apg + from server.cli import cell_control + + client, directory_admin, _ = control_stores + local, _ = local_control + await directory_admin.execute( + "UPDATE loom_router.cells SET mode = 'enforced', placements_max = 7" + ) + await local.execute("INSERT INTO loom.cell_first(is_first, cell) VALUES (true, 'first')") + await local.execute("INSERT INTO loom.tenancy_mode(mode) VALUES ('recording')") + await local.execute("INSERT INTO loom.cell_self(current_fp) VALUES ('{\"v\":1}')") + await apg.take_lock(local, "backout reopen test") + pid = local.get_server_pid() + before_close = opened = placement_error = None + + async def call(path, body): + nonlocal before_close, opened, placement_error + response = await client.post("/cells/v1/operator/cells/first" + path, json=body) + assert response.status_code == 200, response.text + answer = response.json() + if path == "/claim": + # The queued opener has the NEW generation, read before cleanup closes D. + before_close = await read(client) + if path == "/limit": + assert answer["closed"] and answer["placed_orgs"] == 0 + assert await local.fetchval("SELECT EXISTS(SELECT 1 FROM loom.cell_self)") + assert await apg.lock_held(local) + snapshot = await read(client) if fresh_open else before_close + opened = await client.post( + "/cells/v1/operator/cells/first/limit", + json={**expected(snapshot), "closed": False, "placements_max": 99}, + ) + # Actually attempt placement before the acknowledged close returns to cleanup. + try: + await directory.place("between-close-and-cleanup", "first", app.placement_refusal) + except directory.PlacementRefused as exc: + placement_error = str(exc) + return answer + + await cell_control.backout(_owner(local), call, await read(client)) + assert opened.status_code == 409, opened.text + assert "serving" in opened.json()["error"] + assert placement_error and "closed" in placement_error + after = await read(client) + assert after["placed_orgs"] == 0 + assert tuple( + await directory_admin.fetchrow("SELECT closed, placements_max FROM loom_router.cells") + ) == (True, 7) + assert after["mutation_generation"] == before_close["mutation_generation"] + assert not await directory_admin.fetchval("SELECT EXISTS(SELECT 1 FROM loom_router.placements)") + assert ( + await directory_admin.fetchval( + "SELECT count(*) FROM loom_router.audit WHERE action = 'limit'" + ) + == 1 + ) + assert local.get_server_pid() == pid and await apg.lock_held(local) + for table in ("cell_first", "tenancy_mode", "cell_self", "tenancy"): + assert not await local.fetchval(f"SELECT EXISTS(SELECT 1 FROM loom.{table})") + + +@pytest.mark.asyncio +async def test_attaching_open_refusal_preserves_guard_order_and_closed_attach_progress( + control_stores, +): + client, conn, _ = control_stores + original = await read(client) + current = await claim(client, original) + + async def limit(snapshot, **fields): + return await client.post( + "/cells/v1/operator/cells/first/limit", json={**expected(snapshot), **fields} + ) + + stale = await limit(original, closed=False, placements_max=99) + assert stale.status_code == 409 and "generation" in stale.json()["error"] + for field, value in (("host", "other.example.test"), ("current_fp", {"v": 2})): + changed = await limit({**current, field: value}, closed=False, placements_max=99) + assert ( + changed.status_code == 409 and "registration or fingerprints" in changed.json()["error"] + ) + noop = await limit(current, closed=False, placements_max=99) + assert noop.status_code == 409 and "serving" in noop.json()["error"] + assert await conn.fetchval("SELECT placements_max FROM loom_router.cells") is None + assert not await conn.fetchval( + "SELECT EXISTS(SELECT 1 FROM loom_router.audit WHERE action='limit')" + ) + assert (await limit(current, closed=True)).status_code == 200 + assert (await limit(current, placements_max=0)).status_code == 200 + closed = await read(client) + assert tuple(await conn.fetchrow("SELECT closed, placements_max FROM loom_router.cells")) == ( + True, + 0, + ) + assert closed["mutation_generation"] == current["mutation_generation"] + + # An earlier mode request cannot escape the cleanup owner's claim. + stale_mode = await client.post( + "/cells/v1/operator/cells/first/attach/enforced", + json={**expected(original), "deployment_id": "dep-first"}, + ) + assert stale_mode.status_code == 409 and "generation" in stale_mode.json()["error"] + # Existing held-org attach batches intentionally bypass closed/quota. + await conn.execute( + "UPDATE loom_router.cells SET mode='enforced', attach_started_at=now(), current_fp='{\"v\":1}'" + ) + row = await read(client) + values = expected(row, generation=False)["expected"] + control = directory.Control(values, uuid.UUID(row["mutation_generation"])) + assert await directory.attach_place("first", "dep-first", ["held-org"], expected=control) == ( + 1, + 1, + ) + served = await directory.attach_serving("first", "dep-first", expected=control) + assert served.row.closed and served.row.state == "serving" + response = await limit(await read(client), closed=False) + assert response.status_code == 200, response.text + assert not response.json()["closed"] + + +@pytest.mark.asyncio +async def test_pending_attach_batch_cannot_escape_backout_claim( + control_stores, local_control, monkeypatch +): + import asyncpg + + from server.cli import cell_attach_pg as apg + from server.cli import cell_control + + client, directory_admin, _ = control_stores + local, dsn = local_control + await directory_admin.execute( + "UPDATE loom_router.cells SET mode='enforced', attach_started_at=now(), current_fp='{\"v\":1}'" + ) + await local.execute("INSERT INTO loom.cell_first(is_first,cell) VALUES(true,'first')") + await local.execute("INSERT INTO loom.tenancy_mode(mode) VALUES('recording')") + await local.execute("INSERT INTO loom.cell_self(current_fp) VALUES('{\"v\":1}')") + old_backend = await asyncpg.connect(dsn) + await apg.take_lock(old_backend, "old attach batch") + old = await claim(client, await read(client)) + entered, resume = asyncio.Event(), asyncio.Event() + real = directory.attach_place + + async def delayed(*args, **kwargs): + entered.set() + await resume.wait() + return await real(*args, **kwargs) + + monkeypatch.setattr(directory, "attach_place", delayed) + pending = asyncio.create_task( + client.post( + "/cells/v1/operator/cells/first/attach/place", + json={**expected(old), "deployment_id": "dep-first", "orgs": ["old-batch"]}, + ) + ) + + async def call(path, body): + response = await client.post("/cells/v1/operator/cells/first" + path, json=body) + assert response.status_code == 200, response.text + if path == "/limit": + resume.set() + stale = await pending + assert stale.status_code == 409 and "generation" in stale.json()["error"] + return response.json() + + try: + await asyncio.wait_for(entered.wait(), 3) + await local.execute("SELECT pg_terminate_backend($1)", old_backend.get_server_pid()) + await apg.take_lock(local, "backout successor") + await cell_control.backout(_owner(local), call, await read(client)) + assert await apg.lock_held(local) + assert not await local.fetchval("SELECT EXISTS(SELECT 1 FROM loom.cell_self)") + assert not await directory_admin.fetchval( + "SELECT EXISTS(SELECT 1 FROM loom_router.placements)" + ) + assert await directory_admin.fetchval("SELECT closed FROM loom_router.cells") + finally: + resume.set() + await pending + await old_backend.close() diff --git a/tests/integration/test_cell_org_triggers_pg.py b/tests/integration/test_cell_org_triggers_pg.py index 3d412047bf..65e8691df7 100644 --- a/tests/integration/test_cell_org_triggers_pg.py +++ b/tests/integration/test_cell_org_triggers_pg.py @@ -672,7 +672,8 @@ async def test_a_dropped_guard_function_is_named_in_any_state_and_fails_an_enfor await conn.execute(f"DROP FUNCTION {cell_triggers.TENANCY_FUNCTION}()") if state == "enforced": with pytest.raises(asyncpg.exceptions.UndefinedFunctionError): - await cell_triggers.ensure(conn) + async with conn.transaction(): + await cell_triggers.ensure(conn) defects = {(d.kind, d.text) for d in await cell_triggers.missing(conn)} assert ("function", "loom.tenancy_refuse_unmarked() is absent") in defects if state == "enforced": # the failed boot created no guard @@ -714,6 +715,33 @@ async def test_missing_names_a_guard_disabled_narrowed_or_calling_another_functi assert "loom.tenancy_refuse_unmarked() has a body or settings no migration writes" in texts +@pytest.mark.asyncio +async def test_ensure_table_creates_a_recording_first_cells_triggers_only_when_asked() -> None: + """Attach creates a first cell's triggers in `recording`, one table at a time; a boot's + `ensure` never does, and the function passes every row there.""" + async with _cell("recording") as (dsn, conn): + with pytest.raises(RuntimeError, match="recording"): + await cell_triggers.ensure_table(conn, "scope_generations") + ensure = cell_triggers.ensure_table + assert (await ensure(conn, "scope_generations", recording_first=True)).answer == "applied" + assert (await ensure(conn, "tenancy", recording_first=True)).answer == "applied" + assert await _cell_triggers(conn) == { + ("scope_generations", "tenancy_insert", "O"), + ("scope_generations", "tenancy_update", "O"), + ("tenancy", "tenancy_record", "O"), + } + # Again: nothing to do. + assert (await ensure(conn, "scope_generations", recording_first=True)).answer == "applied" + with pytest.raises(ValueError, match="no cell triggers"): + await cell_triggers.ensure_table(conn, "idempotency", recording_first=True) + plain = await _unmarked(dsn) + try: + await plain.execute(_INSERT, "unheld-org", "s") # recording passes every row + await plain.execute(_HOLD, "new-org") # a first cell still recording records + finally: + await plain.close() + + @pytest.mark.asyncio @pytest.mark.parametrize("loom_cell", ["eu-1", ""]) async def test_the_cell_refresh_reads_the_marker_its_pool_sends( diff --git a/tests/integration/test_cell_retirement.py b/tests/integration/test_cell_retirement.py new file mode 100644 index 0000000000..8032268818 --- /dev/null +++ b/tests/integration/test_cell_retirement.py @@ -0,0 +1,178 @@ +"""Certified retirement and pruning against an owned scratch ClickHouse table.""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import os +import sys +from uuid import UUID, uuid4 + +import pytest + +from server.core import provider_process, retirement +from server.core.ch import CHClient, query_tag +from server.core.sql_literal import sql_string +from server.insights.store import provider_participants as store + + +def _write_scratch_owner(tmp_path, owner, **changes): + owner.update(changes) + owner["cleanup_complete"] = owner["drop_confirmed"] and owner["disconnect_confirmed"] + path = tmp_path / "scratch-owner.json" + staging = tmp_path / "scratch-owner.next" + descriptor = os.open(staging, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + json.dump(owner, stream, indent=2) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + os.replace(staging, path) + descriptor = os.open(tmp_path, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _scratch_owner(tmp_path, request, database, table): + owner = { + "schema": "loom.native-scratch.v1", + "case_nodeid": request.node.nodeid, + "database": database, + "table": table, + "ownership_marker": "loom-native-retirement-" + uuid4().hex, + "create_intended": True, + "create_confirmed": False, + "table_created": False, + "drop_confirmed": False, + "disconnect_confirmed": False, + } + _write_scratch_owner(tmp_path, owner) + return owner + + +@pytest.mark.asyncio +async def test_certified_retirement_survives_old_beat_and_prunes_only_certified_history( + monkeypatch, + tmp_path, + request, +): + if not os.environ.get("LOOM_CH_HOST"): + pytest.skip("Set LOOM_CH_HOST to the owned disposable ClickHouse") + database = "retirement_" + uuid4().hex + table = database + ".participants" + owner = _scratch_owner(tmp_path, request, database, table) + client = CHClient() + created = False + with query_tag("test.cell_retirement"): + try: + await client.connect() + await client.execute( + "CREATE DATABASE {database:Identifier} COMMENT " + + sql_string(owner["ownership_marker"]), + {"database": database}, + ) + created = True + _write_scratch_owner(tmp_path, owner, create_confirmed=True) + await client.execute( + "CREATE TABLE {database:Identifier}.participants AS loom._provider_participants", + {"database": database}, + ) + _write_scratch_owner(tmp_path, owner, table_created=True) + old_table = store.TABLE + for name in ( + "TABLE", + "PARTICIPANTS_SQL", + "OWN_VERSION_SQL", + "_PRUNE_IDS_SQL", + "_PRUNE_SQL", + ): + monkeypatch.setattr(store, name, getattr(store, name).replace(old_table, table)) + monkeypatch.setattr( + store, "_PRUNE_SQL", store._PRUNE_SQL + " SETTINGS mutations_sync=2" + ) + stopped, legacy = uuid4(), uuid4() + process = await asyncio.create_subprocess_exec(sys.executable, "-c", "pass") + try: + await asyncio.wait_for(process.wait(), timeout=10) + finally: + if process.returncode is None: + process.kill() + await process.wait() + fields = provider_process.host_proof().split("|") + fields[4:6] = [str(process.pid), "1"] + identity = "|".join(fields) + await client.insert( + table, + [ + { + "participant_id": stopped, + "state": "running", + "version": 1, + "host_proof": identity, + }, + {"participant_id": legacy, "state": "retired", "version": 1, "host_proof": ""}, + ], + ) + assert await store.running_queries(stopped, ch=client) == 0 + proof = retirement.death_proof(stopped, identity, after=1) + assert proof is not None + await store.record_retirement(proof, ch=client) + # A previously in-flight heartbeat cannot outrank the certified final row. + await client.insert( + table, + [ + { + "participant_id": stopped, + "state": "running", + "version": 2, + "host_proof": identity, + } + ], + ) + rows = await client.query_all( + "SELECT participant_id, transition_id, state, version, stale_after_s, host_proof " + f"FROM {table} FINAL" + ) + final = next(row for row in rows if str(row["participant_id"]) == str(stopped)) + assert str(final["transition_id"]) == str(UUID(int=0)) + assert final["state"] == "retired" and final["version"] > 2 + assert final["stale_after_s"] == 0 + assert final["host_proof"] == provider_process.certified(identity) + assert [str(row["participant_id"]) for row in await store.participants(ch=client)] == [ + str(legacy) + ] + # Both are old, but only the certified identity is allowed to disappear. + await client.execute( + f"ALTER TABLE {table} UPDATE heartbeat_at=now64(3)-toIntervalHour(25) " + "WHERE 1 SETTINGS mutations_sync=2" + ) + assert await store.prune_retired(ch=client) == 1 + rows = await client.query_all(f"SELECT participant_id, state FROM {table} FINAL") + assert [(str(row["participant_id"]), row["state"]) for row in rows] == [ + (str(legacy), "retired") + ] + finally: + # Keep the test's failure if required scratch cleanup also fails. + with ( + contextlib.suppress(BaseException) + if sys.exception() is not None + else contextlib.nullcontext() + ): + try: + if created: + await client.execute( + "DROP DATABASE {database:Identifier} SYNC", {"database": database} + ) + _write_scratch_owner(tmp_path, owner, drop_confirmed=True) + finally: + # Disconnect must not replace a failed drop or receipt write. + with ( + contextlib.suppress(BaseException) + if sys.exception() is not None + else contextlib.nullcontext() + ): + await client.disconnect() + _write_scratch_owner(tmp_path, owner, disconnect_confirmed=True) diff --git a/tests/integration/test_cell_scan.py b/tests/integration/test_cell_scan.py index d1892be846..c78bc40331 100644 --- a/tests/integration/test_cell_scan.py +++ b/tests/integration/test_cell_scan.py @@ -166,7 +166,7 @@ def walk(node: dict[str, Any]) -> None: ("Index Only Scan", "tenancy_pkey"), ("Index Only Scan", "tenancy_pkey"), ], scans - assert len(await cell_scan._pg_orgs("loom.tenancy", Budget(60))) == 20000 + assert len(await cell_scan.pg_orgs("loom.tenancy", Budget(60))) == 20000 # Every org-keyed table of the schema has the index the scan needs. names, refusals = await cell_scan.list_postgres(own_pg) assert refusals == [] diff --git a/tests/integration/test_cell_unset_gate_pg.py b/tests/integration/test_cell_unset_gate_pg.py index e94d5adb3f..37e80a7057 100644 --- a/tests/integration/test_cell_unset_gate_pg.py +++ b/tests/integration/test_cell_unset_gate_pg.py @@ -245,22 +245,68 @@ async def test_loom_cell_removes_check_reads_the_first_cells_postgres_alone( ) -> None: """`loom cell remove`'s check of a first cell: one session from LOOM_PG_DSN, open and recording the row's deployment id, or it refuses.""" + import argparse + import asyncpg + import httpx from server.cli import cell as cell_cli + from server.cli.cell_ch import Refused async with own_database() as dsn: monkeypatch.setattr(settings, "pg_dsn", dsn) + monkeypatch.setattr(settings, "cell", "eu-1") + monkeypatch.setattr(settings, "document_s3_bucket", "") monkeypatch.setattr(pg, "_expected", None) + row = { + "cell": "eu-1", + "is_first": True, + "host": "cell.example.test", + "state": "attaching", + "placed_orgs": 0, + "deployment_id": "dep-1", + "bucket": "", + "bucket_prefix": "", + "probe_url": "", + "upstream_url": "", + "current_fp": None, + "next_fp": None, + "mutation_generation": "00000000-0000-0000-0000-000000000001", + } + calls = [] + + def router(request: httpx.Request) -> httpx.Response: + calls.append(request.method) + if request.method == "GET" or request.url.path.endswith("/claim"): + return httpx.Response(200, json=row) + assert request.method == "DELETE" + return httpx.Response(200, json={"cell": "eu-1", "removed": True}) + + args = argparse.Namespace(cell="eu-1") + url = httpx.URL("https://router.example.test") + transport = httpx.MockTransport(router) conn = await asyncpg.connect(dsn) try: await conn.execute( "INSERT INTO loom.authority (authority, epoch, deployment_id) " "VALUES ('postgres', 1, 'dep-1')" ) - assert await cell_cli._first_cell_refusal("dep-1") == "" - assert "records deployment id 'dep-1'" in await cell_cli._first_cell_refusal("dep-2") + assert await cell_cli._remove_locked(args, url, transport) == { + "cell": "eu-1", + "removed": True, + } + assert calls == ["GET", "POST", "DELETE"] + calls.clear() + row["deployment_id"] = "dep-2" + with pytest.raises(Refused, match="do not match the directory registration"): + await cell_cli._remove_locked(args, url, transport) + assert calls == ["GET"] + assert await conn.fetchval("SELECT deployment_id FROM loom.authority") == "dep-1" + calls.clear() + row["deployment_id"] = "dep-1" await _write_state(conn, {"tenancy"}) - assert "reads `partial`" in await cell_cli._first_cell_refusal("dep-1") + with pytest.raises(Refused, match="reads partial"): + await cell_cli._remove_locked(args, url, transport) + assert calls == ["GET"] finally: await conn.close() diff --git a/tests/integration/test_directory_control_cleanup.py b/tests/integration/test_directory_control_cleanup.py new file mode 100644 index 0000000000..c07ce79ac8 --- /dev/null +++ b/tests/integration/test_directory_control_cleanup.py @@ -0,0 +1,510 @@ +"""Directory control closes its actual backend before an interrupted call returns.""" + +from __future__ import annotations + +import asyncio +import os +import time +from collections.abc import AsyncIterator +from typing import Any + +import pytest + +from server.router import directory +from server.settings import settings +from tests.integration._pg_database import create_database, drop_database +from tests.integration.test_pg_owned_cleanup import released, sleeping + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +async def directory_database(monkeypatch) -> AsyncIterator[Any]: + import asyncpg + + admin = os.environ.get("LOOM_PG_DSN") or pytest.skip("LOOM_PG_DSN not set") + name, dsn = await create_database(admin) + monkeypatch.setattr(settings, "router_pg_dsn", dsn) + monkeypatch.setattr(directory, "_pool", None) + observer = await asyncpg.connect(dsn) + await observer.execute("CREATE TABLE probe(id int PRIMARY KEY); INSERT INTO probe VALUES(1)") + try: + yield observer + finally: + await directory.close() + await observer.close(timeout=1) + await drop_database(admin, name) + + +async def locked(conn): + await conn.execute("SELECT pg_advisory_xact_lock($1)", directory._DIRECTORY_LOCK) + await conn.execute("SELECT * FROM probe FOR UPDATE") + + +@pytest.mark.parametrize("interrupt", ["cancel", "deadline"]) +async def test_directory_interrupt_releases_exact_backend_and_global_locks( + directory_database, monkeypatch, interrupt +): + seen = asyncio.Event() + backend = {} + monkeypatch.setattr(directory, "CONTROL_MAX_S", 0.5) + + async def work(conn): + backend["pid"] = conn.get_server_pid() + await locked(conn) + seen.set() + await conn.execute("SELECT pg_sleep(10)") + + task = asyncio.create_task( + directory._control_transaction(work, seconds=directory.CONTROL_MAX_S) + ) + try: + await asyncio.wait_for(seen.wait(), 2) + await sleeping(directory_database, backend["pid"]) + began = time.monotonic() + if interrupt == "cancel": + task.cancel("original directory cancellation") + with pytest.raises( + asyncio.CancelledError if interrupt == "cancel" else directory.DirectoryUnavailable + ): + await asyncio.wait_for(task, 1) + assert time.monotonic() - began < 0.6 + await released(directory_database, backend["pid"]) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_directory_original_failure_never_waits_for_rollback(directory_database, monkeypatch): + import asyncpg + + original = ValueError("original directory work failure") + real = asyncpg.Connection.execute + backend = {} + monkeypatch.setattr(directory, "CONTROL_MAX_S", 0.4) + + async def stalled(self, query, *args, **kwargs): + if query.startswith("ROLLBACK"): + backend["rollback"] = True + await real(self, "SELECT pg_sleep(10)") + return await real(self, query, *args, **kwargs) + + monkeypatch.setattr(asyncpg.Connection, "execute", stalled) + began = time.monotonic() + + async def work(conn): + backend["pid"] = conn.get_server_pid() + await locked(conn) + raise original + + with pytest.raises(ValueError) as caught: + await directory._control_transaction(work, seconds=directory.CONTROL_MAX_S) + assert caught.value is original and "rollback" not in backend + assert time.monotonic() - began < 0.4 + await released(directory_database, backend["pid"]) + + +async def test_directory_normal_commit_closes_before_return(directory_database): + backend = {} + + async def work(conn): + backend["pid"] = conn.get_server_pid() + await locked(conn) + await conn.execute("UPDATE probe SET id=2") + return "acknowledged" + + assert ( + await directory._control_transaction(work, seconds=directory.CONTROL_MAX_S) + == "acknowledged" + ) + assert await directory_database.fetchval("SELECT id FROM probe") == 2 + await released(directory_database, backend["pid"]) + + +async def test_directory_second_cancel_keeps_first_and_close_precedes_worker_cancel( + directory_database, monkeypatch +): + import asyncpg + + monkeypatch.setattr(directory, "CONTROL_MAX_S", 0.4) + real_close = asyncpg.Connection.close + seen, closing, allow_close = asyncio.Event(), asyncio.Event(), asyncio.Event() + backend = {} + events = [] + + async def work(conn): + backend["pid"] = conn.get_server_pid() + await locked(conn) + seen.set() + try: + await conn.execute("SELECT pg_sleep(10)") + finally: + events.append("work stopped") + + async def close(self, **kwargs): + if self.get_server_pid() == backend.get("pid"): + events.append("close started") + await real_close(self, **kwargs) + closing.set() + await allow_close.wait() + else: + await real_close(self, **kwargs) + + monkeypatch.setattr(asyncpg.Connection, "close", close) + task = asyncio.create_task( + directory._control_transaction(work, seconds=directory.CONTROL_MAX_S) + ) + try: + await asyncio.wait_for(seen.wait(), 2) + await sleeping(directory_database, backend["pid"]) + task.cancel("first directory cancellation") + await asyncio.wait_for(closing.wait(), 1) + task.cancel("second directory cancellation") + with pytest.raises(asyncio.CancelledError, match="first directory cancellation"): + await asyncio.wait_for(task, 0.5) + assert events.index("close started") < events.index("work stopped") + assert events.count("close started") == 1 + await released(directory_database, backend["pid"]) + finally: + allow_close.set() + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_directory_commit_acknowledgement_loss_is_not_success( + directory_database, monkeypatch +): + import asyncpg + + real = asyncpg.Connection.execute + backend = {} + original = OSError("commit acknowledgement lost") + + async def execute(self, sql, *args, **kwargs): + value = await real(self, sql, *args, **kwargs) + if sql.startswith("COMMIT") and self.get_server_pid() == backend.get("pid"): + raise original + return value + + async def work(conn): + backend["pid"] = conn.get_server_pid() + await locked(conn) + await conn.execute("UPDATE probe SET id=2") + + monkeypatch.setattr(asyncpg.Connection, "execute", execute) + with pytest.raises(directory.DirectoryUnavailable) as caught: + await directory._control_transaction(work, seconds=directory.CONTROL_MAX_S) + assert caught.value.__cause__ is original + assert await directory_database.fetchval("SELECT id FROM probe") == 2 + await released(directory_database, backend["pid"]) + + +async def test_directory_required_close_failure_refuses_committed_success( + directory_database, monkeypatch +): + import asyncpg + + real_close = asyncpg.Connection.close + backend = {} + original = ValueError("required close failed") + + async def close(self, **kwargs): + ours = self.get_server_pid() == backend.get("pid") + await real_close(self, **kwargs) + if ours: + raise original + + async def work(conn): + backend["pid"] = conn.get_server_pid() + await locked(conn) + await conn.execute("UPDATE probe SET id=2") + + monkeypatch.setattr(asyncpg.Connection, "close", close) + with pytest.raises(ValueError) as caught: + await directory._control_transaction(work, seconds=directory.CONTROL_MAX_S) + assert caught.value is original + assert await directory_database.fetchval("SELECT id FROM probe") == 2 + await released(directory_database, backend["pid"]) + + +@pytest.mark.parametrize("stage", ["acquire", "work", "commit", "close", "release"]) +@pytest.mark.parametrize("kind", ["exit", "keyboard"]) +async def test_directory_fatal_child_is_inert_until_owned_cleanup(directory_database, stage, kind): + import sys + + from tests.integration._pg_database import with_database + + dsn = with_database( + os.environ["LOOM_PG_DSN"], await directory_database.fetchval("SELECT current_database()") + ) + script = r""" +import asyncio, contextlib, os, sys +import asyncpg +from server.router import directory +from server.settings import settings +stage, kind = sys.argv[1:] +original = SystemExit(17) if kind == 'exit' else KeyboardInterrupt('fatal guard') +async def main(): + settings.router_pg_dsn = os.environ['CONTROL_TEST_DSN'] + observer = await asyncpg.connect(settings.router_pg_dsn) + backend = {} + real_close = asyncpg.Connection.close + real_execute = asyncpg.Connection.execute + real_connection = directory._connection + async def execute(self, sql, *args, **kw): + if stage == 'commit' and sql.startswith('COMMIT'): raise original + return await real_execute(self, sql, *args, **kw) + async def close(self, **kw): + ours = self.get_server_pid() == backend.get('pid') + await real_close(self, **kw) + if ours and stage == 'close': raise original + @contextlib.asynccontextmanager + async def connection(): + if stage == 'acquire': raise original + async with real_connection() as conn: + yield conn + if stage == 'release': raise original + async def work(conn): + backend['pid'] = conn.get_server_pid() + await conn.execute('SELECT pg_advisory_xact_lock($1)', directory._DIRECTORY_LOCK) + await conn.execute('SELECT * FROM probe FOR UPDATE') + if stage == 'work': raise original + asyncpg.Connection.close = close + asyncpg.Connection.execute = execute + directory._connection = connection + try: + try: await directory._control_transaction(work, seconds=directory.CONTROL_MAX_S) + except (SystemExit, KeyboardInterrupt) as caught: assert caught is original + else: raise AssertionError('fatal guard returned success') + if stage == 'acquire': + assert not backend and directory._pool is None + else: + assert not await observer.fetchval('SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1)', backend['pid']) + assert not await observer.fetchval('SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)', backend['pid']) + finally: + asyncpg.Connection.close = real_close + asyncpg.Connection.execute = real_execute + await directory.close() + await observer.close(timeout=1) +asyncio.run(main()) +""" + process = await asyncio.create_subprocess_exec( + sys.executable, + "-c", + script, + stage, + kind, + env={**os.environ, "CONTROL_TEST_DSN": dsn}, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, stderr = await asyncio.wait_for(process.communicate(), 10) + assert process.returncode == 0, (stdout.decode(), stderr.decode()) + finally: + if process.returncode is None: + process.kill() + await process.wait() + + +@pytest.mark.parametrize("kind", ["ordinary", "cancel", "exit", "keyboard"]) +@pytest.mark.parametrize("healthy_loan", [False, True]) +@pytest.mark.parametrize("driver_close", ["normal", "exit", "keyboard"]) +async def test_failed_codec_raw_session_is_disposed_without_touching_other_loans( + directory_database, monkeypatch, kind, healthy_loan, driver_close +): + import asyncpg + + original = { + "ordinary": ValueError("fixed codec failed"), + "cancel": asyncio.CancelledError("fixed codec cancelled"), + "exit": SystemExit(17), + "keyboard": KeyboardInterrupt("fixed codec interrupted"), + }[kind] + pool = await directory.pool() + assert pool.get_size() == 0 # Pool object creation does not open a backend. + healthy = await pool.acquire() if healthy_loan else None + healthy_pid = healthy.get_server_pid() if healthy else None + real_codec, real_close = asyncpg.Connection.set_type_codec, asyncpg.Connection.close + failed = {} + attempts = 0 + + async def codec(self, *args, **kwargs): + nonlocal attempts + attempts += 1 + failed["raw"], failed["pid"] = self, self.get_server_pid() + raise original + + async def close(self, **kwargs): + if self is failed.get("raw"): + # This is the failed, never-borrowed raw codec session, not an active + # command session: synchronous quarantine must precede driver cleanup. + assert self.is_closed() + failed["driver close"] = True + if driver_close == "exit": + raise SystemExit(21) + if driver_close == "keyboard": + raise KeyboardInterrupt("secondary closed-raw cleanup failure") + return await real_close(self, **kwargs) + + async def acquire(): + async with directory._connection() as conn: + pytest.fail(f"failed initializer admitted backend {conn.get_server_pid()}") + + monkeypatch.setattr(asyncpg.Connection, "set_type_codec", codec) + monkeypatch.setattr(asyncpg.Connection, "close", close) + try: + if kind in {"exit", "keyboard"}: + result = await directory._control_result(acquire) + assert isinstance(result, directory._ControlFatal) and result.error is original + else: + with pytest.raises(type(original)) as caught: + await acquire() + assert caught.value is original + assert failed["driver close"] and attempts == 1 + await released(directory_database, failed["pid"]) + # Driver background replenishment must not swallow/retry control intent. + for _ in range(3): + await asyncio.sleep(0) + assert attempts == 1 + if healthy: + assert healthy.get_server_pid() == healthy_pid + assert await healthy.fetchval("SELECT 1") == 1 + monkeypatch.setattr(asyncpg.Connection, "set_type_codec", real_codec) + async with directory._connection() as recovered: + recovered_pid = recovered.get_server_pid() + assert recovered_pid not in {failed["pid"], healthy_pid} + assert await recovered.fetchval("SELECT 1") == 1 + async with directory._connection() as cached: + assert cached.get_server_pid() == recovered_pid + finally: + monkeypatch.setattr(asyncpg.Connection, "set_type_codec", real_codec) + if healthy: + await pool.release(healthy) + + +@pytest.mark.parametrize("kind", ["exit", "keyboard"]) +@pytest.mark.parametrize("secondary", ["ordinary", "exit", "keyboard"]) +async def test_codec_quarantine_failure_keeps_native_original_and_unknown_diagnostic( + directory_database, monkeypatch, kind, secondary +): + import asyncpg + + original = SystemExit(17) if kind == "exit" else KeyboardInterrupt("codec interrupted") + second = { + "ordinary": ValueError("termination failed"), + "exit": SystemExit(19), + "keyboard": KeyboardInterrupt("termination interrupted"), + }[secondary] + real_codec, real_terminate = asyncpg.Connection.set_type_codec, asyncpg.Connection.terminate + failed = {} + + async def codec(self, *args, **kwargs): + failed["raw"], failed["pid"] = self, self.get_server_pid() + raise original + + def terminate(self): + if self is failed.get("raw"): + raise second + return real_terminate(self) + + async def acquire(): + async with directory._connection(): + pytest.fail("failed initialization returned success") + + monkeypatch.setattr(asyncpg.Connection, "set_type_codec", codec) + monkeypatch.setattr(asyncpg.Connection, "terminate", terminate) + try: + result = await directory._control_result(acquire) + assert isinstance(result, directory._ControlFatal) and result.error is original + assert original.__notes__ == ["directory initialization session closure is unknown"] + # Termination's request is not the proof; here the driver's subsequent + # ordinary-error public close really completes and the observer proves it. + await released(directory_database, failed["pid"]) + finally: + monkeypatch.setattr(asyncpg.Connection, "set_type_codec", real_codec) + monkeypatch.setattr(asyncpg.Connection, "terminate", real_terminate) + raw = failed.get("raw") + if raw is not None: + real_terminate(raw) + + +@pytest.mark.parametrize("signal_count", [1, 2]) +async def test_directory_signals_keep_real_cancel_request_alive_until_original_end( + directory_database, monkeypatch, signal_count +): + from asyncpg import connect_utils + + seen, cancelling, allow_cancel = asyncio.Event(), asyncio.Event(), asyncio.Event() + backend = {} + first_signal = {} + real_wait = directory._ControlOwner.wait + real_cancel = connect_utils._cancel + seconds = 0.5 + + async def cancel(**kwargs): + if kwargs["backend_pid"] == backend.get("pid"): + cancelling.set() + await allow_cancel.wait() + return await real_cancel(**kwargs) + + async def work(conn): + backend["pid"] = conn.get_server_pid() + await locked(conn) + seen.set() + await conn.execute("SELECT pg_sleep(10)") + + async def remember(self, task, end): + try: + return await real_wait(self, task, end) + except asyncio.CancelledError as error: + first_signal.setdefault("error", error) + raise + + monkeypatch.setattr(directory._ControlOwner, "wait", remember) + monkeypatch.setattr(connect_utils, "_cancel", cancel) + began = time.monotonic() + task = asyncio.create_task(directory._control_transaction(work, seconds=seconds)) + try: + await asyncio.wait_for(seen.wait(), 2) + await sleeping(directory_database, backend["pid"]) + task.cancel("first directory signal") + await asyncio.wait_for(cancelling.wait(), 2) + if signal_count == 2: + task.cancel("second directory signal") + await asyncio.sleep(0.03) + assert not task.done(), "second signal abandoned a live server cancel request" + assert await directory_database.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1 AND wait_event='PgSleep')", + backend["pid"], + ) + assert ( + await directory_database.fetchval( + "SELECT count(*) FROM pg_locks WHERE pid=$1", backend["pid"] + ) + > 0 + ) + allow_cancel.set() + with pytest.raises(asyncio.CancelledError, match="first directory signal") as caught: + await asyncio.wait_for(task, seconds) + assert caught.value is first_signal["error"] + assert time.monotonic() - began <= seconds + 0.1 + await released(directory_database, backend["pid"]) + finally: + allow_cancel.set() + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + if backend: + await directory_database.execute("SELECT pg_cancel_backend($1)", backend["pid"]) + async with asyncio.timeout(2): + for _ in range(200): + if not await directory_database.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)", backend["pid"] + ): + break + await asyncio.sleep(0.01) + else: + pytest.fail("test-owned backend remained after exact cancellation") diff --git a/tests/integration/test_pg_authority.py b/tests/integration/test_pg_authority.py index c426d2d591..eee1707490 100644 --- a/tests/integration/test_pg_authority.py +++ b/tests/integration/test_pg_authority.py @@ -4320,6 +4320,15 @@ async def test_a_redacted_head_is_not_served_until_a_new_value_is_written( search leaves it out and counts it. A new value written over it is served.""" ns = f"pg-{uuid.uuid4().hex[:8]}" first = (await call("memory.set", _set(ns, "k", "the office is in Berlin"))).json() + # A first write answers at its commit; the old search document must exist before + # the head is scrubbed, or there is no stale hit for validation to count. + before = ( + await call( + "memory.search", {"namespace": ns, "query": "office Berlin", "auto_touch": False} + ) + ).json() + assert "k" in [h["memory_key"] for h in before["results"]], before + assert before["stale_dropped"] == 0, before # The head as a profile redaction leaves it: scrubbed in Postgres, its ClickHouse # document not yet. await pg_conn.execute( @@ -5741,3 +5750,112 @@ async def offered(after: Any = None) -> list[pg_memory.Clearable]: } assert a["value"] == "[REDACTED]" and a["redaction_reason"].startswith("leak") assert a["redacted_at"] and a["content_sha256"] != b["content_sha256"] + + +async def test_exact_head_locks_bound_a_large_absent_batch(pg_conn: Any, org: str) -> None: + """Absent keys sharing an identity do not exhaust the commit's statement budget.""" + from server.core.hashing import sha256_value + from server.memory.store import pg_memory + from server.memory.store.rows import ( + IDENTITY_COLUMNS, + MemoryRow, + build_first_version_row, + build_memory_version_row, + written_now, + ) + + namespace = "head-locks" + prefix = "gate-delivery-" + "1" * 32 + first_key = prefix + "-0" + template = pg_memory.encode_row( + build_memory_version_row( + MemoryRow.from_snapshot( + build_first_version_row( + { + "org": org, + "namespace": namespace, + "agent": "bench", + "memory_key": first_key, + "memory_type": "semantic", + "value": json.dumps(""), + "version": 1, + "content_sha256": sha256_value(""), + "actor": "process:head-lock-test", + "source_class": "asserted", + }, + now=written_now(), + ) + ) + ) + ) + assert pg_memory._LOCK_HEADS.count("loom.memory_current") == 1 + query = pg_memory._LOCK_HEADS.replace("loom.memory_current", "pg_temp.head_lock_candidates") + async with pg_conn.transaction(): + await pg_conn.execute( + "CREATE TEMP TABLE head_lock_candidates " + "(LIKE loom.memory_current INCLUDING ALL) WITH (fillfactor = 90) ON COMMIT DROP" + ) + # One statement builds full persisted rows without per-row network calls. + inserted = await pg_conn.execute( + "WITH keys AS (" + " SELECT i, $2::text || '-' || i::text AS key, " + "lpad(to_hex(i + 1), 32, '0')::uuid AS id FROM generate_series(0, 15699) AS i" + "), prefixes AS (" + " SELECT *, 'Cell trigger measurement ' || key || ': ' AS prefix FROM keys" + "), payloads AS (" + " SELECT *, prefix || left(repeat('abcdefghijklmnopqrstuvwxyz ', 4096), " + "4096 - length(prefix)) AS payload FROM prefixes" + "), stored AS (" + " SELECT *, to_json(payload)::text AS value, " + "encode(sha256(convert_to(to_json(payload)::text, 'UTF8')), 'hex') AS hash " + "FROM payloads) " + "INSERT INTO pg_temp.head_lock_candidates " + "(org, namespace, agent, user_id, session_id, memory_key, memory_id, " + "version, seq, content_sha256, row) " + "SELECT $1::jsonb->>'org', $1::jsonb->>'namespace', $1::jsonb->>'agent', " + "$1::jsonb->>'user_id', $1::jsonb->>'session_id', key, id, 1, i + 1, hash, " + "$1::jsonb || jsonb_build_object('memory_key', key, 'memory_id', id::text, " + "'value', value, 'content_sha256', hash) FROM stored", + template, + prefix, + ) + assert inserted == "INSERT 0 15700" + await pg_conn.execute("ANALYZE pg_temp.head_lock_candidates") + await pg_conn.execute("SET LOCAL statement_timeout = 5000") + await pg_conn.execute("SET LOCAL lock_timeout = 2000") + expected = {} + for index in (2, 10, 1): + key = f"{prefix}-{index}" + start = f"Cell trigger measurement {key}: " + payload = start + ("abcdefghijklmnopqrstuvwxyz " * 4096)[: 4096 - len(start)] + row = { + **template, + "memory_key": key, + "memory_id": str(uuid.UUID(int=index + 1)), + "value": json.dumps(payload), + "content_sha256": sha256_value(payload), + } + identity = pg_memory.identity_key(row) + expected[identity] = {**dict(zip(IDENTITY_COLUMNS, identity, strict=True)), "row": row} + matched = await pg_conn.fetch(query, *pg_memory._key_arrays(list(expected))) + assert [dict(row) for row in matched] == [expected[key] for key in sorted(expected)] + identity = next(iter(expected)) + changed = [ + tuple( + "different-" + value if position == axis else value + for position, value in enumerate(identity) + ) + for axis in range(6) + ] + assert await pg_conn.fetch(query, *pg_memory._key_arrays(sorted(changed))) == [] + absent_prefix = "gate-delivery-" + "2" * 32 + absent = sorted( + (org, namespace, "bench", "-", "-", f"{absent_prefix}-{index}") for index in range(2500) + ) + arguments = pg_memory._key_arrays(absent) + plan = await pg_conn.fetchval("EXPLAIN (FORMAT JSON) " + query, *arguments) + try: + assert await pg_conn.fetch(query, *arguments) == [] + except Exception as error: # The synthetic plan must not replace the native failure. + error.add_note("Synthetic head-lock plan before query: " + json.dumps(plan)) + raise diff --git a/tests/integration/test_pg_online_index.py b/tests/integration/test_pg_online_index.py new file mode 100644 index 0000000000..38638ad877 --- /dev/null +++ b/tests/integration/test_pg_online_index.py @@ -0,0 +1,398 @@ +"""Online index retries and migrator coexistence on scratch Postgres databases.""" + +from __future__ import annotations + +import asyncio +import contextlib +import os +import time +from collections.abc import AsyncIterator +from pathlib import Path +from typing import Any + +import pytest + +from server.core import pg +from server.core.paths import ROUTER_MIGRATIONS_DIR +from server.core.schema import pg_migrate, pg_online_index +from server.router import directory +from server.settings import settings +from tests.integration._pg_database import create_database, drop_database + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +async def database(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[tuple[Any, Any]]: + import asyncpg + + admin = os.environ.get("LOOM_PG_DSN") or pytest.skip("LOOM_PG_DSN not set") + name, dsn = await create_database(admin) + monkeypatch.setattr(settings, "router_pg_dsn", dsn) + monkeypatch.setattr(directory, "_pool", None) + + @contextlib.asynccontextmanager + async def connect() -> AsyncIterator[Any]: + conn = await asyncpg.connect(dsn, server_settings={"application_name": "online-index-test"}) + try: + yield conn + finally: + await conn.close() + + try: + async with connect() as conn: + yield conn, connect + finally: + await directory.close() + await drop_database(admin, name) + + +def _prefix(tmp_path: Path) -> Path: + prefix = tmp_path / "old" + prefix.mkdir() + for source in sorted(ROUTER_MIGRATIONS_DIR.glob("*.sql"))[:3]: + (prefix / source.name).write_bytes(source.read_bytes()) + return prefix + + +def _journal() -> list[str]: + return sorted(path.name for path in ROUTER_MIGRATIONS_DIR.glob("*.sql")) + + +def _after_prefix() -> list[str]: + return _journal()[3:] + + +async def _migrate(connect: Any, files: Path = ROUTER_MIGRATIONS_DIR) -> list[str]: + return await pg_migrate.migrate(files, connect=connect, schema="loom_router", lock_key=8917423) + + +async def test_online_migration_installs_and_retries_from_both_schemas(database, tmp_path) -> None: + conn, connect = database + await _migrate(connect, _prefix(tmp_path)) + assert await _migrate(connect) == _after_prefix() + assert await _migrate(connect) == [] + index = pg_online_index.parse( + (ROUTER_MIGRATIONS_DIR / "004_placements_by_cell_org.sql").read_text() + ) + assert index is not None and await pg_online_index.ready(conn, index) is True + assert await conn.fetchval("SELECT to_regclass('loom_router.placements_by_cell') IS NOT NULL") + + +async def test_install_from_empty(database) -> None: + _, connect = database + assert await _migrate(connect) == _journal() + + +async def _until(conn: Any, sql: str, *args: Any) -> None: + async with asyncio.timeout(8): + for _ in range(800): + if await conn.fetchval(sql, *args): + return + await asyncio.sleep(0.01) + pytest.fail("Postgres did not reach the expected migration state") + + +def _old_runner(): + """The shipped blocking runner, kept as source for the rolling-upgrade proof.""" + import importlib.util + + path = Path(__file__).parent / "fixtures" / "pg_migrate_n_minus_one.py" + spec = importlib.util.spec_from_file_location("old_pg_migrate", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +async def test_old_snapshot_old_blocking_migrator_and_two_new_migrators(database, tmp_path) -> None: + conn, connect = database + old_files = _prefix(tmp_path) + await _migrate(connect, old_files) + async with connect() as snapshot: + tx = snapshot.transaction(isolation="repeatable_read") + await tx.start() + await snapshot.fetchval("SELECT count(*) FROM loom_router.placements") + first = asyncio.create_task(_migrate(connect)) + second = None + try: + await _until( + conn, + "SELECT EXISTS(SELECT 1 FROM pg_stat_progress_create_index " + "WHERE datid = (SELECT oid FROM pg_database WHERE datname = current_database()) " + "AND phase = 'waiting for old snapshots')", + ) + second = asyncio.create_task(_migrate(connect)) + # The ordinary key is free while CREATE INDEX waits for this old snapshot. + # Running this original blocking implementation would deadlock if it weren't. + old = _old_runner() + async with asyncio.timeout(2): + assert ( + await old.migrate( + old_files, connect=connect, schema="loom_router", lock_key=8917423 + ) + == [] + ) + assert not first.done() and not second.done() + # Even while the index waits, the old serving path can write and read. + await conn.execute( + "INSERT INTO loom_router.cells " + "(cell, host, state, credential_sha256, deployment_id) " + "VALUES ('a', 'a.example.test', 'serving', repeat('a', 64), 'dep-a')" + ) + from tests.integration.fixtures import router_placement_n_minus_one as serving + + assert await serving.place("org-a", "a", lambda _: "") == ("a", "dep-a") + assert await serving.lookup("org-a") == ("a", "a.example.test") + assert await conn.fetchval("SELECT placed FROM loom_router.cells WHERE cell = 'a'") == 1 + assert ( + await conn.fetchval("SELECT count(*) FROM loom_router.audit WHERE action = 'place'") + == 1 + ) + await tx.rollback() + async with asyncio.timeout(5): + results = await asyncio.gather(first, second) + assert sorted(results) == [[], _after_prefix()] + finally: + if snapshot.is_in_transaction(): + await tx.rollback() + for task in (first, second): + if task is not None and not task.done(): + task.cancel() + await asyncio.gather( + *(t for t in (first, second) if t is not None), return_exceptions=True + ) + + +async def test_cancelled_build_leaves_a_matching_invalid_index_and_retry_repairs_it( + database, tmp_path +) -> None: + conn, connect = database + await _migrate(connect, _prefix(tmp_path)) + async with connect() as snapshot: + tx = snapshot.transaction(isolation="repeatable_read") + await tx.start() + await snapshot.fetchval("SELECT count(*) FROM loom_router.placements") + migrating = asyncio.create_task(_migrate(connect)) + try: + await _until( + conn, + "SELECT EXISTS(SELECT 1 FROM pg_stat_progress_create_index " + "WHERE datid = (SELECT oid FROM pg_database WHERE datname = current_database()) " + "AND phase = 'waiting for old snapshots')", + ) + migrating.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(migrating, 2) + assert await conn.fetchval( + "SELECT NOT indisvalid FROM pg_index " + "WHERE indexrelid = 'loom_router.placements_by_cell_org'::regclass" + ) + assert not await conn.fetchval( + "SELECT EXISTS(SELECT 1 FROM loom_router.pg_migrations " + "WHERE filename = '004_placements_by_cell_org.sql')" + ) + finally: + await tx.rollback() + if not migrating.done(): + migrating.cancel() + await asyncio.gather(migrating, return_exceptions=True) + assert await _migrate(connect) == _after_prefix() + assert await _migrate(connect) == [] + + +@pytest.mark.parametrize( + "definition", + [ + "CREATE TABLE loom_router.placements_by_cell_org (id int)", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (org, cell)", + 'CREATE INDEX placements_by_cell_org ON loom_router.placements (cell, org COLLATE "POSIX")', + "CREATE UNIQUE INDEX placements_by_cell_org ON loom_router.placements (cell, org)", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (cell, org) WHERE org > 'm'", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (cell, org) INCLUDE(placed_at)", + "CREATE INDEX placements_by_cell_org ON loom_router.cells (cell, host)", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (cell DESC, org)", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (cell text_pattern_ops, org)", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (cell, org) WITH (fillfactor=70)", + ], +) +async def test_foreign_same_named_object_is_never_dropped(database, tmp_path, definition) -> None: + conn, connect = database + await _migrate(connect, _prefix(tmp_path)) + await conn.execute(definition) + oid = await conn.fetchval("SELECT 'loom_router.placements_by_cell_org'::regclass::oid") + with pytest.raises(ValueError, match="different definition"): + await _migrate(connect) + assert await conn.fetchval("SELECT 'loom_router.placements_by_cell_org'::regclass::oid") == oid + + +async def test_recorded_invalid_index_and_checksum_drift_refuse(database, tmp_path) -> None: + conn, connect = database + await _migrate(connect) + await conn.execute("DROP INDEX loom_router.placements_by_cell_org") + with pytest.raises(ValueError, match="recorded online-index"): + await _migrate(connect) + await conn.execute( + "UPDATE loom_router.pg_migrations SET checksum = repeat('0', 64) " + "WHERE filename = '004_placements_by_cell_org.sql'" + ) + with pytest.raises(pg_migrate.MigrationDrift): + await _migrate(connect) + + +async def test_cancel_after_catalog_completion_reuses_the_valid_index( + database, tmp_path, monkeypatch +) -> None: + conn, connect = database + await _migrate(connect, _prefix(tmp_path)) + original = pg_online_index.build + + async def completed_then_cancel(*args, **kwargs): + await original(*args, **kwargs) + raise asyncio.CancelledError + + monkeypatch.setattr(pg_online_index, "build", completed_then_cancel) + with pytest.raises(asyncio.CancelledError): + await _migrate(connect) + oid = await conn.fetchval("SELECT 'loom_router.placements_by_cell_org'::regclass::oid") + monkeypatch.setattr(pg_online_index, "build", original) + assert await _migrate(connect) == _after_prefix() + assert await conn.fetchval("SELECT 'loom_router.placements_by_cell_org'::regclass::oid") == oid + + +async def test_keyset_plan_uses_the_ordered_cell_index(database) -> None: + conn, connect = database + await _migrate(connect) + await conn.execute( + "INSERT INTO loom_router.cells (cell, host, state, credential_sha256, deployment_id) " + "SELECT 'cell-' || n, 'cell-' || n || '.example.test', 'serving', " + "md5(n::text) || md5(n::text), 'dep-' || n FROM generate_series(1, 32) n" + ) + await conn.execute( + "INSERT INTO loom_router.placements(org, cell) " + "SELECT 'org-' || lpad(n::text, 5, '0'), 'cell-' || (n % 32 + 1) " + "FROM generate_series(1, 4096) n" + ) + await conn.execute("VACUUM ANALYZE loom_router.placements") + plan = await conn.fetch( + "EXPLAIN SELECT org FROM loom_router.placements " + "WHERE cell = 'cell-2' AND org > 'org-01000' ORDER BY org LIMIT 10" + ) + text = "\n".join(r[0] for r in plan) + assert "placements_by_cell_org" in text and "Sort" not in text + + +async def test_ordinary_files_still_rollback_with_the_ledger(database, tmp_path) -> None: + conn, connect = database + (tmp_path / "001_one.sql").write_text("CREATE TABLE loom_router.one (id int);") + (tmp_path / "002_two.sql").write_text("CREATE TABLE loom_router.two (id int); SELECT 1 / 0;") + import asyncpg + + with pytest.raises(asyncpg.DivisionByZeroError): + await _migrate(connect, tmp_path) + assert await conn.fetchval("SELECT to_regclass('loom_router.one') IS NOT NULL") + assert not await conn.fetchval("SELECT to_regclass('loom_router.two') IS NOT NULL") + assert await conn.fetchval("SELECT count(*) FROM loom_router.pg_migrations") == 1 + (tmp_path / "002_two.sql").write_text("CREATE TABLE loom_router.two (id int);") + assert await _migrate(connect, tmp_path) == ["002_two.sql"] + + +async def test_invalid_online_declaration_fails_before_the_ledger_exists( + database, tmp_path +) -> None: + conn, connect = database + (tmp_path / "001_bad.sql").write_text("-- loom:online-index bad\nSELECT 1;") + with pytest.raises(ValueError, match="online-index"): + await _migrate(connect, tmp_path) + assert not await conn.fetchval("SELECT to_regclass('loom_router.pg_migrations') IS NOT NULL") + + +async def test_cancelled_concurrent_drop_is_retryable(database, tmp_path) -> None: + conn, connect = database + await _migrate(connect, _prefix(tmp_path)) + index = pg_online_index.parse( + (ROUTER_MIGRATIONS_DIR / "004_placements_by_cell_org.sql").read_text() + ) + assert index is not None + await conn.execute(index.sql) + async with connect() as reader: + tx = reader.transaction() + await tx.start() + await reader.execute("SET LOCAL enable_seqscan = off") + plan = await reader.fetch( + "EXPLAIN SELECT org FROM loom_router.placements WHERE cell = 'a' ORDER BY org" + ) + assert "placements_by_cell_org" in "\n".join(r[0] for r in plan) + await reader.fetch("SELECT org FROM loom_router.placements WHERE cell = 'a' ORDER BY org") + + async def direct_drop(): + async with connect() as dropping: + try: + await dropping.execute( + "DROP INDEX CONCURRENTLY loom_router.placements_by_cell_org" + ) + finally: + await pg.close_owned(dropping, end=time.monotonic() + 1) + + dropping = asyncio.create_task(direct_drop()) + try: + query = ( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity " + "WHERE datname = current_database() AND query LIKE 'DROP INDEX CONCURRENTLY%' " + "AND wait_event IS NOT NULL)" + ) + await _until(conn, query) + # Cancel after catalog invalidation, not an earlier lock/I/O wait. + await _until( + conn, + "SELECT NOT indisvalid FROM pg_index " + "WHERE indexrelid='loom_router.placements_by_cell_org'::regclass", + ) + dropping.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(dropping, 2) + assert await pg_online_index.ready(conn, index) is False + dropping = asyncio.create_task(_migrate(connect)) + await _until(conn, query) + dropping.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(dropping, 2) + finally: + await tx.rollback() + if not dropping.done(): + dropping.cancel() + await asyncio.gather(dropping, return_exceptions=True) + assert await _migrate(connect) == _after_prefix() + + +async def test_try_lock_wait_has_no_retained_transaction_or_snapshot(database) -> None: + conn, connect = database + await conn.execute("SELECT pg_advisory_lock(8917423::bigint)") + migrating = asyncio.create_task(_migrate(connect)) + try: + await _until( + conn, + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity " + "WHERE datname = current_database() AND pid <> pg_backend_pid() " + "AND query = 'SELECT pg_try_advisory_lock($1)' " + "AND state = 'idle' AND backend_xmin IS NULL AND xact_start IS NULL)", + ) + assert not migrating.done() + await conn.execute("SELECT pg_advisory_unlock(8917423::bigint)") + assert await asyncio.wait_for(migrating, 5) == _journal() + finally: + await conn.execute("SELECT pg_advisory_unlock(8917423::bigint)") + if not migrating.done(): + migrating.cancel() + await asyncio.gather(migrating, return_exceptions=True) + + +async def test_online_validation_is_repeated_after_later_files(database, tmp_path) -> None: + _, connect = database + for source in ROUTER_MIGRATIONS_DIR.glob("*.sql"): + (tmp_path / source.name).write_bytes(source.read_bytes()) + ( + tmp_path / f"{max(int(name.split('_')[0]) for name in _journal()) + 1:03d}_damage.sql" + ).write_text("DROP INDEX loom_router.placements_by_cell_org;") + with pytest.raises(ValueError, match="recorded online-index"): + await _migrate(connect, tmp_path) diff --git a/tests/integration/test_pg_online_index_durability.py b/tests/integration/test_pg_online_index_durability.py new file mode 100644 index 0000000000..5acb835065 --- /dev/null +++ b/tests/integration/test_pg_online_index_durability.py @@ -0,0 +1,216 @@ +"""A cancelled online-index ledger commit survives a private standby promotion. + +Only its dedicated CI pair enables this destructive drill. Configured failures +are errors; the adopted integration server's replication settings never change. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import os +from collections.abc import AsyncIterator +from pathlib import Path +from urllib.parse import urlsplit + +import asyncpg +import pytest + +from server.core.paths import ROUTER_MIGRATIONS_DIR +from server.core.schema import pg_migrate +from tests.integration._pg_database import create_database + +PRIMARY = "loom-scale-index-drill-pg" +STANDBY = PRIMARY + "-standby" +APPLICATION = "online-index-durability" + + +async def _docker(*args: str) -> str: + process = await asyncio.create_subprocess_exec( + "docker", *args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + try: + async with asyncio.timeout(30): + stdout, stderr = await process.communicate() + except BaseException: + if process.returncode is None: + process.kill() + await asyncio.wait_for(process.wait(), 5) + raise + assert process.returncode == 0, stderr.decode() + return stdout.decode().strip() + + +async def _sql(container: str, database: str, sql: str) -> str: + return await _docker("exec", container, "psql", "-U", "loom", "-d", database, "-Atc", sql) + + +async def _waiting(observer, task) -> int: + async with asyncio.timeout(8): + for _ in range(800): + assert not task.done(), "migration acknowledged before standby flush" + pid = await observer.fetchval( + "SELECT pid FROM pg_stat_activity WHERE application_name = $1 " + "AND wait_event = 'SyncRep'", + APPLICATION, + ) + if pid is not None: + assert not task.done() + return pid + await asyncio.sleep(0.01) + pytest.fail("migration did not reach its synchronous COMMIT") + + +async def _ownership(observer) -> None: + identifier = str(await observer.fetchval("SELECT system_identifier FROM pg_control_system()")) + for container in (PRIMARY, STANDBY): + assert await _docker("inspect", "--format", "{{.Name}}", container) == "/" + container + assert ( + await _sql(container, "loom", "SELECT system_identifier FROM pg_control_system()") + == identifier + ) + assert await _sql(STANDBY, "loom", "SELECT pg_is_in_recovery()") == "t" + assert ( + await observer.fetchval( + "SELECT count(*) FROM pg_stat_replication WHERE application_name = 'loom_standby' " + "AND sync_state = 'sync' AND state = 'streaming'" + ) + == 1 + ) + + +@pytest.mark.asyncio +@pytest.mark.recovery +@pytest.mark.destructive +async def test_online_ledger_retry_survives_physical_standby_promotion( + tmp_path, monkeypatch +) -> None: + if not os.environ.get("LOOM_PG_INDEX_DRILL"): + pytest.skip("LOOM_PG_INDEX_DRILL not set: needs its disposable physical standby") + assert os.environ["LOOM_PG_INDEX_DRILL"] == "1" + dsn = os.environ["LOOM_PG_INDEX_DRILL_DSN"] + observer = await asyncpg.connect(dsn, timeout=5, command_timeout=5) + tasks = [] + paused = False + try: + await _ownership(observer) + _, target = await create_database(dsn) + + @contextlib.asynccontextmanager + async def connect() -> AsyncIterator[asyncpg.Connection]: + conn = await asyncpg.connect( + target, + timeout=5, + server_settings={"application_name": APPLICATION, "synchronous_commit": "on"}, + ) + try: + yield conn + finally: + conn.terminate() + + prefix = tmp_path / "old" + prefix.mkdir() + for source in sorted(ROUTER_MIGRATIONS_DIR.glob("*.sql"))[:3]: + (prefix / source.name).write_bytes(source.read_bytes()) + + async def migrate(files: Path = ROUTER_MIGRATIONS_DIR): + return await pg_migrate.migrate( + files, connect=connect, schema="loom_router", lock_key=8917423 + ) + + await migrate(prefix) + record = pg_migrate._record_online + + async def pause_before_ledger(*args, **kwargs): + nonlocal paused + await _docker("pause", STANDBY) + paused = True + await record(*args, **kwargs) + + monkeypatch.setattr(pg_migrate, "_record_online", pause_before_ledger) + first = asyncio.create_task(migrate()) + tasks.append(first) + pid = await _waiting(observer, first) + # The index is acknowledged; the pending ledger transaction is not visible yet. + async with connect() as conn: + assert ( + await conn.fetchval( + "SELECT indisvalid AND indisready FROM pg_index " + "WHERE indexrelid = 'loom_router.placements_by_cell_org'::regclass" + ) + is True + ) + first.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(first, 3) + async with asyncio.timeout(5): + for _ in range(500): + if not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid = $1)", pid + ): + break + await asyncio.sleep(0.01) + else: + pytest.fail("cancelled migration retained backend locks") + # Cancellation releases the commit wait before other sessions can see the ledger. + async with connect() as conn: + assert ( + await conn.fetchval( + "SELECT count(*) FROM loom_router.pg_migrations " + "WHERE filename = '004_placements_by_cell_org.sql'" + ) + == 1 + ) + monkeypatch.setattr(pg_migrate, "_record_online", record) + retry = asyncio.create_task(migrate()) + tasks.append(retry) + await _waiting(observer, retry) + await _docker("unpause", STANDBY) + paused = False + assert await asyncio.wait_for(retry, 8) == sorted( + path.name + for path in ROUTER_MIGRATIONS_DIR.glob("*.sql") + if path.name > "004_placements_by_cell_org.sql" + ) + observer.terminate() + await _docker("kill", "--signal", "KILL", PRIMARY) + await _docker( + "exec", + "--user", + "postgres", + STANDBY, + "bash", + "-c", + 'pg_ctl -D "$PGDATA" promote -w -t 10', + ) + assert await _sql(STANDBY, "loom", "SELECT pg_is_in_recovery()") == "f" + database = urlsplit(target).path.removeprefix("/") + assert ( + await _sql( + STANDBY, + database, + "SELECT count(*) FROM loom_router.pg_migrations " + "WHERE filename = '004_placements_by_cell_org.sql' AND checksum = '" + + pg_migrate.checksum(ROUTER_MIGRATIONS_DIR / "004_placements_by_cell_org.sql") + + "'", + ) + == "1" + ) + assert ( + await _sql( + STANDBY, + database, + "SELECT indisvalid AND indisready FROM pg_index " + "WHERE indexrelid = 'loom_router.placements_by_cell_org'::regclass", + ) + == "t" + ) + finally: + if paused: + await _docker("unpause", STANDBY) + for task in tasks: + if not task.done(): + task.cancel() + async with asyncio.timeout(5): + await asyncio.gather(*tasks, return_exceptions=True) + observer.terminate() diff --git a/tests/integration/test_pg_owned_cleanup.py b/tests/integration/test_pg_owned_cleanup.py new file mode 100644 index 0000000000..62a339774a --- /dev/null +++ b/tests/integration/test_pg_owned_cleanup.py @@ -0,0 +1,231 @@ +"""Cancellation of owned Postgres work releases its actual backend locks.""" + +from __future__ import annotations + +import asyncio +import contextlib +import os +import time +from collections.abc import AsyncIterator +from pathlib import Path +from typing import Any + +import pytest + +from server.core import pg +from server.core.schema import pg_migrate +from tests.integration._pg_database import create_database, drop_database + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +async def database() -> AsyncIterator[tuple[Any, Any]]: + import asyncpg + + admin = os.environ.get("LOOM_PG_DSN") or pytest.skip("LOOM_PG_DSN not set") + name, dsn = await create_database(admin) + conn = await asyncpg.connect(dsn, server_settings={"statement_timeout": "10000"}) + observer = await asyncpg.connect(dsn) + try: + await conn.execute("CREATE TABLE probe(id int PRIMARY KEY); INSERT INTO probe VALUES(1)") + yield conn, observer + finally: + await conn.close(timeout=1) + await observer.close(timeout=1) + await drop_database(admin, name) + + +async def sleeping(observer: Any, pid: int) -> None: + async with asyncio.timeout(2): + for _ in range(400): + if await observer.fetchval( + "SELECT wait_event='PgSleep' FROM pg_stat_activity WHERE pid=$1", pid + ): + break + await asyncio.sleep(0.005) + else: + pytest.fail("backend did not reach pg_sleep") + assert await observer.fetchval("SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1)", pid) + + +async def released(observer: Any, pid: int) -> None: + assert not await observer.fetchval("SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1)", pid) + assert not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)", pid + ) + + +async def test_owned_migrator_cancellation_releases_server_locks(database, tmp_path: Path) -> None: + conn, observer = database + (tmp_path / "001_sleep.sql").write_text("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10);") + + @contextlib.asynccontextmanager + async def connect() -> AsyncIterator[Any]: + yield conn + + task = asyncio.create_task(pg_migrate.migrate(tmp_path, connect=connect, schema="test")) + await sleeping(observer, conn.get_server_pid()) + pid = conn.get_server_pid() + started = time.monotonic() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, 1.5) + assert time.monotonic() - started < 1.5 + await released(observer, pid) + + +@pytest.mark.parametrize("signals", [1, 2]) +async def test_owned_migrator_keeps_real_cancel_request_alive( + database, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, signals: int +) -> None: + from asyncpg import connect_utils + + conn, observer = database + pid = conn.get_server_pid() + cancelling, allow_cancel = asyncio.Event(), asyncio.Event() + real_cancel = connect_utils._cancel + monkeypatch.setattr(pg, "_RELEASE_TIMEOUT_S", 0.5) + + async def held_cancel(**kwargs: Any) -> None: + if kwargs["backend_pid"] == pid: + cancelling.set() + await allow_cancel.wait() + await real_cancel(**kwargs) + + monkeypatch.setattr(connect_utils, "_cancel", held_cancel) + (tmp_path / "001_sleep.sql").write_text("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10);") + + @contextlib.asynccontextmanager + async def connect() -> AsyncIterator[Any]: + yield conn + + task = asyncio.create_task(pg_migrate.migrate(tmp_path, connect=connect, schema="test")) + try: + await sleeping(observer, pid) + task.cancel("first signal") + await asyncio.wait_for(cancelling.wait(), 2) + if signals == 2: + task.cancel("second signal") + await asyncio.sleep(0.03) + assert not task.done(), ( + await observer.fetchval("SELECT wait_event FROM pg_stat_activity WHERE pid=$1", pid), + await observer.fetchval("SELECT count(*) FROM pg_locks WHERE pid=$1", pid), + ) + allow_cancel.set() + with pytest.raises(asyncio.CancelledError, match="first signal"): + await asyncio.wait_for(task, 0.5) + await released(observer, pid) + finally: + allow_cancel.set() + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + await observer.execute("SELECT pg_cancel_backend($1)", pid) + async with asyncio.timeout(2): + for _ in range(200): + if not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)", pid + ): + break + await asyncio.sleep(0.01) + else: + pytest.fail("owned backend did not end after teardown cancellation") + + +@pytest.mark.parametrize("transaction_context", [False, True]) +async def test_public_owned_close_releases_live_query_before_task_cancel( + database, transaction_context: bool +) -> None: + conn, observer = database + pid = conn.get_server_pid() + + async def work() -> None: + if transaction_context: + async with conn.transaction(): + await conn.execute("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10)") + else: + await conn.execute("BEGIN; SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10)") + + task = asyncio.create_task(work()) + try: + await sleeping(observer, pid) + await pg.close_owned(conn, end=time.monotonic() + 0.5) + await released(observer, pid) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_borrowed_migrator_leaves_failure_cleanup_to_owner(database, tmp_path: Path) -> None: + conn, observer = database + (tmp_path / "001_sleep.sql").write_text("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10);") + task = asyncio.create_task(pg_migrate.migrate_borrowed(conn, tmp_path, schema="test")) + pid = conn.get_server_pid() + try: + await sleeping(observer, pid) + await pg.close_owned(conn, end=time.monotonic() + 0.5) + await released(observer, pid) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_borrowed_migrator_normal_completion_retains_owners_lock(database, tmp_path) -> None: + conn, observer = database + await conn.execute("SELECT pg_advisory_lock(712398)") + (tmp_path / "001_write.sql").write_text("INSERT INTO probe VALUES(2)") + assert await pg_migrate.migrate_borrowed(conn, tmp_path, schema="test") == ["001_write.sql"] + assert not conn.is_closed() + assert await observer.fetchval("SELECT count(*) FROM probe") == 2 + assert await conn.fetchval("SELECT pg_advisory_unlock(712398)") + assert ( + await conn.fetchval( + "SELECT count(*) FROM pg_locks WHERE pid=pg_backend_pid() AND locktype='advisory'" + ) + == 0 + ) + + +async def test_owned_close_interrupts_installed_transaction_rollback(database, monkeypatch) -> None: + import asyncpg + + conn, observer = database + pid = conn.get_server_pid() + execute = asyncpg.Connection.execute + + async def stalled_rollback(self, query, *args, **kwargs): + if self is conn and query.startswith("ROLLBACK"): + await execute(self, "SELECT pg_sleep(10)") + return await execute(self, query, *args, **kwargs) + + monkeypatch.setattr(asyncpg.Connection, "execute", stalled_rollback) + + async def work() -> None: + async with conn.transaction(): + await conn.execute("SELECT * FROM probe FOR UPDATE") + raise ValueError("statement failure before rollback") + + task = asyncio.create_task(work()) + try: + await sleeping(observer, pid) + await pg.close_owned(conn, end=time.monotonic() + 0.5) + await released(observer, pid) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_borrowed_failure_keeps_the_session_for_its_owner(database, tmp_path) -> None: + import asyncpg + + conn, _ = database + (tmp_path / "001_failure.sql").write_text("SELECT * FROM missing_table_for_failure") + with pytest.raises(asyncpg.UndefinedTableError): + await pg_migrate.migrate_borrowed(conn, tmp_path, schema="test") + assert not conn.is_closed() + assert await conn.fetchval("SELECT pg_advisory_unlock($1)", pg_migrate._LOCK_KEY) + assert await conn.fetchval("SELECT count(*) FROM test.pg_migrations") == 0 diff --git a/tests/integration/test_provider_coordination_live_ch.py b/tests/integration/test_provider_coordination_live_ch.py index 9a6c9db1e2..8e74b88368 100644 --- a/tests/integration/test_provider_coordination_live_ch.py +++ b/tests/integration/test_provider_coordination_live_ch.py @@ -1,6 +1,8 @@ """Real participant rows and Keeper recovery on an opt-in disposable database.""" +import contextlib import os +import sys from uuid import uuid4 import pytest @@ -19,11 +21,14 @@ def _live() -> None: @pytest.mark.asyncio -async def test_participant_ack_is_full_row_and_expiry_is_on_the_server_clock(): +async def test_participant_ack_is_full_row_and_expiry_is_on_the_server_clock(monkeypatch): _live() ch = _default_client() await ch.connect() identity, transition = uuid4(), uuid4() + monkeypatch.setattr( + "server.insights.store.provider_participants.process_identity", lambda: identity + ) try: await record_participant(identity, transition, "running", ch=ch) await record_participant(identity, transition, "drained", ch=ch) @@ -39,10 +44,11 @@ async def test_participant_ack_is_full_row_and_expiry_is_on_the_server_clock(): rows = [r for r in await participants(ch=ch) if str(r["participant_id"]) == str(identity)] assert rows[0]["expired"] assert await running_queries(identity, ch=ch) == 0 - await record_participant( - identity, transition, "retired", after=int(rows[0]["version"]), ch=ch - ) - assert all(str(row["participant_id"]) != str(identity) for row in await participants(ch=ch)) + with pytest.raises(ValueError, match="state"): + await record_participant( + identity, transition, "retired", after=int(rows[0]["version"]), ch=ch + ) + assert any(str(row["participant_id"]) == str(identity) for row in await participants(ch=ch)) finally: try: await ch.execute( @@ -54,6 +60,78 @@ async def test_participant_ack_is_full_row_and_expiry_is_on_the_server_clock(): await ch.disconnect() +@pytest.mark.asyncio +async def test_participant_census_excludes_certified_history_before_returning_rows(): + _live() + from server.core.ch import query_tag, sequential_reads + from server.core.provider_process import certified, has_certificate + from server.insights.store import provider_participants as store + + legacy = "test|boot||pidns|7|99||" + proof = certified(legacy) + invalid_pid = proof.replace("|7|99|", "|0|99|") + oversized_pid = proof.replace("|7|99|", "|" + "5" * 5000 + "|99|") + assert has_certificate(proof) + assert not has_certificate(invalid_pid) + assert not has_certificate(oversized_pid) + blockers = { + uuid4(): (state, identity) + for state, identity in ( + ("running", proof), + ("drained", proof), + ("retired", ""), + ("retired", legacy), + ("retired", legacy + "|unknown"), + ("retired", proof + "|unknown"), + ("retired", invalid_pid), + ("retired", oversized_pid), + ) + } + history = {uuid4(): ("retired", proof) for _ in range(256)} + created = {**history, **blockers} + expected = {str(identity): value for identity, value in blockers.items()} + ids = [str(identity) for identity in created] + ch = _default_client() + with query_tag("test.provider_participant_census"): + await ch.connect() + try: + await ch.insert( + store.TABLE, + [ + {"participant_id": identity, "state": state, "version": 1, "host_proof": value} + for identity, (state, value) in created.items() + ], + ) + with sequential_reads(): + raw = await ch.query_all( + "SELECT participant_id, state, host_proof FROM (" + + store.PARTICIPANTS_SQL + + ") WHERE participant_id IN {ids:Array(UUID)}", + {"ids": ids}, + ) + assert { + str(row["participant_id"]): (row["state"], row["host_proof"]) for row in raw + } == expected, "ClickHouse must exclude certified history before returning census rows" + kept = await store.participants(ch=ch) + assert { + str(row["participant_id"]): (row["state"], row["host_proof"]) + for row in kept + if str(row["participant_id"]) in ids + } == expected + finally: + with ( + contextlib.suppress(BaseException) if sys.exception() else contextlib.nullcontext() + ): + try: + await ch.execute( + f"ALTER TABLE {store.TABLE} DELETE WHERE participant_id IN {{ids:Array(UUID)}} " + "SETTINGS mutations_sync=2", + {"ids": ids}, + ) + finally: + await ch.disconnect() + + @pytest.mark.asyncio async def test_keeper_never_steals_from_a_beating_process_even_with_expired_timestamp(): _live() diff --git a/tests/integration/test_router_directory_pg.py b/tests/integration/test_router_directory_pg.py index caf092e9ec..41f8d343c6 100644 --- a/tests/integration/test_router_directory_pg.py +++ b/tests/integration/test_router_directory_pg.py @@ -7,7 +7,6 @@ from __future__ import annotations import asyncio -import contextlib import hashlib import json import os @@ -22,12 +21,31 @@ from server.settings import settings +async def _observed(name: str) -> directory.Control: + try: + found = await directory.attach_read(name) + except directory.NoSuchCell: + return directory.Control({}, uuid.UUID(int=0)) + values = { + key: getattr(found.row, key) if hasattr(found.row, key) else getattr(found, key) + for key in directory.CONTROL_FIELDS + } + return directory.Control(values, found.row.mutation_generation) + + @pytest.fixture -def dsn(monkeypatch: pytest.MonkeyPatch) -> str: - value = os.environ.get("LOOM_ROUTER_PG_DSN") or pytest.skip("LOOM_ROUTER_PG_DSN not set") +async def dsn(monkeypatch: pytest.MonkeyPatch): + from tests.integration._pg_database import create_database, drop_database + + admin_dsn = os.environ.get("LOOM_ROUTER_PG_DSN") or pytest.skip("LOOM_ROUTER_PG_DSN not set") + name, value = await create_database(admin_dsn) monkeypatch.setattr(settings, "router_pg_dsn", value) monkeypatch.setattr(directory, "_pool", None) - return value + try: + yield value + finally: + await directory.close() + await drop_database(admin_dsn, name) @pytest.fixture @@ -228,52 +246,72 @@ async def row(cell: str) -> dict[str, Any]: # The operator's changes: a limit, and a credential rotation with its refusals. # Each writes the fields it names and updated_at, and leaves every other column. before = await row(b) - limited = await directory.set_limit(b, closed=True, placements_max=7) + limited = await directory.set_limit( + b, closed=True, placements_max=7, expected=await _observed(b) + ) assert (limited.closed, limited.placements_max) == (True, 7) after = await row(b) assert after["updated_at"] > before["updated_at"] changed = {"closed": True, "placements_max": 7, "updated_at": after["updated_at"]} assert after == {**before, **changed} - cleared = await directory.set_limit(b, placements_max=None) + cleared = await directory.set_limit(b, placements_max=None, expected=await _observed(b)) assert (cleared.closed, cleared.placements_max) == (True, None) # closed is kept assert await row(b) == {**after, "placements_max": None, "updated_at": ANY} with pytest.raises(ValueError): - await directory.set_limit(b) # nothing to set: no statement + await directory.set_limit( + b, expected=await _observed(b) + ) # nothing to set: no statement with pytest.raises(ValueError): - await directory.rotate_credential(b, "finsh") # no such step: nothing written + await directory.rotate_credential( + b, "finsh", expected=await _observed(b) + ) # no such step: nothing written with pytest.raises(ValueError): - await directory.rotate_credential(b, "next") # no hash to record + await directory.rotate_credential( + b, "next", expected=await _observed(b) + ) # no hash to record with pytest.raises(ValueError): - await directory.rotate_credential(b, "finish", "c" * 64) # a hash finish ignores + await directory.rotate_credential( + b, "finish", "c" * 64, expected=await _observed(b) + ) # a hash finish ignores with pytest.raises(directory.NoSuchCell): - await directory.set_limit(f"nope-{b}", closed=True) + await directory.set_limit( + f"nope-{b}", closed=True, expected=await _observed(f"nope-{b}") + ) sha_b = hashlib.sha256(b.encode()).hexdigest() with pytest.raises(directory.OperatorRefused, match="records that credential hash"): - await directory.rotate_credential(a, "next", sha_b) # b's current hash + await directory.rotate_credential( + a, "next", sha_b, expected=await _observed(a) + ) # b's current hash with pytest.raises(directory.OperatorRefused, match="no next credential"): - await directory.rotate_credential(a, "finish") + await directory.rotate_credential(a, "finish", expected=await _observed(a)) new = hashlib.sha256(f"new-{a}".encode()).hexdigest() before = await row(a) - assert (await directory.rotate_credential(a, "next", new)).credential_next_sha256 == new + assert ( + await directory.rotate_credential(a, "next", new, expected=await _observed(a)) + ).credential_next_sha256 == new assert await row(a) == {**before, "credential_next_sha256": new, "updated_at": ANY} assert (await directory.credentials())[a] == (hashlib.sha256(a.encode()).hexdigest(), new) with pytest.raises(directory.OperatorRefused, match="already cell .* next credential"): - await directory.rotate_credential(a, "next", new) # this change, retried + await directory.rotate_credential( + a, "next", new, expected=await _observed(a) + ) # this change, retried with pytest.raises(directory.OperatorRefused, match="another next credential pending"): - await directory.rotate_credential(a, "next", "e" * 64) + await directory.rotate_credential(a, "next", "e" * 64, expected=await _observed(a)) with pytest.raises(directory.OperatorRefused, match="records that credential hash"): - await directory.rotate_credential(b, "next", new) # a's next hash + await directory.rotate_credential( + b, "next", new, expected=await _observed(b) + ) # a's next hash # Two cells asking for one next hash at once: the directory lock lets one in. raced = hashlib.sha256(f"raced-{a}".encode()).hexdigest() both = await asyncio.gather( - directory.rotate_credential(b, "next", raced), - directory.rotate_credential(q, "next", raced), + directory.rotate_credential(b, "next", raced, expected=await _observed(b)), + directory.rotate_credential(q, "next", raced, expected=await _observed(q)), return_exceptions=True, ) assert sum(isinstance(r, directory.Cell) for r in both) == 1 assert sum(isinstance(r, directory.OperatorRefused) for r in both) == 1 pending = await row(a) - finished = await directory.rotate_credential(a, "finish") + finished = await directory.rotate_credential(a, "finish", expected=await _observed(a)) assert (finished.credential_sha256, finished.credential_next_sha256) == (new, None) assert await row(a) == { **pending, @@ -282,11 +320,13 @@ async def row(cell: str) -> dict[str, Any]: "updated_at": ANY, } winner = b if isinstance(both[0], directory.Cell) else q - reverted = await directory.rotate_credential(winner, "revert") + reverted = await directory.rotate_credential( + winner, "revert", expected=await _observed(winner) + ) assert reverted.credential_next_sha256 is None assert (await directory.credentials())[winner] == (reverted.credential_sha256,) with pytest.raises(directory.OperatorRefused, match="no next credential"): - await directory.rotate_credential(winner, "revert") + await directory.rotate_credential(winner, "revert", expected=await _observed(winner)) def steps(cell: str) -> Any: return conn.fetch( @@ -587,16 +627,22 @@ async def test_the_first_cell_attaches_through_the_operator_calls(own_dsn: str) try: # Nothing before begin; begin writes its start once and the fingerprints. with pytest.raises(directory.OperatorRefused, match="has not begun"): - await directory.attach_enforced(name, dep) + await directory.attach_enforced(name, dep, expected=await _observed(name)) with pytest.raises(directory.OperatorRefused, match="has not begun"): - await directory.attach_place(name, dep, ["a"]) + await directory.attach_place(name, dep, ["a"], expected=await _observed(name)) with pytest.raises(directory.OperatorRefused, match="deployment id"): - await directory.attach_begin(name, directory.Begin("dep-x", _fp(name), None, "k", {})) - found = await directory.attach_begin(name, begin) + await directory.attach_begin( + name, + directory.Begin("dep-x", _fp(name), None, "k", {}), + expected=await _observed(name), + ) + found = await directory.attach_begin(name, begin, expected=await _observed(name)) started = found.attach_started_at assert started is not None and found.row.current_fp == _fp(name) again = await directory.attach_begin( - name, directory.Begin(dep, _fp(name), None, "keys-2", {}) + name, + directory.Begin(dep, _fp(name), None, "keys-2", {}), + expected=await _observed(name), ) assert again.attach_started_at == started stored = await conn.fetchrow( @@ -609,35 +655,45 @@ async def test_the_first_cell_attaches_through_the_operator_calls(own_dsn: str) ) with pytest.raises(directory.OperatorRefused, match="loom cell update"): await directory.attach_begin( - name, directory.Begin(dep, _fp(name, oidc_issuer="x"), None, "k", {}) + name, + directory.Begin(dep, _fp(name, oidc_issuer="x"), None, "k", {}), + expected=await _observed(name), ) # A batch waits for the directory mode `enforced`. with pytest.raises(directory.OperatorRefused, match="attach/enforced first"): - await directory.attach_place(name, dep, ["a"]) - enforced = await directory.attach_enforced(name, dep) + await directory.attach_place(name, dep, ["a"], expected=await _observed(name)) + enforced = await directory.attach_enforced(name, dep, expected=await _observed(name)) assert enforced.row.mode == "enforced" - await directory.attach_enforced(name, dep) # idempotent + await directory.attach_enforced(name, dep, expected=await _observed(name)) # idempotent # A closed cell with a quota of 0 still takes its own orgs; a rerun places none # twice. await conn.execute( "UPDATE loom_router.cells SET closed = true, placements_max = 0 WHERE cell = $1", name ) - assert await directory.attach_place(name, dep, ["b", "a", "a"]) == (2, 2) - assert await directory.attach_place(name, dep, ["a", "c"]) == (1, 3) + assert await directory.attach_place( + name, dep, ["b", "a", "a"], expected=await _observed(name) + ) == (2, 2) + assert await directory.attach_place( + name, dep, ["a", "c"], expected=await _observed(name) + ) == (1, 3) placed = await conn.fetch( "SELECT org FROM loom_router.placements WHERE cell = $1 ORDER BY org", name ) assert [r["org"] for r in placed] == ["a", "b", "c"] with pytest.raises(directory.OrgTooLong): - await directory.attach_place(name, dep, ["d", "x" * 2049]) - served = await directory.attach_serving(name, dep) + await directory.attach_place( + name, dep, ["d", "x" * 2049], expected=await _observed(name) + ) + served = await directory.attach_serving(name, dep, expected=await _observed(name)) assert served.row.state == "serving" assert await conn.fetchval( "SELECT issuers FROM loom_router.cells WHERE cell = $1", name ) == [f"https://{name}.example.test"] - assert (await directory.attach_serving(name, dep)).row.state == "serving" # done + assert ( + await directory.attach_serving(name, dep, expected=await _observed(name)) + ).row.state == "serving" # done with pytest.raises(directory.OperatorRefused, match="serving already"): - await directory.attach_enforced(name, dep) + await directory.attach_enforced(name, dep, expected=await _observed(name)) read = await directory.attach_read(name, audited=True) assert (read.row.state, read.other) == ("serving", None) audit = await conn.fetch( @@ -686,8 +742,10 @@ async def _attaching_first(own_dsn: str) -> tuple[str, str]: name = f"lr-{uuid.uuid4().hex[:8]}" dep = f"dep-{name}" await directory.register(_first(name)) - await directory.attach_begin(name, directory.Begin(dep, _fp(name), None, "k", {})) - await directory.attach_enforced(name, dep) + await directory.attach_begin( + name, directory.Begin(dep, _fp(name), None, "k", {}), expected=await _observed(name) + ) + await directory.attach_enforced(name, dep, expected=await _observed(name)) return name, dep @@ -740,7 +798,9 @@ async def _held(*a: Any) -> None: async with asyncio.timeout(30): live = asyncio.ensure_future(directory.place("acme", name, lambda _row: "")) await inserted.wait() - batch = asyncio.ensure_future(directory.attach_place(name, dep, ["acme", "beta"])) + batch = asyncio.ensure_future( + directory.attach_place(name, dep, ["acme", "beta"], expected=await _observed(name)) + ) await _waits_on_the_row_lock(conn, "FOR UPDATE") release.set() assert await live == (name, dep) @@ -767,12 +827,14 @@ async def test_a_live_placement_racing_a_batch_of_the_same_org_answers_it_placed written, release = asyncio.Event(), asyncio.Event() real = directory._attach_change - @contextlib.asynccontextmanager - async def _held(cell_name: str) -> Any: - async with real(cell_name) as locked: - yield locked + async def _held(cell_name: str, expected: directory.Control, change) -> Any: + async def hold(conn, found): + result = await change(conn, found) written.set() await release.wait() # the batch's writes are done, its commit held + return result + + return await real(cell_name, expected, hold) monkeypatch.setattr(directory, "_attach_change", _held) conn = await asyncpg.connect(own_dsn) @@ -780,7 +842,9 @@ async def _held(cell_name: str) -> Any: # A full quota: a live placement judged on it would be refused. await conn.execute("UPDATE loom_router.cells SET placements_max = 0 WHERE cell = $1", name) async with asyncio.timeout(30): - batch = asyncio.ensure_future(directory.attach_place(name, dep, ["acme"])) + batch = asyncio.ensure_future( + directory.attach_place(name, dep, ["acme"], expected=await _observed(name)) + ) await written.wait() live = asyncio.ensure_future(directory.place("acme", name, lambda _row: "")) await _waits_on_the_row_lock(conn, "FOR UPDATE") @@ -821,23 +885,29 @@ async def test_an_attaching_cell_with_no_placement_is_removed_with_its_hosts( later, ) with pytest.raises(directory.OperatorRefused, match=f"loom cell remove --cell {later}"): - await directory.attach_begin(first, directory.Begin(dep, _fp(first), None, "k", {})) - assert (await directory.remove(later)).cell == later + await directory.attach_begin( + first, + directory.Begin(dep, _fp(first), None, "k", {}), + expected=await _observed(first), + ) + assert (await directory.remove(later, expected=await _observed(later))).cell == later assert ( await conn.fetchval("SELECT count(*) FROM loom_router.hosts WHERE cell = $1", later) == 0 ) with pytest.raises(directory.NoSuchCell): - await directory.remove(later) + await directory.remove(later, expected=await _observed(later)) # The first cell backs out at any directory mode until its first placement. - await directory.attach_begin(first, directory.Begin(dep, _fp(first), None, "k", {})) - await directory.attach_enforced(first, dep) - await directory.attach_place(first, dep, ["only"]) + await directory.attach_begin( + first, directory.Begin(dep, _fp(first), None, "k", {}), expected=await _observed(first) + ) + await directory.attach_enforced(first, dep, expected=await _observed(first)) + await directory.attach_place(first, dep, ["only"], expected=await _observed(first)) with pytest.raises(directory.OperatorRefused, match="never back"): - await directory.remove(first) + await directory.remove(first, expected=await _observed(first)) await conn.execute("UPDATE loom_router.cells SET state = 'serving' WHERE cell = $1", first) with pytest.raises(directory.OperatorRefused, match="stays attached"): - await directory.remove(first) + await directory.remove(first, expected=await _observed(first)) removed = await conn.fetch( "SELECT detail FROM loom_router.audit WHERE action = 'remove' ORDER BY id" ) @@ -860,10 +930,14 @@ async def test_an_attaching_cell_with_no_placement_is_removed_with_its_hosts( await conn.execute("DELETE FROM loom_router.cells") await directory.register(_first(other)) await directory.attach_begin( - other, directory.Begin(f"dep-{other}", _fp(other), None, "k", {}) + other, + directory.Begin(f"dep-{other}", _fp(other), None, "k", {}), + expected=await _observed(other), ) - await directory.attach_enforced(other, f"dep-{other}") # a step-3 lock that never landed - assert (await directory.remove(other)).is_first + await directory.attach_enforced( + other, f"dep-{other}", expected=await _observed(other) + ) # a step-3 lock that never landed + assert (await directory.remove(other, expected=await _observed(other))).is_first assert await conn.fetchval("SELECT count(*) FROM loom_router.cells") == 0 assert await conn.fetchval("SELECT count(*) FROM loom_router.hosts") == 0 finally: diff --git a/tests/unit/cell_control_owner.py b/tests/unit/cell_control_owner.py new file mode 100644 index 0000000000..3a2d009821 --- /dev/null +++ b/tests/unit/cell_control_owner.py @@ -0,0 +1,11 @@ +"""Provide an explicit owner for a test's already-open raw session.""" + +from typing import Any + +from server.cli.cell_control import Owner + + +def owner(conn: Any) -> Owner: + owned = Owner() + owned.conn = conn + return owned diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 7e523ec4ac..cfac2e4fdc 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -95,6 +95,7 @@ async def checked(_ch): return None monkeypatch.setattr(provider_command, "model_command", command) + monkeypatch.setattr(provider_command, "participant_command", command) monkeypatch.setattr(pg_profile, "check_configured", checked) @@ -1167,3 +1168,11 @@ async def _none(org: str, namespaces: list[str], **_: object) -> dict[str, str]: return {} monkeypatch.setattr("server.knowledge.app.mirrors.mirrors_of", _none) + + +@pytest.fixture(autouse=True) +def isolated_retirement_gate(monkeypatch): + """Each test models a fresh process; permanent shutdown cannot leak between them.""" + from server.core import retirement + + monkeypatch.setattr(retirement, "_gate", retirement.RetirementGate()) diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json index c9b48a70c3..40bf10506e 100644 --- a/tests/unit/snapshots/cli_help.json +++ b/tests/unit/snapshots/cli_help.json @@ -2,7 +2,7 @@ "loom": [ "usage: loom [-h] [--version] {code,decisions,knowledge,export-okf,import-okf,okf-lint,up,down,ch-setup,participant,deployment,settings,secrets,signing-key,provider-lock,export-analytics,sql-access,reembed,reextract,deploy-delta,entity-export,audit-namespace,mentor,serve,migrate,reconcile-status,period-digest,demote-activity-narration,recalibrate,import-clickgap,import-recall,import-brain,claim-release,mint-token,revoke-token,sso,bot,doctor,report,drops,namespaces-gc,erasure-re-verify,reconcile,router,cell,tracker-probe,tracker-contract-check,tracker-dupcheck-fit,tracker-embed-reset,tracker-feed,namespaces-adopt,sensitivity-sweep,integrity-repair,rechunk,export-actions,lexical-stats-rebuild,version,install,demo,init,ingest-transcript,memory-search,set-host,login,logout,whoami,call,token} ...", "Loom — vendor-neutral agent memory plane on ClickHouse.", - "positional arguments: {code,decisions,knowledge,export-okf,import-okf,okf-lint,up,down,ch-setup,participant,deployment,settings,secrets,signing-key,provider-lock,export-analytics,sql-access,reembed,reextract,deploy-delta,entity-export,audit-namespace,mentor,serve,migrate,reconcile-status,period-digest,demote-activity-narration,recalibrate,import-clickgap,import-recall,import-brain,claim-release,mint-token,revoke-token,sso,bot,doctor,report,drops,namespaces-gc,erasure-re-verify,reconcile,router,cell,tracker-probe,tracker-contract-check,tracker-dupcheck-fit,tracker-embed-reset,tracker-feed,namespaces-adopt,sensitivity-sweep,integrity-repair,rechunk,export-actions,lexical-stats-rebuild,version,install,demo,init,ingest-transcript,memory-search,set-host,login,logout,whoami,call,token} code Code intelligence plane operations. decisions Decision records mined from documents. knowledge Knowledge documents and their sources. export-okf Render a namespace as an OKF v0.2 bundle (Markdown + frontmatter) — the git-reviewable projection of Loom memory. import-okf Write the concepts of an OKF v0.2 bundle into a namespace via memory.set. okf-lint Lint an OKF bundle directory offline — the compile gate a mirror repository's CI runs. Exit 2 critical, 1 warnings, 0 clean. up Boot the local stack (Loom, its document extractor and ClickHouse) and print how to connect: creates .env and the ClickHouse config from their templates, runs docker compose up --build, waits for /v1/health, then prints the dashboard URL and the first memory.set (curl + Python SDK) and `claude mcp add` snippets — minting a writer token first when auth is on. Re-run any time. down Stop the local stack while preserving data; --delete-data performs a confirmed reset. ch-setup Prepare a self-managed ClickHouse or a ClickHouse Cloud service for Loom: create the loom database and the read-only reader user, or print the SQL for an admin. participant The Loom processes a provider change waits for. deployment The configuration every replica checks itself against. settings Every setting's value in force, and overrides of the live ones. secrets Credentials and signing keys stored sealed in ClickHouse. signing-key The token signing keys beside LOOM_JWT_SECRET, kept in ClickHouse. provider-lock The provider configuration lock. export-analytics Write one UTC day of usage and a namespace snapshot to S3 as Parquet, names hashed under LOOM_ANALYTICS_EXPORT_NAME_KEY. Never replaces an object, so a repeated run is safe. sql-access Make, rescope or drop a ClickHouse user that reads a tenant's memory, conversations and request ledger through the sql_v1_* views. reembed Re-embed vectors whose embedding_model != the configured embedder (run after changing LOOM_EMBEDDING_MODEL — Loom does not auto-re-embed). Covers memory, entity, and conversation vectors. Coded vector tables keep their width: change a model of another width in Settings → Models & keys, which rebuilds them. reextract Replay a namespace's raw evidence (memory.set_from_messages turns) through a running server's write path into ANOTHER namespace — the current extractor over the stored transcripts; the source is never written. deploy-delta What a colour swap changed: the deployment's ledger rows (volume, errors, p50/p95, upstream spend per operation) and its ClickHouse statement shapes (runs, bytes, seconds) for a window on each side of the swap, with flags for what moved past a coarse threshold. entity-export Export a namespace's entity graph (co-occurrence edges + triples) to GraphML or JSON-LD for an external graph or RDF tool. Read-only; every edge carries its provenance and a note that a path is declared, not proven. audit-namespace How a namespace is being used (namespace.audit): exact-key reads vs searches, hash keys, double-encoded and oversized values, hand-rolled index rows, leases stored as memories, version churn. Read-only; prints a table (code, severity, kind, count, share, first sample) or --json. Exit 1 on a critical finding. mentor The store's advice to one agent (mentor.brief): what it called, the planes its namespace offers, the hints with their evidence, and how often past hints were followed. Read-only; prints a table or --json. Exit 1 when a warning-level hint fires. serve Boot the control plane (uvicorn). migrate Apply schema migrations against the configured CH. reconcile-status Re-emit memory_embeddings rows whose status diverged from loom.memory (superseded / conflicted / duplicate marks the pool never saw), compared and re-emitted per identity row (org, namespace, agent, user_id, session_id, key). Dry run by default (counts by status, the rows it would write); --apply writes. period-digest Write period digests — one derived semantic memory over K consecutive session digests of each identity in one namespace. Additive and idempotent. Dry run by default (plans only); --apply writes. demote-activity-narration Find a namespace's extracted facts that narrate tool activity ('User read lines 1-1000 of X', 'Claude ran a command on …') with no durable outcome and demote (confidence x0.25) or label them through memory.apply. Dry run by default; keys an agent read by name in the last --days are left alone. Never deletes. recalibrate Re-probe the embedder + run per-NS calibration immediately (otherwise this runs weekly via the recalibration worker). import-clickgap Import a ClickGap (Detector→Validator→Reviewer) SQLite export into Loom (lessons + typed entities + categories). Reembed-only; preserves kind→memory_type, per-row TTL, recall_count (→ ranking via reinforcement_count), suppression polarity, and provenance. Idempotent via deterministic keys + per-run version. import-recall Import an external vector-recall agent-memory SQLite export into Loom. Introspects the DB at runtime (schema not assumed): tables are classified by column heuristics into memories (→ memory.set) or conversation transcripts (→ conversation.append, grouped by conversation id); unmatched tables/rows are skipped, never crash. Idempotent via deterministic keys. import-brain Import a markdown agent 'brain' directory into Loom: MEMORY.md (always- injected HOT layer) → profile.set sections; memory/contexts/*.md (read-on- demand WARM layer) → knowledge.add docs. Completes the migration alongside import-recall (the deep vector layer). Missing files are skipped, never crash. claim-release Release a self-signup claim: revoke its grants, clear the owner, deactivate it. The name stays reserved. mint-token Mint a JWT locally (requires LOOM_JWT_SECRET). revoke-token Revoke one token by jti or subject (writes loom.revoked_tokens through the revocation module; needs the configured ClickHouse). sso Carry password accounts over to single sign-on. bot Bots and CI jobs that sign in with a key or their CI's identity. doctor Diagnose pre-boot misconfig (Python version, settings, JWT, CH reachability) and print the problem list /v1/health carries, each with its fix. With --client, also examine this machine as a coding-agent client. report Print the bug form's Environment field: versions and the `loom doctor` checks. drops Print the memory ingest drop ledger (loom.memory_drops) for a namespace, by stage/reason — what the pipeline discarded, truncated or gave up on. namespaces-gc List (default) or erase throwaway namespaces — finished bench runs under the bench prefixes that nobody registered and nobody has written to for --idle-hours — in one list-scoped erasure saga per batch. erasure-re-verify Re-verify failed erasure sagas: print each one's last step-named reason and a fresh residue census (default, read-only); --resume re-runs them from their checkpoint past the recovery worker's attempts budget; --orphan-graph lists namespaces whose entity graph outlived their memories and, with --erase, purges them. reconcile After a restore of ClickHouse or Postgres, run again in the restored store the erasures and redactions the other records past the restore point, and (after a ClickHouse restore) publish again every revision committed after it. Stop every server and worker first. Dry run by default. router The router in front of a deployment's cells. cell Operate a deployment's cells through the router's directory. tracker-probe Read-only first contact with the tracker source: counts and newest timestamps per upstream table per credential, breaker states, the configured tracker namespaces. tracker-contract-check Read-only: does each tracker source's schema meet the record contract (docs/features/tracker/record-contract.md)? Names missing tables / columns, optional tables present, feature columns. Exit 0 when every checked source meets it. tracker-dupcheck-fit Refit tracker.duplicate_check's calibrated confidence for one tracker namespace now: replays the duplicate gold at filing time, fits, stores the fit. tracker-embed-reset Clear the tracker_embed pass cursors of one tracker namespace so the next pass re-walks the window and re-embeds every item whose text recipe changed. Dry run by default (prints the cursors); --apply clears them. tracker-feed Maintain a team's GitHub tracker record namespaces-adopt Give namespaces that hold live memory and no registry row an owner row: kind=bench under the bench prefixes, else memory; retention_days 0, no TTL. Dry run unless --apply. sensitivity-sweep Label live rows the sensitivity detector would have labelled on write but a path skipped. Dry run by default: counts by level, probe shape and namespace. --apply writes one new version per row, tombstones a high row's embedding rows and tags a medium row's. The closing line is the high- sensitivity invariant (high rows with a live embedding). integrity-repair Read the row-level integrity gauges (dim-0 rows holding a vector, tombstoned_at/status drift, active rows past valid_to) and, with --apply, re-emit the dim-0 rows with their true dimension so search can return them again. Drift and valid_to are listed, not repaired. rechunk Re-chunk and re-embed a namespace's knowledge documents from stored content through the current chunker, so every document carries section metadata, chunk context and document frequency. Idempotent; one embedding per new chunk. Dry run by default (counts the documents and chunks); --apply re-chunks. export-actions Export the action schema derived from the operation registry — an MCP tools/list manifest with loom extras (json) or the Cedar schema (cedar) a policy gateway governs the surface with. lexical-stats-rebuild Recompute BM25 document frequency for a namespace from its live rows (the term-frequency views only see inserts; supersession and tombstones leave stale vocabulary behind). version Print installed Loom version. install Wire Loom into a local AI client's config. Writes the MCP server block into Claude Desktop / Cursor / VS Code's config file, atomically, merging with anything already there; for Claude Code it runs (or prints) `claude mcp add` and for Codex `codex mcp add`. Restart the client afterwards. demo The first five minutes: ingest this repo's AGENTS.md (or --file) into the knowledge plane the way `make ingest` does, ask it a question, and print the answer with its evidence rows. On the stock fake-LLM stack it prints the top knowledge.search hits and says what a real answer needs. init Create an identity on a Loom server. `--agent NAME` signs an agent up by itself (server needs LOOM_SELF_SIGNUP=open): mints a writer token on a fresh agent- namespace, saves it to ~/.loom/token, and prints the claim code a person enters once, with a password of their own, to claim the namespace. ingest-transcript Pipe a chat-history file (markdown / JSON) into Loom's conversation table as one thread. Generic ingest for tools without an MCP integration (Aider, Continue.dev, JetBrains AI, anything else): export the chat → run this command → the thread shows in the dashboard's Conversations. memory-search Run a memory.search from the shell. Default human output; --json-agent emits a structured envelope agents can pipe through (a `{command, results}` JSON shape). set-host Designate the Loom host(s) for this deployment. Writes the given login handle(s) or email(s) to LOOM_PLATFORM_ADMINS in the .env (--env-file, default ./.env): a handle promotes that password account to platform_admin, an email promotes an SSO sign-in with that verified email. Multiple OK (on-call rotation, co-operators, etc.). Re-login required after to pick up the role. login Sign in to a Loom server on any browser with a one-time code. logout End the stored session on the server and forget it here. whoami Show who the stored session signs you in as. call Call any operation or API route with the stored session (`loom call --help`). token Make, list and revoke your own tokens for CI and agents (`loom token --help`).", + "positional arguments: {code,decisions,knowledge,export-okf,import-okf,okf-lint,up,down,ch-setup,participant,deployment,settings,secrets,signing-key,provider-lock,export-analytics,sql-access,reembed,reextract,deploy-delta,entity-export,audit-namespace,mentor,serve,migrate,reconcile-status,period-digest,demote-activity-narration,recalibrate,import-clickgap,import-recall,import-brain,claim-release,mint-token,revoke-token,sso,bot,doctor,report,drops,namespaces-gc,erasure-re-verify,reconcile,router,cell,tracker-probe,tracker-contract-check,tracker-dupcheck-fit,tracker-embed-reset,tracker-feed,namespaces-adopt,sensitivity-sweep,integrity-repair,rechunk,export-actions,lexical-stats-rebuild,version,install,demo,init,ingest-transcript,memory-search,set-host,login,logout,whoami,call,token} code Code intelligence plane operations. decisions Decision records mined from documents. knowledge Knowledge documents and their sources. export-okf Render a namespace as an OKF v0.2 bundle (Markdown + frontmatter) — the git-reviewable projection of Loom memory. import-okf Write the concepts of an OKF v0.2 bundle into a namespace via memory.set. okf-lint Lint an OKF bundle directory offline — the compile gate a mirror repository's CI runs. Exit 2 critical, 1 warnings, 0 clean. up Boot the local stack (Loom, its document extractor and ClickHouse) and print how to connect: creates .env and the ClickHouse config from their templates, runs docker compose up --build, waits for /v1/health, then prints the dashboard URL and the first memory.set (curl + Python SDK) and `claude mcp add` snippets — minting a writer token first when auth is on. Re-run any time. down Stop the local stack while preserving data; --delete-data performs a confirmed reset. ch-setup Prepare a self-managed ClickHouse or a ClickHouse Cloud service for Loom: create the loom database and the read-only reader user, or print the SQL for an admin. participant The Loom processes a provider change waits for. deployment The configuration every replica checks itself against. settings Every setting's value in force, and overrides of the live ones. secrets Credentials and signing keys stored sealed in ClickHouse. signing-key The token signing keys beside LOOM_JWT_SECRET, kept in ClickHouse. provider-lock The provider configuration lock. export-analytics Write one UTC day of usage and a namespace snapshot to S3 as Parquet, names hashed under LOOM_ANALYTICS_EXPORT_NAME_KEY. Never replaces an object, so a repeated run is safe. sql-access Make, rescope or drop a ClickHouse user that reads a tenant's memory, conversations and request ledger through the sql_v1_* views. reembed Re-embed vectors whose embedding_model != the configured embedder (run after changing LOOM_EMBEDDING_MODEL — Loom does not auto-re-embed). Covers memory, entity, and conversation vectors. Coded vector tables keep their width: change a model of another width in Settings → Models & keys, which rebuilds them. reextract Replay a namespace's raw evidence (memory.set_from_messages turns) through a running server's write path into ANOTHER namespace — the current extractor over the stored transcripts; the source is never written. deploy-delta What a colour swap changed: the deployment's ledger rows (volume, errors, p50/p95, upstream spend per operation) and its ClickHouse statement shapes (runs, bytes, seconds) for a window on each side of the swap, with flags for what moved past a coarse threshold. entity-export Export a namespace's entity graph (co-occurrence edges + triples) to GraphML or JSON-LD for an external graph or RDF tool. Read-only; every edge carries its provenance and a note that a path is declared, not proven. audit-namespace How a namespace is being used (namespace.audit): exact-key reads vs searches, hash keys, double-encoded and oversized values, hand-rolled index rows, leases stored as memories, version churn. Read-only; prints a table (code, severity, kind, count, share, first sample) or --json. Exit 1 on a critical finding. mentor The store's advice to one agent (mentor.brief): what it called, the planes its namespace offers, the hints with their evidence, and how often past hints were followed. Read-only; prints a table or --json. Exit 1 when a warning-level hint fires. serve Boot the control plane (uvicorn). migrate Apply schema migrations against the configured CH. reconcile-status Re-emit memory_embeddings rows whose status diverged from loom.memory (superseded / conflicted / duplicate marks the pool never saw), compared and re-emitted per identity row (org, namespace, agent, user_id, session_id, key). Dry run by default (counts by status, the rows it would write); --apply writes. period-digest Write period digests — one derived semantic memory over K consecutive session digests of each identity in one namespace. Additive and idempotent. Dry run by default (plans only); --apply writes. demote-activity-narration Find a namespace's extracted facts that narrate tool activity ('User read lines 1-1000 of X', 'Claude ran a command on …') with no durable outcome and demote (confidence x0.25) or label them through memory.apply. Dry run by default; keys an agent read by name in the last --days are left alone. Never deletes. recalibrate Re-probe the embedder + run per-NS calibration immediately (otherwise this runs weekly via the recalibration worker). import-clickgap Import a ClickGap (Detector→Validator→Reviewer) SQLite export into Loom (lessons + typed entities + categories). Reembed-only; preserves kind→memory_type, per-row TTL, recall_count (→ ranking via reinforcement_count), suppression polarity, and provenance. Idempotent via deterministic keys + per-run version. import-recall Import an external vector-recall agent-memory SQLite export into Loom. Introspects the DB at runtime (schema not assumed): tables are classified by column heuristics into memories (→ memory.set) or conversation transcripts (→ conversation.append, grouped by conversation id); unmatched tables/rows are skipped, never crash. Idempotent via deterministic keys. import-brain Import a markdown agent 'brain' directory into Loom: MEMORY.md (always- injected HOT layer) → profile.set sections; memory/contexts/*.md (read-on- demand WARM layer) → knowledge.add docs. Completes the migration alongside import-recall (the deep vector layer). Missing files are skipped, never crash. claim-release Release a self-signup claim: revoke its grants, clear the owner, deactivate it. The name stays reserved. mint-token Mint a JWT locally (requires LOOM_JWT_SECRET and verified stores). revoke-token Revoke one token by jti or subject (writes loom.revoked_tokens through the revocation module; needs the configured ClickHouse). sso Carry password accounts over to single sign-on. bot Bots and CI jobs that sign in with a key or their CI's identity. doctor Diagnose pre-boot misconfig (Python version, settings, JWT, CH reachability) and print the problem list /v1/health carries, each with its fix. With --client, also examine this machine as a coding-agent client. report Print the bug form's Environment field: versions and the `loom doctor` checks. drops Print the memory ingest drop ledger (loom.memory_drops) for a namespace, by stage/reason — what the pipeline discarded, truncated or gave up on. namespaces-gc List (default) or erase throwaway namespaces — finished bench runs under the bench prefixes that nobody registered and nobody has written to for --idle-hours — in one list-scoped erasure saga per batch. erasure-re-verify Re-verify failed erasure sagas: print each one's last step-named reason and a fresh residue census (default, read-only); --resume re-runs them from their checkpoint past the recovery worker's attempts budget; --orphan-graph lists namespaces whose entity graph outlived their memories and, with --erase, purges them. reconcile After a restore of ClickHouse or Postgres, run again in the restored store the erasures and redactions the other records past the restore point, and (after a ClickHouse restore) publish again every revision committed after it. Stop every server and worker first. Dry run by default. router The router in front of a deployment's cells. cell Operate a deployment's cells through the router's directory. tracker-probe Read-only first contact with the tracker source: counts and newest timestamps per upstream table per credential, breaker states, the configured tracker namespaces. tracker-contract-check Read-only: does each tracker source's schema meet the record contract (docs/features/tracker/record-contract.md)? Names missing tables / columns, optional tables present, feature columns. Exit 0 when every checked source meets it. tracker-dupcheck-fit Refit tracker.duplicate_check's calibrated confidence for one tracker namespace now: replays the duplicate gold at filing time, fits, stores the fit. tracker-embed-reset Clear the tracker_embed pass cursors of one tracker namespace so the next pass re-walks the window and re-embeds every item whose text recipe changed. Dry run by default (prints the cursors); --apply clears them. tracker-feed Maintain a team's GitHub tracker record namespaces-adopt Give namespaces that hold live memory and no registry row an owner row: kind=bench under the bench prefixes, else memory; retention_days 0, no TTL. Dry run unless --apply. sensitivity-sweep Label live rows the sensitivity detector would have labelled on write but a path skipped. Dry run by default: counts by level, probe shape and namespace. --apply writes one new version per row, tombstones a high row's embedding rows and tags a medium row's. The closing line is the high- sensitivity invariant (high rows with a live embedding). integrity-repair Read the row-level integrity gauges (dim-0 rows holding a vector, tombstoned_at/status drift, active rows past valid_to) and, with --apply, re-emit the dim-0 rows with their true dimension so search can return them again. Drift and valid_to are listed, not repaired. rechunk Re-chunk and re-embed a namespace's knowledge documents from stored content through the current chunker, so every document carries section metadata, chunk context and document frequency. Idempotent; one embedding per new chunk. Dry run by default (counts the documents and chunks); --apply re-chunks. export-actions Export the action schema derived from the operation registry — an MCP tools/list manifest with loom extras (json) or the Cedar schema (cedar) a policy gateway governs the surface with. lexical-stats-rebuild Recompute BM25 document frequency for a namespace from its live rows (the term-frequency views only see inserts; supersession and tombstones leave stale vocabulary behind). version Print installed Loom version. install Wire Loom into a local AI client's config. Writes the MCP server block into Claude Desktop / Cursor / VS Code's config file, atomically, merging with anything already there; for Claude Code it runs (or prints) `claude mcp add` and for Codex `codex mcp add`. Restart the client afterwards. demo The first five minutes: ingest this repo's AGENTS.md (or --file) into the knowledge plane the way `make ingest` does, ask it a question, and print the answer with its evidence rows. On the stock fake-LLM stack it prints the top knowledge.search hits and says what a real answer needs. init Create an identity on a Loom server. `--agent NAME` signs an agent up by itself (server needs LOOM_SELF_SIGNUP=open): mints a writer token on a fresh agent- namespace, saves it to ~/.loom/token, and prints the claim code a person enters once, with a password of their own, to claim the namespace. ingest-transcript Pipe a chat-history file (markdown / JSON) into Loom's conversation table as one thread. Generic ingest for tools without an MCP integration (Aider, Continue.dev, JetBrains AI, anything else): export the chat → run this command → the thread shows in the dashboard's Conversations. memory-search Run a memory.search from the shell. Default human output; --json-agent emits a structured envelope agents can pipe through (a `{command, results}` JSON shape). set-host Designate the Loom host(s) for this deployment. Writes the given login handle(s) or email(s) to LOOM_PLATFORM_ADMINS in the .env (--env-file, default ./.env): a handle promotes that password account to platform_admin, an email promotes an SSO sign-in with that verified email. Multiple OK (on-call rotation, co-operators, etc.). Re-login required after to pick up the role. login Sign in to a Loom server on any browser with a one-time code. logout End the stored session on the server and forget it here. whoami Show who the stored session signs you in as. call Call any operation or API route with the stored session (`loom call --help`). token Make, list and revoke your own tokens for CI and agents (`loom token --help`).", "options: -h, --help show this help message and exit --version show program's version number and exit" ], "loom code": [ @@ -185,12 +185,12 @@ ], "loom participant": [ "usage: loom participant [-h] {retire} ...", - "positional arguments: {retire} retire Retire the provider participant of a process that died without shutting down (its id is `participant_id` in that process's /v1/health).", + "positional arguments: {retire} retire Verify exact physical death on this trusted host and retire its participant after all queries settle (the id is participant_id in /v1/health).", "options: -h, --help show this help message and exit" ], "loom participant retire": [ - "usage: loom participant retire [-h] --participant PARTICIPANT [--yes]", - "options: -h, --help show this help message and exit --participant PARTICIPANT The participant id to retire --yes Its process is confirmed stopped" + "usage: loom participant retire [-h] --participant PARTICIPANT", + "options: -h, --help show this help message and exit --participant PARTICIPANT The participant id to retire" ], "loom deployment": [ "usage: loom deployment [-h] {publish} ...", @@ -397,8 +397,8 @@ "options: -h, --help show this help message and exit --name NAME" ], "loom doctor": [ - "usage: loom doctor [-h] [--json] [--client] [--server-url SERVER_URL] [--token TOKEN]", - "options: -h, --help show this help message and exit --json Emit the checks and the problem list as JSON. --client Also examine this machine as a coding-agent client: the server the hooks would call, the token they would present, whether it is accepted and can write the intended namespace, the Claude Code / Codex wiring, and capture jobs that never reached the server. --server-url SERVER_URL Override the client's server URL. --token TOKEN Override the client's token." + "usage: loom doctor [-h] [--json] [--client] [--server-url SERVER_URL] [--pg-allow-role PG_ALLOW_ROLE] [--stray-orgs] [--stray-orgs-budget STRAY_ORGS_BUDGET] [--token TOKEN]", + "options: -h, --help show this help message and exit --json Emit the checks and the problem list as JSON. --client Also examine this machine as a coding-agent client: the server the hooks would call, the token they would present, whether it is accepted and can write the intended namespace, the Claude Code / Codex wiring, and capture jobs that never reached the server. --server-url SERVER_URL Override the client's server URL. --pg-allow-role PG_ALLOW_ROLE On a cell: a Postgres role that may make an org held (the servers' role). --stray-orgs On a cell: also read the orgs its stores hold without a tenancy row. --stray-orgs-budget STRAY_ORGS_BUDGET Seconds --stray-orgs may read for, smallest tables first (default 60). --token TOKEN Override the client's token." ], "loom report": [ "usage: loom report [-h] [--open]", @@ -434,8 +434,8 @@ "options: -h, --help show this help message and exit" ], "loom cell": [ - "usage: loom cell [-h] {credential,limit,which,remove,prepare} ...", - "positional arguments: {credential,limit,which,remove,prepare} credential Make a router credential, or rotate a cell's. limit Close, reopen or bound a cell to new orgs. which Print the lookup's cell and host for an org (the first cell if not placed). remove Remove the directory row of a cell attaching with no placement. prepare Register this cell in the router's directory and write its rows.", + "usage: loom cell [-h] {credential,limit,which,remove,prepare,attach} ...", + "positional arguments: {credential,limit,which,remove,prepare,attach} credential Make a router credential, or rotate a cell's. limit Close, reopen or bound a cell to new orgs. which Print the lookup's cell and host for an org (the first cell if not placed). remove Remove the directory row of a cell attaching with no placement. prepare Register this cell in the router's directory and write its rows. attach Attach the prepared first cell: place its orgs, serve.", "options: -h, --help show this help message and exit" ], "loom cell credential": [ @@ -445,7 +445,7 @@ ], "loom cell limit": [ "usage: loom cell limit [-h] --cell CELL [--closed | --open] [--placements-max N|none] [--router ROUTER]", - "options: -h, --help show this help message and exit --cell CELL The cell to limit --closed Refuse its new orgs --open Take new orgs again --placements-max N|none Its quota of orgs (none: no quota) --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" + "options: -h, --help show this help message and exit --cell CELL The cell to limit --closed Refuse its new orgs --open Take new orgs again once serving --placements-max N|none Its quota of orgs (none: no quota) --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" ], "loom cell which": [ "usage: loom cell which [-h] --org ORG [--router ROUTER]", @@ -459,6 +459,10 @@ "usage: loom cell prepare [-h] --cell CELL --host HOST --credential-sha256 SHA256 [--probe-url PROBE_URL] [--wait WAIT] [--router ROUTER]", "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --host HOST The host the cell serves its orgs at --credential-sha256 SHA256 The sha256 of the cell's router credential (`loom cell credential new`) --probe-url PROBE_URL A later cell's probe listener address (not the first's) --wait WAIT Seconds to retry a busy or unreachable probe listener --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" ], + "loom cell attach": [ + "usage: loom cell attach [-h] --cell CELL [--wait WAIT] [--scan-budget SCAN_BUDGET] [--dry-run] [--ch-allow-user CH_ALLOW_USER] [--ch-allow-user-directory CH_ALLOW_USER_DIRECTORY] [--ch-allow-object CH_ALLOW_OBJECT] [--ch-allow-replica CH_ALLOW_REPLICA] [--pg-allow-role PG_ALLOW_ROLE] [--router ROUTER]", + "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --wait WAIT Seconds of wall time the waits share; 0 runs one try of each, with no retry --scan-budget SCAN_BUDGET Seconds one full org scan may take --dry-run Run the refusals and the scan and print what would be placed; write nothing (the router's own begin refusals do not run) --ch-allow-user CH_ALLOW_USER A ClickHouse user that may write the configured databases --ch-allow-user-directory CH_ALLOW_USER_DIRECTORY A ClickHouse user directory the check cannot read --ch-allow-object CH_ALLOW_OBJECT A ClickHouse object (db.name) that writes the databases --ch-allow-replica CH_ALLOW_REPLICA A replica outside the cluster (db.table:replica) --pg-allow-role PG_ALLOW_ROLE A Postgres role that may make an org held (the servers' role) --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" + ], "loom tracker-probe": [ "usage: loom tracker-probe [-h]", "options: -h, --help show this help message and exit" diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json index e011a5fdc0..6b63b43d3d 100644 --- a/tests/unit/snapshots/cli_tree.json +++ b/tests/unit/snapshots/cli_tree.json @@ -5707,8 +5707,7 @@ "description": null, "dests": [ "help", - "participant", - "yes" + "participant" ] } ], @@ -5744,21 +5743,6 @@ "required": true, "help": "The participant id to retire", "metavar": null - }, - { - "option_strings": [ - "--yes" - ], - "dest": "yes", - "action": "_StoreTrueAction", - "default": false, - "const": true, - "choices": null, - "nargs": 0, - "type": null, - "required": false, - "help": "Its process is confirmed stopped", - "metavar": null } ] } @@ -5769,7 +5753,7 @@ "choices_help": [ { "dest": "retire", - "help": "Retire the provider participant of a process that died without shutting down (its id is `participant_id` in that process's /v1/health).", + "help": "Verify exact physical death on this trusted host and retire its participant after all queries settle (the id is participant_id in /v1/health).", "metavar": "retire" } ] @@ -11238,6 +11222,9 @@ "json", "client", "server_url", + "pg_allow_role", + "stray_orgs", + "stray_orgs_budget", "token" ] } @@ -11305,6 +11292,51 @@ "help": "Override the client's server URL.", "metavar": null }, + { + "option_strings": [ + "--pg-allow-role" + ], + "dest": "pg_allow_role", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "On a cell: a Postgres role that may make an org held (the servers' role).", + "metavar": null + }, + { + "option_strings": [ + "--stray-orgs" + ], + "dest": "stray_orgs", + "action": "_StoreTrueAction", + "default": false, + "const": true, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "On a cell: also read the orgs its stores hold without a tenancy row.", + "metavar": null + }, + { + "option_strings": [ + "--stray-orgs-budget" + ], + "dest": "stray_orgs_budget", + "action": "_StoreAction", + "default": 60.0, + "const": null, + "choices": null, + "nargs": null, + "type": "builtins:float", + "required": false, + "help": "Seconds --stray-orgs may read for, smallest tables first (default 60).", + "metavar": null + }, { "option_strings": [ "--token" @@ -12599,7 +12631,7 @@ "nargs": 0, "type": null, "required": false, - "help": "Take new orgs again", + "help": "Take new orgs again once serving", "metavar": null }, { @@ -12928,6 +12960,211 @@ "metavar": null } ] + }, + "attach": { + "prog": "loom cell attach", + "usage": null, + "description": null, + "epilog": null, + "formatter_class": "argparse:HelpFormatter", + "add_help": true, + "allow_abbrev": true, + "defaults": { + "func": "server.cli.cell_attach:cmd_attach" + }, + "groups": [ + { + "title": "positional arguments", + "description": null, + "dests": [] + }, + { + "title": "options", + "description": null, + "dests": [ + "help", + "cell", + "wait", + "scan_budget", + "dry_run", + "ch_allow_user", + "ch_allow_user_directory", + "ch_allow_object", + "ch_allow_replica", + "pg_allow_role", + "router" + ] + } + ], + "mutually_exclusive": [], + "arguments": [ + { + "option_strings": [ + "-h", + "--help" + ], + "dest": "help", + "action": "_HelpAction", + "default": "==SUPPRESS==", + "const": null, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "show this help message and exit", + "metavar": null + }, + { + "option_strings": [ + "--cell" + ], + "dest": "cell", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": true, + "help": "This cell's name (LOOM_CELL)", + "metavar": null + }, + { + "option_strings": [ + "--wait" + ], + "dest": "wait", + "action": "_StoreAction", + "default": 600, + "const": null, + "choices": null, + "nargs": null, + "type": "builtins:int", + "required": false, + "help": "Seconds of wall time the waits share; 0 runs one try of each, with no retry", + "metavar": null + }, + { + "option_strings": [ + "--scan-budget" + ], + "dest": "scan_budget", + "action": "_StoreAction", + "default": 7200, + "const": null, + "choices": null, + "nargs": null, + "type": "builtins:int", + "required": false, + "help": "Seconds one full org scan may take", + "metavar": null + }, + { + "option_strings": [ + "--dry-run" + ], + "dest": "dry_run", + "action": "_StoreTrueAction", + "default": false, + "const": true, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "Run the refusals and the scan and print what would be placed; write nothing (the router's own begin refusals do not run)", + "metavar": null + }, + { + "option_strings": [ + "--ch-allow-user" + ], + "dest": "ch_allow_user", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "A ClickHouse user that may write the configured databases", + "metavar": null + }, + { + "option_strings": [ + "--ch-allow-user-directory" + ], + "dest": "ch_allow_user_directory", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "A ClickHouse user directory the check cannot read", + "metavar": null + }, + { + "option_strings": [ + "--ch-allow-object" + ], + "dest": "ch_allow_object", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "A ClickHouse object (db.name) that writes the databases", + "metavar": null + }, + { + "option_strings": [ + "--ch-allow-replica" + ], + "dest": "ch_allow_replica", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "A replica outside the cluster (db.table:replica)", + "metavar": null + }, + { + "option_strings": [ + "--pg-allow-role" + ], + "dest": "pg_allow_role", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "A Postgres role that may make an org held (the servers' role)", + "metavar": null + }, + { + "option_strings": [ + "--router" + ], + "dest": "router", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "The router's URL (default: LOOM_CELL_ROUTER_URL)", + "metavar": null + } + ] } }, "aliases_of": { @@ -12935,7 +13172,8 @@ "limit": "loom cell limit", "which": "loom cell which", "remove": "loom cell remove", - "prepare": "loom cell prepare" + "prepare": "loom cell prepare", + "attach": "loom cell attach" }, "choices_help": [ { @@ -12962,6 +13200,11 @@ "dest": "prepare", "help": "Register this cell in the router's directory and write its rows.", "metavar": "prepare" + }, + { + "dest": "attach", + "help": "Attach the prepared first cell: place its orgs, serve.", + "metavar": "attach" } ] } @@ -16170,7 +16413,7 @@ }, { "dest": "mint-token", - "help": "Mint a JWT locally (requires LOOM_JWT_SECRET).", + "help": "Mint a JWT locally (requires LOOM_JWT_SECRET and verified stores).", "metavar": "mint-token" }, { diff --git a/tests/unit/snapshots/settings_defaults.json b/tests/unit/snapshots/settings_defaults.json index 35fb6568e1..d09b5cb585 100644 --- a/tests/unit/snapshots/settings_defaults.json +++ b/tests/unit/snapshots/settings_defaults.json @@ -963,6 +963,7 @@ "namespace_taxonomy_ttl_s": 300.0, "namespace_tune_document_sample": 50, "namespace_write_marks_flush_interval_s": 5.0, + "native_command_drain_timeout_s": 15.0, "oauth_access_token_ttl_seconds": 3600, "oauth_enabled": false, "oauth_issuer": "", @@ -1267,6 +1268,7 @@ "worker_backoff_cap_s": 60.0, "worker_ledger_rows": true, "worker_partial_retry_s": 3600.0, + "workers_child_retirement_timeout_s": 1.0, "workers_process": "child", "workload_exchanges_per_agent": 300, "workload_exchanges_per_trust": 50000 diff --git a/tests/unit/test_analytics_export.py b/tests/unit/test_analytics_export.py index 6375893585..a2508f2981 100644 --- a/tests/unit/test_analytics_export.py +++ b/tests/unit/test_analytics_export.py @@ -257,7 +257,10 @@ async def disconnect(self) -> None: def test_an_export_of_yesterday_connects_inside_its_deadline_and_hashes_under_the_key( - now: datetime, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + stub_model_command, + now: datetime, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], ) -> None: from server.core import ch as ch_module from server.insights.app import analytics_export as app diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py new file mode 100644 index 0000000000..0cbd369201 --- /dev/null +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -0,0 +1,2333 @@ +"""Offline proofs for the fixed cell trigger timing protocol; no fixture is opened.""" + +from __future__ import annotations + +import argparse +import asyncio +import dataclasses +import json +import socket +import uuid +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import httpx +import pytest + +from bench import cell_trigger_gate as gate + + +@pytest.fixture(autouse=True) +def no_external_io(monkeypatch): + async def forbidden(*_args, **_kwargs): + raise AssertionError("unexpected real I/O") + + monkeypatch.setattr(gate.asyncpg, "connect", forbidden) + monkeypatch.setattr(gate.clickhouse_connect, "get_async_client", forbidden) + monkeypatch.setattr(gate.httpx.AsyncClient, "send", forbidden) + + +def args(**overrides): + return argparse.Namespace( + **{ + "base_url": "http://127.0.0.1:17932", + "pg_dsn": "postgresql://u:secret@127.0.0.1:15790/gate_cell", + "token": "secret-token", + "org": "org", + "namespace": "ns", + "launch_manifest": "unused", + **overrides, + } + ) + + +def role_config(): + from server.settings import Settings + + config = SimpleNamespace(**Settings.model_construct().model_dump(mode="json")) + vars(config).update( + cell_router_token="", + router_operator_token="", + pg_dsn=args().pg_dsn, + ch_host="127.0.0.1", + ch_port=18790, + ch_database="gate_cell", + ch_telemetry_database="gate_cell_t", + cell="gate-cell", + background_workers="off", + storage_authority="postgres", + embedding_provider="fake", + embedding_model="fake-ngram-16", + embedding_dim=16, + embedding_base_url="", + embedding_send_dimensions=False, + llm_provider="fake", + llm_model="fake", + jwt_secret="test-only-root", + deployment=False, + instance_heartbeat_seconds=10, + build_commit="a" * 40, + pg_delivery_batch_rows=500, + ) + + def dump(**kwargs): + return { + k: v + for k, v in vars(config).items() + if k != "model_dump" and ("include" not in kwargs or k in kwargs["include"]) + } + + config.model_dump = dump + return config + + +def fixture_config(): + config = role_config() + manifest = gate.Fixture( + schema=2, + base_url=args().base_url, + pg_host="127.0.0.1", + pg_port=15790, + pg_database="gate_cell", + ch_host=config.ch_host, + ch_port=config.ch_port, + ch_database=config.ch_database, + ch_telemetry_database=config.ch_telemetry_database, + cell=config.cell, + org="org", + namespace="ns", + deployment_id="deployment", + epoch=1, + instance_id="instance", + build_commit=config.build_commit, + source_tree="b" * 40, + launch_manifest_sha256="c" * 64, + ownership_receipt_sha256="d" * 64, + api_configuration_sha256=gate._digest( + {**config.model_dump(), "cell_router_token": "lcr_" + "0" * 128} + ), + cli_configuration_sha256=gate._digest(config.model_dump()), + shared_configuration_sha256=gate._digest( + {k: v for k, v in config.model_dump().items() if k != "cell_router_token"} + ), + background_workers="off", + controller_excluded=True, + pg_internal_port=5432, + ) + return manifest, config + + +def test_gate_keeps_the_original_api_sample_contract(): + assert (gate.CALLS, gate.WARMUP, gate.ROUNDS, gate.BATCHES) == (2000, 50, 5, 5) + assert not hasattr(gate, "_batch") and not hasattr(gate, "_clean") + + +@pytest.mark.parametrize(("batch", "expected"), [(500, 45500), (1590, 100000)]) +def test_complete_fixture_count_uses_the_unchanged_delivery_batch(batch, expected): + assert gate._planned_keys(batch) == expected + + +@pytest.mark.parametrize("batch", [1591, 0, -1, 1.5, True]) +def test_invalid_batch_refuses_before_loading(batch): + with pytest.raises(ValueError): + gate._planned_keys(batch) + + +def test_arm_order_alternates_pair_order_and_has_five_of_each(): + order = gate._arm_order() + assert order == (True, False, False, True, True, False, False, True, True, False) + assert order.count(True) == order.count(False) == 5 + + +@pytest.mark.parametrize( + "url", + [ + "http://localhost:1234", + "http://example.com:1234", + "http://10.0.0.1:1234", + "http://127.0.0.1", + "http://127.0.0.1:1234/other", + "http://u:pw@127.0.0.1:1234", + "http://127.0.0.1:1234?x=1", + "http://127.0.0.1:1234#fragment", + ], +) +def test_http_endpoint_refuses_ambiguous_or_remote_targets(url): + with pytest.raises(gate.Invalid): + gate._endpoint(url) + + +@pytest.mark.parametrize( + "dsn", + [ + "postgresql://u@localhost:15790/scratch", + "postgresql://u@127.0.0.1:15790/loom", + "postgresql://u@127.0.0.1:15790/scratch?host=remote", + "postgresql:///scratch?service=other", + "postgresql://u@127.0.0.1,127.0.0.2:15790/scratch", + "postgresql://u@127.0.0.1:15790/scratch?options=-cdefault_transaction_read_only=off", + ], +) +def test_pg_endpoint_refuses_overrides_defaults_and_alternate_targets(dsn): + with pytest.raises(gate.Invalid): + gate._endpoint(dsn, pg=True) + + +def test_manifest_binds_published_port_without_conflating_internal_port(): + manifest, config = fixture_config() + assert manifest.pg_port == 15790 and manifest.pg_internal_port == 5432 + assert manifest.validate(args(), config) == 45500 + with pytest.raises(gate.Invalid, match="manifest PG endpoint"): + dataclasses.replace(manifest, pg_port=5432).validate(args(), config) + + +@pytest.mark.parametrize( + "change", + [ + {"ch_host": "10.0.0.1"}, + {"ch_database": "loom"}, + {"cell": "other"}, + {"background_workers": "on"}, + {"llm_provider": "openai"}, + {"embedding_provider": "local"}, + {"pg_delivery_batch_rows": 1591}, + ], +) +def test_configuration_drift_and_unsafe_settings_refuse(change): + manifest, config = fixture_config() + for k, v in change.items(): + setattr(config, k, v) + with pytest.raises(gate.Invalid): + manifest.validate(args(), config) + + +def test_artifact_never_overwrites_existing_file_or_symlink(tmp_path): + original = tmp_path / "old.jsonl" + original.write_text("preserve") + with pytest.raises(FileExistsError): + gate.Artifact(str(original)) + link = tmp_path / "link" + link.symlink_to(original) + with pytest.raises(FileExistsError): + gate.Artifact(str(link)) + assert original.read_text() == "preserve" + fresh = tmp_path / "fresh" + artifact = gate.Artifact(str(fresh)) + artifact.emit("test", n=1) + artifact.close() + assert fresh.stat().st_mode & 0o777 == 0o600 + assert json.loads(fresh.read_text()) == {"kind": "test", "n": 1} + + +@pytest.mark.asyncio +async def test_actual_api_loop_sends_unique_four_kib_values_and_full_sample(monkeypatch): + sent, outcomes = [], [] + + class Client: + def __init__(self, **kw): + assert kw["follow_redirects"] is False and kw["trust_env"] is False + + async def __aenter__(self): + return self + + async def __aexit__(self, *_): + return None + + async def post(self, url, *, json): + sent.append(json) + return httpx.Response( + 200, + json={ + "memory_id": str(uuid.uuid4()), + "version": len(sent), + "admission": "ready", + "status": "active", + "content_sha256": gate.sha256_value(json["value"]), + "memory_key": json["memory_key"], + "namespace": json["namespace"], + }, + request=httpx.Request("POST", url), + ) + + monkeypatch.setattr(gate.httpx, "AsyncClient", Client) + rows = await gate._memory_set( + args(), emit=lambda kind, **fields: outcomes.append((kind, fields)) + ) + assert len(sent) == len(rows) == len(outcomes) == 2050 + assert ( + len({body["memory_key"] for body in sent}) == len({body["value"] for body in sent}) == 2050 + ) + assert all(len(body["value"].encode("utf-8")) == 4096 for body in sent) + assert sum(row.measured for row in rows) == 2000 + assert all("value" not in fields and "token" not in fields for _, fields in outcomes) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "status,changes", + [ + (302, {}), + (503, {}), + (200, {"status": "duplicate"}), + (200, {"noop": True}), + (200, {"content_sha256": "wrong"}), + (200, {"version": 0}), + ], +) +async def test_api_refusal_is_recorded_once_and_never_enters_samples(monkeypatch, status, changes): + events = [] + + class Client: + def __init__(self, **_): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *_): + return None + + async def post(self, url, *, json): + body = { + "memory_id": str(uuid.uuid4()), + "version": 1, + "status": "active", + "admission": "ready", + "memory_key": json["memory_key"], + "namespace": json["namespace"], + "content_sha256": gate.sha256_value(json["value"]), + **changes, + } + return httpx.Response(status, json=body, request=httpx.Request("POST", url)) + + monkeypatch.setattr(gate.httpx, "AsyncClient", Client) + with pytest.raises(gate.Invalid): + await gate._memory_set(args(), emit=lambda *a, **kw: events.append((a, kw))) + assert len(events) == 1 and events[0][0] == ("request_error",) + assert "secret-token" not in str(events) and "value" not in events[0][1] + + +def samples(on_p99, off_p99, on_rps=90.1, off_rps=100): + on, off = iter(on_p99), iter(off_p99) + return [ + gate.Samples(arm, [next(on) if arm else next(off)] * gate.CALLS, on_rps if arm else off_rps) + for arm in gate._arm_order() + ] + + +def test_statistics_use_difference_of_arm_medians_and_throughput_not_time(): + result = gate._statistics(samples([2, 12, 22, 32, 102], [1, 6, 51, 61, 71])) + assert result["p99_delta_ms"] == -29 + assert result["paired_p99_differences_ms"] == [1, 6, -29, -29, 31] + assert result["latency_within_limit"] is True + # Time increases 10.99%, but the original rows/second gate passes at 9.9% loss. + assert result["throughput_loss"] == pytest.approx(0.099) + assert result["throughput_within_limit"] is True + failed = gate._statistics(samples([20] * 5, [1] * 5, 89, 100)) + assert failed["latency_within_limit"] is failed["throughput_within_limit"] is False + assert gate._quantile(list(range(2000)), 0.99) == 1979 + + +@pytest.mark.parametrize("value", [0, -1, float("nan"), float("inf")]) +def test_invalid_throughput_cannot_become_passing_number(value): + with pytest.raises(gate.Invalid): + gate._statistics(samples([1] * 5, [1] * 5, value)) + + +@pytest.mark.asyncio +async def test_actual_production_worker_body_runs_five_times_with_unchanged_batch(monkeypatch): + from server.core import idempotency, pg + from server.llm import customer + from server.memory.app import pg_delivery as publish + from server.memory.store import pg_memory + from server.memory.workers import pg_delivery as worker + from server.settings import settings + + monkeypatch.setattr(settings, "pg_delivery_batch_rows", 2) + monkeypatch.setattr(pg, "enabled", lambda: True) + monkeypatch.setattr(worker, "_read_backlog", AsyncMock()) + claim = AsyncMock(return_value=[object(), object()]) + monkeypatch.setattr(pg_memory, "claim_due", claim) + attribution = [] + + async def publish_claims(claims): + attribution.append(customer._attribution.get()) + assert len(claims) == 2 + return 2, 0 + + monkeypatch.setattr(publish, "publish_claims", publish_claims) + for module, name in ( + (idempotency, "sweep_expired"), + (pg_memory, "sweep_erasure_evidence"), + (pg_memory, "sweep_settled"), + (publish, "publish_audit_outbox"), + ): + monkeypatch.setattr(module, name, AsyncMock(return_value=0)) + times = iter([0, 2_000_000_000]) + monkeypatch.setattr(gate.time, "perf_counter_ns", lambda: next(times)) + events = [] + assert ( + await gate._delivery(args(), 2, emit=lambda kind, **kw: events.append((kind, kw)), arm=0) + == 5 + ) + assert claim.await_count == 5 and all(call.args == (2,) for call in claim.await_args_list) + assert len(attribution) == 5 + assert all( + a.org == "org" and a.namespace == "ns" and a.subject == gate.SUBJECT for a in attribution + ) + assert [kind for kind, _ in events] == ["worker"] * 5 + ["delivery_block"] + assert customer._attribution.get() is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "result", + [ + {"claimed": 1, "published": 1}, + {"claimed": 2, "published": 1}, + {"claimed": 2, "published": 2, "failed": 1}, + ], +) +async def test_worker_count_failure_invalidates_block_without_retries(monkeypatch, result): + from server.memory.workers import pg_delivery as worker + from server.settings import settings + + monkeypatch.setattr(settings, "pg_delivery_batch_rows", 2) + run = AsyncMock(return_value=worker.DeliveryResult(**result)) + monkeypatch.setattr(worker, "run_pass", run) + with pytest.raises(gate.Invalid): + await gate._delivery(args(), 2, emit=lambda *_a, **_kw: None, arm=0) + assert run.await_count == 1 + + +@pytest.mark.asyncio +async def test_canonical_full_rows_audits_and_expected_vectors_use_real_fake_embedder(monkeypatch): + import server.embedding + from server.embedding.fake import FakeNGramEmbedder + from server.llm import customer + from server.memory.store import pg_memory + from server.memory.store.rows import MEMORY_PERSISTED_COLS + + embedder = FakeNGramEmbedder(dim=16) + monkeypatch.setattr(server.embedding, "get_embedder", lambda: embedder) + # Keep actual require(): refresh its real switch read through a fake store edge. + import logging + + from server.core.polled_copy import PolledCopy + + monkeypatch.setattr( + customer, + "_switch", + PolledCopy("switch", AsyncMock(return_value=frozenset()), logging.getLogger(__name__)), + ) + block = await gate._build_block(args(), 1) + assert len(block.rows) == len(block.audits) == len(block.documents) == 5 + for row, audit, doc in zip(block.rows, block.audits, block.documents, strict=True): + assert set(MEMORY_PERSISTED_COLS) <= row.keys() + assert len(json.loads(row["value"]).encode()) == 4096 + assert row["actor"] == gate.SUBJECT and row["source_class"] == "asserted" + assert row["content_sha256"] == gate.sha256_value(json.loads(row["value"])) + assert audit["event_type"] == "memory.revision" and len(audit["row_hash"]) == 64 + assert row["value"] not in audit["subject"] + assert len(doc["embedding"]) == doc["embedding_dim"] == 16 + assert doc["content_sha256"] == row["content_sha256"] + assert pg_memory.decode_row(pg_memory.encode_row(row)) == row + assert customer._attribution.get() is None + + +@pytest.mark.asyncio +async def test_stage_calls_real_commit_interface_with_due_fresh_targets(monkeypatch): + from server.memory.store import pg_memory + + block = gate.Block( + [{"memory_id": str(uuid.uuid4()), "org": "org", "namespace": "ns", "memory_key": "key"}], + [{"row_hash": "hash"}], + [], + ) + debt = AsyncMock() + absent = AsyncMock() + monkeypatch.setattr(gate, "_debt", debt) + monkeypatch.setattr(gate, "_absent", absent) + committed = pg_memory.Committed( + rows=block.rows, audits=block.audits, seqs=[("org", "ns", 1)], targets=[7] + ) + commit = AsyncMock(return_value=committed) + monkeypatch.setattr(pg_memory, "commit_versions", commit) + conn = SimpleNamespace(fetchval=AsyncMock(side_effect=[0, 1])) + await gate._stage(conn, object(), block) + assert commit.await_args.kwargs == {"targets": 7, "publisher_lease": False} + assert block.seqs == [("org", "ns", 1)] + assert [call.args[1] for call in debt.await_args_list] == [0, 1] + absent.assert_awaited_once() + + +@pytest.mark.parametrize( + ("field", "expected", "actual"), + [ + ("value", "right", "wrong"), + ("version", 2, 1), + ("content_sha256", "a", "b"), + ("embedding", [1.0, 0.0], [0.0, 1.0]), + ("embedding", [1.0], []), + ("row_hash", "good", "bad"), + ], +) +def test_stored_payload_vector_version_and_audit_mismatch_refuse(field, expected, actual): + with pytest.raises(gate.Invalid): + gate._compare({field: expected}, {field: actual}, [field], "stored") + + +def stored_bfloat16(vector): + """Exercise the installed insert writer and query reader without a server.""" + import array + + from clickhouse_connect.datatypes.base import TypeDef + from clickhouse_connect.datatypes.numeric import BFloat16 + + class Encoded: + def __init__(self, raw): + self.raw = raw + + def read_array(self, kind, count): + result = array.array(kind) + result.frombytes(self.raw) + assert len(result) == count + return result + + kind = BFloat16(TypeDef()) + raw = bytearray() + kind._write_column_binary(vector, raw, SimpleNamespace(column_name="embedding")) + return kind._read_column_binary( + Encoded(raw), len(vector), SimpleNamespace(use_numpy=False), None + ) + + +@pytest.mark.parametrize("field", ["embedding", "memory_key_embedding"]) +def test_storage_vector_matches_installed_bfloat16_and_rejects_corrupted_neighbor(field): + from server.embedding.fake import FakeNGramEmbedder + + vector = FakeNGramEmbedder(dim=1024)._embed_raw(gate._payload("gate-review-owned-1")) + stored = stored_bfloat16(vector) + assert sum(a != b for a, b in zip(vector, stored, strict=True)) > 0 + gate._compare({field: vector}, {field: stored}, [field], "stored") + import struct + + corrupted = list(stored) + index = next(i for i, value in enumerate(stored) if value > 0) + bits = struct.unpack(" 0 + + +@pytest.mark.parametrize("cleanup_failure", [False, True]) +def test_final_verdict_follows_native_cleanup_and_artifact_close( + monkeypatch, tmp_path, capsys, cleanup_failure +): + import sys + + import server.settings + import server.wiring + from server.cli import _ch + + fixture, config = fixture_config() + manifest = tmp_path / "manifest" + manifest.write_text(json.dumps(dataclasses.asdict(fixture))) + output = tmp_path / "result.jsonl" + arguments = args() + argv = ["driver"] + for name, value in { + **vars(arguments), + "fixture_manifest": str(manifest), + "output": str(output), + }.items(): + argv += ["--" + name.replace("_", "-"), str(value)] + order = [] + monkeypatch.setattr(sys, "argv", argv) + monkeypatch.setattr(server.settings, "settings", config) + monkeypatch.setattr(server.wiring, "wire", lambda: order.append("wire")) + monkeypatch.setattr(gate, "_preflight", AsyncMock()) + monkeypatch.setattr(gate, "_source", lambda _: {"clean": True}) + monkeypatch.setattr(gate, "_launch", lambda *_: {}) + + def native(*_a, **_kw): + order.append("native_cleanup") + return { + "latency_within_limit": True, + "throughput_within_limit": True, + "p99_delta_ms": 0.0, + "throughput_loss": 0.0, + } + + monkeypatch.setattr(_ch, "run_connected", native) + + def sync(_fd): + assert capsys.readouterr().out == "" + assert order == ["wire", "native_cleanup"] + if cleanup_failure: + raise OSError("secret-token") + + monkeypatch.setattr(gate.os, "fsync", sync) + assert gate.main() == int(cleanup_failure) + final = capsys.readouterr().out + assert "secret-token" not in final + assert json.loads(final)["verdict"] == ("INVALID" if cleanup_failure else "PASS") + assert '"verdict": "PASS"' not in output.read_text() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", + [ + "", + "projection_value", + "projection_version", + "vector", + "audit_hash", + "audit_subject", + "missing_document", + "targets", + "attempts", + "head", + ], +) +async def test_composed_stored_verification_compares_authority_projection_vectors_and_audits( + monkeypatch, fault +): + import logging + + import server.embedding + from server.core.polled_copy import PolledCopy + from server.embedding.fake import FakeNGramEmbedder + from server.llm import customer + from server.memory.store import pg_memory + + monkeypatch.setattr( + customer, + "_switch", + PolledCopy("switch", AsyncMock(return_value=frozenset()), logging.getLogger(__name__)), + ) + monkeypatch.setattr(server.embedding, "get_embedder", lambda: FakeNGramEmbedder(dim=16)) + block = await gate._build_block(args(), 1) + block.seqs = [("org", "ns", i + 1) for i in range(5)] + records = [ + { + "head": json.dumps(pg_memory.encode_row(row)), + "revision": json.dumps(pg_memory.encode_row(row)), + "audit": json.dumps(pg_memory.encode_audit(audit)), + "seq": i + 1, + "targets": 1 if fault == "targets" else 0, + "attempts": 1 if fault == "attempts" else 0, + "last_error": "", + } + for i, (row, audit) in enumerate(zip(block.rows, block.audits, strict=True)) + ] + projections = [dict(row) for row in block.rows] + documents = [dict(row) for row in block.documents] + for document in documents: + for field in ("embedding", "memory_key_embedding"): + document[field] = stored_bfloat16(document[field]) + audits = [dict(row) for row in block.audits] + if fault == "projection_value": + projections[0]["value"] = '"wrong"' + if fault == "projection_version": + projections[0]["version"] -= 1 + if fault == "vector": + documents[0]["embedding"] = [0.0] * 16 + if fault == "audit_hash": + audits[0]["row_hash"] = "bad" + if fault == "audit_subject": + audits[0]["subject"] = "wrong" + if fault == "missing_document": + documents.pop() + if fault == "head": + records.pop() + conn = SimpleNamespace( + fetch=AsyncMock(return_value=records), + fetchrow=AsyncMock(return_value={"pending": 0, "audits": 0, "errors": 0}), + ) + + async def query(sql, params): + if "FROM loom.memory FINAL" in sql: + return projections + if "FROM loom.memory_embeddings FINAL" in sql: + return documents + if "FROM loom.audit_log" in sql: + return audits + raise AssertionError(sql) + + ch = SimpleNamespace(query_all=query) + if fault: + with pytest.raises(gate.Invalid): + await gate._verify(conn, ch, args(), block) + else: + await gate._verify(conn, ch, args(), block) + assert all( + "org=" in call.args[0] and "namespace=" in call.args[0] + for call in conn.fetch.await_args_list + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", ["", "memory_key", "value", "version", "content_sha256", "status"] +) +async def test_api_authority_failure_names_only_the_failed_field(monkeypatch, fault): + import logging + + import server.embedding + from server.core.polled_copy import PolledCopy + from server.embedding.fake import FakeNGramEmbedder + from server.llm import customer + from server.memory.store import pg_memory + + monkeypatch.setattr( + customer, + "_switch", + PolledCopy("switch", AsyncMock(return_value=frozenset()), logging.getLogger(__name__)), + ) + monkeypatch.setattr(server.embedding, "get_embedder", lambda: FakeNGramEmbedder(dim=16)) + block = await gate._build_block(args(), 1) + row, audit = block.rows[0], block.audits[0] + expected = gate.RequestRow( + str(row["memory_key"]), + str(row["memory_id"]), + row["version"], + row["content_sha256"], + 1.0, + True, + "derived_ready", + ) + head = dict(row) + if fault: + head[fault] = { + "memory_key": "changed-key", + "value": json.dumps("changed-value"), + "version": row["version"] + 1, + "content_sha256": "0" * 64, + "status": "conflicted", + }[fault] + stored = json.dumps(pg_memory.encode_row(head)) + conn = SimpleNamespace( + fetch=AsyncMock( + return_value=[ + { + "head": stored, + "revision": stored, + "audit": json.dumps(pg_memory.encode_audit(audit)), + "seq": 1, + "targets": 0, + "attempts": 0, + "last_error": "", + } + ] + ), + fetchrow=AsyncMock(return_value={"pending": 0, "audits": 0, "errors": 0}), + ) + document = await gate._expected_document(args(), row) + for field in ("embedding", "memory_key_embedding"): + document[field] = stored_bfloat16(document[field]) + + async def query(sql, params): + if "FROM loom.memory FINAL" in sql: + return [row] + if "FROM loom.memory_embeddings FINAL" in sql: + return [document] + if "FROM loom.audit_log" in sql: + return [audit] + raise AssertionError(sql) + + ch = SimpleNamespace(query_all=query) + if fault: + with pytest.raises(gate.Invalid) as raised: + await gate._verify(conn, ch, args(), [expected]) + assert str(raised.value) == "authority API " + fault + else: + await gate._verify(conn, ch, args(), [expected]) + + +@pytest.mark.asyncio +async def test_real_customer_policy_can_refuse_expected_vectors_without_bypass(monkeypatch): + import logging + + import server.embedding + from server.core.polled_copy import PolledCopy + from server.embedding.fake import FakeNGramEmbedder + from server.llm import customer + + monkeypatch.setattr( + customer, + "_switch", + PolledCopy( + "switch", AsyncMock(return_value=frozenset({"org"})), logging.getLogger(__name__) + ), + ) + monkeypatch.setattr(server.embedding, "get_embedder", lambda: FakeNGramEmbedder(dim=16)) + with pytest.raises(customer.ModelCallsDisabled): + await gate._build_block(args(), 1) + assert customer._attribution.get() is None + + +@pytest.mark.asyncio +async def test_partial_trigger_toggle_is_restored_on_original_failure(monkeypatch): + conn = TriggerConn() + original = ValueError("DDL failed") + writes = 0 + + async def execute(sql): + nonlocal writes + writes += 1 + if writes == 2: + raise original + conn.on = sql.startswith("CREATE TRIGGER") + + async def snapshot(c): + return ("memory_current", "tenancy"), "invariant", "org" if c.on else gate._digest([]) + + monkeypatch.setattr(conn, "execute", execute) + monkeypatch.setattr(gate.Triggers, "snapshot", staticmethod(snapshot)) + monkeypatch.setattr(gate.cell_triggers, "missing", AsyncMock(return_value=[])) + triggers = await gate.Triggers.capture(conn) + with pytest.raises(ValueError) as raised: + try: + await triggers.set(conn, False) + finally: + await triggers.restore(conn) + assert raised.value is original and conn.on and triggers.on + + +def test_manifest_unknown_fields_and_credentials_refuse(tmp_path): + fixture, _ = fixture_config() + path = tmp_path / "manifest" + path.write_text(json.dumps({**dataclasses.asdict(fixture), "password": "secret"})) + with pytest.raises(gate.Invalid, match="fixture manifest"): + gate.Fixture.read(str(path)) + + +@pytest.mark.parametrize("fault", ["", "head", "tree", "dirty", "refused"]) +def test_source_provenance_uses_only_fixed_bounded_readonly_git(monkeypatch, fault): + fixture, _ = fixture_config() + calls = [] + + def git(argv, **kwargs): + calls.append(argv) + assert argv[0] == "git" and kwargs["timeout"] == 5 and kwargs["check"] is False + assert kwargs["capture_output"] is True + assert kwargs["cwd"] == Path(__file__).resolve().parents[2] + assert "shell" not in kwargs + if argv[1:] == ["rev-parse", "HEAD"]: + value = "wrong" if fault == "head" else fixture.build_commit + elif argv[1:] == ["rev-parse", "HEAD^{tree}"]: + value = "wrong" if fault == "tree" else fixture.source_tree + else: + assert argv[1:] == ["status", "--porcelain"] + value = " M file" if fault == "dirty" else "" + return SimpleNamespace(returncode=int(fault == "refused"), stdout=value) + + monkeypatch.setattr(gate.subprocess, "run", git) + if fault: + with pytest.raises(gate.Invalid): + gate._source(fixture) + else: + assert gate._source(fixture) == { + "head": fixture.build_commit, + "tree": fixture.source_tree, + "clean": True, + } + assert len(calls) == 3 + + +@pytest.mark.asyncio +async def test_restoration_has_finite_deadline_and_closes_owned_connection(monkeypatch): + conn = ProofPG() + + class Stuck: + async def restore(self, _): + await asyncio.Event().wait() + + monkeypatch.setattr(gate, "RESTORE_S", 0.01) + + async def close(owned, *_): + assert owned is conn + await conn.close() + + monkeypatch.setattr(gate, "_close_pg", close) + artifact = SimpleNamespace(emit=lambda *_a, **_kw: None) + with pytest.raises(TimeoutError): + await gate._cleanup(Stuck(), conn, artifact) + assert conn.closed + + +@pytest.mark.asyncio +async def test_review_delivery_timestamps_pass_real_clickhouse_serialization(monkeypatch): + import logging + + from clickhouse_connect.datatypes.registry import get_from_name + from clickhouse_connect.driver.insert import InsertContext + + import server.embedding + from server.core.ch import _insert_data + from server.core.polled_copy import PolledCopy + from server.embedding.fake import FakeNGramEmbedder + from server.llm import customer + + monkeypatch.setattr( + customer, + "_switch", + PolledCopy("switch", AsyncMock(return_value=frozenset()), logging.getLogger(__name__)), + ) + monkeypatch.setattr(server.embedding, "get_embedder", lambda: FakeNGramEmbedder(dim=16)) + block = await gate._build_block(args(), 1) + for col in ("created_at", "updated_at", "last_reinforced_at"): + actual_data = _insert_data(block.rows, [col], ["DateTime"]) + kind = get_from_name("DateTime") + context = InsertContext("loom.memory", [col], [kind]) + context.start_column(col) + serialized = bytearray() + kind.write_column([row[0] for row in actual_data], serialized, context) + assert len(serialized) == 4 * len(block.rows) + assert all( + row["created_at"] == row["updated_at"] == row["last_reinforced_at"] for row in block.rows + ) + + +@pytest.mark.asyncio +async def test_review_trailing_slash_is_normalized_for_actual_health_and_api_routes(monkeypatch): + from starlette.responses import JSONResponse + from starlette.routing import Route, Router + + fixture, _ = fixture_config() + paths = [] + + async def health(request): + paths.append(request.url.path) + return JSONResponse({}) + + app = Router( + routes=[ + Route("/v1/health", health), + Route("/v1/health/deployment", health), + Route("/v1/memory.set", health, methods=["POST"]), + ] + ) + transport = httpx.ASGITransport(app=app) + + async def send(self, request, **kwargs): + response = await transport.handle_async_request(request) + await response.aread() + return response + + monkeypatch.setattr(httpx.AsyncClient, "send", send) + with pytest.raises(gate.Invalid): + await gate._health(args(base_url=fixture.base_url + "/"), fixture) + with pytest.raises(gate.Invalid): + await gate._memory_set(args(base_url=fixture.base_url + "/")) + assert paths == ["/v1/health", "/v1/health/deployment", "/v1/memory.set"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status,code", [(302, ""), (503, "PUBLICATION_PENDING")]) +async def test_review_failed_request_keeps_raw_status_elapsed_and_classification( + monkeypatch, status, code +): + async def send(self, request, **kwargs): + return httpx.Response( + status, json={"detail": {"code": code, "message": "secret-token"}}, request=request + ) + + monkeypatch.setattr(httpx.AsyncClient, "send", send) + clock = iter([10_000_000, 17_000_000]) + monkeypatch.setattr(gate.time, "perf_counter_ns", lambda: next(clock)) + events = [] + with pytest.raises(gate.Invalid): + await gate._memory_set(args(), emit=lambda kind, **record: events.append((kind, record))) + assert len(events) == 1 and events[0][0] == "request_error" + record = events[0][1] + assert record["status"] == status and record["code"] == code + assert record["elapsed_ms"] == 7 and record["measured"] is False + assert "secret-token" not in json.dumps(events) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", + [ + "", + "settings_fp", + "keys_fp", + "config_fp", + "llm_provider", + "llm_model", + "embedding_model", + "embedding_dim_configured", + "embedding_dim_probed", + "host", + "instance", + "uuid", + "namespace_policy", + ], +) +async def test_review_composed_proof_matches_actual_server_fingerprints_and_health( + monkeypatch, tmp_path, fault +): + import server.settings + from server.core import cell, deployment_config + + fixture, config = fixture_config() + fixture, launch_path, _, launch, _ = launch_files(tmp_path, fixture, config) + monkeypatch.setattr(gate, "_owned_server", lambda _: None) + mine = cell.mine() + mine["deployment_id"] = fixture.deployment_id + fingerprints = deployment_config.fingerprints(config) + + class PG(ProofPG): + async def fetchrow(self, sql, *params): + if "FROM loom.cell_self" in sql: + return {"current_fp": json.dumps(cell.as_value(mine)), "next_fp": None} + return await super().fetchrow(sql, *params) + + class CH(ProofCH): + async def query_all(self, sql, params=None): + if "FROM loom.namespaces" in sql: + assert params == {"org": args().org, "ns": args().namespace} + return [ + { + "default_conflict_policy": "auto_reinforce" + if fault == "namespace_policy" + else "append_only" + } + ] + if "FROM loom.instances" in sql: + return [ + { + "instance_id": fixture.instance_id, + "build_commit": fixture.build_commit, + "workers_lease": "", + "serving": 1, + "host": "other" if fault == "host" else socket.gethostname(), + **fingerprints, + **({fault: "wrong"} if fault.endswith("_fp") else {}), + } + ] + if "FROM loom.cell_heartbeats" in sql: + return [ + { + "cell": fixture.cell, + "tenancy_mode": "enforced", + "fingerprint": json.dumps( + {**cell.as_value(mine), "serves": True, "marked": True} + ), + } + ] + return await super().query_all(sql, params) + + async def send(self, request, **kwargs): + body = ( + {"deployment_id": fixture.deployment_id} + if request.url.path.endswith("/deployment") + else { + "build_commit": fixture.build_commit, + "instance": "other:4242" + if fault == "instance" + else launch["server"]["health_instance"], + "deployment_config": { + "instance_id": "other" if fault == "uuid" else fixture.instance_id + }, + "ch_ok": True, + "pg_ok": True, + "status": "ok", + "fake_embedding": True, + "fake_llm": True, + "pending_migrations": [], + "storage_authority": "postgres", + "llm_provider": config.llm_provider, + "llm_model": config.llm_model, + "embedding_model": config.embedding_model, + "embedding_dim_configured": config.embedding_dim, + "embedding_dim_probed": config.embedding_dim, + **( + {fault: 999 if "dim" in fault else "wrong"} + if fault and not fault.endswith("_fp") + else {} + ), + } + ) + return httpx.Response(200, json=body, request=request) + + monkeypatch.setattr(server.settings, "settings", config) + monkeypatch.setattr(gate, "_source", lambda _: {"clean": True}) + monkeypatch.setattr(cell, "mine", lambda: dict(mine)) + monkeypatch.setattr(httpx.AsyncClient, "send", send) + if fault: + with pytest.raises(gate.Invalid): + await gate._proof(PG(), CH(), args(launch_manifest=str(launch_path)), fixture) + else: + proof = await gate._proof(PG(), CH(), args(launch_manifest=str(launch_path)), fixture) + assert proof["api_configuration_sha256"] == fixture.api_configuration_sha256 + assert proof["cli_configuration_sha256"] == fixture.cli_configuration_sha256 + assert proof["shared_configuration_sha256"] == fixture.shared_configuration_sha256 + assert proof["shared_configuration_excludes"] == ["cell_router_token"] + assert proof["credential_posture"] == receipt_posture() + assert proof["api_configuration_attested"] is True + + +def receipt_posture(): + return { + "api_cell_router_token_present": True, + "cli_cell_router_token_present": False, + "api_router_operator_token_present": False, + "cli_router_operator_token_present": False, + } + + +def launch_files(tmp_path, fixture, config): + import os + import socket + + from server.core import deployment_config + + public = { + k: v + for k, v in config.model_dump().items() + if k.startswith("pg_") + and k != "pg_dsn" + or k.startswith("memory_conflict") + or k + in { + "storage_authority", + "background_workers", + "embedding_provider", + "embedding_model", + "embedding_dim", + "embedding_send_dimensions", + "embedding_worker_enabled", + "embedding_cache_shared", + "embedding_cache_shared_max_chars", + "embedding_cache_size", + "embedding_cache_ttl_s", + "llm_provider", + "llm_model", + } + } + configuration = { + "schema": 2, + "api_configuration_sha256": fixture.api_configuration_sha256, + "cli_configuration_sha256": fixture.cli_configuration_sha256, + "shared_configuration_sha256": fixture.shared_configuration_sha256, + "credential_posture": receipt_posture(), + "workload": public, + "fingerprints": deployment_config.fingerprints(config), + } + artifact_path = tmp_path / "configuration.json" + artifact_bytes = json.dumps(configuration).encode() + artifact_path.write_bytes(artifact_bytes) + source_path = str(__import__("pathlib").Path(gate.__file__).resolve().parents[1]) + launch = { + "schema": 2, + "server": { + "pid": 4242, + "start_ticks": "123", + "boot_id": "fake-boot-id", + "uid": os.geteuid(), + "cwd": source_path, + "health_instance": f"{socket.gethostname()}:4242", + "instance_id": fixture.instance_id, + }, + "source": {"head": fixture.build_commit, "tree": fixture.source_tree}, + "configuration_artifact": { + "path": str(artifact_path), + "sha256": gate.hashlib.sha256(artifact_bytes).hexdigest(), + }, + } + launch_path = tmp_path / "launch.json" + launch_bytes = json.dumps(launch).encode() + launch_path.write_bytes(launch_bytes) + fixture = dataclasses.replace( + fixture, launch_manifest_sha256=gate.hashlib.sha256(launch_bytes).hexdigest() + ) + return fixture, launch_path, artifact_path, launch, configuration + + +@pytest.mark.parametrize( + "fault", + [ + "", + "launch_hash", + "artifact_hash", + "config", + "workload", + "workload_type", + "fingerprints", + "source", + "instance", + "symlink", + ], +) +def test_launch_receipt_binds_named_owned_process_settings_and_nonsecret_artifact( + monkeypatch, tmp_path, fault +): + fixture, config = fixture_config() + fixture, path, artifact_path, launch, configuration = launch_files(tmp_path, fixture, config) + checked = [] + monkeypatch.setattr(gate, "_owned_server", lambda server: checked.append(server), raising=False) + if fault in {"config", "workload", "workload_type", "fingerprints"}: + if fault == "config": + configuration["cli_configuration_sha256"] = "0" * 64 + if fault == "workload": + configuration["workload"]["pg_delivery_batch_rows"] = 501 + if fault == "workload_type": + configuration["workload"]["pg_delivery_batch_rows"] = 500.0 + if fault == "fingerprints": + configuration["fingerprints"]["keys_fp"] = "wrong" + blob = json.dumps(configuration).encode() + artifact_path.write_bytes(blob) + launch["configuration_artifact"]["sha256"] = gate.hashlib.sha256(blob).hexdigest() + if fault == "artifact_hash": + launch["configuration_artifact"]["sha256"] = "0" * 64 + if fault == "source": + launch["source"]["tree"] = "0" * 40 + if fault == "instance": + launch["server"]["instance_id"] = "other" + blob = json.dumps(launch).encode() + path.write_bytes(blob) + fixture = dataclasses.replace( + fixture, + launch_manifest_sha256="0" * 64 + if fault == "launch_hash" + else gate.hashlib.sha256(blob).hexdigest(), + ) + if fault == "symlink": + link = tmp_path / "link.json" + link.symlink_to(path) + path = link + if fault: + with pytest.raises((gate.Invalid, OSError)): + gate._launch(args(launch_manifest=str(path)), fixture, config) + else: + result = gate._launch(args(launch_manifest=str(path)), fixture, config) + assert result["health_instance"] == launch["server"]["health_instance"] + assert result["fingerprints"] == configuration["fingerprints"] + assert checked == [launch["server"]] + + +@pytest.mark.parametrize( + "fault", + ["", "uid", "actual_uid", "pid", "start", "reuse", "boot", "cwd", "stamp_pid", "stamp_host"], +) +def test_owned_server_reads_only_exact_named_birth_metadata(monkeypatch, tmp_path, fault): + import os + from pathlib import Path + + from server.core import provider_process + + fixture, config = fixture_config() + _, _, _, launch, _ = launch_files(tmp_path, fixture, config) + server = dict(launch["server"]) + if fault == "uid": + server["uid"] += 1 + if fault == "pid": + server["pid"] = True + if fault == "stamp_pid": + server["health_instance"] = f"{socket.gethostname()}:4243" + if fault == "stamp_host": + server["health_instance"] = "other:4242" + paths, births = [], [] + real_stat = Path.stat + + def metadata(path, **kwargs): + if str(path).startswith("/proc"): + assert str(path) == "/proc/4242" + paths.append(str(path)) + return SimpleNamespace(st_uid=os.geteuid() + int(fault == "actual_uid")) + return real_stat(path, **kwargs) + + def start(pid): + assert type(pid) is int and pid == 4242 + births.append(pid) + return "wrong" if fault == "start" or fault == "reuse" and len(births) == 2 else "123" + + real_read = Path.read_text + + def read(path, **kwargs): + if str(path).startswith("/proc"): + assert str(path) == "/proc/sys/kernel/random/boot_id" + paths.append(str(path)) + return "wrong" if fault == "boot" else "fake-boot-id" + return real_read(path, **kwargs) + + real_link = os.readlink + + def link(path, **kwargs): + if str(path).startswith("/proc"): + assert str(path) == "/proc/4242/cwd" + paths.append(str(path)) + return "other" if fault == "cwd" else server["cwd"] + return real_link(path, **kwargs) + + monkeypatch.setattr(Path, "stat", metadata) + monkeypatch.setattr(Path, "read_text", read) + monkeypatch.setattr(provider_process, "_start_ticks", start) + monkeypatch.setattr(os, "readlink", link) + if fault: + with pytest.raises(gate.Invalid): + gate._owned_server(server) + else: + gate._owned_server(server) + assert births == [4242, 4242] + assert paths == ["/proc/4242", "/proc/sys/kernel/random/boot_id", "/proc/4242/cwd"] + + +@pytest.mark.asyncio +async def test_failed_network_request_retains_elapsed_with_no_invented_status(monkeypatch): + async def send(self, request, **kwargs): + raise httpx.ConnectError("secret-token", request=request) + + monkeypatch.setattr(httpx.AsyncClient, "send", send) + clock = iter([1_000_000, 3_000_000]) + monkeypatch.setattr(gate.time, "perf_counter_ns", lambda: next(clock)) + records = [] + with pytest.raises(httpx.ConnectError): + await gate._memory_set(args(), emit=lambda kind, **record: records.append(record)) + assert len(records) == 1 and records[0]["elapsed_ms"] == 2 + assert records[0]["status"] is None and records[0]["code"] == "" + assert "secret-token" not in json.dumps(records) + + +def test_nonregular_receipt_open_is_nonblocking_before_type_check(monkeypatch): + import os + + def open_receipt(path, flags): + assert flags & os.O_NONBLOCK + assert flags & os.O_NOFOLLOW + raise gate.Invalid("nonregular receipt") + + monkeypatch.setattr(gate.os, "open", open_receipt) + with pytest.raises(gate.Invalid, match="nonregular"): + gate._receipt("named-owned-receipt") + + +def test_launch_refusal_precedes_any_native_or_store_setup(monkeypatch, tmp_path, capsys): + import sys + + import server.settings + import server.wiring + from server.cli import _ch + + fixture, config = fixture_config() + fixture, launch_path, _, _, _ = launch_files(tmp_path, fixture, config) + fixture = dataclasses.replace(fixture, launch_manifest_sha256="0" * 64) + manifest = tmp_path / "fixture.json" + manifest.write_text(json.dumps(dataclasses.asdict(fixture))) + arguments = vars(args(launch_manifest=str(launch_path))) + arguments.update(fixture_manifest=str(manifest), output=str(tmp_path / "output.jsonl")) + argv = ["driver"] + for name, value in arguments.items(): + argv += ["--" + name.replace("_", "-"), str(value)] + + def forbidden(*_args, **_kwargs): + raise AssertionError("launch refusal must precede native/process setup") + + monkeypatch.setattr(sys, "argv", argv) + monkeypatch.setattr(server.settings, "settings", config) + monkeypatch.setattr(gate, "_source", lambda _: {"clean": True}) + monkeypatch.setattr(gate, "_owned_server", forbidden) + monkeypatch.setattr(gate, "_preflight", forbidden) + monkeypatch.setattr(server.wiring, "wire", forbidden) + monkeypatch.setattr(_ch, "run_connected", forbidden) + assert gate.main() == 1 + verdict = json.loads(capsys.readouterr().out) + assert verdict["verdict"] == "INVALID" and verdict["reason"] == "launch receipt hash" + assert not (tmp_path / "output.jsonl").exists() + + +@pytest.mark.asyncio +async def test_restore_closes_before_installed_transaction_rollback_can_overrun(monkeypatch): + from asyncpg.transaction import Transaction + + events = [] + release, rolling_back = asyncio.Event(), asyncio.Event() + + class Connection: + _pool_release_ctr = 0 + _top_xact = None + _protocol = SimpleNamespace(is_in_transaction=lambda: False) + + def __init__(self): + self.closed = False + self.timeouts = [] + + def is_closed(self): + return self.closed + + def transaction(self): + return Transaction(self, None, False, False) + + async def execute(self, sql, *args): + if sql.startswith("DROP TRIGGER"): + raise ValueError("restoration DDL failed") + if sql.startswith("ROLLBACK"): + rolling_back.set() + try: + while not release.is_set(): + try: + await release.wait() + except asyncio.CancelledError: + continue + finally: + events.append("rollback ended") + + async def close(self, **kwargs): + self.timeouts.append(kwargs["timeout"]) + events.append("public close") + self.closed = True + release.set() + + def terminate(self): + self.closed = True + release.set() + + conn = Connection() + monkeypatch.setattr(gate, "RESTORE_S", 0.04) + artifact = SimpleNamespace(emit=lambda *a, **k: None) + task = asyncio.create_task( + gate._cleanup(gate.Triggers(("memory_current",), "", ""), conn, artifact) + ) + try: + await asyncio.wait_for(rolling_back.wait(), 1) + done, _ = await asyncio.wait({task}, timeout=0.2) + assert done, "restoration deadline waited for rollback before closing" + with pytest.raises(TimeoutError): + task.result() + assert conn.closed + assert events.index("public close") < events.index("rollback ended") + assert conn.timeouts and all(0 < bound <= 0.04 for bound in conn.timeouts) + finally: + release.set() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("kind", ["SystemExit", "KeyboardInterrupt"]) +@pytest.mark.parametrize("phase", ["restore", "close"]) +def test_driver_cleanup_native_failure_stays_inside_its_owner(kind, phase): + import subprocess + import sys + import textwrap + + program = textwrap.dedent( + """ + import argparse + import asyncio + import sys + from types import SimpleNamespace + from unittest.mock import AsyncMock + from bench import cell_trigger_gate as gate + kind, phase = sys.argv[1:] + original = ValueError('original operation failure') + secondary = {'SystemExit': SystemExit, 'KeyboardInterrupt': KeyboardInterrupt}[kind]( + 'cleanup failed') + events = [] + class Connection: + async def close(self, **kwargs): + events.append('close') + if phase == 'close': raise secondary + def terminate(self): events.append('abort') + class Triggers: + tables = ('memory_current',) + invariant = org = 'hash' + async def set(self, *args): pass + async def restore(self, *args): + events.append('restore') + if phase == 'restore': raise secondary + async def main(): + gate.asyncpg.connect = AsyncMock(return_value=Connection()) + gate._proof = AsyncMock(return_value={}) + gate.Triggers.capture = AsyncMock(return_value=Triggers()) + gate._memory_set = AsyncMock(side_effect=original) + artifact = SimpleNamespace(emit=lambda *a, **k: None) + try: + await gate._run(argparse.Namespace(pg_dsn='unused'), None, artifact, object()) + except BaseException as error: + assert error is original, (type(error).__name__, events) + assert events[0] == 'restore' and 'close' in events, events + await asyncio.sleep(0) + print('original-after-driver-cleanup') + return + raise AssertionError('operation failure was swallowed') + asyncio.run(main()) + """ + ) + result = subprocess.run( + [sys.executable, "-c", program, kind, phase], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "original-after-driver-cleanup" in result.stdout + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "first_kind", [ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] +) +@pytest.mark.parametrize( + "close_kind", [RuntimeError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] +) +@pytest.mark.parametrize("diagnostic_kind", [OSError, SystemExit, KeyboardInterrupt]) +async def test_cleanup_diagnostic_cannot_replace_restoration_error( + first_kind, close_kind, diagnostic_kind +): + from server.cli import cell_attach_pg as apg + + first = first_kind("restoration failed first") + events = [] + + class Triggers: + async def restore(self, conn): + raise first + + class Conn: + async def close(self, **kwargs): + events.append("close") + raise close_kind("close failed second") + + def terminate(self): + events.append("abort") + + class Artifact: + def emit(self, event, **kwargs): + events.append(event) + raise diagnostic_kind("diagnostic failed third") + + async def owner(): + try: + await gate._cleanup(Triggers(), Conn(), Artifact()) + except BaseException as error: # noqa: BLE001 — assert the actual owner failure. + assert error is first + assert "connection_cleanup_error" in events + return + raise AssertionError("restoration failure was swallowed") + + result = await apg._work_result(owner) + assert result is None + + +@pytest.mark.asyncio +async def test_successful_restoration_still_fails_when_artifact_write_fails(): + first = OSError("artifact failed") + conn = SimpleNamespace(close=AsyncMock(), terminate=lambda: None) + artifact = SimpleNamespace(emit=lambda *a, **k: (_ for _ in ()).throw(first)) + triggers = SimpleNamespace(restore=AsyncMock()) + with pytest.raises(OSError) as observed: + await gate._cleanup(triggers, conn, artifact) + assert observed.value is first + conn.close.assert_awaited_once() + + +def rewrite_launch(fixture, path, artifact_path, launch, configuration, *, raw=None, level=None): + configuration_raw = raw if level == "configuration" else json.dumps(configuration).encode() + artifact_path.write_bytes(configuration_raw) + launch["configuration_artifact"]["sha256"] = gate.hashlib.sha256(configuration_raw).hexdigest() + launch_raw = raw if level == "launch" else json.dumps(launch).encode() + path.write_bytes(launch_raw) + return dataclasses.replace( + fixture, launch_manifest_sha256=gate.hashlib.sha256(launch_raw).hexdigest() + ) + + +def test_schema_two_local_pair_has_only_one_fixed_credential_difference(monkeypatch, tmp_path): + from server.core import deployment_config + + fixture, config = fixture_config() + fixture, path, _, _, configuration = launch_files(tmp_path, fixture, config) + checked = [] + monkeypatch.setattr(gate, "_owned_server", lambda value: checked.append(value)) + assert fixture.validate(args(), config) == 45500 + assert gate._launch(args(launch_manifest=str(path)), fixture, config) + actual = config.model_dump(mode="json") + api = {**actual, "cell_router_token": "lcr_" + "0" * 128} + assert gate._digest(api) == fixture.api_configuration_sha256 + assert gate._digest(actual) == fixture.cli_configuration_sha256 + assert fixture.api_configuration_sha256 != fixture.cli_configuration_sha256 + assert gate._shared_configuration(config) == { + key: value for key, value in api.items() if key != "cell_router_token" + } + assert configuration["fingerprints"] == deployment_config.fingerprints(config) + assert len(checked) == 1 + + +@pytest.mark.parametrize("field", ["cell_router_token", "router_operator_token", "jwt_secret"]) +@pytest.mark.parametrize("shape", ["missing", "extra"]) +def test_complete_settings_shape_refuses_missing_or_extra_fields(field, shape): + config = role_config() + if shape == "missing": + delattr(config, field) + else: + config.unexpected_setting = "not-an-exclusion" + with pytest.raises(gate.Invalid, match="Settings fields"): + gate._configuration(config) + + +def test_every_other_effective_setting_stays_in_the_common_digest(): + from server.settings import Settings + + fixture, config = fixture_config() + original = config.model_dump(mode="json") + fields = set(original) - {"cell_router_token"} + assert fields == set(Settings.model_fields) - {"cell_router_token"} + configuration = { + "schema": 2, + "api_configuration_sha256": fixture.api_configuration_sha256, + "cli_configuration_sha256": fixture.cli_configuration_sha256, + "shared_configuration_sha256": fixture.shared_configuration_sha256, + "workload": {}, + "fingerprints": {}, + "credential_posture": receipt_posture(), + } + for field in sorted(fields): + try: + setattr(config, field, ["changed-type-and-value", original[field]]) + changed = dataclasses.replace( + fixture, cli_configuration_sha256=gate._digest(config.model_dump(mode="json")) + ) + configuration["cli_configuration_sha256"] = changed.cli_configuration_sha256 + with pytest.raises(gate.Invalid, match="launch shared configuration"): + gate._attested_configuration(configuration, changed, config) + finally: + setattr(config, field, original[field]) + + +@pytest.mark.parametrize("value", [True, 1, 1.0, "1", None, [], {}]) +def test_common_digest_keeps_json_value_types(value): + _, config = fixture_config() + config.pg_delivery_batch_rows = value + assert gate._digest(gate._shared_configuration(config)) != gate._digest( + {**gate._shared_configuration(config), "pg_delivery_batch_rows": "unchanged"} + ) + hashes = {gate._digest({"value": v}) for v in [True, 1, 1.0, "1", None, [], {}]} + assert len(hashes) == 7 + + +@pytest.mark.parametrize("location", ["fixture", "launch", "configuration"]) +@pytest.mark.parametrize("version", [None, 1, True, 2.0, "2"]) +def test_old_missing_and_noninteger_receipt_versions_refuse( + monkeypatch, tmp_path, location, version +): + fixture, config = fixture_config() + fixture, path, artifact_path, launch, configuration = launch_files(tmp_path, fixture, config) + manifest = dataclasses.asdict(fixture) + target = {"fixture": manifest, "launch": launch, "configuration": configuration}[location] + if version is None: + del target["schema"] + else: + target["schema"] = version + if location != "server": + monkeypatch.setattr( + gate, "_owned_server", lambda _: pytest.fail("invalid receipt reached birth") + ) + if location == "fixture": + path.write_text(json.dumps(manifest)) + with pytest.raises(gate.Invalid): + gate.Fixture.read(str(path)) + else: + fixture = rewrite_launch(fixture, path, artifact_path, launch, configuration) + with pytest.raises(gate.Invalid): + gate._launch(args(launch_manifest=str(path)), fixture, config) + + +@pytest.mark.parametrize( + "location", + [ + "fixture", + "launch", + "server", + "source", + "configuration_artifact", + "configuration", + "workload", + "fingerprints", + "credential_posture", + ], +) +@pytest.mark.parametrize("fault", ["duplicate", "unknown", "missing"]) +def test_closed_receipt_objects_reject_duplicate_and_unknown_fields( + monkeypatch, tmp_path, location, fault +): + fixture, config = fixture_config() + fixture, path, artifact_path, launch, configuration = launch_files(tmp_path, fixture, config) + manifest = dataclasses.asdict(fixture) + root = ( + manifest + if location == "fixture" + else configuration + if location in {"configuration", "workload", "fingerprints", "credential_posture"} + else launch + ) + target = root if location in {"fixture", "launch", "configuration"} else root[location] + if fault == "unknown": + target["unexpected"] = True + blob = json.dumps(root).encode() + elif fault == "missing": + del target[next(iter(target))] + blob = json.dumps(root).encode() + else: + key = next(iter(target)) + original = json.dumps(target) + duplicate = original[:-1] + ", " + json.dumps(key) + ": " + json.dumps(target[key]) + "}" + blob = json.dumps(root).replace(original, duplicate, 1).encode() + if location != "server": + monkeypatch.setattr( + gate, "_owned_server", lambda _: pytest.fail("invalid receipt reached birth") + ) + if location == "fixture": + path.write_bytes(blob) + with pytest.raises(gate.Invalid): + gate.Fixture.read(str(path)) + else: + fixture = rewrite_launch( + fixture, + path, + artifact_path, + launch, + configuration, + raw=blob, + level="configuration" if root is configuration else "launch", + ) + with pytest.raises(gate.Invalid): + gate._launch(args(launch_manifest=str(path)), fixture, config) + + +@pytest.mark.parametrize("key", list(receipt_posture())) +@pytest.mark.parametrize("fault", ["wrong", "integer", "missing"]) +def test_posture_requires_exact_boolean_literals(monkeypatch, tmp_path, key, fault): + fixture, config = fixture_config() + fixture, path, artifact_path, launch, configuration = launch_files(tmp_path, fixture, config) + if fault == "missing": + del configuration["credential_posture"][key] + else: + configuration["credential_posture"][key] = ( + not receipt_posture()[key] if fault == "wrong" else int(receipt_posture()[key]) + ) + fixture = rewrite_launch(fixture, path, artifact_path, launch, configuration) + monkeypatch.setattr(gate, "_owned_server", lambda _: pytest.fail("bad posture reached birth")) + with pytest.raises(gate.Invalid, match="credential posture"): + gate._launch(args(launch_manifest=str(path)), fixture, config) + + +@pytest.mark.parametrize("fault", ["equal", "cli", "shared", "swapped"]) +def test_role_digests_cannot_be_equal_swapped_or_mismatched(fault): + fixture, config = fixture_config() + if fault == "equal": + fixture = dataclasses.replace( + fixture, api_configuration_sha256=fixture.cli_configuration_sha256 + ) + elif fault == "swapped": + fixture = dataclasses.replace( + fixture, + api_configuration_sha256=fixture.cli_configuration_sha256, + cli_configuration_sha256=fixture.api_configuration_sha256, + ) + else: + fixture = dataclasses.replace(fixture, **{fault + "_configuration_sha256": "0" * 64}) + with pytest.raises(gate.Invalid): + fixture.validate(args(), config) + + +@pytest.mark.parametrize("location", ["fixture", "configuration"]) +def test_removed_configuration_digest_is_not_a_compatibility_field(monkeypatch, tmp_path, location): + fixture, config = fixture_config() + fixture, path, artifact_path, launch, configuration = launch_files(tmp_path, fixture, config) + target = dataclasses.asdict(fixture) if location == "fixture" else configuration + target["configuration_sha256"] = fixture.cli_configuration_sha256 + if location == "fixture": + path.write_text(json.dumps(target)) + with pytest.raises(gate.Invalid): + gate.Fixture.read(str(path)) + else: + fixture = rewrite_launch(fixture, path, artifact_path, launch, configuration) + with pytest.raises(gate.Invalid): + gate._launch(args(launch_manifest=str(path)), fixture, config) + + +@pytest.mark.parametrize("location", ["fixture", "launch", "configuration"]) +def test_duplicate_json_fields_refuse_without_any_schema_assumption(tmp_path, location): + path = tmp_path / "receipt.json" + path.write_text('{"' + location + '":0,"' + location + '":1}') + with pytest.raises(gate.Invalid, match="duplicate receipt field"): + gate._receipt(str(path)) + + +@pytest.mark.parametrize( + "field", ["api_configuration_sha256", "cli_configuration_sha256", "shared_configuration_sha256"] +) +@pytest.mark.parametrize("value", [None, True, 64, "A" * 64, "f" * 63, "g" * 64]) +def test_role_digest_fields_are_lowercase_sha256_strings(field, value): + fixture, config = fixture_config() + reason = "fixture receipt digest" if type(value) is str else "fixture field types" + with pytest.raises(gate.Invalid, match=reason): + dataclasses.replace(fixture, **{field: value}).validate(args(), config) + + +def test_schema_two_requires_explicit_internal_port_and_preserves_existing_bytes(tmp_path): + fixture, config = fixture_config() + value = dataclasses.asdict(fixture) + del value["pg_internal_port"] + original = json.dumps(value).encode() + path = tmp_path / "fixture.json" + path.write_bytes(original) + with pytest.raises(gate.Invalid): + gate.Fixture.read(str(path)) + assert path.read_bytes() == original + value["pg_internal_port"] = None + path.write_text(json.dumps(value)) + assert gate.Fixture.read(str(path)).validate(args(), config) == 45500 + + +@pytest.mark.parametrize("target", ["fixture", "launch", "configuration"]) +@pytest.mark.parametrize("fault", ["symlink", "oversize", "owner"]) +def test_all_receipt_roles_keep_file_safety(monkeypatch, tmp_path, target, fault): + import os + + fixture, config = fixture_config() + fixture, path, artifact_path, launch, configuration = launch_files(tmp_path, fixture, config) + manifest_path = tmp_path / "fixture.json" + manifest_path.write_text(json.dumps(dataclasses.asdict(fixture))) + selected = {"fixture": manifest_path, "launch": path, "configuration": artifact_path}[target] + original = selected.read_bytes() + if fault == "symlink": + saved = tmp_path / "saved" + saved.write_bytes(original) + selected.unlink() + selected.symlink_to(saved) + elif fault == "oversize": + selected.write_bytes(b" " * 65_537) + else: + actual = os.fstat + + def metadata(fd): + result = actual(fd) + return SimpleNamespace(st_mode=result.st_mode, st_uid=os.geteuid() + 1) + + monkeypatch.setattr(gate.os, "fstat", metadata) + with pytest.raises((gate.Invalid, OSError, ValueError)): + if target == "fixture": + gate.Fixture.read(str(manifest_path)) + else: + gate._launch(args(launch_manifest=str(path)), fixture, config) + if fault != "oversize": + assert selected.read_bytes() == original + + +def native_guard_edges(monkeypatch, config, events): + from contextlib import asynccontextmanager + + import server.settings + from server.auth import tenancy + from server.code.domain.index import repos + from server.core import background, cell, ch, pg, pg_profile + from server.insights.app import provider_command + from server.settings import Settings + + class Client: + connected = False + + async def connect(self): + events.append("native client") + + @asynccontextmanager + async def model_command(): + yield + + native_settings = Settings.model_construct(**config.model_dump(mode="json")) + + async def state(sql): + assert sql == cell._READ + return { + "first": {"is_first": True, "cell": config.cell}, + "mode": "enforced", + "own": {"current_fp": cell.as_value(cell.mine()), "next_fp": None}, + "tenancy": True, + "marked": True, + } + + async def held(sql, org): + assert sql == tenancy._HOLDS and org == "org" + events.append("native held-org guard") + return 1 + + monkeypatch.setattr(server.settings, "settings", config) + monkeypatch.setattr(cell, "settings", native_settings) + monkeypatch.setattr(tenancy, "settings", native_settings) + monkeypatch.setattr(repos, "settings", native_settings) + monkeypatch.setattr(cell, "_view", cell._View()) + monkeypatch.setattr(cell, "_deployment_id", "") + monkeypatch.setattr(cell, "_code_orgs", None) + monkeypatch.setattr(tenancy, "_held", set()) + monkeypatch.setattr(tenancy, "_elsewhere", {}) + monkeypatch.setattr(ch, "get_client", Client) + monkeypatch.setattr(pg_profile, "check_configured", AsyncMock()) + monkeypatch.setattr(pg_profile, "deployment_id", AsyncMock(return_value="deployment")) + monkeypatch.setattr(pg, "fetchrow_reset", state) + monkeypatch.setattr(pg, "fetchval", held) + monkeypatch.setattr(provider_command, "model_command", model_command) + monkeypatch.setattr(background, "register_buffer_drainer", lambda *_: None) + monkeypatch.setattr("server.cli._ch.disconnect", AsyncMock()) + + +@pytest.mark.parametrize("credential", ["", "lcr_" + "0" * 128]) +def test_role_pair_enters_the_real_native_guard_without_changing_it(monkeypatch, credential): + from server.cli import _ch + + fixture, config = fixture_config() + events = [] + native_guard_edges(monkeypatch, config, events) + assert fixture.validate(args(), config) == 45500 + config.cell_router_token = credential + + async def work(_): + events.append("native work") + return 17 + + if credential: + with pytest.raises(_ch.OrgNotHeld, match="LOOM_CELL_ROUTER_TOKEN"): + _ch.run_connected(work, orgs=("org",)) + assert events == ["native client"] + else: + assert _ch.run_connected(work, orgs=("org",)) == 17 + assert events == ["native client", "native held-org guard", "native work"] + + +@pytest.mark.parametrize( + "fault", + [ + "", + "api_configuration", + "operator_token", + "arbitrary_setting", + "legacy_field", + "duplicate", + "schema_type", + "pg_port_type", + ], +) +def test_receipt_orchestration_refuses_before_clients_and_keeps_the_real_native_guard( + monkeypatch, tmp_path, capsys, fault +): + import sys + + import server.wiring + + fixture, config = fixture_config() + fixture, launch_path, _, _, _ = launch_files(tmp_path, fixture, config) + manifest = dataclasses.asdict(fixture) + if fault == "api_configuration": + config.cell_router_token = "lcr_" + "0" * 128 + elif fault == "operator_token": + config.router_operator_token = "lro_" + "1" * 128 + elif fault == "arbitrary_setting": + config.instance_heartbeat_seconds += 1 + manifest["cli_configuration_sha256"] = gate._digest(config.model_dump(mode="json")) + elif fault == "legacy_field": + manifest["configuration_sha256"] = fixture.cli_configuration_sha256 + elif fault == "schema_type": + manifest["schema"] = 2.0 + elif fault == "pg_port_type": + manifest["pg_port"] = float(manifest["pg_port"]) + raw = json.dumps(manifest) + if fault == "duplicate": + raw = raw[:-1] + ', "schema":2}' + manifest_path = tmp_path / "fixture.json" + manifest_path.write_text(raw) + output = tmp_path / "output.jsonl" + argv = ["driver"] + for key, value in vars( + args( + launch_manifest=str(launch_path), + fixture_manifest=str(manifest_path), + output=str(output), + ) + ).items(): + argv += ["--" + key.replace("_", "-"), str(value)] + events = [] + native_guard_edges(monkeypatch, config, events) + monkeypatch.setattr(sys, "argv", argv) + monkeypatch.setattr(gate, "_source", lambda _: {"clean": True}) + monkeypatch.setattr(gate, "_owned_server", lambda _: None) + monkeypatch.setattr( + gate, "_preflight", AsyncMock(side_effect=lambda *_: events.append("readonly")) + ) + monkeypatch.setattr(server.wiring, "wire", lambda: events.append("wire")) + summary = { + "latency_within_limit": True, + "throughput_within_limit": True, + "p99_delta_ms": 0.0, + "throughput_loss": 0.0, + } + + async def run(*_): + events.append("native work") + return summary + + monkeypatch.setattr(gate, "_run", run) + assert gate.main() == int(bool(fault)) + text = capsys.readouterr().out + assert json.loads(text)["verdict"] == ("INVALID" if fault else "PASS") + assert all( + value not in text + for value in [ + args().pg_dsn, + "secret-token", + "test-only-root", + "lcr_" + "0" * 128, + "lro_" + "1" * 128, + ] + ) + if fault: + assert events == [] and not output.exists() + else: + assert events == [ + "readonly", + "wire", + "native client", + "native held-org guard", + "native work", + ] + record = json.loads(output.read_text().splitlines()[0]) + assert record["manifest"]["schema"] == 2 + assert record["shared_configuration_excludes"] == ["cell_router_token"] + assert record["credential_posture"] == receipt_posture() + + +def test_unversioned_single_configuration_fixture_is_no_longer_accepted(tmp_path): + legacy = { + "base_url": args().base_url, + "pg_host": "127.0.0.1", + "pg_port": 15790, + "pg_database": "gate_cell", + "ch_host": "127.0.0.1", + "ch_port": 18790, + "ch_database": "gate_cell", + "ch_telemetry_database": "gate_cell_t", + "cell": "gate-cell", + "org": "org", + "namespace": "ns", + "deployment_id": "deployment", + "epoch": 1, + "instance_id": "instance", + "build_commit": "a" * 40, + "source_tree": "b" * 40, + "launch_manifest_sha256": "c" * 64, + "ownership_receipt_sha256": "d" * 64, + "configuration_sha256": "e" * 64, + "background_workers": "off", + "controller_excluded": True, + "pg_internal_port": 5432, + } + path = tmp_path / "legacy.json" + raw = json.dumps(legacy).encode() + path.write_bytes(raw) + with pytest.raises(gate.Invalid): + gate.Fixture.read(str(path)) + assert path.read_bytes() == raw + + +@pytest.mark.parametrize("field", ["pg_port", "ch_port", "epoch"]) +@pytest.mark.parametrize("kind", [bool, float, str]) +def test_fixture_integer_fields_never_coerce_json_types(field, kind): + fixture, config = fixture_config() + with pytest.raises(gate.Invalid, match="fixture field types"): + dataclasses.replace(fixture, **{field: kind(getattr(fixture, field))}).validate( + args(), config + ) diff --git a/tests/unit/test_cell.py b/tests/unit/test_cell.py index dcd2536ac4..624798a45a 100644 --- a/tests/unit/test_cell.py +++ b/tests/unit/test_cell.py @@ -7,13 +7,14 @@ import json import time from unittest.mock import AsyncMock, call +from uuid import UUID import pytest from pydantic import ValidationError from server import boot, worker_main from server.auth import jwt -from server.core import cell, cell_probe, instances, pg, pg_profile +from server.core import cell, cell_probe, instances, pg, pg_profile, provider_process from server.core import ch as ch_mod from server.settings import Settings, settings from server.transport.model_maintenance import ModelMaintenanceMiddleware @@ -329,6 +330,8 @@ def test_an_on_process_outside_a_cell_holds_from_its_first_step( def test_the_heartbeat_reports_the_cell_and_a_gated_process_is_not_serving( monkeypatch: pytest.MonkeyPatch, ) -> None: + participant = UUID(int=7) + monkeypatch.setattr(provider_process, "process_identity", lambda: participant) monkeypatch.setattr(instances.readiness, "draining", lambda: False) monkeypatch.setattr(instances.deployment_config, "drift", lambda: []) monkeypatch.setattr(settings, "serves_traffic", True) @@ -346,6 +349,7 @@ def test_the_heartbeat_reports_the_cell_and_a_gated_process_is_not_serving( **cell.as_value(cell.mine()), "serves": True, "marked": None, + "participant_id": str(participant), } diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py new file mode 100644 index 0000000000..ce341a619e --- /dev/null +++ b/tests/unit/test_cell_attach.py @@ -0,0 +1,1482 @@ +"""`loom cell attach` (server/cli/cell_attach.py, cell_attach_beats.py, cell_attach_pg.py) on +fakes: every refusal the heartbeats, the Postgres reads and the router's answers make, the +waits and their budget, the placement batches and the rescan's watch. The real Postgres, +ClickHouse and router behaviour is tests/integration/test_cell_attach.py. CH-free.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any + +import httpx +import pytest + +from server.cli import cell_attach, cell_control, cell_scan +from server.cli import cell_attach_beats as beats +from server.cli import cell_attach_pg as apg +from server.cli.cell_ch import Refusal, Refused +from server.cli.cell_scan import Table +from server.core import cell, instances +from server.router.limits import ATTACH_BATCH, MAX_BODY_BYTES +from server.settings import settings +from tests.unit.cell_control_owner import owner as _owner +from tests.unit.clock_fake import ManualClock + +FIELDS = {**cell.DEFAULTS, "deployment_id": "dep-1"} +CURRENT = cell.as_value(FIELDS) +NEXT = cell.as_value({**FIELDS, "oidc_client_id": "next"}) + + +def _value(**keys: Any) -> dict[str, Any]: + return {**CURRENT, "serves": True, "marked": True, "router": True, "router_why": "", **keys} + + +def _beat( + i: str = "p1", + *, + cell_name: str | None = "first", + live: bool = True, + stopped: bool = False, + mode: str = "recording", + value: dict[str, Any] | None = None, + keys_fp: str = "k", +) -> beats.Beat: + return beats.Beat( + instance_id=i, + live=live, + stopped=stopped, + keys_fp=keys_fp, + last_beat="2026-01-01 00:00:00.000", + cell=cell_name, + mode=mode if cell_name is not None else "", + value=(value if value is not None else _value()) if cell_name is not None else None, + ) + + +def _judge(found: list[beats.Beat], stopped_unset: beats.StoppedUnset = "refuse") -> Any: + return beats.judge(found, "first", (CURRENT, NEXT), stopped_unset) + + +def _reasons(verdict: beats.Verdict) -> list[str]: + return [str(r) for r in verdict.refusals] + + +# -- the heartbeats -------------------------------------------------------------------- + + +def test_a_healthy_cell_passes_and_reports_its_keys() -> None: + nxt = {**_value(), **NEXT} + verdict = _judge([_beat("a"), _beat("b", value=nxt)]) + assert (verdict.refusals, verdict.booting, verdict.live, verdict.keys_fps) == ([], [], 2, {"k"}) + + +def test_a_live_process_with_loom_cell_unset_or_another_cell_refuses_at_once() -> None: + verdict = _judge([_beat("a", cell_name=""), _beat("b", cell_name="other")], "wait") + assert _reasons(verdict) == [ + "process a: runs with LOOM_CELL unset: set LOOM_CELL=first on it", + "process b: runs with LOOM_CELL=other: set LOOM_CELL=first on it", + ] + + +def test_a_stopped_unset_process_refuses_or_is_waited_on_as_asked() -> None: + stopped = [ + _beat("a", cell_name="", live=False, stopped=True), + _beat("b", cell_name=None, live=False, stopped=True), + ] + refused = _judge(stopped, "refuse") + assert [r.subject for r in refused.refusals] == ["process a", "process b"] + assert "may still be writing" in refused.refusals[0].reason + waited = _judge(stopped, "wait") + assert (waited.refusals, waited.aging) == ([], ["a", "b"]) + + +def test_a_live_instance_with_no_cell_row_is_booting() -> None: + verdict = _judge([_beat("a", cell_name=None)]) + assert (verdict.refusals, verdict.booting) == ([], ["a"]) + + +def test_a_cell_row_without_the_current_keys_refuses_even_while_booting() -> None: + old = {k: v for k, v in _value().items() if k not in ("serves", "marked", "router")} + verdict = _judge([_beat("a", value=old, mode="")]) + assert _reasons(verdict) == [ + "process a: reports no serves, marked or router key: upgrade it to this release" + ] + + +@pytest.mark.parametrize( + "keys,booting,refused", + [ + ({"marked": False}, [], "lose the tenancy marker"), + ({"marked": None}, ["a"], ""), + ({"router": False, "router_why": "a 404"}, [], "cannot read its router: a 404"), + ({"router": None}, ["a"], ""), + # A workers-only pod places no org: it needs no router credential. + ({"serves": False, "router": False}, [], ""), + ({"serves": False, "router": None}, [], ""), + ], +) +def test_marked_and_router_refuse_or_wait( + keys: dict[str, Any], booting: list[str], refused: str +) -> None: + verdict = _judge([_beat("a", value=_value(**keys))]) + assert verdict.booting == booting + assert (refused in " ".join(_reasons(verdict))) and bool(verdict.refusals) == bool(refused) + + +def test_no_tenancy_mode_yet_is_booting() -> None: + # A pod whose first cell-state read failed: mode '', marked and router null. + verdict = _judge([_beat("a", mode="", value=_value(marked=None, router=None))]) + assert (verdict.refusals, verdict.booting) == ([], ["a"]) + + +def test_a_fingerprint_on_neither_current_nor_next_refuses() -> None: + drifted = {**_value(), **cell.as_value({**FIELDS, "oidc_client_id": "drifted"})} + assert _reasons(_judge([_beat("a", value=drifted)])) == [ + "process a: runs neither loom.cell_self's current nor its next configuration" + ] + + +def test_the_final_refusals() -> None: + scim = {**_value(), **cell.as_value({**FIELDS, "scim": True})} + verdict = beats.judge( + [_beat("a", value=scim)], "first", (cell.as_value({**FIELDS, "scim": True}),), "refuse" + ) + assert [r.reason for r in beats.final_refusals(verdict)] == [ + "SCIM is on: attach with SCIM on needs the per-cell SCIM release" + ] + two = _judge([_beat("a"), _beat("b", keys_fp="other")]) + assert "root-secret fingerprints" in beats.final_refusals(two)[0].reason + assert "no live process" in beats.final_refusals(_judge([]))[0].reason + + +class _Clock: + def __init__(self) -> None: + self.now = 0.0 + + def __call__(self) -> float: + return self.now + + +@pytest.fixture +def clock(monkeypatch: pytest.MonkeyPatch) -> _Clock: + c = _Clock() + + async def _sleep(s: float) -> None: + c.now += s + + monkeypatch.setattr(apg, "_sleep", _sleep) + return c + + +@pytest.mark.asyncio +async def test_settle_waits_for_a_booting_process(clock: _Clock) -> None: + reads = [[_beat("a", cell_name=None)], [_beat("a", cell_name=None)], [_beat("a")]] + + async def read() -> list[beats.Beat]: + return reads.pop(0) + + wait = apg.Wait(10, clock) + verdict = await beats.settle(read, _judge, wait) + assert (verdict.live, wait.spent) == (1, 2.0) + + +@pytest.mark.asyncio +async def test_settle_refuses_at_the_end_of_wait_naming_what_it_waited_on(clock: _Clock) -> None: + async def read() -> list[beats.Beat]: + return [_beat("a", cell_name=None), _beat("b", cell_name="", live=False, stopped=True)] + + with pytest.raises(Refused) as e: + await beats.settle(read, lambda f: _judge(f, "wait"), apg.Wait(3, clock)) + assert [str(r) for r in e.value.refusals] == [ + "process a: still booting at the end of --wait", + "process b: stopped with LOOM_CELL unset, not aged out within --wait", + ] + + +@pytest.mark.asyncio +async def test_settle_extends_once_for_a_process_first_seen_booting(clock: _Clock) -> None: + seen = 0 + + async def read() -> list[beats.Beat]: + nonlocal seen + seen += 1 + # A stopped process aging out first; then one booting from the third read (2 s + # spent), done at the seventh, past the first deadline. + if seen < 3: + return [_beat("a"), _beat("x", cell_name="", live=False, stopped=True)] + return [_beat("a"), _beat("b", cell_name=None)] if seen < 7 else [_beat("a"), _beat("b")] + + wait = apg.Wait(3, clock) + verdict = await beats.settle(read, lambda f: _judge(f, "wait"), wait, extend_s=4) + assert (verdict.live, wait.seconds) == (2, 6) + + async def forever() -> list[beats.Beat]: + return [_beat("b", cell_name=None)] + + wait = apg.Wait(2, clock) + with pytest.raises(Refused, match="still booting"): + await beats.settle(forever, _judge, wait, extend_s=1) + # Seen booting from the start: first sight plus the extension is inside the deadline. + assert wait.seconds == 2 + + +@pytest.mark.asyncio +async def test_wait_is_wall_time_lock_tries_included_and_no_scan(clock: _Clock) -> None: + """Each mode-commit try waits its lock timeout before it times out: the tries and the + pauses between them end within --wait, and time outside the wait is never charged.""" + asked: list[str] = [] + + def lock(sql: str, *_a: Any) -> str: + if "lock_timeout" in sql: + asked.append(sql) + clock.now += int(sql.split("'")[1].removesuffix("ms")) / 1000 + raise _Timeout() + return "OK" + + conn = FakeConn( + [ + ("SET LOCAL lock_timeout", lambda *a: lock(conn.executed[-1])), + ] + ) + + async def nothing() -> None: + pass + + async def ready() -> list[str]: + return [] + + wait = apg.Wait(4, clock) + clock.now += 100 # a scan before the wait + with pytest.raises(Refused, match="not taken within --wait"): + await apg.commit_enforced( + _owner(conn), + nothing, + ready, + wait, + lambda _t: None, + lambda _m: None, + expected_cell="first", + expected_fp=(CURRENT, NEXT), + ) + assert clock.now - 100 <= 4 and wait.spent <= 4 + # A 2 s try, a 1 s pause, then a try that waits only the second left. + assert [a.split("'")[1] for a in asked] == ["2000ms", "1000ms"] + + +@pytest.mark.asyncio +async def test_the_trigger_pass_refuses_at_once_a_table_it_cannot_alter( + monkeypatch: pytest.MonkeyPatch, clock: _Clock +) -> None: + from server.core import cell_triggers + + async def planned(_c: Any) -> list[str]: + return ["scope_generations"] + + async def holders(_c: Any, _t: str) -> list[Any]: + return [] + + async def ensure_table(*_a: Any, **_kw: Any) -> Any: + return cell_triggers.Outcome("denied") + + monkeypatch.setattr(cell_triggers, "planned_tables", planned) + monkeypatch.setattr(cell_triggers, "holders", holders) + monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) + conn = FakeConn([("deadlock_timeout", 1000)]) + wait = apg.Wait(60, clock) + (refusal,) = await apg.ensure_triggers(_owner(conn), wait, lambda _t: None) + assert refusal.subject == "postgres loom.scope_generations" + assert "cannot alter it" in refusal.reason and wait.spent < 1 + + holders = iter(["pid 7", "pid 7", "pid 7", "pid 8"] + ["pid 8"] * 100) + + async def busy(*_a: Any, **_kw: Any) -> Any: + clock.now += 0.1 + return cell_triggers.Outcome("busy", next(holders)) + + monkeypatch.setattr(cell_triggers, "ensure_table", busy) + wait = apg.Wait(10, clock) + logged: list[str] = [] + (refusal,) = await apg.ensure_triggers(_owner(conn), wait, logged.append) + # The last try ran with the floor's lock wait, after --wait was all but spent. + assert "not taken within --wait" in refusal.reason + assert wait.spent <= 10 + apg.TRY_FLOOR_MS / 1000 + 1e-9 + # Said once per change of holders, not once a try. + assert logged == [ + "loom.scope_generations: its lock is busy (pid 7); retrying", + "loom.scope_generations: its lock is busy (pid 8); retrying", + ] + + +# -- the rescan's watch ---------------------------------------------------------------- + + +def test_the_prune_contract_is_an_hour() -> None: + assert instances.PRUNE_AFTER_S == 3600 + + +@pytest.mark.asyncio +async def test_the_watch_refuses_every_unset_process_it_saw( + monkeypatch: pytest.MonkeyPatch, +) -> None: + rounds = [ + [ + _beat("a", cell_name=""), + _beat("c"), + _beat("d", cell_name=None, live=False, stopped=True), + ], + # a's rows were swept; c's cell row alone was swept: it was seen with the cell. + [_beat("c", cell_name=None, live=False, stopped=True)], + ] + + async def read_beats(_ch: Any, *, since: str | None = None) -> list[beats.Beat]: + assert since == "2026-01-01 00:00:00.000" + return rounds.pop(0) + + monkeypatch.setattr(beats, "read_beats", read_beats) + clock = _Clock() + watch = beats.Watch(None, "2026-01-01 00:00:00.000", "first", clock) + await watch.read_once() + await watch.read_once() + assert [r.subject for r in watch.refusals()] == ["process a", "process d"] + clock.now += beats.watch_period_s() + 1 + monkeypatch.setattr(beats, "read_beats", read_beats) + rounds.append([]) + await watch.read_once() + assert watch.refusals()[-1].subject == "heartbeats" + + +def test_the_watch_refuses_up_front_with_a_very_long_heartbeat( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(settings, "instance_heartbeat_seconds", 1180) + assert beats.prune_window_refusals() + monkeypatch.setattr(settings, "instance_heartbeat_seconds", 10) + assert beats.prune_window_refusals() == [] + + +# -- batches, ordering, arguments -------------------------------------------------------- + + +def test_batches_keep_the_count_and_the_byte_bound() -> None: + many = [f"org-{i:05d}" for i in range(ATTACH_BATCH + 1)] + assert [len(b) for b in cell_attach.batches(many, "dep")] == [ATTACH_BATCH, 1] + # Escaping grows a non-ASCII name six times: the encoded body is what counts. + wide = ["é" * 2000 + str(i) for i in range(10)] + out = cell_attach.batches(wide, "dep") + assert len(out) > 1 and sum(map(len, out)) == 10 + for b in out: + body = json.dumps({"deployment_id": "dep", "orgs": b}, separators=(",", ":")) + assert len(body.encode()) <= MAX_BODY_BYTES + + +def test_move_sources_are_scanned_first() -> None: + tables = [Table("loom", n, "MergeTree") for n in ("a", "memory_extraction_queue", "b", "src")] + ordered = cell_scan.order_sources(tables, {"loom.src"}) + assert [t.name for t in ordered] == ["memory_extraction_queue", "src", "a", "b"] + + +def _args(**over: Any) -> argparse.Namespace: + fields: dict[str, Any] = { + "cell": "first", + "wait": 10, + "scan_budget": 100, + "dry_run": False, + "ch_allow_user": [], + "ch_allow_user_directory": [], + "ch_allow_object": [], + "ch_allow_replica": [], + "pg_allow_role": [], + "router": "http://127.0.0.1:7900", + } + return argparse.Namespace(**{**fields, **over}) + + +def test_the_arguments(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "storage_authority", "postgres") + monkeypatch.setattr(settings, "pg_dsn", "postgresql://cell.internal/loom") + assert cell_attach._args_problem(_args()) == "" + assert "LOOM_CELL" in cell_attach._args_problem(_args(cell="other")) + assert "--scan-budget" in cell_attach._args_problem(_args(scan_budget=0)) + monkeypatch.setattr(settings, "storage_authority", "clickhouse") + assert "Postgres" in cell_attach._args_problem(_args()) + + +def test_an_org_past_the_limit_is_named_with_its_sources() -> None: + found = cell_scan.ScanResult() + found.add("postgres loom.tenancy", "x" * (cell.ORG_MAX_BYTES + 1)) + (refusal,) = cell_attach._too_long(found) + assert refusal.subject.startswith(f"org of {cell.ORG_MAX_BYTES + 1} bytes") + assert "found in postgres loom.tenancy" in refusal.reason + + +# -- the router ------------------------------------------------------------------------ + + +def _router(*responses: httpx.Response) -> cell_attach.Router: + answers = list(responses) + + def handler(_request: httpx.Request) -> httpx.Response: + return answers.pop(0) + + return cell_attach.Router( + httpx.URL("http://127.0.0.1:7900"), "first", httpx.MockTransport(handler) + ) + + +@pytest.mark.asyncio +async def test_the_router_calls_refuse_as_the_router_answers() -> None: + ok = {"cell": "first", "state": "attaching"} + assert await _router(httpx.Response(200, json=ok)).call("") == ok + with pytest.raises(Refused, match="not upgraded"): + await _router(httpx.Response(404, json={"error": "no route"})).call("/attach/place", {}) + with pytest.raises(Refused, match=r"refused /attach/begin \(409\): use loom cell update"): + await _router(httpx.Response(409, json={"error": "use loom cell update"})).call( + "/attach/begin", {} + ) + with pytest.raises(Refused, match="did not answer as the router does"): + await _router(httpx.Response(502, text="bad gateway")).call("") + + +# -- the Postgres reads, on a fake session --------------------------------------------- + + +@dataclass +class FakeConn: + """Answers by the first needle the SQL holds; a callable answer is called with the + arguments; an exception answer is raised.""" + + routes: list[tuple[str, Any]] = field(default_factory=list) + executed: list[str] = field(default_factory=list) + + def _answer(self, sql: str, args: tuple[Any, ...]) -> Any: + for needle, answer in self.routes: + if needle in sql: + if isinstance(answer, BaseException): + raise answer + return answer(*args) if callable(answer) else answer + if sql.startswith("SELECT (SELECT is_first FROM loom.cell_first)"): + return { + "is_first": True, + "cell": "first", + "mode": "recording", + "has_own": True, + "has_tenancy": True, + } + if sql == apg._HELD: + return True + if sql == "SELECT mode FROM loom.tenancy_mode": + return "recording" + if sql == "SELECT is_first, cell FROM loom.cell_first": + return {"is_first": True, "cell": "first"} + if sql == "SELECT current_fp, next_fp FROM loom.cell_self": + return {"current_fp": CURRENT, "next_fp": NEXT} + raise AssertionError(f"unrouted: {sql[:80]}") + + async def fetch(self, sql: str, *args: Any) -> Any: + return self._answer(sql, args) + + async def fetchrow(self, sql: str, *args: Any) -> Any: + return self._answer(sql, args) + + async def fetchval(self, sql: str, *args: Any) -> Any: + return self._answer(sql, args) + + async def execute(self, sql: str, *args: Any) -> str: + assert "pg_terminate_backend" not in sql and "pg_cancel_backend" not in sql + self.executed.append(sql) + for needle, answer in self.routes: + if needle in sql: + if isinstance(answer, BaseException): + raise answer + return answer(*args) if callable(answer) else str(answer) + return "OK" + + def transaction(self) -> Any: + conn = self + + class _Tx: + async def start(self) -> None: + await conn.execute("BEGIN") + + async def commit(self) -> None: + await conn.execute("COMMIT") + + async def rollback(self) -> None: + await conn.execute("ROLLBACK") + + async def __aenter__(self) -> None: + await self.start() + + async def __aexit__(self, *exc: Any) -> None: + if exc[0]: + await self.rollback() + else: + await self.commit() + + return _Tx() + + def terminate(self) -> None: + self.executed.append("TERMINATE") + + async def close(self, **kwargs: Any) -> None: + assert kwargs["timeout"] > 0 + self.executed.append("CLOSE") + + def is_closed(self) -> bool: + return bool(self.executed and self.executed[-1] in {"CLOSE", "TERMINATE"}) + + +def _short_phases(monkeypatch: pytest.MonkeyPatch, bounds: list[float] | None = None) -> None: + # The work timer stays real; scheduling cannot consume the fake close budget. + phase_clock = ManualClock() + monkeypatch.setattr(cell_control, "time", phase_clock) + monkeypatch.setattr(cell_control.pg, "time", phase_clock) + supervise = cell_control.Owner.call + + async def bounded(self: Any, work: Any, *, total_s: float) -> Any: + if bounds is not None: + bounds.append(total_s) + return await supervise(self, work, total_s=0.01) + + monkeypatch.setattr(cell_control.Owner, "call", bounded) + + +def _seq(*values: Any) -> Callable[..., Any]: + pending = list(values) + return lambda *_a: pending.pop(0) + + +@pytest.mark.asyncio +async def test_the_session_checks_refuse_a_pooler() -> None: + moved = FakeConn( + [ + ("pg_backend_pid() AS pid", _seq((1, "t0"), (2, "t1"))), + ("current_setting($1, true)", None), + ] + ) + refusals = await apg.direct_refusals(moved, "attach") + assert [r.reason.split(":")[0] for r in refusals] == [ + "attach's session moved between backends", + "attach's session loses loom.tenancy_checked", + ] + direct = FakeConn([("pg_backend_pid() AS pid", (1, "t0")), ("current_setting($1, true)", "on")]) + assert await apg.direct_refusals(direct, "attach") == [] + assert "RESET ALL" in direct.executed + + +@pytest.mark.asyncio +async def test_the_lock_refuses_when_held_or_not_on_this_backend() -> None: + holder = [{"pid": 7, "application_name": "loom", "client": "10.0.0.1", "backend_start": "t"}] + held = FakeConn([("pg_try_advisory_lock", False), ("FROM pg_locks l LEFT JOIN", holder)]) + with pytest.raises(Refused, match="held by pid 7 \\(loom, from 10.0.0.1"): + await apg.take_lock(held, "attach") + pooled = FakeConn([("pg_try_advisory_lock", True), ("SELECT EXISTS", False)]) + with pytest.raises(Refused, match="a pooler"): + await apg.take_lock(pooled, "attach") + assert any("tcp_keepalives_idle" in s for s in pooled.executed) + + +@pytest.mark.asyncio +async def test_a_database_not_utf8_refuses() -> None: + assert await apg.encoding_refusals(FakeConn([("pg_encoding_to_char", "UTF8")])) == [] + (r,) = await apg.encoding_refusals(FakeConn([("pg_encoding_to_char", "SQL_ASCII")])) + assert "SQL_ASCII, not UTF8" in r.reason + + +def test_privilege_holders_outside_the_allowed_refuse() -> None: + holders = apg.Holders( + own=["loom"], + others={"ci": ["may insert into loom.tenancy"], "servers": ["owns loom.memory_current"]}, + ) + assert [r.subject for r in apg.privilege_refusals(holders, ["servers"])] == ["postgres role ci"] + assert apg.privilege_refusals(holders, ["ci", "servers"]) == [] + + +@pytest.mark.asyncio +async def test_settings_and_subscriptions_refuse() -> None: + conn = FakeConn( + [ + ( + "pg_db_role_setting", + [{"role": "ci", "db": "every database", "cfg": "Loom.Tenancy_Checked=on"}], + ), + ("pg_subscription", [{"subname": "mirror", "subenabled": False}]), + ] + ) + refusals = await apg.unmarked_refusals(conn, True) + assert [r.subject for r in refusals] == [ + "postgres unmarked session", + "postgres setting for ci in every database", + "postgres subscription mirror", + ] + + +@pytest.mark.asyncio +async def test_own_role_lacks_and_prepared_transactions_refuse() -> None: + conn = FakeConn( + [ + ("SELECT current_user", "attacher"), + ("WHERE NOT held", [{"need": "effective privileges of pg_read_all_stats"}]), + ("has_table_privilege($1, 'SELECT')", lambda t: t != "loom.memory_current"), + ("max_prepared_transactions", 5), + ] + ) + tables = [ + {"relname": "memory_current", "relkind": "r", "owned": True}, + # Its grants are held, but not its effective owner-role privileges: the pass could + # not alter it, so it refuses before any write. + {"relname": "scope_generations", "relkind": "r", "owned": False}, + {"relname": "tenancy", "relkind": "r", "owned": True}, + ] + refusals = await apg.own_role_refusals(conn, tables) + assert [str(r) for r in refusals] == [ + "postgres role attacher: attach's role lacks effective privileges of pg_read_all_stats", + "postgres role attacher: attach's role lacks SELECT on loom.memory_current", + "postgres role attacher: attach's role lacks effective privileges of the owner role of " + "loom.scope_generations", + "postgres max_prepared_transactions: is 5: a prepared transaction outlives every " + "session, so step 4 could not wait for it; set it to 0", + ] + + +def _row(**over: Any) -> dict[str, Any]: + return { + "pid": 9, + "backend_type": "client backend", + "application_name": "ci", + "xact_start": "t", + "ours": True, + "xid_old": False, + "xmin_old": False, + "hidden": False, + **over, + } + + +@pytest.mark.asyncio +async def test_step_4_refuses_what_it_cannot_wait_for(clock: _Clock) -> None: + base = [("max_prepared_transactions", 0), ("pg_prepared_xacts", 0)] + hidden = FakeConn([*base, ("pg_stat_activity s", [_row(hidden=True)])]) + with pytest.raises(Refused, match="hidden from attach's role"): + await apg.wait_transactions(hidden, 100, apg.Wait(5, clock)) + prepared = FakeConn([("max_prepared_transactions", 0), ("pg_prepared_xacts", 1)]) + with pytest.raises(Refused, match="prepared transaction"): + await apg.wait_transactions(prepared, 100, apg.Wait(5, clock)) + held = FakeConn( + [ + *base, + ( + "pg_stat_activity s", + [_row(xmin_old=True), _row(pid=10, ours=False, xid_old=True), _row(pid=11)], + ), + ] + ) + with pytest.raises(Refused) as e: + await apg.wait_transactions(held, 100, apg.Wait(2, clock)) + assert [r.reason for r in e.value.refusals] == [ + "may have read `recording`: still open", + "another database's transaction, holding the snapshot horizon", + ] + done = FakeConn([*base, ("pg_stat_activity s", [_row(pid=10, ours=False, xid_old=True)])]) + await apg.wait_transactions(done, 100, apg.Wait(0, clock)) + + +class _Timeout(Exception): + sqlstate = "55P03" + + +@pytest.mark.asyncio +async def test_the_mode_commit_retries_a_busy_lock_without_ending_other_backends( + clock: _Clock, +) -> None: + tries = _seq(_Timeout(), "UPDATE 1") + + def lock(*_a: Any) -> str: + answer = tries() + if isinstance(answer, Exception): + raise answer + return "LOCK TABLE" + + conn = FakeConn( + [ + ("LOCK TABLE", lock), + ("UPDATE loom.tenancy_mode", "UPDATE 1"), + ] + ) + confirmed: list[bool] = [] + + async def confirm() -> None: + confirmed.append(True) + + async def ready() -> list[str]: + return [] + + logged: list[str] = [] + await apg.commit_enforced( + _owner(conn), + confirm, + ready, + apg.Wait(10, clock), + logged.append, + lambda _m: None, + expected_cell="first", + expected_fp=(CURRENT, NEXT), + ) + assert logged == ["the mode's lock is busy; retrying"] + # Before each try, with no termination of a different backend. + assert len(confirmed) == 2 + assert conn.executed.count("ROLLBACK") == 1 + assert "TERMINATE" not in conn.executed + assert not any( + "pg_terminate_backend" in sql or "pg_cancel_backend" in sql for sql in conn.executed + ) + assert conn.executed.count("COMMIT") == 1 + assert sum(sql.startswith("UPDATE loom.tenancy_mode") for sql in conn.executed) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("wait_s", [0, 2]) +async def test_the_mode_commit_refuses_past_wait_and_skips_a_try_while_booting( + clock: _Clock, + monkeypatch: pytest.MonkeyPatch, + wait_s: int, +) -> None: + conn = FakeConn() + run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + run.owner = _owner(FakeConn()) + run.own_fp = (CURRENT, NEXT) + + async def read(_ch: Any) -> list[beats.Beat]: + return [_beat("a", cell_name=None), _beat("b", cell_name=None)] + + monkeypatch.setattr(beats, "read_beats", read) + + async def confirm() -> None: + pass + + with pytest.raises(Refused) as error: + await apg.commit_enforced( + _owner(conn), + confirm, + run.ready_for_commit, + apg.Wait(wait_s, clock), + lambda _t: None, + lambda _m: None, + expected_cell="first", + expected_fp=(CURRENT, NEXT), + ) + assert [str(r) for r in error.value.refusals] == [ + "process a: still booting at the end of --wait", + "process b: still booting at the end of --wait", + ] + assert not any("LOCK TABLE" in s for s in conn.executed) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("wait_s,bound_s", [(0, 8.1), (10, 10.0)]) +async def test_the_mode_try_bounds_a_commit_that_never_answers( + monkeypatch: pytest.MonkeyPatch, clock: _Clock, wait_s: int, bound_s: float +) -> None: + class _StalledCommit(FakeConn): + async def execute(self, sql: str, *args: Any) -> str: + result = await super().execute(sql, *args) + if sql == "COMMIT": + await asyncio.Future() + return result + + conn = _StalledCommit([("UPDATE loom.tenancy_mode", "UPDATE 1")]) + bounds: list[float] = [] + _short_phases(monkeypatch, bounds) + marks: list[str] = [] + + async def confirm() -> None: + pass + + async def ready() -> list[str]: + return [] + + with pytest.raises(TimeoutError): + await asyncio.wait_for( + apg.commit_enforced( + _owner(conn), + confirm, + ready, + apg.Wait(wait_s, clock), + lambda _t: None, + marks.append, + expected_cell="first", + expected_fp=(CURRENT, NEXT), + ), + timeout=0.2, + ) + assert "COMMIT" in conn.executed + assert conn.executed[-1] == "CLOSE" + assert "TERMINATE" not in conn.executed + assert bounds == pytest.approx([bound_s], abs=0.001) + assert marks == ["unknown"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stall", ["LOCK TABLE", "UPDATE", "ROLLBACK"]) +async def test_a_mode_deadline_never_waits_for_rollback( + monkeypatch: pytest.MonkeyPatch, stall: str +) -> None: + class _StalledRollback(FakeConn): + async def execute(self, sql: str, *args: Any) -> str: + result = await super().execute(sql, *args) + if stall == "ROLLBACK" and sql.startswith("LOCK TABLE"): + raise _Timeout() + if sql.startswith(stall) or sql == "ROLLBACK": + await asyncio.Future() + return result + + conn = _StalledRollback() + _short_phases(monkeypatch) + marks: list[str] = [] + task = asyncio.create_task( + apg._mode_try( + _owner(conn), 100, marks.append, expected_cell="first", expected_fp=(CURRENT, NEXT) + ) + ) + done, _ = await asyncio.wait([task], timeout=0.2) + if not done: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert task in done, "mode deadline is stuck awaiting rollback" + assert isinstance(task.exception(), TimeoutError) + assert conn.executed[-1] == "CLOSE" + assert "TERMINATE" not in conn.executed + assert marks == (["unknown"] if stall == "UPDATE" else []) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed_read", [1, 2]) +@pytest.mark.parametrize("failure", ["cancel", "error"]) +async def test_a_failed_direct_session_read_never_waits_for_rollback( + failed_read: int, failure: str +) -> None: + class _StalledRollback(FakeConn): + reads = 0 + + async def fetchrow(self, sql: str, *args: Any) -> Any: + self.reads += 1 + if self.reads == failed_read: + if failure == "error": + raise ValueError("identity read failed") + await asyncio.Future() + return (1, "t0") + + async def execute(self, sql: str, *args: Any) -> str: + result = await super().execute(sql, *args) + if sql == "ROLLBACK": + await asyncio.Future() + return result + + conn = _StalledRollback() + task = asyncio.create_task(apg.direct_refusals(conn, "attach")) + await asyncio.sleep(0) + if failure == "cancel": + task.cancel() + done, _ = await asyncio.wait([task], timeout=0.2) + if not done: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert task in done, "direct-session failure is stuck awaiting rollback" + if failure == "cancel": + assert task.cancelled() + else: + assert isinstance(task.exception(), ValueError) + assert "CLOSE" not in conn.executed + assert "TERMINATE" not in conn.executed + assert "ROLLBACK" not in conn.executed + + +@pytest.mark.asyncio +async def test_an_interrupt_waiting_for_the_mode_lock_leaves_recording(clock: _Clock) -> None: + conn = FakeConn([("LOCK TABLE", KeyboardInterrupt())]) + marks: list[str] = [] + + async def confirm() -> None: + pass + + async def ready() -> list[str]: + return [] + + with pytest.raises(cell_control._ForwardFatal) as fatal: + await apg.commit_enforced( + _owner(conn), + confirm, + ready, + apg.Wait(2, clock), + lambda _t: None, + marks.append, + expected_cell="first", + expected_fp=(CURRENT, NEXT), + ) + assert isinstance(fatal.value.fatal.error, KeyboardInterrupt) + assert conn.executed[-1] == "CLOSE" + assert "TERMINATE" not in conn.executed + assert marks == [] + + +@pytest.mark.asyncio +async def test_the_durable_read_must_see_enforced_and_x_above_its_xid( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from server.core import pg_profile + + async def synchronous() -> str: + return "synchronous" + + monkeypatch.setattr(pg_profile, "standby_now", synchronous) + run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + run.owner = _owner(FakeConn()) + run.lock_conn = FakeConn([("pg_snapshot_xmax", "100")]) + + async def read(mode: str, xid: int) -> apg.Durable: + return apg.Durable(mode=mode, first=(True, "first"), xid=xid) + + monkeypatch.setattr(apg, "durable_read", lambda _conn: read("recording", 50)) + with pytest.raises(Refused, match="did not land"): + await run.durable() + monkeypatch.setattr(apg, "durable_read", lambda _conn: read("enforced", 100)) + with pytest.raises(Refused, match="is not above 100"): + await run.durable() + monkeypatch.setattr(apg, "durable_read", lambda _conn: read("enforced", 99)) + assert await run.durable() == 100 + + async def waiting() -> str: + return "waiting" + + monkeypatch.setattr(pg_profile, "standby_now", waiting) + with pytest.raises(Refused, match="not synchronous before"): + await run.durable() + + +@pytest.mark.asyncio +async def test_a_serving_row_is_done_only_once_the_cell_is_enforced( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from contextlib import asynccontextmanager + + from server.core import pg + + state = ["enforced"] + conn = FakeConn([("SELECT deployment_id FROM loom.authority", "dep-1")]) + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "document_s3_bucket", "") + monkeypatch.setattr(settings, "document_s3_prefix", "") + + @asynccontextmanager + async def dedicated(**_kw: Any) -> Any: + yield conn + + async def read_mode(_conn: Any) -> tuple[str, Any]: + return state[0], (True, "first") + + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(apg, "read_mode", read_mode) + run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + run.owner = _owner(FakeConn()) + run.row = {"deployment_id": "dep-1", "cell": "first", "bucket": "", "bucket_prefix": ""} + assert await run.done() + state[0] = "recording" + with pytest.raises(Refused, match="attach lowers and repairs neither"): + await run.done() + + +def test_refusal_text() -> None: + assert str(Refusal("a", "b")) == "a: b" + + +@pytest.mark.asyncio +async def test_a_root_secret_change_during_attach_refuses_at_the_end( + monkeypatch: pytest.MonkeyPatch, +) -> None: + run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + run.owner = _owner(FakeConn()) + run.begun_keys_fp = "before" + + async def settle(*_a: Any, **_kw: Any) -> beats.Verdict: + return beats.Verdict(live=1, keys_fps={"after"}) + + monkeypatch.setattr(beats, "settle", settle) + with pytest.raises(Refused, match="a root-secret rotation"): + await run.final_checks(beats.Watch(None, "", "first")) + run.begun_keys_fp = "after" + await run.final_checks(beats.Watch(None, "", "first")) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", [Refused([Refusal("postgres", "cut")]), OSError("gone")]) +async def test_a_stop_between_a_landed_commit_and_the_durable_read_says_the_outcome_is_unknown( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], failure: Exception +) -> None: + from contextlib import asynccontextmanager + + from server.core import pg + + @asynccontextmanager + async def dedicated(**_kw: Any) -> Any: + yield None + + async def landed(*a: Any, **_kw: Any) -> None: + a[-1]("unknown") # the try sent its UPDATE; the commit landed, its answer may not have + + async def call(_self: Any, _path: str, _body: Any = None) -> dict[str, Any]: + return {"cell": "first"} + + async def run(self: Any) -> int: + self.owner = _owner(FakeConn()) + self.row = {"deployment_id": "dep-1"} + await self.commit_mode() + raise failure # what stops the run before the durable read finishes + + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(apg, "commit_enforced", landed) + monkeypatch.setattr(cell_attach.Router, "call", call) + monkeypatch.setattr(cell_attach.Run, "mutate", call) + monkeypatch.setattr(cell_attach.Run, "run", run) + url = httpx.URL("http://127.0.0.1:7900") + assert await cell_attach.attach(None, _args(), url) == 1 + err = capsys.readouterr().err + assert "the mode commit's outcome is unknown: rerun attach" in err + assert "stays recording" not in err + + +@pytest.mark.asyncio +async def test_doctor_names_both_catalog_checks_when_the_catalog_is_unread( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from server.cli import doctor_cell + + async def before(_ch: Any) -> str: + return "" + + async def marked() -> tuple[str, bool, list[Any]]: + return "enforced", True, [] + + async def unmarked() -> bool: + return False + + async def unread(_roles: Any) -> Any: + raise OSError("the catalog read failed") + + monkeypatch.setattr(doctor_cell, "_deployment_check", before) + monkeypatch.setattr(doctor_cell, "_marked_session", marked) + monkeypatch.setattr(doctor_cell, "_unmarked_session", unmarked) + monkeypatch.setattr(doctor_cell, "_writer_reads", unread) + lines = {n: (g, d) for n, g, d in await doctor_cell._checks(None)} + for name in ("cell_tenancy_privilege", "cell_unchecked_writers"): + assert lines[name] == ("WARN", "not read: OSError: the catalog read failed") + + +@pytest.mark.asyncio +async def test_the_ddl_lock_bound_is_taken_after_the_migration_lock( + monkeypatch: pytest.MonkeyPatch, clock: _Clock +) -> None: + """A migration lock that took the rest of --wait leaves the DDL its floor: one try, never + the full 100 ms when less is left, never none.""" + from server.core import cell_triggers + + def migration_lock(*_a: Any) -> str: + clock.now += 1.95 # all but 50 ms of --wait + return "OK" + + asked: list[str] = [] + + async def ensure_table(*_a: Any, lock_timeout: str, **_kw: Any) -> Any: + asked.append(lock_timeout) + return cell_triggers.Outcome("applied") + + monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) + conn = FakeConn([("pg_advisory_lock", migration_lock), ("pg_advisory_unlock", "OK")]) + wait = apg.Wait(2, clock) + with wait.counting(): + outcome = await apg._ensure_one(_owner(conn), "scope_generations", 500, wait) + assert outcome.answer == "applied" + assert asked == [f"{apg.TRY_FLOOR_MS}ms"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stall", ["migration", "reset", "ddl", "commit", "rollback", "unlock"]) +async def test_a_trigger_try_deadline_covers_transaction_and_lock_cleanup( + monkeypatch: pytest.MonkeyPatch, clock: _Clock, stall: str +) -> None: + import asyncpg + + from server.core import cell_triggers + + class _Stalled(FakeConn): + async def execute(self, sql: str, *args: Any) -> str: + result = await super().execute(sql, *args) + if stall == "rollback" and sql == "CREATE TRIGGER stub": + raise asyncpg.LockNotAvailableError("busy") + needles = { + "migration": "SELECT pg_advisory_lock", + "reset": "RESET lock_timeout", + "ddl": "CREATE TRIGGER stub", + "commit": "COMMIT", + "rollback": "ROLLBACK", + "unlock": "SELECT pg_advisory_unlock", + } + if sql.startswith(needles[stall]) or (stall == "ddl" and sql == "ROLLBACK"): + await asyncio.Future() + return result + + async def ensure_table(conn: Any, table: str, **kwargs: Any) -> Any: + return await cell_triggers._run_ddl( + conn, table, ["CREATE TRIGGER stub"], kwargs["lock_timeout"], 1, warn=False + ) + + _short_phases(monkeypatch) + monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) + conn = _Stalled() + task = asyncio.create_task( + apg._ensure_one(_owner(conn), "scope_generations", 100, apg.Wait(0, clock)) + ) + done, _ = await asyncio.wait([task], timeout=0.2) + try: + assert task in done, f"trigger deadline is stuck at {stall}" + assert isinstance(task.exception(), TimeoutError) + assert conn.executed[-1] == "CLOSE" + assert "TERMINATE" not in conn.executed + if stall in {"migration", "ddl", "commit", "rollback"}: + assert apg._MIGRATION_UNLOCK not in conn.executed + finally: + for _ in range(3): + if task.done(): + break + task.cancel() + await asyncio.wait([task], timeout=0.01) + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed", ["migration", "ddl"]) +async def test_a_failed_trigger_try_does_not_mask_its_error_with_cleanup( + monkeypatch: pytest.MonkeyPatch, clock: _Clock, failed: str +) -> None: + from server.core import cell_triggers + + async def ensure_table(*_a: Any, **_kw: Any) -> Any: + raise ValueError("DDL failed") + + monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) + conn = FakeConn( + [ + ("pg_advisory_lock", ValueError("migration failed") if failed == "migration" else "OK"), + ("RESET", RuntimeError("reset failed") if failed == "migration" else "OK"), + ("pg_advisory_unlock", RuntimeError("unlock failed")), + ] + ) + with pytest.raises(ValueError, match=f"{failed if failed == 'migration' else 'DDL'} failed"): + await apg._ensure_one(_owner(conn), "scope_generations", 100, apg.Wait(0, clock)) + assert conn.executed[-1] == "CLOSE" + assert "TERMINATE" not in conn.executed + + +@pytest.mark.asyncio +async def test_a_busy_migration_lock_is_named_as_such( + monkeypatch: pytest.MonkeyPatch, clock: _Clock +) -> None: + from server.core import cell_triggers + + class _Busy(Exception): + sqlstate = "55P03" + + holder = [{"pid": 42, "application_name": "loom", "backend_start": "t0"}] + conn = FakeConn( + [ + ("pg_advisory_lock", _Busy()), + ("l.locktype = 'advisory'", holder), + ("deadlock_timeout", 1000), + ] + ) + + async def planned(_c: Any) -> list[str]: + return ["scope_generations"] + + async def no_holders(_c: Any, _t: str) -> list[Any]: + return [] + + monkeypatch.setattr(cell_triggers, "planned_tables", planned) + monkeypatch.setattr(cell_triggers, "holders", no_holders) + logged: list[str] = [] + (refusal,) = await apg.ensure_triggers(_owner(conn), apg.Wait(2, clock), logged.append) + named = "the migration lock (held by pid 42, loom, since t0)" + assert logged == [f"loom.scope_generations: its lock is busy ({named}); retrying"] + assert refusal.reason == f"its triggers were not created within --wait: {named}" + + +@pytest.mark.asyncio +async def test_wait_0_runs_one_try_of_each_wait( + clock: _Clock, monkeypatch: pytest.MonkeyPatch +) -> None: + """On an idle database --wait 0 commits the mode and creates the triggers in one try + each, and the heartbeats pass when nothing boots.""" + from server.core import cell_triggers + + conn = FakeConn( + [ + ("LOCK TABLE", "LOCK TABLE"), + ("UPDATE loom.tenancy_mode", "UPDATE 1"), + ] + ) + marks: list[str] = [] + + async def ok() -> list[str]: + return [] + + async def nothing() -> None: + pass + + await apg.commit_enforced( + _owner(conn), + nothing, + ok, + apg.Wait(0, clock), + lambda _t: None, + marks.append, + expected_cell="first", + expected_fp=(CURRENT, NEXT), + ) + assert marks == ["unknown", "enforced"] + assert any(f"lock_timeout = '{apg.TRY_FLOOR_MS}ms'" in sql for sql in conn.executed) + + async def planned(_c: Any) -> list[str]: + return ["scope_generations"] + + async def no_holders(_c: Any, _t: str) -> list[Any]: + return [] + + async def applied(*_a: Any, **_kw: Any) -> Any: + return cell_triggers.Outcome("applied") + + async def no_defects(_c: Any) -> list[Any]: + return [] + + monkeypatch.setattr(cell_triggers, "planned_tables", planned) + monkeypatch.setattr(cell_triggers, "holders", no_holders) + monkeypatch.setattr(cell_triggers, "ensure_table", applied) + monkeypatch.setattr(cell_triggers, "missing", no_defects) + ddl = FakeConn([("deadlock_timeout", 1000)]) + assert await apg.ensure_triggers(_owner(ddl), apg.Wait(0, clock), lambda _t: None) == [] + + async def read() -> list[beats.Beat]: + return [_beat("a")] + + assert (await beats.settle(read, _judge, apg.Wait(0, clock))).live == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("refused_before_any_try", [True, False]) +async def test_a_mode_commit_refused_without_a_landed_try_says_recording( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + clock: _Clock, + refused_before_any_try: bool, +) -> None: + """The closing line says recording when no try sent its UPDATE (the heartbeats refused + before one) and when every try timed out and rolled back.""" + + def lock(*_a: Any) -> str: + raise _Timeout() + + conn = FakeConn([("LOCK TABLE", lock)]) + + async def call(_self: Any, _path: str, _body: Any = None) -> dict[str, Any]: + return {"cell": "first"} + + async def ready(_self: Any) -> list[str]: + if refused_before_any_try: + raise Refused([Refusal("process p1", "runs with LOOM_CELL unset")]) + return [] + + async def confirm(_self: Any) -> None: + pass + + async def run(self: Any) -> int: + self.owner = _owner(FakeConn()) + self.row, self.mode = {"deployment_id": "dep-1"}, "recording" + self.lock_conn = conn + self.owner = _owner(conn) + self.wait = apg.Wait(2, clock) + await self.commit_mode() + return 0 + + from contextlib import asynccontextmanager + + from server.core import pg + + @asynccontextmanager + async def dedicated(**_kw: Any) -> Any: + yield conn + + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(cell_attach.Router, "call", call) + monkeypatch.setattr(cell_attach.Run, "mutate", call) + monkeypatch.setattr(cell_attach.Run, "ready_for_commit", ready) + monkeypatch.setattr(cell_attach.Run, "confirm_lock", confirm) + monkeypatch.setattr(cell_attach.Run, "run", run) + assert await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) == 1 + assert capsys.readouterr().err.splitlines()[-1] == "loom cell attach: the cell stays recording" + + +def _stub_attach(monkeypatch: pytest.MonkeyPatch, body: Any) -> None: + from contextlib import asynccontextmanager + + from server.core import pg + + @asynccontextmanager + async def dedicated(**_kw: Any) -> Any: + yield None + + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(cell_attach.Run, "run", body) + + +@pytest.mark.asyncio +async def test_a_router_refusal_of_attach_enforced_leaves_recording( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + async def refuse(_self: Any, path: str, _body: Any = None) -> dict[str, Any]: + raise Refused([Refusal("router", f"refused {path}")]) + + async def run(self: Any) -> int: + self.owner = _owner(FakeConn()) + self.row, self.mode = {"deployment_id": "dep-1"}, "recording" + await self.commit_mode() + return 0 + + monkeypatch.setattr(cell_attach.Router, "call", refuse) + monkeypatch.setattr(cell_attach.Run, "mutate", refuse) + _stub_attach(monkeypatch, run) + assert await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) == 1 + assert capsys.readouterr().err.splitlines()[-1] == "loom cell attach: the cell stays recording" + + +@pytest.mark.asyncio +async def test_a_durable_read_that_sees_recording_says_recording( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + from server.core import pg_profile + + async def synchronous() -> str: + return "synchronous" + + async def read(_conn: Any) -> apg.Durable: + return apg.Durable(mode="recording", first=(True, "first"), xid=1) + + async def run(self: Any) -> int: + self.owner = _owner(FakeConn()) + self.mode = "unknown" # a mode commit whose outcome was lost + await self.durable() + return 0 + + monkeypatch.setattr(pg_profile, "standby_now", synchronous) + monkeypatch.setattr(apg, "durable_read", read) + _stub_attach(monkeypatch, run) + assert await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) == 1 + assert capsys.readouterr().err.splitlines()[-1] == "loom cell attach: the cell stays recording" + + +@pytest.mark.asyncio +async def test_an_enforced_resume_refused_early_says_enforced( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + from server.core import pg_profile + + async def recorded(_ch: Any) -> dict[str, Any]: + return {"deployment_id": "other"} + + async def health(_self: Any, _host: str) -> str: + return "dep-1" + + async def read_mode(_conn: Any) -> tuple[str, Any]: + return "enforced", (True, "first") + + async def run(self: Any) -> int: + self.owner = _owner(FakeConn()) + self.row = {"is_first": True, "state": "attaching", "deployment_id": "dep-1"} + self.lock_conn = FakeConn([("FROM loom.authority", "dep-1")]) + if refusals := await self.stores_refusals(): + raise Refused(refusals) + return 0 + + monkeypatch.setattr(pg_profile, "recorded_authority", recorded) + monkeypatch.setattr(cell_attach.Router, "health", health) + monkeypatch.setattr(apg, "read_mode", read_mode) + _stub_attach(monkeypatch, run) + assert await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) == 1 + err = capsys.readouterr().err.splitlines() + assert "clickhouse: records another deployment id" in err[0] + assert err[-1] == ( + "loom cell attach: the cell stays enforced and attaching: rerun attach to resume" + ) + + +@pytest.mark.asyncio +async def test_an_interrupt_prints_the_mode_line_and_goes_on( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + async def run(self: Any) -> int: + self.owner = _owner(FakeConn()) + self.mode = "unknown" + raise KeyboardInterrupt + + _stub_attach(monkeypatch, run) + with pytest.raises(KeyboardInterrupt): + await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) + assert "the mode commit's outcome is unknown" in capsys.readouterr().err + + +@pytest.mark.asyncio +async def test_the_stray_read_names_a_refused_table_and_reads_on( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from contextlib import asynccontextmanager + + from server.cli import cell_ch, doctor_cell + from server.core import pg + + async def topology(_ch: Any) -> Any: + return cell_ch.Topology("", 0) + + async def no_tables(*_a: Any) -> tuple[list[Any], list[Any]]: + return [], [] + + async def sizes(*_a: Any) -> dict[str, int]: + return {} + + pg_rows = [{"name": "loom.a_unindexed", "n": 1}, {"name": "loom.b_data", "n": 2}] + conn = FakeConn([("FROM loom.tenancy", []), ("pg_class c", pg_rows)]) + + @asynccontextmanager + async def dedicated(**_kw: Any) -> Any: + yield conn + + async def pg_orgs(table: str, _budget: Any) -> list[str]: + if table == "loom.a_unindexed": + raise Refused([Refusal(f"postgres {table}", "has no plain btree index led by org")]) + return ["stray-org"] + + monkeypatch.setattr(cell_ch, "read_topology", topology) + monkeypatch.setattr(cell_scan, "list_clickhouse", no_tables) + monkeypatch.setattr(doctor_cell, "_table_sizes", sizes) + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(cell_scan, "pg_orgs", pg_orgs) + lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(None, 60)} + assert lines["cell_stray_orgs"] == ("WARN", "postgres loom.b_data: stray-org") + assert lines["cell_stray_orgs_unread"] == ( + "WARN", + "not read: postgres loom.a_unindexed: has no plain btree index led by org", + ) + assert "cell_stray_orgs_unchecked" not in lines diff --git a/tests/unit/test_cell_attach_abort.py b/tests/unit/test_cell_attach_abort.py new file mode 100644 index 0000000000..574fdfaabb --- /dev/null +++ b/tests/unit/test_cell_attach_abort.py @@ -0,0 +1,124 @@ +"""Fallback abort cannot replace the owner's selected first error.""" + +import subprocess +import sys +import textwrap + +import pytest + + +@pytest.mark.parametrize("owner", ["native", "supervise"]) +@pytest.mark.parametrize( + "first_kind", ["ValueError", "CancelledError", "SystemExit", "KeyboardInterrupt"] +) +@pytest.mark.parametrize( + "abort_kind", ["none", "RuntimeError", "CancelledError", "SystemExit", "KeyboardInterrupt"] +) +def test_actual_attach_owner_preserves_first_error_through_abort(owner, first_kind, abort_kind): + program = textwrap.dedent( + """ + import asyncio + import sys + import time + from server.cli import cell_attach_pg as apg + from server.cli import cell_control + from server.core import pg + + owner, first_kind, abort_kind = sys.argv[1:] + kinds = dict(ValueError=ValueError, RuntimeError=RuntimeError, + CancelledError=asyncio.CancelledError, SystemExit=SystemExit, + KeyboardInterrupt=KeyboardInterrupt) + first = kinds[first_kind]('first') + events = [] + deadlines = [] + class Conn: + def is_closed(self): return False + async def close(self, **kwargs): events.append('public close') + def terminate(self): + events.append('abort') + if abort_kind != 'none': raise kinds[abort_kind]('secondary abort') + + async def main(): + reaping, release, entered = asyncio.Event(), asyncio.Event(), asyncio.Event() + children = [] + async def work(): + children.append(asyncio.current_task()) + entered.set() + try: + if owner == 'native' or first_kind != 'CancelledError': raise first + await asyncio.Future() + finally: + if owner == 'supervise' and first_kind == 'CancelledError': + reaping.set() + await release.wait() + async def close(conn, *, end): + deadlines.append(end) + events.append('public close') + if owner == 'native': + children.append(asyncio.current_task()) + raise RuntimeError('secondary close') + original_close = pg.close_owned + pg.close_owned = close + original_wait = asyncio.wait + waits = [] + async def wait(tasks, *, timeout): + waits.append(timeout) + if len(waits) == 2 and first_kind != 'CancelledError': + # Inject a second signal specifically during finalization. + reaping.set() + await release.wait() + return await original_wait(tasks, timeout=timeout) + if owner == 'supervise': asyncio.wait = wait + end = time.monotonic() + 1 + owned = cell_control.Owner() + owned.conn = Conn() + async def operation(): + try: + if owner == 'native': + result = await cell_control._captured( + lambda: owned.call(work, total_s=1) + ) + if isinstance(result, cell_control._Fatal): raise result.error + else: + await apg.supervise(Conn(), work(), end=end) + except BaseException as error: + if first_kind == 'CancelledError' and owner == 'supervise': + assert isinstance(error, asyncio.CancelledError) and error.args == ('first',) + else: + assert error is first, (type(error).__name__, events) + return + raise AssertionError('first error was swallowed') + try: + task = asyncio.create_task(apg._work_result(operation)) + if owner == 'supervise': + await entered.wait() + if first_kind == 'CancelledError': task.cancel('first') + await reaping.wait() + task.cancel('second') + result = await task + if isinstance(result, apg._Interrupted): raise result.error + assert events.count('abort') == 1, events + assert events.index('public close') < events.index('abort'), events + expected_end = owned.phase_end if owner == 'native' else end + assert deadlines == [expected_end], deadlines + if owner == 'native': assert owned.close_end == owned.phase_end + assert all(0 <= bound <= 1 for bound in waits), waits + finally: + pg.close_owned = original_close + asyncio.wait = original_wait + release.set() + await asyncio.gather(*children, return_exceptions=True) + assert all(child.done() and not child._log_traceback for child in children) + print('first preserved and children reaped') + asyncio.run(main()) + """ + ) + result = subprocess.run( + [sys.executable, "-c", program, owner, first_kind, abort_kind], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "first preserved and children reaped" in result.stdout diff --git a/tests/unit/test_cell_attach_cleanup.py b/tests/unit/test_cell_attach_cleanup.py new file mode 100644 index 0000000000..a018784d21 --- /dev/null +++ b/tests/unit/test_cell_attach_cleanup.py @@ -0,0 +1,137 @@ +"""The attach owner preserves failures through bounded public-close cleanup.""" + +import argparse +import asyncio +import time +from unittest.mock import AsyncMock + +import pytest + +from server.cli import cell_attach, cell_control +from tests.unit.cell_control_owner import owner +from tests.unit.test_cell_control import snapshot + + +@pytest.mark.asyncio +async def test_failed_work_keeps_its_error_when_public_close_also_fails(): + events = [] + + class Conn: + async def close(self, **kwargs): + events.append("close") + raise RuntimeError("close failed") + + def terminate(self): + events.append("abort") + + original = ValueError("original work failure") + + async def work(): + raise original + + with pytest.raises(ValueError, match="original work failure") as caught: + await owner(Conn()).call(work, total_s=0.1) + assert caught.value is original + assert events[0] == "close" and events.count("close") == 1 + assert events[1:] and set(events[1:]) == {"abort"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed", [False, True]) +async def test_second_cancellation_preserves_original_failure_and_closes_before_cancel(failed): + entered, closing = asyncio.Event(), asyncio.Event() + events = [] + + class Conn: + async def close(self, **kwargs): + events.append("close") + closing.set() + await asyncio.Future() + + def terminate(self): + events.append("abort") + + def is_closed(self): + return False + + async def work(): + entered.set() + try: + if failed: + raise ValueError("original work failure") + await asyncio.Future() + finally: + events.append("work ended") + + task = asyncio.create_task(owner(Conn()).call(work, total_s=0.2)) + await entered.wait() + if not failed: + task.cancel("first cancellation") + await closing.wait() + task.cancel("second cancellation") + with pytest.raises( + ValueError if failed else asyncio.CancelledError, + match="original work failure" if failed else "first cancellation", + ): + await task + assert events.count("abort") >= 1 + if not failed: + assert events.index("close") < events.index("work ended") + + +@pytest.mark.asyncio +async def test_lock_owner_bounds_close_inside_the_same_cleanup_allowance(): + closing = asyncio.Event() + conn = AsyncMock() + conn.terminate = lambda: None + + async def close(**_): + closing.set() + await asyncio.Future() + + conn.close.side_effect = close + owned = cell_control.Owner(exit_s=0.03) + owned.conn = conn + started = time.monotonic() + with pytest.raises(TimeoutError): + await owned.close() + assert time.monotonic() - started < 0.15 + assert closing.is_set() and owned.closed + conn.close.assert_awaited_once() + await asyncio.sleep(0) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed", [False, True]) +async def test_run_preserves_its_own_failure_without_hiding_normal_close_failure( + failed, monkeypatch +): + router = AsyncMock() + router.call.return_value = {**snapshot(), "state": "attaching"} + run = cell_attach.Run( + None, argparse.Namespace(cell="test", wait=0, dry_run=True), router, lambda _: None + ) + original = ValueError("run failed") + conn = AsyncMock() + conn.terminate = lambda: None + conn.close.side_effect = RuntimeError("close failed") + + async def open(owned, what): + owned.conn = conn + if failed: + raise original + + monkeypatch.setattr(cell_control.Owner, "_open", open) + run.read_own_row = AsyncMock() + run.refusals = AsyncMock(return_value=(None, None, None)) + run.dry_run = AsyncMock(return_value=0) + try: + raise LookupError("unrelated caller exception") + except LookupError: + with pytest.raises( + ValueError if failed else RuntimeError, match="run failed" if failed else "close failed" + ) as caught: + await run.run() + if failed: + assert caught.value is original + conn.close.assert_awaited_once() diff --git a/tests/unit/test_cell_attach_owned_exit.py b/tests/unit/test_cell_attach_owned_exit.py new file mode 100644 index 0000000000..bc8bc89da5 --- /dev/null +++ b/tests/unit/test_cell_attach_owned_exit.py @@ -0,0 +1,165 @@ +"""The shared attach owner finishes before errors reach the command boundary.""" + +import subprocess +import sys +import textwrap + +import pytest + + +@pytest.mark.parametrize("owner", ["lock", "mode"]) +@pytest.mark.parametrize("secondary", ["RuntimeError", "SystemExit", "KeyboardInterrupt"]) +def test_actual_attach_owner_keeps_first_failure(owner, secondary): + program = textwrap.dedent( + """ + import argparse + import asyncio + import sys + from unittest.mock import AsyncMock + from server.cli.cell_attach import Run + from server.cli import cell_attach_pg as apg + from server.cli import cell_control + from tests.unit.test_cell_control import snapshot + + owner, kind = sys.argv[1:] + original = ValueError('operation failed') + secondary = {'RuntimeError': RuntimeError, 'SystemExit': SystemExit, + 'KeyboardInterrupt': KeyboardInterrupt}[kind]('exit failed') + events = [] + class Conn: + def __init__(self): + self.closed = False + self.closes = 0 + def is_closed(self): + return self.closed + async def close(self, **kwargs): + self.closes += 1 + events.append('close') + self.closed = True + raise secondary + def terminate(self): + self.closed = True + events.append('abort') + + async def main(): + conn = Conn() + async def opening(owned, what): owned.conn = conn + async def fail(*args): raise original + cell_control.Owner._open = opening + router = AsyncMock() + router.call.return_value = {**snapshot(), 'state': 'attaching'} + run = Run(None, argparse.Namespace(cell='test', wait=0), router, lambda _: None) + run.row = {'deployment_id': 'test'} + run.mode = 'recording' + run.confirm_lock = AsyncMock() + run.ready_for_commit = AsyncMock(return_value=[]) + run.mutate = AsyncMock() + run.read_own_row = AsyncMock(side_effect=original) + apg._mode_transaction = fail + async def mode(owned): + run.owner, run.lock_conn = owned, owned.conn + return await run.commit_mode() + try: + await (run.run() if owner == 'lock' else cell_control.run('attach', mode)) + except BaseException as error: + assert error is original, (type(error).__name__, events) + assert events.count('close') == 1, events + await asyncio.sleep(0) + print('original-after-owned-exit') + return + raise AssertionError('operation failure was swallowed') + asyncio.run(main()) + """ + ) + result = subprocess.run( + [sys.executable, "-c", program, owner, secondary], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "original-after-owned-exit" in result.stdout + + +@pytest.mark.asyncio +@pytest.mark.parametrize("factory_failure", [False, True]) +async def test_mode_failure_survives_close_construction_and_await_failures( + monkeypatch, factory_failure +): + import argparse + from unittest.mock import AsyncMock + + from server.cli import cell_attach_pg as apg + from server.cli import cell_control + from server.cli.cell_attach import Run + from server.core import pg + + original = ValueError("mode failed") + observed = [] + conn = AsyncMock() + conn.is_closed = lambda: False + conn.terminate = lambda: None + owned = cell_control.Owner() + + async def opening(): + owned.conn = conn + + async def failed_close(): + raise RuntimeError("close failed") + + def close(actual, *, end): + observed.append((actual, end)) + if factory_failure: + raise RuntimeError("close construction failed") + return failed_close() + + monkeypatch.setattr(pg, "close_owned", close) + commit = AsyncMock(side_effect=original) + monkeypatch.setattr(apg, "commit_enforced", commit) + run = Run(None, argparse.Namespace(cell="test", wait=0), AsyncMock(), lambda _: None) + run.owner, run.lock_conn = owned, conn + run.row = {"deployment_id": "test"} + run.mode = "recording" + run.mutate = AsyncMock() + with pytest.raises(ValueError) as result: + await owned.run(opening, run.commit_mode, opening_s=0.3) + assert result.value is original and original.__traceback__ is not None + commit.assert_awaited_once() + assert observed == [(conn, owned.close_end)] + assert owned.closed and owned._close_task.done() + assert not owned._close_task._log_traceback + + +@pytest.mark.asyncio +async def test_successful_mode_work_still_fails_when_required_close_fails(monkeypatch): + import argparse + from unittest.mock import AsyncMock + + from server.cli import cell_attach_pg as apg + from server.cli import cell_control + from server.cli.cell_attach import Run + + original = OSError("public close failed") + conn = AsyncMock() + conn.is_closed = lambda: False + conn.terminate = lambda: None + conn.close.side_effect = original + owned = cell_control.Owner() + + async def opening(): + owned.conn = conn + + async def committed(*args, **kwargs): + args[-1]("unknown") + + monkeypatch.setattr(apg, "commit_enforced", committed) + run = Run(None, argparse.Namespace(cell="test", wait=0), AsyncMock(), lambda _: None) + run.owner, run.lock_conn = owned, conn + run.row = {"deployment_id": "test"} + run.mutate = AsyncMock() + with pytest.raises(OSError) as result: + await owned.run(opening, run.commit_mode, opening_s=0.3) + assert result.value is original + assert run.mode == "unknown" + conn.close.assert_awaited_once() diff --git a/tests/unit/test_cell_attach_replica_ownership.py b/tests/unit/test_cell_attach_replica_ownership.py new file mode 100644 index 0000000000..f2317d31a9 --- /dev/null +++ b/tests/unit/test_cell_attach_replica_ownership.py @@ -0,0 +1,132 @@ +"""The threaded API fixture owns entry and exit until cleanup has finished.""" + +import ast +import asyncio +import contextlib +import sys +import threading +from collections.abc import AsyncIterator +from pathlib import Path + +import pytest + +SOURCE = Path(__file__).resolve().parents[1] / "integration/test_cell_attach_two_replicas.py" + + +def _fixture(context): + # Execute the actual helper alone: no server imports, services or subprocesses. + tree = ast.parse(SOURCE.read_text()) + helper = next(n for n in tree.body if isinstance(n, ast.AsyncFunctionDef) and n.name == "_api") + + def spawn(name, env, *, port): + assert (name, env, port) == ("owned", {"LOOM_INSTANCE_LABEL": "owned"}, 1) + return context + + namespace = { + "asyncio": asyncio, + "contextlib": contextlib, + "sys": sys, + "AsyncIterator": AsyncIterator, + "loom_server": spawn, + } + exec(compile(ast.Module(body=[helper], type_ignores=[]), str(SOURCE), "exec"), namespace) + return namespace["_api"] + + +@pytest.mark.parametrize("phase", ["entry", "exit", "body-error"]) +def test_cancellation_waits_for_owned_context_before_fixture_unwinds(phase): + async def scenario(): + observed = asyncio.Event() + entry_finished = threading.Event() + release = threading.Event() + loop = asyncio.get_running_loop() + unwound = [] + first = ValueError("body failed") if phase == "body-error" else None + + class Context: + alive = False + exits = 0 + + def __enter__(self): + self.alive = True + try: + if phase == "entry": + loop.call_soon_threadsafe(observed.set) + assert release.wait(5), "test did not release entry" + return "http://owned" + finally: + entry_finished.set() + + def __exit__(self, *_): + self.exits += 1 + if phase != "entry": + loop.call_soon_threadsafe(observed.set) + assert release.wait(5), "test did not release exit" + self.alive = False + + context = Context() + api = _fixture(context) + + async def owner(): + try: + async with api("owned", {}, 1) as base: + assert base == "http://owned" + if first is not None: + raise first + finally: + unwound.append(context.alive) + + task = asyncio.create_task(owner()) + try: + await asyncio.wait_for(observed.wait(), timeout=2) + for message in ("first cancel", "second cancel", "third cancel"): + task.cancel(message) + await asyncio.sleep(0) + assert not task.done(), "owner unwound while its context was still live" + assert unwound == [] and context.alive + release.set() + with pytest.raises( + ValueError if first is not None else asyncio.CancelledError + ) as caught: + await asyncio.wait_for(task, timeout=2) + if first is not None: + assert caught.value is first + else: + assert caught.value.args == ("first cancel",) + assert context.exits == 1 and not context.alive + assert unwound == [False] + finally: + release.set() + with contextlib.suppress(BaseException): + await task + assert await asyncio.to_thread(entry_finished.wait, 2) + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("startup_fails", [False, True]) +def test_entry_outcome_selects_exactly_one_exit_or_its_existing_failure_cleanup(startup_fails): + first = ValueError("startup failed after its own cleanup") + exits = [] + + class Context: + def __enter__(self): + if startup_fails: + raise first + return "http://owned" + + def __exit__(self, *_): + exits.append(True) + + async def scenario(): + async with _fixture(Context())("owned", {}, 1) as base: + assert base == "http://owned" + + if startup_fails: + with pytest.raises(ValueError) as caught: + asyncio.run(scenario()) + assert caught.value is first + assert exits == [] + else: + asyncio.run(scenario()) + assert exits == [True] diff --git a/tests/unit/test_cell_attach_serving_identity.py b/tests/unit/test_cell_attach_serving_identity.py new file mode 100644 index 0000000000..469ed16ea3 --- /dev/null +++ b/tests/unit/test_cell_attach_serving_identity.py @@ -0,0 +1,134 @@ +"""A serving shortcut validates the registration without claiming or writing.""" + +from __future__ import annotations + +from contextlib import asynccontextmanager +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import UUID + +import pytest + +from server.cli import cell_attach, cell_control +from server.cli.cell_ch import Refused +from server.core import pg +from server.settings import settings +from tests.unit.test_cell_attach import FakeConn, _args + + +@pytest.fixture +def serving_run(monkeypatch): + def make( + *, + deployment="registered", + configured_cell="first", + bucket="bucket", + prefix="docs/", + first=True, + mode="enforced", + ): + row = { + "cell": "first", + "is_first": first, + "state": "serving", + "host": "first.example", + "deployment_id": "registered", + "bucket": "bucket", + "bucket_prefix": "docs/", + "probe_url": "", + "upstream_url": "", + "current_fp": None, + "next_fp": None, + "mutation_generation": str(UUID(int=1)), + } + monkeypatch.setattr(settings, "cell", configured_cell) + monkeypatch.setattr(settings, "document_s3_bucket", bucket) + monkeypatch.setattr(settings, "document_s3_prefix", prefix) + conn = FakeConn( + [ + ("SELECT deployment_id FROM loom.authority", deployment), + ( + "SELECT (SELECT is_first FROM loom.cell_first)", + { + "is_first": first, + "cell": "first", + "mode": mode, + "has_own": True, + "has_tenancy": True, + }, + ), + ( + "SELECT is_first, cell FROM loom.cell_first", + {"is_first": first, "cell": "first"}, + ), + ] + ) + sessions = [] + + @asynccontextmanager + async def dedicated(**kwargs): + sessions.append("opened") + try: + yield conn + finally: + sessions.append("closed") + + monkeypatch.setattr(pg, "dedicated", dedicated) + owner = AsyncMock(side_effect=AssertionError("serving must not start an owner")) + claim = AsyncMock(side_effect=AssertionError("serving must not claim")) + mutate = AsyncMock(side_effect=AssertionError("serving must not mutate")) + monkeypatch.setattr(cell_control, "run", owner) + monkeypatch.setattr(cell_control, "claim", claim) + monkeypatch.setattr(cell_control, "mutate", mutate) + router = SimpleNamespace(call=AsyncMock(return_value=row)) + run = cell_attach.Run(None, _args(), router, lambda text: None) + return run, conn, router, sessions, (owner, claim, mutate) + + return make + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "overrides", + [ + pytest.param({"deployment": "another-deployment"}, id="deployment"), + pytest.param({"deployment": None}, id="missing-authority"), + pytest.param({"configured_cell": "other"}, id="configured-cell"), + pytest.param({"bucket": "another-bucket"}, id="bucket"), + pytest.param({"prefix": "another-prefix/"}, id="bucket-prefix"), + ], +) +async def test_serving_run_refuses_another_registration_without_writes(serving_run, overrides): + run, conn, router, sessions, mutations = serving_run(**overrides) + with pytest.raises(Refused, match="do not match the directory registration"): + await run.run() + assert "done" not in run.report + assert sessions == ["opened", "closed"] + assert router.call.await_args_list[0].args == ("",) + assert router.call.await_count == 1 + assert conn.executed == [] + assert all(call.await_count == 0 for call in mutations) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("first", [True, False], ids=["first", "later"]) +async def test_matching_serving_registration_is_done_without_writes(serving_run, first): + run, conn, router, sessions, mutations = serving_run(first=first) + assert await run.run() == 0 + assert run.report == {"cell": "first", "state": "serving", "done": True} + assert sessions == ["opened", "closed"] + assert router.call.await_count == 1 + assert conn.executed == [] + assert all(call.await_count == 0 for call in mutations) + + +@pytest.mark.asyncio +async def test_matching_serving_registration_still_refuses_recording(serving_run): + run, conn, router, sessions, mutations = serving_run(mode="recording") + with pytest.raises(Refused, match="attach lowers and repairs neither"): + await run.run() + assert "done" not in run.report + assert sessions == ["opened", "closed"] + assert router.call.await_count == 1 + assert conn.executed == [] + assert all(call.await_count == 0 for call in mutations) diff --git a/tests/unit/test_cell_cli_orgs.py b/tests/unit/test_cell_cli_orgs.py index 91253a27a9..5e4fc8733f 100644 --- a/tests/unit/test_cell_cli_orgs.py +++ b/tests/unit/test_cell_cli_orgs.py @@ -17,7 +17,13 @@ from server.settings import settings ROOT = Path(__file__).resolve().parents[2] -_CONNECTS = {"run_connected", "connect_checked", "connect_declared", "_connected"} +_CONNECTS = { + "run_connected", + "connect_checked", + "connect_declared", + "check_acceptance", + "_connected", +} # `.connect()` receivers that are not the deployment's ClickHouse writer: the setup # command's own admin client and SQLite files an importer reads. _OTHER_CLIENTS = {"preflight", "sqlite3"} @@ -476,6 +482,27 @@ async def connect(self) -> None: assert not [r for r in caplog.records if r.name == "clickhouse_connect"] +@pytest.mark.parametrize("diagnostic_mode", ["open", "recording", "enforced", "partial"]) +def test_setup_identity_probe_neither_installs_nor_uses_acceptance( + monkeypatch: pytest.MonkeyPatch, diagnostic_mode: str +) -> None: + """Bootstrap checks store ownership, never a reusable data-plane admission.""" + from server.core import retirement + + _cell_set(monkeypatch, (diagnostic_mode, "eu-1")) + client = AsyncMock() + monkeypatch.setattr("server.core.ch.get_client", lambda: client) + monkeypatch.setattr(pg_profile, "recorded_authority", AsyncMock(return_value=None)) + acceptance = AsyncMock(side_effect=AssertionError("bootstrap is not admission")) + monkeypatch.setattr(_ch, "check_acceptance", acceptance) + monkeypatch.setattr(cell, "refresh", acceptance) + before = cell._view + asyncio.run(_ch.check_setup_stores()) + assert cell._view is before + assert not retirement.get_gate().enrolled + acceptance.assert_not_awaited() + + def test_the_loom_code_drain_after_a_subcommand_is_a_read() -> None: """The reconnect that drains buffered writes runs after every `loom code` subcommand, reads included; the subcommand's own connect gated its writes, so the @@ -976,6 +1003,7 @@ def test_a_cell_whose_rows_cannot_be_read_refuses_every_declared_org( asyncio.run(_ch.check_orgs([])) +@pytest.mark.usefixtures("stub_model_command") def test_a_command_for_an_org_the_cell_does_not_hold_never_runs( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/test_cell_control.py b/tests/unit/test_cell_control.py new file mode 100644 index 0000000000..ccbbb7b6f9 --- /dev/null +++ b/tests/unit/test_cell_control.py @@ -0,0 +1,307 @@ +"""A control owner cannot survive an uncertain commit or a lost lock session.""" + +from __future__ import annotations + +import asyncio +from typing import Any +from unittest.mock import AsyncMock +from uuid import UUID + +import pytest + +from server.cli import cell_control +from tests.unit.test_cell_attach import FakeConn + + +@pytest.mark.asyncio +async def test_uncertain_commit_closes_before_cancelling_work(monkeypatch): + class Stalled(FakeConn): + async def execute(self, sql: str, *args: Any) -> str: + answer = await super().execute(sql, *args) + if sql == "COMMIT": + try: + await asyncio.Future() + finally: + self.executed.append("WORK ENDED") + return answer + + conn = Stalled() + owner = cell_control.Owner() + owner.conn = conn + monkeypatch.setattr(cell_control, "TRANSACTION_S", 0.04) + with pytest.raises(TimeoutError): + await owner.transaction(AsyncMock()) + assert conn.executed.index("CLOSE") < conn.executed.index("WORK ENDED") + assert "TERMINATE" not in conn.executed and "ROLLBACK" not in conn.executed + assert any("pg_logical_emit_message" in sql for sql in conn.executed) + + +@pytest.mark.asyncio +async def test_failure_never_starts_rollback_or_replaces_original(): + conn = FakeConn() + owner = cell_control.Owner() + owner.conn = conn + original = ValueError("original failure") + with pytest.raises(ValueError) as found: + await owner.transaction(AsyncMock(side_effect=original)) + assert found.value is original + assert conn.executed[-1] == "CLOSE" + assert "ROLLBACK" not in conn.executed and "TERMINATE" not in conn.executed + + +@pytest.mark.asyncio +async def test_cancellation_during_body_closes_before_work_cancel(): + conn = FakeConn() + owner = cell_control.Owner() + owner.conn = conn + entered = asyncio.Event() + + async def body(): + entered.set() + await asyncio.Future() + + task = asyncio.create_task(owner.transaction(body)) + await entered.wait() + task.cancel("original cancellation") + with pytest.raises(asyncio.CancelledError, match="original cancellation"): + await task + assert conn.executed[-1] == "CLOSE" and "ROLLBACK" not in conn.executed + + +def snapshot(): + return { + "cell": "first", + "is_first": True, + "host": "first.example", + "deployment_id": "dep", + "bucket": "", + "bucket_prefix": "", + "probe_url": "", + "upstream_url": "", + "current_fp": None, + "next_fp": None, + "mutation_generation": str(UUID(int=1)), + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("after_claim", [False, True]) +async def test_lost_lock_never_reconnects_or_continues_claim(monkeypatch, after_claim): + lock = AsyncMock( + side_effect=[None, cell_control._refused("lost")] + if after_claim + else cell_control._refused("lost") + ) + monkeypatch.setattr(cell_control, "check_lock", lock) + monkeypatch.setattr(cell_control, "paired", AsyncMock()) + call = AsyncMock(return_value={**snapshot(), "mutation_generation": str(UUID(int=2))}) + with pytest.raises(cell_control.Refused, match="lost"): + await cell_control.claim(object(), call, snapshot()) + assert call.await_count == int(after_claim) + + +@pytest.mark.asyncio +async def test_unknown_claim_acknowledgement_never_reuses_the_observed_generation(monkeypatch): + monkeypatch.setattr(cell_control, "check_lock", AsyncMock()) + monkeypatch.setattr(cell_control, "paired", AsyncMock()) + call = AsyncMock(side_effect=OSError("reply lost")) + with pytest.raises(OSError, match="reply lost"): + await cell_control.claim(object(), call, snapshot()) + assert call.await_count == 1 + assert "mutation_generation" not in call.call_args.args[1]["expected"] + + +def test_older_router_snapshot_cannot_make_a_control_request(): + row = snapshot() + row.pop("mutation_generation") + with pytest.raises(cell_control.Refused, match="not upgraded"): + cell_control.preconditions(row) + + +@pytest.mark.asyncio +async def test_whole_router_request_is_bounded_even_without_a_transport_reply(monkeypatch): + import httpx + + async def stalled(_request): + await asyncio.Future() + + monkeypatch.setattr(cell_control, "REQUEST_S", 0.01) + with pytest.raises(cell_control.Refused, match="reply unknown.*TimeoutError"): + await cell_control.request( + httpx.URL("http://127.0.0.1:1"), + "first", + httpx.MockTransport(stalled), + "/claim", + {}, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sqlstate", ["55P03", "57014"]) +@pytest.mark.parametrize("where", ["UPDATE loom.tenancy_mode", "COMMIT"]) +async def test_mode_contention_is_busy_only_before_commit_after_acknowledged_rollback( + sqlstate, where +): + from server.cli import cell_attach_pg as apg + from tests.unit.cell_control_owner import owner + from tests.unit.test_cell_attach import CURRENT, NEXT + + class Busy(Exception): + pass + + busy = Busy("expected contention") + busy.sqlstate = sqlstate + routes = [(where, busy)] + if where == "COMMIT": + routes.append(("UPDATE loom.tenancy_mode", "UPDATE 1")) + conn = FakeConn(routes) + owned = owner(conn) + marks = [] + if where == "COMMIT": + with pytest.raises(Busy) as found: + await apg._mode_try(owned, 100, marks.append, "first", (CURRENT, NEXT)) + assert found.value is busy + assert "ROLLBACK" not in conn.executed and marks == ["unknown"] + assert owned.closed + else: + assert not await apg._mode_try(owned, 100, marks.append, "first", (CURRENT, NEXT)) + assert conn.executed[-1] == "ROLLBACK" and marks == ["unknown", "recording"] + assert not owned.closed + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed", ["rollback", "ownership", "update-status"]) +async def test_terminal_mode_failure_never_resets_unknown_or_retries(failed): + import asyncpg + + from server.cli import cell_attach_pg as apg + from tests.unit.cell_control_owner import owner + from tests.unit.test_cell_attach import CURRENT, NEXT, _seq + + original = ValueError("rollback failed") + routes = [("UPDATE loom.tenancy_mode", "UPDATE 0")] + if failed != "update-status": + routes = [("UPDATE loom.tenancy_mode", asyncpg.LockNotAvailableError("busy"))] + if failed == "rollback": + routes.append(("ROLLBACK", original)) + if failed == "ownership": + routes.append((apg._HELD, _seq(True, False))) + conn, marks = FakeConn(routes), [] + with pytest.raises(ValueError if failed == "rollback" else cell_control.Refused) as found: + await apg.commit_enforced( + owner(conn), + AsyncMock(), + AsyncMock(return_value=[]), + apg.Wait(0), + lambda _: None, + marks.append, + expected_cell="first", + expected_fp=(CURRENT, NEXT), + ) + if failed == "rollback": + assert found.value is original + assert marks == ["unknown"] and conn.executed[-1] == "CLOSE" + assert conn.executed.count("BEGIN") == 1 + + +@pytest.mark.asyncio +async def test_expired_phase_does_not_receive_another_exit_allowance(monkeypatch): + import time + + class StalledClose(FakeConn): + async def close(self, **kwargs): + self.timeouts = [kwargs["timeout"]] + self.executed.append("CLOSE START") + await asyncio.Future() + + conn = StalledClose() + + async def opening(self, _what): + self.conn = conn + + monkeypatch.setattr(cell_control.Owner, "_open", opening) + monkeypatch.setattr(cell_control, "TRANSACTION_S", 0.04) + + async def body(owned): + return await owned.transaction(lambda: asyncio.Future()) + + started = time.monotonic() + with pytest.raises(TimeoutError): + await cell_control.run("test", body) + assert time.monotonic() - started < 0.3 + assert conn.executed.count("CLOSE START") == 1 + assert 0 < conn.timeouts[0] <= 0.02 + assert "TERMINATE" in conn.executed + + +@pytest.mark.asyncio +async def test_late_connection_is_discarded_without_starting_command_work(monkeypatch): + from server.core import pg + + connected, discarded = asyncio.Event(), asyncio.Event() + + class Late(FakeConn): + def terminate(self): + super().terminate() + discarded.set() + + conn = Late() + + async def connect(**_): + try: + await asyncio.Future() + except asyncio.CancelledError: + while not connected.is_set(): + try: + await connected.wait() + except asyncio.CancelledError: + pass + return conn + + monkeypatch.setattr(pg, "_connect_dedicated", connect) + monkeypatch.setattr(cell_control, "OPEN_S", 0.04) + body = AsyncMock() + with pytest.raises(TimeoutError): + await cell_control.run("late", body) + connected.set() + await asyncio.wait_for(discarded.wait(), 0.3) + body.assert_not_awaited() + assert conn.executed == ["TERMINATE"] + + +@pytest.mark.asyncio +async def test_successful_body_cannot_hide_failed_required_close(monkeypatch): + class Broken(FakeConn): + async def close(self, **_): + raise OSError("required close failed") + + conn = Broken() + + async def opening(self, _what): + self.conn = conn + + monkeypatch.setattr(cell_control.Owner, "_open", opening) + with pytest.raises(OSError, match="required close failed"): + await cell_control.run("test", AsyncMock(return_value=7)) + assert "TERMINATE" in conn.executed + + +@pytest.mark.asyncio +async def test_late_connection_arriving_inside_reserve_is_closed_once(monkeypatch): + from server.core import pg + + conn = FakeConn() + + async def connect(**_): + try: + await asyncio.Future() + except asyncio.CancelledError: + return conn + + monkeypatch.setattr(pg, "_connect_dedicated", connect) + monkeypatch.setattr(cell_control, "OPEN_S", 0.08) + work = AsyncMock() + with pytest.raises(TimeoutError): + await cell_control.run("late", work) + work.assert_not_awaited() + assert conn.executed == ["CLOSE"] diff --git a/tests/unit/test_cell_control_abort.py b/tests/unit/test_cell_control_abort.py new file mode 100644 index 0000000000..890ddcba27 --- /dev/null +++ b/tests/unit/test_cell_control_abort.py @@ -0,0 +1,185 @@ +"""A failed fallback abort cannot replace the native owner's first failure.""" + +from __future__ import annotations + +import asyncio +import time +from unittest.mock import AsyncMock + +import pytest + +from server.cli import cell_control + +pytestmark = pytest.mark.asyncio +ERRORS = (ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt) +ABORT_ERRORS = (RuntimeError, asyncio.CancelledError, SystemExit, KeyboardInterrupt) + + +async def outcome(work): + try: + result = await cell_control._captured(work) + except BaseException as error: # noqa: BLE001 — observe the identical native failure inertly. + return error + assert isinstance(result, cell_control._Fatal), "a failed required close cannot succeed" + return result.error + + +@pytest.mark.parametrize("first_kind", ERRORS) +@pytest.mark.parametrize("abort_kind", ABORT_ERRORS) +async def test_direct_owner_keeps_first_close_failure_and_retrieves_child( + monkeypatch, first_kind, abort_kind +): + first, secondary = first_kind("first close failure"), abort_kind("abort failure") + aborted = [] + + class Connection: + def terminate(self): + aborted.append(True) + raise secondary + + owner = cell_control.Owner(exit_s=0.2) + owner.conn = Connection() + close = AsyncMock(side_effect=first) + monkeypatch.setattr(cell_control.pg, "close_owned", close) + result = await outcome(owner.close) + assert result is first + assert aborted == [True] + close.assert_awaited_once() + assert owner.closed and owner._close_task.done() + assert owner._close_task.cancelled() or not owner._close_task._log_traceback + + +@pytest.mark.parametrize("abort_kind", ABORT_ERRORS) +@pytest.mark.parametrize("first_kind", ERRORS) +async def test_outer_signal_wins_over_later_close_and_abort_failures( + monkeypatch, first_kind, abort_kind +): + entered, finish = asyncio.Event(), asyncio.Event() + first, secondary = first_kind("close failed after signal"), abort_kind("abort failure") + seen = [] + actual_wait = asyncio.wait + owner = cell_control.Owner(exit_s=0.2) + + class Connection: + def terminate(self): + raise secondary + + async def close(conn, *, end): + entered.set() + await finish.wait() + raise first + + async def wait(*args, **kwargs): + try: + return await actual_wait(*args, **kwargs) + except asyncio.CancelledError as error: + seen.append(error) + raise + + owner.conn = Connection() + monkeypatch.setattr(cell_control.pg, "close_owned", close) + monkeypatch.setattr(cell_control.asyncio, "wait", wait) + task = asyncio.create_task(outcome(owner.close)) + try: + await entered.wait() + original_end = owner.close_end + task.cancel("original outer cancellation") + await asyncio.sleep(0) + assert len(seen) == 1 and not task.done() + finish.set() + result = await task + assert result is seen[0] + assert owner.close_end == original_end + assert owner._close_task.done() + assert owner._close_task.cancelled() or not owner._close_task._log_traceback + finally: + finish.set() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("abort_kind", ABORT_ERRORS) +async def test_abort_failure_still_cancels_and_retrieves_expired_close_child( + monkeypatch, abort_kind +): + entered, ended = asyncio.Event(), asyncio.Event() + secondary = abort_kind("abort failure") + + class Connection: + def terminate(self): + raise secondary + + async def close(conn, *, end): + entered.set() + try: + await asyncio.Future() + finally: + ended.set() + raise ValueError("late close failure") + + owner = cell_control.Owner(exit_s=0.02) + owner.conn = Connection() + monkeypatch.setattr(cell_control.pg, "close_owned", close) + started = time.monotonic() + result = await outcome(owner.close) + assert isinstance(result, TimeoutError) + assert time.monotonic() - started < 0.2 + assert entered.is_set() + await asyncio.wait_for(ended.wait(), 0.2) + await asyncio.sleep(0) + assert owner._close_task.done() and not owner._close_task._log_traceback + + +@pytest.mark.parametrize("abort_kind", ABORT_ERRORS) +async def test_self_cancelled_close_is_terminal_even_when_abort_fails(monkeypatch, abort_kind): + first = asyncio.CancelledError("close cancelled itself") + calls = [] + + class Connection: + def terminate(self): + calls.append("abort") + raise abort_kind("abort failure") + + async def close(conn, *, end): + calls.append("close") + raise first + + owner = cell_control.Owner(exit_s=60) + owner.conn = Connection() + monkeypatch.setattr(cell_control.pg, "close_owned", close) + started = time.monotonic() + result = await outcome(owner.close) + assert result is first + assert time.monotonic() - started < 0.2 + assert calls == ["close", "abort"] + + +@pytest.mark.parametrize("wait_kind", ERRORS) +@pytest.mark.parametrize("abort_kind", ABORT_ERRORS) +async def test_failed_reap_abort_cannot_replace_the_callers_first_failure( + monkeypatch, wait_kind, abort_kind +): + owner = cell_control.Owner() + owner.close_end = time.monotonic() + 0.2 + calls = [] + + class Connection: + def terminate(self): + calls.append("abort") + raise abort_kind("abort failed while reaping") + + async def failed_wait(*args, **kwargs): + raise wait_kind("reap wait interrupted") + + owner.conn = Connection() + child = owner._start(lambda: asyncio.Future()) + monkeypatch.setattr(cell_control.asyncio, "wait", failed_wait) + try: + try: + result = await cell_control._captured(lambda: owner._stop({child})) + except BaseException as error: # noqa: BLE001 — the probe must expose secondary native errors. + result = error + assert result is None, "reaping cannot replace an earlier operation failure" + assert calls == ["abort"] + finally: + child.cancel() + await asyncio.gather(child, return_exceptions=True) diff --git a/tests/unit/test_cell_control_census.py b/tests/unit/test_cell_control_census.py new file mode 100644 index 0000000000..64359d1293 --- /dev/null +++ b/tests/unit/test_cell_control_census.py @@ -0,0 +1,582 @@ +"""Inventory cell control SQL writers and the shared boundaries they must retain. + +This recognizes Python string literals (including adjacent literals) and f-strings. It is a +source change gate, not a proof for arbitrary generated SQL; backend/session behavior +is exercised against Postgres in test_cell_control_fence. +""" + +from __future__ import annotations + +import ast +import re +from collections import Counter +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +# Include immutable placement writes: their exemption must remain explicit. +TABLES = r"(?:loom\.(?:cell_self|cell_first|tenancy_mode)|loom_router\.[A-Za-z_]+)" +TARGET = rf"(?:{TABLES}|loom(?:_router)?\.\{{[^}}]+\}}|\{{[^}}]+\}})" +WRITES = re.compile( + rf"(?:^|;|[()])\s*(INSERT\s+INTO|UPDATE|DELETE\s+FROM|TRUNCATE(?:\s+TABLE)?|" + rf"ALTER\s+TABLE(?:\s+IF\s+EXISTS)?|CREATE\s+TABLE(?:\s+IF\s+NOT\s+EXISTS)?|" + rf"DROP\s+TABLE(?:\s+IF\s+EXISTS)?)\s+({TARGET})" + rf"|(?:^|;|[()])\s*((?:CREATE|DROP)\s+TRIGGER)\b[^;]*?\bON\s+({TARGET})", + re.IGNORECASE, +) + + +def mutations(sql: str) -> list[tuple[str, str]]: + sql = re.sub(r"--[^\n]*|/\*.*?\*/", "", sql, flags=re.DOTALL).replace('"', "") + return [(" ".join((a or c).upper().split()), b or d) for a, b, c, d in WRITES.findall(sql)] + + +class Inventory(ast.NodeVisitor): + def __init__(self) -> None: + self.scope: list[str] = [] + self.writes: Counter[tuple[str, str, str]] = Counter() + self.functions: dict[str, ast.AST] = {} + + def visit_FunctionDef(self, node: ast.FunctionDef | ast.AsyncFunctionDef) -> None: + self.scope.append(node.name) + self.functions[".".join(self.scope)] = node + self.generic_visit(node) + self.scope.pop() + + visit_AsyncFunctionDef = visit_FunctionDef + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + self.scope.append(node.name) + self.generic_visit(node) + self.scope.pop() + + def record(self, sql: str) -> None: + for verb, target in mutations(sql): + self.writes[(".".join(self.scope), verb, target)] += 1 + + def visit_Constant(self, node: ast.Constant) -> None: + if isinstance(node.value, str): + self.record(node.value) + + def visit_JoinedStr(self, node: ast.JoinedStr) -> None: + self.record( + "".join( + str(part.value) + if isinstance(part, ast.Constant) + else "{" + ast.unparse(part.value) + "}" + for part in node.values + ) + ) + # Do not count each f-string fragment as another SQL statement. + + +def inventory(source: str) -> Inventory: + found = Inventory() + found.visit(ast.parse(source)) + return found + + +def test_repository_cell_control_writers_are_classified(): + actual = {} + for path in (ROOT / "server").rglob("*.py"): + writes = inventory(path.read_text()).writes + if writes: + actual[str(path.relative_to(ROOT))] = dict(writes) + expected = {} + for group in (GUARDED_WRITERS, EXEMPT_WRITERS, GENERIC_SQL_EXCLUSIONS): + for path, entries in group.items(): + expected.setdefault(path, {}).update(entries) + assert actual == expected + + +def test_cell_control_migrations_are_classified(): + actual = {} + for path in (ROOT / "schemas").rglob("*.sql"): + writes = Counter(mutations(path.read_text())) + if writes: + actual[str(path.relative_to(ROOT))] = dict(writes) + assert actual == MIGRATION_WRITERS + + +# Entries name every sink, including nested callbacks and dynamic table expressions. +# A new sink, including another statement in an existing owner, requires classification. +GUARDED_WRITERS = { + "server/cli/cell_attach_pg.py": {("_mode_transaction", "UPDATE", "loom.tenancy_mode"): 1}, + "server/cli/cell_control.py": { + ("backout.clear_state", "ALTER TABLE", "loom.cell_first"): 3, + ("backout.clear_state", "DELETE FROM", "loom.{table}"): 1, + ("backout.clear_triggers", "DROP TRIGGER", "loom.{table}"): 1, + }, + "server/cli/cell_prepare.py": { + ("_write_rows.write", "INSERT INTO", "loom.cell_first"): 1, + ("_write_rows.write", "INSERT INTO", "loom.tenancy_mode"): 1, + ("_write_rows.write", "INSERT INTO", "loom.cell_self"): 1, + }, + "server/router/directory.py": { + ("set_limit.change", "UPDATE", "loom_router.cells"): 1, + ("rotate_credential.change", "UPDATE", "loom_router.cells"): 1, + ("claim.change_locked", "UPDATE", "loom_router.cells"): 1, + ("attach_begin.change_locked", "UPDATE", "loom_router.cells"): 1, + ("attach_place.change_locked", "INSERT INTO", "loom_router.placements"): 1, + ("attach_place.change_locked", "UPDATE", "loom_router.cells"): 1, + ("attach_place.change_locked", "INSERT INTO", "loom_router.audit"): 1, + ("attach_enforced.change_locked", "UPDATE", "loom_router.cells"): 1, + ("attach_serving.change_locked", "UPDATE", "loom_router.cells"): 1, + ("remove.change_locked", "DELETE FROM", "loom_router.hosts"): 1, + ("remove.change_locked", "DELETE FROM", "loom_router.cells"): 1, + }, +} + +# Registration only bootstraps or validates its immutable rerun. Ordinary placement +# writes an immutable org placement/count under its org and cell locks. Reads write +# audit rows only. Trigger setup only enables/installs guards under the migration lock. +EXEMPT_WRITERS = { + "server/core/cell_triggers.py": { + ("_create", "CREATE TRIGGER", "{qualified}"): 2, + ("ensure", "ALTER TABLE", "loom.cell_first"): 1, + ("_table_statements", "ALTER TABLE", "loom.{_ident(table)}"): 1, + }, + "server/router/directory.py": { + ("register", "INSERT INTO", "loom_router.cells"): 1, + ("register", "INSERT INTO", "loom_router.hosts"): 1, + ("register", "INSERT INTO", "loom_router.audit"): 1, + ("_place_one", "INSERT INTO", "loom_router.placements"): 1, + ("_place_one", "UPDATE", "loom_router.cells"): 1, + ("_place_one", "INSERT INTO", "loom_router.audit"): 1, + ("_audit", "INSERT INTO", "loom_router.audit"): 1, + }, +} + +# Unresolved table SQL outside cell controls is classified by exact function and +# target expression, never by a whole-file or execute/f-string exemption. These are +# ClickHouse content/retention/rebuild stores, Keeper locks/marks and provider tables; +# pg_migrate writes only its named migration ledger, and prepare's nonce writes only +# the probe lock table. None is a cell acceptance or directory control write. +GENERIC_SQL_EXCLUSIONS = { + "server/cli/cell_prepare.py": {("_write_nonces", "INSERT INTO", "{leases.LOCK_TABLE}"): 1}, + "server/code/store/index/prefix_twin.py": {("fill_sql", "INSERT INTO", "{table}"): 1}, + "server/core/ch.py": {("_fenced_statement", "INSERT INTO", "{qualify(table)}"): 1}, + "server/core/leases.py": { + ("", "INSERT INTO", "{LOCK_TABLE}"): 1, + ("", "ALTER TABLE", "{LOCK_TABLE}"): 4, + }, + "server/core/namespace_write_marks.py": {("", "INSERT INTO", "{KEYS_TABLE}"): 1}, + "server/core/schema/migrate_lease.py": {("", "CREATE TABLE IF NOT EXISTS", "{LOCK_TABLE}"): 1}, + "server/core/schema/pg_migrate.py": { + ("_ledger", "CREATE TABLE IF NOT EXISTS", "{schema}"): 1, + ("_record_online", "INSERT INTO", "{schema}"): 1, + ("_apply", "INSERT INTO", "{schema}"): 1, + }, + "server/core/schema/table_rebuild.py": { + ("copy_sql", "INSERT INTO", "loom.{dst}"): 1, + ("apply", "DROP TABLE", "loom.{twin}"): 2, + }, + "server/erasure/store/mutations.py": { + ("run_mutation", "ALTER TABLE", "{table}"): 1, + ("run_lightweight_delete", "DELETE FROM", "{table}"): 1, + ("run_keeper_delete", "ALTER TABLE", "{table}"): 1, + }, + "server/erasure/store/reclaim.py": {("apply_deleted_mask", "ALTER TABLE", "{table}"): 1}, + "server/erasure/store/steps_memory.py": { + ("_scrub_link_cursors", "DELETE FROM", "{_CURSORS}"): 1 + }, + "server/erasure/store/steps_principals.py": {("_rewrite", "INSERT INTO", "{table}"): 1}, + "server/insights/domain/analysis.py": {("entity_sql", "INSERT INTO", "{db}"): 1}, + "server/insights/domain/provider_prepared_ddl.py": { + ("prepared_ddl", "CREATE TABLE", "loom.{prepared}"): 1 + }, + "server/insights/store/provider_cache.py": { + ("invalidate_staged_cache", "TRUNCATE TABLE", "loom.{names.target}"): 1 + }, + "server/insights/store/provider_cleanup.py": { + ("cleanup_rebuild_copies", "DROP TABLE IF EXISTS", "{db}"): 1 + }, + "server/insights/store/provider_participants.py": {("", "ALTER TABLE", "{TABLE}"): 1}, + "server/insights/store/provider_prefix.py": { + ("derive_staged_prefixes", "TRUNCATE TABLE", "loom.{names.target}"): 1, + ("derive_staged_prefixes", "INSERT INTO", "loom.{names.target}"): 1, + }, + "server/insights/store/provider_prepared.py": { + ("prepare_live_table", "DROP TABLE", "loom.{names.prepared}"): 1, + ("prepare_live_table", "INSERT INTO", "loom.{names.prepared}"): 1, + ("prepare_live_table", "ALTER TABLE", "loom.{names.prepared}"): 1, + }, + "server/insights/store/provider_staging.py": { + ("create_snapshot", "CREATE TABLE", "loom.{names.snapshot}"): 1, + ("complete_snapshot", "ALTER TABLE", "loom.{names.snapshot}"): 1, + ("discard_snapshot", "DROP TABLE", "loom.{names.snapshot}"): 1, + ("snapshot_query_running", "CREATE TABLE", "loom.{names.snapshot}"): 1, + }, + "server/insights/store/provider_target.py": { + ("ensure_target", "CREATE TABLE", "loom.{names.target}"): 1, + ("_insert_patches", "INSERT INTO", "loom.{names.target}"): 1, + }, + "server/knowledge/store/blobs.py": { + ("copy_held", "INSERT INTO", "{TABLE}"): 1, + ("unwrite", "DELETE FROM", "{TABLE}"): 1, + ("delete", "DELETE FROM", "{TABLE}"): 1, + ("delete", "DELETE FROM", "{UPLOADS}"): 1, + ("forget_uploads", "DELETE FROM", "{UPLOADS}"): 1, + }, + "server/knowledge/store/chunks.py": { + ("_record_supersessions", "INSERT INTO", "{SUPERSESSIONS_TABLE}"): 1 + }, + "server/knowledge/store/conversations.py": { + ("mark_message_saved", "ALTER TABLE", "{table}"): 1, + ("unsave_message", "DELETE FROM", "{table}"): 1, + ("unsave_message", "ALTER TABLE", "{table}"): 1, + }, + "server/knowledge/store/datasets.py": { + ("forget", "DELETE FROM", "{ROWS}"): 1, + ("forget", "DELETE FROM", "{DATASETS}"): 1, + ("forget_published", "DELETE FROM", "{ROWS}"): 1, + ("forget_published", "DELETE FROM", "{DATASETS}"): 1, + ("copy_snapshot", "INSERT INTO", "{ROWS}"): 1, + ("forget_snapshot", "DELETE FROM", "{ROWS}"): 1, + }, + "server/knowledge/store/gaps.py": { + ("delete_miss", "DELETE FROM", "{MISSES}"): 1, + ("delete_build", "DELETE FROM", "{GAPS}"): 1, + ("delete_misses", "DELETE FROM", "{MISSES}"): 1, + ("delete_builds", "DELETE FROM", "{GAPS}"): 1, + }, + "server/knowledge/store/restore.py": {("unretire", "DELETE FROM", "{SUPERSESSIONS_TABLE}"): 1}, + "server/memory/app/reembed.py": { + ("in_database_sql", "INSERT INTO", "loom.{table}"): 1, + ("delete_off_model", "DELETE FROM", "loom.{table}"): 1, + }, + "server/memory/store/code_links.py": { + ("_purge_tombstones", "DELETE FROM", "{TABLE}"): 1, + ("_drop_gone", "DELETE FROM", "{TABLE}"): 1, + }, + "server/memory/store/graph_projections.py": {("KeeperLayout", "INSERT INTO", "{TABLE}"): 1}, + "server/memory/store/lifecycle.py": {("_tombstone_where", "ALTER TABLE", "{table}"): 1}, + "server/memory/store/redaction.py": {("delete_by_key", "ALTER TABLE", "{table}"): 1}, + "server/memory/store/term_freq.py": {("rebuild", "ALTER TABLE", "{table}"): 1}, + "server/memory/store/webhooks.py": {("mark_dead", "INSERT INTO", "{ATTEMPTS}"): 1}, + "server/memory/workers/usage_fold.py": { + ("_apply_update", "UPDATE", "{table}"): 1, + ("_apply_update", "ALTER TABLE", "{table}"): 1, + }, + "server/storage/retention_overrides.py": { + ("compute_alters", "ALTER TABLE", "{target.table}"): 1, + ("current_ttl_matches", "ALTER TABLE", "{table}"): 1, + ("apply", "ALTER TABLE", "{target.table}"): 1, + }, +} + +# Schema creation/extension and immutable-trigger installation run as migrations, +# never as live acceptance writers. Offline restore generation rotation stays stopped +# and fenced as described in the recovery guide, not a live SQL exemption. +MIGRATION_WRITERS = { + "schemas/pg_migrations/008_cells.sql": { + ("CREATE TABLE IF NOT EXISTS", "loom.cell_first"): 1, + ("DROP TRIGGER", "loom.cell_first"): 2, + ("CREATE TRIGGER", "loom.cell_first"): 2, + ("ALTER TABLE", "loom.cell_first"): 2, + ("CREATE TABLE IF NOT EXISTS", "loom.tenancy_mode"): 1, + ("CREATE TABLE IF NOT EXISTS", "loom.cell_self"): 1, + }, + "schemas/router_migrations/001_directory.sql": { + ("CREATE TABLE IF NOT EXISTS", "loom_router.cells"): 1, + ("CREATE TABLE IF NOT EXISTS", "loom_router.placements"): 1, + ("DROP TRIGGER", "loom_router.placements"): 2, + ("CREATE TRIGGER", "loom_router.placements"): 2, + ("CREATE TABLE IF NOT EXISTS", "loom_router.hosts"): 1, + ("CREATE TABLE IF NOT EXISTS", "loom_router.audit"): 1, + }, + "schemas/router_migrations/002_cell_upstream.sql": {("ALTER TABLE", "loom_router.cells"): 1}, + "schemas/router_migrations/003_cell_next_credential.sql": { + ("ALTER TABLE", "loom_router.cells"): 1 + }, + "schemas/router_migrations/005_control_generation.sql": { + ("ALTER TABLE", "loom_router.cells"): 1 + }, +} + + +def calls(node: ast.AST) -> set[str]: + return {ast.unparse(child) for child in ast.walk(node) if isinstance(child, ast.Call)} + + +def function(path: str, name: str) -> ast.AST: + return inventory((ROOT / path).read_text()).functions[name] + + +@pytest.mark.parametrize( + ("path", "name", "required"), + [ + ( + "server/cli/cell_prepare.py", + "prepare", + [ + 'cell_control.run("prepare", lambda owner: _prepare_locked(ch, args, router, transport, owner))' + ], + ), + ( + "server/cli/cell_prepare.py", + "_prepare_locked", + [ + "cell_control.claim(conn, call, snapshot)", + "owner.call(lambda: _write_rows(conn, rows, answer, current, nxt), total_s=cell_control.TRANSACTION_S)", + ], + ), + ( + "server/cli/cell_prepare.py", + "_write_rows", + ["cell_control.transaction(conn, write)", "_read_rows(conn, count_tenancy=False)"], + ), + ("server/cli/cell_attach.py", "Run.run", ['cell_control.run("attach", self._owned)']), + ( + "server/cli/cell_attach_pg.py", + "_mode_try", + [ + "owner.call(lambda: _mode_transaction(owner.conn, lock_ms, mark, expected_cell, expected_fp), total_s=bound_s)" + ], + ), + ( + "server/cli/cell_control.py", + "Owner.transaction", + ["self.call(lambda: transaction(self.conn, work), total_s=TRANSACTION_S)"], + ), + ( + "server/cli/cell_attach.py", + "Run._owned", + ["cell_control.claim(self.lock_conn, self.router.call, self.row)"], + ), + ( + "server/cli/cell_attach.py", + "Run.commit_mode", + [ + "apg.commit_enforced(self.owner, self.confirm_lock, self.ready_for_commit, self.wait, self.log, mark, expected_cell=self.name, expected_fp=self.own_fp)" + ], + ), + ( + "server/cli/cell_attach_pg.py", + "_mode_transaction", + ["cell_control.check_lock(conn)", "conn.transaction()", "tx.commit()"], + ), + ( + "server/cli/cell_control.py", + "transaction", + ["check_lock(conn)", "tx.commit()"], + ), + ( + "server/cli/cell_control.py", + "claim", + ["check_lock(conn)", "paired(conn, row)"], + ), + ( + "server/cli/cell_control.py", + "backout", + [ + "claim(conn, call, row)", + 'mutate(conn, call, row, "/limit", {"closed": True})', + "owner.transaction(clear_state)", + "owner.transaction(clear_triggers)", + "paired(conn, row)", + ], + ), + ( + "server/cli/cell.py", + "_remove_locked", + [ + 'cell_control.run("remove", remove)', + "cell_control.claim(conn, call, row)", + "cell_control.check_lock(conn)", + ], + ), + ( + "server/router/directory.py", + "_control_transaction", + [ + "owner.start(lambda: owner.work(change))", + "owner.wait(task, owner.work_end)", + "owner.finish(task)", + ], + ), + ( + "server/router/directory.py", + "_ControlOwner.work", + [ + "self.context.__aenter__()", + "self.conn.transaction()", + "tx.start()", + "change(self.conn)", + "tx.commit()", + ], + ), + ( + "server/router/directory.py", + "_attach_change", + [ + "_control_transaction(locked, seconds=CONTROL_MAX_S)", + "control_refusal(attaching, expected, claim=claim)", + "change(conn, attaching)", + ], + ), + ( + "server/router/directory.py", + "_operator_change", + ["_attach_change(cell_name, expected, change_locked)", "change(conn, found.row)"], + ), + ( + "server/router/directory.py", + "set_limit", + ['_operator_change(cell_name, "limit", change, expected)'], + ), + ( + "server/router/directory.py", + "rotate_credential", + ['_operator_change(cell_name, "credential", change, expected)'], + ), + ( + "server/router/directory.py", + "claim", + ["_attach_change(cell_name, expected, change_locked, claim=True)"], + ), + *[ + ( + "server/router/directory.py", + name, + ["_attach_change(cell_name, expected, change_locked)"], + ) + for name in ( + "attach_begin", + "attach_place", + "attach_enforced", + "attach_serving", + "remove", + ) + ], + ], +) +def test_guarded_entrypoints_keep_shared_boundaries(path, name, required): + actual = calls(function(path, name)) + for expression in required: + assert ast.unparse(ast.parse(expression, mode="eval").body) in actual, ( + path, + name, + expression, + ) + + +def test_dynamic_cleanup_scope_cannot_expand_unclassified(): + backout = function("server/cli/cell_control.py", "backout") + cleanup_tables = [ + ast.literal_eval(node.iter) + for node in ast.walk(backout) + if isinstance(node, ast.For) + and ast.unparse(node.target) == "table" + and isinstance(node.iter, ast.Tuple) + ] + assert cleanup_tables == [("cell_first", "tenancy_mode", "cell_self", "tenancy")] + assert "t.tgname IN ('tenancy_insert', 'tenancy_update', 'tenancy_record')" in ast.unparse( + backout + ) + + +def test_reads_registration_and_immutable_placement_are_explicit_exceptions(): + path = "server/router/directory.py" + found = inventory((ROOT / path).read_text()) + for name in ( + "routing", + "credentials", + "first_cell", + "first_cell_named", + "lookup", + "attach_read", + ): + assert name in found.functions + assert not any(owner == name for owner, _verb, _target in found.writes) + # Operator reads may audit, but cannot bypass control guards for a state write. + assert any(call.startswith("_audit(") for call in calls(found.functions["attach_read"])) + assert "_rerun_refusal(held, reg)" in calls(found.functions["register"]) + assert "_place_one(conn, org, cell_name, _cell(locked), refusal)" in calls( + found.functions["place"] + ) + assert "refusal(candidate)" in calls(found.functions["_place_one"]) + for name in ("set_limit", "rotate_credential"): + assert not any(call.startswith("claim(") for call in calls(found.functions[name])) + + +@pytest.mark.parametrize( + ("statement", "expected"), + [ + ('"UPDATE loom.cell_self SET current_fp = $1"', ("UPDATE", "loom.cell_self")), + ( + '"WITH expected AS (SELECT 1) UPDATE loom.cell_self SET current_fp = $1"', + ("UPDATE", "loom.cell_self"), + ), + ( + '"WITH expected AS (SELECT 1) DELETE FROM loom_router.cells WHERE cell = $1"', + ("DELETE FROM", "loom_router.cells"), + ), + ( + '"WITH changed AS (UPDATE loom.cell_self SET current_fp = $1 RETURNING 1) SELECT * FROM changed"', + ("UPDATE", "loom.cell_self"), + ), + ( + '"WITH outer_cte AS (WITH inner_cte AS (SELECT 1) UPDATE loom_router.cells SET closed = true RETURNING cell) SELECT * FROM outer_cte"', + ("UPDATE", "loom_router.cells"), + ), + ( + '"INSERT INTO loom_router.cells (cell) VALUES ($1)"', + ("INSERT INTO", "loom_router.cells"), + ), + ('f"DELETE FROM loom.{table}"', ("DELETE FROM", "loom.{table}")), + ('f"UPDATE {target} SET current_fp = $1"', ("UPDATE", "{target}")), + ('"DROP TABLE IF EXISTS loom.cell_self"', ("DROP TABLE IF EXISTS", "loom.cell_self")), + ('f\'DROP TRIGGER "{name}" ON loom."{table}"\'', ("DROP TRIGGER", "loom.{table}")), + ], +) +def test_inventory_detects_new_literal_and_dynamic_writers(statement, expected): + source = ( + f"async def unclassified(conn, table, target, name):\n await conn.execute({statement})\n" + ) + found = inventory(source).writes + assert found == Counter({("unclassified", *expected): 1}) + assert not any( + "unclassified" == owner + for group in (GUARDED_WRITERS, EXEMPT_WRITERS, GENERIC_SQL_EXCLUSIONS) + for entries in group.values() + for owner, _verb, _target in entries + ) + + +def test_acceptance_writer_callsites_are_classified(): + # Another caller of a checked writer still needs its own claim and ownership proof. + names = {"_write_rows", "_mode_try", "_mode_transaction", "commit_enforced", "backout"} + actual = Counter() + for path in (ROOT / "server").rglob("*.py"): + found = inventory(path.read_text()) + for owner, node in found.functions.items(): + # These owners have no nested functions; walk only the direct callsites. + for child in ast.walk(node): + if isinstance(child, ast.Call): + target = ast.unparse(child.func) + if target.rsplit(".", 1)[-1] in names: + actual[(str(path.relative_to(ROOT)), owner, target)] += 1 + assert actual == Counter( + { + ("server/cli/cell_prepare.py", "_prepare_locked", "_write_rows"): 1, + ("server/cli/cell_attach_pg.py", "commit_enforced", "_mode_try"): 1, + ("server/cli/cell_attach_pg.py", "_mode_try", "_mode_transaction"): 1, + ("server/cli/cell_attach.py", "Run.commit_mode", "apg.commit_enforced"): 1, + } + ) + + +def test_cte_data_modification_classifies_every_literal_sink(): + source = """ +async def unclassified(conn): + await conn.execute("WITH changed AS (DELETE FROM loom.cell_self RETURNING 1) " + "/* body done */ UPDATE loom_router.cells SET closed = true") +""" + assert inventory(source).writes == Counter( + { + ("unclassified", "DELETE FROM", "loom.cell_self"): 1, + ("unclassified", "UPDATE", "loom_router.cells"): 1, + } + ) diff --git a/tests/unit/test_cell_native_http_first_error.py b/tests/unit/test_cell_native_http_first_error.py new file mode 100644 index 0000000000..c9b61bd146 --- /dev/null +++ b/tests/unit/test_cell_native_http_first_error.py @@ -0,0 +1,259 @@ +"""Native HTTP clients preserve the first outcome through required exit.""" + +from __future__ import annotations + +import argparse +import asyncio +from types import SimpleNamespace + +import httpx +import pytest + +from server.cli import cell_attach, cell_control, cell_prepare +from server.settings import settings + +_CLIENT = httpx.AsyncClient +_BOUNDARIES = ("control", "health", "register") +FIRST_ERRORS = ( + ValueError, + httpx.ReadError, + TimeoutError, + asyncio.CancelledError, + SystemExit, + KeyboardInterrupt, +) +EXIT_ERRORS = ( + RuntimeError, + ValueError, + httpx.ReadError, + TimeoutError, + asyncio.CancelledError, + SystemExit, + KeyboardInterrupt, +) + + +@pytest.fixture +def exchange(monkeypatch): + state = SimpleNamespace( + first=None, + second=None, + phase="request", + close_phase="await", + events=[], + exit_args=None, + entered=asyncio.Event(), + block=False, + answer={"org": "acme", "cell": "eu-1", "deployment_id": "dep-eu-1"}, + status=200, + close_entered=asyncio.Event(), + close_block=False, + ) + + async def handler(request): + state.events.append("request") + state.entered.set() + if state.block: + try: + await asyncio.Future() + except asyncio.CancelledError as error: + state.first = error + raise + if state.first is not None: + raise state.first + return httpx.Response(state.status, json=state.answer) + + class Client(_CLIENT): + async def __aenter__(self): + state.events.append("enter") + if state.phase == "enter": + raise state.first + return await super().__aenter__() + + def __aexit__(self, *args): + state.exit_args = args + state.events.append("exit-call") + if state.close_phase == "construct" and state.second is not None: + raise state.second + + async def close(): + state.events.append("exit-await") + state.close_entered.set() + if state.close_block: + await asyncio.Future() + await super(Client, self).__aexit__(*args) + if state.second is not None: + raise state.second + + return close() + + def client(**kwargs): + state.events.append("construct") + if state.phase == "construct": + raise state.first + kwargs["transport"] = httpx.MockTransport(handler) + return Client(**kwargs) + + monkeypatch.setattr(httpx, "AsyncClient", client) + monkeypatch.setattr(settings, "router_operator_token", "inert-test-token") + return state + + +async def invoke(boundary): + url = httpx.URL("https://router.test") + if boundary == "control": + return await cell_control.request(url, "eu-1", None, "/claim", {}) + if boundary == "health": + return await cell_attach.Router(url, "eu-1", None).health("eu-1.example.test") + return await cell_prepare._register(argparse.Namespace(cell="eu-1", wait=0), url, {}, None) + + +async def observed(callback): + # Native fatal values must stay in this task rather than reach the event loop. + try: + return "value", await callback() + except BaseException as error: # noqa: BLE001 — compare original native failure objects. + return "error", error + + +def assert_first(boundary, state, outcome): + kind, result = outcome + if boundary == "health" and isinstance(state.first, (httpx.HTTPError, ValueError)): + assert (kind, result) == ("value", "") + elif (boundary == "control" and isinstance(state.first, (httpx.HTTPError, TimeoutError))) or ( + boundary == "register" + and state.phase == "request" + and isinstance(state.first, httpx.HTTPError) + ): + expected = cell_prepare.Refused if boundary == "register" else cell_control.Refused + assert kind == "error" and isinstance(result, expected) + assert type(state.first).__name__ in str(result) + else: + assert kind == "error" and result is state.first + + +def assert_exit_first(boundary, state): + first = state.exit_args[1] + if boundary == "register" and isinstance(state.first, httpx.HTTPError): + assert isinstance(first, cell_prepare.Refused) + else: + assert first is state.first + + +def assert_close_failure(boundary, state, outcome): + kind, result = outcome + if boundary == "control" and isinstance(state.second, (httpx.HTTPError, TimeoutError)): + assert kind == "error" and isinstance(result, cell_control.Refused) + else: + assert kind == "error" and result is state.second + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +@pytest.mark.parametrize("second_kind", (*EXIT_ERRORS, None)) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +async def test_request_first_outcome_survives_client_exit( + exchange, boundary, first_kind, second_kind, close_phase +): + exchange.first = first_kind("request-first") + exchange.second = second_kind("exit-second") if second_kind is not None else None + exchange.close_phase = close_phase + assert_first(boundary, exchange, await observed(lambda: invoke(boundary))) + assert exchange.events[:3] == ["construct", "enter", "request"] + assert exchange.events.count("exit-call") == 1 + assert_exit_first(boundary, exchange) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +async def test_required_client_exit_failure_cannot_report_success( + exchange, boundary, second_kind, close_phase +): + exchange.second = second_kind("required-exit") + exchange.close_phase = close_phase + assert_close_failure(boundary, exchange, await observed(lambda: invoke(boundary))) + assert exchange.exit_args == (None, None, None) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +async def test_successful_client_neighbor(exchange, boundary): + kind, result = await observed(lambda: invoke(boundary)) + assert kind == "value" + expected = "dep-eu-1" if boundary == "health" else exchange.answer + assert result == expected + assert exchange.events == ["construct", "enter", "request", "exit-call", "exit-await"] + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("phase", ("construct", "enter")) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +async def test_client_acquisition_keeps_its_first_outcome(exchange, boundary, phase, first_kind): + exchange.phase = phase + exchange.first = first_kind("acquire-first") + assert_first(boundary, exchange, await observed(lambda: invoke(boundary))) + assert "request" not in exchange.events and "exit-call" not in exchange.events + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +async def test_actual_task_cancellation_keeps_its_payload(exchange, boundary, second_kind): + exchange.block = True + exchange.second = second_kind("exit-second") + task = asyncio.create_task(observed(lambda: invoke(boundary))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("operator-first-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) + assert result.args == ("operator-first-stop",) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +async def test_repeated_cancellation_during_http_exit_keeps_first_payload(exchange, boundary): + exchange.block = True + exchange.close_block = True + task = asyncio.create_task(observed(lambda: invoke(boundary))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("first-stop") + await asyncio.wait_for(exchange.close_entered.wait(), 1) + task.cancel("second-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) and result.args == ("first-stop",) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +@pytest.mark.parametrize("semantic", ("malformed", "backend", "probe", "refusal")) +async def test_registration_selected_refusal_survives_exit( + monkeypatch, exchange, second_kind, close_phase, semantic +): + selected = cell_prepare.Refused("registration-first") + monkeypatch.setattr(cell_prepare, "Refused", lambda message: selected) + exchange.status = 200 if semantic == "malformed" else 503 + exchange.answer = ( + {"unexpected": True} + if semantic == "malformed" + else { + "error": "registration-first", + "code": {"backend": "BACKEND_UNAVAILABLE", "probe": "PROBE_BUSY", "refusal": "OTHER"}[ + semantic + ], + } + ) + exchange.second = second_kind("exit-second") + exchange.close_phase = close_phase + kind, result = await observed(lambda: invoke("register")) + assert kind == "error" and result is selected + assert exchange.exit_args[1] is selected diff --git a/tests/unit/test_cell_native_nonce_first_error.py b/tests/unit/test_cell_native_nonce_first_error.py new file mode 100644 index 0000000000..94254c3d96 --- /dev/null +++ b/tests/unit/test_cell_native_nonce_first_error.py @@ -0,0 +1,269 @@ +"""Native callers retain their first outcome while deleting their probe nonces.""" + +from __future__ import annotations + +import argparse +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import httpx +import pytest + +from server.cli import cell_attach, cell_control, cell_prepare +from server.core import leases, object_store, pg +from server.settings import settings +from tests.unit import test_cell_native_http_first_error as native_http +from tests.unit.test_cell_attach import FakeConn +from tests.unit.test_cell_control import snapshot + +exchange = native_http.exchange +observed = native_http.observed +FIRST_ERRORS = native_http.FIRST_ERRORS +CALLERS = ("prepare", "begin", "serving") +FATAL_ERRORS = (asyncio.CancelledError, SystemExit, KeyboardInterrupt) +CLEANUP_PHASES = ("postgres", "clickhouse", "bucket") +URL = httpx.URL("http://router.test") + + +@pytest.fixture +def nonce_edges(monkeypatch, exchange): + state = SimpleNamespace( + secondary=None, + phase="postgres", + write_first=False, + events=[], + normalized=[], + cleanup_entered=asyncio.Event(), + cleanup_block=False, + ) + nonces = {"postgres": "pg", "clickhouse": "ch", "bucket": "bucket"} + + async def deletion(phase): + state.events.append(phase) + if phase == state.phase: + state.cleanup_entered.set() + if state.cleanup_block: + await asyncio.Future() + if state.secondary is not None: + raise state.secondary + + async def postgres(*args): + await deletion("postgres") + + async def clickhouse(*args): + assert args[0] == leases.LOCK_RELEASE + await deletion("clickhouse") + + async def read_object(location): + return b"bucket" + + async def bucket(*args): + await deletion("bucket") + + async def write(*args): + if state.write_first: + raise exchange.first + + monkeypatch.setattr(pg, "execute", postgres) + monkeypatch.setattr(object_store, "root_prefix", lambda: "") + monkeypatch.setattr(object_store, "read", read_object) + monkeypatch.setattr(object_store, "delete", bucket) + monkeypatch.setattr(cell_prepare, "_new_nonces", lambda: nonces.copy()) + monkeypatch.setattr(cell_prepare, "_write_nonces", write) + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "document_s3_bucket", "bucket-test") + state.ch = SimpleNamespace(execute=clickhouse) + return state + + +@pytest.fixture +def native_call(monkeypatch, nonce_edges, exchange): + state = nonce_edges + row = {**snapshot(), "is_first": False, "state": "attaching", "mode": "enforced"} + exchange.answer = row.copy() + conn = FakeConn() + rows = SimpleNamespace(deployment_id="dep", first=None, tenancy_rows=0) + monkeypatch.setattr(cell_prepare, "_read_rows", AsyncMock(return_value=rows)) + monkeypatch.setattr(cell_prepare, "_read_beats", AsyncMock(return_value=[])) + monkeypatch.setattr(cell_prepare, "_beat_fields", lambda *args: [{}]) + monkeypatch.setattr(cell_prepare, "_state_rows", lambda rows: 0) + monkeypatch.setattr(cell_prepare, "_prefix_objects", AsyncMock(return_value=0)) + monkeypatch.setattr(cell_prepare, "_answer_problem", lambda *args: "") + monkeypatch.setattr(cell_prepare, "_own_rows", lambda *args: ({}, None)) + state.write_rows = AsyncMock(return_value=[]) + monkeypatch.setattr(cell_prepare, "_write_rows", state.write_rows) + monkeypatch.setattr(cell_control, "claim", AsyncMock(side_effect=lambda conn, call, row: row)) + args = argparse.Namespace( + cell="first", + wait=0, + host="first.example.test", + credential_sha256="hash", + probe_url="", + ch_allow_user=[], + ch_allow_user_directory=[], + ch_allow_object=[], + ch_allow_replica=[], + pg_allow_role=[], + ) + + async def mutate(path, body): + return await cell_control.request(URL, "first", None, path, body) + + run = SimpleNamespace( + ch=state.ch, args=args, row=row, own_fp=({}, None), keys_fp="key", mutate=mutate + ) + + async def invoke(caller, owned): + owner = cell_control.Owner(exit_s=0.5) + direct = SimpleNamespace( + conn=conn, transaction=lambda work: work(), call=lambda work, **kw: work() + ) + + async def work(): + if caller == "prepare": + return await cell_prepare._prepare_locked( + state.ch, args, URL, None, owner if owned else direct + ) + return await getattr(cell_attach.Run, caller)(run) + + async def opening(): + owner.conn = conn + + return await owner.run(opening, work, opening_s=0.5) if owned else await work() + + state.conn = conn + state.invoke = invoke + return state + + +@pytest.fixture(autouse=True) +def selected_normalization(monkeypatch, nonce_edges): + original_prepare = cell_prepare.Refused + original_control = cell_control._refused + + class PrepareRefused(original_prepare): + def __init__(self, message): + super().__init__(message) + nonce_edges.normalized.append(self) + + def control_refused(message): + selected = original_control(message) + nonce_edges.normalized.append(selected) + return selected + + monkeypatch.setattr(cell_prepare, "Refused", PrepareRefused) + monkeypatch.setattr(cell_control, "_refused", control_refused) + + +def assert_first(state, exchange, caller, first_kind, owned, outcome): + kind, result = outcome + assert kind == "error" + normalized = not state.write_first and ( + (caller == "prepare" and issubclass(first_kind, httpx.HTTPError)) + or (caller != "prepare" and issubclass(first_kind, (httpx.HTTPError, TimeoutError))) + ) + if normalized: + assert state.normalized and result is state.normalized[0] + elif owned and first_kind is asyncio.CancelledError: + # asyncio reconstructs cancellation when Owner retrieves its child Task. + assert isinstance(result, asyncio.CancelledError) + else: + assert result is exchange.first + assert state.write_rows.await_count == 0 + assert state.conn.executed.count("CLOSE") == int(owned) + + +@pytest.mark.parametrize("caller", CALLERS) +@pytest.mark.parametrize("owned", (False, True)) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +@pytest.mark.parametrize("cleanup_kind", (*FATAL_ERRORS, None)) +@pytest.mark.parametrize("phase", CLEANUP_PHASES) +@pytest.mark.parametrize("http_exit_failure", (False, True)) +async def test_http_first_outcome_survives_native_nonce_cleanup( + native_call, exchange, caller, owned, first_kind, cleanup_kind, phase, http_exit_failure +): + state = native_call + state.phase = phase + state.secondary = cleanup_kind("nonce-second") if cleanup_kind is not None else None + exchange.first = first_kind("request-first") + exchange.second = KeyboardInterrupt("http-exit-second") if http_exit_failure else None + assert_first( + state, + exchange, + caller, + first_kind, + owned, + await observed(lambda: state.invoke(caller, owned)), + ) + assert phase in state.events and len(exchange.exit_args) == 3 + + +@pytest.mark.parametrize("caller", CALLERS) +@pytest.mark.parametrize("owned", (False, True)) +@pytest.mark.parametrize("cleanup_kind", (*FATAL_ERRORS, ValueError, None)) +@pytest.mark.parametrize("phase", CLEANUP_PHASES) +async def test_standalone_nonce_cleanup_keeps_cancellation_and_success_neighbors( + native_call, exchange, caller, owned, cleanup_kind, phase +): + state = native_call + state.phase = phase + state.secondary = cleanup_kind("nonce-only") if cleanup_kind is not None else None + kind, result = await observed(lambda: state.invoke(caller, owned)) + if cleanup_kind in FATAL_ERRORS: + assert kind == "error" + if owned and cleanup_kind is asyncio.CancelledError: + assert isinstance(result, asyncio.CancelledError) + else: + assert result is state.secondary + assert state.write_rows.await_count == 0 + else: + assert kind == "value" + assert result == ( + 0 if caller == "prepare" else None if caller == "begin" else exchange.answer + ) + assert state.events == list(CLEANUP_PHASES) + assert state.write_rows.await_count == int(caller == "prepare") + assert state.conn.executed.count("CLOSE") == int(owned) + + +@pytest.mark.parametrize("caller", CALLERS) +@pytest.mark.parametrize("owned", (False, True)) +@pytest.mark.parametrize("cleanup_kind", FATAL_ERRORS) +async def test_nonce_write_failure_stays_first(native_call, exchange, caller, owned, cleanup_kind): + state = native_call + state.write_first = True + state.secondary = cleanup_kind("nonce-second") + exchange.first = ValueError("write-first") + assert_first( + state, + exchange, + caller, + ValueError, + owned, + await observed(lambda: state.invoke(caller, owned)), + ) + assert exchange.events == [] and state.events == ["postgres"] + + +@pytest.mark.parametrize("caller", CALLERS) +async def test_repeated_cancellation_keeps_first_stop_through_nonce_deletion( + native_call, exchange, caller +): + state = native_call + state.cleanup_block = True + exchange.block = True + task = asyncio.create_task(observed(lambda: state.invoke(caller, False))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("first-stop") + await asyncio.wait_for(state.cleanup_entered.wait(), 1) + task.cancel("second-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) and result.args == ("first-stop",) + assert state.write_rows.await_count == 0 + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) diff --git a/tests/unit/test_cell_operator.py b/tests/unit/test_cell_operator.py index 63b1e00d26..5c37b78cb0 100644 --- a/tests/unit/test_cell_operator.py +++ b/tests/unit/test_cell_operator.py @@ -8,6 +8,7 @@ import hashlib import json from typing import Any +from uuid import UUID import httpx import pytest @@ -23,6 +24,29 @@ SHA = "a" * 64 +OBSERVED = { + "cell": "eu-1", + "is_first": False, + "host": "eu-1.example", + "deployment_id": "dep-eu-1", + "bucket": "", + "bucket_prefix": "", + "probe_url": "", + "upstream_url": "", + "current_fp": None, + "next_fp": None, + "mutation_generation": str(UUID(int=1)), +} + + +@pytest.fixture(autouse=True) +def control_snapshot(monkeypatch: pytest.MonkeyPatch) -> None: + def read(url: str, **_kw: Any) -> httpx.Response: + return httpx.Response(200, json=OBSERVED, request=httpx.Request("GET", url)) + + monkeypatch.setattr(httpx, "get", read) + + def test_a_new_credential_is_lcr_and_128_hex_with_its_sha256(capsys) -> None: credential, sha = cell_cli.new_credential() assert credential.startswith("lcr_") and len(credential) == 132 @@ -110,7 +134,9 @@ def test_a_rotation_step_calls_the_operator_api(operator, kw, body) -> None: assert cell_cli._cmd_credential(_args(cell="eu-1", **kw)) == 0 (call,) = operator.calls assert call["url"] == "https://router.test/cells/v1/operator/cells/eu-1/credential" - assert call["json"] == body and call["headers"] == {"Authorization": f"Bearer {OPERATOR}"} + assert call["json"] == {**body, "expected": OBSERVED} and call["headers"] == { + "Authorization": f"Bearer {OPERATOR}" + } @pytest.mark.parametrize( @@ -124,7 +150,7 @@ def test_a_rotation_step_calls_the_operator_api(operator, kw, body) -> None: ) def test_a_limit_calls_the_operator_api(operator, kw, body) -> None: assert cell_cli._cmd_limit(_args(cell="eu-1", **kw)) == 0 - assert operator.calls[0]["json"] == body + assert operator.calls[0]["json"] == {**body, "expected": OBSERVED} assert operator.calls[0]["url"].endswith("/cells/v1/operator/cells/eu-1/limit") @@ -345,7 +371,7 @@ def test_a_page_that_is_not_the_routers_answer_fails(monkeypatch, capsys, comman assert cell_cli._cmd_limit(_args(cell="eu-1", closed=True)) == 1 err = capsys.readouterr().err assert "did not answer as the router does" in err - assert ("the change may have been made" in err) is (command == "limit") + assert "the change may have been made" not in err # failed precondition read sent no mutation def _parse(*argv: str) -> argparse.Namespace: @@ -388,6 +414,7 @@ def _row(**over: Any) -> directory.Cell: "placed": 3, "credential_sha256": "b" * 64, "credential_next_sha256": None, + "mutation_generation": UUID(int=1), "deployment_id": "dep-eu-1", "current_fp": None, "next_fp": None, @@ -403,12 +430,18 @@ def api(monkeypatch) -> tuple[TestClient, list[Any]]: calls: list[Any] = [] async def _limit(cell: str, **kw: Any) -> directory.Cell: + expected = kw.pop("expected") + assert expected.generation == UUID(int=1) + assert expected.values == {k: OBSERVED[k] for k in directory.CONTROL_FIELDS} calls.append(("limit", cell, kw)) if cell == "nope": raise directory.NoSuchCell("cell 'nope' is not in the directory") return _row(closed=kw.get("closed") is True) - async def _rotate(cell: str, step: str, sha: str = "") -> directory.Cell: + async def _rotate( + cell: str, step: str, sha: str = "", *, expected: directory.Control + ) -> directory.Cell: + assert expected.generation == UUID(int=1) calls.append(("credential", cell, step, sha)) if sha == "c" * 64: raise directory.OperatorRefused("cell 'eu-2' records that credential hash already") @@ -420,6 +453,8 @@ async def _rotate(cell: str, step: str, sha: str = "") -> directory.Cell: def _post(client: TestClient, path: str, body: Any, token: str = OPERATOR) -> Any: + if isinstance(body, dict): + body = {**body, "expected": OBSERVED} return client.post( f"/cells/v1/operator/cells/{path}", json=body, headers={"Authorization": f"Bearer {token}"} ) @@ -536,7 +571,7 @@ async def _credentials() -> dict[str, tuple[str, ...]]: reads.append(1) return dict(hashes) - async def _rotate(cell: str, step: str, sha: str = "") -> directory.Cell: + async def _rotate(cell: str, step: str, sha: str = "", **_kw: Any) -> directory.Cell: hashes["eu-1"] = ("b" * 64, sha) return _row(credential_next_sha256=sha) diff --git a/tests/unit/test_cell_org_triggers.py b/tests/unit/test_cell_org_triggers.py index 3f670d4014..5d25ba0177 100644 --- a/tests/unit/test_cell_org_triggers.py +++ b/tests/unit/test_cell_org_triggers.py @@ -86,6 +86,14 @@ async def execute(self, sql: str, *_a: Any) -> None: async def fetch(self, _sql: str, *_a: Any) -> list[Any]: return [] + def terminate(self) -> None: + self.statements.append("terminated") + + async def fetchval(self, sql: str, *_a: Any) -> bool: + assert sql in ("SELECT pg_try_advisory_lock($1)", "SELECT pg_advisory_unlock($1)") + self.statements.append(sql) + return True + @pytest.mark.parametrize("name", ["eu-1", ""]) @pytest.mark.parametrize("schema", ["loom", "loom_router"]) @@ -617,11 +625,17 @@ def __init__(self, error: Exception) -> None: self.error = error def transaction(self) -> Any: - @contextlib.asynccontextmanager - async def _tx() -> AsyncIterator[None]: - yield + class _Tx: + async def start(self) -> None: + pass + + async def commit(self) -> None: + pass + + async def rollback(self) -> None: + pass - return _tx() + return _Tx() async def fetchrow(self, _sql: str, *_a: Any) -> dict[str, Any]: return { @@ -695,7 +709,7 @@ def test_doctor_runs_the_cell_checks_only_with_loom_cell( monkeypatch.setattr(doctor, "_doctor_scim_token_check", lambda _s: ("scim_token", "PASS", "")) called: list[bool] = [] monkeypatch.setattr( - doctor_cell, "cell_checks", lambda: called.append(True) or [("cell_state", "PASS", "")] + doctor_cell, "cell_checks", lambda *_: called.append(True) or [("cell_state", "PASS", "")] ) names = [n for n, _g, _d in doctor._run_live_checks()] assert called == ([True] if runs else []) and ("cell_state" in names) == runs diff --git a/tests/unit/test_cell_place_owner.py b/tests/unit/test_cell_place_owner.py new file mode 100644 index 0000000000..b8dac073fb --- /dev/null +++ b/tests/unit/test_cell_place_owner.py @@ -0,0 +1,257 @@ +"""Placement reuses the control owner without replacing its phase budgets.""" + +from __future__ import annotations + +import asyncio +import time +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from server.cli import cell_control +from server.core import pg +from tests.unit.clock_fake import ManualClock +from tests.unit.test_cell_attach import FakeConn + +pytestmark = pytest.mark.asyncio + + +async def test_custom_exit_is_used_only_outside_a_local_phase(): + owner = cell_control.Owner(exit_s=0.09) + conn = FakeConn() + + async def opening(): + owner.conn = conn + + before = time.monotonic() + assert await owner.run(opening, AsyncMock(return_value=17), opening_s=0.3) == 17 + assert before + 0.09 <= owner.close_end <= time.monotonic() + 0.09 + assert conn.executed == ["CLOSE"] + + +@pytest.mark.parametrize("occupied", ["conn", "phase", "task", "closed", "repeated"]) +async def test_command_entry_refuses_reuse_before_any_callback(occupied): + owner = cell_control.Owner() + held = None + if occupied == "conn": + owner.conn = FakeConn() + elif occupied == "phase": + owner.phase_end = time.monotonic() + 1 + elif occupied == "task": + held = owner._start(lambda: asyncio.Future()) + elif occupied == "closed": + await owner.close() + else: + await owner.run(AsyncMock(), AsyncMock(), opening_s=0.1) + opening, body = AsyncMock(), AsyncMock() + try: + with pytest.raises(cell_control.Refused, match="fresh"): + await owner.run(opening, body, opening_s=0.1) + opening.assert_not_called() + body.assert_not_called() + finally: + if held is not None: + held.cancel() + await asyncio.gather(held, return_exceptions=True) + + +@pytest.mark.parametrize("stage", ["opening", "body"]) +async def test_recursive_entry_cannot_replace_the_original_backend(stage): + owner = cell_control.Owner() + original = FakeConn() + other_open, other_body = AsyncMock(), AsyncMock() + + async def recurse(): + with pytest.raises(cell_control.Refused, match="fresh"): + await owner.run(other_open, other_body, opening_s=0.1) + + async def opening(): + if stage == "opening": + await recurse() + owner.conn = original + + async def body(): + if stage == "body": + await recurse() + return 3 + + assert await owner.run(opening, body, opening_s=0.1) == 3 + other_open.assert_not_called() + other_body.assert_not_called() + assert owner.conn is original and original.executed == ["CLOSE"] + + +async def test_native_wrapper_keeps_its_existing_three_budgets(monkeypatch): + observed = {} + actual_run = cell_control.Owner.run + actual_call = cell_control.Owner.call + + async def wrapped_run(self, opening, work, *, opening_s): + observed.update(opening=opening_s, exit=self.exit_s) + return await actual_run(self, opening, work, opening_s=opening_s) + + async def wrapped_call(self, work, *, total_s): + observed.setdefault("phases", []).append(total_s) + return await actual_call(self, work, total_s=total_s) + + async def opening(self, what): + assert what == "native" + self.conn = FakeConn() + + monkeypatch.setattr(cell_control.Owner, "run", wrapped_run) + monkeypatch.setattr(cell_control.Owner, "call", wrapped_call) + monkeypatch.setattr(cell_control.Owner, "_open", opening) + assert ( + await cell_control.run("native", lambda owner: owner.transaction(AsyncMock(return_value=7))) + == 7 + ) + assert observed == {"opening": 75.0, "exit": pg._RELEASE_TIMEOUT_S, "phases": [75.0, 60.0]} + + +async def test_failed_phase_keeps_its_end_instead_of_custom_exit(): + owner = cell_control.Owner(exit_s=99) + conn = FakeConn() + + async def opening(): + owner.conn = conn + + async def body(): + return await owner.call(lambda: asyncio.Future(), total_s=0.04) + + before = time.monotonic() + with pytest.raises(TimeoutError): + await owner.run(opening, body, opening_s=0.1) + assert before < owner.close_end <= before + 0.1 + assert owner.close_end == owner.phase_end + assert conn.executed == ["CLOSE"] + + +async def test_public_close_defers_repeated_signals_until_acknowledged_within_original_end(): + events = [] + closing, finish = asyncio.Event(), asyncio.Event() + + class Connection: + async def close(self, **kwargs): + events.append("close") + closing.set() + await finish.wait() + events.append("ack") + + def terminate(self): + events.append("abort") + + def is_closed(self): + return "ack" in events + + owner = cell_control.Owner(exit_s=0.2) + owner.conn = Connection() + task = asyncio.create_task(owner.close()) + try: + await closing.wait() + original_end = owner.close_end + task.cancel("first signal") + await asyncio.sleep(0) + task.cancel("second signal") + await asyncio.sleep(0) + assert not task.done(), "a second signal must not interrupt owned public close" + assert owner.close_end == original_end + finish.set() + with pytest.raises(asyncio.CancelledError, match="first signal"): + await task + assert events == ["close", "ack"] + finally: + finish.set() + await asyncio.gather(task, return_exceptions=True) + + +async def test_repeated_signals_cannot_extend_an_expired_public_close(monkeypatch): + clock = ManualClock(t=100.0) + local_time = SimpleNamespace(monotonic=clock.monotonic) + monkeypatch.setattr(cell_control, "time", local_time) + monkeypatch.setattr(pg, "time", local_time) + closing = asyncio.Event() + conn = FakeConn() + + async def stalled(**kwargs): + closing.set() + await asyncio.Future() + + conn.close = stalled + owner = cell_control.Owner(exit_s=0.04) + owner.conn = conn + waits = [] + + async def signalled_wait(tasks, **kwargs): + assert tasks == {owner._close_task} + waits.append(kwargs["timeout"]) + assert len(waits) <= 2, "signals cannot grant another cleanup allowance" + await closing.wait() + if len(waits) == 1: + clock.advance(0.01) + else: + clock.t = 100.04 + asyncio.current_task().cancel("first signal" if len(waits) == 1 else "second signal") + await asyncio.sleep(0) + raise AssertionError("cancellation must reach the public close") + + # Replace only this module's wait seam; native close and the event loop stay real. + monkeypatch.setattr( + cell_control, + "asyncio", + SimpleNamespace( + create_task=asyncio.create_task, + CancelledError=asyncio.CancelledError, + wait=signalled_wait, + ), + ) + task = asyncio.create_task(owner.close()) + try: + done, _ = await asyncio.wait({task}, timeout=0.2) + assert done, "repeated signals cannot restart the original cleanup allowance" + with pytest.raises(asyncio.CancelledError, match="first signal"): + await task + assert waits == pytest.approx([0.04, 0.03]) + assert clock.monotonic() - 100.0 == pytest.approx(0.04) + assert owner.close_end == 100.04 + assert "TERMINATE" in conn.executed + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + if owner._close_task is not None: + await asyncio.gather(owner._close_task, return_exceptions=True) + + +async def test_closed_transport_cannot_acknowledge_pending_public_cancellation(): + closing, acknowledged = asyncio.Event(), asyncio.Event() + events = [] + + class Connection: + async def close(self, **kwargs): + closing.set() + await acknowledged.wait() + events.append("ack") + + def is_closed(self): + # An aborted transport is closed even while its cancellation is pending. + return True + + def terminate(self): + events.append("abort") + + owner = cell_control.Owner(exit_s=0.2) + owner.conn = Connection() + task = asyncio.create_task(owner.close()) + try: + await closing.wait() + task.cancel("stop") + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not task.done() and events == [] + acknowledged.set() + with pytest.raises(asyncio.CancelledError, match="stop"): + await task + assert events == ["ack"] + finally: + acknowledged.set() + await asyncio.gather(task, return_exceptions=True) diff --git a/tests/unit/test_cell_prepare.py b/tests/unit/test_cell_prepare.py index e9ea657df6..7c6e8f07fc 100644 --- a/tests/unit/test_cell_prepare.py +++ b/tests/unit/test_cell_prepare.py @@ -14,7 +14,7 @@ import pytest from server.cli import cell as cell_cli -from server.cli import cell_prepare +from server.cli import cell_control, cell_prepare from server.core import cell from server.settings import settings @@ -30,6 +30,21 @@ def _cell(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(settings, "pg_dsn", "postgresql://cell.internal/loom") monkeypatch.setattr(settings, "router_operator_token", OPERATOR) + # The attach lock's session is Postgres's (tests/integration/test_cell_attach.py). + class Owner: + conn = None + + async def transaction(self, work): + return await work() + + async def call(self, work, **_): + return await work() + + async def run(_what, work): + return await work(Owner()) + + monkeypatch.setattr(cell_control, "run", run) + def _args(**over: Any) -> argparse.Namespace: fields = { @@ -366,7 +381,7 @@ async def test_the_processes_live_after_the_wait_are_the_ones_judged(monkeypatch reads = [[_beat("i-old", before)], [_beat("i-new", after)]] written: list[Any] = [] - async def _rows_read() -> cell_prepare._CellRows: + async def _rows_read(_conn: Any) -> cell_prepare._CellRows: return _rows() async def _beats(_ch: Any) -> list[cell_prepare._Beat]: @@ -378,7 +393,7 @@ async def _nothing(*_a: Any) -> Any: async def _register(*_a: Any) -> dict[str, Any]: return {**OK, "registered": True} - async def _write(_rows: Any, _answer: Any, current: Any, nxt: Any) -> list[str]: + async def _write(_conn: Any, _rows: Any, _answer: Any, current: Any, nxt: Any) -> list[str]: written.append((current, nxt)) return ["first", "mode", "own"] @@ -392,6 +407,17 @@ async def _write(_rows: Any, _answer: Any, current: Any, nxt: Any) -> list[str]: ("_write_rows", _write), ): monkeypatch.setattr(cell_prepare, name, fake) + + directory_mode = ["recording"] + + async def snapshot(*_a: Any, **_kw: Any) -> dict[str, Any]: + return {**OK, "mode": directory_mode[0], "current_fp": None, "next_fp": None} + + async def claimed(_conn: Any, _call: Any, row: Any) -> Any: + return row + + monkeypatch.setattr(cell_control, "request", snapshot) + monkeypatch.setattr(cell_control, "claim", claimed) assert await cell_prepare.prepare(None, _args(), ROUTER) == 0 assert written == [(cell.as_value(after), None)] and reads == [] # stdout stays one JSON document; a first cell's next step goes to stderr. @@ -399,6 +425,7 @@ async def _write(_rows: Any, _answer: Any, current: Any, nxt: Any) -> list[str]: assert json.loads(out.out)["is_first"] is True assert out.err.strip() == "loom cell prepare: prepared the first cell; roll out LOOM_CELL next" # A rerun on a cell past recording names no next step. + directory_mode[0] = "enforced" monkeypatch.setattr( cell_prepare, "_register", AsyncMock(return_value={**OK, "mode": "enforced"}) ) @@ -406,6 +433,7 @@ async def _write(_rows: Any, _answer: Any, current: Any, nxt: Any) -> list[str]: assert await cell_prepare.prepare(None, _args(), ROUTER) == 0 assert capsys.readouterr().err == "" # A rerun in recording that writes no first row (the rollout is usually done). + directory_mode[0] = "recording" monkeypatch.setattr(cell_prepare, "_register", AsyncMock(return_value={**OK})) async def _own_only(*_a: Any) -> list[str]: @@ -415,3 +443,16 @@ async def _own_only(*_a: Any) -> list[str]: reads.extend([[_beat("i-new", after)], [_beat("i-new", after)]]) assert await cell_prepare.prepare(None, _args(), ROUTER) == 0 assert capsys.readouterr().err == "" + + # A prior controller committed enforced after registration but before this claim. + directory_mode[0] = "enforced" + posted: list[str] = [] + + async def write_latest(_conn: Any, _rows: Any, answer: Any, *_fp: Any) -> list[str]: + posted.append(answer["mode"]) + return ["mode"] + + monkeypatch.setattr(cell_prepare, "_write_rows", write_latest) + reads.extend([[_beat("i-new", after)], [_beat("i-new", after)]]) + assert await cell_prepare.prepare(None, _args(), ROUTER) == 0 + assert posted == ["enforced"] diff --git a/tests/unit/test_cell_register.py b/tests/unit/test_cell_register.py index 3d26bfa003..16a66b3489 100644 --- a/tests/unit/test_cell_register.py +++ b/tests/unit/test_cell_register.py @@ -9,6 +9,7 @@ import hashlib import json from typing import Any +from uuid import UUID import httpx import pytest @@ -90,6 +91,7 @@ def _row(reg: directory.Registration) -> directory.Cell: current_fp=None, next_fp=None, credential_next_sha256=None, + mutation_generation=UUID(int=1), ) @@ -501,7 +503,23 @@ def test_a_cell_never_registers_with_the_operator_credentials_hash(router) -> No assert r.status_code == 400 and "operator credential's hash" in r.json()["error"] rotated = client.post( "/cells/v1/operator/cells/eu-1/credential", - json={"step": "next", "sha256": operator_sha}, + json={ + "step": "next", + "sha256": operator_sha, + "expected": { + "cell": "eu-1", + "is_first": True, + "host": "loom.example", + "deployment_id": "dep-1", + "bucket": "", + "bucket_prefix": "", + "probe_url": "", + "upstream_url": "", + "current_fp": None, + "next_fp": None, + "mutation_generation": "00000000-0000-0000-0000-000000000001", + }, + }, headers={"Authorization": f"Bearer {OPERATOR}"}, ) assert rotated.status_code == 400 and "operator credential's hash" in rotated.json()["error"] diff --git a/tests/unit/test_cell_retirement.py b/tests/unit/test_cell_retirement.py new file mode 100644 index 0000000000..48517fb11e --- /dev/null +++ b/tests/unit/test_cell_retirement.py @@ -0,0 +1,753 @@ +"""Permanent retirement requires completion, never expiry or an API object's absence.""" + +from unittest.mock import AsyncMock +from uuid import UUID, uuid4 + +import pytest + +from server.core import provider_process +from server.insights.app import provider_barrier as barrier +from server.insights.store import provider_participants as store + + +@pytest.mark.asyncio +async def test_expired_quiet_participant_is_not_manual_death(monkeypatch): + participant = uuid4() + row = { + "participant_id": participant, + "transition_id": UUID(int=0), + "version": 7, + "state": "running", + "expired": True, + "host_proof": provider_process.host_proof(), + } + monkeypatch.setattr(barrier, "participants", AsyncMock(return_value=[row])) + monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) + monkeypatch.setattr(barrier, "record_participant", AsyncMock()) + assert await barrier.retire_participant(participant) == "live" + barrier.record_participant.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_generic_store_cannot_manufacture_retirement(): + ch = AsyncMock() + with pytest.raises(ValueError, match="[Rr]etire|[Ss]tate"): + await store.record_participant(uuid4(), UUID(int=0), "retired", ch=ch) + ch.insert.assert_not_awaited() + + +def test_changed_namespace_does_not_prove_old_container_ended(monkeypatch): + fields = provider_process.host_proof().split("|") + assert len(fields) == 8 + fields[2] = "c" * 64 + fields[3] = "another-namespace" + monkeypatch.setattr(provider_process, "_container", lambda: fields[2]) + assert not provider_process.proves_stopped("|".join(fields)) + + +@pytest.mark.asyncio +async def test_close_is_permanent_through_an_awaited_retirement_write(monkeypatch): + import asyncio + + from server.core import retirement + + gate = retirement.get_gate() + gate.install("test") + gate.enrolled = True + gate.stopped() + entered, finish = asyncio.Event(), asyncio.Event() + + async def write(proof): + assert gate.quiescent + entered.set() + await finish.wait() + + monkeypatch.setattr(barrier, "_OWN", barrier._Own()) + monkeypatch.setattr(barrier, "record_retirement", write) + monkeypatch.setattr(barrier, "drain_task_sets", AsyncMock(return_value=True)) + monkeypatch.setattr(barrier, "flush_registered_buffers", AsyncMock(return_value=True)) + monkeypatch.setattr(barrier, "_server_side_settled", AsyncMock(return_value=True)) + monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) + task = asyncio.create_task(barrier.retire_process()) + await entered.wait() + with pytest.raises(retirement.ProcessStopping): + async with gate.admit(): + pytest.fail("Permanent closure must not reopen while insert is awaited") + with pytest.raises(retirement.ProcessStopping): + await barrier.register_process() + finish.set() + assert await task + + +def _request_app(handler): + from fastapi import FastAPI + + from server.main import attach_browser_middleware + from server.transport import http + from server.transport.model_maintenance import ModelMaintenanceMiddleware + from server.transport.retirement import RetirementMiddleware + + app = FastAPI() + app.add_api_route("/oauth/token", handler, methods=["POST"]) + app.add_middleware(ModelMaintenanceMiddleware) + attach_browser_middleware(app) + http.attach(app) + app.add_middleware(RetirementMiddleware) + return app + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cancel_tail", [False, True]) +async def test_composed_request_tail_holds_counter_after_dispatch_raises(monkeypatch, cancel_tail): + import asyncio + + import httpx + + from server.core import request_tail, retirement + + gate = retirement.get_gate() + entered, finish, canceled = asyncio.Event(), asyncio.Event(), asyncio.Event() + observed = [] + + async def write(): + observed.append(gate.active) + entered.set() + try: + await finish.wait() + except asyncio.CancelledError: + canceled.set() + await finish.wait() # cancellation requested; this writer still exists + + async def handler(): + assert gate.active == 2 + assert request_tail.defer(write) + raise ValueError("before response start") + + app = _request_app(handler) + transport = httpx.ASGITransport(app=app, raise_app_exceptions=False) + monkeypatch.setattr(barrier, "record_retirement", AsyncMock()) + async with httpx.AsyncClient(transport=transport, base_url="http://localhost") as client: + task = asyncio.create_task(client.post("/oauth/token")) + await asyncio.wait_for(entered.wait(), 5) + assert gate.active == 1 # actual handler ended, outer tail remains counted + assert observed == [1] + gate.install("test") + gate.enrolled = True + gate.stopped() + assert not await barrier.retire_process() + assert not task.done() + barrier.record_retirement.assert_not_awaited() + if cancel_tail: + task.cancel() + await asyncio.wait_for(canceled.wait(), 5) + assert gate.active == 1 and not task.done() + assert not await barrier.retire_process() + finish.set() + await task + assert gate.active == 0 + + +@pytest.mark.asyncio +async def test_inner_scope_refuses_when_close_lands_after_outer_admission(): + from server.core import retirement + from server.transport.model_maintenance import ModelMaintenanceMiddleware + from server.transport.retirement import RetirementMiddleware + + gate = retirement.get_gate() + handler, send = AsyncMock(), AsyncMock() + + async def between(scope, receive, send): + assert gate.active == 1 + gate.close() + await ModelMaintenanceMiddleware(handler)(scope, receive, send) + assert gate.active == 1 + + await RetirementMiddleware(between)( + {"type": "http", "path": "/oauth/token", "method": "POST"}, AsyncMock(), send + ) + handler.assert_not_awaited() + assert send.await_args_list[0].args[0]["status"] == 503 + assert gate.active == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path", + [ + "/oauth/token", + "/oauth/callback", + "/v1/auth/login", + "/v1/scim/Users", + "/v1/admin/models/reset", + ], +) +async def test_canceled_control_dispatch_remains_counted_until_actual_completion(path): + import asyncio + + from server.core import retirement + from server.transport.model_maintenance import ModelMaintenanceMiddleware + from server.transport.retirement import RetirementMiddleware + + gate = retirement.get_gate() + entered, canceled, finish = asyncio.Event(), asyncio.Event(), asyncio.Event() + + async def handler(scope, receive, send): + entered.set() + try: + await finish.wait() + except asyncio.CancelledError: + canceled.set() + await finish.wait() + assert gate.active == 2 # work could still mint/write here + + task = asyncio.create_task( + RetirementMiddleware(ModelMaintenanceMiddleware(handler))( + {"type": "http", "path": path, "method": "POST"}, AsyncMock(), AsyncMock() + ) + ) + await entered.wait() + task.cancel() + await canceled.wait() + gate.close() + assert gate.active == 2 and not task.done() + finish.set() + await task + assert gate.active == 0 + + +@pytest.mark.asyncio +async def test_registered_descendant_blocks_after_outer_and_inner_completed(monkeypatch): + import asyncio + + import httpx + + from server.core import retirement + from server.core.background import create_deferred_task + + gate = retirement.get_gate() + entered, finish = asyncio.Event(), asyncio.Event() + tasks = [] + + async def descendant(): + entered.set() + await finish.wait() + + async def handler(): + tasks.append(create_deferred_task(descendant(), name="retirement-test-descendant")) + return {"ok": True} + + app = _request_app(handler) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://localhost" + ) as client: + assert (await client.post("/oauth/token")).status_code == 200 + await entered.wait() + assert gate.active == 0 + gate.install("test") + gate.enrolled = True + gate.stopped() + monkeypatch.setattr(barrier, "record_retirement", AsyncMock()) + assert not await barrier.retire_process() + barrier.record_retirement.assert_not_awaited() + finish.set() + await asyncio.gather(*tasks) + + +@pytest.mark.asyncio +async def test_certificates_fail_closed_in_original_parsers(): + from pathlib import Path + from types import ModuleType + + source = ( + Path(__file__).parents[1] / "fixtures" / "provider_process_before_retirement.py" + ).read_text() + previous = ModuleType("old_provider_process") + exec(compile(source, "old_provider_process.py", "exec"), previous.__dict__) + proof = provider_process.certified(provider_process.host_proof()) + assert len(proof.split("|")) == 9 + assert not previous.proves_stopped(proof) + assert not await previous.pod_gone(proof) + assert provider_process.has_certificate(proof) + assert not provider_process.has_certificate(proof.replace("cell-retirement-v1", "unknown-v2")) + assert not provider_process.has_certificate("||||||||cell-retirement-v1") + + +@pytest.mark.asyncio +async def test_uncertified_retired_rows_remain_census_blockers_and_unprunable(): + from types import SimpleNamespace + + identity = uuid4() + row = {"participant_id": identity, "state": "retired", "version": 7, "host_proof": ""} + ch = SimpleNamespace(query_all=AsyncMock(return_value=[row]), execute=AsyncMock()) + assert await store.participants(ch=ch) == [row] + assert await store.prune_retired(ch=ch) == 0 + ch.execute.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_prune_rechecks_latest_certificate_and_version(): + from types import SimpleNamespace + + identity = uuid4() + proof = provider_process.certified(provider_process.host_proof()) + row = {"participant_id": identity, "state": "retired", "version": 7, "host_proof": proof} + newer = {**row, "state": "running", "version": 8} + ch = SimpleNamespace(query_all=AsyncMock(side_effect=[[row], [newer]]), execute=AsyncMock()) + assert await store.prune_retired(ch=ch) == 0 + ch.execute.assert_not_awaited() + + +def test_production_topology_installs_outer_and_inner_scopes(): + from server.main import verify_retirement_topology + + verify_retirement_topology() + + +@pytest.mark.parametrize("migrate", [False, True]) +@pytest.mark.asyncio +async def test_server_enrolls_after_pair_verification_before_first_cell_read(monkeypatch, migrate): + from types import SimpleNamespace + + from server import boot, main + from server.auth import jwt + from server.core import object_store + from server.observability import instance + from server.settings import settings + + events = [] + + class FirstRead(Exception): + pass + + async def paired(*args, **kwargs): + events.append("paired") + + async def enrolled(): + assert events[-1] == "paired" + events.append("enrolled") + + async def first_read(): + assert events[-1] == "enrolled" + events.append("read") + raise FirstRead + + async def migrations(*args, enabled, **kwargs): + assert enabled is migrate + return 0 # disabled and no-pending-migrations boots both need enrollment + + monkeypatch.setattr(instance, "require_instance_label", lambda *args, **kwargs: None) + monkeypatch.setattr(jwt, "require_real_signing_secret", lambda **kwargs: None) + monkeypatch.setattr(boot, "require_identity_settings", lambda: None) + monkeypatch.setattr(boot, "require_safe_bind", lambda **kwargs: None) + monkeypatch.setattr(object_store, "require_client", lambda: None) + monkeypatch.setattr(main, "get_client", lambda: SimpleNamespace(connect=AsyncMock())) + monkeypatch.setattr(boot, "require_clickhouse_ready", AsyncMock()) + monkeypatch.setattr(boot, "register_readiness_checks", lambda ch: None) + monkeypatch.setattr(boot, "run_fenced_boot_migrations", migrations) + monkeypatch.setattr(boot, "require_keeper", AsyncMock()) + monkeypatch.setattr(boot, "require_storage_authority", paired) + monkeypatch.setattr(boot, "start_cell_gate", first_read) + monkeypatch.setattr(barrier, "register_process", enrolled) + monkeypatch.setattr(settings, "boot_migrate", migrate) + with pytest.raises(FirstRead): + async with main.lifespan(main.app): + pytest.fail("Only the first read is under test") + assert events == ["paired", "enrolled", "read"] + + +def test_cli_connects_verified_then_enrolls_then_reads_acceptance(monkeypatch): + from contextlib import asynccontextmanager + from types import SimpleNamespace + + from server.cli import _ch + from server.core import pg_profile + from server.insights.app import provider_command + + events = [] + client = SimpleNamespace(connected=True, disconnect=AsyncMock()) + + async def verified(ch): + assert ch is client + events.append("verified") + + @asynccontextmanager + async def command(): + events.append("enrolled") + yield + events.append("completed") + + async def accepting(orgs): + assert events == ["verified", "enrolled"] + events.append("read") + + async def work(ch): + assert events[-1] == "read" + events.append("work") + return 7 + + monkeypatch.setattr("server.core.ch.get_client", lambda: client) + monkeypatch.setattr(pg_profile, "check_configured", verified) + monkeypatch.setattr(provider_command, "model_command", command) + monkeypatch.setattr(_ch, "check_orgs", accepting) + assert _ch.run_connected(work, orgs=("example",)) == 7 + assert events == ["verified", "enrolled", "read", "work", "completed"] + + +def test_retirement_writers_and_accepting_command_roots_are_an_explicit_census(): + import ast + from pathlib import Path + + root = Path(__file__).parents[2] / "server" + constructors, writers, commands, stopped, plain = set(), set(), set(), set(), set() + for path in root.rglob("*.py"): + for node in ast.walk(ast.parse(path.read_text())): + if not isinstance(node, ast.Call): + continue + func = node.func + name = ( + func.id + if isinstance(func, ast.Name) + else func.attr + if isinstance(func, ast.Attribute) + else "" + ) + relative = str(path.relative_to(root)) + if name == "RetirementProof": + constructors.add(relative) + if name == "record_retirement": + writers.add(relative) + if name == "model_command": + commands.add(relative) + if name == "participant_command": + plain.add(relative) + if name == "stopped" and isinstance(func, ast.Attribute): + stopped.add(relative) + assert constructors == {"core/retirement.py"} + assert writers == {"insights/app/provider_barrier.py"} + assert commands == { + "cli/_ch.py", + "cli/code_run.py", + "cli/import_clickgap.py", + "cli/import_recall.py", + } + assert plain == { + "cli/sql_access.py", + "cli/analytics_export.py", + "cli/maintenance.py", + "insights/app/provider_command.py", + } + assert stopped == {"main.py", "worker_main.py", "insights/app/provider_command.py"} + + +def test_manual_retirement_has_no_affirmation_bypass(): + import argparse + + from server.cli.participant import register_participant_cli + + parser = argparse.ArgumentParser() + register_participant_cli(parser.add_subparsers(dest="command", required=True)) + with pytest.raises(SystemExit) as error: + parser.parse_args(["participant", "retire", "--participant", str(uuid4()), "--yes"]) + assert error.value.code == 2 + + +def test_fork_cannot_inherit_admission(monkeypatch): + from server.core import retirement + + gate = retirement.get_gate() + gate.install("server") + gate.enrolled = True + monkeypatch.setattr(retirement, "_pid", -1) + child = retirement.get_gate() + assert child is not gate and child.role == "server" and not child.enrolled + + +@pytest.mark.asyncio +@pytest.mark.parametrize("api_state", ["404", "replacement", "Failed", "Succeeded", "unreachable"]) +async def test_pod_object_death_never_retires_work_that_can_resume( + monkeypatch, tmp_path, api_state +): + import asyncio + from pathlib import Path + from types import ModuleType + + import httpx + + from server.core import retirement + + old = ModuleType("old_reaper") + source = ( + Path(__file__).parents[1] / "fixtures" / "provider_process_before_retirement.py" + ).read_text() + exec(compile(source, "old_reaper.py", "exec"), old.__dict__) + uid = str(uuid4()) + proof = "|".join( + ("partitioned-host", "boot", uid + "/loom", "4026", "7", "99", "loom", "old-pod") + ) + (tmp_path / "token").write_text("test-token") + monkeypatch.setattr(old, "_SA", str(tmp_path)) + monkeypatch.setenv("POD_NAMESPACE", "loom") + monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "kubernetes.invalid") + seen = [] + + def answer(request): + seen.append(request) + if api_state == "unreachable": + raise httpx.ConnectError("partitioned", request=request) + return httpx.Response( + 404 if api_state == "404" else 200, + json={ + "metadata": {"uid": str(uuid4()) if api_state == "replacement" else uid}, + "status": {"phase": api_state}, + }, + ) + + real_client = httpx.AsyncClient + monkeypatch.setattr( + httpx, "AsyncClient", lambda **kw: real_client(transport=httpx.MockTransport(answer)) + ) + # Demonstrate the exact old mistake before testing its replacement. + assert await old.pod_gone(proof) is (api_state != "unreachable") + participant = uuid4() + row = { + "participant_id": participant, + "state": "running", + "version": 9, + "expired": True, + "host_proof": proof, + } + monkeypatch.setattr(barrier, "participants", AsyncMock(return_value=[row])) + monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) + monkeypatch.setattr(barrier, "prune_retired", AsyncMock()) + monkeypatch.setattr(barrier, "record_retirement", AsyncMock()) + monkeypatch.setattr(barrier, "_STALE", None) + resume, admitted = asyncio.Event(), asyncio.Event() + effects = [] + + async def paused(): + async with retirement.get_gate().admit(): + admitted.set() + await resume.wait() + effects.append("mint after object deletion") + + task = asyncio.create_task(paused()) + await admitted.wait() + await barrier.sweep_stopped(now=5000) + barrier.record_retirement.assert_not_awaited() + assert barrier.stale_participants()[0]["participant_id"] == str(participant) + assert len(seen) == 1 # the current reaper never asks the API for a death certificate + resume.set() + await task + assert effects == ["mint after object deletion"] + + +@pytest.mark.asyncio +async def test_production_child_task_cancellation_stays_counted_until_completion(): + import asyncio + import contextlib + + import anyio + import httpx + + from server.core import retirement + + entered, finish = asyncio.Event(), asyncio.Event() + gate = retirement.get_gate() + + async def handler(): + entered.set() + with anyio.CancelScope(shield=True): + await finish.wait() + return {"ok": True} + + app = _request_app(handler) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://localhost" + ) as client: + task = asyncio.create_task(client.post("/oauth/token")) + await entered.wait() + task.cancel() + # Let BaseHTTPMiddleware propagate cancellation to its downstream child. + for _ in range(5): + await asyncio.sleep(0) + gate.close() + assert gate.active > 0 + assert not task.done() + finish.set() + with contextlib.suppress(asyncio.CancelledError): + await task + assert gate.active == 0 + + +@pytest.mark.asyncio +async def test_shutdown_budget_does_not_wait_for_cancellation_deferring_drains(monkeypatch): + import asyncio + import time + + from server import boot, main + from server.core import instances, retirement + + gate = retirement.get_gate() + gate.install("test") + gate.enrolled = True + gate.stopped() + canceled, finish = asyncio.Event(), asyncio.Event() + owned = [] + + async def drains(*args): + owned.append(asyncio.current_task()) + try: + await finish.wait() + except asyncio.CancelledError: + canceled.set() + await finish.wait() + + async def leave(*args, **kwargs): + assert not await barrier.retire_process() + + monkeypatch.setattr(instances, "stop", AsyncMock()) + monkeypatch.setattr(main, "_shutdown_drains", drains) + monkeypatch.setattr(boot, "leave_deployment", leave) + monkeypatch.setattr(boot, "SHUTDOWN_DRAIN_BUDGET_S", 0.01) + monkeypatch.setattr(barrier, "record_retirement", AsyncMock()) + started = time.monotonic() + await asyncio.wait_for(main._shut_down(None, None, None, None), timeout=0.5) + assert time.monotonic() - started < 0.5 + await canceled.wait() + assert not owned[0].done() + barrier.record_retirement.assert_not_awaited() + finish.set() + await owned[0] + + +def test_nonmodel_principal_command_enrolls_before_acceptance(monkeypatch): + from types import SimpleNamespace + + from server.cli import _ch, sql_access + from server.core import pg_profile, retirement + from server.insights.app import provider_command + + events = [] + client = SimpleNamespace(connected=True, disconnect=AsyncMock()) + + async def verify(ch): + events.append("verified") + + async def register(): + assert events == ["verified"] + retirement.get_gate().enrolled = True + events.append("registered") + + async def acceptance(orgs): + assert events == ["verified", "registered"] + events.append("acceptance") + + async def work(ch): + assert events[-1] == "acceptance" + assert retirement.get_gate().active == 1 + return 7 + + monkeypatch.setattr("server.core.ch.get_client", lambda: client) + monkeypatch.setattr(pg_profile, "check_configured", verify) + monkeypatch.setattr(provider_command, "register_process", register) + monkeypatch.setattr(provider_command, "drain_task_sets", AsyncMock(return_value=True)) + monkeypatch.setattr(provider_command, "flush_registered_buffers", AsyncMock(return_value=True)) + monkeypatch.setattr(provider_command, "_retire_until", AsyncMock(return_value=True)) + monkeypatch.setattr(_ch, "check_orgs", acceptance) + assert sql_access._connected(work, orgs=("example",)) == 7 + assert retirement.get_gate().closed and retirement.get_gate().active == 0 + + +@pytest.mark.asyncio +async def test_stream_and_response_background_cleanup_remain_counted(monkeypatch): + import asyncio + + import httpx + from starlette.background import BackgroundTask + from starlette.responses import StreamingResponse + + from server.core import retirement + + gate = retirement.get_gate() + body_entered, body_finish = asyncio.Event(), asyncio.Event() + cleanup_entered, cleanup_finish = asyncio.Event(), asyncio.Event() + + async def body(): + yield b"first" + body_entered.set() + await body_finish.wait() + yield b"last" + + async def cleanup(): + cleanup_entered.set() + await cleanup_finish.wait() + + async def handler(): + return StreamingResponse(body(), background=BackgroundTask(cleanup)) + + app = _request_app(handler) + monkeypatch.setattr(barrier, "record_retirement", AsyncMock()) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://localhost" + ) as client: + task = asyncio.create_task(client.post("/oauth/token")) + await body_entered.wait() + assert gate.active == 2 + gate.install("test") + gate.enrolled = True + gate.stopped() + assert not await barrier.retire_process() + body_finish.set() + await cleanup_entered.wait() + assert gate.active == 2 and not task.done() + assert not await barrier.retire_process() + cleanup_finish.set() + assert (await task).content == b"firstlast" + assert gate.active == 0 + + +@pytest.mark.asyncio +async def test_websocket_lifetime_and_shutdown_refusal_share_the_same_counter(): + import asyncio + + from server.core import retirement + + entered, finish = asyncio.Event(), asyncio.Event() + gate = retirement.get_gate() + + async def handler(): + return {} + + from starlette.websockets import WebSocket + + async def websocket(ws: WebSocket): + await ws.accept() + entered.set() + await finish.wait() + await ws.close() + + app = _request_app(handler) + app.add_api_websocket_route("/socket", websocket) + scope = { + "type": "websocket", + "path": "/socket", + "scheme": "ws", + "query_string": b"", + "headers": [(b"host", b"localhost")], + "subprotocols": [], + "client": ("127.0.0.1", 1234), + "server": ("localhost", 80), + } + task = asyncio.create_task( + app(scope, AsyncMock(return_value={"type": "websocket.connect"}), AsyncMock()) + ) + await entered.wait() + assert gate.active == 2 + gate.close() + send = AsyncMock() + await app(scope, AsyncMock(), send) + send.assert_awaited_once_with({"type": "websocket.close", "code": 1013}) + assert gate.active == 2 + finish.set() + await task + assert gate.active == 0 diff --git a/tests/unit/test_cell_trigger_ci_creator.py b/tests/unit/test_cell_trigger_ci_creator.py new file mode 100644 index 0000000000..730f265968 --- /dev/null +++ b/tests/unit/test_cell_trigger_ci_creator.py @@ -0,0 +1,830 @@ +"""A failed fixture may remove only the containers its creator recorded.""" + +import signal +from copy import deepcopy +from pathlib import Path + +import pytest + +from scripts.gates.cell_trigger_acceptance import Stores, native_cleanup_complete, write + + +class FakeStores(Stores): + def __init__(self, folder: Path, rows: dict): + super().__init__(folder, folder, "fixture", {"run_id": "123"}) + self.rows = deepcopy(rows) + self.removed = [] + + def inspect(self, identity, *, cleanup=False): + return self.rows[identity] + + def command(self, *argv, timeout=120, cleanup=False): + assert cleanup + if argv[:3] == ("docker", "rm", "--force"): + self.removed.append(argv[-1]) + return "" + assert argv[:3] == ("docker", "ps", "-a") + identity = next(part.removeprefix("id=") for part in argv if part.startswith("id=")) + return identity if identity in self.rows and identity not in self.removed else "" + + +def container(identity="own", created="birth", label="fixture"): + return { + "Id": identity, + "Created": created, + "Name": "/intent", + "Config": {"Labels": {"loom.ci.trigger": label}}, + } + + +@pytest.mark.parametrize( + "change", + [ + {"Id": "reused"}, + {"Created": "new-birth"}, + {"Config": {"Labels": {"loom.ci.trigger": "foreign"}}}, + ], +) +def test_cleanup_refuses_changed_container_identity(tmp_path, change): + actual = {**container(), **change} + stores = FakeStores(tmp_path, {"own": actual}) + stores.items = [{"role": "pg-primary", "name": "intent", "id": "own", "created": "birth"}] + assert stores.cleanup() + assert stores.removed == [] + + +def test_cleanup_continues_exact_owned_containers_after_a_refusal(tmp_path): + stores = FakeStores(tmp_path, {"own": container(), "foreign": container(label="other")}) + stores.items = [ + {"role": "pg-primary", "name": "intent", "id": "own", "created": "birth"}, + {"role": "pg-standby", "name": "other", "id": "foreign", "created": "birth"}, + ] + assert stores.cleanup() == ["RuntimeError:pg-standby"] + assert stores.removed == ["own"] + + +def test_partial_create_requires_its_exact_name_and_random_label(tmp_path): + stores = FakeStores(tmp_path, {"intent": container()}) + stores.items = [{"role": "clickhouse", "name": "intent", "id": None, "created": None}] + assert stores.cleanup() == [] + assert stores.removed == ["own"] + + +def test_partial_create_never_adopts_a_foreign_matching_name(tmp_path): + stores = FakeStores(tmp_path, {"intent": container(label="other")}) + stores.items = [{"role": "clickhouse", "name": "intent", "id": None, "created": None}] + assert stores.cleanup() == ["RuntimeError:clickhouse"] + assert stores.removed == [] + + +def test_cancellation_is_deferred_until_owned_cleanup_finishes(tmp_path): + stores = FakeStores(tmp_path, {}) + stores.interrupted(signal.SIGTERM, None) + assert stores.cancelled == signal.SIGTERM + stores.terminal = True + with pytest.raises(SystemExit) as caught: + stores.interrupted(signal.SIGINT, None) + assert caught.value.code == 128 + signal.SIGTERM + + +def test_store_removal_requires_acknowledged_native_cleanup(tmp_path): + (tmp_path / "fixture").mkdir() + with pytest.raises(FileNotFoundError): + native_cleanup_complete(tmp_path, "fixture") + write( + tmp_path / "fixture/outer-result.json", + {"schema": 2, "fixture_id": "fixture", "native_cleanup_complete": False}, + ) + with pytest.raises(RuntimeError, match="Native retirement"): + native_cleanup_complete(tmp_path, "fixture") + + +def test_failed_measurement_can_close_only_after_real_native_cleanup(tmp_path): + (tmp_path / "fixture").mkdir() + write( + tmp_path / "fixture/outer-result.json", + { + "schema": 2, + "fixture_id": "fixture", + "candidate_verdict": "FAIL", + "native_cleanup_complete": True, + }, + ) + native_cleanup_complete(tmp_path, "fixture") + + +def test_dependency_metadata_accepts_only_the_actual_python_version(tmp_path): + import hashlib + import json + import sys + + from scripts.gates.testsupport.cell_trigger import fixture + + environment = tmp_path / ".venv" + for name in fixture.CODEC_FILES: + path = environment / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("fixture codec bytes") + (environment / fixture.CODEC_FILES[1]).write_text('version = "1.9.0"\n') + (environment / fixture.CODEC_FILES[4]).write_text('__version__ = "0.32.0"\n') + spec = { + "root": str(tmp_path), + "python": str(environment / "bin/python"), + "python_sha256": "0" * 64, + "lockfile_sha256": "1" * 64, + } + + def check(version): + (environment / "pyvenv.cfg").write_text( + f"implementation = CPython\nversion_info = {version}\n" + "include-system-site-packages = false\n" + ) + receipt = { + "schema": 2, + **spec, + "versions": fixture.CODEC_VERSIONS, + "codec_sources": { + name: {"path": str(environment / name), "sha256": fixture.sha(environment / name)} + for name in fixture.CODEC_FILES + }, + } + receipt.pop("dependency_receipt", None) + path = tmp_path / "receipt.json" + path.write_text(json.dumps(receipt)) + path.chmod(0o600) + spec["dependency_receipt"] = { + "path": str(path), + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + } + fixture.dependencies(spec, verify=True) + + check(".".join(map(str, sys.version_info[:3]))) + check("3.12") + for changed in ("3.13.0", "3.12.invalid", "3.12.999"): + with pytest.raises(fixture.Refused, match="fixed isolated CPython"): + check(changed) + + +@pytest.fixture +def protocol_reader(tmp_path, monkeypatch): + import sys + + from scripts.gates.testsupport.cell_trigger import fixture + + monkeypatch.setitem(sys.modules, "fixture", fixture) + from scripts.gates.testsupport.cell_trigger import capture + + data = (Path(fixture.__file__).parent / fixture.PROTOCOL).read_bytes() + path = tmp_path / fixture.PROTOCOL + path.write_bytes(data) + path.chmod(0o644) + monkeypatch.setattr(capture, "HERE", tmp_path) + return capture, fixture, path, data + + +def test_committed_public_protocol_is_readable_without_private_receipt_mode(protocol_reader): + capture, _, _, _ = protocol_reader + assert capture.protocol()["schema"] == 1 + + +def test_public_protocol_keeps_hash_and_symlink_guards(protocol_reader): + capture, fixture, path, data = protocol_reader + path.write_bytes(data + b"\n") + with pytest.raises(fixture.Refused): + capture.protocol() + path.unlink() + target = path.with_name("target.json") + target.write_bytes(data) + target.chmod(0o644) + path.symlink_to(target) + with pytest.raises((OSError, fixture.Refused)): + capture.protocol() + + +def test_private_receipt_still_refuses_public_read_permissions(protocol_reader): + _, fixture, path, _ = protocol_reader + with pytest.raises(fixture.Refused, match="owned private stream"): + fixture.read(path) + + +def test_replication_waits_for_primary_tcp_after_temporary_socket_ready(tmp_path, monkeypatch): + from scripts.gates import cell_trigger_acceptance as creator + + monkeypatch.setattr(creator, "ports", lambda count: list(range(18000, 18000 + count))) + monkeypatch.setattr(creator.time, "sleep", lambda _: None) + + class InitializingStores(Stores): + def __init__(self): + super().__init__(tmp_path, tmp_path, "fixture", {"run_id": "123"}) + self.tcp_attempts = 0 + self.tcp_ready = False + + def _clickhouse_ready(self, port): + return True + + def create(self, role, args): + return {"id": role} + + def command(self, *argv, timeout=120, cleanup=False): + if "pg_isready" in argv: + assert argv[argv.index("-h") + 1] == "127.0.0.1" + self.tcp_attempts += 1 + if self.tcp_attempts == 1: + raise RuntimeError("simulated TCP startup refusal") + self.tcp_ready = True + return "127.0.0.1:5432 - accepting connections" + if "psql" in argv: + query = argv[-1] + if query == "SELECT 1": + return "1" # The temporary init server accepts Unix sockets only. + assert self.tcp_ready, "Replication configuration raced the temporary init server" + return "sync" if "SELECT sync_state" in query else "" + return "1" if "clickhouse-client" in argv else "network-id" + + stores = InitializingStores() + stores.start("test-image") + assert stores.tcp_attempts == 2 and stores.tcp_ready + + +@pytest.mark.parametrize("setting", ["config_mount", "database", "data_owner"]) +def test_clickhouse_creation_uses_ci_server_configuration(tmp_path, monkeypatch, setting): + from scripts.gates import cell_trigger_acceptance as creator + + monkeypatch.setattr(creator, "ports", lambda count: list(range(18000, 18000 + count))) + + class CapturedCreation(Exception): + pass + + class CapturingStores(Stores): + def command(self, *argv, timeout=120, cleanup=False): + return "network-id" + + def create(self, role, args): + assert role == "clickhouse" + self.creation_args = args + raise CapturedCreation + + stores = CapturingStores(tmp_path, tmp_path, "fixture", {"run_id": "123"}) + with pytest.raises(CapturedCreation): + stores.start("test-image") + args = stores.creation_args + if setting == "config_mount": + mounts = [args[index + 1] for index, value in enumerate(args) if value == "-v"] + for filename, section in ( + ("custom_settings_prefixes.xml", "config.d"), + ("log_queries_readonly.xml", "users.d"), + ): + assert ( + str(tmp_path / "dev/clickhouse-ci" / filename) + + ":/etc/clickhouse-server/" + + section + + "/" + + filename + + ":ro" + ) in mounts + elif setting == "database": + environment = [args[index + 1] for index, value in enumerate(args) if value == "-e"] + assert "CLICKHOUSE_DB=loom" in environment + else: + environment = [args[index + 1] for index, value in enumerate(args) if value == "-e"] + owner = (tmp_path / "clickhouse-data").stat() + assert f"CLICKHOUSE_UID={owner.st_uid}" in environment + assert f"CLICKHOUSE_GID={owner.st_gid}" in environment + assert not any( + value.startswith(("CLICKHOUSE_RUN_AS_ROOT=", "CLICKHOUSE_DO_NOT_CHOWN=")) + for value in environment + ) + + +def test_failure_diagnostics_expose_only_allowlisted_owned_state(tmp_path): + import json + + class DiagnosticStores(Stores): + def command(self, *argv, timeout=120, cleanup=False): + assert cleanup and timeout == 5 + if argv[:2] == ("docker", "inspect"): + value = container() + value["Config"]["Env"] = ["TOKEN=private-value"] + value["State"] = { + "Status": "running", + "Running": True, + "ExitCode": 0, + "OOMKilled": False, + "Error": "private-log", + } + return json.dumps([value]) + return "primary:sync" + + stores = DiagnosticStores(tmp_path, tmp_path, "fixture", {"run_id": "123"}) + stores.phase = "standby_sync" + stores.items = [{"role": "pg-primary", "name": "intent", "id": "own", "created": "birth"}] + result = stores.failure_diagnostics() + assert result == { + "phase": "standby_sync", + "containers": [ + { + "role": "pg-primary", + "available": True, + "status": "running", + "exit_code": 0, + "oom_killed": False, + "pg_state": {"recovery": False, "standby_sync": "sync"}, + } + ], + } + assert "private" not in json.dumps(result) + + +def test_diagnostic_failure_preserves_primary_result_and_runs_cleanup( + tmp_path, monkeypatch, capsys +): + import json + import sys + + from scripts.gates import cell_trigger_acceptance as creator + + cleaned = [] + + class FailingStores(Stores): + def start(self, image): + raise ValueError("private primary failure") + + def failure_diagnostics(self): + raise OSError("private diagnostic failure") + + def cleanup(self): + cleaned.append(True) + return [] + + root = Path(sys.executable).parent.parent.parent + monkeypatch.setenv("GITHUB_ACTIONS", "true") + monkeypatch.setenv("CI", "true") + monkeypatch.setenv("GITHUB_WORKSPACE", str(root)) + monkeypatch.setenv("RUNNER_TEMP", str(tmp_path)) + for name in ("GITHUB_RUN_ID", "GITHUB_RUN_ATTEMPT", "GITHUB_REPOSITORY"): + monkeypatch.setenv(name, "123") + monkeypatch.setattr(sys, "argv", ["creator", "--clickhouse-image", "test-image"]) + monkeypatch.setattr(creator, "Stores", FailingStores) + monkeypatch.setattr(creator, "source", lambda _: {"head": "a" * 40, "tree": "b" * 40}) + monkeypatch.setattr(creator, "source_hashes", lambda _: {}) + monkeypatch.setattr(creator, "resources", lambda _: None) + monkeypatch.setattr(creator.os, "umask", lambda _: 0) + monkeypatch.setattr(creator.signal, "signal", lambda *_: None) + assert creator.main() == 1 + result = json.loads(capsys.readouterr().out) + assert cleaned == [True] + assert result["error_type"] == "ValueError" and result["diagnostic_error_type"] == "OSError" + assert "private" not in json.dumps(result) + published = json.loads((tmp_path / "cell-trigger-artifacts/result.json").read_text()) + assert ( + published["error_type"] == "ValueError" and published["diagnostic_error_type"] == "OSError" + ) + + +@pytest.fixture +def keeper_config_case(tmp_path, monkeypatch): + import json + import sys + from types import SimpleNamespace + + from scripts.gates.testsupport.cell_trigger import fixture + + monkeypatch.setitem(sys.modules, "fixture", fixture) + from scripts.gates.testsupport.cell_trigger import capture + + monkeypatch.setitem(sys.modules, "capture", capture) + from scripts.gates.testsupport.cell_trigger import native + + expected = { + "container_id": "owned-ch", + "created_at": "birth", + "host_pid": 123, + "memory_limit_bytes": 5 * fixture.GIB, + "server_limit_bytes": 9 * fixture.GIB // 2, + "shared_buffers_bytes": 128 * 1024**2, + "ports": [], + "disk_path": str(tmp_path), + "start_ticks": "start", + "uid": 1000, + "boot_id": "boot", + } + actual = { + "Id": "owned-ch", + "Created": "birth", + "State": {"Running": True, "Pid": 123}, + "HostConfig": {"Memory": 5 * fixture.GIB}, + "NetworkSettings": {"Ports": {}}, + "Mounts": [{"Source": str(tmp_path)}], + } + calls = [] + owner = SimpleNamespace(owned=SimpleNamespace(identity=lambda _: (1, "start"))) + owner.prefix = "/keeper_map_tables\n" + + def command(role, argv, env, *, seconds=120, cleanup=False): + calls.append((argv, cleanup)) + log = tmp_path / (role + ".log") + if argv == ["docker", "inspect", "owned-ch"]: + log.write_text(json.dumps([actual])) + else: + assert argv == [ + "docker", + "exec", + "owned-ch", + "clickhouse", + "extract-from-config", + "--config-file", + "/etc/clickhouse-server/config.xml", + "--key", + "keeper_map_path_prefix", + ] + log.write_text(owner.prefix) + return {"log": log} + + owner.command = command + original_stat, original_read = native.os.stat, Path.read_text + + def stat(path, *args, **kwargs): + if str(path) == "/proc/123": + return SimpleNamespace(st_uid=1000) + return original_stat(path, *args, **kwargs) + + def read_text(path, *args, **kwargs): + if str(path) == "/proc/sys/kernel/random/boot_id": + return "boot\n" + return original_read(path, *args, **kwargs) + + monkeypatch.setattr(native.os, "stat", stat) + monkeypatch.setattr(Path, "read_text", read_text) + spec = { + "workdir": str(tmp_path), + "fixture_id": "fixture", + "stores": {"clickhouse": expected}, + "ch": {"keeper_prefix": "/keeper_map_tables"}, + "databases": {"ch_schema": "bench_schema"}, + "pg_standby_dsn": "fake", + } + instance = native.Fixture(spec, owner) + monkeypatch.setattr(instance, "env", lambda _: {}) + return native, instance, owner, actual, calls + + +@pytest.mark.asyncio +async def test_keeper_config_allows_creation_without_sql_server_setting( + keeper_config_case, monkeypatch +): + from contextlib import asynccontextmanager + + import asyncpg + + native, instance, _, _, calls = keeper_config_case + instance.inspect_stores() + commands = [] + queries = [] + + class Postgres: + async def fetchval(self, query, *args): + return { + "SELECT NOT pg_is_in_recovery()": True, + "SELECT pg_is_in_recovery()": True, + "SHOW server_version_num": "160000", + "SHOW max_prepared_transactions": "0", + "SHOW synchronous_standby_names": "FIRST 1 (loom_standby)", + "SELECT pg_size_bytes(current_setting('shared_buffers'))": 128 * 1024**2, + "SELECT EXISTS(SELECT FROM pg_stat_replication WHERE state='streaming' " + "AND sync_state='sync' AND flush_lsn IS NOT NULL)": True, + }[query] + + async def close(self, **kwargs): + assert kwargs["timeout"] > 0 + + class ClickHouse: + async def command(self, query): + commands.append(query) + + async def connect(*args, **kwargs): + return Postgres() + + @asynccontextmanager + async def connections(*args, **kwargs): + yield Postgres(), ClickHouse() + + async def rows(ch, query, parameters=None): + queries.append(query) + if "keeper_map_path_prefix" in query: + return [] # The real config key is absent from declared ServerSettings. + if "max_server_memory_usage" in query: + return [{"value": str(9 * native.GIB // 2)}] + if "count()" in query: + return [{"n": 0}] + if "toString(uuid)" in query: + return [{"id": "database-uuid"}] + assert "system.zookeeper" in query + return [] + + monkeypatch.setattr(asyncpg, "connect", connect) + monkeypatch.setattr(native, "connections", connections) + monkeypatch.setattr(native, "rows", rows) + instance.s["stores"]["pg_primary"] = {"shared_buffers_bytes": 128 * 1024**2} + instance.s["stores"]["pg_standby"] = {"shared_buffers_bytes": 128 * 1024**2} + await native.stores(instance.s, "ch_schema", instance.w / "created.json") + assert commands == ["CREATE DATABASE `bench_schema` ENGINE=Atomic"] + assert len(calls) == 2 and calls[-1][0][-1] == "keeper_map_path_prefix" + assert not any("keeper_map_path_prefix" in query for query in queries) + + +@pytest.mark.parametrize("cleanup", [False, True]) +def test_keeper_config_refuses_wrong_actual_prefix(keeper_config_case, cleanup): + native, instance, owner, _, calls = keeper_config_case + owner.prefix = "/different-prefix\n" + with pytest.raises(native.Refused, match="actual Keeper prefix"): + instance.inspect_stores(cleanup=cleanup) + assert calls[-1][1] is cleanup + + +def test_keeper_config_probe_follows_store_ownership_validation(keeper_config_case): + native, instance, _, actual, calls = keeper_config_case + actual["Created"] = "foreign-birth" + with pytest.raises(native.Refused, match="current owned container"): + instance.inspect_stores() + assert [argv for argv, _ in calls] == [["docker", "inspect", "owned-ch"]] + + +def test_native_setup_registers_independent_append_only_writes(keeper_config_case, monkeypatch): + native, instance, owner, _, _ = keeper_config_case + instance.s.update( + ports={"probe": 7790, "router": 7791, "tls": 7792, "api": 7793}, + pg_admin_dsn="postgresql://u:p@127.0.0.1:15432/postgres", + databases={ + "pg_cell": "bench_cell", + "pg_directory": "bench_directory", + "ch_schema": "bench_schema", + "ch_telemetry": "bench_telemetry", + }, + ch={"port": 18123, "username": "fixture", "password": "test-only"}, + head="a" * 40, + python="unused", + ) + log = instance.w / "credentials.log" + log.write_text("credential: lcr_" + "1" * 128 + "\ncredential: lco_" + "2" * 128 + "\n") + + def action(task, *, role="", **kwargs): + if task == "create": + native.write(instance.w / (role + "-created.json"), {"created": True}) + return {} + + class Registered(Exception): + pass + + def register(path, *, token, body): + assert path == "/v1/namespace.register" + assert token == "fixture-token" + assert body == { + "org": instance.org, + "namespace": instance.namespace, + "owner_email": "benchmark", + "default_conflict_policy": "append_only", + } + raise Registered + + monkeypatch.setattr(instance, "inspect_stores", lambda: None) + monkeypatch.setattr(instance, "action", action) + monkeypatch.setattr(instance, "cli", lambda *args, **kwargs: {"log": log}) + monkeypatch.setattr(instance, "command_receipt", lambda *args, **kwargs: None) + monkeypatch.setattr(instance, "certificate", lambda: None) + monkeypatch.setattr(instance, "start_api", lambda *args: None) + monkeypatch.setattr(instance, "process_receipt", lambda *args: {}) + monkeypatch.setattr(owner, "start", lambda *args, **kwargs: {}, raising=False) + monkeypatch.setattr(native, "mint_metadata", lambda *args: ({}, "fixture-token")) + monkeypatch.setattr(instance, "http", register) + with pytest.raises(Registered): + instance.setup() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy", ["auto_reinforce", "append_only"]) +async def test_native_bootstrap_checks_persisted_independent_write_policy( + keeper_config_case, monkeypatch, policy +): + from contextlib import asynccontextmanager + from types import SimpleNamespace + from unittest.mock import AsyncMock + + native, instance, _, _, _ = keeper_config_case + authority = {"authority": "postgres", "deployment_id": "deployment", "epoch": 1} + conn = SimpleNamespace( + fetchrow=AsyncMock(return_value=authority), fetchval=AsyncMock(return_value=0) + ) + + @asynccontextmanager + async def connected(*args, **kwargs): + yield conn, object() + + async def rows(ch, sql, params=None): + if ".storage_authority" in sql: + return [authority] + if ".namespaces " in sql: + assert params == {"org": instance.org, "ns": instance.namespace} + row = {"owner_email": "benchmark"} + if "default_conflict_policy" in sql: + row["default_conflict_policy"] = policy + return [row] + if ".namespace_grants " in sql: + return [{"subject": "benchmark", "role": "admin"}] + raise AssertionError(sql) + + monkeypatch.setattr(native, "connections", connected) + monkeypatch.setattr(native, "rows", rows) + output = instance.w / "bootstrap.json" + if policy != "append_only": + with pytest.raises(native.Refused, match="native registered namespace"): + await native.proof(instance.s, output, bootstrap=True) + assert not output.exists() + else: + await native.proof(instance.s, output, bootstrap=True) + assert native.read(output)["namespace_registered"] is True + + +@pytest.mark.parametrize("role", ["inspect", "keeper-prefix", "create", "private-role"]) +def test_nonzero_native_command_retains_only_structural_metadata(monkeypatch, role): + import json + from types import SimpleNamespace + + from scripts.gates.testsupport.cell_trigger import fixture + + owner = fixture.Owner({}, None, lambda *args, **kwargs: None) + owner.phase, owner.serial = "setup", 5 + item = {"process": SimpleNamespace(returncode=7)} + monkeypatch.setattr(owner, "start", lambda *args, **kwargs: item) + monkeypatch.setattr(owner, "poll", lambda _: 7) + with pytest.raises(fixture.Refused) as caught: + owner.command(role, ["private-argv"], {"TOKEN": "private-token"}) + owner.select(caught.value) + record = owner.failure_summary()["records"][0] + assert owner.first is caught.value + assert record["command"] == { + "role": role if role != "private-role" else "other", + "serial": 5, + "returncode": 7, + } + assert record["site"]["file"] == "fixture.py" + assert record["site"]["function"] == "command" + assert "private-" not in json.dumps(record) + + +def test_native_failure_site_reaches_published_evidence_without_private_logs( + keeper_config_case, monkeypatch, tmp_path, capsys +): + import json + import sys + + from scripts.gates.cell_trigger_acceptance import publish + from scripts.gates.testsupport.cell_trigger import fixture + + native, instance, owner, _, _ = keeper_config_case + context, output = tmp_path / "failure-context.json", tmp_path / "pg_cell-created.json" + instance.context = context + instance.s["databases"]["pg_cell"] = "bench_cell" + instance.s["python"] = "unused" + native.write(context, instance.s | {"root": str(tmp_path)}) + + async def failed(*args, **kwargs): + raise ValueError("private-native-secret") + + monkeypatch.setattr(native, "stores", failed) + monkeypatch.setattr(sys, "path", list(sys.path)) + monkeypatch.setattr( + sys, + "argv", + [ + "native.py", + "--task", + "create", + "--role", + "pg_cell", + "--context", + str(context), + "--output", + str(output), + ], + ) + assert native.main() == 1 + assert not output.exists() + metadata = native.read(output.with_name(output.name + ".failure.json"), limit=4096) + assert metadata["error_type"] == "ValueError" + assert metadata["site"]["file"] == "native.py" + assert metadata["site"]["function"] == "main" + assert metadata["site"]["line"] > 0 + assert "private-native-secret" not in json.dumps(metadata) + capsys.readouterr().err + original = fixture.Refused("private-command-secret") + original.command = {"role": "create", "serial": 5, "returncode": 1} + + def refused(*args, **kwargs): + raise original + + owner.command = refused + with pytest.raises(fixture.Refused) as caught: + instance.action("create", role="pg_cell") + assert caught.value is original + observer = fixture.Owner({}, None, lambda *args, **kwargs: None) + observer.select(caught.value, phase="setup") + evidence = observer.failure_summary() + assert evidence["records"][0]["native"] == metadata + assert evidence["records"][0]["command"] == original.command + folder = tmp_path / "publication" + (folder / "fixture").mkdir(parents=True) + native.write(folder / "fixture/outer-result.json", {"failure_evidence": evidence}) + (folder / "fixture/private.log").write_text("private-log-secret") + (folder / "fixture/private-settings.json").write_text('{"token":"private-settings-secret"}') + destination = tmp_path / "published" + publish(folder, destination, original, phase="native", diagnostic_error_type=None) + assert {path.name for path in destination.iterdir()} == {"outer-result.json", "result.json"} + published = json.loads((destination / "outer-result.json").read_text()) + assert published["failure_evidence"] == evidence + assert "private-" not in json.dumps(published) + + +@pytest.mark.parametrize( + "contents", + [ + "not-json", + '{"error_type":"private-extra"}', + "x" * 4097, + pytest.param("[" * 1500 + "0" + "]" * 1500, id="deep-nesting"), + ], +) +def test_invalid_native_failure_sidecar_preserves_command_refusal(keeper_config_case, contents): + native, instance, owner, _, _ = keeper_config_case + instance.s["python"] = "unused" + path = instance.w / "pg_cell-created.json.failure.json" + path.write_text(contents) + path.chmod(0o600) + original = native.Refused("original") + + def refused(*args, **kwargs): + raise original + + owner.command = refused + with pytest.raises(native.Refused) as caught: + instance.action("create", role="pg_cell") + assert caught.value is original and original.native is None + + +@pytest.mark.parametrize("kind", [KeyboardInterrupt, SystemExit]) +@pytest.mark.parametrize("site", ["read", "closed"]) +def test_fatal_native_diagnostic_preserves_original_command_refusal( + keeper_config_case, monkeypatch, kind, site +): + native, instance, owner, _, _ = keeper_config_case + instance.s["python"] = "unused" + original = native.Refused("original") + + def refused(*args, **kwargs): + raise original + + def diagnostic(*args, **kwargs): + raise kind("private-diagnostic-secret") + + owner.command = refused + monkeypatch.setattr(native, "read", lambda *args, **kwargs: {}) + monkeypatch.setattr(native, site, diagnostic) + try: + instance.action("create", role="pg_cell") + except BaseException as error: # noqa: BLE001 — assert the actual retained failure, including fatal diagnostics. + assert error is original and original.native is None + else: + pytest.fail("command refusal was swallowed") + + +def test_clickhouse_waits_for_published_http_after_temporary_native_ready(tmp_path, monkeypatch): + from scripts.gates import cell_trigger_acceptance as creator + + monkeypatch.setattr(creator, "ports", lambda count: list(range(18000, 18000 + count))) + monkeypatch.setattr(creator.time, "sleep", lambda _: None) + + class InitializingClickHouse(Stores): + def __init__(self): + super().__init__(tmp_path, tmp_path, "fixture", {"run_id": "123"}) + self.published_attempts = 0 + self.native_attempts = 0 + + def create(self, role, args): + return {"id": role} + + def command(self, *argv, timeout=120, cleanup=False): + if "clickhouse-client" in argv: + self.native_attempts += 1 + return "1" # The temporary loopback-only server already accepts native queries. + if "pg_isready" in argv: + return "127.0.0.1:5432 - accepting connections" + return "sync" if "SELECT sync_state" in argv[-1] else "network-id" + + def _clickhouse_ready(self, port): + assert port == 18000 + self.published_attempts += 1 + return self.published_attempts == 2 + + stores = InitializingClickHouse() + stores.start("test-image") + assert stores.published_attempts == 2 + assert stores.native_attempts == 0 + assert stores.phase == "clickhouse_query" diff --git a/tests/unit/test_cell_trigger_ci_creator_http_ready.py b/tests/unit/test_cell_trigger_ci_creator_http_ready.py new file mode 100644 index 0000000000..8bad85a598 --- /dev/null +++ b/tests/unit/test_cell_trigger_ci_creator_http_ready.py @@ -0,0 +1,79 @@ +"""The fixed creator probe accepts only the published HTTP query response.""" + +import http.client +import sys + +import pytest + +from scripts.gates import cell_trigger_acceptance as creator + + +@pytest.mark.parametrize( + ("status", "body", "failure", "expected"), + [ + (200, b"1\n", None, True), + (200, b"1\nx", None, False), + (200, b"1", None, False), + (200, b"", None, False), + (500, b"1\n", None, False), + (200, b"1\n", "disconnect", False), + (200, b"1\n", "socket", False), + ], +) +def test_fixed_http_probe_checks_response_and_closes_connection( + monkeypatch, capsys, status, body, failure, expected +): + calls = [] + + class Response: + def __init__(self): + self.status = status + + def read(self, amount): + assert amount == 3 + calls.append("read") + return body + + class Connection: + def request(self, method, path, *, body, headers): + assert (method, path, body) == ("POST", "/", "SELECT 1") + assert headers == {"X-ClickHouse-User": "loom", "X-ClickHouse-Key": "loomdev"} + calls.append("request") + if failure == "socket": + raise OSError("not listening") + + def getresponse(self): + calls.append("response") + if failure == "disconnect": + raise http.client.RemoteDisconnected("bootstrap stopped") + return Response() + + def close(self): + calls.append("close") + + def connect(host, port, *, timeout): + assert (host, port, timeout) == ("127.0.0.1", 18000, 2) + return Connection() + + monkeypatch.setattr(http.client, "HTTPConnection", connect) + monkeypatch.setattr(sys, "argv", ["probe", "18000"]) + exec(creator._CLICKHOUSE_READY, {}) + assert capsys.readouterr().out == ("1\n" if expected else "") + assert calls[-1] == "close" + assert calls.count("close") == 1 + assert calls.count("read") == (1 if status == 200 and failure is None else 0) + + +def test_readiness_uses_existing_bounded_creator_command(tmp_path): + class RecordingStores(creator.Stores): + def command(self, *argv, timeout=120, cleanup=False): + self.recorded = (argv, timeout, cleanup) + return "1" + + stores = RecordingStores(tmp_path, tmp_path, "fixture", {"run_id": "123"}) + assert stores._clickhouse_ready(18000) + assert stores.recorded == ( + (sys.executable, "-I", "-S", "-c", creator._CLICKHOUSE_READY, "18000"), + 2, + False, + ) diff --git a/tests/unit/test_cell_trigger_driver_failure_details.py b/tests/unit/test_cell_trigger_driver_failure_details.py new file mode 100644 index 0000000000..15db973026 --- /dev/null +++ b/tests/unit/test_cell_trigger_driver_failure_details.py @@ -0,0 +1,247 @@ +"""The native driver retains finite Postgres failure evidence without payloads.""" + +import asyncio +import dataclasses +import json +import sys +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import asyncpg +import pytest + +import bench.cell_trigger_gate as gate +from tests.unit.test_bench_cell_trigger_gate import args, fixture_config +from tests.unit.test_cell_trigger_ci_creator import keeper_config_case as keeper_config_case +from tests.unit.test_cell_trigger_native_failure_details import detail + + +@pytest.mark.parametrize("fault", ["query_cancel", "connection", "unknown_commit"]) +def test_actual_stage_failure_retains_pg_cause_in_invalid_result( + monkeypatch, tmp_path, capsys, fault +): + output, calls = _main_case(monkeypatch, tmp_path, fault) + assert gate.main() == 1 + verdict = json.loads(capsys.readouterr().out) + assert verdict["verdict"] == "INVALID" + assert verdict["phase"] == "native_driver" + assert verdict["reason"] == "operation refused" + assert verdict["error_type"] == "PostgresUnavailableError" + assert calls == ["wire", "stage", "native_returned"] + assert "secret-trigger-diagnostic-payload" not in json.dumps(verdict) + assert "failure" in verdict + metadata = verdict["failure"] + assert metadata["error_type"] == "PostgresUnavailableError" + assert metadata["site"]["file"] == "pg.py" + assert metadata["site"]["function"] == "_raise_if_unavailable" + assert metadata["site"]["line"] > 0 + assert metadata["driver"]["outcome_unknown"] is (fault == "unknown_commit") + expected = ( + {"error_type": "ConnectionResetError", "errno": 104} + if fault == "connection" + else { + "error_type": "QueryCanceledError", + "errno": None, + "sqlstate": "57014", + "cancel_reason": "unknown", + } + ) + assert metadata["driver"]["causes"] == [expected] + invalid = json.loads(output.read_text().splitlines()[-1]) + assert invalid["kind"] == "invalid" and invalid["failure"] == metadata + + +def test_healthy_native_driver_keeps_pass_without_failure_metadata(monkeypatch, tmp_path, capsys): + output, calls = _main_case(monkeypatch, tmp_path, None) + assert gate.main() == 0 + verdict = json.loads(capsys.readouterr().out) + assert verdict["verdict"] == "PASS" and "failure" not in verdict + assert calls == ["wire", "native_returned"] + assert json.loads(output.read_text().splitlines()[-1])["kind"] == "driver_cleanup" + + +def _main_case(monkeypatch, tmp_path, fault): + import server.settings + import server.wiring + from server.cli import _ch + from server.core import pg + from server.memory.store import pg_memory + + fixture, config = fixture_config() + manifest = tmp_path / "manifest" + manifest.write_text(json.dumps(dataclasses.asdict(fixture))) + output = tmp_path / "driver.jsonl" + argv = ["driver"] + for name, value in { + **vars(args()), + "fixture_manifest": str(manifest), + "output": str(output), + }.items(): + argv += ["--" + name.replace("_", "-"), str(value)] + calls = [] + monkeypatch.setattr(sys, "argv", argv) + monkeypatch.setattr(server.settings, "settings", config) + monkeypatch.setattr(server.wiring, "wire", lambda: calls.append("wire")) + monkeypatch.setattr(gate, "_preflight", AsyncMock()) + monkeypatch.setattr(gate, "_source", lambda _: {"clean": True}) + monkeypatch.setattr(gate, "_launch", lambda *_: {}) + monkeypatch.setattr(gate, "_debt", AsyncMock()) + monkeypatch.setattr(gate, "_absent", AsyncMock()) + + async def commit(*_args, **_kwargs): + calls.append("stage") + cause = ( + ConnectionResetError(104, "secret-trigger-diagnostic-payload") + if fault == "connection" + else asyncpg.QueryCanceledError("secret-trigger-diagnostic-payload") + ) + pg._raise_if_unavailable(cause, outcome_unknown=fault == "unknown_commit") + raise AssertionError("the actual PG wrapper must refuse") + + monkeypatch.setattr(pg_memory, "commit_versions", commit) + + def native(*_args, **_kwargs): + try: + if fault: + block = gate.Block( + rows=[ + { + "org": "owned", + "namespace": "owned", + "memory_id": "00000000-0000-0000-0000-000000000011", + "memory_key": "owned", + } + ], + audits=[], + documents=[], + ) + asyncio.run( + gate._stage( + SimpleNamespace(fetchval=AsyncMock(return_value=0)), object(), block + ) + ) + return { + "latency_within_limit": True, + "throughput_within_limit": True, + "p99_delta_ms": 0.0, + "throughput_loss": 0.0, + } + finally: + calls.append("native_returned") + + monkeypatch.setattr(_ch, "run_connected", native) + return output, calls + + +@pytest.mark.parametrize("cause_state", ["57014", "08006", "secret-query-state", True]) +def test_shared_sanitizer_keeps_only_closed_pg_values(cause_state): + from scripts.gates.testsupport.cell_trigger.fixture import native_failure_metadata + from server.core.pg import PostgresUnavailableError + + cause = OSError(104, "secret-trigger-diagnostic-payload") + cause.sqlstate = cause_state + error = PostgresUnavailableError("secret-trigger-diagnostic-payload", outcome_unknown=True) + error.__cause__ = cause + metadata = native_failure_metadata(error) + assert metadata["driver"]["outcome_unknown"] is True + entry = metadata["driver"]["causes"][0] + assert entry["error_type"] == "ConnectionResetError" and entry["errno"] == 104 + if type(cause_state) is str and cause_state in {"57014", "08006"}: + assert entry["sqlstate"] == cause_state + else: + assert "sqlstate" not in entry + assert "secret-trigger-diagnostic-payload" not in json.dumps(metadata) + assert len(json.dumps(metadata)) < 4096 + + +def test_shared_sanitizer_preserves_existing_transport_shape_and_cycle_bound(): + from clickhouse_connect.driver.exceptions import OperationalError + + from scripts.gates.testsupport.cell_trigger.fixture import native_failure_metadata + + error = OperationalError("secret-trigger-diagnostic-payload", code=159) + cause = ConnectionRefusedError(111, "secret-trigger-diagnostic-payload") + error.__cause__ = cause + cause.__cause__ = error + metadata = native_failure_metadata(error) + assert metadata["driver"] == { + "code": 159, + "causes": [{"error_type": "ConnectionRefusedError", "errno": 111}], + } + assert "secret-trigger-diagnostic-payload" not in json.dumps(metadata) + + +def test_diagnostic_error_cannot_replace_actual_stage_invalid(monkeypatch, tmp_path, capsys): + output, calls = _main_case(monkeypatch, tmp_path, "query_cancel") + + def failed(*_args, **_kwargs): + raise RuntimeError("secret-trigger-diagnostic-payload") + + monkeypatch.setattr(gate, "native_failure_metadata", failed) + assert gate.main() == 1 + verdict = json.loads(capsys.readouterr().out) + assert verdict["verdict"] == "INVALID" and verdict["error_type"] == "PostgresUnavailableError" + assert verdict["phase"] == "native_driver" and verdict["reason"] == "operation refused" + assert "failure" not in verdict + assert calls == ["wire", "stage", "native_returned"] + invalid = json.loads(output.read_text().splitlines()[-1]) + assert invalid["kind"] == "invalid" and "failure" not in invalid + assert "secret-trigger-diagnostic-payload" not in json.dumps(verdict) + + +@pytest.mark.parametrize( + "fault", + [ + "none", + "root_sqlstate", + "cause_sqlstate", + "boolean_sqlstate", + "outcome_unknown", + "site_file", + "site_line", + "site_function", + "site_extra", + "driver_extra", + ], +) +def test_native_reader_validates_new_pg_fields_without_replacing_first_error( + keeper_config_case, fault +): + native, instance, owner, _, _ = keeper_config_case + instance.s["python"] = "unused" + metadata = detail() + driver = metadata["driver"] + driver.update(sqlstate="57014", outcome_unknown=True) + cause = driver["causes"][0] + cause.update( + sqlstate="08006", site={"file": "native.py", "line": 702, "function": "connections"} + ) + if fault == "root_sqlstate": + driver["sqlstate"] = "secret-query" + elif fault == "cause_sqlstate": + cause["sqlstate"] = "secret-query" + elif fault == "boolean_sqlstate": + driver["sqlstate"] = True + elif fault == "outcome_unknown": + driver["outcome_unknown"] = 1 + elif fault == "site_file": + cause["site"]["file"] = "/secret/path/native.py" + elif fault == "site_line": + cause["site"]["line"] = True + elif fault == "site_function": + cause["site"]["function"] = "secret-payload" + elif fault == "site_extra": + cause["site"]["message"] = "secret-payload" + elif fault == "driver_extra": + driver["message"] = "secret-payload" + native.write(instance.w / "pg_cell-created.json.failure.json", metadata) + original = native.Refused("original-command-failure") + + def refused(*_args, **_kwargs): + raise original + + owner.command = refused + with pytest.raises(native.Refused) as caught: + instance.action("create", role="pg_cell") + assert caught.value is original + assert original.native == (metadata if fault == "none" else None) diff --git a/tests/unit/test_cell_trigger_gate_artifact_close.py b/tests/unit/test_cell_trigger_gate_artifact_close.py new file mode 100644 index 0000000000..b403eb4ebb --- /dev/null +++ b/tests/unit/test_cell_trigger_gate_artifact_close.py @@ -0,0 +1,77 @@ +"""Artifact close retains its first flush, sync or file-close failure.""" + +import asyncio +import dataclasses +import json + +import pytest + +from bench import cell_trigger_gate as gate +from tests.unit.test_bench_cell_trigger_gate import fixture_config, receipt_posture + + +@pytest.mark.parametrize("phase", ["flush", "fsync"]) +@pytest.mark.parametrize( + "first_kind", [None, OSError, ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] +) +@pytest.mark.parametrize( + "second_kind", + [None, OSError, ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt], +) +def test_actual_artifact_close_retains_first_flush_sync_or_close_failure( + monkeypatch, tmp_path, phase, first_kind, second_kind +): + first = first_kind("flush or sync failed first") if first_kind is not None else None + second = second_kind("file close failed second") if second_kind is not None else None + artifact = gate.Artifact(str(tmp_path / "raw.jsonl")) + fixture, config = fixture_config() + artifact.emit( + "configuration", + manifest=dataclasses.asdict(fixture), + shared_configuration_excludes=["cell_router_token"], + credential_posture=receipt_posture(), + api_configuration_attested=True, + ) + original = artifact.file + events = [] + + class File: + def flush(self): + events.append("flush") + if phase == "flush" and first is not None: + raise first + return original.flush() + + def close(self): + events.append("file close") + original.close() + if second is not None: + raise second + + def __getattr__(self, name): + return getattr(original, name) + + def sync(_fd): + events.append("fsync") + if phase == "fsync" and first is not None: + raise first + + artifact.file = File() + monkeypatch.setattr(gate.os, "fsync", sync) + selected = first if first is not None else second + if selected is None: + artifact.close() + else: + with pytest.raises(BaseException) as observed: + artifact.close() + assert observed.value is selected + assert original.closed and events[-1] == "file close" + assert events.count("file close") == 1 + assert ("fsync" in events) == (phase != "flush" or first is None) + record = json.loads((tmp_path / "raw.jsonl").read_text()) + assert record["manifest"]["schema"] == 2 + assert record["manifest"]["cli_configuration_sha256"] == gate._digest(config.model_dump()) + assert "configuration_sha256" not in record["manifest"] + assert record["shared_configuration_excludes"] == ["cell_router_token"] + assert record["credential_posture"] == receipt_posture() + assert config.pg_dsn not in json.dumps(record) diff --git a/tests/unit/test_cell_trigger_gate_diagnostics.py b/tests/unit/test_cell_trigger_gate_diagnostics.py new file mode 100644 index 0000000000..580eb3d733 --- /dev/null +++ b/tests/unit/test_cell_trigger_gate_diagnostics.py @@ -0,0 +1,240 @@ +"""Artifact diagnostics retain the first driver failure at the actual owner.""" + +import argparse +import asyncio +import dataclasses +import json +from unittest.mock import AsyncMock + +import pytest + +from bench import cell_trigger_gate as gate +from server.cli import cell_attach_pg as apg + + +@pytest.fixture +def run_driver(monkeypatch, tmp_path): + from server.core import background + + def configure(*, prior=None, cleanup=None, output=None, event="cleanup_error"): + events = [] + + class Conn: + async def close(self, **kwargs): + events.append("public close") + + def terminate(self): + events.append("abort") + + class Triggers: + tables = ("memory_current",) + invariant = org = "digest" + + async def set(self, *args): + pass + + async def restore(self, *args): + events.append("restoration") + if cleanup is not None: + raise cleanup + + artifact = gate.Artifact(str(tmp_path / "raw.jsonl")) + original = artifact.file + + class File: + def write(self, line): + kind = json.loads(line)["kind"] + events.append(kind) + if kind == event and output is not None: + raise output + return original.write(line) + + def __getattr__(self, name): + return getattr(original, name) + + artifact.file = File() + monkeypatch.setattr(gate.asyncpg, "connect", AsyncMock(return_value=Conn())) + monkeypatch.setattr(gate, "_proof", AsyncMock(return_value={})) + monkeypatch.setattr(gate.Triggers, "capture", AsyncMock(return_value=Triggers())) + monkeypatch.setattr(gate, "_memory_set", AsyncMock(side_effect=prior)) + if prior is None: + monkeypatch.setattr(gate, "_arm_order", lambda: ()) + monkeypatch.setattr(gate, "_statistics", lambda samples: {}) + monkeypatch.setattr(background, "drain_task_sets", AsyncMock(return_value=True)) + monkeypatch.setattr(background, "flush_registered_buffers", AsyncMock(return_value=True)) + + async def run(): + try: + return await gate._run( + argparse.Namespace(pg_dsn="unused"), None, artifact, object() + ) + finally: + artifact.close() + + return run, events + + return configure + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "prior_kind", [None, ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] +) +@pytest.mark.parametrize( + "output_kind", + [None, OSError, RuntimeError, asyncio.CancelledError, SystemExit, KeyboardInterrupt], +) +async def test_outer_cleanup_diagnostic_preserves_selected_first_error( + run_driver, prior_kind, output_kind +): + prior = prior_kind("work failed first") if prior_kind is not None else None + cleanup = ValueError("cleanup failed") + output = output_kind("diagnostic failed") if output_kind is not None else None + run, events = run_driver(prior=prior, cleanup=cleanup, output=output) + + async def owner(): + try: + await run() + except BaseException as error: # noqa: BLE001 — assert the actual owner failure. + assert error is (prior if prior is not None else cleanup) + assert events.count("cleanup_error") == 1 + assert events.index("public close") < events.index("cleanup_error") + return + raise AssertionError("selected failure was swallowed") + + assert await apg._work_result(owner) is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("event", ["provisional_statistics", "triggers_restored"]) +@pytest.mark.parametrize( + "output_kind", [OSError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] +) +async def test_required_output_failure_after_successful_work_still_fails( + run_driver, event, output_kind +): + first = output_kind("required artifact failed") + run, events = run_driver(output=first, event=event) + + async def owner(): + try: + await run() + except BaseException as error: # noqa: BLE001 — native errors stay in their owner. + assert error is first + assert "restoration" in events and "public close" in events + return + raise AssertionError("required output failure was swallowed") + + assert await apg._work_result(owner) is None + + +@pytest.mark.asyncio +async def test_successful_driver_and_cleanup_output_returns_normally(run_driver): + run, events = run_driver() + assert await run() == {} + assert events == ["provisional_statistics", "restoration", "triggers_restored", "public close"] + + +@pytest.mark.parametrize("phase", ["invalid", "close"]) +@pytest.mark.parametrize( + "prior_kind", [None, ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] +) +@pytest.mark.parametrize( + "output_kind", + [None, OSError, RuntimeError, asyncio.CancelledError, SystemExit, KeyboardInterrupt], +) +def test_main_preserves_first_invalid_and_requires_successful_artifact_close( + monkeypatch, tmp_path, capsys, phase, prior_kind, output_kind +): + import sys + + import server.settings + import server.wiring + from server.cli import _ch + from tests.unit.test_bench_cell_trigger_gate import args, fixture_config, launch_files + + fixture, config = fixture_config() + fixture, launch_path, _, _, _ = launch_files(tmp_path, fixture, config) + manifest_path = tmp_path / "fixture.json" + manifest_path.write_text(json.dumps(dataclasses.asdict(fixture))) + prior = prior_kind("first native work failure") if prior_kind is not None else None + output = output_kind("secondary artifact failure") if output_kind is not None else None + events = [] + real_artifact = gate.Artifact + + class Artifact(real_artifact): + def __init__(self, path): + super().__init__(path) + original = self.file + + class File: + def write(self, line): + kind = json.loads(line)["kind"] + events.append(kind) + if phase == "invalid" and kind == "invalid" and output is not None: + raise output + return original.write(line) + + def close(self): + events.append("file closed") + return original.close() + + def __getattr__(self, name): + return getattr(original, name) + + self.file = File() + + argv = ["driver"] + for key, value in vars( + args( + fixture_manifest=str(manifest_path), + launch_manifest=str(launch_path), + output=str(tmp_path / "raw.jsonl"), + ) + ).items(): + argv += ["--" + key.replace("_", "-"), str(value)] + monkeypatch.setattr(sys, "argv", argv) + monkeypatch.setattr(server.settings, "settings", config) + monkeypatch.setattr(server.wiring, "wire", lambda: None) + monkeypatch.setattr(gate, "_source", lambda _: {}) + monkeypatch.setattr(gate, "_owned_server", lambda _: None) + monkeypatch.setattr(gate, "_preflight", AsyncMock()) + monkeypatch.setattr(gate, "Artifact", Artifact) + + def native(*_args, **_kwargs): + if prior is not None: + raise prior + return { + "latency_within_limit": True, + "throughput_within_limit": True, + "p99_delta_ms": 0.0, + "throughput_loss": 0.0, + } + + def sync(_fd): + assert capsys.readouterr().out == "" + events.append("file sync") + if phase == "close" and output is not None: + raise output + + monkeypatch.setattr(_ch, "run_connected", native) + monkeypatch.setattr(gate.os, "fsync", sync) + invalid = prior is not None or (phase == "close" and output is not None) + try: + result = gate.main() + except BaseException as error: # noqa: BLE001 — an escaped fatal must fail this case only. + pytest.fail(f"main escaped {type(error).__name__}") + assert result == int(invalid) + verdict = json.loads(capsys.readouterr().out) + assert verdict["verdict"] == ("INVALID" if invalid else "PASS") + if prior is not None: + assert verdict["error_type"] == prior_kind.__name__ + assert verdict["phase"] == "native_driver" + elif invalid: + assert verdict["error_type"] == output_kind.__name__ + assert verdict["phase"] == "artifact_close" + assert events[-2:] == ["file sync", "file closed"] + assert not any( + json.loads(line).get("verdict") == "PASS" + for line in (tmp_path / "raw.jsonl").read_text().splitlines() + ) diff --git a/tests/unit/test_cell_trigger_gate_error_envelope.py b/tests/unit/test_cell_trigger_gate_error_envelope.py new file mode 100644 index 0000000000..2bdf47074e --- /dev/null +++ b/tests/unit/test_cell_trigger_gate_error_envelope.py @@ -0,0 +1,86 @@ +"""Failed benchmark requests classify real wire errors without echoing payloads.""" + +import json + +import httpx +import pytest +from fastapi import FastAPI + +from bench import cell_trigger_gate as gate +from server.core.errors import register_exception_handler +from server.core.pg_lease import PublicationPending +from tests.unit.test_bench_cell_trigger_gate import args + + +@pytest.fixture(autouse=True) +def no_database(monkeypatch): + async def forbidden(*args, **kwargs): + raise AssertionError("error classification must not open a database") + + monkeypatch.setattr(gate.asyncpg, "connect", forbidden) + monkeypatch.setattr(gate.clickhouse_connect, "get_async_client", forbidden) + + +@pytest.mark.asyncio +async def test_publication_pending_from_the_server_handler_retains_its_code(monkeypatch): + app = FastAPI() + register_exception_handler(app) + + @app.post("/v1/memory.set") + async def pending(): + error = PublicationPending() + error.message = "secret-token" + raise error + + transport = httpx.ASGITransport(app=app) + envelopes = [] + + async def send(self, request, **kwargs): + response = await transport.handle_async_request(request) + await response.aread() + envelopes.append(response.json()) + return response + + monkeypatch.setattr(httpx.AsyncClient, "send", send) + events = [] + with pytest.raises(gate.Invalid, match="memory.set HTTP status"): + await gate._memory_set(args(), emit=lambda kind, **row: events.append((kind, row))) + assert len(envelopes) == 1 and "error" not in envelopes[0] + assert envelopes[0]["detail"]["code"] == "PUBLICATION_PENDING" + assert envelopes[0]["detail"]["committed"] is True + assert len(events) == 1 and events[0][0] == "request_error" + assert events[0][1]["status"] == 503 + assert events[0][1]["code"] == "PUBLICATION_PENDING" + assert events[0][1]["measured"] is False + assert "secret-token" not in json.dumps(events) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "candidate,expected", + [ + pytest.param("BAD_CONFIG", "BAD_CONFIG", id="valid-code"), + pytest.param("A" * 64, "A" * 64, id="maximum-length"), + pytest.param("secret-token", "", id="unsafe-text"), + pytest.param("A" * 65, "", id="overlong"), + pytest.param(42, "", id="number"), + pytest.param(None, "", id="null"), + pytest.param([], "", id="list"), + ], +) +async def test_detail_code_uses_the_existing_allowlist(monkeypatch, candidate, expected): + async def send(self, request, **kwargs): + return httpx.Response( + 503, + request=request, + json={"detail": {"code": candidate, "message": "secret-token"}}, + ) + + monkeypatch.setattr(httpx.AsyncClient, "send", send) + events = [] + with pytest.raises(gate.Invalid, match="memory.set HTTP status"): + await gate._memory_set(args(), emit=lambda kind, **row: events.append((kind, row))) + assert len(events) == 1 and events[0][0] == "request_error" + assert events[0][1]["status"] == 503 + assert events[0][1]["code"] == expected + assert "secret-token" not in json.dumps(events) diff --git a/tests/unit/test_cell_trigger_gate_first_error.py b/tests/unit/test_cell_trigger_gate_first_error.py new file mode 100644 index 0000000000..5379b0f619 --- /dev/null +++ b/tests/unit/test_cell_trigger_gate_first_error.py @@ -0,0 +1,332 @@ +"""Driver work retains its error through secondary cleanup and artifact failures.""" + +import asyncio +import dataclasses +import json +import time +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import httpx +import pytest +from asyncpg.transaction import Transaction + +from bench import cell_trigger_gate as gate +from tests.unit.test_bench_cell_trigger_gate import args, fixture_config, receipt_posture + +FIRST = [None, ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] +SECOND = [None, OSError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] + + +def failure(kind): + return None if kind is None else kind("injected failure") + + +async def observed(work): + try: + await work() + except BaseException as error: # noqa: BLE001 — native errors stay inside this test. + return error + return None + + +@pytest.fixture(autouse=True) +def no_external_io(monkeypatch): + async def forbidden(*_args, **_kwargs): + raise AssertionError("unexpected external I/O") + + monkeypatch.setattr(gate.asyncpg, "connect", forbidden) + monkeypatch.setattr(gate.clickhouse_connect, "get_async_client", forbidden) + monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", forbidden) + + +@pytest.fixture +def artifact_output(monkeypatch, tmp_path): + artifacts = [] + + def configure(errors): + artifact = gate.Artifact(str(tmp_path / f"raw-{len(artifacts)}.jsonl")) + artifacts.append(artifact) + fixture, _ = fixture_config() + artifact.emit( + "configuration", + manifest=dataclasses.asdict(fixture), + shared_configuration_excludes=["cell_router_token"], + credential_posture=receipt_posture(), + api_configuration_attested=True, + ) + original = artifact.file + events = [] + + class File: + def write(self, line): + kind = json.loads(line)["kind"] + events.append(kind) + if (error := errors.get(kind)) is not None: + raise error + return original.write(line) + + def __getattr__(self, name): + return getattr(original, name) + + artifact.file = File() + return artifact, events + + yield configure + for artifact in artifacts: + artifact.close() + + +@pytest.fixture +def http_edges(monkeypatch): + import server.settings + + fixture, config = fixture_config() + assert fixture.validate(args(), config) == 45500 + assert config.cell_router_token == config.router_operator_token == "" + monkeypatch.setattr(server.settings, "settings", config) + monkeypatch.setattr(gate, "CALLS", 1) + monkeypatch.setattr(gate, "WARMUP", 0) + client_class = httpx.AsyncClient + + def configure(first=None, closing=None, *, bad_status=False): + events = [] + + class Transport(httpx.AsyncBaseTransport): + async def handle_async_request(self, request): + events.append("request") + if first is not None: + raise first + if request.url.path.endswith("/memory.set"): + sent = json.loads(request.content) + body = { + "memory_id": "00000000-0000-0000-0000-000000000001", + "version": 1, + "status": "active", + "memory_key": sent["memory_key"], + "namespace": sent["namespace"], + "content_sha256": gate.sha256_value(sent["value"]), + } + elif request.url.path.endswith("/deployment"): + body = {"deployment_id": fixture.deployment_id} + else: + body = { + "build_commit": fixture.build_commit, + "deployment_config": {"instance_id": fixture.instance_id}, + "ch_ok": True, + "pg_ok": True, + "status": "ok", + "fake_embedding": True, + "fake_llm": True, + "pending_migrations": [], + "storage_authority": "postgres", + "llm_provider": config.llm_provider, + "llm_model": config.llm_model, + "embedding_model": config.embedding_model, + "embedding_dim_configured": config.embedding_dim, + "embedding_dim_probed": config.embedding_dim, + "instance": "test:1", + } + return httpx.Response(503 if bad_status else 200, json=body) + + async def aclose(self): + events.append("http close") + if closing is not None: + raise closing + + def client(**kwargs): + return client_class(**kwargs, transport=Transport()) + + monkeypatch.setattr(gate.httpx, "AsyncClient", client) + return fixture, events + + return configure + + +@pytest.mark.asyncio +@pytest.mark.parametrize("first_kind", FIRST) +@pytest.mark.parametrize("ch_kind", SECOND) +@pytest.mark.parametrize("pg_kind", SECOND) +async def test_preflight_retains_first_error_and_attempts_both_closes( + monkeypatch, first_kind, ch_kind, pg_kind +): + first, ch_error, pg_error = map(failure, (first_kind, ch_kind, pg_kind)) + events = [] + + async def proof(*_args): + events.append("proof") + if first is not None: + raise first + + async def close_ch(): + events.append("ch close") + if ch_error is not None: + raise ch_error + + async def close_pg(*_args): + events.append("pg close") + if pg_error is not None: + raise pg_error + + monkeypatch.setattr(gate.asyncpg, "connect", AsyncMock(return_value=object())) + monkeypatch.setattr( + gate.ReadOnlyCH, "open", AsyncMock(return_value=SimpleNamespace(close=close_ch)) + ) + monkeypatch.setattr(gate, "_proof", proof) + monkeypatch.setattr(gate, "_close_pg", close_pg) + assert await observed(lambda: gate._preflight(args(), None)) is (first or ch_error or pg_error) + assert events == ["proof", "ch close", "pg close"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("consumer", ["health", "memory"]) +@pytest.mark.parametrize("first_kind", FIRST) +@pytest.mark.parametrize("close_kind", SECOND) +async def test_http_client_exit_retains_work_failure_and_requires_successful_close( + http_edges, consumer, first_kind, close_kind +): + first, closing = failure(first_kind), failure(close_kind) + fixture, events = http_edges(first, closing) + + async def work(): + if consumer == "health": + await gate._health(args(), fixture, {"health_instance": "test:1"}) + else: + await gate._memory_set(args()) + + assert await observed(work) is (first or closing) + assert events[-1] == "http close" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("consumer", ["health", "memory"]) +async def test_semantic_http_refusal_survives_client_close_error(http_edges, consumer): + fixture, events = http_edges(closing=OSError("close failed"), bad_status=True) + + async def work(): + if consumer == "health": + await gate._health(args(), fixture, {"health_instance": "test:1"}) + else: + await gate._memory_set(args()) + + field = "health HTTP status" if consumer == "health" else "memory.set HTTP status" + error = await observed(work) + assert isinstance(error, gate.Invalid) and str(error) == field + assert events[-1] == "http close" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("source", ["request", "request_record"]) +@pytest.mark.parametrize("first_kind", FIRST[1:]) +@pytest.mark.parametrize("output_kind", SECOND) +async def test_request_diagnostic_retains_request_or_required_record_failure( + http_edges, artifact_output, source, first_kind, output_kind +): + first, output = failure(first_kind), failure(output_kind) + _, http_events = http_edges(first if source == "request" else None) + errors = {"request_error": output} + if source == "request_record": + errors["request"] = first + artifact, events = artifact_output(errors) + assert await observed(lambda: gate._memory_set(args(), emit=artifact.emit)) is first + assert events[-1] == "request_error" and http_events[-1] == "http close" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("record", ["worker_error", "worker"]) +@pytest.mark.parametrize("first_kind", FIRST[1:]) +@pytest.mark.parametrize("output_kind", SECOND) +async def test_worker_diagnostics_and_prior_results_retain_worker_failure( + monkeypatch, artifact_output, record, first_kind, output_kind +): + from server.memory.workers import pg_delivery + from server.settings import settings + + first, output = failure(first_kind), failure(output_kind) + good = pg_delivery.DeliveryResult(claimed=1, published=1) + outcomes = [good, first] if record == "worker" else [first] + monkeypatch.setattr(gate, "BATCHES", len(outcomes)) + monkeypatch.setattr(settings, "pg_delivery_batch_rows", 1) + worker = AsyncMock(side_effect=outcomes) + monkeypatch.setattr(pg_delivery, "run_pass", worker) + artifact, events = artifact_output({record: output}) + assert await observed(lambda: gate._delivery(args(), 1, emit=artifact.emit, arm=0)) is first + assert worker.await_count == len(outcomes) and "worker_error" in events + if record == "worker": + assert events == ["worker_error", "worker"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("output_kind", SECOND) +async def test_successful_worker_work_still_requires_result_output( + monkeypatch, artifact_output, output_kind +): + from server.memory.workers import pg_delivery + from server.settings import settings + + output = failure(output_kind) + monkeypatch.setattr(gate, "BATCHES", 1) + monkeypatch.setattr(settings, "pg_delivery_batch_rows", 1) + worker = AsyncMock(return_value=pg_delivery.DeliveryResult(claimed=1, published=1)) + monkeypatch.setattr(pg_delivery, "run_pass", worker) + ticks = iter([0, 1_000_000_000]) + monkeypatch.setattr(gate.time, "perf_counter_ns", lambda: next(ticks)) + artifact, events = artifact_output({"worker": output}) + assert await observed(lambda: gate._delivery(args(), 1, emit=artifact.emit, arm=0)) is output + assert worker.await_count == 1 + assert events == (["worker"] if output is not None else ["worker", "delivery_block"]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method", ["set", "restore"]) +@pytest.mark.parametrize("first_kind", FIRST) +@pytest.mark.parametrize("exit_kind", SECOND) +async def test_trigger_work_retains_error_through_transaction_exit_and_owner_cleanup( + monkeypatch, method, first_kind, exit_kind +): + first, exit_error = failure(first_kind), failure(exit_kind) + events = [] + + class Connection: + _pool_release_ctr = 0 + _top_xact = None + _protocol = SimpleNamespace(is_in_transaction=lambda: False) + + def __init__(self): + self.closed = False + + def transaction(self): + return Transaction(self, None, False, False) + + def is_closed(self): + return self.closed + + async def execute(self, sql): + events.append(sql.split()[0]) + if sql.startswith("DROP") and first is not None: + raise first + if sql.startswith(("ROLLBACK", "COMMIT")) and exit_error is not None: + raise exit_error + + async def close(self, **kwargs): + events.append("public close") + self.closed = True + + def terminate(self): + events.append("abort") + self.closed = True + + triggers = gate.Triggers(("memory_current",), "invariant", "org") + monkeypatch.setattr(triggers, "verify", AsyncMock()) + monkeypatch.setattr(gate.cell_triggers, "missing", AsyncMock(return_value=[])) + conn = Connection() + + async def work(): + if method == "set": + await triggers.set(conn, False) + else: + await gate._restore(triggers, conn, time.monotonic() + 1) + + assert await observed(work) is (first or exit_error) + if method == "restore" and (first is not None or exit_error is not None): + assert conn.closed and "public close" in events diff --git a/tests/unit/test_cell_trigger_gate_json_codec.py b/tests/unit/test_cell_trigger_gate_json_codec.py new file mode 100644 index 0000000000..62c9aefa46 --- /dev/null +++ b/tests/unit/test_cell_trigger_gate_json_codec.py @@ -0,0 +1,201 @@ +"""The full stored-row proof accepts native JSON without weakening other fields.""" + +from __future__ import annotations + +import json +import logging +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from bench import cell_trigger_gate as gate +from tests.unit.test_bench_cell_trigger_gate import ( + args, + stored_bfloat16, +) +from tests.unit.test_bench_cell_trigger_gate import ( + no_external_io as no_external_io, +) + + +def native_json(monkeypatch, value): + from clickhouse_connect.datatypes import dynamic + from clickhouse_connect.datatypes.base import TypeDef + from clickhouse_connect.driver.context import BaseQueryContext + + # A fresh type leaves the installed registry and global format map untouched. + dtype = dynamic.JSON(TypeDef()) + context = BaseQueryContext() + assert dtype.read_format(context) == "native" + dtype.typed_paths = list(value) + dtype.typed_types = [ + SimpleNamespace(read_column_data=lambda *_args, v=v: [v]) for v in value.values() + ] + monkeypatch.setattr( + dynamic, + "SHARED_DATA_TYPE", + SimpleNamespace(read_column_data=lambda *_args: [{}]), + ) + state = dynamic.JSONState(2, [], [None] * len(value), [], None) + decoded = dtype._read_column_binary(object(), 1, context, state)[0] + assert type(decoded) is dict + assert decoded == value + return decoded + + +@pytest.mark.parametrize( + "value", + [ + {}, + {"n": 1}, + {"flag": True}, + {"nullable": None}, + {"nested": {"b": [1, False, None], "a": "text"}}, + {"when": "2026-01-02T03:04:05.123456+01:00"}, + {"text": '\u03bb\n"quoted"'}, + {"z": 0, "a": "1"}, + ], +) +def test_installed_native_json_matches_serialized_authority(monkeypatch, value): + decoded = native_json(monkeypatch, value) + gate._compare( + {"value_json": json.dumps(value, indent=2)}, + {"value_json": decoded}, + ("value_json",), + "full memory projection", + ) + + +@pytest.mark.parametrize( + ("expected", "actual"), + [ + ({}, {"extra": 1}), + ({"n": 1}, {"n": True}), + ({"n": 1}, {"n": "1"}), + ({"v": None}, {}), + ({"v": [1, 2]}, {"v": [2, 1]}), + ({"v": {}}, {"v": []}), + ({"v": "2026-01-02T03:04:05.123456"}, {"v": "2026-01-02T03:04:05.123999"}), + ({"v": "2026-01-02T03:04:05+01:00"}, {"v": "2026-01-02T03:04:05+00:00"}), + ], +) +def test_native_json_rejects_changed_json_value(monkeypatch, expected, actual): + with pytest.raises(gate.Invalid, match="full memory projection value_json"): + gate._compare( + {"value_json": json.dumps(expected)}, + {"value_json": native_json(monkeypatch, actual)}, + ("value_json",), + "full memory projection", + ) + + +@pytest.mark.parametrize("side", ["expected", "actual"]) +def test_malformed_json_is_an_invalid_projection(side): + expected, actual = {"value_json": "{}"}, {"value_json": {}} + (expected if side == "expected" else actual)["value_json"] = "{broken" + with pytest.raises(gate.Invalid, match="full memory projection value_json"): + gate._compare(expected, actual, ("value_json",), "full memory projection") + + +def test_json_normalization_applies_only_to_value_json(): + with pytest.raises(gate.Invalid, match="full memory projection value"): + gate._compare({"value": "{}"}, {"value": {}}, ("value",), "full memory projection") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", + ["", "value_json", "value", "version", "vector", "audit_hash"], + ids=["healthy", "value_json", "value", "version", "vector", "audit_hash"], +) +async def test_actual_verify_keeps_full_proof_with_native_json(monkeypatch, fault): + import server.embedding + from server.core.polled_copy import PolledCopy + from server.embedding.fake import FakeNGramEmbedder + from server.llm import customer + from server.memory.store import pg_memory + from server.memory.store.rows import MEMORY_PERSISTED_COLS + + monkeypatch.setattr( + customer, + "_switch", + PolledCopy("switch", AsyncMock(return_value=frozenset()), logging.getLogger(__name__)), + ) + monkeypatch.setattr(server.embedding, "get_embedder", lambda: FakeNGramEmbedder(dim=16)) + block = await gate._build_block(args(), 1) + block.seqs = [("org", "ns", i + 1) for i in range(5)] + assert len(MEMORY_PERSISTED_COLS) == 61 + assert all(set(row) == set(MEMORY_PERSISTED_COLS) for row in block.rows) + records = [ + { + "head": json.dumps(pg_memory.encode_row(row)), + "revision": json.dumps(pg_memory.encode_row(row)), + "audit": json.dumps(pg_memory.encode_audit(audit)), + "seq": i + 1, + "targets": 0, + "attempts": 0, + "last_error": "", + } + for i, (row, audit) in enumerate(zip(block.rows, block.audits, strict=True)) + ] + projections = [ + {**row, "value_json": native_json(monkeypatch, json.loads(row["value_json"]))} + for row in block.rows + ] + documents = [dict(row) for row in block.documents] + for document in documents: + for field in ("embedding", "memory_key_embedding"): + document[field] = stored_bfloat16(document[field]) + audits = [dict(row) for row in block.audits] + if fault == "value_json": + projections[0]["value_json"] = native_json(monkeypatch, {"wrong": True}) + elif fault == "value": + projections[0]["value"] = '"wrong"' + elif fault == "version": + projections[0]["version"] -= 1 + elif fault == "vector": + documents[0]["embedding"] = [0.0] * 16 + elif fault == "audit_hash": + audits[0]["row_hash"] = "bad" + conn = SimpleNamespace( + fetch=AsyncMock(return_value=records), + fetchrow=AsyncMock(return_value={"pending": 0, "audits": 0, "errors": 0}), + ) + phases = [] + compare = gate._compare + + def full_compare(expected, actual, fields, phase): + if phase == "full memory projection": + assert set(fields) == set(block.rows[0]) and len(fields) == 61 + phases.append(phase) + return compare(expected, actual, fields, phase) + + monkeypatch.setattr(gate, "_compare", full_compare) + + async def query(sql, params): + assert params["org"] == "org" and params["ns"] == "ns" + if "FROM loom.memory FINAL" in sql: + return projections + if "FROM loom.memory_embeddings FINAL" in sql: + return documents + if "FROM loom.audit_log" in sql: + return audits + raise AssertionError(sql) + + if fault: + reason = { + "value_json": "full memory projection value_json", + "value": "full memory projection value$", + "version": "full memory projection version", + "vector": "full search document vector", + "audit_hash": "exact stored audit hashes", + }[fault] + with pytest.raises(gate.Invalid, match=reason): + await gate._verify(conn, SimpleNamespace(query_all=query), args(), block) + else: + await gate._verify(conn, SimpleNamespace(query_all=query), args(), block) + assert phases.count("full memory projection") == 5 + assert phases.count("full search document") == 5 + assert phases.count("full stored audit") == 5 + conn.fetchrow.assert_awaited_once() diff --git a/tests/unit/test_cell_trigger_native_failure_details.py b/tests/unit/test_cell_trigger_native_failure_details.py new file mode 100644 index 0000000000..8f661ac541 --- /dev/null +++ b/tests/unit/test_cell_trigger_native_failure_details.py @@ -0,0 +1,129 @@ +"""Native failure details expose bounded numbers and classes without payloads.""" + +import json +import sys +from copy import deepcopy + +import pytest +from clickhouse_connect.driver.exceptions import OperationalError + +from tests.unit.test_cell_trigger_ci_creator import keeper_config_case as keeper_config_case + + +def detail(): + return { + "error_type": "OperationalError", + "site": {"file": "native.py", "line": 702, "function": "connections"}, + "outcome": "error", + "driver": { + "code": 159, + "causes": [{"error_type": "ConnectionRefusedError", "errno": 111}], + }, + } + + +@pytest.mark.parametrize("shape", ["transport", "bounded-cycle"]) +def test_native_sidecar_keeps_only_numeric_code_and_bounded_causes( + keeper_config_case, monkeypatch, tmp_path, shape +): + native, instance, _, _, _ = keeper_config_case + context, output = tmp_path / "failure-context.json", tmp_path / "pg_cell-created.json" + native.write( + context, + instance.s | {"root": str(tmp_path), "databases": {"pg_cell": "bench_cell"}}, + ) + monkeypatch.setattr(sys, "path", list(sys.path)) + primary = OperationalError("private-SQL-URL-settings-token", code=159) + cause = ConnectionRefusedError(111, "private-network-address") + primary.__cause__ = cause + expected = detail()["driver"] + if shape == "bounded-cycle": + previous = cause + for _ in range(6): + child = OSError(5, "private-payload") + previous.__cause__ = child + previous = child + previous.__cause__ = cause + expected["causes"] += [{"error_type": "OSError", "errno": 5}] * 3 + + async def failed(*args, **kwargs): + raise primary + + monkeypatch.setattr(native, "stores", failed) + monkeypatch.setattr( + sys, + "argv", + [ + "native.py", + "--task", + "create", + "--role", + "pg_cell", + "--context", + str(context), + "--output", + str(output), + ], + ) + assert native.main() == 1 + assert not output.exists() + raw = output.with_name(output.name + ".failure.json").read_bytes() + metadata = native.read(output.with_name(output.name + ".failure.json"), limit=4096) + assert metadata.get("driver") == expected + assert metadata["error_type"] == "OperationalError" + assert len(raw) <= 4096 and b"private-" not in raw + + +def test_valid_driver_detail_reaches_the_original_command_failure(keeper_config_case): + native, instance, owner, _, _ = keeper_config_case + instance.s["python"] = "unused" + metadata = detail() + native.write(instance.w / "pg_cell-created.json.failure.json", metadata) + original = native.Refused("private-command-failure") + + def refused(*args, **kwargs): + raise original + + owner.command = refused + with pytest.raises(native.Refused) as caught: + instance.action("create", role="pg_cell") + assert caught.value is original and original.native == metadata + assert "private-" not in json.dumps(original.native) + + +@pytest.mark.parametrize( + "field,value", + [ + ("code", True), + ("code", -1), + ("code", 2**31), + ("code", "private-code"), + ("causes", [{"error_type": "OSError", "errno": 5}] * 5), + ("class", "private-class-name"), + ("class", "A" * 65), + ("errno", True), + ("errno", 2**31), + ("extra", "private-message"), + ], +) +def test_unsafe_driver_detail_cannot_replace_the_command_failure(keeper_config_case, field, value): + native, instance, owner, _, _ = keeper_config_case + instance.s["python"] = "unused" + metadata = deepcopy(detail()) + driver = metadata["driver"] + if field in {"code", "causes"}: + driver[field] = value + elif field in {"class", "errno"}: + driver["causes"][0]["error_type" if field == "class" else "errno"] = value + else: + driver["message"] = value + native.write(instance.w / "pg_cell-created.json.failure.json", metadata) + original = native.Refused("original") + + def refused(*args, **kwargs): + raise original + + owner.command = refused + with pytest.raises(native.Refused) as caught: + instance.action("create", role="pg_cell") + assert caught.value is original and original.native is None diff --git a/tests/unit/test_cell_trigger_resource_census.py b/tests/unit/test_cell_trigger_resource_census.py new file mode 100644 index 0000000000..fb3cf02058 --- /dev/null +++ b/tests/unit/test_cell_trigger_resource_census.py @@ -0,0 +1,276 @@ +"""A vanished descendant requires a fresh complete resource census.""" + +from collections import Counter +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from scripts.gates.testsupport.cell_trigger import fixture + + +@pytest.fixture +def census(monkeypatch, tmp_path): + class Owned: + pid = 100 + + def __init__(self): + self.round = 0 + self.reads = Counter() + self.sampled_pids = [] + self.rows = {100: (1, "self", "S"), 200: (1, "store", "S")} + self.trees = [[300], [400]] + self.identify = lambda pid: self.rows.get(pid, (100, str(pid), "S")) + self.enumerate = lambda pid: ( + self.trees[min(self.round - 1, len(self.trees) - 1)] if pid == 100 else [] + ) + self.measure = lambda pid: 1 + + def identity(self, pid): + self.reads[pid] += 1 + return self.identify(pid) + + def children(self, pid): + if pid == self.pid: + self.round += 1 + return self.enumerate(pid) + + def rss(self, pid): + self.sampled_pids.append(pid) + return self.measure(pid) + + owned = Owned() + clock = SimpleNamespace(now=10.0) + original_read = Path.read_text + + def read(path, *args, **kwargs): + if str(path) == "/proc/meminfo": + return f"MemAvailable: {20 * 1024**2} kB\n" + if str(path) == "/proc/sys/kernel/random/boot_id": + return "boot" + return original_read(path, *args, **kwargs) + + monkeypatch.setattr(fixture, "time", SimpleNamespace(monotonic=lambda: clock.now)) + monkeypatch.setattr(Path, "read_text", read) + monkeypatch.setattr( + fixture.shutil, "disk_usage", lambda _: SimpleNamespace(free=32 * fixture.GIB) + ) + original_stat = fixture.os.stat + monkeypatch.setattr( + fixture.os, + "stat", + lambda path, *args, **kwargs: ( + SimpleNamespace(st_uid=fixture.os.geteuid()) + if str(path) in {"/proc/100", "/proc/200"} + else original_stat(path, *args, **kwargs) + ), + ) + spec = { + "workdir": str(tmp_path), + "stores": { + "pg": { + "host_pid": 200, + "start_ticks": "store", + "boot_id": "boot", + "uid": fixture.os.geteuid(), + "disk_path": str(tmp_path), + } + }, + } + owner = fixture.Owner(spec, owned, lambda *args, **kwargs: None) + owner.sampled = 7.0 + return owner, owned, clock + + +@pytest.mark.parametrize("stage", ["before", "after", "children"]) +def test_a_vanished_descendant_restarts_the_whole_forest(census, stage): + owner, owned, _ = census + original_identity, original_children = owned.identify, owned.enumerate + + def identity(pid): + if pid == 300 and (stage != "after" or owned.reads[pid] > 1): + return None + return original_identity(pid) + + def children(pid): + if pid == 300 and stage == "children": + raise FileNotFoundError("gone") + return original_children(pid) + + owned.identify, owned.enumerate = identity, children + if stage == "children": + owned.identify = lambda pid: ( + None if pid == 300 and owned.reads[pid] > 1 else original_identity(pid) + ) + owner.sample() + assert owner.first is None + assert owner.sampled == 10.0 + assert owned.round == 2 + assert owned.sampled_pids.count(200) == 2 + assert 400 in owned.sampled_pids + + +def test_a_complete_first_forest_needs_no_retry(census): + owner, owned, _ = census + owner.sample() + assert owner.first is None and owner.sampled == 10.0 + assert owned.round == 1 and set(owned.sampled_pids) == {100, 200, 300} + + +@pytest.mark.parametrize( + "fault", + [ + "churn", + "root-gone", + "store-gone", + "store-birth", + "pid-reuse", + "identity-change", + "permission", + "malformed", + "rss", + "stale", + ], +) +def test_incomplete_or_unsafe_census_never_refreshes_the_sample(census, fault): + owner, owned, clock = census + original = owned.identify + + def identify(pid): + if fault == "root-gone" and pid == 100: + return None + if fault == "store-gone" and pid == 200: + return None + if fault == "store-birth" and pid == 200: + return (1, "changed", "S") + if pid >= 300: + if fault in {"churn", "stale"}: + if fault == "stale": + clock.now = 13.0 + return None + if fault == "pid-reuse" and owned.round == 1: + return None + if fault == "identity-change" and owned.reads[pid] > 1: + return (100, "new-birth", "S") + if fault == "permission": + raise PermissionError("unknown") + if fault == "malformed": + raise ValueError("unknown") + return original(pid) + + owned.identify = identify + if fault == "churn": + owned.trees = [[300], [400], [500], [600]] + if fault == "pid-reuse": + owned.trees = [[300], [300]] + if fault == "rss": + owned.measure = lambda pid: fixture.POLICY["stop_process_rss_bytes"] // 1024 + 1 + owner.sample() + assert owner.first is not None + assert owner.sampled == 7.0 + assert owned.round <= 3 + if fault not in {"churn", "pid-reuse", "stale"}: + assert owned.round <= 1 + + +@pytest.mark.parametrize( + "fault", ["root-birth", "store-birth", "new-child-rss", "live-child-census"] +) +def test_restarted_forest_retains_root_and_descendant_guards(census, fault): + owner, owned, _ = census + original_identity, original_children = owned.identify, owned.enumerate + + def identity(pid): + if pid == 300: + return original_identity(pid) if fault == "live-child-census" else None + if owned.round and ( + (pid == 100 and fault == "root-birth") or (pid == 200 and fault == "store-birth") + ): + return (1, "reused", "S") + return original_identity(pid) + + def children(pid): + if pid == 300 and fault == "live-child-census": + raise FileNotFoundError("unverified disappearance") + return original_children(pid) + + owned.identify, owned.enumerate = identity, children + if fault == "new-child-rss": + owned.measure = lambda pid: ( + fixture.POLICY["stop_process_rss_bytes"] // 1024 + 1 if pid == 400 else 1 + ) + owner.sample() + assert owner.first is not None + assert owner.sampled == 7.0 + if fault == "new-child-rss": + assert owned.round == 2 and 400 in owned.sampled_pids + else: + assert owned.round == 1 + + +def test_census_deadline_stops_before_the_next_descendant(census): + owner, owned, clock = census + original_children = owned.enumerate + owned.enumerate = lambda pid: [301] if pid == 300 else original_children(pid) + + def measure(pid): + if pid == 300: + clock.now = 13.0 + return 1 + + owned.measure = measure + owner.sample() + assert isinstance(owner.first, fixture.Refused) + assert owner.sampled == 7.0 + assert 300 in owned.sampled_pids and 301 not in owned.sampled_pids + + +def test_restarted_forest_refuses_store_uid_drift(census, monkeypatch): + owner, owned, _ = census + original_identity, original_stat = owned.identify, fixture.os.stat + owned.identify = lambda pid: None if pid == 300 else original_identity(pid) + + def stat(path, *args, **kwargs): + if str(path) == "/proc/200" and owned.round: + return SimpleNamespace(st_uid=fixture.os.geteuid() + 1) + return original_stat(path, *args, **kwargs) + + monkeypatch.setattr(fixture.os, "stat", stat) + owner.sample() + assert isinstance(owner.first, fixture.Refused) + assert owner.sampled == 7.0 + assert owned.round == 1 + + +def test_store_identity_refusals_keep_distinct_failure_sites(census, monkeypatch): + owner, owned, _ = census + original_identity, original_stat = owned.identify, fixture.os.stat + sites = [] + for fault in ("missing", "start", "boot", "uid"): + owner.failures.clear() + owner.sampled = 7.0 + owner.spec["stores"]["pg"]["boot_id"] = "changed" if fault == "boot" else "boot" + owned.identify = lambda pid, fault=fault: ( + None + if pid == 200 and fault == "missing" + else (1, "changed", "S") + if pid == 200 and fault == "start" + else original_identity(pid) + ) + + def stat(path, *args, fault=fault, **kwargs): + if str(path) == "/proc/200" and fault == "uid": + return SimpleNamespace(st_uid=fixture.os.geteuid() + 1) + return original_stat(path, *args, **kwargs) + + monkeypatch.setattr(fixture.os, "stat", stat) + owner.sample() + assert isinstance(owner.first, fixture.Refused) + assert str(owner.first) == "resource store birth changed" + assert owner.sampled == 7.0 + record = owner.failure_summary()["records"][0] + assert record["primary"] is True and record["phase"] == "resource" + assert record["site"]["file"] == "fixture.py" + assert record["site"]["function"] == "sample" + sites.append(record["site"]["line"]) + assert len(set(sites)) == 4 diff --git a/tests/unit/test_cell_trigger_stage_cancellation.py b/tests/unit/test_cell_trigger_stage_cancellation.py new file mode 100644 index 0000000000..4fcc55148b --- /dev/null +++ b/tests/unit/test_cell_trigger_stage_cancellation.py @@ -0,0 +1,291 @@ +"""Actual staging failures retain bounded cancellation and fresh-plan evidence.""" + +import asyncio +import json +from copy import deepcopy +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import asyncpg +import pytest + +from bench import cell_trigger_gate as gate +from scripts.gates.testsupport.cell_trigger.fixture import native_failure_metadata +from server.core import pg +from server.memory.store import pg_memory +from tests.unit.test_cell_trigger_ci_creator import keeper_config_case as keeper_config_case +from tests.unit.test_cell_trigger_driver_failure_details import _main_case +from tests.unit.test_cell_trigger_gate_diagnostics import run_driver as run_driver + +SECRET = "secret-stage-SQL-value-condition-credential" +MESSAGES = [ + ("canceling statement due to statement timeout", "statement_timeout"), + ("canceling statement due to user request", "user_request"), + (SECRET, "unknown"), +] +SUMMARY = { + "latency_within_limit": True, + "throughput_within_limit": True, + "p99_delta_ms": 0.0, + "throughput_loss": 0.0, +} + + +def canceled(message): + try: + raise asyncpg.PostgresError.new({"C": "57014", "M": message}) + except asyncpg.QueryCanceledError as cause: + try: + pg._raise_if_unavailable(cause) + except pg.PostgresUnavailableError as error: + return error + raise AssertionError("the real PostgreSQL wrapper must refuse") + + +def plan_node(kind="Hash Join"): + return { + "Node Type": kind, + "Join Type": "Inner", + "Plan Rows": 0, + "Startup Cost": 1.0, + "Total Cost": 2.0, + "Relation Name": SECRET, + "Index Name": SECRET, + "Hash Cond": SECRET, + "Index Cond": SECRET, + "Output": [SECRET], + "SQL": SECRET, + } + + +@pytest.fixture +def stage_case(monkeypatch, tmp_path, run_driver): + def configure(*, fault=None, message=MESSAGES[0][0], plan_fault=None): + artifact_error = OSError(SECRET) if fault == "artifact" else None + cleanup_error = RuntimeError(SECRET) if fault == "explain" else None + run, events = run_driver( + prior=None, cleanup=cleanup_error, output=artifact_error, event="stage_error" + ) + rows = [ + { + "org": "owned", + "namespace": "scratch", + "agent": "bench", + "user_id": "-", + "session_id": "-", + "memory_key": key, + "memory_id": f"00000000-0000-0000-0000-{index:012d}", + } + for index, key in enumerate(("b", "a"), 1) + ] + block = gate.Block(rows=rows, audits=[{}, {}], documents=[]) + first = ( + None + if fault == "healthy" + else ValueError(SECRET) + if fault == "noncancel" + else KeyboardInterrupt(SECRET) + if fault == "fatal" + else canceled(message) + ) + state = SimpleNamespace(first=first, explain=[], commits=[], block=block) + conn = gate.asyncpg.connect.return_value + monkeypatch.setattr(conn, "is_in_transaction", lambda: False, raising=False) + + async def fetchval(sql, *values): + if sql.startswith("EXPLAIN"): + state.explain.append((sql, values)) + assert sql == "EXPLAIN (FORMAT JSON) " + pg_memory._LOCK_HEADS + assert values == ( + ["owned", "owned"], + ["scratch", "scratch"], + ["bench", "bench"], + ["-", "-"], + ["-", "-"], + ["a", "b"], + ) + if fault == "explain": + raise OSError(SECRET) + if plan_fault == "oversized": + return " " * 65_537 + node = plan_node() + if plan_fault == "deep": + for _ in range(9): + node = {**plan_node("Materialize"), "Plans": [node]} + elif plan_fault == "nonfinite": + node["Total Cost"] = float("nan") + else: + node["Plans"] = [plan_node("Index Scan"), plan_node("Function Scan")] + return json.dumps([{"Plan": node, "Private": SECRET}]) + if "fresh" in sql: + raise AssertionError("freshness is an invariant label, never SQL") + if "memory_id=ANY" in sql: + return 0 + assert "targets=7" in sql + return len(rows) + + async def commit(actual_rows, actual_audits, **kwargs): + state.commits.append((actual_rows, actual_audits, kwargs)) + assert actual_rows is block.rows and actual_audits is block.audits + assert kwargs == {"targets": 7, "publisher_lease": False} + if first is not None: + raise first + return SimpleNamespace( + seqs=[("owned", "scratch", index) for index in (1, 2)], + targets=[7, 7], + rows=rows, + audits=block.audits, + ) + + monkeypatch.setattr(conn, "fetchval", fetchval, raising=False) + monkeypatch.setattr(gate, "_arm_order", lambda: (True,)) + monkeypatch.setattr( + gate, + "_memory_set", + AsyncMock(return_value=[SimpleNamespace(measured=True, elapsed_ms=1.0)]), + ) + monkeypatch.setattr(gate, "_settle_api", AsyncMock()) + monkeypatch.setattr(gate, "_build_block", AsyncMock(return_value=block)) + monkeypatch.setattr(gate, "_delivery", AsyncMock(return_value=100.0)) + monkeypatch.setattr(gate, "_verify", AsyncMock()) + monkeypatch.setattr(gate, "_physical", AsyncMock(return_value={})) + monkeypatch.setattr(gate, "_debt", AsyncMock()) + monkeypatch.setattr(gate, "_absent", AsyncMock()) + monkeypatch.setattr(gate, "_statistics", lambda samples: SUMMARY.copy()) + monkeypatch.setattr(pg_memory, "commit_versions", commit) + + async def observe(): + try: + result = await run() + except BaseException as error: # noqa: BLE001 — keep native fatals inside their owner. + result = error + records = [ + json.loads(line) for line in (tmp_path / "raw.jsonl").read_text().splitlines() + ] + assert SECRET not in json.dumps(records) + return result, records + + return run, observe, events, state + + return configure + + +@pytest.mark.asyncio +@pytest.mark.parametrize("message,reason", MESSAGES) +async def test_actual_stage_records_cancellation_before_preserving_original( + stage_case, message, reason +): + _, observe, events, state = stage_case(message=message) + result, records = await observe() + stage = [record for record in records if record["kind"] == "stage_error"] + assert len(stage) == 1 # Baseline fails here after the real _run -> _stage refusal. + assert stage[0]["arm"] == 0 and stage[0]["key_count"] == 2 + assert type(stage[0]["elapsed_ms"]) in {int, float} and stage[0]["elapsed_ms"] >= 0 + assert result is state.first and len(state.commits) == len(state.explain) == 1 + metadata = native_failure_metadata(result) + cause = metadata["driver"]["causes"][0] + assert cause["cancel_reason"] == reason and cause["sqlstate"] == "57014" + assert metadata["driver"]["outcome_unknown"] is False + assert SECRET not in json.dumps(metadata) + fresh = [record for record in records if record["kind"] == "freshplan_after_failure"] + assert len(fresh) == 1 + assert fresh[0]["observer"] == "independent_connection" and fresh[0]["analyze"] is False + assert fresh[0]["key_count"] == 2 + assert fresh[0]["nodes"] == [ + { + "depth": depth, + "node_type": kind, + "join_type": "Inner", + "plan_rows": 0, + "startup_cost": 1.0, + "total_cost": 2.0, + } + for depth, kind in ((0, "Hash Join"), (1, "Index Scan"), (1, "Function Scan")) + ] + assert events.index("stage_error") < events.index("restoration") + assert "triggers_restored" in events and "public close" in events + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fault", ["noncancel", "fatal", "explain", "artifact"]) +async def test_actual_stage_diagnostics_cannot_replace_work_or_cleanup(stage_case, fault): + _, observe, events, state = stage_case(fault=fault) + result, records = await observe() + assert events.count("stage_error") == 1 # Causal baseline failure, not a new helper lookup. + assert result is state.first and len(state.commits) == 1 + assert len(state.explain) == (0 if fault in {"noncancel", "fatal"} else 1) + assert "restoration" in events and "public close" in events + assert ("cleanup_error" in events) is (fault == "explain") + assert any(record["kind"] == "freshplan_after_failure" for record in records) is ( + fault == "artifact" + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("plan_fault", ["oversized", "deep", "nonfinite"]) +async def test_actual_stage_rejects_unsafe_plan_and_retains_cancellation(stage_case, plan_fault): + _, observe, events, state = stage_case(plan_fault=plan_fault) + result, records = await observe() + assert any(record["kind"] == "stage_error" for record in records) + assert result is state.first and len(state.explain) == 1 + assert not any(record["kind"] == "freshplan_after_failure" for record in records) + assert "triggers_restored" in events and "public close" in events + + +@pytest.mark.parametrize("fault", ["healthy", None]) +def test_actual_stage_reaches_main_verdict_and_retains_cleanup( + monkeypatch, tmp_path, capsys, stage_case, fault +): + from server.cli import _ch + + output, _ = _main_case(monkeypatch, tmp_path, None) + run, _, events, state = stage_case(fault=fault) + monkeypatch.setattr(_ch, "run_connected", lambda *_args, **_kwargs: asyncio.run(run())) + assert gate.main() == (0 if fault == "healthy" else 1) + verdict = json.loads(capsys.readouterr().out) + assert verdict["verdict"] == ("PASS" if fault == "healthy" else "INVALID") + assert len(state.commits) == 1 and "triggers_restored" in events and "public close" in events + if fault == "healthy": + assert not state.explain and "settled" in events and "arm" in events + assert not any(kind in events for kind in ("stage_error", "freshplan_after_failure")) + assert json.loads(output.read_text().splitlines()[-1])["kind"] == "driver_cleanup" + else: + assert "stage_error" in events and len(state.explain) == 1 + assert verdict["error_type"] == "PostgresUnavailableError" + assert verdict["failure"]["driver"]["outcome_unknown"] is False + assert json.loads(output.read_text().splitlines()[-1])["kind"] == "invalid" + assert SECRET not in json.dumps(verdict) + + +@pytest.mark.parametrize( + "message,reason,fault", + [(message, reason, None) for message, reason in MESSAGES] + + [(MESSAGES[0][0], "statement_timeout", fault) for fault in ("enum", "sqlstate", "extra")], +) +def test_native_action_reads_only_closed_cancellation_evidence( + keeper_config_case, message, reason, fault +): + native, instance, owner, _, _ = keeper_config_case + instance.s["python"] = "unused" + metadata = native_failure_metadata(canceled(message)) + cause = metadata["driver"]["causes"][0] + assert cause.get("cancel_reason") == reason # Baseline misses the actual server reason. + expected = deepcopy(metadata) + if fault == "enum": + cause["cancel_reason"] = SECRET + elif fault == "sqlstate": + cause["sqlstate"] = "08006" + elif fault == "extra": + cause["message"] = SECRET + native.write(instance.w / "pg_cell-created.json.failure.json", metadata) + original = native.Refused(SECRET) + + def refused(*_args, **_kwargs): + raise original + + owner.command = refused + with pytest.raises(native.Refused) as caught: + instance.action("create", role="pg_cell") + assert caught.value is original + assert original.native == (expected if fault is None else None) + assert SECRET not in json.dumps(original.native) diff --git a/tests/unit/test_cli_hygiene.py b/tests/unit/test_cli_hygiene.py index 5458194131..630ca7d2de 100644 --- a/tests/unit/test_cli_hygiene.py +++ b/tests/unit/test_cli_hygiene.py @@ -171,7 +171,7 @@ def test_mint_token_signs_with_the_keyring_it_reads( pyjwt.decode(token, "k" * 40, algorithms=["HS256"], audience=settings.jwt_audience) -def test_an_offline_mint_signs_with_the_root_secret( +def test_an_offline_native_mint_refuses_without_enrollment( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: _auth_on_with(monkeypatch, "a-real-secret-that-is-not-the-default") @@ -180,10 +180,9 @@ def _unreachable(run: Any, **_declared: object) -> None: raise ConnectionError("no clickhouse") monkeypatch.setattr(access, "run_connected", _unreachable) - assert cli.main(_MINT) == 0 + assert cli.main(_MINT) == 1 out, err = capsys.readouterr() - assert "signing keyring not read; signing as kid" in err - assert pyjwt.get_unverified_header(out.strip())["kid"] == "default" + assert out == "" and "enrolled scope" in err def _doctor(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> tuple[int, str]: @@ -324,8 +323,13 @@ def _refuse(_run: object, **_declared: object) -> bool: raise AuthorityMismatch("this deployment's memory authority is postgres") monkeypatch.setattr(access, "run_connected", _refuse) + parser = argparse.ArgumentParser() + access._add_mint_token_parser(parser.add_subparsers()) + args = parser.parse_args( + ["mint-token", "--subject", "sa:test", "--org", "o", "--platform-admin", "--ttl-days", "1"] + ) with pytest.raises(AuthorityMismatch): - access._platform_admin_rollout_ready() + access._cmd_mint_token(args) def test_a_revocation_for_another_deployment_is_refused_as_such( diff --git a/tests/unit/test_cli_native_completion.py b/tests/unit/test_cli_native_completion.py new file mode 100644 index 0000000000..a2ff54b234 --- /dev/null +++ b/tests/unit/test_cli_native_completion.py @@ -0,0 +1,252 @@ +"""Native completion waits for real retirement checks and observes tracked outcomes.""" + +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from server.cli import _ch +from server.core import retirement, rmt_carry +from server.insights.app import provider_barrier as barrier +from server.insights.app import provider_command as command +from tests.unit.test_cli_native_first_error import FIRST +from tests.unit.test_cli_native_first_error import native as native + + +@pytest.mark.parametrize("first_class", [None, *FIRST]) +def test_healthy_heartbeat_finishes_after_native_owner_requests_stop( + native, monkeypatch, first_class +): + requested = [] + attempts = [] + heartbeat = SimpleNamespace(running=True) + + def stop(): + requested.append(True) + + async def settle(client): + assert client is native.client + assert requested and heartbeat.running + attempts.append(True) + heartbeat.running = False + + heartbeat.signal_stop = stop + monkeypatch.setattr(barrier, "_HEARTBEAT", heartbeat) + monkeypatch.setattr(rmt_carry, "settle", settle) + first = first_class("work") if first_class else None + + async def work(_): + if first is not None: + raise first + return 7 + + if first is None: + assert _ch.run_connected(work, orgs=("o",)) == 7 + else: + with pytest.raises(first_class) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert attempts == [True] + assert not heartbeat.running + assert native.events[-3:] == ["certificate", "pg", "ch"] + + +@pytest.mark.parametrize("first_class", [None, *FIRST]) +def test_permanent_heartbeat_respects_same_absolute_native_allowance( + native, monkeypatch, first_class +): + monkeypatch.setattr(command.settings, "native_command_drain_timeout_s", 0.02) + monkeypatch.setattr( + barrier, "_HEARTBEAT", SimpleNamespace(signal_stop=lambda: None, running=True) + ) + monkeypatch.setattr(rmt_carry, "settle", AsyncMock()) + first = first_class("work") if first_class else None + elapsed = [] + + async def work(_): + loop = asyncio.get_running_loop() + elapsed.append((loop, loop.time())) + if first is not None: + raise first + return 7 + + with pytest.raises(first_class or TimeoutError) as caught: + _ch.run_connected(work, orgs=("o",)) + if first is not None: + assert caught.value is first + loop, start = elapsed[0] + assert loop.time() - start < 0.5 + assert "certificate" not in native.events + assert native.events[-2:] == ["pg", "ch"] + + +@pytest.mark.parametrize("first_class", [None, ValueError, asyncio.CancelledError]) +def test_repeated_owner_cancellation_keeps_first_and_allows_finite_heartbeat_completion( + native, monkeypatch, first_class +): + first = first_class("work") if first_class else None + heartbeat = SimpleNamespace(running=True, signal_stop=lambda: None) + attempts = [] + cancellations = [] + + async def settle(client): + assert client is native.client + owner = asyncio.current_task() + attempts.append(True) + if len(attempts) <= 2: + owner.cancel("first signal" if len(attempts) == 1 else "second signal") + else: + heartbeat.running = False + try: + await asyncio.sleep(0) + except asyncio.CancelledError as error: + cancellations.append(error) + raise + + monkeypatch.setattr(barrier, "_HEARTBEAT", heartbeat) + monkeypatch.setattr(rmt_carry, "settle", settle) + + async def work(_): + if first is not None: + raise first + return 7 + + with pytest.raises(first_class or asyncio.CancelledError) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is (first if first is not None else cancellations[0]) + assert len(attempts) == 3 and len(cancellations) == 2 + assert native.events[-3:] == ["certificate", "pg", "ch"] + + +@pytest.mark.parametrize("first_class", [None, *FIRST]) +def test_real_track_completed_failure_is_observed_before_native_cleanup(native, first_class): + producer_error = OSError("producer") + first = first_class("work") if first_class else None + tasks = [] + + async def work(_): + async def producer(): + raise producer_error + + task = native.gate.track(asyncio.create_task(producer())) + tasks.append(task) + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not native.gate.producers + if first is not None: + raise first + return 7 + + with pytest.raises(first_class or OSError) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is (first if first is not None else producer_error) + assert native.events[-3:] == ["certificate", "pg", "ch"] + assert all(task.done() and not task._log_traceback for task in tasks) + + +@pytest.mark.asyncio +async def test_native_track_retains_only_first_failure_and_no_completed_task_handles(): + gate = retirement.RetirementGate() + gate.install("command") + first = OSError("first producer") + tasks = [] + + async def producer(error=None): + if error is not None: + raise error + + for error in [None] * 100 + [first, ValueError("later")] + [None] * 100: + task = gate.track(asyncio.create_task(producer(error))) + tasks.append(task) + await asyncio.sleep(0) + await asyncio.sleep(0) + assert gate.producer_failure is first + assert gate.producers == set() + assert all(not task._log_traceback for task in tasks) + assert retirement.RetirementGate().producer_failure is None + + +@pytest.mark.asyncio +async def test_intentionally_cancelled_tracked_native_producer_is_not_failed(): + gate = retirement.RetirementGate() + gate.install("command") + task = gate.track(asyncio.create_task(asyncio.Event().wait())) + await asyncio.sleep(0) + task.cancel() + await asyncio.gather(task, return_exceptions=True) + assert task.cancelled() and gate.producers == set() + assert gate.producer_failure is None + + +@pytest.mark.parametrize("role", ["server", "worker"]) +@pytest.mark.asyncio +async def test_other_role_tracking_keeps_existing_outcome_ownership(role): + gate = retirement.RetirementGate() + gate.install(role) + first = OSError("role-owned producer") + + async def producer(): + raise first + + task = gate.track(asyncio.create_task(producer())) + await asyncio.sleep(0) + await asyncio.sleep(0) + assert gate.producers == set() and gate.producer_failure is None + assert task._log_traceback + assert task.exception() is first + + +def test_actual_calibration_consumer_failure_survives_finished_track_callback(native, monkeypatch): + from server.memory.app import embedder_calibration + + first = OSError("calibration producer") + tasks = [] + + async def sleep(_): + raise first + + monkeypatch.setattr(embedder_calibration, "_refresh_task", None) + monkeypatch.setattr( + embedder_calibration, + "asyncio", + SimpleNamespace(sleep=sleep, get_running_loop=asyncio.get_running_loop), + ) + + async def configuration(): + tasks.append(embedder_calibration.start_refresh_loop()) + + monkeypatch.setattr(command, "load_active_configuration", configuration) + + async def work(_): + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not native.gate.producers + return 7 + + with pytest.raises(OSError) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert tasks[0].done() and not tasks[0]._log_traceback + assert native.events[-3:] == ["certificate", "pg", "ch"] + + +@pytest.mark.parametrize("phase", ["flush", "certificate"]) +def test_dependency_cancellation_is_not_retried_as_owner_cancellation(native, monkeypatch, phase): + first = asyncio.CancelledError("cleanup dependency cancelled") + required = AsyncMock( + side_effect=[first, RuntimeError("cleanup retried without owner cancellation")] + ) + target = "flush_registered_buffers" if phase == "flush" else "_retire_until" + monkeypatch.setattr(command, target, required) + + async def work(_): + return 7 + + with pytest.raises(asyncio.CancelledError) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert required.await_count == 1 + assert native.events[-2:] == ["pg", "ch"] + assert not native.client.connected + assert native.gate.closed and native.gate.active == 0 diff --git a/tests/unit/test_cli_native_first_error.py b/tests/unit/test_cli_native_first_error.py new file mode 100644 index 0000000000..3fad11815a --- /dev/null +++ b/tests/unit/test_cli_native_first_error.py @@ -0,0 +1,418 @@ +"""The native command lifetime preserves its first failure through owned cleanup.""" + +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from server.cli import _ch +from server.core import background, retirement +from server.core.model_work_gate import ModelWorkGate +from server.insights.app import provider_barrier as barrier +from server.insights.app import provider_command as command + + +@pytest.fixture +def native(monkeypatch): + from server import boot + from server.core import ch, pg, pg_profile + from server.memory.app import embedder_calibration + + events = [] + gate = retirement.get_gate() + client = SimpleNamespace(connected=False) + errors = {} + model_gate = ModelWorkGate() + + def event(name): + events.append(name) + if name in errors: + raise errors[name] + + async def connect(): + event("connect") + client.connected = True + + async def close_ch(): + client.connected = False + event("ch") + + async def close_pg(): + event("pg") + + async def verify(_): + assert not gate.enrolled + event("verify") + + async def enroll(): + event("enroll") + gate.enrolled = True + + async def acceptance(_): + assert gate.enrolled and gate.active and not gate.closed + event("acceptance") + + async def configuration(): + event("configuration") + + async def flush(): + event("flush") + + async def certificate(proof): + assert gate.quiescent and gate.enrolled and client.connected + assert proof.version > proof.after + assert events[-1] == "flush" + event("certificate") + + client.connect, client.disconnect = connect, close_ch + monkeypatch.setattr(ch, "get_client", lambda: client) + monkeypatch.setattr(command, "get_client", lambda: client, raising=False) + monkeypatch.setattr(pg, "close_pool", close_pg) + monkeypatch.setattr(pg_profile, "check_configured", verify) + monkeypatch.setattr(_ch, "check_orgs", acceptance) + monkeypatch.setattr(command, "register_process", enroll) + monkeypatch.setattr(command, "load_active_configuration", configuration) + monkeypatch.setattr(command, "_adopt_overrides", AsyncMock(return_value=True)) + monkeypatch.setattr(command, "get_model_work_gate", lambda: model_gate) + monkeypatch.setattr(barrier, "get_model_work_gate", lambda: model_gate) + monkeypatch.setattr(background, "_registered_tasks", set()) + monkeypatch.setattr(background, "_buffer_drainers", {}) + monkeypatch.setattr(boot, "drain_writes", flush) + monkeypatch.setattr(barrier, "_OWN", barrier._Own()) + monkeypatch.setattr( + barrier, "_HEARTBEAT", SimpleNamespace(signal_stop=lambda: None, running=False) + ) + monkeypatch.setattr(barrier, "_server_side_settled", AsyncMock(return_value=True)) + monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) + monkeypatch.setattr(barrier, "record_retirement", certificate) + monkeypatch.setattr(embedder_calibration, "stop_refresh_loop", lambda: event("stop")) + return SimpleNamespace( + events=events, errors=errors, client=client, gate=gate, model_gate=model_gate, event=event + ) + + +FIRST = [ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] + + +@pytest.mark.parametrize("first_class", FIRST) +@pytest.mark.parametrize("entry", ["verify", "acceptance", "configuration", "work"]) +@pytest.mark.parametrize("secondary", ["pg", "ch", "certificate"]) +def test_native_first_entry_or_work_error_survives_all_close_boundaries( + native, first_class, entry, secondary +): + first = first_class("first") + native.errors[entry] = first + native.errors[secondary] = OSError("secondary") + + async def work(_): + native.event("work") + return 7 + + with pytest.raises(first_class) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert native.events[-2:] == ["pg", "ch"] + assert not native.client.connected + assert native.model_gate.active == native.gate.active == 0 + if entry != "work": + assert "work" not in native.events + if entry != "verify": + assert "stop" in native.events + + +@pytest.mark.parametrize("first_class", FIRST) +@pytest.mark.parametrize("connector", ["connect_checked", "connect_verified"]) +@pytest.mark.asyncio +async def test_connect_entry_preserves_refusal_and_attempts_both_closes( + native, first_class, connector +): + first = first_class("pairing refusal") + native.errors.update(verify=first, pg=OSError("pg"), ch=RuntimeError("ch")) + call = getattr(_ch, connector) + with pytest.raises(first_class) as caught: + if connector == "connect_checked": + await call(native.client, orgs=("o",)) + else: + await call(native.client) + assert caught.value is first + assert native.events == ["connect", "verify", "pg", "ch"] + + +@pytest.mark.parametrize("first_class", FIRST) +@pytest.mark.asyncio +async def test_disconnect_preserves_first_pg_failure_and_still_closes_ch(native, first_class): + first = first_class("pg close") + native.errors.update(pg=first, ch=OSError("ch close")) + with pytest.raises(first_class) as caught: + await _ch.disconnect(native.client) + assert caught.value is first + assert native.events == ["pg", "ch"] + + +@pytest.mark.parametrize("cleanup", ["stop", "flush", "certificate", "pg", "ch"]) +@pytest.mark.parametrize("first_class", FIRST) +def test_required_native_cleanup_error_after_success_is_fatal(native, cleanup, first_class): + first = first_class(cleanup) + native.errors[cleanup] = first + + async def work(_): + native.event("work") + return 7 + + with pytest.raises(first_class) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert native.events[-2:] == ["pg", "ch"] + assert not native.client.connected + + +def test_first_required_cleanup_error_wins_and_all_safe_cleanup_runs(native, monkeypatch): + first = OSError("keeper") + + async def keeper(_): + raise first + + async def work(_): + await asyncio.sleep(0) + return 7 + + monkeypatch.setattr(command, "_keep_overrides", keeper) + native.errors.update(certificate=ValueError("retirement"), pg=RuntimeError("pg")) + with pytest.raises(OSError) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert "stop" in native.events and "flush" in native.events + assert native.events[-3:] == ["certificate", "pg", "ch"] + + +@pytest.mark.parametrize("first_class", FIRST) +def test_driver_restore_failure_remains_first_when_keeper_and_native_close_fail( + native, monkeypatch, first_class +): + first = first_class("driver work") + + async def keeper(_): + raise OSError("keeper") + + async def work(_): + await asyncio.sleep(0) + try: + raise first + finally: + native.events.append("restore") + + monkeypatch.setattr(command, "_keep_overrides", keeper) + native.errors.update(certificate=OSError("certificate"), pg=OSError("pg"), ch=OSError("ch")) + with pytest.raises(first_class) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert native.events.index("restore") < native.events.index("stop") + assert native.events[-2:] == ["pg", "ch"] + + +@pytest.mark.parametrize("phase", ["drain", "flush", "retire"]) +@pytest.mark.parametrize("first_class", [None, *FIRST]) +def test_unacknowledged_cleanup_refuses_without_early_certification( + native, monkeypatch, phase, first_class +): + calls = [] + first = first_class("work") if first_class else None + + async def unacknowledged(*_, **__): + calls.append(phase) + return False + + target = { + "drain": "drain_task_sets", + "flush": "flush_registered_buffers", + "retire": "_retire_until", + } + monkeypatch.setattr(command, target[phase], unacknowledged) + + async def work(_): + if first is not None: + raise first + return 7 + + with pytest.raises(first_class or RuntimeError) as caught: + _ch.run_connected(work, orgs=("o",)) + if first is not None: + assert caught.value is first + assert calls == [phase] + assert "certificate" not in native.events + assert native.events[-2:] == ["pg", "ch"] + + +def test_actual_deferred_descendant_and_role_producer_end_before_certificate(native): + async def work(_): + async def producer(): + try: + await asyncio.Event().wait() + finally: + await asyncio.sleep(0) + native.events.append("producer-ended") + + native.gate.track(asyncio.create_task(producer())) + await asyncio.sleep(0) + + async def descendant(): + await asyncio.sleep(0) + native.events.append("descendant") + + async def deferred(): + await asyncio.sleep(0) + background.create_deferred_task(descendant()) + + background.create_deferred_task(deferred()) + return 7 + + assert _ch.run_connected(work, orgs=("o",)) == 7 + assert native.events.index("producer-ended") < native.events.index("certificate") + assert native.events.index("descendant") < native.events.index("certificate") + assert native.gate.quiescent and native.model_gate.active == 0 + assert native.events[-3:] == ["certificate", "pg", "ch"] + + +@pytest.mark.parametrize("first_class", [SystemExit, KeyboardInterrupt]) +@pytest.mark.asyncio +async def test_override_keeper_returns_fatal_to_owner_before_loop_shutdown(first_class): + first = first_class("keeper fatal") + + async def sleep(_): + raise first + + assert await command._keep_overrides(True, sleep=sleep) is first + + +@pytest.mark.parametrize("first_class", [SystemExit, KeyboardInterrupt]) +def test_real_override_keeper_fatal_is_retrieved_before_native_cleanup( + native, monkeypatch, first_class +): + first = first_class("keeper fatal") + actual = command._keep_overrides + + async def sleep(_): + raise first + + monkeypatch.setattr(command, "_keep_overrides", lambda loaded: actual(loaded, sleep=sleep)) + + async def work(_): + await asyncio.sleep(0) + return 7 + + with pytest.raises(first_class) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert native.events[-3:] == ["certificate", "pg", "ch"] + assert native.gate.quiescent + + +def test_clean_native_success_has_one_certified_exit_and_no_keeper_left(native, monkeypatch): + keepers = [] + actual = command._keep_overrides + + async def keeper(loaded): + keepers.append(asyncio.current_task()) + return await actual(loaded) + + async def work(_): + await asyncio.sleep(0) + return 7 + + monkeypatch.setattr(command, "_keep_overrides", keeper) + assert _ch.run_connected(work, orgs=("o",)) == 7 + assert native.events.count("certificate") == 1 + assert native.events[-3:] == ["certificate", "pg", "ch"] + assert keepers and all(task.done() for task in keepers) + + +def test_caller_cancellation_during_keeper_stop_is_not_swallowed(native, monkeypatch): + owner = [] + keepers = [] + + async def keeper(_): + keepers.append(asyncio.current_task()) + try: + await asyncio.Event().wait() + finally: + owner[0].cancel("cleanup cancellation") + await asyncio.sleep(0) + + async def work(_): + owner.append(asyncio.current_task()) + await asyncio.sleep(0) + return 7 + + monkeypatch.setattr(command, "_keep_overrides", keeper) + with pytest.raises(asyncio.CancelledError, match="cleanup cancellation"): + _ch.run_connected(work, orgs=("o",)) + assert native.events[-3:] == ["certificate", "pg", "ch"] + assert keepers and all(task.done() for task in keepers) + + +def test_unfinished_role_producer_prevents_success_and_certificate(native, monkeypatch): + producers = [] + monkeypatch.setattr(command.settings, "native_command_drain_timeout_s", 0) + + async def work(_): + async def producer(): + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + await asyncio.Event().wait() + + task = native.gate.track(asyncio.create_task(producer())) + producers.append(task) + await asyncio.sleep(0) + return 7 + + with pytest.raises(RuntimeError, match="producers did not stop"): + _ch.run_connected(work, orgs=("o",)) + assert "certificate" not in native.events + assert not native.gate.producers_stopped + assert producers and all(task.done() for task in producers) + + +def test_first_role_producer_failure_is_retrieved_and_does_not_skip_safe_drain(native, monkeypatch): + first = OSError("producer failed") + producers = [] + + async def work(_): + async def producer(error): + raise error + + # Completion callbacks run before native shutdown snapshots tracked producers. + for error in [first, first]: + task = asyncio.create_task(producer(error)) + native.gate.track(task) + producers.append(task) + await asyncio.sleep(0) + await asyncio.sleep(0) + return 7 + + with pytest.raises(OSError) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert native.events[-3:] == ["certificate", "pg", "ch"] + assert all(task.done() and not task._log_traceback for task in producers) + + +def test_actual_retirement_refusal_stays_registered_and_refuses_success(native, monkeypatch): + monkeypatch.setattr(command.settings, "native_command_drain_timeout_s", 0.02) + stopped = [] + monkeypatch.setattr( + barrier, + "_HEARTBEAT", + SimpleNamespace(signal_stop=lambda: stopped.append(True), running=True), + ) + + async def work(_): + return 7 + + with pytest.raises(TimeoutError): + _ch.run_connected(work, orgs=("o",)) + assert stopped and all(stopped) + assert "certificate" not in native.events + assert native.gate.enrolled and native.gate.quiescent + assert native.events[-2:] == ["pg", "ch"] diff --git a/tests/unit/test_cli_native_wrappers.py b/tests/unit/test_cli_native_wrappers.py new file mode 100644 index 0000000000..72283e6512 --- /dev/null +++ b/tests/unit/test_cli_native_wrappers.py @@ -0,0 +1,316 @@ +"""Actual native entry points preserve first failures across their outer close.""" + +import argparse +import asyncio +import sqlite3 +from functools import partial +from types import SimpleNamespace + +import pytest + +from server.cli import ( + _ch, + analytics_export, + code_run, + import_brain, + import_clickgap, + import_recall, + maintenance, + sql_access, +) +from server.insights.app import analytics_export as export_app +from tests.unit.test_cli_native_first_error import FIRST +from tests.unit.test_cli_native_first_error import native as native + + +@pytest.fixture +def wrappers(native, monkeypatch): + from server.code.app.index import repo_registry + from server.memory.app import reextract + + monkeypatch.setattr(repo_registry, "refresh_db_repos", _nothing) + monkeypatch.setattr(analytics_export, "_refusal", lambda *_: "") + monkeypatch.setattr(export_app, "check_credentials", lambda **_: None) + monkeypatch.setattr(export_app, "name_hash_key", lambda _: b"inert") + monkeypatch.setattr(export_app, "s3_put", lambda *_, **__: None) + monkeypatch.setattr(maintenance, "resolve_token", lambda _: ("", "none")) + monkeypatch.setattr(reextract, "replay", _report) + monkeypatch.setattr(reextract, "render_report", lambda *_, **__: "report") + return native + + +async def _nothing(*_, **__): + return None + + +async def _report(*_, **__): + return SimpleNamespace(failed=0, sessions=1) + + +def _invoke(family, work, monkeypatch): + if family == "sql": + return sql_access._connected(work, orgs=("o",)) + if family == "export": + monkeypatch.setattr(export_app, "export_day", work) + return analytics_export._cmd_export( + argparse.Namespace( + day=None, + role_arn="", + region="", + bucket="scratch", + expected_bucket_owner="", + prefix="", + ) + ) + if family == "reextract": + from server.memory.app import reextract + + monkeypatch.setattr(reextract, "load_sessions", work) + return maintenance._cmd_reextract( + argparse.Namespace( + namespace="n", + into="other", + token="", + dry_run=True, + org="o", + limit_sessions=None, + base_url="", + dates=False, + concurrency=1, + ) + ) + if family == "code": + return asyncio.run(code_run._then_disconnect(work())) + raise AssertionError(family) + + +@pytest.mark.parametrize("family", ["sql", "export", "reextract", "code"]) +@pytest.mark.parametrize("first_class", FIRST) +@pytest.mark.parametrize("second_close", [False, True]) +def test_actual_wrappers_preserve_work_failure( + wrappers, monkeypatch, family, first_class, second_close +): + first = first_class("work") + if second_close: + wrappers.errors.update(pg=OSError("pg close"), ch=RuntimeError("ch close")) + + async def work(*_, **__): + assert wrappers.gate.enrolled and wrappers.gate.active > 0 + raise first + + with pytest.raises(first_class) as caught: + _invoke(family, work, monkeypatch) + assert caught.value is first + assert wrappers.events[-2:] == ["pg", "ch"] + assert not wrappers.client.connected + assert wrappers.gate.active == wrappers.model_gate.active == 0 + + +@pytest.mark.parametrize("family", ["sql", "export", "reextract", "code"]) +@pytest.mark.parametrize("failed_close", [False, True]) +def test_successful_actual_wrappers_still_require_cleanup( + wrappers, monkeypatch, family, failed_close +): + first = OSError("pg close") + if failed_close: + wrappers.errors.update(pg=first, ch=RuntimeError("ch close")) + + async def work(*_, **__): + return [] if family == "export" else ([{}] if family == "reextract" else 7) + + if failed_close: + with pytest.raises(OSError) as caught: + _invoke(family, work, monkeypatch) + assert caught.value is first + else: + assert _invoke(family, work, monkeypatch) == (0 if family in ("export", "reextract") else 7) + assert wrappers.events[-3:] == ["certificate", "pg", "ch"] + + +@pytest.mark.parametrize("first_class", FIRST) +@pytest.mark.parametrize("reopen_failure", [False, True]) +def test_code_reconnect_cannot_replace_subcommand_failure( + wrappers, monkeypatch, first_class, reopen_failure +): + first = first_class("subcommand") + events = [] + + async def reconnect(*_, **__): + events.append("reconnect") + if reopen_failure: + raise OSError("reconnect") + + async def work(): + raise first + + monkeypatch.setattr(code_run, "connect_declared", reconnect) + with pytest.raises(first_class) as caught: + asyncio.run(code_run._then_disconnect(work())) + assert caught.value is first + assert events == ["reconnect"] + assert wrappers.events[-3:] == ["certificate", "pg", "ch"] + + +@pytest.mark.parametrize("first_class", FIRST) +@pytest.mark.parametrize("phase", ["entry", "participant-close"]) +@pytest.mark.asyncio +async def test_actual_recall_writer_entry_and_closer_preserve_first( + wrappers, monkeypatch, first_class, phase +): + first = first_class("first") + if phase == "entry": + wrappers.errors["acceptance"] = first + else: + wrappers.errors["certificate"] = first + wrappers.errors.update(pg=OSError("pg close"), ch=RuntimeError("ch close")) + args = argparse.Namespace(base_url=None, org="o") + with pytest.raises(first_class) as caught: + _, closer = await import_recall._make_writer(args, lambda: {}, timeout_s=1) + await closer() + assert caught.value is first + assert wrappers.events[-2:] == ["pg", "ch"] + assert wrappers.gate.active == wrappers.model_gate.active == 0 + + +@pytest.mark.parametrize("family", ["recall", "brain"]) +@pytest.mark.parametrize("first_class", [asyncio.CancelledError, SystemExit, KeyboardInterrupt]) +def test_actual_importer_outer_closer_preserves_unhandled_work_failure( + wrappers, monkeypatch, tmp_path, family, first_class +): + first = first_class("write cancelled or fatal") + wrappers.errors.update(pg=OSError("pg close"), ch=RuntimeError("ch close")) + + async def handler(_): + raise first + + if family == "brain": + (tmp_path / "MEMORY.md").write_text("## A\ncontent\n") + monkeypatch.setattr(import_brain, "_brain_inproc_ops", lambda: {"profile.set": handler}) + args = argparse.Namespace( + dir=str(tmp_path), dry_run=False, org="o", namespace="n", agent="a", base_url=None + ) + call = import_brain._cmd_import_brain + else: + path = tmp_path / "source.db" + with sqlite3.connect(path): + pass + report = { + "tables": {}, + "memories_imported": 1, + "messages_imported": 0, + "threads": 0, + "skipped_rows": 0, + } + monkeypatch.setattr( + import_recall, + "_recall_plan_writes", + lambda *_, **__: ([{"op": "memory.set", "payload": {}}], report), + ) + monkeypatch.setattr(import_recall, "_recall_inproc_ops", lambda: {"memory.set": handler}) + args = argparse.Namespace( + sqlite=str(path), dry_run=False, org="o", namespace="n", agent="a", base_url=None + ) + call = import_recall._cmd_import_recall + with pytest.raises(first_class) as caught: + call(args) + assert caught.value is first + assert wrappers.events[-2:] == ["pg", "ch"] + + +@pytest.mark.parametrize("first_class", FIRST) +def test_actual_clickgap_outer_disconnect_preserves_admission_failure( + wrappers, monkeypatch, tmp_path, first_class +): + from server import embedding + + first = first_class("admission") + wrappers.errors.update(acceptance=first, pg=OSError("pg close"), ch=RuntimeError("ch close")) + path = tmp_path / "source.db" + with sqlite3.connect(path) as source: + source.execute( + "CREATE TABLE lessons (id TEXT, text TEXT, source TEXT, created_at TEXT, kind TEXT, expires_at TEXT, evidence_count INT, recall_count INT, last_recalled_at TEXT, reasoning TEXT)" + ) + monkeypatch.setattr(import_clickgap, "_clickgap_count_source", lambda *_, **__: {}) + monkeypatch.setattr(import_clickgap, "_clickgap_preload", lambda *_, **__: ({}, [], {}, {})) + monkeypatch.setattr(embedding, "get_embedder", lambda: SimpleNamespace(model_name="inert")) + args = argparse.Namespace( + sqlite=str(path), + org="o", + namespace="n", + agent="a", + dry_run=False, + batch_size=1, + seed_recall_events=False, + ) + with pytest.raises(first_class) as caught: + import_clickgap._cmd_import_clickgap(args) + assert caught.value is first + assert wrappers.events[-2:] == ["pg", "ch"] + + +@pytest.mark.doctor_tier_read +@pytest.mark.parametrize("first_class", [asyncio.CancelledError, SystemExit, KeyboardInterrupt]) +@pytest.mark.parametrize("diagnostic", ["cell", "strays", "registry", "tiers"]) +def test_actual_diagnostic_closes_preserve_unhandled_native_failure( + wrappers, monkeypatch, first_class, diagnostic +): + from server.cli import doctor, doctor_cell + from server.code.app.index import repo_registry, tier_health + from server.core import settings_overrides + from server.settings import settings + + first = first_class("diagnostic") + wrappers.errors.update(pg=OSError("pg close"), ch=RuntimeError("ch close")) + + async def fail(*_, **__): + raise first + + if diagnostic == "cell": + monkeypatch.setattr(doctor_cell, "_checks", fail) + call = doctor_cell.cell_checks + elif diagnostic == "strays": + monkeypatch.setattr(doctor_cell, "_deployment_check", _nothing) + monkeypatch.setattr(doctor_cell, "_stray_reads", fail) + call = partial(doctor_cell.stray_org_checks, 1) + else: + monkeypatch.setattr(settings, "ch_host", "inert") + if diagnostic == "registry": + monkeypatch.setattr(repo_registry, "registry_read", lambda: False) + monkeypatch.setattr(repo_registry, "refresh_db_repos", fail) + call = doctor._read_code_registry + else: + monkeypatch.setattr(settings, "code_intel_enabled", True) + monkeypatch.setattr(doctor, "_read_code_registry", lambda: None) + monkeypatch.setattr(repo_registry, "registry_read", lambda: True) + monkeypatch.setattr(settings_overrides, "refresh", _nothing) + monkeypatch.setattr(tier_health, "refresh_failing", fail) + doctor._doctor_tier_failures.cache_clear() + call = doctor._doctor_tier_failures + with pytest.raises(first_class) as caught: + call() + assert caught.value is first + assert wrappers.events[-2:] == ["pg", "ch"] + doctor._doctor_tier_failures.cache_clear() + + +@pytest.mark.parametrize("first_class", [asyncio.CancelledError, SystemExit, KeyboardInterrupt]) +@pytest.mark.asyncio +async def test_actual_setup_store_diagnostic_preserves_unhandled_failure( + wrappers, monkeypatch, first_class +): + from server.core import pg, pg_profile + from server.settings import settings + + first = first_class("setup diagnostic") + wrappers.errors.update(pg=OSError("pg close"), ch=RuntimeError("ch close")) + monkeypatch.setattr(settings, "cell", "c") + monkeypatch.setattr(pg, "enabled", lambda: True) + + async def refusal(_): + raise first + + monkeypatch.setattr(pg_profile, "recorded_authority", refusal) + with pytest.raises(first_class) as caught: + await _ch.check_setup_stores() + assert caught.value is first + assert wrappers.events[-2:] == ["pg", "ch"] diff --git a/tests/unit/test_code_definition_family.py b/tests/unit/test_code_definition_family.py index 5db75f4efb..1ee5950224 100644 --- a/tests/unit/test_code_definition_family.py +++ b/tests/unit/test_code_definition_family.py @@ -223,6 +223,9 @@ def work(n: int) -> int: def test_reading_many_declarators_of_one_binding_is_linear() -> None: from tests.unit.step_count import steps + # Load grammars and start the parser child before comparing per-file work. + assert file_definitions("warm.js", b"const warm = () => 0;\n") + def work(n: int) -> int: src = "const " + ",\n ".join(f"v{i} = () => {i} // c" for i in range(n)) + ";\n" return steps(lambda: file_definitions("a.js", src.encode())) diff --git a/tests/unit/test_directory_control_cleanup.py b/tests/unit/test_directory_control_cleanup.py new file mode 100644 index 0000000000..c8dbe8ed0e --- /dev/null +++ b/tests/unit/test_directory_control_cleanup.py @@ -0,0 +1,361 @@ +"""A directory control attempt owns acquisition, cleanup and context exit once.""" + +import asyncio +import contextlib +import time +from unittest.mock import AsyncMock, Mock + +import pytest + +from server.router import directory + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +def owned(monkeypatch): + conn = Mock() + conn.transaction.return_value = Mock(start=AsyncMock(), commit=AsyncMock()) + conn.close = AsyncMock() + conn.terminate = Mock() + context = Mock(__aenter__=AsyncMock(return_value=conn), __aexit__=AsyncMock(return_value=None)) + monkeypatch.setattr(directory, "_connection", lambda: context) + monkeypatch.setattr(directory, "CONTROL_MAX_S", 0.08) + return conn, context + + +async def test_success_closes_once_before_pool_exit(owned): + conn, context = owned + events = [] + conn.close.side_effect = lambda **_: events.append("close") + context.__aexit__.side_effect = lambda *_: events.append("exit") + assert ( + await directory._control_transaction( + AsyncMock(return_value=19), seconds=directory.CONTROL_MAX_S + ) + == 19 + ) + assert events == ["close", "exit"] + conn.close.assert_awaited_once() + conn.transaction.return_value.commit.assert_awaited_once() + + +async def test_original_error_survives_close_and_exit_failure(owned): + conn, context = owned + original = ValueError("work failure") + conn.close.side_effect = RuntimeError("close failure") + context.__aexit__.side_effect = LookupError("exit failure") + with pytest.raises(ValueError) as caught: + await directory._control_transaction( + AsyncMock(side_effect=original), seconds=directory.CONTROL_MAX_S + ) + assert caught.value is original + conn.close.assert_awaited_once() + context.__aexit__.assert_awaited_once() + conn.transaction.return_value.commit.assert_not_awaited() + + +async def test_close_failure_survives_exit_failure_after_acknowledged_commit(owned): + conn, context = owned + original = ValueError("close failure") + conn.close.side_effect = original + context.__aexit__.side_effect = RuntimeError("exit failure") + with pytest.raises(ValueError) as caught: + await directory._control_transaction(AsyncMock(), seconds=directory.CONTROL_MAX_S) + assert caught.value is original + conn.transaction.return_value.commit.assert_awaited_once() + + +async def test_exit_cannot_open_an_extra_cleanup_allowance(owned): + conn, context = owned + entered = asyncio.Event() + + async def stalled(*_): + entered.set() + await asyncio.Future() + + context.__aexit__.side_effect = stalled + began = time.monotonic() + with pytest.raises(directory.DirectoryUnavailable): + await directory._control_transaction(AsyncMock(), seconds=directory.CONTROL_MAX_S) + assert entered.is_set() and time.monotonic() - began < 0.2 + conn.close.assert_awaited_once() + + +async def test_acquisition_failure_does_not_close_an_unowned_proxy(owned): + conn, context = owned + original = ValueError("acquisition failure") + context.__aenter__.side_effect = original + with pytest.raises(ValueError) as caught: + await directory._control_transaction(AsyncMock(), seconds=directory.CONTROL_MAX_S) + assert caught.value is original + conn.close.assert_not_awaited() + conn.terminate.assert_not_called() + context.__aexit__.assert_not_awaited() + + +async def test_no_known_proxy_cancellation_does_not_close_another_attempt(owned): + conn, context = owned + entered = asyncio.Event() + + async def acquire(): + entered.set() + await asyncio.Future() + + context.__aenter__.side_effect = acquire + task = asyncio.create_task( + directory._control_transaction(AsyncMock(), seconds=directory.CONTROL_MAX_S) + ) + await entered.wait() + task.cancel("first acquisition cancellation") + with pytest.raises(asyncio.CancelledError, match="first acquisition cancellation"): + await task + conn.close.assert_not_awaited() + conn.terminate.assert_not_called() + context.__aexit__.assert_not_awaited() + context.__aenter__.side_effect = None + assert ( + await directory._control_transaction( + AsyncMock(return_value=2), seconds=directory.CONTROL_MAX_S + ) + == 2 + ) + conn.close.assert_awaited_once() + + +async def test_late_acquired_proxy_is_discarded_without_running_callback(owned): + conn, context = owned + entered, allow = asyncio.Event(), asyncio.Event() + + async def acquire(): + entered.set() + while not allow.is_set(): + with contextlib.suppress(asyncio.CancelledError): + await allow.wait() + return conn + + context.__aenter__.side_effect = acquire + work = AsyncMock() + task = asyncio.create_task( + directory._control_transaction(work, seconds=directory.CONTROL_MAX_S) + ) + await entered.wait() + task.cancel("acquisition cancelled") + with pytest.raises(asyncio.CancelledError, match="acquisition cancelled"): + await task + allow.set() + for _ in range(20): + await asyncio.sleep(0) + work.assert_not_awaited() + conn.transaction.assert_not_called() + conn.terminate.assert_called() + conn.close.assert_not_awaited() # Expired original end cannot create a fresh close budget. + + +@pytest.mark.parametrize( + "ending", ["success", "failure", "self_cancel", "exit", "keyboard", "deadline"] +) +@pytest.mark.parametrize("locally_closed", [False, True]) +async def test_outer_signals_wait_for_the_same_close_and_keep_first( + owned, monkeypatch, ending, locally_closed +): + conn, _ = owned + owner = directory._ControlOwner(0.08 if ending == "deadline" else 10) + conn.is_closed.return_value = locally_closed + conn._lost = locally_closed + owner.conn = conn + end = owner.end + started, allow = asyncio.Event(), asyncio.Event() + first = asyncio.CancelledError("first outer signal") + failure = ValueError("close failure") + real_wait = asyncio.wait + waits = 0 + close_cancelled = [] + calls = [] + began = time.monotonic() + + async def close(_conn, *, end): + calls.append(_conn) + assert _conn is conn and end == owner.end + started.set() + try: + await allow.wait() + except asyncio.CancelledError: + close_cancelled.append(True) + raise + if ending == "failure": + raise failure + if ending == "self_cancel": + raise asyncio.CancelledError("close child cancelled itself") + if ending == "exit": + raise SystemExit(17) + if ending == "keyboard": + raise KeyboardInterrupt("close child interrupted") + + async def interrupted_wait(tasks, **kwargs): + nonlocal waits + waits += 1 + if waits == 1: + await started.wait() + raise first + return await real_wait(tasks, **kwargs) + + monkeypatch.setattr(directory.pg, "close_owned", close) + monkeypatch.setattr(asyncio, "wait", interrupted_wait) + task = asyncio.create_task(owner.close()) + await started.wait() + for _ in range(3): + await asyncio.sleep(0) + assert not task.done(), "an outer signal cancelled independent close before its end" + conn.terminate.assert_not_called() + assert not close_cancelled and owner.end == end + if ending != "deadline": + allow.set() + with pytest.raises(asyncio.CancelledError) as caught: + await task + assert caught.value is first and owner.end == end + assert time.monotonic() - began < 0.2 + assert calls == [conn] and waits < 5 + if ending == "success": + conn.terminate.assert_not_called() + else: + conn.terminate.assert_called_once_with() + + +@pytest.mark.parametrize("already_done", [False, True]) +async def test_self_cancelled_close_is_terminal_before_a_long_end(owned, monkeypatch, already_done): + conn, _ = owned + owner = directory._ControlOwner(10) + owner.conn = conn + original = asyncio.CancelledError("close child self-cancelled") + + async def close(_conn, *, end): + raise original + + monkeypatch.setattr(directory.pg, "close_owned", close) + if already_done: + owner.close_task = owner.start(lambda: close(conn, end=owner.end)) + await asyncio.sleep(0) + assert owner.close_task.done() + began = time.monotonic() + with pytest.raises(asyncio.CancelledError) as caught: + await owner.close() + assert caught.value is original + assert time.monotonic() - began < 0.2 + conn.terminate.assert_called_once_with() + + +@pytest.mark.parametrize("abort_kind", ["ordinary", "cancel", "exit", "keyboard"]) +@pytest.mark.parametrize( + "first_kind", ["ordinary", "cancel", "exit", "keyboard", "outer_cancel", "deadline"] +) +async def test_secondary_abort_never_replaces_first_or_skips_owned_task_settlement( + owned, monkeypatch, abort_kind, first_kind +): + conn, _ = owned + owner = directory._ControlOwner(0.02 if first_kind == "deadline" else 10) + owner.conn = conn + errors = { + "ordinary": ValueError("first close failure"), + "cancel": asyncio.CancelledError("first close cancellation"), + "exit": SystemExit(17), + "keyboard": KeyboardInterrupt("first close interruption"), + "outer_cancel": asyncio.CancelledError("first outer cancellation"), + } + first = errors.get(first_kind) + secondary = { + "ordinary": RuntimeError("secondary abort failure"), + "cancel": asyncio.CancelledError("secondary abort cancellation"), + "exit": SystemExit(19), + "keyboard": KeyboardInterrupt("secondary abort interruption"), + }[abort_kind] + conn.terminate.side_effect = secondary + began = asyncio.Event() + cancelled = asyncio.Event() + real_wait = asyncio.wait + signals = 0 + + async def close(_conn, *, end): + began.set() + if first_kind == "deadline": + try: + await asyncio.Future() + finally: + cancelled.set() + raise ValueError("later close failure") if first_kind == "outer_cancel" else first + + async def interrupted_wait(tasks, **kwargs): + nonlocal signals + signals += 1 + if first_kind == "outer_cancel" and signals == 1: + raise first + return await real_wait(tasks, **kwargs) + + async def observed(): + try: + await owner.close() + except BaseException as error: # noqa: BLE001 — test native failures must remain inert too. + return error.fatal.error if isinstance(error, directory._ControlInterrupted) else error + pytest.fail("failed required cleanup became success") + + monkeypatch.setattr(directory.pg, "close_owned", close) + monkeypatch.setattr(asyncio, "wait", interrupted_wait) + result = await observed() + if first_kind == "deadline": + assert isinstance(result, TimeoutError) + await asyncio.wait_for(cancelled.wait(), 0.1) + assert owner.close_task.cancelled() + else: + assert result is first + assert began.is_set() + conn.terminate.assert_called_once_with() + + +@pytest.mark.parametrize("ending", ["timeout", "cancel", "healthy"]) +async def test_integration_fatal_child_reaps_its_process_after_interruption(monkeypatch, ending): + from tests.integration import test_directory_control_cleanup as native + + original = { + "timeout": TimeoutError("owned child communication timed out"), + "cancel": asyncio.CancelledError("owned child communication cancelled"), + "healthy": None, + }[ending] + observer = Mock(fetchval=AsyncMock(return_value="owned_fake")) + process = Mock(returncode=None) + + async def communicate(): + if original is not None: + raise original + process.returncode = 0 + return b"", b"" + + async def wait(): + process.returncode = -9 + return process.returncode + + process.communicate = AsyncMock(side_effect=communicate) + process.wait = AsyncMock(side_effect=wait) + create = AsyncMock(return_value=process) + monkeypatch.setenv("LOOM_PG_DSN", "postgresql://u:test-only@127.0.0.1:15432/admin") + monkeypatch.setattr(native.asyncio, "create_subprocess_exec", create) + if original is None: + await native.test_directory_fatal_child_is_inert_until_owned_cleanup( + observer, "work", "exit" + ) + else: + with pytest.raises(type(original)) as caught: + await native.test_directory_fatal_child_is_inert_until_owned_cleanup( + observer, "work", "exit" + ) + assert caught.value is original + observer.fetchval.assert_awaited_once_with("SELECT current_database()") + create.assert_awaited_once() + process.communicate.assert_awaited_once() + if original is None: + assert process.returncode == 0 + process.kill.assert_not_called() + process.wait.assert_not_awaited() + else: + process.kill.assert_called_once_with() + process.wait.assert_awaited_once_with() + assert process.returncode == -9 diff --git a/tests/unit/test_directory_pool_init.py b/tests/unit/test_directory_pool_init.py new file mode 100644 index 0000000000..ea8bb3fd37 --- /dev/null +++ b/tests/unit/test_directory_pool_init.py @@ -0,0 +1,160 @@ +"""Failed fixed-codec initialization never admits its unborrowed raw connection.""" + +from __future__ import annotations + +import asyncio +from unittest.mock import AsyncMock, Mock + +import pytest + +from server.router import directory + + +@pytest.mark.parametrize( + "error", [ValueError("codec failure"), asyncio.CancelledError("codec cancelled")] +) +@pytest.mark.parametrize("position", [0, 1]) +async def test_every_failed_initialization_quarantines_its_exact_raw_session(error, position): + conn = Mock(set_type_codec=AsyncMock(side_effect=[None] * position + [error])) + with pytest.raises(type(error)) as caught: + await directory._init(conn) + assert caught.value is error + conn.terminate.assert_called_once_with() + assert conn.set_type_codec.await_count == position + 1 + + +async def test_fixed_builtin_codecs_do_not_execute_sql(): + import json + from types import MethodType, SimpleNamespace + + import asyncpg + + conn = SimpleNamespace( + _check_open=Mock(), + _protocol=Mock(), + _drop_local_statement_cache=Mock(), + _execute=AsyncMock(side_effect=AssertionError("codec initialization executed SQL")), + terminate=Mock(), + ) + conn._introspect_type = MethodType(asyncpg.Connection._introspect_type, conn) + conn.set_type_codec = MethodType(asyncpg.Connection.set_type_codec, conn) + await directory._init(conn) + conn._execute.assert_not_awaited() + conn.terminate.assert_not_called() + calls = conn._protocol.get_settings.return_value.add_python_codec.call_args_list + assert len(calls) == 2 + assert [call.args[1] for call in calls] == ["json", "jsonb"] + assert all(call.args[2] == "pg_catalog" for call in calls) + assert all(call.args[5:7] == (json.dumps, json.loads) for call in calls) + + +@pytest.mark.parametrize("kind", ["ordinary", "cancel", "exit", "keyboard"]) +@pytest.mark.parametrize("secondary", ["ordinary", "cancel", "exit", "keyboard"]) +async def test_secondary_quarantine_failure_keeps_first_and_reports_unknown(kind, secondary): + def error(which): + return { + "ordinary": ValueError("original codec failure"), + "cancel": asyncio.CancelledError("codec cancelled"), + "exit": SystemExit(17), + "keyboard": KeyboardInterrupt("codec interrupted"), + }[which] + + original = error(kind) + conn = Mock( + set_type_codec=AsyncMock(side_effect=original), terminate=Mock(side_effect=error(secondary)) + ) + if isinstance(original, (SystemExit, KeyboardInterrupt)): + with pytest.raises(directory._PoolInitFailure) as caught: + await directory._init(conn) + assert caught.value.error is original + else: + with pytest.raises(type(original)) as caught: + await directory._init(conn) + assert caught.value is original + assert original.__notes__ == ["directory initialization session closure is unknown"] + conn.terminate.assert_called_once_with() + conn.set_type_codec.assert_awaited_once() + + +async def test_diagnostic_failure_cannot_replace_original_native_failure(): + class BrokenNote(SystemExit): + def add_note(self, note): + raise KeyboardInterrupt("secondary diagnostic failure") + + original = BrokenNote(17) + conn = Mock( + set_type_codec=AsyncMock(side_effect=original), terminate=Mock(side_effect=RuntimeError) + ) + with pytest.raises(directory._PoolInitFailure) as caught: + await directory._init(conn) + assert caught.value.error is original + + +async def test_pool_has_no_autonomous_opening_and_keeps_existing_limits(monkeypatch): + import asyncpg + + opened = object() + create = AsyncMock(return_value=opened) + monkeypatch.setattr(directory, "_pool", None) + monkeypatch.setattr(directory, "_require_dsn", lambda: "private-test-dsn") + monkeypatch.setattr(asyncpg, "create_pool", create) + assert await directory.pool() is opened + assert await directory.pool() is opened + create.assert_awaited_once_with( + "private-test-dsn", + min_size=0, + max_size=10, + init=directory._init, + server_settings={ + **directory._SESSION, + "statement_timeout": "10000", + "idle_in_transaction_session_timeout": "20000", + }, + command_timeout=11, + timeout=10, + ) + + +@pytest.mark.parametrize("boundary", ["creation", "acquisition", "lookup"]) +@pytest.mark.parametrize("kind", ["exit", "keyboard"]) +async def test_every_pool_boundary_unwraps_identical_native_after_driver_cleanup( + monkeypatch, boundary, kind +): + from contextlib import asynccontextmanager + + import asyncpg + + original = SystemExit(17) if kind == "exit" else KeyboardInterrupt("codec interrupted") + events = [] + carrier = directory._PoolInitFailure(original) + monkeypatch.setattr(directory, "_pool", None) + monkeypatch.setattr(directory, "_require_dsn", lambda: "private-test-dsn") + + async def create(*args, **kwargs): + events.append("driver creation cleanup") + raise carrier + + @asynccontextmanager + async def acquire(): + try: + raise carrier + yield # pragma: no cover + finally: + events.append("driver acquisition cleanup") + + if boundary == "creation": + monkeypatch.setattr(asyncpg, "create_pool", create) + else: + monkeypatch.setattr(directory, "_pool", Mock(acquire=acquire)) + slots = directory._lookups._value + with pytest.raises(type(original)) as caught: + if boundary == "creation": + await directory.pool() + else: + async with directory._connection(lookup=boundary == "lookup"): + pytest.fail("failed initializer admitted a connection") + assert caught.value is original + assert events == [ + "driver creation cleanup" if boundary == "creation" else "driver acquisition cleanup" + ] + assert directory._lookups._value == slots diff --git a/tests/unit/test_docker_provider_owned_subtree.py b/tests/unit/test_docker_provider_owned_subtree.py new file mode 100644 index 0000000000..05b3fcc7ef --- /dev/null +++ b/tests/unit/test_docker_provider_owned_subtree.py @@ -0,0 +1,261 @@ +"""The native fixture captures and checks its exact controlled process subtree.""" + +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +import scripts.gates.docker_provider_data as data +from server.insights.store import provider_participants +from server.workers.child import child_argv +from tests.unit.test_docker_provider_smoke import _process_fixture + + +def _fixture(monkeypatch, *, background=True, fault=None): + parent, processes, roots = _process_fixture(monkeypatch) + children = {1: "8 9", 8: "18" if background else "", 9: ""} + descendant = { + **processes[8], + "pid": 18, + "parent": 8, + "start": "180", + "command": child_argv(4, 8), + } + processes[18] = descendant + child = { + "participant_id": "background-participant", + "version": 5, + "state": "running", + "host_proof": "h|b|c|1|18|180|||cell-retirement-v1", + } + registrations = [*roots, *([child] if background else [])] + if fault == "root-reused": + processes[8] = {**processes[8], "start": "81"} + elif fault == "root-parent": + processes[8] = {**processes[8], "parent": 2} + elif fault == "child-reused": + processes[18] = {**descendant, "start": "181"} + elif fault == "child-parent": + processes[18] = {**descendant, "parent": 9} + elif fault == "child-namespace": + processes[18] = {**descendant, "namespace": 2} + elif fault == "child-command": + processes[18] = {**descendant, "command": [*descendant["command"], "--unknown"]} + elif fault == "two-background": + processes[19] = {**descendant, "pid": 19, "start": "190"} + children[8] = "18 19" + elif fault == "missing-child": + registrations = roots + elif fault == "duplicate-child": + registrations.append({**child, "participant_id": "ambiguous-background"}) + + def read(path): + parts = str(path).split("/") + if len(parts) == 4 and parts[1] == "proc" and parts[3] == "stat": + process = processes[int(parts[2])] + fields = ["0"] * 20 + fields[0], fields[17], fields[19] = "S", "1", process["start"] + return f"{process['pid']} (fixture worker) " + " ".join(fields) + assert parts[1] == "proc" and parts[3] == "task" and parts[2] == parts[4] + assert parts[5] == "children" + return children[int(parts[2])] + + monkeypatch.setattr(data, "Path", lambda path: SimpleNamespace(read_text=lambda: read(path))) + monkeypatch.setattr(data, "_supervisor", lambda: parent) + + async def query(sql, parameters=None): + selected = registrations + if parameters and "proofs" in parameters: + selected = [r for r in selected if r["host_proof"] in parameters["proofs"]] + if ( + fault == "child-changed-after-query" + and parameters + and child["host_proof"] in parameters.get("proofs", []) + ): + processes[18] = {**descendant, "start": "181"} + return selected + + client = SimpleNamespace(query_all=AsyncMock(side_effect=query)) + return roots, child, client + + +@pytest.mark.asyncio +async def test_actual_workers_capture_the_exact_background_child(monkeypatch): + roots, child, client = _fixture(monkeypatch) + captured = await data.workers(client) + assert [r["participant_id"] for r in captured] == [r["participant_id"] for r in roots] + assert [c["participant_id"] for r in captured for c in r.get("children", [])] == [ + child["participant_id"] + ] + assert len(captured) == 2 + + +@pytest.mark.asyncio +async def test_actual_workers_keep_the_same_two_web_roots_without_a_background_child(monkeypatch): + roots, _, client = _fixture(monkeypatch, background=False) + captured = await data.workers(client) + assert [r["participant_id"] for r in captured] == [r["participant_id"] for r in roots] + assert not any(r.get("children", []) for r in captured) + assert len(captured) == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fault", ["root-reused", "root-parent"]) +async def test_actual_workers_refuse_changed_web_identity(monkeypatch, fault): + _, _, client = _fixture(monkeypatch, fault=fault) + with pytest.raises(AssertionError): + await data.workers(client) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", + [ + "child-reused", + "child-parent", + "child-namespace", + "child-command", + "two-background", + "missing-child", + "duplicate-child", + "child-changed-after-query", + ], +) +async def test_actual_workers_refuse_uncertain_background_identity(monkeypatch, fault): + _, _, client = _fixture(monkeypatch, fault=fault) + with pytest.raises(AssertionError): + await data.workers(client) + + +@pytest.mark.asyncio +async def test_actual_recreated_check_refuses_an_unretired_background_child(monkeypatch): + roots, child, _ = _fixture(monkeypatch) + captured = [{**r, "children": [child] if i == 0 else []} for i, r in enumerate(roots)] + observed = [{**r, "state": "retired", "version": 6} for r in roots] + [ + {**child, "state": "drained", "version": 6} + ] + + async def query(_sql, parameters): + return [r for r in observed if r["participant_id"] in parameters["ids"]] + + client = SimpleNamespace( + connect=AsyncMock(), disconnect=AsyncMock(), query_all=AsyncMock(side_effect=query) + ) + monkeypatch.setattr(data, "get_client", lambda: client) + monkeypatch.setattr(data.sys, "argv", ["fixture", "retired", json.dumps(captured)]) + monkeypatch.setattr(provider_participants, "running_queries", AsyncMock(return_value=0)) + monkeypatch.setattr(data.provider_process, "proves_stopped", lambda _: False) + with pytest.raises(AssertionError, match="Recreated workers did not retire"): + await data.main() + client.disconnect.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_actual_recreated_check_reports_only_closed_role_proof_counts(monkeypatch, capsys): + roots, child, _ = _fixture(monkeypatch) + captured = [{**r, "children": [child] if i == 0 else []} for i, r in enumerate(roots)] + observed = [{**r, "state": "retired", "version": 6} for r in [*roots, child]] + + async def query(_sql, parameters): + return [r for r in observed if r["participant_id"] in parameters["ids"]] + + client = SimpleNamespace( + connect=AsyncMock(), disconnect=AsyncMock(), query_all=AsyncMock(side_effect=query) + ) + monkeypatch.setattr(data, "get_client", lambda: client) + monkeypatch.setattr(data.sys, "argv", ["fixture", "retired", json.dumps(captured)]) + monkeypatch.setattr(provider_participants, "running_queries", AsyncMock(return_value=0)) + monkeypatch.setattr(data.provider_process, "proves_stopped", lambda _: False) + await data.main() + prefix = "Provider subtree retirement diagnostic: " + output = capsys.readouterr().out + assert output.startswith(prefix) + summary = json.loads(output.removeprefix(prefix)) + assert summary == { + "web": { + "captured": 2, + "retirement_proved": 2, + "death_proved": 0, + "query_zero_proved": 2, + "unproved": 0, + "kernel_zombie_single_thread": 0, + "kernel_zombie_multi_thread": 0, + "kernel_present_non_zombie": 2, + "kernel_missing": 0, + "kernel_uninspectable": 0, + }, + "background": { + "captured": 1, + "retirement_proved": 1, + "death_proved": 0, + "query_zero_proved": 1, + "unproved": 0, + "kernel_zombie_single_thread": 0, + "kernel_zombie_multi_thread": 0, + "kernel_present_non_zombie": 1, + "kernel_missing": 0, + "kernel_uninspectable": 0, + }, + } + assert all(row["host_proof"] not in output for row in [*roots, child]) + assert child["participant_id"] not in output + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("state", "threads", "start", "error", "expected"), + [ + ("Z", "1", "180", None, "kernel_zombie_single_thread"), + ("Z", "2", "180", None, "kernel_zombie_multi_thread"), + ("S", "1", "180", None, "kernel_present_non_zombie"), + ("S", "1", "180", FileNotFoundError, "kernel_missing"), + ("S", "1", "180", PermissionError, "kernel_uninspectable"), + ("Z", "1", "181", None, "kernel_uninspectable"), + ], +) +async def test_subtree_kernel_counts_observe_only_the_captured_identity( + monkeypatch, capsys, state, threads, start, error, expected +): + roots, child, _ = _fixture(monkeypatch) + captured = [{**roots[0], "children": [child]}] + fixture_path = data.Path + + def read(path): + if path != "/proc/18/stat": + return fixture_path(path).read_text() + if error is not None: + raise error("kernel record unavailable") + fields = ["0"] * 20 + fields[0], fields[17], fields[19] = state, threads, start + return "18 (fixture child) " + " ".join(fields) + + monkeypatch.setattr(data, "Path", lambda path: SimpleNamespace(read_text=lambda: read(path))) + monkeypatch.setattr(data.provider_process, "proves_stopped", lambda _: False) + client = SimpleNamespace(query_all=AsyncMock(side_effect=AssertionError("no query expected"))) + await data._subtree_diagnostic(client, captured, proved=True) + prefix = "Provider subtree retirement diagnostic: " + output = capsys.readouterr().out + assert output.startswith(prefix) + summary = json.loads(output.removeprefix(prefix)) + common = { + "captured": 1, + "retirement_proved": 1, + "death_proved": 0, + "query_zero_proved": 1, + "unproved": 0, + } + kernel_keys = { + "kernel_zombie_single_thread", + "kernel_zombie_multi_thread", + "kernel_present_non_zombie", + "kernel_missing", + "kernel_uninspectable", + } + assert summary == { + "web": {**common, **{key: int(key == "kernel_present_non_zombie") for key in kernel_keys}}, + "background": {**common, **{key: int(key == expected) for key in kernel_keys}}, + } + assert all(row["host_proof"] not in output for row in [*roots, child]) + assert child["participant_id"] not in output + client.query_all.assert_not_awaited() diff --git a/tests/unit/test_docker_provider_reset_diagnostics.py b/tests/unit/test_docker_provider_reset_diagnostics.py new file mode 100644 index 0000000000..36223ff9b7 --- /dev/null +++ b/tests/unit/test_docker_provider_reset_diagnostics.py @@ -0,0 +1,220 @@ +"""Failure evidence preserves the provider smoke's original reset refusal.""" + +import io +import json +import subprocess +import urllib.error +from unittest.mock import AsyncMock +from uuid import UUID + +import pytest + +import scripts.gates.docker_provider_data as data +import scripts.gates.docker_provider_smoke as smoke + +TRANSITION = str(UUID(int=17)) + + +@pytest.mark.parametrize("diagnostic_failure", [None, RuntimeError, KeyboardInterrupt]) +def test_reset_failure_collects_diagnostic_and_preserves_first_error( + monkeypatch, tmp_path, diagnostic_failure +): + events, failure = _provider_flow(monkeypatch, tmp_path, diagnostic_failure=diagnostic_failure) + with pytest.raises(AssertionError) as caught: + smoke.smoke_providers("http://fixture", events["compose"]) + assert caught.value is failure + assert events["checks"][-1] == ("diagnostic", TRANSITION) + assert events["waits"] == 4 + assert events["resets"] == 1 + assert events["checks"].count(("crash-worker",)) == 1 + assert ( + "retired", + json.dumps([{"participant_id": "first"}, {"participant_id": "second"}]), + ) in events["checks"] + assert not any(args[0] == "environment" for args in events["checks"]) + + +def test_healthy_reset_does_not_collect_diagnostic(monkeypatch, tmp_path): + events, _ = _provider_flow(monkeypatch, tmp_path, healthy=True) + smoke.smoke_providers("http://fixture", events["compose"]) + assert not any(args[0] == "diagnostic" for args in events["checks"]) + assert events["checks"][-1] == ("environment", "8") + assert events["waits"] == 4 + assert events["resets"] == 1 + + +def _provider_flow(monkeypatch, tmp_path, *, diagnostic_failure=None, healthy=False): + override = tmp_path / "override.json" + override.write_text(json.dumps({"services": {"loom": {}}})) + events = { + "compose": ["docker", "compose", "-f", str(override)], + "checks": [], + "waits": 0, + "resets": 0, + } + failure = AssertionError("Timed out waiting for provider activation") + environment = { + "source": "environment", + "embedding_model": "environment", + "embedding_dimensions": 8, + "llm_model": "fake", + "pending": None, + } + active = { + "source": "dashboard", + "pending": None, + "embedding_model": "docker-smoke-16", + "embedding_dimensions": 16, + } + statuses = iter( + [ + { + **active, + "embedding_model": "docker-smoke-8", + "configured_keys": {"compatible": True}, + }, + { + "embedding_model": "docker-smoke-8", + "pending": {"state": "failed", "error_code": "rebuild_rebuild_failed"}, + }, + active, + environment, + ] + ) + model_reads = iter([environment, active]) + applications = iter(["queued", "active", "queued"]) + + def request(_base, path, _payload=None): + if path == "/v1/admin/models": + return next(model_reads) + raise urllib.error.HTTPError("http://fixture", 503, "closed", {}, None) + + def configure(*_args): + raise urllib.error.HTTPError( + "http://fixture", 422, "width", {}, io.BytesIO(b"unsupported_dimensions") + ) + + def dashboard(_base, action, *_args): + events["resets"] += action == "reset" + return { + "state": next(applications) if action == "apply" else "queued", + "config_id": TRANSITION, + } + + def check(_compose, *args): + events["checks"].append(args) + if args[0] == "diagnostic" and diagnostic_failure is not None: + raise diagnostic_failure("diagnostic secret must not replace the original") + return ( + [{"participant_id": "first"}, {"participant_id": "second"}] + if args[0] == "workers" + else None + ) + + def status(_base, predicate): + events["waits"] += 1 + value = next(statuses) + assert predicate(value) + if events["waits"] == 4 and not healthy: + raise failure + return value + + monkeypatch.setattr(smoke, "request", request) + monkeypatch.setattr(smoke, "configure", configure) + monkeypatch.setattr(smoke, "dashboard_action", dashboard) + monkeypatch.setattr(smoke, "data_check", check) + monkeypatch.setattr(smoke, "wait_status", status) + monkeypatch.setattr(smoke, "wait_admission", lambda *_: None) + monkeypatch.setattr(smoke, "control", lambda *_: None) + monkeypatch.setattr(smoke, "run", lambda *_: None) + return events, failure + + +def test_diagnostic_exec_is_bounded_and_discards_stderr(monkeypatch): + calls = [] + monkeypatch.setattr( + smoke.subprocess, "run", lambda *args, **kwargs: calls.append((args, kwargs)) + ) + smoke.data_check(["docker", "compose", "-f", "owned.json"], "diagnostic", TRANSITION) + args, kwargs = calls[0] + assert args[0][-2:] == ["diagnostic", TRANSITION] + assert kwargs["timeout"] == 10 + assert kwargs["check"] is True + assert kwargs["stderr"] == subprocess.DEVNULL + assert "capture_output" not in kwargs + + +@pytest.mark.asyncio +async def test_diagnostic_emits_only_closed_bounded_counts(monkeypatch, capsys): + from types import SimpleNamespace + + row = _counts() + query = AsyncMock(return_value=row) + monkeypatch.setattr( + data.provider_process, "host_proof", lambda: "host|boot|container|namespace|55|90" + ) + await data.diagnostic(SimpleNamespace(query_one=query), TRANSITION) + sql, params = query.await_args.args + assert sql.lstrip().startswith("WITH retained AS (SELECT") + assert "ANY LEFT JOIN" in sql and "LIMIT 2" in sql + for guard in ( + "max_execution_time = 3", + "max_rows_to_read = 10000", + "max_bytes_to_read = 10485760", + "max_result_rows = 1", + "max_result_bytes = 4096", + "max_memory_usage = 134217728", + ): + assert guard in sql + assert params == { + "name": "provider_configuration", + "transition": TRANSITION, + "scope": ["host", "boot", "container", "namespace"], + } + output = capsys.readouterr().out + assert json.loads(output.split(": ", 1)[1]) == row + assert len(output) < 1024 + assert "host|boot" not in output and TRANSITION not in output + assert query.await_count == 1 + + +@pytest.mark.parametrize("fault", ["extra", "bool", "large", "negative", "missing", "none"]) +@pytest.mark.asyncio +async def test_diagnostic_refuses_invalid_aggregate_without_output(monkeypatch, capsys, fault): + from types import SimpleNamespace + + row = _counts() + if fault == "extra": + row["payload"] = "secret" + elif fault == "bool": + row["retained"] = True + elif fault == "large": + row["retained"] = 10001 + elif fault == "negative": + row["retained"] = -1 + elif fault == "missing": + del row["retained"] + else: + row = None + monkeypatch.setattr( + data.provider_process, "host_proof", lambda: "host|boot|container|namespace|55|90" + ) + with pytest.raises(ValueError, match="Invalid provider diagnostic aggregate"): + await data.diagnostic(SimpleNamespace(query_one=AsyncMock(return_value=row)), TRANSITION) + assert capsys.readouterr().out == "" + + +def _counts(): + return { + "retained": 4, + "running": 1, + "matching_drained": 3, + "other_drained": 0, + "retained_retired": 0, + "expired": 1, + "same_scope": 3, + "lease_rows": 1, + "expired_leases": 0, + "retained_holders": 1, + "matching_drained_holders": 1, + } diff --git a/tests/unit/test_docker_provider_smoke.py b/tests/unit/test_docker_provider_smoke.py index a561016e08..a7c59cf446 100644 --- a/tests/unit/test_docker_provider_smoke.py +++ b/tests/unit/test_docker_provider_smoke.py @@ -1,5 +1,6 @@ """The smoke fixture is deterministic and isolates recovery from paid providers.""" +import io import json import threading import urllib.error @@ -138,3 +139,509 @@ async def test_data_check_requires_no_retained_rebuild_copies(monkeypatch): monkeypatch.setattr(data, "get_client", lambda: client) with pytest.raises(AssertionError): await data.verify("model", 8) + + +def test_crash_keeps_the_supervisor_and_verifies_retirement_before_reset(monkeypatch, tmp_path): + override = tmp_path / "override.json" + override.write_text(json.dumps({"services": {"loom": {}}})) + compose = ["docker", "compose", "-f", str(override)] + events = [] + environment = { + "source": "environment", + "embedding_model": "environment", + "embedding_dimensions": 8, + "llm_model": "fake", + } + active = { + "source": "dashboard", + "pending": None, + "embedding_model": "docker-smoke-16", + "embedding_dimensions": 16, + } + statuses = iter( + [ + { + **active, + "embedding_model": "docker-smoke-8", + "configured_keys": {"compatible": True}, + }, + { + "embedding_model": "docker-smoke-8", + "pending": {"state": "failed", "error_code": "rebuild_rebuild_failed"}, + }, + active, + {**environment, "pending": None}, + ] + ) + model_reads = iter([environment, active]) + + def request(_base, path, _payload=None): + if path == "/v1/admin/models": + return next(model_reads) + raise urllib.error.HTTPError("http://fixture", 503, "closed", {}, None) + + def configure(*_args): + raise urllib.error.HTTPError( + "http://fixture", 422, "width", {}, io.BytesIO(b"unsupported_dimensions") + ) + + applications = iter(["queued", "active", "queued"]) + + def dashboard(_base, action, *_args): + events.append(action) + return {"state": next(applications) if action == "apply" else "queued"} + + def check(_compose, *args): + events.append(args[0]) + return ( + [{"participant_id": "first"}, {"participant_id": "second"}] + if args[0] == "workers" + else None + ) + + def status(_base, predicate): + value = next(statuses) + assert predicate(value) + return value + + monkeypatch.setattr(smoke, "request", request) + monkeypatch.setattr(smoke, "configure", configure) + monkeypatch.setattr(smoke, "dashboard_action", dashboard) + monkeypatch.setattr(smoke, "data_check", check) + monkeypatch.setattr(smoke, "wait_status", status) + monkeypatch.setattr(smoke, "wait_admission", lambda *_: None) + monkeypatch.setattr(smoke, "control", lambda *_: None) + monkeypatch.setattr(smoke, "run", lambda *args: events.append(tuple(args))) + smoke.smoke_providers("http://fixture", compose) + assert not any(isinstance(event, tuple) and "SIGKILL" in event for event in events) + assert events.index("retry") < events.index("crash-worker") < events.index("workers") + recreate = [ + i + for i, event in enumerate(events) + if isinstance(event, tuple) and "--force-recreate" in event + ] + assert len(recreate) == 2 + assert events.index("workers") < recreate[-1] < events.index("retired") < events.index("reset") + + +def _process_fixture(monkeypatch): + from multiprocessing import spawn + from types import SimpleNamespace + + parent = { + "pid": 1, + "parent": 0, + "start": "10", + "namespace": 1, + "executable": "/python", + "command": ["python", *data.SUPERVISOR], + } + processes = { + pid: { + "pid": pid, + "parent": 1, + "start": str(pid * 10), + "namespace": 1, + "executable": "/python", + "command": [ + str(value, "utf-8") if isinstance(value, bytes) else value + for value in spawn.get_command_line(tracker_fd=4, pipe_handle=8) + ], + } + for pid in (8, 9) + } + rows = [ + { + "participant_id": str(pid), + "children": [], + "version": 5, + "state": "running", + "host_proof": f"h|b|c|1|{pid}|{pid * 10}|||cell-retirement-v1", + } + for pid in (8, 9) + ] + monkeypatch.setattr(data.provider_process, "host_proof", lambda: "h|b|c|1|99|990||") + monkeypatch.setattr(data, "_process", lambda pid: processes[pid]) + monkeypatch.setattr(data, "Path", lambda _path: SimpleNamespace(read_text=lambda: "8 9")) + return parent, processes, rows + + +def test_worker_selection_joins_kernel_children_without_treating_other_participants_as_workers( + monkeypatch, +): + parent, processes, rows = _process_fixture(monkeypatch) + processes[44] = { + **processes[8], + "pid": 44, + "parent": 0, + "start": "440", + "command": ["python", "-", "seed"], + } + mixed = [ + { + **rows[0], + "participant_id": "native", + "host_proof": "h|b|c|1|44|440|||cell-retirement-v1", + }, + { + **rows[0], + "participant_id": "reused-pid", + "host_proof": "h|b|c|1|8|79|||cell-retirement-v1", + }, + {**rows[0], "participant_id": "gone", "host_proof": "h|b|c|1|66|660|||cell-retirement-v1"}, + *rows, + ] + assert data._workers(mixed, parent) == [ + {k: row[k] for k in ("participant_id", "host_proof", "version")} for row in rows + ] + assert [row["participant_id"] for row in mixed] == ["native", "reused-pid", "gone", "8", "9"] + + +def test_worker_command_uses_exact_inherited_interpreter_flags(monkeypatch): + from multiprocessing import spawn, util + + parent, processes, rows = _process_fixture(monkeypatch) + monkeypatch.setattr(util, "_args_from_interpreter_flags", lambda: ["-B"]) + command = [ + str(value, "utf-8") if isinstance(value, bytes) else value + for value in spawn.get_command_line(tracker_fd=4, pipe_handle=8) + ] + processes[8] = {**processes[8], "command": command} + assert data._worker_process(rows[0]["host_proof"], parent) == processes[8] + for flags in (["-E"], ["-B", "-X", "unknown"]): + processes[8] = {**processes[8], "command": [command[0], *flags, *command[-3:]]} + with pytest.raises(AssertionError, match="interpreter flags"): + data._worker_process(rows[0]["host_proof"], parent) + + +def test_kernel_workers_never_ignore_missing_or_mismatched_current_proofs(monkeypatch): + parent, processes, rows = _process_fixture(monkeypatch) + for proof in ( + None, + "h|b|c|1|8|79|||cell-retirement-v1", + "h|b|c|2|8|80|||cell-retirement-v1", + "h|b|c|1|8|80|||unknown", + ): + changed = rows[1:] if proof is None else [{**rows[0], "host_proof": proof}, rows[1]] + with pytest.raises(AssertionError, match="exact registrations|certificate invalid"): + data._workers(changed, parent) + for field, value in (("namespace", 2), ("executable", "/other-python")): + original = processes[8] + processes[8] = {**original, field: value} + with pytest.raises(AssertionError, match=f"{field} mismatch"): + data._workers(rows, parent) + processes[8] = original + + +def test_worker_selection_requires_registered_supervised_processes(monkeypatch): + parent, processes, rows = _process_fixture(monkeypatch) + foreign = {**rows[0], "host_proof": "h|b|c|2|8|80|||cell-retirement-v1"} + assert data._workers([*rows, foreign], parent) == [ + {k: row[k] for k in ("participant_id", "host_proof", "version")} for row in rows + ] + processes[8] = {**processes[8], "parent": 2} + with pytest.raises(AssertionError): + data._workers(rows, parent) + processes[8] = { + **processes[8], + "parent": 1, + "command": ["python", "-c", "from multiprocessing.resource_tracker import main; main(4)"], + } + with pytest.raises(AssertionError): + data._workers(rows, parent) + + +@pytest.mark.parametrize("fault", ["none", "pid-reused", "supervisor-changed"]) +def test_pidfd_crash_refuses_changed_identity_before_signalling(monkeypatch, fault): + parent, processes, rows = _process_fixture(monkeypatch) + events = [] + observed = iter( + [processes[8], {**processes[8], "start": "81"} if fault == "pid-reused" else processes[8]] + ) + monkeypatch.setattr(data, "_process", lambda _pid: next(observed)) + monkeypatch.setattr( + data, + "_supervisor", + lambda: {**parent, "start": "11"} if fault == "supervisor-changed" else parent, + ) + monkeypatch.setattr(data.os, "pidfd_open", lambda pid: events.append(("open", pid)) or 73) + monkeypatch.setattr(data.os, "close", lambda fd: events.append(("close", fd))) + monkeypatch.setattr( + data.signal, "pidfd_send_signal", lambda fd, sig: events.append(("signal", fd, sig)) + ) + if fault == "none": + data._kill_worker(rows[0], parent) + assert events == [("open", 8), ("signal", 73, data.signal.SIGKILL), ("close", 73)] + else: + with pytest.raises(AssertionError): + data._kill_worker(rows[0], parent) + assert events == [("open", 8), ("close", 73)] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", ["none", "drained", "old-version", "different-proof", "busy", "unknown"] +) +async def test_retirement_requires_exact_newer_certificate_and_known_zero_queries( + monkeypatch, fault +): + from unittest.mock import AsyncMock + + from server.insights.store import provider_participants + + _parent, _processes, rows = _process_fixture(monkeypatch) + observed = {**rows[0], "state": "retired", "version": 6} + if fault == "drained": + observed["state"] = "drained" + elif fault == "old-version": + observed["version"] = 5 + elif fault == "different-proof": + observed["host_proof"] = rows[1]["host_proof"] + client = AsyncMock() + client.query_all.return_value = [observed] + count = 1 if fault == "busy" else None if fault == "unknown" else 0 + monkeypatch.setattr(provider_participants, "running_queries", AsyncMock(return_value=count)) + assert await data.retired(client, [rows[0]]) is (fault == "none") + assert client.query_all.await_args.args[1] == {"ids": [rows[0]["participant_id"]]} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault,reason", + [ + ("death_unknown", '"death": "unproved"'), + ("missing_rows", '"retirement": "row_count"'), + ("queries_unknown", '"retirement": "queries_unknown"'), + ("replacement_mismatch", '"victim_absent": false'), + ("none", None), + ], +) +async def test_crash_observation_distinguishes_unproved_predicates(monkeypatch, fault, reason): + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from server.insights.store import provider_participants + + parent, _processes, rows = _process_fixture(monkeypatch) + monkeypatch.setattr(data, "_supervisor", lambda: parent) + monkeypatch.setattr(data, "workers", AsyncMock(return_value=rows)) + signals = [] + monkeypatch.setattr(data, "_kill_worker", lambda victim, _: signals.append(victim)) + monkeypatch.setattr(data.provider_process, "proves_stopped", lambda _: fault != "death_unknown") + victim = rows[0] + replacement = {**victim, "participant_id": "replacement"} + after = rows if fault == "replacement_mismatch" else [rows[1], replacement] + monkeypatch.setattr(data, "_workers", lambda *_: after) + client = AsyncMock() + client.query_all.side_effect = [ + [rows[0]], + [rows[1]], + [] if fault == "missing_rows" else [{**victim, "state": "retired", "version": 6}], + [victim], + ] + monkeypatch.setattr( + provider_participants, + "running_queries", + AsyncMock(return_value=None if fault == "queries_unknown" else 0), + ) + ticks = iter([0, 0, 241]) + monkeypatch.setattr(data, "time", SimpleNamespace(monotonic=lambda: next(ticks))) + monkeypatch.setattr(data.asyncio, "sleep", AsyncMock()) + if reason is None: + await data.crash_worker(client) + else: + with pytest.raises(AssertionError) as failure: + await data.crash_worker(client) + assert reason in str(failure.value) + assert victim["host_proof"] not in str(failure.value) + assert victim["participant_id"] not in str(failure.value) + assert signals == [victim] + reads = client.query_all.await_args_list + assert [call.args[1] for call in reads[:2]] == [ + {"ids": [row["participant_id"]]} for row in rows + ] + assert all( + "SELECT participant_id, state, version, host_proof" in call.args[0] for call in reads[:2] + ) + if fault == "missing_rows": + assert '"reference_count": 1' in str(failure.value) + assert '"reference_identity": true' in str(failure.value) + assert "SELECT participant_id FROM" in reads[-1].args[0] + else: + assert all("SELECT participant_id FROM" not in call.args[0] for call in reads) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", ["missing-first", "missing-second", "wrong-identity", "reference-error"] +) +async def test_crash_refuses_unreadable_captured_worker_before_signalling(monkeypatch, fault): + from types import SimpleNamespace + from unittest.mock import AsyncMock + + parent, _processes, rows = _process_fixture(monkeypatch) + monkeypatch.setattr(data, "_supervisor", lambda: parent) + monkeypatch.setattr(data, "workers", AsyncMock(return_value=rows)) + signals = [] + monkeypatch.setattr(data, "_kill_worker", lambda victim, _: signals.append(victim)) + monkeypatch.setattr(data.provider_process, "proves_stopped", lambda _: True) + client = AsyncMock() + + async def query(sql, parameters): + selected = next(row for row in rows if [row["participant_id"]] == parameters["ids"]) + if sql.startswith("SELECT participant_id FROM"): + if fault == "reference-error": + raise RuntimeError("private diagnostic failure") + return [selected] + if fault == "missing-second" and selected == rows[0]: + return [selected] + return [rows[1]] if fault == "wrong-identity" else [] + + client.query_all.side_effect = query + ticks = iter([0, 0, 241]) + monkeypatch.setattr(data, "time", SimpleNamespace(monotonic=lambda: next(ticks))) + monkeypatch.setattr(data.asyncio, "sleep", AsyncMock()) + with pytest.raises(AssertionError, match="Captured worker readback failed") as failure: + await data.crash_worker(client) + assert signals == [] + message = str(failure.value) + assert '"reader_identity": false' in message + assert f'"reader_count": {int(fault == "wrong-identity")}' in message + if fault == "reference-error": + assert '"reference_read": "query_error"' in message + assert "private diagnostic failure" not in message + else: + assert '"reference_count": 1' in message + assert '"reference_identity": true' in message + assert all(row["host_proof"] not in message for row in rows) + assert [call.args[1] for call in client.query_all.await_args_list] == ( + [{"ids": [rows[0]["participant_id"]]}] if fault == "missing-second" else [] + ) + [{"ids": [rows[int(fault == "missing-second")]["participant_id"]]}] * 2 + + +@pytest.mark.asyncio +async def test_retirement_reader_preserves_string_identity_from_native_uuid(): + from unittest.mock import AsyncMock + from uuid import UUID + + participant_id = UUID("00112233-4455-6677-8899-aabbccddeeff") + row = { + "participant_id": participant_id, + "state": "retired", + "version": 6, + "host_proof": "proof", + } + client = AsyncMock() + client.query_all.return_value = [row] + captured = [{**row, "participant_id": str(participant_id)}] + assert await data._retirement_rows(client, captured) == captured + assert row["participant_id"] is participant_id + assert client.query_all.await_args.args == ( + "SELECT participant_id, state, version, host_proof " + "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)} LIMIT 5", + {"ids": [str(participant_id)]}, + ) + + +@pytest.mark.parametrize( + "fault", ["gone", "zombie", "dead", "live", "permission", "unreadable", "malformed", "one-live"] +) +def test_worker_census_omits_only_proved_exited_children(monkeypatch, fault): + from types import SimpleNamespace + + parent, _processes, rows = _process_fixture(monkeypatch) + process = data._process + + def observed_process(pid): + if pid == 7: + if fault == "permission": + raise PermissionError("unreadable child executable") + raise FileNotFoundError("disappeared child executable") + return process(pid) + + def read(path): + if path == "/proc/1/task/1/children": + return "7 8" if fault == "one-live" else "7 8 9" + assert path == "/proc/7/stat" + if fault == "gone": + raise FileNotFoundError("reaped child") + if fault == "unreadable": + raise PermissionError("unreadable child stat") + if fault == "malformed": + return "7 (incomplete child)" + state = "X" if fault == "dead" else "R" if fault == "live" else "Z" + return f"7 (exited child) {state} 1" + + monkeypatch.setattr(data, "_process", observed_process) + monkeypatch.setattr(data, "Path", lambda path: SimpleNamespace(read_text=lambda: read(path))) + if fault in {"gone", "zombie", "dead"}: + assert data._workers(rows, parent) == [ + {key: row[key] for key in ("participant_id", "host_proof", "version")} for row in rows + ] + else: + with pytest.raises((OSError, IndexError, AssertionError)): + data._workers(rows, parent) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", ["spawn", "registration", "timeout", "duplicate", "certificate", "namespace"] +) +async def test_crash_waits_only_for_incomplete_replacement_census(monkeypatch, fault): + from types import SimpleNamespace + from unittest.mock import AsyncMock + + parent, processes, rows = _process_fixture(monkeypatch) + replacement = { + **rows[0], + "participant_id": "replacement", + "host_proof": "h|b|c|1|10|100|||cell-retirement-v1", + } + processes[10] = {**processes[8], "pid": 10, "start": "100"} + if fault == "certificate": + replacement["host_proof"] = "h|b|c|1|10|100|||unknown" + if fault == "namespace": + processes[10]["namespace"] = 2 + stage = 0 + + def children(_path): + pending = fault in {"spawn", "timeout"} and stage == 0 + return SimpleNamespace(read_text=lambda: "9" if pending else "9 10") + + async def registered(_client): + if fault == "registration" and stage == 0: + return [rows[1]] + if fault == "duplicate": + return [rows[1], replacement, {**replacement, "participant_id": "ambiguous"}] + return [rows[1], replacement] + + async def pause(_seconds): + nonlocal stage + stage += 1 + + monkeypatch.setattr(data, "Path", children) + monkeypatch.setattr(data, "_supervisor", lambda: parent) + monkeypatch.setattr(data, "workers", AsyncMock(return_value=rows)) + monkeypatch.setattr(data, "_worker_rows", registered) + monkeypatch.setattr(data, "_retirement_rows", AsyncMock(side_effect=[[rows[0]], [rows[1]]])) + monkeypatch.setattr(data, "retired", AsyncMock(return_value=True)) + monkeypatch.setattr(data.provider_process, "proves_stopped", lambda _proof: True) + signals = [] + monkeypatch.setattr(data, "_kill_worker", lambda victim, _parent: signals.append(victim)) + ticks = iter([0, 0, 241] if fault == "timeout" else [0, 0, 1, 241]) + monkeypatch.setattr(data, "time", SimpleNamespace(monotonic=lambda: next(ticks))) + monkeypatch.setattr(data.asyncio, "sleep", pause) + if fault in {"spawn", "registration"}: + await data.crash_worker(object()) + assert stage == 1 + else: + pattern = { + "timeout": "supervised replacement was not proved", + "duplicate": "exact registrations: 2", + "certificate": "certificate invalid", + "namespace": "namespace mismatch", + }[fault] + with pytest.raises(AssertionError, match=pattern): + await data.crash_worker(object()) + assert stage == int(fault == "timeout") + assert signals == [rows[0]] diff --git a/tests/unit/test_doctor_client_checks.py b/tests/unit/test_doctor_client_checks.py index 554220a424..a2e1b0bd01 100644 --- a/tests/unit/test_doctor_client_checks.py +++ b/tests/unit/test_doctor_client_checks.py @@ -572,7 +572,7 @@ def test_doctor_client_flag_appends_the_lines_and_json_carries_them( ) -> None: from server.cli import doctor - monkeypatch.setattr(doctor, "_run_doctor_checks", lambda: [("python", "PASS", "3.12")]) + monkeypatch.setattr(doctor, "_run_doctor_checks", lambda *_: [("python", "PASS", "3.12")]) monkeypatch.setattr(doctor, "_doctor_problems", lambda: []) monkeypatch.setattr( doctor, diff --git a/tests/unit/test_doctor_client_client_only.py b/tests/unit/test_doctor_client_client_only.py index 60ceb323d6..712c10d7cd 100644 --- a/tests/unit/test_doctor_client_client_only.py +++ b/tests/unit/test_doctor_client_client_only.py @@ -56,7 +56,7 @@ def test_clickhouse_line_names_the_probed_host_on_failure(monkeypatch: pytest.Mo assert detail.startswith("https://ch.example.com:8443 → TimeoutError") -def _live() -> list[Check]: +def _live(*_roles: Any) -> list[Check]: return [ ("python_version", "PASS", "3.12"), ("clickhouse_reachable", "FAIL", "https://ch.example.com:8443 → TimeoutError: timed out"), diff --git a/tests/unit/test_integration_suite_targets.py b/tests/unit/test_integration_suite_targets.py index df7a98561a..1665edcd26 100644 --- a/tests/unit/test_integration_suite_targets.py +++ b/tests/unit/test_integration_suite_targets.py @@ -114,6 +114,78 @@ def test_the_gate_catches_each_implicit_target(tmp_path: Path) -> None: ], found +@pytest.mark.parametrize("fault", ["none", "second_create", "migration", "first_drop"]) +def test_cell_attach_owns_only_its_created_clickhouse_databases(monkeypatch, fault): + from server.core.schema import migrate + from server.settings import settings + from tests.integration import test_cell_attach as attach + + for name in ("LOOM_PG_DSN", "LOOM_ROUTER_PG_DSN", "LOOM_CH_HOST"): + monkeypatch.setenv(name, "owned-test-target") + initial = settings.ch_database, settings.ch_telemetry_database + created, dropped, connections, migrations = [], [], [], [] + + class Driver: + closed = False + + def command(self, sql): + assert not self.closed + action, _, name, *_ = sql.split() + if action == "CREATE": + if fault == "second_create" and created: + raise RuntimeError("second create refused") + created.append(name) + else: + assert action == "DROP" and name in created + dropped.append(name) + if fault == "first_drop" and len(dropped) == 1: + raise RuntimeError("first drop refused") + + def close(self): + self.closed = True + + def admin(**kwargs): + assert kwargs == {"database": ""} + driver = Driver() + connections.append(driver) + return driver + + bootstrap = Driver() + monkeypatch.setattr(attach, "ch_client_from_env", admin) + monkeypatch.setattr(migrate, "client", lambda **kwargs: bootstrap) + + def apply(args): + assert args == [] + assert migrate.client() is bootstrap + migrations.append((settings.ch_database, settings.ch_telemetry_database)) + assert migrations[-1] == tuple(created) + assert all(name.startswith("loom_it_attach") for name in created) + return 1 if fault == "migration" else 0 + + monkeypatch.setattr(migrate, "main", apply) + fixture = attach.cell_clickhouse_databases.__wrapped__() + if fault == "second_create": + with pytest.raises(RuntimeError, match="second create refused"): + next(fixture) + assert migrations == [] + elif fault == "migration": + with pytest.raises(AssertionError): + next(fixture) + else: + assert next(fixture) == tuple(created) + assert bootstrap.closed and all(driver.closed for driver in connections) + if fault == "first_drop": + with pytest.raises(RuntimeError, match="first drop refused"): + next(fixture) + else: + with pytest.raises(StopIteration): + next(fixture) + assert dropped == list(reversed(created)) + assert all(driver.closed for driver in connections) + assert (settings.ch_database, settings.ch_telemetry_database) == initial + assert len(migrations) == (0 if fault == "second_create" else 1) + + def test_the_gate_catches_a_clickhouse_cli_without_the_target_fixture(tmp_path: Path) -> None: probe = tmp_path / "test_probe.py" probe.write_text('CLI = ROOT / "dev" / "backup_loom.py"\n') diff --git a/tests/unit/test_native_command_retirement.py b/tests/unit/test_native_command_retirement.py new file mode 100644 index 0000000000..228352a83d --- /dev/null +++ b/tests/unit/test_native_command_retirement.py @@ -0,0 +1,201 @@ +"""The actual native command cannot outlive its certified participant.""" + +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from server.cli import _ch, access +from server.core import retirement +from server.insights.app import provider_barrier as barrier +from server.insights.app import provider_command as command + + +@pytest.fixture +def native(monkeypatch): + from server.core import background, ch, pg, pg_profile + from server.core.model_work_gate import ModelWorkGate + + events = [] + gate = retirement.get_gate() + client = SimpleNamespace(connected=False) + + async def connect(): + client.connected = True + + async def disconnect(): + client.connected = False + + client.connect, client.disconnect = connect, disconnect + monkeypatch.setattr(ch, "get_client", lambda: client) + monkeypatch.setattr(pg, "close_pool", AsyncMock()) + + async def verified(_): + assert not gate.enrolled + events.append("verified") + + async def enroll(): + assert events == ["verified"] + gate.enrolled = True + events.append("enrolled") + + async def acceptance(**_): + assert gate.enrolled and gate.active > 0 and not gate.closed + events.append("acceptance") + + async def certificate(proof): + assert gate.quiescent and client.connected + assert events[-1] == "buffer" + events.append("certificate") + + async def flush(): + events.append("buffer") + return True + + monkeypatch.setattr(pg_profile, "check_configured", verified) + monkeypatch.setattr(command, "register_process", enroll) + monkeypatch.setattr(_ch, "check_acceptance", acceptance) + model_gate = ModelWorkGate() + monkeypatch.setattr(command, "get_model_work_gate", lambda: model_gate) + monkeypatch.setattr(barrier, "get_model_work_gate", lambda: model_gate) + monkeypatch.setattr(command, "load_active_configuration", AsyncMock()) + monkeypatch.setattr(command, "_adopt_overrides", AsyncMock(return_value=True)) + monkeypatch.setattr(background, "_registered_tasks", set()) + monkeypatch.setattr(command, "drain_task_sets", background.drain_task_sets) + monkeypatch.setattr(barrier, "drain_task_sets", background.drain_task_sets) + monkeypatch.setattr(command, "flush_registered_buffers", flush) + monkeypatch.setattr(barrier, "flush_registered_buffers", flush) + monkeypatch.setattr(barrier, "_server_side_settled", AsyncMock(return_value=True)) + monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) + monkeypatch.setattr(barrier, "_OWN", barrier._Own()) + monkeypatch.setattr( + barrier, "_HEARTBEAT", SimpleNamespace(signal_stop=lambda: None, running=False) + ) + monkeypatch.setattr(barrier, "record_retirement", certificate) + return gate, events + + +def test_actual_claim_release_keeps_lookup_and_release_in_one_lifetime(native, monkeypatch): + import asyncio + + from server.core.background import create_deferred_task + from server.identity.app import claim_release + from server.identity.store import namespaces + + gate, events = native + + async def lookup(*_, **__): + assert gate.active > 0 and not gate.closed + events.append("lookup") + return {"owner_email": "owner@example.test"} + + async def release(**_): + assert gate.active > 0 and gate.enrolled and not gate.closed + assert "certificate" not in events + events.append("release") + + async def descendant(): + assert gate.enrolled and gate.closed + assert "certificate" not in events + events.append("descendant") + + async def write(): + await asyncio.sleep(0) + assert gate.enrolled and gate.closed + assert "certificate" not in events + events.append("deferred") + create_deferred_task(descendant()) + + create_deferred_task(write()) + return SimpleNamespace(grants_revoked=2) + + monkeypatch.setattr(namespaces, "fetch_row", lookup) + monkeypatch.setattr(claim_release, "release_claim", release) + assert access._cmd_claim_release(SimpleNamespace(org="o", namespace="n", yes=True)) == 0 + assert events.count("enrolled") == events.count("certificate") == 1 + assert events.index("lookup") < events.index("release") < events.index("certificate") + assert events.index("release") < events.index("deferred") < events.index("descendant") + assert events.index("descendant") < events.index("buffer") < events.index("certificate") + assert gate.quiescent + + +@pytest.mark.parametrize("platform_admin", [False, True]) +def test_actual_mint_readiness_keyring_signing_and_audit_precede_retirement( + native, monkeypatch, platform_admin +): + import argparse + + from server.auth import jwt, keyring, machines + from server.observability import audit + + gate, events = native + + def observed(event): + assert gate.active > 0 and gate.enrolled and not gate.closed + assert "certificate" not in events + events.append(event) + + async def ready(_): + observed("readiness") + return True + + async def refresh(**_): + observed("keyring") + return True + + def mint(**_): + observed("mint") + return "test-token" + + async def record(*_, **__): + observed("audit") + return "audit-id" + + monkeypatch.setattr(machines, "platform_admin_tokens_ready", ready) + monkeypatch.setattr(keyring, "refresh", refresh) + monkeypatch.setattr(jwt, "mint_token", mint) + monkeypatch.setattr(audit, "emit", record) + parser = argparse.ArgumentParser() + access._add_mint_token_parser(parser.add_subparsers(dest="cmd")) + argv = ["mint-token", "--subject", "sa:test", "--org", "o", "--ttl-days", "1"] + argv += ["--platform-admin"] if platform_admin else ["--namespaces", "n"] + assert access._cmd_mint_token(parser.parse_args(argv)) == 0 + assert events.count("enrolled") == events.count("certificate") == 1 + assert events.index("keyring") < events.index("mint") < events.index("certificate") + if platform_admin: + assert events.index("readiness") < events.index("keyring") + assert events.index("mint") < events.index("audit") < events.index("certificate") + assert gate.quiescent + + +@pytest.mark.parametrize("failure", ["unavailable", "wrong-pair"]) +def test_actual_mint_never_signs_without_verified_enrollment(native, monkeypatch, failure): + import argparse + from unittest.mock import Mock + + from server.auth import jwt + from server.core import pg_profile + from server.core.pg import AuthorityMismatch + + gate, events = native + + async def refuse(_): + if failure == "wrong-pair": + raise AuthorityMismatch("different paired stores") + raise ConnectionError("unavailable") + + minted = Mock(side_effect=AssertionError("unenrolled mint")) + monkeypatch.setattr(pg_profile, "check_configured", refuse) + monkeypatch.setattr(jwt, "mint_token", minted) + parser = argparse.ArgumentParser() + access._add_mint_token_parser(parser.add_subparsers(dest="cmd")) + args = parser.parse_args( + ["mint-token", "--subject", "sa:test", "--org", "o", "--namespaces", "n"] + ) + if failure == "wrong-pair": + with pytest.raises(AuthorityMismatch): + access._cmd_mint_token(args) + else: + assert access._cmd_mint_token(args) == 1 + minted.assert_not_called() + assert not gate.enrolled and not gate.closed and events == [] diff --git a/tests/unit/test_pg_close_owned.py b/tests/unit/test_pg_close_owned.py new file mode 100644 index 0000000000..cb5abfa58f --- /dev/null +++ b/tests/unit/test_pg_close_owned.py @@ -0,0 +1,405 @@ +"""Owned-session cleanup delivers cancellation before aborting its transport.""" + +from __future__ import annotations + +import asyncio +import time +from typing import Any + +import pytest + +from server.core import pg + +pytestmark = pytest.mark.asyncio + + +class Connection: + def __init__(self) -> None: + self.started = asyncio.Event() + self.finished = asyncio.Event() + self.aborted = False + self.timeouts: list[float] = [] + + async def close(self, *, timeout: float) -> None: # noqa: ASYNC109 — driver API. + self.timeouts.append(timeout) + self.started.set() + await self.finished.wait() + + def terminate(self) -> None: + self.aborted = True + self.finished.set() + + +async def test_public_close_completes_before_transport_abort() -> None: + conn = Connection() + conn.finished.set() + await pg.close_owned(conn, end=time.monotonic() + 1) + assert conn.timeouts and 0 < conn.timeouts[0] <= 1 + assert not conn.aborted + + +async def test_expired_deadline_aborts_without_starting_close() -> None: + conn = Connection() + with pytest.raises(TimeoutError): + await pg.close_owned(conn, end=time.monotonic() - 1) + assert conn.aborted and not conn.timeouts + + +async def test_close_timeout_aborts_within_the_original_deadline() -> None: + conn = Connection() + started = time.monotonic() + with pytest.raises(TimeoutError): + await pg.close_owned(conn, end=started + 0.03) + assert conn.aborted and time.monotonic() - started < 0.3 + + +async def test_outer_signals_do_not_abort_pending_public_close() -> None: + conn = Connection() + task = asyncio.create_task(pg.close_owned(conn, end=time.monotonic() + 0.5)) + try: + await conn.started.wait() + task.cancel("first close interruption") + await asyncio.sleep(0) + task.cancel("second close interruption") + await asyncio.sleep(0) + assert not task.done() + assert not conn.aborted + conn.finished.set() + with pytest.raises(asyncio.CancelledError, match="first close interruption"): + await task + assert not conn.aborted + assert len(conn.timeouts) == 1 + finally: + conn.finished.set() + await asyncio.gather(task, return_exceptions=True) + + +async def test_self_cancelled_close_child_does_not_spin_to_deadline() -> None: + class SelfCancelled(Connection): + async def close(self, **_: Any) -> None: + raise asyncio.CancelledError("close child cancelled") + + started = time.monotonic() + with pytest.raises(asyncio.CancelledError, match="close child cancelled"): + await pg.close_owned(SelfCancelled(), end=started + 10) + assert time.monotonic() - started < 0.3 + + +@pytest.mark.parametrize( + "later", + [ + ValueError("late close"), + asyncio.CancelledError("close child"), + SystemExit(17), + KeyboardInterrupt(), + ], +) +async def test_first_outer_cancel_survives_a_later_close_failure(later: BaseException) -> None: + class Failing(Connection): + async def close(self, **_: Any) -> None: + self.started.set() + await self.finished.wait() + raise later + + def is_closed(self) -> bool: + return True # An aborted transport cannot prove cancellation acknowledgement. + + conn = Failing() + + async def captured() -> BaseException | None: + try: + await pg.close_owned(conn, end=time.monotonic() + 10) + except BaseException as error: # noqa: BLE001 — keep native failures inside the test task. + return error + return None + + task = asyncio.create_task(captured()) + try: + await conn.started.wait() + task.cancel("first close interruption") + await asyncio.sleep(0) + assert not task.done() + assert not conn.aborted + started = time.monotonic() + conn.finished.set() + result = await task + assert isinstance(result, asyncio.CancelledError) + assert str(result) == "first close interruption" + assert time.monotonic() - started < 0.3 + assert conn.aborted + finally: + conn.finished.set() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("abort_error", [ValueError("abort"), SystemExit(17), KeyboardInterrupt()]) +async def test_abort_failure_cannot_replace_the_close_failure(abort_error: BaseException) -> None: + original = ValueError("original close failure") + + class Broken(Connection): + async def close(self, **_: Any) -> None: + raise original + + def terminate(self) -> None: + raise abort_error + + with pytest.raises(ValueError) as result: + await pg.close_owned(Broken(), end=time.monotonic() + 1) + assert result.value is original + + +async def test_second_cancellation_waits_for_the_original_cleanup_end() -> None: + conn = Connection() + supervisor_started = asyncio.Event() + original = asyncio.CancelledError("operation cancelled") + + async def supervisor() -> None: + try: + supervisor_started.set() + await asyncio.Future() + except asyncio.CancelledError: + try: + await pg.close_owned(conn, end=time.monotonic() + 0.03) + except BaseException: # noqa: BLE001 — preserve the original cancellation. + pass + raise original from None + + task = asyncio.create_task(supervisor()) + await supervisor_started.wait() + task.cancel() + await asyncio.wait_for(conn.started.wait(), 1) + task.cancel() + await asyncio.sleep(0) + assert not conn.aborted + with pytest.raises(asyncio.CancelledError, match="operation cancelled"): + await asyncio.wait_for(task, 0.3) + assert conn.aborted + + +async def test_failed_public_close_aborts_and_reports_failure() -> None: + class Broken(Connection): + async def close(self, **_: Any) -> None: + raise ValueError("close failed") + + conn = Broken() + with pytest.raises(ValueError, match="close failed"): + await pg.close_owned(conn, end=time.monotonic() + 1) + assert conn.aborted + + +async def test_expired_close_child_exception_is_retrieved() -> None: + release = asyncio.Event() + errors: list[dict[str, Any]] = [] + + class LateFailure(Connection): + async def close(self, **_: Any) -> None: + self.started.set() + try: + await release.wait() + except asyncio.CancelledError: + await release.wait() + raise ValueError("late close failure") + + conn = LateFailure() + loop = asyncio.get_running_loop() + old_handler = loop.get_exception_handler() + loop.set_exception_handler(lambda _, context: errors.append(context)) + try: + with pytest.raises(TimeoutError): + await pg.close_owned(conn, end=time.monotonic() + 0.01) + assert conn.aborted + release.set() + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not errors + finally: + release.set() + loop.set_exception_handler(old_handler) + + +async def test_migration_failure_survives_cleanup_failure(monkeypatch, tmp_path) -> None: + import contextlib + + from server.core.schema import pg_migrate + + class Broken(Connection): + async def close(self, **_: Any) -> None: + raise RuntimeError("cleanup failed") + + conn = Broken() + + @contextlib.asynccontextmanager + async def connect(): + yield conn + + async def fail(*_: Any) -> None: + raise ValueError("migration failed") + + monkeypatch.setattr(pg_migrate, "_migrate", fail) + with pytest.raises(ValueError, match="migration failed"): + await pg_migrate.migrate(tmp_path, connect=connect, schema="test") + assert conn.aborted + + +async def test_successful_migration_still_reports_context_exit_failure( + monkeypatch, tmp_path +) -> None: + import contextlib + + from server.core.schema import pg_migrate + + original = ValueError("context exit failed") + + @contextlib.asynccontextmanager + async def connect(): + yield Connection() + raise original + + async def applied(*_: Any) -> list[str]: + return ["001.sql"] + + monkeypatch.setattr(pg_migrate, "_migrate", applied) + with pytest.raises(ValueError) as error: + await pg_migrate.migrate(tmp_path, connect=connect, schema="test") + assert error.value is original + + +@pytest.mark.parametrize("cancel_work", [False, True]) +async def test_second_cancellation_during_migration_context_exit_keeps_first_failure( + monkeypatch, tmp_path, cancel_work +) -> None: + import contextlib + + from server.core.schema import pg_migrate + + conn = Connection() + conn.finished.set() + started, exiting = asyncio.Event(), asyncio.Event() + observed: list[BaseException] = [] + original = ValueError("migration failed") + + @contextlib.asynccontextmanager + async def connect(): + try: + yield conn + except BaseException as error: + observed.append(error) + exiting.set() + await asyncio.Future() + raise + + async def work(*_: Any) -> list[str]: + started.set() + if cancel_work: + await asyncio.Future() + raise original + + monkeypatch.setattr(pg_migrate, "_migrate", work) + monkeypatch.setattr(pg, "_RELEASE_TIMEOUT_S", 0.1) + task = asyncio.create_task(pg_migrate.migrate(tmp_path, connect=connect, schema="test")) + await asyncio.wait_for(started.wait(), 1) + before = time.monotonic() + if cancel_work: + task.cancel("first cancellation") + await asyncio.wait_for(exiting.wait(), 1) + task.cancel("second cancellation") + with pytest.raises(BaseException) as result: + await asyncio.wait_for(task, 0.3) + assert time.monotonic() - before < 0.3 + assert result.value is observed[0] + assert ( + isinstance(result.value, asyncio.CancelledError) + if cancel_work + else result.value is original + ) + assert len(conn.timeouts) == 1 + assert conn.aborted + + +@pytest.mark.parametrize("seam", ["reap", "exit"]) +@pytest.mark.parametrize("original_kind", ["ordinary", "cancel", "exit", "keyboard"]) +@pytest.mark.parametrize("abort_kind", ["ordinary", "cancel", "exit", "keyboard"]) +async def test_failed_migration_abort_preserves_first_and_finishes_context( + monkeypatch, tmp_path, seam, original_kind, abort_kind +): + from server.core.schema import pg_migrate + + def error(kind, message): + return { + "ordinary": ValueError(message), + "cancel": asyncio.CancelledError(message), + "exit": SystemExit(message), + "keyboard": KeyboardInterrupt(message), + }[kind] + + original, secondary = ( + error(original_kind, "migration failed"), + error(abort_kind, "abort failed"), + ) + events, first, exits, ends, retrieved, triples = [], [], [], [], [], [] + conn = Connection() + real_wait, real_failed, real_retrieve = ( + asyncio.wait, + pg_migrate._failed_migration, + pg._retrieve_task, + ) + + class Context: + async def __aenter__(self): + return conn + + async def __aexit__(self, kind, value, traceback): + exits.append((kind, value, traceback)) + events.append("exit") + if seam == "exit": + raise OSError("context exit failed") + + async def migrate(*args): + raise original + + async def close(_conn, *, end): + assert _conn is conn + ends.append(end) + events.append("close") + + async def failed(cm, actual_conn, task, observed, end): + first.append(observed) + triples.append((type(observed), observed, observed.__traceback__)) + return await real_failed(cm, actual_conn, task, observed, end) + + async def wait(tasks, *, timeout): # noqa: ASYNC109 — asyncio.wait test seam. + assert 0 <= timeout <= max(0, ends[0] - time.monotonic()) + 0.01 + if seam == "reap" and "reap" not in events: + events.append("reap") + raise asyncio.CancelledError("second signal during reap") + return await real_wait(tasks, timeout=timeout) + + def abort(): + events.append("abort") + raise secondary + + def retrieve(task): + retrieved.append(task) + real_retrieve(task) + + monkeypatch.setattr(pg_migrate, "_migrate", migrate) + monkeypatch.setattr(pg_migrate, "_failed_migration", failed) + monkeypatch.setattr(pg, "close_owned", close) + monkeypatch.setattr(conn, "terminate", abort) + monkeypatch.setattr(asyncio, "wait", wait) + monkeypatch.setattr(pg, "_retrieve_task", retrieve) + try: + await pg_migrate.migrate(tmp_path, connect=Context, schema="test") + except BaseException as observed: # noqa: BLE001 — native test failures must remain inert. + result = observed + else: + pytest.fail("failed migration returned success") + assert len(first) == 1 and result is first[0] + if original_kind != "cancel": + assert result is original + assert exits == triples + assert len(ends) == 1 and events.count("close") == events.count("abort") == 1 + assert events[-1] == ("exit" if seam == "reap" else "abort") + for _ in range(3): + await asyncio.sleep(0) + assert len(retrieved) == 2 and all(task.done() for task in retrieved) diff --git a/tests/unit/test_pg_online_index.py b/tests/unit/test_pg_online_index.py new file mode 100644 index 0000000000..3bb0c19476 --- /dev/null +++ b/tests/unit/test_pg_online_index.py @@ -0,0 +1,217 @@ +"""Only the declared concurrent btree form escapes a migration transaction.""" + +from __future__ import annotations + +import pytest + +from server.core.schema import pg_online_index + +SQL = """-- loom:online-index loom_router.placements_by_cell_org ON loom_router.placements (cell, org) +CREATE INDEX CONCURRENTLY placements_by_cell_org ON loom_router.placements +USING btree (cell COLLATE "C", org COLLATE "C"); +""" + + +def test_the_online_index_declaration_has_one_exact_target() -> None: + parsed = pg_online_index.parse(SQL) + assert parsed is not None + assert parsed.schema == "loom_router" + assert parsed.name == "placements_by_cell_org" + assert parsed.table == "placements" + assert parsed.columns == ("cell", "org") + assert pg_online_index.parse("CREATE TABLE loom.t (id int);") is None + + +@pytest.mark.parametrize( + "sql", + [ + SQL + "SELECT 1;", + SQL.replace("CREATE INDEX", "CREATE UNIQUE INDEX"), + SQL.replace('org COLLATE "C"', 'org COLLATE "POSIX"'), + SQL.replace("(cell, org)", "(org, cell)"), + SQL.replace("ON loom_router.placements\n", "ON loom_router.other\n"), + SQL.replace("USING btree", "USING hash"), + SQL.replace('org COLLATE "C"', 'lower(org) COLLATE "C"'), + SQL.replace(");", ") WHERE true;"), + SQL.replace("-- loom:online-index", "-- loom:online-index-bad"), + SQL.replace("placements_by_cell_org ON", "other_index ON", 1), + SQL.replace("(cell, org)", "(cell, cell)"), + SQL.replace("CONCURRENTLY", ""), + ], +) +def test_other_online_sql_is_refused(sql: str) -> None: + with pytest.raises(ValueError, match="online.index"): + pg_online_index.parse(sql) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("pending", [True, False]) +async def test_ledger_retries_commit_a_wal_fence_even_when_already_recorded( + tmp_path, pending +) -> None: + import contextlib + + from server.core import pg + from server.core.schema import pg_migrate + + path = tmp_path / "001_index.sql" + path.write_text(SQL) + + class Connection: + def __init__(self): + self.statements = [] + + def transaction(self): + return contextlib.nullcontext() + + async def execute(self, sql, *args): + self.statements.append((sql, args)) + + conn = Connection() + await pg_migrate._record_online(conn, path, "loom_router", pending=pending) + assert conn.statements[-1] == (pg._DURABLE_MESSAGE, ()) + assert len(conn.statements) == (2 if pending else 1) + if pending: + assert conn.statements[0][1] == (path.name, pg_migrate.checksum(path)) + + +@pytest.mark.asyncio +async def test_migrator_preserves_a_borrowed_sessions_statement_budget(tmp_path) -> None: + import contextlib + + from server.core.schema import pg_migrate + + class Connection: + def __init__(self): + self.statements = [] + + async def execute(self, sql, *args): + self.statements.append(sql) + + async def fetch(self, sql): + return [] + + async def fetchval(self, sql, *args): + self.statements.append(sql) + return True + + conn = Connection() + + @contextlib.asynccontextmanager + async def connect(): + yield conn + + assert await pg_migrate.migrate(tmp_path, connect=connect, schema="loom_router") == [] + assert not any("statement_timeout" in sql for sql in conn.statements) + assert "SET synchronous_commit = 'on'" in conn.statements + assert conn.statements[-1] == "SELECT pg_advisory_unlock($1)" + + +@pytest.mark.asyncio +async def test_online_target_must_match_its_ledger_before_connecting(tmp_path) -> None: + from server.core.schema import pg_migrate + + (tmp_path / "001_index.sql").write_text(SQL) + + def never_connect(): + pytest.fail("invalid target opened a connection") + + with pytest.raises(ValueError, match="ledger schema"): + await pg_migrate.migrate(tmp_path, schema="loom", connect=never_connect) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("phase", ["statement", "rollback", "commit"]) +async def test_cancellation_closes_before_installed_asyncpg_transaction_unwind( + tmp_path, phase +) -> None: + import asyncio + import contextlib + from types import SimpleNamespace + + from asyncpg.transaction import Transaction + + from server.core.schema import pg_migrate + + (tmp_path / "001_one.sql").write_text("SELECT stalled_statement;") + reached = asyncio.Event() + terminated = asyncio.Event() + closed = asyncio.Event() + + class Connection: + _pool_release_ctr = 0 + _top_xact = None + _protocol = SimpleNamespace(is_in_transaction=lambda: False) + + def is_closed(self): + return closed.is_set() or terminated.is_set() + + def transaction(self): + return Transaction(self, None, False, False) + + async def close(self, **kwargs): + assert kwargs["timeout"] > 0 + assert not terminated.is_set() + closed.set() + + def terminate(self): + terminated.set() + + async def fetchval(self, sql, *args): + return True + + async def fetch(self, sql): + return [] + + async def execute(self, sql, *args): + if sql == "SELECT stalled_statement;" and phase == "rollback": + raise ValueError("statement failed before rollback") + if ( + (sql == "SELECT stalled_statement;" and phase == "statement") + or (sql.startswith("ROLLBACK") and phase == "rollback") + or (sql.startswith("COMMIT") and phase == "commit") + ): + reached.set() + # Public close must run independently of transaction unwinding. + while not closed.is_set(): + try: + await closed.wait() + except asyncio.CancelledError: + continue + + conn = Connection() + + @contextlib.asynccontextmanager + async def connect(): + yield conn + + migrating = asyncio.create_task( + pg_migrate.migrate(tmp_path, connect=connect, schema="loom_router") + ) + try: + await asyncio.wait_for(reached.wait(), 1) + migrating.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(migrating, 1) + assert closed.is_set() and not terminated.is_set() + finally: + closed.set() + if not migrating.done(): + migrating.cancel() + await asyncio.gather(migrating, return_exceptions=True) + + +def test_postgres_journals_keep_distinct_numbers_and_declared_online_forms() -> None: + from server.core.paths import PG_MIGRATIONS_DIR, ROUTER_MIGRATIONS_DIR + from server.core.schema import pg_migrate + from server.core.schema.journal import migration_number + + for directory in (PG_MIGRATIONS_DIR, ROUTER_MIGRATIONS_DIR): + files = pg_migrate.migration_files(directory) + numbers = [migration_number(path.name) for path in files] + assert None not in numbers and len(numbers) == len(set(numbers)) + for path in files: + sql = path.read_text() + parsed = pg_online_index.parse(sql) + if "CONCURRENTLY" in sql: + assert parsed is not None, f"{path.name} lacks an online-index declaration" diff --git a/tests/unit/test_pg_owned_fatal.py b/tests/unit/test_pg_owned_fatal.py new file mode 100644 index 0000000000..dcf77f560e --- /dev/null +++ b/tests/unit/test_pg_owned_fatal.py @@ -0,0 +1,227 @@ +"""Owned child tasks preserve native failures until their supervisor cleans up.""" + +from __future__ import annotations + +import subprocess +import sys +import textwrap + +import pytest + + +@pytest.mark.parametrize("kind", ["SystemExit", "KeyboardInterrupt"]) +@pytest.mark.parametrize( + "phase", + ["close", "migration", "migration_exit", "migration_ordinary_exit", "migration_exit_stall"], +) +def test_native_failure_reaches_the_owner_after_cleanup(kind: str, phase: str, tmp_path) -> None: + program = textwrap.dedent( + """ + import asyncio + import contextlib + import sys + import time + from pathlib import Path + from server.core import pg + from server.core.schema import pg_migrate + + kind, phase, path = sys.argv[1:] + fatal_type = SystemExit if kind == 'SystemExit' else KeyboardInterrupt + original = ValueError('original') if 'ordinary' in phase else fatal_type('original') + secondary = fatal_type('cleanup') + events = [] + + class Connection: + async def close(self, **kwargs): + events.append('close') + if phase == 'close': + raise original + if phase in {'migration_exit', 'migration_ordinary_exit'}: + raise secondary + def terminate(self): + events.append('abort') + + conn = Connection() + + @contextlib.asynccontextmanager + async def connect(): + try: + yield conn + finally: + events.append('exit') + if phase == 'migration_exit_stall': + await asyncio.Future() + if phase in {'migration_exit', 'migration_ordinary_exit'}: + raise secondary + + async def fail(*args): + events.append('work') + raise original + + async def main(): + pg_migrate._migrate = fail + pg._RELEASE_TIMEOUT_S = 0.05 + started = time.monotonic() + try: + if phase == 'close': + await pg.close_owned(conn, end=time.monotonic() + 1) + else: + await pg_migrate.migrate(Path(path), connect=connect, schema='test') + except BaseException as error: + assert error is original, (type(error).__name__, events) + assert time.monotonic() - started < 0.3, events + assert 'close' in events, events + if phase != 'close': + assert 'exit' in events, events + assert events.index('close') < events.index('exit'), events + else: + assert 'abort' in events, events + await asyncio.sleep(0) + print('owner-received-original-after-cleanup') + return + raise AssertionError('native failure was swallowed') + + asyncio.run(main()) + """ + ) + result = subprocess.run( + [sys.executable, "-c", program, kind, phase, str(tmp_path)], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "owner-received-original-after-cleanup" in result.stdout + + +@pytest.mark.parametrize("connection_context", ["cell", "router"]) +@pytest.mark.parametrize("kind", ["SystemExit", "KeyboardInterrupt", "ValueError"]) +@pytest.mark.parametrize("exit_behavior", ["fatal", "stalled"]) +def test_actual_connection_context_keeps_first_failure_and_cleanup_end( + connection_context: str, kind: str, exit_behavior: str, tmp_path +) -> None: + program = textwrap.dedent( + """ + import asyncio + import sys + import time + from pathlib import Path + import asyncpg + from server.core import pg + from server.core.schema import pg_migrate + from server.router import directory + + context, kind, exit_behavior, path = sys.argv[1:] + original = {'SystemExit': SystemExit, 'KeyboardInterrupt': KeyboardInterrupt, + 'ValueError': ValueError}[kind]('original') + secondary = SystemExit('context failed') + events = [] + + class Connection: + async def close(self, **kwargs): + events.append('close') + if events.count('close') > 1: + if exit_behavior == 'stalled': + await asyncio.Future() + raise secondary + def terminate(self): + events.append('abort') + + conn = Connection() + async def raw(*args, **kwargs): + return conn + async def checked(*args): + pass + async def fail(*args): + raise original + + async def main(): + pg.settings.pg_dsn = "postgresql://unused/test" + pg._check_deployment = checked + pg.init_connection = checked + asyncpg.connect = raw + directory._require_dsn = lambda: 'unused-test-dsn' + pg._RELEASE_TIMEOUT_S = 0.05 + pg_migrate._migrate = fail + connect = None if context == 'cell' else directory._dedicated + started = time.monotonic() + try: + await pg_migrate.migrate(Path(path), connect=connect, schema='test') + except BaseException as error: + assert error is original, (type(error).__name__, events) + assert time.monotonic() - started < 0.3, events + assert events.count('close') == 2, events + if exit_behavior == 'stalled': + assert 'abort' in events, events + await asyncio.sleep(0) + print('first-failure-and-original-end-preserved') + return + raise AssertionError('first migration failure was swallowed') + asyncio.run(main()) + """ + ) + result = subprocess.run( + [sys.executable, "-c", program, connection_context, kind, exit_behavior, str(tmp_path)], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "first-failure-and-original-end-preserved" in result.stdout + + +@pytest.mark.parametrize("exit_error", ["RuntimeError", "SystemExit", "KeyboardInterrupt"]) +def test_synchronous_context_exit_failure_keeps_original_migration_error( + exit_error: str, tmp_path +) -> None: + program = textwrap.dedent( + """ + import asyncio + import sys + from pathlib import Path + from server.core.schema import pg_migrate + + kind, path = sys.argv[1:] + original = ValueError('original migration error') + secondary = {'RuntimeError': RuntimeError, 'SystemExit': SystemExit, + 'KeyboardInterrupt': KeyboardInterrupt}[kind]('context factory failed') + events = [] + class Connection: + async def close(self, **kwargs): + events.append('close') + def terminate(self): + events.append('abort') + class Context: + async def __aenter__(self): + return Connection() + def __aexit__(self, kind, value, traceback): + assert value is original + events.append('exit') + raise secondary + async def fail(*args): + raise original + async def main(): + pg_migrate._migrate = fail + try: + await pg_migrate.migrate(Path(path), connect=Context, schema='test') + except BaseException as error: + assert error is original, (type(error).__name__, events) + assert events[:2] == ['close', 'exit'], events + await asyncio.sleep(0) + print('synchronous-context-failure-contained') + return + raise AssertionError('migration failure was swallowed') + asyncio.run(main()) + """ + ) + result = subprocess.run( + [sys.executable, "-c", program, exit_error, str(tmp_path)], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "synchronous-context-failure-contained" in result.stdout diff --git a/tests/unit/test_platform_admin_bots.py b/tests/unit/test_platform_admin_bots.py index 84fd4c1834..ed2ddb14b2 100644 --- a/tests/unit/test_platform_admin_bots.py +++ b/tests/unit/test_platform_admin_bots.py @@ -13,6 +13,7 @@ import asyncio import json from typing import Any +from unittest.mock import AsyncMock import jwt as pyjwt import pytest @@ -181,13 +182,16 @@ def minting(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, Any]]: monkeypatch.setenv("LOOM_JWT_SECRET", _SECRET) monkeypatch.setattr(settings, "jwt_secret", _SECRET, raising=False) - # An empty keyring: the mint signs with the root secret, reading no ClickHouse. - monkeypatch.setattr(access, "run_connected", lambda run, **_declared: None) + from server.auth import keyring + + monkeypatch.setattr(access, "run_connected", lambda run, **_declared: asyncio.run(run(None))) + monkeypatch.setattr(keyring, "load_for_connected_mint", AsyncMock()) + monkeypatch.setattr("server.cli._ch.check_acceptance", AsyncMock()) # These mint tests run after every replica has the human-admin guards. - monkeypatch.setattr(access, "_platform_admin_rollout_ready", lambda: True) + monkeypatch.setattr(access, "_platform_admin_rollout_ready", AsyncMock(return_value=True)) recorded: list[dict[str, Any]] = [] - def audited(args: Any, summary: dict[str, Any]) -> bool: + async def audited(args: Any, summary: dict[str, Any]) -> bool: recorded.append({"subject": args.subject, "jti": summary["jti"]}) return True @@ -236,7 +240,7 @@ def test_a_platform_admin_mint_the_audit_log_misses_is_not_handed_out( ) -> None: from server.cli import access - monkeypatch.setattr(access, "_audit_platform_admin_mint", lambda args, summary: False) + monkeypatch.setattr(access, "_audit_platform_admin_mint", AsyncMock(return_value=False)) assert cli.main([*_MINT, "--subject", "sa:cp", "--ttl-days", "7"]) == 1 out, err = capsys.readouterr() assert out == "" and "no token" in err @@ -257,7 +261,7 @@ async def emit(event: str, **kw: Any) -> str: monkeypatch.setattr(audit, "emit", emit) monkeypatch.setattr(access, "run_connected", lambda run, **_declared: asyncio.run(run(None))) args = SimpleNamespace(subject="sa:cp", org="acme", ttl_days=7) - assert access._audit_platform_admin_mint(args, {"jti": "j1", "namespaces": ["*"]}) + assert asyncio.run(access._audit_platform_admin_mint(args, {"jti": "j1", "namespaces": ["*"]})) [(event, kw)] = rows assert event == "auth.platform_admin_minted" and kw["org"] == "acme" assert kw["subject"] == {"subject": "sa:cp", "jti": "j1", "namespaces": ["*"], "ttl_days": 7} @@ -266,7 +270,9 @@ async def lost(event: str, **kw: Any) -> str: return "" # audit.emit's answer when its insert failed monkeypatch.setattr(audit, "emit", lost) - assert not access._audit_platform_admin_mint(args, {"jti": "j2", "namespaces": ["*"]}) + assert not asyncio.run( + access._audit_platform_admin_mint(args, {"jti": "j2", "namespaces": ["*"]}) + ) _CP_BODY = { diff --git a/tests/unit/test_platform_admin_rollout.py b/tests/unit/test_platform_admin_rollout.py index 134f54cc8a..6bf52c711b 100644 --- a/tests/unit/test_platform_admin_rollout.py +++ b/tests/unit/test_platform_admin_rollout.py @@ -4,6 +4,7 @@ import time from typing import Any +from unittest.mock import AsyncMock import pytest @@ -87,9 +88,10 @@ def mint(**kw: Any) -> str: monkeypatch.setattr(instances, "live_rows", lambda: [{"schema_version": 76}]) monkeypatch.setattr(instances, "read_live", read_live) - monkeypatch.setattr(keyring, "load_for_offline_mint", lambda **kw: None) + monkeypatch.setattr(keyring, "load_for_connected_mint", AsyncMock()) + monkeypatch.setattr("server.cli._ch.check_acceptance", AsyncMock()) monkeypatch.setattr(access, "run_connected", lambda run, **_declared: asyncio.run(run(None))) - monkeypatch.setattr(access, "_audit_platform_admin_mint", lambda *a: True) + monkeypatch.setattr(access, "_audit_platform_admin_mint", AsyncMock(return_value=True)) monkeypatch.setattr(jwt, "mint_token", mint) code = cli.main( [ @@ -114,7 +116,7 @@ def mint(**kw: Any) -> str: assert minted == [] -def test_cli_admin_mint_refuses_an_unread_fleet(monkeypatch, capsys) -> None: +def test_cli_admin_mint_refuses_when_the_connection_scope_cannot_open(monkeypatch, capsys) -> None: from server.auth import jwt from server.cli import access @@ -139,7 +141,51 @@ def mint(**kw): ] ) out, err = capsys.readouterr() + assert code == 1 and out == "" and "enrolled scope" in err + + +def test_cli_admin_mint_refuses_an_unread_fleet(monkeypatch, capsys) -> None: + import asyncio + + from server.auth import jwt, keyring + from server.cli import access + + connected = object() + reads = [] + minted = [] + + async def unread(ch, *, all_deployments): + assert all_deployments + reads.append(ch) + raise ConnectionError("offline") + + def mint(**kwargs): + minted.append(kwargs) + return "secret-token" + + monkeypatch.setattr(instances, "read_live", unread) + monkeypatch.setattr(keyring, "load_for_connected_mint", AsyncMock()) + monkeypatch.setattr("server.cli._ch.check_acceptance", AsyncMock()) + monkeypatch.setattr(access, "run_connected", lambda run, **kwargs: asyncio.run(run(connected))) + monkeypatch.setattr(access, "_audit_platform_admin_mint", AsyncMock(return_value=True)) + monkeypatch.setattr(jwt, "mint_token", mint) + code = cli.main( + [ + "mint-token", + "--subject", + "sa:controlplane", + "--org", + "acme", + "--platform-admin", + "--ttl-days", + "7", + ] + ) + out, err = capsys.readouterr() + assert reads == [connected] assert code == 1 and out == "" and "rollout" in err + assert "secret-token" not in out + err + assert minted == [] def test_admin_token_floor_matches_the_physical_migration() -> None: diff --git a/tests/unit/test_provider_barrier.py b/tests/unit/test_provider_barrier.py index 8cac6eceb8..551a8c998f 100644 --- a/tests/unit/test_provider_barrier.py +++ b/tests/unit/test_provider_barrier.py @@ -29,6 +29,11 @@ def _row(pid, transition=None, state="running", *, version=1, expired=False): @pytest.fixture def setup(monkeypatch): + from server.core import retirement + + retirement.get_gate().install("test") + retirement.get_gate().enrolled = True + retirement.get_gate().producers_stopped = True identity, transition = uuid4(), uuid4() gate = ModelWorkGate() monkeypatch.setattr(barrier, "get_model_work_gate", lambda: gate) @@ -37,6 +42,7 @@ def setup(monkeypatch): monkeypatch.setattr(barrier, "flush_registered_buffers", AsyncMock(return_value=True)) record = AsyncMock() monkeypatch.setattr(barrier, "record_participant", record) + monkeypatch.setattr(barrier, "record_retirement", AsyncMock()) monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) rows = [_row(identity, transition, "drained")] monkeypatch.setattr(barrier, "participants", AsyncMock(return_value=rows)) @@ -188,7 +194,7 @@ def test_a_heartbeat_carries_the_latest_state_above_any_version_seen(setup): "state": "drained", "version": 10**20 + 1, "stale_after_s": barrier.STALE_AFTER_S, - "host_proof": barrier.host_proof(), + "host_proof": barrier.participant_proof(), } # An acknowledgement stamped after the beat outranks it. assert barrier._OWN.stamp(transition, "drained")[2] > row[3] @@ -205,12 +211,14 @@ async def test_register_writes_the_first_row_and_retire_stops_the_beats(setup, m await barrier.register_process() assert beat.running and len(ch.inserts) == 1 tags: list[str] = [] - record.side_effect = lambda *_a, **_k: tags.append(current_query_tag()) + barrier.record_retirement.side_effect = lambda *_a, **_k: tags.append(current_query_tag()) + beat.signal_stop() + beat.thread.join(timeout=5) assert await barrier.retire_process() assert beat.stopping.is_set() # signalled, not joined: the thread ends after its beat beat.thread.join(timeout=5) assert not beat.running - assert record.call_args.args[2] == "retired" + assert barrier.record_retirement.call_args.args[0].participant == barrier.process_identity() assert tags == ["provider.participant_heartbeat"] and current_query_tag() == "" beat.ch = ch beat.beat() @@ -218,7 +226,7 @@ async def test_register_writes_the_first_row_and_retire_stops_the_beats(setup, m @pytest.mark.asyncio -async def test_a_boot_that_fails_after_enrolling_retires_its_row(setup, monkeypatch): +async def test_a_boot_that_fails_does_not_certify_a_pending_heartbeat(setup, monkeypatch): _, _, _, record = setup monkeypatch.setattr(barrier, "_HEARTBEAT", barrier._Heartbeat(lambda: _SyncCH())) monkeypatch.setattr(barrier, "_registered", False) @@ -232,7 +240,10 @@ async def lifespan(_app): with pytest.raises(RuntimeError, match="migrations failed"): async with barrier.participant_lifespan(lifespan)(None): pytest.fail("a failed boot never serves") - assert record.call_args.args[2] == "retired" + barrier.record_retirement.assert_not_awaited() + barrier._HEARTBEAT.thread.join(timeout=5) + assert await barrier.retire_process() + assert barrier.record_retirement.call_args.args[0].participant == barrier.process_identity() @pytest.mark.asyncio @@ -263,8 +274,9 @@ async def test_retire_participant_refuses_a_live_one(setup, monkeypatch): assert await barrier.retire_participant(other) == "live" record.assert_not_awaited() monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) - assert await barrier.retire_participant(other) == "retired" - record.assert_awaited_once_with(other, UUID(int=0), "retired", after=5, ch=None) + assert await barrier.retire_participant(other) == "live" + record.assert_not_awaited() + barrier.record_retirement.assert_not_awaited() def test_loom_participant_retire_refuses_a_live_one(setup, monkeypatch, capsys): @@ -281,7 +293,7 @@ def test_loom_participant_retire_refuses_a_live_one(setup, monkeypatch, capsys): monkeypatch.setattr(_ch, "run_connected", lambda run, **_declared: asyncio.run(run(None))) args = argparse.Namespace(participant=str(other), yes=True) assert participant._cmd_retire(args) == 1 - assert "still running" in capsys.readouterr().err + assert "no verifiable exact physical-death proof" in capsys.readouterr().err record.assert_not_awaited() @@ -434,14 +446,13 @@ async def _hang(*_a, **_k): @pytest.mark.asyncio -async def test_a_live_row_expires_after_the_heartbeat_window_and_a_retired_one_at_once(): +async def test_a_live_row_expires_after_the_heartbeat_window_and_a_retired_one_at_once(monkeypatch): ch = SimpleNamespace(insert=AsyncMock()) - await pp.record_participant(uuid4(), UUID(int=0), "running", ch=ch) + await pp.record_participant(pp.process_identity(), UUID(int=0), "running", ch=ch) (row,) = ch.insert.call_args.args[1] assert row["stale_after_s"] == pp.STALE_AFTER_S > barrier.HEARTBEAT_S - await pp.record_participant(uuid4(), UUID(int=0), "retired", ch=ch) - (row,) = ch.insert.call_args.args[1] - assert row["stale_after_s"] == 0 + with pytest.raises(ValueError, match="state"): + await pp.record_participant(uuid4(), UUID(int=0), "retired", ch=ch) @pytest.mark.asyncio diff --git a/tests/unit/test_provider_command.py b/tests/unit/test_provider_command.py index 347506b522..645b0fcb3e 100644 --- a/tests/unit/test_provider_command.py +++ b/tests/unit/test_provider_command.py @@ -8,12 +8,25 @@ from server.insights.app import provider_command as command +@pytest.mark.parametrize("value", [0, -1, float("inf"), float("nan")]) +def test_native_drain_budget_is_finite_and_positive(value): + from pydantic import ValidationError + + from server.settings.catalog import RuntimeSettings + + with pytest.raises(ValidationError): + RuntimeSettings(native_command_drain_timeout_s=value) + + @pytest.fixture def setup(monkeypatch): + from server.core import retirement + + retirement.get_gate().enrolled = True gate = ModelWorkGate() monkeypatch.setattr(command, "get_model_work_gate", lambda: gate) monkeypatch.setattr(command, "register_process", AsyncMock()) - monkeypatch.setattr(command, "retire_process", AsyncMock()) + monkeypatch.setattr(command, "_retire_until", AsyncMock(return_value=True)) monkeypatch.setattr(command, "load_active_configuration", AsyncMock()) monkeypatch.setattr(command, "drain_task_sets", AsyncMock(return_value=True)) monkeypatch.setattr(command, "flush_registered_buffers", AsyncMock(return_value=True)) @@ -26,9 +39,9 @@ async def test_command_enrolls_loads_settings_then_admits_and_retires(setup): assert setup.active == 1 command.register_process.assert_awaited_once() command.load_active_configuration.assert_awaited_once() - command.retire_process.assert_not_awaited() + command._retire_until.assert_not_awaited() assert setup.active == 0 - command.retire_process.assert_awaited_once() + command._retire_until.assert_awaited_once() @pytest.mark.asyncio @@ -37,15 +50,16 @@ async def test_pending_transition_never_runs_command(setup): with pytest.raises(ModelChangeInProgress): async with command.model_command(): pytest.fail("closed admission must reject a newborn command") - command.retire_process.assert_awaited_once() + command._retire_until.assert_awaited_once() @pytest.mark.asyncio async def test_undrained_descendants_keep_participant_registered(setup, monkeypatch): monkeypatch.setattr(command, "drain_task_sets", AsyncMock(return_value=False)) - async with command.model_command(): - pass - command.retire_process.assert_not_awaited() + with pytest.raises(RuntimeError, match="deferred writes did not drain"): + async with command.model_command(): + pass + command._retire_until.assert_not_awaited() command.flush_registered_buffers.assert_not_awaited() @@ -54,4 +68,4 @@ async def test_business_failure_does_not_skip_drain(setup): with pytest.raises(ValueError, match="business failure"): async with command.model_command(): raise ValueError("business failure") - command.retire_process.assert_awaited_once() + command._retire_until.assert_awaited_once() diff --git a/tests/unit/test_provider_host_proof.py b/tests/unit/test_provider_host_proof.py index 60cead66a5..295f8dc03f 100644 --- a/tests/unit/test_provider_host_proof.py +++ b/tests/unit/test_provider_host_proof.py @@ -66,11 +66,11 @@ def _eperm(pid: int, sig: int) -> None: assert not pp.proves_stopped(_with(pid=_dead_pid())) -def test_a_container_restarted_into_a_new_pid_namespace_is_proven_stopped( +def test_a_changed_container_pid_namespace_does_not_prove_the_old_process_stopped( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(pp, "_container", lambda: "c" * 64) - assert pp.proves_stopped(_with(container="c" * 64, pidns="1")) + assert not pp.proves_stopped(_with(container="c" * 64, pidns="1")) @pytest.mark.parametrize("theirs", ["d" * 64, ""]) @@ -152,127 +152,73 @@ async def test_this_host_retires_its_dead_participants_once_their_queries_ended( barrier, "running_queries", AsyncMock(side_effect=lambda p: int(str(p) == str(busy))) ) record = AsyncMock() - monkeypatch.setattr(barrier, "record_participant", record) + monkeypatch.setattr(barrier, "record_retirement", record) monkeypatch.setattr(barrier, "_last_here", -float("inf")) assert await barrier.retire_dead_here(now=100.0) == 1 - record.assert_awaited_once_with(dead, UUID(int=0), "retired", after=5) + record.assert_awaited_once() + proof = record.await_args.args[0] + assert proof.participant == dead and proof.after == 5 + assert proof.host_proof == pp.certified(gone) assert await barrier.retire_dead_here(now=100.0 + barrier.HERE_S / 2) == 0 # throttled -# -- a pod Kubernetes deleted ------------------------------------------------ - -_POD_UID = "0b6f2c1e-1111-2222-3333-444455556666" - - -def _pod_proof( - namespace: str = "loom", container: str = f"{_POD_UID}/loom", pod: str = "loom-7d9-abc" +def _fresh_process_stat( + monkeypatch: pytest.MonkeyPatch, text: str, error: type[Exception] | None = None ) -> str: - # The host name is the pod name cut to 63 characters; the lookup uses the pod name. - return "|".join((pod[:63], "boot", container, "4026", "7", "99", namespace, pod)) + """Hold the first birth check steady and supply the subsequent kernel record.""" + from io import StringIO + monkeypatch.setattr(pp.socket, "gethostname", lambda: "host") + monkeypatch.setattr(pp, "_boot_and_pidns", lambda: ("boot", "77")) + monkeypatch.setattr(pp.os, "kill", lambda pid, sig: None) + monkeypatch.setattr(pp, "_start_ticks", lambda pid: "900") -@pytest.fixture -def kube(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> Any: - """A service-account mount, the API's address and a stand-in API: it answers - the (status, body) pairs queued in `answers` and keeps what it saw in `seen`.""" - from types import SimpleNamespace + def fresh_open(path: str, *args: Any, **kwargs: Any) -> StringIO: + assert path == "/proc/424242/stat" + if error is not None: + raise error("uninspectable fresh process record") + return StringIO(text) - import httpx + monkeypatch.setattr(pp, "open", fresh_open, raising=False) + return "host|boot|container|77|424242|900||" - (tmp_path / "token").write_text("tok") - (tmp_path / "ca.crt").write_text("") - monkeypatch.setattr(pp, "_SA", str(tmp_path)) - monkeypatch.setenv("POD_NAMESPACE", "loom") - monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "10.96.0.1") - api = SimpleNamespace(answers=[], seen=[]) - def handler(request: httpx.Request) -> httpx.Response: - api.seen.append(request) - status, body = api.answers.pop(0) - return httpx.Response(status, json=body) +def _process_stat_record(state: str = "Z", threads: str = "1", start: str = "900") -> str: + fields = ["0"] * 20 + fields[0], fields[17], fields[19] = state, threads, start + return "424242 (owned child) " + " ".join(fields) - real = httpx.AsyncClient - monkeypatch.setattr( - httpx, - "AsyncClient", - lambda **kw: real(transport=httpx.MockTransport(handler), timeout=kw.get("timeout")), - ) - return api - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "status,uid,phase,gone", - [ - (404, "", "", True), - (200, "another-uid", "Running", True), - (200, _POD_UID, "Failed", True), # evicted: the object stays, its containers stopped - (200, _POD_UID, "Succeeded", True), - (200, _POD_UID, "Running", False), - (200, _POD_UID, "Pending", False), - (403, "", "", False), - ], -) -async def test_a_pod_is_gone_only_when_the_api_says_so(kube, status, uid, phase, gone) -> None: - kube.answers.append((status, {"metadata": {"uid": uid}, "status": {"phase": phase}})) - assert await pp.pod_gone(_pod_proof()) is gone - assert kube.seen[0].url.path == "/api/v1/namespaces/loom/pods/loom-7d9-abc" - assert kube.seen[0].headers["authorization"] == "Bearer tok" +def test_an_exact_single_thread_zombie_record_proves_exit(monkeypatch: pytest.MonkeyPatch) -> None: + proof = _fresh_process_stat(monkeypatch, _process_stat_record()) + assert pp.proves_stopped(proof) -@pytest.mark.asyncio @pytest.mark.parametrize( - "proof", + "record", [ - _pod_proof(namespace="other"), - _pod_proof(container="a" * 64), - _pod_proof(pod=""), - "h|b|c|p|1|2", - "h|b|" + _POD_UID + "/loom|p|1|2|loom", # a 7-part proof (no pod name) proves nothing - "", + _process_stat_record(threads="0"), + _process_stat_record(threads="2"), + _process_stat_record(threads="-1"), + _process_stat_record(threads="unknown"), + _process_stat_record(threads="1.0"), + *[_process_stat_record(state=state) for state in ("S", "D", "R", "T", "t", "X")], + _process_stat_record(start="901"), + _process_stat_record(start="unknown"), + "424242 (owned child) Z", + "unparseable kernel record", ], ) -async def test_a_pod_elsewhere_or_outside_kubernetes_is_never_gone(kube, proof) -> None: - assert await pp.pod_gone(proof) is False - assert kube.seen == [] # nothing to ask: another namespace, a Docker container, an old row - - -@pytest.mark.asyncio -async def test_the_leader_retires_an_expired_participant_whose_pod_is_gone( - monkeypatch: pytest.MonkeyPatch, +def test_fresh_kernel_record_must_prove_the_same_single_thread_exit( + monkeypatch: pytest.MonkeyPatch, record: str ) -> None: - me, lost, busy, kept = (uuid4() for _ in range(4)) - monkeypatch.setattr(barrier, "process_identity", lambda: me) - rows = [ - _row(me, pp.host_proof()), - _row(lost, _pod_proof(), expired=True, version=9), - _row(busy, _pod_proof(), expired=True), - _row(kept, _pod_proof(), expired=True), - ] - monkeypatch.setattr(barrier, "participants", AsyncMock(return_value=rows)) - monkeypatch.setattr(barrier, "prune_retired", AsyncMock()) - monkeypatch.setattr( - barrier, "running_queries", AsyncMock(side_effect=lambda p: int(str(p) == str(busy))) - ) - asked: list[str] = [] - - async def gone(proof: str) -> bool: - asked.append(proof) - return len(asked) == 1 # the first asked (lost) is gone, kept's pod still exists - - monkeypatch.setattr(barrier, "pod_gone", gone) - record = AsyncMock() - monkeypatch.setattr(barrier, "record_participant", record) - monkeypatch.setattr(barrier, "_STALE", None) - await barrier.sweep_stopped(now=5000.0) - record.assert_awaited_once_with(lost, UUID(int=0), "retired", after=9) - assert len(asked) == 2 # a participant with a query running is never asked about - assert [p["participant_id"] for p in barrier.stale_participants()] == [str(busy), str(kept)] + proof = _fresh_process_stat(monkeypatch, record) + assert not pp.proves_stopped(proof) -@pytest.mark.asyncio -async def test_a_long_pod_name_is_looked_up_whole_not_by_its_cut_host_name(kube) -> None: - long = "loom-memory-platform-production-workers-7d9c8b6f5d-x4kqz-abcdefgh" - kube.answers.append((200, {"metadata": {"uid": _POD_UID}, "status": {"phase": "Running"}})) - assert await pp.pod_gone(_pod_proof(pod=long)) is False - assert kube.seen[0].url.path == f"/api/v1/namespaces/loom/pods/{long}" +@pytest.mark.parametrize("error", [PermissionError, OSError, IndexError]) +def test_unreadable_fresh_kernel_record_never_proves_exit( + monkeypatch: pytest.MonkeyPatch, error: type[Exception] +) -> None: + proof = _fresh_process_stat(monkeypatch, _process_stat_record(), error) + assert not pp.proves_stopped(proof) diff --git a/tests/unit/test_provider_zombie_exit.py b/tests/unit/test_provider_zombie_exit.py new file mode 100644 index 0000000000..db6bd2868f --- /dev/null +++ b/tests/unit/test_provider_zombie_exit.py @@ -0,0 +1,152 @@ +"""Exact single-thread process exit is provable before its parent collects it.""" + +import ctypes +import os +import select +import signal +import threading +import time +from pathlib import Path + +import pytest + +from server.core import provider_process + +pytestmark = pytest.mark.skipif(not provider_process.host_proof(), reason="needs /proc") + + +def _stat(pid: int) -> tuple[str, int, str, int]: + parts = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split() + return parts[0], int(parts[1]), parts[19], int(parts[17]) + + +def _wait_owned(pid: int, start: str) -> int: + """Collect only this fork's child, within a finite cleanup allowance.""" + for force in (False, True): + end = time.monotonic() + 2.0 + if force: + try: + _, parent, now_start, _ = _stat(pid) + except FileNotFoundError: + parent, now_start = None, None + if parent == os.getpid() and now_start == start: + os.kill(pid, signal.SIGKILL) + while time.monotonic() < end: + found, status = os.waitpid(pid, os.WNOHANG) + if found == pid: + return status + time.sleep(0.005) + raise AssertionError("Owned witness child did not settle within cleanup allowance") + + +def test_an_exact_single_thread_zombie_is_proven_stopped_before_wait() -> None: + """A captured child has exited, even while its parent retains the wait status.""" + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: + os.close(write_fd) + os.read(read_fd, 1) + os._exit(0) + os.close(read_fd) + start = None + released = False + try: + state, parent, start, threads = _stat(pid) + assert state != "Z" and parent == os.getpid() and threads == 1 + proof = provider_process.host_proof().split("|") + assert len(proof) == 8 and all(proof[i] for i in (0, 1, 3)) + proof[4:6] = [str(pid), start] + exact = "|".join(proof) + assert not provider_process.proves_stopped(exact), "Live child must refuse" + wrong_namespace = proof.copy() + wrong_namespace[3] = str(int(proof[3]) + 1) + assert not provider_process.proves_stopped("|".join(wrong_namespace)) + os.write(write_fd, b"x") + released = True + end = time.monotonic() + 2.0 + while time.monotonic() < end: + state, parent, observed_start, threads = _stat(pid) + assert parent == os.getpid() and observed_start == start + if state == "Z" and threads == 1: + break + time.sleep(0.005) + assert state == "Z" and threads == 1, ( + "Controlled child must reach single-thread Z before any waitpid" + ) + assert provider_process.proves_stopped(exact) + finally: + if not released: + os.write(write_fd, b"x") + os.close(write_fd) + if start is None: + _, parent, start, _ = _stat(pid) + assert parent == os.getpid() + status = _wait_owned(pid, start) + assert os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0 + assert provider_process.proves_stopped(exact), "After waitpid, absence proves death" + + +def test_a_zombie_leader_is_refused_until_its_last_live_thread_exits() -> None: + """Z alone is insufficient while another thread of that exact process runs.""" + read_fd, write_fd = os.pipe() + ready_read, ready_write = os.pipe() + pid = os.fork() + if pid == 0: + os.close(write_fd) + os.close(ready_read) + + def live_thread() -> None: + os.write(ready_write, b"r") + os.read(read_fd, 1) + os._exit(0) + + threading.Thread(target=live_thread).start() + libc = ctypes.CDLL(None) + libc.pthread_exit.argtypes = [ctypes.c_void_p] + libc.pthread_exit.restype = None + libc.pthread_exit(None) + os._exit(99) # pthread_exit must not return. + os.close(read_fd) + os.close(ready_write) + start = None + released = False + try: + _, parent, start, _ = _stat(pid) + assert parent == os.getpid() + proof = provider_process.host_proof().split("|") + assert len(proof) == 8 and all(proof[i] for i in (0, 1, 3)) + proof[4:6] = [str(pid), start] + exact = "|".join(proof) + assert select.select([ready_read], [], [], 2.0)[0] == [ready_read] + assert os.read(ready_read, 1) == b"r" + end = time.monotonic() + 2.0 + while time.monotonic() < end: + state, parent, observed_start, threads = _stat(pid) + assert parent == os.getpid() and observed_start == start + if state == "Z" and threads > 1: + break + time.sleep(0.005) + assert state == "Z" and threads > 1, "Live-thread leader-Z witness was not established" + assert not provider_process.proves_stopped(exact) + os.write(write_fd, b"x") + released = True + end = time.monotonic() + 2.0 + while time.monotonic() < end: + state, parent, observed_start, threads = _stat(pid) + assert parent == os.getpid() and observed_start == start + if state == "Z" and threads == 1: + break + time.sleep(0.005) + assert state == "Z" and threads == 1, "Controlled group exit must reach Z1 before waitpid" + assert provider_process.proves_stopped(exact) + finally: + if not released: + os.write(write_fd, b"x") + os.close(write_fd) + os.close(ready_read) + if start is None: + _, parent, start, _ = _stat(pid) + assert parent == os.getpid() + status = _wait_owned(pid, start) + assert os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0 + assert provider_process.proves_stopped(exact) diff --git a/tests/unit/test_readiness.py b/tests/unit/test_readiness.py index b7adc50935..49e153bc68 100644 --- a/tests/unit/test_readiness.py +++ b/tests/unit/test_readiness.py @@ -441,16 +441,50 @@ async def insert(self, table, rows, *, settings=None): usage._reset_for_tests() -def test_shutdown_drains_share_a_budget_before_retirement() -> None: +@pytest.mark.asyncio +async def test_shutdown_drains_share_a_budget_before_retirement(monkeypatch) -> None: import inspect + from types import SimpleNamespace from server import main - - src = inspect.getsource(main._shut_down) - budget = src.index("_shutdown_drains(lease, supervisor, emitter), timeout=left") - assert ( - src.index("await _instances.stop(ch)") < budget < src.index("await boot.leave_deployment(") - ) + from server.core import instances + from server.insights.app.provider_barrier import SHUTDOWN_RETIRE_S + + clock = [100.0] + calls = [] + wait_timeouts = [] + ch, lease, supervisor, emitter = object(), object(), object(), object() + + async def stop(client): + assert client is ch + calls.append("stop") + clock[0] += 3.0 + + async def drains(actual_lease, actual_supervisor, actual_emitter): + assert (actual_lease, actual_supervisor, actual_emitter) == (lease, supervisor, emitter) + calls.append("drains") + clock[0] += 2.0 + + async def wait(tasks, **kwargs): + wait_timeouts.append(kwargs["timeout"]) + return await asyncio.wait(tasks, **kwargs) + + async def leave(client, *, budget_s): + assert client is ch + calls.append("retire") + assert budget_s == SHUTDOWN_RETIRE_S + 5.0 + + monkeypatch.setattr(main, "time", SimpleNamespace(monotonic=lambda: clock[0])) + monkeypatch.setattr(main, "asyncio", SimpleNamespace(wait=wait)) + monkeypatch.setattr(instances, "stop", stop) + monkeypatch.setattr(main, "_shutdown_drains", drains) + monkeypatch.setattr(boot, "leave_deployment", leave) + monkeypatch.setattr(boot, "SHUTDOWN_DRAIN_BUDGET_S", 10.0) + + await main._shut_down(ch, lease, supervisor, emitter) + + assert calls == ["stop", "drains", "retire"] + assert wait_timeouts == [7.0] life = inspect.getsource(main.lifespan) assert "await _shut_down(" in life and "ch.disconnect()" not in life diff --git a/tests/unit/test_reextract.py b/tests/unit/test_reextract.py index 852f94d1b0..addaef8003 100644 --- a/tests/unit/test_reextract.py +++ b/tests/unit/test_reextract.py @@ -159,7 +159,9 @@ def test_cli_refuses_the_source_as_target(capsys) -> None: assert "must differ" in capsys.readouterr().err -def test_reextract_token_falls_back_to_the_saved_token(tmp_path, monkeypatch) -> None: +def test_reextract_token_falls_back_to_the_saved_token( + tmp_path, monkeypatch, stub_model_command +) -> None: """--token, else LOOM_API_KEY, else ~/.loom/token: the order every client command uses.""" import argparse diff --git a/tests/unit/test_revocation_web_refresh.py b/tests/unit/test_revocation_web_refresh.py index bbc0f56a5a..8dac8ed351 100644 --- a/tests/unit/test_revocation_web_refresh.py +++ b/tests/unit/test_revocation_web_refresh.py @@ -133,16 +133,28 @@ def _poll_loop(period: float) -> asyncio.Task[None]: t.cancel() -def test_lifespan_starts_the_token_state_and_cancels_it_at_shutdown() -> None: - src = (ROOT / "server" / "main.py").read_text() - lifespan = src[src.index("async def lifespan(") :] - assert "auth_loops = await boot.start_token_state()" in lifespan - assert ( - "for loop in (*cell_loops, *auth_loops, overrides_loop):\n loop.cancel()" - in lifespan +@pytest.mark.asyncio +async def test_lifespan_starts_the_token_state_and_cancels_it_at_shutdown(monkeypatch) -> None: + import ast + + from server import boot, main + from server.code.app.index import mirrors + from server.core import retirement + from server.memory.app import embedder_calibration + + tree = ast.parse((ROOT / "server" / "main.py").read_text()) + lifespan = next( + n for n in ast.walk(tree) if isinstance(n, ast.AsyncFunctionDef) and n.name == "lifespan" ) - # the decorator still sits on lifespan, not on the helper above it - assert "@asynccontextmanager\nasync def lifespan(" in src + calls = {ast.unparse(n.func) for n in ast.walk(lifespan) if isinstance(n, ast.Call)} + assert {"boot.start_token_state", "_stop_background"} <= calls + monkeypatch.setattr(mirrors, "stop_warmup", lambda: None) + monkeypatch.setattr(embedder_calibration, "stop_refresh_loop", lambda: None) + monkeypatch.setattr(boot, "cancel_boot_tasks", lambda: None) + task = retirement.get_gate().track(asyncio.create_task(asyncio.Event().wait())) + await main._stop_background(None) + await asyncio.gather(task, return_exceptions=True) + assert task.cancelled() and retirement.get_gate().closed @pytest.mark.asyncio diff --git a/tests/unit/test_revoke_issued_before.py b/tests/unit/test_revoke_issued_before.py index e7e9bb0af0..fa0c71444d 100644 --- a/tests/unit/test_revoke_issued_before.py +++ b/tests/unit/test_revoke_issued_before.py @@ -20,6 +20,7 @@ from contextlib import asynccontextmanager from datetime import UTC, datetime, timedelta from typing import Any +from unittest.mock import AsyncMock import jwt as pyjwt import pytest @@ -265,10 +266,12 @@ def test_cli_mint_reports_the_jti_but_never_beside_a_saved_token( from server.settings import settings monkeypatch.setattr(settings, "jwt_secret", _SECRET, raising=False) + from server.auth import keyring from server.cli import access - # An empty keyring: the mint signs with the root secret, reading no ClickHouse. - monkeypatch.setattr(access, "run_connected", lambda run, **_declared: None) + monkeypatch.setattr(access, "run_connected", lambda run, **_declared: asyncio.run(run(None))) + monkeypatch.setattr(keyring, "load_for_connected_mint", AsyncMock()) + monkeypatch.setattr("server.cli._ch.check_acceptance", AsyncMock()) rc = cli.main( ["mint-token", "--subject", "sa:x", "--org", "acme", "--namespaces", "shop", "--json"] ) @@ -539,6 +542,6 @@ def test_cli_mint_refuses_a_reserved_subject_without_a_traceback( def _read(**_k: Any) -> None: raise AssertionError("the keyring was read for a subject no token may carry") - monkeypatch.setattr(keyring, "load_for_offline_mint", _read) + monkeypatch.setattr(keyring, "load_for_connected_mint", _read) rc = cli.main(["mint-token", "--subject", subject, "--org", "acme", "--namespaces", "shop"]) assert rc == 2 and "grant principal" in capsys.readouterr().err diff --git a/tests/unit/test_router.py b/tests/unit/test_router.py index 9de51a1e75..84bd4db79d 100644 --- a/tests/unit/test_router.py +++ b/tests/unit/test_router.py @@ -9,6 +9,7 @@ from contextlib import asynccontextmanager from typing import Any from unittest.mock import AsyncMock +from uuid import UUID import httpx import pytest @@ -46,6 +47,7 @@ def _row(**over: Any) -> directory.Cell: "next_fp": None, } values.update(over) + values["mutation_generation"] = UUID(int=1) return directory.Cell(**values) @@ -102,6 +104,8 @@ async def lookup(self, org: str) -> tuple[str, str] | None: async def set_limit(self, name: str, **fields: Any) -> directory.Cell: self._check() + expected = fields.pop("expected") + assert expected.generation == self.cells[name].mutation_generation given = {k: v for k, v in fields.items() if v is not directory.UNSET} self.cells[name] = dataclasses.replace(self.cells[name], **given) return self.cells[name] @@ -964,7 +968,13 @@ def test_a_cell_the_operator_closes_refuses_its_next_new_org( assert _place(client, "before").status_code == 200 closed = client.post( "/cells/v1/operator/cells/eu-1/limit", - json={"closed": True}, + json={ + "closed": True, + "expected": { + **{key: getattr(fake.cells["eu-1"], key, "") for key in directory.CONTROL_FIELDS}, + "mutation_generation": str(fake.cells["eu-1"].mutation_generation), + }, + }, headers={"Authorization": f"Bearer {operator}"}, ) assert closed.status_code == 200 and closed.json()["closed"] is True diff --git a/tests/unit/test_router_attach.py b/tests/unit/test_router_attach.py index efb0b29e80..821272bcdf 100644 --- a/tests/unit/test_router_attach.py +++ b/tests/unit/test_router_attach.py @@ -14,12 +14,14 @@ from datetime import UTC, datetime from pathlib import Path from typing import Any +from uuid import UUID import httpx import pytest from starlette.testclient import TestClient from server.cli import cell as cell_cli +from server.cli import cell_control from server.core import cell from server.router import app as router_app from server.router import directory @@ -44,6 +46,7 @@ def _cell_row(**over: Any) -> dict[str, Any]: row: dict[str, Any] = { "cell": "eu-1", + "mutation_generation": UUID(int=1), "is_first": True, "host": "loom.example", "state": "attaching", @@ -86,6 +89,7 @@ class _Conn: def __init__(self, store: _Store) -> None: self.s = store + self._tx = None def _count(self) -> None: if self.s.down: @@ -102,7 +106,7 @@ def _row(self, name: str, sql: str) -> dict[str, Any] | None: async def execute(self, sql: str, *a: Any) -> str: self._count() cells = self.s.cells - if sql.startswith("SELECT pg_advisory_xact_lock"): + if sql.startswith(("SELECT pg_advisory_xact_lock", "SELECT pg_logical_emit_message")): return "" if "SET attach_started_at" in sql: row = cells[a[0]] @@ -172,17 +176,36 @@ async def fetch(self, sql: str, *a: Any) -> list[Any]: self.s.placements.update(dict.fromkeys(new, a[1])) return [{"org": o} for o in new] + async def close(self, **_kw: Any) -> None: + self.terminate() + + def terminate(self) -> None: + if self._tx is not None: + self.s.cells, self.s.hosts, self.s.placements, self.s.audit = self._tx.saved + self._tx = None + def transaction(self) -> Any: conn = self class _Tx: + async def start(self) -> None: + await self.__aenter__() + + async def commit(self) -> None: + await self.__aexit__(None) + + async def rollback(self) -> None: + await self.__aexit__(Exception) + async def __aenter__(self) -> None: + conn._tx = self conn._count() self.saved = copy.deepcopy( (conn.s.cells, conn.s.hosts, conn.s.placements, conn.s.audit) ) async def __aexit__(self, kind: Any, *_exc: Any) -> None: + conn._tx = None conn.s.statements += 1 # the commit, or the rollback if kind is not None: conn.s.cells, conn.s.hosts, conn.s.placements, conn.s.audit = self.saved @@ -233,7 +256,9 @@ def listener() -> _Listener: @pytest.fixture def client(store: _Store, listener: _Listener) -> TestClient: router = router_app.Router(probe_transport=httpx.MockTransport(listener)) - return TestClient(router_app.build_app(router)) + client = TestClient(router_app.build_app(router)) + client.control_store = store + return client def _call( @@ -241,6 +266,11 @@ def _call( ) -> Any: url = "/cells/v1/operator/cells/eu-1" + (f"/{path}" if path else "") headers = {"Authorization": f"Bearer {token}"} + if method != "GET" and (body is None or isinstance(body, dict)): + row = client.control_store.cells.get("eu-1", _cell_row()) + observed = {name: row[name] for name in directory.CONTROL_FIELDS} + observed["mutation_generation"] = str(row["mutation_generation"]) + body = {**(body or {}), "expected": observed} if body is None: return client.request(method, url, headers=headers) return client.request(method, url, json=body, headers=headers) @@ -249,6 +279,10 @@ def _call( def _begin(**over: Any) -> dict[str, Any]: body: dict[str, Any] = { "deployment_id": "dep-1", + "expected": { + **{name: _cell_row()[name] for name in directory.CONTROL_FIELDS}, + "mutation_generation": str(UUID(int=1)), + }, "current_fp": CURRENT, "next_fp": None, "keys_fp": "keys-1", @@ -292,6 +326,8 @@ def test_the_operator_reads_the_cells_row(client: TestClient, store: _Store) -> "current_fp": None, "next_fp": None, "other": None, + "mutation_generation": str(UUID(int=1)), + **{name: store.cells["eu-1"][name] for name in directory._REGISTERED}, } # Any other row is named: every attach call refuses while it exists. store.cells["eu-2"] = _cell_row(cell="eu-2", is_first=False, host="eu-2.example") @@ -322,6 +358,15 @@ def test_every_new_route_needs_the_operator_credential_and_names_an_unknown_cell assert store.statements == 0 url = "/cells/v1/operator/cells/eu-9" + (f"/{path}" if path else "") headers = {"Authorization": f"Bearer {OPERATOR}"} + if method != "GET": + source = _cell_row() + body = { + **(body or {}), + "expected": { + **{name: source[name] for name in directory.CONTROL_FIELDS}, + "mutation_generation": str(source["mutation_generation"]), + }, + } r = client.request(method, url, headers=headers, json=body) assert (r.status_code, r.json()["code"]) == (404, "NO_SUCH_CELL") bad = client.request(method, url.replace("eu-9", "EU_9"), headers=headers, json=body) @@ -613,7 +658,15 @@ def test_begin_refuses_a_fingerprint_whose_placement_body_would_pass_the_routers # And again under the lock. big = directory.Begin("dep-1", grown, None, "keys", {}) with pytest.raises(directory.OperatorRefused, match="configured_orgs"): - asyncio.run(directory.attach_begin("eu-1", big)) + asyncio.run( + directory.attach_begin( + "eu-1", + big, + expected=directory.Control( + {k: store.cells["eu-1"][k] for k in directory.CONTROL_FIELDS}, UUID(int=1) + ), + ) + ) assert store.cells["eu-1"]["attach_started_at"] is None and store.audit == [] # One that fits, with the longest org escaped, is taken. fits = cell.as_value({**FIELDS, "configured_orgs": [f"org-{i:05d}" for i in range(100)]}) @@ -884,16 +937,33 @@ def sent(call: Any) -> int: asyncio.run(call) return store.statements - before + def observed(name: str = "eu-1") -> directory.Control: + row = store.cells[name] + return directory.Control( + {k: row[k] for k in directory.CONTROL_FIELDS}, row["mutation_generation"] + ) + begin = directory.Begin("dep-1", CURRENT, None, "keys", {}) assert sent(directory.attach_read("eu-1", audited=True)) == directory.OPERATOR_READ_STATEMENTS assert sent(directory.attach_read("eu-1")) == directory.ATTACH_READ_STATEMENTS - assert sent(directory.attach_begin("eu-1", begin)) == directory.ATTACH_CHANGE_STATEMENTS - assert sent(directory.attach_enforced("eu-1", "dep-1")) == directory.ATTACH_CHANGE_STATEMENTS - batch = directory.attach_place("eu-1", "dep-1", [f"o{i}" for i in range(50)]) + assert ( + sent(directory.attach_begin("eu-1", begin, expected=observed())) + == directory.ATTACH_CHANGE_STATEMENTS + ) + assert ( + sent(directory.attach_enforced("eu-1", "dep-1", expected=observed())) + == directory.ATTACH_CHANGE_STATEMENTS + ) + batch = directory.attach_place( + "eu-1", "dep-1", [f"o{i}" for i in range(50)], expected=observed() + ) assert sent(batch) == directory.ATTACH_PLACE_STATEMENTS # whatever the batch's size - assert sent(directory.attach_serving("eu-1", "dep-1")) == directory.ATTACH_CHANGE_STATEMENTS + assert ( + sent(directory.attach_serving("eu-1", "dep-1", expected=observed())) + == directory.ATTACH_CHANGE_STATEMENTS + ) store.cells["eu-2"] = _cell_row(cell="eu-2", is_first=False, host="eu-2.example") - assert sent(directory.remove("eu-2")) == directory.REMOVE_STATEMENTS + assert sent(directory.remove("eu-2", expected=observed("eu-2"))) == directory.REMOVE_STATEMENTS assert directory.REMOVE_MAX_S == directory._bound_s(directory.REMOVE_STATEMENTS) @@ -990,6 +1060,8 @@ def cli_router(monkeypatch: pytest.MonkeyPatch) -> _Router: monkeypatch.setattr(settings, "router_operator_token", OPERATOR) fake = _Router( { + **_cell_row(), + "mutation_generation": str(UUID(int=1)), "cell": "eu-1", "is_first": True, "state": "attaching", @@ -999,6 +1071,23 @@ def cli_router(monkeypatch: pytest.MonkeyPatch) -> _Router: ) monkeypatch.setattr(httpx, "get", fake.get) monkeypatch.setattr(httpx, "delete", fake.delete) + + async def request( + url: Any, name: str, transport: Any, path: str, body: Any = None, *, method: str = "POST" + ) -> Any: + target = f"{url}/cells/v1/operator/cells/{name}" + path + if path == "/claim": + fake.calls.append(("POST", target)) + return dict(fake.row) + response = fake.get(target) if method == "GET" else fake.delete(target) + if response.status_code != 200: + if response.status_code == 404 and response.json().get("code") != "NO_SUCH_CELL": + raise cell_control._refused("a router replica is not upgraded") + raise cell_control._refused(response.json()["error"]) + return response.json() + + monkeypatch.setattr(cell_control, "request", request) + _stores(monkeypatch, "") return fake @@ -1007,13 +1096,33 @@ def _stores(monkeypatch: pytest.MonkeyPatch, refusal: Any) -> list[str]: it was asked about.""" checked: list[str] = [] - async def _check(deployment_id: str) -> str: - checked.append(deployment_id) + async def paired(_conn: Any, row: dict[str, Any]) -> None: + checked.append(row["deployment_id"]) if isinstance(refusal, Exception): raise refusal - return refusal + if refusal: + raise cell_control._refused(refusal) + + async def state(_conn: Any) -> tuple[str, str]: + return "open", "" - monkeypatch.setattr(cell_cli, "_first_cell_refusal", _check) + async def held(_conn: Any) -> None: + pass + + monkeypatch.setattr(cell_control, "paired", paired) + + class Owner: + conn = object() + + async def transaction(self, work): + return await work() + + async def run(_what, work): + return await work(Owner()) + + monkeypatch.setattr(cell_control, "run", run) + monkeypatch.setattr(cell_control, "check_lock", held) + monkeypatch.setattr(cell, "read_state_if_present", state) return checked @@ -1022,9 +1131,9 @@ def test_remove_checks_a_first_cells_postgres_then_deletes_its_row( ) -> None: checked = _stores(monkeypatch, "") assert cell_cli._cmd_remove(_remove_args()) == 0 - assert checked == ["dep-1"] - assert [m for m, _url in cli_router.calls] == ["GET", "DELETE"] - assert cli_router.calls[1][1] == "https://router.test/cells/v1/operator/cells/eu-1" + assert checked == ["dep-1", "dep-1"] + assert [m for m, _url in cli_router.calls] == ["GET", "GET", "POST", "DELETE"] + assert cli_router.calls[-1][1] == "https://router.test/cells/v1/operator/cells/eu-1" assert json.loads(capsys.readouterr().out) == {"cell": "eu-1", "removed": True} @@ -1040,16 +1149,16 @@ def test_remove_refuses_a_first_cell_its_postgres_does_not_clear( ) -> None: _stores(monkeypatch, refusal) assert cell_cli._cmd_remove(_remove_args()) == 1 - assert "refusing" in capsys.readouterr().err - assert [m for m, _url in cli_router.calls] == ["GET"] + assert "stop and rerun" in capsys.readouterr().err + assert [m for m, _url in cli_router.calls] == ["GET", "GET"] -def test_remove_reads_no_store_for_a_later_cell( +def test_remove_checks_a_later_cells_pairing_and_claim( cli_router: _Router, monkeypatch: pytest.MonkeyPatch ) -> None: cli_router.row = {**cli_router.row, "is_first": False, "deployment_id": "dep-2"} - checked = _stores(monkeypatch, "never asked") - assert cell_cli._cmd_remove(_remove_args()) == 0 and checked == [] + checked = _stores(monkeypatch, "") + assert cell_cli._cmd_remove(_remove_args()) == 0 and checked == ["dep-2", "dep-2"] @pytest.mark.parametrize( @@ -1098,7 +1207,7 @@ def test_remove_tells_an_older_router_from_an_unknown_cell_on_the_delete( cli_router.deleted = (404, {"error": "the router answers no such path"}) assert cell_cli._cmd_remove(_remove_args()) == 1 assert "not upgraded" in capsys.readouterr().err - assert [m for m, _url in cli_router.calls] == ["GET", "DELETE"] + assert [m for m, _url in cli_router.calls] == ["GET", "GET", "POST", "DELETE"] @pytest.mark.parametrize( @@ -1112,42 +1221,31 @@ def test_remove_tells_an_older_router_from_an_unknown_cell_on_the_delete( ], ) def test_the_first_cell_check_needs_an_open_postgres_of_the_rows_deployment( - monkeypatch: pytest.MonkeyPatch, state: str, recorded: str, why: str + cli_router: _Router, + monkeypatch: pytest.MonkeyPatch, + capsys: Any, + state: str, + recorded: str, + why: str, ) -> None: - """One Postgres session of its own reads both: no pool, and no ClickHouse.""" - from server.core import pg + async def paired(_conn: Any, row: dict[str, Any]) -> None: + if not recorded: + raise cell_control._refused("records no deployment id") + if recorded != row["deployment_id"]: + raise cell_control._refused(f"records deployment id {recorded!r}") - sessions: list[dict[str, Any]] = [] - - class _Session: - async def fetchval(self, sql: str) -> str: - assert sql == "SELECT deployment_id FROM loom.authority" - return recorded - - @contextlib.asynccontextmanager - async def _dedicated(**kw: Any) -> Any: - sessions.append(kw) - yield _Session() - - async def _state(conn: Any) -> tuple[str, str]: - assert isinstance(conn, _Session) + async def read(_conn: Any) -> tuple[str, str]: return state, "" - def _no_pool(*_a: Any, **_kw: Any) -> None: - raise AssertionError("the check reads no pool") - - # LOOM_PG_DSN alone: the CLI's storage authority setting is not read. - monkeypatch.setattr(settings, "storage_authority", "clickhouse") - monkeypatch.setattr(settings, "pg_dsn", "postgresql://cell.internal/loom") - monkeypatch.setattr(pg, "dedicated", _dedicated) - monkeypatch.setattr(pg, "fetchrow", _no_pool) - monkeypatch.setattr(pg, "fetchval", _no_pool) - monkeypatch.setattr(cell, "read_state_if_present", _state) - refusal = asyncio.run(cell_cli._first_cell_refusal("dep-1")) - assert (why in refusal) if why else refusal == "" - assert sessions == [{"statement_timeout_ms": cell_cli._CHECK_STATEMENT_MS}] - monkeypatch.setattr(settings, "pg_dsn", "") - assert "LOOM_PG_DSN" in asyncio.run(cell_cli._first_cell_refusal("dep-1")) + monkeypatch.setattr(cell_control, "paired", paired) + monkeypatch.setattr(cell, "read_state_if_present", read) + result = cell_cli._cmd_remove(_remove_args()) + if why: + assert result == 1 + assert why.replace("`", "") in capsys.readouterr().err + assert not any(method in ("POST", "DELETE") for method, _url in cli_router.calls) + else: + assert result == 0 def test_remove_never_connects_clickhouse( @@ -1158,13 +1256,7 @@ def test_remove_never_connects_clickhouse( def _never(*_a: Any, **_kw: Any) -> None: raise AssertionError("loom cell remove connects no ClickHouse") - async def _read() -> tuple[str, str]: - return "open", "dep-1" - monkeypatch.setattr(_ch, "run_connected", _never) - monkeypatch.setattr(settings, "storage_authority", "postgres") - monkeypatch.setattr(settings, "pg_dsn", "postgresql://cell.internal/loom") - monkeypatch.setattr(cell_cli, "_read_first_cell", _read) assert cell_cli._cmd_remove(_remove_args()) == 0 @@ -1211,7 +1303,7 @@ def test_a_rotation_drops_this_replicas_cached_state( store.cells["eu-1"]["state"] = "serving" assert _state(client).json()["state"] == "attaching" # cached - async def _rotate(name: str, step: str, sha: str) -> directory.Cell: + async def _rotate(name: str, step: str, sha: str, **_kw: Any) -> directory.Cell: store.cells[name]["credential_next_sha256"] = sha return directory._cell(_Conn(store)._row(name, "")) @@ -1303,13 +1395,14 @@ def test_attach_serving_waits_for_a_state_read_in_flight_and_drops_its_row( after attach/serving runs once that read is done, so the next read answers `serving`.""" router = router_app.Router(probe_transport=httpx.MockTransport(listener)) with TestClient(router_app.build_app(router)) as setup: + setup.control_store = store _enforced(setup) started, go = _blocked_state_read(store, monkeypatch) committed = asyncio.Event() real_serving = directory.attach_serving - async def _serving(*a: Any) -> Any: - found = await real_serving(*a) + async def _serving(*a: Any, **kw: Any) -> Any: + found = await real_serving(*a, **kw) committed.set() return found @@ -1325,7 +1418,14 @@ async def _run() -> None: serving = asyncio.create_task( c.post( "/cells/v1/operator/cells/eu-1/attach/serving", - json={"deployment_id": "dep-1", "nonces": NONCES}, + json={ + "deployment_id": "dep-1", + "nonces": NONCES, + "expected": { + **{k: store.cells["eu-1"][k] for k in directory.CONTROL_FIELDS}, + "mutation_generation": str(UUID(int=1)), + }, + }, headers={"Authorization": f"Bearer {OPERATOR}"}, ) ) @@ -1337,3 +1437,17 @@ async def _run() -> None: assert again.json()["state"] == "serving" asyncio.run(_run()) + + +def test_native_compact_batch_with_control_envelope_fits_public_handler(client, store): + from server.cli.cell_attach import _encoded_size, batches + + _enforced(client) + orgs = [f"{i:03d}" + "x" * 127 for i in range(500)] + batch = batches(orgs, "dep-1")[0] + body = {"deployment_id": "dep-1", "orgs": batch} + assert _encoded_size(body) <= MAX_BODY_BYTES < len(json.dumps(body).encode()) + response = _call(client, "POST", "attach/place", body) + assert response.status_code == 200, response.text + assert response.json()["placed"] == len(batch) + assert set(store.placements) == set(batch) diff --git a/tests/unit/test_settings_overrides.py b/tests/unit/test_settings_overrides.py index e8eabb6761..e14c88c9c2 100644 --- a/tests/unit/test_settings_overrides.py +++ b/tests/unit/test_settings_overrides.py @@ -453,13 +453,18 @@ async def test_a_native_command_chunks_as_the_http_op_does_under_an_override( ) -> None: """`loom rechunk` and knowledge.upsert share `_chunk_text`: inside a native command it reads the stored override, not the startup default.""" + from server.core import retirement from server.insights.app import provider_command as pc from server.knowledge.app import writes async def _noop(*_a: Any, **_k: Any) -> bool: return True - for name in ("register_process", "load_active_configuration", "retire_process"): + async def _register() -> None: + retirement.get_gate().enrolled = True + + monkeypatch.setattr(pc, "register_process", _register) + for name in ("load_active_configuration", "_retire_until"): monkeypatch.setattr(pc, name, _noop) monkeypatch.setattr(pc, "drain_task_sets", _noop) monkeypatch.setattr(pc, "flush_registered_buffers", _noop) diff --git a/tests/unit/test_shutdown_deadline.py b/tests/unit/test_shutdown_deadline.py index e8d53b92f2..8f60ab0419 100644 --- a/tests/unit/test_shutdown_deadline.py +++ b/tests/unit/test_shutdown_deadline.py @@ -125,7 +125,7 @@ def close(self): pass # The parent measures from here to the process's exit, imports left out # (CLOCK_MONOTONIC is one clock for every process on the host). print("retiring", time.monotonic(), flush=True) -assert asyncio.run(barrier.retire_process()) +assert not asyncio.run(barrier.retire_process()) """ diff --git a/tests/unit/test_workers_child.py b/tests/unit/test_workers_child.py index 1f3c1854a6..57269fd3b2 100644 --- a/tests/unit/test_workers_child.py +++ b/tests/unit/test_workers_child.py @@ -12,6 +12,7 @@ import threading import time from types import SimpleNamespace +from unittest.mock import AsyncMock import pytest @@ -83,7 +84,7 @@ async def test_parent_sends_the_deadline_and_stops_the_child(monkeypatch, tmp_pa monkeypatch.setattr( child, "child_argv", lambda fd, _alive_fd: [sys.executable, "-c", _STUB, str(fd), str(out)] ) - workers = child.ChildWorkers(_Lease()) + workers = child.ChildWorkers(_Lease(), retire_child=AsyncMock()) workers.start_all() await _until(lambda: out.exists() and bool(out.read_text())) deadline = float(out.read_text()) @@ -96,7 +97,7 @@ async def test_parent_sends_the_deadline_and_stops_the_child(monkeypatch, tmp_pa def test_the_deadline_stops_moving_once_the_lease_is_not_fresh() -> None: lease = _Lease(remaining=12.0) - workers = child.ChildWorkers(lease) + workers = child.ChildWorkers(lease, retire_child=AsyncMock()) workers._send_deadline() sent = workers._deadline lease.remaining = 0.0 # lost @@ -111,7 +112,7 @@ async def test_a_child_that_ignores_sigterm_is_killed_at_the_deadline_plus_the_d ) -> None: ready = _stubborn(monkeypatch, tmp_path) lease = _Lease(remaining=0.3) - workers = child.ChildWorkers(lease) + workers = child.ChildWorkers(lease, retire_child=AsyncMock()) workers.start_all() await _until(ready.exists) lease.remaining = 0.0 # the lease is lost: the deadline stays where it was @@ -124,7 +125,7 @@ async def test_a_child_that_ignores_sigterm_is_killed_at_the_deadline_plus_the_d @pytest.mark.asyncio async def test_a_cancelled_stop_kills_the_child(monkeypatch, tmp_path) -> None: ready = _stubborn(monkeypatch, tmp_path) - workers = child.ChildWorkers(_Lease(remaining=30.0)) + workers = child.ChildWorkers(_Lease(remaining=30.0), retire_child=AsyncMock()) workers.start_all() await _until(ready.exists) proc = workers._proc @@ -139,7 +140,7 @@ async def test_a_cancelled_stop_kills_the_child(monkeypatch, tmp_path) -> None: async def test_a_child_that_exits_unasked_is_started_again(monkeypatch) -> None: monkeypatch.setattr(child, "RESTART_DELAY_S", 0.01) monkeypatch.setattr(child, "child_argv", lambda fd, _alive_fd: [sys.executable, "-c", "pass"]) - workers = child.ChildWorkers(_Lease()) + workers = child.ChildWorkers(_Lease(), retire_child=AsyncMock()) workers.start_all() await _until(lambda: workers.restarts >= 2) await workers.stop_all() @@ -153,7 +154,7 @@ async def test_a_child_that_keeps_failing_to_start_gives_the_lease_up(monkeypatc child, "child_argv", lambda fd, _alive_fd: [sys.executable, "-c", "exit(3)"] ) lease = _Lease() - workers = child.ChildWorkers(lease) + workers = child.ChildWorkers(lease, retire_child=AsyncMock()) workers.start_all() await _until(lambda: bool(lease.given_up)) assert workers.failed_starts == 3 and workers.restarts == 2 @@ -173,7 +174,7 @@ async def test_a_child_that_cannot_be_created_gives_the_lease_up_and_leaks_no_fd monkeypatch.setattr(child, "child_argv", lambda fd, _alive_fd: [missing]) fds = _open_fds() lease = _Lease() - workers = child.ChildWorkers(lease) + workers = child.ChildWorkers(lease, retire_child=AsyncMock()) workers.start_all() await _until(lambda: bool(lease.given_up)) assert workers.failed_starts == 3 and "could not start" in lease.given_up[0] @@ -204,7 +205,7 @@ async def test_a_child_whose_loop_hangs_is_not_alive(monkeypatch) -> None: monkeypatch.setattr( child, "child_argv", lambda _fd, alive_fd: [sys.executable, "-c", _HANGS, str(alive_fd)] ) - workers = child.ChildWorkers(_Lease()) + workers = child.ChildWorkers(_Lease(), retire_child=AsyncMock()) workers.start_all() await _until(lambda: workers._proc is not None) started = workers._beat_at @@ -304,12 +305,14 @@ def test_an_auto_holder_runs_its_workers_in_a_child(monkeypatch) -> None: lease = SupervisorLease("auto") monkeypatch.setattr(settings, "workers_process", "child") - assert isinstance(child.worker_runner(lease), child.ChildWorkers) + assert isinstance(child.worker_runner(lease, retire_child=AsyncMock()), child.ChildWorkers) assert child.current_runner() is child._RUNNER monkeypatch.setattr(settings, "workers_process", "inline") - assert isinstance(child.worker_runner(lease), WorkerSupervisor) + assert isinstance(child.worker_runner(lease, retire_child=AsyncMock()), WorkerSupervisor) monkeypatch.setattr(settings, "workers_process", "child") - assert isinstance(child.worker_runner(SupervisorLease("on")), WorkerSupervisor) + assert isinstance( + child.worker_runner(SupervisorLease("on"), retire_child=AsyncMock()), WorkerSupervisor + ) def test_the_child_runs_its_own_entry_point_not_the_serving_app() -> None: diff --git a/tests/unit/test_workers_child_retirement.py b/tests/unit/test_workers_child_retirement.py new file mode 100644 index 0000000000..bfa1829226 --- /dev/null +++ b/tests/unit/test_workers_child_retirement.py @@ -0,0 +1,273 @@ +"""The reaped worker child's original process identity gates retirement.""" + +import asyncio +import os +import signal +import time +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import UUID + +import pytest + +from server.core import provider_process +from server.insights.app import provider_barrier as barrier +from server.insights.store import provider_participants as store +from server.settings import Settings +from server.workers import child +from tests.unit.clock_fake import ManualClock + + +@pytest.mark.asyncio +async def test_actual_reap_certifies_its_exact_uncertified_child(monkeypatch): + workers, proc, proof, record, publish = await _owned(monkeypatch) + try: + await workers._reap(time.monotonic() + 10) + assert proc.returncode == -signal.SIGTERM and proc.waits >= 1 + assert record.await_count == 1 + evidence = record.await_args.args[0] + assert evidence.participant == UUID(int=17) + assert evidence.host_proof == proof and evidence.after == 9 + barrier.running_queries.assert_awaited_once_with(UUID(int=17), ch=None) + assert proc.signals == [signal.SIGTERM] + finally: + workers._close_pipe() + publish.cancel() + await asyncio.gather(publish, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_actual_reap_does_not_rewrite_an_already_certified_child(monkeypatch): + workers, proc, _, record, publish = await _owned(monkeypatch, fault="already_retired") + try: + await workers._reap(time.monotonic() + 10) + assert proc.returncode == -signal.SIGTERM + assert not record.await_count + finally: + workers._close_pipe() + publish.cancel() + await asyncio.gather(publish, return_exceptions=True) + + +@pytest.mark.parametrize( + "fault", + [ + "missing", + "duplicate", + "wrong_pid", + "reused_pid_proof", + "different_namespace", + "own_uuid", + "alive", + "queries_running", + "queries_unknown", + "query_error", + "birth_unknown", + "birth_other_parent", + "birth_wrong_namespace", + "parent_changed", + ], +) +@pytest.mark.asyncio +async def test_reap_retirement_refuses_unknown_or_unsettled_identity(monkeypatch, fault): + workers, proc, _, record, publish = await _owned(monkeypatch, fault=fault) + try: + await workers._reap(time.monotonic() + 10) + assert proc.returncode == -signal.SIGTERM + assert not record.await_count + finally: + workers._close_pipe() + publish.cancel() + await asyncio.gather(publish, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_a_live_owned_child_cannot_be_certified_before_its_wait(monkeypatch): + workers, proc, _, record, publish = await _owned(monkeypatch) + try: + await workers._retire_child(proc, time.monotonic() + 10) + assert proc.returncode is None and proc.waits == 0 + assert not record.await_count + finally: + workers._close_pipe() + publish.cancel() + await asyncio.gather(publish, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_exhausted_original_stop_budget_refuses_retirement(monkeypatch): + workers, proc, _, record, publish = await _owned(monkeypatch) + try: + await workers._reap(time.monotonic() - 1) + assert proc.returncode is not None + assert not record.await_count + finally: + workers._close_pipe() + publish.cancel() + await asyncio.gather(publish, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_exact_proof_participant_read_is_bounded_and_parameterized(monkeypatch): + client = SimpleNamespace(query_all=AsyncMock(return_value=[])) + proof = "host|boot|container|77|123|901|||cell-retirement-v1" + assert await store.participants(ch=client, host_proof=proof) == [] + sql, params = client.query_all.await_args.args + assert sql == store.PARTICIPANTS_SQL + " AND host_proof = {proof:String} LIMIT 2" + assert params == {"proof": proof} + assert proof not in sql + + +async def _owned(monkeypatch, *, fault=None): + parent = os.getpid() + pid = parent + 100000 + here = f"host|boot|container|77|{parent}|900||" + proof = f"host|boot|container|77|{pid}|901|||cell-retirement-v1" + rows = [{"participant_id": UUID(int=17), "host_proof": proof, "state": "drained", "version": 9}] + if fault in {"missing", "already_retired"}: + rows = [] + elif fault == "duplicate": + rows *= 2 + elif fault in {"wrong_pid", "reused_pid_proof", "different_namespace"}: + parts = proof.split("|") + parts[{"wrong_pid": 4, "reused_pid_proof": 5, "different_namespace": 3}[fault]] = "999" + rows[0]["host_proof"] = "|".join(parts) + elif fault == "own_uuid": + rows[0]["participant_id"] = UUID(int=18) + participants = AsyncMock(return_value=rows) + monkeypatch.setattr(store, "participants", participants) + monkeypatch.setattr(barrier, "participants", participants) + monkeypatch.setattr(barrier, "process_identity", lambda: UUID(int=18)) + monkeypatch.setattr(barrier, "proves_stopped", lambda _: fault != "alive") + monkeypatch.setattr(provider_process, "proves_stopped", lambda _: fault != "alive") + monkeypatch.setattr(provider_process, "host_proof", lambda: here) + queries = AsyncMock( + return_value=None if fault == "queries_unknown" else int(fault == "queries_running") + ) + if fault == "query_error": + queries.side_effect = RuntimeError("unavailable") + monkeypatch.setattr(barrier, "running_queries", queries) + record = AsyncMock() + monkeypatch.setattr(barrier, "record_retirement", record) + original_read, original_stat = Path.read_text, os.stat + fields = ["S", str(parent)] + ["0"] * 18 + fields[19] = "901" + if fault == "birth_other_parent": + fields[1] = str(parent + 1) + + def read(path, *args, **kwargs): + if path == Path(f"/proc/{pid}/stat"): + if fault == "birth_unknown": + raise PermissionError("uninspectable") + return f"{pid} (python) " + " ".join(fields) + return original_read(path, *args, **kwargs) + + def stat(path, *args, **kwargs): + if str(path) == f"/proc/{pid}/ns/pid": + return SimpleNamespace(st_ino=78 if fault == "birth_wrong_namespace" else 77) + return original_stat(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", read) + monkeypatch.setattr(os, "stat", stat) + + class Process: + returncode = None + waits = 0 + + def __init__(self): + self.pid = pid + self.signals = [] + + def send_signal(self, sig): + self.signals.append(sig) + + def kill(self): + self.returncode = -signal.SIGKILL + + async def wait(self): + self.waits += 1 + self.returncode = -signal.SIGTERM if self.returncode is None else self.returncode + return self.returncode + + proc = Process() + monkeypatch.setattr(child.asyncio, "create_subprocess_exec", AsyncMock(return_value=proc)) + lease = SimpleNamespace(fresh=lambda: True, remaining_s=lambda: 10) + workers = child.ChildWorkers(lease, retire_child=barrier.retire_reaped_process) + await workers._spawn() + if fault == "parent_changed": + monkeypatch.setattr( + provider_process, "host_proof", lambda: f"host|boot|container|77|{parent + 1}|902||" + ) + return workers, proc, proof, record, workers._publish + + +@pytest.mark.asyncio +async def test_naturally_exited_owned_child_is_retired_within_restart_delay(monkeypatch): + workers, _, proof, record, publish = await _owned(monkeypatch) + clock = [time.monotonic()] + sleeps = [] + + class StopLoop(Exception): + pass + + async def record_with_elapsed_proof(_, *, ch=None): + clock[0] += 0.4 + + async def stop_after_restart_delay(delay): + sleeps.append(delay) + raise StopLoop + + record.side_effect = record_with_elapsed_proof + monkeypatch.setattr(workers, "_spawn", AsyncMock()) + monkeypatch.setattr(child, "time", SimpleNamespace(monotonic=lambda: clock[0])) + monkeypatch.setattr(child.asyncio, "sleep", stop_after_restart_delay) + try: + with pytest.raises(StopLoop): + await workers._keep_running() + assert record.await_count == 1 + assert record.await_args.args[0].host_proof == proof + assert sleeps == [pytest.approx(child.RESTART_DELAY_S - 0.4)] + assert workers.restarts == 1 + finally: + workers._close_pipe() + publish.cancel() + await asyncio.gather(publish, return_exceptions=True) + + +@pytest.mark.parametrize("value", [0.0, -1.0, float("nan"), float("inf"), -float("inf")]) +def test_child_retirement_proof_timeout_requires_a_positive_finite_setting(value): + with pytest.raises(ValueError): + Settings(_env_file=None, workers_child_retirement_timeout_s=value) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("configured", "remaining", "expected"), + [(0.125, 2.0, 0.125), (1.0, 0.25, 0.25), (2.0, 0.0625, 0.0625)], +) +async def test_child_retirement_setting_caps_only_the_original_remaining_allowance( + monkeypatch, configured, remaining, expected +): + workers, proc, _, record, publish = await _owned(monkeypatch) + clock = ManualClock(t=100.0) + allowances = [] + original_timeout = asyncio.timeout + + def observed_timeout(delay): + allowances.append(delay) + return original_timeout(delay) + + monkeypatch.setattr(child.settings, "workers_child_retirement_timeout_s", configured) + monkeypatch.setattr(child, "time", clock) + monkeypatch.setattr(child.asyncio, "timeout", observed_timeout) + try: + await workers._reap(clock.monotonic() + remaining) + assert proc.returncode == -signal.SIGTERM and proc.waits >= 1 + assert record.await_count == 1 + assert allowances == [expected] + assert clock.monotonic() == 100.0 + finally: + workers._close_pipe() + publish.cancel() + await asyncio.gather(publish, return_exceptions=True)