From bb8e0412be154087030ecac7cc5c37f7bfbbca6a Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 04:34:38 +0000 Subject: [PATCH 001/114] loom cell attach turns a prepared first cell into a serving one; prepare takes the same lock The attach command holds the attach lock on a direct, marked session of the cell's Postgres for its run and resumes by the directory row and the cell's mode. It refuses before any write on the stores' identity, the session, the roles, settings and subscriptions that would let an unchecked writer through, the database encoding, durability, its own grants, the ClickHouse writers, the table list and the processes' heartbeats; posts attach/begin; creates the triggers under the migration lock; in recording scans and writes the tenancy rows, refusing an org past ORG_MAX_BYTES; commits enforced, proves it durable, waits out every transaction that could have read recording, rescans, places every org in batches and marks the cell serving. A refusal after the mode commit leaves the cell enforced and attaching, and a rerun resumes there. Prepare refuses a session behind a pooler or a database that is not UTF8, and takes the attach lock. Doctor runs attach's Postgres reads again and reads stray orgs with --stray-orgs. instances.read_recent keeps stopped rows inside the widened window, and PRUNE_AFTER_S names the sweep's hour. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- server/cli/cell.py | 4 +- server/cli/cell_attach.py | 756 +++++++++++++++++++ server/cli/cell_attach_beats.py | 324 ++++++++ server/cli/cell_attach_pg.py | 690 +++++++++++++++++ server/cli/cell_prepare.py | 42 +- server/cli/cell_scan.py | 50 +- server/cli/doctor.py | 47 +- server/cli/doctor_cell.py | 166 +++- server/core/cell_triggers.py | 139 ++-- server/core/instances.py | 57 +- tests/integration/test_cell_attach.py | 635 ++++++++++++++++ tests/integration/test_cell_scan.py | 2 +- tests/unit/test_cell_attach.py | 682 +++++++++++++++++ tests/unit/test_cell_org_triggers.py | 2 +- tests/unit/test_cell_prepare.py | 3 + tests/unit/test_doctor_client_checks.py | 2 +- tests/unit/test_doctor_client_client_only.py | 2 +- 17 files changed, 3529 insertions(+), 74 deletions(-) create mode 100644 server/cli/cell_attach.py create mode 100644 server/cli/cell_attach_beats.py create mode 100644 server/cli/cell_attach_pg.py create mode 100644 tests/integration/test_cell_attach.py create mode 100644 tests/unit/test_cell_attach.py diff --git a/server/cli/cell.py b/server/cli/cell.py index 9cdec5c894..65f18990d8 100644 --- a/server/cli/cell.py +++ b/server/cli/cell.py @@ -1,7 +1,7 @@ """`loom cell credential | limit | which | remove`: the operator's commands that touch the router's directory (docs/architecture/cells.md, "Operations"). Only `remove` reads a cell's stores, a first cell's Postgres; the others run while a cell is down. `loom cell prepare` is -`server/cli/cell_prepare.py`. +`server/cli/cell_prepare.py`, `loom cell attach` `server/cli/cell_attach.py`. - `credential new` makes a router credential offline and prints it with its SHA-256; `credential --cell --next | --finish | --revert` rotates a cell's. @@ -459,6 +459,8 @@ def _add_cell_parser(sub: Any) -> None: remove.add_argument("--router", help="The router's URL (default: LOOM_CELL_ROUTER_URL)") remove.set_defaults(func=_cmd_remove) + from server.cli.cell_attach import add_attach_parser from server.cli.cell_prepare import add_prepare_parser add_prepare_parser(csub) + add_attach_parser(csub) diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py new file mode 100644 index 0000000000..368f74ebf8 --- /dev/null +++ b/server/cli/cell_attach.py @@ -0,0 +1,756 @@ +"""`loom cell attach`: turn a prepared first cell into a `serving` one +(docs/architecture/cells.md, "Attaching a cell"; the runbook is +docs/operations/cells.md). + +A run holds the attach lock on a direct session of the cell's Postgres for its life, and +goes, each step idempotent, so a rerun resumes by the directory row and the cell's mode: + +1. the directory row: on `serving` it is done (exit 0) once the cell reads `enforced`; +2. the refusals, before any write: the stores are the registered ones, the session is + direct and marked, the cell's processes (their heartbeats), the roles, settings and + subscriptions that would let an unchecked writer through, the ClickHouse writers, + the table list, durability, attach's own grants; +3. `attach/begin` (the router probes the stores and writes the fingerprints); +4. the mode: `recording`, or `enforced` on a resume; +5. the triggers (`cell_triggers.ensure_table`), under the migration lock one try at a time; +6. in `recording`, step 2's scan, refusing an org past `cell.ORG_MAX_BYTES`, and the + tenancy rows of the orgs found; the triggers again; the pre-commit checks; +7. step 3: `attach/enforced`, then the mode commit (never on a resume): from it on the mode + only rises, so every later refusal leaves the cell `enforced` and `attaching`; +8. the durable read of `enforced`, the heartbeats, step 4's wait for every transaction + that could have read `recording`, the rescan, the tenancy rows and their placements + (`attach/place`), the final checks and `attach/serving`. + +`--wait` bounds the waits, `--scan-budget` each full scan. `--dry-run` runs the refusals +and the scan and prints what would be placed, writing nothing. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sys +import time +from collections.abc import Callable, Iterable +from typing import Any + +import httpx + +from server.cli import cell as cell_cli +from server.cli import cell_attach_beats as beats +from server.cli import cell_attach_pg as apg +from server.cli import cell_scan +from server.cli._ch import writes +from server.cli.cell_ch import Refusal, Refused, read_topology +from server.core import cell, cell_triggers +from server.settings import settings + +_WAIT_S = 600 # not a setting: a flag's default +_SCAN_BUDGET_S = 7200 # not a setting: a flag's default +_LOCK_STATEMENT_MS = 60_000 # not a setting: a bound on the lock session's catalog reads +_HEALTH_TIMEOUT = httpx.Timeout(10.0, connect=5.0) # not a setting: one health read + + +def _args_problem(args: argparse.Namespace) -> str: + if not cell.NAME.fullmatch(args.cell or ""): + return f"{args.cell!r} is no cell name (1 to 32 of a-z, 0-9, -)" + if args.cell != settings.cell: + return f"LOOM_CELL is {settings.cell!r}: attach runs with LOOM_CELL={args.cell}" + if settings.storage_authority != "postgres" or not settings.pg_dsn: + return ( + "a cell keeps its state in Postgres: set LOOM_STORAGE_AUTHORITY=postgres and " + "LOOM_PG_DSN" + ) + if args.wait < 0 or args.scan_budget <= 0: + return "--wait is 0 or more seconds, --scan-budget more than 0" + return "" + + +class Router: + """The router's operator API for one cell.""" + + def __init__( + self, url: httpx.URL, name: str, transport: httpx.AsyncBaseTransport | None + ) -> None: + self.url, self.name, self.transport = url, name, transport + + def _client(self) -> httpx.AsyncClient: + return httpx.AsyncClient( + timeout=cell_cli._OPERATOR_TIMEOUT, + trust_env=self.url.scheme == "https", + transport=self.transport, + headers={"Authorization": f"Bearer {settings.router_operator_token}"}, + ) + + async def call(self, path: str, body: dict[str, Any] | None = None) -> dict[str, Any]: + """The router's 200 answer naming the cell; Refused with its refusal otherwise.""" + url = cell_cli.at(self.url, f"/cells/v1/operator/cells/{self.name}" + path) + try: + async with self._client() as client: + r = await (client.get(url) if body is None else client.post(url, json=body)) + except httpx.HTTPError as e: + raise _refused("router", f"no answer ({type(e).__name__}): rerun attach") from None + answer = cell_cli.router_answer(r, {"cell": self.name}, cell_cli.OPERATOR_REFUSALS) + if answer is None: + raise _refused("router", cell_cli.not_the_router(r)) + if r.status_code == 404 and answer.get("code") != "NO_SUCH_CELL": + raise _refused("router", f"a router replica is not upgraded: it does not answer {url}") + if r.status_code != 200: + raise _refused( + "router", f"refused {path or 'the read'} ({r.status_code}): {answer['error']}" + ) + return answer + + async def health(self, host: str) -> str: + """The deployment id `https:///v1/health/deployment` answers ('' when none).""" + try: + async with httpx.AsyncClient( + timeout=_HEALTH_TIMEOUT, transport=self.transport + ) as client: + r = await client.get(f"https://{host}/v1/health/deployment") + answer = r.json() + except (httpx.HTTPError, ValueError): + return "" + return str(answer.get("deployment_id") or "") if isinstance(answer, dict) else "" + + +def _too_long(found: cell_scan.ScanResult) -> list[Refusal]: + """Step 2's refusal of every org past `cell.ORG_MAX_BYTES`, before any tenancy row.""" + return [ + Refusal( + f"org of {len(org.encode())} bytes ({org[:64]}…)", + f"found in {', '.join(sorted(sources))}: longer than {cell.ORG_MAX_BYTES} bytes; " + "remove it first (docs/operations/cells.md, the over-long org procedure)", + ) + for org, sources in sorted(found.too_long.items()) + ] + + +def _refused(subject: str, reason: str) -> Refused: + return Refused([Refusal(subject, reason)]) + + +def _encoded_size(body: Any) -> int: + return len(json.dumps(body, ensure_ascii=True, separators=(",", ":")).encode()) + + +def batches(orgs: Iterable[str], deployment_id: str) -> list[list[str]]: + """`orgs` in `attach/place` batches: at most ATTACH_BATCH each, and an encoded body + within the router's body bound.""" + from server.router.limits import ATTACH_BATCH, MAX_BODY_BYTES + + out: list[list[str]] = [] + batch: list[str] = [] + for org in sorted(orgs): + if batch and ( + len(batch) >= ATTACH_BATCH + or _encoded_size({"deployment_id": deployment_id, "orgs": [*batch, org]}) + > MAX_BODY_BYTES + ): + out.append(batch) + batch = [] + batch.append(org) + if batch: + out.append(batch) + return out + + +class Run: + """One attach run: its sessions, what it read, and what it prints.""" + + def __init__( + self, + ch: Any, + args: argparse.Namespace, + router: Router, + log: Callable[[str], None], + ) -> None: + self.ch, self.args, self.router, self.log = ch, args, router, log + self.name = args.cell + self.wait = apg.Wait(args.wait) + self.lock_conn: Any = None + self._lock_cm: Any = None + self.row: dict[str, Any] = {} + self.mode = "" + self.own_fp: tuple[dict[str, Any] | None, dict[str, Any] | None] = (None, None) + self.keys_fp = "" + self.report: dict[str, Any] = {"cell": self.name} + + # -- the lock session -------------------------------------------------------------- + + async def open_lock(self) -> None: + from server.core import pg + + self._lock_cm = pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) + self.lock_conn = await self._lock_cm.__aenter__() + if refusals := await apg.direct_refusals(self.lock_conn, "attach"): + raise Refused(refusals) + await apg.take_lock(self.lock_conn, "attach") + + async def close_lock(self) -> None: + if self._lock_cm is not None: + await self._lock_cm.__aexit__(None, None, None) + self._lock_cm = self.lock_conn = None + + async def confirm_lock(self) -> None: + """The lock is held still, or taken again (refusing when another run holds it now), + and the cell's state read again: a cleanup, a prepare rerun or another attach could + have run in the gap.""" + try: + if await apg.lock_held(self.lock_conn): + return + except Exception as e: # noqa: BLE001 — a lost session is taken again below + self.log(f"the lock session was lost ({type(e).__name__}); taking the lock again") + else: + self.log("the attach lock was lost; taking it again") + try: + await self.close_lock() + except Exception: # noqa: BLE001 — the old session is gone either way + self._lock_cm = self.lock_conn = None + await self.open_lock() + state, first = await apg.read_mode(self.lock_conn) + if first != (True, self.name) or (self.mode == "enforced" and state != "enforced"): + raise _refused( + "cell state", + f"changed while the lock was lost (now {state}, first row {first}): rerun attach", + ) + row = await self.router.call("") + if ( + row.get("deployment_id") != self.row.get("deployment_id") + or row.get("state") != "attaching" + ): + raise _refused("directory row", "changed while the lock was lost: rerun attach") + + # -- the refusals ------------------------------------------------------------------ + + async def read_own_row(self) -> None: + own = await self.lock_conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") + if own is None: + raise _refused("postgres loom.cell_self", "absent: run loom cell prepare first") + self.own_fp = (cell.json_object(own["current_fp"]), cell.json_object(own["next_fp"])) + + def judge_beats(self, stopped_unset: str) -> Callable[[list[beats.Beat]], beats.Verdict]: + def _judge(found: list[beats.Beat]) -> beats.Verdict: + if stopped_unset == "ignore": + found = [b for b in found if b.live or b.cell == self.name] + return beats.judge( + found, self.name, self.own_fp, "wait" if stopped_unset == "wait" else "refuse" + ) + + return _judge + + async def check_beats(self, stopped_unset: str) -> beats.Verdict: + verdict = await beats.settle( + lambda: beats.read_beats(self.ch), self.judge_beats(stopped_unset), self.wait + ) + (self.keys_fp,) = verdict.keys_fps + return verdict + + async def stores_refusals(self) -> list[Refusal]: + from server.core import pg_profile + + out = [] + row = self.row + if not row.get("is_first") or row.get("state") != "attaching": + out.append( + Refusal( + "directory row", + f"is {'the first' if row.get('is_first') else 'a later'} cell, " + f"{row.get('state')}: this attach is the first cell's, while attaching", + ) + ) + if row.get("other"): + out.append( + Refusal( + "directory row", + f"another cell, {row['other']}, is registered: remove it first " + "(loom cell remove)", + ) + ) + expected = str(row.get("deployment_id") or "") + recorded_pg = await self.lock_conn.fetchval("SELECT deployment_id FROM loom.authority") + if recorded_pg != expected: + out.append( + Refusal( + "postgres", f"records deployment id {recorded_pg!r}, the directory {expected!r}" + ) + ) + recorded = await pg_profile.recorded_authority(self.ch) + if recorded is None or str(recorded["deployment_id"]) != expected: + out.append(Refusal("clickhouse", f"records another deployment id than {expected!r}")) + state, first = await apg.read_mode(self.lock_conn) + if first != (True, self.name): + out.append( + Refusal("postgres loom.cell_first", f"reads {first}, not (true, {self.name})") + ) + if state not in ("recording", "enforced"): + out.append(Refusal("postgres cell state", f"reads {state}")) + answered = await self.router.health(str(row.get("host") or "")) + if answered != expected: + out.append( + Refusal( + f"https://{row.get('host')}/v1/health/deployment", + f"answers deployment id {answered!r}, not {expected!r}", + ) + ) + return out + + async def durability_refusals(self) -> list[Refusal]: + from server.core import pg, pg_profile + + out = [] + verdict = pg_profile.durability_verdict("synchronous", *await pg_profile.standby_state()) + if verdict.refusal: + out.append(Refusal("postgres durability", verdict.refusal)) + try: + await pg.check_durable_message() + except Exception as e: # noqa: BLE001 — a role that may not write it refuses + out.append(Refusal("postgres durability", f"pg_logical_emit_message refused: {e}")) + if (now := await pg_profile.standby_now()) != "synchronous": + out.append( + Refusal("postgres standbys", f"commits are acknowledged as {now}, not synchronous") + ) + return out + + async def ch_limit_refusals(self) -> list[Refusal]: + """A profile or quota that caps the rows or bytes attach's user reads would end a + scan early.""" + row = await self.ch.query_one( + "SELECT getSetting('max_rows_to_read') AS r, getSetting('max_bytes_to_read') AS b" + ) + out = [] + if row and (int(row["r"]) or int(row["b"])): + out.append( + Refusal("clickhouse user", "a settings profile caps the rows or bytes it reads") + ) + quota = await self.ch.query_all( + "SELECT quota_name FROM system.quota_usage " + "WHERE max_read_rows > 0 OR max_read_bytes > 0" + ) + out += [ + Refusal(f"clickhouse quota {q['quota_name']}", "caps the rows or bytes attach reads") + for q in quota + ] + return out + + async def writer_refusals(self) -> tuple[list[Refusal], Any, Any]: + from server.cli import cell_writers + + topology = await read_topology(self.ch) + if refusals := topology.refusals(): + return refusals, topology, {} + names = await cell_writers.host_names(self.ch, topology) if topology.multi else {} + allowed = cell_writers.Allowed( + users=frozenset(self.args.ch_allow_user), + user_directories=frozenset(self.args.ch_allow_user_directory), + objects=frozenset(self.args.ch_allow_object), + replicas=frozenset(self.args.ch_allow_replica), + ) + return await cell_writers.check_writers(self.ch, topology, names, allowed), topology, names + + async def catalog_refusals(self) -> list[Refusal]: + """The Postgres checks that run again before the mode commit and in step 4.""" + holders = await apg.privilege_holders(self.lock_conn) + self.report["own_roles"] = holders.own + out = apg.privilege_refusals(holders, self.args.pg_allow_role) + out += await apg.unmarked_refusals(self.lock_conn, await apg.session_passes()) + return out + + async def refusals(self) -> tuple[Any, Any, cell_scan.TableList]: + """Every refusal before any write, named together.""" + out = await self.stores_refusals() + out += await apg.encoding_refusals(self.lock_conn) + out += await self.catalog_refusals() + out += await self.durability_refusals() + rows = await self.lock_conn.fetch(cell_triggers.ORG_TABLES) + tables_pg = [f"loom.{r['relname']}" for r in rows] + out += await apg.own_role_refusals(self.lock_conn, tables_pg) + out += await self.ch_limit_refusals() + if out: + raise Refused(out) + state, _ = await apg.read_mode(self.lock_conn) + # A stopped process with LOOM_CELL unset refuses while the cell can still be + # backed out; on a resume in `enforced` step 4 waits for it to age out. + await self.check_beats("refuse" if state == "recording" else "ignore") + writers, topology, names = await self.writer_refusals() + out += writers + listed = await cell_scan.list_tables(self.ch, self.lock_conn, topology, names) + out += listed.refusals + if out: + raise Refused(out) + return topology, names, listed + + # -- begin ------------------------------------------------------------------------- + + async def begin(self) -> None: + from server.cli.cell_prepare import _delete_nonces, _new_nonces, _write_nonces + from server.router.limits import ATTACH_BEGIN_MAX_BYTES + + current, nxt = self.own_fp + nonces = _new_nonces() + body = { + "deployment_id": self.row["deployment_id"], + "current_fp": current, + "next_fp": nxt, + "keys_fp": self.keys_fp, + "ch_allow_users": sorted(self.args.ch_allow_user), + "ch_allow_user_directories": sorted(self.args.ch_allow_user_directory), + "ch_allow_objects": sorted(self.args.ch_allow_object), + "ch_allow_replicas": sorted(self.args.ch_allow_replica), + "pg_allow_roles": sorted(self.args.pg_allow_role), + "nonces": nonces, + } + if (size := _encoded_size(body)) > ATTACH_BEGIN_MAX_BYTES: + raise _refused( + "attach/begin", + f"its body is {size} bytes, past the router's {ATTACH_BEGIN_MAX_BYTES}: the " + "fingerprints or the allow lists are too large", + ) + try: + await _write_nonces(self.ch, nonces) + self.row = {**self.row, **await self.router.call("/attach/begin", body)} + finally: + await _delete_nonces(self.ch, nonces) + + # -- the scans --------------------------------------------------------------------- + + async def scan(self, topology: Any, listed: cell_scan.TableList) -> cell_scan.ScanResult: + budget = cell_scan.Budget(self.args.scan_budget) + current, nxt = self.own_fp + return await cell_scan.scan( + self.ch, listed, topology, budget, cell_scan.configured_orgs(current, nxt) + ) + + def _named(self, found: cell_scan.ScanResult) -> dict[str, Any]: + return { + "malformed": {k: v[:20] for k, v in sorted(found.malformed.items())}, + "too_long": { + f"{o[:64]}… ({len(o.encode())} bytes)": sorted(s) for o, s in found.too_long.items() + }, + } + + async def triggers(self) -> None: + if refusals := await apg.ensure_triggers(self.lock_conn, self.wait, self.log): + raise Refused(refusals) + + # -- step 3 ------------------------------------------------------------------------ + + async def ready_for_commit(self) -> bool: + """The cheap heartbeat refusals right before a mode-commit try: a refusal raises, + a booting process skips the try.""" + verdict = self.judge_beats("refuse")(await beats.read_beats(self.ch)) + if verdict.refusals: + raise Refused(verdict.refusals) + return not verdict.booting + + async def commit_mode(self) -> None: + from server.core import pg + + await self.router.call("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) + try: + async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: + await apg.commit_enforced( + conn, self.confirm_lock, self.ready_for_commit, self.wait, self.log + ) + except Refused: + raise + except Exception as e: # noqa: BLE001 — the durable read says whether it landed + self.log(f"the mode commit's outcome is unknown ({type(e).__name__}: {e})") + async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: + await apg.end_mode_transactions(conn, self.confirm_lock) + + async def durable(self) -> int: + """The durable read of `enforced`, retried within `--wait`; then X, the snapshot's + xmax, above the read's own xid.""" + from server.core import pg_profile + + while True: + try: + if await pg_profile.standby_now() != "synchronous": + raise _refused("postgres standbys", "not synchronous before the durable read") + read = await apg.durable_read() + if await pg_profile.standby_now() != "synchronous": + raise _refused("postgres standbys", "not synchronous after the durable read") + break + except Refused: + raise + except Exception as e: + self.log(f"the durable read failed ({type(e).__name__}); retrying") + if not await self.wait.pause(): + raise _refused("postgres durable read", "did not commit within --wait") from e + if read.first != (True, self.name) or read.mode != "enforced": + raise _refused( + "postgres cell state", + f"the durable read saw mode {read.mode!r} and first row {read.first}: the mode " + "commit did not land; rerun attach", + ) + self.mode = "enforced" + xmax = await apg.snapshot_xmax(self.lock_conn) + if xmax <= read.xid: + raise _refused("postgres snapshot", f"its xmax {xmax} is not above {read.xid}") + return xmax + + # -- step 4 ------------------------------------------------------------------------ + + async def cheap_checks(self, stopped_unset: str) -> None: + from server.core import pg_profile + + out, _, _ = await self.writer_refusals() + out += [ + Refusal("postgres triggers", d.text) + for d in await cell_triggers.missing(self.lock_conn) + ] + out += await self.catalog_refusals() + if out: + raise Refused(out) + await self.check_beats(stopped_unset) + if (now := await pg_profile.standby_now()) != "synchronous": + raise _refused("postgres standbys", f"commits are acknowledged as {now}") + + async def final_checks(self, watch: beats.Watch) -> None: + from server.core import instances + + floor = apg.Wait(max(self.wait.left(), instances.boot_allowance_s())) + + async def _read() -> list[beats.Beat]: + found = await watch.read_once() + if refusals := watch.refusals(): + raise Refused(refusals) + # A cell-correct process whose cell row alone was swept. + return [b for b in found if not (b.cell is None and b.instance_id in watch.cell_ids)] + + verdict = await beats.settle( + _read, self.judge_beats("refuse"), floor, extend_s=instances.boot_allowance_s() + ) + if verdict.keys_fps != {self.keys_fp}: + raise _refused( + "processes", + "the root-secret fingerprint changed during attach (a root-secret rotation): " + "rerun attach", + ) + + async def step4(self, xmax: int, topology: Any, names: Any) -> None: + from server.core import pg + + async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: + await apg.wait_transactions(conn, xmax, self.wait) + await self.cheap_checks("wait") + since = str((await self.ch.query_one("SELECT toString(now64(3)) AS t"))["t"]) + watch = beats.Watch(self.ch, since, self.name) + if refusals := watch.refusals_up_front(): + raise Refused(refusals) + task = asyncio.create_task(watch.run()) + try: + listed = await cell_scan.list_tables(self.ch, self.lock_conn, topology, names) + found = await self.scan(topology, listed) + orgs = set(found.orgs) + self.report["tenancy_rows_written"] = await apg.write_tenancy(orgs) + placed = 0 + for batch in batches(orgs, self.row["deployment_id"]): + await self.confirm_lock() + answer = await self.router.call( + "/attach/place", {"deployment_id": self.row["deployment_id"], "orgs": batch} + ) + placed += int(answer.get("placed") or 0) + self.report.update(orgs_found=len(orgs), placed_now=placed, skipped=self._named(found)) + writers, _, _ = await self.writer_refusals() + if writers: + raise Refused(writers) + finally: + task.cancel() + await self.final_checks(watch) + row = await self.router.call("") + current, nxt = self.own_fp + if (row.get("current_fp"), row.get("next_fp")) != (current, nxt): + raise _refused( + "postgres loom.cell_self", + "differs from the directory's fingerprints (a loom cell update between them?)", + ) + state, _ = await apg.read_mode(self.lock_conn) + if state != "enforced": + raise _refused("postgres cell state", f"reads {state}, not enforced") + + async def serving(self) -> dict[str, Any]: + from server.cli.cell_prepare import _delete_nonces, _new_nonces, _write_nonces + + nonces = _new_nonces() + try: + await _write_nonces(self.ch, nonces) + return await self.router.call( + "/attach/serving", {"deployment_id": self.row["deployment_id"], "nonces": nonces} + ) + finally: + await _delete_nonces(self.ch, nonces) + + # -- the run ----------------------------------------------------------------------- + + async def done(self) -> bool: + """On a `serving` row: True once the cell reads `enforced`; Refused otherwise.""" + from server.core import pg + + async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: + state, _ = await apg.read_mode(conn) + if state != "enforced": + raise _refused( + "postgres cell state", + f"the directory says serving and the cell reads {state}: run loom cell restore", + ) + return True + + async def run(self) -> int: + self.row = await self.router.call("") + if self.row.get("state") == "serving": + await self.done() + self.report.update(state="serving", done=True) + return 0 + await self.open_lock() + try: + await self.read_own_row() + topology, names, listed = await self.refusals() + if self.args.dry_run: + return await self.dry_run(topology, listed) + await self.begin() + state, _ = await apg.read_mode(self.lock_conn) + self.mode = state + await self.triggers() + if state == "recording": + found = await self.scan(topology, listed) + if found.too_long: + raise Refused(_too_long(found)) + written = await apg.write_tenancy(found.orgs) + self.log(f"step 2: {len(found.orgs)} orgs found, {written} tenancy rows written") + await self.triggers() + await self.precommit(found) + await self.commit_mode() + else: + await self.triggers() + await self.router.call( + "/attach/enforced", {"deployment_id": self.row["deployment_id"]} + ) + xmax = await self.durable() + await self.check_beats("ignore") + await self.step4(xmax, topology, names) + answer = await self.serving() + self.report.update(state=answer.get("state"), placed_orgs=answer.get("placed_orgs")) + return 0 + finally: + await self.close_lock() + + async def precommit(self, found: cell_scan.ScanResult) -> None: + from server.core import pg_profile + + out = await self.catalog_refusals() + if (now := await pg_profile.standby_now()) != "synchronous": + out.append(Refusal("postgres standbys", f"commits are acknowledged as {now}")) + if found.elapsed_s > self.args.scan_budget / 2: + out.append( + Refusal( + "--scan-budget", + f"step 2 took {found.elapsed_s:.0f} s, more than half of " + f"{self.args.scan_budget} s, and step 4 reads the same tables again: " + "raise it", + ) + ) + if out: + raise Refused(out) + verdict = await self.check_beats("refuse") + if refusals := beats.final_refusals(verdict): + raise Refused(refusals) + + async def dry_run(self, topology: Any, listed: cell_scan.TableList) -> int: + found = await self.scan(topology, listed) + per_source: dict[str, list[str]] = {} + for org, sources in found.orgs.items(): + for s in sources: + per_source.setdefault(s, []).append(org) + self.report.update( + dry_run=True, + scan_seconds=round(found.elapsed_s, 1), + half_budget_ok=found.elapsed_s <= self.args.scan_budget / 2, + orgs=len(found.orgs), + per_source={ + s: {"count": len(o), "sample": sorted(o)[:5]} for s, o in sorted(per_source.items()) + }, + skipped=self._named(found), + ) + return 0 + + +async def attach( + ch: Any, + args: argparse.Namespace, + url: httpx.URL, + transport: httpx.AsyncBaseTransport | None = None, +) -> int: + def log(text: str) -> None: + print(f"loom cell attach: {text}", file=sys.stderr) + + run = Run(ch, args, Router(url, args.cell, transport), log) + started = time.monotonic() + try: + code = await run.run() + except Refused as e: + for r in e.refusals: + log(f"refusing: {r}") + mode = ( + "enforced, attaching: rerun attach to resume" + if run.mode == "enforced" + else ("unchanged" if not run.mode else f"{run.mode}") + ) + log(f"the cell stays {mode}") + return 1 + run.report["seconds"] = round(time.monotonic() - started, 1) + print(json.dumps(run.report, indent=2, sort_keys=True, default=str)) + return code + + +def cmd_attach(args: argparse.Namespace) -> int: + if problem := _args_problem(args): + print(f"loom cell attach: {problem}", file=sys.stderr) + return 2 + url = cell_cli.operator_router(args) + if isinstance(url, str): + print(f"loom cell attach: {url}", file=sys.stderr) + return 2 + from server.cli._ch import run_connected + from server.core.ch import statement_timeout + + async def _run(ch: Any) -> int: + try: + return await attach(ch, args, url) + except Exception as e: # noqa: BLE001 — a store that fails names itself; a rerun resumes + print( + f"loom cell attach: stopped ({type(e).__name__}: {e}); run it again", + file=sys.stderr, + ) + return 1 + + # Every ClickHouse statement, the connect included, may run as long as a scan may. + with statement_timeout(args.scan_budget): + return run_connected(_run, org_free=writes("attaches the cell; places the orgs it holds")) + + +def add_attach_parser(csub: Any) -> None: + p = csub.add_parser("attach", help="Attach the prepared first cell: place its orgs, serve.") + p.add_argument("--cell", required=True, help="This cell's name (LOOM_CELL)") + p.add_argument("--wait", type=int, default=_WAIT_S, help="Seconds the waits share") + p.add_argument( + "--scan-budget", type=int, default=_SCAN_BUDGET_S, help="Seconds one full org scan may take" + ) + p.add_argument( + "--dry-run", + action="store_true", + help="Run the refusals and the scan and print what would be placed; write nothing " + "(the router's own begin refusals do not run)", + ) + for flag, what in ( + ("--ch-allow-user", "A ClickHouse user that may write the configured databases"), + ("--ch-allow-user-directory", "A ClickHouse user directory the check cannot read"), + ("--ch-allow-object", "A ClickHouse object (db.name) that writes the databases"), + ("--ch-allow-replica", "A replica outside the cluster (db.table:replica)"), + ("--pg-allow-role", "A Postgres role that may make an org held (the servers' role)"), + ): + p.add_argument(flag, action="append", default=[], help=what) + p.add_argument("--router", help="The router's URL (default: LOOM_CELL_ROUTER_URL)") + p.set_defaults(func=cmd_attach) diff --git a/server/cli/cell_attach_beats.py b/server/cli/cell_attach_beats.py new file mode 100644 index 0000000000..a2fc37701d --- /dev/null +++ b/server/cli/cell_attach_beats.py @@ -0,0 +1,324 @@ +"""The first attach's judgment of the cell's processes, from their heartbeats +(docs/architecture/cells.md, "Attaching a cell" and "The heartbeat's keys"). + +Every check reads both heartbeat tables across every deployment sharing the ClickHouse, as +prepare does, stopped rows included (`instances.read_recent`), and judges in one order: + +- `cell` first: a live process whose `LOOM_CELL` is unset or another cell refuses at once; + a stopped one, or a row with no cell row that is no longer live, may still be writing + for the widened window after its stop beat, and is refused, or waited on until it ages + out, as the caller's `stopped_unset` says; +- then the keys a current build reports (`serves`, `marked`, `router`): a cell row + without them refuses, even while booting; +- then booting: a live instance with no cell row yet, no tenancy mode yet, `marked` null, + or `router` null where `serves` is true, is waited on; `marked` false, or `router` false + where `serves` is true, refuses, naming the process and the router's cause; +- and the fingerprint: each live process's matches `loom.cell_self`'s current or next. +""" + +from __future__ import annotations + +import asyncio +import json +import time +from collections.abc import Awaitable, Callable, Iterable, Mapping +from dataclasses import dataclass, field +from typing import Any, Literal + +from server.cli.cell_attach_pg import Wait +from server.cli.cell_ch import Refusal, Refused +from server.core import cell, instances + +StoppedUnset = Literal["refuse", "wait"] + + +@dataclass(frozen=True) +class Beat: + """One process: its instance row (`live`: inside the live window and not stopped) and + its cell row (`cell` None when it wrote none), the fingerprint JSON decoded.""" + + instance_id: str + live: bool + stopped: bool + keys_fp: str + last_beat: str + cell: str | None = None + mode: str = "" + value: dict[str, Any] | None = None + + +async def read_beats(ch: Any, *, since: str | None = None) -> list[Beat]: + rows = await instances.read_recent(ch, since=since) + ids = [str(r["instance_id"]) for r in rows] + cells = await ch.query_all( + f"SELECT instance_id, cell, tenancy_mode, fingerprint FROM {instances.CELL_TABLE} " + "WHERE has({ids:Array(String)}, instance_id)", + {"ids": ids}, + ) + by_id = {str(r["instance_id"]): r for r in cells} + out = [] + for r in rows: + found = by_id.get(str(r["instance_id"])) + beat = Beat( + instance_id=str(r["instance_id"]), + live=bool(r["live"]), + stopped=bool(r["stopped"]), + keys_fp=str(r["keys_fp"]), + last_beat=str(r["last_beat"]), + cell=None if found is None else str(found["cell"]), + mode="" if found is None else str(found["tenancy_mode"]), + value=None if found is None else _decoded(str(found["fingerprint"])), + ) + out.append(beat) + return out + + +def _decoded(text: str) -> dict[str, Any] | None: + try: + value = json.loads(text) + except ValueError: + return None + return value if isinstance(value, dict) else None + + +@dataclass +class Verdict: + """What one read found: the refusals, the processes waited on (booting, or a stopped + process with `LOOM_CELL` unset aging out), the live processes judged, their `keys_fp` + values, and whether one reports SCIM on.""" + + refusals: list[Refusal] = field(default_factory=list) + booting: list[str] = field(default_factory=list) + aging: list[str] = field(default_factory=list) + live: int = 0 + keys_fps: set[str] = field(default_factory=set) + scim: bool = False + + +def judge( + beats: Iterable[Beat], + name: str, + accepted: Iterable[Mapping[str, Any] | None], + stopped_unset: StoppedUnset, +) -> Verdict: + """The verdict on `beats` for the cell `name`, whose processes run `accepted` (its + `loom.cell_self` current and next).""" + values = [v for v in accepted if v is not None] + out = Verdict() + for b in beats: + who = f"process {b.instance_id}" + if b.cell is None: + if b.live: + out.booting.append(b.instance_id) + else: + _stopped(out, b, "stopped before reporting its LOOM_CELL", stopped_unset) + continue + if b.cell != name: + runs = f"runs with LOOM_CELL={b.cell}" if b.cell else "runs with LOOM_CELL unset" + if b.live: + out.refusals.append(Refusal(who, f"{runs}: set LOOM_CELL={name} on it")) + else: + _stopped(out, b, f"{runs} and stopped", stopped_unset) + continue + if not b.live: + continue + value = b.value or {} + if not all(k in value for k in ("serves", "marked", "router")): + out.refusals.append( + Refusal(who, "reports no serves, marked or router key: upgrade it to this release") + ) + continue + if value["marked"] is False: + out.refusals.append( + Refusal( + who, + "its Postgres sessions lose the tenancy marker (a pooler?): point " + "LOOM_PG_DSN straight at Postgres", + ) + ) + continue + serves = value["serves"] is True + if serves and value["router"] is False: + why = value.get("router_why") or "no cause reported" + out.refusals.append(Refusal(who, f"cannot read its router: {why}")) + continue + if value["marked"] is None or (serves and value["router"] is None) or not b.mode: + out.booting.append(b.instance_id) + continue + fields = value.get("fields") + if not isinstance(fields, dict) or not any(cell.matches(fields, v) for v in values): + out.refusals.append( + Refusal(who, "runs neither loom.cell_self's current nor its next configuration") + ) + continue + out.live += 1 + out.keys_fps.add(b.keys_fp) + out.scim = out.scim or bool(fields.get("scim")) + return out + + +def _stopped(out: Verdict, b: Beat, what: str, stopped_unset: StoppedUnset) -> None: + if stopped_unset == "wait": + out.aging.append(b.instance_id) + return + out.refusals.append( + Refusal( + f"process {b.instance_id}", + f"{what} at {b.last_beat}, and may still be writing (its drain and retirement " + "after its stop beat)", + ) + ) + + +def final_refusals(verdict: Verdict, *, scim_ok: bool = False) -> list[Refusal]: + out = [] + if not verdict.live and not verdict.booting: + out.append(Refusal("processes", "no live process of this cell reports a heartbeat")) + if len(verdict.keys_fps) > 1: + out.append( + Refusal( + "processes", + "the live processes report different root-secret fingerprints (keys_fp): " + "finish the root-secret rollout first", + ) + ) + if verdict.scim and not scim_ok: + out.append( + Refusal("processes", "SCIM is on: attach with SCIM on needs the per-cell SCIM release") + ) + return out + + +async def settle( + read: Callable[[], Awaitable[list[Beat]]], + judge_beats: Callable[[list[Beat]], Verdict], + wait: Wait, + *, + extend_s: float = 0.0, +) -> Verdict: + """Read and judge until nothing is waited on: Refused at the first refusal, and once + `wait` runs out naming what was waited on. With `extend_s`, the deadline is extended + once, to `extend_s` past the first sight of the last process first seen booting.""" + first_seen: dict[str, float] = {} + extended = False + while True: + verdict = judge_beats(await read()) + if verdict.refusals: + raise Refused(verdict.refusals) + if not verdict.booting and not verdict.aging: + if refusals := final_refusals(verdict): + raise Refused(refusals) + return verdict + for i in verdict.booting: + first_seen.setdefault(i, wait.spent) + if await wait.pause(): + continue + if extend_s and not extended and verdict.booting: + extended = True + wait.seconds = max(wait.seconds, max(first_seen[i] for i in verdict.booting) + extend_s) + if await wait.pause(): + continue + raise Refused( + [Refusal(f"process {i}", "still booting at the end of --wait") for i in verdict.booting] + + [ + Refusal(f"process {i}", "stopped with LOOM_CELL unset, not aged out within --wait") + for i in verdict.aging + ] + ) + + +# -- the rescan's watch ---------------------------------------------------------------- + +_READ_TIMEOUT_S = 30.0 # not a setting: one heartbeat read's bound +_SKEW_S = 10.0 # not a setting: a margin for clock skew between ClickHouse hosts + + +def watch_period_s() -> float: + """`T'`: how long a row whose window reaches past the rescan's start is kept before the + leader's sweep may drop it, less a margin for clock skew.""" + return instances.PRUNE_AFTER_S - instances.widened_window_s() - _SKEW_S + + +class Watch: + """The processes seen during step 4's rescan, read every `T' / 2` from the rescan's + start (`since`), so a process with `LOOM_CELL` unset that ran during it is refused at + the end even once the leader swept its rows.""" + + def __init__( + self, ch: Any, since: str, name: str, clock: Callable[[], float] = time.monotonic + ) -> None: + self.ch, self.since, self.name = ch, since, name + self._clock = clock + self.cell_ids: set[str] = set() + self.unset: dict[str, str] = {} + self.no_cell: dict[str, str] = {} + self.gap = False + self._good = clock() + + def refusals_up_front(self) -> list[Refusal]: + if watch_period_s() < 2 * _READ_TIMEOUT_S: + return [ + Refusal( + "LOOM_INSTANCE_HEARTBEAT_SECONDS", + "so long that a stopped process's rows can be swept before attach reads " + "them: lower it", + ) + ] + return [] + + async def read_once(self) -> list[Beat]: + try: + async with asyncio.timeout(_READ_TIMEOUT_S): + beats = await read_beats(self.ch, since=self.since) + except Exception: + self._check_gap() + raise + self._check_gap() + self._good = self._clock() + for b in beats: + if b.cell == self.name: + self.cell_ids.add(b.instance_id) + elif b.cell is not None: + self.unset[b.instance_id] = b.last_beat + elif b.stopped: + self.no_cell[b.instance_id] = b.last_beat + return beats + + def _check_gap(self) -> None: + if self._clock() - self._good > watch_period_s(): + self.gap = True + + async def run(self) -> None: + """Read until cancelled; a failed read is retried at the next period.""" + while True: + await asyncio.sleep(watch_period_s() / 2) + try: + await self.read_once() + except Exception: # noqa: BLE001 — a failed read is retried; a gap refuses + continue + + def refusals(self) -> list[Refusal]: + out = [ + Refusal( + f"process {i}", + f"ran with LOOM_CELL unset or another cell during the rescan (last beat {at})", + ) + for i, at in sorted(self.unset.items()) + ] + out += [ + Refusal( + f"process {i}", + f"stopped during the rescan before reporting its LOOM_CELL (last beat {at})", + ) + for i, at in sorted(self.no_cell.items()) + if i not in self.cell_ids + ] + if self.gap: + out.append( + Refusal( + "heartbeats", + "not read for longer than the leader keeps a stopped process's rows: " + "rerun attach", + ) + ) + return out diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py new file mode 100644 index 0000000000..6ee4432907 --- /dev/null +++ b/server/cli/cell_attach_pg.py @@ -0,0 +1,690 @@ +"""The first attach's reads and writes on the cell's Postgres (docs/architecture/cells.md, +"Attaching a cell"): the session checks, the one-run lock, the catalog checks attach and +`loom doctor` share (the roles that can make an org held or lower the mode, the settings +and subscriptions that pass an unchecked writer), the trigger pass, the mode commit, its +durable read and the wait for the transactions that could have read `recording`. + +Every check returns its refusals as `cell_ch.Refusal` (subject, reason), as the org scan +and the ClickHouse writer check do, so attach and doctor print the same names.""" + +from __future__ import annotations + +import asyncio +import time +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from typing import Any + +from server.cli.cell_ch import Refusal, Refused +from server.core import cell, cell_triggers, pg +from server.core.cell import ATTACH_LOCK_KEY + +_sleep = asyncio.sleep # the seam the wait tests replace + +# The pause between polls of a wait. +POLL_S = 1.0 # not a setting: a poll interval +# The migration lock's wait a trigger try (a boot holds it for its migrations). +_MIGRATION_LOCK_TIMEOUT = "10s" +# The mode commit's lock wait a try, and the statement bound above it. +_MODE_LOCK_TIMEOUT = "2s" +_MODE_STATEMENT_TIMEOUT = "5s" +# The server ends the lock session soon after its client is gone, releasing the lock. +_KEEPALIVE = {"tcp_keepalives_idle": "30", "tcp_keepalives_interval": "10"} + + +class Wait: + """`--wait`: the seconds attach's waits share, counting only the time spent waiting + (never scan time).""" + + def __init__(self, seconds: float, clock: Callable[[], float] = time.monotonic) -> None: + self.seconds = float(seconds) + self.spent = 0.0 + self._clock = clock + + def left(self) -> float: + return max(0.0, self.seconds - self.spent) + + def now(self) -> float: + return self._clock() + + async def pause(self, seconds: float = POLL_S) -> bool: + """Wait `seconds` (at most what is left) and count them; False once none is left.""" + step = min(seconds, self.left()) + if step <= 0: + return False + start = self._clock() + await _sleep(step) + self.spent += max(step, self._clock() - start) + return True + + +def _split(key: int) -> str: + """The `pg_locks` columns that hold a bigint advisory key, in this database.""" + return ( + f"l.locktype = 'advisory' AND l.classid = ({key} >> 32)::oid " + f"AND l.objid = ({key} & 4294967295)::oid AND l.objsubid = 1 " + "AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database())" + ) + + +_LOCK_HOLDER = f""" +SELECT l.pid, a.application_name, a.backend_start, a.client_addr::text AS client +FROM pg_locks l LEFT JOIN pg_stat_activity a ON a.pid = l.pid +WHERE {_split(ATTACH_LOCK_KEY)} AND l.granted +""" + +_IDENTITY = ( + "SELECT pg_backend_pid() AS pid, backend_start FROM pg_stat_activity " + "WHERE pid = pg_backend_pid()" +) + + +async def direct_refusals(conn: Any, what: str) -> list[Refusal]: + """Whether `conn` is one direct, marked session: the same backend across two + transactions, and the tenancy marker kept through `RESET ALL` (a pooler that strips + startup parameters drops it). Behind a transaction pooler a session-level lock would + outlive the CLI on the pooler's server connection, and attach's tenancy rows would be + refused as an unchecked writer's.""" + async with conn.transaction(): + first = await conn.fetchrow(_IDENTITY) + async with conn.transaction(): + second = await conn.fetchrow(_IDENTITY) + out = [] + if first is None or second is None or tuple(first) != tuple(second): + out.append( + Refusal( + "postgres session", + f"{what}'s session moved between backends: point LOOM_PG_DSN straight at " + "Postgres, never through a pooler", + ) + ) + await conn.execute("RESET ALL") # a startup value survives it, a SET does not + if await conn.fetchval("SELECT current_setting($1, true)", pg.TENANCY_MARKER) != "on": + out.append( + Refusal( + "postgres session", + f"{what}'s session loses {pg.TENANCY_MARKER}: run it with LOOM_CELL set, " + "LOOM_PG_DSN straight at Postgres", + ) + ) + return out + + +_HELD = f""" +SELECT EXISTS (SELECT 1 FROM pg_locks l + WHERE {_split(ATTACH_LOCK_KEY)} AND l.granted AND l.pid = pg_backend_pid()) +""" + + +async def lock_holder(conn: Any) -> str: + """Who holds the attach lock ('' when no one), never a query's text.""" + rows = await conn.fetch(_LOCK_HOLDER) + return "; ".join( + f"pid {r['pid']} ({r['application_name'] or 'no name'}, from {r['client'] or 'local'}, " + f"since {r['backend_start']})" + for r in rows + ) + + +async def take_lock(conn: Any, what: str) -> None: + """Take the attach lock on `conn` for the session's life, confirmed held by this + backend; Refused naming its holder when another run holds it.""" + for name, value in _KEEPALIVE.items(): + await conn.execute(f"SET {name} = {value}") + if not await conn.fetchval("SELECT pg_try_advisory_lock($1)", ATTACH_LOCK_KEY): + holder = await lock_holder(conn) or "a session that just ended" + raise Refused( + [ + Refusal( + "attach lock", + f"held by {holder}: an attach, prepare or update is running; {what} " + "runs one at a time", + ) + ] + ) + if not await lock_held(conn): + raise Refused( + [ + Refusal( + "attach lock", + "taken but not held by this session's backend: a pooler between " + "this command and Postgres; point LOOM_PG_DSN straight at it", + ) + ] + ) + + +async def lock_held(conn: Any) -> bool: + """Whether `conn`'s backend holds the attach lock.""" + return bool(await conn.fetchval(_HELD)) + + +async def encoding_refusals(conn: Any) -> list[Refusal]: + """A cell's database is UTF8: its `text` then holds no NUL or invalid UTF-8, so every + tenancy row within the length limit is an org a request can name.""" + encoding = await conn.fetchval( + "SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname = current_database()" + ) + if encoding == "UTF8": + return [] + return [ + Refusal( + "postgres database", + f"its encoding is {encoding}, not UTF8: a cell's database is UTF8 " + "(recreate it with ENCODING 'UTF8' and restore into it)", + ) + ] + + +# -- who can make an org held, or lower the mode -------------------------------------- + +_SYSTEM_SCHEMA = ( + "n.nspname NOT IN ('pg_catalog', 'information_schema') " + "AND n.nspname NOT LIKE 'pg\\_toast%' AND n.nspname NOT LIKE 'pg\\_temp\\_%'" +) + +# Every right a login role may hold that lets it make an org held, lower the mode, or run +# its own SQL as the servers on their marked sessions: each row a role and why. +_RIGHTS = f""" +WITH owned AS ( + SELECT c.relowner AS owner, 'owns loom.' || c.relname AS why + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'loom' AND c.relkind IN ('r', 'p') + UNION ALL + SELECT p.proowner, 'owns loom.' || p.proname || '()' FROM pg_proc p + WHERE p.oid IN (to_regprocedure('loom.tenancy_refuse_unheld()'), + to_regprocedure('loom.tenancy_refuse_unmarked()'), + to_regprocedure('loom.cell_first_immutable()')) + UNION ALL + SELECT d.datdba, 'owns the database' FROM pg_database d + WHERE d.datname = current_database() + UNION ALL + SELECT n.nspowner, 'owns schema ' || n.nspname FROM pg_namespace n WHERE {_SYSTEM_SCHEMA} +), +login AS (SELECT oid, rolname FROM pg_roles WHERE rolcanlogin) +SELECT r.rolname AS role, w.why FROM login r, LATERAL ( + SELECT 'may insert into loom.tenancy' + WHERE has_any_column_privilege(r.oid, 'loom.tenancy', 'INSERT') + UNION ALL SELECT 'may update loom.tenancy (org)' + WHERE has_column_privilege(r.oid, 'loom.tenancy', 'org', 'UPDATE') + UNION ALL SELECT 'may update loom.tenancy_mode (mode)' + WHERE has_column_privilege(r.oid, 'loom.tenancy_mode', 'mode', 'UPDATE') + UNION ALL SELECT 'may insert into loom.tenancy_mode' + WHERE has_any_column_privilege(r.oid, 'loom.tenancy_mode', 'INSERT') + UNION ALL SELECT 'holds TRIGGER on loom.' || c.relname + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'loom' AND c.relkind IN ('r', 'p') + AND has_table_privilege(r.oid, c.oid, 'TRIGGER') + UNION ALL SELECT 'may create in the database' + WHERE has_database_privilege(r.oid, current_database(), 'CREATE') + UNION ALL SELECT 'may create in schema ' || n.nspname FROM pg_namespace n + WHERE {_SYSTEM_SCHEMA} AND has_schema_privilege(r.oid, n.oid, 'CREATE') + UNION ALL SELECT 'may set session_replication_role' + WHERE has_parameter_privilege(r.oid, 'session_replication_role', 'SET') + OR has_parameter_privilege(r.oid, 'session_replication_role', 'ALTER SYSTEM') + UNION ALL SELECT 'may ALTER SYSTEM SET {pg.TENANCY_MARKER}' + WHERE has_parameter_privilege(r.oid, '{pg.TENANCY_MARKER}', 'ALTER SYSTEM') + UNION ALL SELECT 'is a member of ' || g.rolname FROM pg_roles g + WHERE g.rolname IN ('pg_write_server_files', 'pg_execute_server_program') + AND pg_has_role(r.oid, g.oid, 'MEMBER') + UNION ALL SELECT o.why || CASE WHEN o.owner = r.oid THEN '' + ELSE ' (as a member of ' || pg_get_userbyid(o.owner) || ')' + END + FROM owned o WHERE pg_has_role(r.oid, o.owner, 'MEMBER') +) w(why) +ORDER BY 1, 2 +""" + +# The roles attach counts as its own: superusers, and a login role that itself owns +# loom.tenancy, loom.tenancy_mode or a guarded table (it can drop or disable the +# triggers, or lower the mode, whatever else it holds). +_OWN_ROLES = """ +SELECT rolname FROM pg_roles WHERE rolcanlogin AND (rolsuper OR oid IN ( + SELECT c.relowner FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'loom' AND c.relkind IN ('r', 'p') + AND (c.relname IN ('tenancy', 'tenancy_mode') OR c.relname = ANY($1::text[])) +)) OR rolname = current_user +ORDER BY 1 +""" + + +@dataclass(frozen=True) +class Holders: + """The login roles holding a right `_RIGHTS` counts: the ones attach counts as its own + (`own`, named in its output) and every other with its reasons.""" + + own: list[str] + others: dict[str, list[str]] + + +async def privilege_holders(conn: Any) -> Holders: + guarded = [t for t in await cell_triggers.planned_tables(conn) if t != "tenancy"] + own = [str(r["rolname"]) for r in await conn.fetch(_OWN_ROLES, guarded)] + others: dict[str, list[str]] = {} + for r in await conn.fetch(_RIGHTS): + if r["role"] not in own: + others.setdefault(str(r["role"]), []).append(str(r["why"])) + return Holders(own, others) + + +def privilege_refusals(holders: Holders, allowed: Iterable[str]) -> list[Refusal]: + allow = set(allowed) + return [ + Refusal( + f"postgres role {role}", + f"{'; '.join(why)}: it can make an org held or lower the mode outside attach; " + "revoke it, or name the role with --pg-allow-role", + ) + for role, why in sorted(holders.others.items()) + if role not in allow + ] + + +# -- what passes an unchecked writer through the triggers ----------------------------- + +_ROLE_SETTINGS = f""" +SELECT coalesce(r.rolname, 'every role') AS role, coalesce(d.datname, 'every database') AS db, + cfg +FROM pg_db_role_setting s +LEFT JOIN pg_roles r ON r.oid = s.setrole +LEFT JOIN pg_database d ON d.oid = s.setdatabase, +LATERAL unnest(s.setconfig) cfg +WHERE s.setdatabase IN (0, (SELECT oid FROM pg_database WHERE datname = current_database())) + AND lower(split_part(cfg, '=', 1)) IN ('{pg.TENANCY_MARKER}', 'session_replication_role') +ORDER BY 1, 2, 3 +""" + +_SUBSCRIPTIONS = """ +SELECT subname, subenabled FROM pg_subscription +WHERE subdbid = (SELECT oid FROM pg_database WHERE datname = current_database()) +ORDER BY 1 +""" + + +async def unmarked_refusals(conn: Any, session_passes: bool) -> list[Refusal]: + """What lets a session that sends no marker pass the triggers: the session read + (`session_passes`, doctor's unmarked session), an `ALTER ROLE … SET` or + `ALTER DATABASE … SET` of the marker or `session_replication_role` for this database or + every one, and any logical-replication subscription into this database (its apply + worker runs as `replica`, firing no origin trigger).""" + out = [] + if session_passes: + out.append( + Refusal( + "postgres unmarked session", + "a session that sends no marker is marked, or runs as replica (a role, " + "database, ALTER SYSTEM or options setting): every unchecked writer passes", + ) + ) + for r in await conn.fetch(_ROLE_SETTINGS): + out.append( + Refusal( + f"postgres setting for {r['role']} in {r['db']}", + f"{r['cfg']}: an unchecked writer under it passes the triggers; reset it", + ) + ) + for r in await conn.fetch(_SUBSCRIPTIONS): + state = "enabled" if r["subenabled"] else "disabled" + out.append( + Refusal( + f"postgres subscription {r['subname']}", + f"{state}: its apply worker fires no origin trigger; drop it", + ) + ) + return out + + +async def session_passes() -> bool: + """Whether a session that sends no marker passes the triggers all the same: marked, + or in the replica role that fires no origin trigger.""" + async with pg.dedicated(statement_timeout_ms=5000, marked=False) as conn: + value = await conn.fetchval("SELECT current_setting($1, true)", pg.TENANCY_MARKER) + role = await conn.fetchval("SELECT current_setting('session_replication_role')") + return bool(value == "on" or role == "replica") + + +# -- attach's own role ------------------------------------------------------------------ + +_OWN_GRANTS = """ +SELECT need, held FROM (VALUES + ('membership in pg_read_all_stats', pg_has_role(current_user, 'pg_read_all_stats', 'MEMBER')), + ('membership in pg_signal_backend', pg_has_role(current_user, 'pg_signal_backend', 'MEMBER')), + ('USAGE on schema loom', has_schema_privilege('loom', 'USAGE')), + ('SELECT on loom.cell_first', has_table_privilege('loom.cell_first', 'SELECT')), + ('SELECT on loom.tenancy_mode', has_table_privilege('loom.tenancy_mode', 'SELECT')), + ('UPDATE on loom.tenancy_mode', has_table_privilege('loom.tenancy_mode', 'UPDATE')), + ('SELECT on loom.cell_self', has_table_privilege('loom.cell_self', 'SELECT')), + ('SELECT on loom.authority', has_table_privilege('loom.authority', 'SELECT')), + ('INSERT on loom.tenancy', has_table_privilege('loom.tenancy', 'INSERT')), + ('SELECT, INSERT, UPDATE and DELETE on loom.cell_probe', + has_table_privilege('loom.cell_probe', 'SELECT, INSERT, UPDATE, DELETE')), + ('TEMPORARY on the database', has_database_privilege(current_database(), 'TEMPORARY')) +) v(need, held) +WHERE NOT held +""" + + +async def own_role_refusals(conn: Any, tables: Iterable[str]) -> list[Refusal]: + """What attach's own role lacks (the guarded tables' owner roles are `missing`'s + `owner` defects), and a server that allows prepared transactions.""" + role = await conn.fetchval("SELECT current_user") + lacks = [str(r["need"]) for r in await conn.fetch(_OWN_GRANTS)] + for table in tables: + if not await conn.fetchval("SELECT has_table_privilege($1, 'SELECT')", table): + lacks.append(f"SELECT on {table}") + out = [Refusal(f"postgres role {role}", f"attach's role lacks {n}") for n in lacks] + out += await prepared_refusals(conn) + return out + + +async def prepared_refusals(conn: Any) -> list[Refusal]: + setting = await conn.fetchval("SELECT current_setting('max_prepared_transactions')::int") + if setting: + return [ + Refusal( + "postgres max_prepared_transactions", + f"is {setting}: a prepared transaction outlives every session, so step 4 " + "could not wait for it; set it to 0", + ) + ] + return [] + + +# -- the triggers ----------------------------------------------------------------------- + +_MIGRATION_LOCK = "SELECT pg_advisory_lock($1)" +_MIGRATION_UNLOCK = "SELECT pg_advisory_unlock($1)" + + +async def ensure_triggers(conn: Any, wait: Wait, log: Callable[[str], None]) -> list[Refusal]: + """Give every planned table its triggers (`cell_triggers.ensure_table` with + `recording_first`), one try at a time under the migration lock, retrying a table whose + lock is busy a second apart within `wait`; one autovacuum holding it is asked to yield + by one try just past `deadlock_timeout`. Then `missing`: any defect refuses.""" + from server.core.schema.pg_migrate import _LOCK_KEY + + deadlock_ms = int( + await conn.fetchval( + "SELECT setting::int * CASE unit WHEN 's' THEN 1000 ELSE 1 END " + "FROM pg_settings WHERE name = 'deadlock_timeout'" + ) + ) + out: list[Refusal] = [] + for table in await cell_triggers.planned_tables(conn): + yielded = False + while True: + lock_timeout = cell_triggers.LOCK_TIMEOUT + holders = await cell_triggers.holders(conn, table) + if not yielded and any(h["backend_type"] == "autovacuum worker" for h in holders): + lock_timeout, yielded = f"{deadlock_ms + 500}ms", True + done = await _ensure_one(conn, table, lock_timeout, _LOCK_KEY) + if done: + break + if not await wait.pause(): + out.append(_busy_refusal(table, await cell_triggers.holders(conn, table))) + break + log(f"loom.{table}: its lock is busy; retrying") + if out: + return out + return [Refusal("postgres triggers", d.text) for d in await cell_triggers.missing(conn)] + + +async def _ensure_one(conn: Any, table: str, lock_timeout: str, key: int) -> bool: + await conn.execute(f"SET lock_timeout = '{_MIGRATION_LOCK_TIMEOUT}'") + try: + await conn.execute(_MIGRATION_LOCK, key) + except Exception as e: + if getattr(e, "sqlstate", None) == "55P03": + return False # a boot migrating: the next try + raise + finally: + await conn.execute("RESET lock_timeout") + try: + return await cell_triggers.ensure_table( + conn, table, recording_first=True, lock_timeout=lock_timeout, attempts=1 + ) + finally: + await conn.execute(_MIGRATION_UNLOCK, key) + + +def _busy_refusal(table: str, holders: list[Any]) -> Refusal: + named = "; ".join( + f"pid {h['pid']} ({'holds' if h['granted'] else 'waiting'}, {h['backend_type']}, " + f"{h['application_name'] or 'no name'}, transaction since {h['xact_start']}" + + (", an anti-wraparound vacuum, which never yields" if h["wraparound"] else "") + + ")" + for h in holders + ) + return Refusal( + f"postgres loom.{table}", + f"its triggers' lock was not taken within --wait; held by {named or 'no one now'}", + ) + + +# -- step 3: the mode commit ------------------------------------------------------------ + +# The mode transaction in flight: the backend holding or waiting for the ExclusiveLock on +# loom.tenancy_mode in this database, which only attach takes. +_MODE_HOLDERS = """ +SELECT l.pid, a.application_name, a.xact_start, r.rolsuper, l.granted +FROM pg_locks l +JOIN pg_stat_activity a ON a.pid = l.pid +LEFT JOIN pg_roles r ON r.oid = a.usesysid +WHERE l.locktype = 'relation' AND l.relation = 'loom.tenancy_mode'::regclass + AND l.mode = 'ExclusiveLock' + AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database()) + AND l.pid <> pg_backend_pid() +""" + +_TIMED_OUT = frozenset({"55P03", "57014"}) + + +async def commit_enforced( + conn: Any, + confirm_lock: Callable[[], Any], + ready: Callable[[], Any], + wait: Wait, + log: Callable[[str], None], +) -> None: + """Raise the mode to `enforced` on `conn` (a session of its own, never the lock + session): an EXCLUSIVE lock on loom.tenancy_mode a try, `_MODE_LOCK_TIMEOUT` each, + within `wait`; `confirm_lock` before each try and each termination (it refuses when + another run holds the attach lock), and `ready` before each try (False skips the try + while a process boots). After a failed try the in-flight mode transaction, found by its + lock, is ended. Raises Refused when `wait` runs out; whether the commit landed is the + durable read's to say.""" + while True: + await confirm_lock() + if await ready(): + try: + async with conn.transaction(): + await conn.execute(f"SET LOCAL lock_timeout = '{_MODE_LOCK_TIMEOUT}'") + await conn.execute(f"SET LOCAL statement_timeout = '{_MODE_STATEMENT_TIMEOUT}'") + await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") + status = await conn.execute( + "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" + ) + if status != "UPDATE 1": + raise Refused( + [ + Refusal( + "postgres loom.tenancy_mode", f"the mode update read {status}" + ) + ] + ) + return + except Refused: + raise + except Exception as e: + if getattr(e, "sqlstate", None) not in _TIMED_OUT: + raise + log("the mode's lock is busy; retrying") + await end_mode_transactions(conn, confirm_lock) + if not await wait.pause(): + raise Refused( + [ + Refusal( + "postgres loom.tenancy_mode", + "its lock was not taken within --wait: a long transaction reads it " + "FOR SHARE; rerun attach", + ) + ] + ) + + +async def end_mode_transactions(conn: Any, confirm_lock: Callable[[], Any]) -> None: + """End an earlier run's mode transaction still in flight and wait until its lock + clears; Refused when its role is a superuser and attach's is not.""" + for _ in range(30): + rows = await conn.fetch(_MODE_HOLDERS) + if not rows: + return + superuser = bool( + await conn.fetchval("SELECT rolsuper FROM pg_roles WHERE rolname = current_user") + ) + for r in rows: + if r["rolsuper"] and not superuser: + raise Refused( + [ + Refusal( + f"postgres backend {r['pid']}", + f"an earlier mode transaction ({r['application_name'] or 'no name'}, " + f"since {r['xact_start']}) runs as a superuser: a superuser must " + "end it (pg_terminate_backend)", + ) + ] + ) + await confirm_lock() + await conn.fetchval("SELECT pg_terminate_backend($1)", r["pid"]) + await _sleep(0.1) + + +# -- the durable read ------------------------------------------------------------------- + +_DURABLE_TRY_S = 10.0 # not a setting: one try's bound; a cancelled one is retried + + +@dataclass(frozen=True) +class Durable: + """What the durable read saw: the mode, the `first` row, and the xid its logical + message took.""" + + mode: str + first: tuple[bool, str] | None + xid: int + + +async def durable_read() -> Durable: + """The mode and the `first` row read in a transaction that emits a transactional + logical message, whose commit waits for the synchronous standbys: once it returns, the + mode read is on them. Refused when the message took no xid.""" + async with asyncio.timeout(_DURABLE_TRY_S), pg.durable_transaction() as conn: + await conn.execute("SELECT pg_logical_emit_message(true, 'loom.attach', '')") + xid = await conn.fetchval("SELECT pg_current_xact_id_if_assigned()::text") + mode = await conn.fetchval("SELECT mode FROM loom.tenancy_mode") + first = await conn.fetchrow("SELECT is_first, cell FROM loom.cell_first") + if xid is None: + raise Refused([Refusal("postgres durable read", "its logical message took no xid")]) + return Durable( + mode=str(mode or ""), + first=None if first is None else (bool(first["is_first"]), str(first["cell"])), + xid=int(xid), + ) + + +async def snapshot_xmax(conn: Any) -> int: + return int(await conn.fetchval("SELECT pg_snapshot_xmax(pg_current_snapshot())::text")) + + +# -- step 4: the transactions that could have read `recording` -------------------------- + +# Every backend but this statement's own: of the cell's database, one whose xid or (but +# an autovacuum's) snapshot is older than X; of another database, one whose xid is older +# (it holds every snapshot's horizon). Both compared through age() against one reference, +# which is exactly `< X`, wraparound-safe, and takes no xid. +_OPEN = """ +WITH x AS (SELECT age($1::text::xid8::xid) AS a) +SELECT s.pid, s.backend_type, s.application_name, s.xact_start::text AS xact_start, + s.datid = (SELECT oid FROM pg_database WHERE datname = current_database()) AS ours, + coalesce(s.backend_xid IS NOT NULL AND age(s.backend_xid) > x.a, false) AS xid_old, + coalesce(s.backend_xmin IS NOT NULL AND age(s.backend_xmin) > x.a + AND s.backend_type <> 'autovacuum worker', false) AS xmin_old, + coalesce(s.query = '', false) AS hidden +FROM pg_stat_activity s, x +WHERE s.pid <> pg_backend_pid() +""" + + +def _named(r: Any) -> str: + return ( + f"pid {r['pid']} ({r['backend_type']}, {r['application_name'] or 'no name'}, " + f"transaction since {r['xact_start']})" + ) + + +async def wait_transactions(conn: Any, xmax: int, wait: Wait) -> None: + """Wait until no backend of the cell's database holds an xid or a snapshot older than + `xmax`; Refused at once on a prepared transaction or an unreadable row, and past + `wait` naming each backend waited on and each other database's horizon holder.""" + while True: + if refusals := await prepared_refusals(conn): + raise Refused(refusals) + if await conn.fetchval("SELECT count(*) FROM pg_prepared_xacts"): + raise Refused( + [Refusal("postgres prepared transaction", "one is open: commit or roll it back")] + ) + rows = await conn.fetch(_OPEN, str(xmax)) + hidden = [r for r in rows if r["ours"] and r["hidden"]] + if hidden: + raise Refused( + [ + Refusal( + f"postgres backend {r['pid']}", + "its transaction state is hidden from attach's role: grant it " + "pg_read_all_stats", + ) + for r in hidden + ] + ) + waited = [r for r in rows if r["ours"] and (r["xid_old"] or r["xmin_old"])] + if not waited: + return + if not await wait.pause(): + horizon = [r for r in rows if not r["ours"] and r["xid_old"]] + raise Refused( + [ + Refusal(f"postgres {_named(r)}", "may have read `recording`: still open") + for r in waited + ] + + [ + Refusal( + f"postgres {_named(r)}", + "another database's transaction, holding the snapshot horizon", + ) + for r in horizon + ] + ) + + +# -- the tenancy rows ------------------------------------------------------------------- + + +async def write_tenancy(orgs: Iterable[str]) -> int: + """The tenancy rows of `orgs`, on a marked session, as one statement per thousand; the + rows written.""" + batch = sorted(orgs) + written = 0 + for i in range(0, len(batch), 1000): + status = await pg.execute( + "INSERT INTO loom.tenancy (org) SELECT unnest($1::text[]) ON CONFLICT DO NOTHING", + batch[i : i + 1000], + ) + written += int(status.split()[-1]) + return written + + +async def read_mode(conn: Any) -> tuple[str, tuple[bool, str] | None]: + """The cell's state (`cell.read_state`) and its `first` row.""" + state = await cell.read_state(conn) + first = await conn.fetchrow("SELECT is_first, cell FROM loom.cell_first") + return state, None if first is None else (bool(first["is_first"]), str(first["cell"])) diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index 8ad83098a9..cf83fa8f27 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -1,7 +1,10 @@ """`loom cell prepare`: register this cell in the router's directory and write the rows its processes read (docs/architecture/cells.md, "Preparing a cell" and "The tenancy check"). -It runs where the cell's stores are configured (`LOOM_CELL` set to the cell it names): +It runs where the cell's stores are configured (`LOOM_CELL` set to the cell it names), +holding the attach lock (`cell.ATTACH_LOCK_KEY`) on a direct session of the cell's Postgres, +after that session's checks (one backend, the tenancy marker kept, a UTF8 database), so it +never runs beside an attach or another prepare: 1. It reads the cell's Postgres (deployment id, `first`, mode and own rows, tenancy rows) and the live processes' heartbeats, and refuses a cell whose rows name another cell @@ -23,11 +26,13 @@ import argparse import asyncio +import contextlib import hashlib import json import secrets import sys import time +from collections.abc import AsyncIterator from dataclasses import dataclass from typing import Any @@ -46,6 +51,9 @@ _WAIT_MAX_S = 86400 # not a setting: a flag's bound +# A bound on the lock session's statements: it reads the catalog and holds the lock. +_LOCK_STATEMENT_MS = 60_000 # not a setting: a bound on one CLI session's reads + _sleep = asyncio.sleep # the seam the retry tests replace @@ -442,6 +450,38 @@ async def prepare( args: argparse.Namespace, router: httpx.URL, transport: httpx.AsyncBaseTransport | None = None, +) -> int: + """Prepare the cell, holding the attach lock for the run (`attach_lock`).""" + async with attach_lock(): + return await _prepare_locked(ch, args, router, transport) + + +@contextlib.asynccontextmanager +async def attach_lock() -> AsyncIterator[None]: + """The attach lock, held on a direct session of the cell's Postgres while the block + runs, after that session's checks: prepare and attach write the same probe nonce, and + one of them runs at a time.""" + from server.cli import cell_attach_pg + from server.cli.cell_ch import Refused as Stopped + from server.core import pg + + async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: + try: + refusals = await cell_attach_pg.direct_refusals(conn, "prepare") + refusals += await cell_attach_pg.encoding_refusals(conn) + if refusals: + raise Stopped(refusals) + await cell_attach_pg.take_lock(conn, "prepare") + except Stopped as e: + raise Refused(str(e)) from None + yield + + +async def _prepare_locked( + ch: Any, + args: argparse.Namespace, + router: httpx.URL, + transport: httpx.AsyncBaseTransport | None, ) -> int: rows = await _read_rows() if not rows.deployment_id: diff --git a/server/cli/cell_scan.py b/server/cli/cell_scan.py index 6968dd49b0..e7d9077a02 100644 --- a/server/cli/cell_scan.py +++ b/server/cli/cell_scan.py @@ -67,6 +67,19 @@ } ) +# Tables a worker drains into another table, deleting what it moved: each is scanned +# before every table that is not one, so a row moved mid-scan is read where it was or where +# it went. A table a materialized view reads is ordered the same way, from +# `system.tables` dependencies (`order_sources`). +MOVE_SOURCES = frozenset( + { + "loom.learnings_queue", + "loom.memory_code_link_queue", + "loom.memory_extraction_queue", + "loom.memory_webhook_outbox", + } +) + # A refreshable view's temporary table while a refresh exchanges its target. _EXCHANGE = re.compile(r"^\.tmp\.inner(_id)?\.") # A provider transition's copies (server/erasure/store/provider_copies.py). @@ -338,6 +351,32 @@ async def list_postgres(conn: Any) -> tuple[list[str], list[Refusal]]: return names, refusals +def order_sources(tables: list[Table], view_sources: Iterable[str]) -> list[Table]: + """`tables` with the move sources (`MOVE_SOURCES`, and every table a view reads, + `view_sources`, canonical names) first, each group by name.""" + first = MOVE_SOURCES | set(view_sources) + return sorted(tables, key=lambda t: (t.canonical not in first, t.canonical)) + + +async def view_sources( + ch: Any, topology: Topology, names: Mapping[str, str], dbs: list[str] +) -> set[str]: + """The tables of `dbs` a view depends on, as canonical names.""" + rows = await per_host( + ch, + topology, + names, + "SELECT hostName() AS h, database, name FROM {source:tables} " + "WHERE database IN {dbs:Array(String)} AND notEmpty(dependencies_table)", + {"dbs": dbs}, + ) + return { + f"{ch_names.logical(str(r['database']))}.{r['name']}" + for found in rows.values() + for r in found + } + + async def list_tables( ch: Any, pg_conn: Any, topology: Topology, names: Mapping[str, str] ) -> TableList: @@ -370,7 +409,8 @@ async def _list_tables( if copies := provider_copies(tables): out.refusals += copies return out - out.clickhouse, problems = judge_clickhouse(tables, classified, topology) + scanned, problems = judge_clickhouse(tables, classified, topology) + out.clickhouse = order_sources(scanned, await view_sources(ch, topology, names, databases())) out.refusals += problems out.refusals += await reverse_clickhouse(ch, tables, classified) out.postgres, problems = await list_postgres(pg_conn) @@ -460,7 +500,7 @@ def pg_skip_scan_sql(table: str) -> str: return _PG_SKIP_SCAN.format(table=f"{pg_quote(schema)}.{pg_quote(name)}") -async def _pg_orgs(table: str, budget: Budget) -> list[str]: +async def pg_orgs(table: str, budget: Budget) -> list[str]: subject = f"postgres {table}" while True: left = budget.require(subject) @@ -615,7 +655,7 @@ async def _read_scan( return _body_rows(b"".join(chunks).decode("utf-8", "replace").splitlines(), "k") -async def _ch_orgs(ch: Any, table: Table, topology: Topology, budget: Budget) -> list[bytes | None]: +async def ch_orgs(ch: Any, table: Table, topology: Topology, budget: Budget) -> list[bytes | None]: subject = f"clickhouse {table.canonical}" while True: budget.require(subject) @@ -697,10 +737,10 @@ async def scan( result.add("configured", org) with query_tag(TAG): for name in tables.postgres: - for org in await _pg_orgs(name, budget): + for org in await pg_orgs(name, budget): result.add(f"postgres {name}", org) for table in tables.clickhouse: - for raw in await _ch_orgs(ch, table, topology, budget): + for raw in await ch_orgs(ch, table, topology, budget): result.add(f"clickhouse {table.canonical}", raw) result.elapsed_s = budget.elapsed() return result diff --git a/server/cli/doctor.py b/server/cli/doctor.py index d1295ffb32..29a9ddabf6 100644 --- a/server/cli/doctor.py +++ b/server/cli/doctor.py @@ -566,10 +566,15 @@ def grade(p: Any) -> str: return [(f"problem:{p.code}", grade(p), f"{p.message} Fix: {p.fix}") for p in problems] -def _run_doctor_checks() -> list[tuple[str, str, str]]: +def _run_doctor_checks(allowed_roles: tuple[str, ...] = ()) -> list[tuple[str, str, str]]: """The live checks (Python, settings, JWT, ClickHouse, LLM) followed - by the shared problem list, as (name, grade, detail).""" - return _run_live_checks() + _code_mirror_checks() + _problem_checks(_doctor_problems()) + by the shared problem list, as (name, grade, detail). `allowed_roles` is + `--pg-allow-role`, for a cell's checks.""" + return ( + _run_live_checks(allowed_roles) + + _code_mirror_checks() + + _problem_checks(_doctor_problems()) + ) def _code_mirror_checks() -> list[tuple[str, str, str]]: @@ -582,7 +587,7 @@ def _code_mirror_checks() -> list[tuple[str, str, str]]: return [("code_mirrors", WARN, f"not checked: {type(e).__name__}: {e}")] -def _run_live_checks() -> list[tuple[str, str, str]]: +def _run_live_checks(allowed_roles: tuple[str, ...] = ()) -> list[tuple[str, str, str]]: """Run the doctor's live checks and return them as (name, grade, detail) with grade one of PASS / WARN / FAIL. @@ -664,7 +669,7 @@ def _run_live_checks() -> list[tuple[str, str, str]]: if settings_ok and ch_ok and settings.cell: from server.cli import doctor_cell - checks += doctor_cell.cell_checks() + checks += doctor_cell.cell_checks(allowed_roles) # 5. LLM provider resolution: what extraction runs on and # what decided it; WARN when the provider was inferred (or named) but its @@ -677,6 +682,17 @@ def _run_live_checks() -> list[tuple[str, str, str]]: return checks +def _stray_org_checks(budget_s: float) -> list[tuple[str, str, str]]: + """`--stray-orgs`, on a cell only (LOOM_CELL set).""" + from server.settings import settings + + if not settings.cell: + return [("cell_stray_orgs", WARN, "not read: LOOM_CELL is unset (no cell)")] + from server.cli import doctor_cell + + return doctor_cell.stray_org_checks(budget_s) + + def _client_checks(args: argparse.Namespace) -> list[tuple[str, str, str]]: """`loom doctor --client`: the checks from the machine a coding agent runs on, with the CLI's own HTTP helpers. A crash in one of @@ -722,7 +738,9 @@ def _cmd_doctor(_args: argparse.Namespace) -> int: operator sees the full picture. Exit 0 = nothing failed (warnings are listed but do not fail); 1 = at least one FAIL. """ - checks = _run_doctor_checks() + checks = _run_doctor_checks(tuple(getattr(_args, "pg_allow_role", None) or ())) + if getattr(_args, "stray_orgs", False): + checks += _stray_org_checks(_args.stray_orgs_budget) # The client's side of the same picture, when asked. client = _client_checks(_args) if getattr(_args, "client", False) else [] checks = _client_machine_grades(checks, client) + client @@ -841,4 +859,21 @@ def _add_doctor_flags(p_doc: argparse.ArgumentParser) -> None: ), ) p_doc.add_argument("--server-url", default=None, help="Override the client's server URL.") + p_doc.add_argument( + "--pg-allow-role", + action="append", + default=[], + help="On a cell: a Postgres role that may make an org held (the servers' role).", + ) + p_doc.add_argument( + "--stray-orgs", + action="store_true", + help="On a cell: also read the orgs its stores hold without a tenancy row.", + ) + p_doc.add_argument( + "--stray-orgs-budget", + type=float, + default=60.0, + help="Seconds --stray-orgs may read for, smallest tables first (default 60).", + ) p_doc.add_argument("--token", default=None, help="Override the client's token.") diff --git a/server/cli/doctor_cell.py b/server/cli/doctor_cell.py index f2d6f80367..c79723bbbc 100644 --- a/server/cli/doctor_cell.py +++ b/server/cli/doctor_cell.py @@ -1,7 +1,11 @@ """`loom doctor`'s cell checks, run only with LOOM_CELL set (docs/architecture/cells.md): the deployment check, a partial cell state, the tenancy marker on this process's -sessions and on every session, and the defects of the org triggers and the `loom.tenancy` -guard (`cell_triggers.missing`). +sessions and on every session, the defects of the org triggers and the `loom.tenancy` +guard (`cell_triggers.missing`), and the point-in-time reads attach makes, run again on +every doctor run: the login roles that can make an org held or lower the mode, and the +role and database settings and subscriptions that pass an unchecked writer +(`cell_attach_pg`). `--stray-orgs` also reads the orgs the cell's stores hold without a +tenancy row (`stray_org_checks`). A refusal (ClickHouse access denied, no LOOM_PG_DSN, another recorded authority or a Postgres of another deployment) fails the doctor's exit and reads no cell state; @@ -78,14 +82,34 @@ async def _unmarked_session() -> bool: """Whether a session that sends no marker passes the triggers all the same (a role, database, ALTER SYSTEM or options setting): marked, or in the replica role that fires no origin trigger.""" + from server.cli import cell_attach_pg + + return await cell_attach_pg.session_passes() + + +async def _writer_reads(allowed_roles: tuple[str, ...]) -> list[tuple[str, str, str]]: + """Attach's point-in-time Postgres reads, again: the login roles outside the owner + roles, the superusers and `--pg-allow-role` that can make an org held or lower the + mode, and the settings and subscriptions that pass an unchecked writer.""" + from server.cli import cell_attach_pg from server.core import pg - async with pg.dedicated(statement_timeout_ms=_STATEMENT_MS, marked=False) as conn: - value: str | None = await conn.fetchval( - "SELECT current_setting($1, true)", pg.TENANCY_MARKER - ) - role: str = await conn.fetchval("SELECT current_setting('session_replication_role')") - return value == "on" or role == "replica" + async with pg.dedicated(statement_timeout_ms=_STATEMENT_MS) as conn: + holders = await cell_attach_pg.privilege_holders(conn) + unchecked = await cell_attach_pg.unmarked_refusals(conn, False) + roles = cell_attach_pg.privilege_refusals(holders, allowed_roles) + return [ + ( + "cell_tenancy_privilege", + WARN if roles else PASS, + "; ".join(str(r) for r in roles) or "no role outside the owner roles can", + ), + ( + "cell_unchecked_writers", + WARN if unchecked else PASS, + "; ".join(str(r) for r in unchecked) or "no setting or subscription passes one", + ), + ] def _grades( @@ -149,7 +173,7 @@ def _grades( return out -async def _checks(ch: Any) -> list[tuple[str, str, str]]: +async def _checks(ch: Any, allowed_roles: tuple[str, ...] = ()) -> list[tuple[str, str, str]]: try: if before := await asyncio.wait_for(_deployment_check(ch), timeout=_SESSION_S): return [("cell_state", PASS, f"cell checks not run: {before}")] @@ -159,7 +183,11 @@ async def _checks(ch: Any) -> list[tuple[str, str, str]]: return [("cell_deployment", FAIL, f"deployment check refused: {e}")] except Exception as e: # noqa: BLE001 — an unreadable cell is named, not a failed run return [("cell_state", WARN, f"not read: {_reason(e)}")] - return _grades(state, marked, marks_all, defects) + try: + reads: list[tuple[str, str, str]] = await _bounded(_writer_reads(allowed_roles)) + except Exception as e: # noqa: BLE001 — an unread catalog is named, not a failed run + reads = [("cell_tenancy_privilege", WARN, f"not read: {_reason(e)}")] + return _grades(state, marked, marks_all, defects) + reads def _reason(e: BaseException) -> str: @@ -167,9 +195,10 @@ def _reason(e: BaseException) -> str: return f"{type(e).__name__}: {str(e) or 'no answer within the bound'}" -def cell_checks() -> list[tuple[str, str, str]]: +def cell_checks(allowed_roles: tuple[str, ...] = ()) -> list[tuple[str, str, str]]: """The cell checks, on the CLI's ClickHouse client connected for them alone (no CLI - deployment check: a deployment not booted yet is named, not refused).""" + deployment check: a deployment not booted yet is named, not refused). `allowed_roles` + is doctor's `--pg-allow-role`.""" from server.cli._ch import connect_declared, disconnect from server.core.ch import get_client @@ -180,10 +209,121 @@ async def _go() -> list[tuple[str, str, str]]: connect_declared(ch, org_free=reads("loom doctor reads a cell's state")), timeout=_SESSION_S, ) - return await _checks(ch) + return await _checks(ch, allowed_roles) except Exception as e: # noqa: BLE001 — a ClickHouse that does not answer is named return [("cell_state", WARN, f"not read: {_reason(e)}")] finally: await disconnect(ch) return asyncio.run(_go()) + + +# -- --stray-orgs -------------------------------------------------------------------------- + + +async def _table_sizes(ch: Any, tables: list[Any]) -> dict[str, int]: + """Each ClickHouse table's active row count, from `system.parts` (KeeperMap: 0).""" + from server.core import ch_names + + rows = await ch.query_all( + "SELECT database, table, sum(rows) AS n FROM system.parts WHERE active " + "AND database IN {dbs:Array(String)} GROUP BY database, table", + {"dbs": sorted({t.database for t in tables})}, + ) + return {f"{ch_names.logical(str(r['database']))}.{r['table']}": int(r["n"]) for r in rows} + + +async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: + from server.cli import cell_scan + from server.cli.cell_ch import Refused, databases, read_topology + from server.core import pg + from server.core.cell_triggers import ORG_TABLES + + topology = await read_topology(ch) + if topology.multi: + from server.cli.cell_writers import host_names + + names = await host_names(ch, topology) + else: + names = {} + listed, problems = await cell_scan.list_clickhouse(ch, topology, names, databases()) + ch_tables = [t for t in listed if t.engine.endswith("MergeTree") or t.engine == "KeeperMap"] + sizes = await _table_sizes(ch, ch_tables) + async with pg.dedicated(statement_timeout_ms=_STATEMENT_MS) as conn: + held = {str(r["org"]) for r in await conn.fetch("SELECT org FROM loom.tenancy")} + pg_rows = await conn.fetch( + "SELECT 'loom.' || c.relname AS name, greatest(c.reltuples, 0)::bigint AS n " + "FROM pg_class c WHERE c.oid IN (SELECT to_regclass('loom.' || t.relname) FROM (" + + ORG_TABLES + + ") t)" + ) + # Smallest first: a budget that runs out leaves the largest unchecked. + work: list[tuple[int, str, Any]] = [(int(r["n"]), "postgres", str(r["name"])) for r in pg_rows] + work += [(sizes.get(t.canonical, 0), "clickhouse", t) for t in ch_tables] + work.sort(key=lambda w: (w[0], str(w[2]))) + budget = cell_scan.Budget(budget_s) + stray: dict[str, list[str]] = {} + unchecked: list[str] = [] + for i, (_n, store, table) in enumerate(work): + found = cell_scan.ScanResult() + name = table if store == "postgres" else table.canonical + try: + if store == "postgres": + for org in await cell_scan.pg_orgs(table, budget): + found.add(name, org) + else: + for raw in await cell_scan.ch_orgs(ch, table, topology, budget): + found.add(name, raw) + except Refused: + unchecked += [w[2] if w[1] == "postgres" else w[2].canonical for w in work[i:]] + break + orgs = sorted(set(found.orgs) - held) + sorted(found.too_long) + bad = [m for ms in found.malformed.values() for m in ms] + if orgs or bad: + log = " (a log table: never placed)" if name in cell_scan.LOG_TABLES else "" + stray[f"{store} {name}{log}"] = orgs[:5] + bad[:5] + out = [ + ( + "cell_stray_orgs", + WARN if stray else PASS, + "; ".join(f"{t}: {', '.join(o)}" for t, o in sorted(stray.items())) + or "every org found has a tenancy row", + ) + ] + if unchecked or problems: + out.append( + ( + "cell_stray_orgs_unchecked", + WARN, + f"not read within --stray-orgs-budget ({budget_s:g} s): " + + ", ".join(unchecked + [str(p) for p in problems]), + ) + ) + return out + + +def stray_org_checks(budget_s: float) -> list[tuple[str, str, str]]: + """`loom doctor --stray-orgs`: the orgs the cell's stores hold without a tenancy row + (log-only orgs, names no request can carry, rows an unchecked writer left), smallest + tables first within one budget, naming every table left unchecked. Never a FAIL: the + stray-org erase command removes what it names.""" + from server.cli._ch import connect_declared, disconnect + from server.core.ch import get_client, statement_timeout + + async def _go() -> list[tuple[str, str, str]]: + ch = get_client() + try: + await asyncio.wait_for( + connect_declared(ch, org_free=reads("loom doctor reads a cell's orgs")), + timeout=_SESSION_S, + ) + if before := await asyncio.wait_for(_deployment_check(ch), timeout=_SESSION_S): + return [("cell_stray_orgs", PASS, f"not read: {before}")] + return await _stray_reads(ch, budget_s) + except Exception as e: # noqa: BLE001 — an unread store is named, not a failed run + return [("cell_stray_orgs", WARN, f"not read: {_reason(e)}")] + finally: + await disconnect(ch) + + with statement_timeout(budget_s): + return asyncio.run(_go()) diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 19cf492b0e..9c2c9755ae 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -36,7 +36,9 @@ CELL_FIRST_TRIGGERS = ("cell_first_immutable", "cell_first_no_truncate") # Org-keyed tables the org triggers skip: the cell's own tenancy rows (`tenancy_record` # guards them instead), and the idempotency rows keyed by the caller's token org (a platform -# admin's replay row for its home org is neither refused nor placed). +# admin's replay row for its home org is not refused). Outside the guard is not outside the +# first attach: its org scan (`server/cli/cell_scan.py`) reads every table with an `org` +# column, these included, and places the orgs it finds. EXCLUDED = frozenset({"tenancy", "idempotency", "idempotency_fences"}) # A table's DDL waits this long for its lock a try, LOCK_ATTEMPTS tries LOCK_RETRY_S # apart: a boot never queues behind a long transaction (and every write of the table @@ -77,10 +79,13 @@ # What a trigger's DDL (SHARE ROW EXCLUSIVE) waited behind on a table, read after the # wait failed (pg_blocking_pids is empty by then): the locks held in a mode it conflicts # with (every mode but the two readers take), and the requests queued ahead of it in -# one, whose own blocker heads the queue. +# one, whose own blocker heads the queue. An anti-wraparound autovacuum, which never yields +# its lock, says so in its activity line (compared here, never returned). _HOLDERS = """ SELECT DISTINCT l.pid, l.granted, a.backend_type, a.state, a.application_name, - a.xact_start, a.wait_event + a.xact_start, a.wait_event, + coalesce(a.backend_type = 'autovacuum worker' + AND a.query LIKE '%(to prevent wraparound)', false) AS wraparound FROM pg_locks l JOIN pg_stat_activity a ON a.pid = l.pid WHERE l.relation = $1::regclass AND l.pid <> pg_backend_pid() AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database()) @@ -208,63 +213,107 @@ async def ensure(conn: Any) -> None: tries; a table whose lock it cannot take is logged with the holders of its locks, one the role cannot alter is logged as such, and either is skipped (the boot serves). A failed read of the state or the catalog raises. Runs on the migration session, which - holds the migration lock.""" + holds the migration lock. `ensure_table` runs one table's part alone.""" state = await cell.read_state(conn) if state == "open": return - # A table's statements, in one transaction: one lock wait, and every statement or none. - statements: dict[str, list[str]] = {} first = await _triggers(conn, CELL_FIRST_TRIGGERS) - for name in CELL_FIRST_TRIGGERS: - row = first.get(("cell_first", name)) - if row is not None and row["enabled"] != "A": - statements.setdefault("cell_first", []).append( - f"ALTER TABLE loom.cell_first ENABLE ALWAYS TRIGGER {name}" + always = [ + f"ALTER TABLE loom.cell_first ENABLE ALWAYS TRIGGER {name}" + for name in CELL_FIRST_TRIGGERS + if (row := first.get(("cell_first", name))) is not None and row["enabled"] != "A" + ] + if always: + await _run_ddl(conn, "cell_first", always) + if state != "enforced": + return + tables = await conn.fetch(ORG_TABLES) + for r in tables: + if r["relkind"] == "p": + # No trigger guards it: named here as missing() names it. + log.warning( + "cell triggers: loom.%s is partitioned: unsupported, unguarded", r["relname"] ) - if state == "enforced": - present = await _triggers(conn, (*ORG_TRIGGERS, TENANCY_TRIGGER)) - tables = await conn.fetch(ORG_TABLES) - for r in tables: - if r["relkind"] == "p": - # No trigger guards it: named here as missing() names it. - log.warning( - "cell triggers: loom.%s is partitioned: unsupported, unguarded", r["relname"] - ) - for r, names in _planned(tables): - table = r["relname"] - for name in names: - row = present.get((table, name)) - if row is None: - statements.setdefault(table, []).append(_create(f"loom.{_ident(table)}", name)) - elif row["enabled"] != "O": - statements.setdefault(table, []).append( - f"ALTER TABLE loom.{_ident(table)} ENABLE TRIGGER {name}" - ) - for table, sqls in statements.items(): - await _run_ddl(conn, table, sqls) + present = await _triggers(conn, (*ORG_TRIGGERS, TENANCY_TRIGGER)) + for r, names in _planned(tables): + if sqls := _table_statements(r["relname"], names, present): + await _run_ddl(conn, r["relname"], sqls) + + +async def planned_tables(conn: Any) -> list[str]: + """The tables `ensure_table` gives triggers: the guarded org tables and `loom.tenancy`.""" + return [r["relname"] for r, _ in _planned(await conn.fetch(ORG_TABLES))] + + +async def ensure_table( + conn: Any, + table: str, + *, + recording_first: bool = False, + lock_timeout: str = LOCK_TIMEOUT, + attempts: int = LOCK_ATTEMPTS, +) -> bool: + """`ensure` for one table of the planned set (`loom.tenancy` its guard, any other its + org triggers), so one table's DDL can run alone: in `enforced`, or with + `recording_first` also in a first cell's `recording`, where the org triggers' function + passes every row and a boot never creates them. False when the DDL was not applied (its + lock not taken in `attempts` tries of `lock_timeout`, a table this role cannot alter); + True when it was, or nothing was needed. Raises for a table outside the planned set, or + in another state.""" + state = await cell.read_state(conn) + if state != "enforced" and not (recording_first and state == "recording"): + raise RuntimeError(f"cell triggers: not created in state {state!r}") + planned = [ + (r, names) for r, names in _planned(await conn.fetch(ORG_TABLES)) if r["relname"] == table + ] + if not planned: + raise ValueError(f"loom.{table} carries no cell triggers") + ((_, names),) = planned + sqls = _table_statements(table, names, await _triggers(conn, names)) + return not sqls or await _run_ddl(conn, table, sqls, lock_timeout, attempts) + + +def _table_statements( + table: str, names: tuple[str, ...], present: dict[tuple[str, str], Any] +) -> list[str]: + """The DDL that gives `table` its named triggers, each present and enabled.""" + out = [] + for name in names: + row = present.get((table, name)) + if row is None: + out.append(_create(f"loom.{_ident(table)}", name)) + elif row["enabled"] != "O": + out.append(f"ALTER TABLE loom.{_ident(table)} ENABLE TRIGGER {name}") + return out -async def _run_ddl(conn: Any, table: str, sqls: list[str]) -> None: - """A table's DDL in one transaction, LOCK_ATTEMPTS short tries. The boot serves past - the two causes cells.md names (its lock not taken, a table this role cannot alter); - any other error fails it, as a failed read of the state does.""" +async def _run_ddl( + conn: Any, + table: str, + sqls: list[str], + lock_timeout: str = LOCK_TIMEOUT, + attempts: int = LOCK_ATTEMPTS, +) -> bool: + """A table's DDL in one transaction, LOCK_ATTEMPTS short tries; whether it applied. The + boot serves past the two causes cells.md names (its lock not taken, a table this role + cannot alter); any other error fails it, as a failed read of the state does.""" import asyncpg - for attempt in range(LOCK_ATTEMPTS): + for attempt in range(attempts): try: async with conn.transaction(): - await conn.execute(f"SET LOCAL lock_timeout = '{LOCK_TIMEOUT}'") + await conn.execute(f"SET LOCAL lock_timeout = '{lock_timeout}'") for sql in sqls: await conn.execute(sql) - return + return True except asyncpg.exceptions.InsufficientPrivilegeError as e: log.warning( "cell triggers: not applied on loom.%s (%s): this role cannot alter it", table, e ) - return + return False except asyncpg.exceptions.LockNotAvailableError as e: # Short tries outlast a short transaction; an autovacuum is left to finish. - if attempt < LOCK_ATTEMPTS - 1: + if attempt < attempts - 1: await asyncio.sleep(LOCK_RETRY_S) continue log.warning( @@ -274,13 +323,19 @@ async def _run_ddl(conn: Any, table: str, sqls: list[str]) -> None: "; ".join(sqls), await _holders(conn, table), ) + return False + + +async def holders(conn: Any, table: str) -> list[Any]: + """The sessions a blocked DDL on `table` waits behind (`_HOLDERS`).""" + return list(await conn.fetch(_HOLDERS, f"loom.{_ident(table)}")) async def _holders(conn: Any, table: str) -> str: """The sessions a blocked DDL waited behind, never their query text (a literal in it may be memory content).""" try: - rows = await conn.fetch(_HOLDERS, f"loom.{_ident(table)}") + rows = await holders(conn, table) except Exception as e: # noqa: BLE001 — naming the holders is best effort log.warning("cell triggers: the lock holders of loom.%s not read: %s", table, e) return "lock holders not read" diff --git a/server/core/instances.py b/server/core/instances.py index 27a35d8ec6..bb47bfd945 100644 --- a/server/core/instances.py +++ b/server/core/instances.py @@ -53,6 +53,10 @@ _LIVE_INTERVALS = 3 # How often the leader drops the KeeperMap keys of processes gone an hour. _PRUNE_EVERY_S = 600.0 +# How old a row's last beat is before the leader drops its key. A contract across builds, +# not a tunable: attach reads stopped rows back that far (`read_recent`) while the leader +# that sweeps them may run another build, so a change is a design change of its own. +PRUNE_AFTER_S = 3600 # (pid, instance id, start time) of this process. _process: tuple[int, str, datetime] | None = None @@ -130,6 +134,22 @@ def live_window_s() -> float: return _LIVE_INTERVALS * float(settings.instance_heartbeat_seconds) +def widened_window_s() -> float: + """The live window plus what a stopping process still does after its stop beat: it + drains its ClickHouse writes, then settles cut-off ones. A process whose last beat, + stopped or not, is younger than this may still write.""" + from server.boot import SHUTDOWN_DRAIN_BUDGET_S + from server.insights.app.provider_barrier import SHUTDOWN_RETIRE_S + + return live_window_s() + SHUTDOWN_DRAIN_BUDGET_S + SHUTDOWN_RETIRE_S + + +def boot_allowance_s() -> float: + """How long a booting process takes to report its first cell row and its first router + answer: two beats and one router read.""" + return 2 * float(settings.instance_heartbeat_seconds) + float(settings.cell_router_timeout_s) + + async def beat(ch: Any, *, stopped: bool = False) -> None: """Write this process's row. A KeeperMap insert replaces the key.""" await ch.insert(TABLE, [row(stopped=stopped)]) @@ -197,6 +217,34 @@ async def read_live(ch: Any, *, all_deployments: bool = False) -> list[dict[str, return out +async def read_recent(ch: Any, *, since: str | None = None) -> list[dict[str, Any]]: + """Every process of every deployment sharing this ClickHouse whose widened window + (`widened_window_s`) reaches past `since` (a ClickHouse `DateTime64(3)` as text), or + past now without one: stopped rows included, which `read_live` drops. `stopped` is its + newest row's flag and `live` whether that row is inside the live window and not + stopped, both judged in ClickHouse.""" + rows = await ch.query_all( + f""" + SELECT instance_id, + argMax(keys_fp, heartbeat_at) AS keys_fp, + argMax(stopped, heartbeat_at) AS stopped, + toString(max(heartbeat_at)) AS last_beat, + max(heartbeat_at) > now64(3) - toIntervalMillisecond({{live:UInt32}}) + AND stopped = 0 AS live + FROM {TABLE} + GROUP BY instance_id + HAVING max(heartbeat_at) + toIntervalMillisecond({{wide:UInt32}}) + > if({{since:String}} = '', now64(3), toDateTime64({{since:String}}, 3)) + """, + params={ + "live": int(live_window_s() * 1000), + "wide": int(widened_window_s() * 1000), + "since": since or "", + }, + ) + return [dict(r) for r in rows] + + def live_rows() -> list[dict[str, Any]]: """The live rows as of the last read; none before the first read or once that read is older than the live window (ClickHouse unreachable).""" @@ -224,14 +272,15 @@ async def _prune(ch: Any) -> None: # Each table's sweep stands alone: one failing leaves the other to run. try: await ch.execute( - "ALTER TABLE loom.instances DELETE WHERE heartbeat_at < now64(3) - toIntervalHour(1)" + "ALTER TABLE loom.instances DELETE " + f"WHERE heartbeat_at < now64(3) - toIntervalSecond({PRUNE_AFTER_S})" ) except Exception as e: # noqa: BLE001 — a later sweep retries log.warning("instance row sweep failed: %s", e) try: await ch.execute( "ALTER TABLE loom.cell_heartbeats DELETE " - "WHERE heartbeat_at < now64(3) - toIntervalHour(1)" + f"WHERE heartbeat_at < now64(3) - toIntervalSecond({PRUNE_AFTER_S})" ) except Exception as e: # noqa: BLE001 — a later sweep retries log.warning("cell heartbeat sweep failed: %s", e) @@ -331,8 +380,10 @@ def _reset_for_tests() -> None: __all__ = [ "CELL_TABLE", + "PRUNE_AFTER_S", "TABLE", "beat", + "boot_allowance_s", "cell_beat", "cell_row", "instance_id", @@ -341,8 +392,10 @@ def _reset_for_tests() -> None: "live_window_s", "publish_config", "read_live", + "read_recent", "schema_version", "start", "stop", "tick", + "widened_window_s", ] diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py new file mode 100644 index 0000000000..406d277646 --- /dev/null +++ b/tests/integration/test_cell_attach.py @@ -0,0 +1,635 @@ +"""`loom cell attach` (server/cli/cell_attach.py) against a real Postgres with a synchronous +standby, a real ClickHouse and the router served in process: a prepared first cell in +`recording` with orgs in both stores attaches to `serving` with every org placed and the +triggers created; a rerun on `serving` exits 0; a refusal after step 3 leaves `enforced` and +a rerun resumes from there; a second run is refused at the attach lock; the Postgres reads +attach makes refuse what they name (an unchecked writer's setting, a subscription, a role +that can make an org held, a transaction that may have read `recording`). + +The cell's Postgres and the directory are databases of the test's own; the configured +ClickHouse databases are scratch ones; the cell's processes are heartbeat rows the test +writes. Needs LOOM_PG_DSN (a server with a synchronous standby), LOOM_ROUTER_PG_DSN and +LOOM_CH_HOST.""" + +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import json +import os +import uuid +from collections.abc import AsyncIterator +from typing import Any + +import httpx +import pytest + +from server.cli import cell as cell_cli +from server.cli import cell_attach, cell_attach_pg, cell_prepare +from server.cli.cell_ch import Refused +from server.core import cell, cell_probe, cell_triggers, instances, pg +from server.core.ch import get_client +from server.core.schema import pg_migrate +from server.router import app as router_app +from server.router import directory +from server.settings import settings +from tests.integration._pg_database import create_database, drop_database + +OPERATOR = "lco_" + "2" * 128 +CREDENTIAL_SHA = hashlib.sha256(b"lcr_first").hexdigest() +ROUTER = "http://127.0.0.1:7901" +HOST = "loom.example" + + +class _Network(httpx.AsyncBaseTransport): + """What attach and the router reach: the router at ROUTER, the shared host's health + route, the cell's upstream main port (:7777) and its probe listener (any other).""" + + def __init__(self) -> None: + self.listener = httpx.ASGITransport(app=cell_probe.listener_app()) + self.router = httpx.ASGITransport( + app=router_app.build_app(router_app.Router(probe_transport=self)) + ) + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + if request.url.host == "127.0.0.1" and request.url.port == 7901: + return await self.router.handle_async_request(request) + if request.url.host == HOST or request.url.port == 7777: + return httpx.Response(200, json=cell_probe.answer()) + return await self.listener.handle_async_request(request) + + +async def _admin(sql: str, *args: Any) -> Any: + import asyncpg + + conn = await asyncpg.connect(os.environ["LOOM_PG_DSN"]) + try: + return await conn.execute(sql, *args) + finally: + await conn.close() + + +@pytest.fixture +async def cell_stores(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[dict[str, Any]]: + cell_dsn = os.environ.get("LOOM_PG_DSN") or pytest.skip("LOOM_PG_DSN not set") + router_dsn = os.environ.get("LOOM_ROUTER_PG_DSN") or pytest.skip("LOOM_ROUTER_PG_DSN not set") + os.environ.get("LOOM_CH_HOST") or pytest.skip("LOOM_CH_HOST not set") + cell_db, own_cell = await create_database(cell_dsn) + router_db, own_router = await create_database(router_dsn) + deployment = f"dep-{uuid.uuid4().hex[:8]}" + tag = f"it-{uuid.uuid4().hex[:8]}-" + for name, value in { + "cell": "first", + "storage_authority": "postgres", + "pg_dsn": own_cell, + "document_s3_bucket": "", + "router_pg_dsn": own_router, + "router_shared_host": HOST, + "router_upstream_probe_url": "http://listener.internal:7790", + "router_upstream_url": "http://upstream.internal:7777", + "router_operator_sha256": hashlib.sha256(OPERATOR.encode()).hexdigest(), + "router_operator_token": OPERATOR, + "ch_cluster": "", + "public_base_url": f"https://{HOST}", + "oauth_issuer": "", + }.items(): + monkeypatch.setattr(settings, name, value) + await pg.close_pool() + monkeypatch.setattr(directory, "_pool", None) + # A step's poll interval and the waits stay short. + monkeypatch.setattr(cell_attach_pg, "POLL_S", 0.2) + ch = get_client() + await ch.connect() + ids: list[str] = [] + try: + await pg_migrate.migrate() + await pg.execute( + "INSERT INTO loom.authority (authority, epoch, deployment_id) " + "VALUES ('postgres', 1, $1)", + deployment, + ) + await directory.migrate() + await ch.insert( + "loom.storage_authority", + [{"authority": "postgres", "epoch": 10**12, "deployment_id": deployment}], + ) + cell.set_deployment_id(deployment) + yield {"ch": ch, "ids": ids, "tag": tag, "deployment": deployment, "db": cell_db} + finally: + cell.set_deployment_id("") + await ch.execute( + "ALTER TABLE loom.storage_authority DELETE WHERE deployment_id = {d:String} " + "SETTINGS mutations_sync = 2", + {"d": deployment}, + ) + for table in (instances.TABLE, instances.CELL_TABLE): + await ch.execute( + f"ALTER TABLE {table} DELETE WHERE has({{ids:Array(String)}}, instance_id)", + {"ids": ids}, + ) + await ch.execute( + "DELETE FROM loom.namespaces WHERE startsWith(org, {p:String})", {"p": tag} + ) + await ch.disconnect() + await pg.close_pool() + await directory.close() + await drop_database(cell_dsn, cell_db) + await drop_database(router_dsn, router_db) + + +def _value(**keys: Any) -> str: + reported = { + **cell.as_value(cell.mine()), + "serves": True, + "marked": True, + "router": True, + "router_why": "", + **keys, + } + return json.dumps(reported, sort_keys=True) + + +async def _beat( + stores: dict[str, Any], + instance_id: str, + *, + cell_name: str | None = "first", + mode: str = "recording", + stopped: bool = False, + **keys: Any, +) -> None: + """One process's heartbeat rows, as `instances.tick` writes them.""" + ch = stores["ch"] + if instance_id not in stores["ids"]: + stores["ids"].append(instance_id) + await ch.insert( + instances.TABLE, [{**instances.row(stopped=stopped), "instance_id": instance_id}] + ) + if cell_name is not None: + await ch.insert( + instances.CELL_TABLE, + [ + { + "instance_id": instance_id, + "cell": cell_name, + "tenancy_mode": mode, + "fingerprint": _value(**keys), + } + ], + ) + + +async def _prepare(stores: dict[str, Any], network: _Network) -> None: + pid = f"it-{uuid.uuid4().hex[:8]}" + await _beat(stores, pid, cell_name="first", mode="open") + args = argparse.Namespace( + cell="first", + host=HOST, + credential_sha256=CREDENTIAL_SHA, + probe_url="", + wait=0, + router=ROUTER, + ) + router = cell_cli.operator_router(args) + assert isinstance(router, httpx.URL) + assert await cell_prepare.prepare(stores["ch"], args, router, network) == 0 + stores["pid"] = pid + await _beat(stores, pid, mode="recording") + + +def _args(**over: Any) -> argparse.Namespace: + fields: dict[str, Any] = { + "cell": "first", + "wait": 20, + "scan_budget": 600, + "dry_run": False, + "ch_allow_user": [settings.ch_user or "default"], + "ch_allow_user_directory": [], + "ch_allow_object": [], + "ch_allow_replica": [], + "pg_allow_role": [], + "router": ROUTER, + } + return argparse.Namespace(**{**fields, **over}) + + +async def _attach(stores: dict[str, Any], network: _Network, **over: Any) -> int: + args = _args(**over) + url = cell_cli.operator_router(args) + assert isinstance(url, httpx.URL) + return await cell_attach.attach(stores["ch"], args, url, network) + + +async def _mode() -> str: + return str(await pg.fetchval("SELECT mode FROM loom.tenancy_mode")) + + +async def _placed() -> set[str]: + import asyncpg + + conn = await asyncpg.connect(settings.router_pg_dsn) + try: + rows = await conn.fetch("SELECT org FROM loom_router.placements WHERE cell = 'first'") + finally: + await conn.close() + return {r["org"] for r in rows} + + +async def _row() -> Any: + return await directory.first_cell_named("first") + + +@pytest.mark.asyncio +async def test_a_recording_first_cell_attaches_to_serving(cell_stores, capsys) -> None: + stores, network = cell_stores, _Network() + tag = stores["tag"] + await _prepare(stores, network) + capsys.readouterr() + # Orgs a recording cell holds: a tenancy row alone, a Postgres data row, a ClickHouse + # row, and one name no request can carry (skipped and named). + await pg.execute("INSERT INTO loom.tenancy (org) VALUES ($1)", tag + "signed-in") + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", tag + "pg") + await stores["ch"].insert( + "loom.namespaces", + [ + {"org": tag + "ch", "namespace": "n", "owner_email": ""}, + {"org": tag + "bad\x00org", "namespace": "n", "owner_email": ""}, + ], + ) + assert await _attach(stores, network) == 0 + report = json.loads(capsys.readouterr().out) + mine = {tag + "signed-in", tag + "pg", tag + "ch"} + assert report["state"] == "serving" + assert await _mode() == "enforced" + placed = await _placed() + assert {o for o in placed if o.startswith(tag)} == mine + held = {r["org"] for r in await pg.fetch("SELECT org FROM loom.tenancy")} + assert held == placed + assert any("clickhouse loom.namespaces" in k for k in report["skipped"]["malformed"]) + row = await _row() + assert (row.state, row.mode, row.placed) == ("serving", "enforced", len(placed)) + async with pg.dedicated() as conn: + assert await cell_triggers.missing(conn) == [] + # A rerun on `serving` is done. + assert await _attach(stores, network) == 0 + assert json.loads(capsys.readouterr().out)["done"] is True + + +@pytest.mark.asyncio +async def test_a_refusal_after_step_3_leaves_enforced_and_a_rerun_resumes( + cell_stores, monkeypatch, capsys +) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + org = stores["tag"] + "a" + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", org) + # A process with LOOM_CELL unset appears once the mode is committed: refused at once. + real = cell_attach.Run.durable + + async def durable_then_unset(self: Any) -> int: + xmax = await real(self) + await _beat(stores, f"it-unset-{uuid.uuid4().hex[:6]}", cell_name="") + return xmax + + monkeypatch.setattr(cell_attach.Run, "durable", durable_then_unset) + assert await _attach(stores, network) == 1 + assert "runs with LOOM_CELL unset" in capsys.readouterr().err + assert await _mode() == "enforced" + row = await _row() + assert (row.state, row.mode) == ("attaching", "enforced") + # Placed live once enforced; the rerun resumes from the durable read. + monkeypatch.setattr(cell_attach.Run, "durable", real) + for i in list(stores["ids"]): + if i.startswith("it-unset-"): + await _beat(stores, i, cell_name="first", mode="enforced") + await _beat(stores, stores["pid"], mode="enforced") + assert await _attach(stores, network) == 0 + assert org in await _placed() + assert (await _row()).state == "serving" + + +@pytest.mark.asyncio +async def test_a_second_run_is_refused_at_the_lock(cell_stores, capsys) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + async with pg.dedicated() as holder: + await cell_attach_pg.take_lock(holder, "the test") + assert await _attach(stores, network) == 1 + assert "attach lock: held by pid" in capsys.readouterr().err + assert await _mode() == "recording" + + +@pytest.mark.asyncio +async def test_a_dry_run_writes_nothing(cell_stores, capsys) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + capsys.readouterr() + org = stores["tag"] + "dry" + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", org) + assert await _attach(stores, network, dry_run=True) == 0 + report = json.loads(capsys.readouterr().out) + assert report["per_source"]["postgres loom.scope_generations"] == { + "count": 1, + "sample": [org], + } + assert await _mode() == "recording" + assert await _placed() == set() + assert await pg.fetchval("SELECT count(*) FROM loom.tenancy") == 0 + async with pg.dedicated() as conn: + assert any(d.kind == "absent" for d in await cell_triggers.missing(conn)) + + +@pytest.mark.asyncio +async def test_an_org_past_the_limit_refuses_before_any_tenancy_row(cell_stores, capsys) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + long = "o" * (cell.ORG_MAX_BYTES + 1) + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", long) + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ('org-ok', '*')") + assert await _attach(stores, network) == 1 + assert f"longer than {cell.ORG_MAX_BYTES} bytes" in capsys.readouterr().err + assert await _mode() == "recording" + assert await pg.fetchval("SELECT count(*) FROM loom.tenancy") == 0 + assert (await _row()).placed == 0 + + +@pytest.mark.asyncio +async def test_settings_and_subscriptions_that_pass_unchecked_writers_refuse( + cell_stores, capsys +) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + db = stores["db"] + role = f"it_role_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + try: + # Stored as first spelled: a fresh session that never set the placeholder. + await _admin(f'ALTER ROLE {role} IN DATABASE "{db}" SET "Loom.Tenancy_Checked" = \'on\'') + async with pg.dedicated() as conn: + setting = await conn.fetchval( + "SELECT setconfig::text FROM pg_db_role_setting s JOIN pg_roles r " + "ON r.oid = s.setrole WHERE r.rolname = $1", + role, + ) + assert "Loom.Tenancy_Checked" in setting + refusals = await cell_attach_pg.unmarked_refusals(conn, False) + assert [r.subject for r in refusals] == [f"postgres setting for {role} in {db}"] + # Another database's entry does not refuse. + await _admin(f'ALTER ROLE {role} IN DATABASE "{db}" RESET ALL') + await _admin( + f"ALTER ROLE {role} IN DATABASE postgres SET session_replication_role = 'replica'" + ) + async with pg.dedicated() as conn: + assert await cell_attach_pg.unmarked_refusals(conn, False) == [] + # A role that can insert a tenancy row, a column grant alone, refuses attach. + await pg.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + await pg.execute(f"GRANT INSERT (org) ON loom.tenancy TO {role}") + assert await _attach(stores, network) == 1 + err = capsys.readouterr().err + assert f"postgres role {role}: may insert into loom.tenancy" in err + assert await _mode() == "recording" + assert await _attach(stores, network, pg_allow_role=[role], dry_run=True) == 0 + finally: + await pg.execute(f"REVOKE ALL ON loom.tenancy FROM {role}") + await pg.execute(f"REVOKE ALL ON SCHEMA loom FROM {role}") + await _admin(f"ALTER ROLE {role} IN DATABASE postgres RESET ALL") + await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_a_disabled_subscription_refuses(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + name = f"it_sub_{uuid.uuid4().hex[:6]}" + await pg.execute( + f"CREATE SUBSCRIPTION {name} CONNECTION 'host=127.0.0.1 dbname=none' " + "PUBLICATION none WITH (connect = false, enabled = false)" + ) + try: + async with pg.dedicated() as conn: + refusals = await cell_attach_pg.unmarked_refusals(conn, False) + assert [str(r) for r in refusals] == [ + f"postgres subscription {name}: disabled: its apply worker fires no origin " + "trigger; drop it" + ] + finally: + await pg.execute(f"ALTER SUBSCRIPTION {name} SET (slot_name = NONE)") + await pg.execute(f"DROP SUBSCRIPTION {name}") + + +@pytest.mark.asyncio +async def test_step_4_waits_for_a_transaction_that_may_have_read_recording(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + async with pg.dedicated() as writer, pg.dedicated() as poller: + tx = writer.transaction() + await tx.start() + await writer.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ('w', '*')") + # X is read after a later transaction completed, as after the durable read. + await poller.execute("BEGIN; SELECT pg_current_xact_id(); COMMIT") + xmax = await cell_attach_pg.snapshot_xmax(poller) + with pytest.raises(Refused, match="may have read `recording`"): + await cell_attach_pg.wait_transactions(poller, xmax, cell_attach_pg.Wait(0.5)) + await tx.rollback() + # A session idle outside a transaction holds nothing: the wait passes. + await cell_attach_pg.wait_transactions(poller, xmax, cell_attach_pg.Wait(0.5)) + + +@pytest.mark.asyncio +async def test_the_session_checks_and_the_encoding(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + async with pg.dedicated() as conn: + assert await cell_attach_pg.direct_refusals(conn, "attach") == [] + assert await cell_attach_pg.encoding_refusals(conn) == [] + async with pg.dedicated(marked=False) as conn: + refusals = await cell_attach_pg.direct_refusals(conn, "attach") + assert [r.reason.split(":")[0] for r in refusals] == [ + f"attach's session loses {pg.TENANCY_MARKER}" + ] + name = f"it_latin_{uuid.uuid4().hex[:6]}" + await _admin( + f"CREATE DATABASE {name} ENCODING 'LATIN1' LC_COLLATE 'C' LC_CTYPE 'C' TEMPLATE template0" + ) + try: + import asyncpg + + from tests.integration._pg_database import with_database + + conn = await asyncpg.connect(with_database(os.environ["LOOM_PG_DSN"], name)) + try: + (refusal,) = await cell_attach_pg.encoding_refusals(conn) + assert "LATIN1, not UTF8" in refusal.reason + finally: + await conn.close() + finally: + await _admin(f"DROP DATABASE {name}") + + +@pytest.mark.asyncio +async def test_the_mode_commit_waits_out_a_for_share_holder(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + logged: list[str] = [] + + async def ok() -> bool: + return True + + async with pg.dedicated() as holder, pg.dedicated() as conn: + tx = holder.transaction() + await tx.start() + await holder.execute("SELECT mode FROM loom.tenancy_mode FOR SHARE") + + async def release() -> None: + await asyncio.sleep(3) + await tx.rollback() + + task = asyncio.create_task(release()) + await cell_attach_pg.commit_enforced(conn, ok, ok, cell_attach_pg.Wait(20), logged.append) + await task + assert await _mode() == "enforced" + assert logged == ["the mode's lock is busy; retrying"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "grant,why", + [ + ("GRANT INSERT (org) ON loom.tenancy TO {r}", "may insert into loom.tenancy"), + ("GRANT UPDATE (org) ON loom.tenancy TO {r}", "may update loom.tenancy (org)"), + ("GRANT UPDATE (mode) ON loom.tenancy_mode TO {r}", "may update loom.tenancy_mode (mode)"), + ("GRANT INSERT (mode) ON loom.tenancy_mode TO {r}", "may insert into loom.tenancy_mode"), + ( + "GRANT INSERT, TRUNCATE ON loom.tenancy_mode TO {r}", + "may insert into loom.tenancy_mode", + ), + ( + "GRANT TRIGGER ON loom.scope_generations TO {r}", + "holds TRIGGER on loom.scope_generations", + ), + ("GRANT CREATE ON SCHEMA public TO {r}", "may create in schema public"), + ("GRANT CREATE ON SCHEMA loom TO {r}", "may create in schema loom"), + ( + "GRANT SET ON PARAMETER session_replication_role TO {r}", + "may set session_replication_role", + ), + ( + "SET loom.tenancy_checked = 'off'; " + "GRANT ALTER SYSTEM ON PARAMETER loom.tenancy_checked TO {r}", + "may ALTER SYSTEM SET loom.tenancy_checked", + ), + ("GRANT pg_write_server_files TO {r}", "is a member of pg_write_server_files"), + # A member of the owner role, ADMIN only: it can grant itself the rest. + ("GRANT {owner} TO {r} WITH ADMIN TRUE, INHERIT FALSE, SET FALSE", "owns loom.tenancy"), + ], +) +async def test_each_right_that_makes_an_org_held_is_named( + cell_stores, grant: str, why: str +) -> None: + await _prepare(cell_stores, _Network()) + role = f"it_r_{uuid.uuid4().hex[:6]}" + owner = await pg.fetchval("SELECT current_user") + await _admin(f"CREATE ROLE {role} LOGIN") + try: + async with pg.dedicated() as conn: + await conn.execute("GRANT USAGE ON SCHEMA loom TO " + role) + for sql in grant.format(r=role, owner=owner).split("; "): + await conn.execute(sql) + holders = await cell_attach_pg.privilege_holders(conn) + assert owner in holders.own + assert any(why in w for w in holders.others.get(role, [])), holders.others + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_a_function_owner_counts_though_it_owns_no_table(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + role = f"it_f_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + try: + async with pg.dedicated() as conn: + await conn.execute(f"ALTER FUNCTION loom.tenancy_refuse_unheld() OWNER TO {role}") + holders = await cell_attach_pg.privilege_holders(conn) + await conn.execute("ALTER FUNCTION loom.tenancy_refuse_unheld() OWNER TO CURRENT_USER") + assert holders.others[role] == ["owns loom.tenancy_refuse_unheld()"] + finally: + await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_a_repeatable_read_snapshot_from_before_x_holds_step_4(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + async with pg.dedicated() as reader, pg.dedicated() as poller: + tx = reader.transaction(isolation="repeatable_read") + await tx.start() + await reader.fetchval("SELECT mode FROM loom.tenancy_mode") # its snapshot, no xid + await poller.execute("BEGIN; SELECT pg_current_xact_id(); COMMIT") + xmax = await cell_attach_pg.snapshot_xmax(poller) + with pytest.raises(Refused, match="may have read `recording`"): + await cell_attach_pg.wait_transactions(poller, xmax, cell_attach_pg.Wait(0.5)) + await tx.rollback() + + +@pytest.mark.asyncio +async def test_prepare_is_refused_while_attach_holds_the_lock(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + args = argparse.Namespace( + cell="first", + host=HOST, + credential_sha256=CREDENTIAL_SHA, + probe_url="", + wait=0, + router=ROUTER, + ) + router = cell_cli.operator_router(args) + assert isinstance(router, httpx.URL) + async with pg.dedicated() as holder: + await cell_attach_pg.take_lock(holder, "attach") + with pytest.raises(cell_prepare.Refused, match="attach lock: held by pid"): + await cell_prepare.prepare(cell_stores["ch"], args, router, _Network()) + + +@pytest.mark.asyncio +async def test_doctor_names_the_same_holders_settings_and_subscriptions(cell_stores) -> None: + from server.cli import doctor_cell + + await _prepare(cell_stores, _Network()) + role = f"it_d_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + try: + await pg.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + await pg.execute(f"GRANT INSERT ON loom.tenancy TO {role}") + await _admin(f"ALTER ROLE {role} SET session_replication_role = 'replica'") + lines = {n: (g, d) for n, g, d in await doctor_cell._writer_reads(())} + assert lines["cell_tenancy_privilege"][0] == "WARN" + assert f"postgres role {role}" in lines["cell_tenancy_privilege"][1] + assert lines["cell_unchecked_writers"][0] == "WARN" + allowed = {n: g for n, g, _d in await doctor_cell._writer_reads((role,))} + assert allowed["cell_tenancy_privilege"] == "PASS" + finally: + await pg.execute(f"REVOKE ALL ON loom.tenancy FROM {role}") + await pg.execute(f"REVOKE ALL ON SCHEMA loom FROM {role}") + await _admin(f"ALTER ROLE {role} RESET ALL") + await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_doctor_names_stray_orgs_smallest_tables_first(cell_stores) -> None: + from server.cli import doctor_cell + + stores = cell_stores + await _prepare(stores, _Network()) + tag = stores["tag"] + await pg.execute("INSERT INTO loom.tenancy (org) VALUES ($1)", tag + "held") + await pg.execute( + "INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", tag + "held" + ) + await pg.execute( + "INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", tag + "stray" + ) + lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 300)} + grade, detail = lines["cell_stray_orgs"] + assert grade == "WARN" and tag + "stray" in detail and tag + "held" not in detail + tight = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 0.001)} + assert tight["cell_stray_orgs_unchecked"][0] == "WARN" diff --git a/tests/integration/test_cell_scan.py b/tests/integration/test_cell_scan.py index d1892be846..c78bc40331 100644 --- a/tests/integration/test_cell_scan.py +++ b/tests/integration/test_cell_scan.py @@ -166,7 +166,7 @@ def walk(node: dict[str, Any]) -> None: ("Index Only Scan", "tenancy_pkey"), ("Index Only Scan", "tenancy_pkey"), ], scans - assert len(await cell_scan._pg_orgs("loom.tenancy", Budget(60))) == 20000 + assert len(await cell_scan.pg_orgs("loom.tenancy", Budget(60))) == 20000 # Every org-keyed table of the schema has the index the scan needs. names, refusals = await cell_scan.list_postgres(own_pg) assert refusals == [] diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py new file mode 100644 index 0000000000..015cebd948 --- /dev/null +++ b/tests/unit/test_cell_attach.py @@ -0,0 +1,682 @@ +"""`loom cell attach` (server/cli/cell_attach.py, cell_attach_beats.py, cell_attach_pg.py) on +fakes: every refusal the heartbeats, the Postgres reads and the router's answers make, the +waits and their budget, the placement batches and the rescan's watch. The real Postgres, +ClickHouse and router behaviour is tests/integration/test_cell_attach.py. CH-free.""" + +from __future__ import annotations + +import argparse +import json +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any + +import httpx +import pytest + +from server.cli import cell_attach, cell_scan +from server.cli import cell_attach_beats as beats +from server.cli import cell_attach_pg as apg +from server.cli.cell_ch import Refusal, Refused +from server.cli.cell_scan import Table +from server.core import cell, instances +from server.router.limits import ATTACH_BATCH, MAX_BODY_BYTES +from server.settings import settings + +FIELDS = {**cell.DEFAULTS, "deployment_id": "dep-1"} +CURRENT = cell.as_value(FIELDS) +NEXT = cell.as_value({**FIELDS, "oidc_client_id": "next"}) + + +def _value(**keys: Any) -> dict[str, Any]: + return {**CURRENT, "serves": True, "marked": True, "router": True, "router_why": "", **keys} + + +def _beat( + i: str = "p1", + *, + cell_name: str | None = "first", + live: bool = True, + stopped: bool = False, + mode: str = "recording", + value: dict[str, Any] | None = None, + keys_fp: str = "k", +) -> beats.Beat: + return beats.Beat( + instance_id=i, + live=live, + stopped=stopped, + keys_fp=keys_fp, + last_beat="2026-01-01 00:00:00.000", + cell=cell_name, + mode=mode if cell_name is not None else "", + value=(value if value is not None else _value()) if cell_name is not None else None, + ) + + +def _judge(found: list[beats.Beat], stopped_unset: beats.StoppedUnset = "refuse") -> Any: + return beats.judge(found, "first", (CURRENT, NEXT), stopped_unset) + + +def _reasons(verdict: beats.Verdict) -> list[str]: + return [str(r) for r in verdict.refusals] + + +# -- the heartbeats -------------------------------------------------------------------- + + +def test_a_healthy_cell_passes_and_reports_its_keys() -> None: + nxt = {**_value(), **NEXT} + verdict = _judge([_beat("a"), _beat("b", value=nxt)]) + assert (verdict.refusals, verdict.booting, verdict.live, verdict.keys_fps) == ([], [], 2, {"k"}) + + +def test_a_live_process_with_loom_cell_unset_or_another_cell_refuses_at_once() -> None: + verdict = _judge([_beat("a", cell_name=""), _beat("b", cell_name="other")], "wait") + assert _reasons(verdict) == [ + "process a: runs with LOOM_CELL unset: set LOOM_CELL=first on it", + "process b: runs with LOOM_CELL=other: set LOOM_CELL=first on it", + ] + + +def test_a_stopped_unset_process_refuses_or_is_waited_on_as_asked() -> None: + stopped = [ + _beat("a", cell_name="", live=False, stopped=True), + _beat("b", cell_name=None, live=False, stopped=True), + ] + refused = _judge(stopped, "refuse") + assert [r.subject for r in refused.refusals] == ["process a", "process b"] + assert "may still be writing" in refused.refusals[0].reason + waited = _judge(stopped, "wait") + assert (waited.refusals, waited.aging) == ([], ["a", "b"]) + + +def test_a_live_instance_with_no_cell_row_is_booting() -> None: + verdict = _judge([_beat("a", cell_name=None)]) + assert (verdict.refusals, verdict.booting) == ([], ["a"]) + + +def test_a_cell_row_without_the_current_keys_refuses_even_while_booting() -> None: + old = {k: v for k, v in _value().items() if k not in ("serves", "marked", "router")} + verdict = _judge([_beat("a", value=old, mode="")]) + assert _reasons(verdict) == [ + "process a: reports no serves, marked or router key: upgrade it to this release" + ] + + +@pytest.mark.parametrize( + "keys,booting,refused", + [ + ({"marked": False}, [], "lose the tenancy marker"), + ({"marked": None}, ["a"], ""), + ({"router": False, "router_why": "a 404"}, [], "cannot read its router: a 404"), + ({"router": None}, ["a"], ""), + # A workers-only pod places no org: it needs no router credential. + ({"serves": False, "router": False}, [], ""), + ({"serves": False, "router": None}, [], ""), + ], +) +def test_marked_and_router_refuse_or_wait( + keys: dict[str, Any], booting: list[str], refused: str +) -> None: + verdict = _judge([_beat("a", value=_value(**keys))]) + assert verdict.booting == booting + assert (refused in " ".join(_reasons(verdict))) and bool(verdict.refusals) == bool(refused) + + +def test_no_tenancy_mode_yet_is_booting() -> None: + # A pod whose first cell-state read failed: mode '', marked and router null. + verdict = _judge([_beat("a", mode="", value=_value(marked=None, router=None))]) + assert (verdict.refusals, verdict.booting) == ([], ["a"]) + + +def test_a_fingerprint_on_neither_current_nor_next_refuses() -> None: + drifted = {**_value(), **cell.as_value({**FIELDS, "oidc_client_id": "drifted"})} + assert _reasons(_judge([_beat("a", value=drifted)])) == [ + "process a: runs neither loom.cell_self's current nor its next configuration" + ] + + +def test_the_final_refusals() -> None: + scim = {**_value(), **cell.as_value({**FIELDS, "scim": True})} + verdict = beats.judge( + [_beat("a", value=scim)], "first", (cell.as_value({**FIELDS, "scim": True}),), "refuse" + ) + assert [r.reason for r in beats.final_refusals(verdict)] == [ + "SCIM is on: attach with SCIM on needs the per-cell SCIM release" + ] + two = _judge([_beat("a"), _beat("b", keys_fp="other")]) + assert "root-secret fingerprints" in beats.final_refusals(two)[0].reason + assert "no live process" in beats.final_refusals(_judge([]))[0].reason + + +class _Clock: + def __init__(self) -> None: + self.now = 0.0 + + def __call__(self) -> float: + return self.now + + +@pytest.fixture +def clock(monkeypatch: pytest.MonkeyPatch) -> _Clock: + c = _Clock() + + async def _sleep(s: float) -> None: + c.now += s + + monkeypatch.setattr(apg, "_sleep", _sleep) + return c + + +@pytest.mark.asyncio +async def test_settle_waits_for_a_booting_process(clock: _Clock) -> None: + reads = [[_beat("a", cell_name=None)], [_beat("a", cell_name=None)], [_beat("a")]] + + async def read() -> list[beats.Beat]: + return reads.pop(0) + + wait = apg.Wait(10, clock) + verdict = await beats.settle(read, _judge, wait) + assert (verdict.live, wait.spent) == (1, 2.0) + + +@pytest.mark.asyncio +async def test_settle_refuses_at_the_end_of_wait_naming_what_it_waited_on(clock: _Clock) -> None: + async def read() -> list[beats.Beat]: + return [_beat("a", cell_name=None), _beat("b", cell_name="", live=False, stopped=True)] + + with pytest.raises(Refused) as e: + await beats.settle(read, lambda f: _judge(f, "wait"), apg.Wait(3, clock)) + assert [str(r) for r in e.value.refusals] == [ + "process a: still booting at the end of --wait", + "process b: stopped with LOOM_CELL unset, not aged out within --wait", + ] + + +@pytest.mark.asyncio +async def test_settle_extends_once_for_a_process_first_seen_booting(clock: _Clock) -> None: + seen = 0 + + async def read() -> list[beats.Beat]: + nonlocal seen + seen += 1 + # A stopped process aging out first; then one booting from the third read (2 s + # spent), done at the seventh, past the first deadline. + if seen < 3: + return [_beat("a"), _beat("x", cell_name="", live=False, stopped=True)] + return [_beat("a"), _beat("b", cell_name=None)] if seen < 7 else [_beat("a"), _beat("b")] + + wait = apg.Wait(3, clock) + verdict = await beats.settle(read, lambda f: _judge(f, "wait"), wait, extend_s=4) + assert (verdict.live, wait.seconds) == (2, 6) + + async def forever() -> list[beats.Beat]: + return [_beat("b", cell_name=None)] + + wait = apg.Wait(2, clock) + with pytest.raises(Refused, match="still booting"): + await beats.settle(forever, _judge, wait, extend_s=1) + # Seen booting from the start: first sight plus the extension is inside the deadline. + assert wait.seconds == 2 + + +@pytest.mark.asyncio +async def test_wait_counts_only_its_own_pauses(clock: _Clock) -> None: + wait = apg.Wait(2, clock) + clock.now += 100 # a scan's time + assert await wait.pause() and await wait.pause() + assert not await wait.pause() + assert wait.spent == 2.0 + + +# -- the rescan's watch ---------------------------------------------------------------- + + +def test_the_prune_contract_is_an_hour() -> None: + assert instances.PRUNE_AFTER_S == 3600 + + +@pytest.mark.asyncio +async def test_the_watch_refuses_every_unset_process_it_saw( + monkeypatch: pytest.MonkeyPatch, +) -> None: + rounds = [ + [ + _beat("a", cell_name=""), + _beat("c"), + _beat("d", cell_name=None, live=False, stopped=True), + ], + # a's rows were swept; c's cell row alone was swept: it was seen with the cell. + [_beat("c", cell_name=None, live=False, stopped=True)], + ] + + async def read_beats(_ch: Any, *, since: str | None = None) -> list[beats.Beat]: + assert since == "2026-01-01 00:00:00.000" + return rounds.pop(0) + + monkeypatch.setattr(beats, "read_beats", read_beats) + clock = _Clock() + watch = beats.Watch(None, "2026-01-01 00:00:00.000", "first", clock) + await watch.read_once() + await watch.read_once() + assert [r.subject for r in watch.refusals()] == ["process a", "process d"] + clock.now += beats.watch_period_s() + 1 + monkeypatch.setattr(beats, "read_beats", read_beats) + rounds.append([]) + await watch.read_once() + assert watch.refusals()[-1].subject == "heartbeats" + + +def test_the_watch_refuses_up_front_with_a_very_long_heartbeat( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(settings, "instance_heartbeat_seconds", 1180) + assert beats.Watch(None, "", "first").refusals_up_front() + monkeypatch.setattr(settings, "instance_heartbeat_seconds", 10) + assert beats.Watch(None, "", "first").refusals_up_front() == [] + + +# -- batches, ordering, arguments -------------------------------------------------------- + + +def test_batches_keep_the_count_and_the_byte_bound() -> None: + many = [f"org-{i:05d}" for i in range(ATTACH_BATCH + 1)] + assert [len(b) for b in cell_attach.batches(many, "dep")] == [ATTACH_BATCH, 1] + # Escaping grows a non-ASCII name six times: the encoded body is what counts. + wide = ["é" * 2000 + str(i) for i in range(10)] + out = cell_attach.batches(wide, "dep") + assert len(out) > 1 and sum(map(len, out)) == 10 + for b in out: + body = json.dumps({"deployment_id": "dep", "orgs": b}, separators=(",", ":")) + assert len(body.encode()) <= MAX_BODY_BYTES + + +def test_move_sources_are_scanned_first() -> None: + tables = [Table("loom", n, "MergeTree") for n in ("a", "memory_extraction_queue", "b", "src")] + ordered = cell_scan.order_sources(tables, {"loom.src"}) + assert [t.name for t in ordered] == ["memory_extraction_queue", "src", "a", "b"] + + +def _args(**over: Any) -> argparse.Namespace: + fields: dict[str, Any] = { + "cell": "first", + "wait": 10, + "scan_budget": 100, + "dry_run": False, + "ch_allow_user": [], + "ch_allow_user_directory": [], + "ch_allow_object": [], + "ch_allow_replica": [], + "pg_allow_role": [], + "router": "http://127.0.0.1:7900", + } + return argparse.Namespace(**{**fields, **over}) + + +def test_the_arguments(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "storage_authority", "postgres") + monkeypatch.setattr(settings, "pg_dsn", "postgresql://cell.internal/loom") + assert cell_attach._args_problem(_args()) == "" + assert "LOOM_CELL" in cell_attach._args_problem(_args(cell="other")) + assert "--scan-budget" in cell_attach._args_problem(_args(scan_budget=0)) + monkeypatch.setattr(settings, "storage_authority", "clickhouse") + assert "Postgres" in cell_attach._args_problem(_args()) + + +def test_an_org_past_the_limit_is_named_with_its_sources() -> None: + found = cell_scan.ScanResult() + found.add("postgres loom.tenancy", "x" * (cell.ORG_MAX_BYTES + 1)) + (refusal,) = cell_attach._too_long(found) + assert refusal.subject.startswith(f"org of {cell.ORG_MAX_BYTES + 1} bytes") + assert "found in postgres loom.tenancy" in refusal.reason + + +# -- the router ------------------------------------------------------------------------ + + +def _router(*responses: httpx.Response) -> cell_attach.Router: + answers = list(responses) + + def handler(_request: httpx.Request) -> httpx.Response: + return answers.pop(0) + + return cell_attach.Router( + httpx.URL("http://127.0.0.1:7900"), "first", httpx.MockTransport(handler) + ) + + +@pytest.mark.asyncio +async def test_the_router_calls_refuse_as_the_router_answers() -> None: + ok = {"cell": "first", "state": "attaching"} + assert await _router(httpx.Response(200, json=ok)).call("") == ok + with pytest.raises(Refused, match="not upgraded"): + await _router(httpx.Response(404, json={"error": "no route"})).call("/attach/place", {}) + with pytest.raises(Refused, match=r"refused /attach/begin \(409\): use loom cell update"): + await _router(httpx.Response(409, json={"error": "use loom cell update"})).call( + "/attach/begin", {} + ) + with pytest.raises(Refused, match="did not answer as the router does"): + await _router(httpx.Response(502, text="bad gateway")).call("") + + +# -- the Postgres reads, on a fake session --------------------------------------------- + + +@dataclass +class FakeConn: + """Answers by the first needle the SQL holds; a callable answer is called with the + arguments; an exception answer is raised.""" + + routes: list[tuple[str, Any]] = field(default_factory=list) + executed: list[str] = field(default_factory=list) + + def _answer(self, sql: str, args: tuple[Any, ...]) -> Any: + for needle, answer in self.routes: + if needle in sql: + if isinstance(answer, BaseException): + raise answer + return answer(*args) if callable(answer) else answer + raise AssertionError(f"unrouted: {sql[:80]}") + + async def fetch(self, sql: str, *args: Any) -> Any: + return self._answer(sql, args) + + async def fetchrow(self, sql: str, *args: Any) -> Any: + return self._answer(sql, args) + + async def fetchval(self, sql: str, *args: Any) -> Any: + return self._answer(sql, args) + + async def execute(self, sql: str, *args: Any) -> str: + self.executed.append(sql) + for needle, answer in self.routes: + if needle in sql: + if isinstance(answer, BaseException): + raise answer + return answer(*args) if callable(answer) else str(answer) + return "OK" + + def transaction(self) -> Any: + conn = self + + class _Tx: + async def __aenter__(self) -> None: + conn.executed.append("BEGIN") + + async def __aexit__(self, *exc: Any) -> None: + conn.executed.append("ROLLBACK" if exc[0] else "COMMIT") + + return _Tx() + + +def _seq(*values: Any) -> Callable[..., Any]: + pending = list(values) + return lambda *_a: pending.pop(0) + + +@pytest.mark.asyncio +async def test_the_session_checks_refuse_a_pooler() -> None: + moved = FakeConn( + [ + ("pg_backend_pid() AS pid", _seq((1, "t0"), (2, "t1"))), + ("current_setting($1, true)", None), + ] + ) + refusals = await apg.direct_refusals(moved, "attach") + assert [r.reason.split(":")[0] for r in refusals] == [ + "attach's session moved between backends", + "attach's session loses loom.tenancy_checked", + ] + direct = FakeConn([("pg_backend_pid() AS pid", (1, "t0")), ("current_setting($1, true)", "on")]) + assert await apg.direct_refusals(direct, "attach") == [] + assert "RESET ALL" in direct.executed + + +@pytest.mark.asyncio +async def test_the_lock_refuses_when_held_or_not_on_this_backend() -> None: + holder = [{"pid": 7, "application_name": "loom", "client": "10.0.0.1", "backend_start": "t"}] + held = FakeConn([("pg_try_advisory_lock", False), ("FROM pg_locks l LEFT JOIN", holder)]) + with pytest.raises(Refused, match="held by pid 7 \\(loom, from 10.0.0.1"): + await apg.take_lock(held, "attach") + pooled = FakeConn([("pg_try_advisory_lock", True), ("SELECT EXISTS", False)]) + with pytest.raises(Refused, match="a pooler"): + await apg.take_lock(pooled, "attach") + assert any("tcp_keepalives_idle" in s for s in pooled.executed) + + +@pytest.mark.asyncio +async def test_a_database_not_utf8_refuses() -> None: + assert await apg.encoding_refusals(FakeConn([("pg_encoding_to_char", "UTF8")])) == [] + (r,) = await apg.encoding_refusals(FakeConn([("pg_encoding_to_char", "SQL_ASCII")])) + assert "SQL_ASCII, not UTF8" in r.reason + + +def test_privilege_holders_outside_the_allowed_refuse() -> None: + holders = apg.Holders( + own=["loom"], + others={"ci": ["may insert into loom.tenancy"], "servers": ["owns loom.memory_current"]}, + ) + assert [r.subject for r in apg.privilege_refusals(holders, ["servers"])] == ["postgres role ci"] + assert apg.privilege_refusals(holders, ["ci", "servers"]) == [] + + +@pytest.mark.asyncio +async def test_settings_and_subscriptions_refuse() -> None: + conn = FakeConn( + [ + ( + "pg_db_role_setting", + [{"role": "ci", "db": "every database", "cfg": "Loom.Tenancy_Checked=on"}], + ), + ("pg_subscription", [{"subname": "mirror", "subenabled": False}]), + ] + ) + refusals = await apg.unmarked_refusals(conn, True) + assert [r.subject for r in refusals] == [ + "postgres unmarked session", + "postgres setting for ci in every database", + "postgres subscription mirror", + ] + + +@pytest.mark.asyncio +async def test_own_role_lacks_and_prepared_transactions_refuse() -> None: + conn = FakeConn( + [ + ("SELECT current_user", "attacher"), + ("WHERE NOT held", [{"need": "membership in pg_read_all_stats"}]), + ("has_table_privilege($1, 'SELECT')", lambda t: t != "loom.memory_current"), + ("max_prepared_transactions", 5), + ] + ) + refusals = await apg.own_role_refusals(conn, ["loom.memory_current", "loom.tenancy"]) + assert [str(r) for r in refusals] == [ + "postgres role attacher: attach's role lacks membership in pg_read_all_stats", + "postgres role attacher: attach's role lacks SELECT on loom.memory_current", + "postgres max_prepared_transactions: is 5: a prepared transaction outlives every " + "session, so step 4 could not wait for it; set it to 0", + ] + + +def _row(**over: Any) -> dict[str, Any]: + return { + "pid": 9, + "backend_type": "client backend", + "application_name": "ci", + "xact_start": "t", + "ours": True, + "xid_old": False, + "xmin_old": False, + "hidden": False, + **over, + } + + +@pytest.mark.asyncio +async def test_step_4_refuses_what_it_cannot_wait_for(clock: _Clock) -> None: + base = [("max_prepared_transactions", 0), ("pg_prepared_xacts", 0)] + hidden = FakeConn([*base, ("pg_stat_activity s", [_row(hidden=True)])]) + with pytest.raises(Refused, match="hidden from attach's role"): + await apg.wait_transactions(hidden, 100, apg.Wait(5, clock)) + prepared = FakeConn([("max_prepared_transactions", 0), ("pg_prepared_xacts", 1)]) + with pytest.raises(Refused, match="prepared transaction"): + await apg.wait_transactions(prepared, 100, apg.Wait(5, clock)) + held = FakeConn( + [ + *base, + ( + "pg_stat_activity s", + [_row(xmin_old=True), _row(pid=10, ours=False, xid_old=True), _row(pid=11)], + ), + ] + ) + with pytest.raises(Refused) as e: + await apg.wait_transactions(held, 100, apg.Wait(2, clock)) + assert [r.reason for r in e.value.refusals] == [ + "may have read `recording`: still open", + "another database's transaction, holding the snapshot horizon", + ] + done = FakeConn([*base, ("pg_stat_activity s", [_row(pid=10, ours=False, xid_old=True)])]) + await apg.wait_transactions(done, 100, apg.Wait(0, clock)) + + +class _Timeout(Exception): + sqlstate = "55P03" + + +@pytest.mark.asyncio +async def test_the_mode_commit_retries_a_busy_lock_and_ends_an_earlier_run(clock: _Clock) -> None: + tries = _seq(_Timeout(), "UPDATE 1") + + def lock(*_a: Any) -> str: + answer = tries() + if isinstance(answer, Exception): + raise answer + return "LOCK TABLE" + + holders = _seq( + [{"pid": 5, "application_name": "attach", "xact_start": "t", "rolsuper": False}], [] + ) + conn = FakeConn( + [ + ("LOCK TABLE", lock), + ("UPDATE loom.tenancy_mode", "UPDATE 1"), + ("l.mode = 'ExclusiveLock'", holders), + ("rolsuper FROM pg_roles", False), + ("pg_terminate_backend", True), + ] + ) + confirmed: list[bool] = [] + + async def confirm() -> None: + confirmed.append(True) + + async def ready() -> bool: + return True + + logged: list[str] = [] + await apg.commit_enforced(conn, confirm, ready, apg.Wait(10, clock), logged.append) + assert logged == ["the mode's lock is busy; retrying"] + # Before each try and the one termination. + assert len(confirmed) == 3 + + +@pytest.mark.asyncio +async def test_a_superusers_mode_transaction_is_named_not_ended() -> None: + conn = FakeConn( + [ + ( + "l.mode = 'ExclusiveLock'", + [{"pid": 5, "application_name": "psql", "xact_start": "t", "rolsuper": True}], + ), + ("rolsuper FROM pg_roles", False), + ] + ) + + async def confirm() -> None: + pass + + with pytest.raises(Refused, match="a superuser must end it"): + await apg.end_mode_transactions(conn, confirm) + + +@pytest.mark.asyncio +async def test_the_mode_commit_refuses_past_wait_and_skips_a_try_while_booting( + clock: _Clock, +) -> None: + conn = FakeConn([("l.mode = 'ExclusiveLock'", [])]) + + async def confirm() -> None: + pass + + async def booting() -> bool: + return False + + with pytest.raises(Refused, match="not taken within --wait"): + await apg.commit_enforced(conn, confirm, booting, apg.Wait(2, clock), lambda _t: None) + assert not any("LOCK TABLE" in s for s in conn.executed) + + +@pytest.mark.asyncio +async def test_the_durable_read_must_see_enforced_and_x_above_its_xid( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from server.core import pg_profile + + async def synchronous() -> str: + return "synchronous" + + monkeypatch.setattr(pg_profile, "standby_now", synchronous) + run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + run.lock_conn = FakeConn([("pg_snapshot_xmax", "100")]) + + async def read(mode: str, xid: int) -> apg.Durable: + return apg.Durable(mode=mode, first=(True, "first"), xid=xid) + + monkeypatch.setattr(apg, "durable_read", lambda: read("recording", 50)) + with pytest.raises(Refused, match="did not land"): + await run.durable() + monkeypatch.setattr(apg, "durable_read", lambda: read("enforced", 100)) + with pytest.raises(Refused, match="is not above 100"): + await run.durable() + monkeypatch.setattr(apg, "durable_read", lambda: read("enforced", 99)) + assert await run.durable() == 100 + + async def waiting() -> str: + return "waiting" + + monkeypatch.setattr(pg_profile, "standby_now", waiting) + with pytest.raises(Refused, match="not synchronous before"): + await run.durable() + + +@pytest.mark.asyncio +async def test_a_serving_row_is_done_only_once_the_cell_is_enforced( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from contextlib import asynccontextmanager + + from server.core import pg + + state = ["enforced"] + + @asynccontextmanager + async def dedicated(**_kw: Any) -> Any: + yield None + + async def read_mode(_conn: Any) -> tuple[str, Any]: + return state[0], (True, "first") + + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(apg, "read_mode", read_mode) + run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + assert await run.done() + state[0] = "recording" + with pytest.raises(Refused, match="run loom cell restore"): + await run.done() + + +def test_refusal_text() -> None: + assert str(Refusal("a", "b")) == "a: b" diff --git a/tests/unit/test_cell_org_triggers.py b/tests/unit/test_cell_org_triggers.py index 3f670d4014..decb365521 100644 --- a/tests/unit/test_cell_org_triggers.py +++ b/tests/unit/test_cell_org_triggers.py @@ -695,7 +695,7 @@ def test_doctor_runs_the_cell_checks_only_with_loom_cell( monkeypatch.setattr(doctor, "_doctor_scim_token_check", lambda _s: ("scim_token", "PASS", "")) called: list[bool] = [] monkeypatch.setattr( - doctor_cell, "cell_checks", lambda: called.append(True) or [("cell_state", "PASS", "")] + doctor_cell, "cell_checks", lambda *_: called.append(True) or [("cell_state", "PASS", "")] ) names = [n for n, _g, _d in doctor._run_live_checks()] assert called == ([True] if runs else []) and ("cell_state" in names) == runs diff --git a/tests/unit/test_cell_prepare.py b/tests/unit/test_cell_prepare.py index e9ea657df6..c978eb94e2 100644 --- a/tests/unit/test_cell_prepare.py +++ b/tests/unit/test_cell_prepare.py @@ -5,6 +5,7 @@ from __future__ import annotations import argparse +import contextlib import json from datetime import UTC, datetime from typing import Any @@ -29,6 +30,8 @@ def _cell(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(settings, "storage_authority", "postgres") monkeypatch.setattr(settings, "pg_dsn", "postgresql://cell.internal/loom") monkeypatch.setattr(settings, "router_operator_token", OPERATOR) + # The attach lock's session is Postgres's (tests/integration/test_cell_attach.py). + monkeypatch.setattr(cell_prepare, "attach_lock", contextlib.nullcontext) def _args(**over: Any) -> argparse.Namespace: diff --git a/tests/unit/test_doctor_client_checks.py b/tests/unit/test_doctor_client_checks.py index 554220a424..a2e1b0bd01 100644 --- a/tests/unit/test_doctor_client_checks.py +++ b/tests/unit/test_doctor_client_checks.py @@ -572,7 +572,7 @@ def test_doctor_client_flag_appends_the_lines_and_json_carries_them( ) -> None: from server.cli import doctor - monkeypatch.setattr(doctor, "_run_doctor_checks", lambda: [("python", "PASS", "3.12")]) + monkeypatch.setattr(doctor, "_run_doctor_checks", lambda *_: [("python", "PASS", "3.12")]) monkeypatch.setattr(doctor, "_doctor_problems", lambda: []) monkeypatch.setattr( doctor, diff --git a/tests/unit/test_doctor_client_client_only.py b/tests/unit/test_doctor_client_client_only.py index 60ceb323d6..712c10d7cd 100644 --- a/tests/unit/test_doctor_client_client_only.py +++ b/tests/unit/test_doctor_client_client_only.py @@ -56,7 +56,7 @@ def test_clickhouse_line_names_the_probed_host_on_failure(monkeypatch: pytest.Mo assert detail.startswith("https://ch.example.com:8443 → TimeoutError") -def _live() -> list[Check]: +def _live(*_roles: Any) -> list[Check]: return [ ("python_version", "PASS", "3.12"), ("clickhouse_reachable", "FAIL", "https://ch.example.com:8443 → TimeoutError: timed out"), From 1f059356679b1a5c20f36770a2bf70f1a702ab12 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 05:19:41 +0000 Subject: [PATCH 002/114] Attach's runbook, the cells.md first-attach steps and the trigger-cost gate measured end to end docs/operations/cell-attach.md is the operator's procedure: the preconditions attach checks and those it relies on, the flags, the point of no return at the mode commit, the orgs attach does not place and the manual removal of an org past the length limit, and doctor's reads after serving. cells.md describes the first attach as it runs (the lock, the refusals, the trigger pass, the mode commit and its durable read, step 4's wait, rescan and final checks), with its stated limits and failure rows. upgrades.md step 18 rotates the ClickHouse writer credential after the LOOM_CELL rollout. bench/cell_trigger_gate.py measures memory.set and a delivery-sized insert through an attached cell with the org triggers on and off; results.md records a local run. Attach compares the final root-secret fingerprint with the one its begin wrote, and the nonce cleanup's message names no single command. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 117 ++++++++++++ docs/architecture/cells.md | 290 +++++++++++++++++++---------- docs/benchmarks/results.md | 31 ++- docs/operations/cell-attach.md | 148 +++++++++++++++ docs/operations/postgres.md | 4 + docs/operations/upgrades.md | 8 + docs/reference/cli.md | 3 +- llms.txt | 1 + server/cli/cell_attach.py | 10 +- server/cli/cell_prepare.py | 2 +- tests/unit/snapshots/cli_help.json | 12 +- tests/unit/snapshots/cli_tree.json | 261 +++++++++++++++++++++++++- tests/unit/test_cell_attach.py | 17 ++ 13 files changed, 791 insertions(+), 113 deletions(-) create mode 100644 bench/cell_trigger_gate.py create mode 100644 docs/operations/cell-attach.md diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py new file mode 100644 index 0000000000..e90fbb32fa --- /dev/null +++ b/bench/cell_trigger_gate.py @@ -0,0 +1,117 @@ +"""The first attach's trigger-cost gate, end to end: `memory.set` p50 and p99 through a server +of an attached cell, and a delivery-sized 1,000-row insert of a held org from a marked and an +unmarked session, with the cell's org triggers on and dropped in turn (the `loom.tenancy` +guard stays). Run it against an attached cell's server, which must not serve anything else, +with a writer token for a placed org; it drops and recreates the cell's org triggers, so +never against a deployment in use. Needs no provider. + + PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py \\ + --base-url http://127.0.0.1:7932 --pg-dsn \\ + --token --org --namespace +""" + +from __future__ import annotations + +import argparse +import asyncio +import statistics +import time +import uuid + +import asyncpg +import httpx + +from server.core import cell_triggers + +CALLS, WARMUP, ROUNDS, INSERTS = 400, 50, 3, 20 + + +async def _triggers(dsn: str, on: bool) -> None: + """Create or drop the org triggers, and check that what was asked holds: every guarded + table carries both triggers enabled at origin, or none does.""" + conn = await asyncpg.connect(dsn) + try: + guarded = [t for t in await cell_triggers.planned_tables(conn) if t != "tenancy"] + for table in guarded: + for name in cell_triggers.ORG_TRIGGERS: + await conn.execute(f'DROP TRIGGER IF EXISTS {name} ON loom."{table}"') + if on: + await conn.execute(cell_triggers._create(f'loom."{table}"', name)) + enabled = await conn.fetchval( + "SELECT count(*) FROM pg_trigger WHERE tgenabled = 'O' AND tgname = ANY($1::text[])", + list(cell_triggers.ORG_TRIGGERS), + ) + finally: + await conn.close() + expected = 2 * len(guarded) if on else 0 + if enabled != expected: + raise SystemExit(f"{enabled} org triggers enabled, expected {expected}") + + +async def _memory_set(args: argparse.Namespace) -> tuple[float, float]: + """p50 and p99 in ms of CALLS sequential `memory.set` calls, each a new key.""" + lat = [] + headers = {"Authorization": f"Bearer {args.token}"} + async with httpx.AsyncClient(timeout=30, headers=headers) as client: + for i in range(CALLS + WARMUP): + body = { + "org": args.org, + "namespace": args.namespace, + "agent": "bench", + "memory_key": f"gate-{uuid.uuid4().hex[:10]}", + "value": f"trigger gate {i}", + } + t = time.perf_counter() + (await client.post(f"{args.base_url}/v1/memory.set", json=body)).raise_for_status() + if i >= WARMUP: + lat.append((time.perf_counter() - t) * 1000) + lat.sort() + return lat[len(lat) // 2], lat[int(0.99 * (len(lat) - 1))] + + +async def _batch(args: argparse.Namespace, marked: bool) -> float: + """The median in ms of INSERTS 1,000-row inserts of the held org, in one statement each.""" + settings = {"loom.tenancy_checked": "on"} if marked else {} + conn = await asyncpg.connect(args.pg_dsn, server_settings=settings) + try: + times = [] + for _ in range(INSERTS): + t = time.perf_counter() + await conn.execute( + "INSERT INTO loom.scope_generations (org, scope) " + "SELECT $1, 'gate-' || g || '-' || $2 FROM generate_series(1, 1000) g", + args.org, + uuid.uuid4().hex[:8], + ) + times.append((time.perf_counter() - t) * 1000) + return statistics.median(times) + finally: + await conn.close() + + +async def main() -> None: + p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + p.add_argument("--base-url", required=True) + p.add_argument("--pg-dsn", required=True) + p.add_argument("--token", required=True) + p.add_argument("--org", required=True) + p.add_argument("--namespace", required=True) + args = p.parse_args() + try: + for _ in range(ROUNDS): + for on in (True, False): + await _triggers(args.pg_dsn, on) + p50, p99 = await _memory_set(args) + marked, unmarked = await _batch(args, True), await _batch(args, False) + print( + f"triggers {'on ' if on else 'off'}: memory.set p50 {p50:.1f} ms " + f"p99 {p99:.1f} ms; 1,000-row insert {marked:.1f} ms marked, " + f"{unmarked:.1f} ms unmarked", + flush=True, + ) + finally: + await _triggers(args.pg_dsn, True) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 78e1d4c689..96580ec7d2 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -389,7 +389,8 @@ open meanwhile, as on a deployment never prepared, since the cell's Postgres, wh holds its state, cannot tell the two apart (ClickHouse carries no cell state: adding it would make the write path read a second store for tenancy) and such a process never reads the directory; where every process has `LOOM_CELL` unset, `loom doctor`'s naming -of it (which reads the directory, and ships with attach) is the only signal. A +of it (which reads the directory, and ships with the router's read-only scope) is the only +signal. A `(true, …)` row with `recording` and the own row: the first cell recording. Both present with the own row otherwise: `enforced`, a later cell's mode reading as `enforced` whatever its row says (its triggers, mode cache and CLI treat it so, and @@ -676,20 +677,18 @@ inserts cannot hold the write off, a new-org request waits on attach for a few s at most, and an attach that stalls holding the write's lock releases it within seconds. Attach takes `LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE` (which waits for running inserts and queues later ones behind it) in short tries, in one transaction with -server-side timeouts, then writes the mode and commits; after any error in that -transaction it ends that session's backend (`pg_terminate_backend`, the backend named by -the pid and `backend_start` attach captured when it connected) and waits until it has -left `pg_stat_activity`, then reads the mode back before it retries or reports; a mode -read back as `enforced` is written again in a fresh transaction, whose commit waits for -the standby, before attach goes on, since a commit ended while it waited is on the -primary alone. The attach command fixes those timeouts, and its tests cover each of -these cases. The mode cache reads the mode with a plain `SELECT`, which takes no row -lock and never waits on attach. An insert that finds `enforced` (a process whose cache +server-side timeouts, then writes the mode and commits; after a failed try it ends the +mode transaction still in flight, found by its `ExclusiveLock` on `loom.tenancy_mode` in +this database (a pid captured at connect is wrong behind a connection path that swaps +backends), and waits until the lock clears. Whether the commit landed is decided by a +durable read of the mode, whose commit waits for the standby, since a commit ended while +it waited is on the primary alone ([attaching a cell](#attaching-a-cell), steps 6 and 7). +The mode cache reads the mode with a plain `SELECT`, which takes no row lock and never +waits on attach. An insert that finds `enforced` (a process whose cache still reads `recording`) writes nothing: the request goes on to the placement call as in -`enforced`, and a CLI command refuses as in `enforced`, naming the org. A process whose -cache still reads `enforced` after step 3 or step 4 returns the cell to `recording` places -through the router's gate until its cache expires. -`loom doctor` lists any tenancy row the directory does not place. +`enforced`, and a CLI command refuses as in `enforced`, naming the org. A first attach +never lowers the mode once it commits `enforced`. `loom doctor`, with the router's +read-only scope, lists any tenancy row the directory does not place. - **It runs after verification, before anything is written for the org.** The op gate (`gates.authorize`) checks the org the transport resolves (`gates.resolve_org`, with @@ -983,51 +982,126 @@ refuses on any failure, naming it: revocation prefix, so a person the identity provider provisions later is admitted, since it will never deprovision a person it does not count as assigned. -The first attach turns the existing deployment into the first cell, in order: - -1. check the mode is `recording`, as prepare set it: every verified request has written - its org's tenancy row if absent while the directory answered that the cell is - `attaching`; -2. in one directory transaction, write `attach_started_at`, the cell's issuers, patterns - and fingerprint (both values of `loom.cell_self`), and the placements, then the - tenancy rows, of every org the cell holds, from one snapshot, straight into the - directory (no other cell exists yet): its configured orgs (of both values), the - distinct `org` of every table of both schemas (generated from the schema, as - erasure's table classification is), and the orgs of catalogued bucket objects; never - `''`, `'-'` or `'*'`; -3. set `enforced` as above, within `--wait`, which the wait below then shares, starting - no try that could commit the mode with less left than that wait's minimum (the mode - cache's lifetime plus the longest request deadline): attach then refuses, reporting - the mode it reads back (`recording`, and a rerun resumes from step 1), or the mode as - unknown when it cannot read it, and a rerun then resumes by the mode it reads; then - wait, within what remains of `--wait` (default 10 minutes, then refuse and set - `recording` back), until every live process reports `enforced` and then the mode - cache's lifetime plus the longest request deadline have passed; then run the SCIM - precondition again (`--revoke-unprovisioned` again if given), for accounts made - before `enforced` took hold; if it refuses, the cell stays `enforced` and - `attaching`, and running attach again resumes from the wait, as a resumed attach does - (below); -4. wait, within what remains of `--wait`, until no transaction that could have read - `recording` is still open, judged from a Postgres snapshot attach takes once it has - read `enforced` back in this run (not from a clock): a transaction on the cell's - database that started before it, from any client, background worker or logical - replication connection, read-only ones included, a writing transaction on another - database of the same server that did (it holds back every snapshot's horizon, and is - named as such), a prepared transaction (which refuses at once), or one whose state - attach cannot read counts as open, and Postgres's own maintenance processes, physical - replication connections (every standby's and a base backup's, which run no SQL) and - attach's own connections do not. Past `--wait` attach refuses, naming what it waited - on, and leaves the cell `enforced`, so a rerun resumes from step 3's wait. The attach - command fixes the exact query, and its tests cover each kind of transaction. Once the - wait passes, place the tenancy rows written since the snapshot and every org step 2's - sources yield when read again (the distinct `org` of every table of both schemas and - the orgs of catalogued bucket objects; a row or an object an unchecked writer, such - as an older CLI or a worker, wrote in `recording` after the snapshot is placed too), - check again that every live process reports `enforced` (refusing and returning to - `recording` as step 3 does if one does not), and mark the cell `serving`. The - directory records the highest mode attach has raised or tried to raise, which - `loom cell restore` writes back only upward: a cell whose step 3 was refused at the - lock is restored `enforced`, and its resumed attach then runs from step 3's wait. +The first attach (`loom cell attach`, its procedure on +[attaching the first cell](../operations/cell-attach.md)) turns the existing deployment +into the first cell. It aliases nothing (an `oauth_issuer_host` on another host refuses at +`attach/begin`), and refuses with SCIM on until per-cell SCIM ships. A run holds an +advisory lock, `cell.ATTACH_LOCK_KEY`, on a session of its own for its life (prepare and +`loom cell update` take the same lock), after checking that the session is one direct +backend that keeps the tenancy marker: a lock left on a pooler's server connection would +refuse every later run, and attach's tenancy rows would be refused as an unchecked +writer's. A lock found lost between steps is taken again (refusing when another run holds +it), and the cell's state is read again. The lock does not make a run safe, since every +step is idempotent and the mode only rises; it keeps a second run's scans from holding the +snapshot horizon that step 4 waits on. + +It refuses, before any write, naming each: + +- stores other than the registered ones: the Postgres deployment id, ClickHouse's + `loom.storage_authority` and the shared host's `/v1/health/deployment` each name the + directory row's deployment id, `loom.cell_first` reads `(true, )`, the row is the + first cell's, `attaching`, with no other directory row; +- a database encoding other than UTF8 (prepare refuses the same), so a tenancy row + within the length limit is always an org a request can name; +- a synchronous standby not attached now, a durability profile that would not refuse + without one, or a role that may not write the durable read's logical message; a role + of attach's own lacking its grants (`pg_read_all_stats` among them) or a server with + `max_prepared_transactions` above 0; +- **the tenancy-privilege check**: a login role, other than attach's own (superusers, + and a role that itself owns `loom.tenancy`, `loom.tenancy_mode` or a guarded table) and + those named with `--pg-allow-role`, that holds or reaches through any membership a right + to make an org held or lower the mode: `INSERT` on any column of `loom.tenancy` or + `UPDATE` of its `org`, `INSERT` on `loom.tenancy_mode` or `UPDATE` of its `mode`, + `TRIGGER` on a `loom` table, `CREATE` on the database or a schema that is not a system + one (a function there can shadow one the servers call unqualified on their marked + session), `SET` or `ALTER SYSTEM` on `session_replication_role`, `ALTER SYSTEM` on + `loom.tenancy_checked`, membership in `pg_write_server_files` or + `pg_execute_server_program`, or ownership of a `loom` table, of the trigger functions or + of `loom.cell_first_immutable()`, the database or a schema; +- **the unmarked-session check**: a session that sends no marker reads one, or runs as + `replica`; a `pg_db_role_setting` entry of `loom.tenancy_checked` or + `session_replication_role` for this database or every one, matched case-insensitively; + any logical-replication subscription into this database, enabled or not (its apply + worker fires no origin trigger); +- the heartbeats (both tables, stopped rows included for the widened window, the live + window plus a stopping pod's drain and retirement): a live process with `LOOM_CELL` + unset or another cell, one without the heartbeat's keys, `marked` false, `router` false + where `serves` is true, a fingerprint on neither `loom.cell_self` value, several + `keys_fp` values, or none live; a booting process (no cell row yet, no mode, `marked` + or `router` null) is waited on within `--wait`; in `recording` a stopped process with + `LOOM_CELL` unset refuses too; +- the ClickHouse writer check and the scan's table list (`server/cli/cell_writers.py`, + `server/cli/cell_scan.py`), and a settings profile or quota that caps what attach's user + reads. + +Then, in order: + +1. `attach/begin`, on every run, before any long step: the router probes the stores + attach wrote its nonces to, checks the upstream, and writes the fingerprints and + `keys_fp`; a directory restored to before it is written again. +2. Read the mode: `recording`, or `enforced` on a resume, which goes from the triggers, + through `attach/enforced` again, to the durable read. +3. Create the triggers itself, on every run, before step 2's scan and again just before the + mode commit (so a table a migration added during the scan is guarded from the commit + on): one table at a time under the migration lock, each DDL waiting 100 ms for its lock + and retried a second apart within `--wait`; an autovacuum holding it gets one try just + past `deadlock_timeout`, so it yields, and an anti-wraparound vacuum is named. Then + `cell_triggers.missing`; any defect refuses. A refused attach in `recording` leaves the + triggers, which pass every row there. +4. In `recording`, the scan: one statement per table, no long transaction (step 4 reads + every source again): the Postgres tables (`loom.tenancy` among them) by skip scans on an + `org`-led btree index, then the ClickHouse tables, each flushed and synced first, with + the configured orgs of both fingerprints. Log tables (`cell_scan.LOG_TABLES`) are not + read: a log row keeps the org a caller typed before any check, so placing a log-only + org would let anyone pin an org name to the cell; `loom doctor --stray-orgs` names such + orgs. An org longer than `cell.ORG_MAX_BYTES` refuses, naming it, before any tenancy row + is written; a name no request can carry is skipped and named. Then the tenancy rows of + the orgs found, on a marked session, and no placement: unscoped callers are answered + from the tenancy rows, and placing now would move the point of no return earlier. +5. The pre-commit checks: the heartbeats, the tenancy-privilege and unmarked-session + checks and the standbys again, and the scan's elapsed time within half of + `--scan-budget`, since step 4 reads the same tables again. Each refuses in + `recording`, which can still be backed out. +6. The mode commit: `attach/enforced`, then on a session of its own an EXCLUSIVE lock on + `loom.tenancy_mode`, 2 s a try within `--wait`, the attach lock confirmed and the + cheap heartbeat refusals run before each try (a booting process skips the try). After a + failed try, the mode transaction an earlier run left in flight is found by its lock + (only attach takes `ExclusiveLock` there; a relation's OID is matched in this database + alone) and ended. From the commit on the mode only rises: every later refusal leaves + the cell `enforced` and `attaching`, and a rerun resumes from the durable read. +7. The durable read, in every run that reads `enforced`: the mode and the `first` row read + in a transaction that emits a transactional logical message, whose commit waits for + the synchronous standbys, with the standbys checked attached just before and after. + Then `X`, the snapshot's xmax, which must exceed that message's xid. A READ COMMITTED + writer that read `recording` holds an xid below `X`, and a REPEATABLE READ one a + snapshot older than `X`: the trigger functions are `VOLATILE`, and the message's xid is + assigned after the mode commit. +8. The heartbeats again: a process with `LOOM_CELL` unset or another cell, without the + keys, unmarked, or with no router where it serves, refuses at once. No process depends + on its cached mode (a request records a new org through `tenancy._RECORD`, which reads + the mode `FOR SHARE`), so neither a cache lifetime nor a request deadline is waited out. +9. Step 4: wait, within `--wait`, while any backend of the cell's database but the polling + one has an xid older than `X`, or any but an autovacuum a snapshot older than `X` + (compared through `age()`, exactly `< X` and wraparound-safe); past `--wait` attach + refuses, naming each, and each backend of another database holding the horizon. A + prepared transaction, or a row of the cell's database whose state is hidden, refuses + at once. Then the cheap checks (the writer check, `missing`, the Postgres checks, the + heartbeats with every stopped process with `LOOM_CELL` unset aged out, the standbys); + the rescan of every source in full, Postgres before ClickHouse (the outbox and delivery + tables move rows to ClickHouse), and the queue and drain tables and the tables a view + reads before the others (`cell_scan.MOVE_SOURCES`); the tenancy rows; the placements, + in `attach/place` batches; the writer check again; and the final checks, under a floor + of their own (the larger of what `--wait` left and `instances.boot_allowance_s()`): + the heartbeats, read through the rescan every half of the time the leader keeps a + stopped row (`instances.PRUNE_AFTER_S`), refusing any process that ran with `LOOM_CELL` + unset during it, `loom.cell_self` equal to the directory's fingerprints, one `keys_fp` + equal to the one `begin` wrote, the mode `enforced`; then `attach/serving`. + +`--wait` (default 10 minutes) is shared by the waits alone; `--scan-budget` (default 2 +hours) bounds each full scan. `--dry-run` runs the refusals and the scan, prints what would +be placed and skipped and the scan's time, and writes nothing; the router's checks at +`attach/begin` do not run. On a `serving` row attach reads the cell's mode: `enforced` is +done (exit 0), anything else refuses, naming `loom cell restore`. A later cell is attached only once the first is `serving`. It must hold no org data (attach refuses otherwise) and starts in `enforced`. Its attach runs, in order: the @@ -1046,13 +1120,18 @@ cell with SCIM on admits no one until its first provisioning cycle has run. A ce nearing its stores' limits is closed to new orgs: its placement calls answer 503, so its issuers' new orgs wait for capacity rather than spill elsewhere. -A resumed first attach, for a cell whose directory row has `attach_started_at`, reruns -every refusal of the list above that applies to the first cell. It then continues by the -cell's mode: in `recording` from step 1; in `enforced`, from step 3's wait (every live -process reports `enforced`, then the cache's lifetime and the longest request deadline -pass) and then its SCIM precondition, so the SCIM check that ends an attach stopped -after setting `enforced`, whatever stopped it, always runs after `enforced` has taken -hold. +A resumed first attach reruns every refusal and continues by the directory row and the +cell's mode: in `recording` from the scan, in `enforced` from the durable read; whenever +the cell reads `enforced` it posts `attach/enforced` again, so a directory restored below +the cell's mode is raised. + +The checks of the stores' writers are point-in-time: a grant, a role setting, a +subscription, a ClickHouse writer or object added after `serving` passes them, as does a +configuration file attach cannot read (`postgresql.conf`, `postgresql.auto.conf`, a +standby's own) or a role that runs code as the server's OS user. `loom doctor` runs the +tenancy-privilege and unmarked-session reads again on every run and warns on what it finds +(`cell_tenancy_privilege`, `cell_unchecked_writers`); the runbook has the operator confirm +the rest. ## What each cell runs @@ -1102,9 +1181,9 @@ closed: every cell keeps serving the orgs it holds, routing continues from each replica's signed snapshot of its last cache (written to shared storage on every directory change), and the placements, lookups of uncached orgs, `loom cell` commands that read or write the directory and the recording cells' state reads answer 503 or -refuse until the directory is back, while `loom doctor` reports its directory checks as -not run and, with `LOOM_CELL` set, exits non-zero; rebuilding a lost directory from the -cells is a design of its own. +refuse until the directory is back, while `loom doctor`, with the router's read-only +scope, reports its directory checks as not run and, with `LOOM_CELL` set, exits +non-zero; rebuilding a lost directory from the cells is a design of its own. Each cell keeps, in its own Postgres, a copy of its directory row's configuration fingerprint (`loom.cell_self`), which its processes compare their configuration with and @@ -1200,30 +1279,36 @@ the directory row. ## Failure modes -| Failure | What happens | -| ------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| The directory is down | requests by host, by issuer and to the first cell are routed; new orgs and uncached lookups answer 503 | -| The router's placement call fails | the request answers 503 and nothing is written; orgs the cell holds are served | -| A cell's issuer places orgs in a flood | from `enforced` on, its placement calls answer 503 with `Retry-After` past its rate, and its calls for new orgs answer 503 past its quota if one is set; other cells place as before | -| A SCIM token of another cell, or an application at the wrong cell's host | the process does not boot, or `/scim/v2` answers 401 and writes nothing | -| A cell's Postgres or ClickHouse | that cell fails as a single deployment does ([storage authority](storage-authority.md#failure)); other cells serve | -| A cell's own rows or tenancy mode are unreadable | with `LOOM_CELL` set a process keeps its last rows and mode within the stale bound, then is gated, a server process whose first read fails does not boot (gated and unready with Postgres migrations pending), and a workers child that never read them is gated; with it unset a server process keeps its verdict with no bound, and a failed first read fails its boot (gated until a read lands with Postgres migrations pending) | -| A `first` or mode row removed, or both beside a tenancy or own row | the org triggers, where a cell has them, act as in `enforced` and every process is gated; `loom doctor` names it, a prepare rerun writes them back | -| No state, tenancy or own row on a cell the directory registers | open; `loom doctor` names it; `loom cell restore` closes it after a restore, a prepare rerun after a stopped prepare | -| A cell's `first` value is not yet read | every request but health probes and the probe route answers 503 (SCIM with `Retry-After`); the process is unready and runs no worker (a workers child, or a server process with Postgres migrations pending; any other server process fails its boot) | -| A request for another cell's org | 409 `WRONG_CELL` after verification, nothing written | -| A platform, `*`-org or principal-less call for an unheld org | on a first cell, 503 while the directory answers `attaching` or cannot be read before a `serving` answer; 409 `WRONG_CELL` once `serving` and on a later cell | -| A pooler that strips the tenancy marker from a cell's sessions | the `cell_marker` readiness check fails in `enforced` and `partial` and warns in `recording`; the heartbeat reports `marked` false and attach refuses the process | -| A cell restored to an earlier time | `loom cell restore` writes placed orgs' rows back before it starts | -| A first cell restored to before its prepare, not yet attached | at processes with `LOOM_CELL` set, every request but health probes and the probe route answers 503 until prepare reruns | -| The directory lost | every cell serves the orgs it holds; placements and uncached lookups answer 503, directory commands refuse, until it is back | -| A cell in `recording`, the directory unreadable | a new org answers 503; the orgs it holds keep serving | -| A process stuck during first attach | the attach refuses after `--wait` and the cell returns to `recording` | -| A recording insert held open during first attach | attach stops trying once too little of `--wait` is left for the wait, reporting the mode it reads back; a rerun resumes | -| Attach stalls while it holds the mode table's lock | Postgres ends its session within seconds and releases the lock; new orgs answer within their deadline | -| A transaction open since before step 4's snapshot | attach refuses once `--wait` runs out (a prepared one at once), naming what it waited on, and leaves `enforced` | -| A CLI command with `LOOM_CELL` unset, started before attach's step 3 | it read the state while the first cell was `recording` and keeps writing ClickHouse after `serving`; the runbook sets `LOOM_CELL` on every CLI before attach | -| A CLI of a build before the state read | it is not refused on an `enforced` cell; every CLI and CI image runs a build with the read before attach ([upgrades](../operations/upgrades.md#upgrade-procedure)) | +| Failure | What happens | +| ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| The directory is down | requests by host, by issuer and to the first cell are routed; new orgs and uncached lookups answer 503 | +| The router's placement call fails | the request answers 503 and nothing is written; orgs the cell holds are served | +| A cell's issuer places orgs in a flood | from `enforced` on, its placement calls answer 503 with `Retry-After` past its rate, and its calls for new orgs answer 503 past its quota if one is set; other cells place as before | +| A SCIM token of another cell, or an application at the wrong cell's host | the process does not boot, or `/scim/v2` answers 401 and writes nothing | +| A cell's Postgres or ClickHouse | that cell fails as a single deployment does ([storage authority](storage-authority.md#failure)); other cells serve | +| A cell's own rows or tenancy mode are unreadable | with `LOOM_CELL` set a process keeps its last rows and mode within the stale bound, then is gated, a server process whose first read fails does not boot (gated and unready with Postgres migrations pending), and a workers child that never read them is gated; with it unset a server process keeps its verdict with no bound, and a failed first read fails its boot (gated until a read lands with Postgres migrations pending) | +| A `first` or mode row removed, or both beside a tenancy or own row | the org triggers, where a cell has them, act as in `enforced` and every process is gated; `loom doctor` names it, a prepare rerun writes them back | +| No state, tenancy or own row on a cell the directory registers | open; `loom doctor` names it; `loom cell restore` closes it after a restore, a prepare rerun after a stopped prepare | +| A cell's `first` value is not yet read | every request but health probes and the probe route answers 503 (SCIM with `Retry-After`); the process is unready and runs no worker (a workers child, or a server process with Postgres migrations pending; any other server process fails its boot) | +| A request for another cell's org | 409 `WRONG_CELL` after verification, nothing written | +| A platform, `*`-org or principal-less call for an unheld org | on a first cell, 503 while the directory answers `attaching` or cannot be read before a `serving` answer; 409 `WRONG_CELL` once `serving` and on a later cell | +| A pooler that strips the tenancy marker from a cell's sessions | the `cell_marker` readiness check fails in `enforced` and `partial` and warns in `recording`; the heartbeat reports `marked` false and attach refuses the process | +| A cell restored to an earlier time | `loom cell restore` writes placed orgs' rows back before it starts | +| A first cell restored to before its prepare, not yet attached | at processes with `LOOM_CELL` set, every request but health probes and the probe route answers 503 until prepare reruns | +| The directory lost | every cell serves the orgs it holds; placements and uncached lookups answer 503, directory commands refuse, until it is back | +| A cell in `recording`, the directory unreadable | a new org answers 503; the orgs it holds keep serving | +| A process stuck during first attach | the attach refuses after `--wait` and the cell returns to `recording` | +| A recording insert held open during first attach | the mode commit's tries wait behind its `FOR SHARE` lock within `--wait`; past it attach refuses and the cell stays `recording`; a rerun resumes | +| Attach stalls while it holds the mode table's lock | Postgres ends its session within seconds and releases the lock; new orgs answer within their deadline | +| A transaction open since before step 4's snapshot | attach refuses once `--wait` runs out (a prepared one at once), naming what it waited on, and leaves `enforced` | +| A CLI command with `LOOM_CELL` unset, started before attach's step 3 | it read the state while the first cell was `recording` and keeps writing ClickHouse after `serving`; the runbook sets `LOOM_CELL` on every CLI before attach, and `loom doctor --stray-orgs` names the orgs it left without a tenancy row | +| A CLI of a build before the state read | it is not refused on an `enforced` cell; every CLI and CI image runs a build with the read before attach ([upgrades](../operations/upgrades.md#upgrade-procedure)) | +| An org found only in a log table of the first cell | never placed; once it is placed in another cell, its log rows here (`agent_events` identities among them) are outside its erasure there until the stray-org erase command removes them; `loom doctor --stray-orgs` names it | +| An org longer than `cell.ORG_MAX_BYTES` written before the limit | the first attach refuses in its scan, naming it, until the runbook's manual procedure removes it; one first found in step 4 is skipped and named | +| A writer, grant, role setting or subscription added after `serving` | attach's checks ran once and pass it; `loom doctor` warns on the Postgres ones on every run | +| `LOOM_SERVES_TRAFFIC=false` on a pod a load balancer routes to | attach does not require its router answer; its new orgs answer 503 | +| A process that changes between attach's last check and the mode commit, or an HA manager that empties `synchronous_standby_names` between the checks around the durable read | not seen by attach: a window of seconds | +| A cell process with a longer `LOOM_INSTANCE_HEARTBEAT_SECONDS` than attach's | it can fall outside attach's live window while live; the runbook requires one value | ## Rolling upgrades @@ -1306,11 +1391,16 @@ from its cache ([upgrades](../operations/upgrades.md#upgrade-procedure)). `loom doctor` names), with their functions in `schemas/pg_migrations/011_tenancy_triggers.sql` and `012_tenancy_guard.sql`; `server/cli/doctor_cell.py`: `loom doctor`'s cell checks; -- `server/cli/cell_scan.py`: the first attach's org scan, a library no command calls: the - tables of both stores with an `org` column, judged by engine and against erasure's - classification, every host of a cluster listing the same, and the distinct orgs of each - table outside `LOG_TABLES`; `server/cli/cell_writers.py`: the ClickHouse writer check (the - users, user directories, objects and replicas outside the operator's allow lists); +- `server/cli/cell_attach.py`: `loom cell attach`; `server/cli/cell_attach_pg.py`: its + Postgres reads and writes (the session checks, the attach lock, the tenancy-privilege + and unmarked-session checks doctor shares, the trigger pass, the mode commit, the + durable read, step 4's wait); `server/cli/cell_attach_beats.py`: its judgment of the + heartbeats and the rescan's watch; +- `server/cli/cell_scan.py`: the first attach's org scan: the tables of both stores with + an `org` column, judged by engine and against erasure's classification, every host of a + cluster listing the same, and the distinct orgs of each table outside `LOG_TABLES`; + `server/cli/cell_writers.py`: the ClickHouse writer check (the users, user directories, + objects and replicas outside the operator's allow lists); `server/cli/cell_ch.py`: what both share (the cluster's hosts, the per-host reads); - `schemas/pg_migrations/008_cells.sql`, `schemas/pg_migrations/009_cell_probe.sql` and `schemas/migrations/094_cell_heartbeats.sql`: the cell's tables; @@ -1386,14 +1476,14 @@ An integration test with two cells and a router in CI's Postgres job: CLI censuses cover the rest); - the first attach places every org with a row in any table, an org first seen during the attach is placed in step 4, and so is one a direct Postgres insert wrote between - step 2's snapshot and `enforced`, a ClickHouse row an older CLI wrote then and a + step 2's scan and `enforced`, a ClickHouse row an older CLI wrote then and a bucket object catalogued then; a process past `LOOM_JWT_REVOCATION_MAX_STALE_SECONDS` of failed reads of its own rows or mode is gated and serves again once a read lands; a pod with `LOOM_CELL` unset is unready, naming the setting, answers 503 to a bearer request, passes its health probe, answers the probe route and runs no worker on an attached cell and on a later cell between prepare and attach, keeps serving on a first cell in `recording`, and keeps the last mode it read through a failed read, - `recording` included after step 3 or step 4 returned the cell to it; a CLI command + `recording` included; a CLI command declaring an org with it unset is refused on an `enforced` or `partial` cell, an org-free one (a token revocation) runs, also with Postgres down on a deployment never prepared, and a `loom cell` store command refuses; `loom cell limit --closed` succeeds diff --git a/docs/benchmarks/results.md b/docs/benchmarks/results.md index 07367d3688..81e099d3c8 100644 --- a/docs/benchmarks/results.md +++ b/docs/benchmarks/results.md @@ -1220,8 +1220,35 @@ a row for the state read and the `loom.tenancy` lookup. Loom's own writes are ma this predicts the ship gate (a `memory.set` p99 rise of at most 10 ms, a delivery pass at most 10% slower) holds by a wide margin; the cost lands on scripts and older builds writing a cell without the check. This measures the triggers on the statements, not a -whole `memory.set` through a server on a cell: the gate itself is measured end to end -with attach. +whole `memory.set` through a server on a cell; the end-to-end gate follows. + +`bench/cell_trigger_gate.py` measures the gate end to end, locally: a first cell attached +by `loom cell attach` (two server processes, a router, Postgres 16 with a synchronous +standby and ClickHouse in Docker on a shared host under other load), 400 sequential +`memory.set` calls of a placed org through one server, and a delivery-sized 1,000-row insert +of the held org from a marked and an unmarked session, the org triggers on and dropped in +turn, three rounds each; the harness checks the triggers it asked for are enabled: + +```bash +PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py --base-url \ + --pg-dsn --token --org --namespace +``` + +| Local, three rounds | Triggers on | Triggers off | +| ----------------------------------- | ---------------------- | ---------------------- | +| `memory.set` p50 | 154.4, 99.3, 98.4 ms | 91.9, 101.2, 101.6 ms | +| `memory.set` p99 | 494.0, 291.7, 257.6 ms | 279.2, 244.7, 233.3 ms | +| 1,000-row insert, marked (median) | 11.0, 9.2, 8.5 ms | 9.7, 11.1, 10.6 ms | +| 1,000-row insert, unmarked (median) | 22.1, 21.4, 21.0 ms | 10.9, 9.6, 11.0 ms | + +The marked batch, which is what a delivery pass of a cell's own processes writes, shows no +cost the rounds can tell from zero (on minus off: +1.3, −1.9, −2.1 ms), within the 10% gate. +The `memory.set` figures move by more between rounds of one configuration than any trigger +cost: on this loaded host the p99 differences within a round (+214.8, +47.0, +24.3 ms +here, and −71.7, −10.0, −42.4 ms in an earlier run of the same measurement) have no +consistent sign, so this run cannot resolve a 10 ms p99 gate; it shows no rise beyond the +host's noise, and the statement-level figures above bound the marked cost at under a +microsecond a row. An unmarked session pays about 11 µs a row, as above. ## Reference computations diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md new file mode 100644 index 0000000000..12a0cbe247 --- /dev/null +++ b/docs/operations/cell-attach.md @@ -0,0 +1,148 @@ +# Attaching the first cell + +`loom cell attach` turns a prepared first cell (`loom cell prepare`, mode `recording`, +directory row `attaching`) into a `serving` one: it places every org the cell holds in the +router's directory, raises the cell's mode to `enforced` and creates its org triggers. This +page is the operator's procedure; the design is +[cells](../architecture/cells.md#attaching-a-cell). + +## Before attach + +Run attach at low load, as one run at a time, and not during a rollout, a provider +transition, a logical backup, a long erasure or a migration: each holds a transaction or a +snapshot long enough for step 4's wait to run out of `--wait`. Steps 2 and 4 run index skip +scans on the Postgres primary (each statement holds vacuum's horizon while it runs) and +read the `org` column of every ClickHouse table, which competes with serving load. + +Every item below is a precondition attach checks, or one it cannot check and relies on: + +1. Every router replica runs this release; every process and every CLI and CI image of + the cell runs it too ([upgrades](upgrades.md#upgrade-procedure), steps 16 and 17). +2. `LOOM_CELL` is set on every process of the cell. Every CLI and CI configuration that + writes for an org has `LOOM_CELL` set without `LOOM_CELL_ROUTER_TOKEN`: in `recording` + it then refuses only new orgs, so nothing has to change at the mode commit. With the + token removed, revoke the `INSERT` on `loom.tenancy` and the `UPDATE (since)` on + `loom.tenancy_mode` the CI role held for recording new orgs. +3. The ClickHouse writer credential is rotated after that rollout + ([upgrades](upgrades.md#upgrade-procedure), step 18). +4. No fingerprinted setting has changed since prepare, and none changes until attach ends: + a change gates every process against `loom.cell_self`. +5. `LOOM_SERVES_TRAFFIC=false` is set only on pods no load balancer routes to: attach + exempts them from the router check. +6. The cell's processes and the attach CLI share one `LOOM_INSTANCE_HEARTBEAT_SECONDS`: + attach judges which processes are live by its own value. +7. `LOOM_PG_DSN` points straight at Postgres, with no pooler of any mode, for the servers + and for attach: attach refuses a session that moves between backends or loses the + tenancy marker, and refuses a server whose heartbeat reports `marked` false. +8. SCIM is off: attach refuses with it on. +9. On a ClickHouse cluster of several hosts, attach's user exists on every host, and the + cluster has an interserver `` or a `remote_servers` user equal to attach's. + One replica that cannot sync blocks attach (`SYSTEM SYNC REPLICA … ON CLUSTER`): an + availability cost, not a safety one. +10. Only Loom's own roles and superusers hold the rights attach's tenancy-privilege check + counts (inserting a tenancy row, changing the mode, creating in a schema, triggers on + `loom` tables, owning them or their trigger functions, `session_replication_role`, + `ALTER SYSTEM` of `loom.tenancy_checked`, `pg_write_server_files`, + `pg_execute_server_program`), and no role that holds one, or owns what it counts, owns + a `SECURITY DEFINER` function another role can execute that does any of that. Attach + checks the catalog; it cannot read a function's body for intent, or a role that runs + code as the server's OS user. +11. `postgresql.conf`, `postgresql.auto.conf` and any pending `ALTER SYSTEM` entry, on the + primary and on every standby, set neither `loom.tenancy_checked` nor + `session_replication_role`. Attach sees a live entry on the primary (its unmarked + session reads it), never a standby's files. +12. Postgres has a synchronous standby attached (`synchronous_standby_names` set) and + `max_prepared_transactions = 0`; the database encoding is UTF8. +13. No org longer than 2048 bytes is in the cell's stores (below). `--dry-run` lists any. + +Attach's Postgres role needs `pg_read_all_stats`, `pg_signal_backend`, `USAGE` on schema +`loom`, `SELECT` on `loom.cell_first`, `loom.tenancy_mode`, `loom.cell_self`, +`loom.authority` and every org table, `UPDATE` on `loom.tenancy_mode`, `INSERT` on +`loom.tenancy`, `SELECT`, `INSERT`, `UPDATE` and `DELETE` on `loom.cell_probe`, `TEMPORARY` +on the database, `EXECUTE` on `pg_logical_emit_message`, and membership in the owner role +of `loom.tenancy` and of every guarded table. Its ClickHouse user needs the grants the +writer check names (`SELECT` on `system.*` and on the configured databases, `SHOW USERS`, +`SHOW ROLES`, `SHOW ROW POLICIES`, `SHOW TABLES`, `SHOW COLUMNS`, `SYSTEM FLUSH ASYNC INSERT +QUEUE`, and on a cluster `SYSTEM SYNC REPLICA`, `CLUSTER` and `READ ON REMOTE`), no settings +profile or quota capping the rows or bytes it reads, and the insert into +`loom._migration_lock` that writes the probe nonce. The CLI needs write access to the +bucket's probe object and `LOOM_ROUTER_OPERATOR_TOKEN`. + +## Run it + +```bash +loom cell attach --cell --dry-run \ + --ch-allow-user --pg-allow-role +loom cell attach --cell \ + --ch-allow-user --pg-allow-role +``` + +`--dry-run` runs the refusals and step 2's scan, prints the count and a sample of the orgs +per source table, the ones it would skip, and whether the scan fit in half of +`--scan-budget`, and writes nothing; the router's own checks at `attach/begin` (the probe, +the hosts, the upstream) do not run. Name every ClickHouse user, user directory, object +and replica that may write the configured databases with `--ch-allow-user`, +`--ch-allow-user-directory`, `--ch-allow-object` and `--ch-allow-replica`, and every +Postgres role besides attach's own that may make an org held with `--pg-allow-role`; each +list is audited by the router. `--wait` (default 600 s) bounds the waits together; +`--scan-budget` (default 7200 s) bounds each full scan, and step 2 must take at most half of +it, since step 4 reads the same tables again. + +A refusal names what to fix. Rerun attach after fixing it: every step is idempotent, and a +rerun resumes by the directory row and the cell's mode. On a `serving` row it prints that it +is done. + +## The point of no return + +Step 3's mode commit is one-way. Before it, a refused attach leaves the cell `recording`, +its tenancy rows and its triggers in place (they pass every row in `recording`), and the +cell can be backed out (the first-cell branch of +[troubleshooting](troubleshooting.md#a-process-with-loom_cell-unset-is-gated)): +`loom cell remove` serves only a first cell that never left `recording`. After it, scoped +principals place their orgs live within seconds, every refusal leaves the cell `enforced` +and `attaching`, and the cell goes forward: fix the cause and rerun attach. Unsetting +`LOOM_CELL` then gates every process, an outage, not a rollback. A row with `placed > 0` +always goes forward. + +From `enforced` on, a `--platform-admin` mint that declares a new org and the admin account +routes cannot bring in a new org (`WRONG_CELL`) until `loom cell place` is available; an +ordinary mint scoped to its org places the org on its first request. A direct call of the +router's operator place route, or a live placement with the cell's credential, cannot read +the cell's Postgres and is accepted once the directory mode is `enforced`; the cell's +credential is the servers' alone. + +## Orgs attach does not place + +- **An org longer than 2048 bytes** refuses step 2, naming it, before any tenancy row is + written. One first found in step 4 is skipped and named. Remove such an org by hand + until the stray-org erase command exists, in this order: its bucket objects, from the + keys of its rows in `loom.documents`, `loom.document_blobs` and `loom.blob_uploads`, so no + memory content is left outside erasure; then its rows in every table with an `org` column + of both ClickHouse databases and of Postgres schema `loom`, the erasure ledger tables + included; then its `loom.tenancy` row, on a session with `loom.tenancy_checked` set. +- **A name no request can carry** (not UTF-8, a NUL) found in a data table is skipped and + named; its rows stay outside placement. +- **An org found only in a log table** (`agent_events` and the tables derived from it, the + code search trace) is never placed: a log row keeps the org an unauthenticated caller + typed. Once that org is placed in another cell, its log rows here are outside its + erasure there. + +`loom doctor --stray-orgs` names the orgs the cell's stores hold without a tenancy row, +smallest tables first within `--stray-orgs-budget` (default 60 s), and every table it left +unchecked. `loom cell place` and `loom cell restore`, which act on these, are not available +yet. + +## After attach + +`loom doctor` runs attach's point-in-time checks again on every run: it warns on a Postgres +role outside the owner roles, the superusers and its own `--pg-allow-role` that can make an +org held or lower the mode (`cell_tenancy_privilege`), and on a role or database setting or +a subscription that passes an unchecked writer (`cell_unchecked_writers`). A grant or a +writer added after `serving` passes attach's checks, which ran once; doctor is how it is +found. + +## Related + +- [Cells](../architecture/cells.md) +- [Troubleshooting: cells](troubleshooting.md#cells) +- [Upgrades](upgrades.md) diff --git a/docs/operations/postgres.md b/docs/operations/postgres.md index d087a158c4..895086c474 100644 --- a/docs/operations/postgres.md +++ b/docs/operations/postgres.md @@ -44,6 +44,10 @@ pooler refuses or drops (its org triggers then refuse Loom's own writes for an o cell holds no row for, and the `loom.tenancy` guard its tenancy row for each newly placed org), and `loom doctor` names a session that lost it. Scripts and bulk loaders never set `session_replication_role = replica` on a cell: it skips the org triggers and the guard. +No role is granted `SET` or `ALTER SYSTEM` on `session_replication_role`, `ALTER SYSTEM` +on `loom.tenancy_checked`, or membership in `pg_write_server_files` or +`pg_execute_server_program` on a cell: `loom cell attach` refuses such a role, and +`loom doctor` warns on one ([attaching the first cell](cell-attach.md)). Every replica of the deployment, and its worker process, sets the same values. The full list, with the admission and delivery settings, is in the diff --git a/docs/operations/upgrades.md b/docs/operations/upgrades.md index 8caf8c789d..50dd5fc46a 100644 --- a/docs/operations/upgrades.md +++ b/docs/operations/upgrades.md @@ -320,6 +320,14 @@ checkout. answers them from one read per replica per `LOOM_ROUTER_DIRECTORY_CACHE_S` ([cells](../architecture/cells.md#the-heartbeats-keys)). +18. Before a first cell's attach, once `LOOM_CELL` is set on every process and every CLI and + CI configuration of the cell (step 16), rotate the ClickHouse credential they write + with: a new user or password for the cell's servers and CLIs, the old one dropped. No + CLI sends a heartbeat, so attach cannot see a job still configured with `LOOM_CELL` + unset; the rotation cuts off any the rollout missed before it can write an org's rows + past attach's last scan. Name the new user with `--ch-allow-user` + ([attaching the first cell](./cell-attach.md)). + While a migration applies, the old build keeps serving: every migration is additive. The rules that let the two builds serve side by side are in [rolling upgrades](../architecture/rolling-upgrades.md). diff --git a/docs/reference/cli.md b/docs/reference/cli.md index ff91f44681..12bac9a65b 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -315,7 +315,7 @@ python scripts/codegen/docs_reference.py | `loom bot create` | Register a bot: its namespaces, and a key (--generate-key or --public-key) and/or a CI trust (--issuer, --audience, --claim). | `--name`, `--org`, `--namespace-role`, `--generate-key`, `--public-key`, `--issuer`, `--audience`, `--claim`, `--jwks-uri`, `--max-token-age`, `--max-access-token-ttl`, `--max-agents`, `--agent-claim`, `--platform-admin`, `--replace` | | `loom bot list` | Every bot, active or disabled. | | | `loom bot disable` | Refuse its proofs and revoke the tokens it holds. | `--name` | -| `loom doctor` | Diagnose pre-boot misconfig (Python version, settings, JWT, CH reachability) and print the problem list /v1/health carries, each with its fix. | `--json`, `--client`, `--server-url`, `--token` | +| `loom doctor` | Diagnose pre-boot misconfig (Python version, settings, JWT, CH reachability) and print the problem list /v1/health carries, each with its fix. | `--json`, `--client`, `--server-url`, `--pg-allow-role`, `--stray-orgs`, `--stray-orgs-budget`, `--token` | | `loom report` | Print the bug form's Environment field: versions and the `loom doctor` checks. | `--open` | | `loom drops` | Print the memory ingest drop ledger (loom.memory_drops) for a namespace, by stage/reason — what the pipeline discarded, truncated or gave up on. | `--namespace`, `--org`, `--hours` | | `loom namespaces-gc` | List (default) or erase throwaway namespaces — finished bench runs under the bench prefixes that nobody registered and nobody has written to for --idle-hours — in one list-scoped erasure saga per… | `--org`, `--prefix`, `--namespace`, `--exclude-prefix`, `--idle-hours`, `--include-registered`, `--erase`, `--no-purge`, `--skip-external`, `--batch`, `--show`, `--requester`, `--worker-pass` | @@ -330,6 +330,7 @@ python scripts/codegen/docs_reference.py | `loom cell which` | Print the lookup's cell and host for an org (the first cell if not placed). | `--org`, `--router` | | `loom cell remove` | Remove the directory row of a cell attaching with no placement. | `--cell`, `--router` | | `loom cell prepare` | Register this cell in the router's directory and write its rows. | `--cell`, `--host`, `--credential-sha256`, `--probe-url`, `--wait`, `--router` | +| `loom cell attach` | Attach the prepared first cell: place its orgs, serve. | `--cell`, `--wait`, `--scan-budget`, `--dry-run`, `--ch-allow-user`, `--ch-allow-user-directory`, `--ch-allow-object`, `--ch-allow-replica`, `--pg-allow-role`, `--router` | | `loom tracker-probe` | Read-only first contact with the tracker source: counts and newest timestamps per upstream table per credential, breaker states, the configured tracker namespaces. | | | `loom tracker-contract-check` | Read-only: does each tracker source's schema meet the record contract (docs/features/tracker/record-contract.md)? | `--source`, `--namespace`, `--json` | | `loom tracker-dupcheck-fit` | Refit tracker.duplicate_check's calibrated confidence for one tracker namespace now: replays the duplicate gold at filing time, fits, stores the fit. | `--namespace`, `--org`, `--negatives`, `--dry-run`, `--out` | diff --git a/llms.txt b/llms.txt index ca465459d3..43b8421b0e 100644 --- a/llms.txt +++ b/llms.txt @@ -104,6 +104,7 @@ Start with [the docs map](docs/index.md). To install Loom, a coding agent reads - [Exporting usage to a data warehouse](docs/operations/analytics-export.md): `loom export-analytics` writes one UTC day of usage and a snapshot of the namespaces to an S3 bucket as Parquet files, one file per entity per day, for a data warehouse to load. - [Backup and recovery](docs/operations/backup-and-recovery.md): A Loom backup is a ClickHouse backup, plus the document bucket while any catalog row names an object in it, and under the Postgres authority a backup of Postgres too. +- [Attaching the first cell](docs/operations/cell-attach.md): `loom cell attach` turns a prepared first cell (`loom cell prepare`, mode `recording`, directory row `attaching`) into a `serving` one: it places every org the cell holds in the router's directory, raises the cell's mode to `enforced` and… - [ClickHouse Cloud and Managed Postgres](docs/operations/clickhouse-cloud.md): This guide runs Loom on Kubernetes over two managed services: ClickHouse Cloud for projections, vectors, history and analytics, and ClickHouse Managed Postgres as the memory authority. - [Code index operations](docs/operations/code-index-operations.md): This page covers how to enable the code index workers, configure repositories and keep the index current; the code index runs only when an operator enables its workers and configures repositories. - [Deployment](docs/operations/deployment.md): Loom is one server image plus ClickHouse (and Postgres, under `LOOM_STORAGE_AUTHORITY=postgres`), with the document extractor service as an image of its own (container images). diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 368f74ebf8..c8b72fb687 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -1,6 +1,6 @@ """`loom cell attach`: turn a prepared first cell into a `serving` one (docs/architecture/cells.md, "Attaching a cell"; the runbook is -docs/operations/cells.md). +docs/operations/cell-attach.md). A run holds the attach lock on a direct session of the cell's Postgres for its life, and goes, each step idempotent, so a rerun resumes by the directory row and the cell's mode: @@ -121,7 +121,7 @@ def _too_long(found: cell_scan.ScanResult) -> list[Refusal]: Refusal( f"org of {len(org.encode())} bytes ({org[:64]}…)", f"found in {', '.join(sorted(sources))}: longer than {cell.ORG_MAX_BYTES} bytes; " - "remove it first (docs/operations/cells.md, the over-long org procedure)", + "remove it first (docs/operations/cell-attach.md, the over-long org procedure)", ) for org, sources in sorted(found.too_long.items()) ] @@ -174,7 +174,8 @@ def __init__( self.row: dict[str, Any] = {} self.mode = "" self.own_fp: tuple[dict[str, Any] | None, dict[str, Any] | None] = (None, None) - self.keys_fp = "" + self.keys_fp = "" # the live processes' one root-secret fingerprint, last read + self.begun_keys_fp = "" # the one this run's `attach/begin` wrote self.report: dict[str, Any] = {"cell": self.name} # -- the lock session -------------------------------------------------------------- @@ -410,6 +411,7 @@ async def begin(self) -> None: try: await _write_nonces(self.ch, nonces) self.row = {**self.row, **await self.router.call("/attach/begin", body)} + self.begun_keys_fp = self.keys_fp finally: await _delete_nonces(self.ch, nonces) @@ -523,7 +525,7 @@ async def _read() -> list[beats.Beat]: verdict = await beats.settle( _read, self.judge_beats("refuse"), floor, extend_s=instances.boot_allowance_s() ) - if verdict.keys_fps != {self.keys_fp}: + if verdict.keys_fps != {self.begun_keys_fp}: raise _refused( "processes", "the root-secret fingerprint changed during attach (a root-secret rotation): " diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index cf83fa8f27..4a820f652b 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -286,7 +286,7 @@ async def _bucket() -> None: try: await step() except Exception as e: # noqa: BLE001 — every nonce is tried; a stale one holds no data - print(f"loom cell prepare: the probe nonce in {where} stays: {e}", file=sys.stderr) + print(f"loom cell: the probe nonce in {where} stays: {e}", file=sys.stderr) def _register_timeout() -> httpx.Timeout: diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json index 2567a90eb5..3e88dbafca 100644 --- a/tests/unit/snapshots/cli_help.json +++ b/tests/unit/snapshots/cli_help.json @@ -397,8 +397,8 @@ "options: -h, --help show this help message and exit --name NAME" ], "loom doctor": [ - "usage: loom doctor [-h] [--json] [--client] [--server-url SERVER_URL] [--token TOKEN]", - "options: -h, --help show this help message and exit --json Emit the checks and the problem list as JSON. --client Also examine this machine as a coding-agent client: the server the hooks would call, the token they would present, whether it is accepted and can write the intended namespace, the Claude Code / Codex wiring, and capture jobs that never reached the server. --server-url SERVER_URL Override the client's server URL. --token TOKEN Override the client's token." + "usage: loom doctor [-h] [--json] [--client] [--server-url SERVER_URL] [--pg-allow-role PG_ALLOW_ROLE] [--stray-orgs] [--stray-orgs-budget STRAY_ORGS_BUDGET] [--token TOKEN]", + "options: -h, --help show this help message and exit --json Emit the checks and the problem list as JSON. --client Also examine this machine as a coding-agent client: the server the hooks would call, the token they would present, whether it is accepted and can write the intended namespace, the Claude Code / Codex wiring, and capture jobs that never reached the server. --server-url SERVER_URL Override the client's server URL. --pg-allow-role PG_ALLOW_ROLE On a cell: a Postgres role that may make an org held (the servers' role). --stray-orgs On a cell: also read the orgs its stores hold without a tenancy row. --stray-orgs-budget STRAY_ORGS_BUDGET Seconds --stray-orgs may read for, smallest tables first (default 60). --token TOKEN Override the client's token." ], "loom report": [ "usage: loom report [-h] [--open]", @@ -434,8 +434,8 @@ "options: -h, --help show this help message and exit" ], "loom cell": [ - "usage: loom cell [-h] {credential,limit,which,remove,prepare} ...", - "positional arguments: {credential,limit,which,remove,prepare} credential Make a router credential, or rotate a cell's. limit Close, reopen or bound a cell to new orgs. which Print the lookup's cell and host for an org (the first cell if not placed). remove Remove the directory row of a cell attaching with no placement. prepare Register this cell in the router's directory and write its rows.", + "usage: loom cell [-h] {credential,limit,which,remove,prepare,attach} ...", + "positional arguments: {credential,limit,which,remove,prepare,attach} credential Make a router credential, or rotate a cell's. limit Close, reopen or bound a cell to new orgs. which Print the lookup's cell and host for an org (the first cell if not placed). remove Remove the directory row of a cell attaching with no placement. prepare Register this cell in the router's directory and write its rows. attach Attach the prepared first cell: place its orgs, serve.", "options: -h, --help show this help message and exit" ], "loom cell credential": [ @@ -459,6 +459,10 @@ "usage: loom cell prepare [-h] --cell CELL --host HOST --credential-sha256 SHA256 [--probe-url PROBE_URL] [--wait WAIT] [--router ROUTER]", "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --host HOST The host the cell serves its orgs at --credential-sha256 SHA256 The sha256 of the cell's router credential (`loom cell credential new`) --probe-url PROBE_URL A later cell's probe listener address (not the first's) --wait WAIT Seconds to retry a busy or unreachable probe listener --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" ], + "loom cell attach": [ + "usage: loom cell attach [-h] --cell CELL [--wait WAIT] [--scan-budget SCAN_BUDGET] [--dry-run] [--ch-allow-user CH_ALLOW_USER] [--ch-allow-user-directory CH_ALLOW_USER_DIRECTORY] [--ch-allow-object CH_ALLOW_OBJECT] [--ch-allow-replica CH_ALLOW_REPLICA] [--pg-allow-role PG_ALLOW_ROLE] [--router ROUTER]", + "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --wait WAIT Seconds the waits share --scan-budget SCAN_BUDGET Seconds one full org scan may take --dry-run Run the refusals and the scan and print what would be placed; write nothing (the router's own begin refusals do not run) --ch-allow-user CH_ALLOW_USER A ClickHouse user that may write the configured databases --ch-allow-user-directory CH_ALLOW_USER_DIRECTORY A ClickHouse user directory the check cannot read --ch-allow-object CH_ALLOW_OBJECT A ClickHouse object (db.name) that writes the databases --ch-allow-replica CH_ALLOW_REPLICA A replica outside the cluster (db.table:replica) --pg-allow-role PG_ALLOW_ROLE A Postgres role that may make an org held (the servers' role) --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" + ], "loom tracker-probe": [ "usage: loom tracker-probe [-h]", "options: -h, --help show this help message and exit" diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json index 2ad62a0df9..3d597c1e03 100644 --- a/tests/unit/snapshots/cli_tree.json +++ b/tests/unit/snapshots/cli_tree.json @@ -11213,6 +11213,9 @@ "json", "client", "server_url", + "pg_allow_role", + "stray_orgs", + "stray_orgs_budget", "token" ] } @@ -11280,6 +11283,51 @@ "help": "Override the client's server URL.", "metavar": null }, + { + "option_strings": [ + "--pg-allow-role" + ], + "dest": "pg_allow_role", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "On a cell: a Postgres role that may make an org held (the servers' role).", + "metavar": null + }, + { + "option_strings": [ + "--stray-orgs" + ], + "dest": "stray_orgs", + "action": "_StoreTrueAction", + "default": false, + "const": true, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "On a cell: also read the orgs its stores hold without a tenancy row.", + "metavar": null + }, + { + "option_strings": [ + "--stray-orgs-budget" + ], + "dest": "stray_orgs_budget", + "action": "_StoreAction", + "default": 60.0, + "const": null, + "choices": null, + "nargs": null, + "type": "builtins:float", + "required": false, + "help": "Seconds --stray-orgs may read for, smallest tables first (default 60).", + "metavar": null + }, { "option_strings": [ "--token" @@ -12903,6 +12951,211 @@ "metavar": null } ] + }, + "attach": { + "prog": "loom cell attach", + "usage": null, + "description": null, + "epilog": null, + "formatter_class": "argparse:HelpFormatter", + "add_help": true, + "allow_abbrev": true, + "defaults": { + "func": "server.cli.cell_attach:cmd_attach" + }, + "groups": [ + { + "title": "positional arguments", + "description": null, + "dests": [] + }, + { + "title": "options", + "description": null, + "dests": [ + "help", + "cell", + "wait", + "scan_budget", + "dry_run", + "ch_allow_user", + "ch_allow_user_directory", + "ch_allow_object", + "ch_allow_replica", + "pg_allow_role", + "router" + ] + } + ], + "mutually_exclusive": [], + "arguments": [ + { + "option_strings": [ + "-h", + "--help" + ], + "dest": "help", + "action": "_HelpAction", + "default": "==SUPPRESS==", + "const": null, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "show this help message and exit", + "metavar": null + }, + { + "option_strings": [ + "--cell" + ], + "dest": "cell", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": true, + "help": "This cell's name (LOOM_CELL)", + "metavar": null + }, + { + "option_strings": [ + "--wait" + ], + "dest": "wait", + "action": "_StoreAction", + "default": 600, + "const": null, + "choices": null, + "nargs": null, + "type": "builtins:int", + "required": false, + "help": "Seconds the waits share", + "metavar": null + }, + { + "option_strings": [ + "--scan-budget" + ], + "dest": "scan_budget", + "action": "_StoreAction", + "default": 7200, + "const": null, + "choices": null, + "nargs": null, + "type": "builtins:int", + "required": false, + "help": "Seconds one full org scan may take", + "metavar": null + }, + { + "option_strings": [ + "--dry-run" + ], + "dest": "dry_run", + "action": "_StoreTrueAction", + "default": false, + "const": true, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "Run the refusals and the scan and print what would be placed; write nothing (the router's own begin refusals do not run)", + "metavar": null + }, + { + "option_strings": [ + "--ch-allow-user" + ], + "dest": "ch_allow_user", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "A ClickHouse user that may write the configured databases", + "metavar": null + }, + { + "option_strings": [ + "--ch-allow-user-directory" + ], + "dest": "ch_allow_user_directory", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "A ClickHouse user directory the check cannot read", + "metavar": null + }, + { + "option_strings": [ + "--ch-allow-object" + ], + "dest": "ch_allow_object", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "A ClickHouse object (db.name) that writes the databases", + "metavar": null + }, + { + "option_strings": [ + "--ch-allow-replica" + ], + "dest": "ch_allow_replica", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "A replica outside the cluster (db.table:replica)", + "metavar": null + }, + { + "option_strings": [ + "--pg-allow-role" + ], + "dest": "pg_allow_role", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "A Postgres role that may make an org held (the servers' role)", + "metavar": null + }, + { + "option_strings": [ + "--router" + ], + "dest": "router", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "The router's URL (default: LOOM_CELL_ROUTER_URL)", + "metavar": null + } + ] } }, "aliases_of": { @@ -12910,7 +13163,8 @@ "limit": "loom cell limit", "which": "loom cell which", "remove": "loom cell remove", - "prepare": "loom cell prepare" + "prepare": "loom cell prepare", + "attach": "loom cell attach" }, "choices_help": [ { @@ -12937,6 +13191,11 @@ "dest": "prepare", "help": "Register this cell in the router's directory and write its rows.", "metavar": "prepare" + }, + { + "dest": "attach", + "help": "Attach the prepared first cell: place its orgs, serve.", + "metavar": "attach" } ] } diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 015cebd948..1c725205e6 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -680,3 +680,20 @@ async def read_mode(_conn: Any) -> tuple[str, Any]: def test_refusal_text() -> None: assert str(Refusal("a", "b")) == "a: b" + + +@pytest.mark.asyncio +async def test_a_root_secret_change_during_attach_refuses_at_the_end( + monkeypatch: pytest.MonkeyPatch, +) -> None: + run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + run.begun_keys_fp = "before" + + async def settle(*_a: Any, **_kw: Any) -> beats.Verdict: + return beats.Verdict(live=1, keys_fps={"after"}) + + monkeypatch.setattr(beats, "settle", settle) + with pytest.raises(Refused, match="a root-secret rotation"): + await run.final_checks(beats.Watch(None, "", "first")) + run.begun_keys_fp = "after" + await run.final_checks(beats.Watch(None, "", "first")) From 55c0918ae463b59d20c3aee84c3e74afd891e1fc Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 05:35:26 +0000 Subject: [PATCH 003/114] The widened heartbeat window lives with attach, so server.core imports no boot or insights module instances.read_recent takes the window it reads back; attach's heartbeat module computes it from the live window and a stopping process's drain and retirement budgets. PRUNE_AFTER_S is marked as the contract it is. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- server/cli/cell_attach_beats.py | 14 ++++++++++++-- server/core/instances.py | 25 ++++++++----------------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/server/cli/cell_attach_beats.py b/server/cli/cell_attach_beats.py index a2fc37701d..d1d938a46b 100644 --- a/server/cli/cell_attach_beats.py +++ b/server/cli/cell_attach_beats.py @@ -32,6 +32,16 @@ StoppedUnset = Literal["refuse", "wait"] +def widened_window_s() -> float: + """The live window plus what a stopping process still does after its stop beat: it + drains its ClickHouse writes, then settles cut-off ones. A process whose last beat, + stopped or not, is younger than this may still write.""" + from server.boot import SHUTDOWN_DRAIN_BUDGET_S + from server.insights.app.provider_barrier import SHUTDOWN_RETIRE_S + + return instances.live_window_s() + SHUTDOWN_DRAIN_BUDGET_S + SHUTDOWN_RETIRE_S + + @dataclass(frozen=True) class Beat: """One process: its instance row (`live`: inside the live window and not stopped) and @@ -48,7 +58,7 @@ class Beat: async def read_beats(ch: Any, *, since: str | None = None) -> list[Beat]: - rows = await instances.read_recent(ch, since=since) + rows = await instances.read_recent(ch, window_s=widened_window_s(), since=since) ids = [str(r["instance_id"]) for r in rows] cells = await ch.query_all( f"SELECT instance_id, cell, tenancy_mode, fingerprint FROM {instances.CELL_TABLE} " @@ -236,7 +246,7 @@ async def settle( def watch_period_s() -> float: """`T'`: how long a row whose window reaches past the rescan's start is kept before the leader's sweep may drop it, less a margin for clock skew.""" - return instances.PRUNE_AFTER_S - instances.widened_window_s() - _SKEW_S + return instances.PRUNE_AFTER_S - widened_window_s() - _SKEW_S class Watch: diff --git a/server/core/instances.py b/server/core/instances.py index bb47bfd945..80e3e76896 100644 --- a/server/core/instances.py +++ b/server/core/instances.py @@ -56,7 +56,7 @@ # How old a row's last beat is before the leader drops its key. A contract across builds, # not a tunable: attach reads stopped rows back that far (`read_recent`) while the leader # that sweeps them may run another build, so a change is a design change of its own. -PRUNE_AFTER_S = 3600 +PRUNE_AFTER_S = 3600 # not a setting: a contract across builds # (pid, instance id, start time) of this process. _process: tuple[int, str, datetime] | None = None @@ -134,16 +134,6 @@ def live_window_s() -> float: return _LIVE_INTERVALS * float(settings.instance_heartbeat_seconds) -def widened_window_s() -> float: - """The live window plus what a stopping process still does after its stop beat: it - drains its ClickHouse writes, then settles cut-off ones. A process whose last beat, - stopped or not, is younger than this may still write.""" - from server.boot import SHUTDOWN_DRAIN_BUDGET_S - from server.insights.app.provider_barrier import SHUTDOWN_RETIRE_S - - return live_window_s() + SHUTDOWN_DRAIN_BUDGET_S + SHUTDOWN_RETIRE_S - - def boot_allowance_s() -> float: """How long a booting process takes to report its first cell row and its first router answer: two beats and one router read.""" @@ -217,10 +207,12 @@ async def read_live(ch: Any, *, all_deployments: bool = False) -> list[dict[str, return out -async def read_recent(ch: Any, *, since: str | None = None) -> list[dict[str, Any]]: - """Every process of every deployment sharing this ClickHouse whose widened window - (`widened_window_s`) reaches past `since` (a ClickHouse `DateTime64(3)` as text), or - past now without one: stopped rows included, which `read_live` drops. `stopped` is its +async def read_recent( + ch: Any, *, window_s: float, since: str | None = None +) -> list[dict[str, Any]]: + """Every process of every deployment sharing this ClickHouse whose last beat plus + `window_s` reaches past `since` (a ClickHouse `DateTime64(3)` as text), or past now + without one: stopped rows included, which `read_live` drops. `stopped` is its newest row's flag and `live` whether that row is inside the live window and not stopped, both judged in ClickHouse.""" rows = await ch.query_all( @@ -238,7 +230,7 @@ async def read_recent(ch: Any, *, since: str | None = None) -> list[dict[str, An """, params={ "live": int(live_window_s() * 1000), - "wide": int(widened_window_s() * 1000), + "wide": int(window_s * 1000), "since": since or "", }, ) @@ -397,5 +389,4 @@ def _reset_for_tests() -> None: "start", "stop", "tick", - "widened_window_s", ] From 4a7bc8804793d7b6832af214f89209adcf3f7c71 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 05:51:59 +0000 Subject: [PATCH 004/114] ensure_table's integration test lands with attach, its caller Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- .../integration/test_cell_org_triggers_pg.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/integration/test_cell_org_triggers_pg.py b/tests/integration/test_cell_org_triggers_pg.py index e49dde4426..a47f4baec5 100644 --- a/tests/integration/test_cell_org_triggers_pg.py +++ b/tests/integration/test_cell_org_triggers_pg.py @@ -714,6 +714,32 @@ async def test_missing_names_a_guard_disabled_narrowed_or_calling_another_functi assert "loom.tenancy_refuse_unmarked() has a body or settings no migration writes" in texts +@pytest.mark.asyncio +async def test_ensure_table_creates_a_recording_first_cells_triggers_only_when_asked() -> None: + """Attach creates a first cell's triggers in `recording`, one table at a time; a boot's + `ensure` never does, and the function passes every row there.""" + async with _cell("recording") as (dsn, conn): + with pytest.raises(RuntimeError, match="recording"): + await cell_triggers.ensure_table(conn, "scope_generations") + assert await cell_triggers.ensure_table(conn, "scope_generations", recording_first=True) + assert await cell_triggers.ensure_table(conn, "tenancy", recording_first=True) + assert await _cell_triggers(conn) == { + ("scope_generations", "tenancy_insert", "O"), + ("scope_generations", "tenancy_update", "O"), + ("tenancy", "tenancy_record", "O"), + } + # Again: nothing to do. + assert await cell_triggers.ensure_table(conn, "scope_generations", recording_first=True) + with pytest.raises(ValueError, match="no cell triggers"): + await cell_triggers.ensure_table(conn, "idempotency", recording_first=True) + plain = await _unmarked(dsn) + try: + await plain.execute(_INSERT, "unheld-org", "s") # recording passes every row + await plain.execute(_HOLD, "new-org") # a first cell still recording records + finally: + await plain.close() + + @pytest.mark.asyncio @pytest.mark.parametrize("loom_cell", ["eu-1", ""]) async def test_the_cell_refresh_reads_the_marker_its_pool_sends( From 4193ffd36fcece18fa4bbb77ceef6f2fbd62ac78 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 07:48:23 +0000 Subject: [PATCH 005/114] Fix a double space in the failure table's pooler row Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- docs/architecture/cells.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 96580ec7d2..46e1439538 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1292,7 +1292,7 @@ the directory row. | A cell's `first` value is not yet read | every request but health probes and the probe route answers 503 (SCIM with `Retry-After`); the process is unready and runs no worker (a workers child, or a server process with Postgres migrations pending; any other server process fails its boot) | | A request for another cell's org | 409 `WRONG_CELL` after verification, nothing written | | A platform, `*`-org or principal-less call for an unheld org | on a first cell, 503 while the directory answers `attaching` or cannot be read before a `serving` answer; 409 `WRONG_CELL` once `serving` and on a later cell | -| A pooler that strips the tenancy marker from a cell's sessions | the `cell_marker` readiness check fails in `enforced` and `partial` and warns in `recording`; the heartbeat reports `marked` false and attach refuses the process | +| A pooler that strips the tenancy marker from a cell's sessions | the `cell_marker` readiness check fails in `enforced` and `partial` and warns in `recording`; the heartbeat reports `marked` false and attach refuses the process | | A cell restored to an earlier time | `loom cell restore` writes placed orgs' rows back before it starts | | A first cell restored to before its prepare, not yet attached | at processes with `LOOM_CELL` set, every request but health probes and the probe route answers 503 until prepare reruns | | The directory lost | every cell serves the orgs it holds; placements and uncached lookups answer 503, directory commands refuse, until it is back | From c6501191d838adfbfee5585f65872f3af659332d Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 08:10:11 +0000 Subject: [PATCH 006/114] Attach refuses a table its role cannot alter up front, charges --wait in wall time, and says when the mode commit's outcome is unknown The refusals name every table the trigger pass would alter whose owner role attach's role is not a member of, before any write; ensure_table and _run_ddl answer applied, busy or denied, so a denial during the pass refuses at once rather than after --wait as a busy lock. --wait is wall time: each bounded wait charges its elapsed time, lock tries included, and every lock try waits at most what is left; scan time stays outside it. From the mode commit on, a stop that comes before the durable read says the commit's outcome is unknown and to rerun attach, for a refusal and for any other error alike. The prune-window check runs with the refusals, before any write; an enforced resume runs one trigger pass; the lock session opens inside the block that closes it; the durable read retries a timeout or an outage only; refusals name no command that does not exist. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- server/cli/cell_attach.py | 78 +++--- server/cli/cell_attach_beats.py | 37 ++- server/cli/cell_attach_pg.py | 235 +++++++++++------- server/core/cell_triggers.py | 41 +-- tests/integration/test_cell_attach.py | 35 +++ .../integration/test_cell_org_triggers_pg.py | 7 +- tests/unit/test_cell_attach.py | 124 ++++++++- 7 files changed, 396 insertions(+), 161 deletions(-) diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index c8b72fb687..ea9cf77d35 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -365,9 +365,9 @@ async def refusals(self) -> tuple[Any, Any, cell_scan.TableList]: out += await self.catalog_refusals() out += await self.durability_refusals() rows = await self.lock_conn.fetch(cell_triggers.ORG_TABLES) - tables_pg = [f"loom.{r['relname']}" for r in rows] - out += await apg.own_role_refusals(self.lock_conn, tables_pg) + out += await apg.own_role_refusals(self.lock_conn, list(rows)) out += await self.ch_limit_refusals() + out += beats.prune_window_refusals() if out: raise Refused(out) state, _ = await apg.read_mode(self.lock_conn) @@ -449,6 +449,8 @@ async def ready_for_commit(self) -> bool: async def commit_mode(self) -> None: from server.core import pg + # From here on the mode may have risen: only the durable read can say. + self.mode = "unknown" await self.router.call("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) try: async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: @@ -465,22 +467,29 @@ async def commit_mode(self) -> None: async def durable(self) -> int: """The durable read of `enforced`, retried within `--wait`; then X, the snapshot's xmax, above the read's own xid.""" - from server.core import pg_profile + from server.core import pg, pg_profile - while True: - try: + with self.wait.counting(): + while True: if await pg_profile.standby_now() != "synchronous": raise _refused("postgres standbys", "not synchronous before the durable read") - read = await apg.durable_read() + try: + read = await apg.durable_read() + except Exception as e: + # A try cut off by its bound, or an outage: its outcome is unknown, and + # the next try reads again. Anything else is no reason to retry. + outage = pg.is_outage(e) or isinstance(e, pg.PostgresUnavailableError) + if not (isinstance(e, TimeoutError) or outage): + raise + self.log(f"the durable read failed ({type(e).__name__}); retrying") + if not await self.wait.pause(): + raise _refused( + "postgres durable read", "did not commit within --wait" + ) from e + continue if await pg_profile.standby_now() != "synchronous": raise _refused("postgres standbys", "not synchronous after the durable read") break - except Refused: - raise - except Exception as e: - self.log(f"the durable read failed ({type(e).__name__}); retrying") - if not await self.wait.pause(): - raise _refused("postgres durable read", "did not commit within --wait") from e if read.first != (True, self.name) or read.mode != "enforced": raise _refused( "postgres cell state", @@ -540,8 +549,6 @@ async def step4(self, xmax: int, topology: Any, names: Any) -> None: await self.cheap_checks("wait") since = str((await self.ch.query_one("SELECT toString(now64(3)) AS t"))["t"]) watch = beats.Watch(self.ch, since, self.name) - if refusals := watch.refusals_up_front(): - raise Refused(refusals) task = asyncio.create_task(watch.run()) try: listed = await cell_scan.list_tables(self.ch, self.lock_conn, topology, names) @@ -567,7 +574,8 @@ async def step4(self, xmax: int, topology: Any, names: Any) -> None: if (row.get("current_fp"), row.get("next_fp")) != (current, nxt): raise _refused( "postgres loom.cell_self", - "differs from the directory's fingerprints (a loom cell update between them?)", + "differs from the directory's fingerprints (a fingerprinted setting changed " + "during attach?)", ) state, _ = await apg.read_mode(self.lock_conn) if state != "enforced": @@ -596,7 +604,9 @@ async def done(self) -> bool: if state != "enforced": raise _refused( "postgres cell state", - f"the directory says serving and the cell reads {state}: run loom cell restore", + f"the directory says serving and the cell reads {state}: the directory and the " + "cell's Postgres disagree, as after a restore of one of them; attach lowers " + "and repairs neither", ) return True @@ -606,8 +616,8 @@ async def run(self) -> int: await self.done() self.report.update(state="serving", done=True) return 0 - await self.open_lock() try: + await self.open_lock() await self.read_own_row() topology, names, listed = await self.refusals() if self.args.dry_run: @@ -626,7 +636,6 @@ async def run(self) -> int: await self.precommit(found) await self.commit_mode() else: - await self.triggers() await self.router.call( "/attach/enforced", {"deployment_id": self.row["deployment_id"]} ) @@ -695,18 +704,30 @@ def log(text: str) -> None: except Refused as e: for r in e.refusals: log(f"refusing: {r}") - mode = ( - "enforced, attaching: rerun attach to resume" - if run.mode == "enforced" - else ("unchanged" if not run.mode else f"{run.mode}") - ) - log(f"the cell stays {mode}") + log(mode_line(run.mode)) + return 1 + except Exception as e: # noqa: BLE001 — a store that fails names itself; a rerun resumes + log(f"stopped ({type(e).__name__}: {e}); run it again") + log(mode_line(run.mode)) return 1 run.report["seconds"] = round(time.monotonic() - started, 1) print(json.dumps(run.report, indent=2, sort_keys=True, default=str)) return code +def mode_line(mode: str) -> str: + """What a stopped run left the cell's mode at, by what it last knew.""" + if mode == "enforced": + return "the cell stays enforced and attaching: rerun attach to resume" + if mode == "unknown": + return ( + "the mode commit's outcome is unknown: rerun attach, which resumes by the mode it reads" + ) + if mode == "recording": + return "the cell stays recording" + return "the cell's state is unchanged" + + def cmd_attach(args: argparse.Namespace) -> int: if problem := _args_problem(args): print(f"loom cell attach: {problem}", file=sys.stderr) @@ -719,14 +740,7 @@ def cmd_attach(args: argparse.Namespace) -> int: from server.core.ch import statement_timeout async def _run(ch: Any) -> int: - try: - return await attach(ch, args, url) - except Exception as e: # noqa: BLE001 — a store that fails names itself; a rerun resumes - print( - f"loom cell attach: stopped ({type(e).__name__}: {e}); run it again", - file=sys.stderr, - ) - return 1 + return await attach(ch, args, url) # Every ClickHouse statement, the connect included, may run as long as a scan may. with statement_timeout(args.scan_budget): diff --git a/server/cli/cell_attach_beats.py b/server/cli/cell_attach_beats.py index d1d938a46b..a9c6e80377 100644 --- a/server/cli/cell_attach_beats.py +++ b/server/cli/cell_attach_beats.py @@ -209,6 +209,16 @@ async def settle( """Read and judge until nothing is waited on: Refused at the first refusal, and once `wait` runs out naming what was waited on. With `extend_s`, the deadline is extended once, to `extend_s` past the first sight of the last process first seen booting.""" + with wait.counting(): + return await _settle(read, judge_beats, wait, extend_s) + + +async def _settle( + read: Callable[[], Awaitable[list[Beat]]], + judge_beats: Callable[[list[Beat]], Verdict], + wait: Wait, + extend_s: float, +) -> Verdict: first_seen: dict[str, float] = {} extended = False while True: @@ -220,7 +230,7 @@ async def settle( raise Refused(refusals) return verdict for i in verdict.booting: - first_seen.setdefault(i, wait.spent) + first_seen.setdefault(i, wait.used()) if await wait.pause(): continue if extend_s and not extended and verdict.booting: @@ -249,6 +259,20 @@ def watch_period_s() -> float: return instances.PRUNE_AFTER_S - widened_window_s() - _SKEW_S +def prune_window_refusals() -> list[Refusal]: + """A heartbeat interval so long that the leader may sweep a stopped process's rows + before step 4's watch reads them refuses before any write.""" + if watch_period_s() < 2 * _READ_TIMEOUT_S: + return [ + Refusal( + "LOOM_INSTANCE_HEARTBEAT_SECONDS", + "so long that a stopped process's rows can be swept before attach reads " + "them: lower it", + ) + ] + return [] + + class Watch: """The processes seen during step 4's rescan, read every `T' / 2` from the rescan's start (`since`), so a process with `LOOM_CELL` unset that ran during it is refused at @@ -265,17 +289,6 @@ def __init__( self.gap = False self._good = clock() - def refusals_up_front(self) -> list[Refusal]: - if watch_period_s() < 2 * _READ_TIMEOUT_S: - return [ - Refusal( - "LOOM_INSTANCE_HEARTBEAT_SECONDS", - "so long that a stopped process's rows can be swept before attach reads " - "them: lower it", - ) - ] - return [] - async def read_once(self) -> list[Beat]: try: async with asyncio.timeout(_READ_TIMEOUT_S): diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 6ee4432907..076dc4a196 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -10,8 +10,9 @@ from __future__ import annotations import asyncio +import contextlib import time -from collections.abc import Callable, Iterable +from collections.abc import Callable, Iterable, Iterator from dataclasses import dataclass from typing import Any @@ -23,38 +24,59 @@ # The pause between polls of a wait. POLL_S = 1.0 # not a setting: a poll interval -# The migration lock's wait a trigger try (a boot holds it for its migrations). -_MIGRATION_LOCK_TIMEOUT = "10s" -# The mode commit's lock wait a try, and the statement bound above it. -_MODE_LOCK_TIMEOUT = "2s" -_MODE_STATEMENT_TIMEOUT = "5s" +# The migration lock's wait a trigger try (a boot holds it for its migrations), in ms; a +# try never waits longer than --wait has left. +_MIGRATION_LOCK_MS = 10_000 # not a setting: one try's bound +# The mode commit's lock wait a try, in ms, and its statement bound's margin above it. +_MODE_LOCK_MS = 2_000 # not a setting: one try's bound +_MODE_STATEMENT_MARGIN_MS = 3_000 # not a setting: a margin over the lock wait # The server ends the lock session soon after its client is gone, releasing the lock. _KEEPALIVE = {"tcp_keepalives_idle": "30", "tcp_keepalives_interval": "10"} class Wait: - """`--wait`: the seconds attach's waits share, counting only the time spent waiting - (never scan time).""" + """`--wait`: the wall-clock seconds attach's bounded waits share. Each wait runs inside + `counting()`, which charges its elapsed time, lock tries included; time outside every + wait (a scan) is never charged.""" def __init__(self, seconds: float, clock: Callable[[], float] = time.monotonic) -> None: self.seconds = float(seconds) self.spent = 0.0 self._clock = clock + self._since: float | None = None + + def used(self) -> float: + """The seconds charged so far, the wait running now included.""" + running = 0.0 if self._since is None else self._clock() - self._since + return self.spent + running def left(self) -> float: - return max(0.0, self.seconds - self.spent) + return max(0.0, self.seconds - self.used()) + + @contextlib.contextmanager + def counting(self) -> Iterator[None]: + """Charge the block's wall time (a block inside another is charged once).""" + if self._since is not None: + yield + return + self._since = self._clock() + try: + yield + finally: + self.spent += self._clock() - self._since + self._since = None - def now(self) -> float: - return self._clock() + def bound_ms(self, most_ms: int) -> int: + """A lock timeout for one try: `most_ms`, or what is left when that is less.""" + return min(most_ms, int(self.left() * 1000)) async def pause(self, seconds: float = POLL_S) -> bool: - """Wait `seconds` (at most what is left) and count them; False once none is left.""" + """Wait `seconds` (at most what is left); False once none is left.""" step = min(seconds, self.left()) if step <= 0: return False - start = self._clock() - await _sleep(step) - self.spent += max(step, self._clock() - start) + with self.counting(): + await _sleep(step) return True @@ -137,7 +159,7 @@ async def take_lock(conn: Any, what: str) -> None: [ Refusal( "attach lock", - f"held by {holder}: an attach, prepare or update is running; {what} " + f"held by {holder}: an attach or a prepare is running; {what} " "runs one at a time", ) ] @@ -364,14 +386,22 @@ async def session_passes() -> bool: """ -async def own_role_refusals(conn: Any, tables: Iterable[str]) -> list[Refusal]: - """What attach's own role lacks (the guarded tables' owner roles are `missing`'s - `owner` defects), and a server that allows prepared transactions.""" +async def own_role_refusals(conn: Any, tables: list[Any]) -> list[Refusal]: + """What attach's own role lacks, `tables` being the rows `cell_triggers.ORG_TABLES` + read: its grants, `SELECT` on every org table, and membership in the owner role of every + table it gives triggers (or the trigger pass could not alter it); and a server that + allows prepared transactions.""" role = await conn.fetchval("SELECT current_user") lacks = [str(r["need"]) for r in await conn.fetch(_OWN_GRANTS)] - for table in tables: - if not await conn.fetchval("SELECT has_table_privilege($1, 'SELECT')", table): - lacks.append(f"SELECT on {table}") + for r in tables: + if not await conn.fetchval( + "SELECT has_table_privilege($1, 'SELECT')", f"loom.{r['relname']}" + ): + lacks.append(f"SELECT on loom.{r['relname']}") + planned = set(cell_triggers.planned_names(tables)) + for r in tables: + if r["relname"] in planned and not r["owned"]: + lacks.append(f"membership in the owner role of loom.{r['relname']}") out = [Refusal(f"postgres role {role}", f"attach's role lacks {n}") for n in lacks] out += await prepared_refusals(conn) return out @@ -400,51 +430,66 @@ async def ensure_triggers(conn: Any, wait: Wait, log: Callable[[str], None]) -> """Give every planned table its triggers (`cell_triggers.ensure_table` with `recording_first`), one try at a time under the migration lock, retrying a table whose lock is busy a second apart within `wait`; one autovacuum holding it is asked to yield - by one try just past `deadlock_timeout`. Then `missing`: any defect refuses.""" - from server.core.schema.pg_migrate import _LOCK_KEY - + by one try just past `deadlock_timeout`. A table this role cannot alter refuses at once. + Then `missing`: any defect refuses.""" deadlock_ms = int( await conn.fetchval( "SELECT setting::int * CASE unit WHEN 's' THEN 1000 ELSE 1 END " "FROM pg_settings WHERE name = 'deadlock_timeout'" ) ) - out: list[Refusal] = [] - for table in await cell_triggers.planned_tables(conn): - yielded = False - while True: - lock_timeout = cell_triggers.LOCK_TIMEOUT - holders = await cell_triggers.holders(conn, table) - if not yielded and any(h["backend_type"] == "autovacuum worker" for h in holders): - lock_timeout, yielded = f"{deadlock_ms + 500}ms", True - done = await _ensure_one(conn, table, lock_timeout, _LOCK_KEY) - if done: - break - if not await wait.pause(): - out.append(_busy_refusal(table, await cell_triggers.holders(conn, table))) - break - log(f"loom.{table}: its lock is busy; retrying") - if out: - return out + with wait.counting(): + for table in await cell_triggers.planned_tables(conn): + if refusal := await _ensure_table(conn, table, deadlock_ms, wait, log): + return [refusal] return [Refusal("postgres triggers", d.text) for d in await cell_triggers.missing(conn)] -async def _ensure_one(conn: Any, table: str, lock_timeout: str, key: int) -> bool: - await conn.execute(f"SET lock_timeout = '{_MIGRATION_LOCK_TIMEOUT}'") +async def _ensure_table( + conn: Any, table: str, deadlock_ms: int, wait: Wait, log: Callable[[str], None] +) -> Refusal | None: + yielded = False + while True: + lock_ms = 100 + holders = await cell_triggers.holders(conn, table) + if not yielded and any(h["backend_type"] == "autovacuum worker" for h in holders): + lock_ms, yielded = deadlock_ms + 500, True + lock_ms = wait.bound_ms(lock_ms) + result = "busy" if lock_ms <= 0 else await _ensure_one(conn, table, lock_ms, wait) + if result == "applied": + return None + if result == "denied": + return Refusal( + f"postgres loom.{table}", + "attach's role cannot alter it: make the role a member of its owner role", + ) + if not await wait.pause(): + return _busy_refusal(table, await cell_triggers.holders(conn, table)) + log(f"loom.{table}: its lock is busy; retrying") + + +async def _ensure_one(conn: Any, table: str, lock_ms: int, wait: Wait) -> str: + """One try under the migration lock, whose own wait is bounded by what `wait` has left.""" + from server.core.schema.pg_migrate import _LOCK_KEY + + migration_ms = wait.bound_ms(_MIGRATION_LOCK_MS) + if migration_ms <= 0: + return "busy" + await conn.execute(f"SET lock_timeout = '{migration_ms}ms'") try: - await conn.execute(_MIGRATION_LOCK, key) + await conn.execute(_MIGRATION_LOCK, _LOCK_KEY) except Exception as e: if getattr(e, "sqlstate", None) == "55P03": - return False # a boot migrating: the next try + return "busy" # a boot migrating: the next try raise finally: await conn.execute("RESET lock_timeout") try: return await cell_triggers.ensure_table( - conn, table, recording_first=True, lock_timeout=lock_timeout, attempts=1 + conn, table, recording_first=True, lock_timeout=f"{lock_ms}ms", attempts=1 ) finally: - await conn.execute(_MIGRATION_UNLOCK, key) + await conn.execute(_MIGRATION_UNLOCK, _LOCK_KEY) def _busy_refusal(table: str, holders: list[Any]) -> Refusal: @@ -487,49 +532,56 @@ async def commit_enforced( log: Callable[[str], None], ) -> None: """Raise the mode to `enforced` on `conn` (a session of its own, never the lock - session): an EXCLUSIVE lock on loom.tenancy_mode a try, `_MODE_LOCK_TIMEOUT` each, - within `wait`; `confirm_lock` before each try and each termination (it refuses when - another run holds the attach lock), and `ready` before each try (False skips the try - while a process boots). After a failed try the in-flight mode transaction, found by its - lock, is ended. Raises Refused when `wait` runs out; whether the commit landed is the - durable read's to say.""" - while True: - await confirm_lock() - if await ready(): - try: - async with conn.transaction(): - await conn.execute(f"SET LOCAL lock_timeout = '{_MODE_LOCK_TIMEOUT}'") - await conn.execute(f"SET LOCAL statement_timeout = '{_MODE_STATEMENT_TIMEOUT}'") - await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") - status = await conn.execute( - "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" - ) - if status != "UPDATE 1": - raise Refused( - [ - Refusal( - "postgres loom.tenancy_mode", f"the mode update read {status}" - ) - ] - ) - return - except Refused: - raise - except Exception as e: - if getattr(e, "sqlstate", None) not in _TIMED_OUT: - raise + session): an EXCLUSIVE lock on loom.tenancy_mode a try, `_MODE_LOCK_MS` each (less + when `wait` has less left), within `wait`; `confirm_lock` before each try and each + termination (it refuses when another run holds the attach lock), and `ready` before + each try (False skips the try while a process boots). After a failed try the in-flight + mode transaction, found by its lock, is ended. Raises Refused when `wait` runs out; + whether the commit landed is the durable read's to say.""" + with wait.counting(): + while True: + await confirm_lock() + lock_ms = wait.bound_ms(_MODE_LOCK_MS) + if lock_ms > 0 and await ready(): + if await _mode_try(conn, lock_ms): + return log("the mode's lock is busy; retrying") - await end_mode_transactions(conn, confirm_lock) - if not await wait.pause(): - raise Refused( - [ - Refusal( - "postgres loom.tenancy_mode", - "its lock was not taken within --wait: a long transaction reads it " - "FOR SHARE; rerun attach", - ) - ] + await end_mode_transactions(conn, confirm_lock) + if not await wait.pause(): + raise Refused( + [ + Refusal( + "postgres loom.tenancy_mode", + "its lock was not taken within --wait: a long transaction reads " + "it FOR SHARE; rerun attach", + ) + ] + ) + + +async def _mode_try(conn: Any, lock_ms: int) -> bool: + """One try of the mode commit, its lock wait `lock_ms`; False when it timed out.""" + try: + async with conn.transaction(): + await conn.execute(f"SET LOCAL lock_timeout = '{lock_ms}ms'") + await conn.execute( + f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" + ) + await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") + status = await conn.execute( + "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" ) + if status != "UPDATE 1": + raise Refused( + [Refusal("postgres loom.tenancy_mode", f"the mode update read {status}")] + ) + return True + except Refused: + raise + except Exception as e: + if getattr(e, "sqlstate", None) not in _TIMED_OUT: + raise + return False async def end_mode_transactions(conn: Any, confirm_lock: Callable[[], Any]) -> None: @@ -626,6 +678,11 @@ async def wait_transactions(conn: Any, xmax: int, wait: Wait) -> None: """Wait until no backend of the cell's database holds an xid or a snapshot older than `xmax`; Refused at once on a prepared transaction or an unreadable row, and past `wait` naming each backend waited on and each other database's horizon holder.""" + with wait.counting(): + await _wait_transactions(conn, xmax, wait) + + +async def _wait_transactions(conn: Any, xmax: int, wait: Wait) -> None: while True: if refusals := await prepared_refusals(conn): raise Refused(refusals) diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 9c2c9755ae..516dee4cc1 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -21,7 +21,7 @@ import dataclasses import logging import re -from typing import Any +from typing import Any, Literal from server.core import cell, pg, readiness @@ -92,6 +92,9 @@ AND l.mode NOT IN ('AccessShareLock', 'RowShareLock') """ +# What a table's DDL came to (`ensure_table`, `_run_ddl`). +Applied = Literal["applied", "busy", "denied"] + # pg_trigger.tgtype bits (src/include/catalog/pg_trigger.h). _ROW, _BEFORE, _INSERT, _UPDATE, _INSTEAD = 1, 2, 4, 16, 64 # The events each trigger must fire on. @@ -242,7 +245,12 @@ async def ensure(conn: Any) -> None: async def planned_tables(conn: Any) -> list[str]: """The tables `ensure_table` gives triggers: the guarded org tables and `loom.tenancy`.""" - return [r["relname"] for r, _ in _planned(await conn.fetch(ORG_TABLES))] + return planned_names(await conn.fetch(ORG_TABLES)) + + +def planned_names(rows: list[Any]) -> list[str]: + """`planned_tables` from rows `ORG_TABLES` read.""" + return [r["relname"] for r, _ in _planned(rows)] async def ensure_table( @@ -252,14 +260,14 @@ async def ensure_table( recording_first: bool = False, lock_timeout: str = LOCK_TIMEOUT, attempts: int = LOCK_ATTEMPTS, -) -> bool: +) -> Applied: """`ensure` for one table of the planned set (`loom.tenancy` its guard, any other its org triggers), so one table's DDL can run alone: in `enforced`, or with `recording_first` also in a first cell's `recording`, where the org triggers' function - passes every row and a boot never creates them. False when the DDL was not applied (its - lock not taken in `attempts` tries of `lock_timeout`, a table this role cannot alter); - True when it was, or nothing was needed. Raises for a table outside the planned set, or - in another state.""" + passes every row and a boot never creates them. `applied` when the DDL ran or nothing + was needed, `busy` when its lock was not taken in `attempts` tries of `lock_timeout`, + `denied` when this role cannot alter the table. Raises for a table outside the planned + set, or in another state.""" state = await cell.read_state(conn) if state != "enforced" and not (recording_first and state == "recording"): raise RuntimeError(f"cell triggers: not created in state {state!r}") @@ -270,7 +278,9 @@ async def ensure_table( raise ValueError(f"loom.{table} carries no cell triggers") ((_, names),) = planned sqls = _table_statements(table, names, await _triggers(conn, names)) - return not sqls or await _run_ddl(conn, table, sqls, lock_timeout, attempts) + if not sqls: + return "applied" + return await _run_ddl(conn, table, sqls, lock_timeout, attempts) def _table_statements( @@ -293,10 +303,11 @@ async def _run_ddl( sqls: list[str], lock_timeout: str = LOCK_TIMEOUT, attempts: int = LOCK_ATTEMPTS, -) -> bool: - """A table's DDL in one transaction, LOCK_ATTEMPTS short tries; whether it applied. The - boot serves past the two causes cells.md names (its lock not taken, a table this role - cannot alter); any other error fails it, as a failed read of the state does.""" +) -> Applied: + """A table's DDL in one transaction, `attempts` short tries: `applied`, or which of the + two causes cells.md names kept it from applying (`busy`: its lock not taken; `denied`: + a table this role cannot alter), which a boot serves past; any other error fails it, as + a failed read of the state does.""" import asyncpg for attempt in range(attempts): @@ -305,12 +316,12 @@ async def _run_ddl( await conn.execute(f"SET LOCAL lock_timeout = '{lock_timeout}'") for sql in sqls: await conn.execute(sql) - return True + return "applied" except asyncpg.exceptions.InsufficientPrivilegeError as e: log.warning( "cell triggers: not applied on loom.%s (%s): this role cannot alter it", table, e ) - return False + return "denied" except asyncpg.exceptions.LockNotAvailableError as e: # Short tries outlast a short transaction; an autovacuum is left to finish. if attempt < attempts - 1: @@ -323,7 +334,7 @@ async def _run_ddl( "; ".join(sqls), await _holders(conn, table), ) - return False + return "busy" async def holders(conn: Any, table: str) -> list[Any]: diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 406d277646..9bfe3fe9bf 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -633,3 +633,38 @@ async def test_doctor_names_stray_orgs_smallest_tables_first(cell_stores) -> Non assert grade == "WARN" and tag + "stray" in detail and tag + "held" not in detail tight = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 0.001)} assert tight["cell_stray_orgs_unchecked"][0] == "WARN" + + +@pytest.mark.asyncio +async def test_a_role_with_the_grants_but_not_the_owner_role_is_refused_before_any_write( + cell_stores, +) -> None: + from server.core import cell_triggers + + await _prepare(cell_stores, _Network()) + role = f"it_o_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + try: + async with pg.dedicated() as conn: + await conn.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + await conn.execute( + f"GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA loom TO {role}" + ) + await conn.execute(f"SET ROLE {role}") + rows = list(await conn.fetch(cell_triggers.ORG_TABLES)) + refusals = await cell_attach_pg.own_role_refusals(conn, rows) + lacking = [r.reason for r in refusals if "owner role" in r.reason] + assert "attach's role lacks membership in the owner role of loom.tenancy" in lacking + assert any("loom.scope_generations" in r for r in lacking) + # The trigger pass tells the cause apart: denied, not a busy lock. + assert ( + await cell_triggers.ensure_table(conn, "scope_generations", recording_first=True) + == "denied" + ) + await conn.execute("RESET ROLE") + assert await _mode() == "recording" + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") diff --git a/tests/integration/test_cell_org_triggers_pg.py b/tests/integration/test_cell_org_triggers_pg.py index a47f4baec5..b2fb66a467 100644 --- a/tests/integration/test_cell_org_triggers_pg.py +++ b/tests/integration/test_cell_org_triggers_pg.py @@ -721,15 +721,16 @@ async def test_ensure_table_creates_a_recording_first_cells_triggers_only_when_a async with _cell("recording") as (dsn, conn): with pytest.raises(RuntimeError, match="recording"): await cell_triggers.ensure_table(conn, "scope_generations") - assert await cell_triggers.ensure_table(conn, "scope_generations", recording_first=True) - assert await cell_triggers.ensure_table(conn, "tenancy", recording_first=True) + ensure = cell_triggers.ensure_table + assert await ensure(conn, "scope_generations", recording_first=True) == "applied" + assert await ensure(conn, "tenancy", recording_first=True) == "applied" assert await _cell_triggers(conn) == { ("scope_generations", "tenancy_insert", "O"), ("scope_generations", "tenancy_update", "O"), ("tenancy", "tenancy_record", "O"), } # Again: nothing to do. - assert await cell_triggers.ensure_table(conn, "scope_generations", recording_first=True) + assert await ensure(conn, "scope_generations", recording_first=True) == "applied" with pytest.raises(ValueError, match="no cell triggers"): await cell_triggers.ensure_table(conn, "idempotency", recording_first=True) plain = await _unmarked(dsn) diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 1c725205e6..b57a40eb57 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -222,12 +222,72 @@ async def forever() -> list[beats.Beat]: @pytest.mark.asyncio -async def test_wait_counts_only_its_own_pauses(clock: _Clock) -> None: - wait = apg.Wait(2, clock) - clock.now += 100 # a scan's time - assert await wait.pause() and await wait.pause() - assert not await wait.pause() - assert wait.spent == 2.0 +async def test_wait_is_wall_time_lock_tries_included_and_no_scan(clock: _Clock) -> None: + """Each mode-commit try waits its lock timeout before it times out: the tries and the + pauses between them end within --wait, and time outside the wait is never charged.""" + asked: list[str] = [] + + def lock(sql: str, *_a: Any) -> str: + if "lock_timeout" in sql: + asked.append(sql) + clock.now += int(sql.split("'")[1].removesuffix("ms")) / 1000 + raise _Timeout() + return "OK" + + conn = FakeConn( + [ + ("SET LOCAL lock_timeout", lambda *a: lock(conn.executed[-1])), + ("l.mode = 'ExclusiveLock'", []), + ] + ) + + async def nothing() -> None: + pass + + async def ready() -> bool: + return True + + wait = apg.Wait(4, clock) + clock.now += 100 # a scan before the wait + with pytest.raises(Refused, match="not taken within --wait"): + await apg.commit_enforced(conn, nothing, ready, wait, lambda _t: None) + assert clock.now - 100 <= 4 and wait.spent <= 4 + # A 2 s try, a 1 s pause, then a try that waits only the second left. + assert [a.split("'")[1] for a in asked] == ["2000ms", "1000ms"] + + +@pytest.mark.asyncio +async def test_the_trigger_pass_refuses_at_once_a_table_it_cannot_alter( + monkeypatch: pytest.MonkeyPatch, clock: _Clock +) -> None: + from server.core import cell_triggers + + async def planned(_c: Any) -> list[str]: + return ["scope_generations"] + + async def holders(_c: Any, _t: str) -> list[Any]: + return [] + + async def ensure_table(*_a: Any, **_kw: Any) -> str: + return "denied" + + monkeypatch.setattr(cell_triggers, "planned_tables", planned) + monkeypatch.setattr(cell_triggers, "holders", holders) + monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) + conn = FakeConn([("deadlock_timeout", 1000)]) + wait = apg.Wait(60, clock) + (refusal,) = await apg.ensure_triggers(conn, wait, lambda _t: None) + assert refusal.subject == "postgres loom.scope_generations" + assert "cannot alter it" in refusal.reason and wait.spent < 1 + + async def busy(*_a: Any, **_kw: Any) -> str: + clock.now += 0.1 + return "busy" + + monkeypatch.setattr(cell_triggers, "ensure_table", busy) + wait = apg.Wait(3, clock) + (refusal,) = await apg.ensure_triggers(conn, wait, lambda _t: None) + assert "not taken within --wait" in refusal.reason and wait.spent <= 3 # -- the rescan's watch ---------------------------------------------------------------- @@ -272,9 +332,9 @@ def test_the_watch_refuses_up_front_with_a_very_long_heartbeat( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(settings, "instance_heartbeat_seconds", 1180) - assert beats.Watch(None, "", "first").refusals_up_front() + assert beats.prune_window_refusals() monkeypatch.setattr(settings, "instance_heartbeat_seconds", 10) - assert beats.Watch(None, "", "first").refusals_up_front() == [] + assert beats.prune_window_refusals() == [] # -- batches, ordering, arguments -------------------------------------------------------- @@ -491,10 +551,19 @@ async def test_own_role_lacks_and_prepared_transactions_refuse() -> None: ("max_prepared_transactions", 5), ] ) - refusals = await apg.own_role_refusals(conn, ["loom.memory_current", "loom.tenancy"]) + tables = [ + {"relname": "memory_current", "relkind": "r", "owned": True}, + # Its grants are held, but not its owner role's membership: the trigger pass could + # not alter it, so it refuses before any write. + {"relname": "scope_generations", "relkind": "r", "owned": False}, + {"relname": "tenancy", "relkind": "r", "owned": True}, + ] + refusals = await apg.own_role_refusals(conn, tables) assert [str(r) for r in refusals] == [ "postgres role attacher: attach's role lacks membership in pg_read_all_stats", "postgres role attacher: attach's role lacks SELECT on loom.memory_current", + "postgres role attacher: attach's role lacks membership in the owner role of " + "loom.scope_generations", "postgres max_prepared_transactions: is 5: a prepared transaction outlives every " "session, so step 4 could not wait for it; set it to 0", ] @@ -674,7 +743,7 @@ async def read_mode(_conn: Any) -> tuple[str, Any]: run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] assert await run.done() state[0] = "recording" - with pytest.raises(Refused, match="run loom cell restore"): + with pytest.raises(Refused, match="attach lowers and repairs neither"): await run.done() @@ -697,3 +766,38 @@ async def settle(*_a: Any, **_kw: Any) -> beats.Verdict: await run.final_checks(beats.Watch(None, "", "first")) run.begun_keys_fp = "after" await run.final_checks(beats.Watch(None, "", "first")) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", [Refused([Refusal("postgres", "cut")]), OSError("gone")]) +async def test_a_stop_between_a_landed_commit_and_the_durable_read_says_the_outcome_is_unknown( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], failure: Exception +) -> None: + from contextlib import asynccontextmanager + + from server.core import pg + + @asynccontextmanager + async def dedicated(**_kw: Any) -> Any: + yield None + + async def landed(*_a: Any) -> None: + pass # the commit landed; its answer may not have + + async def call(_self: Any, _path: str, _body: Any = None) -> dict[str, Any]: + return {"cell": "first"} + + async def run(self: Any) -> int: + self.row = {"deployment_id": "dep-1"} + await self.commit_mode() + raise failure # what stops the run before the durable read finishes + + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(apg, "commit_enforced", landed) + monkeypatch.setattr(cell_attach.Router, "call", call) + monkeypatch.setattr(cell_attach.Run, "run", run) + url = httpx.URL("http://127.0.0.1:7900") + assert await cell_attach.attach(None, _args(), url) == 1 + err = capsys.readouterr().err + assert "the mode commit's outcome is unknown: rerun attach" in err + assert "stays recording" not in err From 5884b04347fd9ec9d839bfbe8c9f780fef3b7607 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 08:17:07 +0000 Subject: [PATCH 007/114] Doctor's stray-org read names a table it cannot read and goes on, and the attach docs describe one step numbering and today's procedure --stray-orgs labels only a spent budget as tables left unchecked; any other refusal names its table and the read continues. A failed catalog read reports both of attach's Postgres checks as not read. The runbook and doctor describe removing a stray or over-long org by hand, and name no command that does not exist yet. cells.md labels the first attach's list with the design's step names, so step 1 to step 4 mean the same everywhere, rewrites the stuck-process failure row and the planned tests for a mode that only rises, and says --wait is wall time. The trigger-cost harness uses cell_triggers.create_statement and deletes the rows its inserts wrote. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 21 +- docs/architecture/cells.md | 405 +++++++++++++------------- docs/operations/cell-attach.md | 19 +- server/cli/cell_attach.py | 41 +-- server/cli/doctor_cell.py | 36 ++- server/core/cell_triggers.py | 5 + tests/integration/test_cell_attach.py | 20 ++ tests/unit/test_cell_attach.py | 27 ++ 8 files changed, 325 insertions(+), 249 deletions(-) diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index e90fbb32fa..4363a810d6 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -1,9 +1,10 @@ """The first attach's trigger-cost gate, end to end: `memory.set` p50 and p99 through a server of an attached cell, and a delivery-sized 1,000-row insert of a held org from a marked and an unmarked session, with the cell's org triggers on and dropped in turn (the `loom.tenancy` -guard stays). Run it against an attached cell's server, which must not serve anything else, -with a writer token for a placed org; it drops and recreates the cell's org triggers, so -never against a deployment in use. Needs no provider. +guard stays), deleting the rows its inserts wrote at the end. Run it against an attached +cell's server, which must not serve anything else, with a writer token for a placed org; +it drops and recreates the cell's org triggers, so never against a deployment in use. +Needs no provider. PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py \\ --base-url http://127.0.0.1:7932 --pg-dsn \\ @@ -36,7 +37,7 @@ async def _triggers(dsn: str, on: bool) -> None: for name in cell_triggers.ORG_TRIGGERS: await conn.execute(f'DROP TRIGGER IF EXISTS {name} ON loom."{table}"') if on: - await conn.execute(cell_triggers._create(f'loom."{table}"', name)) + await conn.execute(cell_triggers.create_statement(table, name)) enabled = await conn.fetchval( "SELECT count(*) FROM pg_trigger WHERE tgenabled = 'O' AND tgname = ANY($1::text[])", list(cell_triggers.ORG_TRIGGERS), @@ -111,6 +112,18 @@ async def main() -> None: ) finally: await _triggers(args.pg_dsn, True) + await _clean(args) + + +async def _clean(args: argparse.Namespace) -> None: + """Delete the rows the inserts wrote (the `memory.set` calls' rows stay the org's).""" + conn = await asyncpg.connect(args.pg_dsn, server_settings={"loom.tenancy_checked": "on"}) + try: + await conn.execute( + "DELETE FROM loom.scope_generations WHERE org = $1 AND scope LIKE 'gate-%'", args.org + ) + finally: + await conn.close() if __name__ == "__main__": diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 46e1439538..3ea7ddab80 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -682,7 +682,8 @@ mode transaction still in flight, found by its `ExclusiveLock` on `loom.tenancy_ this database (a pid captured at connect is wrong behind a connection path that swaps backends), and waits until the lock clears. Whether the commit landed is decided by a durable read of the mode, whose commit waits for the standby, since a commit ended while -it waited is on the primary alone ([attaching a cell](#attaching-a-cell), steps 6 and 7). +it waited is on the primary alone ([attaching a cell](#attaching-a-cell): step 3 and the +durable read). The mode cache reads the mode with a plain `SELECT`, which takes no row lock and never waits on attach. An insert that finds `enforced` (a process whose cache still reads `recording`) writes nothing: the request goes on to the placement call as in @@ -881,14 +882,13 @@ read-only scope, lists any tenancy row the directory does not place. yet, a later cell's configured orgs at attach and a changed one at `loom cell update` included, passes the gate, a closed cell and `placements_max` included, so a later cell closed or full before attach refuses the attach, naming the org. -- **The first cell records until `enforced` as a single deployment does.** In - `recording`, which lasts until step 3 writes `enforced` to the cell's mode (and again - whenever step 3 or step 4 returns the cell to it), a verified request for a new org - writes a tenancy row with no placement call, so the router's rate, `placements_max` - and `loom cell limit --closed` do not apply: new orgs arrive as they do on a single - deployment, bounded by its own signup, issuer and JIT settings, and step 2 or step 4 - places them unrefused. `loom doctor` lists the recorded orgs that hold no data, so the - operator sees what attach will place. +- **The first cell records until `enforced` as a single deployment does.** In `recording`, + which lasts until step 3 writes `enforced` to the cell's mode (the mode only rises after + it), a verified request for a new org writes a tenancy row with no placement call, so the + router's rate, `placements_max` and `loom cell limit --closed` do not apply: new orgs + arrive as they do on a single deployment, bounded by its own signup, issuer and JIT + settings, and step 2 or step 4 places them unrefused. `loom doctor` lists the recorded + orgs that hold no data, so the operator sees what attach will place. - **Configured orgs are placed at attach.** The cell's `oidc_jit_org`, the `self_signup_org` of a cell with signup on, and its code orgs (`LOOM_CODE_ORG` and the org of each `LOOM_CODE_REPOS_JSON` entry the code config keeps) and tracker org are @@ -986,8 +986,8 @@ The first attach (`loom cell attach`, its procedure on [attaching the first cell](../operations/cell-attach.md)) turns the existing deployment into the first cell. It aliases nothing (an `oauth_issuer_host` on another host refuses at `attach/begin`), and refuses with SCIM on until per-cell SCIM ships. A run holds an -advisory lock, `cell.ATTACH_LOCK_KEY`, on a session of its own for its life (prepare and -`loom cell update` take the same lock), after checking that the session is one direct +advisory lock, `cell.ATTACH_LOCK_KEY`, on a session of its own for its life (prepare +takes the same lock), after checking that the session is one direct backend that keeps the tenancy marker: a lock left on a pooler's server connection would refuse every later run, and attach's tenancy rows would be refused as an unchecked writer's. A lock found lost between steps is taken again (refusing when another run holds @@ -1034,74 +1034,76 @@ It refuses, before any write, naming each: `server/cli/cell_scan.py`), and a settings profile or quota that caps what attach's user reads. -Then, in order: - -1. `attach/begin`, on every run, before any long step: the router probes the stores - attach wrote its nonces to, checks the upstream, and writes the fingerprints and - `keys_fp`; a directory restored to before it is written again. -2. Read the mode: `recording`, or `enforced` on a resume, which goes from the triggers, - through `attach/enforced` again, to the durable read. -3. Create the triggers itself, on every run, before step 2's scan and again just before the - mode commit (so a table a migration added during the scan is guarded from the commit - on): one table at a time under the migration lock, each DDL waiting 100 ms for its lock - and retried a second apart within `--wait`; an autovacuum holding it gets one try just - past `deadlock_timeout`, so it yields, and an anti-wraparound vacuum is named. Then - `cell_triggers.missing`; any defect refuses. A refused attach in `recording` leaves the - triggers, which pass every row there. -4. In `recording`, the scan: one statement per table, no long transaction (step 4 reads - every source again): the Postgres tables (`loom.tenancy` among them) by skip scans on an - `org`-led btree index, then the ClickHouse tables, each flushed and synced first, with - the configured orgs of both fingerprints. Log tables (`cell_scan.LOG_TABLES`) are not - read: a log row keeps the org a caller typed before any check, so placing a log-only - org would let anyone pin an org name to the cell; `loom doctor --stray-orgs` names such - orgs. An org longer than `cell.ORG_MAX_BYTES` refuses, naming it, before any tenancy row - is written; a name no request can carry is skipped and named. Then the tenancy rows of - the orgs found, on a marked session, and no placement: unscoped callers are answered - from the tenancy rows, and placing now would move the point of no return earlier. -5. The pre-commit checks: the heartbeats, the tenancy-privilege and unmarked-session - checks and the standbys again, and the scan's elapsed time within half of - `--scan-budget`, since step 4 reads the same tables again. Each refuses in - `recording`, which can still be backed out. -6. The mode commit: `attach/enforced`, then on a session of its own an EXCLUSIVE lock on - `loom.tenancy_mode`, 2 s a try within `--wait`, the attach lock confirmed and the - cheap heartbeat refusals run before each try (a booting process skips the try). After a - failed try, the mode transaction an earlier run left in flight is found by its lock - (only attach takes `ExclusiveLock` there; a relation's OID is matched in this database - alone) and ended. From the commit on the mode only rises: every later refusal leaves - the cell `enforced` and `attaching`, and a rerun resumes from the durable read. -7. The durable read, in every run that reads `enforced`: the mode and the `first` row read - in a transaction that emits a transactional logical message, whose commit waits for - the synchronous standbys, with the standbys checked attached just before and after. - Then `X`, the snapshot's xmax, which must exceed that message's xid. A READ COMMITTED - writer that read `recording` holds an xid below `X`, and a REPEATABLE READ one a - snapshot older than `X`: the trigger functions are `VOLATILE`, and the message's xid is - assigned after the mode commit. -8. The heartbeats again: a process with `LOOM_CELL` unset or another cell, without the - keys, unmarked, or with no router where it serves, refuses at once. No process depends - on its cached mode (a request records a new org through `tenancy._RECORD`, which reads - the mode `FOR SHARE`), so neither a cache lifetime nor a request deadline is waited out. -9. Step 4: wait, within `--wait`, while any backend of the cell's database but the polling - one has an xid older than `X`, or any but an autovacuum a snapshot older than `X` - (compared through `age()`, exactly `< X` and wraparound-safe); past `--wait` attach - refuses, naming each, and each backend of another database holding the horizon. A - prepared transaction, or a row of the cell's database whose state is hidden, refuses - at once. Then the cheap checks (the writer check, `missing`, the Postgres checks, the - heartbeats with every stopped process with `LOOM_CELL` unset aged out, the standbys); - the rescan of every source in full, Postgres before ClickHouse (the outbox and delivery - tables move rows to ClickHouse), and the queue and drain tables and the tables a view - reads before the others (`cell_scan.MOVE_SOURCES`); the tenancy rows; the placements, - in `attach/place` batches; the writer check again; and the final checks, under a floor - of their own (the larger of what `--wait` left and `instances.boot_allowance_s()`): - the heartbeats, read through the rescan every half of the time the leader keeps a - stopped row (`instances.PRUNE_AFTER_S`), refusing any process that ran with `LOOM_CELL` - unset during it, `loom.cell_self` equal to the directory's fingerprints, one `keys_fp` - equal to the one `begin` wrote, the mode `enforced`; then `attach/serving`. - -`--wait` (default 10 minutes) is shared by the waits alone; `--scan-budget` (default 2 -hours) bounds each full scan. `--dry-run` runs the refusals and the scan, prints what would -be placed and skipped and the scan's time, and writes nothing; the router's checks at -`attach/begin` do not run. On a `serving` row attach reads the cell's mode: `enforced` is -done (exit 0), anything else refuses, naming `loom cell restore`. +Then, in order (the step numbers are the ones the rest of this page uses): + +- **`attach/begin`**, on every run, before any long step: the router probes the stores + attach wrote its nonces to, checks the upstream, and writes the fingerprints and + `keys_fp`; a directory restored to before it is written again. +- **Step 1, the mode**: `recording`, or `enforced` on a resume, which goes from the + triggers, through `attach/enforced` again, to the durable read. +- **The triggers**: created by attach itself, on every run, before step 2 and again just + before step 3 (so a table a migration added during the scan is guarded from the commit + on): one table at a time under the migration lock, each DDL waiting 100 ms for its lock + and retried a second apart within `--wait`; an autovacuum holding it gets one try just + past `deadlock_timeout`, so it yields, and an anti-wraparound vacuum is named. Then + `cell_triggers.missing`; any defect refuses. A refused attach in `recording` leaves the + triggers, which pass every row there. +- **Step 2, the scan**, in `recording`: one statement per table, no long transaction (step 4 + reads every source again): the Postgres tables (`loom.tenancy` among them) by skip scans + on an `org`-led btree index, then the ClickHouse tables, each flushed and synced first, + with the configured orgs of both fingerprints. Log tables (`cell_scan.LOG_TABLES`) are not + read: a log row keeps the org a caller typed before any check, so placing a log-only org + would let anyone pin an org name to the cell; `loom doctor --stray-orgs` names such orgs. + An org longer than `cell.ORG_MAX_BYTES` refuses, naming it, before any tenancy row is + written; a name no request can carry is skipped and named. Then the tenancy rows of the + orgs found, on a marked session, and no placement: unscoped callers are answered from the + tenancy rows, and placing now would move the point of no return earlier. +- **The pre-commit checks**: the heartbeats, the tenancy-privilege and unmarked-session + checks and the standbys again, and the scan's elapsed time within half of `--scan-budget`, + since step 4 reads the same tables again. Each refuses in `recording`, which can still be + backed out. +- **Step 3, the mode commit**: `attach/enforced`, then on a session of its own an EXCLUSIVE + lock on `loom.tenancy_mode`, 2 s a try within `--wait`, the attach lock confirmed and the + cheap heartbeat refusals run before each try (a booting process skips the try). After a + failed try, the mode transaction an earlier run left in flight is found by its lock (only + attach takes `ExclusiveLock` there; a relation's OID is matched in this database alone) + and ended. From the commit on the mode only rises: every later refusal leaves the cell + `enforced` and `attaching`, and a rerun resumes from the durable read. +- **The durable read**, in every run that reads `enforced`: the mode and the `first` row + read in a transaction that emits a transactional logical message, whose commit waits for + the synchronous standbys, with the standbys checked attached just before and after. Then + `X`, the snapshot's xmax, which must exceed that message's xid. A READ COMMITTED writer + that read `recording` holds an xid below `X`, and a REPEATABLE READ one a snapshot older + than `X`: the trigger functions are `VOLATILE`, and the message's xid is assigned after + the mode commit. +- **The heartbeats** again: a process with `LOOM_CELL` unset or another cell, without the + keys, unmarked, or with no router where it serves, refuses at once. No process depends on + its cached mode (a request records a new org through `tenancy._RECORD`, which reads the + mode `FOR SHARE`), so neither a cache lifetime nor a request deadline is waited out. +- **Step 4**: wait, within `--wait`, while any backend of the cell's database but the + polling one has an xid older than `X`, or any but an autovacuum a snapshot older than `X` + (compared through `age()`, exactly `< X` and wraparound-safe); past `--wait` attach + refuses, naming each, and each backend of another database holding the horizon. A prepared + transaction, or a row of the cell's database whose state is hidden, refuses at once. Then + the cheap checks (the writer check, `missing`, the Postgres checks, the heartbeats with + every stopped process with `LOOM_CELL` unset aged out, the standbys); the rescan of every + source in full, Postgres before ClickHouse (the outbox and delivery tables move rows to + ClickHouse), and the queue and drain tables and the tables a view reads before the others + (`cell_scan.MOVE_SOURCES`); the tenancy rows; the placements, in `attach/place` batches; + the writer check again; and the final checks, under a floor of their own (the larger of + what `--wait` left and `instances.boot_allowance_s()`): the heartbeats, read through the + rescan every half of the time the leader keeps a stopped row (`instances.PRUNE_AFTER_S`), + refusing any process that ran with `LOOM_CELL` unset during it, `loom.cell_self` equal to + the directory's fingerprints, one `keys_fp` equal to the one `begin` wrote, the mode + `enforced`; then `attach/serving`. + +`--wait` (default 10 minutes) is the wall time the bounded waits share, each lock try +included and never longer than what is left; `--scan-budget` (default 2 hours) bounds each +full scan, which `--wait` does not count. `--dry-run` runs the refusals and the scan, prints +what would be placed and skipped and the scan's time, and writes nothing; the router's +checks at `attach/begin` do not run. On a `serving` row attach reads the cell's mode: +`enforced` is done (exit 0), and any other mode refuses: the directory and the cell's +Postgres disagree, as after a restore of one of them, and attach lowers and repairs neither. A later cell is attached only once the first is `serving`. It must hold no org data (attach refuses otherwise) and starts in `enforced`. Its attach runs, in order: the @@ -1297,13 +1299,13 @@ the directory row. | A first cell restored to before its prepare, not yet attached | at processes with `LOOM_CELL` set, every request but health probes and the probe route answers 503 until prepare reruns | | The directory lost | every cell serves the orgs it holds; placements and uncached lookups answer 503, directory commands refuse, until it is back | | A cell in `recording`, the directory unreadable | a new org answers 503; the orgs it holds keep serving | -| A process stuck during first attach | the attach refuses after `--wait` and the cell returns to `recording` | +| A process stuck during first attach | before step 3, attach refuses after `--wait` and the cell stays `recording`; after it, attach refuses, leaving the cell `enforced` and `attaching`, and a rerun resumes | | A recording insert held open during first attach | the mode commit's tries wait behind its `FOR SHARE` lock within `--wait`; past it attach refuses and the cell stays `recording`; a rerun resumes | | Attach stalls while it holds the mode table's lock | Postgres ends its session within seconds and releases the lock; new orgs answer within their deadline | | A transaction open since before step 4's snapshot | attach refuses once `--wait` runs out (a prepared one at once), naming what it waited on, and leaves `enforced` | | A CLI command with `LOOM_CELL` unset, started before attach's step 3 | it read the state while the first cell was `recording` and keeps writing ClickHouse after `serving`; the runbook sets `LOOM_CELL` on every CLI before attach, and `loom doctor --stray-orgs` names the orgs it left without a tenancy row | | A CLI of a build before the state read | it is not refused on an `enforced` cell; every CLI and CI image runs a build with the read before attach ([upgrades](../operations/upgrades.md#upgrade-procedure)) | -| An org found only in a log table of the first cell | never placed; once it is placed in another cell, its log rows here (`agent_events` identities among them) are outside its erasure there until the stray-org erase command removes them; `loom doctor --stray-orgs` names it | +| An org found only in a log table of the first cell | never placed; once it is placed in another cell, its log rows here (`agent_events` identities among them) are outside its erasure there until they are removed by hand (the runbook's procedure); `loom doctor --stray-orgs` names it | | An org longer than `cell.ORG_MAX_BYTES` written before the limit | the first attach refuses in its scan, naming it, until the runbook's manual procedure removes it; one first found in step 4 is skipped and named | | A writer, grant, role setting or subscription added after `serving` | attach's checks ran once and pass it; `loom doctor` warns on the Postgres ones on every run | | `LOOM_SERVES_TRAFFIC=false` on a pod a load balancer routes to | attach does not require its router answer; its new orgs answer 503 | @@ -1474,126 +1476,117 @@ An integration test with two cells and a router in CI's Postgres job: - every route, the batch entries of MCP and every CLI command included, refuses another cell's org and writes nothing in its Postgres or ClickHouse (the route, op-field and CLI censuses cover the rest); -- the first attach places every org with a row in any table, an org first seen during - the attach is placed in step 4, and so is one a direct Postgres insert wrote between - step 2's scan and `enforced`, a ClickHouse row an older CLI wrote then and a - bucket object catalogued then; a process past `LOOM_JWT_REVOCATION_MAX_STALE_SECONDS` - of failed reads of its own rows or mode is gated and serves again once a read lands; a - pod with `LOOM_CELL` unset is unready, naming the setting, answers 503 to a bearer - request, passes its health probe, answers the probe route and runs no worker on an - attached cell and on a later cell between prepare and attach, keeps serving on a first - cell in `recording`, and keeps the last mode it read through a failed read, - `recording` included; a CLI command - declaring an org with it unset is refused on an `enforced` or `partial` cell, an - org-free one (a token revocation) runs, also with Postgres down on a deployment never - prepared, and a `loom cell` store command refuses; `loom cell limit --closed` succeeds - with the cell's Postgres down; after `loom cell credential --cell --next` both - credentials place and read state, `--finish` refuses the old credential, for a cell - finished before its pods hold the new one the router refuses its calls with 401, its - request answers 503 writing nothing, and it recovers by recording the old hash again, - `--revert` drops a pending `next`, which is then refused, and the old credential keeps - placing, and a second `--next` while one is pending and a hash another cell records as - current or next are refused; `loom cell limit` and `loom cell credential --next`, - `--finish` and `--revert` refuse while the directory is unavailable; concurrent - placements at one under `placements_max` across two router replicas write one; a CLI - command declaring an org with `LOOM_CELL_ROUTER_TOKEN` set refuses, naming it, on a - first cell once it is `enforced` (`attaching` included) and on a later cell, and on a - first cell in `recording` runs and records the org; attach with `placements_max` below - the orgs the first cell holds places them all, and a later cell's attach past its - quota or closed refuses, naming the org; a first cell with `--closed` and - `placements_max` set before step 2 attaches, and an org a request records between step - 2 and `enforced` is placed by step 4 with no 503; after step 3's write of the cell's - mode, a process whose cache still reads `recording` sends a new org's request to the - placement call, which a closed cell answers 503; a CLI command whose insert finds - `enforced` refuses, naming the org, and makes no placement call; after step 3's - refused wait or step 4's recheck returns the cell to `recording`, a new org is - recorded with no placement call, and one at a process whose cache still reads - `enforced` places through the gate until the cache expires; each ordering of attach's - write of the mode with a recording insert holds as stated above (running, starting - during the write, a stream of overlapping inserts each shorter than a try, with attach - setting `enforced` within a try or two, held across several tries, held until too - little of `--wait` is left, a stalled attach, a commit cut off by its timeout, a - commit whose connection is lost while it waits for the standby, read back only once - that backend has gone and written again durably if it reads `enforced`); a - `REPEATABLE READ` transaction whose first statement ran before `enforced` committed, - on attach's role or a second role, holds step 4 until it ends and its write of an - unheld org is then placed, one whose first statement ran after is refused by the - trigger, a transaction that starts after attach takes the lock but before its commit - is visible holds step 4, a transaction opened in `recording` that outlives one run's - `--wait` holds the rerun's step 4, after a refusal and after `loom cell restore` - raised the mode, an autovacuum worker on a loom table does not hold step 4 and a - background worker with a transaction open since `recording` does, a prepared - transaction (after a restart with `max_prepared_transactions` raised during the wait) - makes step 4 refuse at once, naming it, with the mode still `enforced` and a rerun - refused until it is resolved and the setting is back to 0, a write transaction open in - a second database on the same server holds step 4 and is named, a long read on the - synchronous standby, or on a second standby, with `hot_standby_feedback` on does not - hold it, nor does a base backup running across it, a write held open on a +- the first attach places every org with a row in any table, an org first seen during the + attach is placed in step 4, and so is one a direct Postgres insert wrote between step 2's + scan and `enforced`, a ClickHouse row an older CLI wrote then and a bucket object + catalogued then; a process past `LOOM_JWT_REVOCATION_MAX_STALE_SECONDS` of failed reads of + its own rows or mode is gated and serves again once a read lands; a pod with `LOOM_CELL` + unset is unready, naming the setting, answers 503 to a bearer request, passes its health + probe, answers the probe route and runs no worker on an attached cell and on a later cell + between prepare and attach, keeps serving on a first cell in `recording`, and keeps the + last mode it read through a failed read, `recording` included; a CLI command declaring an + org with it unset is refused on an `enforced` or `partial` cell, an org-free one (a token + revocation) runs, also with Postgres down on a deployment never prepared, and a `loom +cell` store command refuses; `loom cell limit --closed` succeeds with the cell's Postgres + down; after `loom cell credential --cell --next` both credentials place and read + state, `--finish` refuses the old credential, for a cell finished before its pods hold the + new one the router refuses its calls with 401, its request answers 503 writing nothing, + and it recovers by recording the old hash again, `--revert` drops a pending `next`, which + is then refused, and the old credential keeps placing, and a second `--next` while one is + pending and a hash another cell records as current or next are refused; `loom cell limit` + and `loom cell credential --next`, `--finish` and `--revert` refuse while the directory is + unavailable; concurrent placements at one under `placements_max` across two router + replicas write one; a CLI command declaring an org with `LOOM_CELL_ROUTER_TOKEN` set + refuses, naming it, on a first cell once it is `enforced` (`attaching` included) and on a + later cell, and on a first cell in `recording` runs and records the org; attach with + `placements_max` below the orgs the first cell holds places them all, and a later cell's + attach past its quota or closed refuses, naming the org; a first cell with `--closed` and + `placements_max` set before step 2 attaches, and an org a request records between step 2 + and `enforced` is placed by step 4 with no 503; after step 3's write of the cell's mode, a + process whose cache still reads `recording` sends a new org's request to the placement + call, which a closed cell answers 503; a CLI command whose insert finds `enforced` + refuses, naming the org, and makes no placement call; a refusal after step 3's write + leaves the cell `enforced` and `attaching`, where a new org is placed through the gate; + each ordering of attach's write of the mode with a recording insert holds as stated above + (running, starting during the write, a stream of overlapping inserts each shorter than a + try, with attach setting `enforced` within a try or two, held across several tries, held + past `--wait` (attach refuses, the cell still `recording`), a stalled attach, a commit cut + off by its timeout, and a commit whose connection is lost while it waits for the standby, + which the durable read decides); a `REPEATABLE READ` transaction whose first statement ran + before `enforced` committed, on attach's role or a second role, holds step 4 until it ends + and its write of an unheld org is then placed, one whose first statement ran after is + refused by the trigger, a transaction that starts after attach takes the lock but before + its commit is visible holds step 4, a transaction opened in `recording` that outlives one + run's `--wait` holds the rerun's step 4 after a refusal, an autovacuum worker on a loom + table does not hold step 4 and a background worker with a transaction open since + `recording` does, a prepared transaction (after a restart with `max_prepared_transactions` + raised during the wait) makes step 4 refuse at once, naming it, with the mode still + `enforced` and a rerun refused until it is resolved and the setting is back to 0, a write + transaction open in a second database on the same server holds step 4 and is named, a long + read on the synchronous standby, or on a second standby, with `hot_standby_feedback` on + does not hold it, nor does a base backup running across it, a write held open on a `replication=database` connection holds step 4, and so does a `REPEATABLE READ` - transaction on one, opened in `recording` and writing nothing before `enforced`, a - long logical dump (a `REPEATABLE READ` read-only transaction) on the cell's database - past `--wait` makes step 4 refuse, naming it, with the cell left `enforced`, and the - first cell's attach without `pg_read_all_stats` or with `max_prepared_transactions` - above 0 refuses before step 1 with the mode still `recording`, while a later cell's - attach does not require either; a missing or disabled org trigger is refused by attach - and named by `loom doctor`, a write for an unheld org on an `enforced` cell whose mode - row was deleted is refused and the state named by `loom doctor`; on that cell a pod - with `LOOM_CELL` unset is unready and runs no worker, and a CLI command declaring an - org with it unset is refused, as a pod is; updating, deleting or truncating the - `first` row on an `enforced` cell is refused, also under - `session_replication_role = replica`, and a missing or disabled `first`-row trigger is - refused by attach and named by `loom doctor`; a running process with `LOOM_CELL` set - on a cell whose mode row is removed answers 503 to a bearer request and runs no worker - pass, and serves again after a rerun of `loom cell prepare` writes the row back while - it runs; a cell the directory registers with no state, tenancy or own row is named by - `loom doctor`, its processes with `LOOM_CELL` set stay unready, a process with it - unset serves as a single deployment, and `loom cell restore` writes both rows back, an - attached cell then serving again and an unattached one after a prepare rerun; a cell - with both rows absent beside a tenancy row, and one beside only its own row, has the - function refuse an unheld org (a boot creates no trigger there), every process gated - and `loom doctor` naming it, and a prepare rerun writes the rows back; a Postgres-only - restore of an attached first cell to before its prepare is named by `loom doctor` and - closed by `loom cell restore`; a prepare rerun during a pending `loom cell update` on - an attached cell, with processes on both values and `loom.cell_self` lost, writes both - values back and keeps both sets of processes serving, and one before attach refuses - the two configurations; a prepare rerun on an attached cell whose `current` is missing - and whose pods' configuration differs from the directory row refuses, and one before - attach whose heartbeats report another bucket prefix refuses; a prepare rerun given a - credential hash that is neither the row's `current` nor its `next` is refused; a - prepare rerun on a `serving` later cell whose prefix holds data writes a removed mode - row back; a cell with a mode row and no `first` row has the function refuse an unheld - org (a boot creates no trigger there), every process gated and `loom doctor` naming - it, and a prepare rerun writes the row back; a later cell whose mode row is set to - `recording` records nothing, its trigger refuses an unheld org, a process with - `LOOM_CELL` set there serves as in `enforced`, a pod with `LOOM_CELL` unset there is - unready and runs no worker, `loom cell prepare` names the row, a CLI command declaring - an org with `LOOM_CELL_ROUTER_TOKEN` set refuses, naming it, and the router refuses - its state read; a build N-1 process's `memory.set` and delivery pass on a - never-prepared deployment after N's migration succeed and write no tenancy row, and on - the first cell in `recording` a boot creates no org trigger, and one created by hand - reads the state and looks nothing up, a write under - `session_replication_role = replica` for an unheld org is skipped (origin triggers), - an `INSERT ... ON CONFLICT DO UPDATE` that inserts a row for an unheld org is refused - and one that hits an existing row of it is not, a marked session's every write passes, - and the triggers on an `enforced` cell cover every table erasure classifies with an - `org` column but the three exclusions; and an org-keyed write while attach holds the - mode table's lock returns at once, with new-org requests answering within their - deadline and mode-cache reads returning at once throughout; `loom cell restore` after - a refusal at step 3's lock leaves the cell `enforced`, and the resumed attach runs - from step 3's wait; a `LOOM_CELL_ROUTER_TOKEN` not of the `lcr_` form is refused at - boot and named by `loom doctor`; `loom cell credential new` prints a credential of - that form and its SHA-256, with the directory and the stores unreachable; prepare and - `--next` refuse an argument that is not 64 lowercase hex characters, the credential - and its bare 128-character secret included, storing nothing; orgs a first cell records - in `recording` with no data are listed by `loom doctor`; a placement flood with no - quota set is held to the rate limit at each router replica, `loom doctor` names the - cell as one with no quota, and a call refused past it at one replica answers 503 with - a `Retry-After` of at most 60 s, after which its retry at that replica, with no other - call of the cell admitted between, is admitted; a cell's first prepare, with no - directory row yet, passes the deployment-id check; a pod or CLI that lost - `LOOM_PG_DSN` or `LOOM_STORAGE_AUTHORITY` on a cell is refused by the authority check; - the first cell's existing tokens, refresh tokens and OAuth clients keep working - through the cutover, a logout through the shared host revokes its refresh token; + transaction on one, opened in `recording` and writing nothing before `enforced`, a long + logical dump (a `REPEATABLE READ` read-only transaction) on the cell's database past + `--wait` makes step 4 refuse, naming it, with the cell left `enforced`, and the first + cell's attach without `pg_read_all_stats` or with `max_prepared_transactions` above 0 + refuses before step 1 with the mode still `recording`, while a later cell's attach does + not require either; a missing or disabled org trigger is refused by attach and named by + `loom doctor`, a write for an unheld org on an `enforced` cell whose mode row was deleted + is refused and the state named by `loom doctor`; on that cell a pod with `LOOM_CELL` unset + is unready and runs no worker, and a CLI command declaring an org with it unset is + refused, as a pod is; updating, deleting or truncating the `first` row on an `enforced` + cell is refused, also under `session_replication_role = replica`, and a missing or + disabled `first`-row trigger is refused by attach and named by `loom doctor`; a running + process with `LOOM_CELL` set on a cell whose mode row is removed answers 503 to a bearer + request and runs no worker pass, and serves again after a rerun of `loom cell prepare` + writes the row back while it runs; a cell the directory registers with no state, tenancy + or own row is named by `loom doctor`, its processes with `LOOM_CELL` set stay unready, a + process with it unset serves as a single deployment, and `loom cell restore` writes both + rows back, an attached cell then serving again and an unattached one after a prepare + rerun; a cell with both rows absent beside a tenancy row, and one beside only its own row, + has the function refuse an unheld org (a boot creates no trigger there), every process + gated and `loom doctor` naming it, and a prepare rerun writes the rows back; a + Postgres-only restore of an attached first cell to before its prepare is named by `loom +doctor` and closed by `loom cell restore`; a prepare rerun during a pending `loom cell +update` on an attached cell, with processes on both values and `loom.cell_self` lost, + writes both values back and keeps both sets of processes serving, and one before attach + refuses the two configurations; a prepare rerun on an attached cell whose `current` is + missing and whose pods' configuration differs from the directory row refuses, and one + before attach whose heartbeats report another bucket prefix refuses; a prepare rerun given + a credential hash that is neither the row's `current` nor its `next` is refused; a prepare + rerun on a `serving` later cell whose prefix holds data writes a removed mode row back; a + cell with a mode row and no `first` row has the function refuse an unheld org (a boot + creates no trigger there), every process gated and `loom doctor` naming it, and a prepare + rerun writes the row back; a later cell whose mode row is set to `recording` records + nothing, its trigger refuses an unheld org, a process with `LOOM_CELL` set there serves as + in `enforced`, a pod with `LOOM_CELL` unset there is unready and runs no worker, `loom +cell prepare` names the row, a CLI command declaring an org with `LOOM_CELL_ROUTER_TOKEN` + set refuses, naming it, and the router refuses its state read; a build N-1 process's + `memory.set` and delivery pass on a never-prepared deployment after N's migration succeed + and write no tenancy row, and on the first cell in `recording` a boot creates no org + trigger, and one created by hand reads the state and looks nothing up, a write under + `session_replication_role = replica` for an unheld org is skipped (origin triggers), an + `INSERT ... ON CONFLICT DO UPDATE` that inserts a row for an unheld org is refused and one + that hits an existing row of it is not, a marked session's every write passes, and the + triggers on an `enforced` cell cover every table erasure classifies with an `org` column + but the three exclusions; and an org-keyed write while attach holds the mode table's lock + returns at once, with new-org requests answering within their deadline and mode-cache + reads returning at once throughout; a refusal at step 3's lock leaves the cell `recording` + with its directory row `enforced`, and the rerun runs step 2 again; a + `LOOM_CELL_ROUTER_TOKEN` not of the `lcr_` form is refused at boot and named by `loom +doctor`; `loom cell credential new` prints a credential of that form and its SHA-256, with + the directory and the stores unreachable; prepare and `--next` refuse an argument that is + not 64 lowercase hex characters, the credential and its bare 128-character secret + included, storing nothing; orgs a first cell records in `recording` with no data are + listed by `loom doctor`; a placement flood with no quota set is held to the rate limit at + each router replica, `loom doctor` names the cell as one with no quota, and a call refused + past it at one replica answers 503 with a `Retry-After` of at most 60 s, after which its + retry at that replica, with no other call of the cell admitted between, is admitted; a + cell's first prepare, with no directory row yet, passes the deployment-id check; a pod or + CLI that lost `LOOM_PG_DSN` or `LOOM_STORAGE_AUTHORITY` on a cell is refused by the + authority check; the first cell's existing tokens, refresh tokens and OAuth clients keep + working through the cutover, a logout through the shared host revokes its refresh token; - attach refuses a shared issuer, root secret, Keeper path, nested bucket prefix or configured org, a second cell that holds data, and a second attach while one runs; - an OIDC sign-in, an MCP client's authorization and a device flow for an org in the @@ -1607,12 +1600,12 @@ An integration test with two cells and a router in CI's Postgres job: application ends the person there, a platform-admin bot still reaches a cell whose SCIM deactivated everyone, and attach refuses a cell with unprovisioned live subjects or sharing another's OIDC client; -- an attach stopped in step 3 after it set `enforced`, before the wait passed, resumes - with the wait before the SCIM precondition, and one refused by the SCIM precondition - resumes from the wait too, the mode still `enforced`; a later attach refused before - its directory transaction leaves `attach_started_at` empty, so a rerun checks a new - cell's refusals again, and a resumed first attach finds its configured orgs placed in - this cell and carries on; +- an attach stopped after step 3's commit landed, before the durable read, says the commit's + outcome is unknown, and its rerun resumes by the mode it reads; one refused after the + durable read resumes from it, the mode still `enforced`; a later attach refused before its + directory transaction leaves `attach_started_at` empty, so a rerun checks a new cell's + refusals again, and a resumed first attach finds its configured orgs placed in this cell + and carries on; - a cell restored to before the first attach's `enforced` gets its mode back from `loom cell restore`; one whose `loom cell restore` was skipped answers 503 for an org it does not hold, another cell's org included, writes no tenancy row for it, and is diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index 12a0cbe247..c51541d06d 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -105,17 +105,16 @@ and `attaching`, and the cell goes forward: fix the cause and rerun attach. Unse always goes forward. From `enforced` on, a `--platform-admin` mint that declares a new org and the admin account -routes cannot bring in a new org (`WRONG_CELL`) until `loom cell place` is available; an -ordinary mint scoped to its org places the org on its first request. A direct call of the -router's operator place route, or a live placement with the cell's credential, cannot read -the cell's Postgres and is accepted once the directory mode is `enforced`; the cell's -credential is the servers' alone. +routes cannot bring in a new org (`WRONG_CELL`); an ordinary mint scoped to its org places +the org on its first request. A direct call of the router's operator place route, or a live +placement with the cell's credential, cannot read the cell's Postgres and is accepted once +the directory mode is `enforced`; the cell's credential is the servers' alone. ## Orgs attach does not place - **An org longer than 2048 bytes** refuses step 2, naming it, before any tenancy row is - written. One first found in step 4 is skipped and named. Remove such an org by hand - until the stray-org erase command exists, in this order: its bucket objects, from the + written. One first found in step 4 is skipped and named. Remove such an org by hand, in + this order: its bucket objects, from the keys of its rows in `loom.documents`, `loom.document_blobs` and `loom.blob_uploads`, so no memory content is left outside erasure; then its rows in every table with an `org` column of both ClickHouse databases and of Postgres schema `loom`, the erasure ledger tables @@ -128,9 +127,9 @@ credential is the servers' alone. erasure there. `loom doctor --stray-orgs` names the orgs the cell's stores hold without a tenancy row, -smallest tables first within `--stray-orgs-budget` (default 60 s), and every table it left -unchecked. `loom cell place` and `loom cell restore`, which act on these, are not available -yet. +smallest tables first within `--stray-orgs-budget` (default 60 s), every table the budget +left unchecked and every table it could not read. Remove a stray org's rows by the same +procedure as an over-long org's. ## After attach diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index ea9cf77d35..52abea19c6 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -3,26 +3,27 @@ docs/operations/cell-attach.md). A run holds the attach lock on a direct session of the cell's Postgres for its life, and -goes, each step idempotent, so a rerun resumes by the directory row and the cell's mode: - -1. the directory row: on `serving` it is done (exit 0) once the cell reads `enforced`; -2. the refusals, before any write: the stores are the registered ones, the session is - direct and marked, the cell's processes (their heartbeats), the roles, settings and - subscriptions that would let an unchecked writer through, the ClickHouse writers, - the table list, durability, attach's own grants; -3. `attach/begin` (the router probes the stores and writes the fingerprints); -4. the mode: `recording`, or `enforced` on a resume; -5. the triggers (`cell_triggers.ensure_table`), under the migration lock one try at a time; -6. in `recording`, step 2's scan, refusing an org past `cell.ORG_MAX_BYTES`, and the - tenancy rows of the orgs found; the triggers again; the pre-commit checks; -7. step 3: `attach/enforced`, then the mode commit (never on a resume): from it on the mode - only rises, so every later refusal leaves the cell `enforced` and `attaching`; -8. the durable read of `enforced`, the heartbeats, step 4's wait for every transaction - that could have read `recording`, the rescan, the tenancy rows and their placements - (`attach/place`), the final checks and `attach/serving`. - -`--wait` bounds the waits, `--scan-budget` each full scan. `--dry-run` runs the refusals -and the scan and prints what would be placed, writing nothing. +goes, each step idempotent, so a rerun resumes by the directory row and the cell's mode. +Steps 1 to 4 are the design's, as cells.md numbers them: + +- the directory row: on `serving` it is done (exit 0) once the cell reads `enforced`; +- the refusals, before any write: the stores are the registered ones, the session is + direct and marked, the cell's processes (their heartbeats), the roles, settings and + subscriptions that would let an unchecked writer through, the ClickHouse writers, the + table list, durability, attach's own grants; +- `attach/begin` (the router probes the stores and writes the fingerprints); +- step 1, the mode: `recording`, or `enforced` on a resume; +- the triggers (`cell_triggers.ensure_table`), under the migration lock one try at a time; +- in `recording`: step 2, the scan, refusing an org past `cell.ORG_MAX_BYTES`, and the + tenancy rows of the orgs found; the triggers again; the pre-commit checks; step 3, + `attach/enforced` and the mode commit, from which on the mode only rises, so every + later refusal leaves the cell `enforced` and `attaching`; +- the durable read of `enforced`, the heartbeats, and step 4: the wait for every + transaction that could have read `recording`, the rescan, the tenancy rows and their + placements (`attach/place`), the final checks and `attach/serving`. + +`--wait` bounds the waits in wall time, `--scan-budget` each full scan. `--dry-run` runs +the refusals and the scan and prints what would be placed, writing nothing. """ from __future__ import annotations diff --git a/server/cli/doctor_cell.py b/server/cli/doctor_cell.py index c79723bbbc..9edb8e275d 100644 --- a/server/cli/doctor_cell.py +++ b/server/cli/doctor_cell.py @@ -186,7 +186,10 @@ async def _checks(ch: Any, allowed_roles: tuple[str, ...] = ()) -> list[tuple[st try: reads: list[tuple[str, str, str]] = await _bounded(_writer_reads(allowed_roles)) except Exception as e: # noqa: BLE001 — an unread catalog is named, not a failed run - reads = [("cell_tenancy_privilege", WARN, f"not read: {_reason(e)}")] + reads = [ + (name, WARN, f"not read: {_reason(e)}") + for name in ("cell_tenancy_privilege", "cell_unchecked_writers") + ] return _grades(state, marked, marks_all, defects) + reads @@ -264,6 +267,7 @@ async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: budget = cell_scan.Budget(budget_s) stray: dict[str, list[str]] = {} unchecked: list[str] = [] + refused: list[str] = [] for i, (_n, store, table) in enumerate(work): found = cell_scan.ScanResult() name = table if store == "postgres" else table.canonical @@ -274,9 +278,15 @@ async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: else: for raw in await cell_scan.ch_orgs(ch, table, topology, budget): found.add(name, raw) - except Refused: - unchecked += [w[2] if w[1] == "postgres" else w[2].canonical for w in work[i:]] - break + except Refused as e: + # The budget spent leaves this table and the larger ones unread; any other + # refusal (a table with no org-led index, a replica that cannot sync) names + # this table alone, and the read goes on. + if budget.left() <= 1: + unchecked += [w[2] if w[1] == "postgres" else w[2].canonical for w in work[i:]] + break + refused += [str(r) for r in e.refusals] + continue orgs = sorted(set(found.orgs) - held) + sorted(found.too_long) bad = [m for ms in found.malformed.values() for m in ms] if orgs or bad: @@ -290,13 +300,20 @@ async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: or "every org found has a tenancy row", ) ] - if unchecked or problems: + if unchecked: out.append( ( "cell_stray_orgs_unchecked", WARN, - f"not read within --stray-orgs-budget ({budget_s:g} s): " - + ", ".join(unchecked + [str(p) for p in problems]), + f"not read within --stray-orgs-budget ({budget_s:g} s): " + ", ".join(unchecked), + ) + ) + if refused or problems: + out.append( + ( + "cell_stray_orgs_unread", + WARN, + "not read: " + "; ".join(refused + [str(p) for p in problems]), ) ) return out @@ -305,8 +322,9 @@ async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: def stray_org_checks(budget_s: float) -> list[tuple[str, str, str]]: """`loom doctor --stray-orgs`: the orgs the cell's stores hold without a tenancy row (log-only orgs, names no request can carry, rows an unchecked writer left), smallest - tables first within one budget, naming every table left unchecked. Never a FAIL: the - stray-org erase command removes what it names.""" + tables first within one budget, naming every table left unchecked and every table it + could not read. Never a FAIL: the operator removes what it names by hand + (docs/operations/cell-attach.md).""" from server.cli._ch import connect_declared, disconnect from server.core.ch import get_client, statement_timeout diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 516dee4cc1..4395138b4d 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -176,6 +176,11 @@ def _function_sql(function: str) -> re.Pattern[str]: ) +def create_statement(table: str, name: str) -> str: + """The CREATE TRIGGER for `name` on `loom.`, as `ensure` writes it.""" + return _create(f"loom.{_ident(table)}", name) + + def _create(qualified: str, name: str) -> str: """The CREATE TRIGGER for `name` on the table `qualified` names.""" if name == TENANCY_TRIGGER: diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 9bfe3fe9bf..e0f96f409f 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -668,3 +668,23 @@ async def test_a_role_with_the_grants_but_not_the_owner_role_is_refused_before_a await conn.execute(f"DROP OWNED BY {role}") await _admin(f"DROP OWNED BY {role}") await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_doctor_names_a_table_it_cannot_read_and_reads_the_rest(cell_stores) -> None: + from server.cli import doctor_cell + + stores = cell_stores + await _prepare(stores, _Network()) + tag = stores["tag"] + # An org table with no org-led index: its skip scan refuses, the read goes on. + await pg.execute("CREATE TABLE loom.it_no_org_index (id int PRIMARY KEY, org text)") + await pg.execute("INSERT INTO loom.it_no_org_index VALUES (1, $1)", tag + "unindexed") + await pg.execute( + "INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", tag + "stray" + ) + lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 300)} + assert tag + "stray" in lines["cell_stray_orgs"][1] + grade, detail = lines["cell_stray_orgs_unread"] + assert grade == "WARN" and "postgres loom.it_no_org_index" in detail + assert "cell_stray_orgs_unchecked" not in lines diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index b57a40eb57..857d12dfe0 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -801,3 +801,30 @@ async def run(self: Any) -> int: err = capsys.readouterr().err assert "the mode commit's outcome is unknown: rerun attach" in err assert "stays recording" not in err + + +@pytest.mark.asyncio +async def test_doctor_names_both_catalog_checks_when_the_catalog_is_unread( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from server.cli import doctor_cell + + async def before(_ch: Any) -> str: + return "" + + async def marked() -> tuple[str, bool, list[Any]]: + return "enforced", True, [] + + async def unmarked() -> bool: + return False + + async def unread(_roles: Any) -> Any: + raise OSError("the catalog read failed") + + monkeypatch.setattr(doctor_cell, "_deployment_check", before) + monkeypatch.setattr(doctor_cell, "_marked_session", marked) + monkeypatch.setattr(doctor_cell, "_unmarked_session", unmarked) + monkeypatch.setattr(doctor_cell, "_writer_reads", unread) + lines = {n: (g, d) for n, g, d in await doctor_cell._checks(None)} + for name in ("cell_tenancy_privilege", "cell_unchecked_writers"): + assert lines[name] == ("WARN", "not read: OSError: the catalog read failed") From 691f4f0eeb11f02b6d31dbc44f49a5d43583536f Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 08:37:50 +0000 Subject: [PATCH 008/114] Attach's closing line names the mode it last read, every DDL try ends inside --wait, and doctor warns on the role it runs as The mode is taken from the first state read, set to unknown only once the router accepted attach/enforced, and set from what the durable read saw; an interrupt prints the mode line and goes on. The trigger DDL's lock bound is taken after the migration lock, from what --wait has left then. An ended mode transaction whose lock does not clear refuses, naming its backend. A busy table's holders come back with ensure_table's answer, and attach says them once per change rather than a boot's warning a try. Attach counts the role it runs as among its own; doctor does not. The dead SCIM recheck and the unused scim_ok parameter are gone. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- server/cli/cell_attach.py | 18 +- server/cli/cell_attach_beats.py | 4 +- server/cli/cell_attach_pg.py | 71 ++++-- server/cli/doctor_cell.py | 2 +- server/core/cell_triggers.py | 55 +++-- tests/integration/test_cell_attach.py | 71 +++++- .../integration/test_cell_org_triggers_pg.py | 6 +- tests/unit/test_cell_attach.py | 208 +++++++++++++++++- 8 files changed, 368 insertions(+), 67 deletions(-) diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 52abea19c6..23449ccff9 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -282,6 +282,8 @@ async def stores_refusals(self) -> list[Refusal]: if recorded is None or str(recorded["deployment_id"]) != expected: out.append(Refusal("clickhouse", f"records another deployment id than {expected!r}")) state, first = await apg.read_mode(self.lock_conn) + if state in cell.MODES: + self.mode = state # what a refusal from here on leaves the cell at if first != (True, self.name): out.append( Refusal("postgres loom.cell_first", f"reads {first}, not (true, {self.name})") @@ -353,7 +355,7 @@ async def writer_refusals(self) -> tuple[list[Refusal], Any, Any]: async def catalog_refusals(self) -> list[Refusal]: """The Postgres checks that run again before the mode commit and in step 4.""" - holders = await apg.privilege_holders(self.lock_conn) + holders = await apg.privilege_holders(self.lock_conn, exempt_self=True) self.report["own_roles"] = holders.own out = apg.privilege_refusals(holders, self.args.pg_allow_role) out += await apg.unmarked_refusals(self.lock_conn, await apg.session_passes()) @@ -450,9 +452,9 @@ async def ready_for_commit(self) -> bool: async def commit_mode(self) -> None: from server.core import pg - # From here on the mode may have risen: only the durable read can say. - self.mode = "unknown" await self.router.call("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) + # From the mode transaction on the mode may have risen: only the durable read can say. + self.mode = "unknown" try: async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: await apg.commit_enforced( @@ -491,6 +493,8 @@ async def durable(self) -> int: if await pg_profile.standby_now() != "synchronous": raise _refused("postgres standbys", "not synchronous after the durable read") break + if read.mode in cell.MODES: + self.mode = read.mode if read.first != (True, self.name) or read.mode != "enforced": raise _refused( "postgres cell state", @@ -666,9 +670,7 @@ async def precommit(self, found: cell_scan.ScanResult) -> None: ) if out: raise Refused(out) - verdict = await self.check_beats("refuse") - if refusals := beats.final_refusals(verdict): - raise Refused(refusals) + await self.check_beats("refuse") async def dry_run(self, topology: Any, listed: cell_scan.TableList) -> int: found = await self.scan(topology, listed) @@ -711,6 +713,10 @@ def log(text: str) -> None: log(f"stopped ({type(e).__name__}: {e}); run it again") log(mode_line(run.mode)) return 1 + except BaseException: + # An interrupt: the mode line, then the interrupt as it came. + log(mode_line(run.mode)) + raise run.report["seconds"] = round(time.monotonic() - started, 1) print(json.dumps(run.report, indent=2, sort_keys=True, default=str)) return code diff --git a/server/cli/cell_attach_beats.py b/server/cli/cell_attach_beats.py index a9c6e80377..c2a8ed2428 100644 --- a/server/cli/cell_attach_beats.py +++ b/server/cli/cell_attach_beats.py @@ -180,7 +180,7 @@ def _stopped(out: Verdict, b: Beat, what: str, stopped_unset: StoppedUnset) -> N ) -def final_refusals(verdict: Verdict, *, scim_ok: bool = False) -> list[Refusal]: +def final_refusals(verdict: Verdict) -> list[Refusal]: out = [] if not verdict.live and not verdict.booting: out.append(Refusal("processes", "no live process of this cell reports a heartbeat")) @@ -192,7 +192,7 @@ def final_refusals(verdict: Verdict, *, scim_ok: bool = False) -> list[Refusal]: "finish the root-secret rollout first", ) ) - if verdict.scim and not scim_ok: + if verdict.scim: out.append( Refusal("processes", "SCIM is on: attach with SCIM on needs the per-cell SCIM release") ) diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 076dc4a196..9c6e4708d6 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -257,15 +257,16 @@ async def encoding_refusals(conn: Any) -> list[Refusal]: ORDER BY 1, 2 """ -# The roles attach counts as its own: superusers, and a login role that itself owns +# The roles attach counts as its own: superusers, a login role that itself owns # loom.tenancy, loom.tenancy_mode or a guarded table (it can drop or disable the -# triggers, or lower the mode, whatever else it holds). +# triggers, or lower the mode, whatever else it holds), and, for attach only, the role it +# runs as ($2): the operator runs attach, while doctor may run as a CI or monitoring role. _OWN_ROLES = """ SELECT rolname FROM pg_roles WHERE rolcanlogin AND (rolsuper OR oid IN ( SELECT c.relowner FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'loom' AND c.relkind IN ('r', 'p') AND (c.relname IN ('tenancy', 'tenancy_mode') OR c.relname = ANY($1::text[])) -)) OR rolname = current_user +)) OR ($2 AND rolname = current_user) ORDER BY 1 """ @@ -279,9 +280,11 @@ class Holders: others: dict[str, list[str]] -async def privilege_holders(conn: Any) -> Holders: +async def privilege_holders(conn: Any, *, exempt_self: bool) -> Holders: + """The holders of a right `_RIGHTS` counts; `exempt_self` counts the role this session + runs as among the own roles (attach's, not doctor's).""" guarded = [t for t in await cell_triggers.planned_tables(conn) if t != "tenancy"] - own = [str(r["rolname"]) for r in await conn.fetch(_OWN_ROLES, guarded)] + own = [str(r["rolname"]) for r in await conn.fetch(_OWN_ROLES, guarded, exempt_self)] others: dict[str, list[str]] = {} for r in await conn.fetch(_RIGHTS): if r["role"] not in own: @@ -449,42 +452,49 @@ async def _ensure_table( conn: Any, table: str, deadlock_ms: int, wait: Wait, log: Callable[[str], None] ) -> Refusal | None: yielded = False + said = "" while True: - lock_ms = 100 + most_ms = 100 holders = await cell_triggers.holders(conn, table) if not yielded and any(h["backend_type"] == "autovacuum worker" for h in holders): - lock_ms, yielded = deadlock_ms + 500, True - lock_ms = wait.bound_ms(lock_ms) - result = "busy" if lock_ms <= 0 else await _ensure_one(conn, table, lock_ms, wait) - if result == "applied": + most_ms, yielded = deadlock_ms + 500, True + outcome = await _ensure_one(conn, table, most_ms, wait) + if outcome.answer == "applied": return None - if result == "denied": + if outcome.answer == "denied": return Refusal( f"postgres loom.{table}", "attach's role cannot alter it: make the role a member of its owner role", ) if not await wait.pause(): return _busy_refusal(table, await cell_triggers.holders(conn, table)) - log(f"loom.{table}: its lock is busy; retrying") + # Said again only when the holders change: a retry a second is otherwise silent. + if outcome.holders != said: + said = outcome.holders + log(f"loom.{table}: its lock is busy ({said or 'no holder named'}); retrying") -async def _ensure_one(conn: Any, table: str, lock_ms: int, wait: Wait) -> str: - """One try under the migration lock, whose own wait is bounded by what `wait` has left.""" +async def _ensure_one(conn: Any, table: str, most_ms: int, wait: Wait) -> cell_triggers.Outcome: + """One try under the migration lock: the migration lock's wait, and then the DDL's lock + wait (at most `most_ms`), each bounded by what `wait` has left when it starts.""" from server.core.schema.pg_migrate import _LOCK_KEY migration_ms = wait.bound_ms(_MIGRATION_LOCK_MS) if migration_ms <= 0: - return "busy" + return cell_triggers.Outcome("busy") await conn.execute(f"SET lock_timeout = '{migration_ms}ms'") try: await conn.execute(_MIGRATION_LOCK, _LOCK_KEY) except Exception as e: if getattr(e, "sqlstate", None) == "55P03": - return "busy" # a boot migrating: the next try + return cell_triggers.Outcome("busy") # a boot migrating: the next try raise finally: await conn.execute("RESET lock_timeout") try: + lock_ms = wait.bound_ms(most_ms) + if lock_ms <= 0: + return cell_triggers.Outcome("busy") return await cell_triggers.ensure_table( conn, table, recording_first=True, lock_timeout=f"{lock_ms}ms", attempts=1 ) @@ -522,6 +532,9 @@ def _busy_refusal(table: str, holders: list[Any]) -> Refusal: """ _TIMED_OUT = frozenset({"55P03", "57014"}) +# How long attach waits for an ended mode transaction's lock to clear. +_END_ROUNDS = 30 # not a setting: checks of the lock +_END_PAUSE_S = 0.1 # not a setting: the pause between them async def commit_enforced( @@ -585,10 +598,13 @@ async def _mode_try(conn: Any, lock_ms: int) -> bool: async def end_mode_transactions(conn: Any, confirm_lock: Callable[[], Any]) -> None: - """End an earlier run's mode transaction still in flight and wait until its lock - clears; Refused when its role is a superuser and attach's is not.""" - for _ in range(30): - rows = await conn.fetch(_MODE_HOLDERS) + """End an earlier run's mode transaction still in flight and wait, for up to + `_END_ROUNDS` checks `_END_PAUSE_S` apart, until its lock clears; Refused when its role + is a superuser and attach's is not, or when a backend still holds the lock after the + last check, naming it.""" + rows: list[Any] = [] + for _ in range(_END_ROUNDS): + rows = list(await conn.fetch(_MODE_HOLDERS)) if not rows: return superuser = bool( @@ -608,7 +624,20 @@ async def end_mode_transactions(conn: Any, confirm_lock: Callable[[], Any]) -> N ) await confirm_lock() await conn.fetchval("SELECT pg_terminate_backend($1)", r["pid"]) - await _sleep(0.1) + await _sleep(_END_PAUSE_S) + if not (rows := list(await conn.fetch(_MODE_HOLDERS))): + return + raise Refused( + [ + Refusal( + f"postgres backend {r['pid']}", + f"still holds loom.tenancy_mode's lock after it was ended " + f"({r['application_name'] or 'no name'}, since {r['xact_start']}): rerun attach " + "once it has gone", + ) + for r in rows + ] + ) # -- the durable read ------------------------------------------------------------------- diff --git a/server/cli/doctor_cell.py b/server/cli/doctor_cell.py index 9edb8e275d..e8a5fc83c1 100644 --- a/server/cli/doctor_cell.py +++ b/server/cli/doctor_cell.py @@ -95,7 +95,7 @@ async def _writer_reads(allowed_roles: tuple[str, ...]) -> list[tuple[str, str, from server.core import pg async with pg.dedicated(statement_timeout_ms=_STATEMENT_MS) as conn: - holders = await cell_attach_pg.privilege_holders(conn) + holders = await cell_attach_pg.privilege_holders(conn, exempt_self=False) unchecked = await cell_attach_pg.unmarked_refusals(conn, False) roles = cell_attach_pg.privilege_refusals(holders, allowed_roles) return [ diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 4395138b4d..7a1ee5e5b7 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -21,7 +21,7 @@ import dataclasses import logging import re -from typing import Any, Literal +from typing import Any, Literal, NamedTuple from server.core import cell, pg, readiness @@ -95,6 +95,14 @@ # What a table's DDL came to (`ensure_table`, `_run_ddl`). Applied = Literal["applied", "busy", "denied"] + +class Outcome(NamedTuple): + """A table's DDL's answer, and with `busy` the sessions it waited behind.""" + + answer: Applied + holders: str = "" + + # pg_trigger.tgtype bits (src/include/catalog/pg_trigger.h). _ROW, _BEFORE, _INSERT, _UPDATE, _INSTEAD = 1, 2, 4, 16, 64 # The events each trigger must fire on. @@ -265,14 +273,15 @@ async def ensure_table( recording_first: bool = False, lock_timeout: str = LOCK_TIMEOUT, attempts: int = LOCK_ATTEMPTS, -) -> Applied: +) -> Outcome: """`ensure` for one table of the planned set (`loom.tenancy` its guard, any other its org triggers), so one table's DDL can run alone: in `enforced`, or with `recording_first` also in a first cell's `recording`, where the org triggers' function passes every row and a boot never creates them. `applied` when the DDL ran or nothing was needed, `busy` when its lock was not taken in `attempts` tries of `lock_timeout`, - `denied` when this role cannot alter the table. Raises for a table outside the planned - set, or in another state.""" + `denied` when this role cannot alter the table; a `busy` answer carries the sessions it + waited behind and logs nothing, so the caller says it as often as it retries. Raises + for a table outside the planned set, or in another state.""" state = await cell.read_state(conn) if state != "enforced" and not (recording_first and state == "recording"): raise RuntimeError(f"cell triggers: not created in state {state!r}") @@ -284,8 +293,8 @@ async def ensure_table( ((_, names),) = planned sqls = _table_statements(table, names, await _triggers(conn, names)) if not sqls: - return "applied" - return await _run_ddl(conn, table, sqls, lock_timeout, attempts) + return Outcome("applied") + return await _run_ddl(conn, table, sqls, lock_timeout, attempts, warn=False) def _table_statements( @@ -308,11 +317,14 @@ async def _run_ddl( sqls: list[str], lock_timeout: str = LOCK_TIMEOUT, attempts: int = LOCK_ATTEMPTS, -) -> Applied: + *, + warn: bool = True, +) -> Outcome: """A table's DDL in one transaction, `attempts` short tries: `applied`, or which of the - two causes cells.md names kept it from applying (`busy`: its lock not taken; `denied`: - a table this role cannot alter), which a boot serves past; any other error fails it, as - a failed read of the state does.""" + two causes cells.md names kept it from applying (`busy`: its lock not taken, with the + sessions it waited behind; `denied`: a table this role cannot alter), which a boot + serves past; any other error fails it, as a failed read of the state does. `warn` logs + a busy table once, after its last try, with its DDL and holders (a boot's `ensure`).""" import asyncpg for attempt in range(attempts): @@ -321,25 +333,28 @@ async def _run_ddl( await conn.execute(f"SET LOCAL lock_timeout = '{lock_timeout}'") for sql in sqls: await conn.execute(sql) - return "applied" + return Outcome("applied") except asyncpg.exceptions.InsufficientPrivilegeError as e: log.warning( "cell triggers: not applied on loom.%s (%s): this role cannot alter it", table, e ) - return "denied" + return Outcome("denied") except asyncpg.exceptions.LockNotAvailableError as e: # Short tries outlast a short transaction; an autovacuum is left to finish. if attempt < attempts - 1: await asyncio.sleep(LOCK_RETRY_S) continue - log.warning( - "cell triggers: not applied on loom.%s (%s): %s; %s", - table, - e, - "; ".join(sqls), - await _holders(conn, table), - ) - return "busy" + held = await _holders(conn, table) + if warn: + log.warning( + "cell triggers: not applied on loom.%s (%s): %s; %s", + table, + e, + "; ".join(sqls), + held, + ) + return Outcome("busy", held) + return Outcome("busy") async def holders(conn: Any, table: str) -> list[Any]: diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index e0f96f409f..d77592d1dc 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -532,7 +532,7 @@ async def test_each_right_that_makes_an_org_held_is_named( await conn.execute("GRANT USAGE ON SCHEMA loom TO " + role) for sql in grant.format(r=role, owner=owner).split("; "): await conn.execute(sql) - holders = await cell_attach_pg.privilege_holders(conn) + holders = await cell_attach_pg.privilege_holders(conn, exempt_self=True) assert owner in holders.own assert any(why in w for w in holders.others.get(role, [])), holders.others finally: @@ -550,7 +550,7 @@ async def test_a_function_owner_counts_though_it_owns_no_table(cell_stores) -> N try: async with pg.dedicated() as conn: await conn.execute(f"ALTER FUNCTION loom.tenancy_refuse_unheld() OWNER TO {role}") - holders = await cell_attach_pg.privilege_holders(conn) + holders = await cell_attach_pg.privilege_holders(conn, exempt_self=True) await conn.execute("ALTER FUNCTION loom.tenancy_refuse_unheld() OWNER TO CURRENT_USER") assert holders.others[role] == ["owns loom.tenancy_refuse_unheld()"] finally: @@ -631,8 +631,40 @@ async def test_doctor_names_stray_orgs_smallest_tables_first(cell_stores) -> Non lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 300)} grade, detail = lines["cell_stray_orgs"] assert grade == "WARN" and tag + "stray" in detail and tag + "held" not in detail - tight = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 0.001)} - assert tight["cell_stray_orgs_unchecked"][0] == "WARN" + + +@pytest.mark.asyncio +async def test_doctor_reads_smallest_first_and_leaves_the_largest_unchecked( + cell_stores, monkeypatch +) -> None: + from server.cli import cell_scan, doctor_cell + + stores = cell_stores + await _prepare(stores, _Network()) + tag = stores["tag"] + await pg.execute("CREATE TABLE loom.it_small (org text PRIMARY KEY)") + await pg.execute("CREATE TABLE loom.it_large (org text PRIMARY KEY)") + await pg.execute("INSERT INTO loom.it_small VALUES ($1)", tag + "small") + await pg.execute( + "INSERT INTO loom.it_large SELECT $1 || g FROM generate_series(1, 20000) g", tag + "l" + ) + await pg.execute("ANALYZE loom.it_small, loom.it_large") + # The budget runs out on the largest table, as it does on a real cell when that table's + # read is what is too slow: every smaller one was read first. + real, read = cell_scan.pg_orgs, [] + + async def pg_orgs(table: str, budget: Any) -> list[str]: + read.append(table) + if table == "loom.it_large": + await asyncio.sleep(budget.left() + 0.1) + return await real(table, budget) + + monkeypatch.setattr(cell_scan, "pg_orgs", pg_orgs) + lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 3)} + assert read[-1] == "loom.it_large" and "loom.it_small" in read + assert tag + "small" in lines["cell_stray_orgs"][1] + grade, detail = lines["cell_stray_orgs_unchecked"] + assert grade == "WARN" and detail.endswith(": loom.it_large") @pytest.mark.asyncio @@ -657,10 +689,10 @@ async def test_a_role_with_the_grants_but_not_the_owner_role_is_refused_before_a assert "attach's role lacks membership in the owner role of loom.tenancy" in lacking assert any("loom.scope_generations" in r for r in lacking) # The trigger pass tells the cause apart: denied, not a busy lock. - assert ( - await cell_triggers.ensure_table(conn, "scope_generations", recording_first=True) - == "denied" + outcome = await cell_triggers.ensure_table( + conn, "scope_generations", recording_first=True ) + assert outcome.answer == "denied" await conn.execute("RESET ROLE") assert await _mode() == "recording" finally: @@ -688,3 +720,28 @@ async def test_doctor_names_a_table_it_cannot_read_and_reads_the_rest(cell_store grade, detail = lines["cell_stray_orgs_unread"] assert grade == "WARN" and "postgres loom.it_no_org_index" in detail assert "cell_stray_orgs_unchecked" not in lines + + +@pytest.mark.asyncio +async def test_attach_exempts_the_role_it_runs_as_and_doctor_does_not(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + role = f"it_ci_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + try: + async with pg.dedicated() as conn: + await conn.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + await conn.execute(f"GRANT SELECT ON ALL TABLES IN SCHEMA loom TO {role}") + await conn.execute(f"GRANT INSERT ON loom.tenancy TO {role}") + await conn.execute(f"SET ROLE {role}") + as_attach = await cell_attach_pg.privilege_holders(conn, exempt_self=True) + as_doctor = await cell_attach_pg.privilege_holders(conn, exempt_self=False) + await conn.execute("RESET ROLE") + # Attach runs as the operator's role; a CI role doctor runs as is warned about. + assert role in as_attach.own and role not in as_attach.others + assert role not in as_doctor.own + assert "may insert into loom.tenancy" in as_doctor.others[role] + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") diff --git a/tests/integration/test_cell_org_triggers_pg.py b/tests/integration/test_cell_org_triggers_pg.py index b2fb66a467..c219d05777 100644 --- a/tests/integration/test_cell_org_triggers_pg.py +++ b/tests/integration/test_cell_org_triggers_pg.py @@ -722,15 +722,15 @@ async def test_ensure_table_creates_a_recording_first_cells_triggers_only_when_a with pytest.raises(RuntimeError, match="recording"): await cell_triggers.ensure_table(conn, "scope_generations") ensure = cell_triggers.ensure_table - assert await ensure(conn, "scope_generations", recording_first=True) == "applied" - assert await ensure(conn, "tenancy", recording_first=True) == "applied" + assert (await ensure(conn, "scope_generations", recording_first=True)).answer == "applied" + assert (await ensure(conn, "tenancy", recording_first=True)).answer == "applied" assert await _cell_triggers(conn) == { ("scope_generations", "tenancy_insert", "O"), ("scope_generations", "tenancy_update", "O"), ("tenancy", "tenancy_record", "O"), } # Again: nothing to do. - assert await ensure(conn, "scope_generations", recording_first=True) == "applied" + assert (await ensure(conn, "scope_generations", recording_first=True)).answer == "applied" with pytest.raises(ValueError, match="no cell triggers"): await cell_triggers.ensure_table(conn, "idempotency", recording_first=True) plain = await _unmarked(dsn) diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 857d12dfe0..b3db51ee02 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -268,8 +268,8 @@ async def planned(_c: Any) -> list[str]: async def holders(_c: Any, _t: str) -> list[Any]: return [] - async def ensure_table(*_a: Any, **_kw: Any) -> str: - return "denied" + async def ensure_table(*_a: Any, **_kw: Any) -> Any: + return cell_triggers.Outcome("denied") monkeypatch.setattr(cell_triggers, "planned_tables", planned) monkeypatch.setattr(cell_triggers, "holders", holders) @@ -280,14 +280,22 @@ async def ensure_table(*_a: Any, **_kw: Any) -> str: assert refusal.subject == "postgres loom.scope_generations" assert "cannot alter it" in refusal.reason and wait.spent < 1 - async def busy(*_a: Any, **_kw: Any) -> str: + holders = iter(["pid 7", "pid 7", "pid 7", "pid 8"] + ["pid 8"] * 100) + + async def busy(*_a: Any, **_kw: Any) -> Any: clock.now += 0.1 - return "busy" + return cell_triggers.Outcome("busy", next(holders)) monkeypatch.setattr(cell_triggers, "ensure_table", busy) - wait = apg.Wait(3, clock) - (refusal,) = await apg.ensure_triggers(conn, wait, lambda _t: None) - assert "not taken within --wait" in refusal.reason and wait.spent <= 3 + wait = apg.Wait(10, clock) + logged: list[str] = [] + (refusal,) = await apg.ensure_triggers(conn, wait, logged.append) + assert "not taken within --wait" in refusal.reason and wait.spent <= 10 + # Said once per change of holders, not once a try. + assert logged == [ + "loom.scope_generations: its lock is busy (pid 7); retrying", + "loom.scope_generations: its lock is busy (pid 8); retrying", + ] # -- the rescan's watch ---------------------------------------------------------------- @@ -828,3 +836,189 @@ async def unread(_roles: Any) -> Any: lines = {n: (g, d) for n, g, d in await doctor_cell._checks(None)} for name in ("cell_tenancy_privilege", "cell_unchecked_writers"): assert lines[name] == ("WARN", "not read: OSError: the catalog read failed") + + +@pytest.mark.asyncio +async def test_the_ddl_lock_bound_is_taken_after_the_migration_lock(clock: _Clock) -> None: + """A migration lock that took the rest of --wait leaves no DDL try.""" + + def migration_lock(*_a: Any) -> str: + clock.now += 2.0 # the rest of --wait + return "OK" + + conn = FakeConn([("pg_advisory_lock", migration_lock), ("pg_advisory_unlock", "OK")]) + wait = apg.Wait(2, clock) + with wait.counting(): + outcome = await apg._ensure_one(conn, "scope_generations", 100, wait) + assert outcome.answer == "busy" + assert not any( + "lock_timeout = '" in s and "ms'" in s and "SET LOCAL" in s for s in conn.executed + ) + + +@pytest.mark.asyncio +async def test_a_mode_transaction_that_will_not_end_is_named(clock: _Clock) -> None: + held = [{"pid": 5, "application_name": "attach", "xact_start": "t", "rolsuper": False}] + conn = FakeConn( + [ + ("l.mode = 'ExclusiveLock'", held), + ("rolsuper FROM pg_roles", False), + ("pg_terminate_backend", True), + ] + ) + + async def confirm() -> None: + pass + + with pytest.raises(Refused) as e: + await apg.end_mode_transactions(conn, confirm) + (refusal,) = e.value.refusals + assert refusal.subject == "postgres backend 5" + assert "still holds loom.tenancy_mode's lock" in refusal.reason + + +def _stub_attach(monkeypatch: pytest.MonkeyPatch, body: Any) -> None: + from contextlib import asynccontextmanager + + from server.core import pg + + @asynccontextmanager + async def dedicated(**_kw: Any) -> Any: + yield None + + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(cell_attach.Run, "run", body) + + +@pytest.mark.asyncio +async def test_a_router_refusal_of_attach_enforced_leaves_recording( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + async def refuse(_self: Any, path: str, _body: Any = None) -> dict[str, Any]: + raise Refused([Refusal("router", f"refused {path}")]) + + async def run(self: Any) -> int: + self.row, self.mode = {"deployment_id": "dep-1"}, "recording" + await self.commit_mode() + return 0 + + monkeypatch.setattr(cell_attach.Router, "call", refuse) + _stub_attach(monkeypatch, run) + assert await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) == 1 + assert capsys.readouterr().err.splitlines()[-1] == "loom cell attach: the cell stays recording" + + +@pytest.mark.asyncio +async def test_a_durable_read_that_sees_recording_says_recording( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + from server.core import pg_profile + + async def synchronous() -> str: + return "synchronous" + + async def read() -> apg.Durable: + return apg.Durable(mode="recording", first=(True, "first"), xid=1) + + async def run(self: Any) -> int: + self.mode = "unknown" # a mode commit whose outcome was lost + await self.durable() + return 0 + + monkeypatch.setattr(pg_profile, "standby_now", synchronous) + monkeypatch.setattr(apg, "durable_read", read) + _stub_attach(monkeypatch, run) + assert await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) == 1 + assert capsys.readouterr().err.splitlines()[-1] == "loom cell attach: the cell stays recording" + + +@pytest.mark.asyncio +async def test_an_enforced_resume_refused_early_says_enforced( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + from server.core import pg_profile + + async def recorded(_ch: Any) -> dict[str, Any]: + return {"deployment_id": "other"} + + async def health(_self: Any, _host: str) -> str: + return "dep-1" + + async def read_mode(_conn: Any) -> tuple[str, Any]: + return "enforced", (True, "first") + + async def run(self: Any) -> int: + self.row = {"is_first": True, "state": "attaching", "deployment_id": "dep-1"} + self.lock_conn = FakeConn([("FROM loom.authority", "dep-1")]) + if refusals := await self.stores_refusals(): + raise Refused(refusals) + return 0 + + monkeypatch.setattr(pg_profile, "recorded_authority", recorded) + monkeypatch.setattr(cell_attach.Router, "health", health) + monkeypatch.setattr(apg, "read_mode", read_mode) + _stub_attach(monkeypatch, run) + assert await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) == 1 + err = capsys.readouterr().err.splitlines() + assert "clickhouse: records another deployment id" in err[0] + assert err[-1] == ( + "loom cell attach: the cell stays enforced and attaching: rerun attach to resume" + ) + + +@pytest.mark.asyncio +async def test_an_interrupt_prints_the_mode_line_and_goes_on( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + async def run(self: Any) -> int: + self.mode = "unknown" + raise KeyboardInterrupt + + _stub_attach(monkeypatch, run) + with pytest.raises(KeyboardInterrupt): + await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) + assert "the mode commit's outcome is unknown" in capsys.readouterr().err + + +@pytest.mark.asyncio +async def test_the_stray_read_names_a_refused_table_and_reads_on( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from contextlib import asynccontextmanager + + from server.cli import cell_ch, doctor_cell + from server.core import pg + + async def topology(_ch: Any) -> Any: + return cell_ch.Topology("", 0) + + async def no_tables(*_a: Any) -> tuple[list[Any], list[Any]]: + return [], [] + + async def sizes(*_a: Any) -> dict[str, int]: + return {} + + pg_rows = [{"name": "loom.a_unindexed", "n": 1}, {"name": "loom.b_data", "n": 2}] + conn = FakeConn([("FROM loom.tenancy", []), ("pg_class c", pg_rows)]) + + @asynccontextmanager + async def dedicated(**_kw: Any) -> Any: + yield conn + + async def pg_orgs(table: str, _budget: Any) -> list[str]: + if table == "loom.a_unindexed": + raise Refused([Refusal(f"postgres {table}", "has no plain btree index led by org")]) + return ["stray-org"] + + monkeypatch.setattr(cell_ch, "read_topology", topology) + monkeypatch.setattr(cell_scan, "list_clickhouse", no_tables) + monkeypatch.setattr(doctor_cell, "_table_sizes", sizes) + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(cell_scan, "pg_orgs", pg_orgs) + lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(None, 60)} + assert lines["cell_stray_orgs"] == ("WARN", "postgres loom.b_data: stray-org") + assert lines["cell_stray_orgs_unread"] == ( + "WARN", + "not read: postgres loom.a_unindexed: has no plain btree index led by org", + ) + assert "cell_stray_orgs_unchecked" not in lines From 4a25c41ba6924963b6352e8fca17272cfae4abf5 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 08:37:50 +0000 Subject: [PATCH 009/114] The attach docs say which role attach exempts, and the trigger gate's figures say what they establish cells.md and the runbook state that attach exempts the role it runs as and doctor does not. results.md gives the marked batch's per-round changes and their mean, says the delivery-pass gate is met on the mean but not established by three rounds alone, and that the memory.set p99 gate is unresolved locally. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- docs/architecture/cells.md | 9 +++++---- docs/benchmarks/results.md | 19 +++++++++++-------- docs/operations/cell-attach.md | 9 +++++---- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 3ea7ddab80..246da73b77 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1007,10 +1007,11 @@ It refuses, before any write, naming each: without one, or a role that may not write the durable read's logical message; a role of attach's own lacking its grants (`pg_read_all_stats` among them) or a server with `max_prepared_transactions` above 0; -- **the tenancy-privilege check**: a login role, other than attach's own (superusers, - and a role that itself owns `loom.tenancy`, `loom.tenancy_mode` or a guarded table) and - those named with `--pg-allow-role`, that holds or reaches through any membership a right - to make an org held or lower the mode: `INSERT` on any column of `loom.tenancy` or +- **the tenancy-privilege check**: a login role, other than attach's own (the role attach + runs as, which the operator runs it with; superusers; and a role that itself owns + `loom.tenancy`, `loom.tenancy_mode` or a guarded table) and those named with + `--pg-allow-role`, that holds or reaches through any membership a right to make an org + held or lower the mode: `INSERT` on any column of `loom.tenancy` or `UPDATE` of its `org`, `INSERT` on `loom.tenancy_mode` or `UPDATE` of its `mode`, `TRIGGER` on a `loom` table, `CREATE` on the database or a schema that is not a system one (a function there can shadow one the servers call unqualified on their marked diff --git a/docs/benchmarks/results.md b/docs/benchmarks/results.md index 81e099d3c8..4814bd7f5f 100644 --- a/docs/benchmarks/results.md +++ b/docs/benchmarks/results.md @@ -1241,14 +1241,17 @@ PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py --base-url | 1,000-row insert, marked (median) | 11.0, 9.2, 8.5 ms | 9.7, 11.1, 10.6 ms | | 1,000-row insert, unmarked (median) | 22.1, 21.4, 21.0 ms | 10.9, 9.6, 11.0 ms | -The marked batch, which is what a delivery pass of a cell's own processes writes, shows no -cost the rounds can tell from zero (on minus off: +1.3, −1.9, −2.1 ms), within the 10% gate. -The `memory.set` figures move by more between rounds of one configuration than any trigger -cost: on this loaded host the p99 differences within a round (+214.8, +47.0, +24.3 ms -here, and −71.7, −10.0, −42.4 ms in an earlier run of the same measurement) have no -consistent sign, so this run cannot resolve a 10 ms p99 gate; it shows no rise beyond the -host's noise, and the statement-level figures above bound the marked cost at under a -microsecond a row. An unmarked session pays about 11 µs a row, as above. +The marked batch is what a delivery pass of a cell's own processes writes. On minus off +within each round it moved +1.3, −1.9 and −2.1 ms (+13.4%, −17.1% and −19.8%); the means are +9.6 ms on and 10.5 ms off, −0.9 ms (−8.6%). The delivery-pass gate (at most 10% slower) is +met on the mean, but one round of three was 13% slower and three rounds give no paired bound +(methodology's paired bootstrap is for per-item labels), so this run does not establish the +gate on its own; the statement-level figures above do, at under a microsecond a row against +about 10 µs a row of insert. The `memory.set` figures move by more between rounds of one +configuration than any trigger cost: on this loaded host the p99 differences within a round +(+214.8, +47.0, +24.3 ms here, and −71.7, −10.0, −42.4 ms in an earlier run of the same +measurement) have no consistent sign: the `memory.set` p99 gate (a rise of at most 10 ms) is +unresolved locally. An unmarked session pays about 11 µs a row, as above. ## Reference computations diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index c51541d06d..481b9a95ef 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -135,10 +135,11 @@ procedure as an over-long org's. `loom doctor` runs attach's point-in-time checks again on every run: it warns on a Postgres role outside the owner roles, the superusers and its own `--pg-allow-role` that can make an -org held or lower the mode (`cell_tenancy_privilege`), and on a role or database setting or -a subscription that passes an unchecked writer (`cell_unchecked_writers`). A grant or a -writer added after `serving` passes attach's checks, which ran once; doctor is how it is -found. +org held or lower the mode (`cell_tenancy_privilege`). Attach exempts the role it runs as, +the operator's; doctor does not exempt the role it runs as, so a CI or monitoring role +holding such a right is named. It warns too on a role or database setting or a subscription +that passes an unchecked writer (`cell_unchecked_writers`). A grant or a writer added after +`serving` passes attach's checks, which ran once; doctor is how it is found. ## Related From 0c37c6860b08744f4bbf9afdf2b0d45681e83855 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 08:51:15 +0000 Subject: [PATCH 010/114] Every attach try runs once whatever --wait has left, the mode reads unknown only while a try's UPDATE is out, and a busy migration lock is named Each trigger and mode-commit try gets a lock wait of at least TRY_FLOOR_MS before --wait is judged spent, so --wait 0 is one try of each wait with no retry, and an idle cell attaches with it. The run's mode turns unknown just before a try sends its UPDATE and back to recording when that try rolls back. A trigger try that finds the migration lock busy names that lock's holder, in the retry line and in the refusal. The step-4 watch task is awaited after its cancel. The ATTACH_LOCK_KEY comment says who holds the lock. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- server/cli/cell_attach.py | 24 +++- server/cli/cell_attach_pg.py | 74 ++++++++--- server/core/cell.py | 6 +- tests/integration/test_cell_attach.py | 17 ++- tests/unit/snapshots/cli_help.json | 2 +- tests/unit/snapshots/cli_tree.json | 2 +- tests/unit/test_cell_attach.py | 176 ++++++++++++++++++++++++-- 7 files changed, 260 insertions(+), 41 deletions(-) diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 23449ccff9..8ed70e2d37 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -30,6 +30,7 @@ import argparse import asyncio +import contextlib import json import sys import time @@ -453,16 +454,22 @@ async def commit_mode(self) -> None: from server.core import pg await self.router.call("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) - # From the mode transaction on the mode may have risen: only the durable read can say. - self.mode = "unknown" + + def mark(mode: str) -> None: + # `unknown` from a try's UPDATE on: only the durable read can say whether it + # landed; `recording` again when that try rolled back. + self.mode = mode + try: async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: await apg.commit_enforced( - conn, self.confirm_lock, self.ready_for_commit, self.wait, self.log + conn, self.confirm_lock, self.ready_for_commit, self.wait, self.log, mark ) except Refused: raise - except Exception as e: # noqa: BLE001 — the durable read says whether it landed + except Exception as e: + if self.mode != "unknown": + raise # no try was in flight: nothing to decide self.log(f"the mode commit's outcome is unknown ({type(e).__name__}: {e})") async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: await apg.end_mode_transactions(conn, self.confirm_lock) @@ -573,6 +580,8 @@ async def step4(self, xmax: int, topology: Any, names: Any) -> None: raise Refused(writers) finally: task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task await self.final_checks(watch) row = await self.router.call("") current, nxt = self.own_fp @@ -757,7 +766,12 @@ async def _run(ch: Any) -> int: def add_attach_parser(csub: Any) -> None: p = csub.add_parser("attach", help="Attach the prepared first cell: place its orgs, serve.") p.add_argument("--cell", required=True, help="This cell's name (LOOM_CELL)") - p.add_argument("--wait", type=int, default=_WAIT_S, help="Seconds the waits share") + p.add_argument( + "--wait", + type=int, + default=_WAIT_S, + help="Seconds of wall time the waits share; 0 runs one try of each, with no retry", + ) p.add_argument( "--scan-budget", type=int, default=_SCAN_BUDGET_S, help="Seconds one full org scan may take" ) diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 9c6e4708d6..0ca9fefcc1 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -24,6 +24,8 @@ # The pause between polls of a wait. POLL_S = 1.0 # not a setting: a poll interval +# The least lock wait a try gets, however little of --wait is left. +TRY_FLOOR_MS = 100 # not a setting: one try's floor # The migration lock's wait a trigger try (a boot holds it for its migrations), in ms; a # try never waits longer than --wait has left. _MIGRATION_LOCK_MS = 10_000 # not a setting: one try's bound @@ -67,8 +69,10 @@ def counting(self) -> Iterator[None]: self._since = None def bound_ms(self, most_ms: int) -> int: - """A lock timeout for one try: `most_ms`, or what is left when that is less.""" - return min(most_ms, int(self.left() * 1000)) + """A lock timeout for one try: `most_ms`, or what is left when that is less, and + never under `TRY_FLOOR_MS`: every try runs once, so `--wait 0` is one try of each + wait with no retry, and only after a try is the wait judged spent.""" + return max(TRY_FLOOR_MS, min(most_ms, int(self.left() * 1000))) async def pause(self, seconds: float = POLL_S) -> bool: """Wait `seconds` (at most what is left); False once none is left.""" @@ -89,6 +93,13 @@ def _split(key: int) -> str: ) +# Who holds an advisory lock (`where` from `_split`), never a query's text. +_ADVISORY_HOLDER = """ +SELECT l.pid, a.application_name, a.backend_start +FROM pg_locks l LEFT JOIN pg_stat_activity a ON a.pid = l.pid +WHERE {where} AND l.granted +""" + _LOCK_HOLDER = f""" SELECT l.pid, a.application_name, a.backend_start, a.client_addr::text AS client FROM pg_locks l LEFT JOIN pg_stat_activity a ON a.pid = l.pid @@ -467,6 +478,11 @@ async def _ensure_table( "attach's role cannot alter it: make the role a member of its owner role", ) if not await wait.pause(): + if outcome.holders.startswith(_MIGRATION_HELD): + return Refusal( + f"postgres loom.{table}", + f"its triggers were not created within --wait: {outcome.holders}", + ) return _busy_refusal(table, await cell_triggers.holders(conn, table)) # Said again only when the holders change: a retry a second is otherwise silent. if outcome.holders != said: @@ -474,29 +490,45 @@ async def _ensure_table( log(f"loom.{table}: its lock is busy ({said or 'no holder named'}); retrying") +_MIGRATION_HELD = "the migration lock" + + +async def _migration_holder(conn: Any, key: int) -> str: + """`the migration lock (held by …)`, naming the session that holds it.""" + try: + rows = await conn.fetch(_ADVISORY_HOLDER.format(where=_split(key))) + except Exception as e: # noqa: BLE001 — naming the holder is best effort + return f"{_MIGRATION_HELD} (its holder not read: {type(e).__name__})" + held = "; ".join( + f"pid {r['pid']}, {r['application_name'] or 'no name'}, since {r['backend_start']}" + for r in rows + ) + return f"{_MIGRATION_HELD} (held by {held or 'a session that just ended'})" + + async def _ensure_one(conn: Any, table: str, most_ms: int, wait: Wait) -> cell_triggers.Outcome: """One try under the migration lock: the migration lock's wait, and then the DDL's lock - wait (at most `most_ms`), each bounded by what `wait` has left when it starts.""" + wait (at most `most_ms`), each bounded by what `wait` has left when it starts, with the + try floor. A busy migration lock answers `busy` naming that lock's holder.""" from server.core.schema.pg_migrate import _LOCK_KEY - migration_ms = wait.bound_ms(_MIGRATION_LOCK_MS) - if migration_ms <= 0: - return cell_triggers.Outcome("busy") - await conn.execute(f"SET lock_timeout = '{migration_ms}ms'") + await conn.execute(f"SET lock_timeout = '{wait.bound_ms(_MIGRATION_LOCK_MS)}ms'") try: await conn.execute(_MIGRATION_LOCK, _LOCK_KEY) except Exception as e: if getattr(e, "sqlstate", None) == "55P03": - return cell_triggers.Outcome("busy") # a boot migrating: the next try + # A boot migrating: the next try. + return cell_triggers.Outcome("busy", await _migration_holder(conn, _LOCK_KEY)) raise finally: await conn.execute("RESET lock_timeout") try: - lock_ms = wait.bound_ms(most_ms) - if lock_ms <= 0: - return cell_triggers.Outcome("busy") return await cell_triggers.ensure_table( - conn, table, recording_first=True, lock_timeout=f"{lock_ms}ms", attempts=1 + conn, + table, + recording_first=True, + lock_timeout=f"{wait.bound_ms(most_ms)}ms", + attempts=1, ) finally: await conn.execute(_MIGRATION_UNLOCK, _LOCK_KEY) @@ -543,21 +575,29 @@ async def commit_enforced( ready: Callable[[], Any], wait: Wait, log: Callable[[str], None], + mark: Callable[[str], None], ) -> None: """Raise the mode to `enforced` on `conn` (a session of its own, never the lock session): an EXCLUSIVE lock on loom.tenancy_mode a try, `_MODE_LOCK_MS` each (less when `wait` has less left), within `wait`; `confirm_lock` before each try and each termination (it refuses when another run holds the attach lock), and `ready` before each try (False skips the try while a process boots). After a failed try the in-flight - mode transaction, found by its lock, is ended. Raises Refused when `wait` runs out; - whether the commit landed is the durable read's to say.""" + mode transaction, found by its lock, is ended. `mark` hears `unknown` just before a try + sends its UPDATE and `recording` when that try rolled back. Raises Refused when `wait` + runs out; whether the commit landed is the durable read's to say.""" with wait.counting(): while True: await confirm_lock() - lock_ms = wait.bound_ms(_MODE_LOCK_MS) - if lock_ms > 0 and await ready(): - if await _mode_try(conn, lock_ms): + if await ready(): + mark("unknown") + try: + done = await _mode_try(conn, wait.bound_ms(_MODE_LOCK_MS)) + except Refused: + mark("recording") # the UPDATE's transaction rolled back + raise + if done: return + mark("recording") log("the mode's lock is busy; retrying") await end_mode_transactions(conn, confirm_lock) if not await wait.pause(): diff --git a/server/core/cell.py b/server/core/cell.py index 2aaea0f529..37e20f99cb 100644 --- a/server/core/cell.py +++ b/server/core/cell.py @@ -62,9 +62,9 @@ # Postgres's btree entry limit (about 2.7 KB), so every org a cell records has a # placement the directory can index. ORG_MAX_BYTES = 2048 # not a setting: cells and the router judge one limit -# The advisory lock on a cell's Postgres that attach will hold, and prepare and update once -# they take it, and that the troubleshooting page's full-state cleanup takes before the -# migration lock. Positive: the page's holder query splits it into two `oid` halves. +# The advisory lock on a cell's Postgres that attach and prepare hold for their run, and that +# the troubleshooting page's full-state cleanup takes before the migration lock. Positive: +# the page's holder query splits it into two `oid` halves. ATTACH_LOCK_KEY = 0x6C6F6F6D_61746368 # "loomatch" diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index d77592d1dc..65064ab365 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -482,7 +482,9 @@ async def release() -> None: await tx.rollback() task = asyncio.create_task(release()) - await cell_attach_pg.commit_enforced(conn, ok, ok, cell_attach_pg.Wait(20), logged.append) + await cell_attach_pg.commit_enforced( + conn, ok, ok, cell_attach_pg.Wait(20), logged.append, lambda _m: None + ) await task assert await _mode() == "enforced" assert logged == ["the mode's lock is busy; retrying"] @@ -745,3 +747,16 @@ async def test_attach_exempts_the_role_it_runs_as_and_doctor_does_not(cell_store await conn.execute(f"DROP OWNED BY {role}") await _admin(f"DROP OWNED BY {role}") await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_wait_0_attaches_an_idle_cell(cell_stores, capsys) -> None: + """--wait 0 is one try of each wait, no retry: an idle cell still reaches serving.""" + stores, network = cell_stores, _Network() + await _prepare(stores, network) + org = stores["tag"] + "w0" + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", org) + capsys.readouterr() + assert await _attach(stores, network, wait=0) == 0, capsys.readouterr().err + assert json.loads(capsys.readouterr().out)["state"] == "serving" + assert org in await _placed() diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json index 3e88dbafca..4e45db4285 100644 --- a/tests/unit/snapshots/cli_help.json +++ b/tests/unit/snapshots/cli_help.json @@ -461,7 +461,7 @@ ], "loom cell attach": [ "usage: loom cell attach [-h] --cell CELL [--wait WAIT] [--scan-budget SCAN_BUDGET] [--dry-run] [--ch-allow-user CH_ALLOW_USER] [--ch-allow-user-directory CH_ALLOW_USER_DIRECTORY] [--ch-allow-object CH_ALLOW_OBJECT] [--ch-allow-replica CH_ALLOW_REPLICA] [--pg-allow-role PG_ALLOW_ROLE] [--router ROUTER]", - "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --wait WAIT Seconds the waits share --scan-budget SCAN_BUDGET Seconds one full org scan may take --dry-run Run the refusals and the scan and print what would be placed; write nothing (the router's own begin refusals do not run) --ch-allow-user CH_ALLOW_USER A ClickHouse user that may write the configured databases --ch-allow-user-directory CH_ALLOW_USER_DIRECTORY A ClickHouse user directory the check cannot read --ch-allow-object CH_ALLOW_OBJECT A ClickHouse object (db.name) that writes the databases --ch-allow-replica CH_ALLOW_REPLICA A replica outside the cluster (db.table:replica) --pg-allow-role PG_ALLOW_ROLE A Postgres role that may make an org held (the servers' role) --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" + "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --wait WAIT Seconds of wall time the waits share; 0 runs one try of each, with no retry --scan-budget SCAN_BUDGET Seconds one full org scan may take --dry-run Run the refusals and the scan and print what would be placed; write nothing (the router's own begin refusals do not run) --ch-allow-user CH_ALLOW_USER A ClickHouse user that may write the configured databases --ch-allow-user-directory CH_ALLOW_USER_DIRECTORY A ClickHouse user directory the check cannot read --ch-allow-object CH_ALLOW_OBJECT A ClickHouse object (db.name) that writes the databases --ch-allow-replica CH_ALLOW_REPLICA A replica outside the cluster (db.table:replica) --pg-allow-role PG_ALLOW_ROLE A Postgres role that may make an org held (the servers' role) --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" ], "loom tracker-probe": [ "usage: loom tracker-probe [-h]", diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json index 3d597c1e03..a31a862106 100644 --- a/tests/unit/snapshots/cli_tree.json +++ b/tests/unit/snapshots/cli_tree.json @@ -13032,7 +13032,7 @@ "nargs": null, "type": "builtins:int", "required": false, - "help": "Seconds the waits share", + "help": "Seconds of wall time the waits share; 0 runs one try of each, with no retry", "metavar": null }, { diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index b3db51ee02..429c2fcfb7 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -250,7 +250,7 @@ async def ready() -> bool: wait = apg.Wait(4, clock) clock.now += 100 # a scan before the wait with pytest.raises(Refused, match="not taken within --wait"): - await apg.commit_enforced(conn, nothing, ready, wait, lambda _t: None) + await apg.commit_enforced(conn, nothing, ready, wait, lambda _t: None, lambda _m: None) assert clock.now - 100 <= 4 and wait.spent <= 4 # A 2 s try, a 1 s pause, then a try that waits only the second left. assert [a.split("'")[1] for a in asked] == ["2000ms", "1000ms"] @@ -290,7 +290,9 @@ async def busy(*_a: Any, **_kw: Any) -> Any: wait = apg.Wait(10, clock) logged: list[str] = [] (refusal,) = await apg.ensure_triggers(conn, wait, logged.append) - assert "not taken within --wait" in refusal.reason and wait.spent <= 10 + # The last try ran with the floor's lock wait, after --wait was all but spent. + assert "not taken within --wait" in refusal.reason + assert wait.spent <= 10 + apg.TRY_FLOOR_MS / 1000 + 1e-9 # Said once per change of holders, not once a try. assert logged == [ "loom.scope_generations: its lock is busy (pid 7); retrying", @@ -654,7 +656,9 @@ async def ready() -> bool: return True logged: list[str] = [] - await apg.commit_enforced(conn, confirm, ready, apg.Wait(10, clock), logged.append) + await apg.commit_enforced( + conn, confirm, ready, apg.Wait(10, clock), logged.append, lambda _m: None + ) assert logged == ["the mode's lock is busy; retrying"] # Before each try and the one termination. assert len(confirmed) == 3 @@ -692,7 +696,9 @@ async def booting() -> bool: return False with pytest.raises(Refused, match="not taken within --wait"): - await apg.commit_enforced(conn, confirm, booting, apg.Wait(2, clock), lambda _t: None) + await apg.commit_enforced( + conn, confirm, booting, apg.Wait(2, clock), lambda _t: None, lambda _m: None + ) assert not any("LOCK TABLE" in s for s in conn.executed) @@ -789,8 +795,8 @@ async def test_a_stop_between_a_landed_commit_and_the_durable_read_says_the_outc async def dedicated(**_kw: Any) -> Any: yield None - async def landed(*_a: Any) -> None: - pass # the commit landed; its answer may not have + async def landed(*a: Any) -> None: + a[-1]("unknown") # the try sent its UPDATE; the commit landed, its answer may not have async def call(_self: Any, _path: str, _body: Any = None) -> dict[str, Any]: return {"cell": "first"} @@ -839,22 +845,166 @@ async def unread(_roles: Any) -> Any: @pytest.mark.asyncio -async def test_the_ddl_lock_bound_is_taken_after_the_migration_lock(clock: _Clock) -> None: - """A migration lock that took the rest of --wait leaves no DDL try.""" +async def test_the_ddl_lock_bound_is_taken_after_the_migration_lock( + monkeypatch: pytest.MonkeyPatch, clock: _Clock +) -> None: + """A migration lock that took the rest of --wait leaves the DDL its floor: one try, never + the full 100 ms when less is left, never none.""" + from server.core import cell_triggers def migration_lock(*_a: Any) -> str: - clock.now += 2.0 # the rest of --wait + clock.now += 1.95 # all but 50 ms of --wait return "OK" + asked: list[str] = [] + + async def ensure_table(*_a: Any, lock_timeout: str, **_kw: Any) -> Any: + asked.append(lock_timeout) + return cell_triggers.Outcome("applied") + + monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) conn = FakeConn([("pg_advisory_lock", migration_lock), ("pg_advisory_unlock", "OK")]) wait = apg.Wait(2, clock) with wait.counting(): - outcome = await apg._ensure_one(conn, "scope_generations", 100, wait) - assert outcome.answer == "busy" - assert not any( - "lock_timeout = '" in s and "ms'" in s and "SET LOCAL" in s for s in conn.executed + outcome = await apg._ensure_one(conn, "scope_generations", 500, wait) + assert outcome.answer == "applied" + assert asked == [f"{apg.TRY_FLOOR_MS}ms"] + + +@pytest.mark.asyncio +async def test_a_busy_migration_lock_is_named_as_such( + monkeypatch: pytest.MonkeyPatch, clock: _Clock +) -> None: + from server.core import cell_triggers + + class _Busy(Exception): + sqlstate = "55P03" + + holder = [{"pid": 42, "application_name": "loom", "backend_start": "t0"}] + conn = FakeConn( + [ + ("pg_advisory_lock", _Busy()), + ("l.locktype = 'advisory'", holder), + ("deadlock_timeout", 1000), + ] ) + async def planned(_c: Any) -> list[str]: + return ["scope_generations"] + + async def no_holders(_c: Any, _t: str) -> list[Any]: + return [] + + monkeypatch.setattr(cell_triggers, "planned_tables", planned) + monkeypatch.setattr(cell_triggers, "holders", no_holders) + logged: list[str] = [] + (refusal,) = await apg.ensure_triggers(conn, apg.Wait(2, clock), logged.append) + named = "the migration lock (held by pid 42, loom, since t0)" + assert logged == [f"loom.scope_generations: its lock is busy ({named}); retrying"] + assert refusal.reason == f"its triggers were not created within --wait: {named}" + + +@pytest.mark.asyncio +async def test_wait_0_runs_one_try_of_each_wait( + clock: _Clock, monkeypatch: pytest.MonkeyPatch +) -> None: + """On an idle database --wait 0 commits the mode and creates the triggers in one try + each, and the heartbeats pass when nothing boots.""" + from server.core import cell_triggers + + conn = FakeConn( + [ + ("LOCK TABLE", "LOCK TABLE"), + ("UPDATE loom.tenancy_mode", "UPDATE 1"), + ("l.mode = 'ExclusiveLock'", []), + ] + ) + marks: list[str] = [] + + async def ok() -> bool: + return True + + async def nothing() -> None: + pass + + await apg.commit_enforced(conn, nothing, ok, apg.Wait(0, clock), lambda _t: None, marks.append) + assert marks == ["unknown"] + assert any(f"lock_timeout = '{apg.TRY_FLOOR_MS}ms'" in sql for sql in conn.executed) + + async def planned(_c: Any) -> list[str]: + return ["scope_generations"] + + async def no_holders(_c: Any, _t: str) -> list[Any]: + return [] + + async def applied(*_a: Any, **_kw: Any) -> Any: + return cell_triggers.Outcome("applied") + + async def no_defects(_c: Any) -> list[Any]: + return [] + + monkeypatch.setattr(cell_triggers, "planned_tables", planned) + monkeypatch.setattr(cell_triggers, "holders", no_holders) + monkeypatch.setattr(cell_triggers, "ensure_table", applied) + monkeypatch.setattr(cell_triggers, "missing", no_defects) + ddl = FakeConn([("deadlock_timeout", 1000)]) + assert await apg.ensure_triggers(ddl, apg.Wait(0, clock), lambda _t: None) == [] + + async def read() -> list[beats.Beat]: + return [_beat("a")] + + assert (await beats.settle(read, _judge, apg.Wait(0, clock))).live == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("refused_before_any_try", [True, False]) +async def test_a_mode_commit_refused_without_a_landed_try_says_recording( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + clock: _Clock, + refused_before_any_try: bool, +) -> None: + """The closing line says recording when no try sent its UPDATE (the heartbeats refused + before one) and when every try timed out and rolled back.""" + + def lock(*_a: Any) -> str: + raise _Timeout() + + conn = FakeConn([("LOCK TABLE", lock), ("l.mode = 'ExclusiveLock'", [])]) + + async def call(_self: Any, _path: str, _body: Any = None) -> dict[str, Any]: + return {"cell": "first"} + + async def ready(_self: Any) -> bool: + if refused_before_any_try: + raise Refused([Refusal("process p1", "runs with LOOM_CELL unset")]) + return True + + async def confirm(_self: Any) -> None: + pass + + async def run(self: Any) -> int: + self.row, self.mode = {"deployment_id": "dep-1"}, "recording" + self.wait = apg.Wait(2, clock) + await self.commit_mode() + return 0 + + from contextlib import asynccontextmanager + + from server.core import pg + + @asynccontextmanager + async def dedicated(**_kw: Any) -> Any: + yield conn + + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(cell_attach.Router, "call", call) + monkeypatch.setattr(cell_attach.Run, "ready_for_commit", ready) + monkeypatch.setattr(cell_attach.Run, "confirm_lock", confirm) + monkeypatch.setattr(cell_attach.Run, "run", run) + assert await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) == 1 + assert capsys.readouterr().err.splitlines()[-1] == "loom cell attach: the cell stays recording" + @pytest.mark.asyncio async def test_a_mode_transaction_that_will_not_end_is_named(clock: _Clock) -> None: From 07f9ea19618558e8b7307402720ae6f73a8fd846 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 08:51:15 +0000 Subject: [PATCH 011/114] The attach docs say what --wait 0 does, and cells.md keeps each code span on one line inside its list items Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- docs/architecture/cells.md | 185 +++++++++++++++++---------------- docs/operations/cell-attach.md | 27 ++--- 2 files changed, 108 insertions(+), 104 deletions(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 246da73b77..53495c7a1d 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1099,12 +1099,14 @@ Then, in order (the step numbers are the ones the rest of this page uses): `enforced`; then `attach/serving`. `--wait` (default 10 minutes) is the wall time the bounded waits share, each lock try -included and never longer than what is left; `--scan-budget` (default 2 hours) bounds each -full scan, which `--wait` does not count. `--dry-run` runs the refusals and the scan, prints -what would be placed and skipped and the scan's time, and writes nothing; the router's -checks at `attach/begin` do not run. On a `serving` row attach reads the cell's mode: -`enforced` is done (exit 0), and any other mode refuses: the directory and the cell's -Postgres disagree, as after a restore of one of them, and attach lowers and repairs neither. +included and never longer than what is left, but for a floor of 100 ms: every try runs once +before the wait is judged spent, so `--wait 0` is one try of each wait with no retry; +`--scan-budget` (default 2 hours) bounds each full scan, which `--wait` does not count. +`--dry-run` runs the refusals and the scan, prints what would be placed and skipped and the +scan's time, and writes nothing; the router's checks at `attach/begin` do not run. On a +`serving` row attach reads the cell's mode: `enforced` is done (exit 0), and any other mode +refuses: the directory and the cell's Postgres disagree, as after a restore of one of them, +and attach lowers and repairs neither. A later cell is attached only once the first is `serving`. It must hold no org data (attach refuses otherwise) and starts in `enforced`. Its attach runs, in order: the @@ -1487,58 +1489,58 @@ An integration test with two cells and a router in CI's Postgres job: between prepare and attach, keeps serving on a first cell in `recording`, and keeps the last mode it read through a failed read, `recording` included; a CLI command declaring an org with it unset is refused on an `enforced` or `partial` cell, an org-free one (a token - revocation) runs, also with Postgres down on a deployment never prepared, and a `loom -cell` store command refuses; `loom cell limit --closed` succeeds with the cell's Postgres - down; after `loom cell credential --cell --next` both credentials place and read - state, `--finish` refuses the old credential, for a cell finished before its pods hold the - new one the router refuses its calls with 401, its request answers 503 writing nothing, - and it recovers by recording the old hash again, `--revert` drops a pending `next`, which - is then refused, and the old credential keeps placing, and a second `--next` while one is - pending and a hash another cell records as current or next are refused; `loom cell limit` - and `loom cell credential --next`, `--finish` and `--revert` refuse while the directory is - unavailable; concurrent placements at one under `placements_max` across two router - replicas write one; a CLI command declaring an org with `LOOM_CELL_ROUTER_TOKEN` set - refuses, naming it, on a first cell once it is `enforced` (`attaching` included) and on a - later cell, and on a first cell in `recording` runs and records the org; attach with - `placements_max` below the orgs the first cell holds places them all, and a later cell's - attach past its quota or closed refuses, naming the org; a first cell with `--closed` and - `placements_max` set before step 2 attaches, and an org a request records between step 2 - and `enforced` is placed by step 4 with no 503; after step 3's write of the cell's mode, a - process whose cache still reads `recording` sends a new org's request to the placement - call, which a closed cell answers 503; a CLI command whose insert finds `enforced` - refuses, naming the org, and makes no placement call; a refusal after step 3's write - leaves the cell `enforced` and `attaching`, where a new org is placed through the gate; - each ordering of attach's write of the mode with a recording insert holds as stated above - (running, starting during the write, a stream of overlapping inserts each shorter than a - try, with attach setting `enforced` within a try or two, held across several tries, held - past `--wait` (attach refuses, the cell still `recording`), a stalled attach, a commit cut - off by its timeout, and a commit whose connection is lost while it waits for the standby, - which the durable read decides); a `REPEATABLE READ` transaction whose first statement ran - before `enforced` committed, on attach's role or a second role, holds step 4 until it ends - and its write of an unheld org is then placed, one whose first statement ran after is - refused by the trigger, a transaction that starts after attach takes the lock but before - its commit is visible holds step 4, a transaction opened in `recording` that outlives one - run's `--wait` holds the rerun's step 4 after a refusal, an autovacuum worker on a loom - table does not hold step 4 and a background worker with a transaction open since - `recording` does, a prepared transaction (after a restart with `max_prepared_transactions` - raised during the wait) makes step 4 refuse at once, naming it, with the mode still - `enforced` and a rerun refused until it is resolved and the setting is back to 0, a write - transaction open in a second database on the same server holds step 4 and is named, a long - read on the synchronous standby, or on a second standby, with `hot_standby_feedback` on - does not hold it, nor does a base backup running across it, a write held open on a - `replication=database` connection holds step 4, and so does a `REPEATABLE READ` - transaction on one, opened in `recording` and writing nothing before `enforced`, a long - logical dump (a `REPEATABLE READ` read-only transaction) on the cell's database past - `--wait` makes step 4 refuse, naming it, with the cell left `enforced`, and the first - cell's attach without `pg_read_all_stats` or with `max_prepared_transactions` above 0 - refuses before step 1 with the mode still `recording`, while a later cell's attach does - not require either; a missing or disabled org trigger is refused by attach and named by - `loom doctor`, a write for an unheld org on an `enforced` cell whose mode row was deleted - is refused and the state named by `loom doctor`; on that cell a pod with `LOOM_CELL` unset - is unready and runs no worker, and a CLI command declaring an org with it unset is - refused, as a pod is; updating, deleting or truncating the `first` row on an `enforced` - cell is refused, also under `session_replication_role = replica`, and a missing or - disabled `first`-row trigger is refused by attach and named by `loom doctor`; a running + revocation) runs, also with Postgres down on a deployment never prepared, and a + `loom cell` store command refuses; `loom cell limit --closed` succeeds with the cell's + Postgres down; after `loom cell credential --cell --next` both credentials place + and read state, `--finish` refuses the old credential, for a cell finished before its pods + hold the new one the router refuses its calls with 401, its request answers 503 writing + nothing, and it recovers by recording the old hash again, `--revert` drops a pending + `next`, which is then refused, and the old credential keeps placing, and a second `--next` + while one is pending and a hash another cell records as current or next are refused; + `loom cell limit` and `loom cell credential --next`, `--finish` and `--revert` refuse + while the directory is unavailable; concurrent placements at one under `placements_max` + across two router replicas write one; a CLI command declaring an org with + `LOOM_CELL_ROUTER_TOKEN` set refuses, naming it, on a first cell once it is `enforced` + (`attaching` included) and on a later cell, and on a first cell in `recording` runs and + records the org; attach with `placements_max` below the orgs the first cell holds places + them all, and a later cell's attach past its quota or closed refuses, naming the org; a + first cell with `--closed` and `placements_max` set before step 2 attaches, and an org a + request records between step 2 and `enforced` is placed by step 4 with no 503; after step + 3's write of the cell's mode, a process whose cache still reads `recording` sends a new + org's request to the placement call, which a closed cell answers 503; a CLI command whose + insert finds `enforced` refuses, naming the org, and makes no placement call; a refusal + after step 3's write leaves the cell `enforced` and `attaching`, where a new org is placed + through the gate; each ordering of attach's write of the mode with a recording insert + holds as stated above (running, starting during the write, a stream of overlapping inserts + each shorter than a try, with attach setting `enforced` within a try or two, held across + several tries, held past `--wait` (attach refuses, the cell still `recording`), a stalled + attach, a commit cut off by its timeout, and a commit whose connection is lost while it + waits for the standby, which the durable read decides); a `REPEATABLE READ` transaction + whose first statement ran before `enforced` committed, on attach's role or a second role, + holds step 4 until it ends and its write of an unheld org is then placed, one whose first + statement ran after is refused by the trigger, a transaction that starts after attach + takes the lock but before its commit is visible holds step 4, a transaction opened in + `recording` that outlives one run's `--wait` holds the rerun's step 4 after a refusal, an + autovacuum worker on a loom table does not hold step 4 and a background worker with a + transaction open since `recording` does, a prepared transaction (after a restart with + `max_prepared_transactions` raised during the wait) makes step 4 refuse at once, naming + it, with the mode still `enforced` and a rerun refused until it is resolved and the + setting is back to 0, a write transaction open in a second database on the same server + holds step 4 and is named, a long read on the synchronous standby, or on a second standby, + with `hot_standby_feedback` on does not hold it, nor does a base backup running across it, + a write held open on a `replication=database` connection holds step 4, and so does a + `REPEATABLE READ` transaction on one, opened in `recording` and writing nothing before + `enforced`, a long logical dump (a `REPEATABLE READ` read-only transaction) on the cell's + database past `--wait` makes step 4 refuse, naming it, with the cell left `enforced`, and + the first cell's attach without `pg_read_all_stats` or with `max_prepared_transactions` + above 0 refuses before step 1 with the mode still `recording`, while a later cell's attach + does not require either; a missing or disabled org trigger is refused by attach and named + by `loom doctor`, a write for an unheld org on an `enforced` cell whose mode row was + deleted is refused and the state named by `loom doctor`; on that cell a pod with + `LOOM_CELL` unset is unready and runs no worker, and a CLI command declaring an org with + it unset is refused, as a pod is; updating, deleting or truncating the `first` row on an + `enforced` cell is refused, also under `session_replication_role = replica`, and a missing + or disabled `first`-row trigger is refused by attach and named by `loom doctor`; a running process with `LOOM_CELL` set on a cell whose mode row is removed answers 503 to a bearer request and runs no worker pass, and serves again after a rerun of `loom cell prepare` writes the row back while it runs; a cell the directory registers with no state, tenancy @@ -1548,12 +1550,12 @@ cell` store command refuses; `loom cell limit --closed` succeeds with the cell's rerun; a cell with both rows absent beside a tenancy row, and one beside only its own row, has the function refuse an unheld org (a boot creates no trigger there), every process gated and `loom doctor` naming it, and a prepare rerun writes the rows back; a - Postgres-only restore of an attached first cell to before its prepare is named by `loom -doctor` and closed by `loom cell restore`; a prepare rerun during a pending `loom cell -update` on an attached cell, with processes on both values and `loom.cell_self` lost, - writes both values back and keeps both sets of processes serving, and one before attach - refuses the two configurations; a prepare rerun on an attached cell whose `current` is - missing and whose pods' configuration differs from the directory row refuses, and one + Postgres-only restore of an attached first cell to before its prepare is named by + `loom doctor` and closed by `loom cell restore`; a prepare rerun during a pending + `loom cell update` on an attached cell, with processes on both values and `loom.cell_self` + lost, writes both values back and keeps both sets of processes serving, and one before + attach refuses the two configurations; a prepare rerun on an attached cell whose `current` + is missing and whose pods' configuration differs from the directory row refuses, and one before attach whose heartbeats report another bucket prefix refuses; a prepare rerun given a credential hash that is neither the row's `current` nor its `next` is refused; a prepare rerun on a `serving` later cell whose prefix holds data writes a removed mode row back; a @@ -1561,33 +1563,34 @@ update` on an attached cell, with processes on both values and `loom.cell_self` creates no trigger there), every process gated and `loom doctor` naming it, and a prepare rerun writes the row back; a later cell whose mode row is set to `recording` records nothing, its trigger refuses an unheld org, a process with `LOOM_CELL` set there serves as - in `enforced`, a pod with `LOOM_CELL` unset there is unready and runs no worker, `loom -cell prepare` names the row, a CLI command declaring an org with `LOOM_CELL_ROUTER_TOKEN` - set refuses, naming it, and the router refuses its state read; a build N-1 process's - `memory.set` and delivery pass on a never-prepared deployment after N's migration succeed - and write no tenancy row, and on the first cell in `recording` a boot creates no org - trigger, and one created by hand reads the state and looks nothing up, a write under - `session_replication_role = replica` for an unheld org is skipped (origin triggers), an - `INSERT ... ON CONFLICT DO UPDATE` that inserts a row for an unheld org is refused and one - that hits an existing row of it is not, a marked session's every write passes, and the - triggers on an `enforced` cell cover every table erasure classifies with an `org` column - but the three exclusions; and an org-keyed write while attach holds the mode table's lock - returns at once, with new-org requests answering within their deadline and mode-cache - reads returning at once throughout; a refusal at step 3's lock leaves the cell `recording` - with its directory row `enforced`, and the rerun runs step 2 again; a - `LOOM_CELL_ROUTER_TOKEN` not of the `lcr_` form is refused at boot and named by `loom -doctor`; `loom cell credential new` prints a credential of that form and its SHA-256, with - the directory and the stores unreachable; prepare and `--next` refuse an argument that is - not 64 lowercase hex characters, the credential and its bare 128-character secret - included, storing nothing; orgs a first cell records in `recording` with no data are - listed by `loom doctor`; a placement flood with no quota set is held to the rate limit at - each router replica, `loom doctor` names the cell as one with no quota, and a call refused - past it at one replica answers 503 with a `Retry-After` of at most 60 s, after which its - retry at that replica, with no other call of the cell admitted between, is admitted; a - cell's first prepare, with no directory row yet, passes the deployment-id check; a pod or - CLI that lost `LOOM_PG_DSN` or `LOOM_STORAGE_AUTHORITY` on a cell is refused by the - authority check; the first cell's existing tokens, refresh tokens and OAuth clients keep - working through the cutover, a logout through the shared host revokes its refresh token; + in `enforced`, a pod with `LOOM_CELL` unset there is unready and runs no worker, + `loom cell prepare` names the row, a CLI command declaring an org with + `LOOM_CELL_ROUTER_TOKEN` set refuses, naming it, and the router refuses its state read; a + build N-1 process's `memory.set` and delivery pass on a never-prepared deployment after + N's migration succeed and write no tenancy row, and on the first cell in `recording` a + boot creates no org trigger, and one created by hand reads the state and looks nothing up, + a write under `session_replication_role = replica` for an unheld org is skipped (origin + triggers), an `INSERT ... ON CONFLICT DO UPDATE` that inserts a row for an unheld org is + refused and one that hits an existing row of it is not, a marked session's every write + passes, and the triggers on an `enforced` cell cover every table erasure classifies with + an `org` column but the three exclusions; and an org-keyed write while attach holds the + mode table's lock returns at once, with new-org requests answering within their deadline + and mode-cache reads returning at once throughout; a refusal at step 3's lock leaves the + cell `recording` with its directory row `enforced`, and the rerun runs step 2 again; a + `LOOM_CELL_ROUTER_TOKEN` not of the `lcr_` form is refused at boot and named by + `loom doctor`; `loom cell credential new` prints a credential of that form and its + SHA-256, with the directory and the stores unreachable; prepare and `--next` refuse an + argument that is not 64 lowercase hex characters, the credential and its bare + 128-character secret included, storing nothing; orgs a first cell records in `recording` + with no data are listed by `loom doctor`; a placement flood with no quota set is held to + the rate limit at each router replica, `loom doctor` names the cell as one with no quota, + and a call refused past it at one replica answers 503 with a `Retry-After` of at most 60 + s, after which its retry at that replica, with no other call of the cell admitted between, + is admitted; a cell's first prepare, with no directory row yet, passes the deployment-id + check; a pod or CLI that lost `LOOM_PG_DSN` or `LOOM_STORAGE_AUTHORITY` on a cell is + refused by the authority check; the first cell's existing tokens, refresh tokens and OAuth + clients keep working through the cutover, a logout through the shared host revokes its + refresh token; - attach refuses a shared issuer, root secret, Keeper path, nested bucket prefix or configured org, a second cell that holds data, and a second attach while one runs; - an OIDC sign-in, an MCP client's authorization and a device flow for an org in the diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index 481b9a95ef..b713aefe0d 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -59,14 +59,14 @@ Attach's Postgres role needs `pg_read_all_stats`, `pg_signal_backend`, `USAGE` o `loom`, `SELECT` on `loom.cell_first`, `loom.tenancy_mode`, `loom.cell_self`, `loom.authority` and every org table, `UPDATE` on `loom.tenancy_mode`, `INSERT` on `loom.tenancy`, `SELECT`, `INSERT`, `UPDATE` and `DELETE` on `loom.cell_probe`, `TEMPORARY` -on the database, `EXECUTE` on `pg_logical_emit_message`, and membership in the owner role -of `loom.tenancy` and of every guarded table. Its ClickHouse user needs the grants the -writer check names (`SELECT` on `system.*` and on the configured databases, `SHOW USERS`, -`SHOW ROLES`, `SHOW ROW POLICIES`, `SHOW TABLES`, `SHOW COLUMNS`, `SYSTEM FLUSH ASYNC INSERT -QUEUE`, and on a cluster `SYSTEM SYNC REPLICA`, `CLUSTER` and `READ ON REMOTE`), no settings -profile or quota capping the rows or bytes it reads, and the insert into -`loom._migration_lock` that writes the probe nonce. The CLI needs write access to the -bucket's probe object and `LOOM_ROUTER_OPERATOR_TOKEN`. +on the database, `EXECUTE` on `pg_logical_emit_message`, and membership in the owner role of +`loom.tenancy` and of every guarded table. Its ClickHouse user needs the grants the writer +check names (`SELECT` on `system.*` and on the configured databases, `SHOW USERS`, +`SHOW ROLES`, `SHOW ROW POLICIES`, `SHOW TABLES`, `SHOW COLUMNS`, +`SYSTEM FLUSH ASYNC INSERT QUEUE`, and on a cluster `SYSTEM SYNC REPLICA`, `CLUSTER` and +`READ ON REMOTE`), no settings profile or quota capping the rows or bytes it reads, and the +insert into `loom._migration_lock` that writes the probe nonce. The CLI needs write access +to the bucket's probe object and `LOOM_ROUTER_OPERATOR_TOKEN`. ## Run it @@ -80,13 +80,14 @@ loom cell attach --cell \ `--dry-run` runs the refusals and step 2's scan, prints the count and a sample of the orgs per source table, the ones it would skip, and whether the scan fit in half of `--scan-budget`, and writes nothing; the router's own checks at `attach/begin` (the probe, -the hosts, the upstream) do not run. Name every ClickHouse user, user directory, object -and replica that may write the configured databases with `--ch-allow-user`, +the hosts, the upstream) do not run. Name every ClickHouse user, user directory, object and +replica that may write the configured databases with `--ch-allow-user`, `--ch-allow-user-directory`, `--ch-allow-object` and `--ch-allow-replica`, and every Postgres role besides attach's own that may make an org held with `--pg-allow-role`; each -list is audited by the router. `--wait` (default 600 s) bounds the waits together; -`--scan-budget` (default 7200 s) bounds each full scan, and step 2 must take at most half of -it, since step 4 reads the same tables again. +list is audited by the router. `--wait` (default 600 s) is the wall time the waits share; +every lock try runs at least once, with a lock wait of at least 100 ms, so `--wait 0` runs +one try of each wait with no retry; `--scan-budget` (default 7200 s) bounds each full scan, +and step 2 must take at most half of it, since step 4 reads the same tables again. A refusal names what to fix. Rerun attach after fixing it: every step is idempotent, and a rerun resumes by the directory row and the cell's mode. On a `serving` row it prints that it From 78c2808934b8cde06ec59202673b3fbe88f3f485 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 20:05:39 +0000 Subject: [PATCH 012/114] Bound cell attach's mode commit and name booting processes Signed-off-by: zlareb1 --- docs/architecture/cells.md | 6 +- docs/operations/cell-attach.md | 6 +- server/cli/cell_attach.py | 4 +- server/cli/cell_attach_pg.py | 34 +++++--- tests/integration/test_cell_attach.py | 4 +- tests/unit/test_cell_attach.py | 111 +++++++++++++++++++++++--- 6 files changed, 134 insertions(+), 31 deletions(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 53495c7a1d..f32815b635 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1065,8 +1065,10 @@ Then, in order (the step numbers are the ones the rest of this page uses): backed out. - **Step 3, the mode commit**: `attach/enforced`, then on a session of its own an EXCLUSIVE lock on `loom.tenancy_mode`, 2 s a try within `--wait`, the attach lock confirmed and the - cheap heartbeat refusals run before each try (a booting process skips the try). After a - failed try, the mode transaction an earlier run left in flight is found by its lock (only + cheap heartbeat refusals run before each try (a booting process skips the try). Each try, + including COMMIT, has a client deadline: the lock wait plus a 3 s statement margin and + a 5 s standby allowance. A commit past it has an unknown outcome until the durable read. + After a failed try, the mode transaction an earlier run left in flight is found by its lock (only attach takes `ExclusiveLock` there; a relation's OID is matched in this database alone) and ended. From the commit on the mode only rises: every later refusal leaves the cell `enforced` and `attaching`, and a rerun resumes from the durable read. diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index b713aefe0d..ec6b6354ec 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -86,8 +86,10 @@ replica that may write the configured databases with `--ch-allow-user`, Postgres role besides attach's own that may make an org held with `--pg-allow-role`; each list is audited by the router. `--wait` (default 600 s) is the wall time the waits share; every lock try runs at least once, with a lock wait of at least 100 ms, so `--wait 0` runs -one try of each wait with no retry; `--scan-budget` (default 7200 s) bounds each full scan, -and step 2 must take at most half of it, since step 4 reads the same tables again. +one try of each wait with no retry. The final heartbeat checks get at least two heartbeats +plus the router timeout, with one extension for a process first seen booting; this floor +is not charged to `--wait`. `--scan-budget` (default 7200 s) bounds each full scan, and +step 2 must take at most half of it, since step 4 reads the same tables again. A refusal names what to fix. Rerun attach after fixing it: every step is idempotent, and a rerun resumes by the directory row and the cell's mode. On a `serving` row it prints that it diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 8ed70e2d37..6ef819796a 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -442,13 +442,13 @@ async def triggers(self) -> None: # -- step 3 ------------------------------------------------------------------------ - async def ready_for_commit(self) -> bool: + async def ready_for_commit(self) -> list[str]: """The cheap heartbeat refusals right before a mode-commit try: a refusal raises, a booting process skips the try.""" verdict = self.judge_beats("refuse")(await beats.read_beats(self.ch)) if verdict.refusals: raise Refused(verdict.refusals) - return not verdict.booting + return verdict.booting async def commit_mode(self) -> None: from server.core import pg diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 0ca9fefcc1..60196363af 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -12,7 +12,7 @@ import asyncio import contextlib import time -from collections.abc import Callable, Iterable, Iterator +from collections.abc import Awaitable, Callable, Iterable, Iterator from dataclasses import dataclass from typing import Any @@ -29,9 +29,13 @@ # The migration lock's wait a trigger try (a boot holds it for its migrations), in ms; a # try never waits longer than --wait has left. _MIGRATION_LOCK_MS = 10_000 # not a setting: one try's bound +# A trigger try yields quickly to active writers. +_TRIGGER_LOCK_MS = 100 # not a setting: one try's bound # The mode commit's lock wait a try, in ms, and its statement bound's margin above it. _MODE_LOCK_MS = 2_000 # not a setting: one try's bound _MODE_STATEMENT_MARGIN_MS = 3_000 # not a setting: a margin over the lock wait +# COMMIT's standby wait is outside Postgres's statement timeout. +_MODE_STANDBY_S = 5.0 # not a setting: one try's standby allowance # The server ends the lock session soon after its client is gone, releasing the lock. _KEEPALIVE = {"tcp_keepalives_idle": "30", "tcp_keepalives_interval": "10"} @@ -465,7 +469,7 @@ async def _ensure_table( yielded = False said = "" while True: - most_ms = 100 + most_ms = _TRIGGER_LOCK_MS holders = await cell_triggers.holders(conn, table) if not yielded and any(h["backend_type"] == "autovacuum worker" for h in holders): most_ms, yielded = deadlock_ms + 500, True @@ -572,7 +576,7 @@ def _busy_refusal(table: str, holders: list[Any]) -> Refusal: async def commit_enforced( conn: Any, confirm_lock: Callable[[], Any], - ready: Callable[[], Any], + ready: Callable[[], Awaitable[list[str]]], wait: Wait, log: Callable[[str], None], mark: Callable[[str], None], @@ -581,17 +585,17 @@ async def commit_enforced( session): an EXCLUSIVE lock on loom.tenancy_mode a try, `_MODE_LOCK_MS` each (less when `wait` has less left), within `wait`; `confirm_lock` before each try and each termination (it refuses when another run holds the attach lock), and `ready` before - each try (False skips the try while a process boots). After a failed try the in-flight + each try (the booting process ids skip the try). After a failed try the in-flight mode transaction, found by its lock, is ended. `mark` hears `unknown` just before a try sends its UPDATE and `recording` when that try rolled back. Raises Refused when `wait` runs out; whether the commit landed is the durable read's to say.""" with wait.counting(): while True: await confirm_lock() - if await ready(): - mark("unknown") + booting = await ready() + if not booting: try: - done = await _mode_try(conn, wait.bound_ms(_MODE_LOCK_MS)) + done = await _mode_try(conn, wait.bound_ms(_MODE_LOCK_MS), mark) except Refused: mark("recording") # the UPDATE's transaction rolled back raise @@ -601,6 +605,13 @@ async def commit_enforced( log("the mode's lock is busy; retrying") await end_mode_transactions(conn, confirm_lock) if not await wait.pause(): + if booting: + raise Refused( + [ + Refusal(f"process {p}", "still booting at the end of --wait") + for p in booting + ] + ) raise Refused( [ Refusal( @@ -612,15 +623,18 @@ async def commit_enforced( ) -async def _mode_try(conn: Any, lock_ms: int) -> bool: - """One try of the mode commit, its lock wait `lock_ms`; False when it timed out.""" +async def _mode_try(conn: Any, lock_ms: int, mark: Callable[[str], None]) -> bool: + """One try including COMMIT, bounded by its lock and statement waits plus the standby + allowance; False on a statement timeout, TimeoutError if COMMIT never answers.""" + bound_s = (lock_ms + _MODE_STATEMENT_MARGIN_MS) / 1000 + _MODE_STANDBY_S try: - async with conn.transaction(): + async with asyncio.timeout(bound_s), conn.transaction(): await conn.execute(f"SET LOCAL lock_timeout = '{lock_ms}ms'") await conn.execute( f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" ) await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") + mark("unknown") status = await conn.execute( "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" ) diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 65064ab365..f6535cdb40 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -469,8 +469,8 @@ async def test_the_mode_commit_waits_out_a_for_share_holder(cell_stores) -> None await _prepare(cell_stores, _Network()) logged: list[str] = [] - async def ok() -> bool: - return True + async def ok() -> list[str]: + return [] async with pg.dedicated() as holder, pg.dedicated() as conn: tx = holder.transaction() diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 429c2fcfb7..95fc006333 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -6,6 +6,7 @@ from __future__ import annotations import argparse +import asyncio import json from collections.abc import Callable from dataclasses import dataclass, field @@ -244,8 +245,8 @@ def lock(sql: str, *_a: Any) -> str: async def nothing() -> None: pass - async def ready() -> bool: - return True + async def ready() -> list[str]: + return [] wait = apg.Wait(4, clock) clock.now += 100 # a scan before the wait @@ -652,8 +653,8 @@ def lock(*_a: Any) -> str: async def confirm() -> None: confirmed.append(True) - async def ready() -> bool: - return True + async def ready() -> list[str]: + return [] logged: list[str] = [] await apg.commit_enforced( @@ -684,24 +685,108 @@ async def confirm() -> None: @pytest.mark.asyncio +@pytest.mark.parametrize("wait_s", [0, 2]) async def test_the_mode_commit_refuses_past_wait_and_skips_a_try_while_booting( clock: _Clock, + monkeypatch: pytest.MonkeyPatch, + wait_s: int, ) -> None: conn = FakeConn([("l.mode = 'ExclusiveLock'", [])]) + run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + run.own_fp = (CURRENT, NEXT) + + async def read(_ch: Any) -> list[beats.Beat]: + return [_beat("a", cell_name=None), _beat("b", cell_name=None)] + + monkeypatch.setattr(beats, "read_beats", read) async def confirm() -> None: pass - async def booting() -> bool: - return False - - with pytest.raises(Refused, match="not taken within --wait"): + with pytest.raises(Refused) as error: await apg.commit_enforced( - conn, confirm, booting, apg.Wait(2, clock), lambda _t: None, lambda _m: None + conn, + confirm, + run.ready_for_commit, + apg.Wait(wait_s, clock), + lambda _t: None, + lambda _m: None, ) + assert [str(r) for r in error.value.refusals] == [ + "process a: still booting at the end of --wait", + "process b: still booting at the end of --wait", + ] assert not any("LOCK TABLE" in s for s in conn.executed) +@pytest.mark.asyncio +@pytest.mark.parametrize("wait_s,bound_s", [(0, 8.1), (10, 10.0)]) +async def test_the_mode_try_bounds_a_commit_that_never_answers( + monkeypatch: pytest.MonkeyPatch, clock: _Clock, wait_s: int, bound_s: float +) -> None: + class _StalledCommit(FakeConn): + def transaction(self) -> Any: + conn = self + + class _Tx: + async def __aenter__(self) -> None: + conn.executed.append("BEGIN") + + async def __aexit__(self, *exc: Any) -> None: + assert not exc[0] + conn.executed.append("COMMIT") + await asyncio.Future() + + return _Tx() + + conn = _StalledCommit([("UPDATE loom.tenancy_mode", "UPDATE 1")]) + bounds: list[float] = [] + timeout = asyncio.timeout + + def bounded(seconds: float) -> Any: + bounds.append(seconds) + return timeout(0) + + monkeypatch.setattr(apg.asyncio, "timeout", bounded) + marks: list[str] = [] + + async def confirm() -> None: + pass + + async def ready() -> list[str]: + return [] + + with pytest.raises(TimeoutError): + await asyncio.wait_for( + apg.commit_enforced( + conn, confirm, ready, apg.Wait(wait_s, clock), lambda _t: None, marks.append + ), + timeout=0.2, + ) + assert conn.executed[-1] == "COMMIT" + assert bounds == [bound_s] + assert marks == ["unknown"] + + +@pytest.mark.asyncio +async def test_an_interrupt_waiting_for_the_mode_lock_leaves_recording(clock: _Clock) -> None: + conn = FakeConn([("LOCK TABLE", KeyboardInterrupt())]) + marks: list[str] = [] + + async def confirm() -> None: + pass + + async def ready() -> list[str]: + return [] + + with pytest.raises(KeyboardInterrupt): + await apg.commit_enforced( + conn, confirm, ready, apg.Wait(2, clock), lambda _t: None, marks.append + ) + assert conn.executed[-1] == "ROLLBACK" + assert marks == [] + + @pytest.mark.asyncio async def test_the_durable_read_must_see_enforced_and_x_above_its_xid( monkeypatch: pytest.MonkeyPatch, @@ -921,8 +1006,8 @@ async def test_wait_0_runs_one_try_of_each_wait( ) marks: list[str] = [] - async def ok() -> bool: - return True + async def ok() -> list[str]: + return [] async def nothing() -> None: pass @@ -975,10 +1060,10 @@ def lock(*_a: Any) -> str: async def call(_self: Any, _path: str, _body: Any = None) -> dict[str, Any]: return {"cell": "first"} - async def ready(_self: Any) -> bool: + async def ready(_self: Any) -> list[str]: if refused_before_any_try: raise Refused([Refusal("process p1", "runs with LOOM_CELL unset")]) - return True + return [] async def confirm(_self: Any) -> None: pass From 62c6b4783ac06200c31d32f735def75e48da017a Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 20:18:57 +0000 Subject: [PATCH 013/114] Require every cell probe grant before attach Check each probe privilege separately because a comma-separated privilege list accepts any one of them. Run the attach integration tests in the Postgres matrix and limit their heartbeat reads to the test's own process IDs while retaining the real queries. Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 1 + server/cli/cell_attach_pg.py | 5 ++- tests/integration/test_cell_attach.py | 50 +++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 564393a0f7..22342c2cab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -898,6 +898,7 @@ jobs: tests/integration/test_router_directory_pg.py tests/integration/test_cell_unset_gate_pg.py tests/integration/test_cell_prepare.py + tests/integration/test_cell_attach.py tests/integration/test_cell_org_triggers_pg.py # The tenant-isolation gates again, against a second server that enforces auth: # the Postgres read paths (heads by key and by id, narrowed scopes) see real tokens. diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 60196363af..583d99775d 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -397,7 +397,10 @@ async def session_passes() -> bool: ('SELECT on loom.authority', has_table_privilege('loom.authority', 'SELECT')), ('INSERT on loom.tenancy', has_table_privilege('loom.tenancy', 'INSERT')), ('SELECT, INSERT, UPDATE and DELETE on loom.cell_probe', - has_table_privilege('loom.cell_probe', 'SELECT, INSERT, UPDATE, DELETE')), + has_table_privilege('loom.cell_probe', 'SELECT') + AND has_table_privilege('loom.cell_probe', 'INSERT') + AND has_table_privilege('loom.cell_probe', 'UPDATE') + AND has_table_privilege('loom.cell_probe', 'DELETE')), ('TEMPORARY on the database', has_database_privilege(current_database(), 'TEMPORARY')) ) v(need, held) WHERE NOT held diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index f6535cdb40..aaa48b8f0e 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -102,6 +102,21 @@ async def cell_stores(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[dict[str ch = get_client() await ch.connect() ids: list[str] = [] + real_live, real_recent = instances.read_live, instances.read_recent + + async def read_live(ch: Any, *, all_deployments: bool = False) -> list[dict[str, Any]]: + rows = await real_live(ch, all_deployments=all_deployments) + return [r for r in rows if r["instance_id"] in ids] + + async def read_recent( + ch: Any, *, window_s: float, since: str | None = None + ) -> list[dict[str, Any]]: + rows = await real_recent(ch, window_s=window_s, since=since) + return [r for r in rows if r["instance_id"] in ids] + + # The integration server shares ClickHouse and reports its own process heartbeats. + monkeypatch.setattr(instances, "read_live", read_live) + monkeypatch.setattr(instances, "read_recent", read_recent) try: await pg_migrate.migrate() await pg.execute( @@ -669,6 +684,41 @@ async def pg_orgs(table: str, budget: Any) -> list[str]: assert grade == "WARN" and detail.endswith(": loom.it_large") +@pytest.mark.asyncio +async def test_attach_requires_every_probe_privilege(cell_stores) -> None: + role = f"it_probe_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + privileges = ("SELECT", "INSERT", "UPDATE", "DELETE") + grants = [ + ("SELECT",), + *(tuple(p for p in privileges if p != missing) for missing in privileges), + privileges, + ] + observed: dict[tuple[str, ...], list[str]] = {} + try: + async with pg.dedicated() as conn: + await conn.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + for held in grants: + await conn.execute(f"REVOKE ALL ON loom.cell_probe FROM {role}") + await conn.execute(f"GRANT {', '.join(held)} ON loom.cell_probe TO {role}") + await conn.execute(f"SET ROLE {role}") + refusals = await cell_attach_pg.own_role_refusals(conn, []) + await conn.execute("RESET ROLE") + observed[held] = [r.reason for r in refusals if "loom.cell_probe" in r.reason] + assert observed == { + held: ( + [] + if held == privileges + else ["attach's role lacks SELECT, INSERT, UPDATE and DELETE on loom.cell_probe"] + ) + for held in grants + } + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") + + @pytest.mark.asyncio async def test_a_role_with_the_grants_but_not_the_owner_role_is_refused_before_any_write( cell_stores, From 95bb74c0fe04dd0981970e9305c4ff447cb5d703 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 20:29:02 +0000 Subject: [PATCH 014/114] Keep cell attach deadlines from waiting on rollback Manage mode and direct-session transactions explicitly so a canceled statement cannot start an unbounded rollback. Roll back ordinary mode statement errors within the try's deadline; terminate synchronously when a try ends with an error or cancellation. Signed-off-by: zlareb1 --- server/cli/cell_attach_pg.py | 64 +++++++++++------- tests/unit/test_cell_attach.py | 116 ++++++++++++++++++++++++++++----- 2 files changed, 139 insertions(+), 41 deletions(-) diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 583d99775d..d58b553d7a 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -122,10 +122,18 @@ async def direct_refusals(conn: Any, what: str) -> list[Refusal]: startup parameters drops it). Behind a transaction pooler a session-level lock would outlive the CLI on the pooler's server connection, and attach's tenancy rows would be refused as an unchecked writer's.""" - async with conn.transaction(): - first = await conn.fetchrow(_IDENTITY) - async with conn.transaction(): - second = await conn.fetchrow(_IDENTITY) + identities = [] + try: + for _ in range(2): + tx = conn.transaction() + await tx.start() + identities.append(await conn.fetchrow(_IDENTITY)) + await tx.commit() + except BaseException: + # A cancelled read must not wait for a transaction context's rollback. + conn.terminate() + raise + first, second = identities out = [] if first is None or second is None or tuple(first) != tuple(second): out.append( @@ -627,31 +635,39 @@ async def commit_enforced( async def _mode_try(conn: Any, lock_ms: int, mark: Callable[[str], None]) -> bool: - """One try including COMMIT, bounded by its lock and statement waits plus the standby - allowance; False on a statement timeout, TimeoutError if COMMIT never answers.""" + """One try including rollback or COMMIT, bounded by its lock and statement waits plus + the standby allowance; False after a statement timeout rolls back, TimeoutError at + the try's deadline.""" bound_s = (lock_ms + _MODE_STATEMENT_MARGIN_MS) / 1000 + _MODE_STANDBY_S try: - async with asyncio.timeout(bound_s), conn.transaction(): - await conn.execute(f"SET LOCAL lock_timeout = '{lock_ms}ms'") - await conn.execute( - f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" - ) - await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") - mark("unknown") - status = await conn.execute( - "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" - ) - if status != "UPDATE 1": - raise Refused( - [Refusal("postgres loom.tenancy_mode", f"the mode update read {status}")] + async with asyncio.timeout(bound_s): + tx = conn.transaction() + await tx.start() + try: + await conn.execute(f"SET LOCAL lock_timeout = '{lock_ms}ms'") + await conn.execute( + f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" ) + await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") + mark("unknown") + status = await conn.execute( + "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" + ) + if status != "UPDATE 1": + raise Refused( + [Refusal("postgres loom.tenancy_mode", f"the mode update read {status}")] + ) + except Exception as e: + await tx.rollback() + if getattr(e, "sqlstate", None) in _TIMED_OUT: + return False + raise + await tx.commit() return True - except Refused: + except BaseException: + # Termination is synchronous: cancellation never starts an unbounded rollback. + conn.terminate() raise - except Exception as e: - if getattr(e, "sqlstate", None) not in _TIMED_OUT: - raise - return False async def end_mode_transactions(conn: Any, confirm_lock: Callable[[], Any]) -> None: diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 95fc006333..065f3a455d 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -473,14 +473,29 @@ def transaction(self) -> Any: conn = self class _Tx: + async def start(self) -> None: + await conn.execute("BEGIN") + + async def commit(self) -> None: + await conn.execute("COMMIT") + + async def rollback(self) -> None: + await conn.execute("ROLLBACK") + async def __aenter__(self) -> None: - conn.executed.append("BEGIN") + await self.start() async def __aexit__(self, *exc: Any) -> None: - conn.executed.append("ROLLBACK" if exc[0] else "COMMIT") + if exc[0]: + await self.rollback() + else: + await self.commit() return _Tx() + def terminate(self) -> None: + self.executed.append("TERMINATE") + def _seq(*values: Any) -> Callable[..., Any]: pending = list(values) @@ -663,6 +678,8 @@ async def ready() -> list[str]: assert logged == ["the mode's lock is busy; retrying"] # Before each try and the one termination. assert len(confirmed) == 3 + assert conn.executed.count("ROLLBACK") == 1 + assert "TERMINATE" not in conn.executed @pytest.mark.asyncio @@ -725,19 +742,11 @@ async def test_the_mode_try_bounds_a_commit_that_never_answers( monkeypatch: pytest.MonkeyPatch, clock: _Clock, wait_s: int, bound_s: float ) -> None: class _StalledCommit(FakeConn): - def transaction(self) -> Any: - conn = self - - class _Tx: - async def __aenter__(self) -> None: - conn.executed.append("BEGIN") - - async def __aexit__(self, *exc: Any) -> None: - assert not exc[0] - conn.executed.append("COMMIT") - await asyncio.Future() - - return _Tx() + async def execute(self, sql: str, *args: Any) -> str: + result = await super().execute(sql, *args) + if sql == "COMMIT": + await asyncio.Future() + return result conn = _StalledCommit([("UPDATE loom.tenancy_mode", "UPDATE 1")]) bounds: list[float] = [] @@ -763,11 +772,84 @@ async def ready() -> list[str]: ), timeout=0.2, ) - assert conn.executed[-1] == "COMMIT" + assert "COMMIT" in conn.executed + assert conn.executed[-1] == "TERMINATE" assert bounds == [bound_s] assert marks == ["unknown"] +@pytest.mark.asyncio +@pytest.mark.parametrize("stall", ["LOCK TABLE", "UPDATE", "ROLLBACK"]) +async def test_a_mode_deadline_never_waits_for_rollback( + monkeypatch: pytest.MonkeyPatch, stall: str +) -> None: + class _StalledRollback(FakeConn): + async def execute(self, sql: str, *args: Any) -> str: + result = await super().execute(sql, *args) + if stall == "ROLLBACK" and sql.startswith("LOCK TABLE"): + raise _Timeout() + if sql.startswith(stall) or sql == "ROLLBACK": + await asyncio.Future() + return result + + conn = _StalledRollback() + timeout = asyncio.timeout + monkeypatch.setattr(apg.asyncio, "timeout", lambda _s: timeout(0)) + marks: list[str] = [] + task = asyncio.create_task(apg._mode_try(conn, 100, marks.append)) + done, _ = await asyncio.wait([task], timeout=0.2) + if not done: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert task in done, "mode deadline is stuck awaiting rollback" + assert isinstance(task.exception(), TimeoutError) + assert conn.executed[-1] == "TERMINATE" + assert marks == (["unknown"] if stall == "UPDATE" else []) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed_read", [1, 2]) +@pytest.mark.parametrize("failure", ["cancel", "error"]) +async def test_a_failed_direct_session_read_never_waits_for_rollback( + failed_read: int, failure: str +) -> None: + class _StalledRollback(FakeConn): + reads = 0 + + async def fetchrow(self, sql: str, *args: Any) -> Any: + self.reads += 1 + if self.reads == failed_read: + if failure == "error": + raise ValueError("identity read failed") + await asyncio.Future() + return (1, "t0") + + async def execute(self, sql: str, *args: Any) -> str: + result = await super().execute(sql, *args) + if sql == "ROLLBACK": + await asyncio.Future() + return result + + conn = _StalledRollback() + task = asyncio.create_task(apg.direct_refusals(conn, "attach")) + await asyncio.sleep(0) + if failure == "cancel": + task.cancel() + done, _ = await asyncio.wait([task], timeout=0.2) + if not done: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert task in done, "direct-session failure is stuck awaiting rollback" + if failure == "cancel": + assert task.cancelled() + else: + assert isinstance(task.exception(), ValueError) + assert conn.executed[-1] == "TERMINATE" + assert "ROLLBACK" not in conn.executed + + @pytest.mark.asyncio async def test_an_interrupt_waiting_for_the_mode_lock_leaves_recording(clock: _Clock) -> None: conn = FakeConn([("LOCK TABLE", KeyboardInterrupt())]) @@ -783,7 +865,7 @@ async def ready() -> list[str]: await apg.commit_enforced( conn, confirm, ready, apg.Wait(2, clock), lambda _t: None, marks.append ) - assert conn.executed[-1] == "ROLLBACK" + assert conn.executed[-1] == "TERMINATE" assert marks == [] From d5ec3b9b118b5943c500f0169323d8569ceb4f4d Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 20:44:09 +0000 Subject: [PATCH 015/114] Bound attach trigger attempts and scope gate cleanup to its run Give each trigger attempt a client deadline covering migration lock acquisition, DDL, commit, rollback and unlock. Use explicit transactions so cancellation terminates the session without waiting for rollback or masking the original error with lock cleanup. Give gate batch scopes a unique prefix per run so cleanup preserves existing rows. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 8 ++- server/cli/cell_attach_pg.py | 47 ++++++++------ server/core/cell_triggers.py | 21 +++++- tests/unit/test_bench_cell_trigger_gate.py | 53 +++++++++++++++ tests/unit/test_cell_attach.py | 75 ++++++++++++++++++++++ tests/unit/test_cell_org_triggers.py | 14 ++-- 6 files changed, 192 insertions(+), 26 deletions(-) create mode 100644 tests/unit/test_bench_cell_trigger_gate.py diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 4363a810d6..8797b4b8e7 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -80,8 +80,9 @@ async def _batch(args: argparse.Namespace, marked: bool) -> float: t = time.perf_counter() await conn.execute( "INSERT INTO loom.scope_generations (org, scope) " - "SELECT $1, 'gate-' || g || '-' || $2 FROM generate_series(1, 1000) g", + "SELECT $1, $2 || g || '-' || $3 FROM generate_series(1, 1000) g", args.org, + args.scope_prefix, uuid.uuid4().hex[:8], ) times.append((time.perf_counter() - t) * 1000) @@ -98,6 +99,7 @@ async def main() -> None: p.add_argument("--org", required=True) p.add_argument("--namespace", required=True) args = p.parse_args() + args.scope_prefix = f"gate-{uuid.uuid4().hex}-" try: for _ in range(ROUNDS): for on in (True, False): @@ -120,7 +122,9 @@ async def _clean(args: argparse.Namespace) -> None: conn = await asyncpg.connect(args.pg_dsn, server_settings={"loom.tenancy_checked": "on"}) try: await conn.execute( - "DELETE FROM loom.scope_generations WHERE org = $1 AND scope LIKE 'gate-%'", args.org + "DELETE FROM loom.scope_generations WHERE org = $1 AND starts_with(scope, $2)", + args.org, + args.scope_prefix, ) finally: await conn.close() diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index d58b553d7a..2af085f86f 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -31,6 +31,7 @@ _MIGRATION_LOCK_MS = 10_000 # not a setting: one try's bound # A trigger try yields quickly to active writers. _TRIGGER_LOCK_MS = 100 # not a setting: one try's bound +_TRIGGER_TRY_MARGIN_S = 8.0 # not a setting: catalog, DDL and standby allowance # The mode commit's lock wait a try, in ms, and its statement bound's margin above it. _MODE_LOCK_MS = 2_000 # not a setting: one try's bound _MODE_STATEMENT_MARGIN_MS = 3_000 # not a setting: a margin over the lock wait @@ -524,29 +525,37 @@ async def _migration_holder(conn: Any, key: int) -> str: async def _ensure_one(conn: Any, table: str, most_ms: int, wait: Wait) -> cell_triggers.Outcome: """One try under the migration lock: the migration lock's wait, and then the DDL's lock wait (at most `most_ms`), each bounded by what `wait` has left when it starts, with the - try floor. A busy migration lock answers `busy` naming that lock's holder.""" + try floor. One client deadline covers the lock, DDL and cleanup, including COMMIT or + rollback. A busy migration lock answers `busy` naming that lock's holder.""" from server.core.schema.pg_migrate import _LOCK_KEY - await conn.execute(f"SET lock_timeout = '{wait.bound_ms(_MIGRATION_LOCK_MS)}ms'") + bound_s = ( + wait.bound_ms(_MIGRATION_LOCK_MS) + wait.bound_ms(most_ms) + ) / 1000 + _TRIGGER_TRY_MARGIN_S try: - await conn.execute(_MIGRATION_LOCK, _LOCK_KEY) - except Exception as e: - if getattr(e, "sqlstate", None) == "55P03": - # A boot migrating: the next try. - return cell_triggers.Outcome("busy", await _migration_holder(conn, _LOCK_KEY)) + async with asyncio.timeout(bound_s): + await conn.execute(f"SET lock_timeout = '{wait.bound_ms(_MIGRATION_LOCK_MS)}ms'") + try: + await conn.execute(_MIGRATION_LOCK, _LOCK_KEY) + except Exception as e: + if getattr(e, "sqlstate", None) != "55P03": + raise + await conn.execute("RESET lock_timeout") + return cell_triggers.Outcome("busy", await _migration_holder(conn, _LOCK_KEY)) + await conn.execute("RESET lock_timeout") + outcome = await cell_triggers.ensure_table( + conn, + table, + recording_first=True, + lock_timeout=f"{wait.bound_ms(most_ms)}ms", + attempts=1, + ) + await conn.execute(_MIGRATION_UNLOCK, _LOCK_KEY) + return outcome + except BaseException: + # Closing the session releases its locks without asynchronous error cleanup. + conn.terminate() raise - finally: - await conn.execute("RESET lock_timeout") - try: - return await cell_triggers.ensure_table( - conn, - table, - recording_first=True, - lock_timeout=f"{wait.bound_ms(most_ms)}ms", - attempts=1, - ) - finally: - await conn.execute(_MIGRATION_UNLOCK, _LOCK_KEY) def _busy_refusal(table: str, holders: list[Any]) -> Refusal: diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 7a1ee5e5b7..1899a46bab 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -328,18 +328,32 @@ async def _run_ddl( import asyncpg for attempt in range(attempts): + rolled_back = False try: - async with conn.transaction(): + tx = conn.transaction() + await tx.start() + try: await conn.execute(f"SET LOCAL lock_timeout = '{lock_timeout}'") for sql in sqls: await conn.execute(sql) + except Exception: + await tx.rollback() + rolled_back = True + raise + await tx.commit() return Outcome("applied") except asyncpg.exceptions.InsufficientPrivilegeError as e: + if not rolled_back: + conn.terminate() + raise log.warning( "cell triggers: not applied on loom.%s (%s): this role cannot alter it", table, e ) return Outcome("denied") except asyncpg.exceptions.LockNotAvailableError as e: + if not rolled_back: + conn.terminate() + raise # Short tries outlast a short transaction; an autovacuum is left to finish. if attempt < attempts - 1: await asyncio.sleep(LOCK_RETRY_S) @@ -354,6 +368,11 @@ async def _run_ddl( held, ) return Outcome("busy", held) + except BaseException: + if not rolled_back: + # Cancellation must not wait for another statement on this session. + conn.terminate() + raise return Outcome("busy") diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py new file mode 100644 index 0000000000..89f32cddde --- /dev/null +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -0,0 +1,53 @@ +"""The trigger gate deletes only the scope rows its own batch inserts wrote.""" + +from __future__ import annotations + +import argparse +import sqlite3 +from typing import Any + +import pytest + +from bench import cell_trigger_gate as gate + + +@pytest.mark.asyncio +async def test_batch_cleanup_preserves_preexisting_gate_scopes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + db = sqlite3.connect(":memory:") + db.execute("ATTACH DATABASE ':memory:' AS loom") + db.execute("CREATE TABLE loom.scope_generations (org text, scope text)") + db.create_function("starts_with", 2, lambda value, prefix: value.startswith(prefix)) + prefix = "gate-own-run-" + sentinel = [("org", "gate-existing"), ("other", prefix + "existing")] + db.executemany("INSERT INTO loom.scope_generations VALUES (?, ?)", sentinel) + + class _Conn: + async def execute(self, sql: str, *args: Any) -> None: + sql = sql.replace( + "FROM generate_series(1, 1000) g", + "FROM (WITH RECURSIVE seq(g) AS (VALUES(1) UNION ALL " + "SELECT g + 1 FROM seq WHERE g < 1000) SELECT g FROM seq)", + ) + db.execute(sql, {str(i): value for i, value in enumerate(args, 1)}) + + async def close(self) -> None: + pass + + async def connect(*_a: Any, **_kw: Any) -> Any: + return _Conn() + + monkeypatch.setattr(gate.asyncpg, "connect", connect) + monkeypatch.setattr(gate, "INSERTS", 1) + args = argparse.Namespace(pg_dsn="unused", org="org", scope_prefix=prefix) + try: + await gate._batch(args, True) + assert db.execute("SELECT count(*) FROM loom.scope_generations").fetchone()[0] == 1002 + await gate._clean(args) + assert ( + db.execute("SELECT org, scope FROM loom.scope_generations ORDER BY org").fetchall() + == sentinel + ) + finally: + db.close() diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 065f3a455d..6d57cbb740 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -1038,6 +1038,81 @@ async def ensure_table(*_a: Any, lock_timeout: str, **_kw: Any) -> Any: assert asked == [f"{apg.TRY_FLOOR_MS}ms"] +@pytest.mark.asyncio +@pytest.mark.parametrize("stall", ["migration", "reset", "ddl", "commit", "rollback", "unlock"]) +async def test_a_trigger_try_deadline_covers_transaction_and_lock_cleanup( + monkeypatch: pytest.MonkeyPatch, clock: _Clock, stall: str +) -> None: + import asyncpg + + from server.core import cell_triggers + + class _Stalled(FakeConn): + async def execute(self, sql: str, *args: Any) -> str: + result = await super().execute(sql, *args) + if stall == "rollback" and sql == "CREATE TRIGGER stub": + raise asyncpg.LockNotAvailableError("busy") + needles = { + "migration": "SELECT pg_advisory_lock", + "reset": "RESET lock_timeout", + "ddl": "CREATE TRIGGER stub", + "commit": "COMMIT", + "rollback": "ROLLBACK", + "unlock": "SELECT pg_advisory_unlock", + } + if sql.startswith(needles[stall]) or (stall == "ddl" and sql == "ROLLBACK"): + await asyncio.Future() + return result + + async def ensure_table(conn: Any, table: str, **kwargs: Any) -> Any: + return await cell_triggers._run_ddl( + conn, table, ["CREATE TRIGGER stub"], kwargs["lock_timeout"], 1, warn=False + ) + + timeout = asyncio.timeout + monkeypatch.setattr(apg.asyncio, "timeout", lambda _s: timeout(0)) + monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) + conn = _Stalled() + task = asyncio.create_task(apg._ensure_one(conn, "scope_generations", 100, apg.Wait(0, clock))) + done, _ = await asyncio.wait([task], timeout=0.2) + try: + assert task in done, f"trigger deadline is stuck at {stall}" + assert isinstance(task.exception(), TimeoutError) + assert conn.executed[-1] == "TERMINATE" + if stall in {"migration", "ddl", "commit", "rollback"}: + assert apg._MIGRATION_UNLOCK not in conn.executed + finally: + for _ in range(3): + if task.done(): + break + task.cancel() + await asyncio.wait([task], timeout=0.01) + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed", ["migration", "ddl"]) +async def test_a_failed_trigger_try_does_not_mask_its_error_with_cleanup( + monkeypatch: pytest.MonkeyPatch, clock: _Clock, failed: str +) -> None: + from server.core import cell_triggers + + async def ensure_table(*_a: Any, **_kw: Any) -> Any: + raise ValueError("DDL failed") + + monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) + conn = FakeConn( + [ + ("pg_advisory_lock", ValueError("migration failed") if failed == "migration" else "OK"), + ("RESET", RuntimeError("reset failed") if failed == "migration" else "OK"), + ("pg_advisory_unlock", RuntimeError("unlock failed")), + ] + ) + with pytest.raises(ValueError, match=f"{failed if failed == 'migration' else 'DDL'} failed"): + await apg._ensure_one(conn, "scope_generations", 100, apg.Wait(0, clock)) + assert conn.executed[-1] == "TERMINATE" + + @pytest.mark.asyncio async def test_a_busy_migration_lock_is_named_as_such( monkeypatch: pytest.MonkeyPatch, clock: _Clock diff --git a/tests/unit/test_cell_org_triggers.py b/tests/unit/test_cell_org_triggers.py index decb365521..769cfe1bb3 100644 --- a/tests/unit/test_cell_org_triggers.py +++ b/tests/unit/test_cell_org_triggers.py @@ -617,11 +617,17 @@ def __init__(self, error: Exception) -> None: self.error = error def transaction(self) -> Any: - @contextlib.asynccontextmanager - async def _tx() -> AsyncIterator[None]: - yield + class _Tx: + async def start(self) -> None: + pass - return _tx() + async def commit(self) -> None: + pass + + async def rollback(self) -> None: + pass + + return _Tx() async def fetchrow(self, _sql: str, *_a: Any) -> dict[str, Any]: return { From 20681aa183677097ec2c1e6a31299a14d4230ed4 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 20:57:56 +0000 Subject: [PATCH 016/114] Check member-role privileges and repeat attach checks after placement Count privileges a login can reach through SET ROLE or an ADMIN regrant, including noninherited group grants. Keep the owner exemptions and explicit role allow list. Repeat the writer, trigger, Postgres catalog and standby checks after the rescan and placement before marking the cell serving. Retain the heartbeat watch and its final wait floor. Signed-off-by: zlareb1 --- docs/architecture/cells.md | 3 +- server/cli/cell_attach.py | 11 ++-- server/cli/cell_attach_pg.py | 8 ++- tests/integration/test_cell_attach.py | 87 +++++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 9 deletions(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index f32815b635..dd1189b696 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1093,7 +1093,8 @@ Then, in order (the step numbers are the ones the rest of this page uses): source in full, Postgres before ClickHouse (the outbox and delivery tables move rows to ClickHouse), and the queue and drain tables and the tables a view reads before the others (`cell_scan.MOVE_SOURCES`); the tenancy rows; the placements, in `attach/place` batches; - the writer check again; and the final checks, under a floor of their own (the larger of + the writer, trigger, Postgres and standby checks again; and the final checks, under a + floor of their own (the larger of what `--wait` left and `instances.boot_allowance_s()`): the heartbeats, read through the rescan every half of the time the leader keeps a stopped row (`instances.PRUNE_AFTER_S`), refusing any process that ran with `LOOM_CELL` unset during it, `loom.cell_self` equal to diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 6ef819796a..bf4e81521e 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -516,7 +516,7 @@ async def durable(self) -> int: # -- step 4 ------------------------------------------------------------------------ - async def cheap_checks(self, stopped_unset: str) -> None: + async def nonheartbeat_checks(self) -> None: from server.core import pg_profile out, _, _ = await self.writer_refusals() @@ -527,10 +527,13 @@ async def cheap_checks(self, stopped_unset: str) -> None: out += await self.catalog_refusals() if out: raise Refused(out) - await self.check_beats(stopped_unset) if (now := await pg_profile.standby_now()) != "synchronous": raise _refused("postgres standbys", f"commits are acknowledged as {now}") + async def cheap_checks(self, stopped_unset: str) -> None: + await self.nonheartbeat_checks() + await self.check_beats(stopped_unset) + async def final_checks(self, watch: beats.Watch) -> None: from server.core import instances @@ -575,9 +578,7 @@ async def step4(self, xmax: int, topology: Any, names: Any) -> None: ) placed += int(answer.get("placed") or 0) self.report.update(orgs_found=len(orgs), placed_now=placed, skipped=self._named(found)) - writers, _, _ = await self.writer_refusals() - if writers: - raise Refused(writers) + await self.nonheartbeat_checks() finally: task.cancel() with contextlib.suppress(asyncio.CancelledError): diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 2af085f86f..56601e72e9 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -231,6 +231,7 @@ async def encoding_refusals(conn: Any) -> list[Refusal]: # Every right a login role may hold that lets it make an org held, lower the mode, or run # its own SQL as the servers on their marked sessions: each row a role and why. +# Membership can supply a right through SET ROLE or an ADMIN regrant without INHERIT. _RIGHTS = f""" WITH owned AS ( SELECT c.relowner AS owner, 'owns loom.' || c.relname AS why @@ -248,7 +249,8 @@ async def encoding_refusals(conn: Any) -> list[Refusal]: SELECT n.nspowner, 'owns schema ' || n.nspname FROM pg_namespace n WHERE {_SYSTEM_SCHEMA} ), login AS (SELECT oid, rolname FROM pg_roles WHERE rolcanlogin) -SELECT r.rolname AS role, w.why FROM login r, LATERAL ( +SELECT DISTINCT l.rolname AS role, w.why FROM login l +JOIN pg_roles r ON pg_has_role(l.oid, r.oid, 'MEMBER'), LATERAL ( SELECT 'may insert into loom.tenancy' WHERE has_any_column_privilege(r.oid, 'loom.tenancy', 'INSERT') UNION ALL SELECT 'may update loom.tenancy (org)' @@ -273,10 +275,10 @@ async def encoding_refusals(conn: Any) -> list[Refusal]: UNION ALL SELECT 'is a member of ' || g.rolname FROM pg_roles g WHERE g.rolname IN ('pg_write_server_files', 'pg_execute_server_program') AND pg_has_role(r.oid, g.oid, 'MEMBER') - UNION ALL SELECT o.why || CASE WHEN o.owner = r.oid THEN '' + UNION ALL SELECT o.why || CASE WHEN o.owner = l.oid THEN '' ELSE ' (as a member of ' || pg_get_userbyid(o.owner) || ')' END - FROM owned o WHERE pg_has_role(r.oid, o.owner, 'MEMBER') + FROM owned o WHERE pg_has_role(l.oid, o.owner, 'MEMBER') ) w(why) ORDER BY 1, 2 """ diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index aaa48b8f0e..168008dfa8 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -324,6 +324,41 @@ async def durable_then_unset(self: Any) -> int: assert (await _row()).state == "serving" +@pytest.mark.asyncio +@pytest.mark.parametrize("defect", ["trigger", "catalog"]) +async def test_a_defect_during_the_rescan_keeps_the_cell_attaching( + cell_stores, monkeypatch, capsys, defect: str +) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + role = f"it_rescan_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + real_scan = cell_attach.Run.scan + + async def scan_then_defect(self: Any, *args: Any) -> Any: + found = await real_scan(self, *args) + if self.mode == "enforced": + if defect == "trigger": + await pg.execute( + f"ALTER TABLE loom.scope_generations DISABLE TRIGGER {cell_triggers.INSERT_TRIGGER}" + ) + else: + await pg.execute(f"GRANT INSERT (org) ON loom.tenancy TO {role}") + return found + + monkeypatch.setattr(cell_attach.Run, "scan", scan_then_defect) + try: + assert await _attach(stores, network) == 1 + refused = capsys.readouterr().err + assert ("postgres triggers" if defect == "trigger" else f"postgres role {role}") in refused + assert await _mode() == "enforced" + assert (await _row()).state == "attaching" + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") + + @pytest.mark.asyncio async def test_a_second_run_is_refused_at_the_lock(cell_stores, capsys) -> None: stores, network = cell_stores, _Network() @@ -505,6 +540,58 @@ async def release() -> None: assert logged == ["the mode's lock is busy; retrying"] +@pytest.mark.asyncio +@pytest.mark.parametrize("membership", ["set", "admin"]) +async def test_a_login_can_lower_mode_through_a_noninherited_group( + cell_stores, membership: str +) -> None: + await _prepare(cell_stores, _Network()) + group = f"it_group_{uuid.uuid4().hex[:6]}" + login = f"it_login_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {group} NOLOGIN") + try: + await _admin(f"CREATE ROLE {login} LOGIN") + async with pg.dedicated() as conn: + await conn.execute(f"GRANT USAGE ON SCHEMA loom TO {group}") + await conn.execute(f"GRANT UPDATE (mode) ON loom.tenancy_mode TO {group}") + flags = ( + "ADMIN FALSE, INHERIT FALSE, SET TRUE" + if membership == "set" + else "ADMIN TRUE, INHERIT FALSE, SET FALSE" + ) + await conn.execute(f"GRANT {group} TO {login} WITH {flags}") + assert not await conn.fetchval( + "SELECT has_column_privilege($1, 'loom.tenancy_mode', 'mode', 'UPDATE')", login + ) + holders = await cell_attach_pg.privilege_holders(conn, exempt_self=True) + await conn.execute(f"SET SESSION AUTHORIZATION {login}") + if membership == "admin": + await conn.execute(f"GRANT {group} TO {login} WITH SET TRUE") + await conn.execute(f"SET ROLE {group}") + assert ( + await conn.execute("UPDATE loom.tenancy_mode SET mode = 'enforced'") == "UPDATE 1" + ) + assert ( + await conn.execute("UPDATE loom.tenancy_mode SET mode = 'recording'") == "UPDATE 1" + ) + await conn.execute("RESET SESSION AUTHORIZATION") + assert "may update loom.tenancy_mode (mode)" in holders.others.get(login, []) + assert login not in holders.own + assert any( + r.subject == f"postgres role {login}" + for r in cell_attach_pg.privilege_refusals(holders, []) + ) + assert not any( + r.subject == f"postgres role {login}" + for r in cell_attach_pg.privilege_refusals(holders, [login]) + ) + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {login}, {group}") + await _admin(f"DROP ROLE IF EXISTS {login}") + await _admin(f"DROP ROLE {group}") + + @pytest.mark.asyncio @pytest.mark.parametrize( "grant,why", From cb95fffcde72511d555dc055d5e3b0f1a3ccbb08 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 21:12:45 +0000 Subject: [PATCH 017/114] Require effective predefined role privileges before attach Check USAGE of pg_read_all_stats and pg_signal_backend because attach uses their privileges without SET ROLE. Keep MEMBER for the separate untrusted-writer reachability checks. Refuse noninherited capabilities before attach writes, and explain the inherited grant requirement in the refusal labels and runbook. Test both inherited neighbors and signaling a scratch backend. Signed-off-by: zlareb1 --- docs/operations/cell-attach.md | 3 +- server/cli/cell_attach_pg.py | 6 ++- tests/integration/test_cell_attach.py | 77 +++++++++++++++++++++++++++ tests/unit/test_cell_attach.py | 4 +- 4 files changed, 85 insertions(+), 5 deletions(-) diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index ec6b6354ec..5cf6f74350 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -55,7 +55,8 @@ Every item below is a precondition attach checks, or one it cannot check and rel `max_prepared_transactions = 0`; the database encoding is UTF8. 13. No org longer than 2048 bytes is in the cell's stores (below). `--dry-run` lists any. -Attach's Postgres role needs `pg_read_all_stats`, `pg_signal_backend`, `USAGE` on schema +Attach's Postgres role needs the privileges of `pg_read_all_stats` and `pg_signal_backend` +without `SET ROLE` (grant their memberships with `INHERIT TRUE`), `USAGE` on schema `loom`, `SELECT` on `loom.cell_first`, `loom.tenancy_mode`, `loom.cell_self`, `loom.authority` and every org table, `UPDATE` on `loom.tenancy_mode`, `INSERT` on `loom.tenancy`, `SELECT`, `INSERT`, `UPDATE` and `DELETE` on `loom.cell_probe`, `TEMPORARY` diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 56601e72e9..b8cc9e6e44 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -398,8 +398,10 @@ async def session_passes() -> bool: _OWN_GRANTS = """ SELECT need, held FROM (VALUES - ('membership in pg_read_all_stats', pg_has_role(current_user, 'pg_read_all_stats', 'MEMBER')), - ('membership in pg_signal_backend', pg_has_role(current_user, 'pg_signal_backend', 'MEMBER')), + ('effective privileges of pg_read_all_stats', + pg_has_role(current_user, 'pg_read_all_stats', 'USAGE')), + ('effective privileges of pg_signal_backend', + pg_has_role(current_user, 'pg_signal_backend', 'USAGE')), ('USAGE on schema loom', has_schema_privilege('loom', 'USAGE')), ('SELECT on loom.cell_first', has_table_privilege('loom.cell_first', 'SELECT')), ('SELECT on loom.tenancy_mode', has_table_privilege('loom.tenancy_mode', 'SELECT')), diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 168008dfa8..894635a4d9 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -771,6 +771,83 @@ async def pg_orgs(table: str, budget: Any) -> list[str]: assert grade == "WARN" and detail.endswith(": loom.it_large") +@pytest.mark.asyncio +async def test_signaling_another_role_requires_an_effective_signal_grant(cell_stores) -> None: + import asyncpg + + caller = f"it_signal_{uuid.uuid4().hex[:6]}" + target = f"it_target_{uuid.uuid4().hex[:6]}" + password = uuid.uuid4().hex + await _admin(f"CREATE ROLE {caller} LOGIN") + try: + await _admin(f"CREATE ROLE {target} LOGIN PASSWORD '{password}'") + victim = await asyncpg.connect(settings.pg_dsn, user=target, password=password) + try: + pid = await victim.fetchval("SELECT pg_backend_pid()") + async with pg.dedicated() as admin, pg.dedicated() as signal: + await admin.execute(f"GRANT pg_signal_backend TO {caller} WITH INHERIT FALSE") + await signal.execute(f"SET SESSION AUTHORIZATION {caller}") + with pytest.raises(asyncpg.InsufficientPrivilegeError): + await signal.fetchval("SELECT pg_terminate_backend($1)", pid) + await admin.execute(f"GRANT pg_signal_backend TO {caller} WITH INHERIT TRUE") + assert await signal.fetchval("SELECT pg_terminate_backend($1)", pid) + finally: + await victim.close() + finally: + await _admin(f"DROP ROLE {caller}") + await _admin(f"DROP ROLE IF EXISTS {target}") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("capability", ["pg_read_all_stats", "pg_signal_backend"]) +async def test_attach_needs_its_predefined_capabilities_without_set_role( + cell_stores, monkeypatch, capsys, capability: str +) -> None: + from contextlib import asynccontextmanager + + stores, network = cell_stores, _Network() + await _prepare(stores, network) + role = f"it_caps_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + real_dedicated = pg.dedicated + + @asynccontextmanager + async def as_attacher(**kwargs: Any) -> AsyncIterator[Any]: + async with real_dedicated(**kwargs) as conn: + await conn.execute(f"SET SESSION AUTHORIZATION {role}") + yield conn + + try: + async with real_dedicated() as conn: + owner = await conn.fetchval("SELECT current_user") + await conn.execute(f"GRANT {owner} TO {role} WITH INHERIT TRUE, SET FALSE") + await conn.execute(f"GRANT pg_read_all_stats, pg_signal_backend TO {role}") + await conn.execute(f"GRANT {capability} TO {role} WITH INHERIT FALSE") + assert await conn.fetchval("SELECT pg_has_role($1, $2, 'MEMBER')", role, capability) + assert not await conn.fetchval("SELECT pg_has_role($1, $2, 'USAGE')", role, capability) + before = await _row() + with monkeypatch.context() as own: + own.setattr(pg, "dedicated", as_attacher) + assert await _attach(stores, network) == 1 + assert ( + f"attach's role lacks effective privileges of {capability}" in capsys.readouterr().err + ) + after = await _row() + assert (after.state, after.mode, after.placed) == (before.state, before.mode, before.placed) + assert await _mode() == "recording" + assert await pg.fetchval("SELECT count(*) FROM loom.tenancy") == 0 + async with real_dedicated() as conn: + await conn.execute(f"GRANT {capability} TO {role} WITH INHERIT TRUE") + with monkeypatch.context() as own: + own.setattr(pg, "dedicated", as_attacher) + assert await _attach(stores, network) == 0, capsys.readouterr().err + assert (await _row()).state == "serving" + finally: + async with real_dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") + + @pytest.mark.asyncio async def test_attach_requires_every_probe_privilege(cell_stores) -> None: role = f"it_probe_{uuid.uuid4().hex[:6]}" diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 6d57cbb740..c0054d0c9e 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -572,7 +572,7 @@ async def test_own_role_lacks_and_prepared_transactions_refuse() -> None: conn = FakeConn( [ ("SELECT current_user", "attacher"), - ("WHERE NOT held", [{"need": "membership in pg_read_all_stats"}]), + ("WHERE NOT held", [{"need": "effective privileges of pg_read_all_stats"}]), ("has_table_privilege($1, 'SELECT')", lambda t: t != "loom.memory_current"), ("max_prepared_transactions", 5), ] @@ -586,7 +586,7 @@ async def test_own_role_lacks_and_prepared_transactions_refuse() -> None: ] refusals = await apg.own_role_refusals(conn, tables) assert [str(r) for r in refusals] == [ - "postgres role attacher: attach's role lacks membership in pg_read_all_stats", + "postgres role attacher: attach's role lacks effective privileges of pg_read_all_stats", "postgres role attacher: attach's role lacks SELECT on loom.memory_current", "postgres role attacher: attach's role lacks membership in the owner role of " "loom.scope_generations", From fde3798cfd2e226d53b20393274b801c58b44ddf Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 21:18:35 +0000 Subject: [PATCH 018/114] Describe the effective owner privileges attach requires Signed-off-by: zlareb1 --- docs/operations/cell-attach.md | 5 +++-- server/cli/cell_attach_pg.py | 9 +++++---- tests/integration/test_cell_attach.py | 5 ++++- tests/unit/test_cell_attach.py | 4 ++-- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index 5cf6f74350..e674da2997 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -60,8 +60,9 @@ without `SET ROLE` (grant their memberships with `INHERIT TRUE`), `USAGE` on sch `loom`, `SELECT` on `loom.cell_first`, `loom.tenancy_mode`, `loom.cell_self`, `loom.authority` and every org table, `UPDATE` on `loom.tenancy_mode`, `INSERT` on `loom.tenancy`, `SELECT`, `INSERT`, `UPDATE` and `DELETE` on `loom.cell_probe`, `TEMPORARY` -on the database, `EXECUTE` on `pg_logical_emit_message`, and membership in the owner role of -`loom.tenancy` and of every guarded table. Its ClickHouse user needs the grants the writer +on the database, `EXECUTE` on `pg_logical_emit_message`, and effective privileges of the owner +role of `loom.tenancy` and every guarded table without `SET ROLE` (grant their memberships +with `INHERIT TRUE`). Its ClickHouse user needs the grants the writer check names (`SELECT` on `system.*` and on the configured databases, `SHOW USERS`, `SHOW ROLES`, `SHOW ROW POLICIES`, `SHOW TABLES`, `SHOW COLUMNS`, `SYSTEM FLUSH ASYNC INSERT QUEUE`, and on a cluster `SYSTEM SYNC REPLICA`, `CLUSTER` and diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index b8cc9e6e44..12e1808f8b 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -422,8 +422,8 @@ async def session_passes() -> bool: async def own_role_refusals(conn: Any, tables: list[Any]) -> list[Refusal]: """What attach's own role lacks, `tables` being the rows `cell_triggers.ORG_TABLES` - read: its grants, `SELECT` on every org table, and membership in the owner role of every - table it gives triggers (or the trigger pass could not alter it); and a server that + read: its grants, `SELECT` on every org table, and effective owner-role privileges for + every table it gives triggers (or the trigger pass could not alter it); and a server that allows prepared transactions.""" role = await conn.fetchval("SELECT current_user") lacks = [str(r["need"]) for r in await conn.fetch(_OWN_GRANTS)] @@ -435,7 +435,7 @@ async def own_role_refusals(conn: Any, tables: list[Any]) -> list[Refusal]: planned = set(cell_triggers.planned_names(tables)) for r in tables: if r["relname"] in planned and not r["owned"]: - lacks.append(f"membership in the owner role of loom.{r['relname']}") + lacks.append(f"effective privileges of the owner role of loom.{r['relname']}") out = [Refusal(f"postgres role {role}", f"attach's role lacks {n}") for n in lacks] out += await prepared_refusals(conn) return out @@ -495,7 +495,8 @@ async def _ensure_table( if outcome.answer == "denied": return Refusal( f"postgres loom.{table}", - "attach's role cannot alter it: make the role a member of its owner role", + "attach's role cannot alter it: grant inherited privileges of its owner role " + "without SET ROLE", ) if not await wait.pause(): if outcome.holders.startswith(_MIGRATION_HELD): diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 894635a4d9..3450558fe4 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -902,7 +902,10 @@ async def test_a_role_with_the_grants_but_not_the_owner_role_is_refused_before_a rows = list(await conn.fetch(cell_triggers.ORG_TABLES)) refusals = await cell_attach_pg.own_role_refusals(conn, rows) lacking = [r.reason for r in refusals if "owner role" in r.reason] - assert "attach's role lacks membership in the owner role of loom.tenancy" in lacking + assert ( + "attach's role lacks effective privileges of the owner role of loom.tenancy" + in lacking + ) assert any("loom.scope_generations" in r for r in lacking) # The trigger pass tells the cause apart: denied, not a busy lock. outcome = await cell_triggers.ensure_table( diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index c0054d0c9e..0ecd3a96b3 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -579,7 +579,7 @@ async def test_own_role_lacks_and_prepared_transactions_refuse() -> None: ) tables = [ {"relname": "memory_current", "relkind": "r", "owned": True}, - # Its grants are held, but not its owner role's membership: the trigger pass could + # Its grants are held, but not its effective owner-role privileges: the pass could # not alter it, so it refuses before any write. {"relname": "scope_generations", "relkind": "r", "owned": False}, {"relname": "tenancy", "relkind": "r", "owned": True}, @@ -588,7 +588,7 @@ async def test_own_role_lacks_and_prepared_transactions_refuse() -> None: assert [str(r) for r in refusals] == [ "postgres role attacher: attach's role lacks effective privileges of pg_read_all_stats", "postgres role attacher: attach's role lacks SELECT on loom.memory_current", - "postgres role attacher: attach's role lacks membership in the owner role of " + "postgres role attacher: attach's role lacks effective privileges of the owner role of " "loom.scope_generations", "postgres max_prepared_transactions: is 5: a prepared transaction outlives every " "session, so step 4 could not wait for it; set it to 0", From cad839bdf5051753a10c7f162e21e104f7b17ef4 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 21:24:41 +0000 Subject: [PATCH 019/114] Build placement indexes outside migration transactions Concurrent index builds release the ordinary migration key so a shipped blocking migrator can finish without retaining a snapshot behind the build. Retry from the catalogs and checksum ledger, and acknowledge a WAL fence on recorded reruns. Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 29 ++ docs/development/migrations.md | 54 ++- .../004_placements_by_cell_org.sql | 3 + server/core/schema/pg_migrate.py | 154 ++++++-- server/core/schema/pg_online_index.py | 125 ++++++ .../fixtures/pg_migrate_n_minus_one.py | 151 +++++++ tests/integration/test_pg_online_index.py | 373 ++++++++++++++++++ .../test_pg_online_index_durability.py | 203 ++++++++++ tests/unit/test_cell_org_triggers.py | 8 + tests/unit/test_pg_online_index.py | 212 ++++++++++ 10 files changed, 1275 insertions(+), 37 deletions(-) create mode 100644 schemas/router_migrations/004_placements_by_cell_org.sql create mode 100644 server/core/schema/pg_online_index.py create mode 100644 tests/integration/fixtures/pg_migrate_n_minus_one.py create mode 100644 tests/integration/test_pg_online_index.py create mode 100644 tests/integration/test_pg_online_index_durability.py create mode 100644 tests/unit/test_pg_online_index.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22342c2cab..a132706be6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -896,10 +896,39 @@ jobs: tests/integration/test_pg_clickhouse_live_ch.py tests/integration/test_cell_tenancy_pg.py tests/integration/test_router_directory_pg.py + tests/integration/test_pg_online_index.py tests/integration/test_cell_unset_gate_pg.py tests/integration/test_cell_prepare.py tests/integration/test_cell_attach.py tests/integration/test_cell_org_triggers_pg.py + - name: pytest online-index durability on a private physical standby + timeout-minutes: 8 + env: + LOOM_PG_IMAGE: postgres:${{ matrix.postgres }} + LOOM_PG_CONTAINER: loom-scale-index-drill-pg + LOOM_PG_HOST_PORT: "55432" + LOOM_PG_INDEX_DRILL: "1" + LOOM_PG_INDEX_DRILL_DSN: postgres://loom:loomdev@localhost:55432/loom + run: | + set -euo pipefail + for name in loom-scale-index-drill-pg loom-scale-index-drill-pg-standby; do + if docker container inspect "$name" >/dev/null 2>&1; then + echo "drill container already exists: $name" >&2 + exit 1 + fi + done + if docker network inspect loom-scale-index-drill-pg-net >/dev/null 2>&1; then + echo "drill network already exists" >&2 + exit 1 + fi + cleanup() { + docker rm -f -v loom-scale-index-drill-pg loom-scale-index-drill-pg-standby || true + docker network rm loom-scale-index-drill-pg-net || true + } + trap cleanup EXIT + bash dev/ci_start_postgres.sh + docker update --memory 1g --memory-swap 1g loom-scale-index-drill-pg loom-scale-index-drill-pg-standby + pytest -q tests/integration/test_pg_online_index_durability.py # The tenant-isolation gates again, against a second server that enforces auth: # the Postgres read paths (heads by key and by id, narrowed scopes) see real tokens. # They run whether or not the memory suites passed (`!cancelled()`). diff --git a/docs/development/migrations.md b/docs/development/migrations.md index bbea754e87..0fcc560a45 100644 --- a/docs/development/migrations.md +++ b/docs/development/migrations.md @@ -157,20 +157,56 @@ operator step ([release process](./release-process.md#release-notes)). edited once merged, and every `CREATE`, `ALTER` or `DROP` of a table names `loom.
`, except a rename's new name, where Postgres takes no schema (`tests/unit/test_storage_layout_rows.py`). A boot applies them under a Postgres -advisory lock, each file in one transaction with its ledger row. A migration waits at +advisory lock, each ordinary file in one transaction with its ledger row. A migration waits at most 5 s for a table lock (`lock_timeout`, then retried), so an `ALTER` never queues every query of the table behind a long transaction. A build serves a ledger that holds a later build's files, with a warning naming them; a ledger file it does not carry numbered at or below its own last file, or one whose checksum changed, refuses the boot. -Every file runs in one transaction, so a migration builds no index on a large table: the -build would hold off the table's writes, every commit answering 503, until it ends. Such -an index needs `CREATE INDEX CONCURRENTLY` outside a transaction, and a boot waiting on -the advisory lock deadlocks with it (a waiting session holds a snapshot the build waits -out): a change that adds such a file also takes the lock by retrying -`pg_try_advisory_lock`. When the first release that writes a table carries the file, the -table is small wherever the file runs, and its index may be built in the file's -transaction (`010_revision_delivered_at.sql`). +Ordinary files run in one transaction, so they build no index on a table that +already ships: a transactional build blocks writes until it ends. An index on a +table introduced by the same release may be transactional because that table is +small wherever the file runs (`010_revision_delivered_at.sql`). + +For an existing table, the shared Postgres runner accepts only this online file +form, in either the cell or router journal: + +```sql +-- loom:online-index loom_router.placements_by_cell_org ON loom_router.placements (cell, org) +CREATE INDEX CONCURRENTLY placements_by_cell_org ON loom_router.placements +USING btree (cell COLLATE "C", org COLLATE "C"); +``` + +The declaration and statement must name the same schema, table, index and ordered +columns. Only ordinary identifiers, a non-unique btree and C collations are +accepted. Expressions, predicates, included columns and additional statements +are refused before the ledger changes. This is not a general escape from +transactional migrations. + +The runner takes session advisory locks by retrying `pg_try_advisory_lock`, with +no transaction or snapshot between attempts. It releases the ordinary migration +key before acquiring a separate schema-scoped online-index key and running +concurrent DDL. A shipped blocking migrator can therefore finish its unchanged +journal while the index waits for old snapshots. After the build finishes, the +runner reacquires the ordinary key, records the file's checksum in the existing +ledger and commits a WAL-message fence. Later files wait for this ledger entry. + +Catalog validation checks the table, ordered columns and collations, btree +method, absence of constraints and expressions, and valid/ready state. A +matching incomplete index is dropped concurrently and rebuilt; a different +same-named object refuses without being dropped. A recorded but absent or +invalid index refuses as corruption. Even when every file is recorded, a build +carrying an online file validates its index and acknowledges a fresh WAL fence: +a primary-visible ledger row after a cancelled synchronous COMMIT is insufficient. +An interrupted build, drop or ledger write is retried from catalogs and the +existing ledger, without a separate job table. Cancellation terminates the +migration session before asyncpg can wait indefinitely on rollback. + +The router's dedicated migration connection has no statement timeout; its +serving pool retains its statement limit. The runner preserves a borrowed +connection's statement budget, including a stopped restore command's deadline. +Concurrent DDL retains the five-second lock timeout and can be interrupted and +retried without changing the file checksum. ## Schema dump and check diff --git a/schemas/router_migrations/004_placements_by_cell_org.sql b/schemas/router_migrations/004_placements_by_cell_org.sql new file mode 100644 index 0000000000..774efeb883 --- /dev/null +++ b/schemas/router_migrations/004_placements_by_cell_org.sql @@ -0,0 +1,3 @@ +-- loom:online-index loom_router.placements_by_cell_org ON loom_router.placements (cell, org) +CREATE INDEX CONCURRENTLY placements_by_cell_org ON loom_router.placements +USING btree (cell COLLATE "C", org COLLATE "C"); diff --git a/server/core/schema/pg_migrate.py b/server/core/schema/pg_migrate.py index 7876c9f178..ebd230036b 100644 --- a/server/core/schema/pg_migrate.py +++ b/server/core/schema/pg_migrate.py @@ -1,6 +1,7 @@ """Postgres migrations: numbered SQL files in `schemas/pg_migrations/`, applied in -order at boot under a session advisory lock, each file in one transaction with -its ledger row (`loom.pg_migrations`, the file's SHA-256). The router's directory +order at boot under a session advisory lock, each ordinary file in a transaction +with its ledger row (`loom.pg_migrations`, the file's SHA-256). Declared online indexes +run concurrently outside transactions under a separate session key. The router's directory (`schemas/router_migrations/`, `server/router/directory.py`) migrates the same way, on its own database, ledger schema and lock. For schema `loom`, `migrate` also marks its session as one that ran the tenancy check and, after the files and before the lock @@ -8,13 +9,14 @@ The rules are the ClickHouse journal's (docs/development/migrations.md): additive only, never edit a merged file. A ledger row whose checksum no longer matches its -file refuses the boot, as does a ledger row for a file this build does not carry -when that file's number is above the build's last one (a newer build's schema is -served, as ClickHouse's is; an unknown lower number means the journal was edited). +file refuses the boot, as does an unknown file numbered at or below this build's +last migration. A build serves a newer schema, warning about unknown files numbered +above its last migration. """ from __future__ import annotations +import asyncio import hashlib import logging from asyncio import sleep @@ -24,7 +26,9 @@ from typing import Any from server.core import pg +from server.core.ch import set_query_tag from server.core.paths import PG_MIGRATIONS_DIR +from server.core.schema import pg_online_index from server.core.schema.journal import migration_number, sorted_migrations log = logging.getLogger(__name__) @@ -105,32 +109,126 @@ async def migrate( holds every read and write of the table behind it; the file is retried. For schema `loom` it marks the session and runs `cell_triggers.ensure` before the lock goes.""" files = migration_files(mig_dir) - done: list[str] = [] - # Closing the connection ends the session, and with it the advisory lock. + # Validate every online declaration before even creating the ledger. + indexes = {f.name: pg_online_index.parse(f.read_text()) for f in files} + if any(index is not None and index.schema != schema for index in indexes.values()): + raise ValueError("online-index target differs from its ledger schema") async with (connect or pg.dedicated)() as conn: - # Another boot migrating holds the lock for as long as its files take. - await conn.execute("SELECT pg_advisory_lock($1)", lock_key) - if schema == "loom": - # A migration's data steps only copy or rewrite rows already stored here: the - # org triggers pass them (server/core/cell_triggers.py). - await conn.execute(f"SET {pg.TENANCY_MARKER} = 'on'") - await conn.execute("SET lock_timeout = '5s'") - await conn.execute(_ledger(schema)) - rows = await conn.fetch(f"SELECT filename, checksum FROM {schema}.pg_migrations") - for name, sql, digest in _read(plan({r["filename"]: r["checksum"] for r in rows}, files)): - await _apply(conn, name, sql, digest, schema=schema) - log.info("applied Postgres migration %s", name) - done.append(name) - if schema == "loom": - # On every migrating boot, after the pending files and before the lock goes. - from server.core import cell_triggers - - await cell_triggers.ensure(conn) - return done + task = asyncio.create_task(_migrate(conn, files, indexes, schema, lock_key)) + try: + return await asyncio.shield(task) + except BaseException: + # Terminate before cancellation can enter asyncpg's unbounded rollback. + conn.terminate() + task.cancel() + await asyncio.wait({task}, timeout=1) + if task.done() and not task.cancelled(): + task.exception() + raise + + +async def _lock(conn: Any, key: int) -> None: + while True: + set_query_tag("migrate.postgres.lock") + if await conn.fetchval("SELECT pg_try_advisory_lock($1)", key): + return + # Each try ends its statement/snapshot before another attempt. + await sleep(0.1) + + +async def _unlock(conn: Any, key: int) -> None: + if not await conn.fetchval("SELECT pg_advisory_unlock($1)", key): + raise RuntimeError("Postgres migration advisory lock was lost") -def _read(files: list[Path]) -> list[tuple[str, str, str]]: - return [(f.name, f.read_text(), checksum(f)) for f in files] +def _online_key(schema: str, lock_key: int) -> int: + value = hashlib.sha256(f"loom.online-index:{schema}:{lock_key}".encode()).digest() + key = int.from_bytes(value[:8], "big") & (2**63 - 1) + if key == lock_key: + raise ValueError("Postgres migration lock keys collide") + return key + + +async def _applied(conn: Any, schema: str, files: list[Path]) -> dict[str, str]: + rows = await conn.fetch(f"SELECT filename, checksum FROM {schema}.pg_migrations") + applied = {r["filename"]: r["checksum"] for r in rows} + plan(applied, files) + return applied + + +async def _record_online(conn: Any, path: Path, schema: str, *, pending: bool) -> None: + async with conn.transaction(): + if pending: + await conn.execute( + f"INSERT INTO {schema}.pg_migrations (filename, checksum) VALUES ($1, $2)", + path.name, + checksum(path), + ) + # A visible ledger from a cancelled synchronous commit needs a fresh flush. + await conn.execute(pg._DURABLE_MESSAGE) + + +async def _online( + conn: Any, + path: Path, + index: pg_online_index.OnlineIndex, + schema: str, + lock_key: int, + files: list[Path], +) -> bool: + await _unlock(conn, lock_key) + online_key = _online_key(schema, lock_key) + await _lock(conn, online_key) + applied = await _applied(conn, schema, files) + await pg_online_index.build(conn, index, recorded=path.name in applied) + # A shipped blocking migrator may wait here; the concurrent build is over. + await _lock(conn, lock_key) + applied = await _applied(conn, schema, files) + pending = path.name not in applied + await _record_online(conn, path, schema, pending=pending) + await _unlock(conn, online_key) + return pending + + +async def _migrate( + conn: Any, + files: list[Path], + indexes: dict[str, pg_online_index.OnlineIndex | None], + schema: str, + lock_key: int, +) -> list[str]: + done: list[str] = [] + await _lock(conn, lock_key) + if schema == "loom": + await conn.execute(f"SET {pg.TENANCY_MARKER} = 'on'") + await conn.execute("SET lock_timeout = '5s'") + await conn.execute("SET synchronous_commit = 'on'") + await conn.execute(_ledger(schema)) + applied = await _applied(conn, schema, files) + for path in files: + set_query_tag("migrate.postgres.apply") + index = indexes[path.name] + if index is not None: + changed = await _online(conn, path, index, schema, lock_key, files) + applied = await _applied(conn, schema, files) + else: + changed = path.name not in applied + if changed: + await _apply(conn, path.name, path.read_text(), checksum(path), schema=schema) + if changed: + log.info("applied Postgres migration %s", path.name) + done.append(path.name) + for index in indexes.values(): + set_query_tag("migrate.postgres.verify") + if index is not None and await pg_online_index.ready(conn, index) is not True: + raise ValueError(f"recorded online-index {index.qualified} is absent or invalid") + if schema == "loom": + from server.core import cell_triggers + + await cell_triggers.ensure(conn) + # A caller may lend its marked session and retain unrelated session keys. + await _unlock(conn, lock_key) + return done async def _apply(conn: Any, name: str, sql: str, digest: str, *, schema: str = "loom") -> None: diff --git a/server/core/schema/pg_online_index.py b/server/core/schema/pg_online_index.py new file mode 100644 index 0000000000..47c2b025a5 --- /dev/null +++ b/server/core/schema/pg_online_index.py @@ -0,0 +1,125 @@ +"""The one nontransactional Postgres migration form: a declared concurrent index.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any + +_ID = r"[a-z_][a-z0-9_]{0,62}" +_DECLARATION = re.compile( + rf"-- loom:online-index (?P{_ID})\.(?P{_ID}) ON " + rf"(?P{_ID})\.(?P
{_ID}) \((?P{_ID}(?:, {_ID})*)\)\n" +) +_CREATE = re.compile( + rf"CREATE\s+INDEX\s+CONCURRENTLY\s+(?P{_ID})\s+ON\s+" + rf"(?P{_ID})\.(?P
{_ID})\s+USING\s+btree\s*" + rf'\((?P{_ID}\s+COLLATE\s+"C"(?:\s*,\s*{_ID}\s+COLLATE\s+"C")*)\)\s*;\s*' +) + + +@dataclass(frozen=True) +class OnlineIndex: + schema: str + name: str + table: str + columns: tuple[str, ...] + sql: str + + @property + def qualified(self) -> str: + return f'"{self.schema}"."{self.name}"' + + +def parse(sql: str) -> OnlineIndex | None: + """Refuse malformed directives before a connection or ledger write is made.""" + if "loom:online-index" not in sql: + return None + declaration = _DECLARATION.match(sql) + create = _CREATE.fullmatch(sql[declaration.end() :]) if declaration else None + if declaration is None or create is None: + raise ValueError("invalid online-index migration declaration or SQL") + columns = tuple(declaration["columns"].split(", ")) + actual = tuple(part.split()[0] for part in create["columns"].split(",")) + if ( + declaration["schema"] != declaration["table_schema"] + or any(declaration[key] != create[key] for key in ("schema", "name", "table")) + or columns != actual + or len(set(columns)) != len(columns) + ): + raise ValueError("online-index SQL does not match its declaration") + return OnlineIndex( + declaration["schema"], + declaration["name"], + declaration["table"], + columns, + sql[declaration.end() :], + ) + + +_CATALOG = """ +SELECT c.relkind::text AS relkind, tn.nspname AS table_schema, t.relname AS table_name, + am.amname, i.indisvalid, i.indisready, i.indislive, + NOT i.indisunique AND NOT i.indisexclusion AND i.indpred IS NULL + AND i.indexprs IS NULL AND i.indnkeyatts = i.indnatts + AND NOT i.indisreplident AND c.reloptions IS NULL AS plain, + ARRAY(SELECT a.attname::text FROM unnest(i.indkey) WITH ORDINALITY k(attnum, n) + JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = k.attnum + ORDER BY k.n) AS columns, + ARRAY(SELECT co.collname::text FROM unnest(i.indcollation) WITH ORDINALITY k(oid, n) + JOIN pg_collation co ON co.oid = k.oid + JOIN pg_namespace ns ON ns.oid = co.collnamespace AND ns.nspname = 'pg_catalog' + ORDER BY k.n) AS collations, + ARRAY(SELECT v FROM unnest(i.indoption) v) AS options, + ARRAY(SELECT op.opcdefault FROM unnest(i.indclass) WITH ORDINALITY k(oid, n) + JOIN pg_opclass op ON op.oid = k.oid ORDER BY k.n) AS default_opclasses, + EXISTS(SELECT 1 FROM pg_constraint WHERE conindid = c.oid) + OR EXISTS(SELECT 1 FROM pg_depend WHERE classid = 'pg_class'::regclass + AND objid = c.oid AND deptype IN ('e', 'x')) + OR EXISTS(SELECT 1 FROM pg_inherits WHERE inhrelid = c.oid) AS dependent +FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace +LEFT JOIN pg_index i ON i.indexrelid = c.oid +LEFT JOIN pg_class t ON t.oid = i.indrelid +LEFT JOIN pg_namespace tn ON tn.oid = t.relnamespace +LEFT JOIN pg_am am ON am.oid = c.relam +WHERE n.nspname = $1 AND c.relname = $2 +""" + + +async def ready(conn: Any, index: OnlineIndex) -> bool | None: + """Absent, matching incomplete, or matching ready; foreign objects always refuse.""" + row = await conn.fetchrow(_CATALOG, index.schema, index.name) + if row is None: + return None + expected = { + "relkind": "i", + "table_schema": index.schema, + "table_name": index.table, + "amname": "btree", + "plain": True, + "columns": list(index.columns), + "collations": ["C"] * len(index.columns), + "options": [0] * len(index.columns), + "default_opclasses": [True] * len(index.columns), + "dependent": False, + } + differences = {key: row[key] for key, value in expected.items() if row[key] != value} + if differences: + raise ValueError( + f"online-index {index.qualified} has a different definition or dependency: " + f"{differences}" + ) + return bool(row["indisvalid"] and row["indisready"] and row["indislive"]) + + +async def build(conn: Any, index: OnlineIndex, *, recorded: bool) -> None: + state = await ready(conn, index) + if recorded and state is not True: + raise ValueError(f"recorded online-index {index.qualified} is absent or invalid") + if state is True: + return + if state is False: + await conn.execute(f"DROP INDEX CONCURRENTLY {index.qualified}") + await conn.execute(index.sql) + if await ready(conn, index) is not True: + raise ValueError(f"online-index {index.qualified} did not become valid and ready") diff --git a/tests/integration/fixtures/pg_migrate_n_minus_one.py b/tests/integration/fixtures/pg_migrate_n_minus_one.py new file mode 100644 index 0000000000..7876c9f178 --- /dev/null +++ b/tests/integration/fixtures/pg_migrate_n_minus_one.py @@ -0,0 +1,151 @@ +"""Postgres migrations: numbered SQL files in `schemas/pg_migrations/`, applied in +order at boot under a session advisory lock, each file in one transaction with +its ledger row (`loom.pg_migrations`, the file's SHA-256). The router's directory +(`schemas/router_migrations/`, `server/router/directory.py`) migrates the same way, +on its own database, ledger schema and lock. For schema `loom`, `migrate` also marks +its session as one that ran the tenancy check and, after the files and before the lock +goes, runs `cell_triggers.ensure` (docs/architecture/cells.md). + +The rules are the ClickHouse journal's (docs/development/migrations.md): additive +only, never edit a merged file. A ledger row whose checksum no longer matches its +file refuses the boot, as does a ledger row for a file this build does not carry +when that file's number is above the build's last one (a newer build's schema is +served, as ClickHouse's is; an unknown lower number means the journal was edited). +""" + +from __future__ import annotations + +import hashlib +import logging +from asyncio import sleep +from collections.abc import Callable +from contextlib import AbstractAsyncContextManager +from pathlib import Path +from typing import Any + +from server.core import pg +from server.core.paths import PG_MIGRATIONS_DIR +from server.core.schema.journal import migration_number, sorted_migrations + +log = logging.getLogger(__name__) + +# A fixed key for pg_advisory_lock: every Loom process migrates under it. +_LOCK_KEY = 0x6C6F6F6D_6D696772 # "loommigr" + + +def _ledger(schema: str) -> str: + # `schema` is one of this build's constants (`loom`, `loom_router`), never input. + return f""" +CREATE SCHEMA IF NOT EXISTS {schema}; +CREATE TABLE IF NOT EXISTS {schema}.pg_migrations ( + filename text PRIMARY KEY, + checksum text NOT NULL, + applied_at timestamptz NOT NULL DEFAULT now() +)""" + + +class MigrationDrift(RuntimeError): + """An applied file changed, or the ledger holds a file the journal lost.""" + + +def migration_files(mig_dir: Path = PG_MIGRATIONS_DIR) -> list[Path]: + return sorted_migrations(mig_dir.glob("*.sql")) + + +def checksum(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def plan(applied: dict[str, str], files: list[Path]) -> list[Path]: + """The files still to apply; raises MigrationDrift on a changed or lost file.""" + known = {f.name: f for f in files} + last = max((migration_number(f.name) or 0 for f in files), default=0) + for name, digest in applied.items(): + f = known.get(name) + if f is None: + if (migration_number(name) or 0) <= last: + raise MigrationDrift(f"applied Postgres migration {name} is not in this build") + continue + if checksum(f) != digest: + raise MigrationDrift(f"applied Postgres migration {name} was edited") + newer = sorted(n for n in applied if n not in known) + if newer: + log.warning("Postgres schema has migrations this build does not carry: %s", newer) + return [f for f in files if f.name not in applied] + + +async def pending(mig_dir: Path = PG_MIGRATIONS_DIR) -> list[str]: + """The files `migrate` would apply, read-only: every file when the ledger is not + there yet. For a process that applies none (`LOOM_BOOT_MIGRATE=0`).""" + import asyncpg + + try: + rows = await pg.fetch("SELECT filename, checksum FROM loom.pg_migrations") + except asyncpg.exceptions.UndefinedTableError: + rows = [] + applied = {r["filename"]: r["checksum"] for r in rows} + return [f.name for f in plan(applied, migration_files(mig_dir))] + + +async def migrate( + mig_dir: Path = PG_MIGRATIONS_DIR, + *, + connect: Callable[[], AbstractAsyncContextManager[Any]] | None = None, + schema: str = "loom", + lock_key: int = _LOCK_KEY, +) -> list[str]: + """Apply every pending file; returns their names. Concurrent boots serialize on + the advisory lock and the later ones find nothing pending. `connect`, `schema` + and `lock_key` name another database's connection, ledger and lock (the router's + directory); the defaults are the memory authority's. + + On a connection of its own, with no client command timeout and no statement + timeout: DDL on a large table runs as long as it takes. A lock it waits for is + bounded (`lock_timeout`), so an `ALTER` queued behind a long transaction never + holds every read and write of the table behind it; the file is retried. For schema + `loom` it marks the session and runs `cell_triggers.ensure` before the lock goes.""" + files = migration_files(mig_dir) + done: list[str] = [] + # Closing the connection ends the session, and with it the advisory lock. + async with (connect or pg.dedicated)() as conn: + # Another boot migrating holds the lock for as long as its files take. + await conn.execute("SELECT pg_advisory_lock($1)", lock_key) + if schema == "loom": + # A migration's data steps only copy or rewrite rows already stored here: the + # org triggers pass them (server/core/cell_triggers.py). + await conn.execute(f"SET {pg.TENANCY_MARKER} = 'on'") + await conn.execute("SET lock_timeout = '5s'") + await conn.execute(_ledger(schema)) + rows = await conn.fetch(f"SELECT filename, checksum FROM {schema}.pg_migrations") + for name, sql, digest in _read(plan({r["filename"]: r["checksum"] for r in rows}, files)): + await _apply(conn, name, sql, digest, schema=schema) + log.info("applied Postgres migration %s", name) + done.append(name) + if schema == "loom": + # On every migrating boot, after the pending files and before the lock goes. + from server.core import cell_triggers + + await cell_triggers.ensure(conn) + return done + + +def _read(files: list[Path]) -> list[tuple[str, str, str]]: + return [(f.name, f.read_text(), checksum(f)) for f in files] + + +async def _apply(conn: Any, name: str, sql: str, digest: str, *, schema: str = "loom") -> None: + import asyncpg + + record = f"INSERT INTO {schema}.pg_migrations (filename, checksum) VALUES ($1, $2)" + attempts = 5 # tries before the boot fails on a table lock held past `lock_timeout` + for attempt in range(1, attempts + 1): + try: + async with conn.transaction(): + await conn.execute(sql) + await conn.execute(record, name, digest) + return + except asyncpg.exceptions.LockNotAvailableError: + if attempt == attempts: + raise + log.warning("Postgres migration %s waited too long for a lock; retrying", name) + await sleep(attempt) diff --git a/tests/integration/test_pg_online_index.py b/tests/integration/test_pg_online_index.py new file mode 100644 index 0000000000..af93dd0937 --- /dev/null +++ b/tests/integration/test_pg_online_index.py @@ -0,0 +1,373 @@ +"""Online index retries and migrator coexistence on scratch Postgres databases.""" + +from __future__ import annotations + +import asyncio +import contextlib +import os +from collections.abc import AsyncIterator +from pathlib import Path +from typing import Any + +import pytest + +from server.core.paths import ROUTER_MIGRATIONS_DIR +from server.core.schema import pg_migrate, pg_online_index +from server.router import directory +from server.settings import settings +from tests.integration._pg_database import create_database, drop_database + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +async def database(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[tuple[Any, Any]]: + import asyncpg + + admin = os.environ.get("LOOM_PG_DSN") or pytest.skip("LOOM_PG_DSN not set") + name, dsn = await create_database(admin) + monkeypatch.setattr(settings, "router_pg_dsn", dsn) + monkeypatch.setattr(directory, "_pool", None) + + @contextlib.asynccontextmanager + async def connect() -> AsyncIterator[Any]: + conn = await asyncpg.connect(dsn, server_settings={"application_name": "online-index-test"}) + try: + yield conn + finally: + await conn.close() + + try: + async with connect() as conn: + yield conn, connect + finally: + await directory.close() + await drop_database(admin, name) + + +def _prefix(tmp_path: Path) -> Path: + prefix = tmp_path / "old" + prefix.mkdir() + for source in sorted(ROUTER_MIGRATIONS_DIR.glob("*.sql"))[:3]: + (prefix / source.name).write_bytes(source.read_bytes()) + return prefix + + +async def _migrate(connect: Any, files: Path = ROUTER_MIGRATIONS_DIR) -> list[str]: + return await pg_migrate.migrate(files, connect=connect, schema="loom_router", lock_key=8917423) + + +async def test_online_migration_installs_and_retries_from_both_schemas(database, tmp_path) -> None: + conn, connect = database + await _migrate(connect, _prefix(tmp_path)) + assert await _migrate(connect) == ["004_placements_by_cell_org.sql"] + assert await _migrate(connect) == [] + index = pg_online_index.parse( + (ROUTER_MIGRATIONS_DIR / "004_placements_by_cell_org.sql").read_text() + ) + assert index is not None and await pg_online_index.ready(conn, index) is True + assert await conn.fetchval("SELECT to_regclass('loom_router.placements_by_cell') IS NOT NULL") + + +async def test_install_from_empty(database) -> None: + _, connect = database + assert len(await _migrate(connect)) == 4 + + +async def _until(conn: Any, sql: str, *args: Any) -> None: + async with asyncio.timeout(8): + for _ in range(800): + if await conn.fetchval(sql, *args): + return + await asyncio.sleep(0.01) + pytest.fail("Postgres did not reach the expected migration state") + + +def _old_runner(): + """The shipped blocking runner, kept as source for the rolling-upgrade proof.""" + import importlib.util + + path = Path(__file__).parent / "fixtures" / "pg_migrate_n_minus_one.py" + spec = importlib.util.spec_from_file_location("old_pg_migrate", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +async def test_old_snapshot_old_blocking_migrator_and_two_new_migrators(database, tmp_path) -> None: + conn, connect = database + old_files = _prefix(tmp_path) + await _migrate(connect, old_files) + async with connect() as snapshot: + tx = snapshot.transaction(isolation="repeatable_read") + await tx.start() + await snapshot.fetchval("SELECT count(*) FROM loom_router.placements") + first = asyncio.create_task(_migrate(connect)) + second = None + try: + await _until( + conn, + "SELECT EXISTS(SELECT 1 FROM pg_stat_progress_create_index " + "WHERE datid = (SELECT oid FROM pg_database WHERE datname = current_database()) " + "AND phase = 'waiting for old snapshots')", + ) + second = asyncio.create_task(_migrate(connect)) + # The ordinary key is free while CREATE INDEX waits for this old snapshot. + # Running this original blocking implementation would deadlock if it weren't. + old = _old_runner() + async with asyncio.timeout(2): + assert ( + await old.migrate( + old_files, connect=connect, schema="loom_router", lock_key=8917423 + ) + == [] + ) + assert not first.done() and not second.done() + # Even while the index waits, the old serving path can write and read. + await conn.execute( + "INSERT INTO loom_router.cells " + "(cell, host, state, credential_sha256, deployment_id) " + "VALUES ('a', 'a.example.test', 'serving', repeat('a', 64), 'dep-a')" + ) + assert await directory.place("org-a", "a", lambda _: "") == ("a", "dep-a") + assert await directory.lookup("org-a") == ("a", "a.example.test") + await tx.rollback() + async with asyncio.timeout(5): + results = await asyncio.gather(first, second) + assert sorted(results) == [[], ["004_placements_by_cell_org.sql"]] + finally: + if snapshot.is_in_transaction(): + await tx.rollback() + for task in (first, second): + if task is not None and not task.done(): + task.cancel() + await asyncio.gather( + *(t for t in (first, second) if t is not None), return_exceptions=True + ) + + +async def test_cancelled_build_leaves_a_matching_invalid_index_and_retry_repairs_it( + database, tmp_path +) -> None: + conn, connect = database + await _migrate(connect, _prefix(tmp_path)) + async with connect() as snapshot: + tx = snapshot.transaction(isolation="repeatable_read") + await tx.start() + await snapshot.fetchval("SELECT count(*) FROM loom_router.placements") + migrating = asyncio.create_task(_migrate(connect)) + try: + await _until( + conn, + "SELECT EXISTS(SELECT 1 FROM pg_stat_progress_create_index " + "WHERE datid = (SELECT oid FROM pg_database WHERE datname = current_database()) " + "AND phase = 'waiting for old snapshots')", + ) + migrating.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(migrating, 2) + assert await conn.fetchval( + "SELECT NOT indisvalid FROM pg_index " + "WHERE indexrelid = 'loom_router.placements_by_cell_org'::regclass" + ) + assert not await conn.fetchval( + "SELECT EXISTS(SELECT 1 FROM loom_router.pg_migrations " + "WHERE filename = '004_placements_by_cell_org.sql')" + ) + finally: + await tx.rollback() + if not migrating.done(): + migrating.cancel() + await asyncio.gather(migrating, return_exceptions=True) + assert await _migrate(connect) == ["004_placements_by_cell_org.sql"] + assert await _migrate(connect) == [] + + +@pytest.mark.parametrize( + "definition", + [ + "CREATE TABLE loom_router.placements_by_cell_org (id int)", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (org, cell)", + 'CREATE INDEX placements_by_cell_org ON loom_router.placements (cell, org COLLATE "POSIX")', + "CREATE UNIQUE INDEX placements_by_cell_org ON loom_router.placements (cell, org)", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (cell, org) WHERE org > 'm'", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (cell, org) INCLUDE(placed_at)", + "CREATE INDEX placements_by_cell_org ON loom_router.cells (cell, host)", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (cell DESC, org)", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (cell text_pattern_ops, org)", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (cell, org) WITH (fillfactor=70)", + ], +) +async def test_foreign_same_named_object_is_never_dropped(database, tmp_path, definition) -> None: + conn, connect = database + await _migrate(connect, _prefix(tmp_path)) + await conn.execute(definition) + oid = await conn.fetchval("SELECT 'loom_router.placements_by_cell_org'::regclass::oid") + with pytest.raises(ValueError, match="different definition"): + await _migrate(connect) + assert await conn.fetchval("SELECT 'loom_router.placements_by_cell_org'::regclass::oid") == oid + + +async def test_recorded_invalid_index_and_checksum_drift_refuse(database, tmp_path) -> None: + conn, connect = database + await _migrate(connect) + await conn.execute("DROP INDEX loom_router.placements_by_cell_org") + with pytest.raises(ValueError, match="recorded online-index"): + await _migrate(connect) + await conn.execute( + "UPDATE loom_router.pg_migrations SET checksum = repeat('0', 64) " + "WHERE filename = '004_placements_by_cell_org.sql'" + ) + with pytest.raises(pg_migrate.MigrationDrift): + await _migrate(connect) + + +async def test_cancel_after_catalog_completion_reuses_the_valid_index( + database, tmp_path, monkeypatch +) -> None: + conn, connect = database + await _migrate(connect, _prefix(tmp_path)) + original = pg_online_index.build + + async def completed_then_cancel(*args, **kwargs): + await original(*args, **kwargs) + raise asyncio.CancelledError + + monkeypatch.setattr(pg_online_index, "build", completed_then_cancel) + with pytest.raises(asyncio.CancelledError): + await _migrate(connect) + oid = await conn.fetchval("SELECT 'loom_router.placements_by_cell_org'::regclass::oid") + monkeypatch.setattr(pg_online_index, "build", original) + assert await _migrate(connect) == ["004_placements_by_cell_org.sql"] + assert await conn.fetchval("SELECT 'loom_router.placements_by_cell_org'::regclass::oid") == oid + + +async def test_keyset_plan_uses_the_ordered_cell_index(database) -> None: + conn, connect = database + await _migrate(connect) + await conn.execute( + "INSERT INTO loom_router.cells (cell, host, state, credential_sha256, deployment_id) " + "SELECT 'cell-' || n, 'cell-' || n || '.example.test', 'serving', " + "md5(n::text) || md5(n::text), 'dep-' || n FROM generate_series(1, 32) n" + ) + await conn.execute( + "INSERT INTO loom_router.placements(org, cell) " + "SELECT 'org-' || lpad(n::text, 5, '0'), 'cell-' || (n % 32 + 1) " + "FROM generate_series(1, 4096) n" + ) + await conn.execute("VACUUM ANALYZE loom_router.placements") + plan = await conn.fetch( + "EXPLAIN SELECT org FROM loom_router.placements " + "WHERE cell = 'cell-2' AND org > 'org-01000' ORDER BY org LIMIT 10" + ) + text = "\n".join(r[0] for r in plan) + assert "placements_by_cell_org" in text and "Sort" not in text + + +async def test_ordinary_files_still_rollback_with_the_ledger(database, tmp_path) -> None: + conn, connect = database + (tmp_path / "001_one.sql").write_text("CREATE TABLE loom_router.one (id int);") + (tmp_path / "002_two.sql").write_text("CREATE TABLE loom_router.two (id int); SELECT 1 / 0;") + import asyncpg + + with pytest.raises(asyncpg.DivisionByZeroError): + await _migrate(connect, tmp_path) + assert await conn.fetchval("SELECT to_regclass('loom_router.one') IS NOT NULL") + assert not await conn.fetchval("SELECT to_regclass('loom_router.two') IS NOT NULL") + assert await conn.fetchval("SELECT count(*) FROM loom_router.pg_migrations") == 1 + (tmp_path / "002_two.sql").write_text("CREATE TABLE loom_router.two (id int);") + assert await _migrate(connect, tmp_path) == ["002_two.sql"] + + +async def test_invalid_online_declaration_fails_before_the_ledger_exists( + database, tmp_path +) -> None: + conn, connect = database + (tmp_path / "001_bad.sql").write_text("-- loom:online-index bad\nSELECT 1;") + with pytest.raises(ValueError, match="online-index"): + await _migrate(connect, tmp_path) + assert not await conn.fetchval("SELECT to_regclass('loom_router.pg_migrations') IS NOT NULL") + + +async def test_cancelled_concurrent_drop_is_retryable(database, tmp_path) -> None: + conn, connect = database + await _migrate(connect, _prefix(tmp_path)) + index = pg_online_index.parse( + (ROUTER_MIGRATIONS_DIR / "004_placements_by_cell_org.sql").read_text() + ) + assert index is not None + await conn.execute(index.sql) + async with connect() as reader: + tx = reader.transaction() + await tx.start() + await reader.execute("SET LOCAL enable_seqscan = off") + plan = await reader.fetch( + "EXPLAIN SELECT org FROM loom_router.placements WHERE cell = 'a' ORDER BY org" + ) + assert "placements_by_cell_org" in "\n".join(r[0] for r in plan) + await reader.fetch("SELECT org FROM loom_router.placements WHERE cell = 'a' ORDER BY org") + + async def direct_drop(): + async with connect() as dropping: + try: + await dropping.execute( + "DROP INDEX CONCURRENTLY loom_router.placements_by_cell_org" + ) + finally: + dropping.terminate() + + dropping = asyncio.create_task(direct_drop()) + try: + query = ( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity " + "WHERE datname = current_database() AND query LIKE 'DROP INDEX CONCURRENTLY%' " + "AND wait_event IS NOT NULL)" + ) + await _until(conn, query) + dropping.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(dropping, 2) + assert await pg_online_index.ready(conn, index) is False + dropping = asyncio.create_task(_migrate(connect)) + await _until(conn, query) + dropping.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(dropping, 2) + finally: + await tx.rollback() + if not dropping.done(): + dropping.cancel() + await asyncio.gather(dropping, return_exceptions=True) + assert await _migrate(connect) == ["004_placements_by_cell_org.sql"] + + +async def test_try_lock_wait_has_no_retained_transaction_or_snapshot(database) -> None: + conn, connect = database + await conn.execute("SELECT pg_advisory_lock(8917423::bigint)") + migrating = asyncio.create_task(_migrate(connect)) + try: + await _until( + conn, + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity " + "WHERE datname = current_database() AND pid <> pg_backend_pid() " + "AND query = 'SELECT pg_try_advisory_lock($1)' " + "AND state = 'idle' AND backend_xmin IS NULL AND xact_start IS NULL)", + ) + assert not migrating.done() + await conn.execute("SELECT pg_advisory_unlock(8917423::bigint)") + assert len(await asyncio.wait_for(migrating, 5)) == 4 + finally: + await conn.execute("SELECT pg_advisory_unlock(8917423::bigint)") + if not migrating.done(): + migrating.cancel() + await asyncio.gather(migrating, return_exceptions=True) + + +async def test_online_validation_is_repeated_after_later_files(database, tmp_path) -> None: + _, connect = database + for source in ROUTER_MIGRATIONS_DIR.glob("*.sql"): + (tmp_path / source.name).write_bytes(source.read_bytes()) + (tmp_path / "005_damage.sql").write_text("DROP INDEX loom_router.placements_by_cell_org;") + with pytest.raises(ValueError, match="recorded online-index"): + await _migrate(connect, tmp_path) diff --git a/tests/integration/test_pg_online_index_durability.py b/tests/integration/test_pg_online_index_durability.py new file mode 100644 index 0000000000..e5644af5af --- /dev/null +++ b/tests/integration/test_pg_online_index_durability.py @@ -0,0 +1,203 @@ +"""A cancelled online-index ledger commit survives a private standby promotion. + +Only its dedicated CI pair enables this destructive drill. Configured failures +are errors; the adopted integration server's replication settings never change. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import os +from collections.abc import AsyncIterator +from pathlib import Path +from urllib.parse import urlsplit + +import asyncpg +import pytest + +from server.core.paths import ROUTER_MIGRATIONS_DIR +from server.core.schema import pg_migrate +from tests.integration._pg_database import create_database + +PRIMARY = "loom-scale-index-drill-pg" +STANDBY = PRIMARY + "-standby" +APPLICATION = "online-index-durability" + + +async def _docker(*args: str) -> str: + process = await asyncio.create_subprocess_exec( + "docker", *args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + try: + async with asyncio.timeout(30): + stdout, stderr = await process.communicate() + except BaseException: + if process.returncode is None: + process.kill() + await asyncio.wait_for(process.wait(), 5) + raise + assert process.returncode == 0, stderr.decode() + return stdout.decode().strip() + + +async def _sql(container: str, database: str, sql: str) -> str: + return await _docker("exec", container, "psql", "-U", "loom", "-d", database, "-Atc", sql) + + +async def _waiting(observer, task) -> int: + async with asyncio.timeout(8): + for _ in range(800): + assert not task.done(), "migration acknowledged before standby flush" + pid = await observer.fetchval( + "SELECT pid FROM pg_stat_activity WHERE application_name = $1 " + "AND wait_event = 'SyncRep'", + APPLICATION, + ) + if pid is not None: + assert not task.done() + return pid + await asyncio.sleep(0.01) + pytest.fail("migration did not reach its synchronous COMMIT") + + +async def _ownership(observer) -> None: + identifier = str(await observer.fetchval("SELECT system_identifier FROM pg_control_system()")) + for container in (PRIMARY, STANDBY): + assert await _docker("inspect", "--format", "{{.Name}}", container) == "/" + container + assert ( + await _sql(container, "loom", "SELECT system_identifier FROM pg_control_system()") + == identifier + ) + assert await _sql(STANDBY, "loom", "SELECT pg_is_in_recovery()") == "t" + assert ( + await observer.fetchval( + "SELECT count(*) FROM pg_stat_replication WHERE application_name = 'loom_standby' " + "AND sync_state = 'sync' AND state = 'streaming'" + ) + == 1 + ) + + +@pytest.mark.asyncio +@pytest.mark.recovery +@pytest.mark.destructive +async def test_online_ledger_retry_survives_physical_standby_promotion( + tmp_path, monkeypatch +) -> None: + if not os.environ.get("LOOM_PG_INDEX_DRILL"): + pytest.skip("LOOM_PG_INDEX_DRILL not set: needs its disposable physical standby") + assert os.environ["LOOM_PG_INDEX_DRILL"] == "1" + dsn = os.environ["LOOM_PG_INDEX_DRILL_DSN"] + observer = await asyncpg.connect(dsn, timeout=5, command_timeout=5) + tasks = [] + paused = False + try: + await _ownership(observer) + _, target = await create_database(dsn) + + @contextlib.asynccontextmanager + async def connect() -> AsyncIterator[asyncpg.Connection]: + conn = await asyncpg.connect( + target, + timeout=5, + server_settings={"application_name": APPLICATION, "synchronous_commit": "on"}, + ) + try: + yield conn + finally: + conn.terminate() + + prefix = tmp_path / "old" + prefix.mkdir() + for source in sorted(ROUTER_MIGRATIONS_DIR.glob("*.sql"))[:3]: + (prefix / source.name).write_bytes(source.read_bytes()) + + async def migrate(files: Path = ROUTER_MIGRATIONS_DIR): + return await pg_migrate.migrate( + files, connect=connect, schema="loom_router", lock_key=8917423 + ) + + await migrate(prefix) + record = pg_migrate._record_online + + async def pause_before_ledger(*args, **kwargs): + nonlocal paused + await _docker("pause", STANDBY) + paused = True + await record(*args, **kwargs) + + monkeypatch.setattr(pg_migrate, "_record_online", pause_before_ledger) + first = asyncio.create_task(migrate()) + tasks.append(first) + pid = await _waiting(observer, first) + # The index is acknowledged already; its ledger COMMIT alone is unflushed. + async with connect() as conn: + assert ( + await conn.fetchval( + "SELECT count(*) FROM loom_router.pg_migrations " + "WHERE filename = '004_placements_by_cell_org.sql'" + ) + == 1 + ) + first.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(first, 3) + async with asyncio.timeout(5): + for _ in range(500): + if not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid = $1)", pid + ): + break + await asyncio.sleep(0.01) + else: + pytest.fail("cancelled migration retained backend locks") + monkeypatch.setattr(pg_migrate, "_record_online", record) + retry = asyncio.create_task(migrate()) + tasks.append(retry) + await _waiting(observer, retry) + await _docker("unpause", STANDBY) + paused = False + assert await asyncio.wait_for(retry, 8) == [] + observer.terminate() + await _docker("kill", "--signal", "KILL", PRIMARY) + await _docker( + "exec", + "--user", + "postgres", + STANDBY, + "bash", + "-c", + 'pg_ctl -D "$PGDATA" promote -w -t 10', + ) + assert await _sql(STANDBY, "loom", "SELECT pg_is_in_recovery()") == "f" + database = urlsplit(target).path.removeprefix("/") + assert ( + await _sql( + STANDBY, + database, + "SELECT count(*) FROM loom_router.pg_migrations " + "WHERE filename = '004_placements_by_cell_org.sql' AND checksum = '" + + pg_migrate.checksum(ROUTER_MIGRATIONS_DIR / "004_placements_by_cell_org.sql") + + "'", + ) + == "1" + ) + assert ( + await _sql( + STANDBY, + database, + "SELECT indisvalid AND indisready FROM pg_index " + "WHERE indexrelid = 'loom_router.placements_by_cell_org'::regclass", + ) + == "t" + ) + finally: + if paused: + await _docker("unpause", STANDBY) + for task in tasks: + if not task.done(): + task.cancel() + async with asyncio.timeout(5): + await asyncio.gather(*tasks, return_exceptions=True) + observer.terminate() diff --git a/tests/unit/test_cell_org_triggers.py b/tests/unit/test_cell_org_triggers.py index 769cfe1bb3..5d25ba0177 100644 --- a/tests/unit/test_cell_org_triggers.py +++ b/tests/unit/test_cell_org_triggers.py @@ -86,6 +86,14 @@ async def execute(self, sql: str, *_a: Any) -> None: async def fetch(self, _sql: str, *_a: Any) -> list[Any]: return [] + def terminate(self) -> None: + self.statements.append("terminated") + + async def fetchval(self, sql: str, *_a: Any) -> bool: + assert sql in ("SELECT pg_try_advisory_lock($1)", "SELECT pg_advisory_unlock($1)") + self.statements.append(sql) + return True + @pytest.mark.parametrize("name", ["eu-1", ""]) @pytest.mark.parametrize("schema", ["loom", "loom_router"]) diff --git a/tests/unit/test_pg_online_index.py b/tests/unit/test_pg_online_index.py new file mode 100644 index 0000000000..7fc43fa2bb --- /dev/null +++ b/tests/unit/test_pg_online_index.py @@ -0,0 +1,212 @@ +"""Only the declared concurrent btree form escapes a migration transaction.""" + +from __future__ import annotations + +import pytest + +from server.core.schema import pg_online_index + +SQL = """-- loom:online-index loom_router.placements_by_cell_org ON loom_router.placements (cell, org) +CREATE INDEX CONCURRENTLY placements_by_cell_org ON loom_router.placements +USING btree (cell COLLATE "C", org COLLATE "C"); +""" + + +def test_the_online_index_declaration_has_one_exact_target() -> None: + parsed = pg_online_index.parse(SQL) + assert parsed is not None + assert parsed.schema == "loom_router" + assert parsed.name == "placements_by_cell_org" + assert parsed.table == "placements" + assert parsed.columns == ("cell", "org") + assert pg_online_index.parse("CREATE TABLE loom.t (id int);") is None + + +@pytest.mark.parametrize( + "sql", + [ + SQL + "SELECT 1;", + SQL.replace("CREATE INDEX", "CREATE UNIQUE INDEX"), + SQL.replace('org COLLATE "C"', 'org COLLATE "POSIX"'), + SQL.replace("(cell, org)", "(org, cell)"), + SQL.replace("ON loom_router.placements\n", "ON loom_router.other\n"), + SQL.replace("USING btree", "USING hash"), + SQL.replace('org COLLATE "C"', 'lower(org) COLLATE "C"'), + SQL.replace(");", ") WHERE true;"), + SQL.replace("-- loom:online-index", "-- loom:online-index-bad"), + SQL.replace("placements_by_cell_org ON", "other_index ON", 1), + SQL.replace("(cell, org)", "(cell, cell)"), + SQL.replace("CONCURRENTLY", ""), + ], +) +def test_other_online_sql_is_refused(sql: str) -> None: + with pytest.raises(ValueError, match="online.index"): + pg_online_index.parse(sql) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("pending", [True, False]) +async def test_ledger_retries_commit_a_wal_fence_even_when_already_recorded( + tmp_path, pending +) -> None: + import contextlib + + from server.core import pg + from server.core.schema import pg_migrate + + path = tmp_path / "001_index.sql" + path.write_text(SQL) + + class Connection: + def __init__(self): + self.statements = [] + + def transaction(self): + return contextlib.nullcontext() + + async def execute(self, sql, *args): + self.statements.append((sql, args)) + + conn = Connection() + await pg_migrate._record_online(conn, path, "loom_router", pending=pending) + assert conn.statements[-1] == (pg._DURABLE_MESSAGE, ()) + assert len(conn.statements) == (2 if pending else 1) + if pending: + assert conn.statements[0][1] == (path.name, pg_migrate.checksum(path)) + + +@pytest.mark.asyncio +async def test_migrator_preserves_a_borrowed_sessions_statement_budget(tmp_path) -> None: + import contextlib + + from server.core.schema import pg_migrate + + class Connection: + def __init__(self): + self.statements = [] + + async def execute(self, sql, *args): + self.statements.append(sql) + + async def fetch(self, sql): + return [] + + async def fetchval(self, sql, *args): + self.statements.append(sql) + return True + + conn = Connection() + + @contextlib.asynccontextmanager + async def connect(): + yield conn + + assert await pg_migrate.migrate(tmp_path, connect=connect, schema="loom_router") == [] + assert not any("statement_timeout" in sql for sql in conn.statements) + assert "SET synchronous_commit = 'on'" in conn.statements + assert conn.statements[-1] == "SELECT pg_advisory_unlock($1)" + + +@pytest.mark.asyncio +async def test_online_target_must_match_its_ledger_before_connecting(tmp_path) -> None: + from server.core.schema import pg_migrate + + (tmp_path / "001_index.sql").write_text(SQL) + + def never_connect(): + pytest.fail("invalid target opened a connection") + + with pytest.raises(ValueError, match="ledger schema"): + await pg_migrate.migrate(tmp_path, schema="loom", connect=never_connect) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("phase", ["statement", "rollback", "commit"]) +async def test_cancellation_terminates_before_installed_asyncpg_transaction_unwind( + tmp_path, phase +) -> None: + import asyncio + import contextlib + from types import SimpleNamespace + + from asyncpg.transaction import Transaction + + from server.core.schema import pg_migrate + + (tmp_path / "001_one.sql").write_text("SELECT stalled_statement;") + reached = asyncio.Event() + terminated = asyncio.Event() + + class Connection: + _pool_release_ctr = 0 + _top_xact = None + _protocol = SimpleNamespace(is_in_transaction=lambda: False) + + def is_closed(self): + return terminated.is_set() + + def transaction(self): + return Transaction(self, None, False, False) + + def terminate(self): + terminated.set() + + async def fetchval(self, sql, *args): + return True + + async def fetch(self, sql): + return [] + + async def execute(self, sql, *args): + if sql == "SELECT stalled_statement;" and phase == "rollback": + raise ValueError("statement failed before rollback") + if ( + (sql == "SELECT stalled_statement;" and phase == "statement") + or (sql.startswith("ROLLBACK") and phase == "rollback") + or (sql.startswith("COMMIT") and phase == "commit") + ): + reached.set() + # Transport termination, rather than a second task cancellation, + # releases a transaction that has already begun async cleanup. + while not terminated.is_set(): + try: + await terminated.wait() + except asyncio.CancelledError: + continue + + conn = Connection() + + @contextlib.asynccontextmanager + async def connect(): + yield conn + + migrating = asyncio.create_task( + pg_migrate.migrate(tmp_path, connect=connect, schema="loom_router") + ) + try: + await asyncio.wait_for(reached.wait(), 1) + migrating.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(migrating, 1) + assert terminated.is_set() + finally: + terminated.set() + if not migrating.done(): + migrating.cancel() + await asyncio.gather(migrating, return_exceptions=True) + + +def test_postgres_journals_keep_distinct_numbers_and_declared_online_forms() -> None: + from server.core.paths import PG_MIGRATIONS_DIR, ROUTER_MIGRATIONS_DIR + from server.core.schema import pg_migrate + from server.core.schema.journal import migration_number + + for directory in (PG_MIGRATIONS_DIR, ROUTER_MIGRATIONS_DIR): + files = pg_migrate.migration_files(directory) + numbers = [migration_number(path.name) for path in files] + assert None not in numbers and len(numbers) == len(set(numbers)) + for path in files: + sql = path.read_text() + parsed = pg_online_index.parse(sql) + if "CONCURRENTLY" in sql: + assert parsed is not None, f"{path.name} lacks an online-index declaration" From cc4ed54549256b07e35101b1bdc17d68e050cb5b Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 22:09:48 +0000 Subject: [PATCH 020/114] Fence cell control mutations across local lock loss Claim a fresh directory UUID before acceptance writes, and compare the generation, registration and exact fingerprint pair on control mutations. Keep every local acceptance transaction on its attach-lock backend and stop after a lost session. The additive router column permits older data-plane reads and inserts. Online control requires turnover of older clients and router handlers; the recovery guide covers claimed state cleanup and offline generation rotation. Signed-off-by: zlareb1 --- docs/architecture/cells.md | 61 ++ docs/operations/cell-control.md | 138 ++++ docs/operations/troubleshooting.md | 12 +- .../005_control_generation.sql | 3 + server/cli/cell.py | 119 ++-- server/cli/cell_attach.py | 126 ++-- server/cli/cell_attach_pg.py | 104 +-- server/cli/cell_control.py | 246 +++++++ server/cli/cell_prepare.py | 85 +-- server/router/app.py | 123 +++- server/router/directory.py | 153 +++-- tests/integration/test_cell_attach.py | 11 +- tests/integration/test_cell_control_fence.py | 613 ++++++++++++++++++ tests/integration/test_router_directory_pg.py | 179 +++-- tests/unit/test_cell_attach.py | 133 ++-- tests/unit/test_cell_control.py | 123 ++++ tests/unit/test_cell_operator.py | 45 +- tests/unit/test_cell_prepare.py | 33 +- tests/unit/test_cell_register.py | 20 +- tests/unit/test_router.py | 12 +- tests/unit/test_router_attach.py | 209 ++++-- 21 files changed, 2054 insertions(+), 494 deletions(-) create mode 100644 docs/operations/cell-control.md create mode 100644 schemas/router_migrations/005_control_generation.sql create mode 100644 server/cli/cell_control.py create mode 100644 tests/integration/test_cell_control_fence.py create mode 100644 tests/unit/test_cell_control.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index dd1189b696..e5419326ed 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1205,6 +1205,67 @@ which writes the directory alone, with the router's operator credential and none cell's stores, so it works while the cell is down; the router judges a placement from the directory row. +## Control mutation fencing + +A cell controller holds the session-level attach lock on one direct, marked, UTF8 +Postgres connection for its whole run. Every write to `loom.cell_self`, `loom.cell_first` +or `loom.tenancy_mode` checks that lock and its observed local state, then writes and +commits on that exact connection. Prepare passes its lock connection to its row writer; +attach commits mode on its lock connection. A lost connection ends the run. It cannot +reconnect and continue an earlier claim. Terminating the client does not prove its +server backend ended: another controller must actually acquire the attach lock before +it starts, and an unresponsive backend can keep that lock until it is positively ended. + +The directory stores one opaque UUID, `cells.mutation_generation`, for each cell. +`POST /cells/v1/operator/cells//claim` takes the directory advisory lock and cell +row lock, compares the full observed registration and exact current/next JSON pair, +generates a fresh UUID on the server, audits it and commits synchronously before +answering. Every execution generates another UUID, including a repeated HTTP request. +The caller never proposes a replacement UUID. Registration assigns a fresh default; +a registration rerun reads the existing value without resetting it. Removing and +registering the same name therefore cannot restore an earlier generation. + +Before its first local acceptance write, prepare, attach or a recovery controller +claims the generation while holding its local lock. Every subsequent directory +mutation carries `expected`: cell, first-cell flag, host, deployment id, bucket, prefix, +probe/upstream addresses, exact current/next pair and the claimed generation. The claim +carries the same fields without a generation. The directory compares them under its +locks before any mutation or no-op success. Canonical JSON comparison distinguishes +`false`, `0` and `0.0`. A stale request cannot release a reservation after its local +backend dies and another controller claims and widens acceptance, even if the directory +pair later returns to its earlier values. A delayed claim can invalidate another +controller's work; it cannot let a dead local session write acceptance state. + +A lost acknowledgement is an unknown outcome. Stop the attempt and rerun: obtain the +local lock, read current state and claim a fresh UUID before recomputing the action. +Do not copy a latest UUID onto an old action. A successful claim or control transaction +waits through synchronous COMMIT. A no-op transaction emits transactional WAL so it +also proves an observed earlier write durable. Acquisition, statements, COMMIT and +cleanup have explicit bounds; an interrupted or uncertain local transaction never +continues on another connection. No operation record, persistent census, phase or lease +is added. The UUID is coordination metadata and holds no memory content. + +The source census for acceptance and operator directory writes is: + +| Writer | Fence | +| --------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | +| `cell_prepare._write_rows` | Fresh claim, observed local rows, exact lock backend | +| `cell_attach_pg._mode_try` | Attach's claim, recording/first/fingerprint comparison, exact lock backend | +| `cell_control.backout` | Fresh claim, fenced close and zero-placement read, observed local rows, exact lock backend | +| `directory.attach_begin`, `attach_place`, `attach_enforced`, `attach_serving`, `remove` | Claimed generation, full registration and exact pair, including retries/no-ops | +| `directory.set_limit`, `rotate_credential` | Read and compare the current generation/registration/pair; do not claim or rotate | +| `directory.register` | Bootstrap default or immutable rerun lookup; never replaces existing acceptance or generation | +| `directory.place` | Immutable placement only; keeps its org/cell locks and quota checks; never writes local acceptance | + +Read-only routes are exempt. Router migrations and an offline directory restore are +maintenance operations. Future restore/update acceptance writers must use the same +claim and lock-session helpers; a new directory mutation must use the common comparison +before its no-op branch as well. Direct privileged SQL is not an online control writer. +The [control upgrade and recovery procedure](../operations/cell-control.md) defines the +required fleet turnover and offline restore of generations. N-1 data-plane builds can +read and insert on the additive schema; old control clients and router handlers cannot +participate in this protocol. + ## Operations - `loom cell restore`, after any store restore and before `loom reconcile`, writes the diff --git a/docs/operations/cell-control.md b/docs/operations/cell-control.md new file mode 100644 index 0000000000..ccd7bf5edf --- /dev/null +++ b/docs/operations/cell-control.md @@ -0,0 +1,138 @@ +# Cell control upgrades and recovery + +Cell control mutations require generation-aware clients and router handlers, and direct +access to the cell's Postgres. This page covers the runtime prerequisite, unknown +outcomes and registered-cell state cleanup. The protocol is described in +[cells](../architecture/cells.md#control-mutation-fencing). + +## Upgrade the control fleet + +Apply the router migrations with `loom router migrate`. The additive UUID column has a +server default and remains readable by N-1 data-plane code. Its volatile default may +rewrite the small cells table and takes a table lock; allow a bounded maintenance window. +Schema compatibility does not let old control programs participate in fencing. + +Before using generation-aware acceptance mutation, inventory and positively terminate +all old router control handlers and prepare, attach, restore, update, remove and manual +cleanup clients, including paused one-offs and outstanding requests. Replace every +launcher/image so those binaries cannot restart against the stores. Use exact process +and start identities, trustworthy evidence that the exact container ended, or independent +node fencing. A heartbeat expiry, missing pod object or one successful request to a router +replica does not prove this inventory. If the inventory is incomplete, stop/fence every +machine that could still run an old control program before resuming control traffic. +Compatible cell data-plane builds may continue serving during this procedure. Every +old router replica that could execute an operator handler must be positively retired; +continuing its data-plane routing alone does not remove that control capability. + +The fencing changes and their dependent CLI changes may be implemented and reviewed as +stacked concerns, merged in dependency order and shipped together. They require no +separate release cut. Running a protected operation requires the capable runtime and +completed control turnover procedure. Missing-generation requests refuse with +`CONTROL_UPGRADE_REQUIRED`; an absent claim route or incomplete operator snapshot tells +the CLI to upgrade the router replica. There is no compatibility fallback or operator +flag that proves a deployment completed its external process inventory. + +## Unknown outcomes + +A timeout, connection loss, cancelled COMMIT or lost acknowledgement ends the current +attempt. Keep the exact cell and router configuration, inspect the reported state and +rerun the command. The rerun must acquire the local attach lock and make a fresh claim +before it changes acceptance. A duplicate claim creates a fresh UUID; it never restores +a caller's earlier token. Do not replay a saved mutation with a copied current token. + +A client connection ending does not prove its Postgres backend ended. If the attach lock +remains held, inspect its exact holder with the query in +[troubleshooting](troubleshooting.md#a-process-with-loom_cell-unset-is-gated). Positively +end that exact owned backend, or wait until it ends. Do not terminate whichever backend +holds a mode-table lock: it may belong to another controller. No control run reconnects +to continue an uncertain attempt. + +## Clear a registered unattached cell's state + +Use this only for the troubleshooting page's eligible unattached cell with no placements, +or a first cell being backed out while its local mode is still `recording`. A first cell +in local `enforced` mode or any cell with placements goes forward. Complete the control +fleet turnover above, retain the existing stop/unset-`LOOM_CELL` procedure, and prevent +accepting or maintenance processes from restarting with the cell setting during cleanup. +Use the table-owning migration role. Never clear a restored copy beside its live source. + +Run the following with this cell's settings, including `LOOM_CELL`, `LOOM_PG_DSN`, its +ClickHouse and bucket configuration, `LOOM_CELL_ROUTER_URL` and the operator credential. +This read-only store pairing check must succeed before the helper claims or clears state. +It does not write participant rows or run the data-plane command gate. + +```python +import argparse +import asyncio + +import httpx + +from server.cli import cell as cell_cli +from server.cli import cell_control +from server.core import pg, pg_profile +from server.core.ch import get_client +from server.settings import settings + + +async def main(): + ch = get_client() + await ch.connect() + try: + recorded = await pg_profile.recorded_authority(ch) + if not recorded or recorded["authority"] != "postgres": + raise RuntimeError("the configured stores do not record Postgres authority") + pg.expect_deployment((str(recorded["deployment_id"]), int(recorded["epoch"]))) + url = cell_cli.operator_router(argparse.Namespace(router=None)) + if not isinstance(url, httpx.URL): + raise RuntimeError("the router URL or operator credential was refused") + + async def call(path, body): + return await cell_control.request(url, settings.cell, None, path, body) + + async with cell_control.session("state cleanup") as conn: + row = await cell_control.request(url, settings.cell, None, "", method="GET") + await cell_control.backout(conn, call, row) + finally: + await ch.disconnect() + + +asyncio.run(main()) +``` + +The helper claims first, closes the cell using the existing limit operation, and requires +that operation's durable placement count to remain zero before any local state cleanup. +A queued placement either wins first and refuses cleanup, or observes the closed row and +cannot create a placement. The helper then compares and deletes only the existing +`cell_first`, `tenancy_mode`, `cell_self` and `tenancy` rows, restores the first-row +triggers and drops only the existing org guard triggers, on the same locked backend. +It takes the migration lock for each bounded local transaction. Even an already-open +retry proves durability before reporting success. + +An abort leaves the cell closed. It never reopens automatically or deletes a placement. +If a placement won, resume attach; inspect the reason before explicitly reopening with +`loom cell limit --open`. Otherwise rerun the cleanup with a fresh claim, then run +`loom cell remove --cell ` with the same cell settings. Removal now holds that +cell's direct Postgres lock and takes a fresh claim for both first and later cells. It +also requires a first cell's durable local state to read `open` before deleting the row. +Only then may the deployment be prepared again. + +## Restore the directory offline + +A directory backup can contain an earlier generation. Stop and positively terminate or +fence all control clients and router processes, including outstanding control requests, +and prevent restarts before restoring it. While control traffic remains stopped, execute +this on the restored directory and wait for its synchronous commit: + +```sql +BEGIN; +SET LOCAL lock_timeout = '5s'; +SET LOCAL statement_timeout = '10s'; +UPDATE loom_router.cells SET mutation_generation = gen_random_uuid(); +COMMIT; +``` + +Bound the client's complete transaction/COMMIT wait too. An uncertain outcome does not +permit APIs to resume: establish the restored writer's actual completion, retry the +rotation while still offline and obtain a known durable commit. Rotate every restored +row before restarting capable router/control binaries. Never copy generations from a +cell backup or rely on an audit retention window to fence restored requests. diff --git a/docs/operations/troubleshooting.md b/docs/operations/troubleshooting.md index 3ede5f4702..1ca3ce0a40 100644 --- a/docs/operations/troubleshooting.md +++ b/docs/operations/troubleshooting.md @@ -295,7 +295,12 @@ each field means is in [monitoring](./monitoring.md#health-checks). - **The deployment is not meant to be a cell** (no directory row, or no router), or **an unattached later cell** (its row reads `is_first = false`, `state = 'attaching'` and `placed = 0`) that you are turning back into a single - deployment: clear the state in one transaction, in a psql session on `LOOM_PG_DSN` + deployment: for a registered cell, use the + [fenced state cleanup](cell-control.md#clear-a-registered-unattached-cells-state). + Do not run the SQL below for a registered cell. For a deployment whose directory + has no row, first positively stop/fence every control client and outstanding router + control request and prevent restarts; with no online acceptance controller, clear + the state in one transaction, in a psql session on `LOOM_PG_DSN` (`\connect` back to it after the check) as the role that owns schema `loom`'s tables (the role that migrates; altering a trigger needs the table's owner). The deployment is then `open`. An unattached later cell's directory row stays live until @@ -353,8 +358,9 @@ each field means is in [monitoring](./monitoring.md#health-checks). `state = 'attaching'` and `placed = 0`, whatever its `mode`, since an attach refused at its step-3 lock leaves it `enforced`; and `SELECT mode FROM loom.tenancy_mode` on its Postgres reads `recording`): back it out in this order. Unset `LOOM_CELL` on every - process, each of which then serves as in `recording`; run the state transaction and the - drop step above; then run `loom cell remove --cell ` with the router's operator + process, each of which then serves as in `recording`; run the + [fenced state cleanup](cell-control.md#clear-a-registered-unattached-cells-state), + with `LOOM_CELL` still set on that control client; then run `loom cell remove --cell ` with the router's operator credential and the cell's `LOOM_PG_DSN`, which refuses until that Postgres reads `open` and records the row's deployment id. Removing the row first would answer every new org 503, and clearing the state first would gate every process with `LOOM_CELL` set. The diff --git a/schemas/router_migrations/005_control_generation.sql b/schemas/router_migrations/005_control_generation.sql new file mode 100644 index 0000000000..82baab882f --- /dev/null +++ b/schemas/router_migrations/005_control_generation.sql @@ -0,0 +1,3 @@ +-- A cell control claim invalidates requests from every earlier local lock owner. +-- Fresh values on insertion and each claim also fence removal/re-registration. +ALTER TABLE loom_router.cells ADD COLUMN mutation_generation uuid NOT NULL DEFAULT gen_random_uuid(); diff --git a/server/cli/cell.py b/server/cli/cell.py index 65f18990d8..6d6f23697f 100644 --- a/server/cli/cell.py +++ b/server/cli/cell.py @@ -1,6 +1,7 @@ """`loom cell credential | limit | which | remove`: the operator's commands that touch the -router's directory (docs/architecture/cells.md, "Operations"). Only `remove` reads a cell's -stores, a first cell's Postgres; the others run while a cell is down. `loom cell prepare` is +router's directory (docs/architecture/cells.md, "Operations"). Removal holds the cell's +Postgres lock and claims its directory generation; limits and credentials need only the +directory. `loom cell prepare` is `server/cli/cell_prepare.py`, `loom cell attach` `server/cli/cell_attach.py`. - `credential new` makes a router credential offline and prints it with its SHA-256; @@ -10,7 +11,8 @@ - `which --org ` prints the cell and host the router's lookup answers for an org (the first cell's for an org with no placement, which it cannot tell apart). - `remove --cell ` removes the directory row of a cell that is `attaching` with no - placement; for the first cell, only once its Postgres reads `open` and records the row's + placement, holding its configured Postgres lock; for the first cell, only once it reads + `open` and records the row's deployment id (the troubleshooting page's back-out runs first). `credential` (except `new`), `limit` and `remove` call the router's operator API with @@ -33,6 +35,7 @@ from server.auth.public_url import is_loopback_host from server.core.cell import NAME, PLACEMENTS_MAX, SHA256, TROUBLESHOOTING from server.router.directory import ( + CONTROL_MAX_S, OPERATOR_CHANGE_MAX_S, OPERATOR_READ_MAX_S, REMOVE_MAX_S, @@ -45,7 +48,7 @@ # A lookup makes up to two directory reads (the org, then the first cell), each a # slot-and-connection wait and a statement. A connect fails fast. _OPERATOR_TIMEOUT = httpx.Timeout( # not a setting: derived - max(OPERATOR_CHANGE_MAX_S, OPERATOR_READ_MAX_S, REMOVE_MAX_S) + 10.0, connect=5.0 + max(CONTROL_MAX_S, OPERATOR_CHANGE_MAX_S, OPERATOR_READ_MAX_S, REMOVE_MAX_S) + 10.0, connect=5.0 ) _LOOKUP_TIMEOUT = httpx.Timeout( # not a setting: derived 2 * (2 * STATEMENT_TIMEOUT_S + 1) + 10.0, connect=5.0 @@ -155,6 +158,18 @@ def _operator_call( if isinstance(found, int): return found router = found + if path in ("limit", "credential"): + from server.cli import cell_control + + row = _read_row(router, args.cell) + if isinstance(row, str): + print(f"loom cell: {row}", file=sys.stderr) + return 1 + try: + body = {**(body or {}), **cell_control.preconditions(row)} + except cell_control.Refused as e: + print(str(e), file=sys.stderr) + return 1 try: r = _send(router, method, args.cell, path, body) except (httpx.ConnectError, httpx.ConnectTimeout, httpx.ProxyError) as e: @@ -306,43 +321,6 @@ def _read_row(router: httpx.URL, cell: str) -> dict[str, Any] | str: return answer -# The first cell's check: one Postgres session, its statements and the whole read bounded. -_CHECK_STATEMENT_MS = 5000 # not a setting: a bound on one CLI read, not on serving -_CHECK_S = 10.0 # not a setting: a bound on one CLI read, not on serving - - -async def _read_first_cell() -> tuple[str, str]: - """The cell state and the deployment id the cell's Postgres records ('' when none), on - one session of its own: no ClickHouse and no pool.""" - from server.core import cell, pg - - async with pg.dedicated(statement_timeout_ms=_CHECK_STATEMENT_MS) as conn: - state, _first = await cell.read_state_if_present(conn) - recorded = await conn.fetchval("SELECT deployment_id FROM loom.authority") - return state, str(recorded or "") - - -async def _first_cell_refusal(deployment_id: str) -> str: - """Why the first cell's row may not be removed yet ('' when it may): its Postgres still - holds a cell state, or is another deployment's. It reads `LOOM_PG_DSN` alone.""" - if not settings.pg_dsn: - return "the first cell keeps its state in Postgres: set LOOM_PG_DSN to the cell's" - state, recorded = await asyncio.wait_for(_read_first_cell(), timeout=_CHECK_S) - if state != "open": - return ( - f"the first cell's Postgres reads `{state}`: unset LOOM_CELL on every process, run " - f"the full-state cleanup ({TROUBLESHOOTING}), then remove the row" - ) - if not recorded: - return "this Postgres records no deployment id: point LOOM_PG_DSN at the cell's own" - if recorded != deployment_id: - return ( - f"this Postgres records deployment id {recorded!r}, and the directory's row " - f"{deployment_id!r}: point LOOM_PG_DSN at the cell's own Postgres" - ) - return "" - - def _cmd_remove(args: argparse.Namespace) -> int: router = _cell_router(args) if isinstance(router, int): @@ -370,24 +348,47 @@ def _cmd_remove(args: argparse.Namespace) -> int: if reason := remove_refusal(args.cell, state, placed): print(f"loom cell remove: refusing: {reason}", file=sys.stderr) return 1 - if row["is_first"]: - try: - refusal = asyncio.run(_first_cell_refusal(row["deployment_id"])) - except Exception as e: # noqa: BLE001 — an unread state refuses the removal - reason = f"no answer within {_CHECK_S:g} s" if isinstance(e, TimeoutError) else str(e) - refusal = f"the first cell's Postgres could not be read ({type(e).__name__}: {reason})" - if refusal: - print(f"loom cell remove: refusing: {refusal}", file=sys.stderr) - return 1 - return _operator_call( - args, - "", - None, - "a retry refused as not in the directory (NO_SUCH_CELL) means the row was removed", - method="DELETE", - router=router, - coded_404=True, - ) + try: + answer = asyncio.run(_remove_locked(args, router, row)) + except Exception as e: # noqa: BLE001 — an unknown control outcome stops the run + print(f"loom cell remove: {type(e).__name__}: {e}; stop and rerun", file=sys.stderr) + return 1 + print(json.dumps(answer, indent=2, sort_keys=True)) + return 0 + + +async def _remove_locked( + args: argparse.Namespace, + router: httpx.URL, + row: dict[str, Any], + transport: httpx.AsyncBaseTransport | None = None, +) -> dict[str, Any]: + from server.cli import cell_control + from server.core import cell + + async with cell_control.session("remove") as conn: + + async def call(path: str, body: dict[str, Any]) -> dict[str, Any]: + return await cell_control.request(router, args.cell, transport, path, body) + + # Re-read under the local lock; the earlier read only checked eligibility. + row = await cell_control.request(router, args.cell, transport, "", method="GET") + if reason := remove_refusal(args.cell, row["state"], row["placed_orgs"]): + raise cell_control._refused(reason) + await cell_control.paired(conn, row) + if row["is_first"]: + async with cell_control.transaction(conn): + state, _ = await cell.read_state_if_present(conn) + if state != "open": + raise cell_control._refused( + f"the first cell's Postgres reads {state}: run the full-state cleanup " + f"({TROUBLESHOOTING}) before removing its directory row" + ) + row = await cell_control.claim(conn, call, row) + await cell_control.check_lock(conn) + return await cell_control.request( + router, args.cell, transport, "", cell_control.preconditions(row), method="DELETE" + ) def _cmd_which(args: argparse.Namespace) -> int: diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index bf4e81521e..b3cb44d9cf 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -42,7 +42,7 @@ from server.cli import cell as cell_cli from server.cli import cell_attach_beats as beats from server.cli import cell_attach_pg as apg -from server.cli import cell_scan +from server.cli import cell_control, cell_scan from server.cli._ch import writes from server.cli.cell_ch import Refusal, Refused, read_topology from server.core import cell, cell_triggers @@ -77,32 +77,15 @@ def __init__( ) -> None: self.url, self.name, self.transport = url, name, transport - def _client(self) -> httpx.AsyncClient: - return httpx.AsyncClient( - timeout=cell_cli._OPERATOR_TIMEOUT, - trust_env=self.url.scheme == "https", - transport=self.transport, - headers={"Authorization": f"Bearer {settings.router_operator_token}"}, - ) - async def call(self, path: str, body: dict[str, Any] | None = None) -> dict[str, Any]: - """The router's 200 answer naming the cell; Refused with its refusal otherwise.""" - url = cell_cli.at(self.url, f"/cells/v1/operator/cells/{self.name}" + path) - try: - async with self._client() as client: - r = await (client.get(url) if body is None else client.post(url, json=body)) - except httpx.HTTPError as e: - raise _refused("router", f"no answer ({type(e).__name__}): rerun attach") from None - answer = cell_cli.router_answer(r, {"cell": self.name}, cell_cli.OPERATOR_REFUSALS) - if answer is None: - raise _refused("router", cell_cli.not_the_router(r)) - if r.status_code == 404 and answer.get("code") != "NO_SUCH_CELL": - raise _refused("router", f"a router replica is not upgraded: it does not answer {url}") - if r.status_code != 200: - raise _refused( - "router", f"refused {path or 'the read'} ({r.status_code}): {answer['error']}" - ) - return answer + return await cell_control.request( + self.url, + self.name, + self.transport, + path, + body, + method="GET" if body is None else "POST", + ) async def health(self, host: str) -> str: """The deployment id `https:///v1/health/deployment` answers ('' when none).""" @@ -183,13 +166,8 @@ def __init__( # -- the lock session -------------------------------------------------------------- async def open_lock(self) -> None: - from server.core import pg - - self._lock_cm = pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) + self._lock_cm = cell_control.session("attach") self.lock_conn = await self._lock_cm.__aenter__() - if refusals := await apg.direct_refusals(self.lock_conn, "attach"): - raise Refused(refusals) - await apg.take_lock(self.lock_conn, "attach") async def close_lock(self) -> None: if self._lock_cm is not None: @@ -197,33 +175,14 @@ async def close_lock(self) -> None: self._lock_cm = self.lock_conn = None async def confirm_lock(self) -> None: - """The lock is held still, or taken again (refusing when another run holds it now), - and the cell's state read again: a cleanup, a prepare rerun or another attach could - have run in the gap.""" - try: - if await apg.lock_held(self.lock_conn): - return - except Exception as e: # noqa: BLE001 — a lost session is taken again below - self.log(f"the lock session was lost ({type(e).__name__}); taking the lock again") - else: - self.log("the attach lock was lost; taking it again") - try: - await self.close_lock() - except Exception: # noqa: BLE001 — the old session is gone either way - self._lock_cm = self.lock_conn = None - await self.open_lock() - state, first = await apg.read_mode(self.lock_conn) - if first != (True, self.name) or (self.mode == "enforced" and state != "enforced"): - raise _refused( - "cell state", - f"changed while the lock was lost (now {state}, first row {first}): rerun attach", - ) - row = await self.router.call("") - if ( - row.get("deployment_id") != self.row.get("deployment_id") - or row.get("state") != "attaching" - ): - raise _refused("directory row", "changed while the lock was lost: rerun attach") + """A lost backend ends the owner; a new run must claim and start again.""" + await cell_control.check_lock(self.lock_conn) + + async def mutate(self, path: str, body: dict[str, Any]) -> dict[str, Any]: + answer = await cell_control.mutate(self.lock_conn, self.router.call, self.row, path, body) + if "mutation_generation" in answer: + self.row = {**self.row, **answer} + return answer # -- the refusals ------------------------------------------------------------------ @@ -406,6 +365,7 @@ async def begin(self) -> None: "pg_allow_roles": sorted(self.args.pg_allow_role), "nonces": nonces, } + body.update(cell_control.preconditions(self.row)) if (size := _encoded_size(body)) > ATTACH_BEGIN_MAX_BYTES: raise _refused( "attach/begin", @@ -414,7 +374,7 @@ async def begin(self) -> None: ) try: await _write_nonces(self.ch, nonces) - self.row = {**self.row, **await self.router.call("/attach/begin", body)} + self.row = {**self.row, **await self.mutate("/attach/begin", body)} self.begun_keys_fp = self.keys_fp finally: await _delete_nonces(self.ch, nonces) @@ -451,28 +411,21 @@ async def ready_for_commit(self) -> list[str]: return verdict.booting async def commit_mode(self) -> None: - from server.core import pg - - await self.router.call("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) + await self.mutate("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) def mark(mode: str) -> None: - # `unknown` from a try's UPDATE on: only the durable read can say whether it - # landed; `recording` again when that try rolled back. self.mode = mode - try: - async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: - await apg.commit_enforced( - conn, self.confirm_lock, self.ready_for_commit, self.wait, self.log, mark - ) - except Refused: - raise - except Exception as e: - if self.mode != "unknown": - raise # no try was in flight: nothing to decide - self.log(f"the mode commit's outcome is unknown ({type(e).__name__}: {e})") - async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: - await apg.end_mode_transactions(conn, self.confirm_lock) + await apg.commit_enforced( + self.lock_conn, + self.confirm_lock, + self.ready_for_commit, + self.wait, + self.log, + mark, + expected_cell=self.name, + expected_fp=self.own_fp, + ) async def durable(self) -> int: """The durable read of `enforced`, retried within `--wait`; then X, the snapshot's @@ -484,13 +437,14 @@ async def durable(self) -> int: if await pg_profile.standby_now() != "synchronous": raise _refused("postgres standbys", "not synchronous before the durable read") try: - read = await apg.durable_read() + read = await apg.durable_read(self.lock_conn) except Exception as e: # A try cut off by its bound, or an outage: its outcome is unknown, and # the next try reads again. Anything else is no reason to retry. outage = pg.is_outage(e) or isinstance(e, pg.PostgresUnavailableError) if not (isinstance(e, TimeoutError) or outage): raise + await self.confirm_lock() self.log(f"the durable read failed ({type(e).__name__}); retrying") if not await self.wait.pause(): raise _refused( @@ -573,7 +527,7 @@ async def step4(self, xmax: int, topology: Any, names: Any) -> None: placed = 0 for batch in batches(orgs, self.row["deployment_id"]): await self.confirm_lock() - answer = await self.router.call( + answer = await self.mutate( "/attach/place", {"deployment_id": self.row["deployment_id"], "orgs": batch} ) placed += int(answer.get("placed") or 0) @@ -586,7 +540,9 @@ async def step4(self, xmax: int, topology: Any, names: Any) -> None: await self.final_checks(watch) row = await self.router.call("") current, nxt = self.own_fp - if (row.get("current_fp"), row.get("next_fp")) != (current, nxt): + if json.dumps((row.get("current_fp"), row.get("next_fp")), sort_keys=True) != json.dumps( + (current, nxt), sort_keys=True + ): raise _refused( "postgres loom.cell_self", "differs from the directory's fingerprints (a fingerprinted setting changed " @@ -602,7 +558,7 @@ async def serving(self) -> dict[str, Any]: nonces = _new_nonces() try: await _write_nonces(self.ch, nonces) - return await self.router.call( + return await self.mutate( "/attach/serving", {"deployment_id": self.row["deployment_id"], "nonces": nonces} ) finally: @@ -627,6 +583,7 @@ async def done(self) -> bool: async def run(self) -> int: self.row = await self.router.call("") + cell_control.preconditions(self.row) if self.row.get("state") == "serving": await self.done() self.report.update(state="serving", done=True) @@ -637,6 +594,7 @@ async def run(self) -> int: topology, names, listed = await self.refusals() if self.args.dry_run: return await self.dry_run(topology, listed) + self.row = await cell_control.claim(self.lock_conn, self.router.call, self.row) await self.begin() state, _ = await apg.read_mode(self.lock_conn) self.mode = state @@ -651,9 +609,7 @@ async def run(self) -> int: await self.precommit(found) await self.commit_mode() else: - await self.router.call( - "/attach/enforced", {"deployment_id": self.row["deployment_id"]} - ) + await self.mutate("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) xmax = await self.durable() await self.check_beats("ignore") await self.step4(xmax, topology, names) diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 12e1808f8b..8ae6436f3f 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -11,6 +11,7 @@ import asyncio import contextlib +import json import time from collections.abc import Awaitable, Callable, Iterable, Iterator from dataclasses import dataclass @@ -581,21 +582,7 @@ def _busy_refusal(table: str, holders: list[Any]) -> Refusal: # The mode transaction in flight: the backend holding or waiting for the ExclusiveLock on # loom.tenancy_mode in this database, which only attach takes. -_MODE_HOLDERS = """ -SELECT l.pid, a.application_name, a.xact_start, r.rolsuper, l.granted -FROM pg_locks l -JOIN pg_stat_activity a ON a.pid = l.pid -LEFT JOIN pg_roles r ON r.oid = a.usesysid -WHERE l.locktype = 'relation' AND l.relation = 'loom.tenancy_mode'::regclass - AND l.mode = 'ExclusiveLock' - AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database()) - AND l.pid <> pg_backend_pid() -""" - _TIMED_OUT = frozenset({"55P03", "57014"}) -# How long attach waits for an ended mode transaction's lock to clear. -_END_ROUNDS = 30 # not a setting: checks of the lock -_END_PAUSE_S = 0.1 # not a setting: the pause between them async def commit_enforced( @@ -605,22 +592,20 @@ async def commit_enforced( wait: Wait, log: Callable[[str], None], mark: Callable[[str], None], + *, + expected_cell: str, + expected_fp: tuple[dict[str, Any] | None, dict[str, Any] | None], ) -> None: - """Raise the mode to `enforced` on `conn` (a session of its own, never the lock - session): an EXCLUSIVE lock on loom.tenancy_mode a try, `_MODE_LOCK_MS` each (less - when `wait` has less left), within `wait`; `confirm_lock` before each try and each - termination (it refuses when another run holds the attach lock), and `ready` before - each try (the booting process ids skip the try). After a failed try the in-flight - mode transaction, found by its lock, is ended. `mark` hears `unknown` just before a try - sends its UPDATE and `recording` when that try rolled back. Raises Refused when `wait` - runs out; whether the commit landed is the durable read's to say.""" + """Raise mode on the exact ATTACH-lock backend; an unknown COMMIT ends this run.""" with wait.counting(): while True: await confirm_lock() booting = await ready() if not booting: try: - done = await _mode_try(conn, wait.bound_ms(_MODE_LOCK_MS), mark) + done = await _mode_try( + conn, wait.bound_ms(_MODE_LOCK_MS), mark, expected_cell, expected_fp + ) except Refused: mark("recording") # the UPDATE's transaction rolled back raise @@ -628,7 +613,6 @@ async def commit_enforced( return mark("recording") log("the mode's lock is busy; retrying") - await end_mode_transactions(conn, confirm_lock) if not await wait.pause(): if booting: raise Refused( @@ -648,7 +632,13 @@ async def commit_enforced( ) -async def _mode_try(conn: Any, lock_ms: int, mark: Callable[[str], None]) -> bool: +async def _mode_try( + conn: Any, + lock_ms: int, + mark: Callable[[str], None], + expected_cell: str, + expected_fp: tuple[dict[str, Any] | None, dict[str, Any] | None], +) -> bool: """One try including rollback or COMMIT, bounded by its lock and statement waits plus the standby allowance; False after a statement timeout rolls back, TimeoutError at the try's deadline.""" @@ -662,7 +652,22 @@ async def _mode_try(conn: Any, lock_ms: int, mark: Callable[[str], None]) -> boo await conn.execute( f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" ) + from server.cli import cell_control + + await cell_control.check_lock(conn) await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") + state, first = await read_mode(conn) + own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") + observed = None if own is None else (own["current_fp"], own["next_fp"]) + if ( + state != "recording" + or first != (True, expected_cell) + or json.dumps(observed, sort_keys=True) + != json.dumps(expected_fp, sort_keys=True) + ): + raise Refused( + [Refusal("postgres acceptance rows", "changed: stop and rerun attach")] + ) mark("unknown") status = await conn.execute( "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" @@ -684,49 +689,6 @@ async def _mode_try(conn: Any, lock_ms: int, mark: Callable[[str], None]) -> boo raise -async def end_mode_transactions(conn: Any, confirm_lock: Callable[[], Any]) -> None: - """End an earlier run's mode transaction still in flight and wait, for up to - `_END_ROUNDS` checks `_END_PAUSE_S` apart, until its lock clears; Refused when its role - is a superuser and attach's is not, or when a backend still holds the lock after the - last check, naming it.""" - rows: list[Any] = [] - for _ in range(_END_ROUNDS): - rows = list(await conn.fetch(_MODE_HOLDERS)) - if not rows: - return - superuser = bool( - await conn.fetchval("SELECT rolsuper FROM pg_roles WHERE rolname = current_user") - ) - for r in rows: - if r["rolsuper"] and not superuser: - raise Refused( - [ - Refusal( - f"postgres backend {r['pid']}", - f"an earlier mode transaction ({r['application_name'] or 'no name'}, " - f"since {r['xact_start']}) runs as a superuser: a superuser must " - "end it (pg_terminate_backend)", - ) - ] - ) - await confirm_lock() - await conn.fetchval("SELECT pg_terminate_backend($1)", r["pid"]) - await _sleep(_END_PAUSE_S) - if not (rows := list(await conn.fetch(_MODE_HOLDERS))): - return - raise Refused( - [ - Refusal( - f"postgres backend {r['pid']}", - f"still holds loom.tenancy_mode's lock after it was ended " - f"({r['application_name'] or 'no name'}, since {r['xact_start']}): rerun attach " - "once it has gone", - ) - for r in rows - ] - ) - - # -- the durable read ------------------------------------------------------------------- _DURABLE_TRY_S = 10.0 # not a setting: one try's bound; a cancelled one is retried @@ -742,11 +704,13 @@ class Durable: xid: int -async def durable_read() -> Durable: +async def durable_read(conn: Any) -> Durable: """The mode and the `first` row read in a transaction that emits a transactional logical message, whose commit waits for the synchronous standbys: once it returns, the mode read is on them. Refused when the message took no xid.""" - async with asyncio.timeout(_DURABLE_TRY_S), pg.durable_transaction() as conn: + from server.cli import cell_control + + async with asyncio.timeout(_DURABLE_TRY_S), cell_control.transaction(conn): await conn.execute("SELECT pg_logical_emit_message(true, 'loom.attach', '')") xid = await conn.fetchval("SELECT pg_current_xact_id_if_assigned()::text") mode = await conn.fetchval("SELECT mode FROM loom.tenancy_mode") diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py new file mode 100644 index 0000000000..3c03a75e2a --- /dev/null +++ b/server/cli/cell_control.py @@ -0,0 +1,246 @@ +"""Fenced cell control: one directory claim and the exact local lock-owning backend.""" + +from __future__ import annotations + +import asyncio +import json +from collections.abc import AsyncIterator, Awaitable, Callable +from contextlib import asynccontextmanager +from typing import Any +from uuid import UUID + +from server.cli import cell_attach_pg as apg +from server.cli.cell_ch import Refusal +from server.cli.cell_ch import Refused as Refused +from server.router.directory import CONTROL_FIELDS, CONTROL_MAX_S +from server.settings import settings + +# A control transaction has few metadata statements; COMMIT's standby wait is bounded too. +TRANSACTION_S = 60.0 # not a setting: a single control transaction's deadline +OPEN_S = 75.0 # not a setting: connection initialization and direct-session checks +REQUEST_S = CONTROL_MAX_S + 30.0 # not a setting: directory bound plus probes/connect/close +Call = Callable[[str, dict[str, Any]], Awaitable[dict[str, Any]]] + + +def _refused(reason: str) -> Refused: + return Refused([Refusal("cell control", reason)]) + + +def preconditions(row: dict[str, Any], *, claim: bool = False) -> dict[str, Any]: + """Carry the whole observed registration and exact fingerprint pair.""" + if any(name not in row for name in (*CONTROL_FIELDS, "mutation_generation")): + raise _refused("a router replica is not upgraded for cell control fencing") + try: + generation = str(UUID(row["mutation_generation"])) + except (ValueError, TypeError, AttributeError): + raise _refused( + "a router replica returned no readable cell control generation: upgrade it" + ) from None + values = {name: row[name] for name in CONTROL_FIELDS} + if not claim: + values["mutation_generation"] = generation + return {"expected": values} + + +async def check_lock(conn: Any) -> None: + try: + held = await apg.lock_held(conn) + except Exception as e: # noqa: BLE001 — an unreadable lock ends this ownership + raise _refused( + f"lock session lost ({type(e).__name__}): stop and rerun with a fresh claim" + ) from None + if not held: + raise _refused("attach lock lost: stop and rerun with a fresh claim") + + +async def paired(conn: Any, row: dict[str, Any]) -> None: + """The locked authority and configured bucket name the observed registration.""" + recorded = await conn.fetchval("SELECT deployment_id FROM loom.authority") + prefix = settings.document_s3_prefix if settings.document_s3_bucket else "" + if ( + not recorded + or recorded != row.get("deployment_id") + or settings.cell != row.get("cell") + or (settings.document_s3_bucket, prefix) != (row.get("bucket"), row.get("bucket_prefix")) + ): + raise _refused( + "this cell configuration and locked Postgres do not match the directory registration" + ) + + +async def claim(conn: Any, call: Call, row: dict[str, Any]) -> dict[str, Any]: + """Claim once; a lost acknowledgement ends the attempt, never restoring an old token.""" + await check_lock(conn) + await paired(conn, row) + answer = await call("/claim", preconditions(row, claim=True)) + await check_lock(conn) + preconditions(answer) + expected = {name: row[name] for name in CONTROL_FIELDS} + if json.dumps(expected, sort_keys=True) != json.dumps( + {name: answer[name] for name in CONTROL_FIELDS}, sort_keys=True + ): + raise _refused( + "claim returned a different registration or fingerprint pair: stop and rerun" + ) + return answer + + +async def mutate( + conn: Any, call: Call, row: dict[str, Any], path: str, body: dict[str, Any] +) -> dict[str, Any]: + """An unknown response stops the caller; a fresh run must claim and recompute.""" + await check_lock(conn) + answer = await call(path, {**body, **preconditions(row)}) + await check_lock(conn) + return answer + + +@asynccontextmanager +async def transaction(conn: Any) -> AsyncIterator[Any]: + """Check, write and durably commit on the lock backend; cancellation ends ownership.""" + settled = False + tx = conn.transaction() + try: + async with asyncio.timeout(TRANSACTION_S): + await tx.start() + try: + await conn.execute("SET LOCAL lock_timeout = '5s'") + await conn.execute("SET LOCAL statement_timeout = '10s'") + await check_lock(conn) + await conn.execute("SELECT pg_logical_emit_message(true, 'loom.control', '')") + yield conn + except Exception: + await tx.rollback() + settled = True + raise + else: + await tx.commit() + settled = True + finally: + if not settled: + # This includes cancellation during COMMIT/rollback. A later session is a new run. + conn.terminate() + + +@asynccontextmanager +async def session(what: str) -> AsyncIterator[Any]: + """A direct, marked, UTF8 session holding ATTACH for its entire control run.""" + from server.core import pg + + cm = pg.dedicated(statement_timeout_ms=60_000) + conn = None + try: + async with asyncio.timeout(OPEN_S): + conn = await cm.__aenter__() + refused = await apg.direct_refusals(conn, what) + refused += await apg.encoding_refusals(conn) + if refused: + raise Refused(refused) + await apg.take_lock(conn, what) + yield conn + finally: + if conn is not None: + # Use dedicated()'s bounded close; this does not prove the server backend ended. + await cm.__aexit__(None, None, None) + + +async def request( + url: Any, + name: str, + transport: Any, + path: str, + body: dict[str, Any] | None = None, + *, + method: str = "POST", +) -> dict[str, Any]: + """Read or mutate one operator row; gateways and lost replies are unknown outcomes.""" + import httpx + + from server.cli import cell as cli + + endpoint = cli.at(url, f"/cells/v1/operator/cells/{name}" + path) + try: + async with ( + asyncio.timeout(REQUEST_S), + httpx.AsyncClient( + timeout=cli._OPERATOR_TIMEOUT, + trust_env=url.scheme == "https", + transport=transport, + headers={"Authorization": f"Bearer {settings.router_operator_token}"}, + ) as client, + ): + response = await client.request(method, endpoint, json=body) + except (httpx.HTTPError, TimeoutError) as e: + raise _refused(f"router reply unknown ({type(e).__name__}): stop and rerun") from None + answer = cli.router_answer(response, {"cell": name}, cli.OPERATOR_REFUSALS) + if answer is None: + raise _refused(cli.not_the_router(response)) + if response.status_code == 404 and answer.get("code") != "NO_SUCH_CELL": + raise _refused("a router replica is not upgraded for cell control fencing") + if response.status_code != 200: + raise _refused( + f"router refused {path or 'the read'} ({response.status_code}): {answer['error']}" + ) + return answer + + +async def _acceptance(conn: Any) -> dict[str, Any]: + return { + "first": dict(row) + if (row := await conn.fetchrow("SELECT is_first, cell FROM loom.cell_first")) + else None, + "mode": await conn.fetchval("SELECT mode FROM loom.tenancy_mode"), + "own": dict(row) + if (row := await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self")) + else None, + } + + +async def backout(conn: Any, call: Call, row: dict[str, Any]) -> None: + """The runbook's exact state cleanup, fenced against queued controllers/placements.""" + from server.core.schema import pg_migrate + + async with transaction(conn): + before = await _acceptance(conn) + first = before["first"] + if first is not None and (first["cell"], first["is_first"]) != (row["cell"], row["is_first"]): + raise _refused("the local first row names another registration") + already_open = not any(before.values()) and not await conn.fetchval( + "SELECT EXISTS(SELECT 1 FROM loom.tenancy)" + ) + if row["is_first"] and before["mode"] != "recording" and not already_open: + raise _refused("first-cell backout requires local recording mode; enforced goes forward") + row = await claim(conn, call, row) + if row.get("state") != "attaching": + raise _refused("a serving cell goes forward; state cleanup refused") + # Closing and reading placed under one directory row lock excludes an ordinary + # placement queued before the claim. A later failure deliberately leaves it closed. + limited = await mutate(conn, call, row, "/limit", {"closed": True}) + if type(limited.get("placed_orgs")) is not int or limited["placed_orgs"] != 0: + raise _refused("cell holds placements: cleanup refused; the cell remains closed") + async with transaction(conn): + await paired(conn, row) + if json.dumps(await _acceptance(conn), sort_keys=True) != json.dumps( + before, sort_keys=True + ): + raise _refused("local acceptance changed: cleanup refused; the cell remains closed") + await conn.execute("SELECT pg_advisory_xact_lock($1)", pg_migrate._LOCK_KEY) + await conn.execute("ALTER TABLE loom.cell_first DISABLE TRIGGER cell_first_immutable") + for table in ("cell_first", "tenancy_mode", "cell_self", "tenancy"): + await conn.execute(f"DELETE FROM loom.{table}") + await conn.execute("ALTER TABLE loom.cell_first ENABLE ALWAYS TRIGGER cell_first_immutable") + await conn.execute( + "ALTER TABLE loom.cell_first ENABLE ALWAYS TRIGGER cell_first_no_truncate" + ) + # Once state is open, a migrating boot cannot recreate the org triggers. The same + # session still holds ATTACH; a retry repeats the durable state cleanup first. + async with transaction(conn): + await conn.execute("SELECT pg_advisory_xact_lock($1)", pg_migrate._LOCK_KEY) + triggers = await conn.fetch( + "SELECT t.tgname, c.relname FROM pg_trigger t JOIN pg_class c ON c.oid = t.tgrelid " + "JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'loom' " + "AND t.tgname IN ('tenancy_insert', 'tenancy_update', 'tenancy_record')" + ) + for trigger in triggers: + name, table = (str(trigger[k]).replace('"', '""') for k in ("tgname", "relname")) + await conn.execute(f'DROP TRIGGER "{name}" ON loom."{table}"') diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index 4a820f652b..c4ac332757 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -39,6 +39,7 @@ import httpx from server.cli import cell as cell_cli +from server.cli import cell_control from server.cli._ch import writes from server.core import cell from server.settings import settings @@ -51,9 +52,6 @@ _WAIT_MAX_S = 86400 # not a setting: a flag's bound -# A bound on the lock session's statements: it reads the catalog and holds the lock. -_LOCK_STATEMENT_MS = 60_000 # not a setting: a bound on one CLI session's reads - _sleep = asyncio.sleep # the seam the retry tests replace @@ -109,14 +107,12 @@ def _args_problem(args: argparse.Namespace) -> str: return "" -async def _read_rows() -> _CellRows: - from server.core import pg, pg_profile - - deployment_id = await pg_profile.deployment_id() - first = await pg.fetchrow("SELECT is_first, cell FROM loom.cell_first") - mode = await pg.fetchval("SELECT mode FROM loom.tenancy_mode") - own = await pg.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") - tenancy_rows = await pg.fetchval("SELECT count(*) FROM loom.tenancy") +async def _read_rows(conn: Any, *, count_tenancy: bool = True) -> _CellRows: + deployment_id = str(await conn.fetchval("SELECT deployment_id FROM loom.authority") or "") + first = await conn.fetchrow("SELECT is_first, cell FROM loom.cell_first") + mode = await conn.fetchval("SELECT mode FROM loom.tenancy_mode") + own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") + tenancy_rows = await conn.fetchval("SELECT count(*) FROM loom.tenancy") if count_tenancy else 0 return _CellRows( deployment_id=deployment_id, first=None if first is None else (bool(first["is_first"]), str(first["cell"])), @@ -382,15 +378,23 @@ def _own_rows( async def _write_rows( - rows: _CellRows, answer: dict[str, Any], current: dict[str, Any], nxt: dict[str, Any] | None + conn: Any, + rows: _CellRows, + answer: dict[str, Any], + current: dict[str, Any], + nxt: dict[str, Any] | None, ) -> list[str]: """Write the `first` and mode rows that are missing and the own row where it differs from `current` and `nxt`, in one transaction; the rows written.""" - from server.core import pg - is_first = bool(answer.get("is_first")) written = [] - async with pg.transaction() as conn: + async with cell_control.transaction(conn): + # A state CAS does not need to count every tenancy row again. + fresh = await _read_rows(conn, count_tenancy=False) + before = (rows.deployment_id, rows.first, rows.mode, rows.current, rows.next) + after = (fresh.deployment_id, fresh.first, fresh.mode, fresh.current, fresh.next) + if json.dumps(before, sort_keys=True) != json.dumps(after, sort_keys=True): + raise Refused("local acceptance rows changed: stop and rerun with a fresh claim") if rows.first is None: await conn.execute( "INSERT INTO loom.cell_first (is_first, cell) VALUES ($1, $2)", @@ -403,7 +407,9 @@ async def _write_rows( "INSERT INTO loom.tenancy_mode (mode) VALUES ($1)", str(answer["mode"]) ) written.append("mode") - if (rows.current, rows.next) != (current, nxt): + if json.dumps((rows.current, rows.next), sort_keys=True) != json.dumps( + (current, nxt), sort_keys=True + ): await conn.execute( "INSERT INTO loom.cell_self (current_fp, next_fp) VALUES ($1, $2) " "ON CONFLICT (id) DO UPDATE SET current_fp = EXCLUDED.current_fp, " @@ -452,29 +458,18 @@ async def prepare( transport: httpx.AsyncBaseTransport | None = None, ) -> int: """Prepare the cell, holding the attach lock for the run (`attach_lock`).""" - async with attach_lock(): - return await _prepare_locked(ch, args, router, transport) + async with attach_lock() as conn: + return await _prepare_locked(ch, args, router, transport, conn) @contextlib.asynccontextmanager -async def attach_lock() -> AsyncIterator[None]: - """The attach lock, held on a direct session of the cell's Postgres while the block - runs, after that session's checks: prepare and attach write the same probe nonce, and - one of them runs at a time.""" - from server.cli import cell_attach_pg - from server.cli.cell_ch import Refused as Stopped - from server.core import pg - - async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: - try: - refusals = await cell_attach_pg.direct_refusals(conn, "prepare") - refusals += await cell_attach_pg.encoding_refusals(conn) - if refusals: - raise Stopped(refusals) - await cell_attach_pg.take_lock(conn, "prepare") - except Stopped as e: - raise Refused(str(e)) from None - yield +async def attach_lock() -> AsyncIterator[Any]: + """Hand the exact lock backend to every acceptance-state read and write.""" + try: + async with cell_control.session("prepare") as conn: + yield conn + except cell_control.Refused as e: + raise Refused(str(e)) from None async def _prepare_locked( @@ -482,8 +477,10 @@ async def _prepare_locked( args: argparse.Namespace, router: httpx.URL, transport: httpx.AsyncBaseTransport | None, + conn: Any, ) -> int: - rows = await _read_rows() + async with cell_control.transaction(conn): + rows = await _read_rows(conn) if not rows.deployment_id: raise Refused("this cell's Postgres records no deployment id: start the cell once first") if rows.first is not None and rows.first[1] != settings.cell: @@ -518,7 +515,19 @@ async def _prepare_locked( if problem := _answer_problem(rows, answer, beats): raise Refused(problem) current, nxt = _own_rows(rows, answer, every) - written = await _write_rows(rows, answer, current, nxt) + + async def call(path: str, body: dict[str, Any]) -> dict[str, Any]: + return await cell_control.request(router, args.cell, transport, path, body) + + snapshot = await cell_control.request(router, args.cell, transport, "", method="GET") + snapshot = await cell_control.claim(conn, call, snapshot) + # A queued earlier controller may have committed before this claim. Its returned + # row, including the highest mode, supersedes the registration response. + answer = {**answer, **snapshot} + if problem := _answer_problem(rows, snapshot, beats): + raise Refused(problem) + current, nxt = _own_rows(rows, snapshot, every) + written = await _write_rows(conn, rows, answer, current, nxt) print( json.dumps( { diff --git a/server/router/app.py b/server/router/app.py index 4e24a7c922..39a1ac23c8 100644 --- a/server/router/app.py +++ b/server/router/app.py @@ -51,6 +51,7 @@ from contextlib import asynccontextmanager from typing import Any, NamedTuple, Protocol from urllib.parse import parse_qs +from uuid import UUID import httpx from starlette.applications import Starlette @@ -542,7 +543,12 @@ async def set_limit(self, request: Request) -> JSONResponse: return refusal if refusal := _malformed_cell_refusal(request): return refusal - body = await _body(request) + body = _parsed(await _bounded(request, ATTACH_BEGIN_MAX_BYTES)) + if not isinstance(body, dict): + return _refused(400, "this call's body is a JSON object") + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected known = {"closed", "placements_max"} if not isinstance(body, dict) or not body or set(body) - known: return _refused(400, "a limit is a JSON object of closed and/or placements_max") @@ -556,7 +562,7 @@ async def set_limit(self, request: Request) -> JSONResponse: return _refused(400, "placements_max is a whole number, 0 or more, or null") return await _operator_answer( lambda: directory.set_limit( - request.path_params["cell"], closed=closed, placements_max=quota + request.path_params["cell"], closed=closed, placements_max=quota, expected=expected ) ) @@ -567,7 +573,12 @@ async def rotate_credential(self, request: Request) -> JSONResponse: return refusal if refusal := _malformed_cell_refusal(request): return refusal - body = await _body(request) + body = _parsed(await _bounded(request, ATTACH_BEGIN_MAX_BYTES)) + if not isinstance(body, dict): + return _refused(400, "this call's body is a JSON object") + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected if not isinstance(body, dict) or set(body) - {"step", "sha256"}: return _refused(400, "a rotation is a JSON object of step and sha256") step, sha = body.get("step"), body.get("sha256", "") @@ -578,7 +589,9 @@ async def rotate_credential(self, request: Request) -> JSONResponse: if step == "next" and _is_operator_hash(sha): return _refused(400, _OPERATOR_HASH_REFUSAL) response = await _operator_answer( - lambda: directory.rotate_credential(request.path_params["cell"], step, sha) + lambda: directory.rotate_credential( + request.path_params["cell"], step, sha, expected=expected + ) ) # This replica reads the hashes again on its next call. A 503 is included: its # change may have committed (BACKEND_UNAVAILABLE). @@ -632,7 +645,7 @@ async def register(self, request: Request) -> JSONResponse: ) async def _operator_body( - self, request: Request, limit: int = MAX_BODY_BYTES + self, request: Request, limit: int = ATTACH_BEGIN_MAX_BYTES ) -> dict[str, Any] | JSONResponse: """The operator call's JSON object, or its refusal: the credential, then the cell's name, then the body (413 past `limit`, 400 when it is no JSON object).""" @@ -660,6 +673,19 @@ async def operator_cell(self, request: Request) -> JSONResponse: lambda found: {**_attach_answer(found), "other": found.other}, ) + async def claim(self, request: Request) -> JSONResponse: + body = await self._operator_body(request, ATTACH_BEGIN_MAX_BYTES) + if isinstance(body, JSONResponse): + return body + expected = _control(body, claim=True) + if isinstance(expected, JSONResponse): + return expected + if body: + return _refused(400, "a control claim contains only expected") + return await _answered( + lambda: directory.claim(request.path_params["cell"], expected), _attach_answer + ) + async def attach_begin(self, request: Request) -> JSONResponse: """`attach/begin`: once the probe listener echoes attach's nonces from the stores the row names and the upstream reports the row's deployment id, write when attach @@ -668,6 +694,9 @@ async def attach_begin(self, request: Request) -> JSONResponse: body = await self._operator_body(request, ATTACH_BEGIN_MAX_BYTES) if isinstance(body, JSONResponse): return body + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected name = request.path_params["cell"] parsed = _begin(body) if isinstance(parsed, JSONResponse): @@ -676,17 +705,29 @@ async def attach_begin(self, request: Request) -> JSONResponse: found = await _read_attaching(name) if isinstance(found, JSONResponse): return found - if reason := directory.begin_refusal(found, begin): + if reason := directory.control_refusal(found, expected) or directory.begin_refusal( + found, begin + ): return _refused(409, reason) if refusal := await self._stores_check(found, nonces): return refusal - return await _answered(lambda: directory.attach_begin(name, begin), _attach_answer) + return await _answered( + lambda: directory.attach_begin(name, begin, expected=expected), _attach_answer + ) async def attach_place(self, request: Request) -> JSONResponse: """`attach/place {deployment_id, orgs}`: a batch of at most ATTACH_BATCH orgs.""" body = await self._operator_body(request) if isinstance(body, JSONResponse): return body + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected + if len(json.dumps(body, ensure_ascii=True).encode()) > MAX_BODY_BYTES: + return _refused( + 413, + f"a placement batch without its control envelope is at most {MAX_BODY_BYTES} bytes", + ) orgs = body.get("orgs") deployment_id = _text(body, "deployment_id", needed=True) if ( @@ -704,7 +745,7 @@ async def attach_place(self, request: Request) -> JSONResponse: name = request.path_params["cell"] batch = [o for o in named if o is not None] return await _answered( - lambda: directory.attach_place(name, deployment_id, batch), + lambda: directory.attach_place(name, deployment_id, batch, expected=expected), lambda counts: {"cell": name, "placed": counts[0], "placed_orgs": counts[1]}, ) @@ -713,12 +754,16 @@ async def attach_enforced(self, request: Request) -> JSONResponse: body = await self._operator_body(request) if isinstance(body, JSONResponse): return body + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected deployment_id = _text(body, "deployment_id", needed=True) if set(body) != {"deployment_id"} or deployment_id is None: return _refused(400, "attach/enforced is a JSON object of deployment_id") name = request.path_params["cell"] return await _answered( - lambda: directory.attach_enforced(name, deployment_id), _attach_answer + lambda: directory.attach_enforced(name, deployment_id, expected=expected), + _attach_answer, ) async def attach_serving(self, request: Request) -> JSONResponse: @@ -728,6 +773,9 @@ async def attach_serving(self, request: Request) -> JSONResponse: body = await self._operator_body(request) if isinstance(body, JSONResponse): return body + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected deployment_id = _text(body, "deployment_id", needed=True) nonces: Any = body.get("nonces") if set(body) != {"deployment_id", "nonces"} or deployment_id is None: @@ -738,13 +786,15 @@ async def attach_serving(self, request: Request) -> JSONResponse: found = await _read_attaching(name) if isinstance(found, JSONResponse): return found + if reason := directory.control_refusal(found, expected): + return _refused(409, reason) if not found.served(deployment_id): if reason := directory.attach_refusal(found, deployment_id, enforced=True): return _refused(409, reason) if refusal := await self._stores_check(found, nonces): return refusal response = await _answered( - lambda: directory.attach_serving(name, deployment_id), _attach_answer + lambda: directory.attach_serving(name, deployment_id, expected=expected), _attach_answer ) # A 503's change may have committed: this replica reads the cell's issuers and state # on its next request either way. @@ -760,13 +810,18 @@ async def remove(self, request: Request) -> JSONResponse: a cell `attaching` with no placement (`directory.remove`). The router cannot read the cell's Postgres: `loom cell remove` checks that a first cell's reads `open`, and a direct call skips that check.""" - if refusal := self._operator(request): - return refusal - if refusal := _malformed_cell_refusal(request): - return refusal + body = await self._operator_body(request, ATTACH_BEGIN_MAX_BYTES) + if isinstance(body, JSONResponse): + return body + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected + if body: + return _refused(400, "a removal contains only expected") name = request.path_params["cell"] response = await _answered( - lambda: directory.remove(name), lambda row: {"cell": row.cell, "removed": True} + lambda: directory.remove(name, expected=expected), + lambda row: {"cell": row.cell, "removed": True}, ) # A 503 is included: the removal may have committed (BACKEND_UNAVAILABLE). if response.status_code in (200, 503): @@ -1315,6 +1370,41 @@ async def _answered[T]( return JSONResponse(render(result)) +def _control(body: dict[str, Any], *, claim: bool = False) -> directory.Control | JSONResponse: + """Remove and validate the exact observed row; no caller may select a new token.""" + value = body.pop("expected", None) + if not isinstance(value, dict): + return _refused( + 409, + "cell control fencing requires upgraded control clients and every router replica", + code="CONTROL_UPGRADE_REQUIRED", + ) + keys = set(directory.CONTROL_FIELDS) | (set() if claim else {"mutation_generation"}) + if set(value) != keys: + return _refused( + 400, + "expected contains the complete registration, fingerprint pair and claimed generation", + ) + if type(value["is_first"]) is not bool or any( + not isinstance(value[name], str) or "\x00" in value[name] + for name in directory.CONTROL_IDENTITY + if name != "is_first" + ): + return _refused(400, "expected registration has invalid field types") + if any( + value[name] is not None and not isinstance(value[name], dict) + for name in ("current_fp", "next_fp") + ) or _holds_nul(value): + return _refused(400, "expected fingerprint pair contains JSON objects or null") + generation = None + if not claim: + try: + generation = UUID(value["mutation_generation"]) + except (TypeError, ValueError, AttributeError): + return _refused(400, "expected mutation_generation is a UUID") + return directory.Control({name: value[name] for name in directory.CONTROL_FIELDS}, generation) + + def _limits(row: directory.Cell) -> dict[str, Any]: return { "cell": row.cell, @@ -1344,6 +1434,8 @@ def _attach_answer(found: directory.Attaching) -> dict[str, Any]: "attach_started_at": started.isoformat() if started else None, "current_fp": row.current_fp, "next_fp": row.next_fp, + "mutation_generation": str(row.mutation_generation), + **{name: getattr(found, name) for name in directory._REGISTERED}, } @@ -1374,6 +1466,7 @@ async def _lifespan(_app: Starlette) -> AsyncIterator[None]: methods=["POST"], ), Route("/cells/v1/operator/cells/{cell}", router.operator_cell, methods=["GET"]), + Route("/cells/v1/operator/cells/{cell}/claim", router.claim, methods=["POST"]), Route("/cells/v1/operator/cells/{cell}", router.remove, methods=["DELETE"]), Route( "/cells/v1/operator/cells/{cell}/attach/begin", diff --git a/server/router/directory.py b/server/router/directory.py index a92967ed86..185c5126d6 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -25,6 +25,7 @@ from dataclasses import dataclass from datetime import datetime from typing import Any +from uuid import UUID from server.core import cell as cell_state from server.core.paths import ROUTER_MIGRATIONS_DIR @@ -59,9 +60,9 @@ def _bound_s(statements: int) -> int: return STATEMENT_TIMEOUT_S + (statements + 1) * (STATEMENT_TIMEOUT_S + 1) -# The longest a limit or credential change can take: eight statements (begin, lock, row, -# check, update, audit, read-back, commit). Registration has its own bound, below. -OPERATOR_CHANGE_MAX_S = _bound_s(8) # not a setting: derived from the bound above +# The longest limit/credential change has ten statements: begin, directory lock, row, +# other row, WAL, credential check, update, audit, read-back and commit. +OPERATOR_CHANGE_MAX_S = _bound_s(10) # not a setting: derived from the bound above # The longest a registration's directory write can take: thirteen statements (begin, lock, # row, five refusal reads, three inserts, read-back, commit); and its read before the probe: # six statements (the row, the first cell, then the credential, deployment id, host and @@ -72,11 +73,11 @@ def _bound_s(statements: int) -> int: # The attach calls' statements, which tests/unit/test_router_attach.py counts against the # code: the operator's read of a cell's row (the row, any other row, audit); the read before # `begin`'s and `serving`'s probe (the row, any other row); `begin`, `enforced` and -# `serving` (begin, lock, row, other row, update, audit, read-back, commit); a batch (begin, -# lock, row, other row, insert, placements elsewhere, count, audit, commit); and the removal -# of an unattached cell (begin, lock, row, delete hosts, delete row, audit, commit). +# `serving` (begin, lock, row, other row, WAL, update, audit, read-back, commit); a batch +# (begin, lock, row, other row, WAL, insert, placements elsewhere, count, audit, commit); +# removal adds other-row and WAL reads before deleting hosts/row and auditing. OPERATOR_READ_STATEMENTS, ATTACH_READ_STATEMENTS = 3, 2 -ATTACH_CHANGE_STATEMENTS, ATTACH_PLACE_STATEMENTS, REMOVE_STATEMENTS = 8, 9, 7 +ATTACH_CHANGE_STATEMENTS, ATTACH_PLACE_STATEMENTS, REMOVE_STATEMENTS = 9, 10, 9 OPERATOR_READ_MAX_S = _bound_s(OPERATOR_READ_STATEMENTS) # not a setting: derived REMOVE_MAX_S = _bound_s(REMOVE_STATEMENTS) # not a setting: derived # Every directory connection commits through the synchronous standby and reads what @@ -107,6 +108,7 @@ class Cell: next_fp: dict[str, Any] | None # The new credential's hash during a rotation (`loom cell credential --next`). credential_next_sha256: str | None + mutation_generation: UUID def holds(self, presented: str) -> bool: """Whether `presented` (a credential's SHA-256) is this cell's, current or next.""" @@ -231,7 +233,7 @@ async def migrate() -> list[str]: # The columns a Cell holds (read by name: `_cell`). _CELL_COLUMNS = ( "cell, is_first, host, state, mode, closed, placements_max, placed, credential_sha256, " - "credential_next_sha256, deployment_id, current_fp, next_fp" + "credential_next_sha256, deployment_id, current_fp, next_fp, mutation_generation" ) _CELL = f"SELECT {_CELL_COLUMNS} FROM loom_router.cells" @@ -302,6 +304,7 @@ async def set_limit( *, closed: bool | _Unset = UNSET, placements_max: int | None | _Unset = UNSET, + expected: Control, ) -> Cell: """`loom cell limit`: close or reopen a cell to new orgs and set or clear its placement quota, on its row alone, audited; the cell's stores are not touched. A @@ -324,10 +327,12 @@ async def change(conn: Any, row: Cell) -> dict[str, Any]: ) return detail - return await _operator_change(cell_name, "limit", change) + return await _operator_change(cell_name, "limit", change, expected) -async def rotate_credential(cell_name: str, step: str, sha256: str = "") -> Cell: +async def rotate_credential( + cell_name: str, step: str, sha256: str = "", *, expected: Control +) -> Cell: """`loom cell credential`: `next` records a new credential's hash beside the current one, refused when any cell holds it as current or next and while one is pending; `finish` makes `next` current and drops the old; `revert` drops `next`. @@ -365,23 +370,18 @@ async def change(conn: Any, row: Cell) -> dict[str, Any]: ) return {"step": step, **({"sha256": sha256} if sha256 else {})} - return await _operator_change(cell_name, "credential", change) + return await _operator_change(cell_name, "credential", change, expected) async def _operator_change( - cell_name: str, action: str, change: Callable[[Any, Cell], Awaitable[dict[str, Any]]] + cell_name: str, + action: str, + change: Callable[[Any, Cell], Awaitable[dict[str, Any]]], + expected: Control, ) -> Cell: - async with _reading(), _connection() as conn, conn.transaction(): - await conn.execute("SELECT pg_advisory_xact_lock($1)", _DIRECTORY_LOCK) - locked = await conn.fetchrow(f"{_CELL} WHERE cell = $1 FOR UPDATE", cell_name) - if locked is None: - raise NoSuchCell(f"cell {cell_name!r} is not in the directory") - detail = await change(conn, _cell(locked)) - await conn.execute( - "INSERT INTO loom_router.audit (actor, action, detail) VALUES ('operator', $1, $2)", - action, - {"cell": cell_name, **detail}, - ) + async with _attach_change(cell_name, expected) as (conn, found): + detail = await change(conn, found.row) + await _audit(conn, action, {"cell": cell_name, **detail}) row = await conn.fetchrow(f"{_CELL} WHERE cell = $1", cell_name) return _cell(row) @@ -866,17 +866,86 @@ async def attach_read(cell_name: str, *, audited: bool = False) -> Attaching: return _attaching(found, other) +# Every acceptance controller compares the full registration and exact JSON pair. +CONTROL_IDENTITY = ("cell", "is_first", "host", "deployment_id", *_REGISTERED) +CONTROL_FIELDS = (*CONTROL_IDENTITY, "current_fp", "next_fp") +CONTROL_MAX_S = _bound_s(10) # not a setting: transaction, commit and pool release bound + + +@dataclass(frozen=True) +class Control: + """The operator's observed registration and pair, with its claimed generation.""" + + values: dict[str, Any] + generation: UUID | None + + +def control_refusal(found: Attaching, expected: Control, *, claim: bool = False) -> str: + if not claim and expected.generation != found.row.mutation_generation: + return "cell control generation changed: stop and rerun the command with a fresh claim" + actual = { + name: getattr(found.row, name) if hasattr(found.row, name) else getattr(found, name) + for name in CONTROL_FIELDS + } + if _as_json(actual) != _as_json(expected.values): + return ( + "cell registration or fingerprints changed: stop and rerun from a fresh directory read" + ) + return "" + + @asynccontextmanager -async def _attach_change(cell_name: str) -> AsyncIterator[tuple[Any, Attaching]]: - """A directory transaction holding the directory lock and then the cell's row lock, with - the row and any other row's name.""" - async with _reading(), _connection() as conn, conn.transaction(): +async def _control_transaction() -> AsyncIterator[Any]: + """Bound acquisition, statements, COMMIT and release; never await rollback on cancel.""" + async with _reading(), asyncio.timeout(CONTROL_MAX_S), _connection() as conn: + settled = False + tx = conn.transaction() + try: + await tx.start() + try: + yield conn + except Exception: + await tx.rollback() + settled = True + raise + else: + await tx.commit() + settled = True + finally: + if not settled: + # A cancelled/unknown COMMIT cannot return this backend to the pool. + conn.terminate() + + +@asynccontextmanager +async def _attach_change( + cell_name: str, expected: Control, *, claim: bool = False +) -> AsyncIterator[tuple[Any, Attaching]]: + async with _control_transaction() as conn: await conn.execute("SELECT pg_advisory_xact_lock($1)", _DIRECTORY_LOCK) found = await conn.fetchrow(f"{_CELL_ATTACH} WHERE cell = $1 FOR UPDATE", cell_name) if found is None: raise NoSuchCell(f"cell {cell_name!r} is not in the directory") other = await conn.fetchval(_OTHER, cell_name) - yield conn, _attaching(found, other) + attaching = _attaching(found, other) + if reason := control_refusal(attaching, expected, claim=claim): + raise OperatorRefused(reason) + # A no-op still has a synchronous WAL commit: a primary-visible earlier write + # whose acknowledgement was lost is not yet a durability proof. + await conn.execute("SELECT pg_logical_emit_message(true, 'loom.control', '')") + yield conn, attaching + + +async def claim(cell_name: str, expected: Control) -> Attaching: + """Rotate on EVERY execution, including an identical HTTP replay.""" + async with _attach_change(cell_name, expected, claim=True) as (conn, _found): + await conn.execute( + "UPDATE loom_router.cells SET mutation_generation = gen_random_uuid() WHERE cell = $1", + cell_name, + ) + await _audit(conn, "control_claim", {"cell": cell_name}) + written = await conn.fetchrow(f"{_CELL_ATTACH} WHERE cell = $1", cell_name) + return _attaching(written, _found.other) async def _audit(conn: Any, action: str, detail: dict[str, Any]) -> None: @@ -887,11 +956,11 @@ async def _audit(conn: Any, action: str, detail: dict[str, Any]) -> None: ) -async def attach_begin(cell_name: str, begin: Begin) -> Attaching: +async def attach_begin(cell_name: str, begin: Begin, *, expected: Control) -> Attaching: """`attach/begin`: write `attach_started_at` once, the fingerprints while the row has none, and the root-secret fingerprint on every call, audited with the allow lists. Idempotent: a rerun posting the same fingerprints writes only `keys_fp`.""" - async with _attach_change(cell_name) as (conn, found): + async with _attach_change(cell_name, expected) as (conn, found): if reason := begin_refusal(found, begin): raise OperatorRefused(reason) await conn.execute( @@ -917,7 +986,9 @@ async def attach_begin(cell_name: str, begin: Begin) -> Attaching: return _attaching(written, None) -async def attach_place(cell_name: str, deployment_id: str, orgs: list[str]) -> tuple[int, int]: +async def attach_place( + cell_name: str, deployment_id: str, orgs: list[str], *, expected: Control +) -> tuple[int, int]: """`attach/place`: place a batch of the first cell's orgs, once its directory mode is `enforced`, past its closed mark and quota (the orgs are the cell's already); the count it placed now and the cell's count. An org already placed in the cell is skipped, so a @@ -927,7 +998,7 @@ async def attach_place(cell_name: str, deployment_id: str, orgs: list[str]) -> t reads `enforced`, judges that.""" _refuse_long(orgs) batch = sorted(set(orgs)) - async with _attach_change(cell_name) as (conn, found): + async with _attach_change(cell_name, expected) as (conn, found): if reason := attach_refusal(found, deployment_id, enforced=True): raise OperatorRefused(reason) new = await conn.fetch( @@ -969,10 +1040,10 @@ async def attach_place(cell_name: str, deployment_id: str, orgs: list[str]) -> t return len(new), int(placed) -async def attach_enforced(cell_name: str, deployment_id: str) -> Attaching: +async def attach_enforced(cell_name: str, deployment_id: str, *, expected: Control) -> Attaching: """`attach/enforced`: record directory mode `enforced` (registration wrote `recording`), after which the cell places new orgs live; idempotent.""" - async with _attach_change(cell_name) as (conn, found): + async with _attach_change(cell_name, expected) as (conn, found): if reason := attach_refusal(found, deployment_id): raise OperatorRefused(reason) await conn.execute( @@ -1001,10 +1072,10 @@ def issuers(row: Cell) -> list[str]: return sorted(found) -async def attach_serving(cell_name: str, deployment_id: str) -> Attaching: +async def attach_serving(cell_name: str, deployment_id: str, *, expected: Control) -> Attaching: """`attach/serving`: mark the first cell `serving`, with the issuers its fingerprints accept computed from the row as locked. A `serving` row is answered as it is.""" - async with _attach_change(cell_name) as (conn, found): + async with _attach_change(cell_name, expected) as (conn, found): row = found.row if found.served(deployment_id): await _audit(conn, "attach_serving", {"cell": cell_name, "serving_already": True}) @@ -1035,7 +1106,7 @@ def remove_refusal(cell_name: str, state: str, placed: int) -> str: return "" -async def remove(cell_name: str) -> Cell: +async def remove(cell_name: str, *, expected: Control) -> Cell: """`DELETE /cells/v1/operator/cells/` (`loom cell remove`): remove a cell's row and its host rows, audited, while it is `attaching` with no placement, whatever its directory mode. A placement in flight either commits first (and the removal refuses) or @@ -1043,12 +1114,8 @@ async def remove(cell_name: str) -> Cell: first cell's Postgres reads `open` first, and a direct call skips that check, leaving a first cell still prepared answering its new orgs 503 and unable to register again until the full back-out has run.""" - async with _reading(), _connection() as conn, conn.transaction(): - await conn.execute("SELECT pg_advisory_xact_lock($1)", _DIRECTORY_LOCK) - found = await conn.fetchrow(f"{_CELL} WHERE cell = $1 FOR UPDATE", cell_name) - if found is None: - raise NoSuchCell(f"cell {cell_name!r} is not in the directory") - row = _cell(found) + async with _attach_change(cell_name, expected) as (conn, found): + row = found.row if reason := remove_refusal(cell_name, row.state, row.placed): raise OperatorRefused(reason) await conn.execute("DELETE FROM loom_router.hosts WHERE cell = $1", cell_name) diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 3450558fe4..573d8fc85a 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -532,8 +532,17 @@ async def release() -> None: await tx.rollback() task = asyncio.create_task(release()) + await cell_attach_pg.take_lock(conn, "mode test") + own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") await cell_attach_pg.commit_enforced( - conn, ok, ok, cell_attach_pg.Wait(20), logged.append, lambda _m: None + conn, + ok, + ok, + cell_attach_pg.Wait(20), + logged.append, + lambda _m: None, + expected_cell="first", + expected_fp=(own["current_fp"], own["next_fp"]), ) await task assert await _mode() == "enforced" diff --git a/tests/integration/test_cell_control_fence.py b/tests/integration/test_cell_control_fence.py new file mode 100644 index 0000000000..a6799d2064 --- /dev/null +++ b/tests/integration/test_cell_control_fence.py @@ -0,0 +1,613 @@ +"""Control requests remain fenced after their cell's lock backend dies.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import os +import uuid +from collections.abc import AsyncIterator +from typing import Any +from urllib.parse import urlsplit, urlunsplit + +import httpx +import pytest + +from server.core import cell +from server.router import app, directory +from server.settings import settings + +TOKEN = "lco_fence-test" + + +@pytest.fixture +async def control_stores(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[Any]: + import asyncpg + + dsn = os.environ.get("LOOM_ROUTER_PG_DSN") or pytest.skip("LOOM_ROUTER_PG_DSN not set") + admin = await asyncpg.connect(dsn) + name = f"loom_fence_{uuid.uuid4().hex[:12]}" + await admin.execute(f'CREATE DATABASE "{name}"') + own = urlunsplit(urlsplit(dsn)._replace(path=f"/{name}")) + monkeypatch.setattr(settings, "router_pg_dsn", own) + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(TOKEN.encode()).hexdigest() + ) + monkeypatch.setattr(directory, "_pool", None) + conn = await asyncpg.connect(own) + try: + await directory.migrate() + await conn.execute( + "INSERT INTO loom_router.cells (cell, is_first, host, deployment_id, credential_sha256, state) " + "VALUES ('first', true, 'first.example.test', 'dep-first', $1, 'attaching')", + "a" * 64, + ) + transport = httpx.ASGITransport(app=app.build_app()) + async with httpx.AsyncClient( + transport=transport, + base_url="http://router", + headers={"Authorization": f"Bearer {TOKEN}"}, + ) as client: + yield client, conn, own + finally: + await conn.close() + await directory.close() + await admin.execute(f'DROP DATABASE "{name}" WITH (FORCE)') + await admin.close() + + +def expected(row: dict[str, Any], *, generation: bool = True) -> dict[str, Any]: + out = { + k: row[k] + for k in ( + "cell", + "is_first", + "host", + "deployment_id", + "bucket", + "bucket_prefix", + "probe_url", + "upstream_url", + "current_fp", + "next_fp", + ) + } + if generation: + out["mutation_generation"] = row["mutation_generation"] + return {"expected": out} + + +async def read(client: httpx.AsyncClient) -> dict[str, Any]: + response = await client.get("/cells/v1/operator/cells/first") + assert response.status_code == 200, response.text + return response.json() + + +async def claim(client: httpx.AsyncClient, row: dict[str, Any]) -> dict[str, Any]: + response = await client.post( + "/cells/v1/operator/cells/first/claim", json=expected(row, generation=False) + ) + assert response.status_code == 200, response.text + return response.json() + + +@pytest.mark.asyncio +async def test_stale_http_after_backend_death_and_pair_aba(control_stores, monkeypatch) -> None: + import asyncpg + + client, conn, own = control_stores + a = await asyncpg.connect(own) + b = await asyncpg.connect(own) + release = asyncio.Event() + entered = asyncio.Event() + task = None + try: + await a.execute("SELECT pg_advisory_lock($1)", cell.ATTACH_LOCK_KEY) + snapshot = await read(client) + old = await claim(client, snapshot) + real = directory.set_limit + + async def delayed(*args, **kwargs): + entered.set() + await release.wait() + return await real(*args, **kwargs) + + monkeypatch.setattr(directory, "set_limit", delayed) + task = asyncio.create_task( + client.post( + "/cells/v1/operator/cells/first/limit", json={**expected(old), "closed": True} + ) + ) + await asyncio.wait_for(entered.wait(), 3) + await conn.execute("SELECT pg_terminate_backend($1)", a.get_server_pid()) + assert await b.fetchval("SELECT pg_try_advisory_lock($1)", cell.ATTACH_LOCK_KEY) + newer = await claim(client, await read(client)) + # A complete pair ABA cannot reauthorize A: only the current generation can write. + await conn.execute( + "UPDATE loom_router.cells SET current_fp = '{\"v\":1}', next_fp = '{\"v\":1}'" + ) + await conn.execute("UPDATE loom_router.cells SET current_fp = NULL, next_fp = NULL") + assert newer["mutation_generation"] != old["mutation_generation"] + release.set() + response = await task + assert response.status_code == 409, response.text + assert not await conn.fetchval("SELECT closed FROM loom_router.cells") + finally: + release.set() + if task is not None: + await task + await a.close() + await b.close() + + +@pytest.mark.asyncio +async def test_replayed_claim_lost_ack_and_noop_wrong_generation(control_stores) -> None: + client, conn, _ = control_stores + snapshot = await read(client) + lost = await claim(client, snapshot) # its caller lost this acknowledgement + current = await claim(client, snapshot) # duplicate HTTP execution must rotate again + assert lost["mutation_generation"] != current["mutation_generation"] + response = await client.post( + "/cells/v1/operator/cells/first/limit", json={**expected(lost), "closed": False} + ) + assert response.status_code == 409, response.text + response = await client.post( + "/cells/v1/operator/cells/first/limit", json={**expected(current), "closed": False} + ) + assert response.status_code == 200, response.text + assert ( + str(await conn.fetchval("SELECT mutation_generation FROM loom_router.cells")) + == current["mutation_generation"] + ) + + +@pytest.fixture +async def local_control(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[Any]: + import asyncpg + + from server.core import pg + from tests.integration._pg_database import own_database + + async with own_database() as dsn: + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "pg_dsn", dsn) + monkeypatch.setattr(settings, "document_s3_bucket", "") + conn = await asyncpg.connect(dsn, server_settings={pg.TENANCY_MARKER: "on"}) + await directory._init(conn) + await conn.execute( + "INSERT INTO loom.authority (authority, epoch, deployment_id) VALUES ('postgres', 1, 'dep-first')" + ) + try: + yield conn, dsn + finally: + await conn.close() + await pg.close_pool() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("writer", ["prepare", "attach"]) +async def test_dead_local_lock_backend_cannot_write_acceptance(local_control, writer) -> None: + import asyncpg + + from server.cli import cell_attach_pg as apg + from server.cli import cell_prepare + + admin, dsn = local_control + dead = await asyncpg.connect(dsn) + await directory._init(dead) + await dead.execute("SELECT pg_advisory_lock($1)", cell.ATTACH_LOCK_KEY) + rows = cell_prepare._CellRows("dep-first", None, "", None, None, 0) + fp = cell.as_value({"deployment_id": "dep-first"}) + await admin.execute("SELECT pg_terminate_backend($1)", dead.get_server_pid()) + successor = await asyncpg.connect(dsn) + try: + assert await successor.fetchval("SELECT pg_try_advisory_lock($1)", cell.ATTACH_LOCK_KEY) + if writer == "prepare": + with pytest.raises( + (asyncpg.InterfaceError, asyncpg.PostgresError, asyncpg.InternalClientError) + ): + await cell_prepare._write_rows( + dead, rows, {"is_first": True, "mode": "recording"}, fp, None + ) + assert not await admin.fetchval("SELECT EXISTS(SELECT 1 FROM loom.cell_self)") + else: + # A dead controller cannot switch to another session to complete its mode write. + await admin.execute( + "INSERT INTO loom.cell_first(is_first, cell) VALUES (true, 'first')" + ) + await admin.execute("INSERT INTO loom.tenancy_mode(mode) VALUES ('recording')") + await admin.execute("INSERT INTO loom.cell_self(current_fp) VALUES ($1)", fp) + with pytest.raises( + (asyncpg.InterfaceError, asyncpg.PostgresError, asyncpg.InternalClientError) + ): + await apg._mode_try(dead, 100, lambda _m: None, "first", (fp, None)) + assert await admin.fetchval("SELECT mode FROM loom.tenancy_mode") == "recording" + finally: + await successor.close() + await dead.close() + + +@pytest.mark.asyncio +async def test_same_session_prepare_and_mode_commit(local_control) -> None: + from server.cli import cell_attach_pg as apg + from server.cli import cell_control, cell_prepare + + conn, _ = local_control + await apg.take_lock(conn, "test") + fp = cell.as_value({"deployment_id": "dep-first"}) + rows = await cell_prepare._read_rows(conn) + written = await cell_prepare._write_rows( + conn, rows, {"is_first": True, "mode": "recording"}, fp, None + ) + assert written == ["first", "mode", "own"] + assert await apg.lock_held(conn) + assert await apg._mode_try(conn, 100, lambda _m: None, "first", (fp, None)) + result = await apg.durable_read(conn) + assert result.mode == "enforced" and result.first == (True, "first") + assert await apg.lock_held(conn) + # No-op prepare remains a same-backend synchronous durability boundary. + rows = await cell_prepare._read_rows(conn) + assert ( + await cell_prepare._write_rows(conn, rows, {"is_first": True, "mode": "enforced"}, fp, None) + == [] + ) + async with cell_control.transaction(conn): + assert await conn.fetchval("SELECT pg_backend_pid()") == conn.get_server_pid() + + +@pytest.mark.asyncio +async def test_recreated_row_fences_previous_generation(control_stores) -> None: + client, conn, _ = control_stores + previous = await claim(client, await read(client)) + response = await client.request( + "DELETE", "/cells/v1/operator/cells/first", json=expected(previous) + ) + assert response.status_code == 200, response.text + await conn.execute( + "INSERT INTO loom_router.cells(cell, is_first, host, state, deployment_id, credential_sha256) VALUES ('first', true, 'first.example.test', 'attaching', 'dep-first', $1)", + "a" * 64, + ) + recreated = await read(client) + assert recreated["mutation_generation"] != previous["mutation_generation"] + response = await client.post( + "/cells/v1/operator/cells/first/limit", json={**expected(previous), "closed": True} + ) + assert response.status_code == 409 + assert not await conn.fetchval("SELECT closed FROM loom_router.cells") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "field,value", + [("current_fp", {"v": 1, "fields": {"x": False}}), ("host", "other.example.test")], +) +async def test_claim_checks_exact_pair_and_registration(control_stores, field, value) -> None: + client, conn, _ = control_stores + original = await read(client) + if field == "current_fp": + await conn.execute('UPDATE loom_router.cells SET current_fp = \'{"v":1,"fields":{"x":0}}\'') + original[field] = value + response = await client.post( + "/cells/v1/operator/cells/first/claim", json=expected(original, generation=False) + ) + assert response.status_code == 409 + + +@pytest.mark.asyncio +async def test_missing_generation_requires_upgrade_after_auth(control_stores) -> None: + client, conn, _ = control_stores + for path in ( + "limit", + "credential", + "attach/begin", + "attach/place", + "attach/enforced", + "attach/serving", + ): + response = await client.post(f"/cells/v1/operator/cells/first/{path}", json={}) + assert (response.status_code, response.json()["code"]) == (409, "CONTROL_UPGRADE_REQUIRED") + response = await client.post( + f"/cells/v1/operator/cells/first/{path}", + content=b"broken", + headers={"Authorization": "Bearer lco_wrong"}, + ) + assert response.status_code == 401 + assert await conn.fetchval("SELECT count(*) FROM loom_router.audit") == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("aba", [False, True]) +async def test_inflight_finalizer_cannot_release_after_new_owner_rewidens( + control_stores, + local_control, + aba, +) -> None: + """Use the production mutation guard with a test-only narrowing HTTP action. + + The update command is a later concern. This exercises its directory boundary without + introducing an update route or implementation in this foundation. + """ + import asyncpg + from starlette.responses import JSONResponse + from starlette.routing import Route + + from server.cli import cell_attach_pg as apg + from server.cli import cell_control + + client, directory_admin, _ = control_stores + local_admin, local_dsn = local_control + current = {"v": 1, "fields": {"deployment_id": "dep-first", "bearer_issuers": ["old"]}} + nxt = {"v": 1, "fields": {"deployment_id": "dep-first", "bearer_issuers": ["new"]}} + await directory_admin.execute( + "UPDATE loom_router.cells SET current_fp = $1::jsonb, next_fp = $2::jsonb", + json.dumps(current), + json.dumps(nxt), + ) + await local_admin.execute("INSERT INTO loom.cell_first(is_first, cell) VALUES (true, 'first')") + await local_admin.execute("INSERT INTO loom.tenancy_mode(mode) VALUES ('recording')") + await local_admin.execute( + "INSERT INTO loom.cell_self(current_fp, next_fp) VALUES ($1, $2)", nxt, None + ) + a = await asyncpg.connect(local_dsn) + b = await asyncpg.connect(local_dsn) + await directory._init(b) + started, resume = asyncio.Event(), asyncio.Event() + task = None + router = app.Router() + + async def finalizer(request): + body = await router._operator_body(request) + if isinstance(body, JSONResponse): + return body + compare = app._control(body) + if isinstance(compare, JSONResponse): + return compare + started.set() + await resume.wait() # request already sent; caller can no longer retract it + + async def change(): + async with directory._attach_change("first", compare) as (conn, found): + await conn.execute( + "UPDATE loom_router.cells SET current_fp = next_fp, next_fp = NULL, issuers = ARRAY['new'] WHERE cell = 'first'" + ) + return found + + return await app._answered(change, app._attach_answer) + + client._transport.app.routes.insert( + 0, Route("/test-finalize/{cell}", finalizer, methods=["POST"]) + ) + try: + await apg.take_lock(a, "old controller") + old = await claim(client, await read(client)) + task = asyncio.create_task(client.post("/test-finalize/first", json=expected(old))) + await asyncio.wait_for(started.wait(), 3) + await local_admin.execute("SELECT pg_terminate_backend($1)", a.get_server_pid()) + await apg.take_lock(b, "new controller") + newer = await claim(client, await read(client)) + # New owner claims BEFORE widening local acceptance, on the owned backend. + async with cell_control.transaction(b): + await b.execute("UPDATE loom.cell_self SET current_fp = $1, next_fp = $2", current, nxt) + if aba: + await directory_admin.execute( + "UPDATE loom_router.cells SET current_fp = next_fp, next_fp = NULL" + ) + await directory_admin.execute( + "UPDATE loom_router.cells SET current_fp = $1::jsonb, next_fp = $2::jsonb", + json.dumps(current), + json.dumps(nxt), + ) + assert newer["mutation_generation"] != old["mutation_generation"] + resume.set() + response = await task + assert response.status_code == 409, response.text + assert (await read(client))["next_fp"] == nxt + assert await local_admin.fetchval("SELECT next_fp FROM loom.cell_self") == nxt + finally: + resume.set() + if task is not None: + await task + await a.close() + await b.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("placement_first", [False, True]) +async def test_backout_closure_serializes_queued_placement( + control_stores, + local_control, + placement_first, +) -> None: + import asyncpg + + from server.cli import cell_attach_pg as apg + from server.cli import cell_control + + client, directory_admin, directory_dsn = control_stores + local_admin, _ = local_control + await directory_admin.execute("UPDATE loom_router.cells SET mode = 'enforced'") + await local_admin.execute("INSERT INTO loom.cell_first(is_first, cell) VALUES (true, 'first')") + await local_admin.execute("INSERT INTO loom.tenancy_mode(mode) VALUES ('recording')") + await local_admin.execute("INSERT INTO loom.cell_self(current_fp) VALUES ('{\"v\":1}')") + await apg.take_lock(local_admin, "backout test") + original = await read(client) + blocker = await asyncpg.connect(directory_dsn) + placement = cleanup = None + + async def call(path, body): + response = await client.post("/cells/v1/operator/cells/first" + path, json=body) + if response.status_code != 200: + raise cell_control._refused(response.json()["error"]) + return response.json() + + async def wait_for_lock(key): + for _ in range(100): + count = await directory_admin.fetchval( + "SELECT count(*) FROM pg_locks WHERE locktype = 'advisory' AND classid::bigint = $1 AND NOT granted", + key, + ) + if count: + return + await asyncio.sleep(0.01) + pytest.fail("owned test request never reached its lock") + + tx = blocker.transaction() + await tx.start() + try: + if placement_first: + await blocker.fetchrow( + "SELECT cell FROM loom_router.cells WHERE cell = 'first' FOR UPDATE" + ) + placement = asyncio.create_task( + directory.place("queued", "first", app.placement_refusal) + ) + # The placement has its org lock and is queued at the cell row before claim. + for _ in range(100): + if await directory_admin.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_locks WHERE locktype='advisory' AND classid::bigint=$1 AND granted)", + directory._PLACE_LOCK, + ): + break + await asyncio.sleep(0.01) + else: + pytest.fail("placement did not acquire its org lock") + cleanup = asyncio.create_task(cell_control.backout(local_admin, call, original)) + for _ in range(100): + waiting = await directory_admin.fetchval( + "SELECT count(*) FROM pg_stat_activity WHERE datname = current_database() " + "AND application_name = 'loom-router' AND wait_event_type = 'Lock'" + ) + if waiting >= 2: + break + await asyncio.sleep(0.01) + else: + pytest.fail("placement and claim did not both queue behind the cell row") + await tx.commit() + assert await placement == ("first", "dep-first") + with pytest.raises(cell_control.Refused, match="holds placements"): + await cleanup + assert await local_admin.fetchval("SELECT EXISTS(SELECT 1 FROM loom.cell_self)") + assert await local_admin.fetchval("SELECT mode FROM loom.tenancy_mode") == "recording" + else: + await blocker.execute( + "SELECT pg_advisory_xact_lock($1, hashtext($2))", directory._PLACE_LOCK, "queued" + ) + placement = asyncio.create_task( + directory.place("queued", "first", app.placement_refusal) + ) + await wait_for_lock(directory._PLACE_LOCK) + await cell_control.backout(local_admin, call, original) + assert not await local_admin.fetchval("SELECT EXISTS(SELECT 1 FROM loom.cell_self)") + await tx.commit() + with pytest.raises(directory.PlacementRefused, match="closed"): + await placement + assert await directory_admin.fetchval("SELECT closed FROM loom_router.cells") + assert (await read(client))["placed_orgs"] == int(placement_first) + finally: + if blocker.is_in_transaction(): + await tx.rollback() + for task in (placement, cleanup): + if task is not None and not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + except Exception: # noqa: BLE001 — preserve the test failure while reaping owned tasks + pass + await blocker.close() + + +@pytest.mark.asyncio +async def test_registration_rerun_keeps_claim_and_prior_column_lists_still_work(control_stores): + client, conn, _ = control_stores + held = await claim(client, await read(client)) + reg = directory.Registration( + cell="first", + is_first=True, + host="first.example.test", + deployment_id="dep-first", + credential_sha256="a" * 64, + bucket="", + bucket_prefix="", + probe_url="", + tenancy_rows=0, + state_rows=0, + prefix_objects=0, + processes_without_cell=0, + ) + row, written = await directory.register(reg) + assert not written and str(row.mutation_generation) == held["mutation_generation"] + # The prior data plane uses explicit lists and does not interpret the added column. + legacy = await conn.fetchrow( + "SELECT cell, is_first, host, state, mode, closed, placements_max, placed, credential_sha256, credential_next_sha256, deployment_id, current_fp, next_fp FROM loom_router.cells" + ) + assert legacy["cell"] == "first" and "mutation_generation" not in legacy + await conn.execute( + "INSERT INTO loom_router.cells(cell,host,state,credential_sha256,deployment_id) VALUES ('later','later.example.test','attaching',$1,'dep-later')", + "b" * 64, + ) + generations = await conn.fetch("SELECT mutation_generation FROM loom_router.cells") + assert len({r["mutation_generation"] for r in generations}) == 2 + + +@pytest.mark.asyncio +async def test_attach_lock_dies_between_confirmation_and_mode_write(local_control): + """The ready callback is a real scheduling gap after the lock check.""" + import argparse + + import asyncpg + + from server.cli import cell_attach + from server.cli import cell_attach_pg as apg + + admin, dsn = local_control + fp = cell.as_value({"deployment_id": "dep-first"}) + await admin.execute("INSERT INTO loom.cell_first(is_first,cell) VALUES(true,'first')") + await admin.execute("INSERT INTO loom.tenancy_mode(mode) VALUES('recording')") + await admin.execute("INSERT INTO loom.cell_self(current_fp) VALUES($1)", fp) + a, b = await asyncpg.connect(dsn), await asyncpg.connect(dsn) + await apg.take_lock(a, "old attach") + + class Router: + async def call(self, path, body): + assert path == "/attach/enforced" + return {"cell": "first"} + + run = cell_attach.Run( + None, argparse.Namespace(cell="first", wait=0), Router(), lambda _text: None + ) + run.lock_conn = a + run.mode = "recording" + run.own_fp = (fp, None) + run.row = { + "cell": "first", + "is_first": True, + "host": "first.example.test", + "deployment_id": "dep-first", + "bucket": "", + "bucket_prefix": "", + "probe_url": "", + "upstream_url": "", + "current_fp": fp, + "next_fp": None, + "mutation_generation": str(uuid.UUID(int=1)), + } + + async def ready(): + await admin.execute("SELECT pg_terminate_backend($1)", a.get_server_pid()) + await apg.take_lock(b, "successor") + return [] + + run.ready_for_commit = ready + try: + with pytest.raises( + (asyncpg.InterfaceError, asyncpg.PostgresError, asyncpg.InternalClientError) + ): + await run.commit_mode() + assert await apg.lock_held(b) + assert await admin.fetchval("SELECT mode FROM loom.tenancy_mode") == "recording" + finally: + await a.close() + await b.close() diff --git a/tests/integration/test_router_directory_pg.py b/tests/integration/test_router_directory_pg.py index caf092e9ec..cbb96ba4cc 100644 --- a/tests/integration/test_router_directory_pg.py +++ b/tests/integration/test_router_directory_pg.py @@ -22,12 +22,31 @@ from server.settings import settings +async def _observed(name: str) -> directory.Control: + try: + found = await directory.attach_read(name) + except directory.NoSuchCell: + return directory.Control({}, uuid.UUID(int=0)) + values = { + key: getattr(found.row, key) if hasattr(found.row, key) else getattr(found, key) + for key in directory.CONTROL_FIELDS + } + return directory.Control(values, found.row.mutation_generation) + + @pytest.fixture -def dsn(monkeypatch: pytest.MonkeyPatch) -> str: - value = os.environ.get("LOOM_ROUTER_PG_DSN") or pytest.skip("LOOM_ROUTER_PG_DSN not set") +async def dsn(monkeypatch: pytest.MonkeyPatch): + from tests.integration._pg_database import create_database, drop_database + + admin_dsn = os.environ.get("LOOM_ROUTER_PG_DSN") or pytest.skip("LOOM_ROUTER_PG_DSN not set") + name, value = await create_database(admin_dsn) monkeypatch.setattr(settings, "router_pg_dsn", value) monkeypatch.setattr(directory, "_pool", None) - return value + try: + yield value + finally: + await directory.close() + await drop_database(admin_dsn, name) @pytest.fixture @@ -228,52 +247,72 @@ async def row(cell: str) -> dict[str, Any]: # The operator's changes: a limit, and a credential rotation with its refusals. # Each writes the fields it names and updated_at, and leaves every other column. before = await row(b) - limited = await directory.set_limit(b, closed=True, placements_max=7) + limited = await directory.set_limit( + b, closed=True, placements_max=7, expected=await _observed(b) + ) assert (limited.closed, limited.placements_max) == (True, 7) after = await row(b) assert after["updated_at"] > before["updated_at"] changed = {"closed": True, "placements_max": 7, "updated_at": after["updated_at"]} assert after == {**before, **changed} - cleared = await directory.set_limit(b, placements_max=None) + cleared = await directory.set_limit(b, placements_max=None, expected=await _observed(b)) assert (cleared.closed, cleared.placements_max) == (True, None) # closed is kept assert await row(b) == {**after, "placements_max": None, "updated_at": ANY} with pytest.raises(ValueError): - await directory.set_limit(b) # nothing to set: no statement + await directory.set_limit( + b, expected=await _observed(b) + ) # nothing to set: no statement with pytest.raises(ValueError): - await directory.rotate_credential(b, "finsh") # no such step: nothing written + await directory.rotate_credential( + b, "finsh", expected=await _observed(b) + ) # no such step: nothing written with pytest.raises(ValueError): - await directory.rotate_credential(b, "next") # no hash to record + await directory.rotate_credential( + b, "next", expected=await _observed(b) + ) # no hash to record with pytest.raises(ValueError): - await directory.rotate_credential(b, "finish", "c" * 64) # a hash finish ignores + await directory.rotate_credential( + b, "finish", "c" * 64, expected=await _observed(b) + ) # a hash finish ignores with pytest.raises(directory.NoSuchCell): - await directory.set_limit(f"nope-{b}", closed=True) + await directory.set_limit( + f"nope-{b}", closed=True, expected=await _observed(f"nope-{b}") + ) sha_b = hashlib.sha256(b.encode()).hexdigest() with pytest.raises(directory.OperatorRefused, match="records that credential hash"): - await directory.rotate_credential(a, "next", sha_b) # b's current hash + await directory.rotate_credential( + a, "next", sha_b, expected=await _observed(a) + ) # b's current hash with pytest.raises(directory.OperatorRefused, match="no next credential"): - await directory.rotate_credential(a, "finish") + await directory.rotate_credential(a, "finish", expected=await _observed(a)) new = hashlib.sha256(f"new-{a}".encode()).hexdigest() before = await row(a) - assert (await directory.rotate_credential(a, "next", new)).credential_next_sha256 == new + assert ( + await directory.rotate_credential(a, "next", new, expected=await _observed(a)) + ).credential_next_sha256 == new assert await row(a) == {**before, "credential_next_sha256": new, "updated_at": ANY} assert (await directory.credentials())[a] == (hashlib.sha256(a.encode()).hexdigest(), new) with pytest.raises(directory.OperatorRefused, match="already cell .* next credential"): - await directory.rotate_credential(a, "next", new) # this change, retried + await directory.rotate_credential( + a, "next", new, expected=await _observed(a) + ) # this change, retried with pytest.raises(directory.OperatorRefused, match="another next credential pending"): - await directory.rotate_credential(a, "next", "e" * 64) + await directory.rotate_credential(a, "next", "e" * 64, expected=await _observed(a)) with pytest.raises(directory.OperatorRefused, match="records that credential hash"): - await directory.rotate_credential(b, "next", new) # a's next hash + await directory.rotate_credential( + b, "next", new, expected=await _observed(b) + ) # a's next hash # Two cells asking for one next hash at once: the directory lock lets one in. raced = hashlib.sha256(f"raced-{a}".encode()).hexdigest() both = await asyncio.gather( - directory.rotate_credential(b, "next", raced), - directory.rotate_credential(q, "next", raced), + directory.rotate_credential(b, "next", raced, expected=await _observed(b)), + directory.rotate_credential(q, "next", raced, expected=await _observed(q)), return_exceptions=True, ) assert sum(isinstance(r, directory.Cell) for r in both) == 1 assert sum(isinstance(r, directory.OperatorRefused) for r in both) == 1 pending = await row(a) - finished = await directory.rotate_credential(a, "finish") + finished = await directory.rotate_credential(a, "finish", expected=await _observed(a)) assert (finished.credential_sha256, finished.credential_next_sha256) == (new, None) assert await row(a) == { **pending, @@ -282,11 +321,13 @@ async def row(cell: str) -> dict[str, Any]: "updated_at": ANY, } winner = b if isinstance(both[0], directory.Cell) else q - reverted = await directory.rotate_credential(winner, "revert") + reverted = await directory.rotate_credential( + winner, "revert", expected=await _observed(winner) + ) assert reverted.credential_next_sha256 is None assert (await directory.credentials())[winner] == (reverted.credential_sha256,) with pytest.raises(directory.OperatorRefused, match="no next credential"): - await directory.rotate_credential(winner, "revert") + await directory.rotate_credential(winner, "revert", expected=await _observed(winner)) def steps(cell: str) -> Any: return conn.fetch( @@ -587,16 +628,22 @@ async def test_the_first_cell_attaches_through_the_operator_calls(own_dsn: str) try: # Nothing before begin; begin writes its start once and the fingerprints. with pytest.raises(directory.OperatorRefused, match="has not begun"): - await directory.attach_enforced(name, dep) + await directory.attach_enforced(name, dep, expected=await _observed(name)) with pytest.raises(directory.OperatorRefused, match="has not begun"): - await directory.attach_place(name, dep, ["a"]) + await directory.attach_place(name, dep, ["a"], expected=await _observed(name)) with pytest.raises(directory.OperatorRefused, match="deployment id"): - await directory.attach_begin(name, directory.Begin("dep-x", _fp(name), None, "k", {})) - found = await directory.attach_begin(name, begin) + await directory.attach_begin( + name, + directory.Begin("dep-x", _fp(name), None, "k", {}), + expected=await _observed(name), + ) + found = await directory.attach_begin(name, begin, expected=await _observed(name)) started = found.attach_started_at assert started is not None and found.row.current_fp == _fp(name) again = await directory.attach_begin( - name, directory.Begin(dep, _fp(name), None, "keys-2", {}) + name, + directory.Begin(dep, _fp(name), None, "keys-2", {}), + expected=await _observed(name), ) assert again.attach_started_at == started stored = await conn.fetchrow( @@ -609,35 +656,45 @@ async def test_the_first_cell_attaches_through_the_operator_calls(own_dsn: str) ) with pytest.raises(directory.OperatorRefused, match="loom cell update"): await directory.attach_begin( - name, directory.Begin(dep, _fp(name, oidc_issuer="x"), None, "k", {}) + name, + directory.Begin(dep, _fp(name, oidc_issuer="x"), None, "k", {}), + expected=await _observed(name), ) # A batch waits for the directory mode `enforced`. with pytest.raises(directory.OperatorRefused, match="attach/enforced first"): - await directory.attach_place(name, dep, ["a"]) - enforced = await directory.attach_enforced(name, dep) + await directory.attach_place(name, dep, ["a"], expected=await _observed(name)) + enforced = await directory.attach_enforced(name, dep, expected=await _observed(name)) assert enforced.row.mode == "enforced" - await directory.attach_enforced(name, dep) # idempotent + await directory.attach_enforced(name, dep, expected=await _observed(name)) # idempotent # A closed cell with a quota of 0 still takes its own orgs; a rerun places none # twice. await conn.execute( "UPDATE loom_router.cells SET closed = true, placements_max = 0 WHERE cell = $1", name ) - assert await directory.attach_place(name, dep, ["b", "a", "a"]) == (2, 2) - assert await directory.attach_place(name, dep, ["a", "c"]) == (1, 3) + assert await directory.attach_place( + name, dep, ["b", "a", "a"], expected=await _observed(name) + ) == (2, 2) + assert await directory.attach_place( + name, dep, ["a", "c"], expected=await _observed(name) + ) == (1, 3) placed = await conn.fetch( "SELECT org FROM loom_router.placements WHERE cell = $1 ORDER BY org", name ) assert [r["org"] for r in placed] == ["a", "b", "c"] with pytest.raises(directory.OrgTooLong): - await directory.attach_place(name, dep, ["d", "x" * 2049]) - served = await directory.attach_serving(name, dep) + await directory.attach_place( + name, dep, ["d", "x" * 2049], expected=await _observed(name) + ) + served = await directory.attach_serving(name, dep, expected=await _observed(name)) assert served.row.state == "serving" assert await conn.fetchval( "SELECT issuers FROM loom_router.cells WHERE cell = $1", name ) == [f"https://{name}.example.test"] - assert (await directory.attach_serving(name, dep)).row.state == "serving" # done + assert ( + await directory.attach_serving(name, dep, expected=await _observed(name)) + ).row.state == "serving" # done with pytest.raises(directory.OperatorRefused, match="serving already"): - await directory.attach_enforced(name, dep) + await directory.attach_enforced(name, dep, expected=await _observed(name)) read = await directory.attach_read(name, audited=True) assert (read.row.state, read.other) == ("serving", None) audit = await conn.fetch( @@ -686,8 +743,10 @@ async def _attaching_first(own_dsn: str) -> tuple[str, str]: name = f"lr-{uuid.uuid4().hex[:8]}" dep = f"dep-{name}" await directory.register(_first(name)) - await directory.attach_begin(name, directory.Begin(dep, _fp(name), None, "k", {})) - await directory.attach_enforced(name, dep) + await directory.attach_begin( + name, directory.Begin(dep, _fp(name), None, "k", {}), expected=await _observed(name) + ) + await directory.attach_enforced(name, dep, expected=await _observed(name)) return name, dep @@ -740,7 +799,9 @@ async def _held(*a: Any) -> None: async with asyncio.timeout(30): live = asyncio.ensure_future(directory.place("acme", name, lambda _row: "")) await inserted.wait() - batch = asyncio.ensure_future(directory.attach_place(name, dep, ["acme", "beta"])) + batch = asyncio.ensure_future( + directory.attach_place(name, dep, ["acme", "beta"], expected=await _observed(name)) + ) await _waits_on_the_row_lock(conn, "FOR UPDATE") release.set() assert await live == (name, dep) @@ -768,8 +829,8 @@ async def test_a_live_placement_racing_a_batch_of_the_same_org_answers_it_placed real = directory._attach_change @contextlib.asynccontextmanager - async def _held(cell_name: str) -> Any: - async with real(cell_name) as locked: + async def _held(cell_name: str, expected: directory.Control) -> Any: + async with real(cell_name, expected) as locked: yield locked written.set() await release.wait() # the batch's writes are done, its commit held @@ -780,7 +841,9 @@ async def _held(cell_name: str) -> Any: # A full quota: a live placement judged on it would be refused. await conn.execute("UPDATE loom_router.cells SET placements_max = 0 WHERE cell = $1", name) async with asyncio.timeout(30): - batch = asyncio.ensure_future(directory.attach_place(name, dep, ["acme"])) + batch = asyncio.ensure_future( + directory.attach_place(name, dep, ["acme"], expected=await _observed(name)) + ) await written.wait() live = asyncio.ensure_future(directory.place("acme", name, lambda _row: "")) await _waits_on_the_row_lock(conn, "FOR UPDATE") @@ -821,23 +884,29 @@ async def test_an_attaching_cell_with_no_placement_is_removed_with_its_hosts( later, ) with pytest.raises(directory.OperatorRefused, match=f"loom cell remove --cell {later}"): - await directory.attach_begin(first, directory.Begin(dep, _fp(first), None, "k", {})) - assert (await directory.remove(later)).cell == later + await directory.attach_begin( + first, + directory.Begin(dep, _fp(first), None, "k", {}), + expected=await _observed(first), + ) + assert (await directory.remove(later, expected=await _observed(later))).cell == later assert ( await conn.fetchval("SELECT count(*) FROM loom_router.hosts WHERE cell = $1", later) == 0 ) with pytest.raises(directory.NoSuchCell): - await directory.remove(later) + await directory.remove(later, expected=await _observed(later)) # The first cell backs out at any directory mode until its first placement. - await directory.attach_begin(first, directory.Begin(dep, _fp(first), None, "k", {})) - await directory.attach_enforced(first, dep) - await directory.attach_place(first, dep, ["only"]) + await directory.attach_begin( + first, directory.Begin(dep, _fp(first), None, "k", {}), expected=await _observed(first) + ) + await directory.attach_enforced(first, dep, expected=await _observed(first)) + await directory.attach_place(first, dep, ["only"], expected=await _observed(first)) with pytest.raises(directory.OperatorRefused, match="never back"): - await directory.remove(first) + await directory.remove(first, expected=await _observed(first)) await conn.execute("UPDATE loom_router.cells SET state = 'serving' WHERE cell = $1", first) with pytest.raises(directory.OperatorRefused, match="stays attached"): - await directory.remove(first) + await directory.remove(first, expected=await _observed(first)) removed = await conn.fetch( "SELECT detail FROM loom_router.audit WHERE action = 'remove' ORDER BY id" ) @@ -860,10 +929,14 @@ async def test_an_attaching_cell_with_no_placement_is_removed_with_its_hosts( await conn.execute("DELETE FROM loom_router.cells") await directory.register(_first(other)) await directory.attach_begin( - other, directory.Begin(f"dep-{other}", _fp(other), None, "k", {}) + other, + directory.Begin(f"dep-{other}", _fp(other), None, "k", {}), + expected=await _observed(other), ) - await directory.attach_enforced(other, f"dep-{other}") # a step-3 lock that never landed - assert (await directory.remove(other)).is_first + await directory.attach_enforced( + other, f"dep-{other}", expected=await _observed(other) + ) # a step-3 lock that never landed + assert (await directory.remove(other, expected=await _observed(other))).is_first assert await conn.fetchval("SELECT count(*) FROM loom_router.cells") == 0 assert await conn.fetchval("SELECT count(*) FROM loom_router.hosts") == 0 finally: diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 0ecd3a96b3..d573d37423 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -251,7 +251,16 @@ async def ready() -> list[str]: wait = apg.Wait(4, clock) clock.now += 100 # a scan before the wait with pytest.raises(Refused, match="not taken within --wait"): - await apg.commit_enforced(conn, nothing, ready, wait, lambda _t: None, lambda _m: None) + await apg.commit_enforced( + conn, + nothing, + ready, + wait, + lambda _t: None, + lambda _m: None, + expected_cell="first", + expected_fp=(CURRENT, NEXT), + ) assert clock.now - 100 <= 4 and wait.spent <= 4 # A 2 s try, a 1 s pause, then a try that waits only the second left. assert [a.split("'")[1] for a in asked] == ["2000ms", "1000ms"] @@ -449,6 +458,22 @@ def _answer(self, sql: str, args: tuple[Any, ...]) -> Any: if isinstance(answer, BaseException): raise answer return answer(*args) if callable(answer) else answer + if sql.startswith("SELECT (SELECT is_first FROM loom.cell_first)"): + return { + "is_first": True, + "cell": "first", + "mode": "recording", + "has_own": True, + "has_tenancy": True, + } + if sql == apg._HELD: + return True + if sql == "SELECT mode FROM loom.tenancy_mode": + return "recording" + if sql == "SELECT is_first, cell FROM loom.cell_first": + return {"is_first": True, "cell": "first"} + if sql == "SELECT current_fp, next_fp FROM loom.cell_self": + return {"current_fp": CURRENT, "next_fp": NEXT} raise AssertionError(f"unrouted: {sql[:80]}") async def fetch(self, sql: str, *args: Any) -> Any: @@ -642,7 +667,9 @@ class _Timeout(Exception): @pytest.mark.asyncio -async def test_the_mode_commit_retries_a_busy_lock_and_ends_an_earlier_run(clock: _Clock) -> None: +async def test_the_mode_commit_retries_a_busy_lock_without_ending_other_backends( + clock: _Clock, +) -> None: tries = _seq(_Timeout(), "UPDATE 1") def lock(*_a: Any) -> str: @@ -673,34 +700,22 @@ async def ready() -> list[str]: logged: list[str] = [] await apg.commit_enforced( - conn, confirm, ready, apg.Wait(10, clock), logged.append, lambda _m: None + conn, + confirm, + ready, + apg.Wait(10, clock), + logged.append, + lambda _m: None, + expected_cell="first", + expected_fp=(CURRENT, NEXT), ) assert logged == ["the mode's lock is busy; retrying"] - # Before each try and the one termination. - assert len(confirmed) == 3 + # Before each try, with no termination of a different backend. + assert len(confirmed) == 2 assert conn.executed.count("ROLLBACK") == 1 assert "TERMINATE" not in conn.executed -@pytest.mark.asyncio -async def test_a_superusers_mode_transaction_is_named_not_ended() -> None: - conn = FakeConn( - [ - ( - "l.mode = 'ExclusiveLock'", - [{"pid": 5, "application_name": "psql", "xact_start": "t", "rolsuper": True}], - ), - ("rolsuper FROM pg_roles", False), - ] - ) - - async def confirm() -> None: - pass - - with pytest.raises(Refused, match="a superuser must end it"): - await apg.end_mode_transactions(conn, confirm) - - @pytest.mark.asyncio @pytest.mark.parametrize("wait_s", [0, 2]) async def test_the_mode_commit_refuses_past_wait_and_skips_a_try_while_booting( @@ -728,6 +743,8 @@ async def confirm() -> None: apg.Wait(wait_s, clock), lambda _t: None, lambda _m: None, + expected_cell="first", + expected_fp=(CURRENT, NEXT), ) assert [str(r) for r in error.value.refusals] == [ "process a: still booting at the end of --wait", @@ -768,7 +785,14 @@ async def ready() -> list[str]: with pytest.raises(TimeoutError): await asyncio.wait_for( apg.commit_enforced( - conn, confirm, ready, apg.Wait(wait_s, clock), lambda _t: None, marks.append + conn, + confirm, + ready, + apg.Wait(wait_s, clock), + lambda _t: None, + marks.append, + expected_cell="first", + expected_fp=(CURRENT, NEXT), ), timeout=0.2, ) @@ -796,7 +820,9 @@ async def execute(self, sql: str, *args: Any) -> str: timeout = asyncio.timeout monkeypatch.setattr(apg.asyncio, "timeout", lambda _s: timeout(0)) marks: list[str] = [] - task = asyncio.create_task(apg._mode_try(conn, 100, marks.append)) + task = asyncio.create_task( + apg._mode_try(conn, 100, marks.append, expected_cell="first", expected_fp=(CURRENT, NEXT)) + ) done, _ = await asyncio.wait([task], timeout=0.2) if not done: task.cancel() @@ -863,7 +889,14 @@ async def ready() -> list[str]: with pytest.raises(KeyboardInterrupt): await apg.commit_enforced( - conn, confirm, ready, apg.Wait(2, clock), lambda _t: None, marks.append + conn, + confirm, + ready, + apg.Wait(2, clock), + lambda _t: None, + marks.append, + expected_cell="first", + expected_fp=(CURRENT, NEXT), ) assert conn.executed[-1] == "TERMINATE" assert marks == [] @@ -885,13 +918,13 @@ async def synchronous() -> str: async def read(mode: str, xid: int) -> apg.Durable: return apg.Durable(mode=mode, first=(True, "first"), xid=xid) - monkeypatch.setattr(apg, "durable_read", lambda: read("recording", 50)) + monkeypatch.setattr(apg, "durable_read", lambda _conn: read("recording", 50)) with pytest.raises(Refused, match="did not land"): await run.durable() - monkeypatch.setattr(apg, "durable_read", lambda: read("enforced", 100)) + monkeypatch.setattr(apg, "durable_read", lambda _conn: read("enforced", 100)) with pytest.raises(Refused, match="is not above 100"): await run.durable() - monkeypatch.setattr(apg, "durable_read", lambda: read("enforced", 99)) + monkeypatch.setattr(apg, "durable_read", lambda _conn: read("enforced", 99)) assert await run.durable() == 100 async def waiting() -> str: @@ -962,7 +995,7 @@ async def test_a_stop_between_a_landed_commit_and_the_durable_read_says_the_outc async def dedicated(**_kw: Any) -> Any: yield None - async def landed(*a: Any) -> None: + async def landed(*a: Any, **_kw: Any) -> None: a[-1]("unknown") # the try sent its UPDATE; the commit landed, its answer may not have async def call(_self: Any, _path: str, _body: Any = None) -> dict[str, Any]: @@ -976,6 +1009,7 @@ async def run(self: Any) -> int: monkeypatch.setattr(pg, "dedicated", dedicated) monkeypatch.setattr(apg, "commit_enforced", landed) monkeypatch.setattr(cell_attach.Router, "call", call) + monkeypatch.setattr(cell_attach.Run, "mutate", call) monkeypatch.setattr(cell_attach.Run, "run", run) url = httpx.URL("http://127.0.0.1:7900") assert await cell_attach.attach(None, _args(), url) == 1 @@ -1169,7 +1203,16 @@ async def ok() -> list[str]: async def nothing() -> None: pass - await apg.commit_enforced(conn, nothing, ok, apg.Wait(0, clock), lambda _t: None, marks.append) + await apg.commit_enforced( + conn, + nothing, + ok, + apg.Wait(0, clock), + lambda _t: None, + marks.append, + expected_cell="first", + expected_fp=(CURRENT, NEXT), + ) assert marks == ["unknown"] assert any(f"lock_timeout = '{apg.TRY_FLOOR_MS}ms'" in sql for sql in conn.executed) @@ -1227,6 +1270,7 @@ async def confirm(_self: Any) -> None: async def run(self: Any) -> int: self.row, self.mode = {"deployment_id": "dep-1"}, "recording" + self.lock_conn = conn self.wait = apg.Wait(2, clock) await self.commit_mode() return 0 @@ -1241,6 +1285,7 @@ async def dedicated(**_kw: Any) -> Any: monkeypatch.setattr(pg, "dedicated", dedicated) monkeypatch.setattr(cell_attach.Router, "call", call) + monkeypatch.setattr(cell_attach.Run, "mutate", call) monkeypatch.setattr(cell_attach.Run, "ready_for_commit", ready) monkeypatch.setattr(cell_attach.Run, "confirm_lock", confirm) monkeypatch.setattr(cell_attach.Run, "run", run) @@ -1248,27 +1293,6 @@ async def dedicated(**_kw: Any) -> Any: assert capsys.readouterr().err.splitlines()[-1] == "loom cell attach: the cell stays recording" -@pytest.mark.asyncio -async def test_a_mode_transaction_that_will_not_end_is_named(clock: _Clock) -> None: - held = [{"pid": 5, "application_name": "attach", "xact_start": "t", "rolsuper": False}] - conn = FakeConn( - [ - ("l.mode = 'ExclusiveLock'", held), - ("rolsuper FROM pg_roles", False), - ("pg_terminate_backend", True), - ] - ) - - async def confirm() -> None: - pass - - with pytest.raises(Refused) as e: - await apg.end_mode_transactions(conn, confirm) - (refusal,) = e.value.refusals - assert refusal.subject == "postgres backend 5" - assert "still holds loom.tenancy_mode's lock" in refusal.reason - - def _stub_attach(monkeypatch: pytest.MonkeyPatch, body: Any) -> None: from contextlib import asynccontextmanager @@ -1295,6 +1319,7 @@ async def run(self: Any) -> int: return 0 monkeypatch.setattr(cell_attach.Router, "call", refuse) + monkeypatch.setattr(cell_attach.Run, "mutate", refuse) _stub_attach(monkeypatch, run) assert await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) == 1 assert capsys.readouterr().err.splitlines()[-1] == "loom cell attach: the cell stays recording" @@ -1309,7 +1334,7 @@ async def test_a_durable_read_that_sees_recording_says_recording( async def synchronous() -> str: return "synchronous" - async def read() -> apg.Durable: + async def read(_conn: Any) -> apg.Durable: return apg.Durable(mode="recording", first=(True, "first"), xid=1) async def run(self: Any) -> int: diff --git a/tests/unit/test_cell_control.py b/tests/unit/test_cell_control.py new file mode 100644 index 0000000000..71d687baf9 --- /dev/null +++ b/tests/unit/test_cell_control.py @@ -0,0 +1,123 @@ +"""A control owner cannot survive an uncertain commit or a lost lock session.""" + +from __future__ import annotations + +import asyncio +from typing import Any +from unittest.mock import AsyncMock +from uuid import UUID + +import pytest + +from server.cli import cell_control +from tests.unit.test_cell_attach import FakeConn + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stall", ["COMMIT", "ROLLBACK"]) +async def test_uncertain_transaction_ends_owned_session_without_unbounded_cleanup( + monkeypatch, stall +): + class Stalled(FakeConn): + async def execute(self, sql: str, *args: Any) -> str: + answer = await super().execute(sql, *args) + if sql == stall: + await asyncio.Future() + return answer + + conn = Stalled() + monkeypatch.setattr(cell_control, "TRANSACTION_S", 0.01) + with pytest.raises(TimeoutError): + async with cell_control.transaction(conn): + if stall == "ROLLBACK": + raise ValueError("refuse") + assert conn.executed[-1] == "TERMINATE" + assert stall in conn.executed + assert any("pg_logical_emit_message" in sql for sql in conn.executed) + + +@pytest.mark.asyncio +async def test_cancellation_during_body_never_waits_for_rollback(): + conn = FakeConn() + entered = asyncio.Event() + + async def work(): + async with cell_control.transaction(conn): + entered.set() + await asyncio.Future() + + task = asyncio.create_task(work()) + await entered.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert conn.executed[-1] == "TERMINATE" + assert "ROLLBACK" not in conn.executed + + +def snapshot(): + return { + "cell": "first", + "is_first": True, + "host": "first.example", + "deployment_id": "dep", + "bucket": "", + "bucket_prefix": "", + "probe_url": "", + "upstream_url": "", + "current_fp": None, + "next_fp": None, + "mutation_generation": str(UUID(int=1)), + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("after_claim", [False, True]) +async def test_lost_lock_never_reconnects_or_continues_claim(monkeypatch, after_claim): + lock = AsyncMock( + side_effect=[None, cell_control._refused("lost")] + if after_claim + else cell_control._refused("lost") + ) + monkeypatch.setattr(cell_control, "check_lock", lock) + monkeypatch.setattr(cell_control, "paired", AsyncMock()) + call = AsyncMock(return_value={**snapshot(), "mutation_generation": str(UUID(int=2))}) + with pytest.raises(cell_control.Refused, match="lost"): + await cell_control.claim(object(), call, snapshot()) + assert call.await_count == int(after_claim) + + +@pytest.mark.asyncio +async def test_unknown_claim_acknowledgement_never_reuses_the_observed_generation(monkeypatch): + monkeypatch.setattr(cell_control, "check_lock", AsyncMock()) + monkeypatch.setattr(cell_control, "paired", AsyncMock()) + call = AsyncMock(side_effect=OSError("reply lost")) + with pytest.raises(OSError, match="reply lost"): + await cell_control.claim(object(), call, snapshot()) + assert call.await_count == 1 + assert "mutation_generation" not in call.call_args.args[1]["expected"] + + +def test_older_router_snapshot_cannot_make_a_control_request(): + row = snapshot() + row.pop("mutation_generation") + with pytest.raises(cell_control.Refused, match="not upgraded"): + cell_control.preconditions(row) + + +@pytest.mark.asyncio +async def test_whole_router_request_is_bounded_even_without_a_transport_reply(monkeypatch): + import httpx + + async def stalled(_request): + await asyncio.Future() + + monkeypatch.setattr(cell_control, "REQUEST_S", 0.01) + with pytest.raises(cell_control.Refused, match="reply unknown.*TimeoutError"): + await cell_control.request( + httpx.URL("http://127.0.0.1:1"), + "first", + httpx.MockTransport(stalled), + "/claim", + {}, + ) diff --git a/tests/unit/test_cell_operator.py b/tests/unit/test_cell_operator.py index 63b1e00d26..5c37b78cb0 100644 --- a/tests/unit/test_cell_operator.py +++ b/tests/unit/test_cell_operator.py @@ -8,6 +8,7 @@ import hashlib import json from typing import Any +from uuid import UUID import httpx import pytest @@ -23,6 +24,29 @@ SHA = "a" * 64 +OBSERVED = { + "cell": "eu-1", + "is_first": False, + "host": "eu-1.example", + "deployment_id": "dep-eu-1", + "bucket": "", + "bucket_prefix": "", + "probe_url": "", + "upstream_url": "", + "current_fp": None, + "next_fp": None, + "mutation_generation": str(UUID(int=1)), +} + + +@pytest.fixture(autouse=True) +def control_snapshot(monkeypatch: pytest.MonkeyPatch) -> None: + def read(url: str, **_kw: Any) -> httpx.Response: + return httpx.Response(200, json=OBSERVED, request=httpx.Request("GET", url)) + + monkeypatch.setattr(httpx, "get", read) + + def test_a_new_credential_is_lcr_and_128_hex_with_its_sha256(capsys) -> None: credential, sha = cell_cli.new_credential() assert credential.startswith("lcr_") and len(credential) == 132 @@ -110,7 +134,9 @@ def test_a_rotation_step_calls_the_operator_api(operator, kw, body) -> None: assert cell_cli._cmd_credential(_args(cell="eu-1", **kw)) == 0 (call,) = operator.calls assert call["url"] == "https://router.test/cells/v1/operator/cells/eu-1/credential" - assert call["json"] == body and call["headers"] == {"Authorization": f"Bearer {OPERATOR}"} + assert call["json"] == {**body, "expected": OBSERVED} and call["headers"] == { + "Authorization": f"Bearer {OPERATOR}" + } @pytest.mark.parametrize( @@ -124,7 +150,7 @@ def test_a_rotation_step_calls_the_operator_api(operator, kw, body) -> None: ) def test_a_limit_calls_the_operator_api(operator, kw, body) -> None: assert cell_cli._cmd_limit(_args(cell="eu-1", **kw)) == 0 - assert operator.calls[0]["json"] == body + assert operator.calls[0]["json"] == {**body, "expected": OBSERVED} assert operator.calls[0]["url"].endswith("/cells/v1/operator/cells/eu-1/limit") @@ -345,7 +371,7 @@ def test_a_page_that_is_not_the_routers_answer_fails(monkeypatch, capsys, comman assert cell_cli._cmd_limit(_args(cell="eu-1", closed=True)) == 1 err = capsys.readouterr().err assert "did not answer as the router does" in err - assert ("the change may have been made" in err) is (command == "limit") + assert "the change may have been made" not in err # failed precondition read sent no mutation def _parse(*argv: str) -> argparse.Namespace: @@ -388,6 +414,7 @@ def _row(**over: Any) -> directory.Cell: "placed": 3, "credential_sha256": "b" * 64, "credential_next_sha256": None, + "mutation_generation": UUID(int=1), "deployment_id": "dep-eu-1", "current_fp": None, "next_fp": None, @@ -403,12 +430,18 @@ def api(monkeypatch) -> tuple[TestClient, list[Any]]: calls: list[Any] = [] async def _limit(cell: str, **kw: Any) -> directory.Cell: + expected = kw.pop("expected") + assert expected.generation == UUID(int=1) + assert expected.values == {k: OBSERVED[k] for k in directory.CONTROL_FIELDS} calls.append(("limit", cell, kw)) if cell == "nope": raise directory.NoSuchCell("cell 'nope' is not in the directory") return _row(closed=kw.get("closed") is True) - async def _rotate(cell: str, step: str, sha: str = "") -> directory.Cell: + async def _rotate( + cell: str, step: str, sha: str = "", *, expected: directory.Control + ) -> directory.Cell: + assert expected.generation == UUID(int=1) calls.append(("credential", cell, step, sha)) if sha == "c" * 64: raise directory.OperatorRefused("cell 'eu-2' records that credential hash already") @@ -420,6 +453,8 @@ async def _rotate(cell: str, step: str, sha: str = "") -> directory.Cell: def _post(client: TestClient, path: str, body: Any, token: str = OPERATOR) -> Any: + if isinstance(body, dict): + body = {**body, "expected": OBSERVED} return client.post( f"/cells/v1/operator/cells/{path}", json=body, headers={"Authorization": f"Bearer {token}"} ) @@ -536,7 +571,7 @@ async def _credentials() -> dict[str, tuple[str, ...]]: reads.append(1) return dict(hashes) - async def _rotate(cell: str, step: str, sha: str = "") -> directory.Cell: + async def _rotate(cell: str, step: str, sha: str = "", **_kw: Any) -> directory.Cell: hashes["eu-1"] = ("b" * 64, sha) return _row(credential_next_sha256=sha) diff --git a/tests/unit/test_cell_prepare.py b/tests/unit/test_cell_prepare.py index c978eb94e2..3071dddccf 100644 --- a/tests/unit/test_cell_prepare.py +++ b/tests/unit/test_cell_prepare.py @@ -15,7 +15,7 @@ import pytest from server.cli import cell as cell_cli -from server.cli import cell_prepare +from server.cli import cell_control, cell_prepare from server.core import cell from server.settings import settings @@ -369,7 +369,7 @@ async def test_the_processes_live_after_the_wait_are_the_ones_judged(monkeypatch reads = [[_beat("i-old", before)], [_beat("i-new", after)]] written: list[Any] = [] - async def _rows_read() -> cell_prepare._CellRows: + async def _rows_read(_conn: Any) -> cell_prepare._CellRows: return _rows() async def _beats(_ch: Any) -> list[cell_prepare._Beat]: @@ -381,7 +381,7 @@ async def _nothing(*_a: Any) -> Any: async def _register(*_a: Any) -> dict[str, Any]: return {**OK, "registered": True} - async def _write(_rows: Any, _answer: Any, current: Any, nxt: Any) -> list[str]: + async def _write(_conn: Any, _rows: Any, _answer: Any, current: Any, nxt: Any) -> list[str]: written.append((current, nxt)) return ["first", "mode", "own"] @@ -395,6 +395,18 @@ async def _write(_rows: Any, _answer: Any, current: Any, nxt: Any) -> list[str]: ("_write_rows", _write), ): monkeypatch.setattr(cell_prepare, name, fake) + monkeypatch.setattr(cell_control, "transaction", lambda _conn: contextlib.nullcontext()) + + directory_mode = ["recording"] + + async def snapshot(*_a: Any, **_kw: Any) -> dict[str, Any]: + return {**OK, "mode": directory_mode[0], "current_fp": None, "next_fp": None} + + async def claimed(_conn: Any, _call: Any, row: Any) -> Any: + return row + + monkeypatch.setattr(cell_control, "request", snapshot) + monkeypatch.setattr(cell_control, "claim", claimed) assert await cell_prepare.prepare(None, _args(), ROUTER) == 0 assert written == [(cell.as_value(after), None)] and reads == [] # stdout stays one JSON document; a first cell's next step goes to stderr. @@ -402,6 +414,7 @@ async def _write(_rows: Any, _answer: Any, current: Any, nxt: Any) -> list[str]: assert json.loads(out.out)["is_first"] is True assert out.err.strip() == "loom cell prepare: prepared the first cell; roll out LOOM_CELL next" # A rerun on a cell past recording names no next step. + directory_mode[0] = "enforced" monkeypatch.setattr( cell_prepare, "_register", AsyncMock(return_value={**OK, "mode": "enforced"}) ) @@ -409,6 +422,7 @@ async def _write(_rows: Any, _answer: Any, current: Any, nxt: Any) -> list[str]: assert await cell_prepare.prepare(None, _args(), ROUTER) == 0 assert capsys.readouterr().err == "" # A rerun in recording that writes no first row (the rollout is usually done). + directory_mode[0] = "recording" monkeypatch.setattr(cell_prepare, "_register", AsyncMock(return_value={**OK})) async def _own_only(*_a: Any) -> list[str]: @@ -418,3 +432,16 @@ async def _own_only(*_a: Any) -> list[str]: reads.extend([[_beat("i-new", after)], [_beat("i-new", after)]]) assert await cell_prepare.prepare(None, _args(), ROUTER) == 0 assert capsys.readouterr().err == "" + + # A prior controller committed enforced after registration but before this claim. + directory_mode[0] = "enforced" + posted: list[str] = [] + + async def write_latest(_conn: Any, _rows: Any, answer: Any, *_fp: Any) -> list[str]: + posted.append(answer["mode"]) + return ["mode"] + + monkeypatch.setattr(cell_prepare, "_write_rows", write_latest) + reads.extend([[_beat("i-new", after)], [_beat("i-new", after)]]) + assert await cell_prepare.prepare(None, _args(), ROUTER) == 0 + assert posted == ["enforced"] diff --git a/tests/unit/test_cell_register.py b/tests/unit/test_cell_register.py index 3d26bfa003..16a66b3489 100644 --- a/tests/unit/test_cell_register.py +++ b/tests/unit/test_cell_register.py @@ -9,6 +9,7 @@ import hashlib import json from typing import Any +from uuid import UUID import httpx import pytest @@ -90,6 +91,7 @@ def _row(reg: directory.Registration) -> directory.Cell: current_fp=None, next_fp=None, credential_next_sha256=None, + mutation_generation=UUID(int=1), ) @@ -501,7 +503,23 @@ def test_a_cell_never_registers_with_the_operator_credentials_hash(router) -> No assert r.status_code == 400 and "operator credential's hash" in r.json()["error"] rotated = client.post( "/cells/v1/operator/cells/eu-1/credential", - json={"step": "next", "sha256": operator_sha}, + json={ + "step": "next", + "sha256": operator_sha, + "expected": { + "cell": "eu-1", + "is_first": True, + "host": "loom.example", + "deployment_id": "dep-1", + "bucket": "", + "bucket_prefix": "", + "probe_url": "", + "upstream_url": "", + "current_fp": None, + "next_fp": None, + "mutation_generation": "00000000-0000-0000-0000-000000000001", + }, + }, headers={"Authorization": f"Bearer {OPERATOR}"}, ) assert rotated.status_code == 400 and "operator credential's hash" in rotated.json()["error"] diff --git a/tests/unit/test_router.py b/tests/unit/test_router.py index 9de51a1e75..84bd4db79d 100644 --- a/tests/unit/test_router.py +++ b/tests/unit/test_router.py @@ -9,6 +9,7 @@ from contextlib import asynccontextmanager from typing import Any from unittest.mock import AsyncMock +from uuid import UUID import httpx import pytest @@ -46,6 +47,7 @@ def _row(**over: Any) -> directory.Cell: "next_fp": None, } values.update(over) + values["mutation_generation"] = UUID(int=1) return directory.Cell(**values) @@ -102,6 +104,8 @@ async def lookup(self, org: str) -> tuple[str, str] | None: async def set_limit(self, name: str, **fields: Any) -> directory.Cell: self._check() + expected = fields.pop("expected") + assert expected.generation == self.cells[name].mutation_generation given = {k: v for k, v in fields.items() if v is not directory.UNSET} self.cells[name] = dataclasses.replace(self.cells[name], **given) return self.cells[name] @@ -964,7 +968,13 @@ def test_a_cell_the_operator_closes_refuses_its_next_new_org( assert _place(client, "before").status_code == 200 closed = client.post( "/cells/v1/operator/cells/eu-1/limit", - json={"closed": True}, + json={ + "closed": True, + "expected": { + **{key: getattr(fake.cells["eu-1"], key, "") for key in directory.CONTROL_FIELDS}, + "mutation_generation": str(fake.cells["eu-1"].mutation_generation), + }, + }, headers={"Authorization": f"Bearer {operator}"}, ) assert closed.status_code == 200 and closed.json()["closed"] is True diff --git a/tests/unit/test_router_attach.py b/tests/unit/test_router_attach.py index efb0b29e80..1471f5882b 100644 --- a/tests/unit/test_router_attach.py +++ b/tests/unit/test_router_attach.py @@ -14,12 +14,14 @@ from datetime import UTC, datetime from pathlib import Path from typing import Any +from uuid import UUID import httpx import pytest from starlette.testclient import TestClient from server.cli import cell as cell_cli +from server.cli import cell_control from server.core import cell from server.router import app as router_app from server.router import directory @@ -44,6 +46,7 @@ def _cell_row(**over: Any) -> dict[str, Any]: row: dict[str, Any] = { "cell": "eu-1", + "mutation_generation": UUID(int=1), "is_first": True, "host": "loom.example", "state": "attaching", @@ -102,7 +105,7 @@ def _row(self, name: str, sql: str) -> dict[str, Any] | None: async def execute(self, sql: str, *a: Any) -> str: self._count() cells = self.s.cells - if sql.startswith("SELECT pg_advisory_xact_lock"): + if sql.startswith(("SELECT pg_advisory_xact_lock", "SELECT pg_logical_emit_message")): return "" if "SET attach_started_at" in sql: row = cells[a[0]] @@ -172,10 +175,22 @@ async def fetch(self, sql: str, *a: Any) -> list[Any]: self.s.placements.update(dict.fromkeys(new, a[1])) return [{"org": o} for o in new] + def terminate(self) -> None: + pass + def transaction(self) -> Any: conn = self class _Tx: + async def start(self) -> None: + await self.__aenter__() + + async def commit(self) -> None: + await self.__aexit__(None) + + async def rollback(self) -> None: + await self.__aexit__(Exception) + async def __aenter__(self) -> None: conn._count() self.saved = copy.deepcopy( @@ -233,7 +248,9 @@ def listener() -> _Listener: @pytest.fixture def client(store: _Store, listener: _Listener) -> TestClient: router = router_app.Router(probe_transport=httpx.MockTransport(listener)) - return TestClient(router_app.build_app(router)) + client = TestClient(router_app.build_app(router)) + client.control_store = store + return client def _call( @@ -241,6 +258,11 @@ def _call( ) -> Any: url = "/cells/v1/operator/cells/eu-1" + (f"/{path}" if path else "") headers = {"Authorization": f"Bearer {token}"} + if method != "GET" and (body is None or isinstance(body, dict)): + row = client.control_store.cells.get("eu-1", _cell_row()) + observed = {name: row[name] for name in directory.CONTROL_FIELDS} + observed["mutation_generation"] = str(row["mutation_generation"]) + body = {**(body or {}), "expected": observed} if body is None: return client.request(method, url, headers=headers) return client.request(method, url, json=body, headers=headers) @@ -249,6 +271,10 @@ def _call( def _begin(**over: Any) -> dict[str, Any]: body: dict[str, Any] = { "deployment_id": "dep-1", + "expected": { + **{name: _cell_row()[name] for name in directory.CONTROL_FIELDS}, + "mutation_generation": str(UUID(int=1)), + }, "current_fp": CURRENT, "next_fp": None, "keys_fp": "keys-1", @@ -292,6 +318,8 @@ def test_the_operator_reads_the_cells_row(client: TestClient, store: _Store) -> "current_fp": None, "next_fp": None, "other": None, + "mutation_generation": str(UUID(int=1)), + **{name: store.cells["eu-1"][name] for name in directory._REGISTERED}, } # Any other row is named: every attach call refuses while it exists. store.cells["eu-2"] = _cell_row(cell="eu-2", is_first=False, host="eu-2.example") @@ -322,6 +350,15 @@ def test_every_new_route_needs_the_operator_credential_and_names_an_unknown_cell assert store.statements == 0 url = "/cells/v1/operator/cells/eu-9" + (f"/{path}" if path else "") headers = {"Authorization": f"Bearer {OPERATOR}"} + if method != "GET": + source = _cell_row() + body = { + **(body or {}), + "expected": { + **{name: source[name] for name in directory.CONTROL_FIELDS}, + "mutation_generation": str(source["mutation_generation"]), + }, + } r = client.request(method, url, headers=headers, json=body) assert (r.status_code, r.json()["code"]) == (404, "NO_SUCH_CELL") bad = client.request(method, url.replace("eu-9", "EU_9"), headers=headers, json=body) @@ -613,7 +650,15 @@ def test_begin_refuses_a_fingerprint_whose_placement_body_would_pass_the_routers # And again under the lock. big = directory.Begin("dep-1", grown, None, "keys", {}) with pytest.raises(directory.OperatorRefused, match="configured_orgs"): - asyncio.run(directory.attach_begin("eu-1", big)) + asyncio.run( + directory.attach_begin( + "eu-1", + big, + expected=directory.Control( + {k: store.cells["eu-1"][k] for k in directory.CONTROL_FIELDS}, UUID(int=1) + ), + ) + ) assert store.cells["eu-1"]["attach_started_at"] is None and store.audit == [] # One that fits, with the longest org escaped, is taken. fits = cell.as_value({**FIELDS, "configured_orgs": [f"org-{i:05d}" for i in range(100)]}) @@ -884,16 +929,33 @@ def sent(call: Any) -> int: asyncio.run(call) return store.statements - before + def observed(name: str = "eu-1") -> directory.Control: + row = store.cells[name] + return directory.Control( + {k: row[k] for k in directory.CONTROL_FIELDS}, row["mutation_generation"] + ) + begin = directory.Begin("dep-1", CURRENT, None, "keys", {}) assert sent(directory.attach_read("eu-1", audited=True)) == directory.OPERATOR_READ_STATEMENTS assert sent(directory.attach_read("eu-1")) == directory.ATTACH_READ_STATEMENTS - assert sent(directory.attach_begin("eu-1", begin)) == directory.ATTACH_CHANGE_STATEMENTS - assert sent(directory.attach_enforced("eu-1", "dep-1")) == directory.ATTACH_CHANGE_STATEMENTS - batch = directory.attach_place("eu-1", "dep-1", [f"o{i}" for i in range(50)]) + assert ( + sent(directory.attach_begin("eu-1", begin, expected=observed())) + == directory.ATTACH_CHANGE_STATEMENTS + ) + assert ( + sent(directory.attach_enforced("eu-1", "dep-1", expected=observed())) + == directory.ATTACH_CHANGE_STATEMENTS + ) + batch = directory.attach_place( + "eu-1", "dep-1", [f"o{i}" for i in range(50)], expected=observed() + ) assert sent(batch) == directory.ATTACH_PLACE_STATEMENTS # whatever the batch's size - assert sent(directory.attach_serving("eu-1", "dep-1")) == directory.ATTACH_CHANGE_STATEMENTS + assert ( + sent(directory.attach_serving("eu-1", "dep-1", expected=observed())) + == directory.ATTACH_CHANGE_STATEMENTS + ) store.cells["eu-2"] = _cell_row(cell="eu-2", is_first=False, host="eu-2.example") - assert sent(directory.remove("eu-2")) == directory.REMOVE_STATEMENTS + assert sent(directory.remove("eu-2", expected=observed("eu-2"))) == directory.REMOVE_STATEMENTS assert directory.REMOVE_MAX_S == directory._bound_s(directory.REMOVE_STATEMENTS) @@ -990,6 +1052,8 @@ def cli_router(monkeypatch: pytest.MonkeyPatch) -> _Router: monkeypatch.setattr(settings, "router_operator_token", OPERATOR) fake = _Router( { + **_cell_row(), + "mutation_generation": str(UUID(int=1)), "cell": "eu-1", "is_first": True, "state": "attaching", @@ -999,6 +1063,23 @@ def cli_router(monkeypatch: pytest.MonkeyPatch) -> _Router: ) monkeypatch.setattr(httpx, "get", fake.get) monkeypatch.setattr(httpx, "delete", fake.delete) + + async def request( + url: Any, name: str, transport: Any, path: str, body: Any = None, *, method: str = "POST" + ) -> Any: + target = f"{url}/cells/v1/operator/cells/{name}" + path + if path == "/claim": + fake.calls.append(("POST", target)) + return dict(fake.row) + response = fake.get(target) if method == "GET" else fake.delete(target) + if response.status_code != 200: + if response.status_code == 404 and response.json().get("code") != "NO_SUCH_CELL": + raise cell_control._refused("a router replica is not upgraded") + raise cell_control._refused(response.json()["error"]) + return response.json() + + monkeypatch.setattr(cell_control, "request", request) + _stores(monkeypatch, "") return fake @@ -1007,13 +1088,24 @@ def _stores(monkeypatch: pytest.MonkeyPatch, refusal: Any) -> list[str]: it was asked about.""" checked: list[str] = [] - async def _check(deployment_id: str) -> str: - checked.append(deployment_id) + async def paired(_conn: Any, row: dict[str, Any]) -> None: + checked.append(row["deployment_id"]) if isinstance(refusal, Exception): raise refusal - return refusal + if refusal: + raise cell_control._refused(refusal) + + async def state(_conn: Any) -> tuple[str, str]: + return "open", "" + + async def held(_conn: Any) -> None: + pass - monkeypatch.setattr(cell_cli, "_first_cell_refusal", _check) + monkeypatch.setattr(cell_control, "paired", paired) + monkeypatch.setattr(cell_control, "session", lambda _what: contextlib.nullcontext(object())) + monkeypatch.setattr(cell_control, "transaction", lambda _conn: contextlib.nullcontext()) + monkeypatch.setattr(cell_control, "check_lock", held) + monkeypatch.setattr(cell, "read_state_if_present", state) return checked @@ -1022,9 +1114,9 @@ def test_remove_checks_a_first_cells_postgres_then_deletes_its_row( ) -> None: checked = _stores(monkeypatch, "") assert cell_cli._cmd_remove(_remove_args()) == 0 - assert checked == ["dep-1"] - assert [m for m, _url in cli_router.calls] == ["GET", "DELETE"] - assert cli_router.calls[1][1] == "https://router.test/cells/v1/operator/cells/eu-1" + assert checked == ["dep-1", "dep-1"] + assert [m for m, _url in cli_router.calls] == ["GET", "GET", "POST", "DELETE"] + assert cli_router.calls[-1][1] == "https://router.test/cells/v1/operator/cells/eu-1" assert json.loads(capsys.readouterr().out) == {"cell": "eu-1", "removed": True} @@ -1040,16 +1132,16 @@ def test_remove_refuses_a_first_cell_its_postgres_does_not_clear( ) -> None: _stores(monkeypatch, refusal) assert cell_cli._cmd_remove(_remove_args()) == 1 - assert "refusing" in capsys.readouterr().err - assert [m for m, _url in cli_router.calls] == ["GET"] + assert "stop and rerun" in capsys.readouterr().err + assert [m for m, _url in cli_router.calls] == ["GET", "GET"] -def test_remove_reads_no_store_for_a_later_cell( +def test_remove_checks_a_later_cells_pairing_and_claim( cli_router: _Router, monkeypatch: pytest.MonkeyPatch ) -> None: cli_router.row = {**cli_router.row, "is_first": False, "deployment_id": "dep-2"} - checked = _stores(monkeypatch, "never asked") - assert cell_cli._cmd_remove(_remove_args()) == 0 and checked == [] + checked = _stores(monkeypatch, "") + assert cell_cli._cmd_remove(_remove_args()) == 0 and checked == ["dep-2", "dep-2"] @pytest.mark.parametrize( @@ -1098,7 +1190,7 @@ def test_remove_tells_an_older_router_from_an_unknown_cell_on_the_delete( cli_router.deleted = (404, {"error": "the router answers no such path"}) assert cell_cli._cmd_remove(_remove_args()) == 1 assert "not upgraded" in capsys.readouterr().err - assert [m for m, _url in cli_router.calls] == ["GET", "DELETE"] + assert [m for m, _url in cli_router.calls] == ["GET", "GET", "POST", "DELETE"] @pytest.mark.parametrize( @@ -1112,42 +1204,31 @@ def test_remove_tells_an_older_router_from_an_unknown_cell_on_the_delete( ], ) def test_the_first_cell_check_needs_an_open_postgres_of_the_rows_deployment( - monkeypatch: pytest.MonkeyPatch, state: str, recorded: str, why: str + cli_router: _Router, + monkeypatch: pytest.MonkeyPatch, + capsys: Any, + state: str, + recorded: str, + why: str, ) -> None: - """One Postgres session of its own reads both: no pool, and no ClickHouse.""" - from server.core import pg - - sessions: list[dict[str, Any]] = [] - - class _Session: - async def fetchval(self, sql: str) -> str: - assert sql == "SELECT deployment_id FROM loom.authority" - return recorded + async def paired(_conn: Any, row: dict[str, Any]) -> None: + if not recorded: + raise cell_control._refused("records no deployment id") + if recorded != row["deployment_id"]: + raise cell_control._refused(f"records deployment id {recorded!r}") - @contextlib.asynccontextmanager - async def _dedicated(**kw: Any) -> Any: - sessions.append(kw) - yield _Session() - - async def _state(conn: Any) -> tuple[str, str]: - assert isinstance(conn, _Session) + async def read(_conn: Any) -> tuple[str, str]: return state, "" - def _no_pool(*_a: Any, **_kw: Any) -> None: - raise AssertionError("the check reads no pool") - - # LOOM_PG_DSN alone: the CLI's storage authority setting is not read. - monkeypatch.setattr(settings, "storage_authority", "clickhouse") - monkeypatch.setattr(settings, "pg_dsn", "postgresql://cell.internal/loom") - monkeypatch.setattr(pg, "dedicated", _dedicated) - monkeypatch.setattr(pg, "fetchrow", _no_pool) - monkeypatch.setattr(pg, "fetchval", _no_pool) - monkeypatch.setattr(cell, "read_state_if_present", _state) - refusal = asyncio.run(cell_cli._first_cell_refusal("dep-1")) - assert (why in refusal) if why else refusal == "" - assert sessions == [{"statement_timeout_ms": cell_cli._CHECK_STATEMENT_MS}] - monkeypatch.setattr(settings, "pg_dsn", "") - assert "LOOM_PG_DSN" in asyncio.run(cell_cli._first_cell_refusal("dep-1")) + monkeypatch.setattr(cell_control, "paired", paired) + monkeypatch.setattr(cell, "read_state_if_present", read) + result = cell_cli._cmd_remove(_remove_args()) + if why: + assert result == 1 + assert why.replace("`", "") in capsys.readouterr().err + assert not any(method in ("POST", "DELETE") for method, _url in cli_router.calls) + else: + assert result == 0 def test_remove_never_connects_clickhouse( @@ -1158,13 +1239,7 @@ def test_remove_never_connects_clickhouse( def _never(*_a: Any, **_kw: Any) -> None: raise AssertionError("loom cell remove connects no ClickHouse") - async def _read() -> tuple[str, str]: - return "open", "dep-1" - monkeypatch.setattr(_ch, "run_connected", _never) - monkeypatch.setattr(settings, "storage_authority", "postgres") - monkeypatch.setattr(settings, "pg_dsn", "postgresql://cell.internal/loom") - monkeypatch.setattr(cell_cli, "_read_first_cell", _read) assert cell_cli._cmd_remove(_remove_args()) == 0 @@ -1211,7 +1286,7 @@ def test_a_rotation_drops_this_replicas_cached_state( store.cells["eu-1"]["state"] = "serving" assert _state(client).json()["state"] == "attaching" # cached - async def _rotate(name: str, step: str, sha: str) -> directory.Cell: + async def _rotate(name: str, step: str, sha: str, **_kw: Any) -> directory.Cell: store.cells[name]["credential_next_sha256"] = sha return directory._cell(_Conn(store)._row(name, "")) @@ -1303,13 +1378,14 @@ def test_attach_serving_waits_for_a_state_read_in_flight_and_drops_its_row( after attach/serving runs once that read is done, so the next read answers `serving`.""" router = router_app.Router(probe_transport=httpx.MockTransport(listener)) with TestClient(router_app.build_app(router)) as setup: + setup.control_store = store _enforced(setup) started, go = _blocked_state_read(store, monkeypatch) committed = asyncio.Event() real_serving = directory.attach_serving - async def _serving(*a: Any) -> Any: - found = await real_serving(*a) + async def _serving(*a: Any, **kw: Any) -> Any: + found = await real_serving(*a, **kw) committed.set() return found @@ -1325,7 +1401,14 @@ async def _run() -> None: serving = asyncio.create_task( c.post( "/cells/v1/operator/cells/eu-1/attach/serving", - json={"deployment_id": "dep-1", "nonces": NONCES}, + json={ + "deployment_id": "dep-1", + "nonces": NONCES, + "expected": { + **{k: store.cells["eu-1"][k] for k in directory.CONTROL_FIELDS}, + "mutation_generation": str(UUID(int=1)), + }, + }, headers={"Authorization": f"Bearer {OPERATOR}"}, ) ) From e0f756bb5c6676002bb8d2740e880d9725fe727e Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 22:13:29 +0000 Subject: [PATCH 021/114] Index cell control recovery and its upgrade refusal Signed-off-by: zlareb1 --- docs/reference/errors.md | 4 ++++ llms.txt | 1 + 2 files changed, 5 insertions(+) diff --git a/docs/reference/errors.md b/docs/reference/errors.md index 62a4a86f46..a2670a9c72 100644 --- a/docs/reference/errors.md +++ b/docs/reference/errors.md @@ -432,6 +432,10 @@ Raised in `server/memory/domain/record_values.py`. Raised in `server/tracker/app/sources.py`. +#### `CONTROL_UPGRADE_REQUIRED` + +Raised in `server/router/app.py`. + #### `CREDENTIAL_NOT_READONLY` Raised in `server/tracker/app/sources.py`. diff --git a/llms.txt b/llms.txt index 43b8421b0e..391bbc9c1d 100644 --- a/llms.txt +++ b/llms.txt @@ -105,6 +105,7 @@ Start with [the docs map](docs/index.md). To install Loom, a coding agent reads - [Exporting usage to a data warehouse](docs/operations/analytics-export.md): `loom export-analytics` writes one UTC day of usage and a snapshot of the namespaces to an S3 bucket as Parquet files, one file per entity per day, for a data warehouse to load. - [Backup and recovery](docs/operations/backup-and-recovery.md): A Loom backup is a ClickHouse backup, plus the document bucket while any catalog row names an object in it, and under the Postgres authority a backup of Postgres too. - [Attaching the first cell](docs/operations/cell-attach.md): `loom cell attach` turns a prepared first cell (`loom cell prepare`, mode `recording`, directory row `attaching`) into a `serving` one: it places every org the cell holds in the router's directory, raises the cell's mode to `enforced` and… +- [Cell control upgrades and recovery](docs/operations/cell-control.md): Cell control mutations require generation-aware clients and router handlers, and direct access to the cell's Postgres. - [ClickHouse Cloud and Managed Postgres](docs/operations/clickhouse-cloud.md): This guide runs Loom on Kubernetes over two managed services: ClickHouse Cloud for projections, vectors, history and analytics, and ClickHouse Managed Postgres as the memory authority. - [Code index operations](docs/operations/code-index-operations.md): This page covers how to enable the code index workers, configure repositories and keep the index current; the code index runs only when an operator enables its workers and configures repositories. - [Deployment](docs/operations/deployment.md): Loom is one server image plus ClickHouse (and Postgres, under `LOOM_STORAGE_AUTHORITY=postgres`), with the document extractor service as an image of its own (container images). From 3f62dfdc92cda9e061b36444e541f438b3102916 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 22:15:54 +0000 Subject: [PATCH 022/114] Validate the cell name before recovery requests Signed-off-by: zlareb1 --- docs/operations/cell-control.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations/cell-control.md b/docs/operations/cell-control.md index ccd7bf5edf..2ef90c33a7 100644 --- a/docs/operations/cell-control.md +++ b/docs/operations/cell-control.md @@ -82,7 +82,7 @@ async def main(): if not recorded or recorded["authority"] != "postgres": raise RuntimeError("the configured stores do not record Postgres authority") pg.expect_deployment((str(recorded["deployment_id"]), int(recorded["epoch"]))) - url = cell_cli.operator_router(argparse.Namespace(router=None)) + url = cell_cli._cell_router(argparse.Namespace(router=None, cell=settings.cell)) if not isinstance(url, httpx.URL): raise RuntimeError("the router URL or operator credential was refused") From db89cb2058cf9d8791c537a166c3335473842f69 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 22:53:14 +0000 Subject: [PATCH 023/114] Keep attaching cells closed through state cleanup Refuse explicit open requests under the guarded attaching row lock, preserving quota atomicity and the durable zero-placement boundary. Inventory control writers and their required boundaries, including dynamic cleanup SQL. Signed-off-by: zlareb1 --- docs/architecture/cells.md | 11 +- docs/operations/cell-control.md | 14 +- server/cli/cell.py | 2 +- server/router/directory.py | 4 + tests/integration/test_cell_control_fence.py | 188 +++++++ tests/unit/snapshots/cli_help.json | 2 +- tests/unit/snapshots/cli_tree.json | 2 +- tests/unit/test_cell_control_census.py | 503 +++++++++++++++++++ 8 files changed, 716 insertions(+), 10 deletions(-) create mode 100644 tests/unit/test_cell_control_census.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index e5419326ed..59ed6fef64 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1203,7 +1203,10 @@ one command, `loom cell limit --cell [--closed | --open] [--placements-max ( | none)]`, which writes the directory alone, with the router's operator credential and none of the cell's stores, so it works while the cell is down; the router judges a placement from -the directory row. +the directory row. An explicit `--open` refuses while the cell is `attaching`, including +an already-open row and a combined quota edit. This behavior break preserves the closed, +zero-placement boundary used by state cleanup. Close and quota-only edits still work; +an attaching cell can complete attach while closed, then reopen once `serving`. ## Control mutation fencing @@ -1245,7 +1248,9 @@ cleanup have explicit bounds; an interrupted or uncertain local transaction neve continues on another connection. No operation record, persistent census, phase or lease is added. The UUID is coordination metadata and holds no memory content. -The source census for acceptance and operator directory writes is: +The source census for acceptance and operator directory writes is checked by +`tests/unit/test_cell_control_census.py`, including dynamic cleanup tables, guarded +entry points and the named exceptions: | Writer | Fence | | --------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | @@ -1274,7 +1279,7 @@ participate in this protocol. there); for an org with no placement it names the first cell, which the lookup does not tell apart. - `loom cell limit --cell --closed` closes a cell to new orgs, `--open` reopens - it, and `--placements-max ` sets its placement quota (`--placements-max none` + a serving cell, and `--placements-max ` sets its placement quota (`--placements-max none` clears it). - `loom cell remove --cell ` removes the directory row of a cell that is `attaching` with no placement (`--cell` is the cell's `LOOM_CELL`, unset by then). For the first cell diff --git a/docs/operations/cell-control.md b/docs/operations/cell-control.md index 2ef90c33a7..b606d9aaf9 100644 --- a/docs/operations/cell-control.md +++ b/docs/operations/cell-control.md @@ -102,15 +102,21 @@ asyncio.run(main()) The helper claims first, closes the cell using the existing limit operation, and requires that operation's durable placement count to remain zero before any local state cleanup. A queued placement either wins first and refuses cleanup, or observes the closed row and -cannot create a placement. The helper then compares and deletes only the existing +cannot create a placement. An explicit `loom cell limit --open` now refuses for every +attaching cell, even one already open; a combined open/quota request changes neither +field. This behavior break keeps concurrent limit clients from reopening the cleanup +boundary. Close and quota-only edits remain available. The helper then compares and +deletes only the existing `cell_first`, `tenancy_mode`, `cell_self` and `tenancy` rows, restores the first-row triggers and drops only the existing org guard triggers, on the same locked backend. It takes the migration lock for each bounded local transaction. Even an already-open retry proves durability before reporting success. -An abort leaves the cell closed. It never reopens automatically or deletes a placement. -If a placement won, resume attach; inspect the reason before explicitly reopening with -`loom cell limit --open`. Otherwise rerun the cleanup with a fresh claim, then run +After the close is acknowledged, an abort leaves the cell closed. It never reopens +automatically or deletes a placement. +If a placement won, resume attach while closed: its existing-org placement batches bypass +the ordinary close/quota gate. Complete attach to `serving` before explicitly reopening +with `loom cell limit --open`. Otherwise rerun the cleanup with a fresh claim, then run `loom cell remove --cell ` with the same cell settings. Removal now holds that cell's direct Postgres lock and takes a fresh claim for both first and later cells. It also requires a first cell's durable local state to read `open` before deleting the row. diff --git a/server/cli/cell.py b/server/cli/cell.py index 6d6f23697f..77b13e8aec 100644 --- a/server/cli/cell.py +++ b/server/cli/cell.py @@ -439,7 +439,7 @@ def _add_cell_parser(sub: Any) -> None: limit.add_argument("--cell", required=True, help="The cell to limit") state = limit.add_mutually_exclusive_group() state.add_argument("--closed", action="store_true", help="Refuse its new orgs") - state.add_argument("--open", action="store_true", help="Take new orgs again") + state.add_argument("--open", action="store_true", help="Take new orgs again once serving") limit.add_argument( "--placements-max", metavar="N|none", help="Its quota of orgs (none: no quota)" ) diff --git a/server/router/directory.py b/server/router/directory.py index 185c5126d6..5158d24cf7 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -314,6 +314,10 @@ async def set_limit( raise ValueError("set_limit sets closed, placements_max or both") async def change(conn: Any, row: Cell) -> dict[str, Any]: + if closed is False and row.state == "attaching": + raise OperatorRefused( + f"cell {cell_name!r} is attaching: it can reopen after attach reaches serving" + ) detail: dict[str, Any] = {} if closed is not UNSET: detail["closed"] = closed diff --git a/tests/integration/test_cell_control_fence.py b/tests/integration/test_cell_control_fence.py index a6799d2064..d96f170fa3 100644 --- a/tests/integration/test_cell_control_fence.py +++ b/tests/integration/test_cell_control_fence.py @@ -144,6 +144,7 @@ async def delayed(*args, **kwargs): @pytest.mark.asyncio async def test_replayed_claim_lost_ack_and_noop_wrong_generation(control_stores) -> None: client, conn, _ = control_stores + await conn.execute("UPDATE loom_router.cells SET state = 'serving'") snapshot = await read(client) lost = await claim(client, snapshot) # its caller lost this acknowledgement current = await claim(client, snapshot) # duplicate HTTP execution must rotate again @@ -611,3 +612,190 @@ async def ready(): finally: await a.close() await b.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fresh_open", [False, True]) +async def test_backout_close_cannot_be_reopened_before_same_session_cleanup( + control_stores, local_control, fresh_open +): + from server.cli import cell_attach_pg as apg + from server.cli import cell_control + + client, directory_admin, _ = control_stores + local, _ = local_control + await directory_admin.execute( + "UPDATE loom_router.cells SET mode = 'enforced', placements_max = 7" + ) + await local.execute("INSERT INTO loom.cell_first(is_first, cell) VALUES (true, 'first')") + await local.execute("INSERT INTO loom.tenancy_mode(mode) VALUES ('recording')") + await local.execute("INSERT INTO loom.cell_self(current_fp) VALUES ('{\"v\":1}')") + await apg.take_lock(local, "backout reopen test") + pid = local.get_server_pid() + before_close = opened = placement_error = None + + async def call(path, body): + nonlocal before_close, opened, placement_error + response = await client.post("/cells/v1/operator/cells/first" + path, json=body) + assert response.status_code == 200, response.text + answer = response.json() + if path == "/claim": + # The queued opener has the NEW generation, read before cleanup closes D. + before_close = await read(client) + if path == "/limit": + assert answer["closed"] and answer["placed_orgs"] == 0 + assert await local.fetchval("SELECT EXISTS(SELECT 1 FROM loom.cell_self)") + assert await apg.lock_held(local) + snapshot = await read(client) if fresh_open else before_close + opened = await client.post( + "/cells/v1/operator/cells/first/limit", + json={**expected(snapshot), "closed": False, "placements_max": 99}, + ) + # Actually attempt placement before the acknowledged close returns to cleanup. + try: + await directory.place("between-close-and-cleanup", "first", app.placement_refusal) + except directory.PlacementRefused as exc: + placement_error = str(exc) + return answer + + await cell_control.backout(local, call, await read(client)) + assert opened.status_code == 409, opened.text + assert "serving" in opened.json()["error"] + assert placement_error and "closed" in placement_error + after = await read(client) + assert after["placed_orgs"] == 0 + assert tuple( + await directory_admin.fetchrow("SELECT closed, placements_max FROM loom_router.cells") + ) == (True, 7) + assert after["mutation_generation"] == before_close["mutation_generation"] + assert not await directory_admin.fetchval("SELECT EXISTS(SELECT 1 FROM loom_router.placements)") + assert ( + await directory_admin.fetchval( + "SELECT count(*) FROM loom_router.audit WHERE action = 'limit'" + ) + == 1 + ) + assert local.get_server_pid() == pid and await apg.lock_held(local) + for table in ("cell_first", "tenancy_mode", "cell_self", "tenancy"): + assert not await local.fetchval(f"SELECT EXISTS(SELECT 1 FROM loom.{table})") + + +@pytest.mark.asyncio +async def test_attaching_open_refusal_preserves_guard_order_and_closed_attach_progress( + control_stores, +): + client, conn, _ = control_stores + original = await read(client) + current = await claim(client, original) + + async def limit(snapshot, **fields): + return await client.post( + "/cells/v1/operator/cells/first/limit", json={**expected(snapshot), **fields} + ) + + stale = await limit(original, closed=False, placements_max=99) + assert stale.status_code == 409 and "generation" in stale.json()["error"] + for field, value in (("host", "other.example.test"), ("current_fp", {"v": 2})): + changed = await limit({**current, field: value}, closed=False, placements_max=99) + assert ( + changed.status_code == 409 and "registration or fingerprints" in changed.json()["error"] + ) + noop = await limit(current, closed=False, placements_max=99) + assert noop.status_code == 409 and "serving" in noop.json()["error"] + assert await conn.fetchval("SELECT placements_max FROM loom_router.cells") is None + assert not await conn.fetchval( + "SELECT EXISTS(SELECT 1 FROM loom_router.audit WHERE action='limit')" + ) + assert (await limit(current, closed=True)).status_code == 200 + assert (await limit(current, placements_max=0)).status_code == 200 + closed = await read(client) + assert tuple(await conn.fetchrow("SELECT closed, placements_max FROM loom_router.cells")) == ( + True, + 0, + ) + assert closed["mutation_generation"] == current["mutation_generation"] + + # An earlier mode request cannot escape the cleanup owner's claim. + stale_mode = await client.post( + "/cells/v1/operator/cells/first/attach/enforced", + json={**expected(original), "deployment_id": "dep-first"}, + ) + assert stale_mode.status_code == 409 and "generation" in stale_mode.json()["error"] + # Existing held-org attach batches intentionally bypass closed/quota. + await conn.execute( + "UPDATE loom_router.cells SET mode='enforced', attach_started_at=now(), current_fp='{\"v\":1}'" + ) + row = await read(client) + values = expected(row, generation=False)["expected"] + control = directory.Control(values, uuid.UUID(row["mutation_generation"])) + assert await directory.attach_place("first", "dep-first", ["held-org"], expected=control) == ( + 1, + 1, + ) + served = await directory.attach_serving("first", "dep-first", expected=control) + assert served.row.closed and served.row.state == "serving" + response = await limit(await read(client), closed=False) + assert response.status_code == 200, response.text + assert not response.json()["closed"] + + +@pytest.mark.asyncio +async def test_pending_attach_batch_cannot_escape_backout_claim( + control_stores, local_control, monkeypatch +): + import asyncpg + + from server.cli import cell_attach_pg as apg + from server.cli import cell_control + + client, directory_admin, _ = control_stores + local, dsn = local_control + await directory_admin.execute( + "UPDATE loom_router.cells SET mode='enforced', attach_started_at=now(), current_fp='{\"v\":1}'" + ) + await local.execute("INSERT INTO loom.cell_first(is_first,cell) VALUES(true,'first')") + await local.execute("INSERT INTO loom.tenancy_mode(mode) VALUES('recording')") + await local.execute("INSERT INTO loom.cell_self(current_fp) VALUES('{\"v\":1}')") + old_backend = await asyncpg.connect(dsn) + await apg.take_lock(old_backend, "old attach batch") + old = await claim(client, await read(client)) + entered, resume = asyncio.Event(), asyncio.Event() + real = directory.attach_place + + async def delayed(*args, **kwargs): + entered.set() + await resume.wait() + return await real(*args, **kwargs) + + monkeypatch.setattr(directory, "attach_place", delayed) + pending = asyncio.create_task( + client.post( + "/cells/v1/operator/cells/first/attach/place", + json={**expected(old), "deployment_id": "dep-first", "orgs": ["old-batch"]}, + ) + ) + + async def call(path, body): + response = await client.post("/cells/v1/operator/cells/first" + path, json=body) + assert response.status_code == 200, response.text + if path == "/limit": + resume.set() + stale = await pending + assert stale.status_code == 409 and "generation" in stale.json()["error"] + return response.json() + + try: + await asyncio.wait_for(entered.wait(), 3) + await local.execute("SELECT pg_terminate_backend($1)", old_backend.get_server_pid()) + await apg.take_lock(local, "backout successor") + await cell_control.backout(local, call, await read(client)) + assert await apg.lock_held(local) + assert not await local.fetchval("SELECT EXISTS(SELECT 1 FROM loom.cell_self)") + assert not await directory_admin.fetchval( + "SELECT EXISTS(SELECT 1 FROM loom_router.placements)" + ) + assert await directory_admin.fetchval("SELECT closed FROM loom_router.cells") + finally: + resume.set() + await pending + await old_backend.close() diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json index 4e45db4285..1e7acd696a 100644 --- a/tests/unit/snapshots/cli_help.json +++ b/tests/unit/snapshots/cli_help.json @@ -445,7 +445,7 @@ ], "loom cell limit": [ "usage: loom cell limit [-h] --cell CELL [--closed | --open] [--placements-max N|none] [--router ROUTER]", - "options: -h, --help show this help message and exit --cell CELL The cell to limit --closed Refuse its new orgs --open Take new orgs again --placements-max N|none Its quota of orgs (none: no quota) --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" + "options: -h, --help show this help message and exit --cell CELL The cell to limit --closed Refuse its new orgs --open Take new orgs again once serving --placements-max N|none Its quota of orgs (none: no quota) --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" ], "loom cell which": [ "usage: loom cell which [-h] --org ORG [--router ROUTER]", diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json index a31a862106..3bc4f24ee5 100644 --- a/tests/unit/snapshots/cli_tree.json +++ b/tests/unit/snapshots/cli_tree.json @@ -12622,7 +12622,7 @@ "nargs": 0, "type": null, "required": false, - "help": "Take new orgs again", + "help": "Take new orgs again once serving", "metavar": null }, { diff --git a/tests/unit/test_cell_control_census.py b/tests/unit/test_cell_control_census.py new file mode 100644 index 0000000000..aec52c908c --- /dev/null +++ b/tests/unit/test_cell_control_census.py @@ -0,0 +1,503 @@ +"""Inventory cell control SQL writers and the shared boundaries they must retain. + +This recognizes Python string literals (including adjacent literals) and f-strings. It is a +source change gate, not a proof for arbitrary generated SQL; backend/session behavior +is exercised against Postgres in test_cell_control_fence. +""" + +from __future__ import annotations + +import ast +import re +from collections import Counter +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +# Include immutable placement writes: their exemption must remain explicit. +TABLES = r"(?:loom\.(?:cell_self|cell_first|tenancy_mode)|loom_router\.[A-Za-z_]+)" +TARGET = rf"(?:{TABLES}|loom(?:_router)?\.\{{[^}}]+\}}|\{{[^}}]+\}})" +WRITES = re.compile( + rf"(?:^|;)\s*(INSERT\s+INTO|UPDATE|DELETE\s+FROM|TRUNCATE(?:\s+TABLE)?|" + rf"ALTER\s+TABLE(?:\s+IF\s+EXISTS)?|CREATE\s+TABLE(?:\s+IF\s+NOT\s+EXISTS)?|" + rf"DROP\s+TABLE(?:\s+IF\s+EXISTS)?)\s+({TARGET})" + rf"|(?:^|;)\s*((?:CREATE|DROP)\s+TRIGGER)\b[^;]*?\bON\s+({TARGET})", + re.IGNORECASE, +) + + +def mutations(sql: str) -> list[tuple[str, str]]: + sql = re.sub(r"--[^\n]*", "", sql).replace('"', "") + return [(" ".join((a or c).upper().split()), b or d) for a, b, c, d in WRITES.findall(sql)] + + +class Inventory(ast.NodeVisitor): + def __init__(self) -> None: + self.scope: list[str] = [] + self.writes: Counter[tuple[str, str, str]] = Counter() + self.functions: dict[str, ast.AST] = {} + + def visit_FunctionDef(self, node: ast.FunctionDef | ast.AsyncFunctionDef) -> None: + self.scope.append(node.name) + self.functions[".".join(self.scope)] = node + self.generic_visit(node) + self.scope.pop() + + visit_AsyncFunctionDef = visit_FunctionDef + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + self.scope.append(node.name) + self.generic_visit(node) + self.scope.pop() + + def record(self, sql: str) -> None: + for verb, target in mutations(sql): + self.writes[(".".join(self.scope), verb, target)] += 1 + + def visit_Constant(self, node: ast.Constant) -> None: + if isinstance(node.value, str): + self.record(node.value) + + def visit_JoinedStr(self, node: ast.JoinedStr) -> None: + self.record( + "".join( + str(part.value) + if isinstance(part, ast.Constant) + else "{" + ast.unparse(part.value) + "}" + for part in node.values + ) + ) + # Do not count each f-string fragment as another SQL statement. + + +def inventory(source: str) -> Inventory: + found = Inventory() + found.visit(ast.parse(source)) + return found + + +def test_repository_cell_control_writers_are_classified(): + actual = {} + for path in (ROOT / "server").rglob("*.py"): + writes = inventory(path.read_text()).writes + if writes: + actual[str(path.relative_to(ROOT))] = dict(writes) + expected = {} + for group in (GUARDED_WRITERS, EXEMPT_WRITERS, GENERIC_SQL_EXCLUSIONS): + for path, entries in group.items(): + expected.setdefault(path, {}).update(entries) + assert actual == expected + + +def test_cell_control_migrations_are_classified(): + actual = {} + for path in (ROOT / "schemas").rglob("*.sql"): + writes = Counter(mutations(path.read_text())) + if writes: + actual[str(path.relative_to(ROOT))] = dict(writes) + assert actual == MIGRATION_WRITERS + + +# Entries name every sink, including nested callbacks and dynamic table expressions. +# A new sink, including another statement in an existing owner, requires classification. +GUARDED_WRITERS = { + "server/cli/cell_attach_pg.py": {("_mode_try", "UPDATE", "loom.tenancy_mode"): 1}, + "server/cli/cell_control.py": { + ("backout", "ALTER TABLE", "loom.cell_first"): 3, + ("backout", "DELETE FROM", "loom.{table}"): 1, + ("backout", "DROP TRIGGER", "loom.{table}"): 1, + }, + "server/cli/cell_prepare.py": { + ("_write_rows", "INSERT INTO", "loom.cell_first"): 1, + ("_write_rows", "INSERT INTO", "loom.tenancy_mode"): 1, + ("_write_rows", "INSERT INTO", "loom.cell_self"): 1, + }, + "server/router/directory.py": { + ("set_limit.change", "UPDATE", "loom_router.cells"): 1, + ("rotate_credential.change", "UPDATE", "loom_router.cells"): 1, + ("claim", "UPDATE", "loom_router.cells"): 1, + ("attach_begin", "UPDATE", "loom_router.cells"): 1, + ("attach_place", "INSERT INTO", "loom_router.placements"): 1, + ("attach_place", "UPDATE", "loom_router.cells"): 1, + ("attach_place", "INSERT INTO", "loom_router.audit"): 1, + ("attach_enforced", "UPDATE", "loom_router.cells"): 1, + ("attach_serving", "UPDATE", "loom_router.cells"): 1, + ("remove", "DELETE FROM", "loom_router.hosts"): 1, + ("remove", "DELETE FROM", "loom_router.cells"): 1, + }, +} + +# Registration only bootstraps or validates its immutable rerun. Ordinary placement +# writes an immutable org placement/count under its org and cell locks. Reads write +# audit rows only. Trigger setup only enables/installs guards under the migration lock. +EXEMPT_WRITERS = { + "server/core/cell_triggers.py": { + ("_create", "CREATE TRIGGER", "{qualified}"): 2, + ("ensure", "ALTER TABLE", "loom.cell_first"): 1, + ("_table_statements", "ALTER TABLE", "loom.{_ident(table)}"): 1, + }, + "server/router/directory.py": { + ("register", "INSERT INTO", "loom_router.cells"): 1, + ("register", "INSERT INTO", "loom_router.hosts"): 1, + ("register", "INSERT INTO", "loom_router.audit"): 1, + ("_place_one", "INSERT INTO", "loom_router.placements"): 1, + ("_place_one", "UPDATE", "loom_router.cells"): 1, + ("_place_one", "INSERT INTO", "loom_router.audit"): 1, + ("_audit", "INSERT INTO", "loom_router.audit"): 1, + }, +} + +# Unresolved table SQL outside cell controls is classified by exact function and +# target expression, never by a whole-file or execute/f-string exemption. These are +# ClickHouse content/retention/rebuild stores, Keeper locks/marks and provider tables; +# pg_migrate writes only its named migration ledger, and prepare's nonce writes only +# the probe lock table. None is a cell acceptance or directory control write. +GENERIC_SQL_EXCLUSIONS = { + "server/cli/cell_prepare.py": {("_write_nonces", "INSERT INTO", "{leases.LOCK_TABLE}"): 1}, + "server/code/store/index/prefix_twin.py": {("fill_sql", "INSERT INTO", "{table}"): 1}, + "server/core/ch.py": {("_fenced_statement", "INSERT INTO", "{qualify(table)}"): 1}, + "server/core/leases.py": { + ("", "INSERT INTO", "{LOCK_TABLE}"): 1, + ("", "ALTER TABLE", "{LOCK_TABLE}"): 4, + }, + "server/core/namespace_write_marks.py": {("", "INSERT INTO", "{KEYS_TABLE}"): 1}, + "server/core/schema/migrate_lease.py": {("", "CREATE TABLE IF NOT EXISTS", "{LOCK_TABLE}"): 1}, + "server/core/schema/pg_migrate.py": { + ("_ledger", "CREATE TABLE IF NOT EXISTS", "{schema}"): 1, + ("_record_online", "INSERT INTO", "{schema}"): 1, + ("_apply", "INSERT INTO", "{schema}"): 1, + }, + "server/core/schema/table_rebuild.py": { + ("copy_sql", "INSERT INTO", "loom.{dst}"): 1, + ("apply", "DROP TABLE", "loom.{twin}"): 2, + }, + "server/erasure/store/mutations.py": { + ("run_mutation", "ALTER TABLE", "{table}"): 1, + ("run_lightweight_delete", "DELETE FROM", "{table}"): 1, + ("run_keeper_delete", "ALTER TABLE", "{table}"): 1, + }, + "server/erasure/store/reclaim.py": {("apply_deleted_mask", "ALTER TABLE", "{table}"): 1}, + "server/erasure/store/steps_memory.py": { + ("_scrub_link_cursors", "DELETE FROM", "{_CURSORS}"): 1 + }, + "server/erasure/store/steps_principals.py": {("_rewrite", "INSERT INTO", "{table}"): 1}, + "server/insights/domain/analysis.py": {("entity_sql", "INSERT INTO", "{db}"): 1}, + "server/insights/domain/provider_prepared_ddl.py": { + ("prepared_ddl", "CREATE TABLE", "loom.{prepared}"): 1 + }, + "server/insights/store/provider_cache.py": { + ("invalidate_staged_cache", "TRUNCATE TABLE", "loom.{names.target}"): 1 + }, + "server/insights/store/provider_cleanup.py": { + ("cleanup_rebuild_copies", "DROP TABLE IF EXISTS", "{db}"): 1 + }, + "server/insights/store/provider_participants.py": {("", "ALTER TABLE", "{TABLE}"): 1}, + "server/insights/store/provider_prefix.py": { + ("derive_staged_prefixes", "TRUNCATE TABLE", "loom.{names.target}"): 1, + ("derive_staged_prefixes", "INSERT INTO", "loom.{names.target}"): 1, + }, + "server/insights/store/provider_prepared.py": { + ("prepare_live_table", "DROP TABLE", "loom.{names.prepared}"): 1, + ("prepare_live_table", "INSERT INTO", "loom.{names.prepared}"): 1, + ("prepare_live_table", "ALTER TABLE", "loom.{names.prepared}"): 1, + }, + "server/insights/store/provider_staging.py": { + ("create_snapshot", "CREATE TABLE", "loom.{names.snapshot}"): 1, + ("complete_snapshot", "ALTER TABLE", "loom.{names.snapshot}"): 1, + ("discard_snapshot", "DROP TABLE", "loom.{names.snapshot}"): 1, + ("snapshot_query_running", "CREATE TABLE", "loom.{names.snapshot}"): 1, + }, + "server/insights/store/provider_target.py": { + ("ensure_target", "CREATE TABLE", "loom.{names.target}"): 1, + ("_insert_patches", "INSERT INTO", "loom.{names.target}"): 1, + }, + "server/knowledge/store/blobs.py": { + ("copy_held", "INSERT INTO", "{TABLE}"): 1, + ("unwrite", "DELETE FROM", "{TABLE}"): 1, + ("delete", "DELETE FROM", "{TABLE}"): 1, + ("delete", "DELETE FROM", "{UPLOADS}"): 1, + ("forget_uploads", "DELETE FROM", "{UPLOADS}"): 1, + }, + "server/knowledge/store/chunks.py": { + ("_record_supersessions", "INSERT INTO", "{SUPERSESSIONS_TABLE}"): 1 + }, + "server/knowledge/store/conversations.py": { + ("mark_message_saved", "ALTER TABLE", "{table}"): 1, + ("unsave_message", "DELETE FROM", "{table}"): 1, + ("unsave_message", "ALTER TABLE", "{table}"): 1, + }, + "server/knowledge/store/datasets.py": { + ("forget", "DELETE FROM", "{ROWS}"): 1, + ("forget", "DELETE FROM", "{DATASETS}"): 1, + ("forget_published", "DELETE FROM", "{ROWS}"): 1, + ("forget_published", "DELETE FROM", "{DATASETS}"): 1, + ("copy_snapshot", "INSERT INTO", "{ROWS}"): 1, + ("forget_snapshot", "DELETE FROM", "{ROWS}"): 1, + }, + "server/knowledge/store/gaps.py": { + ("delete_miss", "DELETE FROM", "{MISSES}"): 1, + ("delete_build", "DELETE FROM", "{GAPS}"): 1, + ("delete_misses", "DELETE FROM", "{MISSES}"): 1, + ("delete_builds", "DELETE FROM", "{GAPS}"): 1, + }, + "server/knowledge/store/restore.py": {("unretire", "DELETE FROM", "{SUPERSESSIONS_TABLE}"): 1}, + "server/memory/app/reembed.py": { + ("in_database_sql", "INSERT INTO", "loom.{table}"): 1, + ("delete_off_model", "DELETE FROM", "loom.{table}"): 1, + }, + "server/memory/store/code_links.py": { + ("_purge_tombstones", "DELETE FROM", "{TABLE}"): 1, + ("_drop_gone", "DELETE FROM", "{TABLE}"): 1, + }, + "server/memory/store/graph_projections.py": {("KeeperLayout", "INSERT INTO", "{TABLE}"): 1}, + "server/memory/store/lifecycle.py": {("_tombstone_where", "ALTER TABLE", "{table}"): 1}, + "server/memory/store/redaction.py": {("delete_by_key", "ALTER TABLE", "{table}"): 1}, + "server/memory/store/term_freq.py": {("rebuild", "ALTER TABLE", "{table}"): 1}, + "server/memory/store/webhooks.py": {("mark_dead", "INSERT INTO", "{ATTEMPTS}"): 1}, + "server/memory/workers/usage_fold.py": { + ("_apply_update", "UPDATE", "{table}"): 1, + ("_apply_update", "ALTER TABLE", "{table}"): 1, + }, + "server/storage/retention_overrides.py": { + ("compute_alters", "ALTER TABLE", "{target.table}"): 1, + ("current_ttl_matches", "ALTER TABLE", "{table}"): 1, + ("apply", "ALTER TABLE", "{target.table}"): 1, + }, +} + +# Schema creation/extension and immutable-trigger installation run as migrations, +# never as live acceptance writers. Offline restore generation rotation stays stopped +# and fenced as described in the recovery guide, not a live SQL exemption. +MIGRATION_WRITERS = { + "schemas/pg_migrations/008_cells.sql": { + ("CREATE TABLE IF NOT EXISTS", "loom.cell_first"): 1, + ("DROP TRIGGER", "loom.cell_first"): 2, + ("CREATE TRIGGER", "loom.cell_first"): 2, + ("ALTER TABLE", "loom.cell_first"): 2, + ("CREATE TABLE IF NOT EXISTS", "loom.tenancy_mode"): 1, + ("CREATE TABLE IF NOT EXISTS", "loom.cell_self"): 1, + }, + "schemas/router_migrations/001_directory.sql": { + ("CREATE TABLE IF NOT EXISTS", "loom_router.cells"): 1, + ("CREATE TABLE IF NOT EXISTS", "loom_router.placements"): 1, + ("DROP TRIGGER", "loom_router.placements"): 2, + ("CREATE TRIGGER", "loom_router.placements"): 2, + ("CREATE TABLE IF NOT EXISTS", "loom_router.hosts"): 1, + ("CREATE TABLE IF NOT EXISTS", "loom_router.audit"): 1, + }, + "schemas/router_migrations/002_cell_upstream.sql": {("ALTER TABLE", "loom_router.cells"): 1}, + "schemas/router_migrations/003_cell_next_credential.sql": { + ("ALTER TABLE", "loom_router.cells"): 1 + }, + "schemas/router_migrations/005_control_generation.sql": { + ("ALTER TABLE", "loom_router.cells"): 1 + }, +} + + +def calls(node: ast.AST) -> set[str]: + return {ast.unparse(child) for child in ast.walk(node) if isinstance(child, ast.Call)} + + +def function(path: str, name: str) -> ast.AST: + return inventory((ROOT / path).read_text()).functions[name] + + +@pytest.mark.parametrize( + ("path", "name", "required"), + [ + ("server/cli/cell_prepare.py", "attach_lock", ['cell_control.session("prepare")']), + ( + "server/cli/cell_prepare.py", + "_prepare_locked", + [ + "cell_control.claim(conn, call, snapshot)", + "_write_rows(conn, rows, answer, current, nxt)", + ], + ), + ( + "server/cli/cell_prepare.py", + "_write_rows", + ["cell_control.transaction(conn)", "_read_rows(conn, count_tenancy=False)"], + ), + ( + "server/cli/cell_attach.py", + "Run.run", + ["self.open_lock()", "cell_control.claim(self.lock_conn, self.router.call, self.row)"], + ), + ( + "server/cli/cell_attach.py", + "Run.commit_mode", + [ + "apg.commit_enforced(self.lock_conn, self.confirm_lock, self.ready_for_commit, self.wait, self.log, mark, expected_cell=self.name, expected_fp=self.own_fp)" + ], + ), + ( + "server/cli/cell_attach_pg.py", + "_mode_try", + ["cell_control.check_lock(conn)", "conn.transaction()", "tx.commit()"], + ), + ( + "server/cli/cell_control.py", + "transaction", + ["check_lock(conn)", "tx.commit()"], + ), + ( + "server/cli/cell_control.py", + "claim", + ["check_lock(conn)", "paired(conn, row)"], + ), + ( + "server/cli/cell_control.py", + "backout", + [ + "claim(conn, call, row)", + 'mutate(conn, call, row, "/limit", {"closed": True})', + "transaction(conn)", + "paired(conn, row)", + ], + ), + ( + "server/cli/cell.py", + "_remove_locked", + [ + 'cell_control.session("remove")', + "cell_control.claim(conn, call, row)", + "cell_control.check_lock(conn)", + ], + ), + ( + "server/router/directory.py", + "_attach_change", + ["_control_transaction()", "control_refusal(attaching, expected, claim=claim)"], + ), + ( + "server/router/directory.py", + "_operator_change", + ["_attach_change(cell_name, expected)", "change(conn, found.row)"], + ), + ( + "server/router/directory.py", + "set_limit", + ['_operator_change(cell_name, "limit", change, expected)'], + ), + ( + "server/router/directory.py", + "rotate_credential", + ['_operator_change(cell_name, "credential", change, expected)'], + ), + ( + "server/router/directory.py", + "claim", + ["_attach_change(cell_name, expected, claim=True)"], + ), + *[ + ("server/router/directory.py", name, ["_attach_change(cell_name, expected)"]) + for name in ( + "attach_begin", + "attach_place", + "attach_enforced", + "attach_serving", + "remove", + ) + ], + ], +) +def test_guarded_entrypoints_keep_shared_boundaries(path, name, required): + actual = calls(function(path, name)) + for expression in required: + assert ast.unparse(ast.parse(expression, mode="eval").body) in actual, ( + path, + name, + expression, + ) + + +def test_dynamic_cleanup_scope_cannot_expand_unclassified(): + backout = function("server/cli/cell_control.py", "backout") + cleanup_tables = [ + ast.literal_eval(node.iter) + for node in ast.walk(backout) + if isinstance(node, ast.For) + and ast.unparse(node.target) == "table" + and isinstance(node.iter, ast.Tuple) + ] + assert cleanup_tables == [("cell_first", "tenancy_mode", "cell_self", "tenancy")] + assert "t.tgname IN ('tenancy_insert', 'tenancy_update', 'tenancy_record')" in ast.unparse( + backout + ) + + +def test_reads_registration_and_immutable_placement_are_explicit_exceptions(): + path = "server/router/directory.py" + found = inventory((ROOT / path).read_text()) + for name in ( + "routing", + "credentials", + "first_cell", + "first_cell_named", + "lookup", + "attach_read", + ): + assert name in found.functions + assert not any(owner == name for owner, _verb, _target in found.writes) + # Operator reads may audit, but cannot bypass control guards for a state write. + assert any(call.startswith("_audit(") for call in calls(found.functions["attach_read"])) + assert "_rerun_refusal(held, reg)" in calls(found.functions["register"]) + assert "_place_one(conn, org, cell_name, _cell(locked), refusal)" in calls( + found.functions["place"] + ) + assert "refusal(candidate)" in calls(found.functions["_place_one"]) + for name in ("set_limit", "rotate_credential"): + assert not any(call.startswith("claim(") for call in calls(found.functions[name])) + + +@pytest.mark.parametrize( + ("statement", "expected"), + [ + ('"UPDATE loom.cell_self SET current_fp = $1"', ("UPDATE", "loom.cell_self")), + ( + '"INSERT INTO loom_router.cells (cell) VALUES ($1)"', + ("INSERT INTO", "loom_router.cells"), + ), + ('f"DELETE FROM loom.{table}"', ("DELETE FROM", "loom.{table}")), + ('f"UPDATE {target} SET current_fp = $1"', ("UPDATE", "{target}")), + ('"DROP TABLE IF EXISTS loom.cell_self"', ("DROP TABLE IF EXISTS", "loom.cell_self")), + ('f\'DROP TRIGGER "{name}" ON loom."{table}"\'', ("DROP TRIGGER", "loom.{table}")), + ], +) +def test_inventory_detects_new_literal_and_dynamic_writers(statement, expected): + source = ( + f"async def unclassified(conn, table, target, name):\n await conn.execute({statement})\n" + ) + found = inventory(source).writes + assert found == Counter({("unclassified", *expected): 1}) + assert not any( + "unclassified" == owner + for group in (GUARDED_WRITERS, EXEMPT_WRITERS, GENERIC_SQL_EXCLUSIONS) + for entries in group.values() + for owner, _verb, _target in entries + ) + + +def test_acceptance_writer_callsites_are_classified(): + # Another caller of a checked writer still needs its own claim and ownership proof. + names = {"_write_rows", "_mode_try", "commit_enforced", "backout"} + actual = Counter() + for path in (ROOT / "server").rglob("*.py"): + found = inventory(path.read_text()) + for owner, node in found.functions.items(): + # These owners have no nested functions; walk only the direct callsites. + for child in ast.walk(node): + if isinstance(child, ast.Call): + target = ast.unparse(child.func) + if target.rsplit(".", 1)[-1] in names: + actual[(str(path.relative_to(ROOT)), owner, target)] += 1 + assert actual == Counter( + { + ("server/cli/cell_prepare.py", "_prepare_locked", "_write_rows"): 1, + ("server/cli/cell_attach_pg.py", "commit_enforced", "_mode_try"): 1, + ("server/cli/cell_attach.py", "Run.commit_mode", "apg.commit_enforced"): 1, + } + ) From 438018601bb9aebe515f7eb3ba280ff4dbfff69a Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 21:24:41 +0000 Subject: [PATCH 024/114] Build placement indexes outside migration transactions Concurrent index builds release the ordinary migration key so a shipped blocking migrator can finish without retaining a snapshot behind the build. Retry from the catalogs and checksum ledger, and acknowledge a WAL fence on recorded reruns. Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 29 ++ docs/development/migrations.md | 54 ++- .../004_placements_by_cell_org.sql | 3 + server/core/schema/pg_migrate.py | 154 ++++++-- server/core/schema/pg_online_index.py | 125 ++++++ .../fixtures/pg_migrate_n_minus_one.py | 151 +++++++ tests/integration/test_pg_online_index.py | 373 ++++++++++++++++++ .../test_pg_online_index_durability.py | 203 ++++++++++ tests/unit/test_cell_org_triggers.py | 8 + tests/unit/test_pg_online_index.py | 212 ++++++++++ 10 files changed, 1275 insertions(+), 37 deletions(-) create mode 100644 schemas/router_migrations/004_placements_by_cell_org.sql create mode 100644 server/core/schema/pg_online_index.py create mode 100644 tests/integration/fixtures/pg_migrate_n_minus_one.py create mode 100644 tests/integration/test_pg_online_index.py create mode 100644 tests/integration/test_pg_online_index_durability.py create mode 100644 tests/unit/test_pg_online_index.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22342c2cab..a132706be6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -896,10 +896,39 @@ jobs: tests/integration/test_pg_clickhouse_live_ch.py tests/integration/test_cell_tenancy_pg.py tests/integration/test_router_directory_pg.py + tests/integration/test_pg_online_index.py tests/integration/test_cell_unset_gate_pg.py tests/integration/test_cell_prepare.py tests/integration/test_cell_attach.py tests/integration/test_cell_org_triggers_pg.py + - name: pytest online-index durability on a private physical standby + timeout-minutes: 8 + env: + LOOM_PG_IMAGE: postgres:${{ matrix.postgres }} + LOOM_PG_CONTAINER: loom-scale-index-drill-pg + LOOM_PG_HOST_PORT: "55432" + LOOM_PG_INDEX_DRILL: "1" + LOOM_PG_INDEX_DRILL_DSN: postgres://loom:loomdev@localhost:55432/loom + run: | + set -euo pipefail + for name in loom-scale-index-drill-pg loom-scale-index-drill-pg-standby; do + if docker container inspect "$name" >/dev/null 2>&1; then + echo "drill container already exists: $name" >&2 + exit 1 + fi + done + if docker network inspect loom-scale-index-drill-pg-net >/dev/null 2>&1; then + echo "drill network already exists" >&2 + exit 1 + fi + cleanup() { + docker rm -f -v loom-scale-index-drill-pg loom-scale-index-drill-pg-standby || true + docker network rm loom-scale-index-drill-pg-net || true + } + trap cleanup EXIT + bash dev/ci_start_postgres.sh + docker update --memory 1g --memory-swap 1g loom-scale-index-drill-pg loom-scale-index-drill-pg-standby + pytest -q tests/integration/test_pg_online_index_durability.py # The tenant-isolation gates again, against a second server that enforces auth: # the Postgres read paths (heads by key and by id, narrowed scopes) see real tokens. # They run whether or not the memory suites passed (`!cancelled()`). diff --git a/docs/development/migrations.md b/docs/development/migrations.md index bbea754e87..0fcc560a45 100644 --- a/docs/development/migrations.md +++ b/docs/development/migrations.md @@ -157,20 +157,56 @@ operator step ([release process](./release-process.md#release-notes)). edited once merged, and every `CREATE`, `ALTER` or `DROP` of a table names `loom.
`, except a rename's new name, where Postgres takes no schema (`tests/unit/test_storage_layout_rows.py`). A boot applies them under a Postgres -advisory lock, each file in one transaction with its ledger row. A migration waits at +advisory lock, each ordinary file in one transaction with its ledger row. A migration waits at most 5 s for a table lock (`lock_timeout`, then retried), so an `ALTER` never queues every query of the table behind a long transaction. A build serves a ledger that holds a later build's files, with a warning naming them; a ledger file it does not carry numbered at or below its own last file, or one whose checksum changed, refuses the boot. -Every file runs in one transaction, so a migration builds no index on a large table: the -build would hold off the table's writes, every commit answering 503, until it ends. Such -an index needs `CREATE INDEX CONCURRENTLY` outside a transaction, and a boot waiting on -the advisory lock deadlocks with it (a waiting session holds a snapshot the build waits -out): a change that adds such a file also takes the lock by retrying -`pg_try_advisory_lock`. When the first release that writes a table carries the file, the -table is small wherever the file runs, and its index may be built in the file's -transaction (`010_revision_delivered_at.sql`). +Ordinary files run in one transaction, so they build no index on a table that +already ships: a transactional build blocks writes until it ends. An index on a +table introduced by the same release may be transactional because that table is +small wherever the file runs (`010_revision_delivered_at.sql`). + +For an existing table, the shared Postgres runner accepts only this online file +form, in either the cell or router journal: + +```sql +-- loom:online-index loom_router.placements_by_cell_org ON loom_router.placements (cell, org) +CREATE INDEX CONCURRENTLY placements_by_cell_org ON loom_router.placements +USING btree (cell COLLATE "C", org COLLATE "C"); +``` + +The declaration and statement must name the same schema, table, index and ordered +columns. Only ordinary identifiers, a non-unique btree and C collations are +accepted. Expressions, predicates, included columns and additional statements +are refused before the ledger changes. This is not a general escape from +transactional migrations. + +The runner takes session advisory locks by retrying `pg_try_advisory_lock`, with +no transaction or snapshot between attempts. It releases the ordinary migration +key before acquiring a separate schema-scoped online-index key and running +concurrent DDL. A shipped blocking migrator can therefore finish its unchanged +journal while the index waits for old snapshots. After the build finishes, the +runner reacquires the ordinary key, records the file's checksum in the existing +ledger and commits a WAL-message fence. Later files wait for this ledger entry. + +Catalog validation checks the table, ordered columns and collations, btree +method, absence of constraints and expressions, and valid/ready state. A +matching incomplete index is dropped concurrently and rebuilt; a different +same-named object refuses without being dropped. A recorded but absent or +invalid index refuses as corruption. Even when every file is recorded, a build +carrying an online file validates its index and acknowledges a fresh WAL fence: +a primary-visible ledger row after a cancelled synchronous COMMIT is insufficient. +An interrupted build, drop or ledger write is retried from catalogs and the +existing ledger, without a separate job table. Cancellation terminates the +migration session before asyncpg can wait indefinitely on rollback. + +The router's dedicated migration connection has no statement timeout; its +serving pool retains its statement limit. The runner preserves a borrowed +connection's statement budget, including a stopped restore command's deadline. +Concurrent DDL retains the five-second lock timeout and can be interrupted and +retried without changing the file checksum. ## Schema dump and check diff --git a/schemas/router_migrations/004_placements_by_cell_org.sql b/schemas/router_migrations/004_placements_by_cell_org.sql new file mode 100644 index 0000000000..774efeb883 --- /dev/null +++ b/schemas/router_migrations/004_placements_by_cell_org.sql @@ -0,0 +1,3 @@ +-- loom:online-index loom_router.placements_by_cell_org ON loom_router.placements (cell, org) +CREATE INDEX CONCURRENTLY placements_by_cell_org ON loom_router.placements +USING btree (cell COLLATE "C", org COLLATE "C"); diff --git a/server/core/schema/pg_migrate.py b/server/core/schema/pg_migrate.py index 7876c9f178..ebd230036b 100644 --- a/server/core/schema/pg_migrate.py +++ b/server/core/schema/pg_migrate.py @@ -1,6 +1,7 @@ """Postgres migrations: numbered SQL files in `schemas/pg_migrations/`, applied in -order at boot under a session advisory lock, each file in one transaction with -its ledger row (`loom.pg_migrations`, the file's SHA-256). The router's directory +order at boot under a session advisory lock, each ordinary file in a transaction +with its ledger row (`loom.pg_migrations`, the file's SHA-256). Declared online indexes +run concurrently outside transactions under a separate session key. The router's directory (`schemas/router_migrations/`, `server/router/directory.py`) migrates the same way, on its own database, ledger schema and lock. For schema `loom`, `migrate` also marks its session as one that ran the tenancy check and, after the files and before the lock @@ -8,13 +9,14 @@ The rules are the ClickHouse journal's (docs/development/migrations.md): additive only, never edit a merged file. A ledger row whose checksum no longer matches its -file refuses the boot, as does a ledger row for a file this build does not carry -when that file's number is above the build's last one (a newer build's schema is -served, as ClickHouse's is; an unknown lower number means the journal was edited). +file refuses the boot, as does an unknown file numbered at or below this build's +last migration. A build serves a newer schema, warning about unknown files numbered +above its last migration. """ from __future__ import annotations +import asyncio import hashlib import logging from asyncio import sleep @@ -24,7 +26,9 @@ from typing import Any from server.core import pg +from server.core.ch import set_query_tag from server.core.paths import PG_MIGRATIONS_DIR +from server.core.schema import pg_online_index from server.core.schema.journal import migration_number, sorted_migrations log = logging.getLogger(__name__) @@ -105,32 +109,126 @@ async def migrate( holds every read and write of the table behind it; the file is retried. For schema `loom` it marks the session and runs `cell_triggers.ensure` before the lock goes.""" files = migration_files(mig_dir) - done: list[str] = [] - # Closing the connection ends the session, and with it the advisory lock. + # Validate every online declaration before even creating the ledger. + indexes = {f.name: pg_online_index.parse(f.read_text()) for f in files} + if any(index is not None and index.schema != schema for index in indexes.values()): + raise ValueError("online-index target differs from its ledger schema") async with (connect or pg.dedicated)() as conn: - # Another boot migrating holds the lock for as long as its files take. - await conn.execute("SELECT pg_advisory_lock($1)", lock_key) - if schema == "loom": - # A migration's data steps only copy or rewrite rows already stored here: the - # org triggers pass them (server/core/cell_triggers.py). - await conn.execute(f"SET {pg.TENANCY_MARKER} = 'on'") - await conn.execute("SET lock_timeout = '5s'") - await conn.execute(_ledger(schema)) - rows = await conn.fetch(f"SELECT filename, checksum FROM {schema}.pg_migrations") - for name, sql, digest in _read(plan({r["filename"]: r["checksum"] for r in rows}, files)): - await _apply(conn, name, sql, digest, schema=schema) - log.info("applied Postgres migration %s", name) - done.append(name) - if schema == "loom": - # On every migrating boot, after the pending files and before the lock goes. - from server.core import cell_triggers - - await cell_triggers.ensure(conn) - return done + task = asyncio.create_task(_migrate(conn, files, indexes, schema, lock_key)) + try: + return await asyncio.shield(task) + except BaseException: + # Terminate before cancellation can enter asyncpg's unbounded rollback. + conn.terminate() + task.cancel() + await asyncio.wait({task}, timeout=1) + if task.done() and not task.cancelled(): + task.exception() + raise + + +async def _lock(conn: Any, key: int) -> None: + while True: + set_query_tag("migrate.postgres.lock") + if await conn.fetchval("SELECT pg_try_advisory_lock($1)", key): + return + # Each try ends its statement/snapshot before another attempt. + await sleep(0.1) + + +async def _unlock(conn: Any, key: int) -> None: + if not await conn.fetchval("SELECT pg_advisory_unlock($1)", key): + raise RuntimeError("Postgres migration advisory lock was lost") -def _read(files: list[Path]) -> list[tuple[str, str, str]]: - return [(f.name, f.read_text(), checksum(f)) for f in files] +def _online_key(schema: str, lock_key: int) -> int: + value = hashlib.sha256(f"loom.online-index:{schema}:{lock_key}".encode()).digest() + key = int.from_bytes(value[:8], "big") & (2**63 - 1) + if key == lock_key: + raise ValueError("Postgres migration lock keys collide") + return key + + +async def _applied(conn: Any, schema: str, files: list[Path]) -> dict[str, str]: + rows = await conn.fetch(f"SELECT filename, checksum FROM {schema}.pg_migrations") + applied = {r["filename"]: r["checksum"] for r in rows} + plan(applied, files) + return applied + + +async def _record_online(conn: Any, path: Path, schema: str, *, pending: bool) -> None: + async with conn.transaction(): + if pending: + await conn.execute( + f"INSERT INTO {schema}.pg_migrations (filename, checksum) VALUES ($1, $2)", + path.name, + checksum(path), + ) + # A visible ledger from a cancelled synchronous commit needs a fresh flush. + await conn.execute(pg._DURABLE_MESSAGE) + + +async def _online( + conn: Any, + path: Path, + index: pg_online_index.OnlineIndex, + schema: str, + lock_key: int, + files: list[Path], +) -> bool: + await _unlock(conn, lock_key) + online_key = _online_key(schema, lock_key) + await _lock(conn, online_key) + applied = await _applied(conn, schema, files) + await pg_online_index.build(conn, index, recorded=path.name in applied) + # A shipped blocking migrator may wait here; the concurrent build is over. + await _lock(conn, lock_key) + applied = await _applied(conn, schema, files) + pending = path.name not in applied + await _record_online(conn, path, schema, pending=pending) + await _unlock(conn, online_key) + return pending + + +async def _migrate( + conn: Any, + files: list[Path], + indexes: dict[str, pg_online_index.OnlineIndex | None], + schema: str, + lock_key: int, +) -> list[str]: + done: list[str] = [] + await _lock(conn, lock_key) + if schema == "loom": + await conn.execute(f"SET {pg.TENANCY_MARKER} = 'on'") + await conn.execute("SET lock_timeout = '5s'") + await conn.execute("SET synchronous_commit = 'on'") + await conn.execute(_ledger(schema)) + applied = await _applied(conn, schema, files) + for path in files: + set_query_tag("migrate.postgres.apply") + index = indexes[path.name] + if index is not None: + changed = await _online(conn, path, index, schema, lock_key, files) + applied = await _applied(conn, schema, files) + else: + changed = path.name not in applied + if changed: + await _apply(conn, path.name, path.read_text(), checksum(path), schema=schema) + if changed: + log.info("applied Postgres migration %s", path.name) + done.append(path.name) + for index in indexes.values(): + set_query_tag("migrate.postgres.verify") + if index is not None and await pg_online_index.ready(conn, index) is not True: + raise ValueError(f"recorded online-index {index.qualified} is absent or invalid") + if schema == "loom": + from server.core import cell_triggers + + await cell_triggers.ensure(conn) + # A caller may lend its marked session and retain unrelated session keys. + await _unlock(conn, lock_key) + return done async def _apply(conn: Any, name: str, sql: str, digest: str, *, schema: str = "loom") -> None: diff --git a/server/core/schema/pg_online_index.py b/server/core/schema/pg_online_index.py new file mode 100644 index 0000000000..47c2b025a5 --- /dev/null +++ b/server/core/schema/pg_online_index.py @@ -0,0 +1,125 @@ +"""The one nontransactional Postgres migration form: a declared concurrent index.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any + +_ID = r"[a-z_][a-z0-9_]{0,62}" +_DECLARATION = re.compile( + rf"-- loom:online-index (?P{_ID})\.(?P{_ID}) ON " + rf"(?P{_ID})\.(?P
{_ID}) \((?P{_ID}(?:, {_ID})*)\)\n" +) +_CREATE = re.compile( + rf"CREATE\s+INDEX\s+CONCURRENTLY\s+(?P{_ID})\s+ON\s+" + rf"(?P{_ID})\.(?P
{_ID})\s+USING\s+btree\s*" + rf'\((?P{_ID}\s+COLLATE\s+"C"(?:\s*,\s*{_ID}\s+COLLATE\s+"C")*)\)\s*;\s*' +) + + +@dataclass(frozen=True) +class OnlineIndex: + schema: str + name: str + table: str + columns: tuple[str, ...] + sql: str + + @property + def qualified(self) -> str: + return f'"{self.schema}"."{self.name}"' + + +def parse(sql: str) -> OnlineIndex | None: + """Refuse malformed directives before a connection or ledger write is made.""" + if "loom:online-index" not in sql: + return None + declaration = _DECLARATION.match(sql) + create = _CREATE.fullmatch(sql[declaration.end() :]) if declaration else None + if declaration is None or create is None: + raise ValueError("invalid online-index migration declaration or SQL") + columns = tuple(declaration["columns"].split(", ")) + actual = tuple(part.split()[0] for part in create["columns"].split(",")) + if ( + declaration["schema"] != declaration["table_schema"] + or any(declaration[key] != create[key] for key in ("schema", "name", "table")) + or columns != actual + or len(set(columns)) != len(columns) + ): + raise ValueError("online-index SQL does not match its declaration") + return OnlineIndex( + declaration["schema"], + declaration["name"], + declaration["table"], + columns, + sql[declaration.end() :], + ) + + +_CATALOG = """ +SELECT c.relkind::text AS relkind, tn.nspname AS table_schema, t.relname AS table_name, + am.amname, i.indisvalid, i.indisready, i.indislive, + NOT i.indisunique AND NOT i.indisexclusion AND i.indpred IS NULL + AND i.indexprs IS NULL AND i.indnkeyatts = i.indnatts + AND NOT i.indisreplident AND c.reloptions IS NULL AS plain, + ARRAY(SELECT a.attname::text FROM unnest(i.indkey) WITH ORDINALITY k(attnum, n) + JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = k.attnum + ORDER BY k.n) AS columns, + ARRAY(SELECT co.collname::text FROM unnest(i.indcollation) WITH ORDINALITY k(oid, n) + JOIN pg_collation co ON co.oid = k.oid + JOIN pg_namespace ns ON ns.oid = co.collnamespace AND ns.nspname = 'pg_catalog' + ORDER BY k.n) AS collations, + ARRAY(SELECT v FROM unnest(i.indoption) v) AS options, + ARRAY(SELECT op.opcdefault FROM unnest(i.indclass) WITH ORDINALITY k(oid, n) + JOIN pg_opclass op ON op.oid = k.oid ORDER BY k.n) AS default_opclasses, + EXISTS(SELECT 1 FROM pg_constraint WHERE conindid = c.oid) + OR EXISTS(SELECT 1 FROM pg_depend WHERE classid = 'pg_class'::regclass + AND objid = c.oid AND deptype IN ('e', 'x')) + OR EXISTS(SELECT 1 FROM pg_inherits WHERE inhrelid = c.oid) AS dependent +FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace +LEFT JOIN pg_index i ON i.indexrelid = c.oid +LEFT JOIN pg_class t ON t.oid = i.indrelid +LEFT JOIN pg_namespace tn ON tn.oid = t.relnamespace +LEFT JOIN pg_am am ON am.oid = c.relam +WHERE n.nspname = $1 AND c.relname = $2 +""" + + +async def ready(conn: Any, index: OnlineIndex) -> bool | None: + """Absent, matching incomplete, or matching ready; foreign objects always refuse.""" + row = await conn.fetchrow(_CATALOG, index.schema, index.name) + if row is None: + return None + expected = { + "relkind": "i", + "table_schema": index.schema, + "table_name": index.table, + "amname": "btree", + "plain": True, + "columns": list(index.columns), + "collations": ["C"] * len(index.columns), + "options": [0] * len(index.columns), + "default_opclasses": [True] * len(index.columns), + "dependent": False, + } + differences = {key: row[key] for key, value in expected.items() if row[key] != value} + if differences: + raise ValueError( + f"online-index {index.qualified} has a different definition or dependency: " + f"{differences}" + ) + return bool(row["indisvalid"] and row["indisready"] and row["indislive"]) + + +async def build(conn: Any, index: OnlineIndex, *, recorded: bool) -> None: + state = await ready(conn, index) + if recorded and state is not True: + raise ValueError(f"recorded online-index {index.qualified} is absent or invalid") + if state is True: + return + if state is False: + await conn.execute(f"DROP INDEX CONCURRENTLY {index.qualified}") + await conn.execute(index.sql) + if await ready(conn, index) is not True: + raise ValueError(f"online-index {index.qualified} did not become valid and ready") diff --git a/tests/integration/fixtures/pg_migrate_n_minus_one.py b/tests/integration/fixtures/pg_migrate_n_minus_one.py new file mode 100644 index 0000000000..7876c9f178 --- /dev/null +++ b/tests/integration/fixtures/pg_migrate_n_minus_one.py @@ -0,0 +1,151 @@ +"""Postgres migrations: numbered SQL files in `schemas/pg_migrations/`, applied in +order at boot under a session advisory lock, each file in one transaction with +its ledger row (`loom.pg_migrations`, the file's SHA-256). The router's directory +(`schemas/router_migrations/`, `server/router/directory.py`) migrates the same way, +on its own database, ledger schema and lock. For schema `loom`, `migrate` also marks +its session as one that ran the tenancy check and, after the files and before the lock +goes, runs `cell_triggers.ensure` (docs/architecture/cells.md). + +The rules are the ClickHouse journal's (docs/development/migrations.md): additive +only, never edit a merged file. A ledger row whose checksum no longer matches its +file refuses the boot, as does a ledger row for a file this build does not carry +when that file's number is above the build's last one (a newer build's schema is +served, as ClickHouse's is; an unknown lower number means the journal was edited). +""" + +from __future__ import annotations + +import hashlib +import logging +from asyncio import sleep +from collections.abc import Callable +from contextlib import AbstractAsyncContextManager +from pathlib import Path +from typing import Any + +from server.core import pg +from server.core.paths import PG_MIGRATIONS_DIR +from server.core.schema.journal import migration_number, sorted_migrations + +log = logging.getLogger(__name__) + +# A fixed key for pg_advisory_lock: every Loom process migrates under it. +_LOCK_KEY = 0x6C6F6F6D_6D696772 # "loommigr" + + +def _ledger(schema: str) -> str: + # `schema` is one of this build's constants (`loom`, `loom_router`), never input. + return f""" +CREATE SCHEMA IF NOT EXISTS {schema}; +CREATE TABLE IF NOT EXISTS {schema}.pg_migrations ( + filename text PRIMARY KEY, + checksum text NOT NULL, + applied_at timestamptz NOT NULL DEFAULT now() +)""" + + +class MigrationDrift(RuntimeError): + """An applied file changed, or the ledger holds a file the journal lost.""" + + +def migration_files(mig_dir: Path = PG_MIGRATIONS_DIR) -> list[Path]: + return sorted_migrations(mig_dir.glob("*.sql")) + + +def checksum(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def plan(applied: dict[str, str], files: list[Path]) -> list[Path]: + """The files still to apply; raises MigrationDrift on a changed or lost file.""" + known = {f.name: f for f in files} + last = max((migration_number(f.name) or 0 for f in files), default=0) + for name, digest in applied.items(): + f = known.get(name) + if f is None: + if (migration_number(name) or 0) <= last: + raise MigrationDrift(f"applied Postgres migration {name} is not in this build") + continue + if checksum(f) != digest: + raise MigrationDrift(f"applied Postgres migration {name} was edited") + newer = sorted(n for n in applied if n not in known) + if newer: + log.warning("Postgres schema has migrations this build does not carry: %s", newer) + return [f for f in files if f.name not in applied] + + +async def pending(mig_dir: Path = PG_MIGRATIONS_DIR) -> list[str]: + """The files `migrate` would apply, read-only: every file when the ledger is not + there yet. For a process that applies none (`LOOM_BOOT_MIGRATE=0`).""" + import asyncpg + + try: + rows = await pg.fetch("SELECT filename, checksum FROM loom.pg_migrations") + except asyncpg.exceptions.UndefinedTableError: + rows = [] + applied = {r["filename"]: r["checksum"] for r in rows} + return [f.name for f in plan(applied, migration_files(mig_dir))] + + +async def migrate( + mig_dir: Path = PG_MIGRATIONS_DIR, + *, + connect: Callable[[], AbstractAsyncContextManager[Any]] | None = None, + schema: str = "loom", + lock_key: int = _LOCK_KEY, +) -> list[str]: + """Apply every pending file; returns their names. Concurrent boots serialize on + the advisory lock and the later ones find nothing pending. `connect`, `schema` + and `lock_key` name another database's connection, ledger and lock (the router's + directory); the defaults are the memory authority's. + + On a connection of its own, with no client command timeout and no statement + timeout: DDL on a large table runs as long as it takes. A lock it waits for is + bounded (`lock_timeout`), so an `ALTER` queued behind a long transaction never + holds every read and write of the table behind it; the file is retried. For schema + `loom` it marks the session and runs `cell_triggers.ensure` before the lock goes.""" + files = migration_files(mig_dir) + done: list[str] = [] + # Closing the connection ends the session, and with it the advisory lock. + async with (connect or pg.dedicated)() as conn: + # Another boot migrating holds the lock for as long as its files take. + await conn.execute("SELECT pg_advisory_lock($1)", lock_key) + if schema == "loom": + # A migration's data steps only copy or rewrite rows already stored here: the + # org triggers pass them (server/core/cell_triggers.py). + await conn.execute(f"SET {pg.TENANCY_MARKER} = 'on'") + await conn.execute("SET lock_timeout = '5s'") + await conn.execute(_ledger(schema)) + rows = await conn.fetch(f"SELECT filename, checksum FROM {schema}.pg_migrations") + for name, sql, digest in _read(plan({r["filename"]: r["checksum"] for r in rows}, files)): + await _apply(conn, name, sql, digest, schema=schema) + log.info("applied Postgres migration %s", name) + done.append(name) + if schema == "loom": + # On every migrating boot, after the pending files and before the lock goes. + from server.core import cell_triggers + + await cell_triggers.ensure(conn) + return done + + +def _read(files: list[Path]) -> list[tuple[str, str, str]]: + return [(f.name, f.read_text(), checksum(f)) for f in files] + + +async def _apply(conn: Any, name: str, sql: str, digest: str, *, schema: str = "loom") -> None: + import asyncpg + + record = f"INSERT INTO {schema}.pg_migrations (filename, checksum) VALUES ($1, $2)" + attempts = 5 # tries before the boot fails on a table lock held past `lock_timeout` + for attempt in range(1, attempts + 1): + try: + async with conn.transaction(): + await conn.execute(sql) + await conn.execute(record, name, digest) + return + except asyncpg.exceptions.LockNotAvailableError: + if attempt == attempts: + raise + log.warning("Postgres migration %s waited too long for a lock; retrying", name) + await sleep(attempt) diff --git a/tests/integration/test_pg_online_index.py b/tests/integration/test_pg_online_index.py new file mode 100644 index 0000000000..af93dd0937 --- /dev/null +++ b/tests/integration/test_pg_online_index.py @@ -0,0 +1,373 @@ +"""Online index retries and migrator coexistence on scratch Postgres databases.""" + +from __future__ import annotations + +import asyncio +import contextlib +import os +from collections.abc import AsyncIterator +from pathlib import Path +from typing import Any + +import pytest + +from server.core.paths import ROUTER_MIGRATIONS_DIR +from server.core.schema import pg_migrate, pg_online_index +from server.router import directory +from server.settings import settings +from tests.integration._pg_database import create_database, drop_database + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +async def database(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[tuple[Any, Any]]: + import asyncpg + + admin = os.environ.get("LOOM_PG_DSN") or pytest.skip("LOOM_PG_DSN not set") + name, dsn = await create_database(admin) + monkeypatch.setattr(settings, "router_pg_dsn", dsn) + monkeypatch.setattr(directory, "_pool", None) + + @contextlib.asynccontextmanager + async def connect() -> AsyncIterator[Any]: + conn = await asyncpg.connect(dsn, server_settings={"application_name": "online-index-test"}) + try: + yield conn + finally: + await conn.close() + + try: + async with connect() as conn: + yield conn, connect + finally: + await directory.close() + await drop_database(admin, name) + + +def _prefix(tmp_path: Path) -> Path: + prefix = tmp_path / "old" + prefix.mkdir() + for source in sorted(ROUTER_MIGRATIONS_DIR.glob("*.sql"))[:3]: + (prefix / source.name).write_bytes(source.read_bytes()) + return prefix + + +async def _migrate(connect: Any, files: Path = ROUTER_MIGRATIONS_DIR) -> list[str]: + return await pg_migrate.migrate(files, connect=connect, schema="loom_router", lock_key=8917423) + + +async def test_online_migration_installs_and_retries_from_both_schemas(database, tmp_path) -> None: + conn, connect = database + await _migrate(connect, _prefix(tmp_path)) + assert await _migrate(connect) == ["004_placements_by_cell_org.sql"] + assert await _migrate(connect) == [] + index = pg_online_index.parse( + (ROUTER_MIGRATIONS_DIR / "004_placements_by_cell_org.sql").read_text() + ) + assert index is not None and await pg_online_index.ready(conn, index) is True + assert await conn.fetchval("SELECT to_regclass('loom_router.placements_by_cell') IS NOT NULL") + + +async def test_install_from_empty(database) -> None: + _, connect = database + assert len(await _migrate(connect)) == 4 + + +async def _until(conn: Any, sql: str, *args: Any) -> None: + async with asyncio.timeout(8): + for _ in range(800): + if await conn.fetchval(sql, *args): + return + await asyncio.sleep(0.01) + pytest.fail("Postgres did not reach the expected migration state") + + +def _old_runner(): + """The shipped blocking runner, kept as source for the rolling-upgrade proof.""" + import importlib.util + + path = Path(__file__).parent / "fixtures" / "pg_migrate_n_minus_one.py" + spec = importlib.util.spec_from_file_location("old_pg_migrate", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +async def test_old_snapshot_old_blocking_migrator_and_two_new_migrators(database, tmp_path) -> None: + conn, connect = database + old_files = _prefix(tmp_path) + await _migrate(connect, old_files) + async with connect() as snapshot: + tx = snapshot.transaction(isolation="repeatable_read") + await tx.start() + await snapshot.fetchval("SELECT count(*) FROM loom_router.placements") + first = asyncio.create_task(_migrate(connect)) + second = None + try: + await _until( + conn, + "SELECT EXISTS(SELECT 1 FROM pg_stat_progress_create_index " + "WHERE datid = (SELECT oid FROM pg_database WHERE datname = current_database()) " + "AND phase = 'waiting for old snapshots')", + ) + second = asyncio.create_task(_migrate(connect)) + # The ordinary key is free while CREATE INDEX waits for this old snapshot. + # Running this original blocking implementation would deadlock if it weren't. + old = _old_runner() + async with asyncio.timeout(2): + assert ( + await old.migrate( + old_files, connect=connect, schema="loom_router", lock_key=8917423 + ) + == [] + ) + assert not first.done() and not second.done() + # Even while the index waits, the old serving path can write and read. + await conn.execute( + "INSERT INTO loom_router.cells " + "(cell, host, state, credential_sha256, deployment_id) " + "VALUES ('a', 'a.example.test', 'serving', repeat('a', 64), 'dep-a')" + ) + assert await directory.place("org-a", "a", lambda _: "") == ("a", "dep-a") + assert await directory.lookup("org-a") == ("a", "a.example.test") + await tx.rollback() + async with asyncio.timeout(5): + results = await asyncio.gather(first, second) + assert sorted(results) == [[], ["004_placements_by_cell_org.sql"]] + finally: + if snapshot.is_in_transaction(): + await tx.rollback() + for task in (first, second): + if task is not None and not task.done(): + task.cancel() + await asyncio.gather( + *(t for t in (first, second) if t is not None), return_exceptions=True + ) + + +async def test_cancelled_build_leaves_a_matching_invalid_index_and_retry_repairs_it( + database, tmp_path +) -> None: + conn, connect = database + await _migrate(connect, _prefix(tmp_path)) + async with connect() as snapshot: + tx = snapshot.transaction(isolation="repeatable_read") + await tx.start() + await snapshot.fetchval("SELECT count(*) FROM loom_router.placements") + migrating = asyncio.create_task(_migrate(connect)) + try: + await _until( + conn, + "SELECT EXISTS(SELECT 1 FROM pg_stat_progress_create_index " + "WHERE datid = (SELECT oid FROM pg_database WHERE datname = current_database()) " + "AND phase = 'waiting for old snapshots')", + ) + migrating.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(migrating, 2) + assert await conn.fetchval( + "SELECT NOT indisvalid FROM pg_index " + "WHERE indexrelid = 'loom_router.placements_by_cell_org'::regclass" + ) + assert not await conn.fetchval( + "SELECT EXISTS(SELECT 1 FROM loom_router.pg_migrations " + "WHERE filename = '004_placements_by_cell_org.sql')" + ) + finally: + await tx.rollback() + if not migrating.done(): + migrating.cancel() + await asyncio.gather(migrating, return_exceptions=True) + assert await _migrate(connect) == ["004_placements_by_cell_org.sql"] + assert await _migrate(connect) == [] + + +@pytest.mark.parametrize( + "definition", + [ + "CREATE TABLE loom_router.placements_by_cell_org (id int)", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (org, cell)", + 'CREATE INDEX placements_by_cell_org ON loom_router.placements (cell, org COLLATE "POSIX")', + "CREATE UNIQUE INDEX placements_by_cell_org ON loom_router.placements (cell, org)", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (cell, org) WHERE org > 'm'", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (cell, org) INCLUDE(placed_at)", + "CREATE INDEX placements_by_cell_org ON loom_router.cells (cell, host)", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (cell DESC, org)", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (cell text_pattern_ops, org)", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (cell, org) WITH (fillfactor=70)", + ], +) +async def test_foreign_same_named_object_is_never_dropped(database, tmp_path, definition) -> None: + conn, connect = database + await _migrate(connect, _prefix(tmp_path)) + await conn.execute(definition) + oid = await conn.fetchval("SELECT 'loom_router.placements_by_cell_org'::regclass::oid") + with pytest.raises(ValueError, match="different definition"): + await _migrate(connect) + assert await conn.fetchval("SELECT 'loom_router.placements_by_cell_org'::regclass::oid") == oid + + +async def test_recorded_invalid_index_and_checksum_drift_refuse(database, tmp_path) -> None: + conn, connect = database + await _migrate(connect) + await conn.execute("DROP INDEX loom_router.placements_by_cell_org") + with pytest.raises(ValueError, match="recorded online-index"): + await _migrate(connect) + await conn.execute( + "UPDATE loom_router.pg_migrations SET checksum = repeat('0', 64) " + "WHERE filename = '004_placements_by_cell_org.sql'" + ) + with pytest.raises(pg_migrate.MigrationDrift): + await _migrate(connect) + + +async def test_cancel_after_catalog_completion_reuses_the_valid_index( + database, tmp_path, monkeypatch +) -> None: + conn, connect = database + await _migrate(connect, _prefix(tmp_path)) + original = pg_online_index.build + + async def completed_then_cancel(*args, **kwargs): + await original(*args, **kwargs) + raise asyncio.CancelledError + + monkeypatch.setattr(pg_online_index, "build", completed_then_cancel) + with pytest.raises(asyncio.CancelledError): + await _migrate(connect) + oid = await conn.fetchval("SELECT 'loom_router.placements_by_cell_org'::regclass::oid") + monkeypatch.setattr(pg_online_index, "build", original) + assert await _migrate(connect) == ["004_placements_by_cell_org.sql"] + assert await conn.fetchval("SELECT 'loom_router.placements_by_cell_org'::regclass::oid") == oid + + +async def test_keyset_plan_uses_the_ordered_cell_index(database) -> None: + conn, connect = database + await _migrate(connect) + await conn.execute( + "INSERT INTO loom_router.cells (cell, host, state, credential_sha256, deployment_id) " + "SELECT 'cell-' || n, 'cell-' || n || '.example.test', 'serving', " + "md5(n::text) || md5(n::text), 'dep-' || n FROM generate_series(1, 32) n" + ) + await conn.execute( + "INSERT INTO loom_router.placements(org, cell) " + "SELECT 'org-' || lpad(n::text, 5, '0'), 'cell-' || (n % 32 + 1) " + "FROM generate_series(1, 4096) n" + ) + await conn.execute("VACUUM ANALYZE loom_router.placements") + plan = await conn.fetch( + "EXPLAIN SELECT org FROM loom_router.placements " + "WHERE cell = 'cell-2' AND org > 'org-01000' ORDER BY org LIMIT 10" + ) + text = "\n".join(r[0] for r in plan) + assert "placements_by_cell_org" in text and "Sort" not in text + + +async def test_ordinary_files_still_rollback_with_the_ledger(database, tmp_path) -> None: + conn, connect = database + (tmp_path / "001_one.sql").write_text("CREATE TABLE loom_router.one (id int);") + (tmp_path / "002_two.sql").write_text("CREATE TABLE loom_router.two (id int); SELECT 1 / 0;") + import asyncpg + + with pytest.raises(asyncpg.DivisionByZeroError): + await _migrate(connect, tmp_path) + assert await conn.fetchval("SELECT to_regclass('loom_router.one') IS NOT NULL") + assert not await conn.fetchval("SELECT to_regclass('loom_router.two') IS NOT NULL") + assert await conn.fetchval("SELECT count(*) FROM loom_router.pg_migrations") == 1 + (tmp_path / "002_two.sql").write_text("CREATE TABLE loom_router.two (id int);") + assert await _migrate(connect, tmp_path) == ["002_two.sql"] + + +async def test_invalid_online_declaration_fails_before_the_ledger_exists( + database, tmp_path +) -> None: + conn, connect = database + (tmp_path / "001_bad.sql").write_text("-- loom:online-index bad\nSELECT 1;") + with pytest.raises(ValueError, match="online-index"): + await _migrate(connect, tmp_path) + assert not await conn.fetchval("SELECT to_regclass('loom_router.pg_migrations') IS NOT NULL") + + +async def test_cancelled_concurrent_drop_is_retryable(database, tmp_path) -> None: + conn, connect = database + await _migrate(connect, _prefix(tmp_path)) + index = pg_online_index.parse( + (ROUTER_MIGRATIONS_DIR / "004_placements_by_cell_org.sql").read_text() + ) + assert index is not None + await conn.execute(index.sql) + async with connect() as reader: + tx = reader.transaction() + await tx.start() + await reader.execute("SET LOCAL enable_seqscan = off") + plan = await reader.fetch( + "EXPLAIN SELECT org FROM loom_router.placements WHERE cell = 'a' ORDER BY org" + ) + assert "placements_by_cell_org" in "\n".join(r[0] for r in plan) + await reader.fetch("SELECT org FROM loom_router.placements WHERE cell = 'a' ORDER BY org") + + async def direct_drop(): + async with connect() as dropping: + try: + await dropping.execute( + "DROP INDEX CONCURRENTLY loom_router.placements_by_cell_org" + ) + finally: + dropping.terminate() + + dropping = asyncio.create_task(direct_drop()) + try: + query = ( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity " + "WHERE datname = current_database() AND query LIKE 'DROP INDEX CONCURRENTLY%' " + "AND wait_event IS NOT NULL)" + ) + await _until(conn, query) + dropping.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(dropping, 2) + assert await pg_online_index.ready(conn, index) is False + dropping = asyncio.create_task(_migrate(connect)) + await _until(conn, query) + dropping.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(dropping, 2) + finally: + await tx.rollback() + if not dropping.done(): + dropping.cancel() + await asyncio.gather(dropping, return_exceptions=True) + assert await _migrate(connect) == ["004_placements_by_cell_org.sql"] + + +async def test_try_lock_wait_has_no_retained_transaction_or_snapshot(database) -> None: + conn, connect = database + await conn.execute("SELECT pg_advisory_lock(8917423::bigint)") + migrating = asyncio.create_task(_migrate(connect)) + try: + await _until( + conn, + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity " + "WHERE datname = current_database() AND pid <> pg_backend_pid() " + "AND query = 'SELECT pg_try_advisory_lock($1)' " + "AND state = 'idle' AND backend_xmin IS NULL AND xact_start IS NULL)", + ) + assert not migrating.done() + await conn.execute("SELECT pg_advisory_unlock(8917423::bigint)") + assert len(await asyncio.wait_for(migrating, 5)) == 4 + finally: + await conn.execute("SELECT pg_advisory_unlock(8917423::bigint)") + if not migrating.done(): + migrating.cancel() + await asyncio.gather(migrating, return_exceptions=True) + + +async def test_online_validation_is_repeated_after_later_files(database, tmp_path) -> None: + _, connect = database + for source in ROUTER_MIGRATIONS_DIR.glob("*.sql"): + (tmp_path / source.name).write_bytes(source.read_bytes()) + (tmp_path / "005_damage.sql").write_text("DROP INDEX loom_router.placements_by_cell_org;") + with pytest.raises(ValueError, match="recorded online-index"): + await _migrate(connect, tmp_path) diff --git a/tests/integration/test_pg_online_index_durability.py b/tests/integration/test_pg_online_index_durability.py new file mode 100644 index 0000000000..e5644af5af --- /dev/null +++ b/tests/integration/test_pg_online_index_durability.py @@ -0,0 +1,203 @@ +"""A cancelled online-index ledger commit survives a private standby promotion. + +Only its dedicated CI pair enables this destructive drill. Configured failures +are errors; the adopted integration server's replication settings never change. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import os +from collections.abc import AsyncIterator +from pathlib import Path +from urllib.parse import urlsplit + +import asyncpg +import pytest + +from server.core.paths import ROUTER_MIGRATIONS_DIR +from server.core.schema import pg_migrate +from tests.integration._pg_database import create_database + +PRIMARY = "loom-scale-index-drill-pg" +STANDBY = PRIMARY + "-standby" +APPLICATION = "online-index-durability" + + +async def _docker(*args: str) -> str: + process = await asyncio.create_subprocess_exec( + "docker", *args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + try: + async with asyncio.timeout(30): + stdout, stderr = await process.communicate() + except BaseException: + if process.returncode is None: + process.kill() + await asyncio.wait_for(process.wait(), 5) + raise + assert process.returncode == 0, stderr.decode() + return stdout.decode().strip() + + +async def _sql(container: str, database: str, sql: str) -> str: + return await _docker("exec", container, "psql", "-U", "loom", "-d", database, "-Atc", sql) + + +async def _waiting(observer, task) -> int: + async with asyncio.timeout(8): + for _ in range(800): + assert not task.done(), "migration acknowledged before standby flush" + pid = await observer.fetchval( + "SELECT pid FROM pg_stat_activity WHERE application_name = $1 " + "AND wait_event = 'SyncRep'", + APPLICATION, + ) + if pid is not None: + assert not task.done() + return pid + await asyncio.sleep(0.01) + pytest.fail("migration did not reach its synchronous COMMIT") + + +async def _ownership(observer) -> None: + identifier = str(await observer.fetchval("SELECT system_identifier FROM pg_control_system()")) + for container in (PRIMARY, STANDBY): + assert await _docker("inspect", "--format", "{{.Name}}", container) == "/" + container + assert ( + await _sql(container, "loom", "SELECT system_identifier FROM pg_control_system()") + == identifier + ) + assert await _sql(STANDBY, "loom", "SELECT pg_is_in_recovery()") == "t" + assert ( + await observer.fetchval( + "SELECT count(*) FROM pg_stat_replication WHERE application_name = 'loom_standby' " + "AND sync_state = 'sync' AND state = 'streaming'" + ) + == 1 + ) + + +@pytest.mark.asyncio +@pytest.mark.recovery +@pytest.mark.destructive +async def test_online_ledger_retry_survives_physical_standby_promotion( + tmp_path, monkeypatch +) -> None: + if not os.environ.get("LOOM_PG_INDEX_DRILL"): + pytest.skip("LOOM_PG_INDEX_DRILL not set: needs its disposable physical standby") + assert os.environ["LOOM_PG_INDEX_DRILL"] == "1" + dsn = os.environ["LOOM_PG_INDEX_DRILL_DSN"] + observer = await asyncpg.connect(dsn, timeout=5, command_timeout=5) + tasks = [] + paused = False + try: + await _ownership(observer) + _, target = await create_database(dsn) + + @contextlib.asynccontextmanager + async def connect() -> AsyncIterator[asyncpg.Connection]: + conn = await asyncpg.connect( + target, + timeout=5, + server_settings={"application_name": APPLICATION, "synchronous_commit": "on"}, + ) + try: + yield conn + finally: + conn.terminate() + + prefix = tmp_path / "old" + prefix.mkdir() + for source in sorted(ROUTER_MIGRATIONS_DIR.glob("*.sql"))[:3]: + (prefix / source.name).write_bytes(source.read_bytes()) + + async def migrate(files: Path = ROUTER_MIGRATIONS_DIR): + return await pg_migrate.migrate( + files, connect=connect, schema="loom_router", lock_key=8917423 + ) + + await migrate(prefix) + record = pg_migrate._record_online + + async def pause_before_ledger(*args, **kwargs): + nonlocal paused + await _docker("pause", STANDBY) + paused = True + await record(*args, **kwargs) + + monkeypatch.setattr(pg_migrate, "_record_online", pause_before_ledger) + first = asyncio.create_task(migrate()) + tasks.append(first) + pid = await _waiting(observer, first) + # The index is acknowledged already; its ledger COMMIT alone is unflushed. + async with connect() as conn: + assert ( + await conn.fetchval( + "SELECT count(*) FROM loom_router.pg_migrations " + "WHERE filename = '004_placements_by_cell_org.sql'" + ) + == 1 + ) + first.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(first, 3) + async with asyncio.timeout(5): + for _ in range(500): + if not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid = $1)", pid + ): + break + await asyncio.sleep(0.01) + else: + pytest.fail("cancelled migration retained backend locks") + monkeypatch.setattr(pg_migrate, "_record_online", record) + retry = asyncio.create_task(migrate()) + tasks.append(retry) + await _waiting(observer, retry) + await _docker("unpause", STANDBY) + paused = False + assert await asyncio.wait_for(retry, 8) == [] + observer.terminate() + await _docker("kill", "--signal", "KILL", PRIMARY) + await _docker( + "exec", + "--user", + "postgres", + STANDBY, + "bash", + "-c", + 'pg_ctl -D "$PGDATA" promote -w -t 10', + ) + assert await _sql(STANDBY, "loom", "SELECT pg_is_in_recovery()") == "f" + database = urlsplit(target).path.removeprefix("/") + assert ( + await _sql( + STANDBY, + database, + "SELECT count(*) FROM loom_router.pg_migrations " + "WHERE filename = '004_placements_by_cell_org.sql' AND checksum = '" + + pg_migrate.checksum(ROUTER_MIGRATIONS_DIR / "004_placements_by_cell_org.sql") + + "'", + ) + == "1" + ) + assert ( + await _sql( + STANDBY, + database, + "SELECT indisvalid AND indisready FROM pg_index " + "WHERE indexrelid = 'loom_router.placements_by_cell_org'::regclass", + ) + == "t" + ) + finally: + if paused: + await _docker("unpause", STANDBY) + for task in tasks: + if not task.done(): + task.cancel() + async with asyncio.timeout(5): + await asyncio.gather(*tasks, return_exceptions=True) + observer.terminate() diff --git a/tests/unit/test_cell_org_triggers.py b/tests/unit/test_cell_org_triggers.py index 769cfe1bb3..5d25ba0177 100644 --- a/tests/unit/test_cell_org_triggers.py +++ b/tests/unit/test_cell_org_triggers.py @@ -86,6 +86,14 @@ async def execute(self, sql: str, *_a: Any) -> None: async def fetch(self, _sql: str, *_a: Any) -> list[Any]: return [] + def terminate(self) -> None: + self.statements.append("terminated") + + async def fetchval(self, sql: str, *_a: Any) -> bool: + assert sql in ("SELECT pg_try_advisory_lock($1)", "SELECT pg_advisory_unlock($1)") + self.statements.append(sql) + return True + @pytest.mark.parametrize("name", ["eu-1", ""]) @pytest.mark.parametrize("schema", ["loom", "loom_router"]) diff --git a/tests/unit/test_pg_online_index.py b/tests/unit/test_pg_online_index.py new file mode 100644 index 0000000000..7fc43fa2bb --- /dev/null +++ b/tests/unit/test_pg_online_index.py @@ -0,0 +1,212 @@ +"""Only the declared concurrent btree form escapes a migration transaction.""" + +from __future__ import annotations + +import pytest + +from server.core.schema import pg_online_index + +SQL = """-- loom:online-index loom_router.placements_by_cell_org ON loom_router.placements (cell, org) +CREATE INDEX CONCURRENTLY placements_by_cell_org ON loom_router.placements +USING btree (cell COLLATE "C", org COLLATE "C"); +""" + + +def test_the_online_index_declaration_has_one_exact_target() -> None: + parsed = pg_online_index.parse(SQL) + assert parsed is not None + assert parsed.schema == "loom_router" + assert parsed.name == "placements_by_cell_org" + assert parsed.table == "placements" + assert parsed.columns == ("cell", "org") + assert pg_online_index.parse("CREATE TABLE loom.t (id int);") is None + + +@pytest.mark.parametrize( + "sql", + [ + SQL + "SELECT 1;", + SQL.replace("CREATE INDEX", "CREATE UNIQUE INDEX"), + SQL.replace('org COLLATE "C"', 'org COLLATE "POSIX"'), + SQL.replace("(cell, org)", "(org, cell)"), + SQL.replace("ON loom_router.placements\n", "ON loom_router.other\n"), + SQL.replace("USING btree", "USING hash"), + SQL.replace('org COLLATE "C"', 'lower(org) COLLATE "C"'), + SQL.replace(");", ") WHERE true;"), + SQL.replace("-- loom:online-index", "-- loom:online-index-bad"), + SQL.replace("placements_by_cell_org ON", "other_index ON", 1), + SQL.replace("(cell, org)", "(cell, cell)"), + SQL.replace("CONCURRENTLY", ""), + ], +) +def test_other_online_sql_is_refused(sql: str) -> None: + with pytest.raises(ValueError, match="online.index"): + pg_online_index.parse(sql) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("pending", [True, False]) +async def test_ledger_retries_commit_a_wal_fence_even_when_already_recorded( + tmp_path, pending +) -> None: + import contextlib + + from server.core import pg + from server.core.schema import pg_migrate + + path = tmp_path / "001_index.sql" + path.write_text(SQL) + + class Connection: + def __init__(self): + self.statements = [] + + def transaction(self): + return contextlib.nullcontext() + + async def execute(self, sql, *args): + self.statements.append((sql, args)) + + conn = Connection() + await pg_migrate._record_online(conn, path, "loom_router", pending=pending) + assert conn.statements[-1] == (pg._DURABLE_MESSAGE, ()) + assert len(conn.statements) == (2 if pending else 1) + if pending: + assert conn.statements[0][1] == (path.name, pg_migrate.checksum(path)) + + +@pytest.mark.asyncio +async def test_migrator_preserves_a_borrowed_sessions_statement_budget(tmp_path) -> None: + import contextlib + + from server.core.schema import pg_migrate + + class Connection: + def __init__(self): + self.statements = [] + + async def execute(self, sql, *args): + self.statements.append(sql) + + async def fetch(self, sql): + return [] + + async def fetchval(self, sql, *args): + self.statements.append(sql) + return True + + conn = Connection() + + @contextlib.asynccontextmanager + async def connect(): + yield conn + + assert await pg_migrate.migrate(tmp_path, connect=connect, schema="loom_router") == [] + assert not any("statement_timeout" in sql for sql in conn.statements) + assert "SET synchronous_commit = 'on'" in conn.statements + assert conn.statements[-1] == "SELECT pg_advisory_unlock($1)" + + +@pytest.mark.asyncio +async def test_online_target_must_match_its_ledger_before_connecting(tmp_path) -> None: + from server.core.schema import pg_migrate + + (tmp_path / "001_index.sql").write_text(SQL) + + def never_connect(): + pytest.fail("invalid target opened a connection") + + with pytest.raises(ValueError, match="ledger schema"): + await pg_migrate.migrate(tmp_path, schema="loom", connect=never_connect) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("phase", ["statement", "rollback", "commit"]) +async def test_cancellation_terminates_before_installed_asyncpg_transaction_unwind( + tmp_path, phase +) -> None: + import asyncio + import contextlib + from types import SimpleNamespace + + from asyncpg.transaction import Transaction + + from server.core.schema import pg_migrate + + (tmp_path / "001_one.sql").write_text("SELECT stalled_statement;") + reached = asyncio.Event() + terminated = asyncio.Event() + + class Connection: + _pool_release_ctr = 0 + _top_xact = None + _protocol = SimpleNamespace(is_in_transaction=lambda: False) + + def is_closed(self): + return terminated.is_set() + + def transaction(self): + return Transaction(self, None, False, False) + + def terminate(self): + terminated.set() + + async def fetchval(self, sql, *args): + return True + + async def fetch(self, sql): + return [] + + async def execute(self, sql, *args): + if sql == "SELECT stalled_statement;" and phase == "rollback": + raise ValueError("statement failed before rollback") + if ( + (sql == "SELECT stalled_statement;" and phase == "statement") + or (sql.startswith("ROLLBACK") and phase == "rollback") + or (sql.startswith("COMMIT") and phase == "commit") + ): + reached.set() + # Transport termination, rather than a second task cancellation, + # releases a transaction that has already begun async cleanup. + while not terminated.is_set(): + try: + await terminated.wait() + except asyncio.CancelledError: + continue + + conn = Connection() + + @contextlib.asynccontextmanager + async def connect(): + yield conn + + migrating = asyncio.create_task( + pg_migrate.migrate(tmp_path, connect=connect, schema="loom_router") + ) + try: + await asyncio.wait_for(reached.wait(), 1) + migrating.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(migrating, 1) + assert terminated.is_set() + finally: + terminated.set() + if not migrating.done(): + migrating.cancel() + await asyncio.gather(migrating, return_exceptions=True) + + +def test_postgres_journals_keep_distinct_numbers_and_declared_online_forms() -> None: + from server.core.paths import PG_MIGRATIONS_DIR, ROUTER_MIGRATIONS_DIR + from server.core.schema import pg_migrate + from server.core.schema.journal import migration_number + + for directory in (PG_MIGRATIONS_DIR, ROUTER_MIGRATIONS_DIR): + files = pg_migrate.migration_files(directory) + numbers = [migration_number(path.name) for path in files] + assert None not in numbers and len(numbers) == len(set(numbers)) + for path in files: + sql = path.read_text() + parsed = pg_online_index.parse(sql) + if "CONCURRENTLY" in sql: + assert parsed is not None, f"{path.name} lacks an online-index declaration" From 417712d4ecde8b788ca060c853ec02e528e42487 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 22:40:22 +0000 Subject: [PATCH 025/114] Close owned Postgres sessions before cancelling their work Send public driver cancellation before aborting the transport, within one absolute cleanup deadline. Separate borrowed migration execution so its operator keeps sole session supervision. Verify backend lock release and preserve interrupted migration errors. Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 1 + server/core/pg.py | 32 ++++ server/core/schema/pg_migrate.py | 52 +++++-- tests/integration/test_pg_online_index.py | 10 +- tests/integration/test_pg_owned_cleanup.py | 173 +++++++++++++++++++++ tests/unit/test_pg_close_owned.py | 144 +++++++++++++++++ tests/unit/test_pg_online_index.py | 21 ++- 7 files changed, 413 insertions(+), 20 deletions(-) create mode 100644 tests/integration/test_pg_owned_cleanup.py create mode 100644 tests/unit/test_pg_close_owned.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a132706be6..6e866c99a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -897,6 +897,7 @@ jobs: tests/integration/test_cell_tenancy_pg.py tests/integration/test_router_directory_pg.py tests/integration/test_pg_online_index.py + tests/integration/test_pg_owned_cleanup.py tests/integration/test_cell_unset_gate_pg.py tests/integration/test_cell_prepare.py tests/integration/test_cell_attach.py diff --git a/server/core/pg.py b/server/core/pg.py index 576b1b1e4b..5daeb33a55 100644 --- a/server/core/pg.py +++ b/server/core/pg.py @@ -21,6 +21,7 @@ import asyncio import json import logging +import time import weakref from collections.abc import AsyncIterator, Mapping from contextlib import asynccontextmanager, suppress @@ -379,6 +380,37 @@ def _terminate(conn: Any) -> None: conn.terminate() +def _retrieve_task(task: asyncio.Task[Any]) -> None: + if not task.cancelled(): + task.exception() + + +async def close_owned(conn: Any, *, end: float) -> None: + """Close an owned session by an absolute time.monotonic deadline. + + Public close sends cancellation before transport cleanup can discard it. Abort + is a bounded fallback, not evidence that the backend has released its locks. + The owner preserves an earlier operation failure when this cleanup fails. + """ + task: asyncio.Task[Any] | None = None + try: + remaining = end - time.monotonic() + if remaining <= 0: + raise TimeoutError("Postgres owned-session cleanup deadline expired") + task = asyncio.create_task(conn.close(timeout=remaining)) + done, _ = await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) + if not done: + raise TimeoutError("Postgres owned-session cleanup deadline expired") + task.result() + except BaseException: + _terminate(conn) + if task is not None: + if not task.done(): + task.cancel() + task.add_done_callback(_retrieve_task) + raise + + async def _release(pool: Any, conn: Any) -> None: """Back to the pool within a bound: a statement still in flight (a client timeout, a cancelled request) has its cancel sent and acknowledged and the connection reset; diff --git a/server/core/schema/pg_migrate.py b/server/core/schema/pg_migrate.py index ebd230036b..371f61a449 100644 --- a/server/core/schema/pg_migrate.py +++ b/server/core/schema/pg_migrate.py @@ -19,6 +19,7 @@ import asyncio import hashlib import logging +import time from asyncio import sleep from collections.abc import Callable from contextlib import AbstractAsyncContextManager @@ -108,25 +109,54 @@ async def migrate( bounded (`lock_timeout`), so an `ALTER` queued behind a long transaction never holds every read and write of the table behind it; the file is retried. For schema `loom` it marks the session and runs `cell_triggers.ensure` before the lock goes.""" - files = migration_files(mig_dir) - # Validate every online declaration before even creating the ledger. - indexes = {f.name: pg_online_index.parse(f.read_text()) for f in files} - if any(index is not None and index.schema != schema for index in indexes.values()): - raise ValueError("online-index target differs from its ledger schema") + files, indexes = _validated(mig_dir, schema) async with (connect or pg.dedicated)() as conn: task = asyncio.create_task(_migrate(conn, files, indexes, schema, lock_key)) try: return await asyncio.shield(task) except BaseException: - # Terminate before cancellation can enter asyncpg's unbounded rollback. - conn.terminate() - task.cancel() - await asyncio.wait({task}, timeout=1) - if task.done() and not task.cancelled(): - task.exception() + end = time.monotonic() + pg._RELEASE_TIMEOUT_S + try: + await pg.close_owned(conn, end=end) + except BaseException: # noqa: BLE001 — preserve the migration failure/cancellation. + pass + if not task.done(): + task.cancel() + task.add_done_callback(pg._retrieve_task) + try: + await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) + except BaseException: # noqa: BLE001 — preserve the original failure on repeated cancel. + pg._terminate(conn) raise +def _validated( + mig_dir: Path, schema: str +) -> tuple[list[Path], dict[str, pg_online_index.OnlineIndex | None]]: + files = migration_files(mig_dir) + # Validate every online declaration before even creating the ledger. + indexes = {f.name: pg_online_index.parse(f.read_text()) for f in files} + if any(index is not None and index.schema != schema for index in indexes.values()): + raise ValueError("online-index target differs from its ledger schema") + return files, indexes + + +async def migrate_borrowed( + conn: Any, + mig_dir: Path = PG_MIGRATIONS_DIR, + *, + schema: str = "loom", + lock_key: int = _LOCK_KEY, +) -> list[str]: + """Apply on the owner's session; its supervisor owns all failure cleanup. + + The caller retains its operation deadline and must close the session before + cancelling this work. Normal completion releases only the migration key. + """ + files, indexes = _validated(mig_dir, schema) + return await _migrate(conn, files, indexes, schema, lock_key) + + async def _lock(conn: Any, key: int) -> None: while True: set_query_tag("migrate.postgres.lock") diff --git a/tests/integration/test_pg_online_index.py b/tests/integration/test_pg_online_index.py index af93dd0937..1995e73bee 100644 --- a/tests/integration/test_pg_online_index.py +++ b/tests/integration/test_pg_online_index.py @@ -5,12 +5,14 @@ import asyncio import contextlib import os +import time from collections.abc import AsyncIterator from pathlib import Path from typing import Any import pytest +from server.core import pg from server.core.paths import ROUTER_MIGRATIONS_DIR from server.core.schema import pg_migrate, pg_online_index from server.router import directory @@ -315,7 +317,7 @@ async def direct_drop(): "DROP INDEX CONCURRENTLY loom_router.placements_by_cell_org" ) finally: - dropping.terminate() + await pg.close_owned(dropping, end=time.monotonic() + 1) dropping = asyncio.create_task(direct_drop()) try: @@ -325,6 +327,12 @@ async def direct_drop(): "AND wait_event IS NOT NULL)" ) await _until(conn, query) + # Cancel after catalog invalidation, not an earlier lock/I/O wait. + await _until( + conn, + "SELECT NOT indisvalid FROM pg_index " + "WHERE indexrelid='loom_router.placements_by_cell_org'::regclass", + ) dropping.cancel() with pytest.raises(asyncio.CancelledError): await asyncio.wait_for(dropping, 2) diff --git a/tests/integration/test_pg_owned_cleanup.py b/tests/integration/test_pg_owned_cleanup.py new file mode 100644 index 0000000000..9085fce011 --- /dev/null +++ b/tests/integration/test_pg_owned_cleanup.py @@ -0,0 +1,173 @@ +"""Cancellation of owned Postgres work releases its actual backend locks.""" + +from __future__ import annotations + +import asyncio +import contextlib +import os +import time +from collections.abc import AsyncIterator +from pathlib import Path +from typing import Any + +import pytest + +from server.core import pg +from server.core.schema import pg_migrate +from tests.integration._pg_database import create_database, drop_database + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +async def database() -> AsyncIterator[tuple[Any, Any]]: + import asyncpg + + admin = os.environ.get("LOOM_PG_DSN") or pytest.skip("LOOM_PG_DSN not set") + name, dsn = await create_database(admin) + conn = await asyncpg.connect(dsn, server_settings={"statement_timeout": "10000"}) + observer = await asyncpg.connect(dsn) + try: + await conn.execute("CREATE TABLE probe(id int PRIMARY KEY); INSERT INTO probe VALUES(1)") + yield conn, observer + finally: + await conn.close(timeout=1) + await observer.close(timeout=1) + await drop_database(admin, name) + + +async def sleeping(observer: Any, pid: int) -> None: + async with asyncio.timeout(2): + for _ in range(400): + if await observer.fetchval( + "SELECT wait_event='PgSleep' FROM pg_stat_activity WHERE pid=$1", pid + ): + break + await asyncio.sleep(0.005) + else: + pytest.fail("backend did not reach pg_sleep") + assert await observer.fetchval("SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1)", pid) + + +async def released(observer: Any, pid: int) -> None: + assert not await observer.fetchval("SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1)", pid) + assert not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)", pid + ) + + +async def test_owned_migrator_cancellation_releases_server_locks(database, tmp_path: Path) -> None: + conn, observer = database + (tmp_path / "001_sleep.sql").write_text("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10);") + + @contextlib.asynccontextmanager + async def connect() -> AsyncIterator[Any]: + yield conn + + task = asyncio.create_task(pg_migrate.migrate(tmp_path, connect=connect, schema="test")) + await sleeping(observer, conn.get_server_pid()) + pid = conn.get_server_pid() + started = time.monotonic() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, 1.5) + assert time.monotonic() - started < 1.5 + await released(observer, pid) + + +@pytest.mark.parametrize("transaction_context", [False, True]) +async def test_public_owned_close_releases_live_query_before_task_cancel( + database, transaction_context: bool +) -> None: + conn, observer = database + pid = conn.get_server_pid() + + async def work() -> None: + if transaction_context: + async with conn.transaction(): + await conn.execute("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10)") + else: + await conn.execute("BEGIN; SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10)") + + task = asyncio.create_task(work()) + try: + await sleeping(observer, pid) + await pg.close_owned(conn, end=time.monotonic() + 0.5) + await released(observer, pid) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_borrowed_migrator_leaves_failure_cleanup_to_owner(database, tmp_path: Path) -> None: + conn, observer = database + (tmp_path / "001_sleep.sql").write_text("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10);") + task = asyncio.create_task(pg_migrate.migrate_borrowed(conn, tmp_path, schema="test")) + pid = conn.get_server_pid() + try: + await sleeping(observer, pid) + await pg.close_owned(conn, end=time.monotonic() + 0.5) + await released(observer, pid) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_borrowed_migrator_normal_completion_retains_owners_lock(database, tmp_path) -> None: + conn, observer = database + await conn.execute("SELECT pg_advisory_lock(712398)") + (tmp_path / "001_write.sql").write_text("INSERT INTO probe VALUES(2)") + assert await pg_migrate.migrate_borrowed(conn, tmp_path, schema="test") == ["001_write.sql"] + assert not conn.is_closed() + assert await observer.fetchval("SELECT count(*) FROM probe") == 2 + assert await conn.fetchval("SELECT pg_advisory_unlock(712398)") + assert ( + await conn.fetchval( + "SELECT count(*) FROM pg_locks WHERE pid=pg_backend_pid() AND locktype='advisory'" + ) + == 0 + ) + + +async def test_owned_close_interrupts_installed_transaction_rollback(database, monkeypatch) -> None: + import asyncpg + + conn, observer = database + pid = conn.get_server_pid() + execute = asyncpg.Connection.execute + + async def stalled_rollback(self, query, *args, **kwargs): + if self is conn and query.startswith("ROLLBACK"): + await execute(self, "SELECT pg_sleep(10)") + return await execute(self, query, *args, **kwargs) + + monkeypatch.setattr(asyncpg.Connection, "execute", stalled_rollback) + + async def work() -> None: + async with conn.transaction(): + await conn.execute("SELECT * FROM probe FOR UPDATE") + raise ValueError("statement failure before rollback") + + task = asyncio.create_task(work()) + try: + await sleeping(observer, pid) + await pg.close_owned(conn, end=time.monotonic() + 0.5) + await released(observer, pid) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_borrowed_failure_keeps_the_session_for_its_owner(database, tmp_path) -> None: + import asyncpg + + conn, _ = database + (tmp_path / "001_failure.sql").write_text("SELECT * FROM missing_table_for_failure") + with pytest.raises(asyncpg.UndefinedTableError): + await pg_migrate.migrate_borrowed(conn, tmp_path, schema="test") + assert not conn.is_closed() + assert await conn.fetchval("SELECT pg_advisory_unlock($1)", pg_migrate._LOCK_KEY) + assert await conn.fetchval("SELECT count(*) FROM test.pg_migrations") == 0 diff --git a/tests/unit/test_pg_close_owned.py b/tests/unit/test_pg_close_owned.py new file mode 100644 index 0000000000..fbbe778a5d --- /dev/null +++ b/tests/unit/test_pg_close_owned.py @@ -0,0 +1,144 @@ +"""Owned-session cleanup delivers cancellation before aborting its transport.""" + +from __future__ import annotations + +import asyncio +import time +from typing import Any + +import pytest + +from server.core import pg + +pytestmark = pytest.mark.asyncio + + +class Connection: + def __init__(self) -> None: + self.started = asyncio.Event() + self.finished = asyncio.Event() + self.aborted = False + self.timeouts: list[float] = [] + + async def close(self, *, timeout: float) -> None: # noqa: ASYNC109 — driver API. + self.timeouts.append(timeout) + self.started.set() + await self.finished.wait() + + def terminate(self) -> None: + self.aborted = True + self.finished.set() + + +async def test_public_close_completes_before_transport_abort() -> None: + conn = Connection() + conn.finished.set() + await pg.close_owned(conn, end=time.monotonic() + 1) + assert conn.timeouts and 0 < conn.timeouts[0] <= 1 + assert not conn.aborted + + +async def test_expired_deadline_aborts_without_starting_close() -> None: + conn = Connection() + with pytest.raises(TimeoutError): + await pg.close_owned(conn, end=time.monotonic() - 1) + assert conn.aborted and not conn.timeouts + + +async def test_close_timeout_aborts_within_the_original_deadline() -> None: + conn = Connection() + started = time.monotonic() + with pytest.raises(TimeoutError): + await pg.close_owned(conn, end=started + 0.03) + assert conn.aborted and time.monotonic() - started < 0.3 + + +async def test_second_cancellation_aborts_without_waiting_for_close_unwind() -> None: + conn = Connection() + supervisor_started = asyncio.Event() + original = asyncio.CancelledError("operation cancelled") + + async def supervisor() -> None: + try: + supervisor_started.set() + await asyncio.Future() + except asyncio.CancelledError: + try: + await pg.close_owned(conn, end=time.monotonic() + 1) + except BaseException: # noqa: BLE001 — preserve the original cancellation. + pass + raise original from None + + task = asyncio.create_task(supervisor()) + await supervisor_started.wait() + task.cancel() + await asyncio.wait_for(conn.started.wait(), 1) + task.cancel() + with pytest.raises(asyncio.CancelledError, match="operation cancelled"): + await asyncio.wait_for(task, 0.3) + assert conn.aborted + + +async def test_failed_public_close_aborts_and_reports_failure() -> None: + class Broken(Connection): + async def close(self, **_: Any) -> None: + raise ValueError("close failed") + + conn = Broken() + with pytest.raises(ValueError, match="close failed"): + await pg.close_owned(conn, end=time.monotonic() + 1) + assert conn.aborted + + +async def test_expired_close_child_exception_is_retrieved() -> None: + release = asyncio.Event() + errors: list[dict[str, Any]] = [] + + class LateFailure(Connection): + async def close(self, **_: Any) -> None: + self.started.set() + try: + await release.wait() + except asyncio.CancelledError: + await release.wait() + raise ValueError("late close failure") + + conn = LateFailure() + loop = asyncio.get_running_loop() + old_handler = loop.get_exception_handler() + loop.set_exception_handler(lambda _, context: errors.append(context)) + try: + with pytest.raises(TimeoutError): + await pg.close_owned(conn, end=time.monotonic() + 0.01) + assert conn.aborted + release.set() + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not errors + finally: + release.set() + loop.set_exception_handler(old_handler) + + +async def test_migration_failure_survives_cleanup_failure(monkeypatch, tmp_path) -> None: + import contextlib + + from server.core.schema import pg_migrate + + class Broken(Connection): + async def close(self, **_: Any) -> None: + raise RuntimeError("cleanup failed") + + conn = Broken() + + @contextlib.asynccontextmanager + async def connect(): + yield conn + + async def fail(*_: Any) -> None: + raise ValueError("migration failed") + + monkeypatch.setattr(pg_migrate, "_migrate", fail) + with pytest.raises(ValueError, match="migration failed"): + await pg_migrate.migrate(tmp_path, connect=connect, schema="test") + assert conn.aborted diff --git a/tests/unit/test_pg_online_index.py b/tests/unit/test_pg_online_index.py index 7fc43fa2bb..3bb0c19476 100644 --- a/tests/unit/test_pg_online_index.py +++ b/tests/unit/test_pg_online_index.py @@ -122,7 +122,7 @@ def never_connect(): @pytest.mark.asyncio @pytest.mark.parametrize("phase", ["statement", "rollback", "commit"]) -async def test_cancellation_terminates_before_installed_asyncpg_transaction_unwind( +async def test_cancellation_closes_before_installed_asyncpg_transaction_unwind( tmp_path, phase ) -> None: import asyncio @@ -136,6 +136,7 @@ async def test_cancellation_terminates_before_installed_asyncpg_transaction_unwi (tmp_path / "001_one.sql").write_text("SELECT stalled_statement;") reached = asyncio.Event() terminated = asyncio.Event() + closed = asyncio.Event() class Connection: _pool_release_ctr = 0 @@ -143,11 +144,16 @@ class Connection: _protocol = SimpleNamespace(is_in_transaction=lambda: False) def is_closed(self): - return terminated.is_set() + return closed.is_set() or terminated.is_set() def transaction(self): return Transaction(self, None, False, False) + async def close(self, **kwargs): + assert kwargs["timeout"] > 0 + assert not terminated.is_set() + closed.set() + def terminate(self): terminated.set() @@ -166,11 +172,10 @@ async def execute(self, sql, *args): or (sql.startswith("COMMIT") and phase == "commit") ): reached.set() - # Transport termination, rather than a second task cancellation, - # releases a transaction that has already begun async cleanup. - while not terminated.is_set(): + # Public close must run independently of transaction unwinding. + while not closed.is_set(): try: - await terminated.wait() + await closed.wait() except asyncio.CancelledError: continue @@ -188,9 +193,9 @@ async def connect(): migrating.cancel() with pytest.raises(asyncio.CancelledError): await asyncio.wait_for(migrating, 1) - assert terminated.is_set() + assert closed.is_set() and not terminated.is_set() finally: - terminated.set() + closed.set() if not migrating.done(): migrating.cancel() await asyncio.gather(migrating, return_exceptions=True) From c601646a118940318e67f16f391695e56328e4e7 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 22:40:22 +0000 Subject: [PATCH 026/114] Close owned Postgres sessions before cancelling their work Send public driver cancellation before aborting the transport, within one absolute cleanup deadline. Separate borrowed migration execution so its operator keeps sole session supervision. Verify backend lock release and preserve interrupted migration errors. Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 1 + server/core/pg.py | 32 ++++ server/core/schema/pg_migrate.py | 52 +++++-- tests/integration/test_pg_online_index.py | 10 +- tests/integration/test_pg_owned_cleanup.py | 173 +++++++++++++++++++++ tests/unit/test_pg_close_owned.py | 144 +++++++++++++++++ tests/unit/test_pg_online_index.py | 21 ++- 7 files changed, 413 insertions(+), 20 deletions(-) create mode 100644 tests/integration/test_pg_owned_cleanup.py create mode 100644 tests/unit/test_pg_close_owned.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a132706be6..6e866c99a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -897,6 +897,7 @@ jobs: tests/integration/test_cell_tenancy_pg.py tests/integration/test_router_directory_pg.py tests/integration/test_pg_online_index.py + tests/integration/test_pg_owned_cleanup.py tests/integration/test_cell_unset_gate_pg.py tests/integration/test_cell_prepare.py tests/integration/test_cell_attach.py diff --git a/server/core/pg.py b/server/core/pg.py index 576b1b1e4b..5daeb33a55 100644 --- a/server/core/pg.py +++ b/server/core/pg.py @@ -21,6 +21,7 @@ import asyncio import json import logging +import time import weakref from collections.abc import AsyncIterator, Mapping from contextlib import asynccontextmanager, suppress @@ -379,6 +380,37 @@ def _terminate(conn: Any) -> None: conn.terminate() +def _retrieve_task(task: asyncio.Task[Any]) -> None: + if not task.cancelled(): + task.exception() + + +async def close_owned(conn: Any, *, end: float) -> None: + """Close an owned session by an absolute time.monotonic deadline. + + Public close sends cancellation before transport cleanup can discard it. Abort + is a bounded fallback, not evidence that the backend has released its locks. + The owner preserves an earlier operation failure when this cleanup fails. + """ + task: asyncio.Task[Any] | None = None + try: + remaining = end - time.monotonic() + if remaining <= 0: + raise TimeoutError("Postgres owned-session cleanup deadline expired") + task = asyncio.create_task(conn.close(timeout=remaining)) + done, _ = await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) + if not done: + raise TimeoutError("Postgres owned-session cleanup deadline expired") + task.result() + except BaseException: + _terminate(conn) + if task is not None: + if not task.done(): + task.cancel() + task.add_done_callback(_retrieve_task) + raise + + async def _release(pool: Any, conn: Any) -> None: """Back to the pool within a bound: a statement still in flight (a client timeout, a cancelled request) has its cancel sent and acknowledged and the connection reset; diff --git a/server/core/schema/pg_migrate.py b/server/core/schema/pg_migrate.py index ebd230036b..371f61a449 100644 --- a/server/core/schema/pg_migrate.py +++ b/server/core/schema/pg_migrate.py @@ -19,6 +19,7 @@ import asyncio import hashlib import logging +import time from asyncio import sleep from collections.abc import Callable from contextlib import AbstractAsyncContextManager @@ -108,25 +109,54 @@ async def migrate( bounded (`lock_timeout`), so an `ALTER` queued behind a long transaction never holds every read and write of the table behind it; the file is retried. For schema `loom` it marks the session and runs `cell_triggers.ensure` before the lock goes.""" - files = migration_files(mig_dir) - # Validate every online declaration before even creating the ledger. - indexes = {f.name: pg_online_index.parse(f.read_text()) for f in files} - if any(index is not None and index.schema != schema for index in indexes.values()): - raise ValueError("online-index target differs from its ledger schema") + files, indexes = _validated(mig_dir, schema) async with (connect or pg.dedicated)() as conn: task = asyncio.create_task(_migrate(conn, files, indexes, schema, lock_key)) try: return await asyncio.shield(task) except BaseException: - # Terminate before cancellation can enter asyncpg's unbounded rollback. - conn.terminate() - task.cancel() - await asyncio.wait({task}, timeout=1) - if task.done() and not task.cancelled(): - task.exception() + end = time.monotonic() + pg._RELEASE_TIMEOUT_S + try: + await pg.close_owned(conn, end=end) + except BaseException: # noqa: BLE001 — preserve the migration failure/cancellation. + pass + if not task.done(): + task.cancel() + task.add_done_callback(pg._retrieve_task) + try: + await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) + except BaseException: # noqa: BLE001 — preserve the original failure on repeated cancel. + pg._terminate(conn) raise +def _validated( + mig_dir: Path, schema: str +) -> tuple[list[Path], dict[str, pg_online_index.OnlineIndex | None]]: + files = migration_files(mig_dir) + # Validate every online declaration before even creating the ledger. + indexes = {f.name: pg_online_index.parse(f.read_text()) for f in files} + if any(index is not None and index.schema != schema for index in indexes.values()): + raise ValueError("online-index target differs from its ledger schema") + return files, indexes + + +async def migrate_borrowed( + conn: Any, + mig_dir: Path = PG_MIGRATIONS_DIR, + *, + schema: str = "loom", + lock_key: int = _LOCK_KEY, +) -> list[str]: + """Apply on the owner's session; its supervisor owns all failure cleanup. + + The caller retains its operation deadline and must close the session before + cancelling this work. Normal completion releases only the migration key. + """ + files, indexes = _validated(mig_dir, schema) + return await _migrate(conn, files, indexes, schema, lock_key) + + async def _lock(conn: Any, key: int) -> None: while True: set_query_tag("migrate.postgres.lock") diff --git a/tests/integration/test_pg_online_index.py b/tests/integration/test_pg_online_index.py index af93dd0937..1995e73bee 100644 --- a/tests/integration/test_pg_online_index.py +++ b/tests/integration/test_pg_online_index.py @@ -5,12 +5,14 @@ import asyncio import contextlib import os +import time from collections.abc import AsyncIterator from pathlib import Path from typing import Any import pytest +from server.core import pg from server.core.paths import ROUTER_MIGRATIONS_DIR from server.core.schema import pg_migrate, pg_online_index from server.router import directory @@ -315,7 +317,7 @@ async def direct_drop(): "DROP INDEX CONCURRENTLY loom_router.placements_by_cell_org" ) finally: - dropping.terminate() + await pg.close_owned(dropping, end=time.monotonic() + 1) dropping = asyncio.create_task(direct_drop()) try: @@ -325,6 +327,12 @@ async def direct_drop(): "AND wait_event IS NOT NULL)" ) await _until(conn, query) + # Cancel after catalog invalidation, not an earlier lock/I/O wait. + await _until( + conn, + "SELECT NOT indisvalid FROM pg_index " + "WHERE indexrelid='loom_router.placements_by_cell_org'::regclass", + ) dropping.cancel() with pytest.raises(asyncio.CancelledError): await asyncio.wait_for(dropping, 2) diff --git a/tests/integration/test_pg_owned_cleanup.py b/tests/integration/test_pg_owned_cleanup.py new file mode 100644 index 0000000000..9085fce011 --- /dev/null +++ b/tests/integration/test_pg_owned_cleanup.py @@ -0,0 +1,173 @@ +"""Cancellation of owned Postgres work releases its actual backend locks.""" + +from __future__ import annotations + +import asyncio +import contextlib +import os +import time +from collections.abc import AsyncIterator +from pathlib import Path +from typing import Any + +import pytest + +from server.core import pg +from server.core.schema import pg_migrate +from tests.integration._pg_database import create_database, drop_database + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +async def database() -> AsyncIterator[tuple[Any, Any]]: + import asyncpg + + admin = os.environ.get("LOOM_PG_DSN") or pytest.skip("LOOM_PG_DSN not set") + name, dsn = await create_database(admin) + conn = await asyncpg.connect(dsn, server_settings={"statement_timeout": "10000"}) + observer = await asyncpg.connect(dsn) + try: + await conn.execute("CREATE TABLE probe(id int PRIMARY KEY); INSERT INTO probe VALUES(1)") + yield conn, observer + finally: + await conn.close(timeout=1) + await observer.close(timeout=1) + await drop_database(admin, name) + + +async def sleeping(observer: Any, pid: int) -> None: + async with asyncio.timeout(2): + for _ in range(400): + if await observer.fetchval( + "SELECT wait_event='PgSleep' FROM pg_stat_activity WHERE pid=$1", pid + ): + break + await asyncio.sleep(0.005) + else: + pytest.fail("backend did not reach pg_sleep") + assert await observer.fetchval("SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1)", pid) + + +async def released(observer: Any, pid: int) -> None: + assert not await observer.fetchval("SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1)", pid) + assert not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)", pid + ) + + +async def test_owned_migrator_cancellation_releases_server_locks(database, tmp_path: Path) -> None: + conn, observer = database + (tmp_path / "001_sleep.sql").write_text("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10);") + + @contextlib.asynccontextmanager + async def connect() -> AsyncIterator[Any]: + yield conn + + task = asyncio.create_task(pg_migrate.migrate(tmp_path, connect=connect, schema="test")) + await sleeping(observer, conn.get_server_pid()) + pid = conn.get_server_pid() + started = time.monotonic() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, 1.5) + assert time.monotonic() - started < 1.5 + await released(observer, pid) + + +@pytest.mark.parametrize("transaction_context", [False, True]) +async def test_public_owned_close_releases_live_query_before_task_cancel( + database, transaction_context: bool +) -> None: + conn, observer = database + pid = conn.get_server_pid() + + async def work() -> None: + if transaction_context: + async with conn.transaction(): + await conn.execute("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10)") + else: + await conn.execute("BEGIN; SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10)") + + task = asyncio.create_task(work()) + try: + await sleeping(observer, pid) + await pg.close_owned(conn, end=time.monotonic() + 0.5) + await released(observer, pid) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_borrowed_migrator_leaves_failure_cleanup_to_owner(database, tmp_path: Path) -> None: + conn, observer = database + (tmp_path / "001_sleep.sql").write_text("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10);") + task = asyncio.create_task(pg_migrate.migrate_borrowed(conn, tmp_path, schema="test")) + pid = conn.get_server_pid() + try: + await sleeping(observer, pid) + await pg.close_owned(conn, end=time.monotonic() + 0.5) + await released(observer, pid) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_borrowed_migrator_normal_completion_retains_owners_lock(database, tmp_path) -> None: + conn, observer = database + await conn.execute("SELECT pg_advisory_lock(712398)") + (tmp_path / "001_write.sql").write_text("INSERT INTO probe VALUES(2)") + assert await pg_migrate.migrate_borrowed(conn, tmp_path, schema="test") == ["001_write.sql"] + assert not conn.is_closed() + assert await observer.fetchval("SELECT count(*) FROM probe") == 2 + assert await conn.fetchval("SELECT pg_advisory_unlock(712398)") + assert ( + await conn.fetchval( + "SELECT count(*) FROM pg_locks WHERE pid=pg_backend_pid() AND locktype='advisory'" + ) + == 0 + ) + + +async def test_owned_close_interrupts_installed_transaction_rollback(database, monkeypatch) -> None: + import asyncpg + + conn, observer = database + pid = conn.get_server_pid() + execute = asyncpg.Connection.execute + + async def stalled_rollback(self, query, *args, **kwargs): + if self is conn and query.startswith("ROLLBACK"): + await execute(self, "SELECT pg_sleep(10)") + return await execute(self, query, *args, **kwargs) + + monkeypatch.setattr(asyncpg.Connection, "execute", stalled_rollback) + + async def work() -> None: + async with conn.transaction(): + await conn.execute("SELECT * FROM probe FOR UPDATE") + raise ValueError("statement failure before rollback") + + task = asyncio.create_task(work()) + try: + await sleeping(observer, pid) + await pg.close_owned(conn, end=time.monotonic() + 0.5) + await released(observer, pid) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_borrowed_failure_keeps_the_session_for_its_owner(database, tmp_path) -> None: + import asyncpg + + conn, _ = database + (tmp_path / "001_failure.sql").write_text("SELECT * FROM missing_table_for_failure") + with pytest.raises(asyncpg.UndefinedTableError): + await pg_migrate.migrate_borrowed(conn, tmp_path, schema="test") + assert not conn.is_closed() + assert await conn.fetchval("SELECT pg_advisory_unlock($1)", pg_migrate._LOCK_KEY) + assert await conn.fetchval("SELECT count(*) FROM test.pg_migrations") == 0 diff --git a/tests/unit/test_pg_close_owned.py b/tests/unit/test_pg_close_owned.py new file mode 100644 index 0000000000..fbbe778a5d --- /dev/null +++ b/tests/unit/test_pg_close_owned.py @@ -0,0 +1,144 @@ +"""Owned-session cleanup delivers cancellation before aborting its transport.""" + +from __future__ import annotations + +import asyncio +import time +from typing import Any + +import pytest + +from server.core import pg + +pytestmark = pytest.mark.asyncio + + +class Connection: + def __init__(self) -> None: + self.started = asyncio.Event() + self.finished = asyncio.Event() + self.aborted = False + self.timeouts: list[float] = [] + + async def close(self, *, timeout: float) -> None: # noqa: ASYNC109 — driver API. + self.timeouts.append(timeout) + self.started.set() + await self.finished.wait() + + def terminate(self) -> None: + self.aborted = True + self.finished.set() + + +async def test_public_close_completes_before_transport_abort() -> None: + conn = Connection() + conn.finished.set() + await pg.close_owned(conn, end=time.monotonic() + 1) + assert conn.timeouts and 0 < conn.timeouts[0] <= 1 + assert not conn.aborted + + +async def test_expired_deadline_aborts_without_starting_close() -> None: + conn = Connection() + with pytest.raises(TimeoutError): + await pg.close_owned(conn, end=time.monotonic() - 1) + assert conn.aborted and not conn.timeouts + + +async def test_close_timeout_aborts_within_the_original_deadline() -> None: + conn = Connection() + started = time.monotonic() + with pytest.raises(TimeoutError): + await pg.close_owned(conn, end=started + 0.03) + assert conn.aborted and time.monotonic() - started < 0.3 + + +async def test_second_cancellation_aborts_without_waiting_for_close_unwind() -> None: + conn = Connection() + supervisor_started = asyncio.Event() + original = asyncio.CancelledError("operation cancelled") + + async def supervisor() -> None: + try: + supervisor_started.set() + await asyncio.Future() + except asyncio.CancelledError: + try: + await pg.close_owned(conn, end=time.monotonic() + 1) + except BaseException: # noqa: BLE001 — preserve the original cancellation. + pass + raise original from None + + task = asyncio.create_task(supervisor()) + await supervisor_started.wait() + task.cancel() + await asyncio.wait_for(conn.started.wait(), 1) + task.cancel() + with pytest.raises(asyncio.CancelledError, match="operation cancelled"): + await asyncio.wait_for(task, 0.3) + assert conn.aborted + + +async def test_failed_public_close_aborts_and_reports_failure() -> None: + class Broken(Connection): + async def close(self, **_: Any) -> None: + raise ValueError("close failed") + + conn = Broken() + with pytest.raises(ValueError, match="close failed"): + await pg.close_owned(conn, end=time.monotonic() + 1) + assert conn.aborted + + +async def test_expired_close_child_exception_is_retrieved() -> None: + release = asyncio.Event() + errors: list[dict[str, Any]] = [] + + class LateFailure(Connection): + async def close(self, **_: Any) -> None: + self.started.set() + try: + await release.wait() + except asyncio.CancelledError: + await release.wait() + raise ValueError("late close failure") + + conn = LateFailure() + loop = asyncio.get_running_loop() + old_handler = loop.get_exception_handler() + loop.set_exception_handler(lambda _, context: errors.append(context)) + try: + with pytest.raises(TimeoutError): + await pg.close_owned(conn, end=time.monotonic() + 0.01) + assert conn.aborted + release.set() + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not errors + finally: + release.set() + loop.set_exception_handler(old_handler) + + +async def test_migration_failure_survives_cleanup_failure(monkeypatch, tmp_path) -> None: + import contextlib + + from server.core.schema import pg_migrate + + class Broken(Connection): + async def close(self, **_: Any) -> None: + raise RuntimeError("cleanup failed") + + conn = Broken() + + @contextlib.asynccontextmanager + async def connect(): + yield conn + + async def fail(*_: Any) -> None: + raise ValueError("migration failed") + + monkeypatch.setattr(pg_migrate, "_migrate", fail) + with pytest.raises(ValueError, match="migration failed"): + await pg_migrate.migrate(tmp_path, connect=connect, schema="test") + assert conn.aborted diff --git a/tests/unit/test_pg_online_index.py b/tests/unit/test_pg_online_index.py index 7fc43fa2bb..3bb0c19476 100644 --- a/tests/unit/test_pg_online_index.py +++ b/tests/unit/test_pg_online_index.py @@ -122,7 +122,7 @@ def never_connect(): @pytest.mark.asyncio @pytest.mark.parametrize("phase", ["statement", "rollback", "commit"]) -async def test_cancellation_terminates_before_installed_asyncpg_transaction_unwind( +async def test_cancellation_closes_before_installed_asyncpg_transaction_unwind( tmp_path, phase ) -> None: import asyncio @@ -136,6 +136,7 @@ async def test_cancellation_terminates_before_installed_asyncpg_transaction_unwi (tmp_path / "001_one.sql").write_text("SELECT stalled_statement;") reached = asyncio.Event() terminated = asyncio.Event() + closed = asyncio.Event() class Connection: _pool_release_ctr = 0 @@ -143,11 +144,16 @@ class Connection: _protocol = SimpleNamespace(is_in_transaction=lambda: False) def is_closed(self): - return terminated.is_set() + return closed.is_set() or terminated.is_set() def transaction(self): return Transaction(self, None, False, False) + async def close(self, **kwargs): + assert kwargs["timeout"] > 0 + assert not terminated.is_set() + closed.set() + def terminate(self): terminated.set() @@ -166,11 +172,10 @@ async def execute(self, sql, *args): or (sql.startswith("COMMIT") and phase == "commit") ): reached.set() - # Transport termination, rather than a second task cancellation, - # releases a transaction that has already begun async cleanup. - while not terminated.is_set(): + # Public close must run independently of transaction unwinding. + while not closed.is_set(): try: - await terminated.wait() + await closed.wait() except asyncio.CancelledError: continue @@ -188,9 +193,9 @@ async def connect(): migrating.cancel() with pytest.raises(asyncio.CancelledError): await asyncio.wait_for(migrating, 1) - assert terminated.is_set() + assert closed.is_set() and not terminated.is_set() finally: - terminated.set() + closed.set() if not migrating.done(): migrating.cancel() await asyncio.gather(migrating, return_exceptions=True) From 51a8ff6498c3796d96c4c35946d4d70d8842d033 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 23:12:55 +0000 Subject: [PATCH 027/114] Close attach sessions before cancelling transaction work Supervise bounded Postgres phases independently, reserving cleanup inside each existing deadline. Send public close while query and transaction work are alive, then cancel and reap within that same bound. Borrowed checks and trigger DDL leave failed-session cleanup to their owner. Bound the lock owner's close and preserve its original failure across cleanup errors or repeated cancellation. Keep wait floors and normal busy retries. Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 1 + docs/architecture/cells.md | 9 +- docs/operations/cell-attach.md | 5 + server/cli/cell_attach.py | 37 +++- server/cli/cell_attach_pg.py | 160 +++++++++++------- server/cli/cell_prepare.py | 16 +- server/core/cell_triggers.py | 10 +- tests/integration/test_cell_attach_cleanup.py | 111 ++++++++++++ tests/unit/test_cell_attach.py | 53 ++++-- tests/unit/test_cell_attach_cleanup.py | 119 +++++++++++++ 10 files changed, 426 insertions(+), 95 deletions(-) create mode 100644 tests/integration/test_cell_attach_cleanup.py create mode 100644 tests/unit/test_cell_attach_cleanup.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e866c99a5..3066d67299 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -901,6 +901,7 @@ jobs: tests/integration/test_cell_unset_gate_pg.py tests/integration/test_cell_prepare.py tests/integration/test_cell_attach.py + tests/integration/test_cell_attach_cleanup.py tests/integration/test_cell_org_triggers_pg.py - name: pytest online-index durability on a private physical standby timeout-minutes: 8 diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index dd1189b696..c738775756 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1067,7 +1067,14 @@ Then, in order (the step numbers are the ones the rest of this page uses): lock on `loom.tenancy_mode`, 2 s a try within `--wait`, the attach lock confirmed and the cheap heartbeat refusals run before each try (a booting process skips the try). Each try, including COMMIT, has a client deadline: the lock wait plus a 3 s statement margin and - a 5 s standby allowance. A commit past it has an unknown outcome until the durable read. + a 5 s standby allowance. Each mode or trigger try reserves the smaller of one second + and one quarter of its original client bound for cleanup. Its independent owner + closes the session through the public driver API while query/transaction work is + still alive, then cancels and reaps that work within the same absolute deadline. + Borrowed checks and trigger DDL leave failed-session cleanup to that owner. Close + failure or exhausted cleanup time aborts the transport and preserves the original + failure; abort alone does not prove that server locks disappeared. A commit past + its deadline has an unknown outcome until the durable read. After a failed try, the mode transaction an earlier run left in flight is found by its lock (only attach takes `ExclusiveLock` there; a relation's OID is matched in this database alone) and ended. From the commit on the mode only rises: every later refusal leaves the cell diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index e674da2997..13c65405f6 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -92,6 +92,11 @@ one try of each wait with no retry. The final heartbeat checks get at least two plus the router timeout, with one extension for a process first seen booting; this floor is not charged to `--wait`. `--scan-budget` (default 7200 s) bounds each full scan, and step 2 must take at most half of it, since step 4 reads the same tables again. +Each Postgres mode or trigger try reserves cleanup time inside its client deadline. +On interruption its owner closes the session before cancelling transaction work. +A close that cannot complete uses transport abort and reports failure; that abort +does not establish that server locks or an unacknowledged commit have disappeared. +Retry the command to resolve an unknown commit through its durable state checks. A refusal names what to fix. Rerun attach after fixing it: every step is idempotent, and a rerun resumes by the directory row and the cell's mode. On a `serving` row it prints that it diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index bf4e81521e..2c3915ae66 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -187,14 +187,39 @@ async def open_lock(self) -> None: self._lock_cm = pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) self.lock_conn = await self._lock_cm.__aenter__() + await apg.supervise( + self.lock_conn, self._take_lock(), end=time.monotonic() + _LOCK_STATEMENT_MS / 1000 + ) + + async def _take_lock(self) -> None: if refusals := await apg.direct_refusals(self.lock_conn, "attach"): raise Refused(refusals) await apg.take_lock(self.lock_conn, "attach") async def close_lock(self) -> None: + from server.core import pg + if self._lock_cm is not None: - await self._lock_cm.__aexit__(None, None, None) + cm, conn = self._lock_cm, self.lock_conn self._lock_cm = self.lock_conn = None + closed = pg._lost(conn) + end = time.monotonic() + (0.0 if closed else pg._RELEASE_TIMEOUT_S) + try: + if not closed: + await pg.close_owned(conn, end=end) + finally: + task = asyncio.create_task(cm.__aexit__(None, None, None)) + task.add_done_callback(pg._retrieve_task) + try: + done, _ = await asyncio.wait({task}, timeout=max(0.0, end - time.monotonic())) + if not done: + task.cancel() + raise TimeoutError("attach lock cleanup deadline expired") + task.result() + except BaseException: + pg._terminate(conn) + task.cancel() + raise async def confirm_lock(self) -> None: """The lock is held still, or taken again (refusing when another run holds it now), @@ -631,6 +656,7 @@ async def run(self) -> int: await self.done() self.report.update(state="serving", done=True) return 0 + failure: BaseException | None = None try: await self.open_lock() await self.read_own_row() @@ -660,8 +686,15 @@ async def run(self) -> int: answer = await self.serving() self.report.update(state=answer.get("state"), placed_orgs=answer.get("placed_orgs")) return 0 + except BaseException as error: + failure = error + raise finally: - await self.close_lock() + try: + await self.close_lock() + except BaseException: + if failure is None: + raise async def precommit(self, found: cell_scan.ScanResult) -> None: from server.core import pg_profile diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 12e1808f8b..c71e1130c7 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -12,7 +12,7 @@ import asyncio import contextlib import time -from collections.abc import Awaitable, Callable, Iterable, Iterator +from collections.abc import Awaitable, Callable, Coroutine, Iterable, Iterator from dataclasses import dataclass from typing import Any @@ -89,6 +89,50 @@ async def pause(self, seconds: float = POLL_S) -> bool: return True +@dataclass +class _Interrupted: + error: BaseException + + +async def _work_result[T](work: Coroutine[Any, Any, T]) -> T | _Interrupted: + try: + return await work + except (KeyboardInterrupt, SystemExit) as error: + # These exceptions otherwise stop asyncio's loop before the owner can close. + return _Interrupted(error) + + +async def supervise[T](conn: Any, work: Coroutine[Any, Any, T], *, end: float) -> T: + """Bound owned work, reserving cleanup inside its original phase deadline. + + A failed borrowed transaction must remain alive until public close sends its + cancellation. Closing the transport first can leave its backend locks held. + """ + task = asyncio.create_task(_work_result(work)) + reserve = min(1.0, max(0.0, end - time.monotonic()) / 4) + try: + done, _ = await asyncio.wait({task}, timeout=max(0.0, end - reserve - time.monotonic())) + if not done: + raise TimeoutError("attach Postgres phase deadline expired") + result = task.result() + if isinstance(result, _Interrupted): + raise result.error + return result + except BaseException: + try: + await pg.close_owned(conn, end=end) + except BaseException: # noqa: BLE001 — preserve the work failure or cancellation. + pass + if not task.done(): + task.cancel() + task.add_done_callback(pg._retrieve_task) + try: + await asyncio.wait({task}, timeout=max(0.0, end - time.monotonic())) + except BaseException: # noqa: BLE001 — repeated cancellation preserves the first failure. + pg._terminate(conn) + raise + + def _split(key: int) -> str: """The `pg_locks` columns that hold a bigint advisory key, in this database.""" return ( @@ -124,16 +168,11 @@ async def direct_refusals(conn: Any, what: str) -> list[Refusal]: outlive the CLI on the pooler's server connection, and attach's tenancy rows would be refused as an unchecked writer's.""" identities = [] - try: - for _ in range(2): - tx = conn.transaction() - await tx.start() - identities.append(await conn.fetchrow(_IDENTITY)) - await tx.commit() - except BaseException: - # A cancelled read must not wait for a transaction context's rollback. - conn.terminate() - raise + for _ in range(2): + tx = conn.transaction() + await tx.start() + identities.append(await conn.fetchrow(_IDENTITY)) + await tx.commit() first, second = identities out = [] if first is None or second is None or tuple(first) != tuple(second): @@ -537,30 +576,32 @@ async def _ensure_one(conn: Any, table: str, most_ms: int, wait: Wait) -> cell_t bound_s = ( wait.bound_ms(_MIGRATION_LOCK_MS) + wait.bound_ms(most_ms) ) / 1000 + _TRIGGER_TRY_MARGIN_S + return await supervise( + conn, _trigger_try(conn, table, most_ms, wait, _LOCK_KEY), end=time.monotonic() + bound_s + ) + + +async def _trigger_try( + conn: Any, table: str, most_ms: int, wait: Wait, key: int +) -> cell_triggers.Outcome: + await conn.execute(f"SET lock_timeout = '{wait.bound_ms(_MIGRATION_LOCK_MS)}ms'") try: - async with asyncio.timeout(bound_s): - await conn.execute(f"SET lock_timeout = '{wait.bound_ms(_MIGRATION_LOCK_MS)}ms'") - try: - await conn.execute(_MIGRATION_LOCK, _LOCK_KEY) - except Exception as e: - if getattr(e, "sqlstate", None) != "55P03": - raise - await conn.execute("RESET lock_timeout") - return cell_triggers.Outcome("busy", await _migration_holder(conn, _LOCK_KEY)) - await conn.execute("RESET lock_timeout") - outcome = await cell_triggers.ensure_table( - conn, - table, - recording_first=True, - lock_timeout=f"{wait.bound_ms(most_ms)}ms", - attempts=1, - ) - await conn.execute(_MIGRATION_UNLOCK, _LOCK_KEY) - return outcome - except BaseException: - # Closing the session releases its locks without asynchronous error cleanup. - conn.terminate() - raise + await conn.execute(_MIGRATION_LOCK, key) + except Exception as e: + if getattr(e, "sqlstate", None) != "55P03": + raise + await conn.execute("RESET lock_timeout") + return cell_triggers.Outcome("busy", await _migration_holder(conn, key)) + await conn.execute("RESET lock_timeout") + outcome = await cell_triggers.ensure_table( + conn, + table, + recording_first=True, + lock_timeout=f"{wait.bound_ms(most_ms)}ms", + attempts=1, + ) + await conn.execute(_MIGRATION_UNLOCK, key) + return outcome def _busy_refusal(table: str, holders: list[Any]) -> Refusal: @@ -653,35 +694,32 @@ async def _mode_try(conn: Any, lock_ms: int, mark: Callable[[str], None]) -> boo the standby allowance; False after a statement timeout rolls back, TimeoutError at the try's deadline.""" bound_s = (lock_ms + _MODE_STATEMENT_MARGIN_MS) / 1000 + _MODE_STANDBY_S + return await supervise( + conn, _mode_transaction(conn, lock_ms, mark), end=time.monotonic() + bound_s + ) + + +async def _mode_transaction(conn: Any, lock_ms: int, mark: Callable[[str], None]) -> bool: + tx = conn.transaction() + await tx.start() try: - async with asyncio.timeout(bound_s): - tx = conn.transaction() - await tx.start() - try: - await conn.execute(f"SET LOCAL lock_timeout = '{lock_ms}ms'") - await conn.execute( - f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" - ) - await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") - mark("unknown") - status = await conn.execute( - "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" - ) - if status != "UPDATE 1": - raise Refused( - [Refusal("postgres loom.tenancy_mode", f"the mode update read {status}")] - ) - except Exception as e: - await tx.rollback() - if getattr(e, "sqlstate", None) in _TIMED_OUT: - return False - raise - await tx.commit() - return True - except BaseException: - # Termination is synchronous: cancellation never starts an unbounded rollback. - conn.terminate() + await conn.execute(f"SET LOCAL lock_timeout = '{lock_ms}ms'") + await conn.execute( + f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" + ) + await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") + mark("unknown") + status = await conn.execute("UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()") + if status != "UPDATE 1": + raise Refused([Refusal("postgres loom.tenancy_mode", f"the mode update read {status}")]) + except Exception as e: + await tx.rollback() + if getattr(e, "sqlstate", None) in _TIMED_OUT: + return False raise + await tx.commit() + + return True async def end_mode_transactions(conn: Any, confirm_lock: Callable[[], Any]) -> None: diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index 4a820f652b..a71a541656 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -467,11 +467,17 @@ async def attach_lock() -> AsyncIterator[None]: async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: try: - refusals = await cell_attach_pg.direct_refusals(conn, "prepare") - refusals += await cell_attach_pg.encoding_refusals(conn) - if refusals: - raise Stopped(refusals) - await cell_attach_pg.take_lock(conn, "prepare") + + async def take() -> None: + refusals = await cell_attach_pg.direct_refusals(conn, "prepare") + refusals += await cell_attach_pg.encoding_refusals(conn) + if refusals: + raise Stopped(refusals) + await cell_attach_pg.take_lock(conn, "prepare") + + await cell_attach_pg.supervise( + conn, take(), end=time.monotonic() + _LOCK_STATEMENT_MS / 1000 + ) except Stopped as e: raise Refused(str(e)) from None yield diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 1899a46bab..68e74777ec 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -324,7 +324,8 @@ async def _run_ddl( two causes cells.md names kept it from applying (`busy`: its lock not taken, with the sessions it waited behind; `denied`: a table this role cannot alter), which a boot serves past; any other error fails it, as a failed read of the state does. `warn` logs - a busy table once, after its last try, with its DDL and holders (a boot's `ensure`).""" + a busy table once, after its last try, with its DDL and holders (a boot's `ensure`). + The borrowing caller owns failed-session cleanup, including cancellation/rollback.""" import asyncpg for attempt in range(attempts): @@ -344,7 +345,6 @@ async def _run_ddl( return Outcome("applied") except asyncpg.exceptions.InsufficientPrivilegeError as e: if not rolled_back: - conn.terminate() raise log.warning( "cell triggers: not applied on loom.%s (%s): this role cannot alter it", table, e @@ -352,7 +352,6 @@ async def _run_ddl( return Outcome("denied") except asyncpg.exceptions.LockNotAvailableError as e: if not rolled_back: - conn.terminate() raise # Short tries outlast a short transaction; an autovacuum is left to finish. if attempt < attempts - 1: @@ -368,11 +367,6 @@ async def _run_ddl( held, ) return Outcome("busy", held) - except BaseException: - if not rolled_back: - # Cancellation must not wait for another statement on this session. - conn.terminate() - raise return Outcome("busy") diff --git a/tests/integration/test_cell_attach_cleanup.py b/tests/integration/test_cell_attach_cleanup.py new file mode 100644 index 0000000000..145be1188d --- /dev/null +++ b/tests/integration/test_cell_attach_cleanup.py @@ -0,0 +1,111 @@ +"""Attach owners close active sessions before cancelling transaction work.""" + +import argparse +import asyncio +import contextlib +import time + +import pytest + +from server.cli import cell_attach +from server.cli import cell_attach_pg as apg +from server.core import cell_triggers, pg +from tests.integration.test_pg_owned_cleanup import database as database +from tests.integration.test_pg_owned_cleanup import released, sleeping + +pytestmark = pytest.mark.asyncio + + +@pytest.mark.parametrize( + "owner,phase", + [ + ("trigger", "query"), + ("trigger", "rollback"), + ("mode", "query"), + ("mode", "rollback"), + ("direct", "query"), + ], +) +@pytest.mark.parametrize("interruption", ["cancel", "deadline", "second-cancel"]) +async def test_attach_owner_releases_backend_locks_before_return( + database, monkeypatch, owner, phase, interruption +): + import asyncpg + + conn, observer = database + await conn.execute( + "CREATE SCHEMA loom; CREATE TABLE loom.tenancy_mode(mode text, since timestamptz); INSERT INTO loom.tenancy_mode VALUES('recording', now())" + ) + pid = conn.get_server_pid() + execute = asyncpg.Connection.execute + closing = asyncio.Event() + close = asyncpg.Connection.close + + async def intercepted(self, query, *args, **kwargs): + if self is conn: + if query.startswith("SET LOCAL statement_timeout"): + return await execute(self, "SET LOCAL statement_timeout = '10000ms'") + if query == "RAISE Python error": + raise ValueError("trigger statement failed") + if phase == "rollback" and query.startswith("ROLLBACK"): + await execute(self, "SELECT pg_sleep(10)") + if owner == "mode" and query.startswith("LOCK TABLE"): + await execute(self, "SELECT * FROM probe FOR UPDATE") + if phase == "rollback": + raise ValueError("mode statement failed") + return await execute(self, "SELECT pg_sleep(10)") + if owner == "direct" and query.startswith("BEGIN"): + await execute(self, query) + return await execute(self, "SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10)") + return await execute(self, query, *args, **kwargs) + + async def closing_session(self, *args, **kwargs): + was_open = not self.is_closed() + await close(self, *args, **kwargs) + if self is conn and was_open and interruption == "second-cancel": + closing.set() + await asyncio.Future() + + async def ensure(conn, table, **kwargs): + statements = ["SELECT * FROM probe FOR UPDATE"] + statements += ["RAISE Python error"] if phase == "rollback" else ["SELECT pg_sleep(10)"] + return await cell_triggers._run_ddl( + conn, table, statements, kwargs["lock_timeout"], 1, warn=False + ) + + @contextlib.asynccontextmanager + async def dedicated(**_): + yield conn + + monkeypatch.setattr(asyncpg.Connection, "execute", intercepted) + monkeypatch.setattr(asyncpg.Connection, "close", closing_session) + monkeypatch.setattr(cell_triggers, "ensure_table", ensure) + monkeypatch.setattr(apg, "_TRIGGER_TRY_MARGIN_S", 0.4) + monkeypatch.setattr(apg, "_MODE_STANDBY_S", 0.5) + monkeypatch.setattr(apg, "_MODE_STATEMENT_MARGIN_MS", 0) + monkeypatch.setattr(cell_attach, "_LOCK_STATEMENT_MS", 600) + monkeypatch.setattr(pg, "dedicated", dedicated) + if owner == "trigger": + work = apg._ensure_one(conn, "probe", 100, apg.Wait(0)) + elif owner == "mode": + work = apg._mode_try(conn, 100, lambda _: None) + else: + run = cell_attach.Run(None, argparse.Namespace(cell="test", wait=0), None, lambda _: None) + work = run.open_lock() + task = asyncio.create_task(work) + try: + await sleeping(observer, pid) + started = time.monotonic() + if interruption != "deadline": + task.cancel() + if interruption == "second-cancel": + await asyncio.wait_for(closing.wait(), 1) + task.cancel() + with pytest.raises(TimeoutError if interruption == "deadline" else asyncio.CancelledError): + await asyncio.wait_for(asyncio.shield(task), 1.2) + assert time.monotonic() - started < 0.7 + await released(observer, pid) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 0ecd3a96b3..05e15b10c2 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -8,6 +8,7 @@ import argparse import asyncio import json +import time from collections.abc import Callable from dataclasses import dataclass, field from typing import Any @@ -496,6 +497,24 @@ async def __aexit__(self, *exc: Any) -> None: def terminate(self) -> None: self.executed.append("TERMINATE") + async def close(self, **kwargs: Any) -> None: + assert kwargs["timeout"] > 0 + self.executed.append("CLOSE") + + def is_closed(self) -> bool: + return bool(self.executed and self.executed[-1] in {"CLOSE", "TERMINATE"}) + + +def _short_phases(monkeypatch: pytest.MonkeyPatch, bounds: list[float] | None = None) -> None: + supervise = apg.supervise + + async def bounded(conn: Any, work: Any, *, end: float) -> Any: + if bounds is not None: + bounds.append(end - time.monotonic()) + return await supervise(conn, work, end=time.monotonic() + 0.01) + + monkeypatch.setattr(apg, "supervise", bounded) + def _seq(*values: Any) -> Callable[..., Any]: pending = list(values) @@ -750,13 +769,7 @@ async def execute(self, sql: str, *args: Any) -> str: conn = _StalledCommit([("UPDATE loom.tenancy_mode", "UPDATE 1")]) bounds: list[float] = [] - timeout = asyncio.timeout - - def bounded(seconds: float) -> Any: - bounds.append(seconds) - return timeout(0) - - monkeypatch.setattr(apg.asyncio, "timeout", bounded) + _short_phases(monkeypatch, bounds) marks: list[str] = [] async def confirm() -> None: @@ -773,8 +786,9 @@ async def ready() -> list[str]: timeout=0.2, ) assert "COMMIT" in conn.executed - assert conn.executed[-1] == "TERMINATE" - assert bounds == [bound_s] + assert conn.executed[-1] == "CLOSE" + assert "TERMINATE" not in conn.executed + assert bounds == pytest.approx([bound_s], abs=0.001) assert marks == ["unknown"] @@ -793,8 +807,7 @@ async def execute(self, sql: str, *args: Any) -> str: return result conn = _StalledRollback() - timeout = asyncio.timeout - monkeypatch.setattr(apg.asyncio, "timeout", lambda _s: timeout(0)) + _short_phases(monkeypatch) marks: list[str] = [] task = asyncio.create_task(apg._mode_try(conn, 100, marks.append)) done, _ = await asyncio.wait([task], timeout=0.2) @@ -804,7 +817,8 @@ async def execute(self, sql: str, *args: Any) -> str: await task assert task in done, "mode deadline is stuck awaiting rollback" assert isinstance(task.exception(), TimeoutError) - assert conn.executed[-1] == "TERMINATE" + assert conn.executed[-1] == "CLOSE" + assert "TERMINATE" not in conn.executed assert marks == (["unknown"] if stall == "UPDATE" else []) @@ -846,7 +860,8 @@ async def execute(self, sql: str, *args: Any) -> str: assert task.cancelled() else: assert isinstance(task.exception(), ValueError) - assert conn.executed[-1] == "TERMINATE" + assert "CLOSE" not in conn.executed + assert "TERMINATE" not in conn.executed assert "ROLLBACK" not in conn.executed @@ -865,7 +880,8 @@ async def ready() -> list[str]: await apg.commit_enforced( conn, confirm, ready, apg.Wait(2, clock), lambda _t: None, marks.append ) - assert conn.executed[-1] == "TERMINATE" + assert conn.executed[-1] == "CLOSE" + assert "TERMINATE" not in conn.executed assert marks == [] @@ -1069,8 +1085,7 @@ async def ensure_table(conn: Any, table: str, **kwargs: Any) -> Any: conn, table, ["CREATE TRIGGER stub"], kwargs["lock_timeout"], 1, warn=False ) - timeout = asyncio.timeout - monkeypatch.setattr(apg.asyncio, "timeout", lambda _s: timeout(0)) + _short_phases(monkeypatch) monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) conn = _Stalled() task = asyncio.create_task(apg._ensure_one(conn, "scope_generations", 100, apg.Wait(0, clock))) @@ -1078,7 +1093,8 @@ async def ensure_table(conn: Any, table: str, **kwargs: Any) -> Any: try: assert task in done, f"trigger deadline is stuck at {stall}" assert isinstance(task.exception(), TimeoutError) - assert conn.executed[-1] == "TERMINATE" + assert conn.executed[-1] == "CLOSE" + assert "TERMINATE" not in conn.executed if stall in {"migration", "ddl", "commit", "rollback"}: assert apg._MIGRATION_UNLOCK not in conn.executed finally: @@ -1110,7 +1126,8 @@ async def ensure_table(*_a: Any, **_kw: Any) -> Any: ) with pytest.raises(ValueError, match=f"{failed if failed == 'migration' else 'DDL'} failed"): await apg._ensure_one(conn, "scope_generations", 100, apg.Wait(0, clock)) - assert conn.executed[-1] == "TERMINATE" + assert conn.executed[-1] == "CLOSE" + assert "TERMINATE" not in conn.executed @pytest.mark.asyncio diff --git a/tests/unit/test_cell_attach_cleanup.py b/tests/unit/test_cell_attach_cleanup.py new file mode 100644 index 0000000000..20e17ac786 --- /dev/null +++ b/tests/unit/test_cell_attach_cleanup.py @@ -0,0 +1,119 @@ +"""The attach owner preserves failures through bounded public-close cleanup.""" + +import argparse +import asyncio +import time +from unittest.mock import AsyncMock + +import pytest + +from server.cli import cell_attach +from server.cli import cell_attach_pg as apg +from server.core import pg + + +@pytest.mark.asyncio +async def test_failed_work_keeps_its_error_when_public_close_also_fails(): + events = [] + + class Conn: + async def close(self, **kwargs): + events.append("close") + raise RuntimeError("close failed") + + def terminate(self): + events.append("abort") + + async def work(): + raise ValueError("original work failure") + + with pytest.raises(ValueError, match="original work failure"): + await apg.supervise(Conn(), work(), end=time.monotonic() + 0.1) + assert events == ["close", "abort"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed", [False, True]) +async def test_second_cancellation_preserves_original_failure_and_closes_before_cancel(failed): + entered, closing = asyncio.Event(), asyncio.Event() + events = [] + + class Conn: + async def close(self, **kwargs): + events.append("close") + closing.set() + await asyncio.Future() + + def terminate(self): + events.append("abort") + + async def work(): + entered.set() + try: + if failed: + raise ValueError("original work failure") + await asyncio.Future() + finally: + events.append("work ended") + + task = asyncio.create_task(apg.supervise(Conn(), work(), end=time.monotonic() + 0.2)) + await entered.wait() + if not failed: + task.cancel("first cancellation") + await closing.wait() + task.cancel("second cancellation") + with pytest.raises( + ValueError if failed else asyncio.CancelledError, + match="original work failure" if failed else "first cancellation", + ): + await task + assert events.count("abort") >= 1 + if not failed: + assert events.index("close") < events.index("work ended") + + +@pytest.mark.asyncio +async def test_lock_owner_bounds_context_exit_inside_the_same_cleanup_allowance(monkeypatch): + conn = AsyncMock() + conn.is_closed = lambda: False + conn.terminate = lambda: None + cm = AsyncMock() + exit_started = asyncio.Event() + + async def exit(*_): + exit_started.set() + await asyncio.Future() + + cm.__aexit__.side_effect = exit + run = cell_attach.Run(None, argparse.Namespace(cell="test", wait=0), None, lambda _: None) + run._lock_cm, run.lock_conn = cm, conn + monkeypatch.setattr(pg, "_RELEASE_TIMEOUT_S", 0.03) + started = time.monotonic() + with pytest.raises(TimeoutError): + await run.close_lock() + assert time.monotonic() - started < 0.15 + assert exit_started.is_set() and run.lock_conn is None + conn.close.assert_awaited_once() + await asyncio.sleep(0) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed", [False, True]) +async def test_run_preserves_its_own_failure_without_hiding_normal_close_failure(failed): + router = AsyncMock() + router.call.return_value = {"state": "attaching"} + run = cell_attach.Run( + None, argparse.Namespace(cell="test", wait=0, dry_run=True), router, lambda _: None + ) + run.open_lock = AsyncMock(side_effect=ValueError("run failed") if failed else None) + run.read_own_row = AsyncMock() + run.refusals = AsyncMock(return_value=(None, None, None)) + run.dry_run = AsyncMock(return_value=0) + run.close_lock = AsyncMock(side_effect=RuntimeError("close failed")) + try: + raise LookupError("unrelated caller exception") + except LookupError: + with pytest.raises( + ValueError if failed else RuntimeError, match="run failed" if failed else "close failed" + ): + await run.run() From 1f69c41935eec5259229393806bdd46bc8d6f758 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 23:15:27 +0000 Subject: [PATCH 028/114] Supervise native cell control cleanup before cancelling work Signed-off-by: zlareb1 --- docs/architecture/cells.md | 9 + docs/operations/cell-control.md | 6 +- server/cli/cell.py | 8 +- server/cli/cell_control.py | 215 +++++++++++++---- server/cli/cell_prepare.py | 38 ++- server/core/pg.py | 27 ++- .../integration/test_cell_control_cleanup.py | 216 ++++++++++++++++++ tests/unit/test_cell_attach.py | 3 + tests/unit/test_cell_control.py | 57 +++-- tests/unit/test_cell_prepare.py | 16 +- 10 files changed, 487 insertions(+), 108 deletions(-) create mode 100644 tests/integration/test_cell_control_cleanup.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 59ed6fef64..d5f24fdabc 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1248,6 +1248,15 @@ cleanup have explicit bounds; an interrupted or uncertain local transaction neve continues on another connection. No operation record, persistent census, phase or lease is added. The UUID is coordination metadata and holds no memory content. +One ephemeral owner supervises the command's raw session and child work. Opening keeps +its 75-second bound and each local control transaction keeps its 60-second bound, +reserving cleanup time inside that bound. On failure it sends public connection close +before cancelling child work, so active statements and rollback cannot prevent bounded +cleanup. An unexpected transaction error propagates immediately; it never waits for +rollback on a session the owner will discard. Uncapped command work retains the existing +one-second session-exit allowance, which cannot extend an expired bounded phase. A +transport abort remains an unknown outcome, not proof that the server backend ended. + The source census for acceptance and operator directory writes is checked by `tests/unit/test_cell_control_census.py`, including dynamic cleanup tables, guarded entry points and the named exceptions: diff --git a/docs/operations/cell-control.md b/docs/operations/cell-control.md index b606d9aaf9..30892abf7f 100644 --- a/docs/operations/cell-control.md +++ b/docs/operations/cell-control.md @@ -89,9 +89,11 @@ async def main(): async def call(path, body): return await cell_control.request(url, settings.cell, None, path, body) - async with cell_control.session("state cleanup") as conn: + async def cleanup(owner): row = await cell_control.request(url, settings.cell, None, "", method="GET") - await cell_control.backout(conn, call, row) + await cell_control.backout(owner, call, row) + + await cell_control.run("state cleanup", cleanup) finally: await ch.disconnect() diff --git a/server/cli/cell.py b/server/cli/cell.py index 77b13e8aec..e43a5a4d07 100644 --- a/server/cli/cell.py +++ b/server/cli/cell.py @@ -366,7 +366,8 @@ async def _remove_locked( from server.cli import cell_control from server.core import cell - async with cell_control.session("remove") as conn: + async def remove(owner: cell_control.Owner) -> dict[str, Any]: + conn = owner.conn async def call(path: str, body: dict[str, Any]) -> dict[str, Any]: return await cell_control.request(router, args.cell, transport, path, body) @@ -377,8 +378,7 @@ async def call(path: str, body: dict[str, Any]) -> dict[str, Any]: raise cell_control._refused(reason) await cell_control.paired(conn, row) if row["is_first"]: - async with cell_control.transaction(conn): - state, _ = await cell.read_state_if_present(conn) + state, _ = await owner.transaction(lambda: cell.read_state_if_present(conn)) if state != "open": raise cell_control._refused( f"the first cell's Postgres reads {state}: run the full-state cleanup " @@ -390,6 +390,8 @@ async def call(path: str, body: dict[str, Any]) -> dict[str, Any]: router, args.cell, transport, "", cell_control.preconditions(row), method="DELETE" ) + return await cell_control.run("remove", remove) + def _cmd_which(args: argparse.Namespace) -> int: router = parse_router(args) diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index 3c03a75e2a..84661747fd 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -4,14 +4,16 @@ import asyncio import json -from collections.abc import AsyncIterator, Awaitable, Callable -from contextlib import asynccontextmanager -from typing import Any +import time +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Any, cast from uuid import UUID from server.cli import cell_attach_pg as apg from server.cli.cell_ch import Refusal from server.cli.cell_ch import Refused as Refused +from server.core import pg from server.router.directory import CONTROL_FIELDS, CONTROL_MAX_S from server.settings import settings @@ -95,53 +97,164 @@ async def mutate( return answer -@asynccontextmanager -async def transaction(conn: Any) -> AsyncIterator[Any]: - """Check, write and durably commit on the lock backend; cancellation ends ownership.""" - settled = False - tx = conn.transaction() +@dataclass(frozen=True) +class _Fatal: + error: SystemExit | KeyboardInterrupt + + +class _ForwardFatal(BaseException): + def __init__(self, fatal: _Fatal) -> None: + self.fatal = fatal + + +async def _captured[T](work: Callable[[], Awaitable[T]]) -> T | _Fatal: try: - async with asyncio.timeout(TRANSACTION_S): - await tx.start() + return await work() + except (SystemExit, KeyboardInterrupt) as error: + return _Fatal(error) + except _ForwardFatal as error: + return error.fatal + + +class Owner: + """One raw backend and one cleanup owner for a native control command.""" + + def __init__(self) -> None: + self.conn: Any = None + self.phase_end: float | None = None + self.close_end: float | None = None + self.closed = False + self._close_task: asyncio.Task[Any] | None = None + self._tasks: set[asyncio.Task[Any]] = set() + + def _start[T](self, work: Callable[[], Awaitable[T]]) -> asyncio.Task[T | _Fatal]: + task = asyncio.create_task(_captured(work)) + self._tasks.add(task) + task.add_done_callback(self._tasks.discard) + task.add_done_callback(pg._retrieve_task) + return task + + async def _open(self, what: str) -> None: + self.conn = await pg._connect_dedicated(statement_timeout_ms=60_000) + if self.closed: + # A connector returning after cancellation cannot publish a usable session. + await pg.close_owned(self.conn, end=self.close_end or 0) + raise _refused("connection arrived after the control attempt ended") + try: + await pg._check_deployment(self.conn) + await pg.init_connection(self.conn) + except Exception as error: + pg._raise_if_unavailable(error) + raise + refused = await apg.direct_refusals(self.conn, what) + refused += await apg.encoding_refusals(self.conn) + if refused: + raise Refused(refused) + await apg.take_lock(self.conn, what) + + async def close(self) -> None: + if not self.closed: + self.closed = True + self.close_end = self.phase_end or (time.monotonic() + pg._RELEASE_TIMEOUT_S) + if self.conn is None: + return + if self._close_task is None: + self._close_task = asyncio.create_task( + _captured(lambda: pg.close_owned(self.conn, end=cast(float, self.close_end))) + ) + self._close_task.add_done_callback(pg._retrieve_task) + try: + done, _ = await asyncio.wait( + {self._close_task}, timeout=max(0, cast(float, self.close_end) - time.monotonic()) + ) + if not done: + raise TimeoutError("cell control cleanup deadline expired; outcome unknown") + result = self._close_task.result() + if isinstance(result, _Fatal): + raise _ForwardFatal(result) + except BaseException: + pg._terminate(self.conn) + if not self._close_task.done(): + self._close_task.cancel() + raise + + async def _stop(self, tasks: set[asyncio.Task[Any]]) -> None: + for task in tasks: + if not task.done(): + task.cancel() + try: + if tasks: + await asyncio.wait(tasks, timeout=max(0, (self.close_end or 0) - time.monotonic())) + except BaseException: # noqa: BLE001 — cleanup cannot replace the original failure. + if self.conn is not None: + pg._terminate(self.conn) + + async def call[T](self, work: Callable[[], Awaitable[T]], *, total_s: float) -> T: + if self.closed or self.phase_end is not None: + raise _refused("a closed or active control phase cannot start another phase") + self.phase_end = time.monotonic() + total_s + task = self._start(work) + try: + done, _ = await asyncio.wait( + {task}, + timeout=max(0, self.phase_end - min(1.0, total_s / 4) - time.monotonic()), + ) + if not done: + raise TimeoutError( + "cell control phase deadline expired; outcome unknown; stop and rerun" + ) + result = task.result() + if isinstance(result, _Fatal): + raise _ForwardFatal(result) + except BaseException: try: - await conn.execute("SET LOCAL lock_timeout = '5s'") - await conn.execute("SET LOCAL statement_timeout = '10s'") - await check_lock(conn) - await conn.execute("SELECT pg_logical_emit_message(true, 'loom.control', '')") - yield conn - except Exception: - await tx.rollback() - settled = True - raise - else: - await tx.commit() - settled = True - finally: - if not settled: - # This includes cancellation during COMMIT/rollback. A later session is a new run. - conn.terminate() + await self.close() + except BaseException: # noqa: BLE001 — keep the original phase failure/cancellation. + pass + await self._stop({task}) + raise + self.phase_end = None + return result + async def transaction[T](self, work: Callable[[], Awaitable[T]]) -> T: + return await self.call(lambda: transaction(self.conn, work), total_s=TRANSACTION_S) -@asynccontextmanager -async def session(what: str) -> AsyncIterator[Any]: - """A direct, marked, UTF8 session holding ATTACH for its entire control run.""" - from server.core import pg - cm = pg.dedicated(statement_timeout_ms=60_000) - conn = None +async def run[T](what: str, work: Callable[[Owner], Awaitable[T]]) -> T: + """Run at the command boundary; fatal guard exceptions leave only after cleanup.""" + owner = Owner() + try: + await owner.call(lambda: owner._open(what), total_s=OPEN_S) + result = await asyncio.shield(owner._start(lambda: work(owner))) + if isinstance(result, _Fatal): + raise _ForwardFatal(result) + except BaseException as original: + try: + await owner.close() + except BaseException: # noqa: BLE001 — preserve the command's original failure. + pass + await owner._stop(set(owner._tasks)) + if isinstance(original, _ForwardFatal): + raise original.fatal.error from None + raise try: - async with asyncio.timeout(OPEN_S): - conn = await cm.__aenter__() - refused = await apg.direct_refusals(conn, what) - refused += await apg.encoding_refusals(conn) - if refused: - raise Refused(refused) - await apg.take_lock(conn, what) - yield conn + await owner.close() finally: - if conn is not None: - # Use dedicated()'s bounded close; this does not prove the server backend ended. - await cm.__aexit__(None, None, None) + await owner._stop(set(owner._tasks)) + return result + + +async def transaction[T](conn: Any, work: Callable[[], Awaitable[T]]) -> T: + """Borrow the owner's backend; every failure discards it without a rollback wait.""" + tx = conn.transaction() + await tx.start() + await conn.execute("SET LOCAL lock_timeout = '5s'") + await conn.execute("SET LOCAL statement_timeout = '10s'") + await check_lock(conn) + await conn.execute("SELECT pg_logical_emit_message(true, 'loom.control', '')") + result = await work() + await tx.commit() + return result async def request( @@ -196,12 +309,12 @@ async def _acceptance(conn: Any) -> dict[str, Any]: } -async def backout(conn: Any, call: Call, row: dict[str, Any]) -> None: +async def backout(owner: Owner, call: Call, row: dict[str, Any]) -> None: """The runbook's exact state cleanup, fenced against queued controllers/placements.""" from server.core.schema import pg_migrate - async with transaction(conn): - before = await _acceptance(conn) + conn = owner.conn + before = await owner.transaction(lambda: _acceptance(conn)) first = before["first"] if first is not None and (first["cell"], first["is_first"]) != (row["cell"], row["is_first"]): raise _refused("the local first row names another registration") @@ -218,7 +331,8 @@ async def backout(conn: Any, call: Call, row: dict[str, Any]) -> None: limited = await mutate(conn, call, row, "/limit", {"closed": True}) if type(limited.get("placed_orgs")) is not int or limited["placed_orgs"] != 0: raise _refused("cell holds placements: cleanup refused; the cell remains closed") - async with transaction(conn): + + async def clear_state() -> None: await paired(conn, row) if json.dumps(await _acceptance(conn), sort_keys=True) != json.dumps( before, sort_keys=True @@ -232,9 +346,12 @@ async def backout(conn: Any, call: Call, row: dict[str, Any]) -> None: await conn.execute( "ALTER TABLE loom.cell_first ENABLE ALWAYS TRIGGER cell_first_no_truncate" ) + + await owner.transaction(clear_state) + # Once state is open, a migrating boot cannot recreate the org triggers. The same # session still holds ATTACH; a retry repeats the durable state cleanup first. - async with transaction(conn): + async def clear_triggers() -> None: await conn.execute("SELECT pg_advisory_xact_lock($1)", pg_migrate._LOCK_KEY) triggers = await conn.fetch( "SELECT t.tgname, c.relname FROM pg_trigger t JOIN pg_class c ON c.oid = t.tgrelid " @@ -244,3 +361,5 @@ async def backout(conn: Any, call: Call, row: dict[str, Any]) -> None: for trigger in triggers: name, table = (str(trigger[k]).replace('"', '""') for k in ("tgname", "relname")) await conn.execute(f'DROP TRIGGER "{name}" ON loom."{table}"') + + await owner.transaction(clear_triggers) diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index c4ac332757..6e94ae1058 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -26,13 +26,11 @@ import argparse import asyncio -import contextlib import hashlib import json import secrets import sys import time -from collections.abc import AsyncIterator from dataclasses import dataclass from typing import Any @@ -388,7 +386,8 @@ async def _write_rows( from `current` and `nxt`, in one transaction; the rows written.""" is_first = bool(answer.get("is_first")) written = [] - async with cell_control.transaction(conn): + + async def write() -> list[str]: # A state CAS does not need to count every tenancy row again. fresh = await _read_rows(conn, count_tenancy=False) before = (rows.deployment_id, rows.first, rows.mode, rows.current, rows.next) @@ -418,7 +417,9 @@ async def _write_rows( nxt, ) written.append("own") - return written + return written + + return await cell_control.transaction(conn, write) def _answer_problem(rows: _CellRows, answer: dict[str, Any], beats: list[_Beat]) -> str: @@ -457,19 +458,13 @@ async def prepare( router: httpx.URL, transport: httpx.AsyncBaseTransport | None = None, ) -> int: - """Prepare the cell, holding the attach lock for the run (`attach_lock`).""" - async with attach_lock() as conn: - return await _prepare_locked(ch, args, router, transport, conn) - - -@contextlib.asynccontextmanager -async def attach_lock() -> AsyncIterator[Any]: - """Hand the exact lock backend to every acceptance-state read and write.""" + """Prepare on one independently supervised ATTACH-lock backend.""" try: - async with cell_control.session("prepare") as conn: - yield conn - except cell_control.Refused as e: - raise Refused(str(e)) from None + return await cell_control.run( + "prepare", lambda owner: _prepare_locked(ch, args, router, transport, owner) + ) + except cell_control.Refused as error: + raise Refused(str(error)) from None async def _prepare_locked( @@ -477,10 +472,10 @@ async def _prepare_locked( args: argparse.Namespace, router: httpx.URL, transport: httpx.AsyncBaseTransport | None, - conn: Any, + owner: cell_control.Owner, ) -> int: - async with cell_control.transaction(conn): - rows = await _read_rows(conn) + conn = owner.conn + rows = await owner.transaction(lambda: _read_rows(conn)) if not rows.deployment_id: raise Refused("this cell's Postgres records no deployment id: start the cell once first") if rows.first is not None and rows.first[1] != settings.cell: @@ -527,7 +522,10 @@ async def call(path: str, body: dict[str, Any]) -> dict[str, Any]: if problem := _answer_problem(rows, snapshot, beats): raise Refused(problem) current, nxt = _own_rows(rows, snapshot, every) - written = await _write_rows(conn, rows, answer, current, nxt) + written = await owner.call( + lambda: _write_rows(conn, rows, answer, current, nxt), + total_s=cell_control.TRANSACTION_S, + ) print( json.dumps( { diff --git a/server/core/pg.py b/server/core/pg.py index 5daeb33a55..d7bdf5b3e4 100644 --- a/server/core/pg.py +++ b/server/core/pg.py @@ -462,20 +462,13 @@ def may_have_committed(e: BaseException) -> bool: return isinstance(e, PostgresUnavailableError) and bool(e.extra.get("outcome_unknown")) -@asynccontextmanager -async def dedicated(*, statement_timeout_ms: int = 0, marked: bool = True) -> AsyncIterator[Any]: - """A connection of its own, outside the pool, with no client timeout and no server - statement timeout unless `statement_timeout_ms` asks for one: for work that runs as - long as it takes (migrations, an erasure's deletes). - A CLI process's first one checks the deployment (`expect_deployment`). - `statement_timeout_ms` bounds its statements as a startup value (a RESET ALL keeps - it); `marked` False leaves the tenancy marker out (`loom doctor` reads a session - without it).""" +async def _connect_dedicated(*, statement_timeout_ms: int = 0, marked: bool = True) -> Any: + """Return the raw dedicated session before its owner initializes it.""" import asyncpg _require_dsn() try: - conn = await asyncpg.connect( + return await asyncpg.connect( settings.pg_dsn, command_timeout=None, server_settings={ @@ -492,6 +485,20 @@ async def dedicated(*, statement_timeout_ms: int = 0, marked: bool = True) -> As except Exception as e: _raise_if_unavailable(e) raise + + +@asynccontextmanager +async def dedicated(*, statement_timeout_ms: int = 0, marked: bool = True) -> AsyncIterator[Any]: + """A connection of its own, outside the pool, with no client timeout and no server + statement timeout unless `statement_timeout_ms` asks for one: for work that runs as + long as it takes (migrations, an erasure's deletes). + A CLI process's first one checks the deployment (`expect_deployment`). + `statement_timeout_ms` bounds its statements as a startup value (a RESET ALL keeps + it); `marked` False leaves the tenancy marker out (`loom doctor` reads a session + without it).""" + import asyncpg + + conn = await _connect_dedicated(statement_timeout_ms=statement_timeout_ms, marked=marked) # Opening includes the deployment check and the codecs (jsonb reads back as a dict, as # the pool's): an outage there is the 503; the block's own errors pass through as # they are (an erasure's deadlock reaches its retry). diff --git a/tests/integration/test_cell_control_cleanup.py b/tests/integration/test_cell_control_cleanup.py new file mode 100644 index 0000000000..2a37ca6170 --- /dev/null +++ b/tests/integration/test_cell_control_cleanup.py @@ -0,0 +1,216 @@ +"""Native cell-control interruption closes its actual Postgres backend and locks.""" + +from __future__ import annotations + +import argparse +import asyncio +import time +from collections.abc import AsyncIterator +from typing import Any + +import httpx +import pytest + +from server.cli import cell_control, cell_prepare +from server.core import pg +from server.settings import settings +from tests.integration._pg_database import own_database +from tests.integration.test_pg_owned_cleanup import released, sleeping + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +async def native_database(monkeypatch) -> AsyncIterator[Any]: + import asyncpg + + async with own_database() as dsn: + monkeypatch.setattr(settings, "pg_dsn", dsn) + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(pg, "_expected", None) + observer = await asyncpg.connect(dsn) + await observer.execute( + "CREATE TABLE probe(id int PRIMARY KEY); INSERT INTO probe VALUES(1)" + ) + try: + yield observer + finally: + await observer.close(timeout=1) + + +@pytest.mark.parametrize("interrupt", ["cancel", "deadline"]) +async def test_native_prepare_interrupt_releases_backend_and_locks( + native_database, monkeypatch, interrupt +): + observed = asyncio.Event() + backend: dict[str, Any] = {} + + async def read(conn, **_): + backend["pid"] = conn.get_server_pid() + observed.set() + await conn.execute("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10)") + + monkeypatch.setattr(cell_prepare, "_read_rows", read) + if interrupt == "deadline": + monkeypatch.setattr(cell_control, "TRANSACTION_S", 0.4) + task = asyncio.create_task( + cell_prepare.prepare(None, argparse.Namespace(), httpx.URL("http://router")) + ) + await asyncio.wait_for(observed.wait(), 3) + await sleeping(native_database, backend["pid"]) + began = time.monotonic() + if interrupt == "cancel": + task.cancel("native prepare cancelled") + with pytest.raises(asyncio.CancelledError if interrupt == "cancel" else TimeoutError): + await asyncio.wait_for(task, 1.5) + assert time.monotonic() - began < 1.5 + await released(native_database, backend["pid"]) + + +async def test_native_prepare_body_failure_is_not_hidden_by_rollback(native_database, monkeypatch): + import asyncpg + + execute = asyncpg.Connection.execute + original = ValueError("original native transaction failure") + backend = {} + + async def read(conn, **_): + backend["pid"] = conn.get_server_pid() + await conn.execute("SELECT * FROM probe FOR UPDATE") + raise original + + async def stalled_rollback(self, query, *args, **kwargs): + if query.startswith("ROLLBACK"): + await execute(self, "SELECT pg_sleep(10)") + return await execute(self, query, *args, **kwargs) + + monkeypatch.setattr(asyncpg.Connection, "execute", stalled_rollback) + monkeypatch.setattr(cell_prepare, "_read_rows", read) + monkeypatch.setattr(cell_control, "TRANSACTION_S", 0.4) + began = time.monotonic() + with pytest.raises(ValueError, match="original native transaction failure") as captured: + await asyncio.wait_for( + cell_prepare.prepare(None, argparse.Namespace(), httpx.URL("http://router")), 1.5 + ) + assert captured.value is original + assert time.monotonic() - began < 0.4 + await released(native_database, backend["pid"]) + + +async def test_native_prepare_second_cancel_preserves_first_and_releases_locks( + native_database, monkeypatch +): + import asyncpg + + real_close = asyncpg.Connection.close + observed, close_started, close_return = asyncio.Event(), asyncio.Event(), asyncio.Event() + backend = {} + + async def read(conn, **_): + backend["conn"], backend["pid"] = conn, conn.get_server_pid() + observed.set() + await conn.execute("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10)") + + async def delayed_close(self, **kwargs): + await real_close(self, **kwargs) + if self is backend.get("conn"): + close_started.set() + await close_return.wait() + + monkeypatch.setattr(cell_prepare, "_read_rows", read) + monkeypatch.setattr(asyncpg.Connection, "close", delayed_close) + task = asyncio.create_task( + cell_prepare.prepare(None, argparse.Namespace(), httpx.URL("http://router")) + ) + try: + await observed.wait() + await sleeping(native_database, backend["pid"]) + task.cancel("first native cancellation") + await asyncio.wait_for(close_started.wait(), 1) + task.cancel("second cancellation") + with pytest.raises(asyncio.CancelledError, match="first native cancellation"): + await asyncio.wait_for(task, 0.5) + await released(native_database, backend["pid"]) + finally: + close_return.set() + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("stage", ["open", "phase", "body"]) +@pytest.mark.parametrize("kind", ["org", "keyboard"]) +async def test_native_fatal_guard_is_inert_until_backend_closed(native_database, stage, kind): + import os + import sys + + # A separate interpreter keeps a regression in child fatal-exception transport + # from stopping pytest's event loop. Proof runs before the native command exits. + from tests.integration._pg_database import with_database + + base = os.environ["LOOM_PG_DSN"] + dsn = with_database(base, await native_database.fetchval("SELECT current_database()")) + script = """ +import argparse, asyncio, json, os, sys +import asyncpg, httpx +from server.cli import cell_control, cell_prepare +from server.cli._ch import OrgNotHeld +from server.core import pg +from server.settings import settings +stage, kind = sys.argv[1:] +settings.pg_dsn = os.environ['LOOM_NATIVE_CLEANUP_DSN'] +settings.cell = 'first' +pg._expected = None +original = OrgNotHeld('org refused') if kind == 'org' else KeyboardInterrupt('guard interrupted') +backend = {} +real_init = pg.init_connection +async def initialize(conn): + await real_init(conn) + backend['pid'] = conn.get_server_pid() + if stage == 'open': + raise original +async def read(conn, **kwargs): + backend['pid'] = conn.get_server_pid() + await conn.execute('SELECT * FROM probe FOR UPDATE') + if stage == 'phase': + raise original + return cell_prepare._CellRows('dep-first', None, '', None, None, 0) +async def beats(ch): + raise original +pg.init_connection = initialize +cell_prepare._read_rows = read +cell_prepare._read_beats = beats +async def main(): + observer = await asyncpg.connect(settings.pg_dsn) + try: + try: + await cell_prepare.prepare(None, argparse.Namespace(), httpx.URL('http://router')) + except BaseException as found: + assert found is original + assert not await observer.fetchval('SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1)', backend['pid']) + assert not await observer.fetchval('SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)', backend['pid']) + print(json.dumps({'original': True, 'backend_absent': True})) + else: + raise AssertionError('fatal guard became success') + finally: + await observer.close(timeout=1) +asyncio.run(main()) +""" + child = await asyncio.create_subprocess_exec( + sys.executable, + "-c", + script, + stage, + kind, + env={**os.environ, "LOOM_NATIVE_CLEANUP_DSN": dsn}, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, _stderr = await asyncio.wait_for(child.communicate(), 10) + assert child.returncode == 0, "native fatal exception interrupted its supervisor" + assert b'"backend_absent": true' in stdout + finally: + if child.returncode is None: + child.kill() + await child.wait() diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index d573d37423..65feba889d 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -518,6 +518,9 @@ async def __aexit__(self, *exc: Any) -> None: return _Tx() + async def close(self, *, timeout: float) -> None: + self.executed.append("CLOSE") + def terminate(self) -> None: self.executed.append("TERMINATE") diff --git a/tests/unit/test_cell_control.py b/tests/unit/test_cell_control.py index 71d687baf9..74ac1c130d 100644 --- a/tests/unit/test_cell_control.py +++ b/tests/unit/test_cell_control.py @@ -14,45 +14,58 @@ @pytest.mark.asyncio -@pytest.mark.parametrize("stall", ["COMMIT", "ROLLBACK"]) -async def test_uncertain_transaction_ends_owned_session_without_unbounded_cleanup( - monkeypatch, stall -): +async def test_uncertain_commit_closes_before_cancelling_work(monkeypatch): class Stalled(FakeConn): async def execute(self, sql: str, *args: Any) -> str: answer = await super().execute(sql, *args) - if sql == stall: - await asyncio.Future() + if sql == "COMMIT": + try: + await asyncio.Future() + finally: + self.executed.append("WORK ENDED") return answer conn = Stalled() - monkeypatch.setattr(cell_control, "TRANSACTION_S", 0.01) + owner = cell_control.Owner() + owner.conn = conn + monkeypatch.setattr(cell_control, "TRANSACTION_S", 0.04) with pytest.raises(TimeoutError): - async with cell_control.transaction(conn): - if stall == "ROLLBACK": - raise ValueError("refuse") - assert conn.executed[-1] == "TERMINATE" - assert stall in conn.executed + await owner.transaction(AsyncMock()) + assert conn.executed.index("CLOSE") < conn.executed.index("WORK ENDED") + assert "TERMINATE" not in conn.executed and "ROLLBACK" not in conn.executed assert any("pg_logical_emit_message" in sql for sql in conn.executed) @pytest.mark.asyncio -async def test_cancellation_during_body_never_waits_for_rollback(): +async def test_failure_never_starts_rollback_or_replaces_original(): conn = FakeConn() + owner = cell_control.Owner() + owner.conn = conn + original = ValueError("original failure") + with pytest.raises(ValueError) as found: + await owner.transaction(AsyncMock(side_effect=original)) + assert found.value is original + assert conn.executed[-1] == "CLOSE" + assert "ROLLBACK" not in conn.executed and "TERMINATE" not in conn.executed + + +@pytest.mark.asyncio +async def test_cancellation_during_body_closes_before_work_cancel(): + conn = FakeConn() + owner = cell_control.Owner() + owner.conn = conn entered = asyncio.Event() - async def work(): - async with cell_control.transaction(conn): - entered.set() - await asyncio.Future() + async def body(): + entered.set() + await asyncio.Future() - task = asyncio.create_task(work()) + task = asyncio.create_task(owner.transaction(body)) await entered.wait() - task.cancel() - with pytest.raises(asyncio.CancelledError): + task.cancel("original cancellation") + with pytest.raises(asyncio.CancelledError, match="original cancellation"): await task - assert conn.executed[-1] == "TERMINATE" - assert "ROLLBACK" not in conn.executed + assert conn.executed[-1] == "CLOSE" and "ROLLBACK" not in conn.executed def snapshot(): diff --git a/tests/unit/test_cell_prepare.py b/tests/unit/test_cell_prepare.py index 3071dddccf..58040a584d 100644 --- a/tests/unit/test_cell_prepare.py +++ b/tests/unit/test_cell_prepare.py @@ -5,7 +5,6 @@ from __future__ import annotations import argparse -import contextlib import json from datetime import UTC, datetime from typing import Any @@ -31,7 +30,19 @@ def _cell(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(settings, "pg_dsn", "postgresql://cell.internal/loom") monkeypatch.setattr(settings, "router_operator_token", OPERATOR) # The attach lock's session is Postgres's (tests/integration/test_cell_attach.py). - monkeypatch.setattr(cell_prepare, "attach_lock", contextlib.nullcontext) + class Owner: + conn = None + + async def transaction(self, work): + return await work() + + async def call(self, work, **_): + return await work() + + async def run(_what, work): + return await work(Owner()) + + monkeypatch.setattr(cell_control, "run", run) def _args(**over: Any) -> argparse.Namespace: @@ -395,7 +406,6 @@ async def _write(_conn: Any, _rows: Any, _answer: Any, current: Any, nxt: Any) - ("_write_rows", _write), ): monkeypatch.setattr(cell_prepare, name, fake) - monkeypatch.setattr(cell_control, "transaction", lambda _conn: contextlib.nullcontext()) directory_mode = ["recording"] From 5f76257d3d34ca983958c36446b265e7579fa8bc Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 23:12:55 +0000 Subject: [PATCH 029/114] Close attach sessions before cancelling transaction work Supervise bounded Postgres phases independently, reserving cleanup inside each existing deadline. Send public close while query and transaction work are alive, then cancel and reap within that same bound. Borrowed checks and trigger DDL leave failed-session cleanup to their owner. Bound the lock owner's close and preserve its original failure across cleanup errors or repeated cancellation. Keep wait floors and normal busy retries. Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 1 + docs/architecture/cells.md | 9 +- docs/operations/cell-attach.md | 5 + server/cli/cell_attach.py | 32 ++++- server/cli/cell_attach_pg.py | 109 +++++++++++----- server/core/cell_triggers.py | 10 +- tests/integration/test_cell_attach_cleanup.py | 111 ++++++++++++++++ tests/unit/test_cell_attach.py | 53 +++++--- tests/unit/test_cell_attach_cleanup.py | 119 ++++++++++++++++++ 9 files changed, 386 insertions(+), 63 deletions(-) create mode 100644 tests/integration/test_cell_attach_cleanup.py create mode 100644 tests/unit/test_cell_attach_cleanup.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e866c99a5..3066d67299 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -901,6 +901,7 @@ jobs: tests/integration/test_cell_unset_gate_pg.py tests/integration/test_cell_prepare.py tests/integration/test_cell_attach.py + tests/integration/test_cell_attach_cleanup.py tests/integration/test_cell_org_triggers_pg.py - name: pytest online-index durability on a private physical standby timeout-minutes: 8 diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index d5f24fdabc..70a203783a 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1067,7 +1067,14 @@ Then, in order (the step numbers are the ones the rest of this page uses): lock on `loom.tenancy_mode`, 2 s a try within `--wait`, the attach lock confirmed and the cheap heartbeat refusals run before each try (a booting process skips the try). Each try, including COMMIT, has a client deadline: the lock wait plus a 3 s statement margin and - a 5 s standby allowance. A commit past it has an unknown outcome until the durable read. + a 5 s standby allowance. Each mode or trigger try reserves the smaller of one second + and one quarter of its original client bound for cleanup. Its independent owner + closes the session through the public driver API while query/transaction work is + still alive, then cancels and reaps that work within the same absolute deadline. + Borrowed checks and trigger DDL leave failed-session cleanup to that owner. Close + failure or exhausted cleanup time aborts the transport and preserves the original + failure; abort alone does not prove that server locks disappeared. A commit past + its deadline has an unknown outcome until the durable read. After a failed try, the mode transaction an earlier run left in flight is found by its lock (only attach takes `ExclusiveLock` there; a relation's OID is matched in this database alone) and ended. From the commit on the mode only rises: every later refusal leaves the cell diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index e674da2997..13c65405f6 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -92,6 +92,11 @@ one try of each wait with no retry. The final heartbeat checks get at least two plus the router timeout, with one extension for a process first seen booting; this floor is not charged to `--wait`. `--scan-budget` (default 7200 s) bounds each full scan, and step 2 must take at most half of it, since step 4 reads the same tables again. +Each Postgres mode or trigger try reserves cleanup time inside its client deadline. +On interruption its owner closes the session before cancelling transaction work. +A close that cannot complete uses transport abort and reports failure; that abort +does not establish that server locks or an unacknowledged commit have disappeared. +Retry the command to resolve an unknown commit through its durable state checks. A refusal names what to fix. Rerun attach after fixing it: every step is idempotent, and a rerun resumes by the directory row and the cell's mode. On a `serving` row it prints that it diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index b3cb44d9cf..ed57bb88da 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -170,9 +170,29 @@ async def open_lock(self) -> None: self.lock_conn = await self._lock_cm.__aenter__() async def close_lock(self) -> None: + from server.core import pg + if self._lock_cm is not None: - await self._lock_cm.__aexit__(None, None, None) + cm, conn = self._lock_cm, self.lock_conn self._lock_cm = self.lock_conn = None + closed = pg._lost(conn) + end = time.monotonic() + (0.0 if closed else pg._RELEASE_TIMEOUT_S) + try: + if not closed: + await pg.close_owned(conn, end=end) + finally: + task = asyncio.create_task(cm.__aexit__(None, None, None)) + task.add_done_callback(pg._retrieve_task) + try: + done, _ = await asyncio.wait({task}, timeout=max(0.0, end - time.monotonic())) + if not done: + task.cancel() + raise TimeoutError("attach lock cleanup deadline expired") + task.result() + except BaseException: + pg._terminate(conn) + task.cancel() + raise async def confirm_lock(self) -> None: """A lost backend ends the owner; a new run must claim and start again.""" @@ -588,6 +608,7 @@ async def run(self) -> int: await self.done() self.report.update(state="serving", done=True) return 0 + failure: BaseException | None = None try: await self.open_lock() await self.read_own_row() @@ -616,8 +637,15 @@ async def run(self) -> int: answer = await self.serving() self.report.update(state=answer.get("state"), placed_orgs=answer.get("placed_orgs")) return 0 + except BaseException as error: + failure = error + raise finally: - await self.close_lock() + try: + await self.close_lock() + except BaseException: + if failure is None: + raise async def precommit(self, found: cell_scan.ScanResult) -> None: from server.core import pg_profile diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 8ae6436f3f..d24f749996 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -13,7 +13,7 @@ import contextlib import json import time -from collections.abc import Awaitable, Callable, Iterable, Iterator +from collections.abc import Awaitable, Callable, Coroutine, Iterable, Iterator from dataclasses import dataclass from typing import Any @@ -90,6 +90,50 @@ async def pause(self, seconds: float = POLL_S) -> bool: return True +@dataclass +class _Interrupted: + error: BaseException + + +async def _work_result[T](work: Coroutine[Any, Any, T]) -> T | _Interrupted: + try: + return await work + except (KeyboardInterrupt, SystemExit) as error: + # These exceptions otherwise stop asyncio's loop before the owner can close. + return _Interrupted(error) + + +async def supervise[T](conn: Any, work: Coroutine[Any, Any, T], *, end: float) -> T: + """Bound owned work, reserving cleanup inside its original phase deadline. + + A failed borrowed transaction must remain alive until public close sends its + cancellation. Closing the transport first can leave its backend locks held. + """ + task = asyncio.create_task(_work_result(work)) + reserve = min(1.0, max(0.0, end - time.monotonic()) / 4) + try: + done, _ = await asyncio.wait({task}, timeout=max(0.0, end - reserve - time.monotonic())) + if not done: + raise TimeoutError("attach Postgres phase deadline expired") + result = task.result() + if isinstance(result, _Interrupted): + raise result.error + return result + except BaseException: + try: + await pg.close_owned(conn, end=end) + except BaseException: # noqa: BLE001 — preserve the work failure or cancellation. + pass + if not task.done(): + task.cancel() + task.add_done_callback(pg._retrieve_task) + try: + await asyncio.wait({task}, timeout=max(0.0, end - time.monotonic())) + except BaseException: # noqa: BLE001 — repeated cancellation preserves the first failure. + pg._terminate(conn) + raise + + def _split(key: int) -> str: """The `pg_locks` columns that hold a bigint advisory key, in this database.""" return ( @@ -125,16 +169,11 @@ async def direct_refusals(conn: Any, what: str) -> list[Refusal]: outlive the CLI on the pooler's server connection, and attach's tenancy rows would be refused as an unchecked writer's.""" identities = [] - try: - for _ in range(2): - tx = conn.transaction() - await tx.start() - identities.append(await conn.fetchrow(_IDENTITY)) - await tx.commit() - except BaseException: - # A cancelled read must not wait for a transaction context's rollback. - conn.terminate() - raise + for _ in range(2): + tx = conn.transaction() + await tx.start() + identities.append(await conn.fetchrow(_IDENTITY)) + await tx.commit() first, second = identities out = [] if first is None or second is None or tuple(first) != tuple(second): @@ -538,30 +577,32 @@ async def _ensure_one(conn: Any, table: str, most_ms: int, wait: Wait) -> cell_t bound_s = ( wait.bound_ms(_MIGRATION_LOCK_MS) + wait.bound_ms(most_ms) ) / 1000 + _TRIGGER_TRY_MARGIN_S + return await supervise( + conn, _trigger_try(conn, table, most_ms, wait, _LOCK_KEY), end=time.monotonic() + bound_s + ) + + +async def _trigger_try( + conn: Any, table: str, most_ms: int, wait: Wait, key: int +) -> cell_triggers.Outcome: + await conn.execute(f"SET lock_timeout = '{wait.bound_ms(_MIGRATION_LOCK_MS)}ms'") try: - async with asyncio.timeout(bound_s): - await conn.execute(f"SET lock_timeout = '{wait.bound_ms(_MIGRATION_LOCK_MS)}ms'") - try: - await conn.execute(_MIGRATION_LOCK, _LOCK_KEY) - except Exception as e: - if getattr(e, "sqlstate", None) != "55P03": - raise - await conn.execute("RESET lock_timeout") - return cell_triggers.Outcome("busy", await _migration_holder(conn, _LOCK_KEY)) - await conn.execute("RESET lock_timeout") - outcome = await cell_triggers.ensure_table( - conn, - table, - recording_first=True, - lock_timeout=f"{wait.bound_ms(most_ms)}ms", - attempts=1, - ) - await conn.execute(_MIGRATION_UNLOCK, _LOCK_KEY) - return outcome - except BaseException: - # Closing the session releases its locks without asynchronous error cleanup. - conn.terminate() - raise + await conn.execute(_MIGRATION_LOCK, key) + except Exception as e: + if getattr(e, "sqlstate", None) != "55P03": + raise + await conn.execute("RESET lock_timeout") + return cell_triggers.Outcome("busy", await _migration_holder(conn, key)) + await conn.execute("RESET lock_timeout") + outcome = await cell_triggers.ensure_table( + conn, + table, + recording_first=True, + lock_timeout=f"{wait.bound_ms(most_ms)}ms", + attempts=1, + ) + await conn.execute(_MIGRATION_UNLOCK, key) + return outcome def _busy_refusal(table: str, holders: list[Any]) -> Refusal: diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 1899a46bab..68e74777ec 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -324,7 +324,8 @@ async def _run_ddl( two causes cells.md names kept it from applying (`busy`: its lock not taken, with the sessions it waited behind; `denied`: a table this role cannot alter), which a boot serves past; any other error fails it, as a failed read of the state does. `warn` logs - a busy table once, after its last try, with its DDL and holders (a boot's `ensure`).""" + a busy table once, after its last try, with its DDL and holders (a boot's `ensure`). + The borrowing caller owns failed-session cleanup, including cancellation/rollback.""" import asyncpg for attempt in range(attempts): @@ -344,7 +345,6 @@ async def _run_ddl( return Outcome("applied") except asyncpg.exceptions.InsufficientPrivilegeError as e: if not rolled_back: - conn.terminate() raise log.warning( "cell triggers: not applied on loom.%s (%s): this role cannot alter it", table, e @@ -352,7 +352,6 @@ async def _run_ddl( return Outcome("denied") except asyncpg.exceptions.LockNotAvailableError as e: if not rolled_back: - conn.terminate() raise # Short tries outlast a short transaction; an autovacuum is left to finish. if attempt < attempts - 1: @@ -368,11 +367,6 @@ async def _run_ddl( held, ) return Outcome("busy", held) - except BaseException: - if not rolled_back: - # Cancellation must not wait for another statement on this session. - conn.terminate() - raise return Outcome("busy") diff --git a/tests/integration/test_cell_attach_cleanup.py b/tests/integration/test_cell_attach_cleanup.py new file mode 100644 index 0000000000..145be1188d --- /dev/null +++ b/tests/integration/test_cell_attach_cleanup.py @@ -0,0 +1,111 @@ +"""Attach owners close active sessions before cancelling transaction work.""" + +import argparse +import asyncio +import contextlib +import time + +import pytest + +from server.cli import cell_attach +from server.cli import cell_attach_pg as apg +from server.core import cell_triggers, pg +from tests.integration.test_pg_owned_cleanup import database as database +from tests.integration.test_pg_owned_cleanup import released, sleeping + +pytestmark = pytest.mark.asyncio + + +@pytest.mark.parametrize( + "owner,phase", + [ + ("trigger", "query"), + ("trigger", "rollback"), + ("mode", "query"), + ("mode", "rollback"), + ("direct", "query"), + ], +) +@pytest.mark.parametrize("interruption", ["cancel", "deadline", "second-cancel"]) +async def test_attach_owner_releases_backend_locks_before_return( + database, monkeypatch, owner, phase, interruption +): + import asyncpg + + conn, observer = database + await conn.execute( + "CREATE SCHEMA loom; CREATE TABLE loom.tenancy_mode(mode text, since timestamptz); INSERT INTO loom.tenancy_mode VALUES('recording', now())" + ) + pid = conn.get_server_pid() + execute = asyncpg.Connection.execute + closing = asyncio.Event() + close = asyncpg.Connection.close + + async def intercepted(self, query, *args, **kwargs): + if self is conn: + if query.startswith("SET LOCAL statement_timeout"): + return await execute(self, "SET LOCAL statement_timeout = '10000ms'") + if query == "RAISE Python error": + raise ValueError("trigger statement failed") + if phase == "rollback" and query.startswith("ROLLBACK"): + await execute(self, "SELECT pg_sleep(10)") + if owner == "mode" and query.startswith("LOCK TABLE"): + await execute(self, "SELECT * FROM probe FOR UPDATE") + if phase == "rollback": + raise ValueError("mode statement failed") + return await execute(self, "SELECT pg_sleep(10)") + if owner == "direct" and query.startswith("BEGIN"): + await execute(self, query) + return await execute(self, "SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10)") + return await execute(self, query, *args, **kwargs) + + async def closing_session(self, *args, **kwargs): + was_open = not self.is_closed() + await close(self, *args, **kwargs) + if self is conn and was_open and interruption == "second-cancel": + closing.set() + await asyncio.Future() + + async def ensure(conn, table, **kwargs): + statements = ["SELECT * FROM probe FOR UPDATE"] + statements += ["RAISE Python error"] if phase == "rollback" else ["SELECT pg_sleep(10)"] + return await cell_triggers._run_ddl( + conn, table, statements, kwargs["lock_timeout"], 1, warn=False + ) + + @contextlib.asynccontextmanager + async def dedicated(**_): + yield conn + + monkeypatch.setattr(asyncpg.Connection, "execute", intercepted) + monkeypatch.setattr(asyncpg.Connection, "close", closing_session) + monkeypatch.setattr(cell_triggers, "ensure_table", ensure) + monkeypatch.setattr(apg, "_TRIGGER_TRY_MARGIN_S", 0.4) + monkeypatch.setattr(apg, "_MODE_STANDBY_S", 0.5) + monkeypatch.setattr(apg, "_MODE_STATEMENT_MARGIN_MS", 0) + monkeypatch.setattr(cell_attach, "_LOCK_STATEMENT_MS", 600) + monkeypatch.setattr(pg, "dedicated", dedicated) + if owner == "trigger": + work = apg._ensure_one(conn, "probe", 100, apg.Wait(0)) + elif owner == "mode": + work = apg._mode_try(conn, 100, lambda _: None) + else: + run = cell_attach.Run(None, argparse.Namespace(cell="test", wait=0), None, lambda _: None) + work = run.open_lock() + task = asyncio.create_task(work) + try: + await sleeping(observer, pid) + started = time.monotonic() + if interruption != "deadline": + task.cancel() + if interruption == "second-cancel": + await asyncio.wait_for(closing.wait(), 1) + task.cancel() + with pytest.raises(TimeoutError if interruption == "deadline" else asyncio.CancelledError): + await asyncio.wait_for(asyncio.shield(task), 1.2) + assert time.monotonic() - started < 0.7 + await released(observer, pid) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 65feba889d..0b3401be88 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -8,6 +8,7 @@ import argparse import asyncio import json +import time from collections.abc import Callable from dataclasses import dataclass, field from typing import Any @@ -524,6 +525,24 @@ async def close(self, *, timeout: float) -> None: def terminate(self) -> None: self.executed.append("TERMINATE") + async def close(self, **kwargs: Any) -> None: + assert kwargs["timeout"] > 0 + self.executed.append("CLOSE") + + def is_closed(self) -> bool: + return bool(self.executed and self.executed[-1] in {"CLOSE", "TERMINATE"}) + + +def _short_phases(monkeypatch: pytest.MonkeyPatch, bounds: list[float] | None = None) -> None: + supervise = apg.supervise + + async def bounded(conn: Any, work: Any, *, end: float) -> Any: + if bounds is not None: + bounds.append(end - time.monotonic()) + return await supervise(conn, work, end=time.monotonic() + 0.01) + + monkeypatch.setattr(apg, "supervise", bounded) + def _seq(*values: Any) -> Callable[..., Any]: pending = list(values) @@ -770,13 +789,7 @@ async def execute(self, sql: str, *args: Any) -> str: conn = _StalledCommit([("UPDATE loom.tenancy_mode", "UPDATE 1")]) bounds: list[float] = [] - timeout = asyncio.timeout - - def bounded(seconds: float) -> Any: - bounds.append(seconds) - return timeout(0) - - monkeypatch.setattr(apg.asyncio, "timeout", bounded) + _short_phases(monkeypatch, bounds) marks: list[str] = [] async def confirm() -> None: @@ -800,8 +813,9 @@ async def ready() -> list[str]: timeout=0.2, ) assert "COMMIT" in conn.executed - assert conn.executed[-1] == "TERMINATE" - assert bounds == [bound_s] + assert conn.executed[-1] == "CLOSE" + assert "TERMINATE" not in conn.executed + assert bounds == pytest.approx([bound_s], abs=0.001) assert marks == ["unknown"] @@ -820,8 +834,7 @@ async def execute(self, sql: str, *args: Any) -> str: return result conn = _StalledRollback() - timeout = asyncio.timeout - monkeypatch.setattr(apg.asyncio, "timeout", lambda _s: timeout(0)) + _short_phases(monkeypatch) marks: list[str] = [] task = asyncio.create_task( apg._mode_try(conn, 100, marks.append, expected_cell="first", expected_fp=(CURRENT, NEXT)) @@ -833,7 +846,8 @@ async def execute(self, sql: str, *args: Any) -> str: await task assert task in done, "mode deadline is stuck awaiting rollback" assert isinstance(task.exception(), TimeoutError) - assert conn.executed[-1] == "TERMINATE" + assert conn.executed[-1] == "CLOSE" + assert "TERMINATE" not in conn.executed assert marks == (["unknown"] if stall == "UPDATE" else []) @@ -875,7 +889,8 @@ async def execute(self, sql: str, *args: Any) -> str: assert task.cancelled() else: assert isinstance(task.exception(), ValueError) - assert conn.executed[-1] == "TERMINATE" + assert "CLOSE" not in conn.executed + assert "TERMINATE" not in conn.executed assert "ROLLBACK" not in conn.executed @@ -901,7 +916,8 @@ async def ready() -> list[str]: expected_cell="first", expected_fp=(CURRENT, NEXT), ) - assert conn.executed[-1] == "TERMINATE" + assert conn.executed[-1] == "CLOSE" + assert "TERMINATE" not in conn.executed assert marks == [] @@ -1106,8 +1122,7 @@ async def ensure_table(conn: Any, table: str, **kwargs: Any) -> Any: conn, table, ["CREATE TRIGGER stub"], kwargs["lock_timeout"], 1, warn=False ) - timeout = asyncio.timeout - monkeypatch.setattr(apg.asyncio, "timeout", lambda _s: timeout(0)) + _short_phases(monkeypatch) monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) conn = _Stalled() task = asyncio.create_task(apg._ensure_one(conn, "scope_generations", 100, apg.Wait(0, clock))) @@ -1115,7 +1130,8 @@ async def ensure_table(conn: Any, table: str, **kwargs: Any) -> Any: try: assert task in done, f"trigger deadline is stuck at {stall}" assert isinstance(task.exception(), TimeoutError) - assert conn.executed[-1] == "TERMINATE" + assert conn.executed[-1] == "CLOSE" + assert "TERMINATE" not in conn.executed if stall in {"migration", "ddl", "commit", "rollback"}: assert apg._MIGRATION_UNLOCK not in conn.executed finally: @@ -1147,7 +1163,8 @@ async def ensure_table(*_a: Any, **_kw: Any) -> Any: ) with pytest.raises(ValueError, match=f"{failed if failed == 'migration' else 'DDL'} failed"): await apg._ensure_one(conn, "scope_generations", 100, apg.Wait(0, clock)) - assert conn.executed[-1] == "TERMINATE" + assert conn.executed[-1] == "CLOSE" + assert "TERMINATE" not in conn.executed @pytest.mark.asyncio diff --git a/tests/unit/test_cell_attach_cleanup.py b/tests/unit/test_cell_attach_cleanup.py new file mode 100644 index 0000000000..20e17ac786 --- /dev/null +++ b/tests/unit/test_cell_attach_cleanup.py @@ -0,0 +1,119 @@ +"""The attach owner preserves failures through bounded public-close cleanup.""" + +import argparse +import asyncio +import time +from unittest.mock import AsyncMock + +import pytest + +from server.cli import cell_attach +from server.cli import cell_attach_pg as apg +from server.core import pg + + +@pytest.mark.asyncio +async def test_failed_work_keeps_its_error_when_public_close_also_fails(): + events = [] + + class Conn: + async def close(self, **kwargs): + events.append("close") + raise RuntimeError("close failed") + + def terminate(self): + events.append("abort") + + async def work(): + raise ValueError("original work failure") + + with pytest.raises(ValueError, match="original work failure"): + await apg.supervise(Conn(), work(), end=time.monotonic() + 0.1) + assert events == ["close", "abort"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed", [False, True]) +async def test_second_cancellation_preserves_original_failure_and_closes_before_cancel(failed): + entered, closing = asyncio.Event(), asyncio.Event() + events = [] + + class Conn: + async def close(self, **kwargs): + events.append("close") + closing.set() + await asyncio.Future() + + def terminate(self): + events.append("abort") + + async def work(): + entered.set() + try: + if failed: + raise ValueError("original work failure") + await asyncio.Future() + finally: + events.append("work ended") + + task = asyncio.create_task(apg.supervise(Conn(), work(), end=time.monotonic() + 0.2)) + await entered.wait() + if not failed: + task.cancel("first cancellation") + await closing.wait() + task.cancel("second cancellation") + with pytest.raises( + ValueError if failed else asyncio.CancelledError, + match="original work failure" if failed else "first cancellation", + ): + await task + assert events.count("abort") >= 1 + if not failed: + assert events.index("close") < events.index("work ended") + + +@pytest.mark.asyncio +async def test_lock_owner_bounds_context_exit_inside_the_same_cleanup_allowance(monkeypatch): + conn = AsyncMock() + conn.is_closed = lambda: False + conn.terminate = lambda: None + cm = AsyncMock() + exit_started = asyncio.Event() + + async def exit(*_): + exit_started.set() + await asyncio.Future() + + cm.__aexit__.side_effect = exit + run = cell_attach.Run(None, argparse.Namespace(cell="test", wait=0), None, lambda _: None) + run._lock_cm, run.lock_conn = cm, conn + monkeypatch.setattr(pg, "_RELEASE_TIMEOUT_S", 0.03) + started = time.monotonic() + with pytest.raises(TimeoutError): + await run.close_lock() + assert time.monotonic() - started < 0.15 + assert exit_started.is_set() and run.lock_conn is None + conn.close.assert_awaited_once() + await asyncio.sleep(0) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed", [False, True]) +async def test_run_preserves_its_own_failure_without_hiding_normal_close_failure(failed): + router = AsyncMock() + router.call.return_value = {"state": "attaching"} + run = cell_attach.Run( + None, argparse.Namespace(cell="test", wait=0, dry_run=True), router, lambda _: None + ) + run.open_lock = AsyncMock(side_effect=ValueError("run failed") if failed else None) + run.read_own_row = AsyncMock() + run.refusals = AsyncMock(return_value=(None, None, None)) + run.dry_run = AsyncMock(return_value=0) + run.close_lock = AsyncMock(side_effect=RuntimeError("close failed")) + try: + raise LookupError("unrelated caller exception") + except LookupError: + with pytest.raises( + ValueError if failed else RuntimeError, match="run failed" if failed else "close failed" + ): + await run.run() From f4702f64809e33a2f4c01a312fa99b82a64131e9 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 23:47:36 +0000 Subject: [PATCH 030/114] Use one owner for native cell control sessions Supervise attach, prepare, removal and backout on their exact lock backend. Keep cleanup inside phase deadlines, forward fatal guard exceptions after close, and retry mode contention only after acknowledged rollback and ownership checks. Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 2 + docs/architecture/cells.md | 6 +- server/cli/cell_attach.py | 131 ++++------- server/cli/cell_attach_pg.py | 206 +++++++----------- server/cli/cell_control.py | 2 + tests/integration/test_cell_attach.py | 19 +- tests/integration/test_cell_attach_cleanup.py | 28 +-- .../integration/test_cell_control_cleanup.py | 79 +++++++ tests/integration/test_cell_control_fence.py | 30 ++- tests/unit/cell_control_owner.py | 11 + tests/unit/test_cell_attach.py | 65 +++--- tests/unit/test_cell_control.py | 150 +++++++++++++ tests/unit/test_cell_control_census.py | 55 +++-- tests/unit/test_cell_prepare.py | 1 + tests/unit/test_router_attach.py | 13 +- 15 files changed, 503 insertions(+), 295 deletions(-) create mode 100644 tests/unit/cell_control_owner.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3066d67299..3b5b7aced8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -902,6 +902,8 @@ jobs: tests/integration/test_cell_prepare.py tests/integration/test_cell_attach.py tests/integration/test_cell_attach_cleanup.py + tests/integration/test_cell_control_cleanup.py + tests/integration/test_cell_control_fence.py tests/integration/test_cell_org_triggers_pg.py - name: pytest online-index durability on a private physical standby timeout-minutes: 8 diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 70a203783a..c0a4115499 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1263,6 +1263,10 @@ cleanup. An unexpected transaction error propagates immediately; it never waits rollback on a session the owner will discard. Uncapped command work retains the existing one-second session-exit allowance, which cannot extend an expired bounded phase. A transport abort remains an unknown outcome, not proof that the server backend ended. +Mode-lock contention retries only after an acknowledged pre-COMMIT rollback and a check +that the same backend still holds ATTACH. A COMMIT, rollback or ownership-check failure +never becomes a busy retry. After attempting the mode update, the diagnostic stays +unknown until rollback plus ownership or COMMIT is acknowledged. The source census for acceptance and operator directory writes is checked by `tests/unit/test_cell_control_census.py`, including dynamic cleanup tables, guarded @@ -1271,7 +1275,7 @@ entry points and the named exceptions: | Writer | Fence | | --------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | | `cell_prepare._write_rows` | Fresh claim, observed local rows, exact lock backend | -| `cell_attach_pg._mode_try` | Attach's claim, recording/first/fingerprint comparison, exact lock backend | +| `cell_attach_pg._mode_transaction` | Attach's claim, supervised mode try, recording/first/fingerprint comparison, exact lock backend | | `cell_control.backout` | Fresh claim, fenced close and zero-placement read, observed local rows, exact lock backend | | `directory.attach_begin`, `attach_place`, `attach_enforced`, `attach_serving`, `remove` | Claimed generation, full registration and exact pair, including retries/no-ops | | `directory.set_limit`, `rotate_credential` | Read and compare the current generation/registration/pair; do not claim or rotate | diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index ed57bb88da..a8a78279fd 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -155,7 +155,7 @@ def __init__( self.name = args.cell self.wait = apg.Wait(args.wait) self.lock_conn: Any = None - self._lock_cm: Any = None + self.owner: cell_control.Owner self.row: dict[str, Any] = {} self.mode = "" self.own_fp: tuple[dict[str, Any] | None, dict[str, Any] | None] = (None, None) @@ -165,35 +165,6 @@ def __init__( # -- the lock session -------------------------------------------------------------- - async def open_lock(self) -> None: - self._lock_cm = cell_control.session("attach") - self.lock_conn = await self._lock_cm.__aenter__() - - async def close_lock(self) -> None: - from server.core import pg - - if self._lock_cm is not None: - cm, conn = self._lock_cm, self.lock_conn - self._lock_cm = self.lock_conn = None - closed = pg._lost(conn) - end = time.monotonic() + (0.0 if closed else pg._RELEASE_TIMEOUT_S) - try: - if not closed: - await pg.close_owned(conn, end=end) - finally: - task = asyncio.create_task(cm.__aexit__(None, None, None)) - task.add_done_callback(pg._retrieve_task) - try: - done, _ = await asyncio.wait({task}, timeout=max(0.0, end - time.monotonic())) - if not done: - task.cancel() - raise TimeoutError("attach lock cleanup deadline expired") - task.result() - except BaseException: - pg._terminate(conn) - task.cancel() - raise - async def confirm_lock(self) -> None: """A lost backend ends the owner; a new run must claim and start again.""" await cell_control.check_lock(self.lock_conn) @@ -417,7 +388,7 @@ def _named(self, found: cell_scan.ScanResult) -> dict[str, Any]: } async def triggers(self) -> None: - if refusals := await apg.ensure_triggers(self.lock_conn, self.wait, self.log): + if refusals := await apg.ensure_triggers(self.owner, self.wait, self.log): raise Refused(refusals) # -- step 3 ------------------------------------------------------------------------ @@ -437,7 +408,7 @@ def mark(mode: str) -> None: self.mode = mode await apg.commit_enforced( - self.lock_conn, + self.owner, self.confirm_lock, self.ready_for_commit, self.wait, @@ -448,32 +419,16 @@ def mark(mode: str) -> None: ) async def durable(self) -> int: - """The durable read of `enforced`, retried within `--wait`; then X, the snapshot's + """The bounded durable read of `enforced`; then X, the snapshot's xmax, above the read's own xid.""" - from server.core import pg, pg_profile + from server.core import pg_profile with self.wait.counting(): - while True: - if await pg_profile.standby_now() != "synchronous": - raise _refused("postgres standbys", "not synchronous before the durable read") - try: - read = await apg.durable_read(self.lock_conn) - except Exception as e: - # A try cut off by its bound, or an outage: its outcome is unknown, and - # the next try reads again. Anything else is no reason to retry. - outage = pg.is_outage(e) or isinstance(e, pg.PostgresUnavailableError) - if not (isinstance(e, TimeoutError) or outage): - raise - await self.confirm_lock() - self.log(f"the durable read failed ({type(e).__name__}); retrying") - if not await self.wait.pause(): - raise _refused( - "postgres durable read", "did not commit within --wait" - ) from e - continue - if await pg_profile.standby_now() != "synchronous": - raise _refused("postgres standbys", "not synchronous after the durable read") - break + if await pg_profile.standby_now() != "synchronous": + raise _refused("postgres standbys", "not synchronous before the durable read") + read = await apg.durable_read(self.owner) + if await pg_profile.standby_now() != "synchronous": + raise _refused("postgres standbys", "not synchronous after the durable read") if read.mode in cell.MODES: self.mode = read.mode if read.first != (True, self.name) or read.mode != "enforced": @@ -608,44 +563,36 @@ async def run(self) -> int: await self.done() self.report.update(state="serving", done=True) return 0 - failure: BaseException | None = None - try: - await self.open_lock() - await self.read_own_row() - topology, names, listed = await self.refusals() - if self.args.dry_run: - return await self.dry_run(topology, listed) - self.row = await cell_control.claim(self.lock_conn, self.router.call, self.row) - await self.begin() - state, _ = await apg.read_mode(self.lock_conn) - self.mode = state + return await cell_control.run("attach", self._owned) + + async def _owned(self, owner: cell_control.Owner) -> int: + self.owner, self.lock_conn = owner, owner.conn + await self.read_own_row() + topology, names, listed = await self.refusals() + if self.args.dry_run: + return await self.dry_run(topology, listed) + self.row = await cell_control.claim(self.lock_conn, self.router.call, self.row) + await self.begin() + state, _ = await apg.read_mode(self.lock_conn) + self.mode = state + await self.triggers() + if state == "recording": + found = await self.scan(topology, listed) + if found.too_long: + raise Refused(_too_long(found)) + written = await apg.write_tenancy(found.orgs) + self.log(f"step 2: {len(found.orgs)} orgs found, {written} tenancy rows written") await self.triggers() - if state == "recording": - found = await self.scan(topology, listed) - if found.too_long: - raise Refused(_too_long(found)) - written = await apg.write_tenancy(found.orgs) - self.log(f"step 2: {len(found.orgs)} orgs found, {written} tenancy rows written") - await self.triggers() - await self.precommit(found) - await self.commit_mode() - else: - await self.mutate("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) - xmax = await self.durable() - await self.check_beats("ignore") - await self.step4(xmax, topology, names) - answer = await self.serving() - self.report.update(state=answer.get("state"), placed_orgs=answer.get("placed_orgs")) - return 0 - except BaseException as error: - failure = error - raise - finally: - try: - await self.close_lock() - except BaseException: - if failure is None: - raise + await self.precommit(found) + await self.commit_mode() + else: + await self.mutate("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) + xmax = await self.durable() + await self.check_beats("ignore") + await self.step4(xmax, topology, names) + answer = await self.serving() + self.report.update(state=answer.get("state"), placed_orgs=answer.get("placed_orgs")) + return 0 async def precommit(self, found: cell_scan.ScanResult) -> None: from server.core import pg_profile diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index d24f749996..d18ff6f628 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -13,14 +13,17 @@ import contextlib import json import time -from collections.abc import Awaitable, Callable, Coroutine, Iterable, Iterator +from collections.abc import Awaitable, Callable, Iterable, Iterator from dataclasses import dataclass -from typing import Any +from typing import TYPE_CHECKING, Any from server.cli.cell_ch import Refusal, Refused from server.core import cell, cell_triggers, pg from server.core.cell import ATTACH_LOCK_KEY +if TYPE_CHECKING: + from server.cli.cell_control import Owner + _sleep = asyncio.sleep # the seam the wait tests replace # The pause between polls of a wait. @@ -90,50 +93,6 @@ async def pause(self, seconds: float = POLL_S) -> bool: return True -@dataclass -class _Interrupted: - error: BaseException - - -async def _work_result[T](work: Coroutine[Any, Any, T]) -> T | _Interrupted: - try: - return await work - except (KeyboardInterrupt, SystemExit) as error: - # These exceptions otherwise stop asyncio's loop before the owner can close. - return _Interrupted(error) - - -async def supervise[T](conn: Any, work: Coroutine[Any, Any, T], *, end: float) -> T: - """Bound owned work, reserving cleanup inside its original phase deadline. - - A failed borrowed transaction must remain alive until public close sends its - cancellation. Closing the transport first can leave its backend locks held. - """ - task = asyncio.create_task(_work_result(work)) - reserve = min(1.0, max(0.0, end - time.monotonic()) / 4) - try: - done, _ = await asyncio.wait({task}, timeout=max(0.0, end - reserve - time.monotonic())) - if not done: - raise TimeoutError("attach Postgres phase deadline expired") - result = task.result() - if isinstance(result, _Interrupted): - raise result.error - return result - except BaseException: - try: - await pg.close_owned(conn, end=end) - except BaseException: # noqa: BLE001 — preserve the work failure or cancellation. - pass - if not task.done(): - task.cancel() - task.add_done_callback(pg._retrieve_task) - try: - await asyncio.wait({task}, timeout=max(0.0, end - time.monotonic())) - except BaseException: # noqa: BLE001 — repeated cancellation preserves the first failure. - pg._terminate(conn) - raise - - def _split(key: int) -> str: """The `pg_locks` columns that hold a bigint advisory key, in this database.""" return ( @@ -500,12 +459,13 @@ async def prepared_refusals(conn: Any) -> list[Refusal]: _MIGRATION_UNLOCK = "SELECT pg_advisory_unlock($1)" -async def ensure_triggers(conn: Any, wait: Wait, log: Callable[[str], None]) -> list[Refusal]: +async def ensure_triggers(owner: Owner, wait: Wait, log: Callable[[str], None]) -> list[Refusal]: """Give every planned table its triggers (`cell_triggers.ensure_table` with `recording_first`), one try at a time under the migration lock, retrying a table whose lock is busy a second apart within `wait`; one autovacuum holding it is asked to yield by one try just past `deadlock_timeout`. A table this role cannot alter refuses at once. Then `missing`: any defect refuses.""" + conn = owner.conn deadlock_ms = int( await conn.fetchval( "SELECT setting::int * CASE unit WHEN 's' THEN 1000 ELSE 1 END " @@ -514,14 +474,15 @@ async def ensure_triggers(conn: Any, wait: Wait, log: Callable[[str], None]) -> ) with wait.counting(): for table in await cell_triggers.planned_tables(conn): - if refusal := await _ensure_table(conn, table, deadlock_ms, wait, log): + if refusal := await _ensure_table(owner, table, deadlock_ms, wait, log): return [refusal] return [Refusal("postgres triggers", d.text) for d in await cell_triggers.missing(conn)] async def _ensure_table( - conn: Any, table: str, deadlock_ms: int, wait: Wait, log: Callable[[str], None] + owner: Owner, table: str, deadlock_ms: int, wait: Wait, log: Callable[[str], None] ) -> Refusal | None: + conn = owner.conn yielded = False said = "" while True: @@ -529,7 +490,7 @@ async def _ensure_table( holders = await cell_triggers.holders(conn, table) if not yielded and any(h["backend_type"] == "autovacuum worker" for h in holders): most_ms, yielded = deadlock_ms + 500, True - outcome = await _ensure_one(conn, table, most_ms, wait) + outcome = await _ensure_one(owner, table, most_ms, wait) if outcome.answer == "applied": return None if outcome.answer == "denied": @@ -567,7 +528,7 @@ async def _migration_holder(conn: Any, key: int) -> str: return f"{_MIGRATION_HELD} (held by {held or 'a session that just ended'})" -async def _ensure_one(conn: Any, table: str, most_ms: int, wait: Wait) -> cell_triggers.Outcome: +async def _ensure_one(owner: Owner, table: str, most_ms: int, wait: Wait) -> cell_triggers.Outcome: """One try under the migration lock: the migration lock's wait, and then the DDL's lock wait (at most `most_ms`), each bounded by what `wait` has left when it starts, with the try floor. One client deadline covers the lock, DDL and cleanup, including COMMIT or @@ -577,8 +538,8 @@ async def _ensure_one(conn: Any, table: str, most_ms: int, wait: Wait) -> cell_t bound_s = ( wait.bound_ms(_MIGRATION_LOCK_MS) + wait.bound_ms(most_ms) ) / 1000 + _TRIGGER_TRY_MARGIN_S - return await supervise( - conn, _trigger_try(conn, table, most_ms, wait, _LOCK_KEY), end=time.monotonic() + bound_s + return await owner.call( + lambda: _trigger_try(owner.conn, table, most_ms, wait, _LOCK_KEY), total_s=bound_s ) @@ -627,7 +588,7 @@ def _busy_refusal(table: str, holders: list[Any]) -> Refusal: async def commit_enforced( - conn: Any, + owner: Owner, confirm_lock: Callable[[], Any], ready: Callable[[], Awaitable[list[str]]], wait: Wait, @@ -643,16 +604,11 @@ async def commit_enforced( await confirm_lock() booting = await ready() if not booting: - try: - done = await _mode_try( - conn, wait.bound_ms(_MODE_LOCK_MS), mark, expected_cell, expected_fp - ) - except Refused: - mark("recording") # the UPDATE's transaction rolled back - raise + done = await _mode_try( + owner, wait.bound_ms(_MODE_LOCK_MS), mark, expected_cell, expected_fp + ) if done: return - mark("recording") log("the mode's lock is busy; retrying") if not await wait.pause(): if booting: @@ -674,65 +630,69 @@ async def commit_enforced( async def _mode_try( - conn: Any, + owner: Owner, lock_ms: int, mark: Callable[[str], None], expected_cell: str, expected_fp: tuple[dict[str, Any] | None, dict[str, Any] | None], ) -> bool: - """One try including rollback or COMMIT, bounded by its lock and statement waits plus - the standby allowance; False after a statement timeout rolls back, TimeoutError at - the try's deadline.""" + """Keep work, acknowledged contention rollback and cleanup in one try budget.""" bound_s = (lock_ms + _MODE_STATEMENT_MARGIN_MS) / 1000 + _MODE_STANDBY_S + return await owner.call( + lambda: _mode_transaction(owner.conn, lock_ms, mark, expected_cell, expected_fp), + total_s=bound_s, + ) + + +async def _mode_transaction( + conn: Any, + lock_ms: int, + mark: Callable[[str], None], + expected_cell: str, + expected_fp: tuple[dict[str, Any] | None, dict[str, Any] | None], +) -> bool: + from server.cli import cell_control + + tx = conn.transaction() + await tx.start() + updated = False try: - async with asyncio.timeout(bound_s): - tx = conn.transaction() - await tx.start() - try: - await conn.execute(f"SET LOCAL lock_timeout = '{lock_ms}ms'") - await conn.execute( - f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" - ) - from server.cli import cell_control - - await cell_control.check_lock(conn) - await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") - state, first = await read_mode(conn) - own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") - observed = None if own is None else (own["current_fp"], own["next_fp"]) - if ( - state != "recording" - or first != (True, expected_cell) - or json.dumps(observed, sort_keys=True) - != json.dumps(expected_fp, sort_keys=True) - ): - raise Refused( - [Refusal("postgres acceptance rows", "changed: stop and rerun attach")] - ) - mark("unknown") - status = await conn.execute( - "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" - ) - if status != "UPDATE 1": - raise Refused( - [Refusal("postgres loom.tenancy_mode", f"the mode update read {status}")] - ) - except Exception as e: - await tx.rollback() - if getattr(e, "sqlstate", None) in _TIMED_OUT: - return False - raise - await tx.commit() - return True - except BaseException: - # Termination is synchronous: cancellation never starts an unbounded rollback. - conn.terminate() - raise + await conn.execute(f"SET LOCAL lock_timeout = '{lock_ms}ms'") + await conn.execute( + f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" + ) + await cell_control.check_lock(conn) + await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") + state, first = await read_mode(conn) + own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") + observed = None if own is None else (own["current_fp"], own["next_fp"]) + if ( + state != "recording" + or first != (True, expected_cell) + or json.dumps(observed, sort_keys=True) != json.dumps(expected_fp, sort_keys=True) + ): + raise Refused([Refusal("postgres acceptance rows", "changed: stop and rerun attach")]) + mark("unknown") + updated = True + status = await conn.execute("UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()") + if status != "UPDATE 1": + raise Refused([Refusal("postgres loom.tenancy_mode", f"the mode update read {status}")]) + except Exception as error: + if getattr(error, "sqlstate", None) not in _TIMED_OUT: + raise + await tx.rollback() + await cell_control.check_lock(conn) + if updated: + mark("recording") + return False + await tx.commit() + mark("enforced") + return True # -- the durable read ------------------------------------------------------------------- -_DURABLE_TRY_S = 10.0 # not a setting: one try's bound; a cancelled one is retried +_DURABLE_TRY_S = 10.0 # not a setting: the state/WAL/COMMIT phase includes its cleanup @dataclass(frozen=True) @@ -745,24 +705,26 @@ class Durable: xid: int -async def durable_read(conn: Any) -> Durable: - """The mode and the `first` row read in a transaction that emits a transactional - logical message, whose commit waits for the synchronous standbys: once it returns, the - mode read is on them. Refused when the message took no xid.""" +async def durable_read(owner: Owner) -> Durable: + """Read and synchronously fence state on the owner's exact ATTACH backend.""" from server.cli import cell_control - async with asyncio.timeout(_DURABLE_TRY_S), cell_control.transaction(conn): + conn = owner.conn + + async def read() -> Durable: await conn.execute("SELECT pg_logical_emit_message(true, 'loom.attach', '')") xid = await conn.fetchval("SELECT pg_current_xact_id_if_assigned()::text") mode = await conn.fetchval("SELECT mode FROM loom.tenancy_mode") first = await conn.fetchrow("SELECT is_first, cell FROM loom.cell_first") - if xid is None: - raise Refused([Refusal("postgres durable read", "its logical message took no xid")]) - return Durable( - mode=str(mode or ""), - first=None if first is None else (bool(first["is_first"]), str(first["cell"])), - xid=int(xid), - ) + if xid is None: + raise Refused([Refusal("postgres durable read", "its logical message took no xid")]) + return Durable( + mode=str(mode or ""), + first=None if first is None else (bool(first["is_first"]), str(first["cell"])), + xid=int(xid), + ) + + return await owner.call(lambda: cell_control.transaction(conn, read), total_s=_DURABLE_TRY_S) async def snapshot_xmax(conn: Any) -> int: diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index 84661747fd..7fd92e28ab 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -239,6 +239,8 @@ async def run[T](what: str, work: Callable[[Owner], Awaitable[T]]) -> T: raise try: await owner.close() + except _ForwardFatal as interrupted: + raise interrupted.fatal.error from None finally: await owner._stop(set(owner._tasks)) return result diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 573d8fc85a..6cbc595bd5 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -35,6 +35,7 @@ from server.router import directory from server.settings import settings from tests.integration._pg_database import create_database, drop_database +from tests.unit.cell_control_owner import owner as _owner OPERATOR = "lco_" + "2" * 128 CREDENTIAL_SHA = hashlib.sha256(b"lcr_first").hexdigest() @@ -535,7 +536,7 @@ async def release() -> None: await cell_attach_pg.take_lock(conn, "mode test") own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") await cell_attach_pg.commit_enforced( - conn, + _owner(conn), ok, ok, cell_attach_pg.Wait(20), @@ -812,19 +813,17 @@ async def test_signaling_another_role_requires_an_effective_signal_grant(cell_st async def test_attach_needs_its_predefined_capabilities_without_set_role( cell_stores, monkeypatch, capsys, capability: str ) -> None: - from contextlib import asynccontextmanager - stores, network = cell_stores, _Network() await _prepare(stores, network) role = f"it_caps_{uuid.uuid4().hex[:6]}" await _admin(f"CREATE ROLE {role} LOGIN") real_dedicated = pg.dedicated + real_connect = pg._connect_dedicated - @asynccontextmanager - async def as_attacher(**kwargs: Any) -> AsyncIterator[Any]: - async with real_dedicated(**kwargs) as conn: - await conn.execute(f"SET SESSION AUTHORIZATION {role}") - yield conn + async def as_attacher(**kwargs: Any) -> Any: + conn = await real_connect(**kwargs) + await conn.execute(f"SET SESSION AUTHORIZATION {role}") + return conn try: async with real_dedicated() as conn: @@ -836,7 +835,7 @@ async def as_attacher(**kwargs: Any) -> AsyncIterator[Any]: assert not await conn.fetchval("SELECT pg_has_role($1, $2, 'USAGE')", role, capability) before = await _row() with monkeypatch.context() as own: - own.setattr(pg, "dedicated", as_attacher) + own.setattr(pg, "_connect_dedicated", as_attacher) assert await _attach(stores, network) == 1 assert ( f"attach's role lacks effective privileges of {capability}" in capsys.readouterr().err @@ -848,7 +847,7 @@ async def as_attacher(**kwargs: Any) -> AsyncIterator[Any]: async with real_dedicated() as conn: await conn.execute(f"GRANT {capability} TO {role} WITH INHERIT TRUE") with monkeypatch.context() as own: - own.setattr(pg, "dedicated", as_attacher) + own.setattr(pg, "_connect_dedicated", as_attacher) assert await _attach(stores, network) == 0, capsys.readouterr().err assert (await _row()).state == "serving" finally: diff --git a/tests/integration/test_cell_attach_cleanup.py b/tests/integration/test_cell_attach_cleanup.py index 145be1188d..3b65c81b7d 100644 --- a/tests/integration/test_cell_attach_cleanup.py +++ b/tests/integration/test_cell_attach_cleanup.py @@ -1,17 +1,16 @@ """Attach owners close active sessions before cancelling transaction work.""" -import argparse import asyncio -import contextlib import time import pytest -from server.cli import cell_attach from server.cli import cell_attach_pg as apg +from server.cli import cell_control from server.core import cell_triggers, pg from tests.integration.test_pg_owned_cleanup import database as database from tests.integration.test_pg_owned_cleanup import released, sleeping +from tests.unit.cell_control_owner import owner as _owner pytestmark = pytest.mark.asyncio @@ -52,7 +51,7 @@ async def intercepted(self, query, *args, **kwargs): if owner == "mode" and query.startswith("LOCK TABLE"): await execute(self, "SELECT * FROM probe FOR UPDATE") if phase == "rollback": - raise ValueError("mode statement failed") + raise asyncpg.LockNotAvailableError("mode statement busy") return await execute(self, "SELECT pg_sleep(10)") if owner == "direct" and query.startswith("BEGIN"): await execute(self, query) @@ -73,9 +72,8 @@ async def ensure(conn, table, **kwargs): conn, table, statements, kwargs["lock_timeout"], 1, warn=False ) - @contextlib.asynccontextmanager - async def dedicated(**_): - yield conn + async def connect(**_): + return conn monkeypatch.setattr(asyncpg.Connection, "execute", intercepted) monkeypatch.setattr(asyncpg.Connection, "close", closing_session) @@ -83,15 +81,19 @@ async def dedicated(**_): monkeypatch.setattr(apg, "_TRIGGER_TRY_MARGIN_S", 0.4) monkeypatch.setattr(apg, "_MODE_STANDBY_S", 0.5) monkeypatch.setattr(apg, "_MODE_STATEMENT_MARGIN_MS", 0) - monkeypatch.setattr(cell_attach, "_LOCK_STATEMENT_MS", 600) - monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(cell_control, "OPEN_S", 0.6) + monkeypatch.setattr(pg, "_connect_dedicated", connect) + await apg.take_lock(conn, "cleanup test") if owner == "trigger": - work = apg._ensure_one(conn, "probe", 100, apg.Wait(0)) + work = apg._ensure_one(_owner(conn), "probe", 100, apg.Wait(0)) elif owner == "mode": - work = apg._mode_try(conn, 100, lambda _: None) + work = apg._mode_try(_owner(conn), 100, lambda _: None, "first", (None, None)) else: - run = cell_attach.Run(None, argparse.Namespace(cell="test", wait=0), None, lambda _: None) - work = run.open_lock() + + async def body(_owner): + raise AssertionError("direct checks unexpectedly completed") + + work = cell_control.run("test", body) task = asyncio.create_task(work) try: await sleeping(observer, pid) diff --git a/tests/integration/test_cell_control_cleanup.py b/tests/integration/test_cell_control_cleanup.py index 2a37ca6170..f66eeb5169 100644 --- a/tests/integration/test_cell_control_cleanup.py +++ b/tests/integration/test_cell_control_cleanup.py @@ -214,3 +214,82 @@ async def main(): if child.returncode is None: child.kill() await child.wait() + + +@pytest.mark.parametrize("sqlstate", ["55P03", "57014"]) +async def test_mode_contention_retries_only_after_real_rollback_on_same_owned_backend( + native_database, monkeypatch, sqlstate +): + import asyncpg + + from server.cli import cell_attach_pg as apg + from tests.unit.cell_control_owner import owner + + conn = await pg._connect_dedicated(statement_timeout_ms=10_000) + await pg.init_connection(conn) + await apg.take_lock(conn, "real mode contention") + await conn.execute("INSERT INTO loom.cell_first(is_first,cell) VALUES(true,'first')") + await conn.execute("INSERT INTO loom.tenancy_mode(mode) VALUES('recording')") + fp = {"v": 1} + await conn.execute("INSERT INTO loom.cell_self(current_fp) VALUES($1)", fp) + pid, marks, injected = conn.get_server_pid(), [], False + execute = asyncpg.Connection.execute + + async def contend(self, query, *args, **kwargs): + nonlocal injected + if self is conn and query.startswith("UPDATE loom.tenancy_mode") and not injected: + injected = True + await execute( + self, + "DO $$ BEGIN RAISE EXCEPTION USING ERRCODE='" + + sqlstate + + "', MESSAGE='test contention'; END $$", + ) + return await execute(self, query, *args, **kwargs) + + monkeypatch.setattr(asyncpg.Connection, "execute", contend) + owned = owner(conn) + try: + assert not await apg._mode_try(owned, 100, marks.append, "first", (fp, None)) + assert conn.get_server_pid() == pid and await apg.lock_held(conn) + assert await conn.fetchval("SELECT mode FROM loom.tenancy_mode") == "recording" + assert marks == ["unknown", "recording"] and not owned.closed + assert await apg._mode_try(owned, 100, marks.append, "first", (fp, None)) + assert await native_database.fetchval("SELECT mode FROM loom.tenancy_mode") == "enforced" + assert marks == ["unknown", "recording", "unknown", "enforced"] + assert conn.get_server_pid() == pid and await apg.lock_held(conn) + finally: + await owned.close() + await released(native_database, pid) + + +async def test_native_attach_run_body_uses_sole_owner_on_cancellation(native_database, monkeypatch): + from uuid import UUID + + from server.cli import cell_attach + from server.router.directory import CONTROL_FIELDS + + observed, backend = asyncio.Event(), {} + + class Router: + async def call(self, _path): + return { + **dict.fromkeys(CONTROL_FIELDS, ""), + "state": "attaching", + "mutation_generation": str(UUID(int=1)), + } + + async def read(self): + backend["pid"] = self.lock_conn.get_server_pid() + observed.set() + await self.lock_conn.execute("BEGIN; SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10)") + + monkeypatch.setattr(cell_attach.Run, "read_own_row", read) + run = cell_attach.Run(None, argparse.Namespace(cell="first", wait=0), Router(), lambda _: None) + task = asyncio.create_task(run.run()) + await observed.wait() + await sleeping(native_database, backend["pid"]) + task.cancel("native attach cancellation") + with pytest.raises(asyncio.CancelledError, match="native attach cancellation"): + await asyncio.wait_for(task, 1.3) + await released(native_database, backend["pid"]) diff --git a/tests/integration/test_cell_control_fence.py b/tests/integration/test_cell_control_fence.py index d96f170fa3..fa375eb12a 100644 --- a/tests/integration/test_cell_control_fence.py +++ b/tests/integration/test_cell_control_fence.py @@ -17,6 +17,7 @@ from server.core import cell from server.router import app, directory from server.settings import settings +from tests.unit.cell_control_owner import owner as _owner TOKEN = "lco_fence-test" @@ -222,7 +223,7 @@ async def test_dead_local_lock_backend_cannot_write_acceptance(local_control, wr with pytest.raises( (asyncpg.InterfaceError, asyncpg.PostgresError, asyncpg.InternalClientError) ): - await apg._mode_try(dead, 100, lambda _m: None, "first", (fp, None)) + await apg._mode_try(_owner(dead), 100, lambda _m: None, "first", (fp, None)) assert await admin.fetchval("SELECT mode FROM loom.tenancy_mode") == "recording" finally: await successor.close() @@ -232,7 +233,7 @@ async def test_dead_local_lock_backend_cannot_write_acceptance(local_control, wr @pytest.mark.asyncio async def test_same_session_prepare_and_mode_commit(local_control) -> None: from server.cli import cell_attach_pg as apg - from server.cli import cell_control, cell_prepare + from server.cli import cell_prepare conn, _ = local_control await apg.take_lock(conn, "test") @@ -243,8 +244,8 @@ async def test_same_session_prepare_and_mode_commit(local_control) -> None: ) assert written == ["first", "mode", "own"] assert await apg.lock_held(conn) - assert await apg._mode_try(conn, 100, lambda _m: None, "first", (fp, None)) - result = await apg.durable_read(conn) + assert await apg._mode_try(_owner(conn), 100, lambda _m: None, "first", (fp, None)) + result = await apg.durable_read(_owner(conn)) assert result.mode == "enforced" and result.first == (True, "first") assert await apg.lock_held(conn) # No-op prepare remains a same-backend synchronous durability boundary. @@ -253,8 +254,10 @@ async def test_same_session_prepare_and_mode_commit(local_control) -> None: await cell_prepare._write_rows(conn, rows, {"is_first": True, "mode": "enforced"}, fp, None) == [] ) - async with cell_control.transaction(conn): - assert await conn.fetchval("SELECT pg_backend_pid()") == conn.get_server_pid() + assert ( + await _owner(conn).transaction(lambda: conn.fetchval("SELECT pg_backend_pid()")) + == conn.get_server_pid() + ) @pytest.mark.asyncio @@ -334,7 +337,6 @@ async def test_inflight_finalizer_cannot_release_after_new_owner_rewidens( from starlette.routing import Route from server.cli import cell_attach_pg as apg - from server.cli import cell_control client, directory_admin, _ = control_stores local_admin, local_dsn = local_control @@ -387,9 +389,11 @@ async def change(): await local_admin.execute("SELECT pg_terminate_backend($1)", a.get_server_pid()) await apg.take_lock(b, "new controller") newer = await claim(client, await read(client)) + # New owner claims BEFORE widening local acceptance, on the owned backend. - async with cell_control.transaction(b): + async def widen(): await b.execute("UPDATE loom.cell_self SET current_fp = $1, next_fp = $2", current, nxt) + if aba: await directory_admin.execute( "UPDATE loom_router.cells SET current_fp = next_fp, next_fp = NULL" @@ -399,6 +403,7 @@ async def change(): json.dumps(current), json.dumps(nxt), ) + await _owner(b).transaction(widen) assert newer["mutation_generation"] != old["mutation_generation"] resume.set() response = await task @@ -473,7 +478,7 @@ async def wait_for_lock(key): await asyncio.sleep(0.01) else: pytest.fail("placement did not acquire its org lock") - cleanup = asyncio.create_task(cell_control.backout(local_admin, call, original)) + cleanup = asyncio.create_task(cell_control.backout(_owner(local_admin), call, original)) for _ in range(100): waiting = await directory_admin.fetchval( "SELECT count(*) FROM pg_stat_activity WHERE datname = current_database() " @@ -498,7 +503,7 @@ async def wait_for_lock(key): directory.place("queued", "first", app.placement_refusal) ) await wait_for_lock(directory._PLACE_LOCK) - await cell_control.backout(local_admin, call, original) + await cell_control.backout(_owner(local_admin), call, original) assert not await local_admin.fetchval("SELECT EXISTS(SELECT 1 FROM loom.cell_self)") await tx.commit() with pytest.raises(directory.PlacementRefused, match="closed"): @@ -580,6 +585,7 @@ async def call(self, path, body): None, argparse.Namespace(cell="first", wait=0), Router(), lambda _text: None ) run.lock_conn = a + run.owner = _owner(a) run.mode = "recording" run.own_fp = (fp, None) run.row = { @@ -658,7 +664,7 @@ async def call(path, body): placement_error = str(exc) return answer - await cell_control.backout(local, call, await read(client)) + await cell_control.backout(_owner(local), call, await read(client)) assert opened.status_code == 409, opened.text assert "serving" in opened.json()["error"] assert placement_error and "closed" in placement_error @@ -788,7 +794,7 @@ async def call(path, body): await asyncio.wait_for(entered.wait(), 3) await local.execute("SELECT pg_terminate_backend($1)", old_backend.get_server_pid()) await apg.take_lock(local, "backout successor") - await cell_control.backout(local, call, await read(client)) + await cell_control.backout(_owner(local), call, await read(client)) assert await apg.lock_held(local) assert not await local.fetchval("SELECT EXISTS(SELECT 1 FROM loom.cell_self)") assert not await directory_admin.fetchval( diff --git a/tests/unit/cell_control_owner.py b/tests/unit/cell_control_owner.py new file mode 100644 index 0000000000..3a2d009821 --- /dev/null +++ b/tests/unit/cell_control_owner.py @@ -0,0 +1,11 @@ +"""Provide an explicit owner for a test's already-open raw session.""" + +from typing import Any + +from server.cli.cell_control import Owner + + +def owner(conn: Any) -> Owner: + owned = Owner() + owned.conn = conn + return owned diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 0b3401be88..a69c5c5c8f 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -8,7 +8,6 @@ import argparse import asyncio import json -import time from collections.abc import Callable from dataclasses import dataclass, field from typing import Any @@ -16,7 +15,7 @@ import httpx import pytest -from server.cli import cell_attach, cell_scan +from server.cli import cell_attach, cell_control, cell_scan from server.cli import cell_attach_beats as beats from server.cli import cell_attach_pg as apg from server.cli.cell_ch import Refusal, Refused @@ -24,6 +23,7 @@ from server.core import cell, instances from server.router.limits import ATTACH_BATCH, MAX_BODY_BYTES from server.settings import settings +from tests.unit.cell_control_owner import owner as _owner FIELDS = {**cell.DEFAULTS, "deployment_id": "dep-1"} CURRENT = cell.as_value(FIELDS) @@ -253,7 +253,7 @@ async def ready() -> list[str]: clock.now += 100 # a scan before the wait with pytest.raises(Refused, match="not taken within --wait"): await apg.commit_enforced( - conn, + _owner(conn), nothing, ready, wait, @@ -287,7 +287,7 @@ async def ensure_table(*_a: Any, **_kw: Any) -> Any: monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) conn = FakeConn([("deadlock_timeout", 1000)]) wait = apg.Wait(60, clock) - (refusal,) = await apg.ensure_triggers(conn, wait, lambda _t: None) + (refusal,) = await apg.ensure_triggers(_owner(conn), wait, lambda _t: None) assert refusal.subject == "postgres loom.scope_generations" assert "cannot alter it" in refusal.reason and wait.spent < 1 @@ -300,7 +300,7 @@ async def busy(*_a: Any, **_kw: Any) -> Any: monkeypatch.setattr(cell_triggers, "ensure_table", busy) wait = apg.Wait(10, clock) logged: list[str] = [] - (refusal,) = await apg.ensure_triggers(conn, wait, logged.append) + (refusal,) = await apg.ensure_triggers(_owner(conn), wait, logged.append) # The last try ran with the floor's lock wait, after --wait was all but spent. assert "not taken within --wait" in refusal.reason assert wait.spent <= 10 + apg.TRY_FLOOR_MS / 1000 + 1e-9 @@ -519,9 +519,6 @@ async def __aexit__(self, *exc: Any) -> None: return _Tx() - async def close(self, *, timeout: float) -> None: - self.executed.append("CLOSE") - def terminate(self) -> None: self.executed.append("TERMINATE") @@ -534,14 +531,14 @@ def is_closed(self) -> bool: def _short_phases(monkeypatch: pytest.MonkeyPatch, bounds: list[float] | None = None) -> None: - supervise = apg.supervise + supervise = cell_control.Owner.call - async def bounded(conn: Any, work: Any, *, end: float) -> Any: + async def bounded(self: Any, work: Any, *, total_s: float) -> Any: if bounds is not None: - bounds.append(end - time.monotonic()) - return await supervise(conn, work, end=time.monotonic() + 0.01) + bounds.append(total_s) + return await supervise(self, work, total_s=0.01) - monkeypatch.setattr(apg, "supervise", bounded) + monkeypatch.setattr(cell_control.Owner, "call", bounded) def _seq(*values: Any) -> Callable[..., Any]: @@ -722,7 +719,7 @@ async def ready() -> list[str]: logged: list[str] = [] await apg.commit_enforced( - conn, + _owner(conn), confirm, ready, apg.Wait(10, clock), @@ -747,6 +744,7 @@ async def test_the_mode_commit_refuses_past_wait_and_skips_a_try_while_booting( ) -> None: conn = FakeConn([("l.mode = 'ExclusiveLock'", [])]) run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + run.owner = _owner(FakeConn()) run.own_fp = (CURRENT, NEXT) async def read(_ch: Any) -> list[beats.Beat]: @@ -759,7 +757,7 @@ async def confirm() -> None: with pytest.raises(Refused) as error: await apg.commit_enforced( - conn, + _owner(conn), confirm, run.ready_for_commit, apg.Wait(wait_s, clock), @@ -801,7 +799,7 @@ async def ready() -> list[str]: with pytest.raises(TimeoutError): await asyncio.wait_for( apg.commit_enforced( - conn, + _owner(conn), confirm, ready, apg.Wait(wait_s, clock), @@ -837,7 +835,9 @@ async def execute(self, sql: str, *args: Any) -> str: _short_phases(monkeypatch) marks: list[str] = [] task = asyncio.create_task( - apg._mode_try(conn, 100, marks.append, expected_cell="first", expected_fp=(CURRENT, NEXT)) + apg._mode_try( + _owner(conn), 100, marks.append, expected_cell="first", expected_fp=(CURRENT, NEXT) + ) ) done, _ = await asyncio.wait([task], timeout=0.2) if not done: @@ -905,9 +905,9 @@ async def confirm() -> None: async def ready() -> list[str]: return [] - with pytest.raises(KeyboardInterrupt): + with pytest.raises(cell_control._ForwardFatal) as fatal: await apg.commit_enforced( - conn, + _owner(conn), confirm, ready, apg.Wait(2, clock), @@ -916,6 +916,7 @@ async def ready() -> list[str]: expected_cell="first", expected_fp=(CURRENT, NEXT), ) + assert isinstance(fatal.value.fatal.error, KeyboardInterrupt) assert conn.executed[-1] == "CLOSE" assert "TERMINATE" not in conn.executed assert marks == [] @@ -932,6 +933,7 @@ async def synchronous() -> str: monkeypatch.setattr(pg_profile, "standby_now", synchronous) run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + run.owner = _owner(FakeConn()) run.lock_conn = FakeConn([("pg_snapshot_xmax", "100")]) async def read(mode: str, xid: int) -> apg.Durable: @@ -974,6 +976,7 @@ async def read_mode(_conn: Any) -> tuple[str, Any]: monkeypatch.setattr(pg, "dedicated", dedicated) monkeypatch.setattr(apg, "read_mode", read_mode) run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + run.owner = _owner(FakeConn()) assert await run.done() state[0] = "recording" with pytest.raises(Refused, match="attach lowers and repairs neither"): @@ -989,6 +992,7 @@ async def test_a_root_secret_change_during_attach_refuses_at_the_end( monkeypatch: pytest.MonkeyPatch, ) -> None: run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + run.owner = _owner(FakeConn()) run.begun_keys_fp = "before" async def settle(*_a: Any, **_kw: Any) -> beats.Verdict: @@ -1021,6 +1025,7 @@ async def call(_self: Any, _path: str, _body: Any = None) -> dict[str, Any]: return {"cell": "first"} async def run(self: Any) -> int: + self.owner = _owner(FakeConn()) self.row = {"deployment_id": "dep-1"} await self.commit_mode() raise failure # what stops the run before the durable read finishes @@ -1086,7 +1091,7 @@ async def ensure_table(*_a: Any, lock_timeout: str, **_kw: Any) -> Any: conn = FakeConn([("pg_advisory_lock", migration_lock), ("pg_advisory_unlock", "OK")]) wait = apg.Wait(2, clock) with wait.counting(): - outcome = await apg._ensure_one(conn, "scope_generations", 500, wait) + outcome = await apg._ensure_one(_owner(conn), "scope_generations", 500, wait) assert outcome.answer == "applied" assert asked == [f"{apg.TRY_FLOOR_MS}ms"] @@ -1125,7 +1130,9 @@ async def ensure_table(conn: Any, table: str, **kwargs: Any) -> Any: _short_phases(monkeypatch) monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) conn = _Stalled() - task = asyncio.create_task(apg._ensure_one(conn, "scope_generations", 100, apg.Wait(0, clock))) + task = asyncio.create_task( + apg._ensure_one(_owner(conn), "scope_generations", 100, apg.Wait(0, clock)) + ) done, _ = await asyncio.wait([task], timeout=0.2) try: assert task in done, f"trigger deadline is stuck at {stall}" @@ -1162,7 +1169,7 @@ async def ensure_table(*_a: Any, **_kw: Any) -> Any: ] ) with pytest.raises(ValueError, match=f"{failed if failed == 'migration' else 'DDL'} failed"): - await apg._ensure_one(conn, "scope_generations", 100, apg.Wait(0, clock)) + await apg._ensure_one(_owner(conn), "scope_generations", 100, apg.Wait(0, clock)) assert conn.executed[-1] == "CLOSE" assert "TERMINATE" not in conn.executed @@ -1194,7 +1201,7 @@ async def no_holders(_c: Any, _t: str) -> list[Any]: monkeypatch.setattr(cell_triggers, "planned_tables", planned) monkeypatch.setattr(cell_triggers, "holders", no_holders) logged: list[str] = [] - (refusal,) = await apg.ensure_triggers(conn, apg.Wait(2, clock), logged.append) + (refusal,) = await apg.ensure_triggers(_owner(conn), apg.Wait(2, clock), logged.append) named = "the migration lock (held by pid 42, loom, since t0)" assert logged == [f"loom.scope_generations: its lock is busy ({named}); retrying"] assert refusal.reason == f"its triggers were not created within --wait: {named}" @@ -1224,7 +1231,7 @@ async def nothing() -> None: pass await apg.commit_enforced( - conn, + _owner(conn), nothing, ok, apg.Wait(0, clock), @@ -1233,7 +1240,7 @@ async def nothing() -> None: expected_cell="first", expected_fp=(CURRENT, NEXT), ) - assert marks == ["unknown"] + assert marks == ["unknown", "enforced"] assert any(f"lock_timeout = '{apg.TRY_FLOOR_MS}ms'" in sql for sql in conn.executed) async def planned(_c: Any) -> list[str]: @@ -1253,7 +1260,7 @@ async def no_defects(_c: Any) -> list[Any]: monkeypatch.setattr(cell_triggers, "ensure_table", applied) monkeypatch.setattr(cell_triggers, "missing", no_defects) ddl = FakeConn([("deadlock_timeout", 1000)]) - assert await apg.ensure_triggers(ddl, apg.Wait(0, clock), lambda _t: None) == [] + assert await apg.ensure_triggers(_owner(ddl), apg.Wait(0, clock), lambda _t: None) == [] async def read() -> list[beats.Beat]: return [_beat("a")] @@ -1289,8 +1296,10 @@ async def confirm(_self: Any) -> None: pass async def run(self: Any) -> int: + self.owner = _owner(FakeConn()) self.row, self.mode = {"deployment_id": "dep-1"}, "recording" self.lock_conn = conn + self.owner = _owner(conn) self.wait = apg.Wait(2, clock) await self.commit_mode() return 0 @@ -1334,6 +1343,7 @@ async def refuse(_self: Any, path: str, _body: Any = None) -> dict[str, Any]: raise Refused([Refusal("router", f"refused {path}")]) async def run(self: Any) -> int: + self.owner = _owner(FakeConn()) self.row, self.mode = {"deployment_id": "dep-1"}, "recording" await self.commit_mode() return 0 @@ -1358,6 +1368,7 @@ async def read(_conn: Any) -> apg.Durable: return apg.Durable(mode="recording", first=(True, "first"), xid=1) async def run(self: Any) -> int: + self.owner = _owner(FakeConn()) self.mode = "unknown" # a mode commit whose outcome was lost await self.durable() return 0 @@ -1385,6 +1396,7 @@ async def read_mode(_conn: Any) -> tuple[str, Any]: return "enforced", (True, "first") async def run(self: Any) -> int: + self.owner = _owner(FakeConn()) self.row = {"is_first": True, "state": "attaching", "deployment_id": "dep-1"} self.lock_conn = FakeConn([("FROM loom.authority", "dep-1")]) if refusals := await self.stores_refusals(): @@ -1408,6 +1420,7 @@ async def test_an_interrupt_prints_the_mode_line_and_goes_on( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: async def run(self: Any) -> int: + self.owner = _owner(FakeConn()) self.mode = "unknown" raise KeyboardInterrupt diff --git a/tests/unit/test_cell_control.py b/tests/unit/test_cell_control.py index 74ac1c130d..6b2b821b91 100644 --- a/tests/unit/test_cell_control.py +++ b/tests/unit/test_cell_control.py @@ -134,3 +134,153 @@ async def stalled(_request): "/claim", {}, ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sqlstate", ["55P03", "57014"]) +@pytest.mark.parametrize("where", ["UPDATE loom.tenancy_mode", "COMMIT"]) +async def test_mode_contention_is_busy_only_before_commit_after_acknowledged_rollback( + sqlstate, where +): + from server.cli import cell_attach_pg as apg + from tests.unit.cell_control_owner import owner + from tests.unit.test_cell_attach import CURRENT, NEXT + + class Busy(Exception): + pass + + busy = Busy("expected contention") + busy.sqlstate = sqlstate + routes = [(where, busy)] + if where == "COMMIT": + routes.append(("UPDATE loom.tenancy_mode", "UPDATE 1")) + conn = FakeConn(routes) + owned = owner(conn) + marks = [] + if where == "COMMIT": + with pytest.raises(Busy) as found: + await apg._mode_try(owned, 100, marks.append, "first", (CURRENT, NEXT)) + assert found.value is busy + assert "ROLLBACK" not in conn.executed and marks == ["unknown"] + assert owned.closed + else: + assert not await apg._mode_try(owned, 100, marks.append, "first", (CURRENT, NEXT)) + assert conn.executed[-1] == "ROLLBACK" and marks == ["unknown", "recording"] + assert not owned.closed + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed", ["rollback", "ownership", "update-status"]) +async def test_terminal_mode_failure_never_resets_unknown_or_retries(failed): + import asyncpg + + from server.cli import cell_attach_pg as apg + from tests.unit.cell_control_owner import owner + from tests.unit.test_cell_attach import CURRENT, NEXT, _seq + + original = ValueError("rollback failed") + routes = [("UPDATE loom.tenancy_mode", "UPDATE 0")] + if failed != "update-status": + routes = [("UPDATE loom.tenancy_mode", asyncpg.LockNotAvailableError("busy"))] + if failed == "rollback": + routes.append(("ROLLBACK", original)) + if failed == "ownership": + routes.append((apg._HELD, _seq(True, False))) + conn, marks = FakeConn(routes), [] + with pytest.raises(ValueError if failed == "rollback" else cell_control.Refused) as found: + await apg.commit_enforced( + owner(conn), + AsyncMock(), + AsyncMock(return_value=[]), + apg.Wait(0), + lambda _: None, + marks.append, + expected_cell="first", + expected_fp=(CURRENT, NEXT), + ) + if failed == "rollback": + assert found.value is original + assert marks == ["unknown"] and conn.executed[-1] == "CLOSE" + assert conn.executed.count("BEGIN") == 1 + + +@pytest.mark.asyncio +async def test_expired_phase_does_not_receive_another_exit_allowance(monkeypatch): + import time + + class StalledClose(FakeConn): + async def close(self, **kwargs): + self.timeouts = [kwargs["timeout"]] + self.executed.append("CLOSE START") + await asyncio.Future() + + conn = StalledClose() + + async def opening(self, _what): + self.conn = conn + + monkeypatch.setattr(cell_control.Owner, "_open", opening) + monkeypatch.setattr(cell_control, "TRANSACTION_S", 0.04) + + async def body(owned): + return await owned.transaction(lambda: asyncio.Future()) + + started = time.monotonic() + with pytest.raises(TimeoutError): + await cell_control.run("test", body) + assert time.monotonic() - started < 0.3 + assert conn.executed.count("CLOSE START") == 1 + assert 0 < conn.timeouts[0] <= 0.02 + assert "TERMINATE" in conn.executed + + +@pytest.mark.asyncio +async def test_late_connection_is_discarded_without_starting_command_work(monkeypatch): + from server.core import pg + + connected, discarded = asyncio.Event(), asyncio.Event() + + class Late(FakeConn): + def terminate(self): + super().terminate() + discarded.set() + + conn = Late() + + async def connect(**_): + try: + await asyncio.Future() + except asyncio.CancelledError: + while not connected.is_set(): + try: + await connected.wait() + except asyncio.CancelledError: + pass + return conn + + monkeypatch.setattr(pg, "_connect_dedicated", connect) + monkeypatch.setattr(cell_control, "OPEN_S", 0.04) + body = AsyncMock() + with pytest.raises(TimeoutError): + await cell_control.run("late", body) + connected.set() + await asyncio.wait_for(discarded.wait(), 0.3) + body.assert_not_awaited() + assert conn.executed == ["TERMINATE"] + + +@pytest.mark.asyncio +async def test_successful_body_cannot_hide_failed_required_close(monkeypatch): + class Broken(FakeConn): + async def close(self, **_): + raise OSError("required close failed") + + conn = Broken() + + async def opening(self, _what): + self.conn = conn + + monkeypatch.setattr(cell_control.Owner, "_open", opening) + with pytest.raises(OSError, match="required close failed"): + await cell_control.run("test", AsyncMock(return_value=7)) + assert "TERMINATE" in conn.executed diff --git a/tests/unit/test_cell_control_census.py b/tests/unit/test_cell_control_census.py index aec52c908c..0c32387393 100644 --- a/tests/unit/test_cell_control_census.py +++ b/tests/unit/test_cell_control_census.py @@ -102,16 +102,16 @@ def test_cell_control_migrations_are_classified(): # Entries name every sink, including nested callbacks and dynamic table expressions. # A new sink, including another statement in an existing owner, requires classification. GUARDED_WRITERS = { - "server/cli/cell_attach_pg.py": {("_mode_try", "UPDATE", "loom.tenancy_mode"): 1}, + "server/cli/cell_attach_pg.py": {("_mode_transaction", "UPDATE", "loom.tenancy_mode"): 1}, "server/cli/cell_control.py": { - ("backout", "ALTER TABLE", "loom.cell_first"): 3, - ("backout", "DELETE FROM", "loom.{table}"): 1, - ("backout", "DROP TRIGGER", "loom.{table}"): 1, + ("backout.clear_state", "ALTER TABLE", "loom.cell_first"): 3, + ("backout.clear_state", "DELETE FROM", "loom.{table}"): 1, + ("backout.clear_triggers", "DROP TRIGGER", "loom.{table}"): 1, }, "server/cli/cell_prepare.py": { - ("_write_rows", "INSERT INTO", "loom.cell_first"): 1, - ("_write_rows", "INSERT INTO", "loom.tenancy_mode"): 1, - ("_write_rows", "INSERT INTO", "loom.cell_self"): 1, + ("_write_rows.write", "INSERT INTO", "loom.cell_first"): 1, + ("_write_rows.write", "INSERT INTO", "loom.tenancy_mode"): 1, + ("_write_rows.write", "INSERT INTO", "loom.cell_self"): 1, }, "server/router/directory.py": { ("set_limit.change", "UPDATE", "loom_router.cells"): 1, @@ -307,35 +307,54 @@ def function(path: str, name: str) -> ast.AST: @pytest.mark.parametrize( ("path", "name", "required"), [ - ("server/cli/cell_prepare.py", "attach_lock", ['cell_control.session("prepare")']), + ( + "server/cli/cell_prepare.py", + "prepare", + [ + 'cell_control.run("prepare", lambda owner: _prepare_locked(ch, args, router, transport, owner))' + ], + ), ( "server/cli/cell_prepare.py", "_prepare_locked", [ "cell_control.claim(conn, call, snapshot)", - "_write_rows(conn, rows, answer, current, nxt)", + "owner.call(lambda: _write_rows(conn, rows, answer, current, nxt), total_s=cell_control.TRANSACTION_S)", ], ), ( "server/cli/cell_prepare.py", "_write_rows", - ["cell_control.transaction(conn)", "_read_rows(conn, count_tenancy=False)"], + ["cell_control.transaction(conn, write)", "_read_rows(conn, count_tenancy=False)"], + ), + ("server/cli/cell_attach.py", "Run.run", ['cell_control.run("attach", self._owned)']), + ( + "server/cli/cell_attach_pg.py", + "_mode_try", + [ + "owner.call(lambda: _mode_transaction(owner.conn, lock_ms, mark, expected_cell, expected_fp), total_s=bound_s)" + ], + ), + ( + "server/cli/cell_control.py", + "Owner.transaction", + ["self.call(lambda: transaction(self.conn, work), total_s=TRANSACTION_S)"], ), ( "server/cli/cell_attach.py", - "Run.run", - ["self.open_lock()", "cell_control.claim(self.lock_conn, self.router.call, self.row)"], + "Run._owned", + ["cell_control.claim(self.lock_conn, self.router.call, self.row)"], ), ( "server/cli/cell_attach.py", "Run.commit_mode", [ - "apg.commit_enforced(self.lock_conn, self.confirm_lock, self.ready_for_commit, self.wait, self.log, mark, expected_cell=self.name, expected_fp=self.own_fp)" + "apg.commit_enforced(self.owner, self.confirm_lock, self.ready_for_commit, self.wait, self.log, mark, expected_cell=self.name, expected_fp=self.own_fp)" ], ), ( "server/cli/cell_attach_pg.py", - "_mode_try", + "_mode_transaction", ["cell_control.check_lock(conn)", "conn.transaction()", "tx.commit()"], ), ( @@ -354,7 +373,8 @@ def function(path: str, name: str) -> ast.AST: [ "claim(conn, call, row)", 'mutate(conn, call, row, "/limit", {"closed": True})', - "transaction(conn)", + "owner.transaction(clear_state)", + "owner.transaction(clear_triggers)", "paired(conn, row)", ], ), @@ -362,7 +382,7 @@ def function(path: str, name: str) -> ast.AST: "server/cli/cell.py", "_remove_locked", [ - 'cell_control.session("remove")', + 'cell_control.run("remove", remove)', "cell_control.claim(conn, call, row)", "cell_control.check_lock(conn)", ], @@ -483,7 +503,7 @@ def test_inventory_detects_new_literal_and_dynamic_writers(statement, expected): def test_acceptance_writer_callsites_are_classified(): # Another caller of a checked writer still needs its own claim and ownership proof. - names = {"_write_rows", "_mode_try", "commit_enforced", "backout"} + names = {"_write_rows", "_mode_try", "_mode_transaction", "commit_enforced", "backout"} actual = Counter() for path in (ROOT / "server").rglob("*.py"): found = inventory(path.read_text()) @@ -498,6 +518,7 @@ def test_acceptance_writer_callsites_are_classified(): { ("server/cli/cell_prepare.py", "_prepare_locked", "_write_rows"): 1, ("server/cli/cell_attach_pg.py", "commit_enforced", "_mode_try"): 1, + ("server/cli/cell_attach_pg.py", "_mode_try", "_mode_transaction"): 1, ("server/cli/cell_attach.py", "Run.commit_mode", "apg.commit_enforced"): 1, } ) diff --git a/tests/unit/test_cell_prepare.py b/tests/unit/test_cell_prepare.py index 58040a584d..7c6e8f07fc 100644 --- a/tests/unit/test_cell_prepare.py +++ b/tests/unit/test_cell_prepare.py @@ -29,6 +29,7 @@ def _cell(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(settings, "storage_authority", "postgres") monkeypatch.setattr(settings, "pg_dsn", "postgresql://cell.internal/loom") monkeypatch.setattr(settings, "router_operator_token", OPERATOR) + # The attach lock's session is Postgres's (tests/integration/test_cell_attach.py). class Owner: conn = None diff --git a/tests/unit/test_router_attach.py b/tests/unit/test_router_attach.py index 1471f5882b..226a1b54c0 100644 --- a/tests/unit/test_router_attach.py +++ b/tests/unit/test_router_attach.py @@ -1102,8 +1102,17 @@ async def held(_conn: Any) -> None: pass monkeypatch.setattr(cell_control, "paired", paired) - monkeypatch.setattr(cell_control, "session", lambda _what: contextlib.nullcontext(object())) - monkeypatch.setattr(cell_control, "transaction", lambda _conn: contextlib.nullcontext()) + + class Owner: + conn = object() + + async def transaction(self, work): + return await work() + + async def run(_what, work): + return await work(Owner()) + + monkeypatch.setattr(cell_control, "run", run) monkeypatch.setattr(cell_control, "check_lock", held) monkeypatch.setattr(cell, "read_state_if_present", state) return checked From f9986b542078721e0e555e8bd861392f6b0e746b Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 23:53:53 +0000 Subject: [PATCH 031/114] Propagate unexpected trigger errors before rollback cleanup Signed-off-by: zlareb1 --- server/core/cell_triggers.py | 5 ++- tests/integration/test_cell_attach_cleanup.py | 44 ++++++++++++++++++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 68e74777ec..784351d215 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -337,7 +337,10 @@ async def _run_ddl( await conn.execute(f"SET LOCAL lock_timeout = '{lock_timeout}'") for sql in sqls: await conn.execute(sql) - except Exception: + except ( + asyncpg.exceptions.InsufficientPrivilegeError, + asyncpg.exceptions.LockNotAvailableError, + ): await tx.rollback() rolled_back = True raise diff --git a/tests/integration/test_cell_attach_cleanup.py b/tests/integration/test_cell_attach_cleanup.py index 145be1188d..ca34ee7629 100644 --- a/tests/integration/test_cell_attach_cleanup.py +++ b/tests/integration/test_cell_attach_cleanup.py @@ -46,7 +46,7 @@ async def intercepted(self, query, *args, **kwargs): if query.startswith("SET LOCAL statement_timeout"): return await execute(self, "SET LOCAL statement_timeout = '10000ms'") if query == "RAISE Python error": - raise ValueError("trigger statement failed") + raise asyncpg.LockNotAvailableError("trigger statement busy") if phase == "rollback" and query.startswith("ROLLBACK"): await execute(self, "SELECT pg_sleep(10)") if owner == "mode" and query.startswith("LOCK TABLE"): @@ -109,3 +109,45 @@ async def dedicated(**_): if not task.done(): task.cancel() await asyncio.gather(task, return_exceptions=True) + + +async def test_unexpected_trigger_error_reaches_owner_before_rollback(database, monkeypatch): + import asyncpg + + from server.core import cell + + conn, observer = database + pid = conn.get_server_pid() + await conn.execute("SELECT pg_advisory_lock($1)", cell.ATTACH_LOCK_KEY) + execute = asyncpg.Connection.execute + original = ValueError("unexpected trigger operation failed") + + async def intercepted(self, query, *args, **kwargs): + if self is conn: + if query == "RAISE unexpected Python error": + assert await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1 AND locktype='advisory')", + pid, + ) + raise original + if query.startswith("ROLLBACK"): + await execute(self, "SELECT pg_sleep(10)") + return await execute(self, query, *args, **kwargs) + + monkeypatch.setattr(asyncpg.Connection, "execute", intercepted) + started = time.monotonic() + with pytest.raises(ValueError) as error: + await apg.supervise( + conn, + cell_triggers._run_ddl( + conn, + "probe", + ["SELECT * FROM probe FOR UPDATE", "RAISE unexpected Python error"], + attempts=1, + warn=False, + ), + end=started + 0.5, + ) + assert error.value is original + assert time.monotonic() - started < 0.5 + await released(observer, pid) From 5a6207098742d4d1a2b87b3af59942eb6f4f070e Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 23:53:53 +0000 Subject: [PATCH 032/114] Propagate unexpected trigger errors before rollback cleanup Signed-off-by: zlareb1 --- server/core/cell_triggers.py | 5 ++- tests/integration/test_cell_attach_cleanup.py | 43 ++++++++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 68e74777ec..784351d215 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -337,7 +337,10 @@ async def _run_ddl( await conn.execute(f"SET LOCAL lock_timeout = '{lock_timeout}'") for sql in sqls: await conn.execute(sql) - except Exception: + except ( + asyncpg.exceptions.InsufficientPrivilegeError, + asyncpg.exceptions.LockNotAvailableError, + ): await tx.rollback() rolled_back = True raise diff --git a/tests/integration/test_cell_attach_cleanup.py b/tests/integration/test_cell_attach_cleanup.py index 3b65c81b7d..77b3e297ce 100644 --- a/tests/integration/test_cell_attach_cleanup.py +++ b/tests/integration/test_cell_attach_cleanup.py @@ -45,7 +45,7 @@ async def intercepted(self, query, *args, **kwargs): if query.startswith("SET LOCAL statement_timeout"): return await execute(self, "SET LOCAL statement_timeout = '10000ms'") if query == "RAISE Python error": - raise ValueError("trigger statement failed") + raise asyncpg.LockNotAvailableError("trigger statement busy") if phase == "rollback" and query.startswith("ROLLBACK"): await execute(self, "SELECT pg_sleep(10)") if owner == "mode" and query.startswith("LOCK TABLE"): @@ -111,3 +111,44 @@ async def body(_owner): if not task.done(): task.cancel() await asyncio.gather(task, return_exceptions=True) + + +async def test_unexpected_trigger_error_reaches_owner_before_rollback(database, monkeypatch): + import asyncpg + + from server.core import cell + + conn, observer = database + pid = conn.get_server_pid() + await conn.execute("SELECT pg_advisory_lock($1)", cell.ATTACH_LOCK_KEY) + execute = asyncpg.Connection.execute + original = ValueError("unexpected trigger operation failed") + + async def intercepted(self, query, *args, **kwargs): + if self is conn: + if query == "RAISE unexpected Python error": + assert await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1 AND locktype='advisory')", + pid, + ) + raise original + if query.startswith("ROLLBACK"): + await execute(self, "SELECT pg_sleep(10)") + return await execute(self, query, *args, **kwargs) + + monkeypatch.setattr(asyncpg.Connection, "execute", intercepted) + started = time.monotonic() + with pytest.raises(ValueError) as error: + await _owner(conn).call( + lambda: cell_triggers._run_ddl( + conn, + "probe", + ["SELECT * FROM probe FOR UPDATE", "RAISE unexpected Python error"], + attempts=1, + warn=False, + ), + total_s=0.5, + ) + assert error.value is original + assert time.monotonic() - started < 0.5 + await released(observer, pid) From deacbaa8c36f2691d461669d1474e068dccec8a3 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 00:14:57 +0000 Subject: [PATCH 033/114] Close directory control sessions before cancelling their work Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 1 + docs/architecture/cells.md | 19 +- server/router/directory.py | 228 +++++++++++--- tests/integration/test_cell_control_fence.py | 6 +- .../test_directory_control_cleanup.py | 296 ++++++++++++++++++ tests/integration/test_router_directory_pg.py | 11 +- tests/unit/test_cell_attach_cleanup.py | 62 ++-- tests/unit/test_cell_control_census.py | 54 +++- tests/unit/test_directory_control_cleanup.py | 136 ++++++++ tests/unit/test_router_attach.py | 10 +- 10 files changed, 735 insertions(+), 88 deletions(-) create mode 100644 tests/integration/test_directory_control_cleanup.py create mode 100644 tests/unit/test_directory_control_cleanup.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b5b7aced8..1abff33cbe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -904,6 +904,7 @@ jobs: tests/integration/test_cell_attach_cleanup.py tests/integration/test_cell_control_cleanup.py tests/integration/test_cell_control_fence.py + tests/integration/test_directory_control_cleanup.py tests/integration/test_cell_org_triggers_pg.py - name: pytest online-index durability on a private physical standby timeout-minutes: 8 diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index c0a4115499..a0079ac900 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -990,10 +990,12 @@ advisory lock, `cell.ATTACH_LOCK_KEY`, on a session of its own for its life (pre takes the same lock), after checking that the session is one direct backend that keeps the tenancy marker: a lock left on a pooler's server connection would refuse every later run, and attach's tenancy rows would be refused as an unchecked -writer's. A lock found lost between steps is taken again (refusing when another run holds -it), and the cell's state is read again. The lock does not make a run safe, since every -step is idempotent and the mode only rises; it keeps a second run's scans from holding the -snapshot horizon that step 4 waits on. +writer's. A lost or unreadable lock session ends the run: it never reconnects or +reacquires the lock. Rerun attach to obtain a fresh directory claim and recompute the +remaining work. Every local acceptance write uses this exact backend, so losing the +lock also ends its authority to change acceptance. The lock serializes prepare, attach +and update, and keeps concurrent scans from holding the snapshot horizon that step 4 +waits on. It refuses, before any write, naming each: @@ -1268,6 +1270,15 @@ that the same backend still holds ATTACH. A COMMIT, rollback or ownership-check never becomes a busy retry. After attempting the mode update, the diagnostic stays unknown until rollback plus ownership or COMMIT is acknowledged. +Directory control mutations supervise the complete callback transaction independently, +including acquisition and COMMIT. One 131-second bound includes public close, child +settlement and pool-context exit, with cleanup time reserved inside it. Both a failed +attempt and an acknowledged successful transaction close only their own acquired +connection before releasing the pool context. Unexpected errors never wait for rollback +on that discarded session. A late acquisition cannot start mutation work or obtain a +fresh cleanup allowance; acquisition without a known proxy remains an unknown outcome. +Ordinary live placement retains its separate transaction and connection handling. + The source census for acceptance and operator directory writes is checked by `tests/unit/test_cell_control_census.py`, including dynamic cleanup tables, guarded entry points and the named exceptions: diff --git a/server/router/directory.py b/server/router/directory.py index 5158d24cf7..42c48ff1fa 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -20,6 +20,7 @@ import asyncio import hmac import json +import time from collections.abc import AsyncIterator, Awaitable, Callable, Iterable, Mapping from contextlib import AsyncExitStack, asynccontextmanager from dataclasses import dataclass @@ -28,6 +29,7 @@ from uuid import UUID from server.core import cell as cell_state +from server.core import pg from server.core.paths import ROUTER_MIGRATIONS_DIR from server.router.limits import MAX_BODY_BYTES from server.settings import settings @@ -383,11 +385,13 @@ async def _operator_change( change: Callable[[Any, Cell], Awaitable[dict[str, Any]]], expected: Control, ) -> Cell: - async with _attach_change(cell_name, expected) as (conn, found): + async def change_locked(conn: Any, found: Attaching) -> Cell: detail = await change(conn, found.row) await _audit(conn, action, {"cell": cell_name, **detail}) row = await conn.fetchrow(f"{_CELL} WHERE cell = $1", cell_name) - return _cell(row) + return _cell(row) + + return await _attach_change(cell_name, expected, change_locked) @dataclass(frozen=True) @@ -898,34 +902,159 @@ def control_refusal(found: Attaching, expected: Control, *, claim: bool = False) return "" -@asynccontextmanager -async def _control_transaction() -> AsyncIterator[Any]: - """Bound acquisition, statements, COMMIT and release; never await rollback on cancel.""" - async with _reading(), asyncio.timeout(CONTROL_MAX_S), _connection() as conn: - settled = False - tx = conn.transaction() +@dataclass(frozen=True) +class _ControlFatal: + error: SystemExit | KeyboardInterrupt + + +async def _control_result[T](work: Callable[[], Awaitable[T]]) -> T | _ControlFatal: + try: + return await work() + except (SystemExit, KeyboardInterrupt) as error: + return _ControlFatal(error) + except _ControlInterrupted as error: + return error.fatal + + +@dataclass(frozen=True) +class _ControlConnection: + conn: Any + + async def close(self, **kwargs: Any) -> None: try: - await tx.start() + await self.conn.close(**kwargs) + except (SystemExit, KeyboardInterrupt) as error: + # pg.close_owned creates its own public-close child. + raise _ControlInterrupted(_ControlFatal(error)) from None + + def terminate(self) -> None: + pg._terminate(self.conn) + + +class _ControlOwner: + """Own only this operator attempt's proxy, through close and pool release.""" + + def __init__(self) -> None: + self.end = time.monotonic() + CONTROL_MAX_S + self.work_end = self.end - min(1.0, CONTROL_MAX_S / 4) + self.context = _connection() + self.conn: Any = None + self.entered = False + self.closing = False + self.close_task: asyncio.Task[Any] | None = None + self.exit_task: asyncio.Task[Any] | None = None + + def start[T](self, work: Callable[[], Awaitable[T]]) -> asyncio.Task[T | _ControlFatal]: + task = asyncio.create_task(_control_result(work)) + task.add_done_callback(pg._retrieve_task) + return task + + async def wait[T](self, task: asyncio.Task[T | _ControlFatal], end: float) -> T: + done, _ = await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) + if not done: + raise TimeoutError("directory control deadline expired; outcome unknown") + result = task.result() + if isinstance(result, _ControlFatal): + # Keep fatal exceptions inert in nested acquisition/cleanup children. + raise _ControlInterrupted(result) + return result + + async def close(self) -> None: + if self.conn is None: + return + if self.close_task is None: + self.close_task = self.start( + lambda: pg.close_owned(_ControlConnection(self.conn), end=self.end) + ) + try: + await self.wait(self.close_task, self.end) + except BaseException: + pg._terminate(self.conn) + self.close_task.cancel() + raise + + async def release(self) -> None: + if self.entered: + if self.exit_task is None: + self.exit_task = self.start(lambda: self.context.__aexit__(None, None, None)) try: - yield conn - except Exception: - await tx.rollback() - settled = True + await self.wait(self.exit_task, self.end) + except BaseException: + self.exit_task.cancel() raise - else: - await tx.commit() - settled = True - finally: - if not settled: - # A cancelled/unknown COMMIT cannot return this backend to the pool. - conn.terminate() + + async def work[T](self, change: Callable[[Any], Awaitable[T]]) -> T: + self.conn = await self.context.__aenter__() + self.entered = True + if self.closing: + # A cancellation-resistant acquire cannot hand a late proxy to a mutation. + try: + await self.close() + finally: + await self.release() + raise TimeoutError("directory connection arrived after its attempt ended") + tx = self.conn.transaction() + await tx.start() + result = await change(self.conn) + await tx.commit() + return result + + async def finish(self, task: asyncio.Task[Any]) -> None: + self.closing = True + failure: BaseException | None = None + try: + await self.close() + except BaseException as error: # noqa: BLE001 — release must preserve the first failure. + failure = error + if not task.done(): + task.cancel() + try: + await asyncio.wait({task}, timeout=max(0, self.end - time.monotonic())) + except BaseException as error: # noqa: BLE001 — cleanup keeps the first failure. + failure = failure or error + try: + await self.release() + except BaseException as error: # noqa: BLE001 — cleanup keeps the first failure. + failure = failure or error + if failure is not None: + raise failure -@asynccontextmanager -async def _attach_change( - cell_name: str, expected: Control, *, claim: bool = False -) -> AsyncIterator[tuple[Any, Attaching]]: - async with _control_transaction() as conn: +class _ControlInterrupted(BaseException): + def __init__(self, fatal: _ControlFatal) -> None: + self.fatal = fatal + + +async def _control_transaction[T](change: Callable[[Any], Awaitable[T]]) -> T: + """Bound the whole callback, close first, then settle work and pool ownership.""" + async with _reading(): + owner = _ControlOwner() + task = owner.start(lambda: owner.work(change)) + try: + result = await owner.wait(task, owner.work_end) + except BaseException as original: + try: + await owner.finish(task) + except BaseException: # noqa: BLE001 — preserve the original failure/cancellation. + pass + if isinstance(original, _ControlInterrupted): + raise original.fatal.error from None + raise + try: + await owner.finish(task) + except _ControlInterrupted as interrupted: + raise interrupted.fatal.error from None + return result + + +async def _attach_change[T]( + cell_name: str, + expected: Control, + change: Callable[[Any, Attaching], Awaitable[T]], + *, + claim: bool = False, +) -> T: + async def locked(conn: Any) -> T: await conn.execute("SELECT pg_advisory_xact_lock($1)", _DIRECTORY_LOCK) found = await conn.fetchrow(f"{_CELL_ATTACH} WHERE cell = $1 FOR UPDATE", cell_name) if found is None: @@ -934,22 +1063,26 @@ async def _attach_change( attaching = _attaching(found, other) if reason := control_refusal(attaching, expected, claim=claim): raise OperatorRefused(reason) - # A no-op still has a synchronous WAL commit: a primary-visible earlier write - # whose acknowledgement was lost is not yet a durability proof. + # Even a no-op needs synchronous WAL acknowledgement. await conn.execute("SELECT pg_logical_emit_message(true, 'loom.control', '')") - yield conn, attaching + return await change(conn, attaching) + + return await _control_transaction(locked) async def claim(cell_name: str, expected: Control) -> Attaching: """Rotate on EVERY execution, including an identical HTTP replay.""" - async with _attach_change(cell_name, expected, claim=True) as (conn, _found): + + async def change_locked(conn: Any, _found: Attaching) -> Attaching: await conn.execute( "UPDATE loom_router.cells SET mutation_generation = gen_random_uuid() WHERE cell = $1", cell_name, ) await _audit(conn, "control_claim", {"cell": cell_name}) written = await conn.fetchrow(f"{_CELL_ATTACH} WHERE cell = $1", cell_name) - return _attaching(written, _found.other) + return _attaching(written, _found.other) + + return await _attach_change(cell_name, expected, change_locked, claim=True) async def _audit(conn: Any, action: str, detail: dict[str, Any]) -> None: @@ -964,7 +1097,8 @@ async def attach_begin(cell_name: str, begin: Begin, *, expected: Control) -> At """`attach/begin`: write `attach_started_at` once, the fingerprints while the row has none, and the root-secret fingerprint on every call, audited with the allow lists. Idempotent: a rerun posting the same fingerprints writes only `keys_fp`.""" - async with _attach_change(cell_name, expected) as (conn, found): + + async def change_locked(conn: Any, found: Attaching) -> Attaching: if reason := begin_refusal(found, begin): raise OperatorRefused(reason) await conn.execute( @@ -987,7 +1121,9 @@ async def attach_begin(cell_name: str, begin: Begin, *, expected: Control) -> At }, ) written = await conn.fetchrow(f"{_CELL_ATTACH} WHERE cell = $1", cell_name) - return _attaching(written, None) + return _attaching(written, None) + + return await _attach_change(cell_name, expected, change_locked) async def attach_place( @@ -1002,7 +1138,8 @@ async def attach_place( reads `enforced`, judges that.""" _refuse_long(orgs) batch = sorted(set(orgs)) - async with _attach_change(cell_name, expected) as (conn, found): + + async def change_locked(conn: Any, found: Attaching) -> tuple[int, int]: if reason := attach_refusal(found, deployment_id, enforced=True): raise OperatorRefused(reason) new = await conn.fetch( @@ -1041,13 +1178,16 @@ async def attach_place( len(batch), len(new), ) - return len(new), int(placed) + return len(new), int(placed) + + return await _attach_change(cell_name, expected, change_locked) async def attach_enforced(cell_name: str, deployment_id: str, *, expected: Control) -> Attaching: """`attach/enforced`: record directory mode `enforced` (registration wrote `recording`), after which the cell places new orgs live; idempotent.""" - async with _attach_change(cell_name, expected) as (conn, found): + + async def change_locked(conn: Any, found: Attaching) -> Attaching: if reason := attach_refusal(found, deployment_id): raise OperatorRefused(reason) await conn.execute( @@ -1061,7 +1201,9 @@ async def attach_enforced(cell_name: str, deployment_id: str, *, expected: Contr {"cell": cell_name, "changed": found.row.mode != "enforced"}, ) written = await conn.fetchrow(f"{_CELL_ATTACH} WHERE cell = $1", cell_name) - return _attaching(written, None) + return _attaching(written, None) + + return await _attach_change(cell_name, expected, change_locked) def issuers(row: Cell) -> list[str]: @@ -1079,7 +1221,8 @@ def issuers(row: Cell) -> list[str]: async def attach_serving(cell_name: str, deployment_id: str, *, expected: Control) -> Attaching: """`attach/serving`: mark the first cell `serving`, with the issuers its fingerprints accept computed from the row as locked. A `serving` row is answered as it is.""" - async with _attach_change(cell_name, expected) as (conn, found): + + async def change_locked(conn: Any, found: Attaching) -> Attaching: row = found.row if found.served(deployment_id): await _audit(conn, "attach_serving", {"cell": cell_name, "serving_already": True}) @@ -1095,7 +1238,9 @@ async def attach_serving(cell_name: str, deployment_id: str, *, expected: Contro ) await _audit(conn, "attach_serving", {"cell": cell_name, "issuers": accepted}) written = await conn.fetchrow(f"{_CELL_ATTACH} WHERE cell = $1", cell_name) - return _attaching(written, None) + return _attaching(written, None) + + return await _attach_change(cell_name, expected, change_locked) def remove_refusal(cell_name: str, state: str, placed: int) -> str: @@ -1118,7 +1263,8 @@ async def remove(cell_name: str, *, expected: Control) -> Cell: first cell's Postgres reads `open` first, and a direct call skips that check, leaving a first cell still prepared answering its new orgs 503 and unable to register again until the full back-out has run.""" - async with _attach_change(cell_name, expected) as (conn, found): + + async def change_locked(conn: Any, found: Attaching) -> Cell: row = found.row if reason := remove_refusal(cell_name, row.state, row.placed): raise OperatorRefused(reason) @@ -1135,7 +1281,9 @@ async def remove(cell_name: str, *, expected: Control) -> Cell: "mode": row.mode, }, ) - return row + return row + + return await _attach_change(cell_name, expected, change_locked) async def lookup(org: str) -> tuple[str, str] | None: diff --git a/tests/integration/test_cell_control_fence.py b/tests/integration/test_cell_control_fence.py index fa375eb12a..e14eab1220 100644 --- a/tests/integration/test_cell_control_fence.py +++ b/tests/integration/test_cell_control_fence.py @@ -370,11 +370,13 @@ async def finalizer(request): await resume.wait() # request already sent; caller can no longer retract it async def change(): - async with directory._attach_change("first", compare) as (conn, found): + async def mutate(conn, found): await conn.execute( "UPDATE loom_router.cells SET current_fp = next_fp, next_fp = NULL, issuers = ARRAY['new'] WHERE cell = 'first'" ) - return found + return found + + return await directory._attach_change("first", compare, mutate) return await app._answered(change, app._attach_answer) diff --git a/tests/integration/test_directory_control_cleanup.py b/tests/integration/test_directory_control_cleanup.py new file mode 100644 index 0000000000..9970d6b5ee --- /dev/null +++ b/tests/integration/test_directory_control_cleanup.py @@ -0,0 +1,296 @@ +"""Directory control closes its actual backend before an interrupted call returns.""" + +from __future__ import annotations + +import asyncio +import os +import time +from collections.abc import AsyncIterator +from typing import Any + +import pytest + +from server.router import directory +from server.settings import settings +from tests.integration._pg_database import create_database, drop_database +from tests.integration.test_pg_owned_cleanup import released, sleeping + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +async def directory_database(monkeypatch) -> AsyncIterator[Any]: + import asyncpg + + admin = os.environ.get("LOOM_PG_DSN") or pytest.skip("LOOM_PG_DSN not set") + name, dsn = await create_database(admin) + monkeypatch.setattr(settings, "router_pg_dsn", dsn) + monkeypatch.setattr(directory, "_pool", None) + observer = await asyncpg.connect(dsn) + await observer.execute("CREATE TABLE probe(id int PRIMARY KEY); INSERT INTO probe VALUES(1)") + try: + yield observer + finally: + await directory.close() + await observer.close(timeout=1) + await drop_database(admin, name) + + +async def locked(conn): + await conn.execute("SELECT pg_advisory_xact_lock($1)", directory._DIRECTORY_LOCK) + await conn.execute("SELECT * FROM probe FOR UPDATE") + + +@pytest.mark.parametrize("interrupt", ["cancel", "deadline"]) +async def test_directory_interrupt_releases_exact_backend_and_global_locks( + directory_database, monkeypatch, interrupt +): + seen = asyncio.Event() + backend = {} + monkeypatch.setattr(directory, "CONTROL_MAX_S", 0.5) + + async def work(conn): + backend["pid"] = conn.get_server_pid() + await locked(conn) + seen.set() + await conn.execute("SELECT pg_sleep(10)") + + task = asyncio.create_task(directory._control_transaction(work)) + try: + await asyncio.wait_for(seen.wait(), 2) + await sleeping(directory_database, backend["pid"]) + began = time.monotonic() + if interrupt == "cancel": + task.cancel("original directory cancellation") + with pytest.raises( + asyncio.CancelledError if interrupt == "cancel" else directory.DirectoryUnavailable + ): + await asyncio.wait_for(task, 1) + assert time.monotonic() - began < 0.6 + await released(directory_database, backend["pid"]) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_directory_original_failure_never_waits_for_rollback(directory_database, monkeypatch): + import asyncpg + + original = ValueError("original directory work failure") + real = asyncpg.Connection.execute + backend = {} + monkeypatch.setattr(directory, "CONTROL_MAX_S", 0.4) + + async def stalled(self, query, *args, **kwargs): + if query.startswith("ROLLBACK"): + backend["rollback"] = True + await real(self, "SELECT pg_sleep(10)") + return await real(self, query, *args, **kwargs) + + monkeypatch.setattr(asyncpg.Connection, "execute", stalled) + began = time.monotonic() + + async def work(conn): + backend["pid"] = conn.get_server_pid() + await locked(conn) + raise original + + with pytest.raises(ValueError) as caught: + await directory._control_transaction(work) + assert caught.value is original and "rollback" not in backend + assert time.monotonic() - began < 0.4 + await released(directory_database, backend["pid"]) + + +async def test_directory_normal_commit_closes_before_return(directory_database): + backend = {} + + async def work(conn): + backend["pid"] = conn.get_server_pid() + await locked(conn) + await conn.execute("UPDATE probe SET id=2") + return "acknowledged" + + assert await directory._control_transaction(work) == "acknowledged" + assert await directory_database.fetchval("SELECT id FROM probe") == 2 + await released(directory_database, backend["pid"]) + + +async def test_directory_second_cancel_keeps_first_and_close_precedes_worker_cancel( + directory_database, monkeypatch +): + import asyncpg + + real_close = asyncpg.Connection.close + seen, closing, allow_close = asyncio.Event(), asyncio.Event(), asyncio.Event() + backend = {} + events = [] + + async def work(conn): + backend["pid"] = conn.get_server_pid() + await locked(conn) + seen.set() + try: + await conn.execute("SELECT pg_sleep(10)") + finally: + events.append("work stopped") + + async def close(self, **kwargs): + if self.get_server_pid() == backend.get("pid"): + events.append("close started") + await real_close(self, **kwargs) + closing.set() + await allow_close.wait() + else: + await real_close(self, **kwargs) + + monkeypatch.setattr(asyncpg.Connection, "close", close) + task = asyncio.create_task(directory._control_transaction(work)) + try: + await asyncio.wait_for(seen.wait(), 2) + await sleeping(directory_database, backend["pid"]) + task.cancel("first directory cancellation") + await asyncio.wait_for(closing.wait(), 1) + task.cancel("second directory cancellation") + with pytest.raises(asyncio.CancelledError, match="first directory cancellation"): + await asyncio.wait_for(task, 0.5) + assert events.index("close started") < events.index("work stopped") + assert events.count("close started") == 1 + await released(directory_database, backend["pid"]) + finally: + allow_close.set() + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_directory_commit_acknowledgement_loss_is_not_success( + directory_database, monkeypatch +): + import asyncpg + + real = asyncpg.Connection.execute + backend = {} + original = OSError("commit acknowledgement lost") + + async def execute(self, sql, *args, **kwargs): + value = await real(self, sql, *args, **kwargs) + if sql.startswith("COMMIT") and self.get_server_pid() == backend.get("pid"): + raise original + return value + + async def work(conn): + backend["pid"] = conn.get_server_pid() + await locked(conn) + await conn.execute("UPDATE probe SET id=2") + + monkeypatch.setattr(asyncpg.Connection, "execute", execute) + with pytest.raises(directory.DirectoryUnavailable) as caught: + await directory._control_transaction(work) + assert caught.value.__cause__ is original + assert await directory_database.fetchval("SELECT id FROM probe") == 2 + await released(directory_database, backend["pid"]) + + +async def test_directory_required_close_failure_refuses_committed_success( + directory_database, monkeypatch +): + import asyncpg + + real_close = asyncpg.Connection.close + backend = {} + original = ValueError("required close failed") + + async def close(self, **kwargs): + ours = self.get_server_pid() == backend.get("pid") + await real_close(self, **kwargs) + if ours: + raise original + + async def work(conn): + backend["pid"] = conn.get_server_pid() + await locked(conn) + await conn.execute("UPDATE probe SET id=2") + + monkeypatch.setattr(asyncpg.Connection, "close", close) + with pytest.raises(ValueError) as caught: + await directory._control_transaction(work) + assert caught.value is original + assert await directory_database.fetchval("SELECT id FROM probe") == 2 + await released(directory_database, backend["pid"]) + + +@pytest.mark.parametrize("stage", ["acquire", "work", "commit", "close", "release"]) +@pytest.mark.parametrize("kind", ["exit", "keyboard"]) +async def test_directory_fatal_child_is_inert_until_owned_cleanup(directory_database, stage, kind): + import sys + + from tests.integration._pg_database import with_database + + dsn = with_database( + os.environ["LOOM_PG_DSN"], await directory_database.fetchval("SELECT current_database()") + ) + script = r""" +import asyncio, contextlib, os, sys +import asyncpg +from server.router import directory +from server.settings import settings +stage, kind = sys.argv[1:] +original = SystemExit(17) if kind == 'exit' else KeyboardInterrupt('fatal guard') +async def main(): + settings.router_pg_dsn = os.environ['CONTROL_TEST_DSN'] + observer = await asyncpg.connect(settings.router_pg_dsn) + backend = {} + real_close = asyncpg.Connection.close + real_execute = asyncpg.Connection.execute + real_connection = directory._connection + async def execute(self, sql, *args, **kw): + if stage == 'commit' and sql.startswith('COMMIT'): raise original + return await real_execute(self, sql, *args, **kw) + async def close(self, **kw): + ours = self.get_server_pid() == backend.get('pid') + await real_close(self, **kw) + if ours and stage == 'close': raise original + @contextlib.asynccontextmanager + async def connection(): + if stage == 'acquire': raise original + async with real_connection() as conn: + yield conn + if stage == 'release': raise original + async def work(conn): + backend['pid'] = conn.get_server_pid() + await conn.execute('SELECT pg_advisory_xact_lock($1)', directory._DIRECTORY_LOCK) + await conn.execute('SELECT * FROM probe FOR UPDATE') + if stage == 'work': raise original + asyncpg.Connection.close = close + asyncpg.Connection.execute = execute + directory._connection = connection + try: + try: await directory._control_transaction(work) + except (SystemExit, KeyboardInterrupt) as caught: assert caught is original + else: raise AssertionError('fatal guard returned success') + if stage == 'acquire': + assert not backend and directory._pool is None + else: + assert not await observer.fetchval('SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1)', backend['pid']) + assert not await observer.fetchval('SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)', backend['pid']) + finally: + asyncpg.Connection.close = real_close + asyncpg.Connection.execute = real_execute + await directory.close() + await observer.close(timeout=1) +asyncio.run(main()) +""" + process = await asyncio.create_subprocess_exec( + sys.executable, + "-c", + script, + stage, + kind, + env={**os.environ, "CONTROL_TEST_DSN": dsn}, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await asyncio.wait_for(process.communicate(), 10) + assert process.returncode == 0, (stdout.decode(), stderr.decode()) diff --git a/tests/integration/test_router_directory_pg.py b/tests/integration/test_router_directory_pg.py index cbb96ba4cc..41f8d343c6 100644 --- a/tests/integration/test_router_directory_pg.py +++ b/tests/integration/test_router_directory_pg.py @@ -7,7 +7,6 @@ from __future__ import annotations import asyncio -import contextlib import hashlib import json import os @@ -828,12 +827,14 @@ async def test_a_live_placement_racing_a_batch_of_the_same_org_answers_it_placed written, release = asyncio.Event(), asyncio.Event() real = directory._attach_change - @contextlib.asynccontextmanager - async def _held(cell_name: str, expected: directory.Control) -> Any: - async with real(cell_name, expected) as locked: - yield locked + async def _held(cell_name: str, expected: directory.Control, change) -> Any: + async def hold(conn, found): + result = await change(conn, found) written.set() await release.wait() # the batch's writes are done, its commit held + return result + + return await real(cell_name, expected, hold) monkeypatch.setattr(directory, "_attach_change", _held) conn = await asyncpg.connect(own_dsn) diff --git a/tests/unit/test_cell_attach_cleanup.py b/tests/unit/test_cell_attach_cleanup.py index 20e17ac786..11bf1bf094 100644 --- a/tests/unit/test_cell_attach_cleanup.py +++ b/tests/unit/test_cell_attach_cleanup.py @@ -7,9 +7,10 @@ import pytest -from server.cli import cell_attach -from server.cli import cell_attach_pg as apg +from server.cli import cell_attach, cell_control from server.core import pg +from tests.unit.cell_control_owner import owner +from tests.unit.test_cell_control import snapshot @pytest.mark.asyncio @@ -24,12 +25,16 @@ async def close(self, **kwargs): def terminate(self): events.append("abort") + original = ValueError("original work failure") + async def work(): - raise ValueError("original work failure") + raise original - with pytest.raises(ValueError, match="original work failure"): - await apg.supervise(Conn(), work(), end=time.monotonic() + 0.1) - assert events == ["close", "abort"] + with pytest.raises(ValueError, match="original work failure") as caught: + await owner(Conn()).call(work, total_s=0.1) + assert caught.value is original + assert events[0] == "close" and events.count("close") == 1 + assert events[1:] and set(events[1:]) == {"abort"} @pytest.mark.asyncio @@ -56,7 +61,7 @@ async def work(): finally: events.append("work ended") - task = asyncio.create_task(apg.supervise(Conn(), work(), end=time.monotonic() + 0.2)) + task = asyncio.create_task(owner(Conn()).call(work, total_s=0.2)) await entered.wait() if not failed: task.cancel("first cancellation") @@ -73,47 +78,58 @@ async def work(): @pytest.mark.asyncio -async def test_lock_owner_bounds_context_exit_inside_the_same_cleanup_allowance(monkeypatch): +async def test_lock_owner_bounds_close_inside_the_same_cleanup_allowance(monkeypatch): + closing = asyncio.Event() conn = AsyncMock() - conn.is_closed = lambda: False conn.terminate = lambda: None - cm = AsyncMock() - exit_started = asyncio.Event() - async def exit(*_): - exit_started.set() + async def close(**_): + closing.set() await asyncio.Future() - cm.__aexit__.side_effect = exit - run = cell_attach.Run(None, argparse.Namespace(cell="test", wait=0), None, lambda _: None) - run._lock_cm, run.lock_conn = cm, conn + conn.close.side_effect = close + owned = owner(conn) monkeypatch.setattr(pg, "_RELEASE_TIMEOUT_S", 0.03) started = time.monotonic() with pytest.raises(TimeoutError): - await run.close_lock() + await owned.close() assert time.monotonic() - started < 0.15 - assert exit_started.is_set() and run.lock_conn is None + assert closing.is_set() and owned.closed conn.close.assert_awaited_once() await asyncio.sleep(0) @pytest.mark.asyncio @pytest.mark.parametrize("failed", [False, True]) -async def test_run_preserves_its_own_failure_without_hiding_normal_close_failure(failed): +async def test_run_preserves_its_own_failure_without_hiding_normal_close_failure( + failed, monkeypatch +): router = AsyncMock() - router.call.return_value = {"state": "attaching"} + router.call.return_value = {**snapshot(), "state": "attaching"} run = cell_attach.Run( None, argparse.Namespace(cell="test", wait=0, dry_run=True), router, lambda _: None ) - run.open_lock = AsyncMock(side_effect=ValueError("run failed") if failed else None) + original = ValueError("run failed") + conn = AsyncMock() + conn.terminate = lambda: None + conn.close.side_effect = RuntimeError("close failed") + + async def open(owned, what): + owned.conn = conn + if failed: + raise original + + monkeypatch.setattr(cell_control.Owner, "_open", open) run.read_own_row = AsyncMock() run.refusals = AsyncMock(return_value=(None, None, None)) run.dry_run = AsyncMock(return_value=0) - run.close_lock = AsyncMock(side_effect=RuntimeError("close failed")) try: raise LookupError("unrelated caller exception") except LookupError: with pytest.raises( ValueError if failed else RuntimeError, match="run failed" if failed else "close failed" - ): + ) as caught: await run.run() + if failed: + assert caught.value is original + conn.close.assert_awaited_once() diff --git a/tests/unit/test_cell_control_census.py b/tests/unit/test_cell_control_census.py index 0c32387393..0c358f2c38 100644 --- a/tests/unit/test_cell_control_census.py +++ b/tests/unit/test_cell_control_census.py @@ -116,15 +116,15 @@ def test_cell_control_migrations_are_classified(): "server/router/directory.py": { ("set_limit.change", "UPDATE", "loom_router.cells"): 1, ("rotate_credential.change", "UPDATE", "loom_router.cells"): 1, - ("claim", "UPDATE", "loom_router.cells"): 1, - ("attach_begin", "UPDATE", "loom_router.cells"): 1, - ("attach_place", "INSERT INTO", "loom_router.placements"): 1, - ("attach_place", "UPDATE", "loom_router.cells"): 1, - ("attach_place", "INSERT INTO", "loom_router.audit"): 1, - ("attach_enforced", "UPDATE", "loom_router.cells"): 1, - ("attach_serving", "UPDATE", "loom_router.cells"): 1, - ("remove", "DELETE FROM", "loom_router.hosts"): 1, - ("remove", "DELETE FROM", "loom_router.cells"): 1, + ("claim.change_locked", "UPDATE", "loom_router.cells"): 1, + ("attach_begin.change_locked", "UPDATE", "loom_router.cells"): 1, + ("attach_place.change_locked", "INSERT INTO", "loom_router.placements"): 1, + ("attach_place.change_locked", "UPDATE", "loom_router.cells"): 1, + ("attach_place.change_locked", "INSERT INTO", "loom_router.audit"): 1, + ("attach_enforced.change_locked", "UPDATE", "loom_router.cells"): 1, + ("attach_serving.change_locked", "UPDATE", "loom_router.cells"): 1, + ("remove.change_locked", "DELETE FROM", "loom_router.hosts"): 1, + ("remove.change_locked", "DELETE FROM", "loom_router.cells"): 1, }, } @@ -387,15 +387,39 @@ def function(path: str, name: str) -> ast.AST: "cell_control.check_lock(conn)", ], ), + ( + "server/router/directory.py", + "_control_transaction", + [ + "owner.start(lambda: owner.work(change))", + "owner.wait(task, owner.work_end)", + "owner.finish(task)", + ], + ), + ( + "server/router/directory.py", + "_ControlOwner.work", + [ + "self.context.__aenter__()", + "self.conn.transaction()", + "tx.start()", + "change(self.conn)", + "tx.commit()", + ], + ), ( "server/router/directory.py", "_attach_change", - ["_control_transaction()", "control_refusal(attaching, expected, claim=claim)"], + [ + "_control_transaction(locked)", + "control_refusal(attaching, expected, claim=claim)", + "change(conn, attaching)", + ], ), ( "server/router/directory.py", "_operator_change", - ["_attach_change(cell_name, expected)", "change(conn, found.row)"], + ["_attach_change(cell_name, expected, change_locked)", "change(conn, found.row)"], ), ( "server/router/directory.py", @@ -410,10 +434,14 @@ def function(path: str, name: str) -> ast.AST: ( "server/router/directory.py", "claim", - ["_attach_change(cell_name, expected, claim=True)"], + ["_attach_change(cell_name, expected, change_locked, claim=True)"], ), *[ - ("server/router/directory.py", name, ["_attach_change(cell_name, expected)"]) + ( + "server/router/directory.py", + name, + ["_attach_change(cell_name, expected, change_locked)"], + ) for name in ( "attach_begin", "attach_place", diff --git a/tests/unit/test_directory_control_cleanup.py b/tests/unit/test_directory_control_cleanup.py new file mode 100644 index 0000000000..1a66b20c26 --- /dev/null +++ b/tests/unit/test_directory_control_cleanup.py @@ -0,0 +1,136 @@ +"""A directory control attempt owns acquisition, cleanup and context exit once.""" + +import asyncio +import contextlib +import time +from unittest.mock import AsyncMock, Mock + +import pytest + +from server.router import directory + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +def owned(monkeypatch): + conn = Mock() + conn.transaction.return_value = Mock(start=AsyncMock(), commit=AsyncMock()) + conn.close = AsyncMock() + conn.terminate = Mock() + context = Mock(__aenter__=AsyncMock(return_value=conn), __aexit__=AsyncMock(return_value=None)) + monkeypatch.setattr(directory, "_connection", lambda: context) + monkeypatch.setattr(directory, "CONTROL_MAX_S", 0.08) + return conn, context + + +async def test_success_closes_once_before_pool_exit(owned): + conn, context = owned + events = [] + conn.close.side_effect = lambda **_: events.append("close") + context.__aexit__.side_effect = lambda *_: events.append("exit") + assert await directory._control_transaction(AsyncMock(return_value=19)) == 19 + assert events == ["close", "exit"] + conn.close.assert_awaited_once() + conn.transaction.return_value.commit.assert_awaited_once() + + +async def test_original_error_survives_close_and_exit_failure(owned): + conn, context = owned + original = ValueError("work failure") + conn.close.side_effect = RuntimeError("close failure") + context.__aexit__.side_effect = LookupError("exit failure") + with pytest.raises(ValueError) as caught: + await directory._control_transaction(AsyncMock(side_effect=original)) + assert caught.value is original + conn.close.assert_awaited_once() + context.__aexit__.assert_awaited_once() + conn.transaction.return_value.commit.assert_not_awaited() + + +async def test_close_failure_survives_exit_failure_after_acknowledged_commit(owned): + conn, context = owned + original = ValueError("close failure") + conn.close.side_effect = original + context.__aexit__.side_effect = RuntimeError("exit failure") + with pytest.raises(ValueError) as caught: + await directory._control_transaction(AsyncMock()) + assert caught.value is original + conn.transaction.return_value.commit.assert_awaited_once() + + +async def test_exit_cannot_open_an_extra_cleanup_allowance(owned): + conn, context = owned + entered = asyncio.Event() + + async def stalled(*_): + entered.set() + await asyncio.Future() + + context.__aexit__.side_effect = stalled + began = time.monotonic() + with pytest.raises(directory.DirectoryUnavailable): + await directory._control_transaction(AsyncMock()) + assert entered.is_set() and time.monotonic() - began < 0.2 + conn.close.assert_awaited_once() + + +async def test_acquisition_failure_does_not_close_an_unowned_proxy(owned): + conn, context = owned + original = ValueError("acquisition failure") + context.__aenter__.side_effect = original + with pytest.raises(ValueError) as caught: + await directory._control_transaction(AsyncMock()) + assert caught.value is original + conn.close.assert_not_awaited() + conn.terminate.assert_not_called() + context.__aexit__.assert_not_awaited() + + +async def test_no_known_proxy_cancellation_does_not_close_another_attempt(owned): + conn, context = owned + entered = asyncio.Event() + + async def acquire(): + entered.set() + await asyncio.Future() + + context.__aenter__.side_effect = acquire + task = asyncio.create_task(directory._control_transaction(AsyncMock())) + await entered.wait() + task.cancel("first acquisition cancellation") + with pytest.raises(asyncio.CancelledError, match="first acquisition cancellation"): + await task + conn.close.assert_not_awaited() + conn.terminate.assert_not_called() + context.__aexit__.assert_not_awaited() + context.__aenter__.side_effect = None + assert await directory._control_transaction(AsyncMock(return_value=2)) == 2 + conn.close.assert_awaited_once() + + +async def test_late_acquired_proxy_is_discarded_without_running_callback(owned): + conn, context = owned + entered, allow = asyncio.Event(), asyncio.Event() + + async def acquire(): + entered.set() + while not allow.is_set(): + with contextlib.suppress(asyncio.CancelledError): + await allow.wait() + return conn + + context.__aenter__.side_effect = acquire + work = AsyncMock() + task = asyncio.create_task(directory._control_transaction(work)) + await entered.wait() + task.cancel("acquisition cancelled") + with pytest.raises(asyncio.CancelledError, match="acquisition cancelled"): + await task + allow.set() + for _ in range(20): + await asyncio.sleep(0) + work.assert_not_awaited() + conn.transaction.assert_not_called() + conn.terminate.assert_called() + conn.close.assert_not_awaited() # Expired original end cannot create a fresh close budget. diff --git a/tests/unit/test_router_attach.py b/tests/unit/test_router_attach.py index 226a1b54c0..bffc8aefea 100644 --- a/tests/unit/test_router_attach.py +++ b/tests/unit/test_router_attach.py @@ -89,6 +89,7 @@ class _Conn: def __init__(self, store: _Store) -> None: self.s = store + self._tx = None def _count(self) -> None: if self.s.down: @@ -175,8 +176,13 @@ async def fetch(self, sql: str, *a: Any) -> list[Any]: self.s.placements.update(dict.fromkeys(new, a[1])) return [{"org": o} for o in new] + async def close(self, **_kw: Any) -> None: + self.terminate() + def terminate(self) -> None: - pass + if self._tx is not None: + self.s.cells, self.s.hosts, self.s.placements, self.s.audit = self._tx.saved + self._tx = None def transaction(self) -> Any: conn = self @@ -192,12 +198,14 @@ async def rollback(self) -> None: await self.__aexit__(Exception) async def __aenter__(self) -> None: + conn._tx = self conn._count() self.saved = copy.deepcopy( (conn.s.cells, conn.s.hosts, conn.s.placements, conn.s.audit) ) async def __aexit__(self, kind: Any, *_exc: Any) -> None: + conn._tx = None conn.s.statements += 1 # the commit, or the rollback if kind is not None: conn.s.cells, conn.s.hosts, conn.s.placements, conn.s.audit = self.saved From 0c106360020e8869752e188d60ae5ab2344b4618 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 00:19:39 +0000 Subject: [PATCH 034/114] Measure cell triggers with the original API and delivery workload Use five runs per arm with 2,000 sequential 4-KiB values and five real worker batches at the configured size. Verify the owned fixture and stored results, retain raw samples, and restore triggers before a final verdict. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 1306 ++++++++++++++++++-- tests/unit/test_bench_cell_trigger_gate.py | 990 ++++++++++++++- 2 files changed, 2162 insertions(+), 134 deletions(-) diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 8797b4b8e7..5bd3bfd3db 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -1,134 +1,1244 @@ -"""The first attach's trigger-cost gate, end to end: `memory.set` p50 and p99 through a server -of an attached cell, and a delivery-sized 1,000-row insert of a held org from a marked and an -unmarked session, with the cell's org triggers on and dropped in turn (the `loom.tenancy` -guard stays), deleting the rows its inserts wrote at the end. Run it against an attached -cell's server, which must not serve anything else, with a writer token for a placed org; -it drops and recreates the cell's org triggers, so never against a deployment in use. -Needs no provider. - - PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py \\ - --base-url http://127.0.0.1:7932 --pg-dsn \\ - --token --org --namespace +"""Measure API p99 and real delivery throughput with cell org triggers on and off. + +Requires an independently owned, attached scratch fixture and its source/ownership +manifest. The outer fixture owner stops its processes and drops its exact databases. +This driver restores triggers but never deletes fixture rows or changes providers. """ from __future__ import annotations import argparse import asyncio +import hashlib +import ipaddress +import json +import math +import os +import re import statistics +import subprocess +import sys import time import uuid +from collections.abc import Callable, Mapping, Sequence +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, TextIO, cast +from urllib.parse import unquote, urlsplit import asyncpg +import clickhouse_connect import httpx from server.core import cell_triggers +from server.core.hashing import hash_text, sha256_value + +CALLS, WARMUP, ROUNDS, BATCHES = 2000, 50, 5, 5 +VALUE_BYTES, KEY_CEILING = 4096, 100_000 +IO_S, SETTLE_S, RESTORE_S = 30.0, 60.0, 15.0 +SUBJECT = "process:benchmark" +_SHA = re.compile(r"[a-f0-9]{64}") +_DB = re.compile(r"[A-Za-z_][A-Za-z0-9_-]*") +_DEFAULT_DBS = { + "loom", + "loom_router", + "loom_telemetry", + "default", + "system", + "postgres", + "template0", + "template1", + "information_schema", +} + + +class Invalid(ValueError): + """The fixture or a sample did not meet the declared measurement contract.""" + + +def _require(ok: object, field: str) -> None: + if not ok: + raise Invalid(field) + + +def _planned_keys(batch: int) -> int: + _require(type(batch) is int and batch > 0, "delivery batch") + count = 2 * ROUNDS * (CALLS + WARMUP + BATCHES * batch) + _require(count <= KEY_CEILING, "100000 key ceiling") + return count + + +def _arm_order() -> tuple[bool, ...]: + return tuple( + on for pair in range(ROUNDS) for on in ((True, False) if pair % 2 == 0 else (False, True)) + ) + -CALLS, WARMUP, ROUNDS, INSERTS = 400, 50, 3, 20 +def _payload(key: str) -> str: + prefix = f"Cell trigger measurement {key}: " + _require(len(prefix) < VALUE_BYTES and prefix.isascii(), "payload key") + return prefix + ("abcdefghijklmnopqrstuvwxyz " * VALUE_BYTES)[: VALUE_BYTES - len(prefix)] -async def _triggers(dsn: str, on: bool) -> None: - """Create or drop the org triggers, and check that what was asked holds: every guarded - table carries both triggers enabled at origin, or none does.""" - conn = await asyncpg.connect(dsn) +def _endpoint(value: str, *, pg: bool = False) -> tuple[str, str, int, str]: + """Validate the literal target without resolving names or accepting DSN overrides.""" try: - guarded = [t for t in await cell_triggers.planned_tables(conn) if t != "tenancy"] - for table in guarded: - for name in cell_triggers.ORG_TRIGGERS: - await conn.execute(f'DROP TRIGGER IF EXISTS {name} ON loom."{table}"') - if on: - await conn.execute(cell_triggers.create_statement(table, name)) - enabled = await conn.fetchval( - "SELECT count(*) FROM pg_trigger WHERE tgenabled = 'O' AND tgname = ANY($1::text[])", - list(cell_triggers.ORG_TRIGGERS), + u = urlsplit(value) + host, port = u.hostname, u.port + _require( + u.scheme in ({"postgres", "postgresql"} if pg else {"http", "https"}), "endpoint scheme" + ) + _require( + host and ipaddress.ip_address(host).is_loopback and port and 0 < port < 65536, + "numeric loopback endpoint", ) + _require(not u.query and not u.fragment, "endpoint overrides") + if not pg: + _require(not u.username and not u.password and u.path in ("", "/"), "HTTP endpoint") + database = unquote(u.path.removeprefix("/")) if pg else "" + if pg: + _scratch(database) + return u.scheme, str(ipaddress.ip_address(str(host))), int(port or 0), database + except (ValueError, TypeError): + raise Invalid("invalid local endpoint") from None + + +def _scratch(value: str) -> None: + _require( + bool(_DB.fullmatch(value)) and value.lower() not in _DEFAULT_DBS, + "explicit scratch database", + ) + + +def _digest(value: Any) -> str: + return hashlib.sha256( + json.dumps(value, sort_keys=True, default=str, separators=(",", ":")).encode() + ).hexdigest() + + +@dataclass(frozen=True) +class Fixture: + """Nonsecret receipt from the owner that created and launched this exact fixture.""" + + base_url: str + pg_host: str + pg_port: int + pg_database: str + ch_host: str + ch_port: int + ch_database: str + ch_telemetry_database: str + cell: str + org: str + namespace: str + deployment_id: str + epoch: int + instance_id: str + build_commit: str + source_tree: str + launch_manifest_sha256: str + ownership_receipt_sha256: str + configuration_sha256: str + background_workers: str + controller_excluded: bool + pg_internal_port: int | None = None + + @classmethod + def read(cls, path: str) -> Fixture: + try: + with Path(path).open("rb") as manifest_file: + raw = manifest_file.read(16_385) + _require(len(raw) <= 16_384, "manifest size") + return cls(**json.loads(raw)) + except (TypeError, ValueError, OSError): + raise Invalid("fixture manifest") from None + + def validate(self, args: argparse.Namespace, config: Any) -> int: + supplied = _endpoint(args.pg_dsn, pg=True) + configured = _endpoint(config.pg_dsn, pg=True) + _require(supplied == configured, "configured PG endpoint") + _require( + supplied[1:] == (self.pg_host, self.pg_port, self.pg_database), "manifest PG endpoint" + ) + _require(_endpoint(args.base_url) == _endpoint(self.base_url), "manifest HTTP endpoint") + for host, port in ((self.ch_host, self.ch_port), (config.ch_host, config.ch_port)): + _endpoint(f"http://[{host}]:{port}" if ":" in host else f"http://{host}:{port}") + _require( + (self.ch_host, self.ch_port) == (config.ch_host, config.ch_port), "manifest CH endpoint" + ) + for name in (self.pg_database, self.ch_database, self.ch_telemetry_database): + _scratch(name) + _require( + (self.ch_database, self.ch_telemetry_database) + == (config.ch_database, config.ch_telemetry_database), + "manifest CH aliases", + ) + _require( + (self.org, self.namespace, self.cell) == (args.org, args.namespace, config.cell), + "manifest identity", + ) + _require( + all((self.org, self.namespace, self.cell, self.instance_id, self.deployment_id)), + "manifest identity fields", + ) + _require( + self.background_workers == config.background_workers == "off" + and self.controller_excluded is True, + "exclusive fixture", + ) + _require( + config.storage_authority == "postgres" + and config.embedding_provider == "fake" + and config.llm_provider == "fake", + "explicit fake providers", + ) + _require( + self.build_commit == config.build_commit + and bool(re.fullmatch(r"[a-f0-9]{40}", self.build_commit)), + "source build", + ) + _require(bool(re.fullmatch(r"[a-f0-9]{40}", self.source_tree)), "source tree") + for value in ( + self.launch_manifest_sha256, + self.ownership_receipt_sha256, + self.configuration_sha256, + ): + _require(bool(_SHA.fullmatch(value)), "fixture receipt digest") + _require( + self.configuration_sha256 == _digest(_configuration(config)), "fixture configuration" + ) + _require(type(self.epoch) is int and self.epoch > 0, "authority epoch") + _require( + self.pg_internal_port is None + or type(self.pg_internal_port) is int + and 0 < self.pg_internal_port < 65536, + "internal PG port", + ) + return _planned_keys(config.pg_delivery_batch_rows) + + +def _configuration(config: Any) -> dict[str, Any]: + # A complete digest catches a setting change without publishing credentials. + return cast(dict[str, Any], config.model_dump(mode="json")) + + +def _source(fixture: Fixture) -> dict[str, Any]: + """Fixed read-only Git commands bind the launched build to this clean checkout.""" + root = Path(__file__).resolve().parents[1] + values = [] + for command in (("rev-parse", "HEAD"), ("rev-parse", "HEAD^{tree}"), ("status", "--porcelain")): + result = subprocess.run( + ["git", *command], cwd=root, capture_output=True, text=True, timeout=5, check=False + ) + _require(result.returncode == 0, "source Git read") + _require(len(result.stdout) <= 65_536, "source status size") + values.append(result.stdout.strip()) + head, tree, status = values + _require( + head == fixture.build_commit and tree == fixture.source_tree and not status, + "clean source head/tree", + ) + return {"head": head, "tree": tree, "clean": True} + + +class Artifact: + def __init__(self, path: str): + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + self.file: TextIO = os.fdopen(fd, "w", encoding="utf-8") + self.records = 0 + + def emit(self, kind: str, **record: Any) -> None: + self.records += 1 + _require(self.records <= KEY_CEILING + 1000, "output record ceiling") + line = json.dumps({"kind": kind, **record}, sort_keys=True, allow_nan=False) + _require(len(line) <= 65_536, "output record size") + self.file.write(line + "\n") + + def close(self) -> None: + try: + self.file.flush() + os.fsync(self.file.fileno()) + finally: + self.file.close() + + +async def _close_pg(conn: Any) -> None: + from server.core import pg + + await pg.close_owned(conn, end=asyncio.get_running_loop().time() + IO_S) + + +class ReadOnlyCH: + """Public ClickHouse client restricted at connection startup, before any query.""" + + def __init__(self, client: Any): + self.client = client + + @classmethod + async def open(cls) -> ReadOnlyCH: + from server.core.ch_names import NamedClient + from server.core.ch_target import connection_kwargs + + raw = await clickhouse_connect.get_async_client( + **connection_kwargs(), + settings={ + "readonly": 1, + "max_execution_time": int(IO_S), + "log_comment": "bench.cell_trigger_gate.preflight", + }, + connect_timeout=IO_S, + send_receive_timeout=IO_S, + ) + return cls(NamedClient(raw)) + + async def query_all( + self, sql: str, params: dict[str, Any] | None = None + ) -> list[dict[str, Any]]: + result = await self.client.query(sql, parameters=params or {}) + return list(result.named_results()) + + async def close(self) -> None: + async with asyncio.timeout(IO_S): + await self.client.close() + + +async def _health(args: argparse.Namespace, fixture: Fixture) -> None: + async with httpx.AsyncClient(timeout=IO_S, follow_redirects=False, trust_env=False) as client: + health = await client.get(f"{args.base_url}/v1/health") + deployment = await client.get(f"{args.base_url}/v1/health/deployment") + _require(health.status_code == deployment.status_code == 200, "health HTTP status") + h, d = health.json(), deployment.json() + _require( + h.get("build_commit") == fixture.build_commit + and d.get("deployment_id") == fixture.deployment_id, + "serving source/deployment", + ) + _require( + h.get("deployment_config", {}).get("instance_id") == fixture.instance_id, + "serving instance", + ) + _require( + h.get("ch_ok") is True and h.get("pg_ok") is True and h.get("status") == "ok", + "serving health", + ) + _require( + h.get("fake_embedding") is True + and h.get("fake_llm") is True + and h.get("pending_migrations") == [], + "serving provider/schema", + ) + _require(h.get("storage_authority") == "postgres", "serving authority") + + +def _object(value: Any) -> dict[str, Any]: + if isinstance(value, str): + value = json.loads(value) + _require(isinstance(value, dict), "metadata JSON object") + return cast(dict[str, Any], value) + + +async def _proof( + conn: Any, ch: Any, args: argparse.Namespace, fixture: Fixture, *, native: bool = False +) -> dict[str, Any]: + from server.core import cell, instances + from server.settings import settings + + fixture.validate(args, settings) + source = _source(fixture) + actual = await conn.fetchrow( + "SELECT current_database() AS db, inet_server_port() AS port, " + "pg_is_in_recovery() AS recovery" + ) + _require( + actual and actual["db"] == fixture.pg_database and actual["recovery"] is False, + "actual PG identity/primary", + ) + if fixture.pg_internal_port is not None: + _require(actual["port"] == fixture.pg_internal_port, "PG internal port") + authority = await conn.fetchrow("SELECT authority, deployment_id, epoch FROM loom.authority") + _require( + authority + and (authority["authority"], authority["deployment_id"], authority["epoch"]) + == ("postgres", fixture.deployment_id, fixture.epoch), + "PG authority identity", + ) + first = await conn.fetchrow("SELECT is_first, cell FROM loom.cell_first") + mode = await conn.fetchval("SELECT mode FROM loom.tenancy_mode") + own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") + held = await conn.fetchval( + "SELECT EXISTS (SELECT 1 FROM loom.tenancy WHERE org = $1)", args.org + ) + _require( + first and first["cell"] == fixture.cell and mode == "enforced" and held is True, + "enforced held org", + ) + _require(own and own["next_fp"] is None, "stable accepted fingerprint") + current = _object(own["current_fp"]) + mine = cell.mine() + mine["deployment_id"] = fixture.deployment_id + _require(cell.matches(mine, current), "accepted local fingerprint") + db = await ch.query_all("SELECT currentDatabase() AS db") + _require(db == [{"db": fixture.ch_database}], "actual CH database") + records = await ch.query_all( + "SELECT authority, epoch, deployment_id FROM loom.storage_authority FINAL" + ) + _require(len(records) == 1 and records[0] == dict(authority), "paired CH authority") + for table in ("provider_configuration", "runtime_overrides"): + rows = await ch.query_all(f"SELECT count() AS n FROM loom.{table}") + _require(rows == [{"n": 0}], f"fresh {table}") + live = await instances.read_live(ch, all_deployments=True) + _require( + len(live) == 1 + and live[0]["instance_id"] == fixture.instance_id + and live[0]["build_commit"] == fixture.build_commit + and not live[0]["workers_lease"], + "exclusive serving source", + ) + beats = await ch.query_all( + "SELECT instance_id, cell, tenancy_mode, fingerprint FROM " + "loom.cell_heartbeats WHERE instance_id = {id:String}", + {"id": fixture.instance_id}, + ) + _require( + len(beats) == 1 + and beats[0]["cell"] == fixture.cell + and beats[0]["tenancy_mode"] == "enforced", + "serving cell heartbeat", + ) + reported = _object(beats[0]["fingerprint"]) + _require( + reported + and cell.matches(mine, reported) + and reported.get("serves") is True + and reported.get("marked") is True, + "serving fingerprint", + ) + await _debt(conn, 0) + await _health(args, fixture) + if native: + await cell.refresh() + _require(not cell.cli_gate_reason(), "native fresh cell acceptance") + return { + "deployment_id": fixture.deployment_id, + "epoch": fixture.epoch, + "instance_id": fixture.instance_id, + "fingerprint_sha256": _digest(current), + "configuration_sha256": fixture.configuration_sha256, + "source": source, + } + + +async def _preflight(args: argparse.Namespace, fixture: Fixture) -> None: + conn = ch = None + try: + conn = await asyncpg.connect( + args.pg_dsn, + timeout=IO_S, + command_timeout=IO_S, + server_settings={"default_transaction_read_only": "on"}, + ) + ch = await ReadOnlyCH.open() + async with asyncio.timeout(IO_S): + await _proof(conn, ch, args, fixture) finally: - await conn.close() - expected = 2 * len(guarded) if on else 0 - if enabled != expected: - raise SystemExit(f"{enabled} org triggers enabled, expected {expected}") + try: + if ch is not None: + await ch.close() + finally: + if conn is not None: + await _close_pg(conn) + + +async def _debt(conn: Any, expected: int) -> None: + row = await conn.fetchrow( + "SELECT (SELECT count(*) FROM loom.memory_deliveries WHERE targets <> " + "0) AS pending, (SELECT count(*) FROM loom.audit_outbox) AS audits, " + "(SELECT count(*) FROM loom.memory_deliveries WHERE attempts <> 0 OR " + "last_error <> '') AS errors" + ) + _require( + row and row["pending"] == expected and row["audits"] == row["errors"] == 0, + "fixture delivery/audit debt", + ) + + +@dataclass +class Triggers: + tables: tuple[str, ...] + invariant: str + org: str + on: bool = True + + @staticmethod + async def snapshot(conn: Any) -> tuple[tuple[str, ...], str, str]: + tables = tuple(await cell_triggers.planned_tables(conn)) + rows = await conn.fetch( + "SELECT c.relname AS table_name, t.tgname AS name, t.tgenabled AS " + "enabled, pg_get_triggerdef(t.oid) AS definition FROM pg_trigger t " + "JOIN pg_class c ON c.oid = t.tgrelid JOIN pg_namespace n ON n.oid " + "= c.relnamespace WHERE n.nspname = 'loom' AND NOT t.tgisinternal " + "ORDER BY c.relname, t.tgname" + ) + functions = await conn.fetch( + "SELECT p.proname, pg_get_functiondef(p.oid) AS definition FROM " + "pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace WHERE " + "n.nspname = 'loom' AND p.prokind = 'f' ORDER BY p.proname, p.oid" + ) + org = [dict(r) for r in rows if r["name"] in cell_triggers.ORG_TRIGGERS] + other = [dict(r) for r in rows if r["name"] not in cell_triggers.ORG_TRIGGERS] + return ( + tables, + _digest({"functions": [dict(r) for r in functions], "triggers": other}), + _digest(org), + ) + + @classmethod + async def capture(cls, conn: Any) -> Triggers: + _require(not await cell_triggers.missing(conn), "canonical ON trigger baseline") + tables, invariant, org = await cls.snapshot(conn) + _require(tables, "guarded tables") + return cls(tables, invariant, org) + + async def verify(self, conn: Any) -> None: + tables, invariant, org = await self.snapshot(conn) + _require( + tables == self.tables and invariant == self.invariant, + "trigger table/function/tenancy drift", + ) + _require(org == (self.org if self.on else _digest([])), "prior trigger treatment") + + async def set(self, conn: Any, on: bool) -> None: + await self.verify(conn) + async with conn.transaction(): + for table in self.tables: + if table == "tenancy": + continue + _require(bool(_DB.fullmatch(table)), "guarded table identifier") + for name in cell_triggers.ORG_TRIGGERS: + await conn.execute(f'DROP TRIGGER IF EXISTS {name} ON loom."{table}"') + if on: + await conn.execute(cell_triggers.create_statement(table, name)) + self.on = on + await self.verify(conn) + + async def restore(self, conn: Any) -> None: + # Recreate the captured exact org guards even after a partial failed toggle. + async with conn.transaction(): + for table in self.tables: + if table == "tenancy": + continue + for name in cell_triggers.ORG_TRIGGERS: + await conn.execute(f'DROP TRIGGER IF EXISTS {name} ON loom."{table}"') + await conn.execute(cell_triggers.create_statement(table, name)) + self.on = True + await self.verify(conn) + _require(not await cell_triggers.missing(conn), "restored canonical triggers") -async def _memory_set(args: argparse.Namespace) -> tuple[float, float]: - """p50 and p99 in ms of CALLS sequential `memory.set` calls, each a new key.""" - lat = [] - headers = {"Authorization": f"Bearer {args.token}"} - async with httpx.AsyncClient(timeout=30, headers=headers) as client: +@dataclass +class Samples: + on: bool + api_ms: list[float] + delivery_rps: float + + +def _quantile(values: Sequence[float], fraction: float) -> float: + _require(values and all(math.isfinite(v) and v >= 0 for v in values), "finite latency samples") + return sorted(values)[math.ceil(fraction * len(values)) - 1] + + +def _statistics(samples: Sequence[Samples]) -> dict[str, Any]: + _require(len(samples) == 2 * ROUNDS, "complete arm samples") + arms: dict[bool, dict[str, Any]] = {} + for on in (True, False): + arm = [s for s in samples if s.on is on] + _require( + len(arm) == ROUNDS and all(len(s.api_ms) == CALLS for s in arm), "complete run samples" + ) + p99 = [_quantile(s.api_ms, 0.99) for s in arm] + rps = [s.delivery_rps for s in arm] + _require(all(math.isfinite(r) and r > 0 for r in rps), "positive finite throughput") + arms[on] = { + "p99_ms": p99, + "median_p99_ms": statistics.median(p99), + "min_p99_ms": min(p99), + "max_p99_ms": max(p99), + "rps": rps, + "median_rps": statistics.median(rps), + } + delta = arms[True]["median_p99_ms"] - arms[False]["median_p99_ms"] + loss = 1 - arms[True]["median_rps"] / arms[False]["median_rps"] + paired = [] + for i in range(0, len(samples), 2): + a, b = samples[i : i + 2] + _require(a.on is not b.on, "paired arm order") + paired.append((_quantile(a.api_ms, 0.99) - _quantile(b.api_ms, 0.99)) * (1 if a.on else -1)) + return { + "on": arms[True], + "off": arms[False], + "p99_delta_ms": delta, + "throughput_loss": loss, + "latency_within_limit": delta <= 10, + "throughput_within_limit": loss <= 0.10, + "paired_p99_differences_ms": paired, + } + + +@dataclass +class RequestRow: + key: str + memory_id: str + version: int + content_sha256: str + elapsed_ms: float + measured: bool + admission: str + + +async def _memory_set( + args: argparse.Namespace, *, arm: int = 0, emit: Callable[..., None] = lambda *_a, **_kw: None +) -> list[RequestRow]: + rows = [] + prefix = f"gate-{uuid.uuid4().hex}-{arm}" + async with httpx.AsyncClient( + timeout=IO_S, + headers={"Authorization": f"Bearer {args.token}"}, + follow_redirects=False, + trust_env=False, + ) as client: for i in range(CALLS + WARMUP): - body = { - "org": args.org, - "namespace": args.namespace, - "agent": "bench", - "memory_key": f"gate-{uuid.uuid4().hex[:10]}", - "value": f"trigger gate {i}", - } - t = time.perf_counter() - (await client.post(f"{args.base_url}/v1/memory.set", json=body)).raise_for_status() - if i >= WARMUP: - lat.append((time.perf_counter() - t) * 1000) - lat.sort() - return lat[len(lat) // 2], lat[int(0.99 * (len(lat) - 1))] - - -async def _batch(args: argparse.Namespace, marked: bool) -> float: - """The median in ms of INSERTS 1,000-row inserts of the held org, in one statement each.""" - settings = {"loom.tenancy_checked": "on"} if marked else {} - conn = await asyncpg.connect(args.pg_dsn, server_settings=settings) + key = f"{prefix}-{i}" + value = _payload(key) + started = time.perf_counter_ns() + try: + response = await client.post( + f"{args.base_url}/v1/memory.set", + json={ + "org": args.org, + "namespace": args.namespace, + "agent": "bench", + "memory_key": key, + "memory_type": "semantic", + "value": value, + }, + ) + _require(response.status_code == 200, "memory.set HTTP status") + out = response.json() + _require( + isinstance(out, dict) + and out.get("status") == "active" + and not out.get("noop") + and not out.get("degraded") + and not out.get("duplicate_of") + and not out.get("merged_into_key"), + "memory.set outcome", + ) + memory_id = str(uuid.UUID(out["memory_id"])) + _require( + type(out.get("version")) is int and out["version"] > 0, "memory.set version" + ) + _require( + out.get("memory_key") == key + and out.get("namespace") == args.namespace + and out.get("content_sha256") == sha256_value(value), + "memory.set identity/hash", + ) + admission = "derived_ready" + elapsed = (time.perf_counter_ns() - started) / 1_000_000 + _require(math.isfinite(elapsed) and elapsed >= 0, "memory.set elapsed") + row = RequestRow( + key, + memory_id, + out["version"], + sha256_value(value), + elapsed, + i >= WARMUP, + admission, + ) + rows.append(row) + emit("request", arm=arm, index=i, status=200, **asdict(row)) + except BaseException as error: + emit("request_error", arm=arm, index=i, key=key, error_type=type(error).__name__) + raise + return rows + + +@dataclass +class Block: + rows: list[dict[str, Any]] + audits: list[dict[str, Any]] + documents: list[dict[str, Any]] + seqs: list[tuple[str, str, int]] = field(default_factory=list) + + +async def _build_block(args: argparse.Namespace, batch: int) -> Block: + from server.llm import customer + from server.memory.store import pg_memory, pg_publish + from server.memory.store.rows import ( + MemoryRow, + build_memory_version_row, + next_version, + ) + + _planned_keys(batch) + prefix = f"gate-delivery-{uuid.uuid4().hex}" + rows, audits, documents = [], [], [] + with customer.for_org(args.org, namespace=args.namespace, subject=SUBJECT): + for index in range(BATCHES * batch): + key = f"{prefix}-{index}" + value = _payload(key) + stored = json.dumps(value, sort_keys=True, separators=(",", ":")) + row = build_memory_version_row( + MemoryRow.from_snapshot( + { + "org": args.org, + "namespace": args.namespace, + "agent": "bench", + "memory_key": key, + "memory_type": "semantic", + "value": stored, + "version": next_version(), + "content_sha256": sha256_value(value), + "actor": SUBJECT, + "source_class": "asserted", + } + ) + ) + row = pg_memory.decode_row(pg_memory.encode_row(row)) + document = await _expected_document(args, row, worker=True) + rows.append(row) + audits.append(pg_publish.audit_row_for(row, request_id="", writer=SUBJECT)) + documents.append(document) + return Block(rows, audits, documents) + + +async def _expected_document( + args: argparse.Namespace, row: dict[str, Any], *, worker: bool = False +) -> dict[str, Any]: + from server.embedding import get_embedder + from server.llm import customer + from server.memory.domain.embed_text import embedding_texts + from server.memory.store.rows import parse_memory_value, search_document_row + + embedder = get_embedder() + raw = str(row["value"]) + excerpt, text, key_text = embedding_texts( + row["memory_type"], parse_memory_value(raw), raw, row["memory_key"] + ) + wanted = sorted({t for t in (text, key_text) if t}) + with customer.for_org(args.org, namespace=args.namespace, subject=SUBJECT): + vectors = dict(zip(wanted, await embedder.embed_many(wanted), strict=True)) + from server.memory.store.conflict_resolver import OWED_MARK + from server.settings import settings + + return search_document_row( + row, + conflict_checked_at=OWED_MARK + if worker and settings.memory_conflict_resolver_enabled + else None, + embedding_model=embedder.model_name, + embedding=vectors.get(text, []), + key_embedding=vectors.get(key_text, [0.0] * embedder.dim), + key_text=key_text, + excerpt=excerpt, + ) + + +async def _stage(conn: Any, ch: Any, block: Block) -> None: + from server.memory.store import pg_memory + + await _debt(conn, 0) + for offset in range(0, len(block.rows), 100): + rows = block.rows[offset : offset + 100] + fresh = await conn.fetchval( + "SELECT count(*) FROM loom.memory_current WHERE org=$1 AND namespace=$2 " + "AND (memory_id=ANY($3::uuid[]) OR memory_key=ANY($4::text[]))", + rows[0]["org"], + rows[0]["namespace"], + [uuid.UUID(str(row["memory_id"])) for row in rows], + [row["memory_key"] for row in rows], + ) + _require(fresh == 0, "fresh absent authority identity") + await _absent(ch, block) + committed = await pg_memory.commit_versions( + block.rows, + block.audits, + targets=pg_memory.MEMORY_ROW | pg_memory.SEARCH_DOC | pg_memory.AUDIT_ROW, + publisher_lease=False, + ) + _require( + len(committed.seqs) == len(block.rows) and all(t == 7 for t in committed.targets), + "staged delivery count/targets", + ) + block.seqs = committed.seqs + # The authority may canonicalize the full persisted row; retain its exact result. + _require( + len(committed.rows) == len(block.rows) and len(committed.audits) == len(block.audits), + "staged row/audit count", + ) + for expected, actual in zip(block.rows, committed.rows, strict=True): + _compare(expected, actual, tuple(expected), "committed full row") + for expected, actual in zip(block.audits, committed.audits, strict=True): + _compare(expected, actual, tuple(expected), "committed full audit") + await _debt(conn, len(block.rows)) + due = await conn.fetchval( + "SELECT count(*) FROM loom.memory_deliveries " + "WHERE targets=7 AND lease_until<=clock_timestamp() AND NOT held AND attempts=0" + ) + _require(due == len(block.rows), "exact fresh due deliveries") + + +async def _delivery( + args: argparse.Namespace, batch: int, *, emit: Callable[..., None], arm: int +) -> float: + from server.llm import customer + from server.memory.workers import pg_delivery + from server.settings import settings + + results: list[dict[str, Any]] = [] + _index = 0 + with customer.for_org(args.org, namespace=args.namespace, subject=SUBJECT): + started = time.perf_counter_ns() + try: + for _index in range(BATCHES): + _require(settings.pg_delivery_batch_rows == batch, "unchanged delivery batch") + result = await pg_delivery.run_pass() + results.append(asdict(result)) + _require( + result.claimed == result.published == batch and result.failed == 0, + "real worker claimed/published/failure", + ) + except BaseException as error: + emit("worker_error", arm=arm, index=_index, error_type=type(error).__name__) + raise + finally: + elapsed = (time.perf_counter_ns() - started) / 1_000_000_000 + for result_index, outcome in enumerate(results): + emit("worker", arm=arm, index=result_index, **outcome) + _require(math.isfinite(elapsed) and elapsed > 0, "positive worker block elapsed") + rps = BATCHES * batch / elapsed + emit("delivery_block", arm=arm, elapsed_seconds=elapsed, rows=BATCHES * batch, rps=rps) + return rps + + +def _normal(value: Any) -> Any: + from datetime import datetime + + if isinstance(value, (bytes, bytearray)): + return hash_text(value) + if isinstance(value, uuid.UUID): + return str(value) + if isinstance(value, str) and re.fullmatch(r"\d{4}-\d\d-\d\dT.*", value): + try: + value = datetime.fromisoformat(value) + except ValueError: + pass + if isinstance(value, datetime): + return value.replace(tzinfo=None).isoformat(timespec="milliseconds") + if isinstance(value, Mapping): + return {k: _normal(v) for k, v in value.items()} + if isinstance(value, (tuple, list)): + return [_normal(v) for v in value] + return value + + +def _compare( + expected: Mapping[str, Any], actual: Mapping[str, Any], fields: Sequence[str], phase: str +) -> None: + for field_name in fields: + left, right = expected.get(field_name), actual.get(field_name) + if field_name in ("embedding", "memory_key_embedding"): + _require( + isinstance(left, list) + and isinstance(right, list) + and len(left) == len(right) + and all( + math.isclose(a, b, rel_tol=1e-6, abs_tol=1e-7) + for a, b in zip(left, right, strict=True) + ), + phase + " vector", + ) + else: + _require(_normal(left) == _normal(right), phase + " " + field_name) + + +async def _absent(ch: Any, block: Block) -> None: + for offset in range(0, len(block.rows), 100): + rows = block.rows[offset : offset + 100] + params = { + "org": rows[0]["org"], + "ns": rows[0]["namespace"], + "ids": [str(r["memory_id"]) for r in rows], + "keys": [r["memory_key"] for r in rows], + "hashes": [a["row_hash"] for a in block.audits[offset : offset + 100]], + } + for table, predicate in ( + ("memory", "memory_id IN {ids:Array(UUID)}"), + ("memory_embeddings", "memory_key IN {keys:Array(String)}"), + ("audit_log", "row_hash IN {hashes:Array(String)}"), + ): + final = "" if table == "audit_log" else " FINAL" + found = await ch.query_all( + f"SELECT count() AS n FROM loom.{table}{final} WHERE org = " + f"{{org:String}} AND namespace = {{ns:String}} AND {predicate}", + params, + ) + _require(found == [{"n": 0}], "fresh absent " + table) + + +async def _verify( + conn: Any, ch: Any, args: argparse.Namespace, identities: Sequence[RequestRow] | Block +) -> None: + block = identities if isinstance(identities, Block) else None + wanted = identities.rows if isinstance(identities, Block) else [asdict(r) for r in identities] + for offset in range(0, len(wanted), 100): + group = wanted[offset : offset + 100] + ids = [uuid.UUID(str(r["memory_id"])) for r in group] + stored = await conn.fetch( + "SELECT c.row::text AS head, r.row::text AS revision, " + "r.audit::text AS audit, r.seq, d.targets, d.attempts, " + "d.last_error FROM loom.memory_current c JOIN " + "loom.memory_revisions r ON " + "(r.org,r.namespace,r.seq)=(c.org,c.namespace,c.seq) LEFT JOIN " + "loom.memory_deliveries d ON " + "(d.org,d.namespace,d.seq)=(r.org,r.namespace,r.seq) WHERE " + "c.org=$1 AND c.namespace=$2 AND c.memory_id=ANY($3::uuid[])", + args.org, + args.namespace, + ids, + ) + _require(len(stored) == len(group), "exact authority head/revision count") + by_id = {json.loads(r["head"])["memory_id"]: r for r in stored} + params = { + "org": args.org, + "ns": args.namespace, + "ids": [str(i) for i in ids], + "keys": [r.get("memory_key", r.get("key")) for r in group], + } + projected = await ch.query_all( + "SELECT * FROM loom.memory FINAL WHERE org={org:String} AND " + "namespace={ns:String} AND memory_id IN {ids:Array(UUID)}", + params, + ) + docs = await ch.query_all( + "SELECT * FROM loom.memory_embeddings FINAL WHERE org={org:String} " + "AND namespace={ns:String} AND memory_key IN {keys:Array(String)}", + params, + ) + _require(len(projected) == len(docs) == len(group), "exact projection/vector counts") + projections = {str(r["memory_id"]): r for r in projected} + documents = {r["memory_key"]: r for r in docs} + audit_hashes = [] + authority_audits = {} + for index, expected in enumerate(group): + id = str(expected["memory_id"]) + _require(id in by_id and id in projections, "authority/projection identity") + record = by_id[id] + _require( + record["targets"] in (None, 0) + and record["attempts"] in (None, 0) + and record["last_error"] in (None, ""), + "delivery settled without failure", + ) + head, revision, audit = ( + json.loads(record[name]) for name in ("head", "revision", "audit") + ) + _compare(head, revision, tuple(head), "authority full row") + if block: + _compare(expected, head, tuple(expected), "authority staged row") + _compare( + block.audits[offset + index], audit, tuple(audit), "authority staged audit" + ) + _require( + (args.org, args.namespace, record["seq"]) == block.seqs[offset + index], + "authority staged sequence", + ) + else: + key = str(expected["key"]) + _require( + head["memory_key"] == key + and json.loads(head["value"]) == _payload(key) + and head["version"] == expected["version"] + and head["content_sha256"] == expected["content_sha256"] + and head["status"] == "active", + "authority API payload/version", + ) + _compare(head, projections[id], tuple(head), "full memory projection") + key = str(head["memory_key"]) + _require(key in documents, "search document identity") + document = documents[key] + if block: + expected_document = block.documents[offset + index] + _compare( + expected_document, + document, + tuple(k for k in expected_document if k != "created_at"), + "full search document", + ) + else: + expected_document = await _expected_document(args, head) + _compare( + expected_document, + document, + tuple( + k + for k in expected_document + if k not in ("created_at", "conflict_checked_at") + ), + "full API search document", + ) + audit_hashes.append(audit["row_hash"]) + authority_audits[audit["row_hash"]] = audit + params["hashes"] = audit_hashes + audits = await ch.query_all( + "SELECT * FROM loom.audit_log WHERE org={org:String} AND " + "namespace={ns:String} AND row_hash IN {hashes:Array(String)}", + params, + ) + _require( + sorted(hash_text(r["row_hash"]) for r in audits) == sorted(audit_hashes), + "exact stored audit hashes", + ) + for audit in audits: + expected_audit = authority_audits[hash_text(audit["row_hash"])] + _compare(expected_audit, audit, tuple(expected_audit), "full stored audit") + await _debt(conn, 0) + + +async def _settle_api( + conn: Any, ch: Any, args: argparse.Namespace, rows: Sequence[RequestRow] +) -> None: + end = asyncio.get_running_loop().time() + SETTLE_S + while True: + pending = await conn.fetchval( + "SELECT count(*) FROM loom.memory_deliveries WHERE org=$1 AND " + "namespace=$2 AND targets<>0", + args.org, + args.namespace, + ) + if pending == 0: + break + _require(asyncio.get_running_loop().time() < end, "API publication settlement deadline") + await asyncio.sleep(0.1) + await _verify(conn, ch, args, rows) + + +async def _physical(conn: Any, ch: Any) -> dict[str, Any]: + from server.core.ch_names import database, telemetry_database + + pg = await conn.fetch( + "SELECT relname, n_live_tup AS estimated_rows, " + "pg_total_relation_size(relid) AS bytes FROM pg_stat_user_tables WHERE " + "schemaname='loom' ORDER BY relname" + ) + parts = await ch.query_all( + "SELECT database, table, sum(rows) AS rows, sum(bytes_on_disk) AS " + "bytes FROM system.parts WHERE active AND database IN " + "{dbs:Array(String)} GROUP BY database, table ORDER BY database, table", + {"dbs": [database(), telemetry_database()]}, + ) + exact = {} + for table in ("memory_current", "memory_revisions", "memory_deliveries", "audit_outbox"): + exact[table] = await conn.fetchval(f"SELECT count(*) FROM loom.{table}") + return { + "postgres_rows": exact, + "postgres_estimates": [dict(r) for r in pg], + "clickhouse_parts": parts, + } + + +async def _run( + args: argparse.Namespace, fixture: Fixture, artifact: Artifact, ch: Any +) -> dict[str, Any]: + from server.core.ch import set_query_tag + from server.settings import settings + + set_query_tag("bench.cell_trigger_gate") + batch = settings.pg_delivery_batch_rows + conn = await asyncpg.connect( + args.pg_dsn, + timeout=IO_S, + command_timeout=IO_S, + server_settings={"loom.tenancy_checked": "on"}, + ) + triggers = None + samples = [] try: - times = [] - for _ in range(INSERTS): - t = time.perf_counter() - await conn.execute( - "INSERT INTO loom.scope_generations (org, scope) " - "SELECT $1, $2 || g || '-' || $3 FROM generate_series(1, 1000) g", - args.org, - args.scope_prefix, - uuid.uuid4().hex[:8], + async with asyncio.timeout(IO_S): + await _proof(conn, ch, args, fixture, native=True) + triggers = await Triggers.capture(conn) + for arm, on in enumerate(_arm_order()): + async with asyncio.timeout(IO_S): + proof = await _proof(conn, ch, args, fixture, native=True) + await triggers.set(conn, on) + artifact.emit( + "arm_proof", + arm=arm, + on=on, + **proof, + tables=triggers.tables, + invariant_sha256=triggers.invariant, + org_triggers_sha256=triggers.org, + ) + api = await _memory_set(args, arm=arm, emit=artifact.emit) + async with asyncio.timeout(SETTLE_S): + await _settle_api(conn, ch, args, api) + block = await _build_block(args, batch) + async with asyncio.timeout(SETTLE_S): + await _stage(conn, ch, block) + rps = await _delivery(args, batch, emit=artifact.emit, arm=arm) + async with asyncio.timeout(SETTLE_S): + await _verify(conn, ch, args, block) + artifact.emit( + "settled", + arm=arm, + api_rows=len(api), + delivery_rows=len(block.rows), + physical=await _physical(conn, ch), + ) + latencies = [r.elapsed_ms for r in api if r.measured] + samples.append(Samples(on, latencies, rps)) + artifact.emit( + "arm", + arm=arm, + on=on, + p50_ms=_quantile(latencies, 0.5), + p99_ms=_quantile(latencies, 0.99), + delivery_rps=rps, ) - times.append((time.perf_counter() - t) * 1000) - return statistics.median(times) + from server.core.background import drain_task_sets, flush_registered_buffers + + _require(await drain_task_sets(timeout_s=IO_S), "driver descendant drain") + _require(await flush_registered_buffers(), "driver buffer drain") + summary = _statistics(samples) + artifact.emit("provisional_statistics", **summary) + return summary finally: - await conn.close() - - -async def main() -> None: - p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - p.add_argument("--base-url", required=True) - p.add_argument("--pg-dsn", required=True) - p.add_argument("--token", required=True) - p.add_argument("--org", required=True) - p.add_argument("--namespace", required=True) - args = p.parse_args() - args.scope_prefix = f"gate-{uuid.uuid4().hex}-" + prior = sys.exception() + cleanup = asyncio.create_task( + _cleanup(triggers, conn, artifact), name="trigger-gate-cleanup" + ) + interrupted = False + while not cleanup.done(): + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError: + interrupted = True + except BaseException: # noqa: BLE001 — preserve an earlier operation failure + break + try: + cleanup.result() + except BaseException as error: + try: + artifact.emit("cleanup_error", error_type=type(error).__name__) + except (OSError, ValueError): + if prior is None: + raise + if prior is None: + raise + if interrupted and prior is None: + raise asyncio.CancelledError + + +async def _cleanup(triggers: Triggers | None, conn: Any, artifact: Artifact) -> None: try: - for _ in range(ROUNDS): - for on in (True, False): - await _triggers(args.pg_dsn, on) - p50, p99 = await _memory_set(args) - marked, unmarked = await _batch(args, True), await _batch(args, False) - print( - f"triggers {'on ' if on else 'off'}: memory.set p50 {p50:.1f} ms " - f"p99 {p99:.1f} ms; 1,000-row insert {marked:.1f} ms marked, " - f"{unmarked:.1f} ms unmarked", - flush=True, - ) + if triggers is not None: + await _restore(triggers, conn) + artifact.emit("triggers_restored", verified=True) finally: - await _triggers(args.pg_dsn, True) - await _clean(args) + prior = sys.exception() + try: + await _close_pg(conn) + except BaseException: + if prior is None: + raise + artifact.emit("connection_cleanup_error", error_type=type(sys.exception()).__name__) + + +async def _restore(triggers: Triggers, conn: Any) -> None: + async with asyncio.timeout(RESTORE_S): + await triggers.restore(conn) -async def _clean(args: argparse.Namespace) -> None: - """Delete the rows the inserts wrote (the `memory.set` calls' rows stay the org's).""" - conn = await asyncpg.connect(args.pg_dsn, server_settings={"loom.tenancy_checked": "on"}) +def main() -> int: + from server.cli import _ch + from server.settings import settings + from server.wiring import wire + + parser = argparse.ArgumentParser(description=__doc__) + for name in ("base-url", "pg-dsn", "token", "org", "namespace", "fixture-manifest", "output"): + parser.add_argument("--" + name, required=True) + args = parser.parse_args() + artifact = None + verdict: dict[str, Any] = {"verdict": "INVALID"} + phase = "local_manifest" try: - await conn.execute( - "DELETE FROM loom.scope_generations WHERE org = $1 AND starts_with(scope, $2)", - args.org, - args.scope_prefix, + fixture = Fixture.read(args.fixture_manifest) + total = fixture.validate(args, settings) + source = _source(fixture) + artifact = Artifact(args.output) + artifact.emit( + "configuration", + manifest=asdict(fixture), + source=source, + manifest_sha256=_digest(asdict(fixture)), + calls=CALLS, + warmup=WARMUP, + rounds=ROUNDS, + batches=BATCHES, + batch_rows=settings.pg_delivery_batch_rows, + logical_keys=total, + value_bytes=VALUE_BYTES, + order=_arm_order(), + driver_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), ) + phase = "readonly_preflight" + asyncio.run(_preflight(args, fixture)) + phase = "native_driver" + wire() + + async def run(ch: Any) -> dict[str, Any]: + return await _run(args, fixture, artifact, ch) + + summary = _ch.run_connected(run, orgs=(args.org,)) + artifact.emit("driver_cleanup", native_returned=True, outer_database_cleanup_required=True) + verdict = { + "verdict": "PASS" + if summary["latency_within_limit"] and summary["throughput_within_limit"] + else "FAIL", + "p99_delta_ms": summary["p99_delta_ms"], + "throughput_loss": summary["throughput_loss"], + "outer_database_cleanup_required": True, + } + except BaseException as error: # noqa: BLE001 — every failed run must emit INVALID + # Never serialize exception text: clients may include credentials or values. + reason = str(error) if isinstance(error, Invalid) else "operation refused" + verdict = { + "verdict": "INVALID", + "phase": phase, + "reason": reason, + "error_type": type(error).__name__, + } + if artifact is not None: + try: + artifact.emit( + "invalid", phase=phase, reason=reason, error_type=type(error).__name__ + ) + except (OSError, ValueError): + pass finally: - await conn.close() + if artifact is not None: + try: + artifact.close() + except (OSError, ValueError) as error: + verdict = { + "verdict": "INVALID", + "error_type": type(error).__name__, + "phase": "artifact_close", + } + print(json.dumps(verdict, sort_keys=True, allow_nan=False), flush=True) + return 0 if verdict["verdict"] == "PASS" else 1 if __name__ == "__main__": - asyncio.run(main()) + raise SystemExit(main()) diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py index 89f32cddde..a27f9560f3 100644 --- a/tests/unit/test_bench_cell_trigger_gate.py +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -1,53 +1,971 @@ -"""The trigger gate deletes only the scope rows its own batch inserts wrote.""" +"""Offline proofs for the fixed cell trigger timing protocol; no fixture is opened.""" from __future__ import annotations import argparse -import sqlite3 -from typing import Any +import asyncio +import dataclasses +import json +import uuid +from contextlib import asynccontextmanager +from types import SimpleNamespace +from unittest.mock import AsyncMock +import httpx import pytest from bench import cell_trigger_gate as gate +@pytest.fixture(autouse=True) +def no_external_io(monkeypatch): + async def forbidden(*_args, **_kwargs): + raise AssertionError("unexpected real I/O") + + monkeypatch.setattr(gate.asyncpg, "connect", forbidden) + monkeypatch.setattr(gate.clickhouse_connect, "get_async_client", forbidden) + monkeypatch.setattr(gate.httpx.AsyncClient, "send", forbidden) + + +def args(**overrides): + return argparse.Namespace( + **{ + "base_url": "http://127.0.0.1:17932", + "pg_dsn": "postgresql://u:secret@127.0.0.1:15790/gate_cell", + "token": "secret-token", + "org": "org", + "namespace": "ns", + **overrides, + } + ) + + +def fixture_config(): + config = SimpleNamespace( + pg_dsn=args().pg_dsn, + ch_host="127.0.0.1", + ch_port=18790, + ch_database="gate_cell", + ch_telemetry_database="gate_cell_t", + cell="gate-cell", + background_workers="off", + storage_authority="postgres", + embedding_provider="fake", + llm_provider="fake", + build_commit="a" * 40, + pg_delivery_batch_rows=500, + ) + config.model_dump = lambda **_: {k: v for k, v in vars(config).items() if k != "model_dump"} + manifest = gate.Fixture( + base_url=args().base_url, + pg_host="127.0.0.1", + pg_port=15790, + pg_database="gate_cell", + ch_host=config.ch_host, + ch_port=config.ch_port, + ch_database=config.ch_database, + ch_telemetry_database=config.ch_telemetry_database, + cell=config.cell, + org="org", + namespace="ns", + deployment_id="deployment", + epoch=1, + instance_id="instance", + build_commit=config.build_commit, + source_tree="b" * 40, + launch_manifest_sha256="c" * 64, + ownership_receipt_sha256="d" * 64, + configuration_sha256=gate._digest(config.model_dump()), + background_workers="off", + controller_excluded=True, + pg_internal_port=5432, + ) + return manifest, config + + +def test_gate_keeps_the_original_api_sample_contract(): + assert (gate.CALLS, gate.WARMUP, gate.ROUNDS, gate.BATCHES) == (2000, 50, 5, 5) + assert not hasattr(gate, "_batch") and not hasattr(gate, "_clean") + + +@pytest.mark.parametrize(("batch", "expected"), [(500, 45500), (1590, 100000)]) +def test_complete_fixture_count_uses_the_unchanged_delivery_batch(batch, expected): + assert gate._planned_keys(batch) == expected + + +@pytest.mark.parametrize("batch", [1591, 0, -1, 1.5, True]) +def test_invalid_batch_refuses_before_loading(batch): + with pytest.raises(ValueError): + gate._planned_keys(batch) + + +def test_arm_order_alternates_pair_order_and_has_five_of_each(): + order = gate._arm_order() + assert order == (True, False, False, True, True, False, False, True, True, False) + assert order.count(True) == order.count(False) == 5 + + +@pytest.mark.parametrize( + "url", + [ + "http://localhost:1234", + "http://example.com:1234", + "http://10.0.0.1:1234", + "http://127.0.0.1", + "http://127.0.0.1:1234/other", + "http://u:pw@127.0.0.1:1234", + "http://127.0.0.1:1234?x=1", + "http://127.0.0.1:1234#fragment", + ], +) +def test_http_endpoint_refuses_ambiguous_or_remote_targets(url): + with pytest.raises(gate.Invalid): + gate._endpoint(url) + + +@pytest.mark.parametrize( + "dsn", + [ + "postgresql://u@localhost:15790/scratch", + "postgresql://u@127.0.0.1:15790/loom", + "postgresql://u@127.0.0.1:15790/scratch?host=remote", + "postgresql:///scratch?service=other", + "postgresql://u@127.0.0.1,127.0.0.2:15790/scratch", + "postgresql://u@127.0.0.1:15790/scratch?options=-cdefault_transaction_read_only=off", + ], +) +def test_pg_endpoint_refuses_overrides_defaults_and_alternate_targets(dsn): + with pytest.raises(gate.Invalid): + gate._endpoint(dsn, pg=True) + + +def test_manifest_binds_published_port_without_conflating_internal_port(): + manifest, config = fixture_config() + assert manifest.pg_port == 15790 and manifest.pg_internal_port == 5432 + assert manifest.validate(args(), config) == 45500 + with pytest.raises(gate.Invalid, match="manifest PG endpoint"): + dataclasses.replace(manifest, pg_port=5432).validate(args(), config) + + +@pytest.mark.parametrize( + "change", + [ + {"ch_host": "10.0.0.1"}, + {"ch_database": "loom"}, + {"cell": "other"}, + {"background_workers": "on"}, + {"llm_provider": "openai"}, + {"embedding_provider": "local"}, + {"pg_delivery_batch_rows": 1591}, + ], +) +def test_configuration_drift_and_unsafe_settings_refuse(change): + manifest, config = fixture_config() + for k, v in change.items(): + setattr(config, k, v) + with pytest.raises(gate.Invalid): + manifest.validate(args(), config) + + +def test_artifact_never_overwrites_existing_file_or_symlink(tmp_path): + original = tmp_path / "old.jsonl" + original.write_text("preserve") + with pytest.raises(FileExistsError): + gate.Artifact(str(original)) + link = tmp_path / "link" + link.symlink_to(original) + with pytest.raises(FileExistsError): + gate.Artifact(str(link)) + assert original.read_text() == "preserve" + fresh = tmp_path / "fresh" + artifact = gate.Artifact(str(fresh)) + artifact.emit("test", n=1) + artifact.close() + assert fresh.stat().st_mode & 0o777 == 0o600 + assert json.loads(fresh.read_text()) == {"kind": "test", "n": 1} + + @pytest.mark.asyncio -async def test_batch_cleanup_preserves_preexisting_gate_scopes( - monkeypatch: pytest.MonkeyPatch, -) -> None: - db = sqlite3.connect(":memory:") - db.execute("ATTACH DATABASE ':memory:' AS loom") - db.execute("CREATE TABLE loom.scope_generations (org text, scope text)") - db.create_function("starts_with", 2, lambda value, prefix: value.startswith(prefix)) - prefix = "gate-own-run-" - sentinel = [("org", "gate-existing"), ("other", prefix + "existing")] - db.executemany("INSERT INTO loom.scope_generations VALUES (?, ?)", sentinel) - - class _Conn: - async def execute(self, sql: str, *args: Any) -> None: - sql = sql.replace( - "FROM generate_series(1, 1000) g", - "FROM (WITH RECURSIVE seq(g) AS (VALUES(1) UNION ALL " - "SELECT g + 1 FROM seq WHERE g < 1000) SELECT g FROM seq)", +async def test_actual_api_loop_sends_unique_four_kib_values_and_full_sample(monkeypatch): + sent, outcomes = [], [] + + class Client: + def __init__(self, **kw): + assert kw["follow_redirects"] is False and kw["trust_env"] is False + + async def __aenter__(self): + return self + + async def __aexit__(self, *_): + return None + + async def post(self, url, *, json): + sent.append(json) + return httpx.Response( + 200, + json={ + "memory_id": str(uuid.uuid4()), + "version": len(sent), + "admission": "ready", + "status": "active", + "content_sha256": gate.sha256_value(json["value"]), + "memory_key": json["memory_key"], + "namespace": json["namespace"], + }, + request=httpx.Request("POST", url), ) - db.execute(sql, {str(i): value for i, value in enumerate(args, 1)}) - async def close(self) -> None: + monkeypatch.setattr(gate.httpx, "AsyncClient", Client) + rows = await gate._memory_set( + args(), emit=lambda kind, **fields: outcomes.append((kind, fields)) + ) + assert len(sent) == len(rows) == len(outcomes) == 2050 + assert ( + len({body["memory_key"] for body in sent}) == len({body["value"] for body in sent}) == 2050 + ) + assert all(len(body["value"].encode("utf-8")) == 4096 for body in sent) + assert sum(row.measured for row in rows) == 2000 + assert all("value" not in fields and "token" not in fields for _, fields in outcomes) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "status,changes", + [ + (302, {}), + (503, {}), + (200, {"status": "duplicate"}), + (200, {"noop": True}), + (200, {"content_sha256": "wrong"}), + (200, {"version": 0}), + ], +) +async def test_api_refusal_is_recorded_once_and_never_enters_samples(monkeypatch, status, changes): + events = [] + + class Client: + def __init__(self, **_): pass - async def connect(*_a: Any, **_kw: Any) -> Any: - return _Conn() + async def __aenter__(self): + return self - monkeypatch.setattr(gate.asyncpg, "connect", connect) - monkeypatch.setattr(gate, "INSERTS", 1) - args = argparse.Namespace(pg_dsn="unused", org="org", scope_prefix=prefix) - try: - await gate._batch(args, True) - assert db.execute("SELECT count(*) FROM loom.scope_generations").fetchone()[0] == 1002 - await gate._clean(args) - assert ( - db.execute("SELECT org, scope FROM loom.scope_generations ORDER BY org").fetchall() - == sentinel + async def __aexit__(self, *_): + return None + + async def post(self, url, *, json): + body = { + "memory_id": str(uuid.uuid4()), + "version": 1, + "status": "active", + "admission": "ready", + "memory_key": json["memory_key"], + "namespace": json["namespace"], + "content_sha256": gate.sha256_value(json["value"]), + **changes, + } + return httpx.Response(status, json=body, request=httpx.Request("POST", url)) + + monkeypatch.setattr(gate.httpx, "AsyncClient", Client) + with pytest.raises(gate.Invalid): + await gate._memory_set(args(), emit=lambda *a, **kw: events.append((a, kw))) + assert len(events) == 1 and events[0][0] == ("request_error",) + assert "secret-token" not in str(events) and "value" not in events[0][1] + + +def samples(on_p99, off_p99, on_rps=90.1, off_rps=100): + on, off = iter(on_p99), iter(off_p99) + return [ + gate.Samples(arm, [next(on) if arm else next(off)] * gate.CALLS, on_rps if arm else off_rps) + for arm in gate._arm_order() + ] + + +def test_statistics_use_difference_of_arm_medians_and_throughput_not_time(): + result = gate._statistics(samples([2, 12, 22, 32, 102], [1, 6, 51, 61, 71])) + assert result["p99_delta_ms"] == -29 + assert result["paired_p99_differences_ms"] == [1, 6, -29, -29, 31] + assert result["latency_within_limit"] is True + # Time increases 10.99%, but the original rows/second gate passes at 9.9% loss. + assert result["throughput_loss"] == pytest.approx(0.099) + assert result["throughput_within_limit"] is True + failed = gate._statistics(samples([20] * 5, [1] * 5, 89, 100)) + assert failed["latency_within_limit"] is failed["throughput_within_limit"] is False + assert gate._quantile(list(range(2000)), 0.99) == 1979 + + +@pytest.mark.parametrize("value", [0, -1, float("nan"), float("inf")]) +def test_invalid_throughput_cannot_become_passing_number(value): + with pytest.raises(gate.Invalid): + gate._statistics(samples([1] * 5, [1] * 5, value)) + + +@pytest.mark.asyncio +async def test_actual_production_worker_body_runs_five_times_with_unchanged_batch(monkeypatch): + from server.core import idempotency, pg + from server.llm import customer + from server.memory.app import pg_delivery as publish + from server.memory.store import pg_memory + from server.memory.workers import pg_delivery as worker + from server.settings import settings + + monkeypatch.setattr(settings, "pg_delivery_batch_rows", 2) + monkeypatch.setattr(pg, "enabled", lambda: True) + monkeypatch.setattr(worker, "_read_backlog", AsyncMock()) + claim = AsyncMock(return_value=[object(), object()]) + monkeypatch.setattr(pg_memory, "claim_due", claim) + attribution = [] + + async def publish_claims(claims): + attribution.append(customer._attribution.get()) + assert len(claims) == 2 + return 2, 0 + + monkeypatch.setattr(publish, "publish_claims", publish_claims) + for module, name in ( + (idempotency, "sweep_expired"), + (pg_memory, "sweep_erasure_evidence"), + (pg_memory, "sweep_settled"), + (publish, "publish_audit_outbox"), + ): + monkeypatch.setattr(module, name, AsyncMock(return_value=0)) + times = iter([0, 2_000_000_000]) + monkeypatch.setattr(gate.time, "perf_counter_ns", lambda: next(times)) + events = [] + assert ( + await gate._delivery(args(), 2, emit=lambda kind, **kw: events.append((kind, kw)), arm=0) + == 5 + ) + assert claim.await_count == 5 and all(call.args == (2,) for call in claim.await_args_list) + assert len(attribution) == 5 + assert all( + a.org == "org" and a.namespace == "ns" and a.subject == gate.SUBJECT for a in attribution + ) + assert [kind for kind, _ in events] == ["worker"] * 5 + ["delivery_block"] + assert customer._attribution.get() is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "result", + [ + {"claimed": 1, "published": 1}, + {"claimed": 2, "published": 1}, + {"claimed": 2, "published": 2, "failed": 1}, + ], +) +async def test_worker_count_failure_invalidates_block_without_retries(monkeypatch, result): + from server.memory.workers import pg_delivery as worker + from server.settings import settings + + monkeypatch.setattr(settings, "pg_delivery_batch_rows", 2) + run = AsyncMock(return_value=worker.DeliveryResult(**result)) + monkeypatch.setattr(worker, "run_pass", run) + with pytest.raises(gate.Invalid): + await gate._delivery(args(), 2, emit=lambda *_a, **_kw: None, arm=0) + assert run.await_count == 1 + + +@pytest.mark.asyncio +async def test_canonical_full_rows_audits_and_expected_vectors_use_real_fake_embedder(monkeypatch): + import server.embedding + from server.embedding.fake import FakeNGramEmbedder + from server.llm import customer + from server.memory.store import pg_memory + from server.memory.store.rows import MEMORY_PERSISTED_COLS + + embedder = FakeNGramEmbedder(dim=16) + monkeypatch.setattr(server.embedding, "get_embedder", lambda: embedder) + # Keep actual require(): refresh its real switch read through a fake store edge. + import logging + + from server.core.polled_copy import PolledCopy + + monkeypatch.setattr( + customer, + "_switch", + PolledCopy("switch", AsyncMock(return_value=frozenset()), logging.getLogger(__name__)), + ) + block = await gate._build_block(args(), 1) + assert len(block.rows) == len(block.audits) == len(block.documents) == 5 + for row, audit, doc in zip(block.rows, block.audits, block.documents, strict=True): + assert set(MEMORY_PERSISTED_COLS) <= row.keys() + assert len(json.loads(row["value"]).encode()) == 4096 + assert row["actor"] == gate.SUBJECT and row["source_class"] == "asserted" + assert row["content_sha256"] == gate.sha256_value(json.loads(row["value"])) + assert audit["event_type"] == "memory.revision" and len(audit["row_hash"]) == 64 + assert row["value"] not in audit["subject"] + assert len(doc["embedding"]) == doc["embedding_dim"] == 16 + assert doc["content_sha256"] == row["content_sha256"] + assert pg_memory.decode_row(pg_memory.encode_row(row)) == row + assert customer._attribution.get() is None + + +@pytest.mark.asyncio +async def test_stage_calls_real_commit_interface_with_due_fresh_targets(monkeypatch): + from server.memory.store import pg_memory + + block = gate.Block( + [{"memory_id": str(uuid.uuid4()), "org": "org", "namespace": "ns", "memory_key": "key"}], + [{"row_hash": "hash"}], + [], + ) + debt = AsyncMock() + absent = AsyncMock() + monkeypatch.setattr(gate, "_debt", debt) + monkeypatch.setattr(gate, "_absent", absent) + committed = pg_memory.Committed( + rows=block.rows, audits=block.audits, seqs=[("org", "ns", 1)], targets=[7] + ) + commit = AsyncMock(return_value=committed) + monkeypatch.setattr(pg_memory, "commit_versions", commit) + conn = SimpleNamespace(fetchval=AsyncMock(side_effect=[0, 1])) + await gate._stage(conn, object(), block) + assert commit.await_args.kwargs == {"targets": 7, "publisher_lease": False} + assert block.seqs == [("org", "ns", 1)] + assert [call.args[1] for call in debt.await_args_list] == [0, 1] + absent.assert_awaited_once() + + +@pytest.mark.parametrize( + ("field", "expected", "actual"), + [ + ("value", "right", "wrong"), + ("version", 2, 1), + ("content_sha256", "a", "b"), + ("embedding", [1.0, 0.0], [0.0, 1.0]), + ("embedding", [1.0], []), + ("row_hash", "good", "bad"), + ], +) +def test_stored_payload_vector_version_and_audit_mismatch_refuse(field, expected, actual): + with pytest.raises(gate.Invalid): + gate._compare({field: expected}, {field: actual}, [field], "stored") + + +class TriggerConn: + def __init__(self): + self.on = True + self.drift = False + self.statements = [] + + @asynccontextmanager + async def transaction(self): + yield + + async def execute(self, sql): + self.statements.append(sql) + if sql.startswith("DROP TRIGGER"): + self.on = False + elif sql.startswith("CREATE TRIGGER"): + self.on = True + + +@pytest.mark.asyncio +async def test_captured_trigger_state_allows_off_to_off_and_restores_exact_tables(monkeypatch): + conn = TriggerConn() + + async def snapshot(c): + return ( + ("memory_current", "tenancy"), + "drift" if c.drift else "invariant", + "org" if c.on else gate._digest([]), + ) + + monkeypatch.setattr(gate.Triggers, "snapshot", staticmethod(snapshot)) + monkeypatch.setattr(gate.cell_triggers, "missing", AsyncMock(return_value=[])) + triggers = await gate.Triggers.capture(conn) + await triggers.set(conn, False) + await triggers.set(conn, False) + assert triggers.on is False + await triggers.restore(conn) + assert triggers.on is True + assert conn.statements and all('ON loom."memory_current"' in sql for sql in conn.statements) + conn.drift = True + with pytest.raises(gate.Invalid, match="drift"): + await triggers.set(conn, False) + + +class ProofPG: + def __init__(self, fault=""): + self.fault = fault + self.closed = False + self.sql = [] + + async def fetchrow(self, sql, *params): + self.sql.append(sql) + if "current_database()" in sql: + return { + "db": "wrong" if self.fault == "database" else "gate_cell", + "port": 5432, + "recovery": self.fault == "recovery", + } + if "FROM loom.authority" in sql: + return { + "authority": "postgres", + "deployment_id": "wrong" if self.fault == "pair" else "deployment", + "epoch": 1, + } + if "FROM loom.cell_first" in sql: + return None if self.fault == "partial" else {"cell": "gate-cell", "is_first": False} + if "FROM loom.cell_self" in sql: + return { + "current_fp": json.dumps({"v": 1, "fields": {}}), + "next_fp": {} if self.fault == "transition" else None, + } + if "AS pending" in sql: + return {"pending": int(self.fault == "debt"), "audits": 0, "errors": 0} + raise AssertionError(sql) + + async def fetchval(self, sql, *params): + self.sql.append(sql) + if "FROM loom.tenancy_mode" in sql: + return "recording" if self.fault == "recording" else "enforced" + if "FROM loom.tenancy WHERE" in sql: + return self.fault != "held" + raise AssertionError(sql) + + async def close(self, **kwargs): + self.closed = True + + +class ProofCH: + def __init__(self, fault=""): + self.fault = fault + self.closed = False + self.sql = [] + + async def query_all(self, sql, parameters=None): + self.sql.append(sql) + if "currentDatabase()" in sql: + return [{"db": "wrong" if self.fault == "ch_database" else "gate_cell"}] + if "FROM loom.storage_authority" in sql: + return [ + { + "authority": "postgres", + "deployment_id": "wrong" if self.fault == "ch_pair" else "deployment", + "epoch": 1, + } + ] + if "FROM loom.provider_configuration" in sql: + return [{"n": int(self.fault == "saved_provider")}] + if "FROM loom.runtime_overrides" in sql: + return [{"n": int(self.fault == "overrides")}] + if "FROM loom.cell_heartbeats" in sql: + return [ + { + "cell": "gate-cell", + "tenancy_mode": "enforced", + "fingerprint": json.dumps( + {"serves": True, "marked": self.fault != "fingerprint"} + ), + } + ] + raise AssertionError(sql) + + async def close(self): + self.closed = True + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", + [ + "database", + "recovery", + "pair", + "partial", + "recording", + "held", + "transition", + "debt", + "ch_database", + "ch_pair", + "saved_provider", + "overrides", + "fingerprint", + "health", + "competing_worker", + ], +) +async def test_read_only_preflight_faults_refuse_before_native_setup(monkeypatch, fault): + import server.settings + from server.cli import _ch + from server.core import cell, instances + + fixture, config = fixture_config() + pg, ch = ProofPG(fault), ProofCH(fault) + connections = [] + + async def connect(dsn, **kwargs): + connections.append(kwargs) + assert kwargs["server_settings"] == {"default_transaction_read_only": "on"} + return pg + + async def close(conn): + await conn.close() + + async def live(*_args, **_kw): + return ( + [] + if fault == "competing_worker" + else [{"instance_id": "instance", "build_commit": "a" * 40, "workers_lease": ""}] ) - finally: - db.close() + + async def health(*_args): + if fault == "health": + raise gate.Invalid("health source") + + native = AsyncMock(side_effect=AssertionError("native setup before proof")) + monkeypatch.setattr(server.settings, "settings", config) + monkeypatch.setattr(gate.asyncpg, "connect", connect) + monkeypatch.setattr(gate.ReadOnlyCH, "open", AsyncMock(return_value=ch)) + monkeypatch.setattr(gate, "_close_pg", close) + monkeypatch.setattr(gate, "_health", health) + monkeypatch.setattr(gate, "_source", lambda _: {"clean": True}) + monkeypatch.setattr(instances, "read_live", live) + monkeypatch.setattr(cell, "mine", lambda: {}) + monkeypatch.setattr(cell, "matches", lambda mine, value: value is not None) + monkeypatch.setattr(_ch, "run_connected", native) + with pytest.raises(gate.Invalid): + await gate._preflight(args(), fixture) + assert pg.closed and ch.closed + native.assert_not_called() + assert connections and all(sql.startswith("SELECT ") for sql in pg.sql + ch.sql) + assert not any("model_transitions" in sql for sql in ch.sql) + + +@pytest.mark.asyncio +async def test_readonly_ch_startup_restricts_connection_before_first_query(monkeypatch): + client = SimpleNamespace(close=AsyncMock()) + opened = AsyncMock(return_value=client) + monkeypatch.setattr(gate.clickhouse_connect, "get_async_client", opened) + ch = await gate.ReadOnlyCH.open() + assert opened.await_args.kwargs["settings"]["readonly"] == 1 + await ch.close() + client.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_health_never_follows_redirects(monkeypatch): + fixture, _ = fixture_config() + + class Client: + def __init__(self, **kwargs): + assert kwargs["follow_redirects"] is False and kwargs["trust_env"] is False + + async def __aenter__(self): + return self + + async def __aexit__(self, *_): + return None + + async def get(self, url): + return httpx.Response( + 302, + headers={"Location": "http://external.invalid"}, + request=httpx.Request("GET", url), + ) + + monkeypatch.setattr(gate.httpx, "AsyncClient", Client) + with pytest.raises(gate.Invalid, match="health HTTP status"): + await gate._health(args(), fixture) + + +@pytest.mark.asyncio +async def test_restore_preserves_primary_failure_and_survives_second_cancellation(monkeypatch): + from server.settings import settings + + fixture, _ = fixture_config() + conn = ProofPG() + restoring, release = asyncio.Event(), asyncio.Event() + failure = ValueError("primary-secret") + events = [] + + class T: + tables = ("memory_current",) + invariant = org = "digest" + + async def set(self, *_): + pass + + async def restore(*_): + restoring.set() + await release.wait() + + async def close(c): + c.closed = True + + monkeypatch.setattr(gate.asyncpg, "connect", AsyncMock(return_value=conn)) + monkeypatch.setattr(gate.Triggers, "capture", AsyncMock(return_value=T())) + monkeypatch.setattr(gate, "_proof", AsyncMock(return_value={})) + monkeypatch.setattr(gate, "_memory_set", AsyncMock(side_effect=failure)) + monkeypatch.setattr(gate, "_restore", restore) + monkeypatch.setattr(gate, "_close_pg", close) + artifact = SimpleNamespace(emit=lambda kind, **kw: events.append(kind)) + task = asyncio.create_task(gate._run(args(), fixture, artifact, object())) + await restoring.wait() + task.cancel() + await asyncio.sleep(0) + task.cancel() + release.set() + with pytest.raises(ValueError) as raised: + await task + assert raised.value is failure and conn.closed + assert "triggers_restored" in events + assert settings.pg_delivery_batch_rows > 0 + + +@pytest.mark.parametrize("cleanup_failure", [False, True]) +def test_final_verdict_follows_native_cleanup_and_artifact_close( + monkeypatch, tmp_path, capsys, cleanup_failure +): + import sys + + import server.settings + import server.wiring + from server.cli import _ch + + fixture, config = fixture_config() + manifest = tmp_path / "manifest" + manifest.write_text(json.dumps(dataclasses.asdict(fixture))) + output = tmp_path / "result.jsonl" + arguments = args() + argv = ["driver"] + for name, value in { + **vars(arguments), + "fixture_manifest": str(manifest), + "output": str(output), + }.items(): + argv += ["--" + name.replace("_", "-"), str(value)] + order = [] + monkeypatch.setattr(sys, "argv", argv) + monkeypatch.setattr(server.settings, "settings", config) + monkeypatch.setattr(server.wiring, "wire", lambda: order.append("wire")) + monkeypatch.setattr(gate, "_preflight", AsyncMock()) + monkeypatch.setattr(gate, "_source", lambda _: {"clean": True}) + + def native(*_a, **_kw): + order.append("native_cleanup") + return { + "latency_within_limit": True, + "throughput_within_limit": True, + "p99_delta_ms": 0.0, + "throughput_loss": 0.0, + } + + monkeypatch.setattr(_ch, "run_connected", native) + + def sync(_fd): + assert capsys.readouterr().out == "" + assert order == ["wire", "native_cleanup"] + if cleanup_failure: + raise OSError("secret-token") + + monkeypatch.setattr(gate.os, "fsync", sync) + assert gate.main() == int(cleanup_failure) + final = capsys.readouterr().out + assert "secret-token" not in final + assert json.loads(final)["verdict"] == ("INVALID" if cleanup_failure else "PASS") + assert '"verdict": "PASS"' not in output.read_text() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", + [ + "", + "projection_value", + "projection_version", + "vector", + "audit_hash", + "audit_subject", + "missing_document", + "targets", + "attempts", + "head", + ], +) +async def test_composed_stored_verification_compares_authority_projection_vectors_and_audits( + monkeypatch, fault +): + import logging + + import server.embedding + from server.core.polled_copy import PolledCopy + from server.embedding.fake import FakeNGramEmbedder + from server.llm import customer + from server.memory.store import pg_memory + + monkeypatch.setattr( + customer, + "_switch", + PolledCopy("switch", AsyncMock(return_value=frozenset()), logging.getLogger(__name__)), + ) + monkeypatch.setattr(server.embedding, "get_embedder", lambda: FakeNGramEmbedder(dim=16)) + block = await gate._build_block(args(), 1) + block.seqs = [("org", "ns", i + 1) for i in range(5)] + records = [ + { + "head": json.dumps(pg_memory.encode_row(row)), + "revision": json.dumps(pg_memory.encode_row(row)), + "audit": json.dumps(pg_memory.encode_audit(audit)), + "seq": i + 1, + "targets": 1 if fault == "targets" else 0, + "attempts": 1 if fault == "attempts" else 0, + "last_error": "", + } + for i, (row, audit) in enumerate(zip(block.rows, block.audits, strict=True)) + ] + projections = [dict(row) for row in block.rows] + documents = [dict(row) for row in block.documents] + audits = [dict(row) for row in block.audits] + if fault == "projection_value": + projections[0]["value"] = '"wrong"' + if fault == "projection_version": + projections[0]["version"] -= 1 + if fault == "vector": + documents[0]["embedding"] = [0.0] * 16 + if fault == "audit_hash": + audits[0]["row_hash"] = "bad" + if fault == "audit_subject": + audits[0]["subject"] = "wrong" + if fault == "missing_document": + documents.pop() + if fault == "head": + records.pop() + conn = SimpleNamespace( + fetch=AsyncMock(return_value=records), + fetchrow=AsyncMock(return_value={"pending": 0, "audits": 0, "errors": 0}), + ) + + async def query(sql, params): + if "FROM loom.memory FINAL" in sql: + return projections + if "FROM loom.memory_embeddings FINAL" in sql: + return documents + if "FROM loom.audit_log" in sql: + return audits + raise AssertionError(sql) + + ch = SimpleNamespace(query_all=query) + if fault: + with pytest.raises(gate.Invalid): + await gate._verify(conn, ch, args(), block) + else: + await gate._verify(conn, ch, args(), block) + assert all( + "org=" in call.args[0] and "namespace=" in call.args[0] + for call in conn.fetch.await_args_list + ) + + +@pytest.mark.asyncio +async def test_real_customer_policy_can_refuse_expected_vectors_without_bypass(monkeypatch): + import logging + + import server.embedding + from server.core.polled_copy import PolledCopy + from server.embedding.fake import FakeNGramEmbedder + from server.llm import customer + + monkeypatch.setattr( + customer, + "_switch", + PolledCopy( + "switch", AsyncMock(return_value=frozenset({"org"})), logging.getLogger(__name__) + ), + ) + monkeypatch.setattr(server.embedding, "get_embedder", lambda: FakeNGramEmbedder(dim=16)) + with pytest.raises(customer.ModelCallsDisabled): + await gate._build_block(args(), 1) + assert customer._attribution.get() is None + + +@pytest.mark.asyncio +async def test_partial_trigger_toggle_is_restored_on_original_failure(monkeypatch): + conn = TriggerConn() + original = ValueError("DDL failed") + writes = 0 + + async def execute(sql): + nonlocal writes + writes += 1 + if writes == 2: + raise original + conn.on = sql.startswith("CREATE TRIGGER") + + async def snapshot(c): + return ("memory_current", "tenancy"), "invariant", "org" if c.on else gate._digest([]) + + monkeypatch.setattr(conn, "execute", execute) + monkeypatch.setattr(gate.Triggers, "snapshot", staticmethod(snapshot)) + monkeypatch.setattr(gate.cell_triggers, "missing", AsyncMock(return_value=[])) + triggers = await gate.Triggers.capture(conn) + with pytest.raises(ValueError) as raised: + try: + await triggers.set(conn, False) + finally: + await triggers.restore(conn) + assert raised.value is original and conn.on and triggers.on + + +def test_manifest_unknown_fields_and_credentials_refuse(tmp_path): + fixture, _ = fixture_config() + path = tmp_path / "manifest" + path.write_text(json.dumps({**dataclasses.asdict(fixture), "password": "secret"})) + with pytest.raises(gate.Invalid, match="fixture manifest"): + gate.Fixture.read(str(path)) + + +@pytest.mark.parametrize("fault", ["", "head", "tree", "dirty", "refused"]) +def test_source_provenance_uses_only_fixed_bounded_readonly_git(monkeypatch, fault): + fixture, _ = fixture_config() + calls = [] + + def git(argv, **kwargs): + calls.append(argv) + assert argv[0] == "git" and kwargs["timeout"] == 5 and kwargs["check"] is False + assert kwargs["capture_output"] is True and kwargs["cwd"].name == "scale-attachcli" + assert "shell" not in kwargs + if argv[1:] == ["rev-parse", "HEAD"]: + value = "wrong" if fault == "head" else fixture.build_commit + elif argv[1:] == ["rev-parse", "HEAD^{tree}"]: + value = "wrong" if fault == "tree" else fixture.source_tree + else: + assert argv[1:] == ["status", "--porcelain"] + value = " M file" if fault == "dirty" else "" + return SimpleNamespace(returncode=int(fault == "refused"), stdout=value) + + monkeypatch.setattr(gate.subprocess, "run", git) + if fault: + with pytest.raises(gate.Invalid): + gate._source(fixture) + else: + assert gate._source(fixture) == { + "head": fixture.build_commit, + "tree": fixture.source_tree, + "clean": True, + } + assert len(calls) == 3 + + +@pytest.mark.asyncio +async def test_restoration_has_finite_deadline_and_closes_owned_connection(monkeypatch): + conn = ProofPG() + + class Stuck: + async def restore(self, _): + await asyncio.Event().wait() + + monkeypatch.setattr(gate, "RESTORE_S", 0.01) + + async def close(owned): + assert owned is conn + await conn.close() + + monkeypatch.setattr(gate, "_close_pg", close) + artifact = SimpleNamespace(emit=lambda *_a, **_kw: None) + with pytest.raises(TimeoutError): + await gate._cleanup(Stuck(), conn, artifact) + assert conn.closed From 459dc34e49e94b19d5023472efe72bee27b3e8ad Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 00:31:50 +0000 Subject: [PATCH 035/114] Fix control batch bounds and schema coexistence checks Signed-off-by: zlareb1 --- docs/architecture/cells.md | 44 +++--- docs/operations/cell-attach.md | 4 +- server/cli/cell_attach_pg.py | 5 +- server/cli/cell_control.py | 2 +- server/router/app.py | 5 +- .../fixtures/router_placement_n_minus_one.py | 141 ++++++++++++++++++ tests/integration/test_cell_attach.py | 4 +- .../integration/test_cell_control_cleanup.py | 119 +++++++++++++++ tests/integration/test_pg_online_index.py | 37 +++-- .../test_pg_online_index_durability.py | 6 +- tests/unit/test_cell_attach.py | 18 +-- tests/unit/test_cell_control.py | 21 +++ tests/unit/test_cell_control_census.py | 36 ++++- tests/unit/test_router_attach.py | 14 ++ 14 files changed, 401 insertions(+), 55 deletions(-) create mode 100644 tests/integration/fixtures/router_placement_n_minus_one.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index a0079ac900..1954a69930 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -677,13 +677,13 @@ inserts cannot hold the write off, a new-org request waits on attach for a few s at most, and an attach that stalls holding the write's lock releases it within seconds. Attach takes `LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE` (which waits for running inserts and queues later ones behind it) in short tries, in one transaction with -server-side timeouts, then writes the mode and commits; after a failed try it ends the -mode transaction still in flight, found by its `ExclusiveLock` on `loom.tenancy_mode` in -this database (a pid captured at connect is wrong behind a connection path that swaps -backends), and waits until the lock clears. Whether the commit landed is decided by a -durable read of the mode, whose commit waits for the standby, since a commit ended while -it waited is on the primary alone ([attaching a cell](#attaching-a-cell): step 3 and the -durable read). +server-side timeouts, then writes the mode and commits on its exact ATTACH backend. +Only recognized pre-COMMIT contention can retry, after acknowledged rollback and a check +that this backend still holds ATTACH. Unexpected failure or an uncertain COMMIT closes +the owned session and ends the run. Attach neither finds nor ends another backend. A +fresh run takes the lock and claims again; whether an earlier commit landed is decided +by a durable mode read whose COMMIT waits for the standby +([attaching a cell](#attaching-a-cell): step 3 and the durable read). The mode cache reads the mode with a plain `SELECT`, which takes no row lock and never waits on attach. An insert that finds `enforced` (a process whose cache still reads `recording`) writes nothing: the request goes on to the placement call as in @@ -1065,7 +1065,7 @@ Then, in order (the step numbers are the ones the rest of this page uses): checks and the standbys again, and the scan's elapsed time within half of `--scan-budget`, since step 4 reads the same tables again. Each refuses in `recording`, which can still be backed out. -- **Step 3, the mode commit**: `attach/enforced`, then on a session of its own an EXCLUSIVE +- **Step 3, the mode commit**: `attach/enforced`, then on the same ATTACH session an EXCLUSIVE lock on `loom.tenancy_mode`, 2 s a try within `--wait`, the attach lock confirmed and the cheap heartbeat refusals run before each try (a booting process skips the try). Each try, including COMMIT, has a client deadline: the lock wait plus a 3 s statement margin and @@ -1077,10 +1077,11 @@ Then, in order (the step numbers are the ones the rest of this page uses): failure or exhausted cleanup time aborts the transport and preserves the original failure; abort alone does not prove that server locks disappeared. A commit past its deadline has an unknown outcome until the durable read. - After a failed try, the mode transaction an earlier run left in flight is found by its lock (only - attach takes `ExclusiveLock` there; a relation's OID is matched in this database alone) - and ended. From the commit on the mode only rises: every later refusal leaves the cell - `enforced` and `attaching`, and a rerun resumes from the durable read. + Only acknowledged pre-COMMIT rollback with ATTACH still held permits another try on + that backend. Any other failure ends the run without terminating another backend; + rerun to take the lock and claim afresh. From an acknowledged commit on, the mode + only rises: every later refusal leaves the cell `enforced` and `attaching`, and a + rerun resumes from the durable read. - **The durable read**, in every run that reads `enforced`: the mode and the `first` row read in a transaction that emits a transactional logical message, whose commit waits for the synchronous standbys, with the standbys checked attached just before and after. Then @@ -1312,17 +1313,18 @@ participate in this protocol. - `loom cell limit --cell --closed` closes a cell to new orgs, `--open` reopens a serving cell, and `--placements-max ` sets its placement quota (`--placements-max none` clears it). -- `loom cell remove --cell ` removes the directory row of a cell that is `attaching` - with no placement (`--cell` is the cell's `LOOM_CELL`, unset by then). For the first cell - it reads the cell's Postgres alone, on one session from `LOOM_PG_DSN` with no ClickHouse, - and refuses unless it reads `open` and records the row's deployment id, which enforces the - back-out order: unset `LOOM_CELL` on every process (each serves as in `recording`), then - the full-state cleanup, then the removal +- `loom cell remove --cell ` removes the directory row of an `attaching` cell + with no placement. For both first and later cells, keep `LOOM_CELL=` and the + registered store settings on the control client. Removal takes the direct marked + ATTACH backend, verifies the paired deployment and registration, and claims a fresh + directory generation before deleting the row. The first cell must also read `open`, + enforcing the back-out order: stop or unset `LOOM_CELL` on the accepting processes, + complete full-state cleanup, then remove the directory row ([troubleshooting](../operations/troubleshooting.md#a-process-with-loom_cell-unset-is-gated)). Removing the row first answers every new org 503; clearing the state first gates every - process with `LOOM_CELL` set. A first cell with a placement goes forward, never back. A - removed cell's stores keep whatever state rows they hold, so the full-state cleanup runs - before they are prepared again. + accepting process with `LOOM_CELL` set. A first cell with a placement goes forward, + never back. Removal leaves the stores' state rows intact; full-state cleanup must + precede preparing those stores again. - The router's operator credential is `lco_` and 128 lowercase hex (a router with no `LOOM_ROUTER_OPERATOR_SHA256` answers the operator API 503 with `NO_OPERATOR_CREDENTIAL`), which `loom cell credential new --operator` makes; the diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index 13c65405f6..20b541aec3 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -55,8 +55,8 @@ Every item below is a precondition attach checks, or one it cannot check and rel `max_prepared_transactions = 0`; the database encoding is UTF8. 13. No org longer than 2048 bytes is in the cell's stores (below). `--dry-run` lists any. -Attach's Postgres role needs the privileges of `pg_read_all_stats` and `pg_signal_backend` -without `SET ROLE` (grant their memberships with `INHERIT TRUE`), `USAGE` on schema +Attach's Postgres role needs the privileges of `pg_read_all_stats` +without `SET ROLE` (grant its membership with `INHERIT TRUE`), `USAGE` on schema `loom`, `SELECT` on `loom.cell_first`, `loom.tenancy_mode`, `loom.cell_self`, `loom.authority` and every org table, `UPDATE` on `loom.tenancy_mode`, `INSERT` on `loom.tenancy`, `SELECT`, `INSERT`, `UPDATE` and `DELETE` on `loom.cell_probe`, `TEMPORARY` diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index d18ff6f628..abf23bbf43 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -399,8 +399,6 @@ async def session_passes() -> bool: SELECT need, held FROM (VALUES ('effective privileges of pg_read_all_stats', pg_has_role(current_user, 'pg_read_all_stats', 'USAGE')), - ('effective privileges of pg_signal_backend', - pg_has_role(current_user, 'pg_signal_backend', 'USAGE')), ('USAGE on schema loom', has_schema_privilege('loom', 'USAGE')), ('SELECT on loom.cell_first', has_table_privilege('loom.cell_first', 'SELECT')), ('SELECT on loom.tenancy_mode', has_table_privilege('loom.tenancy_mode', 'SELECT')), @@ -582,8 +580,7 @@ def _busy_refusal(table: str, holders: list[Any]) -> Refusal: # -- step 3: the mode commit ------------------------------------------------------------ -# The mode transaction in flight: the backend holding or waiting for the ExclusiveLock on -# loom.tenancy_mode in this database, which only attach takes. +# Only acknowledged pre-COMMIT contention permits another try on the same session. _TIMED_OUT = frozenset({"55P03", "57014"}) diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index 7fd92e28ab..c4360839ed 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -138,7 +138,7 @@ async def _open(self, what: str) -> None: self.conn = await pg._connect_dedicated(statement_timeout_ms=60_000) if self.closed: # A connector returning after cancellation cannot publish a usable session. - await pg.close_owned(self.conn, end=self.close_end or 0) + await self.close() raise _refused("connection arrived after the control attempt ended") try: await pg._check_deployment(self.conn) diff --git a/server/router/app.py b/server/router/app.py index 39a1ac23c8..52c9d3d157 100644 --- a/server/router/app.py +++ b/server/router/app.py @@ -723,7 +723,10 @@ async def attach_place(self, request: Request) -> JSONResponse: expected = _control(body) if isinstance(expected, JSONResponse): return expected - if len(json.dumps(body, ensure_ascii=True).encode()) > MAX_BODY_BYTES: + if ( + len(json.dumps(body, ensure_ascii=True, separators=(",", ":")).encode()) + > MAX_BODY_BYTES + ): return _refused( 413, f"a placement batch without its control envelope is at most {MAX_BODY_BYTES} bytes", diff --git a/tests/integration/fixtures/router_placement_n_minus_one.py b/tests/integration/fixtures/router_placement_n_minus_one.py new file mode 100644 index 0000000000..dea9c92c22 --- /dev/null +++ b/tests/integration/fixtures/router_placement_n_minus_one.py @@ -0,0 +1,141 @@ +"""Frozen pre-generation router placement/query code for schema coexistence tests. + +The serving statements and row model are copied from the checkpoint before the +control generation column. Connection acquisition/error handling are injected from +the test directory; no current placement projection or writer is imported. +""" + +from __future__ import annotations + +import hmac +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from typing import Any + +from server.core import cell as cell_state +from server.router.directory import OrgTooLong, PlacementRefused, _connection, _reading + +_PLACE_LOCK = 0x6C726F70 # "lrop" + + +@dataclass(frozen=True) +class Cell: + """What the router's APIs read of a cell's row.""" + + cell: str + is_first: bool + host: str + state: str + mode: str + closed: bool + placements_max: int | None + placed: int + credential_sha256: str + deployment_id: str + current_fp: dict[str, Any] | None + next_fp: dict[str, Any] | None + # The new credential's hash during a rotation (`loom cell credential --next`). + credential_next_sha256: str | None + + def holds(self, presented: str) -> bool: + """Whether `presented` (a credential's SHA-256) is this cell's, current or next.""" + hashes = (self.credential_sha256, self.credential_next_sha256) + return holds(presented, [h for h in hashes if h]) + + +def holds(presented: str, hashes: Iterable[str]) -> bool: + """Whether `presented` (a credential's SHA-256) is one of `hashes` (each set), in + constant time each.""" + return any(hmac.compare_digest(presented, h) for h in hashes) + + +_CELL_COLUMNS = ( + "cell, is_first, host, state, mode, closed, placements_max, placed, credential_sha256, " + "credential_next_sha256, deployment_id, current_fp, next_fp" +) + +_CELL = f"SELECT {_CELL_COLUMNS} FROM loom_router.cells" + +_WINNER = ( + "SELECT p.cell, c.deployment_id FROM loom_router.placements AS p " + "JOIN loom_router.cells AS c USING (cell) WHERE p.org = $1" +) + + +def _cell(row: Any) -> Cell: + return Cell(**dict(row)) + + +async def place(org: str, cell_name: str, refusal: Callable[[Cell], str]) -> tuple[str, str]: + """Place `org` in `cell_name` unless it is placed already; the (cell, deployment id) + that holds it. An org placed already answers its cell, whatever the cell's state. + Otherwise the cell's row is locked, the winner read again (an attach batch holds the row + lock, not the org's), and the row judged (`refusal`, then its quota), so a change to it + waits for the placement and the quota holds across router replicas; a refused placement + raises PlacementRefused and writes nothing, and a new org name longer than + `cell.ORG_MAX_BYTES` raises OrgTooLong (one placed before the limit answers its cell, as + any placed org does).""" + async with _reading(), _connection() as conn, conn.transaction(): + # Placements of one org run one at a time, whichever cell asks: the later one + # reads the earlier one's winner. + await conn.execute("SELECT pg_advisory_xact_lock($1, hashtext($2))", _PLACE_LOCK, org) + row = await conn.fetchrow(_WINNER, org) + if row is None: + _refuse_long([org]) + locked = await conn.fetchrow(f"{_CELL} WHERE cell = $1 FOR UPDATE", cell_name) + if locked is None: + raise PlacementRefused(f"cell {cell_name!r} is not in the directory") + # An attach batch of this cell, which holds the row lock and not the org's, may + # have placed the org since the first read: it is answered as placed. + row = await conn.fetchrow(_WINNER, org) + if row is None: + await _place_one(conn, org, cell_name, _cell(locked), refusal) + row = await conn.fetchrow(_WINNER, org) + return str(row["cell"]), str(row["deployment_id"]) + + +async def _place_one( + conn: Any, org: str, cell_name: str, candidate: Cell, refusal: Callable[[Cell], str] +) -> None: + """Judge the locked row (`refusal`, then its quota) and insert the placement; its count + and audit row only when the insert wrote it.""" + if reason := refusal(candidate): + raise PlacementRefused(reason) + quota = candidate.placements_max + if quota is not None and candidate.placed >= quota: + raise PlacementRefused(f"cell {cell_name!r} holds its quota of orgs") + inserted = await conn.fetchval( + "INSERT INTO loom_router.placements (org, cell) VALUES ($1, $2) " + "ON CONFLICT DO NOTHING RETURNING org", + org, + cell_name, + ) + if inserted is None: + return + await conn.execute( + "UPDATE loom_router.cells SET placed = placed + 1 WHERE cell = $1", cell_name + ) + await conn.execute( + "INSERT INTO loom_router.audit (actor, action, detail) VALUES ($1, 'place', $2)", + f"cell:{cell_name}", + {"org": org, "cell": cell_name}, + ) + + +def _refuse_long(orgs: Iterable[str]) -> None: + """OrgTooLong for the first org longer than `cell.ORG_MAX_BYTES`: the one check every + route that inserts a placement runs, so no route places such an org.""" + for org in orgs: + if reason := cell_state.org_name_refusal(org): + raise OrgTooLong(reason) + + +async def lookup(org: str) -> tuple[str, str] | None: + """The (cell, host) an org is placed in, or None.""" + async with _reading(), _connection(lookup=True) as conn: + row = await conn.fetchrow( + "SELECT c.cell, c.host FROM loom_router.placements AS p " + "JOIN loom_router.cells AS c USING (cell) WHERE p.org = $1", + org, + ) + return (str(row["cell"]), str(row["host"])) if row else None diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 6cbc595bd5..a84474806e 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -809,7 +809,7 @@ async def test_signaling_another_role_requires_an_effective_signal_grant(cell_st @pytest.mark.asyncio -@pytest.mark.parametrize("capability", ["pg_read_all_stats", "pg_signal_backend"]) +@pytest.mark.parametrize("capability", ["pg_read_all_stats"]) async def test_attach_needs_its_predefined_capabilities_without_set_role( cell_stores, monkeypatch, capsys, capability: str ) -> None: @@ -829,7 +829,7 @@ async def as_attacher(**kwargs: Any) -> Any: async with real_dedicated() as conn: owner = await conn.fetchval("SELECT current_user") await conn.execute(f"GRANT {owner} TO {role} WITH INHERIT TRUE, SET FALSE") - await conn.execute(f"GRANT pg_read_all_stats, pg_signal_backend TO {role}") + await conn.execute(f"GRANT pg_read_all_stats TO {role}") await conn.execute(f"GRANT {capability} TO {role} WITH INHERIT FALSE") assert await conn.fetchval("SELECT pg_has_role($1, $2, 'MEMBER')", role, capability) assert not await conn.fetchval("SELECT pg_has_role($1, $2, 'USAGE')", role, capability) diff --git a/tests/integration/test_cell_control_cleanup.py b/tests/integration/test_cell_control_cleanup.py index f66eeb5169..e057fef323 100644 --- a/tests/integration/test_cell_control_cleanup.py +++ b/tests/integration/test_cell_control_cleanup.py @@ -293,3 +293,122 @@ async def read(self): with pytest.raises(asyncio.CancelledError, match="native attach cancellation"): await asyncio.wait_for(task, 1.3) await released(native_database, backend["pid"]) + + +async def test_attach_role_needs_stats_but_not_backend_signal_privilege(native_database): + import uuid + + from server.cli import cell_attach_pg as apg + + role = f"it_attach_grants_{uuid.uuid4().hex[:10]}" + conn = native_database + database = await conn.fetchval("SELECT current_database()") + await conn.execute(f"CREATE ROLE {role}") + try: + await conn.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + await conn.execute( + f"GRANT SELECT ON loom.cell_first, loom.tenancy_mode, loom.cell_self, loom.authority TO {role}" + ) + await conn.execute(f"GRANT UPDATE ON loom.tenancy_mode TO {role}") + await conn.execute(f"GRANT INSERT ON loom.tenancy TO {role}") + await conn.execute(f"GRANT SELECT, INSERT, UPDATE, DELETE ON loom.cell_probe TO {role}") + await conn.execute(f'GRANT TEMPORARY ON DATABASE "{database}" TO {role}') + await conn.execute(f"GRANT pg_read_all_stats TO {role} WITH INHERIT FALSE") + await conn.execute(f"SET SESSION AUTHORIZATION {role}") + assert not await conn.fetchval( + "SELECT pg_has_role(current_user, 'pg_signal_backend', 'USAGE')" + ) + refused = await apg.own_role_refusals(conn, []) + assert [r.reason for r in refused] == [ + "attach's role lacks effective privileges of pg_read_all_stats" + ] + await conn.execute("RESET SESSION AUTHORIZATION") + await conn.execute(f"GRANT pg_read_all_stats TO {role} WITH INHERIT TRUE") + await conn.execute(f"SET SESSION AUTHORIZATION {role}") + assert not await conn.fetchval( + "SELECT pg_has_role(current_user, 'pg_signal_backend', 'USAGE')" + ) + assert await apg.own_role_refusals(conn, []) == [] + finally: + await conn.execute("RESET SESSION AUTHORIZATION") + await conn.execute(f"DROP OWNED BY {role}") + await conn.execute(f"DROP ROLE {role}") + + +@pytest.mark.parametrize("late", [False, True]) +@pytest.mark.parametrize("kind", ["exit", "keyboard"]) +async def test_native_public_close_fatal_stays_inside_owner(native_database, late, kind): + import os + import sys + + from tests.integration._pg_database import with_database + + dsn = with_database( + os.environ["LOOM_PG_DSN"], await native_database.fetchval("SELECT current_database()") + ) + script = r""" +import asyncio, os, sys +import asyncpg +from server.cli import cell_control +from server.core import pg +from server.settings import settings +late, kind = sys.argv[1:] +settings.pg_dsn = os.environ['LOOM_NATIVE_CLEANUP_DSN'] +settings.cell = 'first' +pg._expected = None +original = SystemExit(17) if kind == 'exit' else KeyboardInterrupt('close interrupted') +real_connect, real_close = pg._connect_dedicated, asyncpg.Connection.close +backend = {} +async def connect(**kwargs): + conn = await real_connect(**kwargs) + backend['pid'] = conn.get_server_pid() + if late == 'True': + try: await asyncio.Future() + except asyncio.CancelledError: pass + return conn +async def close(self, **kwargs): + ours = self.get_server_pid() == backend.get('pid') + await real_close(self, **kwargs) + if ours: + backend['closes'] = backend.get('closes', 0) + 1 + raise original +async def body(owner): + await owner.conn.execute('SELECT * FROM probe FOR UPDATE') + return 19 +pg._connect_dedicated = connect +asyncpg.Connection.close = close +if late == 'True': cell_control.OPEN_S = .5 +async def main(): + observer = await asyncpg.connect(settings.pg_dsn) + try: + try: await cell_control.run('native close', body) + except BaseException as caught: + if late == 'True': assert isinstance(caught, TimeoutError), type(caught) + else: assert caught is original, type(caught) + assert backend['closes'] == 1 + assert not await observer.fetchval('SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1)', backend['pid']) + assert not await observer.fetchval('SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)', backend['pid']) + print('LOOP_STILL_RUNNING') + else: raise AssertionError('required close fatal became success') + finally: + asyncpg.Connection.close = real_close + await observer.close(timeout=1) +asyncio.run(main()) +""" + child = await asyncio.create_subprocess_exec( + sys.executable, + "-c", + script, + str(late), + kind, + env={**os.environ, "LOOM_NATIVE_CLEANUP_DSN": dsn}, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, stderr = await asyncio.wait_for(child.communicate(), 10) + assert child.returncode == 0 and b"LOOP_STILL_RUNNING" in stdout, stderr.decode() + finally: + if child.returncode is None: + child.kill() + await child.wait() diff --git a/tests/integration/test_pg_online_index.py b/tests/integration/test_pg_online_index.py index 1995e73bee..38638ad877 100644 --- a/tests/integration/test_pg_online_index.py +++ b/tests/integration/test_pg_online_index.py @@ -55,6 +55,14 @@ def _prefix(tmp_path: Path) -> Path: return prefix +def _journal() -> list[str]: + return sorted(path.name for path in ROUTER_MIGRATIONS_DIR.glob("*.sql")) + + +def _after_prefix() -> list[str]: + return _journal()[3:] + + async def _migrate(connect: Any, files: Path = ROUTER_MIGRATIONS_DIR) -> list[str]: return await pg_migrate.migrate(files, connect=connect, schema="loom_router", lock_key=8917423) @@ -62,7 +70,7 @@ async def _migrate(connect: Any, files: Path = ROUTER_MIGRATIONS_DIR) -> list[st async def test_online_migration_installs_and_retries_from_both_schemas(database, tmp_path) -> None: conn, connect = database await _migrate(connect, _prefix(tmp_path)) - assert await _migrate(connect) == ["004_placements_by_cell_org.sql"] + assert await _migrate(connect) == _after_prefix() assert await _migrate(connect) == [] index = pg_online_index.parse( (ROUTER_MIGRATIONS_DIR / "004_placements_by_cell_org.sql").read_text() @@ -73,7 +81,7 @@ async def test_online_migration_installs_and_retries_from_both_schemas(database, async def test_install_from_empty(database) -> None: _, connect = database - assert len(await _migrate(connect)) == 4 + assert await _migrate(connect) == _journal() async def _until(conn: Any, sql: str, *args: Any) -> None: @@ -132,12 +140,19 @@ async def test_old_snapshot_old_blocking_migrator_and_two_new_migrators(database "(cell, host, state, credential_sha256, deployment_id) " "VALUES ('a', 'a.example.test', 'serving', repeat('a', 64), 'dep-a')" ) - assert await directory.place("org-a", "a", lambda _: "") == ("a", "dep-a") - assert await directory.lookup("org-a") == ("a", "a.example.test") + from tests.integration.fixtures import router_placement_n_minus_one as serving + + assert await serving.place("org-a", "a", lambda _: "") == ("a", "dep-a") + assert await serving.lookup("org-a") == ("a", "a.example.test") + assert await conn.fetchval("SELECT placed FROM loom_router.cells WHERE cell = 'a'") == 1 + assert ( + await conn.fetchval("SELECT count(*) FROM loom_router.audit WHERE action = 'place'") + == 1 + ) await tx.rollback() async with asyncio.timeout(5): results = await asyncio.gather(first, second) - assert sorted(results) == [[], ["004_placements_by_cell_org.sql"]] + assert sorted(results) == [[], _after_prefix()] finally: if snapshot.is_in_transaction(): await tx.rollback() @@ -182,7 +197,7 @@ async def test_cancelled_build_leaves_a_matching_invalid_index_and_retry_repairs if not migrating.done(): migrating.cancel() await asyncio.gather(migrating, return_exceptions=True) - assert await _migrate(connect) == ["004_placements_by_cell_org.sql"] + assert await _migrate(connect) == _after_prefix() assert await _migrate(connect) == [] @@ -241,7 +256,7 @@ async def completed_then_cancel(*args, **kwargs): await _migrate(connect) oid = await conn.fetchval("SELECT 'loom_router.placements_by_cell_org'::regclass::oid") monkeypatch.setattr(pg_online_index, "build", original) - assert await _migrate(connect) == ["004_placements_by_cell_org.sql"] + assert await _migrate(connect) == _after_prefix() assert await conn.fetchval("SELECT 'loom_router.placements_by_cell_org'::regclass::oid") == oid @@ -347,7 +362,7 @@ async def direct_drop(): if not dropping.done(): dropping.cancel() await asyncio.gather(dropping, return_exceptions=True) - assert await _migrate(connect) == ["004_placements_by_cell_org.sql"] + assert await _migrate(connect) == _after_prefix() async def test_try_lock_wait_has_no_retained_transaction_or_snapshot(database) -> None: @@ -364,7 +379,7 @@ async def test_try_lock_wait_has_no_retained_transaction_or_snapshot(database) - ) assert not migrating.done() await conn.execute("SELECT pg_advisory_unlock(8917423::bigint)") - assert len(await asyncio.wait_for(migrating, 5)) == 4 + assert await asyncio.wait_for(migrating, 5) == _journal() finally: await conn.execute("SELECT pg_advisory_unlock(8917423::bigint)") if not migrating.done(): @@ -376,6 +391,8 @@ async def test_online_validation_is_repeated_after_later_files(database, tmp_pat _, connect = database for source in ROUTER_MIGRATIONS_DIR.glob("*.sql"): (tmp_path / source.name).write_bytes(source.read_bytes()) - (tmp_path / "005_damage.sql").write_text("DROP INDEX loom_router.placements_by_cell_org;") + ( + tmp_path / f"{max(int(name.split('_')[0]) for name in _journal()) + 1:03d}_damage.sql" + ).write_text("DROP INDEX loom_router.placements_by_cell_org;") with pytest.raises(ValueError, match="recorded online-index"): await _migrate(connect, tmp_path) diff --git a/tests/integration/test_pg_online_index_durability.py b/tests/integration/test_pg_online_index_durability.py index e5644af5af..f38453e747 100644 --- a/tests/integration/test_pg_online_index_durability.py +++ b/tests/integration/test_pg_online_index_durability.py @@ -158,7 +158,11 @@ async def pause_before_ledger(*args, **kwargs): await _waiting(observer, retry) await _docker("unpause", STANDBY) paused = False - assert await asyncio.wait_for(retry, 8) == [] + assert await asyncio.wait_for(retry, 8) == sorted( + path.name + for path in ROUTER_MIGRATIONS_DIR.glob("*.sql") + if path.name > "004_placements_by_cell_org.sql" + ) observer.terminate() await _docker("kill", "--signal", "KILL", PRIMARY) await _docker( diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index a69c5c5c8f..5188641547 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -239,7 +239,6 @@ def lock(sql: str, *_a: Any) -> str: conn = FakeConn( [ ("SET LOCAL lock_timeout", lambda *a: lock(conn.executed[-1])), - ("l.mode = 'ExclusiveLock'", []), ] ) @@ -487,6 +486,7 @@ async def fetchval(self, sql: str, *args: Any) -> Any: return self._answer(sql, args) async def execute(self, sql: str, *args: Any) -> str: + assert "pg_terminate_backend" not in sql and "pg_cancel_backend" not in sql self.executed.append(sql) for needle, answer in self.routes: if needle in sql: @@ -697,16 +697,10 @@ def lock(*_a: Any) -> str: raise answer return "LOCK TABLE" - holders = _seq( - [{"pid": 5, "application_name": "attach", "xact_start": "t", "rolsuper": False}], [] - ) conn = FakeConn( [ ("LOCK TABLE", lock), ("UPDATE loom.tenancy_mode", "UPDATE 1"), - ("l.mode = 'ExclusiveLock'", holders), - ("rolsuper FROM pg_roles", False), - ("pg_terminate_backend", True), ] ) confirmed: list[bool] = [] @@ -733,6 +727,11 @@ async def ready() -> list[str]: assert len(confirmed) == 2 assert conn.executed.count("ROLLBACK") == 1 assert "TERMINATE" not in conn.executed + assert not any( + "pg_terminate_backend" in sql or "pg_cancel_backend" in sql for sql in conn.executed + ) + assert conn.executed.count("COMMIT") == 1 + assert sum(sql.startswith("UPDATE loom.tenancy_mode") for sql in conn.executed) == 1 @pytest.mark.asyncio @@ -742,7 +741,7 @@ async def test_the_mode_commit_refuses_past_wait_and_skips_a_try_while_booting( monkeypatch: pytest.MonkeyPatch, wait_s: int, ) -> None: - conn = FakeConn([("l.mode = 'ExclusiveLock'", [])]) + conn = FakeConn() run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] run.owner = _owner(FakeConn()) run.own_fp = (CURRENT, NEXT) @@ -1219,7 +1218,6 @@ async def test_wait_0_runs_one_try_of_each_wait( [ ("LOCK TABLE", "LOCK TABLE"), ("UPDATE loom.tenancy_mode", "UPDATE 1"), - ("l.mode = 'ExclusiveLock'", []), ] ) marks: list[str] = [] @@ -1282,7 +1280,7 @@ async def test_a_mode_commit_refused_without_a_landed_try_says_recording( def lock(*_a: Any) -> str: raise _Timeout() - conn = FakeConn([("LOCK TABLE", lock), ("l.mode = 'ExclusiveLock'", [])]) + conn = FakeConn([("LOCK TABLE", lock)]) async def call(_self: Any, _path: str, _body: Any = None) -> dict[str, Any]: return {"cell": "first"} diff --git a/tests/unit/test_cell_control.py b/tests/unit/test_cell_control.py index 6b2b821b91..ccbbb7b6f9 100644 --- a/tests/unit/test_cell_control.py +++ b/tests/unit/test_cell_control.py @@ -284,3 +284,24 @@ async def opening(self, _what): with pytest.raises(OSError, match="required close failed"): await cell_control.run("test", AsyncMock(return_value=7)) assert "TERMINATE" in conn.executed + + +@pytest.mark.asyncio +async def test_late_connection_arriving_inside_reserve_is_closed_once(monkeypatch): + from server.core import pg + + conn = FakeConn() + + async def connect(**_): + try: + await asyncio.Future() + except asyncio.CancelledError: + return conn + + monkeypatch.setattr(pg, "_connect_dedicated", connect) + monkeypatch.setattr(cell_control, "OPEN_S", 0.08) + work = AsyncMock() + with pytest.raises(TimeoutError): + await cell_control.run("late", work) + work.assert_not_awaited() + assert conn.executed == ["CLOSE"] diff --git a/tests/unit/test_cell_control_census.py b/tests/unit/test_cell_control_census.py index 0c358f2c38..45b8e85baa 100644 --- a/tests/unit/test_cell_control_census.py +++ b/tests/unit/test_cell_control_census.py @@ -19,16 +19,16 @@ TABLES = r"(?:loom\.(?:cell_self|cell_first|tenancy_mode)|loom_router\.[A-Za-z_]+)" TARGET = rf"(?:{TABLES}|loom(?:_router)?\.\{{[^}}]+\}}|\{{[^}}]+\}})" WRITES = re.compile( - rf"(?:^|;)\s*(INSERT\s+INTO|UPDATE|DELETE\s+FROM|TRUNCATE(?:\s+TABLE)?|" + rf"(?:^|;|[()])\s*(INSERT\s+INTO|UPDATE|DELETE\s+FROM|TRUNCATE(?:\s+TABLE)?|" rf"ALTER\s+TABLE(?:\s+IF\s+EXISTS)?|CREATE\s+TABLE(?:\s+IF\s+NOT\s+EXISTS)?|" rf"DROP\s+TABLE(?:\s+IF\s+EXISTS)?)\s+({TARGET})" - rf"|(?:^|;)\s*((?:CREATE|DROP)\s+TRIGGER)\b[^;]*?\bON\s+({TARGET})", + rf"|(?:^|;|[()])\s*((?:CREATE|DROP)\s+TRIGGER)\b[^;]*?\bON\s+({TARGET})", re.IGNORECASE, ) def mutations(sql: str) -> list[tuple[str, str]]: - sql = re.sub(r"--[^\n]*", "", sql).replace('"', "") + sql = re.sub(r"--[^\n]*|/\*.*?\*/", "", sql, flags=re.DOTALL).replace('"', "") return [(" ".join((a or c).upper().split()), b or d) for a, b, c, d in WRITES.findall(sql)] @@ -505,6 +505,22 @@ def test_reads_registration_and_immutable_placement_are_explicit_exceptions(): ("statement", "expected"), [ ('"UPDATE loom.cell_self SET current_fp = $1"', ("UPDATE", "loom.cell_self")), + ( + '"WITH expected AS (SELECT 1) UPDATE loom.cell_self SET current_fp = $1"', + ("UPDATE", "loom.cell_self"), + ), + ( + '"WITH expected AS (SELECT 1) DELETE FROM loom_router.cells WHERE cell = $1"', + ("DELETE FROM", "loom_router.cells"), + ), + ( + '"WITH changed AS (UPDATE loom.cell_self SET current_fp = $1 RETURNING 1) SELECT * FROM changed"', + ("UPDATE", "loom.cell_self"), + ), + ( + '"WITH outer_cte AS (WITH inner_cte AS (SELECT 1) UPDATE loom_router.cells SET closed = true RETURNING cell) SELECT * FROM outer_cte"', + ("UPDATE", "loom_router.cells"), + ), ( '"INSERT INTO loom_router.cells (cell) VALUES ($1)"', ("INSERT INTO", "loom_router.cells"), @@ -550,3 +566,17 @@ def test_acceptance_writer_callsites_are_classified(): ("server/cli/cell_attach.py", "Run.commit_mode", "apg.commit_enforced"): 1, } ) + + +def test_cte_data_modification_classifies_every_literal_sink(): + source = """ +async def unclassified(conn): + await conn.execute("WITH changed AS (DELETE FROM loom.cell_self RETURNING 1) " + "/* body done */ UPDATE loom_router.cells SET closed = true") +""" + assert inventory(source).writes == Counter( + { + ("unclassified", "DELETE FROM", "loom.cell_self"): 1, + ("unclassified", "UPDATE", "loom_router.cells"): 1, + } + ) diff --git a/tests/unit/test_router_attach.py b/tests/unit/test_router_attach.py index bffc8aefea..821272bcdf 100644 --- a/tests/unit/test_router_attach.py +++ b/tests/unit/test_router_attach.py @@ -1437,3 +1437,17 @@ async def _run() -> None: assert again.json()["state"] == "serving" asyncio.run(_run()) + + +def test_native_compact_batch_with_control_envelope_fits_public_handler(client, store): + from server.cli.cell_attach import _encoded_size, batches + + _enforced(client) + orgs = [f"{i:03d}" + "x" * 127 for i in range(500)] + batch = batches(orgs, "dep-1")[0] + body = {"deployment_id": "dep-1", "orgs": batch} + assert _encoded_size(body) <= MAX_BODY_BYTES < len(json.dumps(body).encode()) + response = _call(client, "POST", "attach/place", body) + assert response.status_code == 200, response.text + assert response.json()["placed"] == len(batch) + assert set(store.placements) == set(batch) From 9caeb23ec169886d6cc028fc4330045591783677 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 00:31:05 +0000 Subject: [PATCH 036/114] Preserve native migration failures through owned cleanup Capture native failures inside owned work and public-close child tasks so their supervisors can clean up before rethrowing. Keep the first migration failure through context exit within its original cleanup deadline. Signed-off-by: zlareb1 --- server/core/pg.py | 14 ++- server/core/schema/pg_migrate.py | 75 +++++++++---- tests/unit/test_pg_close_owned.py | 75 +++++++++++++ tests/unit/test_pg_owned_fatal.py | 172 ++++++++++++++++++++++++++++++ 4 files changed, 315 insertions(+), 21 deletions(-) create mode 100644 tests/unit/test_pg_owned_fatal.py diff --git a/server/core/pg.py b/server/core/pg.py index d7bdf5b3e4..e70c702582 100644 --- a/server/core/pg.py +++ b/server/core/pg.py @@ -392,16 +392,26 @@ async def close_owned(conn: Any, *, end: float) -> None: is a bounded fallback, not evidence that the backend has released its locks. The owner preserves an earlier operation failure when this cleanup fails. """ + + async def close() -> SystemExit | KeyboardInterrupt | None: + try: + await conn.close(timeout=max(0, end - time.monotonic())) + except (SystemExit, KeyboardInterrupt) as error: + # A native fatal in a child Task stops the loop before its owner can clean up. + return error + return None + task: asyncio.Task[Any] | None = None try: remaining = end - time.monotonic() if remaining <= 0: raise TimeoutError("Postgres owned-session cleanup deadline expired") - task = asyncio.create_task(conn.close(timeout=remaining)) + task = asyncio.create_task(close()) done, _ = await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) if not done: raise TimeoutError("Postgres owned-session cleanup deadline expired") - task.result() + if isinstance(result := task.result(), (SystemExit, KeyboardInterrupt)): + raise result except BaseException: _terminate(conn) if task is not None: diff --git a/server/core/schema/pg_migrate.py b/server/core/schema/pg_migrate.py index 371f61a449..edc5cdb927 100644 --- a/server/core/schema/pg_migrate.py +++ b/server/core/schema/pg_migrate.py @@ -21,7 +21,7 @@ import logging import time from asyncio import sleep -from collections.abc import Callable +from collections.abc import Awaitable, Callable from contextlib import AbstractAsyncContextManager from pathlib import Path from typing import Any @@ -110,24 +110,61 @@ async def migrate( holds every read and write of the table behind it; the file is retried. For schema `loom` it marks the session and runs `cell_triggers.ensure` before the lock goes.""" files, indexes = _validated(mig_dir, schema) - async with (connect or pg.dedicated)() as conn: - task = asyncio.create_task(_migrate(conn, files, indexes, schema, lock_key)) - try: - return await asyncio.shield(task) - except BaseException: - end = time.monotonic() + pg._RELEASE_TIMEOUT_S - try: - await pg.close_owned(conn, end=end) - except BaseException: # noqa: BLE001 — preserve the migration failure/cancellation. - pass - if not task.done(): - task.cancel() - task.add_done_callback(pg._retrieve_task) - try: - await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) - except BaseException: # noqa: BLE001 — preserve the original failure on repeated cancel. - pg._terminate(conn) - raise + cm = (connect or pg.dedicated)() + conn = await cm.__aenter__() + task = asyncio.create_task(_owned_result(_migrate(conn, files, indexes, schema, lock_key))) + try: + result = await asyncio.shield(task) + if isinstance(result, (SystemExit, KeyboardInterrupt)): + raise result + except BaseException as error: + end = time.monotonic() + pg._RELEASE_TIMEOUT_S + await _failed_migration(cm, conn, task, error, end) + raise + await cm.__aexit__(None, None, None) + return result + + +async def _owned_result[T](work: Awaitable[T]) -> T | SystemExit | KeyboardInterrupt: + try: + return await work + except (SystemExit, KeyboardInterrupt) as error: + # Transport native failures through child Tasks without stopping their supervisor. + return error + + +async def _failed_migration( + cm: AbstractAsyncContextManager[Any], + conn: Any, + task: asyncio.Task[Any], + error: BaseException, + end: float, +) -> None: + try: + await pg.close_owned(conn, end=end) + except BaseException: # noqa: BLE001 — preserve the migration failure/cancellation. + pass + if not task.done(): + task.cancel() + task.add_done_callback(pg._retrieve_task) + try: + await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) + except BaseException: # noqa: BLE001 — preserve the first failure on repeated cancellation. + pg._terminate(conn) + release = asyncio.create_task( + _owned_result(cm.__aexit__(type(error), error, error.__traceback__)) + ) + try: + done, _ = await asyncio.wait({release}, timeout=max(0, end - time.monotonic())) + if not done: + raise TimeoutError("migration context cleanup deadline expired") + release.result() + except BaseException: # noqa: BLE001 — a context exit cannot replace the migration failure. + pg._terminate(conn) + finally: + if not release.done(): + release.cancel() + release.add_done_callback(pg._retrieve_task) def _validated( diff --git a/tests/unit/test_pg_close_owned.py b/tests/unit/test_pg_close_owned.py index fbbe778a5d..1ba7ab1f6a 100644 --- a/tests/unit/test_pg_close_owned.py +++ b/tests/unit/test_pg_close_owned.py @@ -142,3 +142,78 @@ async def fail(*_: Any) -> None: with pytest.raises(ValueError, match="migration failed"): await pg_migrate.migrate(tmp_path, connect=connect, schema="test") assert conn.aborted + + +async def test_successful_migration_still_reports_context_exit_failure( + monkeypatch, tmp_path +) -> None: + import contextlib + + from server.core.schema import pg_migrate + + original = ValueError("context exit failed") + + @contextlib.asynccontextmanager + async def connect(): + yield Connection() + raise original + + async def applied(*_: Any) -> list[str]: + return ["001.sql"] + + monkeypatch.setattr(pg_migrate, "_migrate", applied) + with pytest.raises(ValueError) as error: + await pg_migrate.migrate(tmp_path, connect=connect, schema="test") + assert error.value is original + + +@pytest.mark.parametrize("cancel_work", [False, True]) +async def test_second_cancellation_during_migration_context_exit_keeps_first_failure( + monkeypatch, tmp_path, cancel_work +) -> None: + import contextlib + + from server.core.schema import pg_migrate + + conn = Connection() + conn.finished.set() + started, exiting = asyncio.Event(), asyncio.Event() + observed: list[BaseException] = [] + original = ValueError("migration failed") + + @contextlib.asynccontextmanager + async def connect(): + try: + yield conn + except BaseException as error: + observed.append(error) + exiting.set() + await asyncio.Future() + raise + + async def work(*_: Any) -> list[str]: + started.set() + if cancel_work: + await asyncio.Future() + raise original + + monkeypatch.setattr(pg_migrate, "_migrate", work) + monkeypatch.setattr(pg, "_RELEASE_TIMEOUT_S", 0.1) + task = asyncio.create_task(pg_migrate.migrate(tmp_path, connect=connect, schema="test")) + await asyncio.wait_for(started.wait(), 1) + before = time.monotonic() + if cancel_work: + task.cancel("first cancellation") + await asyncio.wait_for(exiting.wait(), 1) + task.cancel("second cancellation") + with pytest.raises(BaseException) as result: + await asyncio.wait_for(task, 0.3) + assert time.monotonic() - before < 0.3 + assert result.value is observed[0] + assert ( + isinstance(result.value, asyncio.CancelledError) + if cancel_work + else result.value is original + ) + assert len(conn.timeouts) == 1 + assert conn.aborted diff --git a/tests/unit/test_pg_owned_fatal.py b/tests/unit/test_pg_owned_fatal.py new file mode 100644 index 0000000000..683dbca782 --- /dev/null +++ b/tests/unit/test_pg_owned_fatal.py @@ -0,0 +1,172 @@ +"""Owned child tasks preserve native failures until their supervisor cleans up.""" + +from __future__ import annotations + +import subprocess +import sys +import textwrap + +import pytest + + +@pytest.mark.parametrize("kind", ["SystemExit", "KeyboardInterrupt"]) +@pytest.mark.parametrize( + "phase", + ["close", "migration", "migration_exit", "migration_ordinary_exit", "migration_exit_stall"], +) +def test_native_failure_reaches_the_owner_after_cleanup(kind: str, phase: str, tmp_path) -> None: + program = textwrap.dedent( + """ + import asyncio + import contextlib + import sys + import time + from pathlib import Path + from server.core import pg + from server.core.schema import pg_migrate + + kind, phase, path = sys.argv[1:] + fatal_type = SystemExit if kind == 'SystemExit' else KeyboardInterrupt + original = ValueError('original') if 'ordinary' in phase else fatal_type('original') + secondary = fatal_type('cleanup') + events = [] + + class Connection: + async def close(self, **kwargs): + events.append('close') + if phase == 'close': + raise original + if phase in {'migration_exit', 'migration_ordinary_exit'}: + raise secondary + def terminate(self): + events.append('abort') + + conn = Connection() + + @contextlib.asynccontextmanager + async def connect(): + try: + yield conn + finally: + events.append('exit') + if phase == 'migration_exit_stall': + await asyncio.Future() + if phase in {'migration_exit', 'migration_ordinary_exit'}: + raise secondary + + async def fail(*args): + events.append('work') + raise original + + async def main(): + pg_migrate._migrate = fail + pg._RELEASE_TIMEOUT_S = 0.05 + started = time.monotonic() + try: + if phase == 'close': + await pg.close_owned(conn, end=time.monotonic() + 1) + else: + await pg_migrate.migrate(Path(path), connect=connect, schema='test') + except BaseException as error: + assert error is original, (type(error).__name__, events) + assert time.monotonic() - started < 0.3, events + assert 'close' in events, events + if phase != 'close': + assert 'exit' in events, events + assert events.index('close') < events.index('exit'), events + else: + assert 'abort' in events, events + await asyncio.sleep(0) + print('owner-received-original-after-cleanup') + return + raise AssertionError('native failure was swallowed') + + asyncio.run(main()) + """ + ) + result = subprocess.run( + [sys.executable, "-c", program, kind, phase, str(tmp_path)], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "owner-received-original-after-cleanup" in result.stdout + + +@pytest.mark.parametrize("connection_context", ["cell", "router"]) +@pytest.mark.parametrize("kind", ["SystemExit", "KeyboardInterrupt", "ValueError"]) +@pytest.mark.parametrize("exit_behavior", ["fatal", "stalled"]) +def test_actual_connection_context_keeps_first_failure_and_cleanup_end( + connection_context: str, kind: str, exit_behavior: str, tmp_path +) -> None: + program = textwrap.dedent( + """ + import asyncio + import sys + import time + from pathlib import Path + import asyncpg + from server.core import pg + from server.core.schema import pg_migrate + from server.router import directory + + context, kind, exit_behavior, path = sys.argv[1:] + original = {'SystemExit': SystemExit, 'KeyboardInterrupt': KeyboardInterrupt, + 'ValueError': ValueError}[kind]('original') + secondary = SystemExit('context failed') + events = [] + + class Connection: + async def close(self, **kwargs): + events.append('close') + if events.count('close') > 1: + if exit_behavior == 'stalled': + await asyncio.Future() + raise secondary + def terminate(self): + events.append('abort') + + conn = Connection() + async def raw(*args, **kwargs): + return conn + async def checked(*args): + pass + async def fail(*args): + raise original + + async def main(): + pg.settings.pg_dsn = "postgresql://unused/test" + pg._check_deployment = checked + pg.init_connection = checked + asyncpg.connect = raw + directory._require_dsn = lambda: 'unused-test-dsn' + pg._RELEASE_TIMEOUT_S = 0.05 + pg_migrate._migrate = fail + connect = None if context == 'cell' else directory._dedicated + started = time.monotonic() + try: + await pg_migrate.migrate(Path(path), connect=connect, schema='test') + except BaseException as error: + assert error is original, (type(error).__name__, events) + assert time.monotonic() - started < 0.3, events + assert events.count('close') == 2, events + if exit_behavior == 'stalled': + assert 'abort' in events, events + await asyncio.sleep(0) + print('first-failure-and-original-end-preserved') + return + raise AssertionError('first migration failure was swallowed') + asyncio.run(main()) + """ + ) + result = subprocess.run( + [sys.executable, "-c", program, connection_context, kind, exit_behavior, str(tmp_path)], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "first-failure-and-original-end-preserved" in result.stdout From d71bfc96a0ffa8c426057b1d531cacaea88e1e3f Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 00:31:05 +0000 Subject: [PATCH 037/114] Preserve native migration failures through owned cleanup Capture native failures inside owned work and public-close child tasks so their supervisors can clean up before rethrowing. Keep the first migration failure through context exit within its original cleanup deadline. Signed-off-by: zlareb1 --- server/core/pg.py | 14 ++- server/core/schema/pg_migrate.py | 75 +++++++++---- tests/unit/test_pg_close_owned.py | 75 +++++++++++++ tests/unit/test_pg_owned_fatal.py | 172 ++++++++++++++++++++++++++++++ 4 files changed, 315 insertions(+), 21 deletions(-) create mode 100644 tests/unit/test_pg_owned_fatal.py diff --git a/server/core/pg.py b/server/core/pg.py index 5daeb33a55..1e51975dfb 100644 --- a/server/core/pg.py +++ b/server/core/pg.py @@ -392,16 +392,26 @@ async def close_owned(conn: Any, *, end: float) -> None: is a bounded fallback, not evidence that the backend has released its locks. The owner preserves an earlier operation failure when this cleanup fails. """ + + async def close() -> SystemExit | KeyboardInterrupt | None: + try: + await conn.close(timeout=max(0, end - time.monotonic())) + except (SystemExit, KeyboardInterrupt) as error: + # A native fatal in a child Task stops the loop before its owner can clean up. + return error + return None + task: asyncio.Task[Any] | None = None try: remaining = end - time.monotonic() if remaining <= 0: raise TimeoutError("Postgres owned-session cleanup deadline expired") - task = asyncio.create_task(conn.close(timeout=remaining)) + task = asyncio.create_task(close()) done, _ = await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) if not done: raise TimeoutError("Postgres owned-session cleanup deadline expired") - task.result() + if isinstance(result := task.result(), (SystemExit, KeyboardInterrupt)): + raise result except BaseException: _terminate(conn) if task is not None: diff --git a/server/core/schema/pg_migrate.py b/server/core/schema/pg_migrate.py index 371f61a449..edc5cdb927 100644 --- a/server/core/schema/pg_migrate.py +++ b/server/core/schema/pg_migrate.py @@ -21,7 +21,7 @@ import logging import time from asyncio import sleep -from collections.abc import Callable +from collections.abc import Awaitable, Callable from contextlib import AbstractAsyncContextManager from pathlib import Path from typing import Any @@ -110,24 +110,61 @@ async def migrate( holds every read and write of the table behind it; the file is retried. For schema `loom` it marks the session and runs `cell_triggers.ensure` before the lock goes.""" files, indexes = _validated(mig_dir, schema) - async with (connect or pg.dedicated)() as conn: - task = asyncio.create_task(_migrate(conn, files, indexes, schema, lock_key)) - try: - return await asyncio.shield(task) - except BaseException: - end = time.monotonic() + pg._RELEASE_TIMEOUT_S - try: - await pg.close_owned(conn, end=end) - except BaseException: # noqa: BLE001 — preserve the migration failure/cancellation. - pass - if not task.done(): - task.cancel() - task.add_done_callback(pg._retrieve_task) - try: - await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) - except BaseException: # noqa: BLE001 — preserve the original failure on repeated cancel. - pg._terminate(conn) - raise + cm = (connect or pg.dedicated)() + conn = await cm.__aenter__() + task = asyncio.create_task(_owned_result(_migrate(conn, files, indexes, schema, lock_key))) + try: + result = await asyncio.shield(task) + if isinstance(result, (SystemExit, KeyboardInterrupt)): + raise result + except BaseException as error: + end = time.monotonic() + pg._RELEASE_TIMEOUT_S + await _failed_migration(cm, conn, task, error, end) + raise + await cm.__aexit__(None, None, None) + return result + + +async def _owned_result[T](work: Awaitable[T]) -> T | SystemExit | KeyboardInterrupt: + try: + return await work + except (SystemExit, KeyboardInterrupt) as error: + # Transport native failures through child Tasks without stopping their supervisor. + return error + + +async def _failed_migration( + cm: AbstractAsyncContextManager[Any], + conn: Any, + task: asyncio.Task[Any], + error: BaseException, + end: float, +) -> None: + try: + await pg.close_owned(conn, end=end) + except BaseException: # noqa: BLE001 — preserve the migration failure/cancellation. + pass + if not task.done(): + task.cancel() + task.add_done_callback(pg._retrieve_task) + try: + await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) + except BaseException: # noqa: BLE001 — preserve the first failure on repeated cancellation. + pg._terminate(conn) + release = asyncio.create_task( + _owned_result(cm.__aexit__(type(error), error, error.__traceback__)) + ) + try: + done, _ = await asyncio.wait({release}, timeout=max(0, end - time.monotonic())) + if not done: + raise TimeoutError("migration context cleanup deadline expired") + release.result() + except BaseException: # noqa: BLE001 — a context exit cannot replace the migration failure. + pg._terminate(conn) + finally: + if not release.done(): + release.cancel() + release.add_done_callback(pg._retrieve_task) def _validated( diff --git a/tests/unit/test_pg_close_owned.py b/tests/unit/test_pg_close_owned.py index fbbe778a5d..1ba7ab1f6a 100644 --- a/tests/unit/test_pg_close_owned.py +++ b/tests/unit/test_pg_close_owned.py @@ -142,3 +142,78 @@ async def fail(*_: Any) -> None: with pytest.raises(ValueError, match="migration failed"): await pg_migrate.migrate(tmp_path, connect=connect, schema="test") assert conn.aborted + + +async def test_successful_migration_still_reports_context_exit_failure( + monkeypatch, tmp_path +) -> None: + import contextlib + + from server.core.schema import pg_migrate + + original = ValueError("context exit failed") + + @contextlib.asynccontextmanager + async def connect(): + yield Connection() + raise original + + async def applied(*_: Any) -> list[str]: + return ["001.sql"] + + monkeypatch.setattr(pg_migrate, "_migrate", applied) + with pytest.raises(ValueError) as error: + await pg_migrate.migrate(tmp_path, connect=connect, schema="test") + assert error.value is original + + +@pytest.mark.parametrize("cancel_work", [False, True]) +async def test_second_cancellation_during_migration_context_exit_keeps_first_failure( + monkeypatch, tmp_path, cancel_work +) -> None: + import contextlib + + from server.core.schema import pg_migrate + + conn = Connection() + conn.finished.set() + started, exiting = asyncio.Event(), asyncio.Event() + observed: list[BaseException] = [] + original = ValueError("migration failed") + + @contextlib.asynccontextmanager + async def connect(): + try: + yield conn + except BaseException as error: + observed.append(error) + exiting.set() + await asyncio.Future() + raise + + async def work(*_: Any) -> list[str]: + started.set() + if cancel_work: + await asyncio.Future() + raise original + + monkeypatch.setattr(pg_migrate, "_migrate", work) + monkeypatch.setattr(pg, "_RELEASE_TIMEOUT_S", 0.1) + task = asyncio.create_task(pg_migrate.migrate(tmp_path, connect=connect, schema="test")) + await asyncio.wait_for(started.wait(), 1) + before = time.monotonic() + if cancel_work: + task.cancel("first cancellation") + await asyncio.wait_for(exiting.wait(), 1) + task.cancel("second cancellation") + with pytest.raises(BaseException) as result: + await asyncio.wait_for(task, 0.3) + assert time.monotonic() - before < 0.3 + assert result.value is observed[0] + assert ( + isinstance(result.value, asyncio.CancelledError) + if cancel_work + else result.value is original + ) + assert len(conn.timeouts) == 1 + assert conn.aborted diff --git a/tests/unit/test_pg_owned_fatal.py b/tests/unit/test_pg_owned_fatal.py new file mode 100644 index 0000000000..683dbca782 --- /dev/null +++ b/tests/unit/test_pg_owned_fatal.py @@ -0,0 +1,172 @@ +"""Owned child tasks preserve native failures until their supervisor cleans up.""" + +from __future__ import annotations + +import subprocess +import sys +import textwrap + +import pytest + + +@pytest.mark.parametrize("kind", ["SystemExit", "KeyboardInterrupt"]) +@pytest.mark.parametrize( + "phase", + ["close", "migration", "migration_exit", "migration_ordinary_exit", "migration_exit_stall"], +) +def test_native_failure_reaches_the_owner_after_cleanup(kind: str, phase: str, tmp_path) -> None: + program = textwrap.dedent( + """ + import asyncio + import contextlib + import sys + import time + from pathlib import Path + from server.core import pg + from server.core.schema import pg_migrate + + kind, phase, path = sys.argv[1:] + fatal_type = SystemExit if kind == 'SystemExit' else KeyboardInterrupt + original = ValueError('original') if 'ordinary' in phase else fatal_type('original') + secondary = fatal_type('cleanup') + events = [] + + class Connection: + async def close(self, **kwargs): + events.append('close') + if phase == 'close': + raise original + if phase in {'migration_exit', 'migration_ordinary_exit'}: + raise secondary + def terminate(self): + events.append('abort') + + conn = Connection() + + @contextlib.asynccontextmanager + async def connect(): + try: + yield conn + finally: + events.append('exit') + if phase == 'migration_exit_stall': + await asyncio.Future() + if phase in {'migration_exit', 'migration_ordinary_exit'}: + raise secondary + + async def fail(*args): + events.append('work') + raise original + + async def main(): + pg_migrate._migrate = fail + pg._RELEASE_TIMEOUT_S = 0.05 + started = time.monotonic() + try: + if phase == 'close': + await pg.close_owned(conn, end=time.monotonic() + 1) + else: + await pg_migrate.migrate(Path(path), connect=connect, schema='test') + except BaseException as error: + assert error is original, (type(error).__name__, events) + assert time.monotonic() - started < 0.3, events + assert 'close' in events, events + if phase != 'close': + assert 'exit' in events, events + assert events.index('close') < events.index('exit'), events + else: + assert 'abort' in events, events + await asyncio.sleep(0) + print('owner-received-original-after-cleanup') + return + raise AssertionError('native failure was swallowed') + + asyncio.run(main()) + """ + ) + result = subprocess.run( + [sys.executable, "-c", program, kind, phase, str(tmp_path)], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "owner-received-original-after-cleanup" in result.stdout + + +@pytest.mark.parametrize("connection_context", ["cell", "router"]) +@pytest.mark.parametrize("kind", ["SystemExit", "KeyboardInterrupt", "ValueError"]) +@pytest.mark.parametrize("exit_behavior", ["fatal", "stalled"]) +def test_actual_connection_context_keeps_first_failure_and_cleanup_end( + connection_context: str, kind: str, exit_behavior: str, tmp_path +) -> None: + program = textwrap.dedent( + """ + import asyncio + import sys + import time + from pathlib import Path + import asyncpg + from server.core import pg + from server.core.schema import pg_migrate + from server.router import directory + + context, kind, exit_behavior, path = sys.argv[1:] + original = {'SystemExit': SystemExit, 'KeyboardInterrupt': KeyboardInterrupt, + 'ValueError': ValueError}[kind]('original') + secondary = SystemExit('context failed') + events = [] + + class Connection: + async def close(self, **kwargs): + events.append('close') + if events.count('close') > 1: + if exit_behavior == 'stalled': + await asyncio.Future() + raise secondary + def terminate(self): + events.append('abort') + + conn = Connection() + async def raw(*args, **kwargs): + return conn + async def checked(*args): + pass + async def fail(*args): + raise original + + async def main(): + pg.settings.pg_dsn = "postgresql://unused/test" + pg._check_deployment = checked + pg.init_connection = checked + asyncpg.connect = raw + directory._require_dsn = lambda: 'unused-test-dsn' + pg._RELEASE_TIMEOUT_S = 0.05 + pg_migrate._migrate = fail + connect = None if context == 'cell' else directory._dedicated + started = time.monotonic() + try: + await pg_migrate.migrate(Path(path), connect=connect, schema='test') + except BaseException as error: + assert error is original, (type(error).__name__, events) + assert time.monotonic() - started < 0.3, events + assert events.count('close') == 2, events + if exit_behavior == 'stalled': + assert 'abort' in events, events + await asyncio.sleep(0) + print('first-failure-and-original-end-preserved') + return + raise AssertionError('first migration failure was swallowed') + asyncio.run(main()) + """ + ) + result = subprocess.run( + [sys.executable, "-c", program, connection_context, kind, exit_behavior, str(tmp_path)], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "first-failure-and-original-end-preserved" in result.stdout From 92d6d9be91565e9c68222ce89889462e6540786a Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 00:33:56 +0000 Subject: [PATCH 038/114] Use common fatal transport for directory session close Signed-off-by: zlareb1 --- server/router/directory.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/server/router/directory.py b/server/router/directory.py index 42c48ff1fa..d4f3f62583 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -916,21 +916,6 @@ async def _control_result[T](work: Callable[[], Awaitable[T]]) -> T | _ControlFa return error.fatal -@dataclass(frozen=True) -class _ControlConnection: - conn: Any - - async def close(self, **kwargs: Any) -> None: - try: - await self.conn.close(**kwargs) - except (SystemExit, KeyboardInterrupt) as error: - # pg.close_owned creates its own public-close child. - raise _ControlInterrupted(_ControlFatal(error)) from None - - def terminate(self) -> None: - pg._terminate(self.conn) - - class _ControlOwner: """Own only this operator attempt's proxy, through close and pool release.""" @@ -963,9 +948,7 @@ async def close(self) -> None: if self.conn is None: return if self.close_task is None: - self.close_task = self.start( - lambda: pg.close_owned(_ControlConnection(self.conn), end=self.end) - ) + self.close_task = self.start(lambda: pg.close_owned(self.conn, end=self.end)) try: await self.wait(self.close_task, self.end) except BaseException: From a13eb96b9d6a2be94268ed53471c620cc18896c5 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 00:43:33 +0000 Subject: [PATCH 039/114] Construct migration cleanup work inside its error capture A custom context can raise before returning its exit awaitable. Defer that invocation to the owned child so it cannot replace the first migration error. Signed-off-by: zlareb1 --- server/core/schema/pg_migrate.py | 10 +++--- tests/unit/test_pg_owned_fatal.py | 55 +++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/server/core/schema/pg_migrate.py b/server/core/schema/pg_migrate.py index edc5cdb927..1722da5a9b 100644 --- a/server/core/schema/pg_migrate.py +++ b/server/core/schema/pg_migrate.py @@ -112,7 +112,9 @@ async def migrate( files, indexes = _validated(mig_dir, schema) cm = (connect or pg.dedicated)() conn = await cm.__aenter__() - task = asyncio.create_task(_owned_result(_migrate(conn, files, indexes, schema, lock_key))) + task = asyncio.create_task( + _owned_result(lambda: _migrate(conn, files, indexes, schema, lock_key)) + ) try: result = await asyncio.shield(task) if isinstance(result, (SystemExit, KeyboardInterrupt)): @@ -125,9 +127,9 @@ async def migrate( return result -async def _owned_result[T](work: Awaitable[T]) -> T | SystemExit | KeyboardInterrupt: +async def _owned_result[T](work: Callable[[], Awaitable[T]]) -> T | SystemExit | KeyboardInterrupt: try: - return await work + return await work() except (SystemExit, KeyboardInterrupt) as error: # Transport native failures through child Tasks without stopping their supervisor. return error @@ -152,7 +154,7 @@ async def _failed_migration( except BaseException: # noqa: BLE001 — preserve the first failure on repeated cancellation. pg._terminate(conn) release = asyncio.create_task( - _owned_result(cm.__aexit__(type(error), error, error.__traceback__)) + _owned_result(lambda: cm.__aexit__(type(error), error, error.__traceback__)) ) try: done, _ = await asyncio.wait({release}, timeout=max(0, end - time.monotonic())) diff --git a/tests/unit/test_pg_owned_fatal.py b/tests/unit/test_pg_owned_fatal.py index 683dbca782..dcf77f560e 100644 --- a/tests/unit/test_pg_owned_fatal.py +++ b/tests/unit/test_pg_owned_fatal.py @@ -170,3 +170,58 @@ async def main(): ) assert result.returncode == 0, result.stdout + result.stderr assert "first-failure-and-original-end-preserved" in result.stdout + + +@pytest.mark.parametrize("exit_error", ["RuntimeError", "SystemExit", "KeyboardInterrupt"]) +def test_synchronous_context_exit_failure_keeps_original_migration_error( + exit_error: str, tmp_path +) -> None: + program = textwrap.dedent( + """ + import asyncio + import sys + from pathlib import Path + from server.core.schema import pg_migrate + + kind, path = sys.argv[1:] + original = ValueError('original migration error') + secondary = {'RuntimeError': RuntimeError, 'SystemExit': SystemExit, + 'KeyboardInterrupt': KeyboardInterrupt}[kind]('context factory failed') + events = [] + class Connection: + async def close(self, **kwargs): + events.append('close') + def terminate(self): + events.append('abort') + class Context: + async def __aenter__(self): + return Connection() + def __aexit__(self, kind, value, traceback): + assert value is original + events.append('exit') + raise secondary + async def fail(*args): + raise original + async def main(): + pg_migrate._migrate = fail + try: + await pg_migrate.migrate(Path(path), connect=Context, schema='test') + except BaseException as error: + assert error is original, (type(error).__name__, events) + assert events[:2] == ['close', 'exit'], events + await asyncio.sleep(0) + print('synchronous-context-failure-contained') + return + raise AssertionError('migration failure was swallowed') + asyncio.run(main()) + """ + ) + result = subprocess.run( + [sys.executable, "-c", program, exit_error, str(tmp_path)], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "synchronous-context-failure-contained" in result.stdout From 0410333de02afbf47bc4e182c5ca6dce7c88fcfe Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 00:45:20 +0000 Subject: [PATCH 040/114] Validate serving configuration and serialize fresh delivery rows Build required timestamps through the first-version helper and check retained launch evidence against the owned serving process and live fingerprints. Normalize accepted URLs and retain failed HTTP status and elapsed samples. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 242 +++++++++- tests/unit/test_bench_cell_trigger_gate.py | 491 ++++++++++++++++++++- 2 files changed, 712 insertions(+), 21 deletions(-) diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 5bd3bfd3db..7321583095 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -15,6 +15,8 @@ import math import os import re +import socket +import stat import statistics import subprocess import sys @@ -241,6 +243,145 @@ def _source(fixture: Fixture) -> dict[str, Any]: return {"head": head, "tree": tree, "clean": True} +_PUBLIC_WORKLOAD = frozenset( + { + "storage_authority", + "background_workers", + "embedding_provider", + "embedding_model", + "embedding_dim", + "embedding_send_dimensions", + "embedding_worker_enabled", + "embedding_cache_shared", + "embedding_cache_shared_max_chars", + "embedding_cache_size", + "embedding_cache_ttl_s", + "llm_provider", + "llm_model", + } +) + + +def _workload(config: Any) -> dict[str, Any]: + return { + key: value + for key, value in _configuration(config).items() + if (key.startswith("pg_") and key != "pg_dsn") + or key.startswith("memory_conflict") + or key in _PUBLIC_WORKLOAD + } + + +def _receipt(path: str) -> tuple[dict[str, Any], str]: + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, "rb") as source: + metadata = os.fstat(source.fileno()) + _require( + stat.S_ISREG(metadata.st_mode) and metadata.st_uid == os.geteuid(), + "owned regular receipt", + ) + raw = source.read(65_537) + _require(len(raw) <= 65_536, "receipt size") + value = json.loads(raw) + _require(isinstance(value, dict), "receipt object") + return value, hashlib.sha256(raw).hexdigest() + + +def _owned_server(server: dict[str, Any]) -> None: + from server.core.provider_process import _start_ticks + + _require( + set(server) + == {"pid", "start_ticks", "boot_id", "uid", "cwd", "health_instance", "instance_id"}, + "launch server fields", + ) + pid = server["pid"] + _require( + type(pid) is int + and pid > 1 + and type(server["uid"]) is int + and server["uid"] == os.geteuid(), + "launch process ownership", + ) + _require( + isinstance(server["start_ticks"], str) and server["start_ticks"].isdigit(), + "launch process birth", + ) + root = Path(__file__).resolve().parents[1] + _require( + server["cwd"] == str(root) and server["health_instance"] == f"{socket.gethostname()}:{pid}", + "launch process source/hostname/PID", + ) + proc = Path("/proc") / str(pid) + _require(proc.stat().st_uid == os.geteuid(), "actual process ownership") + birth = _start_ticks(pid) + _require(birth == server["start_ticks"], "actual process birth") + boot = Path("/proc/sys/kernel/random/boot_id").read_text().strip() + _require( + boot == server["boot_id"] and os.readlink(proc / "cwd") == str(root), + "actual process boot/source", + ) + _require(_start_ticks(pid) == birth, "stable process birth") + + +def _launch(args: argparse.Namespace, fixture: Fixture, config: Any) -> dict[str, Any]: + from server.core import deployment_config + + receipt, digest = _receipt(args.launch_manifest) + _require(digest == fixture.launch_manifest_sha256, "launch receipt hash") + _require( + set(receipt) == {"schema", "server", "source", "configuration_artifact"} + and type(receipt["schema"]) is int + and receipt["schema"] == 1, + "launch receipt schema", + ) + server, source, artifact = ( + receipt[key] for key in ("server", "source", "configuration_artifact") + ) + _require( + isinstance(server, dict) and isinstance(source, dict) and isinstance(artifact, dict), + "launch receipt objects", + ) + _require( + source == {"head": fixture.build_commit, "tree": fixture.source_tree}, + "launch source identity", + ) + _require(server.get("instance_id") == fixture.instance_id, "launch server UUID") + _require( + set(artifact) == {"path", "sha256"} + and isinstance(artifact["path"], str) + and Path(artifact["path"]).is_absolute(), + "configuration artifact path", + ) + configuration, artifact_digest = _receipt(artifact["path"]) + _require(artifact_digest == artifact["sha256"], "configuration artifact hash") + _require( + set(configuration) == {"schema", "configuration_sha256", "workload", "fingerprints"} + and type(configuration["schema"]) is int + and configuration["schema"] == 1, + "configuration artifact schema", + ) + _require( + configuration["configuration_sha256"] + == fixture.configuration_sha256 + == _digest(_configuration(config)), + "launch effective configuration", + ) + _require( + _digest(configuration["workload"]) == _digest(_workload(config)), "launch workload settings" + ) + fingerprints = deployment_config.fingerprints(config) + _require(configuration["fingerprints"] == fingerprints, "launch configuration fingerprints") + _owned_server(server) + return { + "health_instance": server["health_instance"], + "fingerprints": fingerprints, + "launch_sha256": digest, + "configuration_artifact_sha256": artifact_digest, + "birth_sha256": _digest(server), + } + + class Artifact: def __init__(self, path: str): fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) @@ -302,10 +443,14 @@ async def close(self) -> None: await self.client.close() -async def _health(args: argparse.Namespace, fixture: Fixture) -> None: +async def _health( + args: argparse.Namespace, fixture: Fixture, launch: Mapping[str, Any] | None = None +) -> None: + from server.settings import settings + async with httpx.AsyncClient(timeout=IO_S, follow_redirects=False, trust_env=False) as client: - health = await client.get(f"{args.base_url}/v1/health") - deployment = await client.get(f"{args.base_url}/v1/health/deployment") + health = await client.get(f"{args.base_url.rstrip('/')}/v1/health") + deployment = await client.get(f"{args.base_url.rstrip('/')}/v1/health/deployment") _require(health.status_code == deployment.status_code == 200, "health HTTP status") h, d = health.json(), deployment.json() _require( @@ -328,6 +473,18 @@ async def _health(args: argparse.Namespace, fixture: Fixture) -> None: "serving provider/schema", ) _require(h.get("storage_authority") == "postgres", "serving authority") + for field in ("llm_provider", "llm_model", "embedding_model"): + _require(h.get(field) == getattr(settings, field), "serving " + field) + _require( + h.get("llm_provider") == "fake" + and type(h.get("embedding_dim_configured")) is int + and h["embedding_dim_configured"] == settings.embedding_dim + and type(h.get("embedding_dim_probed")) is int + and h["embedding_dim_probed"] == settings.embedding_dim, + "serving embedding dimensions/fake provider", + ) + launch = launch if launch is not None else _launch(args, fixture, settings) + _require(h.get("instance") == launch["health_instance"], "serving owned process") def _object(value: Any) -> dict[str, Any]: @@ -345,6 +502,7 @@ async def _proof( fixture.validate(args, settings) source = _source(fixture) + launch = _launch(args, fixture, settings) actual = await conn.fetchrow( "SELECT current_database() AS db, inet_server_port() AS port, " "pg_is_in_recovery() AS recovery" @@ -391,9 +549,15 @@ async def _proof( len(live) == 1 and live[0]["instance_id"] == fixture.instance_id and live[0]["build_commit"] == fixture.build_commit + and live[0].get("serving") == 1 + and live[0].get("host") == socket.gethostname() and not live[0]["workers_lease"], "exclusive serving source", ) + _require( + all(live[0].get(key) == value for key, value in launch["fingerprints"].items()), + "serving configuration fingerprints", + ) beats = await ch.query_all( "SELECT instance_id, cell, tenancy_mode, fingerprint FROM " "loom.cell_heartbeats WHERE instance_id = {id:String}", @@ -414,7 +578,7 @@ async def _proof( "serving fingerprint", ) await _debt(conn, 0) - await _health(args, fixture) + await _health(args, fixture, launch) if native: await cell.refresh() _require(not cell.cli_gate_reason(), "native fresh cell acceptance") @@ -425,6 +589,7 @@ async def _proof( "fingerprint_sha256": _digest(current), "configuration_sha256": fixture.configuration_sha256, "source": source, + "launch": launch, } @@ -610,9 +775,10 @@ async def _memory_set( key = f"{prefix}-{i}" value = _payload(key) started = time.perf_counter_ns() + response = None try: response = await client.post( - f"{args.base_url}/v1/memory.set", + f"{args.base_url.rstrip('/')}/v1/memory.set", json={ "org": args.org, "namespace": args.namespace, @@ -658,7 +824,29 @@ async def _memory_set( rows.append(row) emit("request", arm=arm, index=i, status=200, **asdict(row)) except BaseException as error: - emit("request_error", arm=arm, index=i, key=key, error_type=type(error).__name__) + elapsed = (time.perf_counter_ns() - started) / 1_000_000 + code = "" + if response is not None: + try: + envelope = response.json() + candidate = envelope.get("error", {}).get("code", "") + if isinstance(candidate, str) and re.fullmatch( + r"[A-Z][A-Z0-9_]{0,63}", candidate + ): + code = candidate + except (ValueError, TypeError, AttributeError): + pass + emit( + "request_error", + arm=arm, + index=i, + key=key, + measured=i >= WARMUP, + elapsed_ms=elapsed, + status=response.status_code if response is not None else None, + code=code, + error_type=type(error).__name__, + ) raise return rows @@ -676,8 +864,10 @@ async def _build_block(args: argparse.Namespace, batch: int) -> Block: from server.memory.store import pg_memory, pg_publish from server.memory.store.rows import ( MemoryRow, + build_first_version_row, build_memory_version_row, next_version, + written_now, ) _planned_keys(batch) @@ -690,18 +880,21 @@ async def _build_block(args: argparse.Namespace, batch: int) -> Block: stored = json.dumps(value, sort_keys=True, separators=(",", ":")) row = build_memory_version_row( MemoryRow.from_snapshot( - { - "org": args.org, - "namespace": args.namespace, - "agent": "bench", - "memory_key": key, - "memory_type": "semantic", - "value": stored, - "version": next_version(), - "content_sha256": sha256_value(value), - "actor": SUBJECT, - "source_class": "asserted", - } + build_first_version_row( + { + "org": args.org, + "namespace": args.namespace, + "agent": "bench", + "memory_key": key, + "memory_type": "semantic", + "value": stored, + "version": next_version(), + "content_sha256": sha256_value(value), + "actor": SUBJECT, + "source_class": "asserted", + }, + now=written_now(), + ) ) ) row = pg_memory.decode_row(pg_memory.encode_row(row)) @@ -1166,7 +1359,16 @@ def main() -> int: from server.wiring import wire parser = argparse.ArgumentParser(description=__doc__) - for name in ("base-url", "pg-dsn", "token", "org", "namespace", "fixture-manifest", "output"): + for name in ( + "base-url", + "pg-dsn", + "token", + "org", + "namespace", + "fixture-manifest", + "launch-manifest", + "output", + ): parser.add_argument("--" + name, required=True) args = parser.parse_args() artifact = None @@ -1176,11 +1378,13 @@ def main() -> int: fixture = Fixture.read(args.fixture_manifest) total = fixture.validate(args, settings) source = _source(fixture) + launch = _launch(args, fixture, settings) artifact = Artifact(args.output) artifact.emit( "configuration", manifest=asdict(fixture), source=source, + launch=launch, manifest_sha256=_digest(asdict(fixture)), calls=CALLS, warmup=WARMUP, diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py index a27f9560f3..31986733a8 100644 --- a/tests/unit/test_bench_cell_trigger_gate.py +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -6,6 +6,7 @@ import asyncio import dataclasses import json +import socket import uuid from contextlib import asynccontextmanager from types import SimpleNamespace @@ -35,6 +36,7 @@ def args(**overrides): "token": "secret-token", "org": "org", "namespace": "ns", + "launch_manifest": "unused", **overrides, } ) @@ -51,11 +53,27 @@ def fixture_config(): background_workers="off", storage_authority="postgres", embedding_provider="fake", + embedding_model="fake-ngram-16", + embedding_dim=16, + embedding_base_url="", + embedding_send_dimensions=False, llm_provider="fake", + llm_model="fake", + jwt_secret="test-only-root", + deployment=False, + instance_heartbeat_seconds=10, build_commit="a" * 40, pg_delivery_batch_rows=500, ) - config.model_dump = lambda **_: {k: v for k, v in vars(config).items() if k != "model_dump"} + + def dump(**kwargs): + return { + k: v + for k, v in vars(config).items() + if k != "model_dump" and ("include" not in kwargs or k in kwargs["include"]) + } + + config.model_dump = dump manifest = gate.Fixture( base_url=args().base_url, pg_host="127.0.0.1", @@ -609,7 +627,16 @@ async def live(*_args, **_kw): return ( [] if fault == "competing_worker" - else [{"instance_id": "instance", "build_commit": "a" * 40, "workers_lease": ""}] + else [ + { + "instance_id": "instance", + "build_commit": "a" * 40, + "workers_lease": "", + "serving": 1, + "host": socket.gethostname(), + **fingerprints, + } + ] ) async def health(*_args): @@ -623,6 +650,10 @@ async def health(*_args): monkeypatch.setattr(gate, "_close_pg", close) monkeypatch.setattr(gate, "_health", health) monkeypatch.setattr(gate, "_source", lambda _: {"clean": True}) + from server.core import deployment_config + + fingerprints = deployment_config.fingerprints(config) + monkeypatch.setattr(gate, "_launch", lambda *_: {"fingerprints": fingerprints}) monkeypatch.setattr(instances, "read_live", live) monkeypatch.setattr(cell, "mine", lambda: {}) monkeypatch.setattr(cell, "matches", lambda mine, value: value is not None) @@ -744,6 +775,7 @@ def test_final_verdict_follows_native_cleanup_and_artifact_close( monkeypatch.setattr(server.wiring, "wire", lambda: order.append("wire")) monkeypatch.setattr(gate, "_preflight", AsyncMock()) monkeypatch.setattr(gate, "_source", lambda _: {"clean": True}) + monkeypatch.setattr(gate, "_launch", lambda *_: {}) def native(*_a, **_kw): order.append("native_cleanup") @@ -969,3 +1001,458 @@ async def close(owned): with pytest.raises(TimeoutError): await gate._cleanup(Stuck(), conn, artifact) assert conn.closed + + +@pytest.mark.asyncio +async def test_review_delivery_timestamps_pass_real_clickhouse_serialization(monkeypatch): + import logging + + from clickhouse_connect.datatypes.registry import get_from_name + from clickhouse_connect.driver.insert import InsertContext + + import server.embedding + from server.core.ch import _insert_data + from server.core.polled_copy import PolledCopy + from server.embedding.fake import FakeNGramEmbedder + from server.llm import customer + + monkeypatch.setattr( + customer, + "_switch", + PolledCopy("switch", AsyncMock(return_value=frozenset()), logging.getLogger(__name__)), + ) + monkeypatch.setattr(server.embedding, "get_embedder", lambda: FakeNGramEmbedder(dim=16)) + block = await gate._build_block(args(), 1) + for col in ("created_at", "updated_at", "last_reinforced_at"): + actual_data = _insert_data(block.rows, [col], ["DateTime"]) + kind = get_from_name("DateTime") + context = InsertContext("loom.memory", [col], [kind]) + context.start_column(col) + serialized = bytearray() + kind.write_column([row[0] for row in actual_data], serialized, context) + assert len(serialized) == 4 * len(block.rows) + assert all( + row["created_at"] == row["updated_at"] == row["last_reinforced_at"] for row in block.rows + ) + + +@pytest.mark.asyncio +async def test_review_trailing_slash_is_normalized_for_actual_health_and_api_routes(monkeypatch): + from starlette.responses import JSONResponse + from starlette.routing import Route, Router + + fixture, _ = fixture_config() + paths = [] + + async def health(request): + paths.append(request.url.path) + return JSONResponse({}) + + app = Router( + routes=[ + Route("/v1/health", health), + Route("/v1/health/deployment", health), + Route("/v1/memory.set", health, methods=["POST"]), + ] + ) + transport = httpx.ASGITransport(app=app) + + async def send(self, request, **kwargs): + response = await transport.handle_async_request(request) + await response.aread() + return response + + monkeypatch.setattr(httpx.AsyncClient, "send", send) + with pytest.raises(gate.Invalid): + await gate._health(args(base_url=fixture.base_url + "/"), fixture) + with pytest.raises(gate.Invalid): + await gate._memory_set(args(base_url=fixture.base_url + "/")) + assert paths == ["/v1/health", "/v1/health/deployment", "/v1/memory.set"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status,code", [(302, ""), (503, "PUBLICATION_PENDING")]) +async def test_review_failed_request_keeps_raw_status_elapsed_and_classification( + monkeypatch, status, code +): + async def send(self, request, **kwargs): + return httpx.Response( + status, json={"error": {"code": code, "message": "secret-token"}}, request=request + ) + + monkeypatch.setattr(httpx.AsyncClient, "send", send) + clock = iter([10_000_000, 17_000_000]) + monkeypatch.setattr(gate.time, "perf_counter_ns", lambda: next(clock)) + events = [] + with pytest.raises(gate.Invalid): + await gate._memory_set(args(), emit=lambda kind, **record: events.append((kind, record))) + assert len(events) == 1 and events[0][0] == "request_error" + record = events[0][1] + assert record["status"] == status and record["code"] == code + assert record["elapsed_ms"] == 7 and record["measured"] is False + assert "secret-token" not in json.dumps(events) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", + [ + "", + "settings_fp", + "keys_fp", + "config_fp", + "llm_provider", + "llm_model", + "embedding_model", + "embedding_dim_configured", + "embedding_dim_probed", + "host", + "instance", + "uuid", + ], +) +async def test_review_composed_proof_matches_actual_server_fingerprints_and_health( + monkeypatch, tmp_path, fault +): + import server.settings + from server.core import cell, deployment_config + + fixture, config = fixture_config() + fixture, launch_path, _, launch, _ = launch_files(tmp_path, fixture, config) + monkeypatch.setattr(gate, "_owned_server", lambda _: None) + mine = cell.mine() + mine["deployment_id"] = fixture.deployment_id + fingerprints = deployment_config.fingerprints(config) + + class PG(ProofPG): + async def fetchrow(self, sql, *params): + if "FROM loom.cell_self" in sql: + return {"current_fp": json.dumps(cell.as_value(mine)), "next_fp": None} + return await super().fetchrow(sql, *params) + + class CH(ProofCH): + async def query_all(self, sql, params=None): + if "FROM loom.instances" in sql: + return [ + { + "instance_id": fixture.instance_id, + "build_commit": fixture.build_commit, + "workers_lease": "", + "serving": 1, + "host": "other" if fault == "host" else socket.gethostname(), + **fingerprints, + **({fault: "wrong"} if fault.endswith("_fp") else {}), + } + ] + if "FROM loom.cell_heartbeats" in sql: + return [ + { + "cell": fixture.cell, + "tenancy_mode": "enforced", + "fingerprint": json.dumps( + {**cell.as_value(mine), "serves": True, "marked": True} + ), + } + ] + return await super().query_all(sql, params) + + async def send(self, request, **kwargs): + body = ( + {"deployment_id": fixture.deployment_id} + if request.url.path.endswith("/deployment") + else { + "build_commit": fixture.build_commit, + "instance": "other:4242" + if fault == "instance" + else launch["server"]["health_instance"], + "deployment_config": { + "instance_id": "other" if fault == "uuid" else fixture.instance_id + }, + "ch_ok": True, + "pg_ok": True, + "status": "ok", + "fake_embedding": True, + "fake_llm": True, + "pending_migrations": [], + "storage_authority": "postgres", + "llm_provider": config.llm_provider, + "llm_model": config.llm_model, + "embedding_model": config.embedding_model, + "embedding_dim_configured": config.embedding_dim, + "embedding_dim_probed": config.embedding_dim, + **( + {fault: 999 if "dim" in fault else "wrong"} + if fault and not fault.endswith("_fp") + else {} + ), + } + ) + return httpx.Response(200, json=body, request=request) + + monkeypatch.setattr(server.settings, "settings", config) + monkeypatch.setattr(gate, "_source", lambda _: {"clean": True}) + monkeypatch.setattr(cell, "mine", lambda: dict(mine)) + monkeypatch.setattr(httpx.AsyncClient, "send", send) + if fault: + with pytest.raises(gate.Invalid): + await gate._proof(PG(), CH(), args(launch_manifest=str(launch_path)), fixture) + else: + proof = await gate._proof(PG(), CH(), args(launch_manifest=str(launch_path)), fixture) + assert proof["configuration_sha256"] == fixture.configuration_sha256 + + +def launch_files(tmp_path, fixture, config): + import os + import socket + + from server.core import deployment_config + + public = { + k: v + for k, v in config.model_dump().items() + if k.startswith("pg_") + and k != "pg_dsn" + or k.startswith("memory_conflict") + or k + in { + "storage_authority", + "background_workers", + "embedding_provider", + "embedding_model", + "embedding_dim", + "embedding_send_dimensions", + "embedding_worker_enabled", + "embedding_cache_shared", + "embedding_cache_shared_max_chars", + "embedding_cache_size", + "embedding_cache_ttl_s", + "llm_provider", + "llm_model", + } + } + configuration = { + "schema": 1, + "configuration_sha256": fixture.configuration_sha256, + "workload": public, + "fingerprints": deployment_config.fingerprints(config), + } + artifact_path = tmp_path / "configuration.json" + artifact_bytes = json.dumps(configuration).encode() + artifact_path.write_bytes(artifact_bytes) + source_path = str(__import__("pathlib").Path(gate.__file__).resolve().parents[1]) + launch = { + "schema": 1, + "server": { + "pid": 4242, + "start_ticks": "123", + "boot_id": "fake-boot-id", + "uid": os.geteuid(), + "cwd": source_path, + "health_instance": f"{socket.gethostname()}:4242", + "instance_id": fixture.instance_id, + }, + "source": {"head": fixture.build_commit, "tree": fixture.source_tree}, + "configuration_artifact": { + "path": str(artifact_path), + "sha256": gate.hashlib.sha256(artifact_bytes).hexdigest(), + }, + } + launch_path = tmp_path / "launch.json" + launch_bytes = json.dumps(launch).encode() + launch_path.write_bytes(launch_bytes) + fixture = dataclasses.replace( + fixture, launch_manifest_sha256=gate.hashlib.sha256(launch_bytes).hexdigest() + ) + return fixture, launch_path, artifact_path, launch, configuration + + +@pytest.mark.parametrize( + "fault", + [ + "", + "launch_hash", + "artifact_hash", + "config", + "workload", + "workload_type", + "fingerprints", + "source", + "instance", + "symlink", + ], +) +def test_launch_receipt_binds_named_owned_process_settings_and_nonsecret_artifact( + monkeypatch, tmp_path, fault +): + fixture, config = fixture_config() + fixture, path, artifact_path, launch, configuration = launch_files(tmp_path, fixture, config) + checked = [] + monkeypatch.setattr(gate, "_owned_server", lambda server: checked.append(server), raising=False) + if fault in {"config", "workload", "workload_type", "fingerprints"}: + if fault == "config": + configuration["configuration_sha256"] = "0" * 64 + if fault == "workload": + configuration["workload"]["pg_delivery_batch_rows"] = 501 + if fault == "workload_type": + configuration["workload"]["pg_delivery_batch_rows"] = 500.0 + if fault == "fingerprints": + configuration["fingerprints"]["keys_fp"] = "wrong" + blob = json.dumps(configuration).encode() + artifact_path.write_bytes(blob) + launch["configuration_artifact"]["sha256"] = gate.hashlib.sha256(blob).hexdigest() + if fault == "artifact_hash": + launch["configuration_artifact"]["sha256"] = "0" * 64 + if fault == "source": + launch["source"]["tree"] = "0" * 40 + if fault == "instance": + launch["server"]["instance_id"] = "other" + blob = json.dumps(launch).encode() + path.write_bytes(blob) + fixture = dataclasses.replace( + fixture, + launch_manifest_sha256="0" * 64 + if fault == "launch_hash" + else gate.hashlib.sha256(blob).hexdigest(), + ) + if fault == "symlink": + link = tmp_path / "link.json" + link.symlink_to(path) + path = link + if fault: + with pytest.raises((gate.Invalid, OSError)): + gate._launch(args(launch_manifest=str(path)), fixture, config) + else: + result = gate._launch(args(launch_manifest=str(path)), fixture, config) + assert result["health_instance"] == launch["server"]["health_instance"] + assert result["fingerprints"] == configuration["fingerprints"] + assert checked == [launch["server"]] + + +@pytest.mark.parametrize( + "fault", + ["", "uid", "actual_uid", "pid", "start", "reuse", "boot", "cwd", "stamp_pid", "stamp_host"], +) +def test_owned_server_reads_only_exact_named_birth_metadata(monkeypatch, tmp_path, fault): + import os + from pathlib import Path + + from server.core import provider_process + + fixture, config = fixture_config() + _, _, _, launch, _ = launch_files(tmp_path, fixture, config) + server = dict(launch["server"]) + if fault == "uid": + server["uid"] += 1 + if fault == "pid": + server["pid"] = True + if fault == "stamp_pid": + server["health_instance"] = f"{socket.gethostname()}:4243" + if fault == "stamp_host": + server["health_instance"] = "other:4242" + paths, births = [], [] + real_stat = Path.stat + + def metadata(path, **kwargs): + if str(path).startswith("/proc"): + assert str(path) == "/proc/4242" + paths.append(str(path)) + return SimpleNamespace(st_uid=os.geteuid() + int(fault == "actual_uid")) + return real_stat(path, **kwargs) + + def start(pid): + assert type(pid) is int and pid == 4242 + births.append(pid) + return "wrong" if fault == "start" or fault == "reuse" and len(births) == 2 else "123" + + real_read = Path.read_text + + def read(path, **kwargs): + if str(path).startswith("/proc"): + assert str(path) == "/proc/sys/kernel/random/boot_id" + paths.append(str(path)) + return "wrong" if fault == "boot" else "fake-boot-id" + return real_read(path, **kwargs) + + real_link = os.readlink + + def link(path, **kwargs): + if str(path).startswith("/proc"): + assert str(path) == "/proc/4242/cwd" + paths.append(str(path)) + return "other" if fault == "cwd" else server["cwd"] + return real_link(path, **kwargs) + + monkeypatch.setattr(Path, "stat", metadata) + monkeypatch.setattr(Path, "read_text", read) + monkeypatch.setattr(provider_process, "_start_ticks", start) + monkeypatch.setattr(os, "readlink", link) + if fault: + with pytest.raises(gate.Invalid): + gate._owned_server(server) + else: + gate._owned_server(server) + assert births == [4242, 4242] + assert paths == ["/proc/4242", "/proc/sys/kernel/random/boot_id", "/proc/4242/cwd"] + + +@pytest.mark.asyncio +async def test_failed_network_request_retains_elapsed_with_no_invented_status(monkeypatch): + async def send(self, request, **kwargs): + raise httpx.ConnectError("secret-token", request=request) + + monkeypatch.setattr(httpx.AsyncClient, "send", send) + clock = iter([1_000_000, 3_000_000]) + monkeypatch.setattr(gate.time, "perf_counter_ns", lambda: next(clock)) + records = [] + with pytest.raises(httpx.ConnectError): + await gate._memory_set(args(), emit=lambda kind, **record: records.append(record)) + assert len(records) == 1 and records[0]["elapsed_ms"] == 2 + assert records[0]["status"] is None and records[0]["code"] == "" + assert "secret-token" not in json.dumps(records) + + +def test_nonregular_receipt_open_is_nonblocking_before_type_check(monkeypatch): + import os + + def open_receipt(path, flags): + assert flags & os.O_NONBLOCK + assert flags & os.O_NOFOLLOW + raise gate.Invalid("nonregular receipt") + + monkeypatch.setattr(gate.os, "open", open_receipt) + with pytest.raises(gate.Invalid, match="nonregular"): + gate._receipt("named-owned-receipt") + + +def test_launch_refusal_precedes_any_native_or_store_setup(monkeypatch, tmp_path, capsys): + import sys + + import server.settings + import server.wiring + from server.cli import _ch + + fixture, config = fixture_config() + fixture, launch_path, _, _, _ = launch_files(tmp_path, fixture, config) + fixture = dataclasses.replace(fixture, launch_manifest_sha256="0" * 64) + manifest = tmp_path / "fixture.json" + manifest.write_text(json.dumps(dataclasses.asdict(fixture))) + arguments = vars(args(launch_manifest=str(launch_path))) + arguments.update(fixture_manifest=str(manifest), output=str(tmp_path / "output.jsonl")) + argv = ["driver"] + for name, value in arguments.items(): + argv += ["--" + name.replace("_", "-"), str(value)] + + def forbidden(*_args, **_kwargs): + raise AssertionError("launch refusal must precede native/process setup") + + monkeypatch.setattr(sys, "argv", argv) + monkeypatch.setattr(server.settings, "settings", config) + monkeypatch.setattr(gate, "_source", lambda _: {"clean": True}) + monkeypatch.setattr(gate, "_owned_server", forbidden) + monkeypatch.setattr(gate, "_preflight", forbidden) + monkeypatch.setattr(server.wiring, "wire", forbidden) + monkeypatch.setattr(_ch, "run_connected", forbidden) + assert gate.main() == 1 + verdict = json.loads(capsys.readouterr().out) + assert verdict["verdict"] == "INVALID" and verdict["reason"] == "launch receipt hash" + assert not (tmp_path / "output.jsonl").exists() From 8c0320060dd4709ac75772fcf1f6e99776e3efc9 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 00:43:33 +0000 Subject: [PATCH 041/114] Construct migration cleanup work inside its error capture A custom context can raise before returning its exit awaitable. Defer that invocation to the owned child so it cannot replace the first migration error. Signed-off-by: zlareb1 (cherry picked from commit 613209b4ab0398601e9375d433d490a08b9dc110) --- server/core/schema/pg_migrate.py | 10 +++--- tests/unit/test_pg_owned_fatal.py | 55 +++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/server/core/schema/pg_migrate.py b/server/core/schema/pg_migrate.py index edc5cdb927..1722da5a9b 100644 --- a/server/core/schema/pg_migrate.py +++ b/server/core/schema/pg_migrate.py @@ -112,7 +112,9 @@ async def migrate( files, indexes = _validated(mig_dir, schema) cm = (connect or pg.dedicated)() conn = await cm.__aenter__() - task = asyncio.create_task(_owned_result(_migrate(conn, files, indexes, schema, lock_key))) + task = asyncio.create_task( + _owned_result(lambda: _migrate(conn, files, indexes, schema, lock_key)) + ) try: result = await asyncio.shield(task) if isinstance(result, (SystemExit, KeyboardInterrupt)): @@ -125,9 +127,9 @@ async def migrate( return result -async def _owned_result[T](work: Awaitable[T]) -> T | SystemExit | KeyboardInterrupt: +async def _owned_result[T](work: Callable[[], Awaitable[T]]) -> T | SystemExit | KeyboardInterrupt: try: - return await work + return await work() except (SystemExit, KeyboardInterrupt) as error: # Transport native failures through child Tasks without stopping their supervisor. return error @@ -152,7 +154,7 @@ async def _failed_migration( except BaseException: # noqa: BLE001 — preserve the first failure on repeated cancellation. pg._terminate(conn) release = asyncio.create_task( - _owned_result(cm.__aexit__(type(error), error, error.__traceback__)) + _owned_result(lambda: cm.__aexit__(type(error), error, error.__traceback__)) ) try: done, _ = await asyncio.wait({release}, timeout=max(0, end - time.monotonic())) diff --git a/tests/unit/test_pg_owned_fatal.py b/tests/unit/test_pg_owned_fatal.py index 683dbca782..dcf77f560e 100644 --- a/tests/unit/test_pg_owned_fatal.py +++ b/tests/unit/test_pg_owned_fatal.py @@ -170,3 +170,58 @@ async def main(): ) assert result.returncode == 0, result.stdout + result.stderr assert "first-failure-and-original-end-preserved" in result.stdout + + +@pytest.mark.parametrize("exit_error", ["RuntimeError", "SystemExit", "KeyboardInterrupt"]) +def test_synchronous_context_exit_failure_keeps_original_migration_error( + exit_error: str, tmp_path +) -> None: + program = textwrap.dedent( + """ + import asyncio + import sys + from pathlib import Path + from server.core.schema import pg_migrate + + kind, path = sys.argv[1:] + original = ValueError('original migration error') + secondary = {'RuntimeError': RuntimeError, 'SystemExit': SystemExit, + 'KeyboardInterrupt': KeyboardInterrupt}[kind]('context factory failed') + events = [] + class Connection: + async def close(self, **kwargs): + events.append('close') + def terminate(self): + events.append('abort') + class Context: + async def __aenter__(self): + return Connection() + def __aexit__(self, kind, value, traceback): + assert value is original + events.append('exit') + raise secondary + async def fail(*args): + raise original + async def main(): + pg_migrate._migrate = fail + try: + await pg_migrate.migrate(Path(path), connect=Context, schema='test') + except BaseException as error: + assert error is original, (type(error).__name__, events) + assert events[:2] == ['close', 'exit'], events + await asyncio.sleep(0) + print('synchronous-context-failure-contained') + return + raise AssertionError('migration failure was swallowed') + asyncio.run(main()) + """ + ) + result = subprocess.run( + [sys.executable, "-c", program, exit_error, str(tmp_path)], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "synchronous-context-failure-contained" in result.stdout From 8018e30c02f902dd360e5589cfe659919d387900 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 01:12:50 +0000 Subject: [PATCH 042/114] Keep attach cleanup inside its owning deadline Close the restoration session before cancelling transaction rollback, and capture context-exit failures without replacing the first operation error. Describe the required timing workload separately from the small diagnostic measurements. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 24 ++-- docs/benchmarks/results.md | 56 +++++--- server/cli/cell_attach.py | 43 ++---- server/cli/cell_attach_pg.py | 53 ++++++- tests/unit/test_bench_cell_trigger_gate.py | 131 ++++++++++++++++- tests/unit/test_cell_attach_owned_exit.py | 160 +++++++++++++++++++++ 6 files changed, 403 insertions(+), 64 deletions(-) create mode 100644 tests/unit/test_cell_attach_owned_exit.py diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 7321583095..b1874e1fd9 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -403,10 +403,10 @@ def close(self) -> None: self.file.close() -async def _close_pg(conn: Any) -> None: +async def _close_pg(conn: Any, end: float | None = None) -> None: from server.core import pg - await pg.close_owned(conn, end=asyncio.get_running_loop().time() + IO_S) + await pg.close_owned(conn, end=time.monotonic() + IO_S if end is None else end) class ReadOnlyCH: @@ -1307,9 +1307,12 @@ async def _run( artifact.emit("provisional_statistics", **summary) return summary finally: + from server.cli import cell_attach_pg as apg + prior = sys.exception() cleanup = asyncio.create_task( - _cleanup(triggers, conn, artifact), name="trigger-gate-cleanup" + apg._work_result(lambda: _cleanup(triggers, conn, artifact)), + name="trigger-gate-cleanup", ) interrupted = False while not cleanup.done(): @@ -1320,7 +1323,8 @@ async def _run( except BaseException: # noqa: BLE001 — preserve an earlier operation failure break try: - cleanup.result() + if isinstance(result := cleanup.result(), apg._Interrupted): + raise result.error except BaseException as error: try: artifact.emit("cleanup_error", error_type=type(error).__name__) @@ -1334,23 +1338,25 @@ async def _run( async def _cleanup(triggers: Triggers | None, conn: Any, artifact: Artifact) -> None: + end = time.monotonic() + RESTORE_S try: if triggers is not None: - await _restore(triggers, conn) + await _restore(triggers, conn, end) artifact.emit("triggers_restored", verified=True) finally: prior = sys.exception() try: - await _close_pg(conn) + await _close_pg(conn, end) except BaseException: if prior is None: raise artifact.emit("connection_cleanup_error", error_type=type(sys.exception()).__name__) -async def _restore(triggers: Triggers, conn: Any) -> None: - async with asyncio.timeout(RESTORE_S): - await triggers.restore(conn) +async def _restore(triggers: Triggers, conn: Any, end: float) -> None: + from server.cli import cell_attach_pg as apg + + await apg.supervise(conn, triggers.restore(conn), end=end) def main() -> int: diff --git a/docs/benchmarks/results.md b/docs/benchmarks/results.md index 4814bd7f5f..0f9bd094b1 100644 --- a/docs/benchmarks/results.md +++ b/docs/benchmarks/results.md @@ -1222,17 +1222,11 @@ at most 10% slower) holds by a wide margin; the cost lands on scripts and older writing a cell without the check. This measures the triggers on the statements, not a whole `memory.set` through a server on a cell; the end-to-end gate follows. -`bench/cell_trigger_gate.py` measures the gate end to end, locally: a first cell attached -by `loom cell attach` (two server processes, a router, Postgres 16 with a synchronous -standby and ClickHouse in Docker on a shared host under other load), 400 sequential -`memory.set` calls of a placed org through one server, and a delivery-sized 1,000-row insert -of the held org from a marked and an unmarked session, the org triggers on and dropped in -turn, three rounds each; the harness checks the triggers it asked for are enabled: - -```bash -PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py --base-url \ - --pg-dsn --token --org --namespace -``` +A small local diagnostic on an attached first cell records 400 sequential `memory.set` +calls per run and 1,000-row statement inserts from marked and unmarked sessions, three +rounds with org triggers on and off. Its fixture has two server processes, a router, +Postgres 16 with a synchronous standby and ClickHouse in Docker on a shared host under +other load. These are diagnostic figures, not the end-to-end acceptance workload: | Local, three rounds | Triggers on | Triggers off | | ----------------------------------- | ---------------------- | ---------------------- | @@ -1241,17 +1235,35 @@ PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py --base-url | 1,000-row insert, marked (median) | 11.0, 9.2, 8.5 ms | 9.7, 11.1, 10.6 ms | | 1,000-row insert, unmarked (median) | 22.1, 21.4, 21.0 ms | 10.9, 9.6, 11.0 ms | -The marked batch is what a delivery pass of a cell's own processes writes. On minus off -within each round it moved +1.3, −1.9 and −2.1 ms (+13.4%, −17.1% and −19.8%); the means are -9.6 ms on and 10.5 ms off, −0.9 ms (−8.6%). The delivery-pass gate (at most 10% slower) is -met on the mean, but one round of three was 13% slower and three rounds give no paired bound -(methodology's paired bootstrap is for per-item labels), so this run does not establish the -gate on its own; the statement-level figures above do, at under a microsecond a row against -about 10 µs a row of insert. The `memory.set` figures move by more between rounds of one -configuration than any trigger cost: on this loaded host the p99 differences within a round -(+214.8, +47.0, +24.3 ms here, and −71.7, −10.0, −42.4 ms in an earlier run of the same -measurement) have no consistent sign: the `memory.set` p99 gate (a rise of at most 10 ms) is -unresolved locally. An unmarked session pays about 11 µs a row, as above. +The marked statement medians average 9.6 ms on and 10.5 ms off. A statement insert omits +the delivery worker's claim, embedding, publication and settlement work, so these figures +establish no delivery-throughput gate. The API p99 differences within each round are ++214.8, +47.0 and +24.3 ms; another small diagnostic has differences of −71.7, −10.0 and +−42.4 ms. Their opposing signs on the loaded host leave the API gate unresolved. Neither +end-to-end gate is established by these measurements. + +`bench/cell_trigger_gate.py` measures the acceptance workload on an independently owned, +attached scratch fixture with explicit fake providers, enforced mode and an already held +org. Each API run has 50 unmeasured warmups and 2,000 sequential measured requests with +4,096-byte values, with five runs per arm in counterbalanced order. Each delivery block +times five complete `pg_delivery.run_pass` calls at the unchanged configured batch size, +with five blocks per arm. The latency gate is `median(p99_on) - median(p99_off) <= 10 ms`; +the throughput gate is `1 - median(rows_per_second_on) / median(rows_per_second_off) <= 0.10`. +The driver retains every run and raw outcome; it does not substitute statement timings. + +```bash +PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py \ + --base-url --pg-dsn --token \ + --org --namespace --fixture-manifest \ + --launch-manifest --output +``` + +The fixture owner supplies the exact source, launch, configuration and ownership receipts, +excludes competing controllers and workers, and stops its processes before deleting its +exact scratch databases. The output path must not exist. The driver verifies those +receipts and the live fixture before native setup and every arm, restores the org triggers +within its cleanup bound and emits a final verdict only after cleanup and artifact sync. +Source tests and the diagnostic figures above are not evidence that this workload passes. ## Reference computations diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 2c3915ae66..c56f7b25f4 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -196,30 +196,11 @@ async def _take_lock(self) -> None: raise Refused(refusals) await apg.take_lock(self.lock_conn, "attach") - async def close_lock(self) -> None: - from server.core import pg - + async def close_lock(self, error: BaseException | None = None) -> None: if self._lock_cm is not None: cm, conn = self._lock_cm, self.lock_conn self._lock_cm = self.lock_conn = None - closed = pg._lost(conn) - end = time.monotonic() + (0.0 if closed else pg._RELEASE_TIMEOUT_S) - try: - if not closed: - await pg.close_owned(conn, end=end) - finally: - task = asyncio.create_task(cm.__aexit__(None, None, None)) - task.add_done_callback(pg._retrieve_task) - try: - done, _ = await asyncio.wait({task}, timeout=max(0.0, end - time.monotonic())) - if not done: - task.cancel() - raise TimeoutError("attach lock cleanup deadline expired") - task.result() - except BaseException: - pg._terminate(conn) - task.cancel() - raise + await apg.close_context(cm, conn, error=error) async def confirm_lock(self) -> None: """The lock is held still, or taken again (refusing when another run holds it now), @@ -485,16 +466,22 @@ def mark(mode: str) -> None: # landed; `recording` again when that try rolled back. self.mode = mode + cm = pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) + conn = await cm.__aenter__() + failure: BaseException | None = None try: - async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: - await apg.commit_enforced( - conn, self.confirm_lock, self.ready_for_commit, self.wait, self.log, mark - ) + await apg.commit_enforced( + conn, self.confirm_lock, self.ready_for_commit, self.wait, self.log, mark + ) + except BaseException as error: # noqa: BLE001 — exit preserves the work failure. + failure = error + try: + await apg.close_context(cm, conn, error=failure) except Refused: raise except Exception as e: - if self.mode != "unknown": - raise # no try was in flight: nothing to decide + if failure is None or self.mode != "unknown": + raise # Only interrupted work needs the unknown-outcome recovery. self.log(f"the mode commit's outcome is unknown ({type(e).__name__}: {e})") async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: await apg.end_mode_transactions(conn, self.confirm_lock) @@ -691,7 +678,7 @@ async def run(self) -> int: raise finally: try: - await self.close_lock() + await self.close_lock(failure) except BaseException: if failure is None: raise diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index c71e1130c7..ca6fedf794 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -94,9 +94,9 @@ class _Interrupted: error: BaseException -async def _work_result[T](work: Coroutine[Any, Any, T]) -> T | _Interrupted: +async def _work_result[T](work: Callable[[], Awaitable[T]]) -> T | _Interrupted: try: - return await work + return await work() except (KeyboardInterrupt, SystemExit) as error: # These exceptions otherwise stop asyncio's loop before the owner can close. return _Interrupted(error) @@ -108,7 +108,7 @@ async def supervise[T](conn: Any, work: Coroutine[Any, Any, T], *, end: float) - A failed borrowed transaction must remain alive until public close sends its cancellation. Closing the transport first can leave its backend locks held. """ - task = asyncio.create_task(_work_result(work)) + task = asyncio.create_task(_work_result(lambda: work)) reserve = min(1.0, max(0.0, end - time.monotonic()) / 4) try: done, _ = await asyncio.wait({task}, timeout=max(0.0, end - reserve - time.monotonic())) @@ -133,6 +133,53 @@ async def supervise[T](conn: Any, work: Coroutine[Any, Any, T], *, end: float) - raise +async def close_context( + cm: contextlib.AbstractAsyncContextManager[Any], + conn: Any, + *, + error: BaseException | None = None, +) -> None: + """Close an owned session and its context without replacing its first failure. + + Work supervision may have closed it already; context exit then gets no fresh + allowance. Both exit invocation and awaiting run inside the fatal capture. + """ + closed = pg._lost(conn) + end = time.monotonic() + (0.0 if closed else pg._RELEASE_TIMEOUT_S) + failure = error + try: + if not closed: + await pg.close_owned(conn, end=end) + except BaseException as caught: # noqa: BLE001 — keep the first operation failure. + if failure is None: + failure = caught + task = asyncio.create_task( + _work_result( + lambda: cm.__aexit__( + type(failure) if failure is not None else None, + failure, + failure.__traceback__ if failure is not None else None, + ) + ) + ) + try: + done, _ = await asyncio.wait({task}, timeout=max(0.0, end - time.monotonic())) + if not done: + raise TimeoutError("attach context cleanup deadline expired") + if isinstance(result := task.result(), _Interrupted): + raise result.error + except BaseException as caught: # noqa: BLE001 — exit cannot replace the first failure. + pg._terminate(conn) + if failure is None: + failure = caught + finally: + if not task.done(): + task.cancel() + task.add_done_callback(pg._retrieve_task) + if failure is not None: + raise failure + + def _split(key: int) -> str: """The `pg_locks` columns that hold a bigint advisory key, in this database.""" return ( diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py index 31986733a8..29dcce134b 100644 --- a/tests/unit/test_bench_cell_trigger_gate.py +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -724,7 +724,7 @@ async def restore(*_): restoring.set() await release.wait() - async def close(c): + async def close(c, *_): c.closed = True monkeypatch.setattr(gate.asyncpg, "connect", AsyncMock(return_value=conn)) @@ -992,7 +992,7 @@ async def restore(self, _): monkeypatch.setattr(gate, "RESTORE_S", 0.01) - async def close(owned): + async def close(owned, *_): assert owned is conn await conn.close() @@ -1456,3 +1456,130 @@ def forbidden(*_args, **_kwargs): verdict = json.loads(capsys.readouterr().out) assert verdict["verdict"] == "INVALID" and verdict["reason"] == "launch receipt hash" assert not (tmp_path / "output.jsonl").exists() + + +@pytest.mark.asyncio +async def test_restore_closes_before_installed_transaction_rollback_can_overrun(monkeypatch): + from asyncpg.transaction import Transaction + + events = [] + release, rolling_back = asyncio.Event(), asyncio.Event() + + class Connection: + _pool_release_ctr = 0 + _top_xact = None + _protocol = SimpleNamespace(is_in_transaction=lambda: False) + + def __init__(self): + self.closed = False + self.timeouts = [] + + def is_closed(self): + return self.closed + + def transaction(self): + return Transaction(self, None, False, False) + + async def execute(self, sql, *args): + if sql.startswith("DROP TRIGGER"): + raise ValueError("restoration DDL failed") + if sql.startswith("ROLLBACK"): + rolling_back.set() + try: + while not release.is_set(): + try: + await release.wait() + except asyncio.CancelledError: + continue + finally: + events.append("rollback ended") + + async def close(self, **kwargs): + self.timeouts.append(kwargs["timeout"]) + events.append("public close") + self.closed = True + release.set() + + def terminate(self): + self.closed = True + release.set() + + conn = Connection() + monkeypatch.setattr(gate, "RESTORE_S", 0.04) + artifact = SimpleNamespace(emit=lambda *a, **k: None) + task = asyncio.create_task( + gate._cleanup(gate.Triggers(("memory_current",), "", ""), conn, artifact) + ) + try: + await asyncio.wait_for(rolling_back.wait(), 1) + done, _ = await asyncio.wait({task}, timeout=0.2) + assert done, "restoration deadline waited for rollback before closing" + with pytest.raises(TimeoutError): + task.result() + assert conn.closed + assert events.index("public close") < events.index("rollback ended") + assert conn.timeouts and all(0 < bound <= 0.04 for bound in conn.timeouts) + finally: + release.set() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("kind", ["SystemExit", "KeyboardInterrupt"]) +@pytest.mark.parametrize("phase", ["restore", "close"]) +def test_driver_cleanup_native_failure_stays_inside_its_owner(kind, phase): + import subprocess + import sys + import textwrap + + program = textwrap.dedent( + """ + import argparse + import asyncio + import sys + from types import SimpleNamespace + from unittest.mock import AsyncMock + from bench import cell_trigger_gate as gate + kind, phase = sys.argv[1:] + original = ValueError('original operation failure') + secondary = {'SystemExit': SystemExit, 'KeyboardInterrupt': KeyboardInterrupt}[kind]( + 'cleanup failed') + events = [] + class Connection: + async def close(self, **kwargs): + events.append('close') + if phase == 'close': raise secondary + def terminate(self): events.append('abort') + class Triggers: + tables = ('memory_current',) + invariant = org = 'hash' + async def set(self, *args): pass + async def restore(self, *args): + events.append('restore') + if phase == 'restore': raise secondary + async def main(): + gate.asyncpg.connect = AsyncMock(return_value=Connection()) + gate._proof = AsyncMock(return_value={}) + gate.Triggers.capture = AsyncMock(return_value=Triggers()) + gate._memory_set = AsyncMock(side_effect=original) + artifact = SimpleNamespace(emit=lambda *a, **k: None) + try: + await gate._run(argparse.Namespace(pg_dsn='unused'), None, artifact, object()) + except BaseException as error: + assert error is original, (type(error).__name__, events) + assert events[0] == 'restore' and 'close' in events, events + await asyncio.sleep(0) + print('original-after-driver-cleanup') + return + raise AssertionError('operation failure was swallowed') + asyncio.run(main()) + """ + ) + result = subprocess.run( + [sys.executable, "-c", program, kind, phase], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "original-after-driver-cleanup" in result.stdout diff --git a/tests/unit/test_cell_attach_owned_exit.py b/tests/unit/test_cell_attach_owned_exit.py new file mode 100644 index 0000000000..9bdaedb76d --- /dev/null +++ b/tests/unit/test_cell_attach_owned_exit.py @@ -0,0 +1,160 @@ +"""Owned attach contexts finish before an error reaches the command boundary.""" + +import subprocess +import sys +import textwrap + +import pytest + + +@pytest.mark.parametrize("owner", ["lock", "mode"]) +@pytest.mark.parametrize("secondary", ["RuntimeError", "SystemExit", "KeyboardInterrupt"]) +def test_actual_owned_context_keeps_first_failure(owner, secondary): + program = textwrap.dedent( + """ + import argparse + import asyncio + import sys + from unittest.mock import AsyncMock + import asyncpg + from server.cli.cell_attach import Run + from server.cli import cell_attach_pg as apg + from server.core import pg + from server.settings import settings + + owner, kind = sys.argv[1:] + original = ValueError('operation failed') + secondary = {'RuntimeError': RuntimeError, 'SystemExit': SystemExit, + 'KeyboardInterrupt': KeyboardInterrupt}[kind]('exit failed') + events = [] + class Conn: + def __init__(self): + self.closed = False + self.closes = 0 + def is_closed(self): + return self.closed + async def close(self, **kwargs): + self.closes += 1 + events.append('close') + self.closed = True + if self.closes == 2: + raise secondary + def terminate(self): + self.closed = True + events.append('abort') + + async def main(): + conn = Conn() + async def raw(*args, **kwargs): return conn + async def checked(*args): pass + async def fail(*args): raise original + asyncpg.connect = raw + pg._check_deployment = checked + pg.init_connection = checked + settings.pg_dsn = 'postgresql://unused/test' + router = AsyncMock() + router.call.return_value = {'state': 'attaching'} + run = Run(None, argparse.Namespace(cell='test', wait=0), router, lambda _: None) + run.row = {'deployment_id': 'test'} + run.mode = 'recording' + run.confirm_lock = AsyncMock() + run.ready_for_commit = AsyncMock(return_value=[]) + async def open_lock(): + run._lock_cm = pg.dedicated() + run.lock_conn = await run._lock_cm.__aenter__() + run.open_lock = open_lock + run.read_own_row = AsyncMock(side_effect=original) + apg._mode_transaction = fail + try: + await (run.run() if owner == 'lock' else run.commit_mode()) + except BaseException as error: + assert error is original, (type(error).__name__, events) + assert events.count('close') == 2, events + await asyncio.sleep(0) + print('original-after-owned-exit') + return + raise AssertionError('operation failure was swallowed') + asyncio.run(main()) + """ + ) + result = subprocess.run( + [sys.executable, "-c", program, owner, secondary], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "original-after-owned-exit" in result.stdout + + +@pytest.mark.asyncio +@pytest.mark.parametrize("factory_failure", [False, True]) +async def test_mode_failure_context_receives_the_actual_exception(monkeypatch, factory_failure): + import argparse + from unittest.mock import AsyncMock + + from server.cli import cell_attach_pg as apg + from server.cli.cell_attach import Run + from server.core import pg + + original = ValueError("mode failed") + observed = [] + conn = AsyncMock() + conn.is_closed = lambda: False + conn.terminate = lambda: None + + class Context: + async def __aenter__(self): + return conn + + def __aexit__(self, kind, error, traceback): + observed.append((kind, error, traceback)) + if factory_failure: + raise RuntimeError("exit construction failed") + return AsyncMock(side_effect=RuntimeError("exit failed"))() + + monkeypatch.setattr(pg, "dedicated", lambda **_: Context()) + monkeypatch.setattr(apg, "commit_enforced", AsyncMock(side_effect=original)) + run = Run(None, argparse.Namespace(cell="test", wait=0), AsyncMock(), lambda _: None) + run.row = {"deployment_id": "test"} + run.mode = "recording" + with pytest.raises(ValueError) as result: + await run.commit_mode() + assert result.value is original + assert len(observed) == 1 + assert observed[0][0] is ValueError and observed[0][1] is original + assert observed[0][2] is not None + + +@pytest.mark.asyncio +async def test_successful_mode_work_still_fails_when_required_close_fails(monkeypatch): + import argparse + from contextlib import asynccontextmanager + from unittest.mock import AsyncMock + + from server.cli import cell_attach_pg as apg + from server.cli.cell_attach import Run + from server.core import pg + + original = OSError("public close failed") + conn = AsyncMock() + conn.is_closed = lambda: False + conn.terminate = lambda: None + conn.close.side_effect = original + + @asynccontextmanager + async def dedicated(**_): + yield conn + + monkeypatch.setattr(pg, "dedicated", dedicated) + + async def committed(*args): + args[-1]("unknown") + + monkeypatch.setattr(apg, "commit_enforced", committed) + run = Run(None, argparse.Namespace(cell="test", wait=0), AsyncMock(), lambda _: None) + run.row = {"deployment_id": "test"} + with pytest.raises(OSError) as result: + await run.commit_mode() + assert result.value is original From 6a1d01819bad753473ff0c5420fec5bb877106d5 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 01:34:22 +0000 Subject: [PATCH 043/114] Compose shared directory cleanup and foreground pool initialization Signed-off-by: zlareb1 --- docs/architecture/cells.md | 11 + server/router/directory.py | 116 ++++++--- .../test_directory_control_cleanup.py | 223 +++++++++++++++++- tests/unit/test_cell_control_census.py | 2 +- tests/unit/test_directory_control_cleanup.py | 191 ++++++++++++++- tests/unit/test_directory_pool_init.py | 160 +++++++++++++ 6 files changed, 655 insertions(+), 48 deletions(-) create mode 100644 tests/unit/test_directory_pool_init.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 1954a69930..c501cf0e4c 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1279,6 +1279,17 @@ connection before releasing the pool context. Unexpected errors never wait for r on that discarded session. A late acquisition cannot start mutation work or obtain a fresh cleanup allowance; acquisition without a known proxy remains an unknown outcome. Ordinary live placement retains its separate transaction and connection handling. +Repeated interruption keeps the same close task and original end; a completed child +cancellation remains a terminal failure. Fallback abort errors cannot replace the first +operation or cleanup failure. + +The shared directory pool opens physical sessions only for foreground acquisitions; +pool object creation does not establish connectivity. Healthy released sessions are +cached under the existing size cap, recycling and idle expiry. A later cold acquisition +opens a new session. Fixed JSON codec initialization quarantines its never-borrowed raw +session on failure; native interruptions return to that foreground consumer after driver +cleanup, with no autonomous opening or replenishment task. If quarantine fails, the +original failure records that session closure is unknown. The source census for acceptance and operator directory writes is checked by `tests/unit/test_cell_control_census.py`, including dynamic cleanup tables, guarded diff --git a/server/router/directory.py b/server/router/directory.py index d4f3f62583..bdf9f4bdd6 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -152,13 +152,37 @@ def _require_dsn() -> str: return settings.router_pg_dsn +class _PoolInitFailure(Exception): + """Carry native control intent through the driver's ordinary-error cleanup.""" + + def __init__(self, error: SystemExit | KeyboardInterrupt) -> None: + super().__init__("directory codec initialization interrupted") + self.error = error + + async def _init(conn: Any) -> None: - for typ in ("json", "jsonb"): - await conn.set_type_codec(typ, encoder=json.dumps, decoder=json.loads, schema="pg_catalog") + try: + for typ in ("json", "jsonb"): + await conn.set_type_codec( + typ, encoder=json.dumps, decoder=json.loads, schema="pg_catalog" + ) + except BaseException as original: + # Only builtin codecs run here: this raw session has never been borrowed or + # held command locks. Quarantine before the driver can lose its ownership. + try: + conn.terminate() + except BaseException: # noqa: BLE001 — disposal must preserve the first init failure. + try: + original.add_note("directory initialization session closure is unknown") + except BaseException: # noqa: BLE001 — diagnostics must preserve the first failure. + pass + if isinstance(original, (SystemExit, KeyboardInterrupt)): + raise _PoolInitFailure(original) from None + raise async def pool() -> Any: - """The directory's pool, opened on first use.""" + """The directory's pool; physical sessions open only for foreground consumers.""" global _pool if _pool is not None: return _pool @@ -166,19 +190,22 @@ async def pool() -> Any: async with _opening: if _pool is None: - _pool = await asyncpg.create_pool( - _require_dsn(), - min_size=1, - max_size=POOL_MAX, - init=_init, - server_settings={ - **_SESSION, - "statement_timeout": str(STATEMENT_TIMEOUT_S * 1000), - "idle_in_transaction_session_timeout": str(STATEMENT_TIMEOUT_S * 2000), - }, - command_timeout=STATEMENT_TIMEOUT_S + 1, - timeout=STATEMENT_TIMEOUT_S, - ) + try: + _pool = await asyncpg.create_pool( + _require_dsn(), + min_size=0, + max_size=POOL_MAX, + init=_init, + server_settings={ + **_SESSION, + "statement_timeout": str(STATEMENT_TIMEOUT_S * 1000), + "idle_in_transaction_session_timeout": str(STATEMENT_TIMEOUT_S * 2000), + }, + command_timeout=STATEMENT_TIMEOUT_S + 1, + timeout=STATEMENT_TIMEOUT_S, + ) + except _PoolInitFailure as failed: + raise failed.error from None return _pool @@ -195,13 +222,17 @@ async def _connection(*, lookup: bool = False) -> AsyncIterator[Any]: slot, the pool's opening and the connection); a lookup's also takes one of LOOKUP_SLOTS. A wait past the bound raises TimeoutError, an OSError, which `_reading` answers as the directory unavailable.""" - async with AsyncExitStack() as stack: - async with asyncio.timeout(STATEMENT_TIMEOUT_S): - if lookup: - await _lookups.acquire() - stack.callback(_lookups.release) - conn = await stack.enter_async_context((await pool()).acquire()) - yield conn + try: + async with AsyncExitStack() as stack: + async with asyncio.timeout(STATEMENT_TIMEOUT_S): + if lookup: + await _lookups.acquire() + stack.callback(_lookups.release) + conn = await stack.enter_async_context((await pool()).acquire()) + yield conn + except _PoolInitFailure as failed: + # Driver acquisition and context cleanup finish before control intent returns. + raise failed.error from None @asynccontextmanager @@ -919,9 +950,9 @@ async def _control_result[T](work: Callable[[], Awaitable[T]]) -> T | _ControlFa class _ControlOwner: """Own only this operator attempt's proxy, through close and pool release.""" - def __init__(self) -> None: - self.end = time.monotonic() + CONTROL_MAX_S - self.work_end = self.end - min(1.0, CONTROL_MAX_S / 4) + def __init__(self, seconds: float) -> None: + self.end = time.monotonic() + seconds + self.work_end = self.end - min(1.0, seconds / 4) self.context = _connection() self.conn: Any = None self.entered = False @@ -949,12 +980,33 @@ async def close(self) -> None: return if self.close_task is None: self.close_task = self.start(lambda: pg.close_owned(self.conn, end=self.end)) + interrupted: asyncio.CancelledError | None = None try: - await self.wait(self.close_task, self.end) + while not self.close_task.done(): + remaining = self.end - time.monotonic() + if remaining <= 0: + raise TimeoutError("directory control deadline expired; outcome unknown") + try: + await asyncio.wait({self.close_task}, timeout=remaining) + except asyncio.CancelledError as error: + # Another signal cannot abandon the public cancellation in flight. + interrupted = interrupted or error + result = self.close_task.result() + if isinstance(result, _ControlFatal): + raise _ControlInterrupted(result) except BaseException: - pg._terminate(self.conn) - self.close_task.cancel() + try: + pg._terminate(self.conn) + except BaseException: # noqa: BLE001 — abort must preserve the first cleanup failure. + pass + finally: + if not self.close_task.done(): + self.close_task.cancel() + if interrupted is not None: + raise interrupted from None raise + if interrupted is not None: + raise interrupted async def release(self) -> None: if self.entered: @@ -1008,10 +1060,10 @@ def __init__(self, fatal: _ControlFatal) -> None: self.fatal = fatal -async def _control_transaction[T](change: Callable[[Any], Awaitable[T]]) -> T: +async def _control_transaction[T](change: Callable[[Any], Awaitable[T]], *, seconds: float) -> T: """Bound the whole callback, close first, then settle work and pool ownership.""" async with _reading(): - owner = _ControlOwner() + owner = _ControlOwner(seconds) task = owner.start(lambda: owner.work(change)) try: result = await owner.wait(task, owner.work_end) @@ -1050,7 +1102,7 @@ async def locked(conn: Any) -> T: await conn.execute("SELECT pg_logical_emit_message(true, 'loom.control', '')") return await change(conn, attaching) - return await _control_transaction(locked) + return await _control_transaction(locked, seconds=CONTROL_MAX_S) async def claim(cell_name: str, expected: Control) -> Attaching: diff --git a/tests/integration/test_directory_control_cleanup.py b/tests/integration/test_directory_control_cleanup.py index 9970d6b5ee..2df016414a 100644 --- a/tests/integration/test_directory_control_cleanup.py +++ b/tests/integration/test_directory_control_cleanup.py @@ -55,7 +55,9 @@ async def work(conn): seen.set() await conn.execute("SELECT pg_sleep(10)") - task = asyncio.create_task(directory._control_transaction(work)) + task = asyncio.create_task( + directory._control_transaction(work, seconds=directory.CONTROL_MAX_S) + ) try: await asyncio.wait_for(seen.wait(), 2) await sleeping(directory_database, backend["pid"]) @@ -97,7 +99,7 @@ async def work(conn): raise original with pytest.raises(ValueError) as caught: - await directory._control_transaction(work) + await directory._control_transaction(work, seconds=directory.CONTROL_MAX_S) assert caught.value is original and "rollback" not in backend assert time.monotonic() - began < 0.4 await released(directory_database, backend["pid"]) @@ -112,7 +114,10 @@ async def work(conn): await conn.execute("UPDATE probe SET id=2") return "acknowledged" - assert await directory._control_transaction(work) == "acknowledged" + assert ( + await directory._control_transaction(work, seconds=directory.CONTROL_MAX_S) + == "acknowledged" + ) assert await directory_database.fetchval("SELECT id FROM probe") == 2 await released(directory_database, backend["pid"]) @@ -122,6 +127,7 @@ async def test_directory_second_cancel_keeps_first_and_close_precedes_worker_can ): import asyncpg + monkeypatch.setattr(directory, "CONTROL_MAX_S", 0.4) real_close = asyncpg.Connection.close seen, closing, allow_close = asyncio.Event(), asyncio.Event(), asyncio.Event() backend = {} @@ -146,7 +152,9 @@ async def close(self, **kwargs): await real_close(self, **kwargs) monkeypatch.setattr(asyncpg.Connection, "close", close) - task = asyncio.create_task(directory._control_transaction(work)) + task = asyncio.create_task( + directory._control_transaction(work, seconds=directory.CONTROL_MAX_S) + ) try: await asyncio.wait_for(seen.wait(), 2) await sleeping(directory_database, backend["pid"]) @@ -187,7 +195,7 @@ async def work(conn): monkeypatch.setattr(asyncpg.Connection, "execute", execute) with pytest.raises(directory.DirectoryUnavailable) as caught: - await directory._control_transaction(work) + await directory._control_transaction(work, seconds=directory.CONTROL_MAX_S) assert caught.value.__cause__ is original assert await directory_database.fetchval("SELECT id FROM probe") == 2 await released(directory_database, backend["pid"]) @@ -215,7 +223,7 @@ async def work(conn): monkeypatch.setattr(asyncpg.Connection, "close", close) with pytest.raises(ValueError) as caught: - await directory._control_transaction(work) + await directory._control_transaction(work, seconds=directory.CONTROL_MAX_S) assert caught.value is original assert await directory_database.fetchval("SELECT id FROM probe") == 2 await released(directory_database, backend["pid"]) @@ -267,7 +275,7 @@ async def work(conn): asyncpg.Connection.execute = execute directory._connection = connection try: - try: await directory._control_transaction(work) + try: await directory._control_transaction(work, seconds=directory.CONTROL_MAX_S) except (SystemExit, KeyboardInterrupt) as caught: assert caught is original else: raise AssertionError('fatal guard returned success') if stage == 'acquire': @@ -294,3 +302,204 @@ async def work(conn): ) stdout, stderr = await asyncio.wait_for(process.communicate(), 10) assert process.returncode == 0, (stdout.decode(), stderr.decode()) + + +@pytest.mark.parametrize("kind", ["ordinary", "cancel", "exit", "keyboard"]) +@pytest.mark.parametrize("healthy_loan", [False, True]) +@pytest.mark.parametrize("driver_close", ["normal", "exit", "keyboard"]) +async def test_failed_codec_raw_session_is_disposed_without_touching_other_loans( + directory_database, monkeypatch, kind, healthy_loan, driver_close +): + import asyncpg + + original = { + "ordinary": ValueError("fixed codec failed"), + "cancel": asyncio.CancelledError("fixed codec cancelled"), + "exit": SystemExit(17), + "keyboard": KeyboardInterrupt("fixed codec interrupted"), + }[kind] + pool = await directory.pool() + assert pool.get_size() == 0 # Pool object creation does not open a backend. + healthy = await pool.acquire() if healthy_loan else None + healthy_pid = healthy.get_server_pid() if healthy else None + real_codec, real_close = asyncpg.Connection.set_type_codec, asyncpg.Connection.close + failed = {} + attempts = 0 + + async def codec(self, *args, **kwargs): + nonlocal attempts + attempts += 1 + failed["raw"], failed["pid"] = self, self.get_server_pid() + raise original + + async def close(self, **kwargs): + if self is failed.get("raw"): + # This is the failed, never-borrowed raw codec session, not an active + # command session: synchronous quarantine must precede driver cleanup. + assert self.is_closed() + failed["driver close"] = True + if driver_close == "exit": + raise SystemExit(21) + if driver_close == "keyboard": + raise KeyboardInterrupt("secondary closed-raw cleanup failure") + return await real_close(self, **kwargs) + + async def acquire(): + async with directory._connection() as conn: + pytest.fail(f"failed initializer admitted backend {conn.get_server_pid()}") + + monkeypatch.setattr(asyncpg.Connection, "set_type_codec", codec) + monkeypatch.setattr(asyncpg.Connection, "close", close) + try: + if kind in {"exit", "keyboard"}: + result = await directory._control_result(acquire) + assert isinstance(result, directory._ControlFatal) and result.error is original + else: + with pytest.raises(type(original)) as caught: + await acquire() + assert caught.value is original + assert failed["driver close"] and attempts == 1 + await released(directory_database, failed["pid"]) + # Driver background replenishment must not swallow/retry control intent. + for _ in range(3): + await asyncio.sleep(0) + assert attempts == 1 + if healthy: + assert healthy.get_server_pid() == healthy_pid + assert await healthy.fetchval("SELECT 1") == 1 + monkeypatch.setattr(asyncpg.Connection, "set_type_codec", real_codec) + async with directory._connection() as recovered: + recovered_pid = recovered.get_server_pid() + assert recovered_pid not in {failed["pid"], healthy_pid} + assert await recovered.fetchval("SELECT 1") == 1 + async with directory._connection() as cached: + assert cached.get_server_pid() == recovered_pid + finally: + monkeypatch.setattr(asyncpg.Connection, "set_type_codec", real_codec) + if healthy: + await pool.release(healthy) + + +@pytest.mark.parametrize("kind", ["exit", "keyboard"]) +@pytest.mark.parametrize("secondary", ["ordinary", "exit", "keyboard"]) +async def test_codec_quarantine_failure_keeps_native_original_and_unknown_diagnostic( + directory_database, monkeypatch, kind, secondary +): + import asyncpg + + original = SystemExit(17) if kind == "exit" else KeyboardInterrupt("codec interrupted") + second = { + "ordinary": ValueError("termination failed"), + "exit": SystemExit(19), + "keyboard": KeyboardInterrupt("termination interrupted"), + }[secondary] + real_codec, real_terminate = asyncpg.Connection.set_type_codec, asyncpg.Connection.terminate + failed = {} + + async def codec(self, *args, **kwargs): + failed["raw"], failed["pid"] = self, self.get_server_pid() + raise original + + def terminate(self): + if self is failed.get("raw"): + raise second + return real_terminate(self) + + async def acquire(): + async with directory._connection(): + pytest.fail("failed initialization returned success") + + monkeypatch.setattr(asyncpg.Connection, "set_type_codec", codec) + monkeypatch.setattr(asyncpg.Connection, "terminate", terminate) + try: + result = await directory._control_result(acquire) + assert isinstance(result, directory._ControlFatal) and result.error is original + assert original.__notes__ == ["directory initialization session closure is unknown"] + # Termination's request is not the proof; here the driver's subsequent + # ordinary-error public close really completes and the observer proves it. + await released(directory_database, failed["pid"]) + finally: + monkeypatch.setattr(asyncpg.Connection, "set_type_codec", real_codec) + monkeypatch.setattr(asyncpg.Connection, "terminate", real_terminate) + raw = failed.get("raw") + if raw is not None: + real_terminate(raw) + + +@pytest.mark.parametrize("signal_count", [1, 2]) +async def test_directory_signals_keep_real_cancel_request_alive_until_original_end( + directory_database, monkeypatch, signal_count +): + from asyncpg import connect_utils + + seen, cancelling, allow_cancel = asyncio.Event(), asyncio.Event(), asyncio.Event() + backend = {} + first_signal = {} + real_wait = directory._ControlOwner.wait + real_cancel = connect_utils._cancel + seconds = 0.5 + + async def cancel(**kwargs): + if kwargs["backend_pid"] == backend.get("pid"): + cancelling.set() + await allow_cancel.wait() + return await real_cancel(**kwargs) + + async def work(conn): + backend["pid"] = conn.get_server_pid() + await locked(conn) + seen.set() + await conn.execute("SELECT pg_sleep(10)") + + async def remember(self, task, end): + try: + return await real_wait(self, task, end) + except asyncio.CancelledError as error: + first_signal.setdefault("error", error) + raise + + monkeypatch.setattr(directory._ControlOwner, "wait", remember) + monkeypatch.setattr(connect_utils, "_cancel", cancel) + began = time.monotonic() + task = asyncio.create_task(directory._control_transaction(work, seconds=seconds)) + try: + await asyncio.wait_for(seen.wait(), 2) + await sleeping(directory_database, backend["pid"]) + task.cancel("first directory signal") + await asyncio.wait_for(cancelling.wait(), 2) + if signal_count == 2: + task.cancel("second directory signal") + await asyncio.sleep(0.03) + assert not task.done(), "second signal abandoned a live server cancel request" + assert await directory_database.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1 AND wait_event='PgSleep')", + backend["pid"], + ) + assert ( + await directory_database.fetchval( + "SELECT count(*) FROM pg_locks WHERE pid=$1", backend["pid"] + ) + > 0 + ) + allow_cancel.set() + with pytest.raises(asyncio.CancelledError, match="first directory signal") as caught: + await asyncio.wait_for(task, seconds) + assert caught.value is first_signal["error"] + assert time.monotonic() - began <= seconds + 0.1 + await released(directory_database, backend["pid"]) + finally: + allow_cancel.set() + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + if backend: + await directory_database.execute("SELECT pg_cancel_backend($1)", backend["pid"]) + async with asyncio.timeout(2): + for _ in range(200): + if not await directory_database.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)", backend["pid"] + ): + break + await asyncio.sleep(0.01) + else: + pytest.fail("test-owned backend remained after exact cancellation") diff --git a/tests/unit/test_cell_control_census.py b/tests/unit/test_cell_control_census.py index 45b8e85baa..64359d1293 100644 --- a/tests/unit/test_cell_control_census.py +++ b/tests/unit/test_cell_control_census.py @@ -411,7 +411,7 @@ def function(path: str, name: str) -> ast.AST: "server/router/directory.py", "_attach_change", [ - "_control_transaction(locked)", + "_control_transaction(locked, seconds=CONTROL_MAX_S)", "control_refusal(attaching, expected, claim=claim)", "change(conn, attaching)", ], diff --git a/tests/unit/test_directory_control_cleanup.py b/tests/unit/test_directory_control_cleanup.py index 1a66b20c26..be6fee8329 100644 --- a/tests/unit/test_directory_control_cleanup.py +++ b/tests/unit/test_directory_control_cleanup.py @@ -29,7 +29,12 @@ async def test_success_closes_once_before_pool_exit(owned): events = [] conn.close.side_effect = lambda **_: events.append("close") context.__aexit__.side_effect = lambda *_: events.append("exit") - assert await directory._control_transaction(AsyncMock(return_value=19)) == 19 + assert ( + await directory._control_transaction( + AsyncMock(return_value=19), seconds=directory.CONTROL_MAX_S + ) + == 19 + ) assert events == ["close", "exit"] conn.close.assert_awaited_once() conn.transaction.return_value.commit.assert_awaited_once() @@ -41,7 +46,9 @@ async def test_original_error_survives_close_and_exit_failure(owned): conn.close.side_effect = RuntimeError("close failure") context.__aexit__.side_effect = LookupError("exit failure") with pytest.raises(ValueError) as caught: - await directory._control_transaction(AsyncMock(side_effect=original)) + await directory._control_transaction( + AsyncMock(side_effect=original), seconds=directory.CONTROL_MAX_S + ) assert caught.value is original conn.close.assert_awaited_once() context.__aexit__.assert_awaited_once() @@ -54,7 +61,7 @@ async def test_close_failure_survives_exit_failure_after_acknowledged_commit(own conn.close.side_effect = original context.__aexit__.side_effect = RuntimeError("exit failure") with pytest.raises(ValueError) as caught: - await directory._control_transaction(AsyncMock()) + await directory._control_transaction(AsyncMock(), seconds=directory.CONTROL_MAX_S) assert caught.value is original conn.transaction.return_value.commit.assert_awaited_once() @@ -70,7 +77,7 @@ async def stalled(*_): context.__aexit__.side_effect = stalled began = time.monotonic() with pytest.raises(directory.DirectoryUnavailable): - await directory._control_transaction(AsyncMock()) + await directory._control_transaction(AsyncMock(), seconds=directory.CONTROL_MAX_S) assert entered.is_set() and time.monotonic() - began < 0.2 conn.close.assert_awaited_once() @@ -80,7 +87,7 @@ async def test_acquisition_failure_does_not_close_an_unowned_proxy(owned): original = ValueError("acquisition failure") context.__aenter__.side_effect = original with pytest.raises(ValueError) as caught: - await directory._control_transaction(AsyncMock()) + await directory._control_transaction(AsyncMock(), seconds=directory.CONTROL_MAX_S) assert caught.value is original conn.close.assert_not_awaited() conn.terminate.assert_not_called() @@ -96,7 +103,9 @@ async def acquire(): await asyncio.Future() context.__aenter__.side_effect = acquire - task = asyncio.create_task(directory._control_transaction(AsyncMock())) + task = asyncio.create_task( + directory._control_transaction(AsyncMock(), seconds=directory.CONTROL_MAX_S) + ) await entered.wait() task.cancel("first acquisition cancellation") with pytest.raises(asyncio.CancelledError, match="first acquisition cancellation"): @@ -105,7 +114,12 @@ async def acquire(): conn.terminate.assert_not_called() context.__aexit__.assert_not_awaited() context.__aenter__.side_effect = None - assert await directory._control_transaction(AsyncMock(return_value=2)) == 2 + assert ( + await directory._control_transaction( + AsyncMock(return_value=2), seconds=directory.CONTROL_MAX_S + ) + == 2 + ) conn.close.assert_awaited_once() @@ -122,7 +136,9 @@ async def acquire(): context.__aenter__.side_effect = acquire work = AsyncMock() - task = asyncio.create_task(directory._control_transaction(work)) + task = asyncio.create_task( + directory._control_transaction(work, seconds=directory.CONTROL_MAX_S) + ) await entered.wait() task.cancel("acquisition cancelled") with pytest.raises(asyncio.CancelledError, match="acquisition cancelled"): @@ -134,3 +150,162 @@ async def acquire(): conn.transaction.assert_not_called() conn.terminate.assert_called() conn.close.assert_not_awaited() # Expired original end cannot create a fresh close budget. + + +@pytest.mark.parametrize( + "ending", ["success", "failure", "self_cancel", "exit", "keyboard", "deadline"] +) +@pytest.mark.parametrize("locally_closed", [False, True]) +async def test_outer_signals_wait_for_the_same_close_and_keep_first( + owned, monkeypatch, ending, locally_closed +): + conn, _ = owned + owner = directory._ControlOwner(0.08 if ending == "deadline" else 10) + conn.is_closed.return_value = locally_closed + conn._lost = locally_closed + owner.conn = conn + end = owner.end + started, allow = asyncio.Event(), asyncio.Event() + first = asyncio.CancelledError("first outer signal") + failure = ValueError("close failure") + real_wait = asyncio.wait + waits = 0 + close_cancelled = [] + calls = [] + began = time.monotonic() + + async def close(_conn, *, end): + calls.append(_conn) + assert _conn is conn and end == owner.end + started.set() + try: + await allow.wait() + except asyncio.CancelledError: + close_cancelled.append(True) + raise + if ending == "failure": + raise failure + if ending == "self_cancel": + raise asyncio.CancelledError("close child cancelled itself") + if ending == "exit": + raise SystemExit(17) + if ending == "keyboard": + raise KeyboardInterrupt("close child interrupted") + + async def interrupted_wait(tasks, **kwargs): + nonlocal waits + waits += 1 + if waits == 1: + await started.wait() + raise first + return await real_wait(tasks, **kwargs) + + monkeypatch.setattr(directory.pg, "close_owned", close) + monkeypatch.setattr(asyncio, "wait", interrupted_wait) + task = asyncio.create_task(owner.close()) + await started.wait() + for _ in range(3): + await asyncio.sleep(0) + assert not task.done(), "an outer signal cancelled independent close before its end" + conn.terminate.assert_not_called() + assert not close_cancelled and owner.end == end + if ending != "deadline": + allow.set() + with pytest.raises(asyncio.CancelledError) as caught: + await task + assert caught.value is first and owner.end == end + assert time.monotonic() - began < 0.2 + assert calls == [conn] and waits < 5 + if ending == "success": + conn.terminate.assert_not_called() + else: + conn.terminate.assert_called_once_with() + + +@pytest.mark.parametrize("already_done", [False, True]) +async def test_self_cancelled_close_is_terminal_before_a_long_end(owned, monkeypatch, already_done): + conn, _ = owned + owner = directory._ControlOwner(10) + owner.conn = conn + original = asyncio.CancelledError("close child self-cancelled") + + async def close(_conn, *, end): + raise original + + monkeypatch.setattr(directory.pg, "close_owned", close) + if already_done: + owner.close_task = owner.start(lambda: close(conn, end=owner.end)) + await asyncio.sleep(0) + assert owner.close_task.done() + began = time.monotonic() + with pytest.raises(asyncio.CancelledError) as caught: + await owner.close() + assert caught.value is original + assert time.monotonic() - began < 0.2 + conn.terminate.assert_called_once_with() + + +@pytest.mark.parametrize("abort_kind", ["ordinary", "cancel", "exit", "keyboard"]) +@pytest.mark.parametrize( + "first_kind", ["ordinary", "cancel", "exit", "keyboard", "outer_cancel", "deadline"] +) +async def test_secondary_abort_never_replaces_first_or_skips_owned_task_settlement( + owned, monkeypatch, abort_kind, first_kind +): + conn, _ = owned + owner = directory._ControlOwner(0.02 if first_kind == "deadline" else 10) + owner.conn = conn + errors = { + "ordinary": ValueError("first close failure"), + "cancel": asyncio.CancelledError("first close cancellation"), + "exit": SystemExit(17), + "keyboard": KeyboardInterrupt("first close interruption"), + "outer_cancel": asyncio.CancelledError("first outer cancellation"), + } + first = errors.get(first_kind) + secondary = { + "ordinary": RuntimeError("secondary abort failure"), + "cancel": asyncio.CancelledError("secondary abort cancellation"), + "exit": SystemExit(19), + "keyboard": KeyboardInterrupt("secondary abort interruption"), + }[abort_kind] + conn.terminate.side_effect = secondary + began = asyncio.Event() + cancelled = asyncio.Event() + real_wait = asyncio.wait + signals = 0 + + async def close(_conn, *, end): + began.set() + if first_kind == "deadline": + try: + await asyncio.Future() + finally: + cancelled.set() + raise ValueError("later close failure") if first_kind == "outer_cancel" else first + + async def interrupted_wait(tasks, **kwargs): + nonlocal signals + signals += 1 + if first_kind == "outer_cancel" and signals == 1: + raise first + return await real_wait(tasks, **kwargs) + + async def observed(): + try: + await owner.close() + except BaseException as error: # noqa: BLE001 — test native failures must remain inert too. + return error.fatal.error if isinstance(error, directory._ControlInterrupted) else error + pytest.fail("failed required cleanup became success") + + monkeypatch.setattr(directory.pg, "close_owned", close) + monkeypatch.setattr(asyncio, "wait", interrupted_wait) + result = await observed() + if first_kind == "deadline": + assert isinstance(result, TimeoutError) + await asyncio.wait_for(cancelled.wait(), 0.1) + assert owner.close_task.cancelled() + else: + assert result is first + assert began.is_set() + conn.terminate.assert_called_once_with() diff --git a/tests/unit/test_directory_pool_init.py b/tests/unit/test_directory_pool_init.py new file mode 100644 index 0000000000..ea8bb3fd37 --- /dev/null +++ b/tests/unit/test_directory_pool_init.py @@ -0,0 +1,160 @@ +"""Failed fixed-codec initialization never admits its unborrowed raw connection.""" + +from __future__ import annotations + +import asyncio +from unittest.mock import AsyncMock, Mock + +import pytest + +from server.router import directory + + +@pytest.mark.parametrize( + "error", [ValueError("codec failure"), asyncio.CancelledError("codec cancelled")] +) +@pytest.mark.parametrize("position", [0, 1]) +async def test_every_failed_initialization_quarantines_its_exact_raw_session(error, position): + conn = Mock(set_type_codec=AsyncMock(side_effect=[None] * position + [error])) + with pytest.raises(type(error)) as caught: + await directory._init(conn) + assert caught.value is error + conn.terminate.assert_called_once_with() + assert conn.set_type_codec.await_count == position + 1 + + +async def test_fixed_builtin_codecs_do_not_execute_sql(): + import json + from types import MethodType, SimpleNamespace + + import asyncpg + + conn = SimpleNamespace( + _check_open=Mock(), + _protocol=Mock(), + _drop_local_statement_cache=Mock(), + _execute=AsyncMock(side_effect=AssertionError("codec initialization executed SQL")), + terminate=Mock(), + ) + conn._introspect_type = MethodType(asyncpg.Connection._introspect_type, conn) + conn.set_type_codec = MethodType(asyncpg.Connection.set_type_codec, conn) + await directory._init(conn) + conn._execute.assert_not_awaited() + conn.terminate.assert_not_called() + calls = conn._protocol.get_settings.return_value.add_python_codec.call_args_list + assert len(calls) == 2 + assert [call.args[1] for call in calls] == ["json", "jsonb"] + assert all(call.args[2] == "pg_catalog" for call in calls) + assert all(call.args[5:7] == (json.dumps, json.loads) for call in calls) + + +@pytest.mark.parametrize("kind", ["ordinary", "cancel", "exit", "keyboard"]) +@pytest.mark.parametrize("secondary", ["ordinary", "cancel", "exit", "keyboard"]) +async def test_secondary_quarantine_failure_keeps_first_and_reports_unknown(kind, secondary): + def error(which): + return { + "ordinary": ValueError("original codec failure"), + "cancel": asyncio.CancelledError("codec cancelled"), + "exit": SystemExit(17), + "keyboard": KeyboardInterrupt("codec interrupted"), + }[which] + + original = error(kind) + conn = Mock( + set_type_codec=AsyncMock(side_effect=original), terminate=Mock(side_effect=error(secondary)) + ) + if isinstance(original, (SystemExit, KeyboardInterrupt)): + with pytest.raises(directory._PoolInitFailure) as caught: + await directory._init(conn) + assert caught.value.error is original + else: + with pytest.raises(type(original)) as caught: + await directory._init(conn) + assert caught.value is original + assert original.__notes__ == ["directory initialization session closure is unknown"] + conn.terminate.assert_called_once_with() + conn.set_type_codec.assert_awaited_once() + + +async def test_diagnostic_failure_cannot_replace_original_native_failure(): + class BrokenNote(SystemExit): + def add_note(self, note): + raise KeyboardInterrupt("secondary diagnostic failure") + + original = BrokenNote(17) + conn = Mock( + set_type_codec=AsyncMock(side_effect=original), terminate=Mock(side_effect=RuntimeError) + ) + with pytest.raises(directory._PoolInitFailure) as caught: + await directory._init(conn) + assert caught.value.error is original + + +async def test_pool_has_no_autonomous_opening_and_keeps_existing_limits(monkeypatch): + import asyncpg + + opened = object() + create = AsyncMock(return_value=opened) + monkeypatch.setattr(directory, "_pool", None) + monkeypatch.setattr(directory, "_require_dsn", lambda: "private-test-dsn") + monkeypatch.setattr(asyncpg, "create_pool", create) + assert await directory.pool() is opened + assert await directory.pool() is opened + create.assert_awaited_once_with( + "private-test-dsn", + min_size=0, + max_size=10, + init=directory._init, + server_settings={ + **directory._SESSION, + "statement_timeout": "10000", + "idle_in_transaction_session_timeout": "20000", + }, + command_timeout=11, + timeout=10, + ) + + +@pytest.mark.parametrize("boundary", ["creation", "acquisition", "lookup"]) +@pytest.mark.parametrize("kind", ["exit", "keyboard"]) +async def test_every_pool_boundary_unwraps_identical_native_after_driver_cleanup( + monkeypatch, boundary, kind +): + from contextlib import asynccontextmanager + + import asyncpg + + original = SystemExit(17) if kind == "exit" else KeyboardInterrupt("codec interrupted") + events = [] + carrier = directory._PoolInitFailure(original) + monkeypatch.setattr(directory, "_pool", None) + monkeypatch.setattr(directory, "_require_dsn", lambda: "private-test-dsn") + + async def create(*args, **kwargs): + events.append("driver creation cleanup") + raise carrier + + @asynccontextmanager + async def acquire(): + try: + raise carrier + yield # pragma: no cover + finally: + events.append("driver acquisition cleanup") + + if boundary == "creation": + monkeypatch.setattr(asyncpg, "create_pool", create) + else: + monkeypatch.setattr(directory, "_pool", Mock(acquire=acquire)) + slots = directory._lookups._value + with pytest.raises(type(original)) as caught: + if boundary == "creation": + await directory.pool() + else: + async with directory._connection(lookup=boundary == "lookup"): + pytest.fail("failed initializer admitted a connection") + assert caught.value is original + assert events == [ + "driver creation cleanup" if boundary == "creation" else "driver acquisition cleanup" + ] + assert directory._lookups._value == slots From cdce8ec301c3e2446781be6add0145469de3aaa2 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 01:21:49 +0000 Subject: [PATCH 044/114] Keep owned cancellation alive through repeated signals Signed-off-by: zlareb1 (cherry picked from commit 96df6ab6bc91198f867d0c2acedcb0aca0e044c8) --- server/core/pg.py | 26 +++++- tests/integration/test_pg_owned_cleanup.py | 58 ++++++++++++ tests/unit/test_pg_close_owned.py | 101 ++++++++++++++++++++- 3 files changed, 179 insertions(+), 6 deletions(-) diff --git a/server/core/pg.py b/server/core/pg.py index 1e51975dfb..8f1d8192c7 100644 --- a/server/core/pg.py +++ b/server/core/pg.py @@ -402,23 +402,41 @@ async def close() -> SystemExit | KeyboardInterrupt | None: return None task: asyncio.Task[Any] | None = None + interrupted: asyncio.CancelledError | None = None try: remaining = end - time.monotonic() if remaining <= 0: raise TimeoutError("Postgres owned-session cleanup deadline expired") task = asyncio.create_task(close()) - done, _ = await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) - if not done: - raise TimeoutError("Postgres owned-session cleanup deadline expired") + while True: + remaining = end - time.monotonic() + if remaining <= 0: + raise TimeoutError("Postgres owned-session cleanup deadline expired") + try: + done, _ = await asyncio.wait({task}, timeout=remaining) + except asyncio.CancelledError as error: + # Keep cancellation delivery alive; another signal cannot reset its end. + interrupted = interrupted or error + continue + if not done: + raise TimeoutError("Postgres owned-session cleanup deadline expired") + break if isinstance(result := task.result(), (SystemExit, KeyboardInterrupt)): raise result except BaseException: - _terminate(conn) + try: + _terminate(conn) + except BaseException: # noqa: BLE001 — abort cannot replace the first cleanup failure. + pass if task is not None: if not task.done(): task.cancel() task.add_done_callback(_retrieve_task) + if interrupted is not None: + raise interrupted from None raise + if interrupted is not None: + raise interrupted async def _release(pool: Any, conn: Any) -> None: diff --git a/tests/integration/test_pg_owned_cleanup.py b/tests/integration/test_pg_owned_cleanup.py index 9085fce011..62a339774a 100644 --- a/tests/integration/test_pg_owned_cleanup.py +++ b/tests/integration/test_pg_owned_cleanup.py @@ -75,6 +75,64 @@ async def connect() -> AsyncIterator[Any]: await released(observer, pid) +@pytest.mark.parametrize("signals", [1, 2]) +async def test_owned_migrator_keeps_real_cancel_request_alive( + database, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, signals: int +) -> None: + from asyncpg import connect_utils + + conn, observer = database + pid = conn.get_server_pid() + cancelling, allow_cancel = asyncio.Event(), asyncio.Event() + real_cancel = connect_utils._cancel + monkeypatch.setattr(pg, "_RELEASE_TIMEOUT_S", 0.5) + + async def held_cancel(**kwargs: Any) -> None: + if kwargs["backend_pid"] == pid: + cancelling.set() + await allow_cancel.wait() + await real_cancel(**kwargs) + + monkeypatch.setattr(connect_utils, "_cancel", held_cancel) + (tmp_path / "001_sleep.sql").write_text("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10);") + + @contextlib.asynccontextmanager + async def connect() -> AsyncIterator[Any]: + yield conn + + task = asyncio.create_task(pg_migrate.migrate(tmp_path, connect=connect, schema="test")) + try: + await sleeping(observer, pid) + task.cancel("first signal") + await asyncio.wait_for(cancelling.wait(), 2) + if signals == 2: + task.cancel("second signal") + await asyncio.sleep(0.03) + assert not task.done(), ( + await observer.fetchval("SELECT wait_event FROM pg_stat_activity WHERE pid=$1", pid), + await observer.fetchval("SELECT count(*) FROM pg_locks WHERE pid=$1", pid), + ) + allow_cancel.set() + with pytest.raises(asyncio.CancelledError, match="first signal"): + await asyncio.wait_for(task, 0.5) + await released(observer, pid) + finally: + allow_cancel.set() + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + await observer.execute("SELECT pg_cancel_backend($1)", pid) + async with asyncio.timeout(2): + for _ in range(200): + if not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)", pid + ): + break + await asyncio.sleep(0.01) + else: + pytest.fail("owned backend did not end after teardown cancellation") + + @pytest.mark.parametrize("transaction_context", [False, True]) async def test_public_owned_close_releases_live_query_before_task_cancel( database, transaction_context: bool diff --git a/tests/unit/test_pg_close_owned.py b/tests/unit/test_pg_close_owned.py index 1ba7ab1f6a..eba8142aff 100644 --- a/tests/unit/test_pg_close_owned.py +++ b/tests/unit/test_pg_close_owned.py @@ -53,7 +53,102 @@ async def test_close_timeout_aborts_within_the_original_deadline() -> None: assert conn.aborted and time.monotonic() - started < 0.3 -async def test_second_cancellation_aborts_without_waiting_for_close_unwind() -> None: +async def test_outer_signals_do_not_abort_pending_public_close() -> None: + conn = Connection() + task = asyncio.create_task(pg.close_owned(conn, end=time.monotonic() + 0.5)) + try: + await conn.started.wait() + task.cancel("first close interruption") + await asyncio.sleep(0) + task.cancel("second close interruption") + await asyncio.sleep(0) + assert not task.done() + assert not conn.aborted + conn.finished.set() + with pytest.raises(asyncio.CancelledError, match="first close interruption"): + await task + assert not conn.aborted + assert len(conn.timeouts) == 1 + finally: + conn.finished.set() + await asyncio.gather(task, return_exceptions=True) + + +async def test_self_cancelled_close_child_does_not_spin_to_deadline() -> None: + class SelfCancelled(Connection): + async def close(self, **_: Any) -> None: + raise asyncio.CancelledError("close child cancelled") + + started = time.monotonic() + with pytest.raises(asyncio.CancelledError, match="close child cancelled"): + await pg.close_owned(SelfCancelled(), end=started + 10) + assert time.monotonic() - started < 0.3 + + +@pytest.mark.parametrize( + "later", + [ + ValueError("late close"), + asyncio.CancelledError("close child"), + SystemExit(17), + KeyboardInterrupt(), + ], +) +async def test_first_outer_cancel_survives_a_later_close_failure(later: BaseException) -> None: + class Failing(Connection): + async def close(self, **_: Any) -> None: + self.started.set() + await self.finished.wait() + raise later + + def is_closed(self) -> bool: + return True # An aborted transport cannot prove cancellation acknowledgement. + + conn = Failing() + + async def captured() -> BaseException | None: + try: + await pg.close_owned(conn, end=time.monotonic() + 10) + except BaseException as error: # noqa: BLE001 — keep native failures inside the test task. + return error + return None + + task = asyncio.create_task(captured()) + try: + await conn.started.wait() + task.cancel("first close interruption") + await asyncio.sleep(0) + assert not task.done() + assert not conn.aborted + started = time.monotonic() + conn.finished.set() + result = await task + assert isinstance(result, asyncio.CancelledError) + assert str(result) == "first close interruption" + assert time.monotonic() - started < 0.3 + assert conn.aborted + finally: + conn.finished.set() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("abort_error", [ValueError("abort"), SystemExit(17), KeyboardInterrupt()]) +async def test_abort_failure_cannot_replace_the_close_failure(abort_error: BaseException) -> None: + original = ValueError("original close failure") + + class Broken(Connection): + async def close(self, **_: Any) -> None: + raise original + + def terminate(self) -> None: + raise abort_error + + with pytest.raises(ValueError) as result: + await pg.close_owned(Broken(), end=time.monotonic() + 1) + assert result.value is original + + +async def test_second_cancellation_waits_for_the_original_cleanup_end() -> None: conn = Connection() supervisor_started = asyncio.Event() original = asyncio.CancelledError("operation cancelled") @@ -64,7 +159,7 @@ async def supervisor() -> None: await asyncio.Future() except asyncio.CancelledError: try: - await pg.close_owned(conn, end=time.monotonic() + 1) + await pg.close_owned(conn, end=time.monotonic() + 0.03) except BaseException: # noqa: BLE001 — preserve the original cancellation. pass raise original from None @@ -74,6 +169,8 @@ async def supervisor() -> None: task.cancel() await asyncio.wait_for(conn.started.wait(), 1) task.cancel() + await asyncio.sleep(0) + assert not conn.aborted with pytest.raises(asyncio.CancelledError, match="operation cancelled"): await asyncio.wait_for(task, 0.3) assert conn.aborted From 70c21f2cd4312a5d6c8d210ec89d73114d9fccf4 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 01:31:46 +0000 Subject: [PATCH 045/114] Migration fallback aborts preserve the first failure Signed-off-by: zlareb1 (cherry picked from commit 30a4ac5d4a0f01045db153af727ca11e4ed9066f) --- server/core/schema/pg_migrate.py | 10 +++- tests/unit/test_pg_close_owned.py | 89 +++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) diff --git a/server/core/schema/pg_migrate.py b/server/core/schema/pg_migrate.py index 1722da5a9b..a62b467c0c 100644 --- a/server/core/schema/pg_migrate.py +++ b/server/core/schema/pg_migrate.py @@ -152,7 +152,10 @@ async def _failed_migration( try: await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) except BaseException: # noqa: BLE001 — preserve the first failure on repeated cancellation. - pg._terminate(conn) + try: + pg._terminate(conn) + except BaseException: # noqa: BLE001 — fallback abort cannot replace the migration failure. + pass release = asyncio.create_task( _owned_result(lambda: cm.__aexit__(type(error), error, error.__traceback__)) ) @@ -162,7 +165,10 @@ async def _failed_migration( raise TimeoutError("migration context cleanup deadline expired") release.result() except BaseException: # noqa: BLE001 — a context exit cannot replace the migration failure. - pg._terminate(conn) + try: + pg._terminate(conn) + except BaseException: # noqa: BLE001 — fallback abort cannot replace the migration failure. + pass finally: if not release.done(): release.cancel() diff --git a/tests/unit/test_pg_close_owned.py b/tests/unit/test_pg_close_owned.py index eba8142aff..cb5abfa58f 100644 --- a/tests/unit/test_pg_close_owned.py +++ b/tests/unit/test_pg_close_owned.py @@ -314,3 +314,92 @@ async def work(*_: Any) -> list[str]: ) assert len(conn.timeouts) == 1 assert conn.aborted + + +@pytest.mark.parametrize("seam", ["reap", "exit"]) +@pytest.mark.parametrize("original_kind", ["ordinary", "cancel", "exit", "keyboard"]) +@pytest.mark.parametrize("abort_kind", ["ordinary", "cancel", "exit", "keyboard"]) +async def test_failed_migration_abort_preserves_first_and_finishes_context( + monkeypatch, tmp_path, seam, original_kind, abort_kind +): + from server.core.schema import pg_migrate + + def error(kind, message): + return { + "ordinary": ValueError(message), + "cancel": asyncio.CancelledError(message), + "exit": SystemExit(message), + "keyboard": KeyboardInterrupt(message), + }[kind] + + original, secondary = ( + error(original_kind, "migration failed"), + error(abort_kind, "abort failed"), + ) + events, first, exits, ends, retrieved, triples = [], [], [], [], [], [] + conn = Connection() + real_wait, real_failed, real_retrieve = ( + asyncio.wait, + pg_migrate._failed_migration, + pg._retrieve_task, + ) + + class Context: + async def __aenter__(self): + return conn + + async def __aexit__(self, kind, value, traceback): + exits.append((kind, value, traceback)) + events.append("exit") + if seam == "exit": + raise OSError("context exit failed") + + async def migrate(*args): + raise original + + async def close(_conn, *, end): + assert _conn is conn + ends.append(end) + events.append("close") + + async def failed(cm, actual_conn, task, observed, end): + first.append(observed) + triples.append((type(observed), observed, observed.__traceback__)) + return await real_failed(cm, actual_conn, task, observed, end) + + async def wait(tasks, *, timeout): # noqa: ASYNC109 — asyncio.wait test seam. + assert 0 <= timeout <= max(0, ends[0] - time.monotonic()) + 0.01 + if seam == "reap" and "reap" not in events: + events.append("reap") + raise asyncio.CancelledError("second signal during reap") + return await real_wait(tasks, timeout=timeout) + + def abort(): + events.append("abort") + raise secondary + + def retrieve(task): + retrieved.append(task) + real_retrieve(task) + + monkeypatch.setattr(pg_migrate, "_migrate", migrate) + monkeypatch.setattr(pg_migrate, "_failed_migration", failed) + monkeypatch.setattr(pg, "close_owned", close) + monkeypatch.setattr(conn, "terminate", abort) + monkeypatch.setattr(asyncio, "wait", wait) + monkeypatch.setattr(pg, "_retrieve_task", retrieve) + try: + await pg_migrate.migrate(tmp_path, connect=Context, schema="test") + except BaseException as observed: # noqa: BLE001 — native test failures must remain inert. + result = observed + else: + pytest.fail("failed migration returned success") + assert len(first) == 1 and result is first[0] + if original_kind != "cancel": + assert result is original + assert exits == triples + assert len(ends) == 1 and events.count("close") == events.count("abort") == 1 + assert events[-1] == ("exit" if seam == "reap" else "abort") + for _ in range(3): + await asyncio.sleep(0) + assert len(retrieved) == 2 and all(task.done() for task in retrieved) From 9d18614e60c404a998e8bb656a8d6f947259e96f Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 01:21:49 +0000 Subject: [PATCH 046/114] Keep owned cancellation alive through repeated signals Signed-off-by: zlareb1 (cherry picked from commit 96df6ab6bc91198f867d0c2acedcb0aca0e044c8) --- server/core/pg.py | 26 +++++- tests/integration/test_pg_owned_cleanup.py | 58 ++++++++++++ tests/unit/test_pg_close_owned.py | 101 ++++++++++++++++++++- 3 files changed, 179 insertions(+), 6 deletions(-) diff --git a/server/core/pg.py b/server/core/pg.py index e70c702582..f672c9edde 100644 --- a/server/core/pg.py +++ b/server/core/pg.py @@ -402,23 +402,41 @@ async def close() -> SystemExit | KeyboardInterrupt | None: return None task: asyncio.Task[Any] | None = None + interrupted: asyncio.CancelledError | None = None try: remaining = end - time.monotonic() if remaining <= 0: raise TimeoutError("Postgres owned-session cleanup deadline expired") task = asyncio.create_task(close()) - done, _ = await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) - if not done: - raise TimeoutError("Postgres owned-session cleanup deadline expired") + while True: + remaining = end - time.monotonic() + if remaining <= 0: + raise TimeoutError("Postgres owned-session cleanup deadline expired") + try: + done, _ = await asyncio.wait({task}, timeout=remaining) + except asyncio.CancelledError as error: + # Keep cancellation delivery alive; another signal cannot reset its end. + interrupted = interrupted or error + continue + if not done: + raise TimeoutError("Postgres owned-session cleanup deadline expired") + break if isinstance(result := task.result(), (SystemExit, KeyboardInterrupt)): raise result except BaseException: - _terminate(conn) + try: + _terminate(conn) + except BaseException: # noqa: BLE001 — abort cannot replace the first cleanup failure. + pass if task is not None: if not task.done(): task.cancel() task.add_done_callback(_retrieve_task) + if interrupted is not None: + raise interrupted from None raise + if interrupted is not None: + raise interrupted async def _release(pool: Any, conn: Any) -> None: diff --git a/tests/integration/test_pg_owned_cleanup.py b/tests/integration/test_pg_owned_cleanup.py index 9085fce011..62a339774a 100644 --- a/tests/integration/test_pg_owned_cleanup.py +++ b/tests/integration/test_pg_owned_cleanup.py @@ -75,6 +75,64 @@ async def connect() -> AsyncIterator[Any]: await released(observer, pid) +@pytest.mark.parametrize("signals", [1, 2]) +async def test_owned_migrator_keeps_real_cancel_request_alive( + database, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, signals: int +) -> None: + from asyncpg import connect_utils + + conn, observer = database + pid = conn.get_server_pid() + cancelling, allow_cancel = asyncio.Event(), asyncio.Event() + real_cancel = connect_utils._cancel + monkeypatch.setattr(pg, "_RELEASE_TIMEOUT_S", 0.5) + + async def held_cancel(**kwargs: Any) -> None: + if kwargs["backend_pid"] == pid: + cancelling.set() + await allow_cancel.wait() + await real_cancel(**kwargs) + + monkeypatch.setattr(connect_utils, "_cancel", held_cancel) + (tmp_path / "001_sleep.sql").write_text("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10);") + + @contextlib.asynccontextmanager + async def connect() -> AsyncIterator[Any]: + yield conn + + task = asyncio.create_task(pg_migrate.migrate(tmp_path, connect=connect, schema="test")) + try: + await sleeping(observer, pid) + task.cancel("first signal") + await asyncio.wait_for(cancelling.wait(), 2) + if signals == 2: + task.cancel("second signal") + await asyncio.sleep(0.03) + assert not task.done(), ( + await observer.fetchval("SELECT wait_event FROM pg_stat_activity WHERE pid=$1", pid), + await observer.fetchval("SELECT count(*) FROM pg_locks WHERE pid=$1", pid), + ) + allow_cancel.set() + with pytest.raises(asyncio.CancelledError, match="first signal"): + await asyncio.wait_for(task, 0.5) + await released(observer, pid) + finally: + allow_cancel.set() + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + await observer.execute("SELECT pg_cancel_backend($1)", pid) + async with asyncio.timeout(2): + for _ in range(200): + if not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)", pid + ): + break + await asyncio.sleep(0.01) + else: + pytest.fail("owned backend did not end after teardown cancellation") + + @pytest.mark.parametrize("transaction_context", [False, True]) async def test_public_owned_close_releases_live_query_before_task_cancel( database, transaction_context: bool diff --git a/tests/unit/test_pg_close_owned.py b/tests/unit/test_pg_close_owned.py index 1ba7ab1f6a..eba8142aff 100644 --- a/tests/unit/test_pg_close_owned.py +++ b/tests/unit/test_pg_close_owned.py @@ -53,7 +53,102 @@ async def test_close_timeout_aborts_within_the_original_deadline() -> None: assert conn.aborted and time.monotonic() - started < 0.3 -async def test_second_cancellation_aborts_without_waiting_for_close_unwind() -> None: +async def test_outer_signals_do_not_abort_pending_public_close() -> None: + conn = Connection() + task = asyncio.create_task(pg.close_owned(conn, end=time.monotonic() + 0.5)) + try: + await conn.started.wait() + task.cancel("first close interruption") + await asyncio.sleep(0) + task.cancel("second close interruption") + await asyncio.sleep(0) + assert not task.done() + assert not conn.aborted + conn.finished.set() + with pytest.raises(asyncio.CancelledError, match="first close interruption"): + await task + assert not conn.aborted + assert len(conn.timeouts) == 1 + finally: + conn.finished.set() + await asyncio.gather(task, return_exceptions=True) + + +async def test_self_cancelled_close_child_does_not_spin_to_deadline() -> None: + class SelfCancelled(Connection): + async def close(self, **_: Any) -> None: + raise asyncio.CancelledError("close child cancelled") + + started = time.monotonic() + with pytest.raises(asyncio.CancelledError, match="close child cancelled"): + await pg.close_owned(SelfCancelled(), end=started + 10) + assert time.monotonic() - started < 0.3 + + +@pytest.mark.parametrize( + "later", + [ + ValueError("late close"), + asyncio.CancelledError("close child"), + SystemExit(17), + KeyboardInterrupt(), + ], +) +async def test_first_outer_cancel_survives_a_later_close_failure(later: BaseException) -> None: + class Failing(Connection): + async def close(self, **_: Any) -> None: + self.started.set() + await self.finished.wait() + raise later + + def is_closed(self) -> bool: + return True # An aborted transport cannot prove cancellation acknowledgement. + + conn = Failing() + + async def captured() -> BaseException | None: + try: + await pg.close_owned(conn, end=time.monotonic() + 10) + except BaseException as error: # noqa: BLE001 — keep native failures inside the test task. + return error + return None + + task = asyncio.create_task(captured()) + try: + await conn.started.wait() + task.cancel("first close interruption") + await asyncio.sleep(0) + assert not task.done() + assert not conn.aborted + started = time.monotonic() + conn.finished.set() + result = await task + assert isinstance(result, asyncio.CancelledError) + assert str(result) == "first close interruption" + assert time.monotonic() - started < 0.3 + assert conn.aborted + finally: + conn.finished.set() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("abort_error", [ValueError("abort"), SystemExit(17), KeyboardInterrupt()]) +async def test_abort_failure_cannot_replace_the_close_failure(abort_error: BaseException) -> None: + original = ValueError("original close failure") + + class Broken(Connection): + async def close(self, **_: Any) -> None: + raise original + + def terminate(self) -> None: + raise abort_error + + with pytest.raises(ValueError) as result: + await pg.close_owned(Broken(), end=time.monotonic() + 1) + assert result.value is original + + +async def test_second_cancellation_waits_for_the_original_cleanup_end() -> None: conn = Connection() supervisor_started = asyncio.Event() original = asyncio.CancelledError("operation cancelled") @@ -64,7 +159,7 @@ async def supervisor() -> None: await asyncio.Future() except asyncio.CancelledError: try: - await pg.close_owned(conn, end=time.monotonic() + 1) + await pg.close_owned(conn, end=time.monotonic() + 0.03) except BaseException: # noqa: BLE001 — preserve the original cancellation. pass raise original from None @@ -74,6 +169,8 @@ async def supervisor() -> None: task.cancel() await asyncio.wait_for(conn.started.wait(), 1) task.cancel() + await asyncio.sleep(0) + assert not conn.aborted with pytest.raises(asyncio.CancelledError, match="operation cancelled"): await asyncio.wait_for(task, 0.3) assert conn.aborted From 0835f33b6b759fa22e8609125b6f016338cb18b6 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 01:31:46 +0000 Subject: [PATCH 047/114] Migration fallback aborts preserve the first failure Signed-off-by: zlareb1 (cherry picked from commit 30a4ac5d4a0f01045db153af727ca11e4ed9066f) --- server/core/schema/pg_migrate.py | 10 +++- tests/unit/test_pg_close_owned.py | 89 +++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) diff --git a/server/core/schema/pg_migrate.py b/server/core/schema/pg_migrate.py index 1722da5a9b..a62b467c0c 100644 --- a/server/core/schema/pg_migrate.py +++ b/server/core/schema/pg_migrate.py @@ -152,7 +152,10 @@ async def _failed_migration( try: await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) except BaseException: # noqa: BLE001 — preserve the first failure on repeated cancellation. - pg._terminate(conn) + try: + pg._terminate(conn) + except BaseException: # noqa: BLE001 — fallback abort cannot replace the migration failure. + pass release = asyncio.create_task( _owned_result(lambda: cm.__aexit__(type(error), error, error.__traceback__)) ) @@ -162,7 +165,10 @@ async def _failed_migration( raise TimeoutError("migration context cleanup deadline expired") release.result() except BaseException: # noqa: BLE001 — a context exit cannot replace the migration failure. - pg._terminate(conn) + try: + pg._terminate(conn) + except BaseException: # noqa: BLE001 — fallback abort cannot replace the migration failure. + pass finally: if not release.done(): release.cancel() diff --git a/tests/unit/test_pg_close_owned.py b/tests/unit/test_pg_close_owned.py index eba8142aff..cb5abfa58f 100644 --- a/tests/unit/test_pg_close_owned.py +++ b/tests/unit/test_pg_close_owned.py @@ -314,3 +314,92 @@ async def work(*_: Any) -> list[str]: ) assert len(conn.timeouts) == 1 assert conn.aborted + + +@pytest.mark.parametrize("seam", ["reap", "exit"]) +@pytest.mark.parametrize("original_kind", ["ordinary", "cancel", "exit", "keyboard"]) +@pytest.mark.parametrize("abort_kind", ["ordinary", "cancel", "exit", "keyboard"]) +async def test_failed_migration_abort_preserves_first_and_finishes_context( + monkeypatch, tmp_path, seam, original_kind, abort_kind +): + from server.core.schema import pg_migrate + + def error(kind, message): + return { + "ordinary": ValueError(message), + "cancel": asyncio.CancelledError(message), + "exit": SystemExit(message), + "keyboard": KeyboardInterrupt(message), + }[kind] + + original, secondary = ( + error(original_kind, "migration failed"), + error(abort_kind, "abort failed"), + ) + events, first, exits, ends, retrieved, triples = [], [], [], [], [], [] + conn = Connection() + real_wait, real_failed, real_retrieve = ( + asyncio.wait, + pg_migrate._failed_migration, + pg._retrieve_task, + ) + + class Context: + async def __aenter__(self): + return conn + + async def __aexit__(self, kind, value, traceback): + exits.append((kind, value, traceback)) + events.append("exit") + if seam == "exit": + raise OSError("context exit failed") + + async def migrate(*args): + raise original + + async def close(_conn, *, end): + assert _conn is conn + ends.append(end) + events.append("close") + + async def failed(cm, actual_conn, task, observed, end): + first.append(observed) + triples.append((type(observed), observed, observed.__traceback__)) + return await real_failed(cm, actual_conn, task, observed, end) + + async def wait(tasks, *, timeout): # noqa: ASYNC109 — asyncio.wait test seam. + assert 0 <= timeout <= max(0, ends[0] - time.monotonic()) + 0.01 + if seam == "reap" and "reap" not in events: + events.append("reap") + raise asyncio.CancelledError("second signal during reap") + return await real_wait(tasks, timeout=timeout) + + def abort(): + events.append("abort") + raise secondary + + def retrieve(task): + retrieved.append(task) + real_retrieve(task) + + monkeypatch.setattr(pg_migrate, "_migrate", migrate) + monkeypatch.setattr(pg_migrate, "_failed_migration", failed) + monkeypatch.setattr(pg, "close_owned", close) + monkeypatch.setattr(conn, "terminate", abort) + monkeypatch.setattr(asyncio, "wait", wait) + monkeypatch.setattr(pg, "_retrieve_task", retrieve) + try: + await pg_migrate.migrate(tmp_path, connect=Context, schema="test") + except BaseException as observed: # noqa: BLE001 — native test failures must remain inert. + result = observed + else: + pytest.fail("failed migration returned success") + assert len(first) == 1 and result is first[0] + if original_kind != "cancel": + assert result is original + assert exits == triples + assert len(ends) == 1 and events.count("close") == events.count("abort") == 1 + assert events[-1] == ("exit" if seam == "reap" else "abort") + for _ in range(3): + await asyncio.sleep(0) + assert len(retrieved) == 2 and all(task.done() for task in retrieved) From 723109d27c75236e2d719fd998ac7d3dbf6d165a Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 01:40:26 +0000 Subject: [PATCH 048/114] Compose native control ownership with bounded abort cleanup Signed-off-by: zlareb1 --- server/cli/cell_control.py | 95 ++++--- .../integration/test_cell_control_cleanup.py | 12 +- tests/unit/test_cell_attach_cleanup.py | 10 +- tests/unit/test_cell_control_abort.py | 185 ++++++++++++++ tests/unit/test_cell_place_owner.py | 233 ++++++++++++++++++ 5 files changed, 497 insertions(+), 38 deletions(-) create mode 100644 tests/unit/test_cell_control_abort.py create mode 100644 tests/unit/test_cell_place_owner.py diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index c4360839ed..1966b9e62b 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -119,7 +119,8 @@ async def _captured[T](work: Callable[[], Awaitable[T]]) -> T | _Fatal: class Owner: """One raw backend and one cleanup owner for a native control command.""" - def __init__(self) -> None: + def __init__(self, *, exit_s: float = pg._RELEASE_TIMEOUT_S) -> None: + self.exit_s = exit_s self.conn: Any = None self.phase_end: float | None = None self.close_end: float | None = None @@ -155,7 +156,7 @@ async def _open(self, what: str) -> None: async def close(self) -> None: if not self.closed: self.closed = True - self.close_end = self.phase_end or (time.monotonic() + pg._RELEASE_TIMEOUT_S) + self.close_end = self.phase_end or (time.monotonic() + self.exit_s) if self.conn is None: return if self._close_task is None: @@ -163,20 +164,33 @@ async def close(self) -> None: _captured(lambda: pg.close_owned(self.conn, end=cast(float, self.close_end))) ) self._close_task.add_done_callback(pg._retrieve_task) + interrupted: asyncio.CancelledError | None = None try: - done, _ = await asyncio.wait( - {self._close_task}, timeout=max(0, cast(float, self.close_end) - time.monotonic()) - ) - if not done: - raise TimeoutError("cell control cleanup deadline expired; outcome unknown") + while not self._close_task.done(): + remaining = cast(float, self.close_end) - time.monotonic() + if remaining <= 0: + raise TimeoutError("cell control cleanup deadline expired; outcome unknown") + try: + await asyncio.wait({self._close_task}, timeout=remaining) + except asyncio.CancelledError as error: + # Signals cannot abort the public cancellation already in flight. + interrupted = interrupted or error result = self._close_task.result() if isinstance(result, _Fatal): raise _ForwardFatal(result) except BaseException: - pg._terminate(self.conn) - if not self._close_task.done(): - self._close_task.cancel() + try: + pg._terminate(self.conn) + except BaseException: # noqa: BLE001 — abort cannot replace the first close failure. + pass + finally: + if not self._close_task.done(): + self._close_task.cancel() + if interrupted is not None: + raise interrupted from None raise + if interrupted is not None: + raise interrupted async def _stop(self, tasks: set[asyncio.Task[Any]]) -> None: for task in tasks: @@ -187,7 +201,10 @@ async def _stop(self, tasks: set[asyncio.Task[Any]]) -> None: await asyncio.wait(tasks, timeout=max(0, (self.close_end or 0) - time.monotonic())) except BaseException: # noqa: BLE001 — cleanup cannot replace the original failure. if self.conn is not None: - pg._terminate(self.conn) + try: + pg._terminate(self.conn) + except BaseException: # noqa: BLE001 — failed abort preserves the caller's failure. + pass async def call[T](self, work: Callable[[], Awaitable[T]], *, total_s: float) -> T: if self.closed or self.phase_end is not None: @@ -219,31 +236,43 @@ async def call[T](self, work: Callable[[], Awaitable[T]], *, total_s: float) -> async def transaction[T](self, work: Callable[[], Awaitable[T]]) -> T: return await self.call(lambda: transaction(self.conn, work), total_s=TRANSACTION_S) + async def run[T]( + self, + opening: Callable[[], Awaitable[None]], + work: Callable[[], Awaitable[T]], + *, + opening_s: float, + ) -> T: + """Own one command; fatal guard failures leave only after bounded cleanup.""" + if self.conn is not None or self.phase_end is not None or self._tasks or self.closed: + raise _refused("a command needs a fresh control owner") + try: + await self.call(opening, total_s=opening_s) + result = await asyncio.shield(self._start(work)) + if isinstance(result, _Fatal): + raise _ForwardFatal(result) + except BaseException as original: + try: + await self.close() + except BaseException: # noqa: BLE001 — preserve the command's original failure. + pass + await self._stop(set(self._tasks)) + if isinstance(original, _ForwardFatal): + raise original.fatal.error from None + raise + try: + await self.close() + except _ForwardFatal as interrupted: + raise interrupted.fatal.error from None + finally: + await self._stop(set(self._tasks)) + return result + async def run[T](what: str, work: Callable[[Owner], Awaitable[T]]) -> T: - """Run at the command boundary; fatal guard exceptions leave only after cleanup.""" + """Run native control with its existing opening, transaction and exit allowances.""" owner = Owner() - try: - await owner.call(lambda: owner._open(what), total_s=OPEN_S) - result = await asyncio.shield(owner._start(lambda: work(owner))) - if isinstance(result, _Fatal): - raise _ForwardFatal(result) - except BaseException as original: - try: - await owner.close() - except BaseException: # noqa: BLE001 — preserve the command's original failure. - pass - await owner._stop(set(owner._tasks)) - if isinstance(original, _ForwardFatal): - raise original.fatal.error from None - raise - try: - await owner.close() - except _ForwardFatal as interrupted: - raise interrupted.fatal.error from None - finally: - await owner._stop(set(owner._tasks)) - return result + return await owner.run(lambda: owner._open(what), lambda: work(owner), opening_s=OPEN_S) async def transaction[T](conn: Any, work: Callable[[], Awaitable[T]]) -> T: diff --git a/tests/integration/test_cell_control_cleanup.py b/tests/integration/test_cell_control_cleanup.py index e057fef323..8f68939b62 100644 --- a/tests/integration/test_cell_control_cleanup.py +++ b/tests/integration/test_cell_control_cleanup.py @@ -105,6 +105,8 @@ async def test_native_prepare_second_cancel_preserves_first_and_releases_locks( real_close = asyncpg.Connection.close observed, close_started, close_return = asyncio.Event(), asyncio.Event(), asyncio.Event() backend = {} + close_interrupted = asyncio.Event() + monkeypatch.setattr(cell_control, "TRANSACTION_S", 0.4) async def read(conn, **_): backend["conn"], backend["pid"] = conn, conn.get_server_pid() @@ -115,7 +117,11 @@ async def delayed_close(self, **kwargs): await real_close(self, **kwargs) if self is backend.get("conn"): close_started.set() - await close_return.wait() + try: + await close_return.wait() + except asyncio.CancelledError: + close_interrupted.set() + raise monkeypatch.setattr(cell_prepare, "_read_rows", read) monkeypatch.setattr(asyncpg.Connection, "close", delayed_close) @@ -128,6 +134,10 @@ async def delayed_close(self, **kwargs): task.cancel("first native cancellation") await asyncio.wait_for(close_started.wait(), 1) task.cancel("second cancellation") + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not close_interrupted.is_set(), "second signal cannot interrupt public close" + assert not task.done() with pytest.raises(asyncio.CancelledError, match="first native cancellation"): await asyncio.wait_for(task, 0.5) await released(native_database, backend["pid"]) diff --git a/tests/unit/test_cell_attach_cleanup.py b/tests/unit/test_cell_attach_cleanup.py index 11bf1bf094..a018784d21 100644 --- a/tests/unit/test_cell_attach_cleanup.py +++ b/tests/unit/test_cell_attach_cleanup.py @@ -8,7 +8,6 @@ import pytest from server.cli import cell_attach, cell_control -from server.core import pg from tests.unit.cell_control_owner import owner from tests.unit.test_cell_control import snapshot @@ -52,6 +51,9 @@ async def close(self, **kwargs): def terminate(self): events.append("abort") + def is_closed(self): + return False + async def work(): entered.set() try: @@ -78,7 +80,7 @@ async def work(): @pytest.mark.asyncio -async def test_lock_owner_bounds_close_inside_the_same_cleanup_allowance(monkeypatch): +async def test_lock_owner_bounds_close_inside_the_same_cleanup_allowance(): closing = asyncio.Event() conn = AsyncMock() conn.terminate = lambda: None @@ -88,8 +90,8 @@ async def close(**_): await asyncio.Future() conn.close.side_effect = close - owned = owner(conn) - monkeypatch.setattr(pg, "_RELEASE_TIMEOUT_S", 0.03) + owned = cell_control.Owner(exit_s=0.03) + owned.conn = conn started = time.monotonic() with pytest.raises(TimeoutError): await owned.close() diff --git a/tests/unit/test_cell_control_abort.py b/tests/unit/test_cell_control_abort.py new file mode 100644 index 0000000000..890ddcba27 --- /dev/null +++ b/tests/unit/test_cell_control_abort.py @@ -0,0 +1,185 @@ +"""A failed fallback abort cannot replace the native owner's first failure.""" + +from __future__ import annotations + +import asyncio +import time +from unittest.mock import AsyncMock + +import pytest + +from server.cli import cell_control + +pytestmark = pytest.mark.asyncio +ERRORS = (ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt) +ABORT_ERRORS = (RuntimeError, asyncio.CancelledError, SystemExit, KeyboardInterrupt) + + +async def outcome(work): + try: + result = await cell_control._captured(work) + except BaseException as error: # noqa: BLE001 — observe the identical native failure inertly. + return error + assert isinstance(result, cell_control._Fatal), "a failed required close cannot succeed" + return result.error + + +@pytest.mark.parametrize("first_kind", ERRORS) +@pytest.mark.parametrize("abort_kind", ABORT_ERRORS) +async def test_direct_owner_keeps_first_close_failure_and_retrieves_child( + monkeypatch, first_kind, abort_kind +): + first, secondary = first_kind("first close failure"), abort_kind("abort failure") + aborted = [] + + class Connection: + def terminate(self): + aborted.append(True) + raise secondary + + owner = cell_control.Owner(exit_s=0.2) + owner.conn = Connection() + close = AsyncMock(side_effect=first) + monkeypatch.setattr(cell_control.pg, "close_owned", close) + result = await outcome(owner.close) + assert result is first + assert aborted == [True] + close.assert_awaited_once() + assert owner.closed and owner._close_task.done() + assert owner._close_task.cancelled() or not owner._close_task._log_traceback + + +@pytest.mark.parametrize("abort_kind", ABORT_ERRORS) +@pytest.mark.parametrize("first_kind", ERRORS) +async def test_outer_signal_wins_over_later_close_and_abort_failures( + monkeypatch, first_kind, abort_kind +): + entered, finish = asyncio.Event(), asyncio.Event() + first, secondary = first_kind("close failed after signal"), abort_kind("abort failure") + seen = [] + actual_wait = asyncio.wait + owner = cell_control.Owner(exit_s=0.2) + + class Connection: + def terminate(self): + raise secondary + + async def close(conn, *, end): + entered.set() + await finish.wait() + raise first + + async def wait(*args, **kwargs): + try: + return await actual_wait(*args, **kwargs) + except asyncio.CancelledError as error: + seen.append(error) + raise + + owner.conn = Connection() + monkeypatch.setattr(cell_control.pg, "close_owned", close) + monkeypatch.setattr(cell_control.asyncio, "wait", wait) + task = asyncio.create_task(outcome(owner.close)) + try: + await entered.wait() + original_end = owner.close_end + task.cancel("original outer cancellation") + await asyncio.sleep(0) + assert len(seen) == 1 and not task.done() + finish.set() + result = await task + assert result is seen[0] + assert owner.close_end == original_end + assert owner._close_task.done() + assert owner._close_task.cancelled() or not owner._close_task._log_traceback + finally: + finish.set() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("abort_kind", ABORT_ERRORS) +async def test_abort_failure_still_cancels_and_retrieves_expired_close_child( + monkeypatch, abort_kind +): + entered, ended = asyncio.Event(), asyncio.Event() + secondary = abort_kind("abort failure") + + class Connection: + def terminate(self): + raise secondary + + async def close(conn, *, end): + entered.set() + try: + await asyncio.Future() + finally: + ended.set() + raise ValueError("late close failure") + + owner = cell_control.Owner(exit_s=0.02) + owner.conn = Connection() + monkeypatch.setattr(cell_control.pg, "close_owned", close) + started = time.monotonic() + result = await outcome(owner.close) + assert isinstance(result, TimeoutError) + assert time.monotonic() - started < 0.2 + assert entered.is_set() + await asyncio.wait_for(ended.wait(), 0.2) + await asyncio.sleep(0) + assert owner._close_task.done() and not owner._close_task._log_traceback + + +@pytest.mark.parametrize("abort_kind", ABORT_ERRORS) +async def test_self_cancelled_close_is_terminal_even_when_abort_fails(monkeypatch, abort_kind): + first = asyncio.CancelledError("close cancelled itself") + calls = [] + + class Connection: + def terminate(self): + calls.append("abort") + raise abort_kind("abort failure") + + async def close(conn, *, end): + calls.append("close") + raise first + + owner = cell_control.Owner(exit_s=60) + owner.conn = Connection() + monkeypatch.setattr(cell_control.pg, "close_owned", close) + started = time.monotonic() + result = await outcome(owner.close) + assert result is first + assert time.monotonic() - started < 0.2 + assert calls == ["close", "abort"] + + +@pytest.mark.parametrize("wait_kind", ERRORS) +@pytest.mark.parametrize("abort_kind", ABORT_ERRORS) +async def test_failed_reap_abort_cannot_replace_the_callers_first_failure( + monkeypatch, wait_kind, abort_kind +): + owner = cell_control.Owner() + owner.close_end = time.monotonic() + 0.2 + calls = [] + + class Connection: + def terminate(self): + calls.append("abort") + raise abort_kind("abort failed while reaping") + + async def failed_wait(*args, **kwargs): + raise wait_kind("reap wait interrupted") + + owner.conn = Connection() + child = owner._start(lambda: asyncio.Future()) + monkeypatch.setattr(cell_control.asyncio, "wait", failed_wait) + try: + try: + result = await cell_control._captured(lambda: owner._stop({child})) + except BaseException as error: # noqa: BLE001 — the probe must expose secondary native errors. + result = error + assert result is None, "reaping cannot replace an earlier operation failure" + assert calls == ["abort"] + finally: + child.cancel() + await asyncio.gather(child, return_exceptions=True) diff --git a/tests/unit/test_cell_place_owner.py b/tests/unit/test_cell_place_owner.py new file mode 100644 index 0000000000..2cce1d25cf --- /dev/null +++ b/tests/unit/test_cell_place_owner.py @@ -0,0 +1,233 @@ +"""Placement reuses the control owner without replacing its phase budgets.""" + +from __future__ import annotations + +import asyncio +import time +from unittest.mock import AsyncMock + +import pytest + +from server.cli import cell_control +from server.core import pg +from tests.unit.test_cell_attach import FakeConn + +pytestmark = pytest.mark.asyncio + + +async def test_custom_exit_is_used_only_outside_a_local_phase(): + owner = cell_control.Owner(exit_s=0.09) + conn = FakeConn() + + async def opening(): + owner.conn = conn + + before = time.monotonic() + assert await owner.run(opening, AsyncMock(return_value=17), opening_s=0.3) == 17 + assert before + 0.09 <= owner.close_end <= time.monotonic() + 0.09 + assert conn.executed == ["CLOSE"] + + +@pytest.mark.parametrize("occupied", ["conn", "phase", "task", "closed", "repeated"]) +async def test_command_entry_refuses_reuse_before_any_callback(occupied): + owner = cell_control.Owner() + held = None + if occupied == "conn": + owner.conn = FakeConn() + elif occupied == "phase": + owner.phase_end = time.monotonic() + 1 + elif occupied == "task": + held = owner._start(lambda: asyncio.Future()) + elif occupied == "closed": + await owner.close() + else: + await owner.run(AsyncMock(), AsyncMock(), opening_s=0.1) + opening, body = AsyncMock(), AsyncMock() + try: + with pytest.raises(cell_control.Refused, match="fresh"): + await owner.run(opening, body, opening_s=0.1) + opening.assert_not_called() + body.assert_not_called() + finally: + if held is not None: + held.cancel() + await asyncio.gather(held, return_exceptions=True) + + +@pytest.mark.parametrize("stage", ["opening", "body"]) +async def test_recursive_entry_cannot_replace_the_original_backend(stage): + owner = cell_control.Owner() + original = FakeConn() + other_open, other_body = AsyncMock(), AsyncMock() + + async def recurse(): + with pytest.raises(cell_control.Refused, match="fresh"): + await owner.run(other_open, other_body, opening_s=0.1) + + async def opening(): + if stage == "opening": + await recurse() + owner.conn = original + + async def body(): + if stage == "body": + await recurse() + return 3 + + assert await owner.run(opening, body, opening_s=0.1) == 3 + other_open.assert_not_called() + other_body.assert_not_called() + assert owner.conn is original and original.executed == ["CLOSE"] + + +async def test_native_wrapper_keeps_its_existing_three_budgets(monkeypatch): + observed = {} + actual_run = cell_control.Owner.run + actual_call = cell_control.Owner.call + + async def wrapped_run(self, opening, work, *, opening_s): + observed.update(opening=opening_s, exit=self.exit_s) + return await actual_run(self, opening, work, opening_s=opening_s) + + async def wrapped_call(self, work, *, total_s): + observed.setdefault("phases", []).append(total_s) + return await actual_call(self, work, total_s=total_s) + + async def opening(self, what): + assert what == "native" + self.conn = FakeConn() + + monkeypatch.setattr(cell_control.Owner, "run", wrapped_run) + monkeypatch.setattr(cell_control.Owner, "call", wrapped_call) + monkeypatch.setattr(cell_control.Owner, "_open", opening) + assert ( + await cell_control.run("native", lambda owner: owner.transaction(AsyncMock(return_value=7))) + == 7 + ) + assert observed == {"opening": 75.0, "exit": pg._RELEASE_TIMEOUT_S, "phases": [75.0, 60.0]} + + +async def test_failed_phase_keeps_its_end_instead_of_custom_exit(): + owner = cell_control.Owner(exit_s=99) + conn = FakeConn() + + async def opening(): + owner.conn = conn + + async def body(): + return await owner.call(lambda: asyncio.Future(), total_s=0.04) + + before = time.monotonic() + with pytest.raises(TimeoutError): + await owner.run(opening, body, opening_s=0.1) + assert before < owner.close_end <= before + 0.1 + assert owner.close_end == owner.phase_end + assert conn.executed == ["CLOSE"] + + +async def test_public_close_defers_repeated_signals_until_acknowledged_within_original_end(): + events = [] + closing, finish = asyncio.Event(), asyncio.Event() + + class Connection: + async def close(self, **kwargs): + events.append("close") + closing.set() + await finish.wait() + events.append("ack") + + def terminate(self): + events.append("abort") + + def is_closed(self): + return "ack" in events + + owner = cell_control.Owner(exit_s=0.2) + owner.conn = Connection() + task = asyncio.create_task(owner.close()) + try: + await closing.wait() + original_end = owner.close_end + task.cancel("first signal") + await asyncio.sleep(0) + task.cancel("second signal") + await asyncio.sleep(0) + assert not task.done(), "a second signal must not interrupt owned public close" + assert owner.close_end == original_end + finish.set() + with pytest.raises(asyncio.CancelledError, match="first signal"): + await task + assert events == ["close", "ack"] + finally: + finish.set() + await asyncio.gather(task, return_exceptions=True) + + +async def test_repeated_signals_cannot_extend_an_expired_public_close(): + closing = asyncio.Event() + conn = FakeConn() + + async def stalled(**kwargs): + closing.set() + await asyncio.Future() + + conn.close = stalled + owner = cell_control.Owner(exit_s=0.04) + owner.conn = conn + started = time.monotonic() + task = asyncio.create_task(owner.close()) + await closing.wait() + end = owner.close_end + + async def signals(): + while not task.done(): + task.cancel("first signal") + await asyncio.sleep(0.001) + + signal_task = asyncio.create_task(signals()) + try: + done, _ = await asyncio.wait({task}, timeout=0.2) + assert done, "repeated signals cannot restart the original cleanup allowance" + with pytest.raises(asyncio.CancelledError, match="first signal"): + await task + assert time.monotonic() - started < 0.2 + assert owner.close_end == end + assert "TERMINATE" in conn.executed + finally: + signal_task.cancel() + await asyncio.gather(signal_task, task, return_exceptions=True) + + +async def test_closed_transport_cannot_acknowledge_pending_public_cancellation(): + closing, acknowledged = asyncio.Event(), asyncio.Event() + events = [] + + class Connection: + async def close(self, **kwargs): + closing.set() + await acknowledged.wait() + events.append("ack") + + def is_closed(self): + # An aborted transport is closed even while its cancellation is pending. + return True + + def terminate(self): + events.append("abort") + + owner = cell_control.Owner(exit_s=0.2) + owner.conn = Connection() + task = asyncio.create_task(owner.close()) + try: + await closing.wait() + task.cancel("stop") + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not task.done() and events == [] + acknowledged.set() + with pytest.raises(asyncio.CancelledError, match="stop"): + await task + assert events == ["ack"] + finally: + acknowledged.set() + await asyncio.gather(task, return_exceptions=True) From f378a281cc5095a50b3ac08adff6138ed4e69f14 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 01:50:13 +0000 Subject: [PATCH 049/114] Match persisted vector precision and preserve attach cleanup errors Compare expected vectors at their BFloat16 storage precision. Keep the first attach failure when fallback abort or a cleanup diagnostic also fails. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 29 +++-- server/cli/cell_attach_pg.py | 10 +- tests/unit/test_bench_cell_trigger_gate.py | 107 ++++++++++++++++++ tests/unit/test_cell_attach_abort.py | 119 +++++++++++++++++++++ 4 files changed, 256 insertions(+), 9 deletions(-) create mode 100644 tests/unit/test_cell_attach_abort.py diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index b1874e1fd9..17c0230ef7 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -18,6 +18,7 @@ import socket import stat import statistics +import struct import subprocess import sys import time @@ -1035,20 +1036,31 @@ def _normal(value: Any) -> Any: return value +def _stored_vector(value: Any) -> list[float]: + _require( + isinstance(value, list) + and all(isinstance(x, (int, float)) and math.isfinite(x) for x in value), + "finite vector", + ) + try: + # The insert driver stores the high 16 bits of each Float32 component. + return [struct.unpack(" None: for field_name in fields: left, right = expected.get(field_name), actual.get(field_name) if field_name in ("embedding", "memory_key_embedding"): + left = _stored_vector(left) _require( - isinstance(left, list) - and isinstance(right, list) + isinstance(right, list) and len(left) == len(right) - and all( - math.isclose(a, b, rel_tol=1e-6, abs_tol=1e-7) - for a, b in zip(left, right, strict=True) - ), + and all(isinstance(x, (int, float)) and math.isfinite(x) for x in right) + and left == right, phase + " vector", ) else: @@ -1350,7 +1362,10 @@ async def _cleanup(triggers: Triggers | None, conn: Any, artifact: Artifact) -> except BaseException: if prior is None: raise - artifact.emit("connection_cleanup_error", error_type=type(sys.exception()).__name__) + try: + artifact.emit("connection_cleanup_error", error_type=type(sys.exception()).__name__) + except BaseException: # noqa: BLE001 — diagnostics cannot replace restoration failure. + pass async def _restore(triggers: Triggers, conn: Any, end: float) -> None: diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index ca6fedf794..271631265f 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -129,7 +129,10 @@ async def supervise[T](conn: Any, work: Coroutine[Any, Any, T], *, end: float) - try: await asyncio.wait({task}, timeout=max(0.0, end - time.monotonic())) except BaseException: # noqa: BLE001 — repeated cancellation preserves the first failure. - pg._terminate(conn) + try: + pg._terminate(conn) + except BaseException: # noqa: BLE001 — abort cannot replace the work failure. + pass raise @@ -169,9 +172,12 @@ async def close_context( if isinstance(result := task.result(), _Interrupted): raise result.error except BaseException as caught: # noqa: BLE001 — exit cannot replace the first failure. - pg._terminate(conn) if failure is None: failure = caught + try: + pg._terminate(conn) + except BaseException: # noqa: BLE001 — abort cannot replace the selected failure. + pass finally: if not task.done(): task.cancel() diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py index 29dcce134b..824808214a 100644 --- a/tests/unit/test_bench_cell_trigger_gate.py +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -461,6 +461,52 @@ def test_stored_payload_vector_version_and_audit_mismatch_refuse(field, expected gate._compare({field: expected}, {field: actual}, [field], "stored") +def stored_bfloat16(vector): + """Exercise the installed insert writer and query reader without a server.""" + import array + + from clickhouse_connect.datatypes.base import TypeDef + from clickhouse_connect.datatypes.numeric import BFloat16 + + class Encoded: + def __init__(self, raw): + self.raw = raw + + def read_array(self, kind, count): + result = array.array(kind) + result.frombytes(self.raw) + assert len(result) == count + return result + + kind = BFloat16(TypeDef()) + raw = bytearray() + kind._write_column_binary(vector, raw, SimpleNamespace(column_name="embedding")) + return kind._read_column_binary( + Encoded(raw), len(vector), SimpleNamespace(use_numpy=False), None + ) + + +@pytest.mark.parametrize("field", ["embedding", "memory_key_embedding"]) +def test_storage_vector_matches_installed_bfloat16_and_rejects_corrupted_neighbor(field): + from server.embedding.fake import FakeNGramEmbedder + + vector = FakeNGramEmbedder(dim=1024)._embed_raw(gate._payload("gate-review-owned-1")) + stored = stored_bfloat16(vector) + assert sum(a != b for a, b in zip(vector, stored, strict=True)) > 0 + gate._compare({field: vector}, {field: stored}, [field], "stored") + import struct + + corrupted = list(stored) + index = next(i for i, value in enumerate(stored) if value > 0) + bits = struct.unpack(" Date: Tue, 17 Nov 2026 02:10:47 +0000 Subject: [PATCH 050/114] WIP: Preserve selected driver errors through artifact diagnostics Secondary reporting failures retain the selected work or cleanup failure. Required output failures still refuse an otherwise successful run. The internal artifact-close matrix and final focused gates remain pending. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 20 +- .../test_cell_trigger_gate_diagnostics.py | 233 ++++++++++++++++++ 2 files changed, 243 insertions(+), 10 deletions(-) create mode 100644 tests/unit/test_cell_trigger_gate_diagnostics.py diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 17c0230ef7..15f9c258ff 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -1340,9 +1340,8 @@ async def _run( except BaseException as error: try: artifact.emit("cleanup_error", error_type=type(error).__name__) - except (OSError, ValueError): - if prior is None: - raise + except BaseException: # noqa: BLE001 — diagnostics cannot replace work or cleanup failure. + pass if prior is None: raise if interrupted and prior is None: @@ -1449,18 +1448,19 @@ async def run(ch: Any) -> dict[str, Any]: artifact.emit( "invalid", phase=phase, reason=reason, error_type=type(error).__name__ ) - except (OSError, ValueError): + except BaseException: # noqa: BLE001 — secondary output cannot replace INVALID. pass finally: if artifact is not None: try: artifact.close() - except (OSError, ValueError) as error: - verdict = { - "verdict": "INVALID", - "error_type": type(error).__name__, - "phase": "artifact_close", - } + except BaseException as error: # noqa: BLE001 — required close fails without losing INVALID. + if verdict["verdict"] != "INVALID": + verdict = { + "verdict": "INVALID", + "error_type": type(error).__name__, + "phase": "artifact_close", + } print(json.dumps(verdict, sort_keys=True, allow_nan=False), flush=True) return 0 if verdict["verdict"] == "PASS" else 1 diff --git a/tests/unit/test_cell_trigger_gate_diagnostics.py b/tests/unit/test_cell_trigger_gate_diagnostics.py new file mode 100644 index 0000000000..b2f4594ef8 --- /dev/null +++ b/tests/unit/test_cell_trigger_gate_diagnostics.py @@ -0,0 +1,233 @@ +"""Artifact diagnostics retain the first driver failure at the actual owner.""" + +import argparse +import asyncio +import json +from unittest.mock import AsyncMock + +import pytest + +from bench import cell_trigger_gate as gate +from server.cli import cell_attach_pg as apg + + +@pytest.fixture +def run_driver(monkeypatch, tmp_path): + from server.core import background + + def configure(*, prior=None, cleanup=None, output=None, event="cleanup_error"): + events = [] + + class Conn: + async def close(self, **kwargs): + events.append("public close") + + def terminate(self): + events.append("abort") + + class Triggers: + tables = ("memory_current",) + invariant = org = "digest" + + async def set(self, *args): + pass + + async def restore(self, *args): + events.append("restoration") + if cleanup is not None: + raise cleanup + + artifact = gate.Artifact(str(tmp_path / "raw.jsonl")) + original = artifact.file + + class File: + def write(self, line): + kind = json.loads(line)["kind"] + events.append(kind) + if kind == event and output is not None: + raise output + return original.write(line) + + def __getattr__(self, name): + return getattr(original, name) + + artifact.file = File() + monkeypatch.setattr(gate.asyncpg, "connect", AsyncMock(return_value=Conn())) + monkeypatch.setattr(gate, "_proof", AsyncMock(return_value={})) + monkeypatch.setattr(gate.Triggers, "capture", AsyncMock(return_value=Triggers())) + monkeypatch.setattr(gate, "_memory_set", AsyncMock(side_effect=prior)) + if prior is None: + monkeypatch.setattr(gate, "_arm_order", lambda: ()) + monkeypatch.setattr(gate, "_statistics", lambda samples: {}) + monkeypatch.setattr(background, "drain_task_sets", AsyncMock(return_value=True)) + monkeypatch.setattr(background, "flush_registered_buffers", AsyncMock(return_value=True)) + + async def run(): + try: + return await gate._run( + argparse.Namespace(pg_dsn="unused"), None, artifact, object() + ) + finally: + artifact.close() + + return run, events + + return configure + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "prior_kind", [None, ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] +) +@pytest.mark.parametrize( + "output_kind", + [None, OSError, RuntimeError, asyncio.CancelledError, SystemExit, KeyboardInterrupt], +) +async def test_outer_cleanup_diagnostic_preserves_selected_first_error( + run_driver, prior_kind, output_kind +): + prior = prior_kind("work failed first") if prior_kind is not None else None + cleanup = ValueError("cleanup failed") + output = output_kind("diagnostic failed") if output_kind is not None else None + run, events = run_driver(prior=prior, cleanup=cleanup, output=output) + + async def owner(): + try: + await run() + except BaseException as error: # noqa: BLE001 — assert the actual owner failure. + assert error is (prior if prior is not None else cleanup) + assert events.count("cleanup_error") == 1 + assert events.index("public close") < events.index("cleanup_error") + return + raise AssertionError("selected failure was swallowed") + + assert await apg._work_result(owner) is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("event", ["provisional_statistics", "triggers_restored"]) +@pytest.mark.parametrize( + "output_kind", [OSError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] +) +async def test_required_output_failure_after_successful_work_still_fails( + run_driver, event, output_kind +): + first = output_kind("required artifact failed") + run, events = run_driver(output=first, event=event) + + async def owner(): + try: + await run() + except BaseException as error: # noqa: BLE001 — native errors stay in their owner. + assert error is first + assert "restoration" in events and "public close" in events + return + raise AssertionError("required output failure was swallowed") + + assert await apg._work_result(owner) is None + + +@pytest.mark.asyncio +async def test_successful_driver_and_cleanup_output_returns_normally(run_driver): + run, events = run_driver() + assert await run() == {} + assert events == ["provisional_statistics", "restoration", "triggers_restored", "public close"] + + +@pytest.mark.parametrize("phase", ["invalid", "close"]) +@pytest.mark.parametrize( + "prior_kind", [None, ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] +) +@pytest.mark.parametrize( + "output_kind", + [None, OSError, RuntimeError, asyncio.CancelledError, SystemExit, KeyboardInterrupt], +) +def test_main_preserves_first_invalid_and_requires_successful_artifact_close( + monkeypatch, tmp_path, capsys, phase, prior_kind, output_kind +): + import sys + + import server.settings + import server.wiring + from server.cli import _ch + from tests.unit.test_bench_cell_trigger_gate import args, fixture_config + + fixture, config = fixture_config() + prior = prior_kind("first native work failure") if prior_kind is not None else None + output = output_kind("secondary artifact failure") if output_kind is not None else None + events = [] + real_artifact = gate.Artifact + + class Artifact(real_artifact): + def __init__(self, path): + super().__init__(path) + original = self.file + + class File: + def write(self, line): + kind = json.loads(line)["kind"] + events.append(kind) + if phase == "invalid" and kind == "invalid" and output is not None: + raise output + return original.write(line) + + def close(self): + events.append("file closed") + return original.close() + + def __getattr__(self, name): + return getattr(original, name) + + self.file = File() + + argv = ["driver"] + for key, value in vars( + args(fixture_manifest="unused", output=str(tmp_path / "raw.jsonl")) + ).items(): + argv += ["--" + key.replace("_", "-"), str(value)] + monkeypatch.setattr(sys, "argv", argv) + monkeypatch.setattr(server.settings, "settings", config) + monkeypatch.setattr(server.wiring, "wire", lambda: None) + monkeypatch.setattr(gate.Fixture, "read", lambda _: fixture) + monkeypatch.setattr(gate, "_source", lambda _: {}) + monkeypatch.setattr(gate, "_launch", lambda *_: {}) + monkeypatch.setattr(gate, "_preflight", AsyncMock()) + monkeypatch.setattr(gate, "Artifact", Artifact) + + def native(*_args, **_kwargs): + if prior is not None: + raise prior + return { + "latency_within_limit": True, + "throughput_within_limit": True, + "p99_delta_ms": 0.0, + "throughput_loss": 0.0, + } + + def sync(_fd): + assert capsys.readouterr().out == "" + events.append("file sync") + if phase == "close" and output is not None: + raise output + + monkeypatch.setattr(_ch, "run_connected", native) + monkeypatch.setattr(gate.os, "fsync", sync) + invalid = prior is not None or (phase == "close" and output is not None) + try: + result = gate.main() + except BaseException as error: # noqa: BLE001 — an escaped fatal must fail this case only. + pytest.fail(f"main escaped {type(error).__name__}") + assert result == int(invalid) + verdict = json.loads(capsys.readouterr().out) + assert verdict["verdict"] == ("INVALID" if invalid else "PASS") + if prior is not None: + assert verdict["error_type"] == prior_kind.__name__ + assert verdict["phase"] == "native_driver" + elif invalid: + assert verdict["error_type"] == output_kind.__name__ + assert verdict["phase"] == "artifact_close" + assert events[-2:] == ["file sync", "file closed"] + assert not any( + json.loads(line).get("verdict") == "PASS" + for line in (tmp_path / "raw.jsonl").read_text().splitlines() + ) From 15f842af44fb49d67a320877f479bfbceac38c44 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 02:21:26 +0000 Subject: [PATCH 051/114] Preserve artifact sync failures through file close A secondary file-close error must retain the first flush or sync failure. A first close failure after successful sync still fails the run. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 8 ++- .../test_cell_trigger_gate_artifact_close.py | 60 +++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test_cell_trigger_gate_artifact_close.py diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 15f9c258ff..ed7f59ec54 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -400,7 +400,13 @@ def close(self) -> None: try: self.file.flush() os.fsync(self.file.fileno()) - finally: + except BaseException: # Retain the flush or sync failure through final close. + try: + self.file.close() + except BaseException: # noqa: BLE001 — close cannot replace failed artifact sync. + pass + raise + else: self.file.close() diff --git a/tests/unit/test_cell_trigger_gate_artifact_close.py b/tests/unit/test_cell_trigger_gate_artifact_close.py new file mode 100644 index 0000000000..3234dc45f5 --- /dev/null +++ b/tests/unit/test_cell_trigger_gate_artifact_close.py @@ -0,0 +1,60 @@ +"""Artifact close retains its first flush, sync or file-close failure.""" + +import asyncio + +import pytest + +from bench import cell_trigger_gate as gate + + +@pytest.mark.parametrize("phase", ["flush", "fsync"]) +@pytest.mark.parametrize( + "first_kind", [None, OSError, ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] +) +@pytest.mark.parametrize( + "second_kind", + [None, OSError, ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt], +) +def test_actual_artifact_close_retains_first_flush_sync_or_close_failure( + monkeypatch, tmp_path, phase, first_kind, second_kind +): + first = first_kind("flush or sync failed first") if first_kind is not None else None + second = second_kind("file close failed second") if second_kind is not None else None + artifact = gate.Artifact(str(tmp_path / "raw.jsonl")) + artifact.emit("control") + original = artifact.file + events = [] + + class File: + def flush(self): + events.append("flush") + if phase == "flush" and first is not None: + raise first + return original.flush() + + def close(self): + events.append("file close") + original.close() + if second is not None: + raise second + + def __getattr__(self, name): + return getattr(original, name) + + def sync(_fd): + events.append("fsync") + if phase == "fsync" and first is not None: + raise first + + artifact.file = File() + monkeypatch.setattr(gate.os, "fsync", sync) + selected = first if first is not None else second + if selected is None: + artifact.close() + else: + with pytest.raises(BaseException) as observed: + artifact.close() + assert observed.value is selected + assert original.closed and events[-1] == "file close" + assert events.count("file close") == 1 + assert ("fsync" in events) == (phase != "flush" or first is None) From a9580c1e0193bc58f0c1ea089652a542b2e001bf Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 02:49:01 +0000 Subject: [PATCH 052/114] The cell trigger driver retains its first failure through cleanup Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 110 ++++-- tests/unit/test_bench_cell_trigger_gate.py | 6 +- .../test_cell_trigger_gate_first_error.py | 321 ++++++++++++++++++ 3 files changed, 408 insertions(+), 29 deletions(-) create mode 100644 tests/unit/test_cell_trigger_gate_first_error.py diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index ed7f59ec54..32803bd38d 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -455,7 +455,9 @@ async def _health( ) -> None: from server.settings import settings - async with httpx.AsyncClient(timeout=IO_S, follow_redirects=False, trust_env=False) as client: + context = httpx.AsyncClient(timeout=IO_S, follow_redirects=False, trust_env=False) + client = await context.__aenter__() + try: health = await client.get(f"{args.base_url.rstrip('/')}/v1/health") deployment = await client.get(f"{args.base_url.rstrip('/')}/v1/health/deployment") _require(health.status_code == deployment.status_code == 200, "health HTTP status") @@ -492,6 +494,14 @@ async def _health( ) launch = launch if launch is not None else _launch(args, fixture, settings) _require(h.get("instance") == launch["health_instance"], "serving owned process") + except BaseException as error: + try: + await context.__aexit__(type(error), error, error.__traceback__) + except BaseException: # noqa: BLE001 — HTTP cleanup cannot replace work failure. + pass + raise + else: + await context.__aexit__(None, None, None) def _object(value: Any) -> dict[str, Any]: @@ -602,6 +612,7 @@ async def _proof( async def _preflight(args: argparse.Namespace, fixture: Fixture) -> None: conn = ch = None + failure: BaseException | None = None try: conn = await asyncpg.connect( args.pg_dsn, @@ -612,13 +623,22 @@ async def _preflight(args: argparse.Namespace, fixture: Fixture) -> None: ch = await ReadOnlyCH.open() async with asyncio.timeout(IO_S): await _proof(conn, ch, args, fixture) - finally: - try: - if ch is not None: - await ch.close() - finally: - if conn is not None: - await _close_pg(conn) + except BaseException as error: # noqa: BLE001 — cleanup must retain any preflight failure. + failure = error + try: + if ch is not None: + await ch.close() + except BaseException as error: # noqa: BLE001 — both metadata clients must be closed. + if failure is None: + failure = error + try: + if conn is not None: + await _close_pg(conn) + except BaseException as error: # noqa: BLE001 — preserve the first work or close failure. + if failure is None: + failure = error + if failure is not None: + raise failure async def _debt(conn: Any, expected: int) -> None: @@ -681,7 +701,9 @@ async def verify(self, conn: Any) -> None: async def set(self, conn: Any, on: bool) -> None: await self.verify(conn) - async with conn.transaction(): + transaction = conn.transaction() + await transaction.start() + try: for table in self.tables: if table == "tenancy": continue @@ -690,18 +712,34 @@ async def set(self, conn: Any, on: bool) -> None: await conn.execute(f'DROP TRIGGER IF EXISTS {name} ON loom."{table}"') if on: await conn.execute(cell_triggers.create_statement(table, name)) + except BaseException: + try: + await transaction.rollback() + except BaseException: # noqa: BLE001 — rollback cannot replace the DDL failure. + pass + raise + await transaction.commit() self.on = on await self.verify(conn) async def restore(self, conn: Any) -> None: # Recreate the captured exact org guards even after a partial failed toggle. - async with conn.transaction(): + transaction = conn.transaction() + await transaction.start() + try: for table in self.tables: if table == "tenancy": continue for name in cell_triggers.ORG_TRIGGERS: await conn.execute(f'DROP TRIGGER IF EXISTS {name} ON loom."{table}"') await conn.execute(cell_triggers.create_statement(table, name)) + except BaseException: + try: + await transaction.rollback() + except BaseException: # noqa: BLE001 — rollback cannot replace the DDL failure. + pass + raise + await transaction.commit() self.on = True await self.verify(conn) _require(not await cell_triggers.missing(conn), "restored canonical triggers") @@ -772,12 +810,14 @@ async def _memory_set( ) -> list[RequestRow]: rows = [] prefix = f"gate-{uuid.uuid4().hex}-{arm}" - async with httpx.AsyncClient( + context = httpx.AsyncClient( timeout=IO_S, headers={"Authorization": f"Bearer {args.token}"}, follow_redirects=False, trust_env=False, - ) as client: + ) + client = await context.__aenter__() + try: for i in range(CALLS + WARMUP): key = f"{prefix}-{i}" value = _payload(key) @@ -843,18 +883,29 @@ async def _memory_set( code = candidate except (ValueError, TypeError, AttributeError): pass - emit( - "request_error", - arm=arm, - index=i, - key=key, - measured=i >= WARMUP, - elapsed_ms=elapsed, - status=response.status_code if response is not None else None, - code=code, - error_type=type(error).__name__, - ) + try: + emit( + "request_error", + arm=arm, + index=i, + key=key, + measured=i >= WARMUP, + elapsed_ms=elapsed, + status=response.status_code if response is not None else None, + code=code, + error_type=type(error).__name__, + ) + except BaseException: # noqa: BLE001 — diagnostics cannot replace the request failure. + pass raise + except BaseException as error: + try: + await context.__aexit__(type(error), error, error.__traceback__) + except BaseException: # noqa: BLE001 — HTTP cleanup cannot replace work failure. + pass + raise + else: + await context.__aexit__(None, None, None) return rows @@ -997,6 +1048,7 @@ async def _delivery( results: list[dict[str, Any]] = [] _index = 0 + failure: BaseException | None = None with customer.for_org(args.org, namespace=args.namespace, subject=SUBJECT): started = time.perf_counter_ns() try: @@ -1009,12 +1061,20 @@ async def _delivery( "real worker claimed/published/failure", ) except BaseException as error: - emit("worker_error", arm=arm, index=_index, error_type=type(error).__name__) + failure = error + try: + emit("worker_error", arm=arm, index=_index, error_type=type(error).__name__) + except BaseException: # noqa: BLE001 — diagnostics cannot replace the worker failure. + pass raise finally: elapsed = (time.perf_counter_ns() - started) / 1_000_000_000 for result_index, outcome in enumerate(results): - emit("worker", arm=arm, index=result_index, **outcome) + try: + emit("worker", arm=arm, index=result_index, **outcome) + except BaseException: # Prior results cannot replace failed work. + if failure is None: + raise _require(math.isfinite(elapsed) and elapsed > 0, "positive worker block elapsed") rps = BATCHES * batch / elapsed emit("delivery_block", arm=arm, elapsed_seconds=elapsed, rows=BATCHES * batch, rps=rps) diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py index 824808214a..a625ab5a27 100644 --- a/tests/unit/test_bench_cell_trigger_gate.py +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -8,7 +8,6 @@ import json import socket import uuid -from contextlib import asynccontextmanager from types import SimpleNamespace from unittest.mock import AsyncMock @@ -513,9 +512,8 @@ def __init__(self): self.drift = False self.statements = [] - @asynccontextmanager - async def transaction(self): - yield + def transaction(self): + return SimpleNamespace(start=AsyncMock(), rollback=AsyncMock(), commit=AsyncMock()) async def execute(self, sql): self.statements.append(sql) diff --git a/tests/unit/test_cell_trigger_gate_first_error.py b/tests/unit/test_cell_trigger_gate_first_error.py new file mode 100644 index 0000000000..7a3ccdf41b --- /dev/null +++ b/tests/unit/test_cell_trigger_gate_first_error.py @@ -0,0 +1,321 @@ +"""Driver work retains its error through secondary cleanup and artifact failures.""" + +import asyncio +import json +import time +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import httpx +import pytest +from asyncpg.transaction import Transaction + +from bench import cell_trigger_gate as gate +from tests.unit.test_bench_cell_trigger_gate import args, fixture_config + +FIRST = [None, ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] +SECOND = [None, OSError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] + + +def failure(kind): + return None if kind is None else kind("injected failure") + + +async def observed(work): + try: + await work() + except BaseException as error: # noqa: BLE001 — native errors stay inside this test. + return error + return None + + +@pytest.fixture(autouse=True) +def no_external_io(monkeypatch): + async def forbidden(*_args, **_kwargs): + raise AssertionError("unexpected external I/O") + + monkeypatch.setattr(gate.asyncpg, "connect", forbidden) + monkeypatch.setattr(gate.clickhouse_connect, "get_async_client", forbidden) + monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", forbidden) + + +@pytest.fixture +def artifact_output(monkeypatch, tmp_path): + artifacts = [] + + def configure(errors): + artifact = gate.Artifact(str(tmp_path / f"raw-{len(artifacts)}.jsonl")) + artifacts.append(artifact) + original = artifact.file + events = [] + + class File: + def write(self, line): + kind = json.loads(line)["kind"] + events.append(kind) + if (error := errors.get(kind)) is not None: + raise error + return original.write(line) + + def __getattr__(self, name): + return getattr(original, name) + + artifact.file = File() + return artifact, events + + yield configure + for artifact in artifacts: + artifact.close() + + +@pytest.fixture +def http_edges(monkeypatch): + import server.settings + + fixture, config = fixture_config() + monkeypatch.setattr(server.settings, "settings", config) + monkeypatch.setattr(gate, "CALLS", 1) + monkeypatch.setattr(gate, "WARMUP", 0) + client_class = httpx.AsyncClient + + def configure(first=None, closing=None, *, bad_status=False): + events = [] + + class Transport(httpx.AsyncBaseTransport): + async def handle_async_request(self, request): + events.append("request") + if first is not None: + raise first + if request.url.path.endswith("/memory.set"): + sent = json.loads(request.content) + body = { + "memory_id": "00000000-0000-0000-0000-000000000001", + "version": 1, + "status": "active", + "memory_key": sent["memory_key"], + "namespace": sent["namespace"], + "content_sha256": gate.sha256_value(sent["value"]), + } + elif request.url.path.endswith("/deployment"): + body = {"deployment_id": fixture.deployment_id} + else: + body = { + "build_commit": fixture.build_commit, + "deployment_config": {"instance_id": fixture.instance_id}, + "ch_ok": True, + "pg_ok": True, + "status": "ok", + "fake_embedding": True, + "fake_llm": True, + "pending_migrations": [], + "storage_authority": "postgres", + "llm_provider": config.llm_provider, + "llm_model": config.llm_model, + "embedding_model": config.embedding_model, + "embedding_dim_configured": config.embedding_dim, + "embedding_dim_probed": config.embedding_dim, + "instance": "test:1", + } + return httpx.Response(503 if bad_status else 200, json=body) + + async def aclose(self): + events.append("http close") + if closing is not None: + raise closing + + def client(**kwargs): + return client_class(**kwargs, transport=Transport()) + + monkeypatch.setattr(gate.httpx, "AsyncClient", client) + return fixture, events + + return configure + + +@pytest.mark.asyncio +@pytest.mark.parametrize("first_kind", FIRST) +@pytest.mark.parametrize("ch_kind", SECOND) +@pytest.mark.parametrize("pg_kind", SECOND) +async def test_preflight_retains_first_error_and_attempts_both_closes( + monkeypatch, first_kind, ch_kind, pg_kind +): + first, ch_error, pg_error = map(failure, (first_kind, ch_kind, pg_kind)) + events = [] + + async def proof(*_args): + events.append("proof") + if first is not None: + raise first + + async def close_ch(): + events.append("ch close") + if ch_error is not None: + raise ch_error + + async def close_pg(*_args): + events.append("pg close") + if pg_error is not None: + raise pg_error + + monkeypatch.setattr(gate.asyncpg, "connect", AsyncMock(return_value=object())) + monkeypatch.setattr( + gate.ReadOnlyCH, "open", AsyncMock(return_value=SimpleNamespace(close=close_ch)) + ) + monkeypatch.setattr(gate, "_proof", proof) + monkeypatch.setattr(gate, "_close_pg", close_pg) + assert await observed(lambda: gate._preflight(args(), None)) is (first or ch_error or pg_error) + assert events == ["proof", "ch close", "pg close"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("consumer", ["health", "memory"]) +@pytest.mark.parametrize("first_kind", FIRST) +@pytest.mark.parametrize("close_kind", SECOND) +async def test_http_client_exit_retains_work_failure_and_requires_successful_close( + http_edges, consumer, first_kind, close_kind +): + first, closing = failure(first_kind), failure(close_kind) + fixture, events = http_edges(first, closing) + + async def work(): + if consumer == "health": + await gate._health(args(), fixture, {"health_instance": "test:1"}) + else: + await gate._memory_set(args()) + + assert await observed(work) is (first or closing) + assert events[-1] == "http close" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("consumer", ["health", "memory"]) +async def test_semantic_http_refusal_survives_client_close_error(http_edges, consumer): + fixture, events = http_edges(closing=OSError("close failed"), bad_status=True) + + async def work(): + if consumer == "health": + await gate._health(args(), fixture, {"health_instance": "test:1"}) + else: + await gate._memory_set(args()) + + field = "health HTTP status" if consumer == "health" else "memory.set HTTP status" + error = await observed(work) + assert isinstance(error, gate.Invalid) and str(error) == field + assert events[-1] == "http close" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("source", ["request", "request_record"]) +@pytest.mark.parametrize("first_kind", FIRST[1:]) +@pytest.mark.parametrize("output_kind", SECOND) +async def test_request_diagnostic_retains_request_or_required_record_failure( + http_edges, artifact_output, source, first_kind, output_kind +): + first, output = failure(first_kind), failure(output_kind) + _, http_events = http_edges(first if source == "request" else None) + errors = {"request_error": output} + if source == "request_record": + errors["request"] = first + artifact, events = artifact_output(errors) + assert await observed(lambda: gate._memory_set(args(), emit=artifact.emit)) is first + assert events[-1] == "request_error" and http_events[-1] == "http close" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("record", ["worker_error", "worker"]) +@pytest.mark.parametrize("first_kind", FIRST[1:]) +@pytest.mark.parametrize("output_kind", SECOND) +async def test_worker_diagnostics_and_prior_results_retain_worker_failure( + monkeypatch, artifact_output, record, first_kind, output_kind +): + from server.memory.workers import pg_delivery + from server.settings import settings + + first, output = failure(first_kind), failure(output_kind) + good = pg_delivery.DeliveryResult(claimed=1, published=1) + outcomes = [good, first] if record == "worker" else [first] + monkeypatch.setattr(gate, "BATCHES", len(outcomes)) + monkeypatch.setattr(settings, "pg_delivery_batch_rows", 1) + worker = AsyncMock(side_effect=outcomes) + monkeypatch.setattr(pg_delivery, "run_pass", worker) + artifact, events = artifact_output({record: output}) + assert await observed(lambda: gate._delivery(args(), 1, emit=artifact.emit, arm=0)) is first + assert worker.await_count == len(outcomes) and "worker_error" in events + if record == "worker": + assert events == ["worker_error", "worker"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("output_kind", SECOND) +async def test_successful_worker_work_still_requires_result_output( + monkeypatch, artifact_output, output_kind +): + from server.memory.workers import pg_delivery + from server.settings import settings + + output = failure(output_kind) + monkeypatch.setattr(gate, "BATCHES", 1) + monkeypatch.setattr(settings, "pg_delivery_batch_rows", 1) + worker = AsyncMock(return_value=pg_delivery.DeliveryResult(claimed=1, published=1)) + monkeypatch.setattr(pg_delivery, "run_pass", worker) + ticks = iter([0, 1_000_000_000]) + monkeypatch.setattr(gate.time, "perf_counter_ns", lambda: next(ticks)) + artifact, events = artifact_output({"worker": output}) + assert await observed(lambda: gate._delivery(args(), 1, emit=artifact.emit, arm=0)) is output + assert worker.await_count == 1 + assert events == (["worker"] if output is not None else ["worker", "delivery_block"]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method", ["set", "restore"]) +@pytest.mark.parametrize("first_kind", FIRST) +@pytest.mark.parametrize("exit_kind", SECOND) +async def test_trigger_work_retains_error_through_transaction_exit_and_owner_cleanup( + monkeypatch, method, first_kind, exit_kind +): + first, exit_error = failure(first_kind), failure(exit_kind) + events = [] + + class Connection: + _pool_release_ctr = 0 + _top_xact = None + _protocol = SimpleNamespace(is_in_transaction=lambda: False) + + def __init__(self): + self.closed = False + + def transaction(self): + return Transaction(self, None, False, False) + + def is_closed(self): + return self.closed + + async def execute(self, sql): + events.append(sql.split()[0]) + if sql.startswith("DROP") and first is not None: + raise first + if sql.startswith(("ROLLBACK", "COMMIT")) and exit_error is not None: + raise exit_error + + async def close(self, **kwargs): + events.append("public close") + self.closed = True + + def terminate(self): + events.append("abort") + self.closed = True + + triggers = gate.Triggers(("memory_current",), "invariant", "org") + monkeypatch.setattr(triggers, "verify", AsyncMock()) + monkeypatch.setattr(gate.cell_triggers, "missing", AsyncMock(return_value=[])) + conn = Connection() + + async def work(): + if method == "set": + await triggers.set(conn, False) + else: + await gate._restore(triggers, conn, time.monotonic() + 1) + + assert await observed(work) is (first or exit_error) + if method == "restore" and (first is not None or exit_error is not None): + assert conn.closed and "public close" in events From 2ff89318f015595fcd034671bbc1e2630317852f Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 02:58:21 +0000 Subject: [PATCH 053/114] Describe trigger creation before the attach mode commit Signed-off-by: zlareb1 --- docs/architecture/cells.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index c738775756..07115e4b5a 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -789,9 +789,9 @@ read-only scope, lists any tenancy row the directory does not place. serves past a table whose lock it cannot take, logged with the holders of its locks, or one the role cannot alter, logged as such (the third exception under Invariants). A first cell is `recording` until attach sets `enforced` while it runs, so no boot can - have created them: attach creates them itself with the same pass - (`cell_triggers.ensure`) once it has set `enforced`, and the cell goes `serving` only - once `cell_triggers.missing` names no defect. A later cell is `enforced` from prepare + have created them: attach creates them with `cell_triggers.ensure_table`, passing + `recording_first=True`, before step 2 and again before setting `enforced`. + The cell goes `serving` only once `cell_triggers.missing` names no defect. A later cell is `enforced` from prepare and gets them at its first migrating boot after prepare, which its attach requires the same way. `loom doctor` names a disabled or altered one in any state, a missing one on an `enforced` cell, and a partitioned table with an `org` column as unsupported, since From 69e9f2dd00b8e93c008dadb0fe2d2cf1a38fce4e Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 03:28:23 +0000 Subject: [PATCH 054/114] Exercise attach cleanup with a fenced directory snapshot Signed-off-by: zlareb1 --- tests/unit/test_cell_attach_owned_exit.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_cell_attach_owned_exit.py b/tests/unit/test_cell_attach_owned_exit.py index 0a8ba4d2f1..bc8bc89da5 100644 --- a/tests/unit/test_cell_attach_owned_exit.py +++ b/tests/unit/test_cell_attach_owned_exit.py @@ -19,6 +19,7 @@ def test_actual_attach_owner_keeps_first_failure(owner, secondary): from server.cli.cell_attach import Run from server.cli import cell_attach_pg as apg from server.cli import cell_control + from tests.unit.test_cell_control import snapshot owner, kind = sys.argv[1:] original = ValueError('operation failed') @@ -46,7 +47,7 @@ async def opening(owned, what): owned.conn = conn async def fail(*args): raise original cell_control.Owner._open = opening router = AsyncMock() - router.call.return_value = {'state': 'attaching'} + router.call.return_value = {**snapshot(), 'state': 'attaching'} run = Run(None, argparse.Namespace(cell='test', wait=0), router, lambda _: None) run.row = {'deployment_id': 'test'} run.mode = 'recording' From 01621af7a22909ac133eeb8a6b4812b397cce132 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 03:37:54 +0000 Subject: [PATCH 055/114] Check trigger gate provenance in its repository checkout Signed-off-by: zlareb1 --- tests/unit/test_bench_cell_trigger_gate.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py index a625ab5a27..211dfd88c9 100644 --- a/tests/unit/test_bench_cell_trigger_gate.py +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -8,6 +8,7 @@ import json import socket import uuid +from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock @@ -1005,7 +1006,8 @@ def test_source_provenance_uses_only_fixed_bounded_readonly_git(monkeypatch, fau def git(argv, **kwargs): calls.append(argv) assert argv[0] == "git" and kwargs["timeout"] == 5 and kwargs["check"] is False - assert kwargs["capture_output"] is True and kwargs["cwd"].name == "scale-attachcli" + assert kwargs["capture_output"] is True + assert kwargs["cwd"] == Path(__file__).resolve().parents[2] assert "shell" not in kwargs if argv[1:] == ["rev-parse", "HEAD"]: value = "wrong" if fault == "head" else fixture.build_commit From 5d043871956df9ef0d0b622c412f1cd1b6384f37 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 03:37:56 +0000 Subject: [PATCH 056/114] Check trigger gate provenance in its repository checkout Signed-off-by: zlareb1 --- tests/unit/test_bench_cell_trigger_gate.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py index a625ab5a27..211dfd88c9 100644 --- a/tests/unit/test_bench_cell_trigger_gate.py +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -8,6 +8,7 @@ import json import socket import uuid +from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock @@ -1005,7 +1006,8 @@ def test_source_provenance_uses_only_fixed_bounded_readonly_git(monkeypatch, fau def git(argv, **kwargs): calls.append(argv) assert argv[0] == "git" and kwargs["timeout"] == 5 and kwargs["check"] is False - assert kwargs["capture_output"] is True and kwargs["cwd"].name == "scale-attachcli" + assert kwargs["capture_output"] is True + assert kwargs["cwd"] == Path(__file__).resolve().parents[2] assert "shell" not in kwargs if argv[1:] == ["rev-parse", "HEAD"]: value = "wrong" if fault == "head" else fixture.build_commit From dfa83dd05308c9d8f017cb9e520960d859e2ba46 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 03:46:54 +0000 Subject: [PATCH 057/114] Preserve native control request failures through HTTP cleanup Signed-off-by: zlareb1 --- server/cli/cell_attach.py | 15 +- server/cli/cell_control.py | 20 +- server/cli/cell_prepare.py | 17 +- .../unit/test_cell_native_http_first_error.py | 263 ++++++++++++++++++ 4 files changed, 303 insertions(+), 12 deletions(-) create mode 100644 tests/unit/test_cell_native_http_first_error.py diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index a8a78279fd..8fd6e7ebee 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -90,10 +90,19 @@ async def call(self, path: str, body: dict[str, Any] | None = None) -> dict[str, async def health(self, host: str) -> str: """The deployment id `https:///v1/health/deployment` answers ('' when none).""" try: - async with httpx.AsyncClient( - timeout=_HEALTH_TIMEOUT, transport=self.transport - ) as client: + manager = httpx.AsyncClient(timeout=_HEALTH_TIMEOUT, transport=self.transport) + client = await manager.__aenter__() + try: r = await client.get(f"https://{host}/v1/health/deployment") + except BaseException: # Client exit preserves the request's first failure. + with contextlib.suppress(BaseException): + await manager.__aexit__(*sys.exc_info()) + raise + except (httpx.HTTPError, ValueError): + return "" + # A required close failure after a successful read must reach the owner. + await manager.__aexit__(None, None, None) + try: answer = r.json() except (httpx.HTTPError, ValueError): return "" diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index 1966b9e62b..730c36bd42 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -3,7 +3,9 @@ from __future__ import annotations import asyncio +import contextlib import json +import sys import time from collections.abc import Awaitable, Callable from dataclasses import dataclass @@ -304,16 +306,22 @@ async def request( endpoint = cli.at(url, f"/cells/v1/operator/cells/{name}" + path) try: - async with ( - asyncio.timeout(REQUEST_S), - httpx.AsyncClient( + async with asyncio.timeout(REQUEST_S): + manager = httpx.AsyncClient( timeout=cli._OPERATOR_TIMEOUT, trust_env=url.scheme == "https", transport=transport, headers={"Authorization": f"Bearer {settings.router_operator_token}"}, - ) as client, - ): - response = await client.request(method, endpoint, json=body) + ) + client = await manager.__aenter__() + try: + response = await client.request(method, endpoint, json=body) + except BaseException: # Client exit preserves the request's first failure. + with contextlib.suppress(BaseException): + await manager.__aexit__(*sys.exc_info()) + raise + else: + await manager.__aexit__(None, None, None) except (httpx.HTTPError, TimeoutError) as e: raise _refused(f"router reply unknown ({type(e).__name__}): stop and rerun") from None answer = cli.router_answer(response, {"cell": name}, cli.OPERATOR_REFUSALS) diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index 6e94ae1058..4310b4c9bb 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -26,6 +26,7 @@ import argparse import asyncio +import contextlib import hashlib import json import secrets @@ -301,9 +302,11 @@ async def _register( token = settings.router_operator_token url = cell_cli.at(router, f"/cells/v1/operator/cells/{args.cell}/register") deadline = time.monotonic() + args.wait - async with httpx.AsyncClient( + manager = httpx.AsyncClient( timeout=_register_timeout(), trust_env=router.scheme == "https", transport=transport - ) as client: + ) + client = await manager.__aenter__() + try: while True: try: r = await client.post(url, json=body, headers={"Authorization": f"Bearer {token}"}) @@ -318,7 +321,7 @@ async def _register( f"{cell_cli.not_the_router(r)}; run prepare again once the router answers" ) if r.status_code == 200: - return answer + break code = answer.get("code") if code == "BACKEND_UNAVAILABLE": raise Refused( @@ -340,6 +343,14 @@ async def _register( print(f"loom cell prepare: {answer['error']}; retrying", file=sys.stderr) await _sleep(pause) + except BaseException: # Client exit preserves the first request or refusal. + with contextlib.suppress(BaseException): + await manager.__aexit__(*sys.exc_info()) + raise + else: + await manager.__aexit__(None, None, None) + return answer + def _own_rows( rows: _CellRows, answer: dict[str, Any], every: list[dict[str, Any]] diff --git a/tests/unit/test_cell_native_http_first_error.py b/tests/unit/test_cell_native_http_first_error.py new file mode 100644 index 0000000000..4634e7d69c --- /dev/null +++ b/tests/unit/test_cell_native_http_first_error.py @@ -0,0 +1,263 @@ +"""Native HTTP clients preserve the first outcome through required exit.""" + +from __future__ import annotations + +import argparse +import asyncio +from types import SimpleNamespace + +import httpx +import pytest + +from server.cli import cell_attach, cell_control, cell_prepare +from server.settings import settings + +_CLIENT = httpx.AsyncClient +_BOUNDARIES = ("control", "health", "register") +FIRST_ERRORS = ( + ValueError, + httpx.ReadError, + TimeoutError, + asyncio.CancelledError, + SystemExit, + KeyboardInterrupt, +) +EXIT_ERRORS = ( + RuntimeError, + ValueError, + httpx.ReadError, + TimeoutError, + asyncio.CancelledError, + SystemExit, + KeyboardInterrupt, +) + + +@pytest.fixture +def exchange(monkeypatch): + state = SimpleNamespace( + first=None, + second=None, + phase="request", + close_phase="await", + events=[], + exit_args=None, + entered=asyncio.Event(), + block=False, + answer={"org": "acme", "cell": "eu-1", "deployment_id": "dep-eu-1"}, + status=200, + close_entered=asyncio.Event(), + close_block=False, + ) + + async def handler(request): + state.events.append("request") + state.entered.set() + if state.block: + try: + await asyncio.Future() + except asyncio.CancelledError as error: + state.first = error + raise + if state.first is not None: + raise state.first + return httpx.Response(state.status, json=state.answer) + + class Client(_CLIENT): + async def __aenter__(self): + state.events.append("enter") + if state.phase == "enter": + raise state.first + return await super().__aenter__() + + def __aexit__(self, *args): + state.exit_args = args + state.events.append("exit-call") + if state.close_phase == "construct" and state.second is not None: + raise state.second + + async def close(): + state.events.append("exit-await") + state.close_entered.set() + if state.close_block: + await asyncio.Future() + await super(Client, self).__aexit__(*args) + if state.second is not None: + raise state.second + + return close() + + def client(**kwargs): + state.events.append("construct") + if state.phase == "construct": + raise state.first + kwargs["transport"] = httpx.MockTransport(handler) + return Client(**kwargs) + + monkeypatch.setattr(httpx, "AsyncClient", client) + monkeypatch.setattr(settings, "router_operator_token", "inert-test-token") + return state + + +async def invoke(boundary): + url = httpx.URL("https://router.test") + if boundary == "control": + return await cell_control.request(url, "eu-1", None, "/claim", {}) + if boundary == "health": + return await cell_attach.Router(url, "eu-1", None).health("eu-1.example.test") + return await cell_prepare._register(argparse.Namespace(cell="eu-1", wait=0), url, {}, None) + + +async def observed(callback): + # Native fatal values must stay in this task rather than reach the event loop. + try: + return "value", await callback() + except BaseException as error: # noqa: BLE001 — compare original native failure objects. + return "error", error + + +def assert_first(boundary, state, outcome): + kind, result = outcome + if boundary == "health" and isinstance(state.first, (httpx.HTTPError, ValueError)): + assert (kind, result) == ("value", "") + elif (boundary == "control" and isinstance(state.first, (httpx.HTTPError, TimeoutError))) or ( + boundary == "register" + and state.phase == "request" + and isinstance(state.first, httpx.HTTPError) + ): + expected = cell_prepare.Refused if boundary == "register" else cell_control.Refused + assert kind == "error" and isinstance(result, expected) + assert type(state.first).__name__ in str(result) + else: + assert kind == "error" and result is state.first + + +def assert_exit_first(boundary, state): + first = state.exit_args[1] + if boundary == "register" and isinstance(state.first, httpx.HTTPError): + assert isinstance(first, cell_prepare.Refused) + else: + assert first is state.first + + +def assert_close_failure(boundary, state, outcome): + kind, result = outcome + if boundary == "control" and isinstance(state.second, (httpx.HTTPError, TimeoutError)): + assert kind == "error" and isinstance(result, cell_control.Refused) + else: + assert kind == "error" and result is state.second + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +@pytest.mark.parametrize("second_kind", (*EXIT_ERRORS, None)) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +async def test_request_first_outcome_survives_client_exit( + exchange, boundary, first_kind, second_kind, close_phase +): + exchange.first = first_kind("request-first") + exchange.second = second_kind("exit-second") if second_kind is not None else None + exchange.close_phase = close_phase + assert_first(boundary, exchange, await observed(lambda: invoke(boundary))) + assert exchange.events[:3] == ["construct", "enter", "request"] + assert exchange.events.count("exit-call") == 1 + assert_exit_first(boundary, exchange) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +async def test_required_client_exit_failure_cannot_report_success( + exchange, boundary, second_kind, close_phase +): + exchange.second = second_kind("required-exit") + exchange.close_phase = close_phase + assert_close_failure(boundary, exchange, await observed(lambda: invoke(boundary))) + assert exchange.exit_args == (None, None, None) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +async def test_successful_client_neighbor(exchange, boundary): + kind, result = await observed(lambda: invoke(boundary)) + assert kind == "value" + expected = ( + None + if boundary == "placement" + else ("dep-eu-1" if boundary == "health" else exchange.answer) + ) + assert result == expected + assert exchange.events == ["construct", "enter", "request", "exit-call", "exit-await"] + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("phase", ("construct", "enter")) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +async def test_client_acquisition_keeps_its_first_outcome(exchange, boundary, phase, first_kind): + exchange.phase = phase + exchange.first = first_kind("acquire-first") + assert_first(boundary, exchange, await observed(lambda: invoke(boundary))) + assert "request" not in exchange.events and "exit-call" not in exchange.events + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +async def test_actual_task_cancellation_keeps_its_payload(exchange, boundary, second_kind): + exchange.block = True + exchange.second = second_kind("exit-second") + task = asyncio.create_task(observed(lambda: invoke(boundary))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("operator-first-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) + assert result.args == ("operator-first-stop",) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +async def test_repeated_cancellation_during_http_exit_keeps_first_payload(exchange, boundary): + exchange.block = True + exchange.close_block = True + task = asyncio.create_task(observed(lambda: invoke(boundary))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("first-stop") + await asyncio.wait_for(exchange.close_entered.wait(), 1) + task.cancel("second-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) and result.args == ("first-stop",) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +@pytest.mark.parametrize("semantic", ("malformed", "backend", "probe", "refusal")) +async def test_registration_selected_refusal_survives_exit( + monkeypatch, exchange, second_kind, close_phase, semantic +): + selected = cell_prepare.Refused("registration-first") + monkeypatch.setattr(cell_prepare, "Refused", lambda message: selected) + exchange.status = 200 if semantic == "malformed" else 503 + exchange.answer = ( + {"unexpected": True} + if semantic == "malformed" + else { + "error": "registration-first", + "code": {"backend": "BACKEND_UNAVAILABLE", "probe": "PROBE_BUSY", "refusal": "OTHER"}[ + semantic + ], + } + ) + exchange.second = second_kind("exit-second") + exchange.close_phase = close_phase + kind, result = await observed(lambda: invoke("register")) + assert kind == "error" and result is selected + assert exchange.exit_args[1] is selected From 6f61b3606c80d8622e778aa23a46adde613c412c Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 21:54:26 +0000 Subject: [PATCH 058/114] Retire participants only after permanent quiescence Keep request tails, actual application tasks, workers and native commands under one permanently closing process gate. Enroll before acceptance reads and retain uncertified retired rows until exact process death can be verified. Remove participant retire --yes and the retireLostPods chart value. Pod API object state and heartbeat expiry cannot authorize retirement or pruning. Signed-off-by: zlareb1 --- deploy/helm/loom/templates/_helpers.tpl | 5 +- .../helm/loom/templates/analytics-export.yaml | 2 +- deploy/helm/loom/templates/deployment.yaml | 2 +- deploy/helm/loom/templates/hooks.yaml | 4 +- deploy/helm/loom/templates/networkpolicy.yaml | 4 - deploy/helm/loom/templates/rbac.yaml | 32 - .../helm/loom/templates/serviceaccount.yaml | 2 +- deploy/helm/loom/templates/tests/health.yaml | 2 +- deploy/helm/loom/templates/workers.yaml | 2 +- deploy/helm/loom/values.yaml | 12 +- docs/architecture/cells.md | 50 +- docs/architecture/failure-and-recovery.md | 21 +- docs/architecture/provider-reconfiguration.md | 35 +- docs/operations/backup-and-recovery.md | 10 +- docs/operations/kubernetes.md | 26 +- docs/operations/replicas.md | 24 +- docs/reference/cli.md | 2 +- docs/reference/errors.md | 4 + server/auth/tenancy.py | 4 +- server/boot.py | 3 + server/cli/_ch.py | 40 +- server/cli/analytics_export.py | 17 +- server/cli/code_run.py | 3 +- server/cli/import_clickgap.py | 5 +- server/cli/import_recall.py | 13 +- server/cli/maintenance.py | 55 +- server/cli/participant.py | 27 +- server/cli/sql_access.py | 9 +- server/core/cell.py | 6 + server/core/instances.py | 3 + server/core/provider_process.py | 76 +- server/core/retirement.py | 114 +++ server/insights/app/provider_barrier.py | 82 +- server/insights/app/provider_command.py | 54 +- .../insights/store/provider_participants.py | 85 +- server/main.py | 130 ++- server/memory/app/embedder_calibration.py | 6 +- server/observability/events.py | 3 +- server/observability/problems.py | 5 +- server/transport/model_maintenance.py | 16 +- server/transport/retirement.py | 28 + server/worker_main.py | 18 +- server/workers/supervisor.py | 6 +- .../provider_process_before_retirement.py | 173 ++++ tests/integration/test_cell_retirement.py | 103 +++ .../test_provider_coordination_live_ch.py | 14 +- tests/unit/conftest.py | 9 + tests/unit/snapshots/cli_help.json | 6 +- tests/unit/snapshots/cli_tree.json | 20 +- tests/unit/test_analytics_export.py | 5 +- tests/unit/test_cell_cli_orgs.py | 9 +- tests/unit/test_cell_retirement.py | 752 ++++++++++++++++++ tests/unit/test_provider_barrier.py | 37 +- tests/unit/test_provider_command.py | 3 + tests/unit/test_provider_host_proof.py | 128 +-- tests/unit/test_reextract.py | 4 +- tests/unit/test_revocation_web_refresh.py | 30 +- tests/unit/test_shutdown_deadline.py | 2 +- 58 files changed, 1851 insertions(+), 491 deletions(-) delete mode 100644 deploy/helm/loom/templates/rbac.yaml create mode 100644 server/core/retirement.py create mode 100644 server/transport/retirement.py create mode 100644 tests/fixtures/provider_process_before_retirement.py create mode 100644 tests/integration/test_cell_retirement.py create mode 100644 tests/unit/test_cell_retirement.py diff --git a/deploy/helm/loom/templates/_helpers.tpl b/deploy/helm/loom/templates/_helpers.tpl index d0fee03c7a..fbbbdbac19 100644 --- a/deploy/helm/loom/templates/_helpers.tpl +++ b/deploy/helm/loom/templates/_helpers.tpl @@ -564,14 +564,11 @@ initContainers: {{/* Pod spec fields the Deployments, the hook Jobs and the analytics export share; `serviceAccount` names another account than the release's. */}} -{{/* `leads`: the pod may hold the workers lease, which alone reads the -Kubernetes API (retireLostPods). */}} {{- define "loom.podCommon" -}} -{{- $leads := .leads -}} {{- $sa := .serviceAccount -}} {{- with .root -}} serviceAccountName: {{ default (include "loom.serviceAccountName" .) $sa }} -automountServiceAccountToken: {{ or .Values.serviceAccount.automountToken (and .Values.retireLostPods $leads) }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountToken }} # Service links would inject LOOM_SERVICE_HOST and the like into Loom's env namespace. enableServiceLinks: false {{- if not .Values.sandbox.hostUsers }} diff --git a/deploy/helm/loom/templates/analytics-export.yaml b/deploy/helm/loom/templates/analytics-export.yaml index e6b0355650..403d7f0009 100644 --- a/deploy/helm/loom/templates/analytics-export.yaml +++ b/deploy/helm/loom/templates/analytics-export.yaml @@ -81,7 +81,7 @@ spec: app.kubernetes.io/component: analytics-export spec: restartPolicy: Never - {{- include "loom.podCommon" (dict "root" $ "leads" false "serviceAccount" $name) | nindent 10 }} + {{- include "loom.podCommon" (dict "root" $ "serviceAccount" $name) | nindent 10 }} {{- include "loom.waitForDevClickhouse" $ | nindent 10 }} containers: - name: analytics-export diff --git a/deploy/helm/loom/templates/deployment.yaml b/deploy/helm/loom/templates/deployment.yaml index 3ee7d1e98c..c53b91de59 100644 --- a/deploy/helm/loom/templates/deployment.yaml +++ b/deploy/helm/loom/templates/deployment.yaml @@ -73,7 +73,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: - {{- include "loom.podCommon" (dict "root" . "leads" (not .Values.workers.enabled)) | nindent 6 }} + {{- include "loom.podCommon" (dict "root" .) | nindent 6 }} {{- if .Values.topologySpread.enabled }} topologySpreadConstraints: - maxSkew: {{ .Values.topologySpread.zoneSkew }} diff --git a/deploy/helm/loom/templates/hooks.yaml b/deploy/helm/loom/templates/hooks.yaml index cdd65cccc1..d55dbe9a64 100644 --- a/deploy/helm/loom/templates/hooks.yaml +++ b/deploy/helm/loom/templates/hooks.yaml @@ -65,7 +65,7 @@ spec: app.kubernetes.io/component: {{ .name }} spec: restartPolicy: Never - {{- include "loom.podCommon" (dict "root" $ "leads" false) | nindent 6 }} + {{- include "loom.podCommon" (dict "root" $) | nindent 6 }} containers: - name: {{ .name }} image: {{ include "loom.image" $ }} @@ -83,7 +83,7 @@ spec: value: {{ $v | toString | quote }} {{- end }} # Its participant proof names the pod, so a hook pod lost mid-run is - # retired like any other (provider_process.pod_gone). + # verified against the same exact process identity as every participant. - name: POD_NAME valueFrom: { fieldRef: { fieldPath: metadata.name } } - name: POD_NAMESPACE diff --git a/deploy/helm/loom/templates/networkpolicy.yaml b/deploy/helm/loom/templates/networkpolicy.yaml index 6a1ed604cb..14903323dc 100644 --- a/deploy/helm/loom/templates/networkpolicy.yaml +++ b/deploy/helm/loom/templates/networkpolicy.yaml @@ -87,10 +87,6 @@ spec: {{- include "loom.anywhereBut" . | nindent 8 }} ports: - port: 443 - {{- if .Values.retireLostPods }} - # The Kubernetes API (443 on EKS; 6443 on most other clusters). - - port: 6443 - {{- end }} {{- if and (not .Values.devClickhouse.enabled) (ne (int .Values.clickhouse.port) 443) }} - port: {{ .Values.clickhouse.port }} {{- end }} diff --git a/deploy/helm/loom/templates/rbac.yaml b/deploy/helm/loom/templates/rbac.yaml deleted file mode 100644 index 72a0ff9d90..0000000000 --- a/deploy/helm/loom/templates/rbac.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- if .Values.retireLostPods }} -{{- if not (or .Values.serviceAccount.create .Values.serviceAccount.name) }} -{{- fail "retireLostPods grants `get pods` to the release's service account: set serviceAccount.create or serviceAccount.name, not the namespace's default" }} -{{- end }} -{{- /* Only `get` on pods: the workers leader asks whether a lost participant's -pod still exists (server/core/provider_process.py `pod_gone`). */}} -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "loom.fullname" . }} - labels: - {{- include "loom.labels" . | nindent 4 }} -rules: - - apiGroups: [""] - resources: [pods] - verbs: [get] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "loom.fullname" . }} - labels: - {{- include "loom.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ include "loom.fullname" . }} -subjects: - - kind: ServiceAccount - name: {{ include "loom.serviceAccountName" . }} - namespace: {{ .Release.Namespace }} -{{- end }} diff --git a/deploy/helm/loom/templates/serviceaccount.yaml b/deploy/helm/loom/templates/serviceaccount.yaml index 4536a11c5c..0c5c212fd8 100644 --- a/deploy/helm/loom/templates/serviceaccount.yaml +++ b/deploy/helm/loom/templates/serviceaccount.yaml @@ -9,5 +9,5 @@ metadata: annotations: {{- toYaml . | nindent 4 }} {{- end }} -automountServiceAccountToken: {{ or .Values.serviceAccount.automountToken .Values.retireLostPods }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountToken }} {{- end }} diff --git a/deploy/helm/loom/templates/tests/health.yaml b/deploy/helm/loom/templates/tests/health.yaml index 98a37832ac..ee21e05380 100644 --- a/deploy/helm/loom/templates/tests/health.yaml +++ b/deploy/helm/loom/templates/tests/health.yaml @@ -10,7 +10,7 @@ metadata: helm.sh/hook-delete-policy: before-hook-creation spec: restartPolicy: Never - {{- include "loom.podCommon" (dict "root" . "leads" false) | nindent 2 }} + {{- include "loom.podCommon" (dict "root" .) | nindent 2 }} containers: - name: health image: {{ include "loom.image" . }} diff --git a/deploy/helm/loom/templates/workers.yaml b/deploy/helm/loom/templates/workers.yaml index b89cce170b..710a38079c 100644 --- a/deploy/helm/loom/templates/workers.yaml +++ b/deploy/helm/loom/templates/workers.yaml @@ -42,7 +42,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: - {{- include "loom.podCommon" (dict "root" . "leads" true) | nindent 6 }} + {{- include "loom.podCommon" (dict "root" .) | nindent 6 }} {{- if .Values.topologySpread.enabled }} # Leader and standby on different nodes. topologySpreadConstraints: diff --git a/deploy/helm/loom/values.yaml b/deploy/helm/loom/values.yaml index 73c0ee9d30..5fe18416a7 100644 --- a/deploy/helm/loom/values.yaml +++ b/deploy/helm/loom/values.yaml @@ -205,15 +205,9 @@ serviceAccount: name: "" # IRSA (eks.amazonaws.com/role-arn) or other cloud identity annotations. annotations: {} - # Mount the service-account token (retireLostPods mounts it too). + # Mount the service-account token for workloads that explicitly need it. automountToken: false -# The workers leader retires the provider participant of a pod Kubernetes has -# deleted (lost with its node, evicted past its grace period), which would -# otherwise hold the next model change until `loom participant retire`. Grants -# the service account `get` on pods in this namespace and mounts its token. -retireLostPods: true - podAnnotations: {} podLabels: {} # amd64 by default: the C++ compiler tier's scip-clang ships for x86_64 only. The @@ -411,8 +405,8 @@ assistant: # Ingress to the Loom port from `ingressFrom` only. Egress to DNS, the # release's web pods, the extractor's pods (with extractor.enabled), the dev # ClickHouse (with devClickhouse.enabled), the EKS Pod Identity agent, and TCP -# 443, the ClickHouse port, postgres.port (with postgres.host) and 6443 (with -# retireLostPods) anywhere except `egressExcept` (providers, the IdP and git remotes +# 443, the ClickHouse port and postgres.port (with postgres.host) anywhere except +# `egressExcept` (providers, the IdP and git remotes # have no fixed range); a ClickHouse or Postgres inside the cluster needs an # extraEgress rule. Off, `guardMetadata` gives the Loom pods an egress-only policy # that allows every pod and every address but `egressExcept`. diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 78e1d4c689..9cd44ad6f4 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -18,6 +18,42 @@ that every transport enforces (`auth_actually_required()`, and `LOOM_MCP_AUTH_MO refuses to boot without them. [Storage authority](storage-authority.md) describes what a cell holds and how its two stores agree. +## Participant lifecycle baseline + +A participant advertising the ninth `host_proof` field `cell-retirement-v1` +enrolls before its first fresh primary acceptance read and uses permanent guarded +retirement. A serving cell heartbeat includes its `participant_id` beside the +fingerprint, allowing a replacement to be joined to the participant registry. +Workers children enroll independently and do not publish serving instance rows. +The capability identifies installed code; it does not prove that every older +process or unsafe retirement client has disappeared. + +Before online configuration finalization is supported, stop every unsafe manual +retirement and pruning writer and prevent its binary/image from restarting. +Inventory and positively terminate every accepting server, workers child, +standalone worker and native command that overlapped those writers. Include paused +boots, already retired/pruned identities and lifecycle-capable processes started +during the overlap: old maintenance code could have hidden their rows too. Only +then start replacement accepting and maintenance processes with this lifecycle. +A shared image tag, pod deletion, readiness or heartbeat expiry is insufficient. +Keep exact PID/boot/namespace/start or trusted runtime termination evidence with +the deployment's upgrade record. No CLI flag or stored acknowledgement substitutes +for that evidence, and the runtime census does not verify the external inventory. + +Census retained legacy rows while unsafe writers are stopped. Recertify a retained +identity only after exact verifiable physical death and known-zero cross-replica +queries. Keep empty or unknown proofs quarantined. If the legacy inventory is +incomplete, stop all accepting/native-command and retirement/pruning processes, +positively terminate or fence every runtime that could execute an old binary, and +prevent old restarts. Only after that physical quiescence may the operator request +an exact audited cleanup of selected unknown legacy participant identities. There +is no automatic age-based cleanup or affirmation bypass. Restart only guarded +lifecycle processes afterward. Unknown or partitioned runtimes remain blockers. + +This baseline is an operational prerequisite, not a release-cut requirement. +Configuration finalization also needs the directory's controller-fencing protocol; +the lifecycle certificate alone cannot authorize reservation release. + ## Why whole deployments A cell shares no store, key, lease or worker with another. Every guarantee a single @@ -485,11 +521,13 @@ with the same configuration, or `--finish`, first writes `next` wherever it is m It reruns attach's overlap refusals against every other cell's directory row, counting both values of each, writes the new fingerprint as `next` to the directory and then to `loom.cell_self`, so old and new processes both serve while the new configuration rolls -out. `loom cell update --finish` waits, at most `--wait` (default 10 minutes), until -every live process reports `next` and the longest request deadline has passed; when the -change turns SCIM on or changes the cell's OIDC client it then runs attach's SCIM -precondition (refusing, or `--revoke-unprovisioned`); only then does it make `next` the -only value, in `loom.cell_self` first and the directory last. A widened issuer or +out. Finalization first narrows the cell's accepted value durably while the directory +keeps both reservations. It then requires a complete post-cutoff participant census, +permanent guarded retirement or verified exact death for every captured identity, +and a serving target replacement outside that captured set. Heartbeat expiry and +request deadlines do not prove this. Only afterward may the directory release its +old reservation. Online finalization needs the [lifecycle baseline](#participant-lifecycle-baseline) +and controller fencing; SCIM-on changes remain gated on actual per-cell SCIM support. A widened issuer or pattern is reserved in the directory before any process accepts it, and a narrowed one is released only after none does. A refusal or a wait that runs out keeps both values and prints the processes not yet reporting `next`; rerunning resumes, and @@ -514,6 +552,8 @@ version and fields, and beside them keys that no fingerprint compares: they are needs no column for them (`server.core.instances.cell_row`). Their presence proves the build that reports them; there is no generation key. +- `participant_id`: this server's provider participant UUID, for joining its actual + process lifecycle to the serving instance row. A workers child publishes no instance row. - `serves`: `LOOM_SERVES_TRAFFIC`. - `marked`: whether this process's pooled Postgres sessions keep the tenancy marker (`loom.tenancy_checked`). The cell refresh reads it with the state rows, after `RESET ALL` diff --git a/docs/architecture/failure-and-recovery.md b/docs/architecture/failure-and-recovery.md index 40f8e4b57e..d54fb025b8 100644 --- a/docs/architecture/failure-and-recovery.md +++ b/docs/architecture/failure-and-recovery.md @@ -186,17 +186,16 @@ node. Nothing else counts: an unreachable process is not a stopped one. TTL is 60 s, renewed every 20 s; a contender takes it over once it expired and the holder's participant counts as stopped. `loom provider-lock clear --holder ` frees one whose holder has no participant row, which never counts as stopped. -- **Killed mid-reconfiguration.** The transition waits on a participant whose heartbeat - expired, since a paused process may resume with admitted work, until a process on - its host proves it gone (its container restarted, or its pid exited) and retires it. - The workers leader - deletes the rows of participants retired a day ago and lists, once a minute, those a - transition would wait on as `provider_participants_stale` in its `/v1/health`. Retire - one by hand with `loom participant retire --participant --yes` once its process - is confirmed gone; the command refuses a participant whose heartbeat is fresh or whose - queries still run. -- **A boot that fails after enrolling** retires its row; a crash-looping container leaves - none behind. +- **Killed mid-reconfiguration.** An expired heartbeat remains a blocker until the + exact process dies or completes guarded shutdown. A trusted local process can + verify its PID, namespace, boot and start identity, then retire it after all + cross-replica queries settle. Use `loom participant retire --participant ` on + that trusted host. Pod deletion or replacement and namespace changes alone do + not prove death. The workers leader prunes only certified permanent retirements + older than a day; unknown legacy rows remain visible. +- **A boot that fails after enrolling** remains nonretired unless all its actual + producers and writes stop. A shutdown deadline does not turn cancellation into + a retirement certificate. A process frozen past its heartbeat (a stopped container, a VM pause) holds the transition until it resumes and drains, or an operator retires it. diff --git a/docs/architecture/provider-reconfiguration.md b/docs/architecture/provider-reconfiguration.md index 71fa716069..622fb99dc4 100644 --- a/docs/architecture/provider-reconfiguration.md +++ b/docs/architecture/provider-reconfiguration.md @@ -36,21 +36,38 @@ stopped only once it retired itself (a graceful shutdown), a later process on it host proved it gone, or an operator retired it. An expired heartbeat is no proof: a paused process can hold admitted work outside ClickHouse and write it with the old configuration once it resumes, so the transition waits for it to beat again and -drain. Each row carries its host name, kernel boot, container id (Docker's, or a -Kubernetes pod's and container name), pid namespace, pid and start time; every 10 s -each process retires the participants of its own host and boot whose pid no longer -runs that process, or that ran in its own container before the container restarted -into a new pid namespace, once no query of theirs runs. That covers a killed process -whose container or worker restarts in place; two containers that share a host name -(host networking, two containers of one pod) never retire each other. An operator who confirmed its -process is gone retires it with `loom participant retire --participant --yes`, -which refuses one whose heartbeat is fresh or whose queries still run. The workers +drain. Each row carries its host name, kernel boot, container identity, pid namespace, +pid and start time. A later process on the same trusted host and pid namespace can +prove that the exact PID exited or was reused. A changed namespace, a missing pod +object, another pod UID or a terminal API phase does not prove physical termination. +`loom participant retire --participant ` performs that exact local verification +and refuses when evidence is unavailable or any cross-replica query remains. +The workers leader's `/v1/health` lists every participant a transition would wait on as `provider_participants_stale` (`heartbeat_expired` or `queries_running`). `system.processes` is read over `clusterAllReplicas(LOOM_CH_CLUSTER)`; a server without that cluster answers for itself, and any other failure of the read counts the participant as running. +Permanent retirement uses a separate process gate that never reopens. The outer +HTTP/WebSocket scope covers request-tail settlement; an inner scope covers actual +application tasks across middleware child-task boundaries, including OAuth and +control handlers. Worker passes and native command bodies use the same counter. +Shutdown closes admission, stops producers, waits for actual scope and registered +descendant completion, flushes buffers and checks carried and server-side writes. +Canceling a coroutine or exhausting the shutdown budget leaves its row nonretired. +A failed startup likewise cannot certify unfinished producers. Model-transition +`drained` is reversible and does not certify permanent retirement. + +The ninth `host_proof` field, `cell-retirement-v1`, identifies this lifecycle. +Only the guarded shutdown or verified-death writer publishes certified retirement. +The generic heartbeat writer cannot retire or reopen a retired identity; a +replacement process draws a fresh UUID. Pruning checks the latest certified row +before deleting all versions. Uncertified legacy retired rows stay visible and +cannot disappear through an age-based prune. Old reapers fail closed on the ninth +field, but old manual/pruning binaries must be stopped before the +[cells lifecycle baseline](cells.md#participant-lifecycle-baseline) is eligible. + Each bounded rebuild step holds the shared configuration Keeper key, renewed every 20 s (TTL 60 s). A contender takes over an expired key only when its holder's participant counts as stopped; an elapsed TTL alone is no proof. `loom diff --git a/docs/operations/backup-and-recovery.md b/docs/operations/backup-and-recovery.md index 02de0a5c91..b2dfa6a12c 100644 --- a/docs/operations/backup-and-recovery.md +++ b/docs/operations/backup-and-recovery.md @@ -504,11 +504,11 @@ Bounded exceptions: - The events stream (budget violations) is drained on `SIGTERM`. A `SIGKILL` or OOM loses at most one flush interval of that telemetry, never audit or memory rows. -- Provider reconfiguration waits on a dead process until a process on its host proves it - gone, or, on Kubernetes, until the workers leader sees its pod gone. Elsewhere, one - whose host is gone too (a move to a new host) is retired by hand with - `loom participant retire --participant --yes` - ([provider reconfiguration](../architecture/provider-reconfiguration.md)). +- Provider reconfiguration waits on a process until guarded retirement or exact + physical-death proof with settled queries. On its trusted host, run + `loom participant retire --participant `. Pod deletion and heartbeat expiry + are insufficient. If its identity is unknown, keep the row until the + [physically quiescent lifecycle recovery](../architecture/cells.md#participant-lifecycle-baseline). - Rate-limit bucket fill is per host. It refills in seconds, so a new host starts with full buckets. diff --git a/docs/operations/kubernetes.md b/docs/operations/kubernetes.md index 8eb2749050..3e31ef253e 100644 --- a/docs/operations/kubernetes.md +++ b/docs/operations/kubernetes.md @@ -189,17 +189,19 @@ graceful stop, and within the 30 s lease TTL when a pod or node dies. Every pod labelled `deploy:k8s:`, so a new leader runs the workers whose last pass is overdue rather than waiting out their interval again. -A pod that dies without stopping (node loss, eviction past the grace period) leaves its -provider participant row, which would hold the next embedding or model change from the -dashboard. With `retireLostPods` (the default) the workers leader retires it once its -heartbeat expired, no query of it runs and Kubernetes says the pod is gone: its object -deleted or replaced, or the pod `Failed` (evicted). That is a minute or two after a -deletion or an eviction, and after a node loss once Kubernetes deletes the node's pods -(five minutes of `NotReady` by default). It needs `get` on pods in the namespace (a Role -the chart creates for the release's service account) and that account's token, which only -the pods that may hold the workers lease mount. Without it, `/v1/health` names the row; -once the pod is gone, retire it with -`loom participant retire --participant --yes` ([replicas](./replicas.md#stopping-a-replica)). +A pod that dies without stopping leaves its provider participant row. A missing +or replaced pod object, a Failed/Succeeded API phase and a completed rollout do +not prove that an old process on a partitioned node stopped. Such rows remain +blockers until guarded shutdown or verified exact physical death with settled +queries. Loom requests no pod-read permission or service-account token for +participant retirement. The explicit `serviceAccount.automountToken` value stays +available for workloads that need a token for another integration. + +Use `loom participant retire --participant ` on the trusted host where its +exact PID/namespace/start identity is verifiable. If that evidence is unavailable, +use the [quiescent lifecycle recovery](../architecture/cells.md#participant-lifecycle-baseline) +after positively terminating or fencing every relevant runtime. API object +cleanup alone cannot authorize retirement. There is no migration Job. Every pod migrates at boot under a ClickHouse lease, so a Job would only repeat that step, and a pre-upgrade hook would run the new schema while @@ -388,7 +390,7 @@ NetworkPolicy whose one rule allows every pod and every address but those (none release: Helm refuses to take over an object it does not own, so the upgrade fails. - An engine that keeps address ranges off node IPs (Cilium's default) blocks host-network endpoints and the nodes' own addresses: NodeLocal DNSCache (so DNS), the EKS Pod - Identity agent, and the API server (which `retireLostPods` calls) where the engine + Identity agent, where the engine gives it an identity of its own. Turn on the engine's node matching, or set `guardMetadata: false` and allow them in your own policy. diff --git a/docs/operations/replicas.md b/docs/operations/replicas.md index f4c8729ef4..e635c0a490 100644 --- a/docs/operations/replicas.md +++ b/docs/operations/replicas.md @@ -101,14 +101,22 @@ it. ## Stopping a replica -Stop it gracefully (`docker stop`, SIGTERM): it marks its heartbeat row stopped -and retires its provider participant row. A forced kill leaves both; the -heartbeat row stops counting after three intervals. A process that restarts in -the same container retires the dead one's participant row itself. If a provider -reconfiguration waits on a dead process's row from elsewhere, retire it with the -`participant_id` its `/v1/health` reported, once you have confirmed it is gone -(the command refuses one whose heartbeat is fresh or whose queries still run): +Stop it gracefully (`docker stop`, SIGTERM). It first closes admission permanently, +stops its producers, and waits for actual request, worker and registered descendant +completion. It retires only after buffered, carried and server-side writes settle. +A canceled task that keeps executing or an exhausted shutdown budget leaves the +participant nonretired. A stopped or expired instance heartbeat is diagnostic. + +A trusted process on the same host and pid namespace can prove the exact old PID +exited or was reused, then retire it after all cross-replica queries settle: ```bash -loom participant retire --participant --yes +loom participant retire --participant ``` + +The command verifies retained boot, PID namespace and process-start identity; it +refuses remote or unknown identity evidence and live/uncountable queries. A +changed container namespace or Kubernetes API object is insufficient. An operator +assertion does not override these checks. For an inaccessible host, positively +terminate or fence the exact runtime before the documented +[quiescent lifecycle recovery](../architecture/cells.md#participant-lifecycle-baseline). diff --git a/docs/reference/cli.md b/docs/reference/cli.md index ff91f44681..3e0b9068ea 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -266,7 +266,7 @@ python scripts/codegen/docs_reference.py | `loom down` | Stop the local stack while preserving data; --delete-data performs a confirmed reset. | `--root`, `--file`, `--env-file`, `--delete-data`, `-y` | | `loom ch-setup` | Prepare a self-managed ClickHouse or a ClickHouse Cloud service for Loom: create the loom database and the read-only reader user, or print the SQL for an admin. | `--print-sql`, `--print-analyst-sql` | | `loom participant` | The Loom processes a provider change waits for. | | -| `loom participant retire` | Retire the provider participant of a process that died without shutting down (its id is `participant_id` in that process's /v1/health). | `--participant`, `--yes` | +| `loom participant retire` | Verify exact physical death on this trusted host and retire its participant after all queries settle (the id is participant_id in /v1/health). | `--participant` | | `loom deployment` | The configuration every replica checks itself against. | | | `loom deployment publish` | Publish this env's configuration as the deployment's, replacing its root secret or LOOM_EMBEDDING_MODEL/DIM on purpose. | `--yes`, `--check` | | `loom settings` | Every setting's value in force, and overrides of the live ones. | | diff --git a/docs/reference/errors.md b/docs/reference/errors.md index 62a4a86f46..792a3cd4ba 100644 --- a/docs/reference/errors.md +++ b/docs/reference/errors.md @@ -752,6 +752,10 @@ Raised in `server/router/app.py`. Category `capacity`, retryable: yes. Raised in `server/router/app.py`. Category `dependency`, retryable: yes. +#### `PROCESS_STOPPING` + +Raised in `server/transport/model_maintenance.py`, `server/transport/retirement.py`. + #### `PROJECTION_BEHIND` HTTP 503, raised as `server.memory.app.pg_admission.ProjectionBehindError`. 503 — earlier commits in this namespace are not yet in ClickHouse, so a read of ClickHouse that must see every write the namespace committed would miss them. Category `capacity`, retryable: yes for an idempotent op, no for another op once it ran. diff --git a/server/auth/tenancy.py b/server/auth/tenancy.py index 8c6b4c9437..5042cb22b5 100644 --- a/server/auth/tenancy.py +++ b/server/auth/tenancy.py @@ -262,7 +262,9 @@ async def _place_once(org: str) -> tuple[str, str]: request cancelled leaves the call running for the others.""" task = _placing.get(org) if task is None: - task = asyncio.create_task(_place(org)) + from server.core.background import create_deferred_task + + task = create_deferred_task(_place(org), name="tenancy-placement") _placing[org] = task task.add_done_callback(functools.partial(_placed, org)) return await asyncio.shield(task) diff --git a/server/boot.py b/server/boot.py index 69029d5b34..90ba996694 100644 --- a/server/boot.py +++ b/server/boot.py @@ -463,6 +463,9 @@ def _spawn_boot_loop(coro: Coroutine[Any, Any, Any], *, name: str) -> asyncio.Ta """A task that runs for the process's life: cancelled at shutdown, never registered as deferred, since a model transition's drain waits for those.""" task = asyncio.get_running_loop().create_task(coro, name=name) + from server.core.retirement import get_gate + + get_gate().track(task) _BOOT_TASKS.add(task) task.add_done_callback(_BOOT_TASKS.discard) return task diff --git a/server/cli/_ch.py b/server/cli/_ch.py index 042d8d1de0..184aa1f6eb 100644 --- a/server/cli/_ch.py +++ b/server/cli/_ch.py @@ -295,10 +295,7 @@ async def connect_declared( from server.core.pg_profile import check_configured await check_configured(ch) - if org_free is not None and not named: - await check_org_free(org_free) - else: - await check_orgs(named) + await check_acceptance(orgs=named, org_free=org_free) except BaseException: await disconnect(ch) raise @@ -315,6 +312,38 @@ async def connect_code_plane(ch: CHClient, args: argparse.Namespace) -> None: await connect_declared(ch, orgs=() if getattr(args, "dry_run", False) else (org,)) +async def check_acceptance( + *, orgs: Iterable[str | None] | None = None, org_free: OrgFree | None = None +) -> None: + """Fresh post-enrollment gate check on already verified, open paired stores.""" + named = _declared(orgs, org_free) + if org_free is not None and not named: + await check_org_free(org_free) + else: + await check_orgs(named) + + +async def connect_verified(ch: CHClient) -> None: + """Verify paired stores without obtaining an acceptance view or admitting work. + + Native commands enroll next, then repeat their accepting org/cell checks. + Operator-only commands may keep using the org-free checked composition. + """ + from server.core.pg_profile import check_configured + + if not ch.connected: + await ch.connect() + try: + await check_configured(ch) + except BaseException: + await disconnect(ch) + raise + from server import boot + from server.core.background import register_buffer_drainer + + register_buffer_drainer("boot-writes", boot.drain_writes) + + async def connect_checked( ch: CHClient, *, orgs: Iterable[str | None] | None = None, org_free: OrgFree | None = None ) -> None: @@ -354,9 +383,10 @@ def run_connected[T]( async def _go() -> T: ch = get_client() - await connect_checked(ch, orgs=named, org_free=org_free) + await connect_verified(ch) try: async with model_command(): + await check_acceptance(orgs=named, org_free=org_free) return await run(ch) finally: await disconnect(ch) diff --git a/server/cli/analytics_export.py b/server/cli/analytics_export.py index f69e3f9396..5f5abf8bb3 100644 --- a/server/cli/analytics_export.py +++ b/server/cli/analytics_export.py @@ -9,7 +9,7 @@ from datetime import date, timedelta from typing import Any -from server.cli._ch import connect_declared, disconnect, reads +from server.cli._ch import check_acceptance, connect_verified, disconnect, reads from server.core.datetimes import utc_now as _now # The first botocore whose put_object takes IfNoneMatch (the `aws` extra's floor). @@ -90,15 +90,20 @@ def new_client() -> Any: check_credentials(role_arn=args.role_arn, region=args.region) put = s3_put(new_client, args.bucket, expected_owner=args.expected_bucket_owner) - # No model calls and no writes to ClickHouse: a plain connection, outside - # the provider barrier that `run_connected` enters. It connects inside the - # deadline's scope, so the driver's read timeout is the deadline too. + # Analytics export consumes org data even without inference; its entire native + # body participates in permanent retirement without taking model admission. + from server.insights.app.provider_command import participant_command + async def _go() -> list[Exported]: with statement_timeout(STATEMENT_TIMEOUT_S): ch = get_client() - await connect_declared(ch, org_free=reads("reads; writes only the export bucket")) + await connect_verified(ch) try: - return await export_day(day, prefix=args.prefix, put=put, name_key=name_key, ch=ch) + async with participant_command(): + await check_acceptance(org_free=reads("reads; writes only the export bucket")) + return await export_day( + day, prefix=args.prefix, put=put, name_key=name_key, ch=ch + ) finally: await disconnect(ch) diff --git a/server/cli/code_run.py b/server/cli/code_run.py index 5479c3d851..1074951fa1 100644 --- a/server/cli/code_run.py +++ b/server/cli/code_run.py @@ -117,10 +117,11 @@ async def _then_disconnect(coro: asyncio.Future[int] | Coroutine[Any, Any, int]) try: # A repository added from the Repositories page lives in the registry, not in # LOOM_CODE_REPOS_JSON: read it before the subcommand resolves its namespace. + from server.cli._ch import connect_verified from server.code.app.index.repo_registry import REGISTRY_UNREAD_MESSAGE, refresh_db_repos from server.code.domain.index.repos import RegistryUnread - await connect_declared(get_client(), org_free=reads("reads the repository registry")) + await connect_verified(get_client()) async with model_command(): # Never raises; while it has never succeeded, `configured_repos` refuses. await refresh_db_repos(force=True) diff --git a/server/cli/import_clickgap.py b/server/cli/import_clickgap.py index 60f3fdc6fc..983ce671cd 100644 --- a/server/cli/import_clickgap.py +++ b/server/cli/import_clickgap.py @@ -716,15 +716,16 @@ async def _clickgap_write_category_links(w: _ClickgapWrite) -> None: async def _clickgap_enroll(stack: Any, org: str) -> Any: """Connect and enroll before selecting the importer's effective embedder.""" from server import boot - from server.cli._ch import connect_checked + from server.cli._ch import check_acceptance, connect_verified from server.core.background import register_buffer_drainer from server.core.ch import get_client from server.embedding import get_embedder from server.insights.app.provider_command import model_command - await connect_checked(get_client(), orgs=(org,)) + await connect_verified(get_client()) register_buffer_drainer("boot-writes", boot.drain_writes) await stack.enter_async_context(model_command()) + await check_acceptance(orgs=(org,)) return get_embedder() diff --git a/server/cli/import_recall.py b/server/cli/import_recall.py index fd0925accb..b24c9371b5 100644 --- a/server/cli/import_recall.py +++ b/server/cli/import_recall.py @@ -781,22 +781,29 @@ async def _http_writer(op: str, payload: dict[str, Any]) -> None: return _http_writer, client.aclose - from server.cli._ch import connect_checked, disconnect + from server.cli._ch import check_acceptance, connect_verified, disconnect from server.core.ch import get_client handlers = inproc_ops() ch = get_client() - await connect_checked(ch, orgs=(args.org,)) + await connect_verified(ch) from server import boot from server.core.background import register_buffer_drainer from server.insights.app.provider_command import model_command register_buffer_drainer("boot-writes", boot.drain_writes) command = model_command() + entered = False try: await command.__aenter__() + entered = True + await check_acceptance(orgs=(args.org,)) except BaseException: - await disconnect(ch) + try: + if entered: + await command.__aexit__(None, None, None) + finally: + await disconnect(ch) raise async def _inproc_writer(op: str, payload: dict[str, Any]) -> None: diff --git a/server/cli/maintenance.py b/server/cli/maintenance.py index 53cb8e4004..9a6cdb2437 100644 --- a/server/cli/maintenance.py +++ b/server/cli/maintenance.py @@ -207,8 +207,9 @@ def _cmd_reextract(args: argparse.Namespace) -> int: the stored transcripts, the source untouched.""" import asyncio - from server.cli._ch import connect_declared, disconnect + from server.cli._ch import check_acceptance, connect_verified, disconnect from server.core.ch import get_client + from server.insights.app.provider_command import participant_command from server.memory.app.reextract import load_sessions, render_report, replay if args.into == args.namespace: @@ -225,33 +226,37 @@ def _cmd_reextract(args: argparse.Namespace) -> int: async def _go() -> int: ch = get_client() - await connect_declared( - ch, org_free=reads("reads; the replay writes through the server at --base-url") - ) + await connect_verified(ch) try: - sessions = await load_sessions(ch, org=args.org, namespace=args.namespace) + async with participant_command(): + await check_acceptance( + org_free=reads("reads; replay writes through the HTTP server") + ) + sessions = await load_sessions(ch, org=args.org, namespace=args.namespace) + if args.limit_sessions: + sessions = sessions[: args.limit_sessions] + if not sessions: + print( + f" no set_from_messages evidence in {args.org}/{args.namespace}", + file=sys.stderr, + ) + return 1 + rep = await replay( + sessions, + base_url=args.base_url, + token=token, + org=args.org, + into=args.into, + dates=args.dates, + concurrency=args.concurrency, + dry_run=args.dry_run, + ) + print( + render_report(rep, source=args.namespace, into=args.into, dry_run=args.dry_run) + ) + return 1 if rep.failed and rep.failed == rep.sessions else 0 finally: await disconnect(ch) - if args.limit_sessions: - sessions = sessions[: args.limit_sessions] - if not sessions: - print( - f" no set_from_messages evidence in {args.org}/{args.namespace}", - file=sys.stderr, - ) - return 1 - rep = await replay( - sessions, - base_url=args.base_url, - token=token, - org=args.org, - into=args.into, - dates=args.dates, - concurrency=args.concurrency, - dry_run=args.dry_run, - ) - print(render_report(rep, source=args.namespace, into=args.into, dry_run=args.dry_run)) - return 1 if rep.failed and rep.failed == rep.sessions else 0 return asyncio.run(_go()) diff --git a/server/cli/participant.py b/server/cli/participant.py index e42832b6af..9154bf4be4 100644 --- a/server/cli/participant.py +++ b/server/cli/participant.py @@ -1,10 +1,9 @@ """`loom participant retire`: the manual recovery for a Loom process that died without shutting down. -Every Loom process registers a provider participant row, and a provider -reconfiguration waits until each live participant acknowledges the drain. A -participant that stops beating expires by itself; this retires one at once, -once an operator has confirmed its process is gone. +A participant retires only after permanent guarded shutdown or verifiable exact +physical death on a trusted host, with its cross-replica queries settled. +Heartbeat age and an operator affirmation are not evidence of death. """ from __future__ import annotations @@ -17,8 +16,7 @@ def _cmd_retire(args: argparse.Namespace) -> int: - """Retire a provider participant row whose process died without shutting - down, so a provider reconfiguration need not wait for it to expire.""" + """Verify exact process death and retire the participant after its queries settle.""" from uuid import UUID from server.cli._ch import run_connected @@ -28,13 +26,6 @@ def _cmd_retire(args: argparse.Namespace) -> int: except ValueError: print(f" ✗ {args.participant!r} is not a participant id", file=sys.stderr) return 2 - if not args.yes: - print( - " ✗ retiring a participant that is still running lets a reconfiguration proceed " - "under its writes; confirm its process is stopped, then pass --yes", - file=sys.stderr, - ) - return 2 async def _go(ch: Any) -> str: from server.insights.app.provider_barrier import retire_participant @@ -47,8 +38,9 @@ async def _go(ch: Any) -> str: return 1 if outcome == "live": print( - f" ✗ participant {participant} is still running: its heartbeat is fresh or " - "a query of its own still runs", + f" ✗ participant {participant} has no verifiable exact physical-death proof " + "or its cross-replica queries have not settled; run on its trusted host after " + "the exact process stops. Expiry and Kubernetes object deletion are not proof", file=sys.stderr, ) return 1 @@ -61,9 +53,8 @@ def register_participant_cli(sub: Any) -> None: ssub = p.add_subparsers(dest="participant_cmd", required=True) r = ssub.add_parser( "retire", - help="Retire the provider participant of a process that died without shutting " - "down (its id is `participant_id` in that process's /v1/health).", + help="Verify exact physical death on this trusted host and retire its participant " + "after all queries settle (the id is participant_id in /v1/health).", ) r.add_argument("--participant", required=True, help="The participant id to retire") - r.add_argument("--yes", action="store_true", help="Its process is confirmed stopped") r.set_defaults(func=_cmd_retire) diff --git a/server/cli/sql_access.py b/server/cli/sql_access.py index 8499eed5cd..6a727c9e7b 100644 --- a/server/cli/sql_access.py +++ b/server/cli/sql_access.py @@ -70,14 +70,17 @@ def _connected( ) -> Any: """Run `work(ch)` on one plain connection: the command calls no model. `orgs` or `org_free` as `server.cli._ch.connect_declared` takes them.""" - from server.cli._ch import connect_declared, disconnect + from server.cli._ch import check_acceptance, connect_verified, disconnect from server.core.ch import get_client + from server.insights.app.provider_command import participant_command async def _go() -> Any: ch = get_client() - await connect_declared(ch, orgs=orgs, org_free=org_free) + await connect_verified(ch) try: - return await work(ch) + async with participant_command(): + await check_acceptance(orgs=orgs, org_free=org_free) + return await work(ch) finally: await disconnect(ch) diff --git a/server/core/cell.py b/server/core/cell.py index 2aaea0f529..a37ce9e029 100644 --- a/server/core/cell.py +++ b/server/core/cell.py @@ -272,6 +272,12 @@ def state(self) -> str: _awaiting_first_read = False +def discard_inherited_view() -> None: + """A child enrolls and reads primary state before using acceptance state.""" + global _view, _memo, _awaiting_first_read + _view, _memo, _awaiting_first_read = _View(), None, True + + def expect_first_read() -> None: global _awaiting_first_read _awaiting_first_read = True diff --git a/server/core/instances.py b/server/core/instances.py index 27a35d8ec6..ecbe666c07 100644 --- a/server/core/instances.py +++ b/server/core/instances.py @@ -142,9 +142,12 @@ def cell_row() -> dict[str, Any]: `marked` (whether this process's pooled sessions keep the tenancy marker, null until a cell-state read lands), and with LOOM_CELL set, until the `first` row is read and then on a first cell only, `router` and `router_why` (`cell_router.router_key`).""" + from server.core.provider_process import process_identity + reported = { **cell.as_value(cell.mine()), "serves": bool(settings.serves_traffic), + "participant_id": str(process_identity()), "marked": cell.marked(), **cell_router.router_key(), } diff --git a/server/core/provider_process.py b/server/core/provider_process.py index 6ef0d6af5a..a2c6ad6328 100644 --- a/server/core/provider_process.py +++ b/server/core/provider_process.py @@ -5,11 +5,9 @@ still has a query running. """ -import asyncio import os import re import socket -from pathlib import Path from uuid import UUID, uuid4 _identity: tuple[int, UUID] | None = None @@ -99,11 +97,14 @@ def host_proof() -> str: def proves_stopped(proof: str) -> bool: """Whether the process `proof` names has stopped, judged from this one: same host name and kernel boot, and either its pid is free or runs another process than - the one that started then (same pid namespace), or it ran in this container - before the container restarted into a new pid namespace (which kills every - process of the old one). Two containers that share a host name never prove - each other stopped; a process that cannot be inspected is not proven stopped.""" + the one that started then (same pid namespace). Another namespace or API + object proves nothing. Sharing a host name or container identity alone does + not prove death; an uninspectable process is not proven stopped.""" parts = proof.split("|") + if len(parts) == 9: + if parts[8] != LIFECYCLE: + return False + parts = parts[:8] if len(parts) not in (6, 7, 8): return False host, boot, container, pidns, pid, start = parts[:6] @@ -114,7 +115,7 @@ def proves_stopped(proof: str) -> bool: if host != socket.gethostname() or boot != my_boot: return False if pidns != my_pidns: - return bool(container) and container == _container() + return False # namespace identity alone says nothing about old processes # Only positive evidence: no such process, or another one under its pid. # A pid this process may not inspect (hidepid, a permission error) proves nothing. try: @@ -129,45 +130,36 @@ def proves_stopped(proof: str) -> bool: return False -_SA = "/var/run/secrets/kubernetes.io/serviceaccount" -_POD_CONTAINER = re.compile(r"^([0-9a-f-]{36})/") +LIFECYCLE = "cell-retirement-v1" -async def pod_gone(proof: str, *, timeout_s: float = 5.0) -> bool: - """Whether Kubernetes says the pod `proof` names is gone: no pod of that name - in its namespace, one with another UID, or this one in phase Failed or - Succeeded (an evicted pod keeps its object; a pod that restarts its - containers reaches those phases only once every one stopped for good). The - kubelet removes a pod's object once its containers stopped (or an operator - force-deleted it), so each is positive evidence a lost pod's process ended. - Asked only of this pod's own namespace, with its service-account token; - False wherever the API or the proof does not say.""" +def certified(proof: str) -> str: + """Preserve the exact captured identity and stamp the installed protocol.""" parts = proof.split("|") - found = _POD_CONTAINER.match(parts[2]) if len(parts) == 8 else None - namespace = os.environ.get("POD_NAMESPACE", "") - if not found or not namespace or parts[6] != namespace or not parts[7]: + if len(parts) == 9 and parts[8] == LIFECYCLE: + parts = parts[:8] + if len(parts) not in (6, 7, 8): + raise ValueError("Unrecognized participant identity proof") + parts += [""] * (8 - len(parts)) + if not all(parts[i] for i in (0, 1, 3, 4, 5)): + raise ValueError("Incomplete participant identity proof") + if not parts[4].isdecimal() or int(parts[4]) <= 0 or not parts[5].isdecimal(): + raise ValueError("Invalid participant process identity") + return "|".join((*parts, LIFECYCLE)) + + +def has_certificate(proof: str) -> bool: + if len(proof.split("|")) != 9: return False - host = os.environ.get("KUBERNETES_SERVICE_HOST", "") - port = os.environ.get("KUBERNETES_SERVICE_PORT", "443") try: - token = (await asyncio.to_thread(Path(f"{_SA}/token").read_text, "ascii")).strip() - except OSError: - return False - if not host or not token: + return certified(proof) == proof + except ValueError: return False - import httpx - url = f"https://{host}:{port}/api/v1/namespaces/{namespace}/pods/{parts[7]}" - try: - async with httpx.AsyncClient(verify=f"{_SA}/ca.crt", timeout=timeout_s) as client: - r = await client.get(url, headers={"Authorization": f"Bearer {token}"}) - except (httpx.HTTPError, OSError): - return False - if r.status_code == 404: - return True - if r.status_code != 200: - return False - pod = r.json() - if str(pod.get("metadata", {}).get("uid", "")) != found.group(1): - return True - return str(pod.get("status", {}).get("phase", "")) in ("Failed", "Succeeded") + +def participant_proof() -> str: + """Advertise capability only in a composition with installed lifecycle guards.""" + from server.core.retirement import get_gate + + proof = host_proof() + return certified(proof) if proof and get_gate().role else proof diff --git a/server/core/retirement.py b/server/core/retirement.py new file mode 100644 index 0000000000..025a437362 --- /dev/null +++ b/server/core/retirement.py @@ -0,0 +1,114 @@ +"""Permanent process admission and the evidence passed to the retirement writer. + +Model maintenance can reopen. This gate never does: the participant identity is +finished once shutdown begins, even when its shutdown budget expires. +""" + +from __future__ import annotations + +import asyncio +import os +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from dataclasses import dataclass +from typing import Any +from uuid import UUID + +from server.core import provider_process + + +class ProcessStopping(RuntimeError): + """This participant can never accept another unit of work.""" + + +class RetirementGate: + """One event loop; checking admission and incrementing never await.""" + + def __init__(self) -> None: + self.active = 0 + self.closed = False + self.role = "" + self.enrolled = False + self.producers_stopped = False + self.producers: set[asyncio.Task[Any]] = set() + + def install(self, role: str) -> None: + if self.closed or self.role not in ("", role): + raise ProcessStopping("A retired participant cannot enroll again") + self.role = role + + def close(self) -> None: + self.closed = True + + def track[T](self, task: asyncio.Task[T]) -> asyncio.Task[T]: + """A process-lifetime producer must actually end, not merely be canceled.""" + self.producers.add(task) + task.add_done_callback(self.producers.discard) + if self.closed: + task.cancel() + raise ProcessStopping("Cannot start a producer during shutdown") + return task + + def stopped(self) -> None: + """Composition calls this after stopping every role-specific producer.""" + self.close() + self.producers_stopped = True + + @property + def quiescent(self) -> bool: + return ( + self.closed + and bool(self.role) + and self.producers_stopped + and self.active == 0 + and all(task.done() for task in self.producers) + ) + + @asynccontextmanager + async def admit(self) -> AsyncIterator[None]: + if self.closed or (self.role and not self.enrolled): + raise ProcessStopping("This process is stopping or has not enrolled") + self.active += 1 + try: + yield + finally: + self.active -= 1 + + +_pid = os.getpid() +_gate = RetirementGate() + + +def get_gate() -> RetirementGate: + """A fork cannot inherit an accepting view or its parent's retirement state.""" + global _pid, _gate + if _pid != os.getpid(): + role = _gate.role + _pid, _gate = os.getpid(), RetirementGate() + _gate.role = role + return _gate + + +@dataclass(frozen=True) +class RetirementProof: + """Internal evidence; only guarded shutdown or verified death constructs it.""" + + participant: UUID + host_proof: str + after: int + version: int | None = None + + +def guarded_proof(participant: UUID, *, version: int) -> RetirementProof: + gate = get_gate() + proof = provider_process.host_proof() + if not gate.quiescent or not gate.enrolled or not proof: + raise ProcessStopping("Permanent guarded quiescence is not established") + return RetirementProof(participant, provider_process.certified(proof), version - 1, version) + + +def death_proof(participant: UUID, proof: str, *, after: int) -> RetirementProof | None: + """Called only after the complete cross-replica running-query census is zero.""" + if not provider_process.proves_stopped(proof): + return None + return RetirementProof(participant, provider_process.certified(proof), after) diff --git a/server/insights/app/provider_barrier.py b/server/insights/app/provider_barrier.py index 503cd021e2..193fcc9335 100644 --- a/server/insights/app/provider_barrier.py +++ b/server/insights/app/provider_barrier.py @@ -11,6 +11,7 @@ import asyncio import contextlib import logging +import os import threading from collections.abc import AsyncIterator, Callable from contextlib import AbstractAsyncContextManager, asynccontextmanager @@ -18,10 +19,11 @@ from typing import Any from uuid import UUID +from server.core import retirement from server.core.background import drain_task_sets, flush_registered_buffers from server.core.ch import query_tag from server.core.model_work_gate import get_model_work_gate -from server.core.provider_process import host_proof, pod_gone, process_identity, proves_stopped +from server.core.provider_process import participant_proof, process_identity, proves_stopped from server.insights.store.provider_participants import ( OWN_VERSION_SQL, SEQUENTIAL, @@ -30,6 +32,7 @@ participants, prune_retired, record_participant, + record_retirement, running_queries, ) @@ -62,6 +65,8 @@ def stamp( ) -> tuple[UUID, str, int]: with self.lock: if transition is not None and state is not None: + if self.state == "retired" and state != "retired": + raise retirement.ProcessStopping("A retired UUID cannot re-enroll") self.transition, self.state = transition, state self.version = max(time_ns(), self.version + 1, floor + 1) return self.transition, self.state, self.version @@ -109,7 +114,7 @@ def beat(self) -> None: transition, state, version = _OWN.stamp(floor=seen) if state == "retired": return - row = [me, transition, state, version, STALE_AFTER_S, host_proof()] + row = [me, transition, state, version, STALE_AFTER_S, participant_proof()] self.ch.insert(TABLE, [row], column_names=_COLUMNS, settings=_TAG) def _run(self) -> None: @@ -146,6 +151,7 @@ def signal_stop(self) -> None: _HEARTBEAT = _Heartbeat() _registered = False +_process_pid = os.getpid() async def register_process() -> None: @@ -155,11 +161,20 @@ async def register_process() -> None: startup loads the pending gate or the committed replacement before admission. A second call (boot enrolls before its migrations) keeps the state. """ - global _registered + global _registered, _process_pid, _OWN, _HEARTBEAT + if _process_pid != os.getpid(): + _process_pid = os.getpid() + _registered, _OWN, _HEARTBEAT = False, _Own(), _Heartbeat() + from server.core import cell + + cell.discard_inherited_view() + if retirement.get_gate().closed: + raise retirement.ProcessStopping("A retired participant cannot re-enroll") if not _registered: _OWN.stamp(UUID(int=0), "running") await asyncio.to_thread(_HEARTBEAT.start) _registered = True + retirement.get_gate().enrolled = True async def enroll_before_migrations(ch: Any) -> None: @@ -216,18 +231,24 @@ async def distributed_drain(transition: UUID) -> bool: async def retire_process() -> bool: - """Retire only when no admitted or registered deferred writer remains, and no - write of this process can still land on the server.""" - global _registered - if get_model_work_gate().active or not await drain_task_sets(timeout_s=0): + """Permanently close admission; certify only actually completed role work.""" + gate = retirement.get_gate() + gate.close() + _HEARTBEAT.signal_stop() + if not gate.quiescent or get_model_work_gate().active: + return False + if not await drain_task_sets(timeout_s=0): + return False + if not await flush_registered_buffers() or not await drain_task_sets(timeout_s=0): return False if not await _server_side_settled(): return False - _HEARTBEAT.signal_stop() + if _HEARTBEAT.running or await running_queries(str(process_identity())) != 0: + return False _, _, version = _OWN.stamp(UUID(int=0), "retired") + proof = retirement.guarded_proof(process_identity(), version=version) with query_tag(_TAG["log_comment"]): - await record_participant(process_identity(), UUID(int=0), "retired", version=version) - _registered = False + await record_retirement(proof) return True @@ -256,7 +277,7 @@ async def retire_at_shutdown(ch: Any, *, budget_s: float = SHUTDOWN_RETIRE_S) -> log.warning( "provider participant not retired at shutdown: %d carried write(s) unsettled " "or an insert of this process still running; retire it with " - "`loom participant retire --participant %s --yes` once it stopped", + "`loom participant retire --participant %s` on its trusted host once it stopped", rmt_carry.unsettled(), process_identity(), ) @@ -268,7 +289,7 @@ async def retire_at_shutdown(ch: Any, *, budget_s: float = SHUTDOWN_RETIRE_S) -> def participant_lifespan[T]( lifespan: Callable[[T], AbstractAsyncContextManager[None]], ) -> Callable[[T], AbstractAsyncContextManager[None]]: - """A boot that fails after enrolling retires its row instead of leaving one per attempt.""" + """A failed boot stops known producers but never certifies an incomplete shutdown.""" @asynccontextmanager async def wrapped(app: T) -> AsyncIterator[None]: @@ -279,6 +300,10 @@ async def wrapped(app: T) -> AsyncIterator[None]: yield except BaseException: if not started and _registered: + gate = retirement.get_gate() + gate.close() + for producer in tuple(gate.producers): + producer.cancel() try: await retire_process() except Exception as e: # noqa: BLE001 — the row expires by heartbeat instead @@ -299,11 +324,16 @@ async def retire_participant(participant: UUID, *, ch: Any = None) -> str: return "absent" if str(participant) == str(process_identity()): return "live" - if not row.get("expired"): + if not proves_stopped(str(row.get("host_proof") or "")): return "live" if await running_queries(participant, ch=ch) != 0: return "live" - await record_participant(participant, UUID(int=0), "retired", after=int(row["version"]), ch=ch) + proof = retirement.death_proof( + participant, str(row.get("host_proof") or ""), after=int(row["version"]) + ) + if proof is None: + return "live" + await record_retirement(proof, ch=ch) return "retired" @@ -328,9 +358,12 @@ async def retire_dead_here(*, now: float | None = None) -> int: continue if await running_queries(participant) != 0: continue - await record_participant( - UUID(participant), UUID(int=0), "retired", after=int(row["version"]) + proof = retirement.death_proof( + UUID(participant), str(row.get("host_proof") or ""), after=int(row["version"]) ) + if proof is None: + continue + await record_retirement(proof) log.info("provider participant %s retired: its process on this host is gone", participant) retired += 1 return retired @@ -348,10 +381,7 @@ def stale_participants() -> tuple[dict[str, str], ...] | None: async def sweep_stopped(*, now: float | None = None) -> None: - """Retire the expired participants with no query left whose Kubernetes pod is - gone (`pod_gone`), and remember the rest that block a reconfiguration (only - their process, Kubernetes or an operator can say they stopped). The workers - leader's job.""" + """Remember stale blockers; only exact verified death authorizes retirement.""" global _STALE, _last_sweep, _last_prune t = monotonic() if now is None else now if _STALE is not None and t - _last_sweep < SWEEP_S: @@ -369,11 +399,15 @@ async def sweep_stopped(*, now: float | None = None) -> None: entry = {"participant_id": participant, "heartbeat_at": str(row.get("heartbeat_at", ""))} if row.get("expired"): running = await running_queries(participant) != 0 - if not running and await pod_gone(str(row.get("host_proof") or "")): - await record_participant( - UUID(participant), UUID(int=0), "retired", after=int(row["version"]) + proof = ( + None + if running + else retirement.death_proof( + UUID(participant), str(row.get("host_proof") or ""), after=int(row["version"]) ) - log.info("provider participant %s retired: Kubernetes deleted its pod", participant) + ) + if proof is not None: + await record_retirement(proof) continue stale.append({**entry, "reason": "queries_running" if running else "heartbeat_expired"}) _STALE = tuple(stale) diff --git a/server/insights/app/provider_command.py b/server/insights/app/provider_command.py index 46ec2bb9a5..b89c418e17 100644 --- a/server/insights/app/provider_command.py +++ b/server/insights/app/provider_command.py @@ -6,7 +6,7 @@ from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import asynccontextmanager -from server.core import settings_overrides +from server.core import retirement, settings_overrides from server.core.background import drain_task_sets, flush_registered_buffers from server.core.ch import set_query_tag from server.core.model_work_gate import get_model_work_gate @@ -16,6 +16,8 @@ log = logging.getLogger(__name__) +COMMAND_DRAIN_S = 15.0 # bounded cleanup; timeout leaves the participant unretired + async def _adopt_overrides(*, failing: bool = False) -> bool: """Whether the stored overrides loaded. A failure is logged when a run of them @@ -41,22 +43,44 @@ async def _keep_overrides( @asynccontextmanager -async def model_command() -> AsyncIterator[None]: - """A running command blocks a rebuild; a newborn loads the gate first. The - deployment's live-setting overrides apply before the work starts, as they do - in every server process, so a native command and the HTTP op agree.""" +async def participant_command() -> AsyncIterator[None]: + """Register all native org/principal work, including commands without models.""" + retirement.get_gate().install("command") await register_process() - keeper: asyncio.Task[None] | None = None try: - await load_active_configuration() - loaded = await _adopt_overrides() - keeper = asyncio.create_task(_keep_overrides(loaded), name="loom-cli-settings-overrides") - async with get_model_work_gate().admit(): + async with retirement.get_gate().admit(): yield finally: - if keeper is not None: - keeper.cancel() - with contextlib.suppress(asyncio.CancelledError): - await keeper - if await drain_task_sets(timeout_s=15) and await flush_registered_buffers(): + gate = retirement.get_gate() + gate.close() + from server.memory.app import embedder_calibration + + embedder_calibration.stop_refresh_loop() + producers = tuple(gate.producers) + for producer in producers: + producer.cancel() + if producers: + await asyncio.wait(producers, timeout=COMMAND_DRAIN_S) + gate.stopped() + if await drain_task_sets(timeout_s=COMMAND_DRAIN_S) and await flush_registered_buffers(): await retire_process() + + +@asynccontextmanager +async def model_command() -> AsyncIterator[None]: + """The permanent lifecycle also holds the reversible model-work admission.""" + async with participant_command(): + keeper: asyncio.Task[None] | None = None + try: + await load_active_configuration() + loaded = await _adopt_overrides() + keeper = asyncio.create_task( + _keep_overrides(loaded), name="loom-cli-settings-overrides" + ) + async with get_model_work_gate().admit(): + yield + finally: + if keeper is not None: + keeper.cancel() + with contextlib.suppress(asyncio.CancelledError): + await keeper diff --git a/server/insights/store/provider_participants.py b/server/insights/store/provider_participants.py index 256a40babb..da04df218d 100644 --- a/server/insights/store/provider_participants.py +++ b/server/insights/store/provider_participants.py @@ -6,12 +6,18 @@ from server.core.ch import sequential_reads from server.core.ch_errors import error_code -from server.core.provider_process import host_proof, participant_tag, process_identity +from server.core.provider_process import ( + has_certificate, + participant_proof, + participant_tag, + process_identity, +) from server.core.repo_base import _CHClient, _default_client +from server.core.retirement import RetirementProof, get_gate from server.settings import settings TABLE = "loom._provider_participants" -# A participant whose last heartbeat is older than this may count as stopped. +# Heartbeat age is diagnostic; it never establishes permanent retirement. STALE_AFTER_S = 60 # `expired` is decided on the server's clock, like the leases. @@ -19,7 +25,7 @@ "SELECT participant_id, transition_id, state, version, heartbeat_at, " "host_proof, " "heartbeat_at + toIntervalSecond(stale_after_s) < now64(3) AS expired " - f"FROM {TABLE} FINAL WHERE state != 'retired'" + f"FROM {TABLE} FINAL" ) ONE_SQL = ( "SELECT state, version, " @@ -58,7 +64,12 @@ def running_sql(*, inserts: bool = False) -> str: # Retired participants whose last row is older than this are deleted. PRUNE_AFTER_H = 24 _PRUNE_IDS_SQL = ( - f"SELECT participant_id FROM {TABLE} FINAL WHERE state = 'retired' " + f"SELECT participant_id, state, version, host_proof FROM {TABLE} FINAL WHERE state = 'retired' " + "AND length(splitByChar('|', host_proof)) = 9 " + "AND arrayElement(splitByChar('|', host_proof), 9) = 'cell-retirement-v1' " + "AND arrayAll(i -> notEmpty(arrayElement(splitByChar('|', host_proof), i)), [1,2,4,5,6]) " + "AND match(arrayElement(splitByChar('|', host_proof), 5), '^[1-9][0-9]*$') " + "AND match(arrayElement(splitByChar('|', host_proof), 6), '^[0-9]+$') " "AND heartbeat_at < now64(3) - toIntervalHour({hours:UInt32}) LIMIT 1000" ) _PRUNE_SQL = f"ALTER TABLE {TABLE} DELETE WHERE participant_id IN {{ids:Array(UUID)}}" @@ -66,7 +77,12 @@ def running_sql(*, inserts: bool = False) -> str: async def participants(*, ch: _CHClient | None = None) -> list[dict[str, Any]]: with sequential_reads(): - return await (ch or _default_client()).query_all(PARTICIPANTS_SQL) + rows = await (ch or _default_client()).query_all(PARTICIPANTS_SQL) + return [ + r + for r in rows + if r["state"] != "retired" or not has_certificate(str(r.get("host_proof") or "")) + ] async def record_participant( @@ -80,9 +96,13 @@ async def record_participant( ) -> None: """`heartbeat_at` takes the server's clock (its column default); this process's own row carries its `host_proof`.""" - if state not in ("running", "drained", "retired"): + if state not in ("running", "drained"): raise ValueError("Unknown participant state") own = str(identity) == str(process_identity()) + if not own: + raise ValueError("A process writes only its own running/drained registration") + if get_gate().closed: + raise ValueError("A permanently closed participant cannot re-enroll") await (ch or _default_client()).insert( TABLE, [ @@ -94,21 +114,58 @@ async def record_participant( # A retired row is stopped at once, so every build's `provider-lock clear` # frees its holder without waiting out the heartbeat window. "stale_after_s": 0 if state == "retired" else STALE_AFTER_S, - "host_proof": host_proof() if own else "", + "host_proof": participant_proof() if own else "", + } + ], + ) + + +async def record_retirement(proof: RetirementProof, *, ch: _CHClient | None = None) -> None: + """The sole retired-row writer, passed evidence from the guarded lifecycle.""" + if not isinstance(proof, RetirementProof) or not has_certificate(proof.host_proof): + raise ValueError("Retirement requires verified process evidence") + await (ch or _default_client()).insert( + TABLE, + [ + { + "participant_id": proof.participant, + "transition_id": UUID(int=0), + "state": "retired", + "version": max(time_ns(), proof.after + 1, proof.version or 0), + "stale_after_s": 0, + "host_proof": proof.host_proof, } ], ) async def prune_retired(*, ch: _CHClient | None = None) -> int: - """Delete every row of the participants retired `PRUNE_AFTER_H` ago (each - process start leaves one); a process that was only paused enrolls again - with its next beat. Returns how many.""" + """Age selects already certified permanent retirements, never unknown rows. + + Retired UUIDs cannot re-enroll. An earlier in-flight beat is below the final + version; removing every version cannot resurrect that beat. + """ client = ch or _default_client() - rows = await client.query_all(_PRUNE_IDS_SQL, {"hours": PRUNE_AFTER_H}) - ids = [r["participant_id"] for r in rows] - if ids: - await client.execute(_PRUNE_SQL, {"ids": ids}) + with sequential_reads(): + rows = await client.query_all(_PRUNE_IDS_SQL, {"hours": PRUNE_AFTER_H}) + ids = [r["participant_id"] for r in rows if has_certificate(str(r.get("host_proof") or ""))] + if not ids: + return 0 + latest = await client.query_all( + f"SELECT participant_id, state, version, host_proof FROM {TABLE} FINAL " + "WHERE participant_id IN {ids:Array(UUID)}", + {"ids": ids}, + ) + expected = {str(r["participant_id"]): int(r["version"]) for r in rows} + ids = [ + r["participant_id"] + for r in latest + if r["state"] == "retired" + and int(r["version"]) == expected.get(str(r["participant_id"])) + and has_certificate(str(r.get("host_proof") or "")) + ] + if ids: + await client.execute(_PRUNE_SQL, {"ids": ids}) return len(ids) diff --git a/server/main.py b/server/main.py index 647e18b1e5..d7a3ffc2f2 100644 --- a/server/main.py +++ b/server/main.py @@ -29,6 +29,7 @@ ) from server.code.app.index.repo_modes import repo_umask from server.core import readiness as _readiness +from server.core import retirement from server.core.ch import get_client, probe_reader_user from server.core.errors import classified from server.core.logging_config import configure_logging @@ -46,6 +47,22 @@ wire() +def verify_retirement_topology() -> None: + """Capability requires both sides of the request-tail and child-task boundaries.""" + from server.core.request_tail import RequestTailMiddleware + from server.transport.model_maintenance import ModelMaintenanceMiddleware + from server.transport.retirement import RetirementMiddleware + + classes: list[object] = [middleware.cls for middleware in app.user_middleware] + if not ( + bool(classes) + and all(cls in classes for cls in (RequestTailMiddleware, ModelMaintenanceMiddleware)) + and classes[0] is RetirementMiddleware + and classes.index(RequestTailMiddleware) < classes.index(ModelMaintenanceMiddleware) + ): + raise RuntimeError("Retirement middleware must wrap the request tail and application") + + async def _shut_down(ch: Any, lease: Any, supervisor: Any, emitter: Any) -> None: """The bounded part of a shutdown: the instance row's stop write and the drains share one budget, and the final cleanup gets what they left unused, so the whole @@ -56,24 +73,35 @@ async def _shut_down(ch: Any, lease: Any, supervisor: Any, emitter: Any) -> None started = time.monotonic() await _instances.stop(ch) left = max(0.0, boot.SHUTDOWN_DRAIN_BUDGET_S - (time.monotonic() - started)) - try: - await asyncio.wait_for(_shutdown_drains(lease, supervisor, emitter), timeout=left) - except TimeoutError: + from server.core.background import create_deferred_task + + drains = create_deferred_task( + _shutdown_drains(lease, supervisor, emitter), name="shutdown-drains" + ) + _, pending = await asyncio.wait({drains}, timeout=left) + if pending: + drains.cancel() + logging.getLogger(__name__).warning( - "shutdown drains exceeded %.0f s; retiring anyway", boot.SHUTDOWN_DRAIN_BUDGET_S + "shutdown drains exceeded %.0f s; permanent retirement remains unproved", + boot.SHUTDOWN_DRAIN_BUDGET_S, ) + else: + await drains unused = max(0.0, boot.SHUTDOWN_DRAIN_BUDGET_S - (time.monotonic() - started)) await boot.leave_deployment(ch, budget_s=SHUTDOWN_RETIRE_S + unused) async def _shutdown_drains(lease: Any, supervisor: Any, emitter: Any) -> None: """The lifespan's shutdown drains, in order; it bounds them all together.""" - from server.workers import leader as _leader - # An agent claim's last cutoff, which ends what an older replica's sign-in # with the code issued while the claim was landing: started first, so it # runs beside the drains below. - claims, started = asyncio.create_task(_agent_claim.drain()), time.monotonic() + from server.core.background import create_deferred_task + from server.workers import leader as _leader + + claims = create_deferred_task(_agent_claim.drain(), name="shutdown-agent-claims") + started = time.monotonic() try: await _leader.shutdown(lease, supervisor) # The service-side writes still in flight land while what they use is @@ -101,10 +129,29 @@ async def _shutdown_drains(lease: Any, supervisor: Any, emitter: Any) -> None: finally: claims.cancel() # a no-op once it is done await emitter.stop() + producers = tuple(retirement.get_gate().producers) + if producers: + await asyncio.gather(*producers, return_exceptions=True) _otel.shutdown() from server.clients import _http as _http_clients await _http_clients.aclose() + retirement.get_gate().stopped() + + +async def _stop_background(probe_listener: Any) -> None: + """Stop admission and signal every producer; retirement still waits for completion.""" + from server.code.app.index import mirrors + + retirement.get_gate().close() + _readiness.begin_drain() + for producer in tuple(retirement.get_gate().producers): + producer.cancel() + if probe_listener is not None: + await probe_listener.stop() + mirrors.stop_warmup() + embedder_calibration.stop_refresh_loop() + boot.cancel_boot_tasks() @asynccontextmanager @@ -141,6 +188,9 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: weak_keys=weak_keys(), ) + verify_retirement_topology() + retirement.get_gate().install("server") + track = retirement.get_gate().track from server.core.object_store import require_client require_client() @@ -174,7 +224,11 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: await boot.require_storage_authority(ch) # The deployment's cell state, before the first heartbeat reports this process # serving and before the lease is decided on it. - cell_loops = await boot.start_cell_gate() + from server.insights.app.provider_barrier import register_process + + await register_process() + for task in await boot.start_cell_gate(): + track(task) await boot.warn_newer_schema() await boot.warn_collection_named_namespaces() # Nothing here writes a column a migration just added while another process @@ -187,21 +241,21 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: from server.core import instances as _instances from server.workers import leader as _leader - instances_loop = await _instances.start( - ch, - _leader.leads_deployment, - label=_instance_mod.instance_label() or "", - workers_lease=_leader.advertised_lease, - on_beat=_leader.read_holder, + track( + await _instances.start( + ch, + _leader.leads_deployment, + label=_instance_mod.instance_label() or "", + workers_lease=_leader.advertised_lease, + on_beat=_leader.read_holder, + ) ) from server.core.background import register_buffer_drainer - from server.insights.app.provider_barrier import register_process register_buffer_drainer("boot-writes", boot.drain_writes) from server.insights.app.provider_runtime import load_active_configuration, refresh_loop - await register_process() await load_active_configuration() emitter = get_emitter() @@ -234,24 +288,29 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: boot.publish_deployment_config(ch) # This process verifies tokens: it loads the revoked set and the signing # keyring now and keeps both fresh. - auth_loops = await boot.start_token_state() + for task in await boot.start_token_state(): + track(task) probe_listener = boot.start_probe_listener() - overrides_loop = boot.start_runtime_overrides() + track(boot.start_runtime_overrides()) await probe_reader_user(ch) from server.insights.app import analysis as _analysis # Provisioned and probed in the background: the lease can wait for minutes. - analysis_loop = _analysis.start_probe_loop() + track(_analysis.start_probe_loop()) from server.assistant.app import service as _assistant assistant_loop = _assistant.start_setup_loop() - tracker_sources_loop = boot.start_tracker_sources_refresher(ch) + if assistant_loop is not None: + track(assistant_loop) + track(boot.start_tracker_sources_refresher(ch)) from server.code.app.index import repo_registry as _repo_registry - code_repos_loop = await _repo_registry.start_refresher(ch) + track(await _repo_registry.start_refresher(ch)) from server.code.app.index import mirrors as _mirrors - _mirrors.start_warmup() # this host's git mirrors, in the background + mirror_loop = _mirrors.start_warmup() + if mirror_loop is not None: + retirement.get_gate().track(mirror_loop) boot.publish_frontend_assets(ch) boot.mark_process_start(ch) await boot.probe_sandbox() @@ -269,37 +328,22 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: "WorkerSupervisor: not started in pid %s — %s", os.getpid(), lease.reason ) lease_loop = boot.start_lease_loop(lease, supervisor, ch) + if lease_loop is not None: + track(lease_loop) if not get_model_work_gate().closed: await boot.provider_report() await boot.probe_vector_capabilities(ch) await boot.calibrate_similarity_gates() - provider_refresh = asyncio.create_task(refresh_loop(), name="provider-configuration-refresh") + track(asyncio.create_task(refresh_loop(), name="provider-configuration-refresh")) # SIGTERM: not ready first, the port closes LOOM_SHUTDOWN_READY_DELAY_S later. boot.install_drain_on_sigterm() try: yield finally: - _readiness.begin_drain() - provider_refresh.cancel() - await asyncio.gather(provider_refresh, return_exceptions=True) - if lease_loop is not None: - lease_loop.cancel() - for loop in (*cell_loops, *auth_loops, overrides_loop): - loop.cancel() - if probe_listener is not None: - await probe_listener.stop() - tracker_sources_loop.cancel() - code_repos_loop.cancel() - analysis_loop.cancel() - if assistant_loop is not None: - assistant_loop.cancel() - _mirrors.stop_warmup() - embedder_calibration.stop_refresh_loop() - instances_loop.cancel() - boot.cancel_boot_tasks() + await _stop_background(probe_listener) await _shut_down(ch, lease, supervisor, emitter) await boot.close_storage_authority() @@ -495,6 +539,10 @@ def attach_browser_middleware(target: FastAPI) -> None: http_transport.attach(app) mcp_transport.attach(app) +from server.transport.retirement import RetirementMiddleware # noqa: E402 + +app.add_middleware(RetirementMiddleware) + from server.auth import login as _login # noqa: E402 from server.auth import oauth as _oauth # noqa: E402 from server.auth import scim_routes as _scim_routes # noqa: E402 diff --git a/server/memory/app/embedder_calibration.py b/server/memory/app/embedder_calibration.py index cc9569ba6b..8ddf1f4eb5 100644 --- a/server/memory/app/embedder_calibration.py +++ b/server/memory/app/embedder_calibration.py @@ -592,7 +592,11 @@ async def _loop() -> None: if not get_model_work_gate().closed: await refresh_shared() - _refresh_task = asyncio.get_running_loop().create_task(_loop(), name="loom-calibration-refresh") + from server.core.retirement import get_gate + + _refresh_task = get_gate().track( + asyncio.get_running_loop().create_task(_loop(), name="loom-calibration-refresh") + ) return _refresh_task diff --git a/server/observability/events.py b/server/observability/events.py index da4de2c333..a89e648a0c 100644 --- a/server/observability/events.py +++ b/server/observability/events.py @@ -194,8 +194,9 @@ async def start(self) -> None: # Process-lifetime task: it must not inherit the tag, request id or # auth of whichever request first started it. from server.core.background import detached_task + from server.core.retirement import get_gate - self._task = detached_task(self._drain_loop(), name="loom-events-drain") + self._task = get_gate().track(detached_task(self._drain_loop(), name="loom-events-drain")) async def stop(self) -> None: self._running = False diff --git a/server/observability/problems.py b/server/observability/problems.py index 07571de3c6..6e43eccfc8 100644 --- a/server/observability/problems.py +++ b/server/observability/problems.py @@ -936,8 +936,9 @@ def _participant_problems(state: HealthState) -> list[Problem]: f"A provider reconfiguration would wait on these participants: {ids}. " "`heartbeat_expired` ones stopped heartbeating (they may be paused), " "and `queries_running` ones did so with a query still running.", - "Once a participant's process is confirmed stopped, retire it with " - "`loom participant retire --participant --yes`.", + "On its trusted host, verify exact physical death and settled queries with " + "`loom participant retire --participant `. Expiry or a deleted pod " + "object does not prove death.", source="runtime", ) ] diff --git a/server/transport/model_maintenance.py b/server/transport/model_maintenance.py index 7f59ba2e2c..f27092672d 100644 --- a/server/transport/model_maintenance.py +++ b/server/transport/model_maintenance.py @@ -6,7 +6,7 @@ from starlette.responses import JSONResponse from starlette.types import ASGIApp, Receive, Scope, Send -from server.core import cell, deployment_config +from server.core import cell, deployment_config, retirement from server.core.errors import classified from server.core.model_work_gate import ModelChangeInProgress, ModelWorkGate, get_model_work_gate from server.settings import settings @@ -111,6 +111,20 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: if scope["type"] not in ("http", "websocket"): await self.app(scope, receive, send) return + try: + async with retirement.get_gate().admit(): + await self._dispatch(scope, receive, send) + except retirement.ProcessStopping: + await _refuse( + scope, + receive, + send, + code="PROCESS_STOPPING", + message="This server is stopping. Retry another server.", + retry_after="1", + ) + + async def _dispatch(self, scope: Scope, receive: Receive, send: Send) -> None: if (reason := cell.gate_reason()) and not _health_probe(scope): await _refuse_cell(scope, receive, send, reason) return diff --git a/server/transport/retirement.py b/server/transport/retirement.py new file mode 100644 index 0000000000..396a304554 --- /dev/null +++ b/server/transport/retirement.py @@ -0,0 +1,28 @@ +"""Outer admission includes request-tail settlement after inner dispatch ends.""" + +from starlette.types import ASGIApp, Receive, Scope, Send + +from server.core import retirement +from server.transport.model_maintenance import _refuse + + +class RetirementMiddleware: + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] not in ("http", "websocket"): + await self.app(scope, receive, send) + return + try: + async with retirement.get_gate().admit(): + await self.app(scope, receive, send) + except retirement.ProcessStopping: + await _refuse( + scope, + receive, + send, + code="PROCESS_STOPPING", + message="This server is stopping. Retry another server.", + retry_after="1", + ) diff --git a/server/worker_main.py b/server/worker_main.py index 6854be4082..c265c7c124 100644 --- a/server/worker_main.py +++ b/server/worker_main.py @@ -20,6 +20,7 @@ from typing import Any from server import boot +from server.core import retirement from server.settings import settings from server.workers import child, leader from server.workers.supervisor import get_supervisor @@ -46,20 +47,24 @@ async def _trackers() -> None: # `with`, not set_query_tag: the supervisor and the shutdown run in this context after. with query_tag("boot.workers_child"): register_buffer_drainer("boot-writes", boot.drain_writes) + retirement.get_gate().install("worker") await register_process() await load_active_configuration() await get_emitter().start() await registry_sources.refresh_db_sources(force=True, ch=ch) if not get_model_work_gate().closed: await boot.calibrate_similarity_gates() + track = retirement.get_gate().track tasks = [ - asyncio.create_task(refresh_loop(), name="provider-configuration-refresh"), - asyncio.create_task(_trackers(), name="loom-tracker-sources-refresh"), - await repo_registry.start_refresher(ch), + track(asyncio.create_task(refresh_loop(), name="provider-configuration-refresh")), + track(asyncio.create_task(_trackers(), name="loom-tracker-sources-refresh")), + track(await repo_registry.start_refresher(ch)), # The workers read LIVE settings too: their passes see the deployment's overrides. - boot.start_runtime_overrides(), + track(boot.start_runtime_overrides()), ] await boot.await_warm_up(settings.boot_warmup_timeout_s, ("runtime_overrides",)) + for task in tasks: + retirement.get_gate().track(task) return tasks @@ -113,13 +118,18 @@ async def _serve(fd: int, alive_fd: int) -> None: supervisor.start_all() await stop.wait() log.info("workers child %s stopping", os.getpid()) + retirement.get_gate().close() embedder_calibration.stop_refresh_loop() for task in services: task.cancel() await asyncio.gather(*services, return_exceptions=True) await supervisor.stop_all(drain_timeout_s=leader.drain_timeout_s()) await _drain() + retirement.get_gate().stopped() finally: + retirement.get_gate().close() + for producer in tuple(retirement.get_gate().producers): + producer.cancel() beat.cancel() await boot.leave_deployment(ch, budget_s=SHUTDOWN_RETIRE_S) await boot.close_storage_authority() diff --git a/server/workers/supervisor.py b/server/workers/supervisor.py index f951132452..b2b08946c5 100644 --- a/server/workers/supervisor.py +++ b/server/workers/supervisor.py @@ -30,6 +30,7 @@ from typing import Any from server.core.model_work_gate import get_model_work_gate +from server.core.retirement import ProcessStopping, get_gate from server.observability.traffic_class import bind_background from server.settings import settings from server.workers import leader, pass_context, status_store @@ -139,6 +140,7 @@ def _spawn(self, spec: WorkerSpec) -> None: _publish_streak(spec.name, 0) _publish_partial(spec.name, 0) task = asyncio.create_task(self._supervise_as_background(spec), name=_task_name(spec)) + get_gate().track(task) self._tasks.append(task) def _restart(self, spec: WorkerSpec, _ended: asyncio.Task[None]) -> None: @@ -257,7 +259,7 @@ async def _supervise(self, spec: WorkerSpec) -> None: try: # A pass past its deadline is cancelled and counted as a # failure (TimeoutError). - async with get_model_work_gate().admit(wait=True): + async with get_gate().admit(), get_model_work_gate().admit(wait=True): # Maintenance waiting consumes neither the pass deadline # nor its duration/accounting. An admitted pass finishes # normally; changing models never cancels it mid-write. @@ -278,6 +280,8 @@ async def _supervise(self, spec: WorkerSpec) -> None: spec, ctx, partial=bool(ctx.count), streak=consecutive_failures ) ) + except ProcessStopping: + return except pass_context.PassStopped: # The pass saw a stop or a stale lease between batches. Recorded # like a cancel; the loop top waits until the lease is fresh. diff --git a/tests/fixtures/provider_process_before_retirement.py b/tests/fixtures/provider_process_before_retirement.py new file mode 100644 index 0000000000..6ef0d6af5a --- /dev/null +++ b/tests/fixtures/provider_process_before_retirement.py @@ -0,0 +1,173 @@ +"""This process's provider-participant id and the tag its ClickHouse queries carry. + +Every ClickHouse client the process opens names the id in its HTTP User-Agent, +so `system.processes` says whether a participant that stopped heartbeating +still has a query running. +""" + +import asyncio +import os +import re +import socket +from pathlib import Path +from uuid import UUID, uuid4 + +_identity: tuple[int, UUID] | None = None + + +def process_identity() -> UUID: + """One id per process; a forked child draws its own.""" + global _identity + if _identity is None or _identity[0] != os.getpid(): + _identity = (os.getpid(), uuid4()) + return _identity[1] + + +def participant_tag(identity: UUID | str) -> str: + """What `system.processes.http_user_agent` contains for that participant's queries.""" + return f"loom-participant/{UUID(str(identity)).hex}" + + +def client_name() -> str: + """The `client_name` every ClickHouse client of this process is opened with.""" + return participant_tag(process_identity()) + + +# host name | kernel boot id | container | pid namespace | pid | process start +# (clock ticks after boot) | Kubernetes namespace and pod name (POD_NAMESPACE, +# POD_NAME: the host name is the pod name cut to 63 characters; '' elsewhere) +_proof: tuple[int, str] | None = None +# Docker mounts /etc/hostname from its container's directory; a Kubernetes container +# mounts its termination log from the pod's, under the container's name. +_CONTAINER = ( + re.compile(r"/containers/([0-9a-f]{64})/hostname "), + re.compile(r"/pods/([0-9a-f-]{36})/containers/([^/ ]+)/"), +) + + +def _start_ticks(pid: int | str) -> str: + with open(f"/proc/{pid}/stat", encoding="ascii") as f: + return f.read().rsplit(")", 1)[1].split()[19] + + +def _boot_and_pidns() -> tuple[str, str]: + with open("/proc/sys/kernel/random/boot_id", encoding="ascii") as f: + boot = f.read().strip() + return boot, str(os.stat("/proc/self/ns/pid").st_ino) + + +def _container() -> str: + """This container's id, stable across its restarts; '' outside one, or where + the mounts do not say.""" + try: + with open("/proc/self/mountinfo", encoding="utf-8") as f: + mounts = f.read() + except OSError: + return "" + for pattern in _CONTAINER: + found = pattern.search(mounts) + if found: + return "/".join(found.groups()) + return "" + + +def host_proof() -> str: + """What lets a later process on this host prove this one stopped (`proves_stopped`); + '' where /proc does not say.""" + global _proof + if _proof is None or _proof[0] != os.getpid(): + try: + boot, pidns = _boot_and_pidns() + value = "|".join( + ( + socket.gethostname(), + boot, + _container(), + pidns, + str(os.getpid()), + _start_ticks("self"), + os.environ.get("POD_NAMESPACE", ""), + os.environ.get("POD_NAME", ""), + ) + ) + except (OSError, IndexError): + value = "" + _proof = (os.getpid(), value) + return _proof[1] + + +def proves_stopped(proof: str) -> bool: + """Whether the process `proof` names has stopped, judged from this one: same host + name and kernel boot, and either its pid is free or runs another process than + the one that started then (same pid namespace), or it ran in this container + before the container restarted into a new pid namespace (which kills every + process of the old one). Two containers that share a host name never prove + each other stopped; a process that cannot be inspected is not proven stopped.""" + parts = proof.split("|") + if len(parts) not in (6, 7, 8): + return False + host, boot, container, pidns, pid, start = parts[:6] + try: + my_boot, my_pidns = _boot_and_pidns() + except OSError: + return False + if host != socket.gethostname() or boot != my_boot: + return False + if pidns != my_pidns: + return bool(container) and container == _container() + # Only positive evidence: no such process, or another one under its pid. + # A pid this process may not inspect (hidepid, a permission error) proves nothing. + try: + os.kill(int(pid), 0) + except ProcessLookupError: + return True + except (OSError, ValueError): + return False + try: + return _start_ticks(pid) != start + except (OSError, IndexError): + return False + + +_SA = "/var/run/secrets/kubernetes.io/serviceaccount" +_POD_CONTAINER = re.compile(r"^([0-9a-f-]{36})/") + + +async def pod_gone(proof: str, *, timeout_s: float = 5.0) -> bool: + """Whether Kubernetes says the pod `proof` names is gone: no pod of that name + in its namespace, one with another UID, or this one in phase Failed or + Succeeded (an evicted pod keeps its object; a pod that restarts its + containers reaches those phases only once every one stopped for good). The + kubelet removes a pod's object once its containers stopped (or an operator + force-deleted it), so each is positive evidence a lost pod's process ended. + Asked only of this pod's own namespace, with its service-account token; + False wherever the API or the proof does not say.""" + parts = proof.split("|") + found = _POD_CONTAINER.match(parts[2]) if len(parts) == 8 else None + namespace = os.environ.get("POD_NAMESPACE", "") + if not found or not namespace or parts[6] != namespace or not parts[7]: + return False + host = os.environ.get("KUBERNETES_SERVICE_HOST", "") + port = os.environ.get("KUBERNETES_SERVICE_PORT", "443") + try: + token = (await asyncio.to_thread(Path(f"{_SA}/token").read_text, "ascii")).strip() + except OSError: + return False + if not host or not token: + return False + import httpx + + url = f"https://{host}:{port}/api/v1/namespaces/{namespace}/pods/{parts[7]}" + try: + async with httpx.AsyncClient(verify=f"{_SA}/ca.crt", timeout=timeout_s) as client: + r = await client.get(url, headers={"Authorization": f"Bearer {token}"}) + except (httpx.HTTPError, OSError): + return False + if r.status_code == 404: + return True + if r.status_code != 200: + return False + pod = r.json() + if str(pod.get("metadata", {}).get("uid", "")) != found.group(1): + return True + return str(pod.get("status", {}).get("phase", "")) in ("Failed", "Succeeded") diff --git a/tests/integration/test_cell_retirement.py b/tests/integration/test_cell_retirement.py new file mode 100644 index 0000000000..0652b4403f --- /dev/null +++ b/tests/integration/test_cell_retirement.py @@ -0,0 +1,103 @@ +"""Certified retirement and pruning against an owned scratch ClickHouse table.""" + +from __future__ import annotations + +import asyncio +import os +import sys +from uuid import UUID, uuid4 + +import pytest + +from server.core import provider_process, retirement +from server.core.ch import CHClient, query_tag +from server.insights.store import provider_participants as store + + +@pytest.mark.asyncio +async def test_certified_retirement_survives_old_beat_and_prunes_only_certified_history( + monkeypatch, +): + if not os.environ.get("LOOM_CH_HOST"): + pytest.skip("Set LOOM_CH_HOST to the owned disposable ClickHouse") + database = "retirement_" + uuid4().hex + table = database + ".participants" + client = CHClient() + await client.connect() + with query_tag("test.cell_retirement"): + await client.execute("CREATE DATABASE {database:Identifier}", {"database": database}) + try: + await client.execute( + "CREATE TABLE {database:Identifier}.participants AS loom._provider_participants", + {"database": database}, + ) + old_table = store.TABLE + for name in ( + "TABLE", + "PARTICIPANTS_SQL", + "OWN_VERSION_SQL", + "_PRUNE_IDS_SQL", + "_PRUNE_SQL", + ): + monkeypatch.setattr(store, name, getattr(store, name).replace(old_table, table)) + monkeypatch.setattr( + store, "_PRUNE_SQL", store._PRUNE_SQL + " SETTINGS mutations_sync=2" + ) + stopped, legacy = uuid4(), uuid4() + process = await asyncio.create_subprocess_exec(sys.executable, "-c", "pass") + await asyncio.wait_for(process.wait(), timeout=10) + fields = provider_process.host_proof().split("|") + fields[4:6] = [str(process.pid), "1"] + identity = "|".join(fields) + await client.insert( + table, + [ + { + "participant_id": stopped, + "state": "running", + "version": 1, + "host_proof": identity, + }, + {"participant_id": legacy, "state": "retired", "version": 1, "host_proof": ""}, + ], + ) + assert await store.running_queries(stopped, ch=client) == 0 + proof = retirement.death_proof(stopped, identity, after=1) + assert proof is not None + await store.record_retirement(proof, ch=client) + # A previously in-flight heartbeat cannot outrank the certified final row. + await client.insert( + table, + [ + { + "participant_id": stopped, + "state": "running", + "version": 2, + "host_proof": identity, + } + ], + ) + rows = await client.query_all( + f"SELECT participant_id, transition_id, state, version, stale_after_s, host_proof FROM {table} FINAL" + ) + final = next(row for row in rows if str(row["participant_id"]) == str(stopped)) + assert str(final["transition_id"]) == str(UUID(int=0)) + assert final["state"] == "retired" and final["version"] > 2 + assert final["stale_after_s"] == 0 + assert final["host_proof"] == provider_process.certified(identity) + assert [str(row["participant_id"]) for row in await store.participants(ch=client)] == [ + str(legacy) + ] + # Both are old, but only the certified identity is allowed to disappear. + await client.execute( + f"ALTER TABLE {table} UPDATE heartbeat_at=now64(3)-toIntervalHour(25) " + "WHERE 1 SETTINGS mutations_sync=2" + ) + assert await store.prune_retired(ch=client) == 1 + rows = await client.query_all(f"SELECT participant_id, state FROM {table} FINAL") + assert [(str(row["participant_id"]), row["state"]) for row in rows] == [ + (str(legacy), "retired") + ] + finally: + await client.execute("DROP DATABASE {database:Identifier} SYNC", {"database": database}) + await client.disconnect() diff --git a/tests/integration/test_provider_coordination_live_ch.py b/tests/integration/test_provider_coordination_live_ch.py index 9a6c9db1e2..16c9704fa8 100644 --- a/tests/integration/test_provider_coordination_live_ch.py +++ b/tests/integration/test_provider_coordination_live_ch.py @@ -19,11 +19,14 @@ def _live() -> None: @pytest.mark.asyncio -async def test_participant_ack_is_full_row_and_expiry_is_on_the_server_clock(): +async def test_participant_ack_is_full_row_and_expiry_is_on_the_server_clock(monkeypatch): _live() ch = _default_client() await ch.connect() identity, transition = uuid4(), uuid4() + monkeypatch.setattr( + "server.insights.store.provider_participants.process_identity", lambda: identity + ) try: await record_participant(identity, transition, "running", ch=ch) await record_participant(identity, transition, "drained", ch=ch) @@ -39,10 +42,11 @@ async def test_participant_ack_is_full_row_and_expiry_is_on_the_server_clock(): rows = [r for r in await participants(ch=ch) if str(r["participant_id"]) == str(identity)] assert rows[0]["expired"] assert await running_queries(identity, ch=ch) == 0 - await record_participant( - identity, transition, "retired", after=int(rows[0]["version"]), ch=ch - ) - assert all(str(row["participant_id"]) != str(identity) for row in await participants(ch=ch)) + with pytest.raises(ValueError, match="state"): + await record_participant( + identity, transition, "retired", after=int(rows[0]["version"]), ch=ch + ) + assert any(str(row["participant_id"]) == str(identity) for row in await participants(ch=ch)) finally: try: await ch.execute( diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index ad5291a171..7a1f0ded28 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -80,6 +80,7 @@ async def checked(_ch): return None monkeypatch.setattr(provider_command, "model_command", command) + monkeypatch.setattr(provider_command, "participant_command", command) monkeypatch.setattr(pg_profile, "check_configured", checked) @@ -1152,3 +1153,11 @@ async def _none(org: str, namespaces: list[str], **_: object) -> dict[str, str]: return {} monkeypatch.setattr("server.knowledge.app.mirrors.mirrors_of", _none) + + +@pytest.fixture(autouse=True) +def isolated_retirement_gate(monkeypatch): + """Each test models a fresh process; permanent shutdown cannot leak between them.""" + from server.core import retirement + + monkeypatch.setattr(retirement, "_gate", retirement.RetirementGate()) diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json index 2567a90eb5..37e0e5e39e 100644 --- a/tests/unit/snapshots/cli_help.json +++ b/tests/unit/snapshots/cli_help.json @@ -185,12 +185,12 @@ ], "loom participant": [ "usage: loom participant [-h] {retire} ...", - "positional arguments: {retire} retire Retire the provider participant of a process that died without shutting down (its id is `participant_id` in that process's /v1/health).", + "positional arguments: {retire} retire Verify exact physical death on this trusted host and retire its participant after all queries settle (the id is participant_id in /v1/health).", "options: -h, --help show this help message and exit" ], "loom participant retire": [ - "usage: loom participant retire [-h] --participant PARTICIPANT [--yes]", - "options: -h, --help show this help message and exit --participant PARTICIPANT The participant id to retire --yes Its process is confirmed stopped" + "usage: loom participant retire [-h] --participant PARTICIPANT", + "options: -h, --help show this help message and exit --participant PARTICIPANT The participant id to retire" ], "loom deployment": [ "usage: loom deployment [-h] {publish} ...", diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json index 2ad62a0df9..8216d4de36 100644 --- a/tests/unit/snapshots/cli_tree.json +++ b/tests/unit/snapshots/cli_tree.json @@ -5682,8 +5682,7 @@ "description": null, "dests": [ "help", - "participant", - "yes" + "participant" ] } ], @@ -5719,21 +5718,6 @@ "required": true, "help": "The participant id to retire", "metavar": null - }, - { - "option_strings": [ - "--yes" - ], - "dest": "yes", - "action": "_StoreTrueAction", - "default": false, - "const": true, - "choices": null, - "nargs": 0, - "type": null, - "required": false, - "help": "Its process is confirmed stopped", - "metavar": null } ] } @@ -5744,7 +5728,7 @@ "choices_help": [ { "dest": "retire", - "help": "Retire the provider participant of a process that died without shutting down (its id is `participant_id` in that process's /v1/health).", + "help": "Verify exact physical death on this trusted host and retire its participant after all queries settle (the id is participant_id in /v1/health).", "metavar": "retire" } ] diff --git a/tests/unit/test_analytics_export.py b/tests/unit/test_analytics_export.py index 7967d8edec..79412dc567 100644 --- a/tests/unit/test_analytics_export.py +++ b/tests/unit/test_analytics_export.py @@ -252,7 +252,10 @@ async def disconnect(self) -> None: def test_an_export_of_yesterday_connects_inside_its_deadline_and_hashes_under_the_key( - now: datetime, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + stub_model_command, + now: datetime, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], ) -> None: from server.core import ch as ch_module from server.insights.app import analytics_export as app diff --git a/tests/unit/test_cell_cli_orgs.py b/tests/unit/test_cell_cli_orgs.py index 60abc95312..6d14350e03 100644 --- a/tests/unit/test_cell_cli_orgs.py +++ b/tests/unit/test_cell_cli_orgs.py @@ -17,7 +17,13 @@ from server.settings import settings ROOT = Path(__file__).resolve().parents[2] -_CONNECTS = {"run_connected", "connect_checked", "connect_declared", "_connected"} +_CONNECTS = { + "run_connected", + "connect_checked", + "connect_declared", + "check_acceptance", + "_connected", +} # `.connect()` receivers that are not the deployment's ClickHouse writer: the setup # command's own admin client and SQLite files an importer reads. _OTHER_CLIENTS = {"preflight", "sqlite3"} @@ -969,6 +975,7 @@ def test_a_cell_whose_rows_cannot_be_read_refuses_every_declared_org( asyncio.run(_ch.check_orgs([])) +@pytest.mark.usefixtures("stub_model_command") def test_a_command_for_an_org_the_cell_does_not_hold_never_runs( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/test_cell_retirement.py b/tests/unit/test_cell_retirement.py new file mode 100644 index 0000000000..a5a3c8dcce --- /dev/null +++ b/tests/unit/test_cell_retirement.py @@ -0,0 +1,752 @@ +"""Permanent retirement requires completion, never expiry or an API object's absence.""" + +from unittest.mock import AsyncMock +from uuid import UUID, uuid4 + +import pytest + +from server.core import provider_process +from server.insights.app import provider_barrier as barrier +from server.insights.store import provider_participants as store + + +@pytest.mark.asyncio +async def test_expired_quiet_participant_is_not_manual_death(monkeypatch): + participant = uuid4() + row = { + "participant_id": participant, + "transition_id": UUID(int=0), + "version": 7, + "state": "running", + "expired": True, + "host_proof": provider_process.host_proof(), + } + monkeypatch.setattr(barrier, "participants", AsyncMock(return_value=[row])) + monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) + monkeypatch.setattr(barrier, "record_participant", AsyncMock()) + assert await barrier.retire_participant(participant) == "live" + barrier.record_participant.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_generic_store_cannot_manufacture_retirement(): + ch = AsyncMock() + with pytest.raises(ValueError, match="[Rr]etire|[Ss]tate"): + await store.record_participant(uuid4(), UUID(int=0), "retired", ch=ch) + ch.insert.assert_not_awaited() + + +def test_changed_namespace_does_not_prove_old_container_ended(monkeypatch): + fields = provider_process.host_proof().split("|") + assert len(fields) == 8 + fields[2] = "c" * 64 + fields[3] = "another-namespace" + monkeypatch.setattr(provider_process, "_container", lambda: fields[2]) + assert not provider_process.proves_stopped("|".join(fields)) + + +@pytest.mark.asyncio +async def test_close_is_permanent_through_an_awaited_retirement_write(monkeypatch): + import asyncio + + from server.core import retirement + + gate = retirement.get_gate() + gate.install("test") + gate.enrolled = True + gate.stopped() + entered, finish = asyncio.Event(), asyncio.Event() + + async def write(proof): + assert gate.quiescent + entered.set() + await finish.wait() + + monkeypatch.setattr(barrier, "record_retirement", write) + monkeypatch.setattr(barrier, "drain_task_sets", AsyncMock(return_value=True)) + monkeypatch.setattr(barrier, "flush_registered_buffers", AsyncMock(return_value=True)) + monkeypatch.setattr(barrier, "_server_side_settled", AsyncMock(return_value=True)) + monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) + task = asyncio.create_task(barrier.retire_process()) + await entered.wait() + with pytest.raises(retirement.ProcessStopping): + async with gate.admit(): + pytest.fail("Permanent closure must not reopen while insert is awaited") + with pytest.raises(retirement.ProcessStopping): + await barrier.register_process() + finish.set() + assert await task + + +def _request_app(handler): + from fastapi import FastAPI + + from server.main import attach_browser_middleware + from server.transport import http + from server.transport.model_maintenance import ModelMaintenanceMiddleware + from server.transport.retirement import RetirementMiddleware + + app = FastAPI() + app.add_api_route("/oauth/token", handler, methods=["POST"]) + app.add_middleware(ModelMaintenanceMiddleware) + attach_browser_middleware(app) + http.attach(app) + app.add_middleware(RetirementMiddleware) + return app + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cancel_tail", [False, True]) +async def test_composed_request_tail_holds_counter_after_dispatch_raises(monkeypatch, cancel_tail): + import asyncio + + import httpx + + from server.core import request_tail, retirement + + gate = retirement.get_gate() + entered, finish, canceled = asyncio.Event(), asyncio.Event(), asyncio.Event() + observed = [] + + async def write(): + observed.append(gate.active) + entered.set() + try: + await finish.wait() + except asyncio.CancelledError: + canceled.set() + await finish.wait() # cancellation requested; this writer still exists + + async def handler(): + assert gate.active == 2 + assert request_tail.defer(write) + raise ValueError("before response start") + + app = _request_app(handler) + transport = httpx.ASGITransport(app=app, raise_app_exceptions=False) + monkeypatch.setattr(barrier, "record_retirement", AsyncMock()) + async with httpx.AsyncClient(transport=transport, base_url="http://localhost") as client: + task = asyncio.create_task(client.post("/oauth/token")) + await asyncio.wait_for(entered.wait(), 5) + assert gate.active == 1 # actual handler ended, outer tail remains counted + assert observed == [1] + gate.install("test") + gate.enrolled = True + gate.stopped() + assert not await barrier.retire_process() + assert not task.done() + barrier.record_retirement.assert_not_awaited() + if cancel_tail: + task.cancel() + await asyncio.wait_for(canceled.wait(), 5) + assert gate.active == 1 and not task.done() + assert not await barrier.retire_process() + finish.set() + await task + assert gate.active == 0 + + +@pytest.mark.asyncio +async def test_inner_scope_refuses_when_close_lands_after_outer_admission(): + from server.core import retirement + from server.transport.model_maintenance import ModelMaintenanceMiddleware + from server.transport.retirement import RetirementMiddleware + + gate = retirement.get_gate() + handler, send = AsyncMock(), AsyncMock() + + async def between(scope, receive, send): + assert gate.active == 1 + gate.close() + await ModelMaintenanceMiddleware(handler)(scope, receive, send) + assert gate.active == 1 + + await RetirementMiddleware(between)( + {"type": "http", "path": "/oauth/token", "method": "POST"}, AsyncMock(), send + ) + handler.assert_not_awaited() + assert send.await_args_list[0].args[0]["status"] == 503 + assert gate.active == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path", + [ + "/oauth/token", + "/oauth/callback", + "/v1/auth/login", + "/v1/scim/Users", + "/v1/admin/models/reset", + ], +) +async def test_canceled_control_dispatch_remains_counted_until_actual_completion(path): + import asyncio + + from server.core import retirement + from server.transport.model_maintenance import ModelMaintenanceMiddleware + from server.transport.retirement import RetirementMiddleware + + gate = retirement.get_gate() + entered, canceled, finish = asyncio.Event(), asyncio.Event(), asyncio.Event() + + async def handler(scope, receive, send): + entered.set() + try: + await finish.wait() + except asyncio.CancelledError: + canceled.set() + await finish.wait() + assert gate.active == 2 # work could still mint/write here + + task = asyncio.create_task( + RetirementMiddleware(ModelMaintenanceMiddleware(handler))( + {"type": "http", "path": path, "method": "POST"}, AsyncMock(), AsyncMock() + ) + ) + await entered.wait() + task.cancel() + await canceled.wait() + gate.close() + assert gate.active == 2 and not task.done() + finish.set() + await task + assert gate.active == 0 + + +@pytest.mark.asyncio +async def test_registered_descendant_blocks_after_outer_and_inner_completed(monkeypatch): + import asyncio + + import httpx + + from server.core import retirement + from server.core.background import create_deferred_task + + gate = retirement.get_gate() + entered, finish = asyncio.Event(), asyncio.Event() + tasks = [] + + async def descendant(): + entered.set() + await finish.wait() + + async def handler(): + tasks.append(create_deferred_task(descendant(), name="retirement-test-descendant")) + return {"ok": True} + + app = _request_app(handler) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://localhost" + ) as client: + assert (await client.post("/oauth/token")).status_code == 200 + await entered.wait() + assert gate.active == 0 + gate.install("test") + gate.enrolled = True + gate.stopped() + monkeypatch.setattr(barrier, "record_retirement", AsyncMock()) + assert not await barrier.retire_process() + barrier.record_retirement.assert_not_awaited() + finish.set() + await asyncio.gather(*tasks) + + +@pytest.mark.asyncio +async def test_certificates_fail_closed_in_original_parsers(): + from pathlib import Path + from types import ModuleType + + source = ( + Path(__file__).parents[1] / "fixtures" / "provider_process_before_retirement.py" + ).read_text() + previous = ModuleType("old_provider_process") + exec(compile(source, "old_provider_process.py", "exec"), previous.__dict__) + proof = provider_process.certified(provider_process.host_proof()) + assert len(proof.split("|")) == 9 + assert not previous.proves_stopped(proof) + assert not await previous.pod_gone(proof) + assert provider_process.has_certificate(proof) + assert not provider_process.has_certificate(proof.replace("cell-retirement-v1", "unknown-v2")) + assert not provider_process.has_certificate("||||||||cell-retirement-v1") + + +@pytest.mark.asyncio +async def test_uncertified_retired_rows_remain_census_blockers_and_unprunable(): + from types import SimpleNamespace + + identity = uuid4() + row = {"participant_id": identity, "state": "retired", "version": 7, "host_proof": ""} + ch = SimpleNamespace(query_all=AsyncMock(return_value=[row]), execute=AsyncMock()) + assert await store.participants(ch=ch) == [row] + assert await store.prune_retired(ch=ch) == 0 + ch.execute.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_prune_rechecks_latest_certificate_and_version(): + from types import SimpleNamespace + + identity = uuid4() + proof = provider_process.certified(provider_process.host_proof()) + row = {"participant_id": identity, "state": "retired", "version": 7, "host_proof": proof} + newer = {**row, "state": "running", "version": 8} + ch = SimpleNamespace(query_all=AsyncMock(side_effect=[[row], [newer]]), execute=AsyncMock()) + assert await store.prune_retired(ch=ch) == 0 + ch.execute.assert_not_awaited() + + +def test_production_topology_installs_outer_and_inner_scopes(): + from server.main import verify_retirement_topology + + verify_retirement_topology() + + +@pytest.mark.parametrize("migrate", [False, True]) +@pytest.mark.asyncio +async def test_server_enrolls_after_pair_verification_before_first_cell_read(monkeypatch, migrate): + from types import SimpleNamespace + + from server import boot, main + from server.auth import jwt + from server.core import object_store + from server.observability import instance + from server.settings import settings + + events = [] + + class FirstRead(Exception): + pass + + async def paired(*args, **kwargs): + events.append("paired") + + async def enrolled(): + assert events[-1] == "paired" + events.append("enrolled") + + async def first_read(): + assert events[-1] == "enrolled" + events.append("read") + raise FirstRead + + async def migrations(*args, enabled, **kwargs): + assert enabled is migrate + return 0 # disabled and no-pending-migrations boots both need enrollment + + monkeypatch.setattr(instance, "require_instance_label", lambda *args, **kwargs: None) + monkeypatch.setattr(jwt, "require_real_signing_secret", lambda **kwargs: None) + monkeypatch.setattr(boot, "require_identity_settings", lambda: None) + monkeypatch.setattr(boot, "require_safe_bind", lambda **kwargs: None) + monkeypatch.setattr(object_store, "require_client", lambda: None) + monkeypatch.setattr(main, "get_client", lambda: SimpleNamespace(connect=AsyncMock())) + monkeypatch.setattr(boot, "require_clickhouse_ready", AsyncMock()) + monkeypatch.setattr(boot, "register_readiness_checks", lambda ch: None) + monkeypatch.setattr(boot, "run_fenced_boot_migrations", migrations) + monkeypatch.setattr(boot, "require_keeper", AsyncMock()) + monkeypatch.setattr(boot, "require_storage_authority", paired) + monkeypatch.setattr(boot, "start_cell_gate", first_read) + monkeypatch.setattr(barrier, "register_process", enrolled) + monkeypatch.setattr(settings, "boot_migrate", migrate) + with pytest.raises(FirstRead): + async with main.lifespan(main.app): + pytest.fail("Only the first read is under test") + assert events == ["paired", "enrolled", "read"] + + +def test_cli_connects_verified_then_enrolls_then_reads_acceptance(monkeypatch): + from contextlib import asynccontextmanager + from types import SimpleNamespace + + from server.cli import _ch + from server.core import pg_profile + from server.insights.app import provider_command + + events = [] + client = SimpleNamespace(connected=True, disconnect=AsyncMock()) + + async def verified(ch): + assert ch is client + events.append("verified") + + @asynccontextmanager + async def command(): + events.append("enrolled") + yield + events.append("completed") + + async def accepting(orgs): + assert events == ["verified", "enrolled"] + events.append("read") + + async def work(ch): + assert events[-1] == "read" + events.append("work") + return 7 + + monkeypatch.setattr("server.core.ch.get_client", lambda: client) + monkeypatch.setattr(pg_profile, "check_configured", verified) + monkeypatch.setattr(provider_command, "model_command", command) + monkeypatch.setattr(_ch, "check_orgs", accepting) + assert _ch.run_connected(work, orgs=("example",)) == 7 + assert events == ["verified", "enrolled", "read", "work", "completed"] + + +def test_retirement_writers_and_accepting_command_roots_are_an_explicit_census(): + import ast + from pathlib import Path + + root = Path(__file__).parents[2] / "server" + constructors, writers, commands, stopped, plain = set(), set(), set(), set(), set() + for path in root.rglob("*.py"): + for node in ast.walk(ast.parse(path.read_text())): + if not isinstance(node, ast.Call): + continue + func = node.func + name = ( + func.id + if isinstance(func, ast.Name) + else func.attr + if isinstance(func, ast.Attribute) + else "" + ) + relative = str(path.relative_to(root)) + if name == "RetirementProof": + constructors.add(relative) + if name == "record_retirement": + writers.add(relative) + if name == "model_command": + commands.add(relative) + if name == "participant_command": + plain.add(relative) + if name == "stopped": + stopped.add(relative) + assert constructors == {"core/retirement.py"} + assert writers == {"insights/app/provider_barrier.py"} + assert commands == { + "cli/_ch.py", + "cli/code_run.py", + "cli/import_clickgap.py", + "cli/import_recall.py", + } + assert plain == { + "cli/sql_access.py", + "cli/analytics_export.py", + "cli/maintenance.py", + "insights/app/provider_command.py", + } + assert stopped == {"main.py", "worker_main.py", "insights/app/provider_command.py"} + + +def test_manual_retirement_has_no_affirmation_bypass(): + import argparse + + from server.cli.participant import register_participant_cli + + parser = argparse.ArgumentParser() + register_participant_cli(parser.add_subparsers(dest="command", required=True)) + with pytest.raises(SystemExit) as error: + parser.parse_args(["participant", "retire", "--participant", str(uuid4()), "--yes"]) + assert error.value.code == 2 + + +def test_fork_cannot_inherit_admission(monkeypatch): + from server.core import retirement + + gate = retirement.get_gate() + gate.install("server") + gate.enrolled = True + monkeypatch.setattr(retirement, "_pid", -1) + child = retirement.get_gate() + assert child is not gate and child.role == "server" and not child.enrolled + + +@pytest.mark.asyncio +@pytest.mark.parametrize("api_state", ["404", "replacement", "Failed", "Succeeded", "unreachable"]) +async def test_pod_object_death_never_retires_work_that_can_resume( + monkeypatch, tmp_path, api_state +): + import asyncio + from pathlib import Path + from types import ModuleType + + import httpx + + from server.core import retirement + + old = ModuleType("old_reaper") + source = ( + Path(__file__).parents[1] / "fixtures" / "provider_process_before_retirement.py" + ).read_text() + exec(compile(source, "old_reaper.py", "exec"), old.__dict__) + uid = str(uuid4()) + proof = "|".join( + ("partitioned-host", "boot", uid + "/loom", "4026", "7", "99", "loom", "old-pod") + ) + (tmp_path / "token").write_text("test-token") + monkeypatch.setattr(old, "_SA", str(tmp_path)) + monkeypatch.setenv("POD_NAMESPACE", "loom") + monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "kubernetes.invalid") + seen = [] + + def answer(request): + seen.append(request) + if api_state == "unreachable": + raise httpx.ConnectError("partitioned", request=request) + return httpx.Response( + 404 if api_state == "404" else 200, + json={ + "metadata": {"uid": str(uuid4()) if api_state == "replacement" else uid}, + "status": {"phase": api_state}, + }, + ) + + real_client = httpx.AsyncClient + monkeypatch.setattr( + httpx, "AsyncClient", lambda **kw: real_client(transport=httpx.MockTransport(answer)) + ) + # Demonstrate the exact old mistake before testing its replacement. + assert await old.pod_gone(proof) is (api_state != "unreachable") + participant = uuid4() + row = { + "participant_id": participant, + "state": "running", + "version": 9, + "expired": True, + "host_proof": proof, + } + monkeypatch.setattr(barrier, "participants", AsyncMock(return_value=[row])) + monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) + monkeypatch.setattr(barrier, "prune_retired", AsyncMock()) + monkeypatch.setattr(barrier, "record_retirement", AsyncMock()) + monkeypatch.setattr(barrier, "_STALE", None) + resume, admitted = asyncio.Event(), asyncio.Event() + effects = [] + + async def paused(): + async with retirement.get_gate().admit(): + admitted.set() + await resume.wait() + effects.append("mint after object deletion") + + task = asyncio.create_task(paused()) + await admitted.wait() + await barrier.sweep_stopped(now=5000) + barrier.record_retirement.assert_not_awaited() + assert barrier.stale_participants()[0]["participant_id"] == str(participant) + assert len(seen) == 1 # the current reaper never asks the API for a death certificate + resume.set() + await task + assert effects == ["mint after object deletion"] + + +@pytest.mark.asyncio +async def test_production_child_task_cancellation_stays_counted_until_completion(): + import asyncio + import contextlib + + import anyio + import httpx + + from server.core import retirement + + entered, finish = asyncio.Event(), asyncio.Event() + gate = retirement.get_gate() + + async def handler(): + entered.set() + with anyio.CancelScope(shield=True): + await finish.wait() + return {"ok": True} + + app = _request_app(handler) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://localhost" + ) as client: + task = asyncio.create_task(client.post("/oauth/token")) + await entered.wait() + task.cancel() + # Let BaseHTTPMiddleware propagate cancellation to its downstream child. + for _ in range(5): + await asyncio.sleep(0) + gate.close() + assert gate.active > 0 + assert not task.done() + finish.set() + with contextlib.suppress(asyncio.CancelledError): + await task + assert gate.active == 0 + + +@pytest.mark.asyncio +async def test_shutdown_budget_does_not_wait_for_cancellation_deferring_drains(monkeypatch): + import asyncio + import time + + from server import boot, main + from server.core import instances, retirement + + gate = retirement.get_gate() + gate.install("test") + gate.enrolled = True + gate.stopped() + canceled, finish = asyncio.Event(), asyncio.Event() + owned = [] + + async def drains(*args): + owned.append(asyncio.current_task()) + try: + await finish.wait() + except asyncio.CancelledError: + canceled.set() + await finish.wait() + + async def leave(*args, **kwargs): + assert not await barrier.retire_process() + + monkeypatch.setattr(instances, "stop", AsyncMock()) + monkeypatch.setattr(main, "_shutdown_drains", drains) + monkeypatch.setattr(boot, "leave_deployment", leave) + monkeypatch.setattr(boot, "SHUTDOWN_DRAIN_BUDGET_S", 0.01) + monkeypatch.setattr(barrier, "record_retirement", AsyncMock()) + started = time.monotonic() + await asyncio.wait_for(main._shut_down(None, None, None, None), timeout=0.5) + assert time.monotonic() - started < 0.5 + await canceled.wait() + assert not owned[0].done() + barrier.record_retirement.assert_not_awaited() + finish.set() + await owned[0] + + +def test_nonmodel_principal_command_enrolls_before_acceptance(monkeypatch): + from types import SimpleNamespace + + from server.cli import _ch, sql_access + from server.core import pg_profile, retirement + from server.insights.app import provider_command + + events = [] + client = SimpleNamespace(connected=True, disconnect=AsyncMock()) + + async def verify(ch): + events.append("verified") + + async def register(): + assert events == ["verified"] + retirement.get_gate().enrolled = True + events.append("registered") + + async def acceptance(orgs): + assert events == ["verified", "registered"] + events.append("acceptance") + + async def work(ch): + assert events[-1] == "acceptance" + assert retirement.get_gate().active == 1 + return 7 + + monkeypatch.setattr("server.core.ch.get_client", lambda: client) + monkeypatch.setattr(pg_profile, "check_configured", verify) + monkeypatch.setattr(provider_command, "register_process", register) + monkeypatch.setattr(provider_command, "drain_task_sets", AsyncMock(return_value=True)) + monkeypatch.setattr(provider_command, "flush_registered_buffers", AsyncMock(return_value=True)) + monkeypatch.setattr(provider_command, "retire_process", AsyncMock(return_value=True)) + monkeypatch.setattr(_ch, "check_orgs", acceptance) + assert sql_access._connected(work, orgs=("example",)) == 7 + assert retirement.get_gate().closed and retirement.get_gate().active == 0 + + +@pytest.mark.asyncio +async def test_stream_and_response_background_cleanup_remain_counted(monkeypatch): + import asyncio + + import httpx + from starlette.background import BackgroundTask + from starlette.responses import StreamingResponse + + from server.core import retirement + + gate = retirement.get_gate() + body_entered, body_finish = asyncio.Event(), asyncio.Event() + cleanup_entered, cleanup_finish = asyncio.Event(), asyncio.Event() + + async def body(): + yield b"first" + body_entered.set() + await body_finish.wait() + yield b"last" + + async def cleanup(): + cleanup_entered.set() + await cleanup_finish.wait() + + async def handler(): + return StreamingResponse(body(), background=BackgroundTask(cleanup)) + + app = _request_app(handler) + monkeypatch.setattr(barrier, "record_retirement", AsyncMock()) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://localhost" + ) as client: + task = asyncio.create_task(client.post("/oauth/token")) + await body_entered.wait() + assert gate.active == 2 + gate.install("test") + gate.enrolled = True + gate.stopped() + assert not await barrier.retire_process() + body_finish.set() + await cleanup_entered.wait() + assert gate.active == 2 and not task.done() + assert not await barrier.retire_process() + cleanup_finish.set() + assert (await task).content == b"firstlast" + assert gate.active == 0 + + +@pytest.mark.asyncio +async def test_websocket_lifetime_and_shutdown_refusal_share_the_same_counter(): + import asyncio + + from server.core import retirement + + entered, finish = asyncio.Event(), asyncio.Event() + gate = retirement.get_gate() + + async def handler(): + return {} + + from starlette.websockets import WebSocket + + async def websocket(ws: WebSocket): + await ws.accept() + entered.set() + await finish.wait() + await ws.close() + + app = _request_app(handler) + app.add_api_websocket_route("/socket", websocket) + scope = { + "type": "websocket", + "path": "/socket", + "scheme": "ws", + "query_string": b"", + "headers": [(b"host", b"localhost")], + "subprotocols": [], + "client": ("127.0.0.1", 1234), + "server": ("localhost", 80), + } + task = asyncio.create_task( + app(scope, AsyncMock(return_value={"type": "websocket.connect"}), AsyncMock()) + ) + await entered.wait() + assert gate.active == 2 + gate.close() + send = AsyncMock() + await app(scope, AsyncMock(), send) + send.assert_awaited_once_with({"type": "websocket.close", "code": 1013}) + assert gate.active == 2 + finish.set() + await task + assert gate.active == 0 diff --git a/tests/unit/test_provider_barrier.py b/tests/unit/test_provider_barrier.py index 8cac6eceb8..551a8c998f 100644 --- a/tests/unit/test_provider_barrier.py +++ b/tests/unit/test_provider_barrier.py @@ -29,6 +29,11 @@ def _row(pid, transition=None, state="running", *, version=1, expired=False): @pytest.fixture def setup(monkeypatch): + from server.core import retirement + + retirement.get_gate().install("test") + retirement.get_gate().enrolled = True + retirement.get_gate().producers_stopped = True identity, transition = uuid4(), uuid4() gate = ModelWorkGate() monkeypatch.setattr(barrier, "get_model_work_gate", lambda: gate) @@ -37,6 +42,7 @@ def setup(monkeypatch): monkeypatch.setattr(barrier, "flush_registered_buffers", AsyncMock(return_value=True)) record = AsyncMock() monkeypatch.setattr(barrier, "record_participant", record) + monkeypatch.setattr(barrier, "record_retirement", AsyncMock()) monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) rows = [_row(identity, transition, "drained")] monkeypatch.setattr(barrier, "participants", AsyncMock(return_value=rows)) @@ -188,7 +194,7 @@ def test_a_heartbeat_carries_the_latest_state_above_any_version_seen(setup): "state": "drained", "version": 10**20 + 1, "stale_after_s": barrier.STALE_AFTER_S, - "host_proof": barrier.host_proof(), + "host_proof": barrier.participant_proof(), } # An acknowledgement stamped after the beat outranks it. assert barrier._OWN.stamp(transition, "drained")[2] > row[3] @@ -205,12 +211,14 @@ async def test_register_writes_the_first_row_and_retire_stops_the_beats(setup, m await barrier.register_process() assert beat.running and len(ch.inserts) == 1 tags: list[str] = [] - record.side_effect = lambda *_a, **_k: tags.append(current_query_tag()) + barrier.record_retirement.side_effect = lambda *_a, **_k: tags.append(current_query_tag()) + beat.signal_stop() + beat.thread.join(timeout=5) assert await barrier.retire_process() assert beat.stopping.is_set() # signalled, not joined: the thread ends after its beat beat.thread.join(timeout=5) assert not beat.running - assert record.call_args.args[2] == "retired" + assert barrier.record_retirement.call_args.args[0].participant == barrier.process_identity() assert tags == ["provider.participant_heartbeat"] and current_query_tag() == "" beat.ch = ch beat.beat() @@ -218,7 +226,7 @@ async def test_register_writes_the_first_row_and_retire_stops_the_beats(setup, m @pytest.mark.asyncio -async def test_a_boot_that_fails_after_enrolling_retires_its_row(setup, monkeypatch): +async def test_a_boot_that_fails_does_not_certify_a_pending_heartbeat(setup, monkeypatch): _, _, _, record = setup monkeypatch.setattr(barrier, "_HEARTBEAT", barrier._Heartbeat(lambda: _SyncCH())) monkeypatch.setattr(barrier, "_registered", False) @@ -232,7 +240,10 @@ async def lifespan(_app): with pytest.raises(RuntimeError, match="migrations failed"): async with barrier.participant_lifespan(lifespan)(None): pytest.fail("a failed boot never serves") - assert record.call_args.args[2] == "retired" + barrier.record_retirement.assert_not_awaited() + barrier._HEARTBEAT.thread.join(timeout=5) + assert await barrier.retire_process() + assert barrier.record_retirement.call_args.args[0].participant == barrier.process_identity() @pytest.mark.asyncio @@ -263,8 +274,9 @@ async def test_retire_participant_refuses_a_live_one(setup, monkeypatch): assert await barrier.retire_participant(other) == "live" record.assert_not_awaited() monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) - assert await barrier.retire_participant(other) == "retired" - record.assert_awaited_once_with(other, UUID(int=0), "retired", after=5, ch=None) + assert await barrier.retire_participant(other) == "live" + record.assert_not_awaited() + barrier.record_retirement.assert_not_awaited() def test_loom_participant_retire_refuses_a_live_one(setup, monkeypatch, capsys): @@ -281,7 +293,7 @@ def test_loom_participant_retire_refuses_a_live_one(setup, monkeypatch, capsys): monkeypatch.setattr(_ch, "run_connected", lambda run, **_declared: asyncio.run(run(None))) args = argparse.Namespace(participant=str(other), yes=True) assert participant._cmd_retire(args) == 1 - assert "still running" in capsys.readouterr().err + assert "no verifiable exact physical-death proof" in capsys.readouterr().err record.assert_not_awaited() @@ -434,14 +446,13 @@ async def _hang(*_a, **_k): @pytest.mark.asyncio -async def test_a_live_row_expires_after_the_heartbeat_window_and_a_retired_one_at_once(): +async def test_a_live_row_expires_after_the_heartbeat_window_and_a_retired_one_at_once(monkeypatch): ch = SimpleNamespace(insert=AsyncMock()) - await pp.record_participant(uuid4(), UUID(int=0), "running", ch=ch) + await pp.record_participant(pp.process_identity(), UUID(int=0), "running", ch=ch) (row,) = ch.insert.call_args.args[1] assert row["stale_after_s"] == pp.STALE_AFTER_S > barrier.HEARTBEAT_S - await pp.record_participant(uuid4(), UUID(int=0), "retired", ch=ch) - (row,) = ch.insert.call_args.args[1] - assert row["stale_after_s"] == 0 + with pytest.raises(ValueError, match="state"): + await pp.record_participant(uuid4(), UUID(int=0), "retired", ch=ch) @pytest.mark.asyncio diff --git a/tests/unit/test_provider_command.py b/tests/unit/test_provider_command.py index 347506b522..c6ba785ab4 100644 --- a/tests/unit/test_provider_command.py +++ b/tests/unit/test_provider_command.py @@ -10,6 +10,9 @@ @pytest.fixture def setup(monkeypatch): + from server.core import retirement + + retirement.get_gate().enrolled = True gate = ModelWorkGate() monkeypatch.setattr(command, "get_model_work_gate", lambda: gate) monkeypatch.setattr(command, "register_process", AsyncMock()) diff --git a/tests/unit/test_provider_host_proof.py b/tests/unit/test_provider_host_proof.py index 60cead66a5..d1ee7960e3 100644 --- a/tests/unit/test_provider_host_proof.py +++ b/tests/unit/test_provider_host_proof.py @@ -70,7 +70,7 @@ def test_a_container_restarted_into_a_new_pid_namespace_is_proven_stopped( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(pp, "_container", lambda: "c" * 64) - assert pp.proves_stopped(_with(container="c" * 64, pidns="1")) + assert not pp.proves_stopped(_with(container="c" * 64, pidns="1")) @pytest.mark.parametrize("theirs", ["d" * 64, ""]) @@ -152,127 +152,11 @@ async def test_this_host_retires_its_dead_participants_once_their_queries_ended( barrier, "running_queries", AsyncMock(side_effect=lambda p: int(str(p) == str(busy))) ) record = AsyncMock() - monkeypatch.setattr(barrier, "record_participant", record) + monkeypatch.setattr(barrier, "record_retirement", record) monkeypatch.setattr(barrier, "_last_here", -float("inf")) assert await barrier.retire_dead_here(now=100.0) == 1 - record.assert_awaited_once_with(dead, UUID(int=0), "retired", after=5) + record.assert_awaited_once() + proof = record.await_args.args[0] + assert proof.participant == dead and proof.after == 5 + assert proof.host_proof == pp.certified(gone) assert await barrier.retire_dead_here(now=100.0 + barrier.HERE_S / 2) == 0 # throttled - - -# -- a pod Kubernetes deleted ------------------------------------------------ - -_POD_UID = "0b6f2c1e-1111-2222-3333-444455556666" - - -def _pod_proof( - namespace: str = "loom", container: str = f"{_POD_UID}/loom", pod: str = "loom-7d9-abc" -) -> str: - # The host name is the pod name cut to 63 characters; the lookup uses the pod name. - return "|".join((pod[:63], "boot", container, "4026", "7", "99", namespace, pod)) - - -@pytest.fixture -def kube(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> Any: - """A service-account mount, the API's address and a stand-in API: it answers - the (status, body) pairs queued in `answers` and keeps what it saw in `seen`.""" - from types import SimpleNamespace - - import httpx - - (tmp_path / "token").write_text("tok") - (tmp_path / "ca.crt").write_text("") - monkeypatch.setattr(pp, "_SA", str(tmp_path)) - monkeypatch.setenv("POD_NAMESPACE", "loom") - monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "10.96.0.1") - api = SimpleNamespace(answers=[], seen=[]) - - def handler(request: httpx.Request) -> httpx.Response: - api.seen.append(request) - status, body = api.answers.pop(0) - return httpx.Response(status, json=body) - - real = httpx.AsyncClient - monkeypatch.setattr( - httpx, - "AsyncClient", - lambda **kw: real(transport=httpx.MockTransport(handler), timeout=kw.get("timeout")), - ) - return api - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "status,uid,phase,gone", - [ - (404, "", "", True), - (200, "another-uid", "Running", True), - (200, _POD_UID, "Failed", True), # evicted: the object stays, its containers stopped - (200, _POD_UID, "Succeeded", True), - (200, _POD_UID, "Running", False), - (200, _POD_UID, "Pending", False), - (403, "", "", False), - ], -) -async def test_a_pod_is_gone_only_when_the_api_says_so(kube, status, uid, phase, gone) -> None: - kube.answers.append((status, {"metadata": {"uid": uid}, "status": {"phase": phase}})) - assert await pp.pod_gone(_pod_proof()) is gone - assert kube.seen[0].url.path == "/api/v1/namespaces/loom/pods/loom-7d9-abc" - assert kube.seen[0].headers["authorization"] == "Bearer tok" - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "proof", - [ - _pod_proof(namespace="other"), - _pod_proof(container="a" * 64), - _pod_proof(pod=""), - "h|b|c|p|1|2", - "h|b|" + _POD_UID + "/loom|p|1|2|loom", # a 7-part proof (no pod name) proves nothing - "", - ], -) -async def test_a_pod_elsewhere_or_outside_kubernetes_is_never_gone(kube, proof) -> None: - assert await pp.pod_gone(proof) is False - assert kube.seen == [] # nothing to ask: another namespace, a Docker container, an old row - - -@pytest.mark.asyncio -async def test_the_leader_retires_an_expired_participant_whose_pod_is_gone( - monkeypatch: pytest.MonkeyPatch, -) -> None: - me, lost, busy, kept = (uuid4() for _ in range(4)) - monkeypatch.setattr(barrier, "process_identity", lambda: me) - rows = [ - _row(me, pp.host_proof()), - _row(lost, _pod_proof(), expired=True, version=9), - _row(busy, _pod_proof(), expired=True), - _row(kept, _pod_proof(), expired=True), - ] - monkeypatch.setattr(barrier, "participants", AsyncMock(return_value=rows)) - monkeypatch.setattr(barrier, "prune_retired", AsyncMock()) - monkeypatch.setattr( - barrier, "running_queries", AsyncMock(side_effect=lambda p: int(str(p) == str(busy))) - ) - asked: list[str] = [] - - async def gone(proof: str) -> bool: - asked.append(proof) - return len(asked) == 1 # the first asked (lost) is gone, kept's pod still exists - - monkeypatch.setattr(barrier, "pod_gone", gone) - record = AsyncMock() - monkeypatch.setattr(barrier, "record_participant", record) - monkeypatch.setattr(barrier, "_STALE", None) - await barrier.sweep_stopped(now=5000.0) - record.assert_awaited_once_with(lost, UUID(int=0), "retired", after=9) - assert len(asked) == 2 # a participant with a query running is never asked about - assert [p["participant_id"] for p in barrier.stale_participants()] == [str(busy), str(kept)] - - -@pytest.mark.asyncio -async def test_a_long_pod_name_is_looked_up_whole_not_by_its_cut_host_name(kube) -> None: - long = "loom-memory-platform-production-workers-7d9c8b6f5d-x4kqz-abcdefgh" - kube.answers.append((200, {"metadata": {"uid": _POD_UID}, "status": {"phase": "Running"}})) - assert await pp.pod_gone(_pod_proof(pod=long)) is False - assert kube.seen[0].url.path == f"/api/v1/namespaces/loom/pods/{long}" diff --git a/tests/unit/test_reextract.py b/tests/unit/test_reextract.py index 852f94d1b0..addaef8003 100644 --- a/tests/unit/test_reextract.py +++ b/tests/unit/test_reextract.py @@ -159,7 +159,9 @@ def test_cli_refuses_the_source_as_target(capsys) -> None: assert "must differ" in capsys.readouterr().err -def test_reextract_token_falls_back_to_the_saved_token(tmp_path, monkeypatch) -> None: +def test_reextract_token_falls_back_to_the_saved_token( + tmp_path, monkeypatch, stub_model_command +) -> None: """--token, else LOOM_API_KEY, else ~/.loom/token: the order every client command uses.""" import argparse diff --git a/tests/unit/test_revocation_web_refresh.py b/tests/unit/test_revocation_web_refresh.py index bbc0f56a5a..8dac8ed351 100644 --- a/tests/unit/test_revocation_web_refresh.py +++ b/tests/unit/test_revocation_web_refresh.py @@ -133,16 +133,28 @@ def _poll_loop(period: float) -> asyncio.Task[None]: t.cancel() -def test_lifespan_starts_the_token_state_and_cancels_it_at_shutdown() -> None: - src = (ROOT / "server" / "main.py").read_text() - lifespan = src[src.index("async def lifespan(") :] - assert "auth_loops = await boot.start_token_state()" in lifespan - assert ( - "for loop in (*cell_loops, *auth_loops, overrides_loop):\n loop.cancel()" - in lifespan +@pytest.mark.asyncio +async def test_lifespan_starts_the_token_state_and_cancels_it_at_shutdown(monkeypatch) -> None: + import ast + + from server import boot, main + from server.code.app.index import mirrors + from server.core import retirement + from server.memory.app import embedder_calibration + + tree = ast.parse((ROOT / "server" / "main.py").read_text()) + lifespan = next( + n for n in ast.walk(tree) if isinstance(n, ast.AsyncFunctionDef) and n.name == "lifespan" ) - # the decorator still sits on lifespan, not on the helper above it - assert "@asynccontextmanager\nasync def lifespan(" in src + calls = {ast.unparse(n.func) for n in ast.walk(lifespan) if isinstance(n, ast.Call)} + assert {"boot.start_token_state", "_stop_background"} <= calls + monkeypatch.setattr(mirrors, "stop_warmup", lambda: None) + monkeypatch.setattr(embedder_calibration, "stop_refresh_loop", lambda: None) + monkeypatch.setattr(boot, "cancel_boot_tasks", lambda: None) + task = retirement.get_gate().track(asyncio.create_task(asyncio.Event().wait())) + await main._stop_background(None) + await asyncio.gather(task, return_exceptions=True) + assert task.cancelled() and retirement.get_gate().closed @pytest.mark.asyncio diff --git a/tests/unit/test_shutdown_deadline.py b/tests/unit/test_shutdown_deadline.py index e8d53b92f2..8f60ab0419 100644 --- a/tests/unit/test_shutdown_deadline.py +++ b/tests/unit/test_shutdown_deadline.py @@ -125,7 +125,7 @@ def close(self): pass # The parent measures from here to the process's exit, imports left out # (CLOCK_MONOTONIC is one clock for every process on the host). print("retiring", time.monotonic(), flush=True) -assert asyncio.run(barrier.retire_process()) +assert not asyncio.run(barrier.retire_process()) """ From 5db501c8e7893f0cccb217011abe4ee1068b402e Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 22:52:01 +0000 Subject: [PATCH 059/114] Hold native principal work through command retirement Keep claim lookup and release in one enrolled async body. Run token readiness, keyring loading, signing and audit before the sole command drain and retirement. Refuse native offline minting when paired-store verification or enrollment fails; a keyring-only read failure inside enrolled work can still use the root key. Remove unreachable registration-row branches and duplicate worker task tracking, and name the changed-namespace death regression after its fail-closed contract. Signed-off-by: zlareb1 --- docs/architecture/cells.md | 11 + docs/guides/tokens-and-grants.md | 7 +- docs/reference/cli.md | 7 +- server/auth/keyring.py | 23 +- server/cli/_ch.py | 4 +- server/cli/access.py | 153 +++++++------ .../insights/store/provider_participants.py | 6 +- server/worker_main.py | 2 - tests/unit/snapshots/cli_help.json | 2 +- tests/unit/snapshots/cli_tree.json | 2 +- tests/unit/test_cell_cli_orgs.py | 21 ++ tests/unit/test_cli_hygiene.py | 14 +- tests/unit/test_native_command_retirement.py | 201 ++++++++++++++++++ tests/unit/test_platform_admin_bots.py | 20 +- tests/unit/test_platform_admin_rollout.py | 8 +- tests/unit/test_provider_host_proof.py | 2 +- tests/unit/test_revoke_issued_before.py | 9 +- 17 files changed, 395 insertions(+), 97 deletions(-) create mode 100644 tests/unit/test_native_command_retirement.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 9cd44ad6f4..7fccee817d 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -25,6 +25,17 @@ enrolls before its first fresh primary acceptance read and uses permanent guarde retirement. A serving cell heartbeat includes its `participant_id` beside the fingerprint, allowing a replacement to be joined to the participant registry. Workers children enroll independently and do not publish serving instance rows. +Native commands hold one admission scope through their complete org or principal +work and drain: claim lookup and release share that scope; token readiness, keyring +read, signing and audit share another. Connections and helper calls do not retire +a command that still has principal work to perform. Store pairing is verified +before enrollment and fresh acceptance. Org-free diagnostics do not install an +accepting view. The pre-schema `ch-setup` ownership probe discards its diagnostic +mode result and checks only store/cell identity before raw database user/profile +setup; its optional analyst-row work opens a separate verified, enrolled scope +and obtains fresh acceptance. That bootstrap exception admits no tenant or token +work. + The capability identifies installed code; it does not prove that every older process or unsafe retirement client has disappeared. diff --git a/docs/guides/tokens-and-grants.md b/docs/guides/tokens-and-grants.md index a5bf21c44a..63c919f0d7 100644 --- a/docs/guides/tokens-and-grants.md +++ b/docs/guides/tokens-and-grants.md @@ -26,7 +26,12 @@ loom mint-token --subject my-bot --org acme --namespaces support --roles writer `loom` ships with the server package and the Docker image, not with the SDK; in a container run `docker exec loom mint-token ...`. Minting needs the signing key, -`LOOM_JWT_SECRET`. A fresh clone gets a development secret from `.env.example`. +`LOOM_JWT_SECRET`, and access to the deployment's migrated stores. The command verifies +the paired stores and enrolls before reading the accepting cell state or signing. +Offline native minting and minting before migrations refuse. If only the keyring read +fails after successful enrollment, an ordinary mint can still sign with the root key. +Platform-admin minting also requires a fresh readable fleet and a recorded audit event. A fresh clone gets a development +secret from `.env.example`. `loom mint-token` and `loom doctor` warn while it is in use, and a deployment with auth on refuses to start with it (except the `make dev` stack). diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 3e0b9068ea..aa4218fbde 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -38,7 +38,10 @@ Commands fall into two kinds: whatever `--token` says, `loom init` refuses before it signs anything up, and `loom call` exchanges the workload token itself. -`loom mint-token` needs only `LOOM_JWT_SECRET`: it signs a token locally. +`loom mint-token` signs locally with `LOOM_JWT_SECRET` after verifying its paired +stores and enrolling as a participant. It refuses when those stores cannot be verified +or enrollment fails, including offline and before the deployment is migrated. Run it +against the deployment's migrated stores. **Session commands** (`login`, `logout`, `whoami`, `call`, `token`) are the Python SDK's own (`loom.cli`; `python -m loom …` with only the SDK installed), so `loom` @@ -304,7 +307,7 @@ python scripts/codegen/docs_reference.py | `loom import-recall` | Import an external vector-recall agent-memory SQLite export into Loom. | `--sqlite`, `--org`, `--namespace`, `--agent`, `--base-url`, `--token`, `--dry-run` | | `loom import-brain` | Import a markdown agent 'brain' directory into Loom: MEMORY.md (always-injected HOT layer) → profile.set sections; memory/contexts/*.md (read-on-demand WARM layer) → knowledge.add docs. | `--dir`, `--org`, `--namespace`, `--agent`, `--base-url`, `--token`, `--dry-run` | | `loom claim-release` | Release a self-signup claim: revoke its grants, clear the owner, deactivate it. | `--org`, `--namespace`, `-y` | -| `loom mint-token` | Mint a JWT locally (requires LOOM_JWT_SECRET). | `--subject`, `--org`, `--namespaces`, `--roles`, `--platform-admin`, `--namespace-role`, `--ttl-days`, `--save`, `--read-scope`, `--read-user-id`, `--traffic-class`, `--consumer`, `--service-account`, `--user-token`, `--json` | +| `loom mint-token` | Mint a JWT locally (requires LOOM_JWT_SECRET and verified stores). | `--subject`, `--org`, `--namespaces`, `--roles`, `--platform-admin`, `--namespace-role`, `--ttl-days`, `--save`, `--read-scope`, `--read-user-id`, `--traffic-class`, `--consumer`, `--service-account`, `--user-token`, `--json` | | `loom revoke-token` | Revoke one token by jti or subject (writes loom.revoked_tokens through the revocation module; needs the configured ClickHouse). | `--jti`, `--subject`, `--list`, `--issued-before`, `--scim`, `--reason`, `--expires-at`, `--json` | | `loom sso` | Carry password accounts over to single sign-on. | | | `loom sso adopt` | Let SSO sign-ins reach the password accounts whose email is at --domain: each person then signs in with the identity provider to their own account, its namespaces and data. | `--domain`, `--org`, `--skip`, `--keep-tokens`, `--apply`, `--dry-run` | diff --git a/server/auth/keyring.py b/server/auth/keyring.py index d4624a4369..ff5c413d4b 100644 --- a/server/auth/keyring.py +++ b/server/auth/keyring.py @@ -162,8 +162,9 @@ def sign_with_root_only() -> None: def load_for_offline_mint(*, run: Callable[..., Any] | None = None) -> bool: """For a process that mints without serving (a CLI, a dev or bench script): read the keyring once, or sign as kid `default` when ClickHouse - cannot be read. `run` is the CLI's `run_connected`; without it the read - uses a client of its own. True when it was read. Raises ValueError when + cannot be read. `run` can supply a connected read; without it the read + uses a client of its own. Native CLI minting uses `load_for_connected_mint` + inside its enrolled command instead. True when it was read. Raises ValueError when the keyring was recorded under another root secret: no server would accept what this process signs; through `run`, raises AuthorityMismatch when the shell is configured for another deployment (`connect_checked`).""" @@ -191,6 +192,24 @@ async def _read() -> bool: raise # a shell configured for another deployment: refused, not signed by root except Exception: # noqa: BLE001 — an offline mint signs with the root secret read = False + return _offline_mint_result(read) + + +async def load_for_connected_mint(*, ch: Any) -> bool: + """Read signing keys inside an already verified, enrolled native command.""" + from server.core.pg import AuthorityMismatch + from server.core.retirement import ProcessStopping + + try: + read = await refresh(ch=ch) + except (AuthorityMismatch, ProcessStopping): + raise + except Exception: # noqa: BLE001 — an enrolled mint can sign with the root secret + read = False + return _offline_mint_result(read) + + +def _offline_mint_result(read: bool) -> bool: if not read: print(" · signing keyring not read; signing as kid 'default'", file=sys.stderr) sign_with_root_only() diff --git a/server/cli/_ch.py b/server/cli/_ch.py index 184aa1f6eb..c67479b965 100644 --- a/server/cli/_ch.py +++ b/server/cli/_ch.py @@ -193,7 +193,9 @@ async def check_setup_stores() -> None: gated connect (`loom ch-setup`), the gate a gated connect would run, before it writes: its ClickHouse, where it already records a deployment, must record its Postgres's, and the Postgres `first` row must not name another cell. A ClickHouse with no Loom - database or record yet (a service set up before the first boot) passes.""" + database or record yet (a service set up before the first boot) passes. This + diagnostic discards the mode result and never installs an accepting view; + subsequent Loom row work must verify, enroll and check acceptance afresh.""" from server.core import pg, pg_profile from server.core.ch import get_client from server.core.ch_errors import error_code, error_line diff --git a/server/cli/access.py b/server/cli/access.py index 25cfc05667..482aea7b1a 100644 --- a/server/cli/access.py +++ b/server/cli/access.py @@ -6,6 +6,7 @@ from __future__ import annotations import argparse +import asyncio import json import sys from datetime import UTC @@ -39,31 +40,34 @@ async def _lookup_owner(ch: Any) -> str | None: return None return str(row.get("owner_email") or "") or "(unknown)" - owner = run_connected(_lookup_owner, org_free=reads("reads")) - if owner is None: - print( - f"no claim found for {args.org}/{args.namespace}", - file=sys.stderr, - ) - return 1 - if not args.yes: - print( - f"This will release the claim on {args.org}/{args.namespace}" - f" (currently owned by {owner!r})." - ) - print( - "Grants are revoked and the namespace deactivated. Its memories are " - "kept, and the name stays reserved: nobody can claim it again." - ) - try: - resp = input("Continue? [y/N] ").strip().lower() - except EOFError: # no terminal to answer (a pipe, a cron): the answer is no - resp = "" - if resp not in ("y", "yes"): - print("aborted") - return 1 - async def _go(ch: Any) -> int: + from server.cli._ch import check_acceptance + + owner = await _lookup_owner(ch) + if owner is None: + print( + f"no claim found for {args.org}/{args.namespace}", + file=sys.stderr, + ) + return 1 + if not args.yes: + print( + f"This will release the claim on {args.org}/{args.namespace}" + f" (currently owned by {owner!r})." + ) + print( + "Grants are revoked and the namespace deactivated. Its memories are " + "kept, and the name stays reserved: nobody can claim it again." + ) + try: + resp = (await asyncio.to_thread(input, "Continue? [y/N] ")).strip().lower() + except EOFError: # no terminal to answer (a pipe, a cron): the answer is no + resp = "" + if resp not in ("y", "yes"): + print("aborted") + return 1 + # Confirmation can wait while the cell changes; check again before mutation. + await check_acceptance(orgs=(args.org,)) out = await release_claim( org=args.org, namespace=args.namespace, @@ -110,8 +114,6 @@ def _invalid_namespace_names(names: list[str]) -> list[str]: def _cmd_mint_token(args: argparse.Namespace) -> int: from server.auth import keyring - from server.auth.authz import GRANT_ROLES - from server.auth.jwt import mint_token, new_jti from server.core.principals import RESERVED_SUBJECT, is_reserved_subject if is_reserved_subject(args.subject): @@ -122,19 +124,49 @@ def _cmd_mint_token(args: argparse.Namespace) -> int: if refused: print(f" ✗ {refused}", file=sys.stderr) return 2 - if args.platform_admin and not _platform_admin_rollout_ready(): + + async def _go(ch: Any) -> int: + from server.cli._ch import check_acceptance + + if args.platform_admin and not await _platform_admin_rollout_ready(ch): + print( + " ✗ platform-admin tokens require a readable fleet with the rollout complete", + file=sys.stderr, + ) + return 1 + try: + await keyring.load_for_connected_mint(ch=ch) + except ValueError as e: # another root: no server would accept the token + print(f" ✗ {e}", file=sys.stderr) + return 1 + # Keyring reads can wait; signing must use a current accepting view too. + await check_acceptance(orgs=(args.org,)) + minted = _mint_token(args) + if isinstance(minted, int): + return minted + token, summary = minted + if args.platform_admin and not await _audit_platform_admin_mint(args, summary): + print(" ✗ the mint could not be recorded in the audit log; no token", file=sys.stderr) + return 1 + return _emit_minted_token(args, token, summary) + + try: + return run_connected(_go, orgs=(args.org,)) + except AuthorityMismatch: + raise + except Exception as e: # noqa: BLE001 — a mint without verified enrollment is refused print( - " ✗ platform-admin tokens require a readable fleet with the rollout complete", + f" ✗ token mint did not complete in its enrolled scope: {type(e).__name__}", file=sys.stderr, ) return 1 - try: - keyring.load_for_offline_mint( - run=lambda work: run_connected(work, org_free=reads("reads the signing keyring")) - ) - except ValueError as e: # another root: no server would accept the token - print(f" ✗ {e}", file=sys.stderr) - return 1 + + +def _mint_token(args: argparse.Namespace) -> tuple[str, dict[str, Any]] | int: + """Build and sign only inside the command's enrolled admission scope.""" + from server.auth.authz import GRANT_ROLES + from server.auth.jwt import mint_token, new_jti + ttl_seconds = (args.ttl_days * 86400) if args.ttl_days else None # Every minted token carries a jti; generated here so # the summary can name it beside (or instead of) the token — it is the @@ -238,11 +270,7 @@ def _cmd_mint_token(args: argparse.Namespace) -> int: "consumer": args.consumer or "", } - if args.platform_admin and not _audit_platform_admin_mint(args, summary): - # Never handed out unrecorded: the role spans every org. - print(" ✗ the mint could not be recorded in the audit log; no token", file=sys.stderr) - return 1 - return _emit_minted_token(args, token, summary) + return token, summary # not a setting: the longest a minted platform-admin token may live; longer is a bot's job @@ -272,45 +300,40 @@ def _platform_admin_refusal(args: argparse.Namespace) -> str | None: return None -def _platform_admin_rollout_ready() -> bool: +async def _platform_admin_rollout_ready(ch: Any) -> bool: from server.auth import machines try: - return bool( - run_connected( - machines.platform_admin_tokens_ready, org_free=reads("reads the live instances") - ) - ) + return bool(await machines.platform_admin_tokens_ready(ch)) except AuthorityMismatch: - raise # a shell configured for another deployment: its own one-line refusal + raise except Exception as e: # noqa: BLE001 — an unread fleet refuses an admin token print(f" ✗ fleet state could not be read: {type(e).__name__}", file=sys.stderr) return False -def _audit_platform_admin_mint(args: argparse.Namespace, summary: dict[str, Any]) -> bool: +async def _audit_platform_admin_mint(args: argparse.Namespace, summary: dict[str, Any]) -> bool: """Record a platform-admin mint before it is handed out; False when the audit row could not be written.""" import getpass from server.observability import audit - async def _emit(ch: Any) -> str: - return await audit.emit( - "auth.platform_admin_minted", - org=args.org, - namespace="-", - actor=f"cli:{getpass.getuser()}", - subject={ - "subject": args.subject, - "jti": summary["jti"], - "namespaces": summary["namespaces"], - "ttl_days": args.ttl_days, - }, - ) - try: - return bool(run_connected(_emit, orgs=(args.org,))) + return bool( + await audit.emit( + "auth.platform_admin_minted", + org=args.org, + namespace="-", + actor=f"cli:{getpass.getuser()}", + subject={ + "subject": args.subject, + "jti": summary["jti"], + "namespaces": summary["namespaces"], + "ttl_days": args.ttl_days, + }, + ) + ) except Exception as e: # noqa: BLE001 — an unrecorded mint is refused, never handed out print(f" ✗ audit log not reached: {type(e).__name__}: {e}", file=sys.stderr) return False @@ -714,7 +737,9 @@ def _add_revoke_token_parser(sub: Any) -> None: def _add_mint_token_parser(sub: Any) -> None: """The `mint-token` subcommand.""" - p_mint = sub.add_parser("mint-token", help="Mint a JWT locally (requires LOOM_JWT_SECRET).") + p_mint = sub.add_parser( + "mint-token", help="Mint a JWT locally (requires LOOM_JWT_SECRET and verified stores)." + ) p_mint.add_argument("--subject", required=True) p_mint.add_argument("--org", required=True) p_mint.add_argument( diff --git a/server/insights/store/provider_participants.py b/server/insights/store/provider_participants.py index da04df218d..faa8bc1d54 100644 --- a/server/insights/store/provider_participants.py +++ b/server/insights/store/provider_participants.py @@ -111,10 +111,8 @@ async def record_participant( "transition_id": transition, "state": state, "version": version if version is not None else max(time_ns(), after + 1), - # A retired row is stopped at once, so every build's `provider-lock clear` - # frees its holder without waiting out the heartbeat window. - "stale_after_s": 0 if state == "retired" else STALE_AFTER_S, - "host_proof": participant_proof() if own else "", + "stale_after_s": STALE_AFTER_S, + "host_proof": participant_proof(), } ], ) diff --git a/server/worker_main.py b/server/worker_main.py index c265c7c124..4a606a4291 100644 --- a/server/worker_main.py +++ b/server/worker_main.py @@ -63,8 +63,6 @@ async def _trackers() -> None: track(boot.start_runtime_overrides()), ] await boot.await_warm_up(settings.boot_warmup_timeout_s, ("runtime_overrides",)) - for task in tasks: - retirement.get_gate().track(task) return tasks diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json index 37e0e5e39e..f0f3075ad1 100644 --- a/tests/unit/snapshots/cli_help.json +++ b/tests/unit/snapshots/cli_help.json @@ -2,7 +2,7 @@ "loom": [ "usage: loom [-h] [--version] {code,decisions,knowledge,export-okf,import-okf,okf-lint,up,down,ch-setup,participant,deployment,settings,secrets,signing-key,provider-lock,export-analytics,sql-access,reembed,reextract,deploy-delta,entity-export,audit-namespace,mentor,serve,migrate,reconcile-status,period-digest,demote-activity-narration,recalibrate,import-clickgap,import-recall,import-brain,claim-release,mint-token,revoke-token,sso,bot,doctor,report,drops,namespaces-gc,erasure-re-verify,reconcile,router,cell,tracker-probe,tracker-contract-check,tracker-dupcheck-fit,tracker-embed-reset,namespaces-adopt,sensitivity-sweep,integrity-repair,rechunk,export-actions,lexical-stats-rebuild,version,install,demo,init,ingest-transcript,memory-search,set-host,login,logout,whoami,call,token} ...", "Loom — vendor-neutral agent memory plane on ClickHouse.", - "positional arguments: {code,decisions,knowledge,export-okf,import-okf,okf-lint,up,down,ch-setup,participant,deployment,settings,secrets,signing-key,provider-lock,export-analytics,sql-access,reembed,reextract,deploy-delta,entity-export,audit-namespace,mentor,serve,migrate,reconcile-status,period-digest,demote-activity-narration,recalibrate,import-clickgap,import-recall,import-brain,claim-release,mint-token,revoke-token,sso,bot,doctor,report,drops,namespaces-gc,erasure-re-verify,reconcile,router,cell,tracker-probe,tracker-contract-check,tracker-dupcheck-fit,tracker-embed-reset,namespaces-adopt,sensitivity-sweep,integrity-repair,rechunk,export-actions,lexical-stats-rebuild,version,install,demo,init,ingest-transcript,memory-search,set-host,login,logout,whoami,call,token} code Code intelligence plane operations. decisions Decision records mined from documents. knowledge Knowledge documents and their sources. export-okf Render a namespace as an OKF v0.2 bundle (Markdown + frontmatter) — the git-reviewable projection of Loom memory. import-okf Write the concepts of an OKF v0.2 bundle into a namespace via memory.set. okf-lint Lint an OKF bundle directory offline — the compile gate a mirror repository's CI runs. Exit 2 critical, 1 warnings, 0 clean. up Boot the local stack (Loom, its document extractor and ClickHouse) and print how to connect: creates .env and the ClickHouse config from their templates, runs docker compose up --build, waits for /v1/health, then prints the dashboard URL and the first memory.set (curl + Python SDK) and `claude mcp add` snippets — minting a writer token first when auth is on. Re-run any time. down Stop the local stack while preserving data; --delete-data performs a confirmed reset. ch-setup Prepare a self-managed ClickHouse or a ClickHouse Cloud service for Loom: create the loom database and the read-only reader user, or print the SQL for an admin. participant The Loom processes a provider change waits for. deployment The configuration every replica checks itself against. settings Every setting's value in force, and overrides of the live ones. secrets Credentials and signing keys stored sealed in ClickHouse. signing-key The token signing keys beside LOOM_JWT_SECRET, kept in ClickHouse. provider-lock The provider configuration lock. export-analytics Write one UTC day of usage and a namespace snapshot to S3 as Parquet, names hashed under LOOM_ANALYTICS_EXPORT_NAME_KEY. Never replaces an object, so a repeated run is safe. sql-access Make, rescope or drop a ClickHouse user that reads a tenant's memory, conversations and request ledger through the sql_v1_* views. reembed Re-embed vectors whose embedding_model != the configured embedder (run after changing LOOM_EMBEDDING_MODEL — Loom does not auto-re-embed). Covers memory, entity, and conversation vectors. Coded vector tables keep their width: change a model of another width in Settings → Models & keys, which rebuilds them. reextract Replay a namespace's raw evidence (memory.set_from_messages turns) through a running server's write path into ANOTHER namespace — the current extractor over the stored transcripts; the source is never written. deploy-delta What a colour swap changed: the deployment's ledger rows (volume, errors, p50/p95, upstream spend per operation) and its ClickHouse statement shapes (runs, bytes, seconds) for a window on each side of the swap, with flags for what moved past a coarse threshold. entity-export Export a namespace's entity graph (co-occurrence edges + triples) to GraphML or JSON-LD for an external graph or RDF tool. Read-only; every edge carries its provenance and a note that a path is declared, not proven. audit-namespace How a namespace is being used (namespace.audit): exact-key reads vs searches, hash keys, double-encoded and oversized values, hand-rolled index rows, leases stored as memories, version churn. Read-only; prints a table (code, severity, kind, count, share, first sample) or --json. Exit 1 on a critical finding. mentor The store's advice to one agent (mentor.brief): what it called, the planes its namespace offers, the hints with their evidence, and how often past hints were followed. Read-only; prints a table or --json. Exit 1 when a warning-level hint fires. serve Boot the control plane (uvicorn). migrate Apply schema migrations against the configured CH. reconcile-status Re-emit memory_embeddings rows whose status diverged from loom.memory (superseded / conflicted / duplicate marks the pool never saw), compared and re-emitted per identity row (org, namespace, agent, user_id, session_id, key). Dry run by default (counts by status, the rows it would write); --apply writes. period-digest Write period digests — one derived semantic memory over K consecutive session digests of each identity in one namespace. Additive and idempotent. Dry run by default (plans only); --apply writes. demote-activity-narration Find a namespace's extracted facts that narrate tool activity ('User read lines 1-1000 of X', 'Claude ran a command on …') with no durable outcome and demote (confidence x0.25) or label them through memory.apply. Dry run by default; keys an agent read by name in the last --days are left alone. Never deletes. recalibrate Re-probe the embedder + run per-NS calibration immediately (otherwise this runs weekly via the recalibration worker). import-clickgap Import a ClickGap (Detector→Validator→Reviewer) SQLite export into Loom (lessons + typed entities + categories). Reembed-only; preserves kind→memory_type, per-row TTL, recall_count (→ ranking via reinforcement_count), suppression polarity, and provenance. Idempotent via deterministic keys + per-run version. import-recall Import an external vector-recall agent-memory SQLite export into Loom. Introspects the DB at runtime (schema not assumed): tables are classified by column heuristics into memories (→ memory.set) or conversation transcripts (→ conversation.append, grouped by conversation id); unmatched tables/rows are skipped, never crash. Idempotent via deterministic keys. import-brain Import a markdown agent 'brain' directory into Loom: MEMORY.md (always- injected HOT layer) → profile.set sections; memory/contexts/*.md (read-on- demand WARM layer) → knowledge.add docs. Completes the migration alongside import-recall (the deep vector layer). Missing files are skipped, never crash. claim-release Release a self-signup claim: revoke its grants, clear the owner, deactivate it. The name stays reserved. mint-token Mint a JWT locally (requires LOOM_JWT_SECRET). revoke-token Revoke one token by jti or subject (writes loom.revoked_tokens through the revocation module; needs the configured ClickHouse). sso Carry password accounts over to single sign-on. bot Bots and CI jobs that sign in with a key or their CI's identity. doctor Diagnose pre-boot misconfig (Python version, settings, JWT, CH reachability) and print the problem list /v1/health carries, each with its fix. With --client, also examine this machine as a coding-agent client. report Print the bug form's Environment field: versions and the `loom doctor` checks. drops Print the memory ingest drop ledger (loom.memory_drops) for a namespace, by stage/reason — what the pipeline discarded, truncated or gave up on. namespaces-gc List (default) or erase throwaway namespaces — finished bench runs under the bench prefixes that nobody registered and nobody has written to for --idle-hours — in one list-scoped erasure saga per batch. erasure-re-verify Re-verify failed erasure sagas: print each one's last step-named reason and a fresh residue census (default, read-only); --resume re-runs them from their checkpoint past the recovery worker's attempts budget; --orphan-graph lists namespaces whose entity graph outlived their memories and, with --erase, purges them. reconcile After a restore of ClickHouse or Postgres, run again in the restored store the erasures and redactions the other records past the restore point, and (after a ClickHouse restore) publish again every revision committed after it. Stop every server and worker first. Dry run by default. router The router in front of a deployment's cells. cell Operate a deployment's cells through the router's directory. tracker-probe Read-only first contact with the tracker source: counts and newest timestamps per upstream table per credential, breaker states, the configured tracker namespaces. tracker-contract-check Read-only: does each tracker source's schema meet the record contract (docs/features/tracker/record-contract.md)? Names missing tables / columns, optional tables present, feature columns. Exit 0 when every checked source meets it. tracker-dupcheck-fit Refit tracker.duplicate_check's calibrated confidence for one tracker namespace now: replays the duplicate gold at filing time, fits, stores the fit. tracker-embed-reset Clear the tracker_embed pass cursors of one tracker namespace so the next pass re-walks the window and re-embeds every item whose text recipe changed. Dry run by default (prints the cursors); --apply clears them. namespaces-adopt Give namespaces that hold live memory and no registry row an owner row: kind=bench under the bench prefixes, else memory; retention_days 0, no TTL. Dry run unless --apply. sensitivity-sweep Label live rows the sensitivity detector would have labelled on write but a path skipped. Dry run by default: counts by level, probe shape and namespace. --apply writes one new version per row, tombstones a high row's embedding rows and tags a medium row's. The closing line is the high- sensitivity invariant (high rows with a live embedding). integrity-repair Read the row-level integrity gauges (dim-0 rows holding a vector, tombstoned_at/status drift, active rows past valid_to) and, with --apply, re-emit the dim-0 rows with their true dimension so search can return them again. Drift and valid_to are listed, not repaired. rechunk Re-chunk and re-embed a namespace's knowledge documents from stored content through the current chunker, so every document carries section metadata, chunk context and document frequency. Idempotent; one embedding per new chunk. Dry run by default (counts the documents and chunks); --apply re-chunks. export-actions Export the action schema derived from the operation registry — an MCP tools/list manifest with loom extras (json) or the Cedar schema (cedar) a policy gateway governs the surface with. lexical-stats-rebuild Recompute BM25 document frequency for a namespace from its live rows (the term-frequency views only see inserts; supersession and tombstones leave stale vocabulary behind). version Print installed Loom version. install Wire Loom into a local AI client's config. Writes the MCP server block into Claude Desktop / Cursor / VS Code's config file, atomically, merging with anything already there; for Claude Code it runs (or prints) `claude mcp add` and for Codex `codex mcp add`. Restart the client afterwards. demo The first five minutes: ingest this repo's AGENTS.md (or --file) into the knowledge plane the way `make ingest` does, ask it a question, and print the answer with its evidence rows. On the stock fake-LLM stack it prints the top knowledge.search hits and says what a real answer needs. init Create an identity on a Loom server. `--agent NAME` signs an agent up by itself (server needs LOOM_SELF_SIGNUP=open): mints a writer token on a fresh agent- namespace, saves it to ~/.loom/token, and prints the claim code a person enters once, with a password of their own, to claim the namespace. ingest-transcript Pipe a chat-history file (markdown / JSON) into Loom's conversation table as one thread. Generic ingest for tools without an MCP integration (Aider, Continue.dev, JetBrains AI, anything else): export the chat → run this command → the thread shows in the dashboard's Conversations. memory-search Run a memory.search from the shell. Default human output; --json-agent emits a structured envelope agents can pipe through (a `{command, results}` JSON shape). set-host Designate the Loom host(s) for this deployment. Writes the given login handle(s) or email(s) to LOOM_PLATFORM_ADMINS in the .env (--env-file, default ./.env): a handle promotes that password account to platform_admin, an email promotes an SSO sign-in with that verified email. Multiple OK (on-call rotation, co-operators, etc.). Re-login required after to pick up the role. login Sign in to a Loom server on any browser with a one-time code. logout End the stored session on the server and forget it here. whoami Show who the stored session signs you in as. call Call any operation or API route with the stored session (`loom call --help`). token Make, list and revoke your own tokens for CI and agents (`loom token --help`).", + "positional arguments: {code,decisions,knowledge,export-okf,import-okf,okf-lint,up,down,ch-setup,participant,deployment,settings,secrets,signing-key,provider-lock,export-analytics,sql-access,reembed,reextract,deploy-delta,entity-export,audit-namespace,mentor,serve,migrate,reconcile-status,period-digest,demote-activity-narration,recalibrate,import-clickgap,import-recall,import-brain,claim-release,mint-token,revoke-token,sso,bot,doctor,report,drops,namespaces-gc,erasure-re-verify,reconcile,router,cell,tracker-probe,tracker-contract-check,tracker-dupcheck-fit,tracker-embed-reset,namespaces-adopt,sensitivity-sweep,integrity-repair,rechunk,export-actions,lexical-stats-rebuild,version,install,demo,init,ingest-transcript,memory-search,set-host,login,logout,whoami,call,token} code Code intelligence plane operations. decisions Decision records mined from documents. knowledge Knowledge documents and their sources. export-okf Render a namespace as an OKF v0.2 bundle (Markdown + frontmatter) — the git-reviewable projection of Loom memory. import-okf Write the concepts of an OKF v0.2 bundle into a namespace via memory.set. okf-lint Lint an OKF bundle directory offline — the compile gate a mirror repository's CI runs. Exit 2 critical, 1 warnings, 0 clean. up Boot the local stack (Loom, its document extractor and ClickHouse) and print how to connect: creates .env and the ClickHouse config from their templates, runs docker compose up --build, waits for /v1/health, then prints the dashboard URL and the first memory.set (curl + Python SDK) and `claude mcp add` snippets — minting a writer token first when auth is on. Re-run any time. down Stop the local stack while preserving data; --delete-data performs a confirmed reset. ch-setup Prepare a self-managed ClickHouse or a ClickHouse Cloud service for Loom: create the loom database and the read-only reader user, or print the SQL for an admin. participant The Loom processes a provider change waits for. deployment The configuration every replica checks itself against. settings Every setting's value in force, and overrides of the live ones. secrets Credentials and signing keys stored sealed in ClickHouse. signing-key The token signing keys beside LOOM_JWT_SECRET, kept in ClickHouse. provider-lock The provider configuration lock. export-analytics Write one UTC day of usage and a namespace snapshot to S3 as Parquet, names hashed under LOOM_ANALYTICS_EXPORT_NAME_KEY. Never replaces an object, so a repeated run is safe. sql-access Make, rescope or drop a ClickHouse user that reads a tenant's memory, conversations and request ledger through the sql_v1_* views. reembed Re-embed vectors whose embedding_model != the configured embedder (run after changing LOOM_EMBEDDING_MODEL — Loom does not auto-re-embed). Covers memory, entity, and conversation vectors. Coded vector tables keep their width: change a model of another width in Settings → Models & keys, which rebuilds them. reextract Replay a namespace's raw evidence (memory.set_from_messages turns) through a running server's write path into ANOTHER namespace — the current extractor over the stored transcripts; the source is never written. deploy-delta What a colour swap changed: the deployment's ledger rows (volume, errors, p50/p95, upstream spend per operation) and its ClickHouse statement shapes (runs, bytes, seconds) for a window on each side of the swap, with flags for what moved past a coarse threshold. entity-export Export a namespace's entity graph (co-occurrence edges + triples) to GraphML or JSON-LD for an external graph or RDF tool. Read-only; every edge carries its provenance and a note that a path is declared, not proven. audit-namespace How a namespace is being used (namespace.audit): exact-key reads vs searches, hash keys, double-encoded and oversized values, hand-rolled index rows, leases stored as memories, version churn. Read-only; prints a table (code, severity, kind, count, share, first sample) or --json. Exit 1 on a critical finding. mentor The store's advice to one agent (mentor.brief): what it called, the planes its namespace offers, the hints with their evidence, and how often past hints were followed. Read-only; prints a table or --json. Exit 1 when a warning-level hint fires. serve Boot the control plane (uvicorn). migrate Apply schema migrations against the configured CH. reconcile-status Re-emit memory_embeddings rows whose status diverged from loom.memory (superseded / conflicted / duplicate marks the pool never saw), compared and re-emitted per identity row (org, namespace, agent, user_id, session_id, key). Dry run by default (counts by status, the rows it would write); --apply writes. period-digest Write period digests — one derived semantic memory over K consecutive session digests of each identity in one namespace. Additive and idempotent. Dry run by default (plans only); --apply writes. demote-activity-narration Find a namespace's extracted facts that narrate tool activity ('User read lines 1-1000 of X', 'Claude ran a command on …') with no durable outcome and demote (confidence x0.25) or label them through memory.apply. Dry run by default; keys an agent read by name in the last --days are left alone. Never deletes. recalibrate Re-probe the embedder + run per-NS calibration immediately (otherwise this runs weekly via the recalibration worker). import-clickgap Import a ClickGap (Detector→Validator→Reviewer) SQLite export into Loom (lessons + typed entities + categories). Reembed-only; preserves kind→memory_type, per-row TTL, recall_count (→ ranking via reinforcement_count), suppression polarity, and provenance. Idempotent via deterministic keys + per-run version. import-recall Import an external vector-recall agent-memory SQLite export into Loom. Introspects the DB at runtime (schema not assumed): tables are classified by column heuristics into memories (→ memory.set) or conversation transcripts (→ conversation.append, grouped by conversation id); unmatched tables/rows are skipped, never crash. Idempotent via deterministic keys. import-brain Import a markdown agent 'brain' directory into Loom: MEMORY.md (always- injected HOT layer) → profile.set sections; memory/contexts/*.md (read-on- demand WARM layer) → knowledge.add docs. Completes the migration alongside import-recall (the deep vector layer). Missing files are skipped, never crash. claim-release Release a self-signup claim: revoke its grants, clear the owner, deactivate it. The name stays reserved. mint-token Mint a JWT locally (requires LOOM_JWT_SECRET and verified stores). revoke-token Revoke one token by jti or subject (writes loom.revoked_tokens through the revocation module; needs the configured ClickHouse). sso Carry password accounts over to single sign-on. bot Bots and CI jobs that sign in with a key or their CI's identity. doctor Diagnose pre-boot misconfig (Python version, settings, JWT, CH reachability) and print the problem list /v1/health carries, each with its fix. With --client, also examine this machine as a coding-agent client. report Print the bug form's Environment field: versions and the `loom doctor` checks. drops Print the memory ingest drop ledger (loom.memory_drops) for a namespace, by stage/reason — what the pipeline discarded, truncated or gave up on. namespaces-gc List (default) or erase throwaway namespaces — finished bench runs under the bench prefixes that nobody registered and nobody has written to for --idle-hours — in one list-scoped erasure saga per batch. erasure-re-verify Re-verify failed erasure sagas: print each one's last step-named reason and a fresh residue census (default, read-only); --resume re-runs them from their checkpoint past the recovery worker's attempts budget; --orphan-graph lists namespaces whose entity graph outlived their memories and, with --erase, purges them. reconcile After a restore of ClickHouse or Postgres, run again in the restored store the erasures and redactions the other records past the restore point, and (after a ClickHouse restore) publish again every revision committed after it. Stop every server and worker first. Dry run by default. router The router in front of a deployment's cells. cell Operate a deployment's cells through the router's directory. tracker-probe Read-only first contact with the tracker source: counts and newest timestamps per upstream table per credential, breaker states, the configured tracker namespaces. tracker-contract-check Read-only: does each tracker source's schema meet the record contract (docs/features/tracker/record-contract.md)? Names missing tables / columns, optional tables present, feature columns. Exit 0 when every checked source meets it. tracker-dupcheck-fit Refit tracker.duplicate_check's calibrated confidence for one tracker namespace now: replays the duplicate gold at filing time, fits, stores the fit. tracker-embed-reset Clear the tracker_embed pass cursors of one tracker namespace so the next pass re-walks the window and re-embeds every item whose text recipe changed. Dry run by default (prints the cursors); --apply clears them. namespaces-adopt Give namespaces that hold live memory and no registry row an owner row: kind=bench under the bench prefixes, else memory; retention_days 0, no TTL. Dry run unless --apply. sensitivity-sweep Label live rows the sensitivity detector would have labelled on write but a path skipped. Dry run by default: counts by level, probe shape and namespace. --apply writes one new version per row, tombstones a high row's embedding rows and tags a medium row's. The closing line is the high- sensitivity invariant (high rows with a live embedding). integrity-repair Read the row-level integrity gauges (dim-0 rows holding a vector, tombstoned_at/status drift, active rows past valid_to) and, with --apply, re-emit the dim-0 rows with their true dimension so search can return them again. Drift and valid_to are listed, not repaired. rechunk Re-chunk and re-embed a namespace's knowledge documents from stored content through the current chunker, so every document carries section metadata, chunk context and document frequency. Idempotent; one embedding per new chunk. Dry run by default (counts the documents and chunks); --apply re-chunks. export-actions Export the action schema derived from the operation registry — an MCP tools/list manifest with loom extras (json) or the Cedar schema (cedar) a policy gateway governs the surface with. lexical-stats-rebuild Recompute BM25 document frequency for a namespace from its live rows (the term-frequency views only see inserts; supersession and tombstones leave stale vocabulary behind). version Print installed Loom version. install Wire Loom into a local AI client's config. Writes the MCP server block into Claude Desktop / Cursor / VS Code's config file, atomically, merging with anything already there; for Claude Code it runs (or prints) `claude mcp add` and for Codex `codex mcp add`. Restart the client afterwards. demo The first five minutes: ingest this repo's AGENTS.md (or --file) into the knowledge plane the way `make ingest` does, ask it a question, and print the answer with its evidence rows. On the stock fake-LLM stack it prints the top knowledge.search hits and says what a real answer needs. init Create an identity on a Loom server. `--agent NAME` signs an agent up by itself (server needs LOOM_SELF_SIGNUP=open): mints a writer token on a fresh agent- namespace, saves it to ~/.loom/token, and prints the claim code a person enters once, with a password of their own, to claim the namespace. ingest-transcript Pipe a chat-history file (markdown / JSON) into Loom's conversation table as one thread. Generic ingest for tools without an MCP integration (Aider, Continue.dev, JetBrains AI, anything else): export the chat → run this command → the thread shows in the dashboard's Conversations. memory-search Run a memory.search from the shell. Default human output; --json-agent emits a structured envelope agents can pipe through (a `{command, results}` JSON shape). set-host Designate the Loom host(s) for this deployment. Writes the given login handle(s) or email(s) to LOOM_PLATFORM_ADMINS in the .env (--env-file, default ./.env): a handle promotes that password account to platform_admin, an email promotes an SSO sign-in with that verified email. Multiple OK (on-call rotation, co-operators, etc.). Re-login required after to pick up the role. login Sign in to a Loom server on any browser with a one-time code. logout End the stored session on the server and forget it here. whoami Show who the stored session signs you in as. call Call any operation or API route with the stored session (`loom call --help`). token Make, list and revoke your own tokens for CI and agents (`loom token --help`).", "options: -h, --help show this help message and exit --version show program's version number and exit" ], "loom code": [ diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json index 8216d4de36..d689f7fdf6 100644 --- a/tests/unit/snapshots/cli_tree.json +++ b/tests/unit/snapshots/cli_tree.json @@ -15607,7 +15607,7 @@ }, { "dest": "mint-token", - "help": "Mint a JWT locally (requires LOOM_JWT_SECRET).", + "help": "Mint a JWT locally (requires LOOM_JWT_SECRET and verified stores).", "metavar": "mint-token" }, { diff --git a/tests/unit/test_cell_cli_orgs.py b/tests/unit/test_cell_cli_orgs.py index 6d14350e03..2e1fc087ee 100644 --- a/tests/unit/test_cell_cli_orgs.py +++ b/tests/unit/test_cell_cli_orgs.py @@ -475,6 +475,27 @@ async def connect(self) -> None: assert not [r for r in caplog.records if r.name == "clickhouse_connect"] +@pytest.mark.parametrize("diagnostic_mode", ["open", "recording", "enforced", "partial"]) +def test_setup_identity_probe_neither_installs_nor_uses_acceptance( + monkeypatch: pytest.MonkeyPatch, diagnostic_mode: str +) -> None: + """Bootstrap checks store ownership, never a reusable data-plane admission.""" + from server.core import retirement + + _cell_set(monkeypatch, (diagnostic_mode, "eu-1")) + client = AsyncMock() + monkeypatch.setattr("server.core.ch.get_client", lambda: client) + monkeypatch.setattr(pg_profile, "recorded_authority", AsyncMock(return_value=None)) + acceptance = AsyncMock(side_effect=AssertionError("bootstrap is not admission")) + monkeypatch.setattr(_ch, "check_acceptance", acceptance) + monkeypatch.setattr(cell, "refresh", acceptance) + before = cell._view + asyncio.run(_ch.check_setup_stores()) + assert cell._view is before + assert not retirement.get_gate().enrolled + acceptance.assert_not_awaited() + + def test_the_loom_code_drain_after_a_subcommand_is_a_read() -> None: """The reconnect that drains buffered writes runs after every `loom code` subcommand, reads included; the subcommand's own connect gated its writes, so the diff --git a/tests/unit/test_cli_hygiene.py b/tests/unit/test_cli_hygiene.py index 5458194131..630ca7d2de 100644 --- a/tests/unit/test_cli_hygiene.py +++ b/tests/unit/test_cli_hygiene.py @@ -171,7 +171,7 @@ def test_mint_token_signs_with_the_keyring_it_reads( pyjwt.decode(token, "k" * 40, algorithms=["HS256"], audience=settings.jwt_audience) -def test_an_offline_mint_signs_with_the_root_secret( +def test_an_offline_native_mint_refuses_without_enrollment( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: _auth_on_with(monkeypatch, "a-real-secret-that-is-not-the-default") @@ -180,10 +180,9 @@ def _unreachable(run: Any, **_declared: object) -> None: raise ConnectionError("no clickhouse") monkeypatch.setattr(access, "run_connected", _unreachable) - assert cli.main(_MINT) == 0 + assert cli.main(_MINT) == 1 out, err = capsys.readouterr() - assert "signing keyring not read; signing as kid" in err - assert pyjwt.get_unverified_header(out.strip())["kid"] == "default" + assert out == "" and "enrolled scope" in err def _doctor(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> tuple[int, str]: @@ -324,8 +323,13 @@ def _refuse(_run: object, **_declared: object) -> bool: raise AuthorityMismatch("this deployment's memory authority is postgres") monkeypatch.setattr(access, "run_connected", _refuse) + parser = argparse.ArgumentParser() + access._add_mint_token_parser(parser.add_subparsers()) + args = parser.parse_args( + ["mint-token", "--subject", "sa:test", "--org", "o", "--platform-admin", "--ttl-days", "1"] + ) with pytest.raises(AuthorityMismatch): - access._platform_admin_rollout_ready() + access._cmd_mint_token(args) def test_a_revocation_for_another_deployment_is_refused_as_such( diff --git a/tests/unit/test_native_command_retirement.py b/tests/unit/test_native_command_retirement.py new file mode 100644 index 0000000000..228352a83d --- /dev/null +++ b/tests/unit/test_native_command_retirement.py @@ -0,0 +1,201 @@ +"""The actual native command cannot outlive its certified participant.""" + +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from server.cli import _ch, access +from server.core import retirement +from server.insights.app import provider_barrier as barrier +from server.insights.app import provider_command as command + + +@pytest.fixture +def native(monkeypatch): + from server.core import background, ch, pg, pg_profile + from server.core.model_work_gate import ModelWorkGate + + events = [] + gate = retirement.get_gate() + client = SimpleNamespace(connected=False) + + async def connect(): + client.connected = True + + async def disconnect(): + client.connected = False + + client.connect, client.disconnect = connect, disconnect + monkeypatch.setattr(ch, "get_client", lambda: client) + monkeypatch.setattr(pg, "close_pool", AsyncMock()) + + async def verified(_): + assert not gate.enrolled + events.append("verified") + + async def enroll(): + assert events == ["verified"] + gate.enrolled = True + events.append("enrolled") + + async def acceptance(**_): + assert gate.enrolled and gate.active > 0 and not gate.closed + events.append("acceptance") + + async def certificate(proof): + assert gate.quiescent and client.connected + assert events[-1] == "buffer" + events.append("certificate") + + async def flush(): + events.append("buffer") + return True + + monkeypatch.setattr(pg_profile, "check_configured", verified) + monkeypatch.setattr(command, "register_process", enroll) + monkeypatch.setattr(_ch, "check_acceptance", acceptance) + model_gate = ModelWorkGate() + monkeypatch.setattr(command, "get_model_work_gate", lambda: model_gate) + monkeypatch.setattr(barrier, "get_model_work_gate", lambda: model_gate) + monkeypatch.setattr(command, "load_active_configuration", AsyncMock()) + monkeypatch.setattr(command, "_adopt_overrides", AsyncMock(return_value=True)) + monkeypatch.setattr(background, "_registered_tasks", set()) + monkeypatch.setattr(command, "drain_task_sets", background.drain_task_sets) + monkeypatch.setattr(barrier, "drain_task_sets", background.drain_task_sets) + monkeypatch.setattr(command, "flush_registered_buffers", flush) + monkeypatch.setattr(barrier, "flush_registered_buffers", flush) + monkeypatch.setattr(barrier, "_server_side_settled", AsyncMock(return_value=True)) + monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) + monkeypatch.setattr(barrier, "_OWN", barrier._Own()) + monkeypatch.setattr( + barrier, "_HEARTBEAT", SimpleNamespace(signal_stop=lambda: None, running=False) + ) + monkeypatch.setattr(barrier, "record_retirement", certificate) + return gate, events + + +def test_actual_claim_release_keeps_lookup_and_release_in_one_lifetime(native, monkeypatch): + import asyncio + + from server.core.background import create_deferred_task + from server.identity.app import claim_release + from server.identity.store import namespaces + + gate, events = native + + async def lookup(*_, **__): + assert gate.active > 0 and not gate.closed + events.append("lookup") + return {"owner_email": "owner@example.test"} + + async def release(**_): + assert gate.active > 0 and gate.enrolled and not gate.closed + assert "certificate" not in events + events.append("release") + + async def descendant(): + assert gate.enrolled and gate.closed + assert "certificate" not in events + events.append("descendant") + + async def write(): + await asyncio.sleep(0) + assert gate.enrolled and gate.closed + assert "certificate" not in events + events.append("deferred") + create_deferred_task(descendant()) + + create_deferred_task(write()) + return SimpleNamespace(grants_revoked=2) + + monkeypatch.setattr(namespaces, "fetch_row", lookup) + monkeypatch.setattr(claim_release, "release_claim", release) + assert access._cmd_claim_release(SimpleNamespace(org="o", namespace="n", yes=True)) == 0 + assert events.count("enrolled") == events.count("certificate") == 1 + assert events.index("lookup") < events.index("release") < events.index("certificate") + assert events.index("release") < events.index("deferred") < events.index("descendant") + assert events.index("descendant") < events.index("buffer") < events.index("certificate") + assert gate.quiescent + + +@pytest.mark.parametrize("platform_admin", [False, True]) +def test_actual_mint_readiness_keyring_signing_and_audit_precede_retirement( + native, monkeypatch, platform_admin +): + import argparse + + from server.auth import jwt, keyring, machines + from server.observability import audit + + gate, events = native + + def observed(event): + assert gate.active > 0 and gate.enrolled and not gate.closed + assert "certificate" not in events + events.append(event) + + async def ready(_): + observed("readiness") + return True + + async def refresh(**_): + observed("keyring") + return True + + def mint(**_): + observed("mint") + return "test-token" + + async def record(*_, **__): + observed("audit") + return "audit-id" + + monkeypatch.setattr(machines, "platform_admin_tokens_ready", ready) + monkeypatch.setattr(keyring, "refresh", refresh) + monkeypatch.setattr(jwt, "mint_token", mint) + monkeypatch.setattr(audit, "emit", record) + parser = argparse.ArgumentParser() + access._add_mint_token_parser(parser.add_subparsers(dest="cmd")) + argv = ["mint-token", "--subject", "sa:test", "--org", "o", "--ttl-days", "1"] + argv += ["--platform-admin"] if platform_admin else ["--namespaces", "n"] + assert access._cmd_mint_token(parser.parse_args(argv)) == 0 + assert events.count("enrolled") == events.count("certificate") == 1 + assert events.index("keyring") < events.index("mint") < events.index("certificate") + if platform_admin: + assert events.index("readiness") < events.index("keyring") + assert events.index("mint") < events.index("audit") < events.index("certificate") + assert gate.quiescent + + +@pytest.mark.parametrize("failure", ["unavailable", "wrong-pair"]) +def test_actual_mint_never_signs_without_verified_enrollment(native, monkeypatch, failure): + import argparse + from unittest.mock import Mock + + from server.auth import jwt + from server.core import pg_profile + from server.core.pg import AuthorityMismatch + + gate, events = native + + async def refuse(_): + if failure == "wrong-pair": + raise AuthorityMismatch("different paired stores") + raise ConnectionError("unavailable") + + minted = Mock(side_effect=AssertionError("unenrolled mint")) + monkeypatch.setattr(pg_profile, "check_configured", refuse) + monkeypatch.setattr(jwt, "mint_token", minted) + parser = argparse.ArgumentParser() + access._add_mint_token_parser(parser.add_subparsers(dest="cmd")) + args = parser.parse_args( + ["mint-token", "--subject", "sa:test", "--org", "o", "--namespaces", "n"] + ) + if failure == "wrong-pair": + with pytest.raises(AuthorityMismatch): + access._cmd_mint_token(args) + else: + assert access._cmd_mint_token(args) == 1 + minted.assert_not_called() + assert not gate.enrolled and not gate.closed and events == [] diff --git a/tests/unit/test_platform_admin_bots.py b/tests/unit/test_platform_admin_bots.py index 84fd4c1834..ed2ddb14b2 100644 --- a/tests/unit/test_platform_admin_bots.py +++ b/tests/unit/test_platform_admin_bots.py @@ -13,6 +13,7 @@ import asyncio import json from typing import Any +from unittest.mock import AsyncMock import jwt as pyjwt import pytest @@ -181,13 +182,16 @@ def minting(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, Any]]: monkeypatch.setenv("LOOM_JWT_SECRET", _SECRET) monkeypatch.setattr(settings, "jwt_secret", _SECRET, raising=False) - # An empty keyring: the mint signs with the root secret, reading no ClickHouse. - monkeypatch.setattr(access, "run_connected", lambda run, **_declared: None) + from server.auth import keyring + + monkeypatch.setattr(access, "run_connected", lambda run, **_declared: asyncio.run(run(None))) + monkeypatch.setattr(keyring, "load_for_connected_mint", AsyncMock()) + monkeypatch.setattr("server.cli._ch.check_acceptance", AsyncMock()) # These mint tests run after every replica has the human-admin guards. - monkeypatch.setattr(access, "_platform_admin_rollout_ready", lambda: True) + monkeypatch.setattr(access, "_platform_admin_rollout_ready", AsyncMock(return_value=True)) recorded: list[dict[str, Any]] = [] - def audited(args: Any, summary: dict[str, Any]) -> bool: + async def audited(args: Any, summary: dict[str, Any]) -> bool: recorded.append({"subject": args.subject, "jti": summary["jti"]}) return True @@ -236,7 +240,7 @@ def test_a_platform_admin_mint_the_audit_log_misses_is_not_handed_out( ) -> None: from server.cli import access - monkeypatch.setattr(access, "_audit_platform_admin_mint", lambda args, summary: False) + monkeypatch.setattr(access, "_audit_platform_admin_mint", AsyncMock(return_value=False)) assert cli.main([*_MINT, "--subject", "sa:cp", "--ttl-days", "7"]) == 1 out, err = capsys.readouterr() assert out == "" and "no token" in err @@ -257,7 +261,7 @@ async def emit(event: str, **kw: Any) -> str: monkeypatch.setattr(audit, "emit", emit) monkeypatch.setattr(access, "run_connected", lambda run, **_declared: asyncio.run(run(None))) args = SimpleNamespace(subject="sa:cp", org="acme", ttl_days=7) - assert access._audit_platform_admin_mint(args, {"jti": "j1", "namespaces": ["*"]}) + assert asyncio.run(access._audit_platform_admin_mint(args, {"jti": "j1", "namespaces": ["*"]})) [(event, kw)] = rows assert event == "auth.platform_admin_minted" and kw["org"] == "acme" assert kw["subject"] == {"subject": "sa:cp", "jti": "j1", "namespaces": ["*"], "ttl_days": 7} @@ -266,7 +270,9 @@ async def lost(event: str, **kw: Any) -> str: return "" # audit.emit's answer when its insert failed monkeypatch.setattr(audit, "emit", lost) - assert not access._audit_platform_admin_mint(args, {"jti": "j2", "namespaces": ["*"]}) + assert not asyncio.run( + access._audit_platform_admin_mint(args, {"jti": "j2", "namespaces": ["*"]}) + ) _CP_BODY = { diff --git a/tests/unit/test_platform_admin_rollout.py b/tests/unit/test_platform_admin_rollout.py index 134f54cc8a..93d3a5eae5 100644 --- a/tests/unit/test_platform_admin_rollout.py +++ b/tests/unit/test_platform_admin_rollout.py @@ -4,6 +4,7 @@ import time from typing import Any +from unittest.mock import AsyncMock import pytest @@ -87,9 +88,10 @@ def mint(**kw: Any) -> str: monkeypatch.setattr(instances, "live_rows", lambda: [{"schema_version": 76}]) monkeypatch.setattr(instances, "read_live", read_live) - monkeypatch.setattr(keyring, "load_for_offline_mint", lambda **kw: None) + monkeypatch.setattr(keyring, "load_for_connected_mint", AsyncMock()) + monkeypatch.setattr("server.cli._ch.check_acceptance", AsyncMock()) monkeypatch.setattr(access, "run_connected", lambda run, **_declared: asyncio.run(run(None))) - monkeypatch.setattr(access, "_audit_platform_admin_mint", lambda *a: True) + monkeypatch.setattr(access, "_audit_platform_admin_mint", AsyncMock(return_value=True)) monkeypatch.setattr(jwt, "mint_token", mint) code = cli.main( [ @@ -139,7 +141,7 @@ def mint(**kw): ] ) out, err = capsys.readouterr() - assert code == 1 and out == "" and "rollout" in err + assert code == 1 and out == "" and "enrolled scope" in err def test_admin_token_floor_matches_the_physical_migration() -> None: diff --git a/tests/unit/test_provider_host_proof.py b/tests/unit/test_provider_host_proof.py index d1ee7960e3..0e70602dd7 100644 --- a/tests/unit/test_provider_host_proof.py +++ b/tests/unit/test_provider_host_proof.py @@ -66,7 +66,7 @@ def _eperm(pid: int, sig: int) -> None: assert not pp.proves_stopped(_with(pid=_dead_pid())) -def test_a_container_restarted_into_a_new_pid_namespace_is_proven_stopped( +def test_a_changed_container_pid_namespace_does_not_prove_the_old_process_stopped( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(pp, "_container", lambda: "c" * 64) diff --git a/tests/unit/test_revoke_issued_before.py b/tests/unit/test_revoke_issued_before.py index e7e9bb0af0..fa0c71444d 100644 --- a/tests/unit/test_revoke_issued_before.py +++ b/tests/unit/test_revoke_issued_before.py @@ -20,6 +20,7 @@ from contextlib import asynccontextmanager from datetime import UTC, datetime, timedelta from typing import Any +from unittest.mock import AsyncMock import jwt as pyjwt import pytest @@ -265,10 +266,12 @@ def test_cli_mint_reports_the_jti_but_never_beside_a_saved_token( from server.settings import settings monkeypatch.setattr(settings, "jwt_secret", _SECRET, raising=False) + from server.auth import keyring from server.cli import access - # An empty keyring: the mint signs with the root secret, reading no ClickHouse. - monkeypatch.setattr(access, "run_connected", lambda run, **_declared: None) + monkeypatch.setattr(access, "run_connected", lambda run, **_declared: asyncio.run(run(None))) + monkeypatch.setattr(keyring, "load_for_connected_mint", AsyncMock()) + monkeypatch.setattr("server.cli._ch.check_acceptance", AsyncMock()) rc = cli.main( ["mint-token", "--subject", "sa:x", "--org", "acme", "--namespaces", "shop", "--json"] ) @@ -539,6 +542,6 @@ def test_cli_mint_refuses_a_reserved_subject_without_a_traceback( def _read(**_k: Any) -> None: raise AssertionError("the keyring was read for a subject no token may carry") - monkeypatch.setattr(keyring, "load_for_offline_mint", _read) + monkeypatch.setattr(keyring, "load_for_connected_mint", _read) rc = cli.main(["mint-token", "--subject", subject, "--org", "acme", "--namespaces", "shop"]) assert rc == 2 and "grant principal" in capsys.readouterr().err From 6badd7de542684cf508bd0745c947fc8e43d2b49 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 00:38:19 +0000 Subject: [PATCH 060/114] Check the participant identity in cell heartbeat tests Keep the full heartbeat assertion aligned with the provider identity used to join serving replacements to their retirement records. Signed-off-by: zlareb1 --- server/core/instances.py | 3 ++- tests/unit/test_cell.py | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/server/core/instances.py b/server/core/instances.py index ecbe666c07..42109b0d35 100644 --- a/server/core/instances.py +++ b/server/core/instances.py @@ -139,7 +139,8 @@ def cell_row() -> dict[str, Any]: """What this process reports about its cell (`loom.cell_heartbeats`). Beside the fingerprint's version and fields, its JSON carries keys no fingerprint compares (docs/architecture/cells.md, "The heartbeat's keys"): `serves` (LOOM_SERVES_TRAFFIC), - `marked` (whether this process's pooled sessions keep the tenancy marker, null until a + `participant_id` (this process's registered provider identity), `marked` + (whether this process's pooled sessions keep the tenancy marker, null until a cell-state read lands), and with LOOM_CELL set, until the `first` row is read and then on a first cell only, `router` and `router_why` (`cell_router.router_key`).""" from server.core.provider_process import process_identity diff --git a/tests/unit/test_cell.py b/tests/unit/test_cell.py index dcd2536ac4..624798a45a 100644 --- a/tests/unit/test_cell.py +++ b/tests/unit/test_cell.py @@ -7,13 +7,14 @@ import json import time from unittest.mock import AsyncMock, call +from uuid import UUID import pytest from pydantic import ValidationError from server import boot, worker_main from server.auth import jwt -from server.core import cell, cell_probe, instances, pg, pg_profile +from server.core import cell, cell_probe, instances, pg, pg_profile, provider_process from server.core import ch as ch_mod from server.settings import Settings, settings from server.transport.model_maintenance import ModelMaintenanceMiddleware @@ -329,6 +330,8 @@ def test_an_on_process_outside_a_cell_holds_from_its_first_step( def test_the_heartbeat_reports_the_cell_and_a_gated_process_is_not_serving( monkeypatch: pytest.MonkeyPatch, ) -> None: + participant = UUID(int=7) + monkeypatch.setattr(provider_process, "process_identity", lambda: participant) monkeypatch.setattr(instances.readiness, "draining", lambda: False) monkeypatch.setattr(instances.deployment_config, "drift", lambda: []) monkeypatch.setattr(settings, "serves_traffic", True) @@ -346,6 +349,7 @@ def test_the_heartbeat_reports_the_cell_and_a_gated_process_is_not_serving( **cell.as_value(cell.mine()), "serves": True, "marked": None, + "participant_id": str(participant), } From 06355a3b74eb2bb5ae95505b3d98a07cc8811183 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 04:35:26 +0000 Subject: [PATCH 061/114] Bind trigger timing receipts to distinct API and CLI configurations The serving API keeps its cell token while the native writing CLI requires it empty. Bind both complete configurations and their common projection excluding only that token, with closed schema 2 receipts and fixed credential posture. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 150 ++++- tests/unit/test_bench_cell_trigger_gate.py | 567 +++++++++++++++++- .../test_cell_trigger_gate_artifact_close.py | 19 +- .../test_cell_trigger_gate_diagnostics.py | 15 +- .../test_cell_trigger_gate_first_error.py | 13 +- 5 files changed, 717 insertions(+), 47 deletions(-) diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 32803bd38d..19db375e4b 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -123,6 +123,7 @@ def _digest(value: Any) -> str: class Fixture: """Nonsecret receipt from the owner that created and launched this exact fixture.""" + schema: int base_url: str pg_host: str pg_port: int @@ -141,22 +142,36 @@ class Fixture: source_tree: str launch_manifest_sha256: str ownership_receipt_sha256: str - configuration_sha256: str + api_configuration_sha256: str + cli_configuration_sha256: str + shared_configuration_sha256: str background_workers: str controller_excluded: bool - pg_internal_port: int | None = None + pg_internal_port: int | None @classmethod def read(cls, path: str) -> Fixture: try: - with Path(path).open("rb") as manifest_file: - raw = manifest_file.read(16_385) - _require(len(raw) <= 16_384, "manifest size") - return cls(**json.loads(raw)) + value, _ = _receipt(path, max_bytes=16_384) + _require(set(value) == set(cls.__dataclass_fields__), "fixture fields") + _require(type(value["schema"]) is int and value["schema"] == 2, "fixture schema") + return cls(**value) except (TypeError, ValueError, OSError): raise Invalid("fixture manifest") from None def validate(self, args: argparse.Namespace, config: Any) -> int: + _require(type(self.schema) is int and self.schema == 2, "fixture schema") + integers = {"schema", "pg_port", "ch_port", "epoch"} + _require( + all(type(getattr(self, key)) is int for key in integers) + and type(self.controller_excluded) is bool + and all( + type(value) is str + for key, value in asdict(self).items() + if key not in integers | {"controller_excluded", "pg_internal_port"} + ), + "fixture field types", + ) supplied = _endpoint(args.pg_dsn, pg=True) configured = _endpoint(config.pg_dsn, pg=True) _require(supplied == configured, "configured PG endpoint") @@ -204,11 +219,24 @@ def validate(self, args: argparse.Namespace, config: Any) -> int: for value in ( self.launch_manifest_sha256, self.ownership_receipt_sha256, - self.configuration_sha256, + self.api_configuration_sha256, + self.cli_configuration_sha256, + self.shared_configuration_sha256, ): - _require(bool(_SHA.fullmatch(value)), "fixture receipt digest") + _require(type(value) is str and bool(_SHA.fullmatch(value)), "fixture receipt digest") + actual = _configuration(config) + _require( + actual["cell_router_token"] == actual["router_operator_token"] == "", + "empty native CLI credentials", + ) + _require( + self.api_configuration_sha256 != self.cli_configuration_sha256, + "distinct API and CLI configurations", + ) + _require(self.cli_configuration_sha256 == _digest(actual), "fixture CLI configuration") _require( - self.configuration_sha256 == _digest(_configuration(config)), "fixture configuration" + self.shared_configuration_sha256 == _digest(_shared_configuration(config)), + "fixture shared configuration", ) _require(type(self.epoch) is int and self.epoch > 0, "authority epoch") _require( @@ -221,8 +249,17 @@ def validate(self, args: argparse.Namespace, config: Any) -> int: def _configuration(config: Any) -> dict[str, Any]: - # A complete digest catches a setting change without publishing credentials. - return cast(dict[str, Any], config.model_dump(mode="json")) + from server.settings import Settings + + values = config.model_dump(mode="json") + _require(type(values) is dict and set(values) == set(Settings.model_fields), "Settings fields") + return cast(dict[str, Any], values) + + +def _shared_configuration(config: Any) -> dict[str, Any]: + values = dict(_configuration(config)) + del values["cell_router_token"] + return values def _source(fixture: Fixture) -> dict[str, Any]: @@ -273,7 +310,15 @@ def _workload(config: Any) -> dict[str, Any]: } -def _receipt(path: str) -> tuple[dict[str, Any], str]: +def _receipt_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + _require(key not in result, "duplicate receipt field") + result[key] = value + return result + + +def _receipt(path: str, *, max_bytes: int = 65_536) -> tuple[dict[str, Any], str]: fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) with os.fdopen(fd, "rb") as source: metadata = os.fstat(source.fileno()) @@ -281,9 +326,9 @@ def _receipt(path: str) -> tuple[dict[str, Any], str]: stat.S_ISREG(metadata.st_mode) and metadata.st_uid == os.geteuid(), "owned regular receipt", ) - raw = source.read(65_537) - _require(len(raw) <= 65_536, "receipt size") - value = json.loads(raw) + raw = source.read(max_bytes + 1) + _require(len(raw) <= max_bytes, "receipt size") + value = json.loads(raw, object_pairs_hook=_receipt_object) _require(isinstance(value, dict), "receipt object") return value, hashlib.sha256(raw).hexdigest() @@ -325,6 +370,52 @@ def _owned_server(server: dict[str, Any]) -> None: _require(_start_ticks(pid) == birth, "stable process birth") +_CREDENTIAL_POSTURE = { + "api_cell_router_token_present": True, + "cli_cell_router_token_present": False, + "api_router_operator_token_present": False, + "cli_router_operator_token_present": False, +} + + +def _attested_configuration(configuration: dict[str, Any], fixture: Fixture, config: Any) -> None: + from server.core import deployment_config + + hashes = { + "api_configuration_sha256": fixture.api_configuration_sha256, + "cli_configuration_sha256": fixture.cli_configuration_sha256, + "shared_configuration_sha256": fixture.shared_configuration_sha256, + } + _require( + set(configuration) == {"schema", "workload", "fingerprints", "credential_posture", *hashes} + and type(configuration["schema"]) is int + and configuration["schema"] == 2, + "configuration artifact schema", + ) + _require(all(configuration[key] == value for key, value in hashes.items()), "launch digests") + _require( + fixture.cli_configuration_sha256 == _digest(_configuration(config)), + "launch CLI configuration", + ) + _require( + fixture.shared_configuration_sha256 == _digest(_shared_configuration(config)), + "launch shared configuration", + ) + _require( + type(configuration["credential_posture"]) is dict + and all(type(value) is bool for value in configuration["credential_posture"].values()) + and configuration["credential_posture"] == _CREDENTIAL_POSTURE, + "launch credential posture", + ) + _require( + _digest(configuration["workload"]) == _digest(_workload(config)), "launch workload settings" + ) + _require( + configuration["fingerprints"] == deployment_config.fingerprints(config), + "launch configuration fingerprints", + ) + + def _launch(args: argparse.Namespace, fixture: Fixture, config: Any) -> dict[str, Any]: from server.core import deployment_config @@ -333,7 +424,7 @@ def _launch(args: argparse.Namespace, fixture: Fixture, config: Any) -> dict[str _require( set(receipt) == {"schema", "server", "source", "configuration_artifact"} and type(receipt["schema"]) is int - and receipt["schema"] == 1, + and receipt["schema"] == 2, "launch receipt schema", ) server, source, artifact = ( @@ -356,23 +447,8 @@ def _launch(args: argparse.Namespace, fixture: Fixture, config: Any) -> dict[str ) configuration, artifact_digest = _receipt(artifact["path"]) _require(artifact_digest == artifact["sha256"], "configuration artifact hash") - _require( - set(configuration) == {"schema", "configuration_sha256", "workload", "fingerprints"} - and type(configuration["schema"]) is int - and configuration["schema"] == 1, - "configuration artifact schema", - ) - _require( - configuration["configuration_sha256"] - == fixture.configuration_sha256 - == _digest(_configuration(config)), - "launch effective configuration", - ) - _require( - _digest(configuration["workload"]) == _digest(_workload(config)), "launch workload settings" - ) + _attested_configuration(configuration, fixture, config) fingerprints = deployment_config.fingerprints(config) - _require(configuration["fingerprints"] == fingerprints, "launch configuration fingerprints") _owned_server(server) return { "health_instance": server["health_instance"], @@ -604,7 +680,12 @@ async def _proof( "epoch": fixture.epoch, "instance_id": fixture.instance_id, "fingerprint_sha256": _digest(current), - "configuration_sha256": fixture.configuration_sha256, + "api_configuration_sha256": fixture.api_configuration_sha256, + "cli_configuration_sha256": fixture.cli_configuration_sha256, + "shared_configuration_sha256": fixture.shared_configuration_sha256, + "shared_configuration_excludes": ["cell_router_token"], + "credential_posture": _CREDENTIAL_POSTURE, + "api_configuration_attested": True, "source": source, "launch": launch, } @@ -1469,6 +1550,9 @@ def main() -> int: artifact.emit( "configuration", manifest=asdict(fixture), + shared_configuration_excludes=["cell_router_token"], + credential_posture=_CREDENTIAL_POSTURE, + api_configuration_attested=True, source=source, launch=launch, manifest_sha256=_digest(asdict(fixture)), diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py index 211dfd88c9..9d9ce8f180 100644 --- a/tests/unit/test_bench_cell_trigger_gate.py +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -42,8 +42,13 @@ def args(**overrides): ) -def fixture_config(): - config = SimpleNamespace( +def role_config(): + from server.settings import Settings + + config = SimpleNamespace(**Settings.model_construct().model_dump(mode="json")) + vars(config).update( + cell_router_token="", + router_operator_token="", pg_dsn=args().pg_dsn, ch_host="127.0.0.1", ch_port=18790, @@ -74,7 +79,13 @@ def dump(**kwargs): } config.model_dump = dump + return config + + +def fixture_config(): + config = role_config() manifest = gate.Fixture( + schema=2, base_url=args().base_url, pg_host="127.0.0.1", pg_port=15790, @@ -93,7 +104,13 @@ def dump(**kwargs): source_tree="b" * 40, launch_manifest_sha256="c" * 64, ownership_receipt_sha256="d" * 64, - configuration_sha256=gate._digest(config.model_dump()), + api_configuration_sha256=gate._digest( + {**config.model_dump(), "cell_router_token": "lcr_" + "0" * 128} + ), + cli_configuration_sha256=gate._digest(config.model_dump()), + shared_configuration_sha256=gate._digest( + {k: v for k, v in config.model_dump().items() if k != "cell_router_token"} + ), background_workers="off", controller_excluded=True, pg_internal_port=5432, @@ -1247,7 +1264,21 @@ async def send(self, request, **kwargs): await gate._proof(PG(), CH(), args(launch_manifest=str(launch_path)), fixture) else: proof = await gate._proof(PG(), CH(), args(launch_manifest=str(launch_path)), fixture) - assert proof["configuration_sha256"] == fixture.configuration_sha256 + assert proof["api_configuration_sha256"] == fixture.api_configuration_sha256 + assert proof["cli_configuration_sha256"] == fixture.cli_configuration_sha256 + assert proof["shared_configuration_sha256"] == fixture.shared_configuration_sha256 + assert proof["shared_configuration_excludes"] == ["cell_router_token"] + assert proof["credential_posture"] == receipt_posture() + assert proof["api_configuration_attested"] is True + + +def receipt_posture(): + return { + "api_cell_router_token_present": True, + "cli_cell_router_token_present": False, + "api_router_operator_token_present": False, + "cli_router_operator_token_present": False, + } def launch_files(tmp_path, fixture, config): @@ -1280,8 +1311,11 @@ def launch_files(tmp_path, fixture, config): } } configuration = { - "schema": 1, - "configuration_sha256": fixture.configuration_sha256, + "schema": 2, + "api_configuration_sha256": fixture.api_configuration_sha256, + "cli_configuration_sha256": fixture.cli_configuration_sha256, + "shared_configuration_sha256": fixture.shared_configuration_sha256, + "credential_posture": receipt_posture(), "workload": public, "fingerprints": deployment_config.fingerprints(config), } @@ -1290,7 +1324,7 @@ def launch_files(tmp_path, fixture, config): artifact_path.write_bytes(artifact_bytes) source_path = str(__import__("pathlib").Path(gate.__file__).resolve().parents[1]) launch = { - "schema": 1, + "schema": 2, "server": { "pid": 4242, "start_ticks": "123", @@ -1339,7 +1373,7 @@ def test_launch_receipt_binds_named_owned_process_settings_and_nonsecret_artifac monkeypatch.setattr(gate, "_owned_server", lambda server: checked.append(server), raising=False) if fault in {"config", "workload", "workload_type", "fingerprints"}: if fault == "config": - configuration["configuration_sha256"] = "0" * 64 + configuration["cli_configuration_sha256"] = "0" * 64 if fault == "workload": configuration["workload"]["pg_delivery_batch_rows"] = 501 if fault == "workload_type": @@ -1690,3 +1724,520 @@ async def test_successful_restoration_still_fails_when_artifact_write_fails(): await gate._cleanup(triggers, conn, artifact) assert observed.value is first conn.close.assert_awaited_once() + + +def rewrite_launch(fixture, path, artifact_path, launch, configuration, *, raw=None, level=None): + configuration_raw = raw if level == "configuration" else json.dumps(configuration).encode() + artifact_path.write_bytes(configuration_raw) + launch["configuration_artifact"]["sha256"] = gate.hashlib.sha256(configuration_raw).hexdigest() + launch_raw = raw if level == "launch" else json.dumps(launch).encode() + path.write_bytes(launch_raw) + return dataclasses.replace( + fixture, launch_manifest_sha256=gate.hashlib.sha256(launch_raw).hexdigest() + ) + + +def test_schema_two_local_pair_has_only_one_fixed_credential_difference(monkeypatch, tmp_path): + from server.core import deployment_config + + fixture, config = fixture_config() + fixture, path, _, _, configuration = launch_files(tmp_path, fixture, config) + checked = [] + monkeypatch.setattr(gate, "_owned_server", lambda value: checked.append(value)) + assert fixture.validate(args(), config) == 45500 + assert gate._launch(args(launch_manifest=str(path)), fixture, config) + actual = config.model_dump(mode="json") + api = {**actual, "cell_router_token": "lcr_" + "0" * 128} + assert gate._digest(api) == fixture.api_configuration_sha256 + assert gate._digest(actual) == fixture.cli_configuration_sha256 + assert fixture.api_configuration_sha256 != fixture.cli_configuration_sha256 + assert gate._shared_configuration(config) == { + key: value for key, value in api.items() if key != "cell_router_token" + } + assert configuration["fingerprints"] == deployment_config.fingerprints(config) + assert len(checked) == 1 + + +@pytest.mark.parametrize("field", ["cell_router_token", "router_operator_token", "jwt_secret"]) +@pytest.mark.parametrize("shape", ["missing", "extra"]) +def test_complete_settings_shape_refuses_missing_or_extra_fields(field, shape): + config = role_config() + if shape == "missing": + delattr(config, field) + else: + config.unexpected_setting = "not-an-exclusion" + with pytest.raises(gate.Invalid, match="Settings fields"): + gate._configuration(config) + + +def test_every_other_effective_setting_stays_in_the_common_digest(): + from server.settings import Settings + + fixture, config = fixture_config() + original = config.model_dump(mode="json") + fields = set(original) - {"cell_router_token"} + assert fields == set(Settings.model_fields) - {"cell_router_token"} + configuration = { + "schema": 2, + "api_configuration_sha256": fixture.api_configuration_sha256, + "cli_configuration_sha256": fixture.cli_configuration_sha256, + "shared_configuration_sha256": fixture.shared_configuration_sha256, + "workload": {}, + "fingerprints": {}, + "credential_posture": receipt_posture(), + } + for field in sorted(fields): + try: + setattr(config, field, ["changed-type-and-value", original[field]]) + changed = dataclasses.replace( + fixture, cli_configuration_sha256=gate._digest(config.model_dump(mode="json")) + ) + configuration["cli_configuration_sha256"] = changed.cli_configuration_sha256 + with pytest.raises(gate.Invalid, match="launch shared configuration"): + gate._attested_configuration(configuration, changed, config) + finally: + setattr(config, field, original[field]) + + +@pytest.mark.parametrize("value", [True, 1, 1.0, "1", None, [], {}]) +def test_common_digest_keeps_json_value_types(value): + _, config = fixture_config() + config.pg_delivery_batch_rows = value + assert gate._digest(gate._shared_configuration(config)) != gate._digest( + {**gate._shared_configuration(config), "pg_delivery_batch_rows": "unchanged"} + ) + hashes = {gate._digest({"value": v}) for v in [True, 1, 1.0, "1", None, [], {}]} + assert len(hashes) == 7 + + +@pytest.mark.parametrize("location", ["fixture", "launch", "configuration"]) +@pytest.mark.parametrize("version", [None, 1, True, 2.0, "2"]) +def test_old_missing_and_noninteger_receipt_versions_refuse( + monkeypatch, tmp_path, location, version +): + fixture, config = fixture_config() + fixture, path, artifact_path, launch, configuration = launch_files(tmp_path, fixture, config) + manifest = dataclasses.asdict(fixture) + target = {"fixture": manifest, "launch": launch, "configuration": configuration}[location] + if version is None: + del target["schema"] + else: + target["schema"] = version + if location != "server": + monkeypatch.setattr( + gate, "_owned_server", lambda _: pytest.fail("invalid receipt reached birth") + ) + if location == "fixture": + path.write_text(json.dumps(manifest)) + with pytest.raises(gate.Invalid): + gate.Fixture.read(str(path)) + else: + fixture = rewrite_launch(fixture, path, artifact_path, launch, configuration) + with pytest.raises(gate.Invalid): + gate._launch(args(launch_manifest=str(path)), fixture, config) + + +@pytest.mark.parametrize( + "location", + [ + "fixture", + "launch", + "server", + "source", + "configuration_artifact", + "configuration", + "workload", + "fingerprints", + "credential_posture", + ], +) +@pytest.mark.parametrize("fault", ["duplicate", "unknown", "missing"]) +def test_closed_receipt_objects_reject_duplicate_and_unknown_fields( + monkeypatch, tmp_path, location, fault +): + fixture, config = fixture_config() + fixture, path, artifact_path, launch, configuration = launch_files(tmp_path, fixture, config) + manifest = dataclasses.asdict(fixture) + root = ( + manifest + if location == "fixture" + else configuration + if location in {"configuration", "workload", "fingerprints", "credential_posture"} + else launch + ) + target = root if location in {"fixture", "launch", "configuration"} else root[location] + if fault == "unknown": + target["unexpected"] = True + blob = json.dumps(root).encode() + elif fault == "missing": + del target[next(iter(target))] + blob = json.dumps(root).encode() + else: + key = next(iter(target)) + original = json.dumps(target) + duplicate = original[:-1] + ", " + json.dumps(key) + ": " + json.dumps(target[key]) + "}" + blob = json.dumps(root).replace(original, duplicate, 1).encode() + if location != "server": + monkeypatch.setattr( + gate, "_owned_server", lambda _: pytest.fail("invalid receipt reached birth") + ) + if location == "fixture": + path.write_bytes(blob) + with pytest.raises(gate.Invalid): + gate.Fixture.read(str(path)) + else: + fixture = rewrite_launch( + fixture, + path, + artifact_path, + launch, + configuration, + raw=blob, + level="configuration" if root is configuration else "launch", + ) + with pytest.raises(gate.Invalid): + gate._launch(args(launch_manifest=str(path)), fixture, config) + + +@pytest.mark.parametrize("key", list(receipt_posture())) +@pytest.mark.parametrize("fault", ["wrong", "integer", "missing"]) +def test_posture_requires_exact_boolean_literals(monkeypatch, tmp_path, key, fault): + fixture, config = fixture_config() + fixture, path, artifact_path, launch, configuration = launch_files(tmp_path, fixture, config) + if fault == "missing": + del configuration["credential_posture"][key] + else: + configuration["credential_posture"][key] = ( + not receipt_posture()[key] if fault == "wrong" else int(receipt_posture()[key]) + ) + fixture = rewrite_launch(fixture, path, artifact_path, launch, configuration) + monkeypatch.setattr(gate, "_owned_server", lambda _: pytest.fail("bad posture reached birth")) + with pytest.raises(gate.Invalid, match="credential posture"): + gate._launch(args(launch_manifest=str(path)), fixture, config) + + +@pytest.mark.parametrize("fault", ["equal", "cli", "shared", "swapped"]) +def test_role_digests_cannot_be_equal_swapped_or_mismatched(fault): + fixture, config = fixture_config() + if fault == "equal": + fixture = dataclasses.replace( + fixture, api_configuration_sha256=fixture.cli_configuration_sha256 + ) + elif fault == "swapped": + fixture = dataclasses.replace( + fixture, + api_configuration_sha256=fixture.cli_configuration_sha256, + cli_configuration_sha256=fixture.api_configuration_sha256, + ) + else: + fixture = dataclasses.replace(fixture, **{fault + "_configuration_sha256": "0" * 64}) + with pytest.raises(gate.Invalid): + fixture.validate(args(), config) + + +@pytest.mark.parametrize("location", ["fixture", "configuration"]) +def test_removed_configuration_digest_is_not_a_compatibility_field(monkeypatch, tmp_path, location): + fixture, config = fixture_config() + fixture, path, artifact_path, launch, configuration = launch_files(tmp_path, fixture, config) + target = dataclasses.asdict(fixture) if location == "fixture" else configuration + target["configuration_sha256"] = fixture.cli_configuration_sha256 + if location == "fixture": + path.write_text(json.dumps(target)) + with pytest.raises(gate.Invalid): + gate.Fixture.read(str(path)) + else: + fixture = rewrite_launch(fixture, path, artifact_path, launch, configuration) + with pytest.raises(gate.Invalid): + gate._launch(args(launch_manifest=str(path)), fixture, config) + + +@pytest.mark.parametrize("location", ["fixture", "launch", "configuration"]) +def test_duplicate_json_fields_refuse_without_any_schema_assumption(tmp_path, location): + path = tmp_path / "receipt.json" + path.write_text('{"' + location + '":0,"' + location + '":1}') + with pytest.raises(gate.Invalid, match="duplicate receipt field"): + gate._receipt(str(path)) + + +@pytest.mark.parametrize( + "field", ["api_configuration_sha256", "cli_configuration_sha256", "shared_configuration_sha256"] +) +@pytest.mark.parametrize("value", [None, True, 64, "A" * 64, "f" * 63, "g" * 64]) +def test_role_digest_fields_are_lowercase_sha256_strings(field, value): + fixture, config = fixture_config() + reason = "fixture receipt digest" if type(value) is str else "fixture field types" + with pytest.raises(gate.Invalid, match=reason): + dataclasses.replace(fixture, **{field: value}).validate(args(), config) + + +def test_schema_two_requires_explicit_internal_port_and_preserves_existing_bytes(tmp_path): + fixture, config = fixture_config() + value = dataclasses.asdict(fixture) + del value["pg_internal_port"] + original = json.dumps(value).encode() + path = tmp_path / "fixture.json" + path.write_bytes(original) + with pytest.raises(gate.Invalid): + gate.Fixture.read(str(path)) + assert path.read_bytes() == original + value["pg_internal_port"] = None + path.write_text(json.dumps(value)) + assert gate.Fixture.read(str(path)).validate(args(), config) == 45500 + + +@pytest.mark.parametrize("target", ["fixture", "launch", "configuration"]) +@pytest.mark.parametrize("fault", ["symlink", "oversize", "owner"]) +def test_all_receipt_roles_keep_file_safety(monkeypatch, tmp_path, target, fault): + import os + + fixture, config = fixture_config() + fixture, path, artifact_path, launch, configuration = launch_files(tmp_path, fixture, config) + manifest_path = tmp_path / "fixture.json" + manifest_path.write_text(json.dumps(dataclasses.asdict(fixture))) + selected = {"fixture": manifest_path, "launch": path, "configuration": artifact_path}[target] + original = selected.read_bytes() + if fault == "symlink": + saved = tmp_path / "saved" + saved.write_bytes(original) + selected.unlink() + selected.symlink_to(saved) + elif fault == "oversize": + selected.write_bytes(b" " * 65_537) + else: + actual = os.fstat + + def metadata(fd): + result = actual(fd) + return SimpleNamespace(st_mode=result.st_mode, st_uid=os.geteuid() + 1) + + monkeypatch.setattr(gate.os, "fstat", metadata) + with pytest.raises((gate.Invalid, OSError, ValueError)): + if target == "fixture": + gate.Fixture.read(str(manifest_path)) + else: + gate._launch(args(launch_manifest=str(path)), fixture, config) + if fault != "oversize": + assert selected.read_bytes() == original + + +def native_guard_edges(monkeypatch, config, events): + from contextlib import asynccontextmanager + + import server.settings + from server.auth import tenancy + from server.code.domain.index import repos + from server.core import background, cell, ch, pg, pg_profile + from server.insights.app import provider_command + from server.settings import Settings + + class Client: + connected = False + + async def connect(self): + events.append("native client") + + @asynccontextmanager + async def model_command(): + yield + + native_settings = Settings.model_construct(**config.model_dump(mode="json")) + + async def state(sql): + assert sql == cell._READ + return { + "first": {"is_first": True, "cell": config.cell}, + "mode": "enforced", + "own": {"current_fp": cell.as_value(cell.mine()), "next_fp": None}, + "tenancy": True, + "marked": True, + } + + async def held(sql, org): + assert sql == tenancy._HOLDS and org == "org" + events.append("native held-org guard") + return 1 + + monkeypatch.setattr(server.settings, "settings", config) + monkeypatch.setattr(cell, "settings", native_settings) + monkeypatch.setattr(tenancy, "settings", native_settings) + monkeypatch.setattr(repos, "settings", native_settings) + monkeypatch.setattr(cell, "_view", cell._View()) + monkeypatch.setattr(cell, "_deployment_id", "") + monkeypatch.setattr(cell, "_code_orgs", None) + monkeypatch.setattr(tenancy, "_held", set()) + monkeypatch.setattr(tenancy, "_elsewhere", {}) + monkeypatch.setattr(ch, "get_client", Client) + monkeypatch.setattr(pg_profile, "check_configured", AsyncMock()) + monkeypatch.setattr(pg_profile, "deployment_id", AsyncMock(return_value="deployment")) + monkeypatch.setattr(pg, "fetchrow_reset", state) + monkeypatch.setattr(pg, "fetchval", held) + monkeypatch.setattr(provider_command, "model_command", model_command) + monkeypatch.setattr(background, "register_buffer_drainer", lambda *_: None) + monkeypatch.setattr("server.cli._ch.disconnect", AsyncMock()) + + +@pytest.mark.parametrize("credential", ["", "lcr_" + "0" * 128]) +def test_role_pair_enters_the_real_native_guard_without_changing_it(monkeypatch, credential): + from server.cli import _ch + + fixture, config = fixture_config() + events = [] + native_guard_edges(monkeypatch, config, events) + assert fixture.validate(args(), config) == 45500 + config.cell_router_token = credential + + async def work(_): + events.append("native work") + return 17 + + if credential: + with pytest.raises(_ch.OrgNotHeld, match="LOOM_CELL_ROUTER_TOKEN"): + _ch.run_connected(work, orgs=("org",)) + assert events == ["native client"] + else: + assert _ch.run_connected(work, orgs=("org",)) == 17 + assert events == ["native client", "native held-org guard", "native work"] + + +@pytest.mark.parametrize( + "fault", + [ + "", + "api_configuration", + "operator_token", + "arbitrary_setting", + "legacy_field", + "duplicate", + "schema_type", + "pg_port_type", + ], +) +def test_receipt_orchestration_refuses_before_clients_and_keeps_the_real_native_guard( + monkeypatch, tmp_path, capsys, fault +): + import sys + + import server.wiring + + fixture, config = fixture_config() + fixture, launch_path, _, _, _ = launch_files(tmp_path, fixture, config) + manifest = dataclasses.asdict(fixture) + if fault == "api_configuration": + config.cell_router_token = "lcr_" + "0" * 128 + elif fault == "operator_token": + config.router_operator_token = "lro_" + "1" * 128 + elif fault == "arbitrary_setting": + config.instance_heartbeat_seconds += 1 + manifest["cli_configuration_sha256"] = gate._digest(config.model_dump(mode="json")) + elif fault == "legacy_field": + manifest["configuration_sha256"] = fixture.cli_configuration_sha256 + elif fault == "schema_type": + manifest["schema"] = 2.0 + elif fault == "pg_port_type": + manifest["pg_port"] = float(manifest["pg_port"]) + raw = json.dumps(manifest) + if fault == "duplicate": + raw = raw[:-1] + ', "schema":2}' + manifest_path = tmp_path / "fixture.json" + manifest_path.write_text(raw) + output = tmp_path / "output.jsonl" + argv = ["driver"] + for key, value in vars( + args( + launch_manifest=str(launch_path), + fixture_manifest=str(manifest_path), + output=str(output), + ) + ).items(): + argv += ["--" + key.replace("_", "-"), str(value)] + events = [] + native_guard_edges(monkeypatch, config, events) + monkeypatch.setattr(sys, "argv", argv) + monkeypatch.setattr(gate, "_source", lambda _: {"clean": True}) + monkeypatch.setattr(gate, "_owned_server", lambda _: None) + monkeypatch.setattr( + gate, "_preflight", AsyncMock(side_effect=lambda *_: events.append("readonly")) + ) + monkeypatch.setattr(server.wiring, "wire", lambda: events.append("wire")) + summary = { + "latency_within_limit": True, + "throughput_within_limit": True, + "p99_delta_ms": 0.0, + "throughput_loss": 0.0, + } + + async def run(*_): + events.append("native work") + return summary + + monkeypatch.setattr(gate, "_run", run) + assert gate.main() == int(bool(fault)) + text = capsys.readouterr().out + assert json.loads(text)["verdict"] == ("INVALID" if fault else "PASS") + assert all( + value not in text + for value in [ + args().pg_dsn, + "secret-token", + "test-only-root", + "lcr_" + "0" * 128, + "lro_" + "1" * 128, + ] + ) + if fault: + assert events == [] and not output.exists() + else: + assert events == [ + "readonly", + "wire", + "native client", + "native held-org guard", + "native work", + ] + record = json.loads(output.read_text().splitlines()[0]) + assert record["manifest"]["schema"] == 2 + assert record["shared_configuration_excludes"] == ["cell_router_token"] + assert record["credential_posture"] == receipt_posture() + + +def test_unversioned_single_configuration_fixture_is_no_longer_accepted(tmp_path): + legacy = { + "base_url": args().base_url, + "pg_host": "127.0.0.1", + "pg_port": 15790, + "pg_database": "gate_cell", + "ch_host": "127.0.0.1", + "ch_port": 18790, + "ch_database": "gate_cell", + "ch_telemetry_database": "gate_cell_t", + "cell": "gate-cell", + "org": "org", + "namespace": "ns", + "deployment_id": "deployment", + "epoch": 1, + "instance_id": "instance", + "build_commit": "a" * 40, + "source_tree": "b" * 40, + "launch_manifest_sha256": "c" * 64, + "ownership_receipt_sha256": "d" * 64, + "configuration_sha256": "e" * 64, + "background_workers": "off", + "controller_excluded": True, + "pg_internal_port": 5432, + } + path = tmp_path / "legacy.json" + raw = json.dumps(legacy).encode() + path.write_bytes(raw) + with pytest.raises(gate.Invalid): + gate.Fixture.read(str(path)) + assert path.read_bytes() == raw + + +@pytest.mark.parametrize("field", ["pg_port", "ch_port", "epoch"]) +@pytest.mark.parametrize("kind", [bool, float, str]) +def test_fixture_integer_fields_never_coerce_json_types(field, kind): + fixture, config = fixture_config() + with pytest.raises(gate.Invalid, match="fixture field types"): + dataclasses.replace(fixture, **{field: kind(getattr(fixture, field))}).validate( + args(), config + ) diff --git a/tests/unit/test_cell_trigger_gate_artifact_close.py b/tests/unit/test_cell_trigger_gate_artifact_close.py index 3234dc45f5..b403eb4ebb 100644 --- a/tests/unit/test_cell_trigger_gate_artifact_close.py +++ b/tests/unit/test_cell_trigger_gate_artifact_close.py @@ -1,10 +1,13 @@ """Artifact close retains its first flush, sync or file-close failure.""" import asyncio +import dataclasses +import json import pytest from bench import cell_trigger_gate as gate +from tests.unit.test_bench_cell_trigger_gate import fixture_config, receipt_posture @pytest.mark.parametrize("phase", ["flush", "fsync"]) @@ -21,7 +24,14 @@ def test_actual_artifact_close_retains_first_flush_sync_or_close_failure( first = first_kind("flush or sync failed first") if first_kind is not None else None second = second_kind("file close failed second") if second_kind is not None else None artifact = gate.Artifact(str(tmp_path / "raw.jsonl")) - artifact.emit("control") + fixture, config = fixture_config() + artifact.emit( + "configuration", + manifest=dataclasses.asdict(fixture), + shared_configuration_excludes=["cell_router_token"], + credential_posture=receipt_posture(), + api_configuration_attested=True, + ) original = artifact.file events = [] @@ -58,3 +68,10 @@ def sync(_fd): assert original.closed and events[-1] == "file close" assert events.count("file close") == 1 assert ("fsync" in events) == (phase != "flush" or first is None) + record = json.loads((tmp_path / "raw.jsonl").read_text()) + assert record["manifest"]["schema"] == 2 + assert record["manifest"]["cli_configuration_sha256"] == gate._digest(config.model_dump()) + assert "configuration_sha256" not in record["manifest"] + assert record["shared_configuration_excludes"] == ["cell_router_token"] + assert record["credential_posture"] == receipt_posture() + assert config.pg_dsn not in json.dumps(record) diff --git a/tests/unit/test_cell_trigger_gate_diagnostics.py b/tests/unit/test_cell_trigger_gate_diagnostics.py index b2f4594ef8..580eb3d733 100644 --- a/tests/unit/test_cell_trigger_gate_diagnostics.py +++ b/tests/unit/test_cell_trigger_gate_diagnostics.py @@ -2,6 +2,7 @@ import argparse import asyncio +import dataclasses import json from unittest.mock import AsyncMock @@ -150,9 +151,12 @@ def test_main_preserves_first_invalid_and_requires_successful_artifact_close( import server.settings import server.wiring from server.cli import _ch - from tests.unit.test_bench_cell_trigger_gate import args, fixture_config + from tests.unit.test_bench_cell_trigger_gate import args, fixture_config, launch_files fixture, config = fixture_config() + fixture, launch_path, _, _, _ = launch_files(tmp_path, fixture, config) + manifest_path = tmp_path / "fixture.json" + manifest_path.write_text(json.dumps(dataclasses.asdict(fixture))) prior = prior_kind("first native work failure") if prior_kind is not None else None output = output_kind("secondary artifact failure") if output_kind is not None else None events = [] @@ -182,15 +186,18 @@ def __getattr__(self, name): argv = ["driver"] for key, value in vars( - args(fixture_manifest="unused", output=str(tmp_path / "raw.jsonl")) + args( + fixture_manifest=str(manifest_path), + launch_manifest=str(launch_path), + output=str(tmp_path / "raw.jsonl"), + ) ).items(): argv += ["--" + key.replace("_", "-"), str(value)] monkeypatch.setattr(sys, "argv", argv) monkeypatch.setattr(server.settings, "settings", config) monkeypatch.setattr(server.wiring, "wire", lambda: None) - monkeypatch.setattr(gate.Fixture, "read", lambda _: fixture) monkeypatch.setattr(gate, "_source", lambda _: {}) - monkeypatch.setattr(gate, "_launch", lambda *_: {}) + monkeypatch.setattr(gate, "_owned_server", lambda _: None) monkeypatch.setattr(gate, "_preflight", AsyncMock()) monkeypatch.setattr(gate, "Artifact", Artifact) diff --git a/tests/unit/test_cell_trigger_gate_first_error.py b/tests/unit/test_cell_trigger_gate_first_error.py index 7a3ccdf41b..5379b0f619 100644 --- a/tests/unit/test_cell_trigger_gate_first_error.py +++ b/tests/unit/test_cell_trigger_gate_first_error.py @@ -1,6 +1,7 @@ """Driver work retains its error through secondary cleanup and artifact failures.""" import asyncio +import dataclasses import json import time from types import SimpleNamespace @@ -11,7 +12,7 @@ from asyncpg.transaction import Transaction from bench import cell_trigger_gate as gate -from tests.unit.test_bench_cell_trigger_gate import args, fixture_config +from tests.unit.test_bench_cell_trigger_gate import args, fixture_config, receipt_posture FIRST = [None, ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] SECOND = [None, OSError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] @@ -46,6 +47,14 @@ def artifact_output(monkeypatch, tmp_path): def configure(errors): artifact = gate.Artifact(str(tmp_path / f"raw-{len(artifacts)}.jsonl")) artifacts.append(artifact) + fixture, _ = fixture_config() + artifact.emit( + "configuration", + manifest=dataclasses.asdict(fixture), + shared_configuration_excludes=["cell_router_token"], + credential_posture=receipt_posture(), + api_configuration_attested=True, + ) original = artifact.file events = [] @@ -73,6 +82,8 @@ def http_edges(monkeypatch): import server.settings fixture, config = fixture_config() + assert fixture.validate(args(), config) == 45500 + assert config.cell_router_token == config.router_operator_token == "" monkeypatch.setattr(server.settings, "settings", config) monkeypatch.setattr(gate, "CALLS", 1) monkeypatch.setattr(gate, "WARMUP", 0) From c884dd174ea6b07236bf886ff7129e4ef15c6989 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 04:51:38 +0000 Subject: [PATCH 062/114] Preserve native errors through nonce cleanup Signed-off-by: zlareb1 --- server/cli/cell.py | 3 +- server/cli/cell_attach.py | 15 +- server/cli/cell_prepare.py | 6 +- .../unit/test_cell_native_http_first_error.py | 6 +- .../test_cell_native_nonce_first_error.py | 269 ++++++++++++++++++ 5 files changed, 288 insertions(+), 11 deletions(-) create mode 100644 tests/unit/test_cell_native_nonce_first_error.py diff --git a/server/cli/cell.py b/server/cli/cell.py index e43a5a4d07..fb1b04db95 100644 --- a/server/cli/cell.py +++ b/server/cli/cell.py @@ -349,7 +349,7 @@ def _cmd_remove(args: argparse.Namespace) -> int: print(f"loom cell remove: refusing: {reason}", file=sys.stderr) return 1 try: - answer = asyncio.run(_remove_locked(args, router, row)) + answer = asyncio.run(_remove_locked(args, router)) except Exception as e: # noqa: BLE001 — an unknown control outcome stops the run print(f"loom cell remove: {type(e).__name__}: {e}; stop and rerun", file=sys.stderr) return 1 @@ -360,7 +360,6 @@ def _cmd_remove(args: argparse.Namespace) -> int: async def _remove_locked( args: argparse.Namespace, router: httpx.URL, - row: dict[str, Any], transport: httpx.AsyncBaseTransport | None = None, ) -> dict[str, Any]: from server.cli import cell_control diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 8fd6e7ebee..821a77448b 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -376,7 +376,11 @@ async def begin(self) -> None: await _write_nonces(self.ch, nonces) self.row = {**self.row, **await self.mutate("/attach/begin", body)} self.begun_keys_fp = self.keys_fp - finally: + except BaseException: # Nonce cleanup preserves the first failure or cancellation. + with contextlib.suppress(BaseException): + await _delete_nonces(self.ch, nonces) + raise + else: await _delete_nonces(self.ch, nonces) # -- the scans --------------------------------------------------------------------- @@ -542,11 +546,16 @@ async def serving(self) -> dict[str, Any]: nonces = _new_nonces() try: await _write_nonces(self.ch, nonces) - return await self.mutate( + answer = await self.mutate( "/attach/serving", {"deployment_id": self.row["deployment_id"], "nonces": nonces} ) - finally: + except BaseException: # Nonce cleanup preserves the first failure or cancellation. + with contextlib.suppress(BaseException): + await _delete_nonces(self.ch, nonces) + raise + else: await _delete_nonces(self.ch, nonces) + return answer # -- the run ----------------------------------------------------------------------- diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index 4310b4c9bb..517a461840 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -513,7 +513,11 @@ async def _prepare_locked( try: await _write_nonces(ch, body["nonces"]) answer = await _register(args, router, body, transport) - finally: + except BaseException: # Nonce cleanup preserves the first failure or cancellation. + with contextlib.suppress(BaseException): + await _delete_nonces(ch, body["nonces"]) + raise + else: await _delete_nonces(ch, body["nonces"]) # The processes may have changed while the probe was retried: judge the ones live now. beats = await _read_beats(ch) diff --git a/tests/unit/test_cell_native_http_first_error.py b/tests/unit/test_cell_native_http_first_error.py index 4634e7d69c..c9b61bd146 100644 --- a/tests/unit/test_cell_native_http_first_error.py +++ b/tests/unit/test_cell_native_http_first_error.py @@ -180,11 +180,7 @@ async def test_required_client_exit_failure_cannot_report_success( async def test_successful_client_neighbor(exchange, boundary): kind, result = await observed(lambda: invoke(boundary)) assert kind == "value" - expected = ( - None - if boundary == "placement" - else ("dep-eu-1" if boundary == "health" else exchange.answer) - ) + expected = "dep-eu-1" if boundary == "health" else exchange.answer assert result == expected assert exchange.events == ["construct", "enter", "request", "exit-call", "exit-await"] diff --git a/tests/unit/test_cell_native_nonce_first_error.py b/tests/unit/test_cell_native_nonce_first_error.py new file mode 100644 index 0000000000..94254c3d96 --- /dev/null +++ b/tests/unit/test_cell_native_nonce_first_error.py @@ -0,0 +1,269 @@ +"""Native callers retain their first outcome while deleting their probe nonces.""" + +from __future__ import annotations + +import argparse +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import httpx +import pytest + +from server.cli import cell_attach, cell_control, cell_prepare +from server.core import leases, object_store, pg +from server.settings import settings +from tests.unit import test_cell_native_http_first_error as native_http +from tests.unit.test_cell_attach import FakeConn +from tests.unit.test_cell_control import snapshot + +exchange = native_http.exchange +observed = native_http.observed +FIRST_ERRORS = native_http.FIRST_ERRORS +CALLERS = ("prepare", "begin", "serving") +FATAL_ERRORS = (asyncio.CancelledError, SystemExit, KeyboardInterrupt) +CLEANUP_PHASES = ("postgres", "clickhouse", "bucket") +URL = httpx.URL("http://router.test") + + +@pytest.fixture +def nonce_edges(monkeypatch, exchange): + state = SimpleNamespace( + secondary=None, + phase="postgres", + write_first=False, + events=[], + normalized=[], + cleanup_entered=asyncio.Event(), + cleanup_block=False, + ) + nonces = {"postgres": "pg", "clickhouse": "ch", "bucket": "bucket"} + + async def deletion(phase): + state.events.append(phase) + if phase == state.phase: + state.cleanup_entered.set() + if state.cleanup_block: + await asyncio.Future() + if state.secondary is not None: + raise state.secondary + + async def postgres(*args): + await deletion("postgres") + + async def clickhouse(*args): + assert args[0] == leases.LOCK_RELEASE + await deletion("clickhouse") + + async def read_object(location): + return b"bucket" + + async def bucket(*args): + await deletion("bucket") + + async def write(*args): + if state.write_first: + raise exchange.first + + monkeypatch.setattr(pg, "execute", postgres) + monkeypatch.setattr(object_store, "root_prefix", lambda: "") + monkeypatch.setattr(object_store, "read", read_object) + monkeypatch.setattr(object_store, "delete", bucket) + monkeypatch.setattr(cell_prepare, "_new_nonces", lambda: nonces.copy()) + monkeypatch.setattr(cell_prepare, "_write_nonces", write) + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "document_s3_bucket", "bucket-test") + state.ch = SimpleNamespace(execute=clickhouse) + return state + + +@pytest.fixture +def native_call(monkeypatch, nonce_edges, exchange): + state = nonce_edges + row = {**snapshot(), "is_first": False, "state": "attaching", "mode": "enforced"} + exchange.answer = row.copy() + conn = FakeConn() + rows = SimpleNamespace(deployment_id="dep", first=None, tenancy_rows=0) + monkeypatch.setattr(cell_prepare, "_read_rows", AsyncMock(return_value=rows)) + monkeypatch.setattr(cell_prepare, "_read_beats", AsyncMock(return_value=[])) + monkeypatch.setattr(cell_prepare, "_beat_fields", lambda *args: [{}]) + monkeypatch.setattr(cell_prepare, "_state_rows", lambda rows: 0) + monkeypatch.setattr(cell_prepare, "_prefix_objects", AsyncMock(return_value=0)) + monkeypatch.setattr(cell_prepare, "_answer_problem", lambda *args: "") + monkeypatch.setattr(cell_prepare, "_own_rows", lambda *args: ({}, None)) + state.write_rows = AsyncMock(return_value=[]) + monkeypatch.setattr(cell_prepare, "_write_rows", state.write_rows) + monkeypatch.setattr(cell_control, "claim", AsyncMock(side_effect=lambda conn, call, row: row)) + args = argparse.Namespace( + cell="first", + wait=0, + host="first.example.test", + credential_sha256="hash", + probe_url="", + ch_allow_user=[], + ch_allow_user_directory=[], + ch_allow_object=[], + ch_allow_replica=[], + pg_allow_role=[], + ) + + async def mutate(path, body): + return await cell_control.request(URL, "first", None, path, body) + + run = SimpleNamespace( + ch=state.ch, args=args, row=row, own_fp=({}, None), keys_fp="key", mutate=mutate + ) + + async def invoke(caller, owned): + owner = cell_control.Owner(exit_s=0.5) + direct = SimpleNamespace( + conn=conn, transaction=lambda work: work(), call=lambda work, **kw: work() + ) + + async def work(): + if caller == "prepare": + return await cell_prepare._prepare_locked( + state.ch, args, URL, None, owner if owned else direct + ) + return await getattr(cell_attach.Run, caller)(run) + + async def opening(): + owner.conn = conn + + return await owner.run(opening, work, opening_s=0.5) if owned else await work() + + state.conn = conn + state.invoke = invoke + return state + + +@pytest.fixture(autouse=True) +def selected_normalization(monkeypatch, nonce_edges): + original_prepare = cell_prepare.Refused + original_control = cell_control._refused + + class PrepareRefused(original_prepare): + def __init__(self, message): + super().__init__(message) + nonce_edges.normalized.append(self) + + def control_refused(message): + selected = original_control(message) + nonce_edges.normalized.append(selected) + return selected + + monkeypatch.setattr(cell_prepare, "Refused", PrepareRefused) + monkeypatch.setattr(cell_control, "_refused", control_refused) + + +def assert_first(state, exchange, caller, first_kind, owned, outcome): + kind, result = outcome + assert kind == "error" + normalized = not state.write_first and ( + (caller == "prepare" and issubclass(first_kind, httpx.HTTPError)) + or (caller != "prepare" and issubclass(first_kind, (httpx.HTTPError, TimeoutError))) + ) + if normalized: + assert state.normalized and result is state.normalized[0] + elif owned and first_kind is asyncio.CancelledError: + # asyncio reconstructs cancellation when Owner retrieves its child Task. + assert isinstance(result, asyncio.CancelledError) + else: + assert result is exchange.first + assert state.write_rows.await_count == 0 + assert state.conn.executed.count("CLOSE") == int(owned) + + +@pytest.mark.parametrize("caller", CALLERS) +@pytest.mark.parametrize("owned", (False, True)) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +@pytest.mark.parametrize("cleanup_kind", (*FATAL_ERRORS, None)) +@pytest.mark.parametrize("phase", CLEANUP_PHASES) +@pytest.mark.parametrize("http_exit_failure", (False, True)) +async def test_http_first_outcome_survives_native_nonce_cleanup( + native_call, exchange, caller, owned, first_kind, cleanup_kind, phase, http_exit_failure +): + state = native_call + state.phase = phase + state.secondary = cleanup_kind("nonce-second") if cleanup_kind is not None else None + exchange.first = first_kind("request-first") + exchange.second = KeyboardInterrupt("http-exit-second") if http_exit_failure else None + assert_first( + state, + exchange, + caller, + first_kind, + owned, + await observed(lambda: state.invoke(caller, owned)), + ) + assert phase in state.events and len(exchange.exit_args) == 3 + + +@pytest.mark.parametrize("caller", CALLERS) +@pytest.mark.parametrize("owned", (False, True)) +@pytest.mark.parametrize("cleanup_kind", (*FATAL_ERRORS, ValueError, None)) +@pytest.mark.parametrize("phase", CLEANUP_PHASES) +async def test_standalone_nonce_cleanup_keeps_cancellation_and_success_neighbors( + native_call, exchange, caller, owned, cleanup_kind, phase +): + state = native_call + state.phase = phase + state.secondary = cleanup_kind("nonce-only") if cleanup_kind is not None else None + kind, result = await observed(lambda: state.invoke(caller, owned)) + if cleanup_kind in FATAL_ERRORS: + assert kind == "error" + if owned and cleanup_kind is asyncio.CancelledError: + assert isinstance(result, asyncio.CancelledError) + else: + assert result is state.secondary + assert state.write_rows.await_count == 0 + else: + assert kind == "value" + assert result == ( + 0 if caller == "prepare" else None if caller == "begin" else exchange.answer + ) + assert state.events == list(CLEANUP_PHASES) + assert state.write_rows.await_count == int(caller == "prepare") + assert state.conn.executed.count("CLOSE") == int(owned) + + +@pytest.mark.parametrize("caller", CALLERS) +@pytest.mark.parametrize("owned", (False, True)) +@pytest.mark.parametrize("cleanup_kind", FATAL_ERRORS) +async def test_nonce_write_failure_stays_first(native_call, exchange, caller, owned, cleanup_kind): + state = native_call + state.write_first = True + state.secondary = cleanup_kind("nonce-second") + exchange.first = ValueError("write-first") + assert_first( + state, + exchange, + caller, + ValueError, + owned, + await observed(lambda: state.invoke(caller, owned)), + ) + assert exchange.events == [] and state.events == ["postgres"] + + +@pytest.mark.parametrize("caller", CALLERS) +async def test_repeated_cancellation_keeps_first_stop_through_nonce_deletion( + native_call, exchange, caller +): + state = native_call + state.cleanup_block = True + exchange.block = True + task = asyncio.create_task(observed(lambda: state.invoke(caller, False))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("first-stop") + await asyncio.wait_for(state.cleanup_entered.wait(), 1) + task.cancel("second-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) and result.args == ("first-stop",) + assert state.write_rows.await_count == 0 + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) From 0a94de8d45f42697f8caa1f62736b3ca7c005f26 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 05:14:56 +0000 Subject: [PATCH 063/114] Native commands preserve their first failure through cleanup Connection and participant cleanup preserve entry and work errors. Successful commands require stopped producers, drained writes, flushed buffers and a positive retirement acknowledgement. Signed-off-by: zlareb1 --- server/cli/_ch.py | 22 +- server/insights/app/provider_command.py | 102 ++++-- tests/unit/test_cli_native_first_error.py | 415 ++++++++++++++++++++++ tests/unit/test_provider_command.py | 7 +- 4 files changed, 516 insertions(+), 30 deletions(-) create mode 100644 tests/unit/test_cli_native_first_error.py diff --git a/server/cli/_ch.py b/server/cli/_ch.py index c67479b965..7772fbdb1a 100644 --- a/server/cli/_ch.py +++ b/server/cli/_ch.py @@ -23,6 +23,7 @@ import logging import sys from collections.abc import Awaitable, Callable, Iterable +from contextlib import suppress from dataclasses import dataclass from typing import TYPE_CHECKING, Literal @@ -299,7 +300,8 @@ async def connect_declared( await check_configured(ch) await check_acceptance(orgs=named, org_free=org_free) except BaseException: - await disconnect(ch) + with suppress(BaseException): + await disconnect(ch) raise @@ -338,7 +340,8 @@ async def connect_verified(ch: CHClient) -> None: try: await check_configured(ch) except BaseException: - await disconnect(ch) + with suppress(BaseException): + await disconnect(ch) raise from server import boot from server.core.background import register_buffer_drainer @@ -362,7 +365,11 @@ async def disconnect(ch: CHClient) -> None: pg.expect_deployment(None) try: await pg.close_pool() - finally: + except BaseException: + with suppress(BaseException): + await ch.disconnect() + raise + else: await ch.disconnect() @@ -389,8 +396,13 @@ async def _go() -> T: try: async with model_command(): await check_acceptance(orgs=named, org_free=org_free) - return await run(ch) - finally: + result = await run(ch) + except BaseException: + with suppress(BaseException): + await disconnect(ch) + raise + else: await disconnect(ch) + return result return asyncio.run(_go()) diff --git a/server/insights/app/provider_command.py b/server/insights/app/provider_command.py index b89c418e17..3b9f7d8a4f 100644 --- a/server/insights/app/provider_command.py +++ b/server/insights/app/provider_command.py @@ -3,6 +3,7 @@ import asyncio import contextlib import logging +import sys from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import asynccontextmanager @@ -34,12 +35,63 @@ async def _adopt_overrides(*, failing: bool = False) -> bool: async def _keep_overrides( loaded: bool, sleep: Callable[[float], Awaitable[None]] = asyncio.sleep -) -> None: +) -> SystemExit | KeyboardInterrupt | None: """A long command (a rechunk, a re-embed) follows an override set while it runs.""" set_query_tag("cli.settings_overrides") - while True: - await sleep(max(1.0, float(settings.runtime_overrides_refresh_seconds))) - loaded = await _adopt_overrides(failing=not loaded) + try: + while True: + await sleep(max(1.0, float(settings.runtime_overrides_refresh_seconds))) + loaded = await _adopt_overrides(failing=not loaded) + except (SystemExit, KeyboardInterrupt) as error: + # A task's fatal exception must reach its owner before the loop stops. + return error + + +async def _finish_participant() -> None: + gate = retirement.get_gate() + gate.close() + from server.memory.app import embedder_calibration + + first: BaseException | None = None + stopped = False + try: + embedder_calibration.stop_refresh_loop() + stopped = True + except BaseException as error: # noqa: BLE001 — finish safe cleanup, then raise the first failure + first = error + producers = tuple(gate.producers) + for producer in producers: + producer.cancel() + try: + if producers: + _, pending = await asyncio.wait(producers, timeout=COMMAND_DRAIN_S) + if pending: + raise RuntimeError("Native command producers did not stop") + except BaseException as error: # noqa: BLE001 — finish safe cleanup, then raise the first failure + if first is None: + first = error + stopped = False + for producer in producers: + if producer.done() and not producer.cancelled(): + try: + producer.result() + except BaseException as error: # noqa: BLE001 — finish safe cleanup, then raise the first failure + if first is None: + first = error + if stopped: + gate.stopped() + try: + if not await drain_task_sets(timeout_s=COMMAND_DRAIN_S): + raise RuntimeError("Native command deferred writes did not drain") + if not await flush_registered_buffers(): + raise RuntimeError("Native command buffers did not flush") + if await retire_process() is not True: + raise RuntimeError("Native command retirement was not certified") + except BaseException as error: # noqa: BLE001 — finish safe cleanup, then raise the first failure + if first is None: + first = error + if first is not None: + raise first @asynccontextmanager @@ -50,27 +102,19 @@ async def participant_command() -> AsyncIterator[None]: try: async with retirement.get_gate().admit(): yield - finally: - gate = retirement.get_gate() - gate.close() - from server.memory.app import embedder_calibration - - embedder_calibration.stop_refresh_loop() - producers = tuple(gate.producers) - for producer in producers: - producer.cancel() - if producers: - await asyncio.wait(producers, timeout=COMMAND_DRAIN_S) - gate.stopped() - if await drain_task_sets(timeout_s=COMMAND_DRAIN_S) and await flush_registered_buffers(): - await retire_process() + except BaseException: + with contextlib.suppress(BaseException): + await _finish_participant() + raise + else: + await _finish_participant() @asynccontextmanager async def model_command() -> AsyncIterator[None]: """The permanent lifecycle also holds the reversible model-work admission.""" async with participant_command(): - keeper: asyncio.Task[None] | None = None + keeper: asyncio.Task[SystemExit | KeyboardInterrupt | None] | None = None try: await load_active_configuration() loaded = await _adopt_overrides() @@ -80,7 +124,21 @@ async def model_command() -> AsyncIterator[None]: async with get_model_work_gate().admit(): yield finally: + first = sys.exception() if keeper is not None: - keeper.cancel() - with contextlib.suppress(asyncio.CancelledError): - await keeper + owner = asyncio.current_task() + assert owner is not None + cancelling = owner.cancelling() + requested = keeper.cancel() + try: + try: + fatal = await keeper + except asyncio.CancelledError: + if not requested or owner.cancelling() != cancelling: + raise + else: + if fatal is not None: + raise fatal + except BaseException: + if first is None: + raise diff --git a/tests/unit/test_cli_native_first_error.py b/tests/unit/test_cli_native_first_error.py new file mode 100644 index 0000000000..35f71b60fb --- /dev/null +++ b/tests/unit/test_cli_native_first_error.py @@ -0,0 +1,415 @@ +"""The native command lifetime preserves its first failure through owned cleanup.""" + +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from server.cli import _ch +from server.core import background, retirement +from server.core.model_work_gate import ModelWorkGate +from server.insights.app import provider_barrier as barrier +from server.insights.app import provider_command as command + + +@pytest.fixture +def native(monkeypatch): + from server import boot + from server.core import ch, pg, pg_profile + from server.memory.app import embedder_calibration + + events = [] + gate = retirement.get_gate() + client = SimpleNamespace(connected=False) + errors = {} + model_gate = ModelWorkGate() + + def event(name): + events.append(name) + if name in errors: + raise errors[name] + + async def connect(): + event("connect") + client.connected = True + + async def close_ch(): + client.connected = False + event("ch") + + async def close_pg(): + event("pg") + + async def verify(_): + assert not gate.enrolled + event("verify") + + async def enroll(): + event("enroll") + gate.enrolled = True + + async def acceptance(_): + assert gate.enrolled and gate.active and not gate.closed + event("acceptance") + + async def configuration(): + event("configuration") + + async def flush(): + event("flush") + + async def certificate(proof): + assert gate.quiescent and gate.enrolled and client.connected + assert proof.version > proof.after + assert events[-1] == "flush" + event("certificate") + + client.connect, client.disconnect = connect, close_ch + monkeypatch.setattr(ch, "get_client", lambda: client) + monkeypatch.setattr(pg, "close_pool", close_pg) + monkeypatch.setattr(pg_profile, "check_configured", verify) + monkeypatch.setattr(_ch, "check_orgs", acceptance) + monkeypatch.setattr(command, "register_process", enroll) + monkeypatch.setattr(command, "load_active_configuration", configuration) + monkeypatch.setattr(command, "_adopt_overrides", AsyncMock(return_value=True)) + monkeypatch.setattr(command, "get_model_work_gate", lambda: model_gate) + monkeypatch.setattr(barrier, "get_model_work_gate", lambda: model_gate) + monkeypatch.setattr(background, "_registered_tasks", set()) + monkeypatch.setattr(background, "_buffer_drainers", {}) + monkeypatch.setattr(boot, "drain_writes", flush) + monkeypatch.setattr(barrier, "_OWN", barrier._Own()) + monkeypatch.setattr( + barrier, "_HEARTBEAT", SimpleNamespace(signal_stop=lambda: None, running=False) + ) + monkeypatch.setattr(barrier, "_server_side_settled", AsyncMock(return_value=True)) + monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) + monkeypatch.setattr(barrier, "record_retirement", certificate) + monkeypatch.setattr(embedder_calibration, "stop_refresh_loop", lambda: event("stop")) + return SimpleNamespace( + events=events, errors=errors, client=client, gate=gate, model_gate=model_gate, event=event + ) + + +FIRST = [ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] + + +@pytest.mark.parametrize("first_class", FIRST) +@pytest.mark.parametrize("entry", ["verify", "acceptance", "configuration", "work"]) +@pytest.mark.parametrize("secondary", ["pg", "ch", "certificate"]) +def test_native_first_entry_or_work_error_survives_all_close_boundaries( + native, first_class, entry, secondary +): + first = first_class("first") + native.errors[entry] = first + native.errors[secondary] = OSError("secondary") + + async def work(_): + native.event("work") + return 7 + + with pytest.raises(first_class) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert native.events[-2:] == ["pg", "ch"] + assert not native.client.connected + assert native.model_gate.active == native.gate.active == 0 + if entry != "work": + assert "work" not in native.events + if entry != "verify": + assert "stop" in native.events + + +@pytest.mark.parametrize("first_class", FIRST) +@pytest.mark.parametrize("connector", ["connect_checked", "connect_verified"]) +@pytest.mark.asyncio +async def test_connect_entry_preserves_refusal_and_attempts_both_closes( + native, first_class, connector +): + first = first_class("pairing refusal") + native.errors.update(verify=first, pg=OSError("pg"), ch=RuntimeError("ch")) + call = getattr(_ch, connector) + with pytest.raises(first_class) as caught: + if connector == "connect_checked": + await call(native.client, orgs=("o",)) + else: + await call(native.client) + assert caught.value is first + assert native.events == ["connect", "verify", "pg", "ch"] + + +@pytest.mark.parametrize("first_class", FIRST) +@pytest.mark.asyncio +async def test_disconnect_preserves_first_pg_failure_and_still_closes_ch(native, first_class): + first = first_class("pg close") + native.errors.update(pg=first, ch=OSError("ch close")) + with pytest.raises(first_class) as caught: + await _ch.disconnect(native.client) + assert caught.value is first + assert native.events == ["pg", "ch"] + + +@pytest.mark.parametrize("cleanup", ["stop", "flush", "certificate", "pg", "ch"]) +@pytest.mark.parametrize("first_class", FIRST) +def test_required_native_cleanup_error_after_success_is_fatal(native, cleanup, first_class): + first = first_class(cleanup) + native.errors[cleanup] = first + + async def work(_): + native.event("work") + return 7 + + with pytest.raises(first_class) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert native.events[-2:] == ["pg", "ch"] + assert not native.client.connected + + +def test_first_required_cleanup_error_wins_and_all_safe_cleanup_runs(native, monkeypatch): + first = OSError("keeper") + + async def keeper(_): + raise first + + async def work(_): + await asyncio.sleep(0) + return 7 + + monkeypatch.setattr(command, "_keep_overrides", keeper) + native.errors.update(certificate=ValueError("retirement"), pg=RuntimeError("pg")) + with pytest.raises(OSError) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert "stop" in native.events and "flush" in native.events + assert native.events[-3:] == ["certificate", "pg", "ch"] + + +@pytest.mark.parametrize("first_class", FIRST) +def test_driver_restore_failure_remains_first_when_keeper_and_native_close_fail( + native, monkeypatch, first_class +): + first = first_class("driver work") + + async def keeper(_): + raise OSError("keeper") + + async def work(_): + await asyncio.sleep(0) + try: + raise first + finally: + native.events.append("restore") + + monkeypatch.setattr(command, "_keep_overrides", keeper) + native.errors.update(certificate=OSError("certificate"), pg=OSError("pg"), ch=OSError("ch")) + with pytest.raises(first_class) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert native.events.index("restore") < native.events.index("stop") + assert native.events[-2:] == ["pg", "ch"] + + +@pytest.mark.parametrize("phase", ["drain", "flush", "retire"]) +@pytest.mark.parametrize("first_class", [None, *FIRST]) +def test_unacknowledged_cleanup_refuses_without_early_certification( + native, monkeypatch, phase, first_class +): + calls = [] + first = first_class("work") if first_class else None + + async def unacknowledged(**_): + calls.append(phase) + return False + + target = { + "drain": "drain_task_sets", + "flush": "flush_registered_buffers", + "retire": "retire_process", + } + monkeypatch.setattr(command, target[phase], unacknowledged) + + async def work(_): + if first is not None: + raise first + return 7 + + with pytest.raises(first_class or RuntimeError) as caught: + _ch.run_connected(work, orgs=("o",)) + if first is not None: + assert caught.value is first + assert calls == [phase] + assert "certificate" not in native.events + assert native.events[-2:] == ["pg", "ch"] + + +def test_actual_deferred_descendant_and_role_producer_end_before_certificate(native): + async def work(_): + async def producer(): + try: + await asyncio.Event().wait() + finally: + await asyncio.sleep(0) + native.events.append("producer-ended") + + native.gate.track(asyncio.create_task(producer())) + await asyncio.sleep(0) + + async def descendant(): + await asyncio.sleep(0) + native.events.append("descendant") + + async def deferred(): + await asyncio.sleep(0) + background.create_deferred_task(descendant()) + + background.create_deferred_task(deferred()) + return 7 + + assert _ch.run_connected(work, orgs=("o",)) == 7 + assert native.events.index("producer-ended") < native.events.index("certificate") + assert native.events.index("descendant") < native.events.index("certificate") + assert native.gate.quiescent and native.model_gate.active == 0 + assert native.events[-3:] == ["certificate", "pg", "ch"] + + +@pytest.mark.parametrize("first_class", [SystemExit, KeyboardInterrupt]) +@pytest.mark.asyncio +async def test_override_keeper_returns_fatal_to_owner_before_loop_shutdown(first_class): + first = first_class("keeper fatal") + + async def sleep(_): + raise first + + assert await command._keep_overrides(True, sleep=sleep) is first + + +@pytest.mark.parametrize("first_class", [SystemExit, KeyboardInterrupt]) +def test_real_override_keeper_fatal_is_retrieved_before_native_cleanup( + native, monkeypatch, first_class +): + first = first_class("keeper fatal") + actual = command._keep_overrides + + async def sleep(_): + raise first + + monkeypatch.setattr(command, "_keep_overrides", lambda loaded: actual(loaded, sleep=sleep)) + + async def work(_): + await asyncio.sleep(0) + return 7 + + with pytest.raises(first_class) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert native.events[-3:] == ["certificate", "pg", "ch"] + assert native.gate.quiescent + + +def test_clean_native_success_has_one_certified_exit_and_no_keeper_left(native, monkeypatch): + keepers = [] + actual = command._keep_overrides + + async def keeper(loaded): + keepers.append(asyncio.current_task()) + return await actual(loaded) + + async def work(_): + await asyncio.sleep(0) + return 7 + + monkeypatch.setattr(command, "_keep_overrides", keeper) + assert _ch.run_connected(work, orgs=("o",)) == 7 + assert native.events.count("certificate") == 1 + assert native.events[-3:] == ["certificate", "pg", "ch"] + assert keepers and all(task.done() for task in keepers) + + +def test_caller_cancellation_during_keeper_stop_is_not_swallowed(native, monkeypatch): + owner = [] + keepers = [] + + async def keeper(_): + keepers.append(asyncio.current_task()) + try: + await asyncio.Event().wait() + finally: + owner[0].cancel("cleanup cancellation") + await asyncio.sleep(0) + + async def work(_): + owner.append(asyncio.current_task()) + await asyncio.sleep(0) + return 7 + + monkeypatch.setattr(command, "_keep_overrides", keeper) + with pytest.raises(asyncio.CancelledError, match="cleanup cancellation"): + _ch.run_connected(work, orgs=("o",)) + assert native.events[-3:] == ["certificate", "pg", "ch"] + assert keepers and all(task.done() for task in keepers) + + +def test_unfinished_role_producer_prevents_success_and_certificate(native, monkeypatch): + producers = [] + monkeypatch.setattr(command, "COMMAND_DRAIN_S", 0) + + async def work(_): + async def producer(): + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + await asyncio.Event().wait() + + task = native.gate.track(asyncio.create_task(producer())) + producers.append(task) + await asyncio.sleep(0) + return 7 + + with pytest.raises(RuntimeError, match="producers did not stop"): + _ch.run_connected(work, orgs=("o",)) + assert "certificate" not in native.events + assert not native.gate.producers_stopped + assert producers and all(task.done() for task in producers) + + +def test_first_role_producer_failure_is_retrieved_and_does_not_skip_safe_drain(native, monkeypatch): + first = OSError("producer failed") + producers = [] + + async def work(_): + async def producer(error): + raise error + + # Hold completed producer handles until the composition retrieves both errors. + for error in [first, first]: + task = asyncio.create_task(producer(error)) + native.gate.producers.add(task) + producers.append(task) + await asyncio.sleep(0) + return 7 + + with pytest.raises(OSError) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert native.events[-3:] == ["certificate", "pg", "ch"] + assert all(task.done() and not task._log_traceback for task in producers) + + +def test_actual_retirement_refusal_stays_registered_and_refuses_success(native, monkeypatch): + stopped = [] + monkeypatch.setattr( + barrier, + "_HEARTBEAT", + SimpleNamespace(signal_stop=lambda: stopped.append(True), running=True), + ) + + async def work(_): + return 7 + + with pytest.raises(RuntimeError, match="retirement was not certified"): + _ch.run_connected(work, orgs=("o",)) + assert stopped == [True] + assert "certificate" not in native.events + assert native.gate.enrolled and native.gate.quiescent + assert native.events[-2:] == ["pg", "ch"] diff --git a/tests/unit/test_provider_command.py b/tests/unit/test_provider_command.py index c6ba785ab4..2a1d22da2d 100644 --- a/tests/unit/test_provider_command.py +++ b/tests/unit/test_provider_command.py @@ -16,7 +16,7 @@ def setup(monkeypatch): gate = ModelWorkGate() monkeypatch.setattr(command, "get_model_work_gate", lambda: gate) monkeypatch.setattr(command, "register_process", AsyncMock()) - monkeypatch.setattr(command, "retire_process", AsyncMock()) + monkeypatch.setattr(command, "retire_process", AsyncMock(return_value=True)) monkeypatch.setattr(command, "load_active_configuration", AsyncMock()) monkeypatch.setattr(command, "drain_task_sets", AsyncMock(return_value=True)) monkeypatch.setattr(command, "flush_registered_buffers", AsyncMock(return_value=True)) @@ -46,8 +46,9 @@ async def test_pending_transition_never_runs_command(setup): @pytest.mark.asyncio async def test_undrained_descendants_keep_participant_registered(setup, monkeypatch): monkeypatch.setattr(command, "drain_task_sets", AsyncMock(return_value=False)) - async with command.model_command(): - pass + with pytest.raises(RuntimeError, match="deferred writes did not drain"): + async with command.model_command(): + pass command.retire_process.assert_not_awaited() command.flush_registered_buffers.assert_not_awaited() From c11f8eaf2d5306758f0af35b76e4112785b210d1 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 13:50:40 +0000 Subject: [PATCH 064/114] Native commands retain failures through participant cleanup Cleanup preserves the first failure and retries cancellation only when the owning task receives another cancellation request. Native wrappers and timing receipts retain the same completion boundary. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 9 + server/cli/_ch.py | 7 +- server/cli/analytics_export.py | 7 +- server/cli/code_run.py | 18 +- server/cli/doctor.py | 14 +- server/cli/doctor_cell.py | 15 +- server/cli/import_brain.py | 7 +- server/cli/import_clickgap.py | 7 +- server/cli/import_recall.py | 58 ++-- server/cli/maintenance.py | 7 +- server/cli/sql_access.py | 7 +- server/core/retirement.py | 12 +- server/insights/app/provider_barrier.py | 37 +- server/insights/app/provider_command.py | 57 ++-- .../unit/test_cell_trigger_gate_json_codec.py | 201 +++++++++++ tests/unit/test_cli_native_completion.py | 252 ++++++++++++++ tests/unit/test_cli_native_first_error.py | 15 +- tests/unit/test_cli_native_wrappers.py | 316 ++++++++++++++++++ tests/unit/test_provider_command.py | 12 +- 19 files changed, 975 insertions(+), 83 deletions(-) create mode 100644 tests/unit/test_cell_trigger_gate_json_codec.py create mode 100644 tests/unit/test_cli_native_completion.py create mode 100644 tests/unit/test_cli_native_wrappers.py diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 19db375e4b..3753ab4f90 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -1210,6 +1210,15 @@ def _compare( and left == right, phase + " vector", ) + elif field_name == "value_json": + try: + # PostgreSQL keeps JSON text; the native ClickHouse decoder returns its value. + left = json.loads(left) if isinstance(left, str) else left + right = json.loads(right) if isinstance(right, str) else right + same = sha256_value(left) == sha256_value(right) + except (ValueError, TypeError) as error: + raise Invalid(phase + " " + field_name) from error + _require(same, phase + " " + field_name) else: _require(_normal(left) == _normal(right), phase + " " + field_name) diff --git a/server/cli/_ch.py b/server/cli/_ch.py index 7772fbdb1a..bca6dadbaa 100644 --- a/server/cli/_ch.py +++ b/server/cli/_ch.py @@ -228,7 +228,12 @@ async def check_setup_stores() -> None: pg.expect_deployment((str(recorded["deployment_id"]), int(recorded["epoch"]))) await check_org_free(writes("ClickHouse users and grants, no tenant rows")) finally: - await disconnect(ch) + first = sys.exception() + try: + await disconnect(ch) + except BaseException: # a close failure must not replace the store-check refusal + if first is None: + raise async def check_orgs(orgs: Iterable[str]) -> None: diff --git a/server/cli/analytics_export.py b/server/cli/analytics_export.py index 5f5abf8bb3..63bd9732dd 100644 --- a/server/cli/analytics_export.py +++ b/server/cli/analytics_export.py @@ -105,7 +105,12 @@ async def _go() -> list[Exported]: day, prefix=args.prefix, put=put, name_key=name_key, ch=ch ) finally: - await disconnect(ch) + first = sys.exception() + try: + await disconnect(ch) + except BaseException: # a disconnect failure must not replace the command failure + if first is None: + raise for e in asyncio.run(_go()): # A kept object's size is the earlier run's, which this run never read. diff --git a/server/cli/code_run.py b/server/cli/code_run.py index 1074951fa1..5c11a94f60 100644 --- a/server/cli/code_run.py +++ b/server/cli/code_run.py @@ -134,13 +134,23 @@ async def _then_disconnect(coro: asyncio.Future[int] | Coroutine[Any, Any, int]) # Some native commands close their own client. Reopen it for # buffered-write drain/retirement before leaving enrollment. # The subcommand's own connect gated its writes; a drain checks nothing new. - await connect_declared( - get_client(), org_free=reads("drains the writes its own connect gated") - ) + first = sys.exception() + try: + await connect_declared( + get_client(), org_free=reads("drains the writes its own connect gated") + ) + except BaseException: # keep the subcommand failure if drain reconnect also fails + if first is None: + raise finally: + first = sys.exception() if not started and asyncio.iscoroutine(coro): coro.close() - await disconnect(get_client()) + try: + await disconnect(get_client()) + except BaseException: # a disconnect failure must not replace the subcommand failure + if first is None: + raise def _run(name: str, coro: asyncio.Future[int] | Coroutine[Any, Any, int]) -> int: diff --git a/server/cli/doctor.py b/server/cli/doctor.py index 29a9ddabf6..b4814b7978 100644 --- a/server/cli/doctor.py +++ b/server/cli/doctor.py @@ -482,7 +482,12 @@ async def read() -> tuple[Any, ...] | None: await tier_health.refresh_failing() return tier_health.problem_rows() finally: - await disconnect(client) + first = sys.exception() + try: + await disconnect(client) + except BaseException: # keep the diagnostic failure when close also fails + if first is None: + raise _read_code_registry() # the self-service entries too; a no-op once a read succeeded try: @@ -509,7 +514,12 @@ async def read() -> None: await connect_declared(client, org_free=reads("reads")) await repo_registry.refresh_db_repos(force=True) finally: - await disconnect(client) + first = sys.exception() + try: + await disconnect(client) + except BaseException: # keep the diagnostic failure when close also fails + if first is None: + raise try: asyncio.run(asyncio.wait_for(read(), timeout=5)) diff --git a/server/cli/doctor_cell.py b/server/cli/doctor_cell.py index e8a5fc83c1..79fbebb5af 100644 --- a/server/cli/doctor_cell.py +++ b/server/cli/doctor_cell.py @@ -15,6 +15,7 @@ from __future__ import annotations import asyncio +import sys from typing import Any from server.cli._ch import reads @@ -216,7 +217,12 @@ async def _go() -> list[tuple[str, str, str]]: except Exception as e: # noqa: BLE001 — a ClickHouse that does not answer is named return [("cell_state", WARN, f"not read: {_reason(e)}")] finally: - await disconnect(ch) + first = sys.exception() + try: + await disconnect(ch) + except BaseException: # keep the diagnostic failure when close also fails + if first is None: + raise return asyncio.run(_go()) @@ -341,7 +347,12 @@ async def _go() -> list[tuple[str, str, str]]: except Exception as e: # noqa: BLE001 — an unread store is named, not a failed run return [("cell_stray_orgs", WARN, f"not read: {_reason(e)}")] finally: - await disconnect(ch) + first = sys.exception() + try: + await disconnect(ch) + except BaseException: # keep the diagnostic failure when close also fails + if first is None: + raise with statement_timeout(budget_s): return asyncio.run(_go()) diff --git a/server/cli/import_brain.py b/server/cli/import_brain.py index 8247dcf699..10bead6dba 100644 --- a/server/cli/import_brain.py +++ b/server/cli/import_brain.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import contextlib import logging from typing import Any @@ -157,7 +158,11 @@ async def _run() -> int: except Exception as e: # noqa: BLE001 — one document must not sink the rest fail += 1 log.warning("import-brain: knowledge.add %s failed: %s", uri, e) - finally: + except BaseException: # an importer failure must survive its writer close + with contextlib.suppress(BaseException): + await closer() + raise + else: await closer() print(f"import-brain: wrote {ok} item(s), {fail} failed") return 1 if fail and ok == 0 else 0 diff --git a/server/cli/import_clickgap.py b/server/cli/import_clickgap.py index 983ce671cd..cc44181142 100644 --- a/server/cli/import_clickgap.py +++ b/server/cli/import_clickgap.py @@ -901,7 +901,12 @@ async def _run_then_disconnect() -> int: async with command_stack: return await _run() finally: - await disconnect(get_client()) + first = sys.exception() + try: + await disconnect(get_client()) + except BaseException: # a disconnect failure must not replace the command failure + if first is None: + raise from contextlib import AsyncExitStack diff --git a/server/cli/import_recall.py b/server/cli/import_recall.py index b24c9371b5..6e49c95b84 100644 --- a/server/cli/import_recall.py +++ b/server/cli/import_recall.py @@ -731,7 +731,11 @@ async def _run() -> int: f" ! write failed ({w['op']} {ident}): {e}", file=sys.stderr, ) - finally: + except BaseException: # an importer failure must survive its writer close + with contextlib.suppress(BaseException): + await closer() + raise + else: await closer() print(f"✓ import complete: {ok} writes ok, {failed} failed") @@ -755,6 +759,27 @@ def _publication_pending(r: Any) -> bool: return isinstance(detail, dict) and detail.get("code") == PublicationPending.code +def _make_http_writer( + args: argparse.Namespace, base_url: str, *, timeout_s: float +) -> tuple[Any, Any]: + import httpx + + token, source = resolve_token(args.token) + if refused_for_agent(source): + raise SystemExit(2) + headers = {"Authorization": f"Bearer {token}"} if token else {} + client = httpx.AsyncClient(timeout=timeout_s, headers=headers) + base = base_url.rstrip("/") + + async def _http_writer(op: str, payload: dict[str, Any]) -> None: + r = await client.post(f"{base}/v1/{op}", json=_recall_jsonable(payload)) + if _publication_pending(r): + return # committed; the server's delivery worker publishes it + r.raise_for_status() + + return _http_writer, client.aclose + + async def _make_writer( args: argparse.Namespace, inproc_ops: Callable[[], _OpHandlers], *, timeout_s: float ) -> tuple[Any, Any]: @@ -764,22 +789,7 @@ async def _make_writer( its handler) on a connected ClickHouse client.""" base_url = getattr(args, "base_url", None) if base_url: - import httpx - - token, source = resolve_token(args.token) - if refused_for_agent(source): - raise SystemExit(2) - headers = {"Authorization": f"Bearer {token}"} if token else {} - client = httpx.AsyncClient(timeout=timeout_s, headers=headers) - base = base_url.rstrip("/") - - async def _http_writer(op: str, payload: dict[str, Any]) -> None: - r = await client.post(f"{base}/v1/{op}", json=_recall_jsonable(payload)) - if _publication_pending(r): - return # committed; the server's delivery worker publishes it - r.raise_for_status() - - return _http_writer, client.aclose + return _make_http_writer(args, base_url, timeout_s=timeout_s) from server.cli._ch import check_acceptance, connect_verified, disconnect from server.core.ch import get_client @@ -798,11 +808,11 @@ async def _http_writer(op: str, payload: dict[str, Any]) -> None: await command.__aenter__() entered = True await check_acceptance(orgs=(args.org,)) - except BaseException: - try: - if entered: + except BaseException: # entry failure must survive participant and connection cleanup + if entered: + with contextlib.suppress(BaseException): await command.__aexit__(None, None, None) - finally: + with contextlib.suppress(BaseException): await disconnect(ch) raise @@ -816,7 +826,11 @@ async def _inproc_writer(op: str, payload: dict[str, Any]) -> None: async def close() -> None: try: await command.__aexit__(None, None, None) - finally: + except BaseException: # participant exit failure must survive connection cleanup + with contextlib.suppress(BaseException): + await disconnect(ch) + raise + else: await disconnect(ch) return _inproc_writer, close diff --git a/server/cli/maintenance.py b/server/cli/maintenance.py index 9a6cdb2437..9e9a3bbd0a 100644 --- a/server/cli/maintenance.py +++ b/server/cli/maintenance.py @@ -256,7 +256,12 @@ async def _go() -> int: ) return 1 if rep.failed and rep.failed == rep.sessions else 0 finally: - await disconnect(ch) + first = sys.exception() + try: + await disconnect(ch) + except BaseException: # a disconnect failure must not replace the command failure + if first is None: + raise return asyncio.run(_go()) diff --git a/server/cli/sql_access.py b/server/cli/sql_access.py index 6a727c9e7b..c34dc37f25 100644 --- a/server/cli/sql_access.py +++ b/server/cli/sql_access.py @@ -82,7 +82,12 @@ async def _go() -> Any: await check_acceptance(orgs=orgs, org_free=org_free) return await work(ch) finally: - await disconnect(ch) + first = sys.exception() + try: + await disconnect(ch) + except BaseException: # a disconnect failure must not replace the command failure + if first is None: + raise return asyncio.run(_go()) diff --git a/server/core/retirement.py b/server/core/retirement.py index 025a437362..f3dd7fc865 100644 --- a/server/core/retirement.py +++ b/server/core/retirement.py @@ -31,6 +31,9 @@ def __init__(self) -> None: self.enrolled = False self.producers_stopped = False self.producers: set[asyncio.Task[Any]] = set() + # Native completion needs the first failed producer after its Task is discarded. + # Fresh gates, including fork reset, discard this one command's outcome. + self.producer_failure: BaseException | None = None def install(self, role: str) -> None: if self.closed or self.role not in ("", role): @@ -43,12 +46,19 @@ def close(self) -> None: def track[T](self, task: asyncio.Task[T]) -> asyncio.Task[T]: """A process-lifetime producer must actually end, not merely be canceled.""" self.producers.add(task) - task.add_done_callback(self.producers.discard) + task.add_done_callback(self._producer_done) if self.closed: task.cancel() raise ProcessStopping("Cannot start a producer during shutdown") return task + def _producer_done(self, task: asyncio.Task[Any]) -> None: + self.producers.discard(task) + if self.role == "command" and not task.cancelled(): + error = task.exception() + if self.producer_failure is None: + self.producer_failure = error + def stopped(self) -> None: """Composition calls this after stopping every role-specific producer.""" self.close() diff --git a/server/insights/app/provider_barrier.py b/server/insights/app/provider_barrier.py index 193fcc9335..794b457649 100644 --- a/server/insights/app/provider_barrier.py +++ b/server/insights/app/provider_barrier.py @@ -264,15 +264,8 @@ async def retire_at_shutdown(ch: Any, *, budget_s: float = SHUTDOWN_RETIRE_S) -> Every attempt, settle and pause ends by `budget_s` from now.""" from server.core import rmt_carry - deadline = monotonic() + budget_s try: - async with asyncio.timeout(budget_s): - while True: - if await retire_process(): - return True - with contextlib.suppress(Exception): - await rmt_carry.settle(ch) - await asyncio.sleep(min(1.0, max(0.0, deadline - monotonic()))) + return await _retire_until(ch, monotonic() + budget_s) except TimeoutError: log.warning( "provider participant not retired at shutdown: %d carried write(s) unsettled " @@ -286,6 +279,19 @@ async def retire_at_shutdown(ch: Any, *, budget_s: float = SHUTDOWN_RETIRE_S) -> return False +async def _retire_until(ch: Any, deadline: float) -> bool: + """Retry the existing settlement checks under the caller's absolute deadline.""" + from server.core import rmt_carry + + async with asyncio.timeout_at(deadline): + while True: + if await retire_process(): + return True + with contextlib.suppress(Exception): + await rmt_carry.settle(ch) + await asyncio.sleep(min(1.0, max(0.0, deadline - monotonic()))) + + def participant_lifespan[T]( lifespan: Callable[[T], AbstractAsyncContextManager[None]], ) -> Callable[[T], AbstractAsyncContextManager[None]]: @@ -314,8 +320,12 @@ async def wrapped(app: T) -> AsyncIterator[None]: async def retire_participant(participant: UUID, *, ch: Any = None) -> str: - """`retired`, `absent` (no live row), or `live`: its heartbeat is fresh or a - query tagged with its id still runs.""" + """`retired`, `absent` (no row), or `live` (not proved stopped). + + Retirement needs the recorded identity proved absent in the same host, boot + and PID namespace, with tagged queries known zero. Unknown evidence blocks; + heartbeat expiry alone proves nothing. + """ row = next( (r for r in await participants(ch=ch) if str(r["participant_id"]) == str(participant)), None, @@ -342,9 +352,10 @@ async def retire_participant(participant: UUID, *, ch: Any = None) -> str: async def retire_dead_here(*, now: float | None = None) -> int: - """Retire the participants this host proves stopped (`proves_stopped`: a - restarted container or a respawned worker) with no query of theirs left - running. Every process's job, every `HERE_S`; how many it retired.""" + """Retire recorded process-start identities proved absent in this host, + boot and PID namespace, with tagged queries known zero. Unknown evidence + blocks. Run at most every `HERE_S`; return how many were retired. + """ global _last_here t = monotonic() if now is None else now if t - _last_here < HERE_S: diff --git a/server/insights/app/provider_command.py b/server/insights/app/provider_command.py index 3b9f7d8a4f..3e16b03c48 100644 --- a/server/insights/app/provider_command.py +++ b/server/insights/app/provider_command.py @@ -9,9 +9,9 @@ from server.core import retirement, settings_overrides from server.core.background import drain_task_sets, flush_registered_buffers -from server.core.ch import set_query_tag +from server.core.ch import get_client, set_query_tag from server.core.model_work_gate import get_model_work_gate -from server.insights.app.provider_barrier import register_process, retire_process +from server.insights.app.provider_barrier import _retire_until, register_process from server.insights.app.provider_runtime import load_active_configuration from server.settings import settings @@ -47,49 +47,64 @@ async def _keep_overrides( return error +async def _drain_participant_until(end: float) -> None: + loop = asyncio.get_running_loop() + async with asyncio.timeout_at(end): + if not await drain_task_sets(timeout_s=max(0.0, end - loop.time())): + raise RuntimeError("Native command deferred writes did not drain") + if not await flush_registered_buffers(): + raise RuntimeError("Native command buffers did not flush") + if await _retire_until(get_client(), end) is not True: + raise RuntimeError("Native command retirement was not certified") + + async def _finish_participant() -> None: gate = retirement.get_gate() gate.close() from server.memory.app import embedder_calibration - first: BaseException | None = None + loop = asyncio.get_running_loop() + end = loop.time() + COMMAND_DRAIN_S + first = gate.producer_failure stopped = False try: embedder_calibration.stop_refresh_loop() stopped = True except BaseException as error: # noqa: BLE001 — finish safe cleanup, then raise the first failure - first = error + if first is None: + first = error producers = tuple(gate.producers) for producer in producers: producer.cancel() try: if producers: - _, pending = await asyncio.wait(producers, timeout=COMMAND_DRAIN_S) + _, pending = await asyncio.wait(producers, timeout=max(0.0, end - loop.time())) if pending: raise RuntimeError("Native command producers did not stop") except BaseException as error: # noqa: BLE001 — finish safe cleanup, then raise the first failure if first is None: first = error stopped = False - for producer in producers: - if producer.done() and not producer.cancelled(): + if first is None: + first = gate.producer_failure + if stopped: + gate.stopped() + owner = asyncio.current_task() + while True: + cancelling = owner.cancelling() if owner is not None else 0 try: - producer.result() - except BaseException as error: # noqa: BLE001 — finish safe cleanup, then raise the first failure + if loop.time() >= end: + raise TimeoutError("Native command cleanup deadline expired") + await _drain_participant_until(end) + except asyncio.CancelledError as error: if first is None: first = error - if stopped: - gate.stopped() - try: - if not await drain_task_sets(timeout_s=COMMAND_DRAIN_S): - raise RuntimeError("Native command deferred writes did not drain") - if not await flush_registered_buffers(): - raise RuntimeError("Native command buffers did not flush") - if await retire_process() is not True: - raise RuntimeError("Native command retirement was not certified") - except BaseException as error: # noqa: BLE001 — finish safe cleanup, then raise the first failure - if first is None: - first = error + if owner is not None and owner.cancelling() > cancelling: + continue # a new owner cancellation still allows cleanup within the same end + except BaseException as error: # noqa: BLE001 — report the first required cleanup failure + if first is None: + first = error + break if first is not None: raise first diff --git a/tests/unit/test_cell_trigger_gate_json_codec.py b/tests/unit/test_cell_trigger_gate_json_codec.py new file mode 100644 index 0000000000..62c9aefa46 --- /dev/null +++ b/tests/unit/test_cell_trigger_gate_json_codec.py @@ -0,0 +1,201 @@ +"""The full stored-row proof accepts native JSON without weakening other fields.""" + +from __future__ import annotations + +import json +import logging +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from bench import cell_trigger_gate as gate +from tests.unit.test_bench_cell_trigger_gate import ( + args, + stored_bfloat16, +) +from tests.unit.test_bench_cell_trigger_gate import ( + no_external_io as no_external_io, +) + + +def native_json(monkeypatch, value): + from clickhouse_connect.datatypes import dynamic + from clickhouse_connect.datatypes.base import TypeDef + from clickhouse_connect.driver.context import BaseQueryContext + + # A fresh type leaves the installed registry and global format map untouched. + dtype = dynamic.JSON(TypeDef()) + context = BaseQueryContext() + assert dtype.read_format(context) == "native" + dtype.typed_paths = list(value) + dtype.typed_types = [ + SimpleNamespace(read_column_data=lambda *_args, v=v: [v]) for v in value.values() + ] + monkeypatch.setattr( + dynamic, + "SHARED_DATA_TYPE", + SimpleNamespace(read_column_data=lambda *_args: [{}]), + ) + state = dynamic.JSONState(2, [], [None] * len(value), [], None) + decoded = dtype._read_column_binary(object(), 1, context, state)[0] + assert type(decoded) is dict + assert decoded == value + return decoded + + +@pytest.mark.parametrize( + "value", + [ + {}, + {"n": 1}, + {"flag": True}, + {"nullable": None}, + {"nested": {"b": [1, False, None], "a": "text"}}, + {"when": "2026-01-02T03:04:05.123456+01:00"}, + {"text": '\u03bb\n"quoted"'}, + {"z": 0, "a": "1"}, + ], +) +def test_installed_native_json_matches_serialized_authority(monkeypatch, value): + decoded = native_json(monkeypatch, value) + gate._compare( + {"value_json": json.dumps(value, indent=2)}, + {"value_json": decoded}, + ("value_json",), + "full memory projection", + ) + + +@pytest.mark.parametrize( + ("expected", "actual"), + [ + ({}, {"extra": 1}), + ({"n": 1}, {"n": True}), + ({"n": 1}, {"n": "1"}), + ({"v": None}, {}), + ({"v": [1, 2]}, {"v": [2, 1]}), + ({"v": {}}, {"v": []}), + ({"v": "2026-01-02T03:04:05.123456"}, {"v": "2026-01-02T03:04:05.123999"}), + ({"v": "2026-01-02T03:04:05+01:00"}, {"v": "2026-01-02T03:04:05+00:00"}), + ], +) +def test_native_json_rejects_changed_json_value(monkeypatch, expected, actual): + with pytest.raises(gate.Invalid, match="full memory projection value_json"): + gate._compare( + {"value_json": json.dumps(expected)}, + {"value_json": native_json(monkeypatch, actual)}, + ("value_json",), + "full memory projection", + ) + + +@pytest.mark.parametrize("side", ["expected", "actual"]) +def test_malformed_json_is_an_invalid_projection(side): + expected, actual = {"value_json": "{}"}, {"value_json": {}} + (expected if side == "expected" else actual)["value_json"] = "{broken" + with pytest.raises(gate.Invalid, match="full memory projection value_json"): + gate._compare(expected, actual, ("value_json",), "full memory projection") + + +def test_json_normalization_applies_only_to_value_json(): + with pytest.raises(gate.Invalid, match="full memory projection value"): + gate._compare({"value": "{}"}, {"value": {}}, ("value",), "full memory projection") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", + ["", "value_json", "value", "version", "vector", "audit_hash"], + ids=["healthy", "value_json", "value", "version", "vector", "audit_hash"], +) +async def test_actual_verify_keeps_full_proof_with_native_json(monkeypatch, fault): + import server.embedding + from server.core.polled_copy import PolledCopy + from server.embedding.fake import FakeNGramEmbedder + from server.llm import customer + from server.memory.store import pg_memory + from server.memory.store.rows import MEMORY_PERSISTED_COLS + + monkeypatch.setattr( + customer, + "_switch", + PolledCopy("switch", AsyncMock(return_value=frozenset()), logging.getLogger(__name__)), + ) + monkeypatch.setattr(server.embedding, "get_embedder", lambda: FakeNGramEmbedder(dim=16)) + block = await gate._build_block(args(), 1) + block.seqs = [("org", "ns", i + 1) for i in range(5)] + assert len(MEMORY_PERSISTED_COLS) == 61 + assert all(set(row) == set(MEMORY_PERSISTED_COLS) for row in block.rows) + records = [ + { + "head": json.dumps(pg_memory.encode_row(row)), + "revision": json.dumps(pg_memory.encode_row(row)), + "audit": json.dumps(pg_memory.encode_audit(audit)), + "seq": i + 1, + "targets": 0, + "attempts": 0, + "last_error": "", + } + for i, (row, audit) in enumerate(zip(block.rows, block.audits, strict=True)) + ] + projections = [ + {**row, "value_json": native_json(monkeypatch, json.loads(row["value_json"]))} + for row in block.rows + ] + documents = [dict(row) for row in block.documents] + for document in documents: + for field in ("embedding", "memory_key_embedding"): + document[field] = stored_bfloat16(document[field]) + audits = [dict(row) for row in block.audits] + if fault == "value_json": + projections[0]["value_json"] = native_json(monkeypatch, {"wrong": True}) + elif fault == "value": + projections[0]["value"] = '"wrong"' + elif fault == "version": + projections[0]["version"] -= 1 + elif fault == "vector": + documents[0]["embedding"] = [0.0] * 16 + elif fault == "audit_hash": + audits[0]["row_hash"] = "bad" + conn = SimpleNamespace( + fetch=AsyncMock(return_value=records), + fetchrow=AsyncMock(return_value={"pending": 0, "audits": 0, "errors": 0}), + ) + phases = [] + compare = gate._compare + + def full_compare(expected, actual, fields, phase): + if phase == "full memory projection": + assert set(fields) == set(block.rows[0]) and len(fields) == 61 + phases.append(phase) + return compare(expected, actual, fields, phase) + + monkeypatch.setattr(gate, "_compare", full_compare) + + async def query(sql, params): + assert params["org"] == "org" and params["ns"] == "ns" + if "FROM loom.memory FINAL" in sql: + return projections + if "FROM loom.memory_embeddings FINAL" in sql: + return documents + if "FROM loom.audit_log" in sql: + return audits + raise AssertionError(sql) + + if fault: + reason = { + "value_json": "full memory projection value_json", + "value": "full memory projection value$", + "version": "full memory projection version", + "vector": "full search document vector", + "audit_hash": "exact stored audit hashes", + }[fault] + with pytest.raises(gate.Invalid, match=reason): + await gate._verify(conn, SimpleNamespace(query_all=query), args(), block) + else: + await gate._verify(conn, SimpleNamespace(query_all=query), args(), block) + assert phases.count("full memory projection") == 5 + assert phases.count("full search document") == 5 + assert phases.count("full stored audit") == 5 + conn.fetchrow.assert_awaited_once() diff --git a/tests/unit/test_cli_native_completion.py b/tests/unit/test_cli_native_completion.py new file mode 100644 index 0000000000..6b27052577 --- /dev/null +++ b/tests/unit/test_cli_native_completion.py @@ -0,0 +1,252 @@ +"""Native completion waits for real retirement checks and observes tracked outcomes.""" + +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from server.cli import _ch +from server.core import retirement, rmt_carry +from server.insights.app import provider_barrier as barrier +from server.insights.app import provider_command as command +from tests.unit.test_cli_native_first_error import FIRST +from tests.unit.test_cli_native_first_error import native as native + + +@pytest.mark.parametrize("first_class", [None, *FIRST]) +def test_healthy_heartbeat_finishes_after_native_owner_requests_stop( + native, monkeypatch, first_class +): + requested = [] + attempts = [] + heartbeat = SimpleNamespace(running=True) + + def stop(): + requested.append(True) + + async def settle(client): + assert client is native.client + assert requested and heartbeat.running + attempts.append(True) + heartbeat.running = False + + heartbeat.signal_stop = stop + monkeypatch.setattr(barrier, "_HEARTBEAT", heartbeat) + monkeypatch.setattr(rmt_carry, "settle", settle) + first = first_class("work") if first_class else None + + async def work(_): + if first is not None: + raise first + return 7 + + if first is None: + assert _ch.run_connected(work, orgs=("o",)) == 7 + else: + with pytest.raises(first_class) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert attempts == [True] + assert not heartbeat.running + assert native.events[-3:] == ["certificate", "pg", "ch"] + + +@pytest.mark.parametrize("first_class", [None, *FIRST]) +def test_permanent_heartbeat_respects_same_absolute_native_allowance( + native, monkeypatch, first_class +): + monkeypatch.setattr(command, "COMMAND_DRAIN_S", 0.02) + monkeypatch.setattr( + barrier, "_HEARTBEAT", SimpleNamespace(signal_stop=lambda: None, running=True) + ) + monkeypatch.setattr(rmt_carry, "settle", AsyncMock()) + first = first_class("work") if first_class else None + elapsed = [] + + async def work(_): + loop = asyncio.get_running_loop() + elapsed.append((loop, loop.time())) + if first is not None: + raise first + return 7 + + with pytest.raises(first_class or TimeoutError) as caught: + _ch.run_connected(work, orgs=("o",)) + if first is not None: + assert caught.value is first + loop, start = elapsed[0] + assert loop.time() - start < 0.5 + assert "certificate" not in native.events + assert native.events[-2:] == ["pg", "ch"] + + +@pytest.mark.parametrize("first_class", [None, ValueError, asyncio.CancelledError]) +def test_repeated_owner_cancellation_keeps_first_and_allows_finite_heartbeat_completion( + native, monkeypatch, first_class +): + first = first_class("work") if first_class else None + heartbeat = SimpleNamespace(running=True, signal_stop=lambda: None) + attempts = [] + cancellations = [] + + async def settle(client): + assert client is native.client + owner = asyncio.current_task() + attempts.append(True) + if len(attempts) <= 2: + owner.cancel("first signal" if len(attempts) == 1 else "second signal") + else: + heartbeat.running = False + try: + await asyncio.sleep(0) + except asyncio.CancelledError as error: + cancellations.append(error) + raise + + monkeypatch.setattr(barrier, "_HEARTBEAT", heartbeat) + monkeypatch.setattr(rmt_carry, "settle", settle) + + async def work(_): + if first is not None: + raise first + return 7 + + with pytest.raises(first_class or asyncio.CancelledError) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is (first if first is not None else cancellations[0]) + assert len(attempts) == 3 and len(cancellations) == 2 + assert native.events[-3:] == ["certificate", "pg", "ch"] + + +@pytest.mark.parametrize("first_class", [None, *FIRST]) +def test_real_track_completed_failure_is_observed_before_native_cleanup(native, first_class): + producer_error = OSError("producer") + first = first_class("work") if first_class else None + tasks = [] + + async def work(_): + async def producer(): + raise producer_error + + task = native.gate.track(asyncio.create_task(producer())) + tasks.append(task) + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not native.gate.producers + if first is not None: + raise first + return 7 + + with pytest.raises(first_class or OSError) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is (first if first is not None else producer_error) + assert native.events[-3:] == ["certificate", "pg", "ch"] + assert all(task.done() and not task._log_traceback for task in tasks) + + +@pytest.mark.asyncio +async def test_native_track_retains_only_first_failure_and_no_completed_task_handles(): + gate = retirement.RetirementGate() + gate.install("command") + first = OSError("first producer") + tasks = [] + + async def producer(error=None): + if error is not None: + raise error + + for error in [None] * 100 + [first, ValueError("later")] + [None] * 100: + task = gate.track(asyncio.create_task(producer(error))) + tasks.append(task) + await asyncio.sleep(0) + await asyncio.sleep(0) + assert gate.producer_failure is first + assert gate.producers == set() + assert all(not task._log_traceback for task in tasks) + assert retirement.RetirementGate().producer_failure is None + + +@pytest.mark.asyncio +async def test_intentionally_cancelled_tracked_native_producer_is_not_failed(): + gate = retirement.RetirementGate() + gate.install("command") + task = gate.track(asyncio.create_task(asyncio.Event().wait())) + await asyncio.sleep(0) + task.cancel() + await asyncio.gather(task, return_exceptions=True) + assert task.cancelled() and gate.producers == set() + assert gate.producer_failure is None + + +@pytest.mark.parametrize("role", ["server", "worker"]) +@pytest.mark.asyncio +async def test_other_role_tracking_keeps_existing_outcome_ownership(role): + gate = retirement.RetirementGate() + gate.install(role) + first = OSError("role-owned producer") + + async def producer(): + raise first + + task = gate.track(asyncio.create_task(producer())) + await asyncio.sleep(0) + await asyncio.sleep(0) + assert gate.producers == set() and gate.producer_failure is None + assert task._log_traceback + assert task.exception() is first + + +def test_actual_calibration_consumer_failure_survives_finished_track_callback(native, monkeypatch): + from server.memory.app import embedder_calibration + + first = OSError("calibration producer") + tasks = [] + + async def sleep(_): + raise first + + monkeypatch.setattr(embedder_calibration, "_refresh_task", None) + monkeypatch.setattr( + embedder_calibration, + "asyncio", + SimpleNamespace(sleep=sleep, get_running_loop=asyncio.get_running_loop), + ) + + async def configuration(): + tasks.append(embedder_calibration.start_refresh_loop()) + + monkeypatch.setattr(command, "load_active_configuration", configuration) + + async def work(_): + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not native.gate.producers + return 7 + + with pytest.raises(OSError) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert tasks[0].done() and not tasks[0]._log_traceback + assert native.events[-3:] == ["certificate", "pg", "ch"] + + +@pytest.mark.parametrize("phase", ["flush", "certificate"]) +def test_dependency_cancellation_is_not_retried_as_owner_cancellation(native, monkeypatch, phase): + first = asyncio.CancelledError("cleanup dependency cancelled") + required = AsyncMock( + side_effect=[first, RuntimeError("cleanup retried without owner cancellation")] + ) + target = "flush_registered_buffers" if phase == "flush" else "_retire_until" + monkeypatch.setattr(command, target, required) + + async def work(_): + return 7 + + with pytest.raises(asyncio.CancelledError) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert required.await_count == 1 + assert native.events[-2:] == ["pg", "ch"] + assert not native.client.connected + assert native.gate.closed and native.gate.active == 0 diff --git a/tests/unit/test_cli_native_first_error.py b/tests/unit/test_cli_native_first_error.py index 35f71b60fb..c77d29ba83 100644 --- a/tests/unit/test_cli_native_first_error.py +++ b/tests/unit/test_cli_native_first_error.py @@ -67,6 +67,7 @@ async def certificate(proof): client.connect, client.disconnect = connect, close_ch monkeypatch.setattr(ch, "get_client", lambda: client) + monkeypatch.setattr(command, "get_client", lambda: client, raising=False) monkeypatch.setattr(pg, "close_pool", close_pg) monkeypatch.setattr(pg_profile, "check_configured", verify) monkeypatch.setattr(_ch, "check_orgs", acceptance) @@ -218,14 +219,14 @@ def test_unacknowledged_cleanup_refuses_without_early_certification( calls = [] first = first_class("work") if first_class else None - async def unacknowledged(**_): + async def unacknowledged(*_, **__): calls.append(phase) return False target = { "drain": "drain_task_sets", "flush": "flush_registered_buffers", - "retire": "retire_process", + "retire": "_retire_until", } monkeypatch.setattr(command, target[phase], unacknowledged) @@ -381,12 +382,13 @@ async def work(_): async def producer(error): raise error - # Hold completed producer handles until the composition retrieves both errors. + # Completion callbacks run before native shutdown snapshots tracked producers. for error in [first, first]: task = asyncio.create_task(producer(error)) - native.gate.producers.add(task) + native.gate.track(task) producers.append(task) await asyncio.sleep(0) + await asyncio.sleep(0) return 7 with pytest.raises(OSError) as caught: @@ -397,6 +399,7 @@ async def producer(error): def test_actual_retirement_refusal_stays_registered_and_refuses_success(native, monkeypatch): + monkeypatch.setattr(command, "COMMAND_DRAIN_S", 0.02) stopped = [] monkeypatch.setattr( barrier, @@ -407,9 +410,9 @@ def test_actual_retirement_refusal_stays_registered_and_refuses_success(native, async def work(_): return 7 - with pytest.raises(RuntimeError, match="retirement was not certified"): + with pytest.raises(TimeoutError): _ch.run_connected(work, orgs=("o",)) - assert stopped == [True] + assert stopped and all(stopped) assert "certificate" not in native.events assert native.gate.enrolled and native.gate.quiescent assert native.events[-2:] == ["pg", "ch"] diff --git a/tests/unit/test_cli_native_wrappers.py b/tests/unit/test_cli_native_wrappers.py new file mode 100644 index 0000000000..72283e6512 --- /dev/null +++ b/tests/unit/test_cli_native_wrappers.py @@ -0,0 +1,316 @@ +"""Actual native entry points preserve first failures across their outer close.""" + +import argparse +import asyncio +import sqlite3 +from functools import partial +from types import SimpleNamespace + +import pytest + +from server.cli import ( + _ch, + analytics_export, + code_run, + import_brain, + import_clickgap, + import_recall, + maintenance, + sql_access, +) +from server.insights.app import analytics_export as export_app +from tests.unit.test_cli_native_first_error import FIRST +from tests.unit.test_cli_native_first_error import native as native + + +@pytest.fixture +def wrappers(native, monkeypatch): + from server.code.app.index import repo_registry + from server.memory.app import reextract + + monkeypatch.setattr(repo_registry, "refresh_db_repos", _nothing) + monkeypatch.setattr(analytics_export, "_refusal", lambda *_: "") + monkeypatch.setattr(export_app, "check_credentials", lambda **_: None) + monkeypatch.setattr(export_app, "name_hash_key", lambda _: b"inert") + monkeypatch.setattr(export_app, "s3_put", lambda *_, **__: None) + monkeypatch.setattr(maintenance, "resolve_token", lambda _: ("", "none")) + monkeypatch.setattr(reextract, "replay", _report) + monkeypatch.setattr(reextract, "render_report", lambda *_, **__: "report") + return native + + +async def _nothing(*_, **__): + return None + + +async def _report(*_, **__): + return SimpleNamespace(failed=0, sessions=1) + + +def _invoke(family, work, monkeypatch): + if family == "sql": + return sql_access._connected(work, orgs=("o",)) + if family == "export": + monkeypatch.setattr(export_app, "export_day", work) + return analytics_export._cmd_export( + argparse.Namespace( + day=None, + role_arn="", + region="", + bucket="scratch", + expected_bucket_owner="", + prefix="", + ) + ) + if family == "reextract": + from server.memory.app import reextract + + monkeypatch.setattr(reextract, "load_sessions", work) + return maintenance._cmd_reextract( + argparse.Namespace( + namespace="n", + into="other", + token="", + dry_run=True, + org="o", + limit_sessions=None, + base_url="", + dates=False, + concurrency=1, + ) + ) + if family == "code": + return asyncio.run(code_run._then_disconnect(work())) + raise AssertionError(family) + + +@pytest.mark.parametrize("family", ["sql", "export", "reextract", "code"]) +@pytest.mark.parametrize("first_class", FIRST) +@pytest.mark.parametrize("second_close", [False, True]) +def test_actual_wrappers_preserve_work_failure( + wrappers, monkeypatch, family, first_class, second_close +): + first = first_class("work") + if second_close: + wrappers.errors.update(pg=OSError("pg close"), ch=RuntimeError("ch close")) + + async def work(*_, **__): + assert wrappers.gate.enrolled and wrappers.gate.active > 0 + raise first + + with pytest.raises(first_class) as caught: + _invoke(family, work, monkeypatch) + assert caught.value is first + assert wrappers.events[-2:] == ["pg", "ch"] + assert not wrappers.client.connected + assert wrappers.gate.active == wrappers.model_gate.active == 0 + + +@pytest.mark.parametrize("family", ["sql", "export", "reextract", "code"]) +@pytest.mark.parametrize("failed_close", [False, True]) +def test_successful_actual_wrappers_still_require_cleanup( + wrappers, monkeypatch, family, failed_close +): + first = OSError("pg close") + if failed_close: + wrappers.errors.update(pg=first, ch=RuntimeError("ch close")) + + async def work(*_, **__): + return [] if family == "export" else ([{}] if family == "reextract" else 7) + + if failed_close: + with pytest.raises(OSError) as caught: + _invoke(family, work, monkeypatch) + assert caught.value is first + else: + assert _invoke(family, work, monkeypatch) == (0 if family in ("export", "reextract") else 7) + assert wrappers.events[-3:] == ["certificate", "pg", "ch"] + + +@pytest.mark.parametrize("first_class", FIRST) +@pytest.mark.parametrize("reopen_failure", [False, True]) +def test_code_reconnect_cannot_replace_subcommand_failure( + wrappers, monkeypatch, first_class, reopen_failure +): + first = first_class("subcommand") + events = [] + + async def reconnect(*_, **__): + events.append("reconnect") + if reopen_failure: + raise OSError("reconnect") + + async def work(): + raise first + + monkeypatch.setattr(code_run, "connect_declared", reconnect) + with pytest.raises(first_class) as caught: + asyncio.run(code_run._then_disconnect(work())) + assert caught.value is first + assert events == ["reconnect"] + assert wrappers.events[-3:] == ["certificate", "pg", "ch"] + + +@pytest.mark.parametrize("first_class", FIRST) +@pytest.mark.parametrize("phase", ["entry", "participant-close"]) +@pytest.mark.asyncio +async def test_actual_recall_writer_entry_and_closer_preserve_first( + wrappers, monkeypatch, first_class, phase +): + first = first_class("first") + if phase == "entry": + wrappers.errors["acceptance"] = first + else: + wrappers.errors["certificate"] = first + wrappers.errors.update(pg=OSError("pg close"), ch=RuntimeError("ch close")) + args = argparse.Namespace(base_url=None, org="o") + with pytest.raises(first_class) as caught: + _, closer = await import_recall._make_writer(args, lambda: {}, timeout_s=1) + await closer() + assert caught.value is first + assert wrappers.events[-2:] == ["pg", "ch"] + assert wrappers.gate.active == wrappers.model_gate.active == 0 + + +@pytest.mark.parametrize("family", ["recall", "brain"]) +@pytest.mark.parametrize("first_class", [asyncio.CancelledError, SystemExit, KeyboardInterrupt]) +def test_actual_importer_outer_closer_preserves_unhandled_work_failure( + wrappers, monkeypatch, tmp_path, family, first_class +): + first = first_class("write cancelled or fatal") + wrappers.errors.update(pg=OSError("pg close"), ch=RuntimeError("ch close")) + + async def handler(_): + raise first + + if family == "brain": + (tmp_path / "MEMORY.md").write_text("## A\ncontent\n") + monkeypatch.setattr(import_brain, "_brain_inproc_ops", lambda: {"profile.set": handler}) + args = argparse.Namespace( + dir=str(tmp_path), dry_run=False, org="o", namespace="n", agent="a", base_url=None + ) + call = import_brain._cmd_import_brain + else: + path = tmp_path / "source.db" + with sqlite3.connect(path): + pass + report = { + "tables": {}, + "memories_imported": 1, + "messages_imported": 0, + "threads": 0, + "skipped_rows": 0, + } + monkeypatch.setattr( + import_recall, + "_recall_plan_writes", + lambda *_, **__: ([{"op": "memory.set", "payload": {}}], report), + ) + monkeypatch.setattr(import_recall, "_recall_inproc_ops", lambda: {"memory.set": handler}) + args = argparse.Namespace( + sqlite=str(path), dry_run=False, org="o", namespace="n", agent="a", base_url=None + ) + call = import_recall._cmd_import_recall + with pytest.raises(first_class) as caught: + call(args) + assert caught.value is first + assert wrappers.events[-2:] == ["pg", "ch"] + + +@pytest.mark.parametrize("first_class", FIRST) +def test_actual_clickgap_outer_disconnect_preserves_admission_failure( + wrappers, monkeypatch, tmp_path, first_class +): + from server import embedding + + first = first_class("admission") + wrappers.errors.update(acceptance=first, pg=OSError("pg close"), ch=RuntimeError("ch close")) + path = tmp_path / "source.db" + with sqlite3.connect(path) as source: + source.execute( + "CREATE TABLE lessons (id TEXT, text TEXT, source TEXT, created_at TEXT, kind TEXT, expires_at TEXT, evidence_count INT, recall_count INT, last_recalled_at TEXT, reasoning TEXT)" + ) + monkeypatch.setattr(import_clickgap, "_clickgap_count_source", lambda *_, **__: {}) + monkeypatch.setattr(import_clickgap, "_clickgap_preload", lambda *_, **__: ({}, [], {}, {})) + monkeypatch.setattr(embedding, "get_embedder", lambda: SimpleNamespace(model_name="inert")) + args = argparse.Namespace( + sqlite=str(path), + org="o", + namespace="n", + agent="a", + dry_run=False, + batch_size=1, + seed_recall_events=False, + ) + with pytest.raises(first_class) as caught: + import_clickgap._cmd_import_clickgap(args) + assert caught.value is first + assert wrappers.events[-2:] == ["pg", "ch"] + + +@pytest.mark.doctor_tier_read +@pytest.mark.parametrize("first_class", [asyncio.CancelledError, SystemExit, KeyboardInterrupt]) +@pytest.mark.parametrize("diagnostic", ["cell", "strays", "registry", "tiers"]) +def test_actual_diagnostic_closes_preserve_unhandled_native_failure( + wrappers, monkeypatch, first_class, diagnostic +): + from server.cli import doctor, doctor_cell + from server.code.app.index import repo_registry, tier_health + from server.core import settings_overrides + from server.settings import settings + + first = first_class("diagnostic") + wrappers.errors.update(pg=OSError("pg close"), ch=RuntimeError("ch close")) + + async def fail(*_, **__): + raise first + + if diagnostic == "cell": + monkeypatch.setattr(doctor_cell, "_checks", fail) + call = doctor_cell.cell_checks + elif diagnostic == "strays": + monkeypatch.setattr(doctor_cell, "_deployment_check", _nothing) + monkeypatch.setattr(doctor_cell, "_stray_reads", fail) + call = partial(doctor_cell.stray_org_checks, 1) + else: + monkeypatch.setattr(settings, "ch_host", "inert") + if diagnostic == "registry": + monkeypatch.setattr(repo_registry, "registry_read", lambda: False) + monkeypatch.setattr(repo_registry, "refresh_db_repos", fail) + call = doctor._read_code_registry + else: + monkeypatch.setattr(settings, "code_intel_enabled", True) + monkeypatch.setattr(doctor, "_read_code_registry", lambda: None) + monkeypatch.setattr(repo_registry, "registry_read", lambda: True) + monkeypatch.setattr(settings_overrides, "refresh", _nothing) + monkeypatch.setattr(tier_health, "refresh_failing", fail) + doctor._doctor_tier_failures.cache_clear() + call = doctor._doctor_tier_failures + with pytest.raises(first_class) as caught: + call() + assert caught.value is first + assert wrappers.events[-2:] == ["pg", "ch"] + doctor._doctor_tier_failures.cache_clear() + + +@pytest.mark.parametrize("first_class", [asyncio.CancelledError, SystemExit, KeyboardInterrupt]) +@pytest.mark.asyncio +async def test_actual_setup_store_diagnostic_preserves_unhandled_failure( + wrappers, monkeypatch, first_class +): + from server.core import pg, pg_profile + from server.settings import settings + + first = first_class("setup diagnostic") + wrappers.errors.update(pg=OSError("pg close"), ch=RuntimeError("ch close")) + monkeypatch.setattr(settings, "cell", "c") + monkeypatch.setattr(pg, "enabled", lambda: True) + + async def refusal(_): + raise first + + monkeypatch.setattr(pg_profile, "recorded_authority", refusal) + with pytest.raises(first_class) as caught: + await _ch.check_setup_stores() + assert caught.value is first + assert wrappers.events[-2:] == ["pg", "ch"] diff --git a/tests/unit/test_provider_command.py b/tests/unit/test_provider_command.py index 2a1d22da2d..9a70d03b37 100644 --- a/tests/unit/test_provider_command.py +++ b/tests/unit/test_provider_command.py @@ -16,7 +16,7 @@ def setup(monkeypatch): gate = ModelWorkGate() monkeypatch.setattr(command, "get_model_work_gate", lambda: gate) monkeypatch.setattr(command, "register_process", AsyncMock()) - monkeypatch.setattr(command, "retire_process", AsyncMock(return_value=True)) + monkeypatch.setattr(command, "_retire_until", AsyncMock(return_value=True)) monkeypatch.setattr(command, "load_active_configuration", AsyncMock()) monkeypatch.setattr(command, "drain_task_sets", AsyncMock(return_value=True)) monkeypatch.setattr(command, "flush_registered_buffers", AsyncMock(return_value=True)) @@ -29,9 +29,9 @@ async def test_command_enrolls_loads_settings_then_admits_and_retires(setup): assert setup.active == 1 command.register_process.assert_awaited_once() command.load_active_configuration.assert_awaited_once() - command.retire_process.assert_not_awaited() + command._retire_until.assert_not_awaited() assert setup.active == 0 - command.retire_process.assert_awaited_once() + command._retire_until.assert_awaited_once() @pytest.mark.asyncio @@ -40,7 +40,7 @@ async def test_pending_transition_never_runs_command(setup): with pytest.raises(ModelChangeInProgress): async with command.model_command(): pytest.fail("closed admission must reject a newborn command") - command.retire_process.assert_awaited_once() + command._retire_until.assert_awaited_once() @pytest.mark.asyncio @@ -49,7 +49,7 @@ async def test_undrained_descendants_keep_participant_registered(setup, monkeypa with pytest.raises(RuntimeError, match="deferred writes did not drain"): async with command.model_command(): pass - command.retire_process.assert_not_awaited() + command._retire_until.assert_not_awaited() command.flush_registered_buffers.assert_not_awaited() @@ -58,4 +58,4 @@ async def test_business_failure_does_not_skip_drain(setup): with pytest.raises(ValueError, match="business failure"): async with command.model_command(): raise ValueError("business failure") - command.retire_process.assert_awaited_once() + command._retire_until.assert_awaited_once() From b253487b6875bdc89cd4fa12057e2a5b8ecbdca1 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 16:11:59 +0000 Subject: [PATCH 065/114] WIP: preserve reviewed attach lifecycle corrections Signed-off-by: zlareb1 --- docs/operations/upgrades.md | 14 ++-- server/cli/_ch.py | 6 +- server/cli/cell_attach_beats.py | 2 +- server/cli/cell_control.py | 2 +- server/core/schema/pg_migrate.py | 2 +- server/insights/app/provider_command.py | 4 +- server/router/directory.py | 2 +- tests/integration/test_cell_retirement.py | 80 +++++++++++++++++++++-- tests/unit/test_cell_retirement.py | 2 +- 9 files changed, 93 insertions(+), 21 deletions(-) diff --git a/docs/operations/upgrades.md b/docs/operations/upgrades.md index 50dd5fc46a..02e45f6c80 100644 --- a/docs/operations/upgrades.md +++ b/docs/operations/upgrades.md @@ -306,9 +306,10 @@ checkout. refused as a permission error (SQLSTATE 42501) rather than `LT001`. 16. Before a first cell's attach, every CLI and CI image runs a build with that state read, - and every CLI that writes for an org has `LOOM_CELL` set. No CLI sends a heartbeat, so - attach cannot check either, and a CLI of an older build is not refused on an `enforced` - cell ([cells](../architecture/cells.md#failure-modes)). + and every CLI that writes for an org has `LOOM_CELL` set. A CLI publishes no instance + or cell heartbeat for attach to read, so attach cannot check either, and a CLI of an + older build is not refused on an `enforced` cell + ([cells](../architecture/cells.md#failure-modes)). 17. Moving to a build whose first-cell processes read their cell's state from the router on a timer: every router replica (`loom router serve`) runs this build before any cell's @@ -322,9 +323,10 @@ checkout. 18. Before a first cell's attach, once `LOOM_CELL` is set on every process and every CLI and CI configuration of the cell (step 16), rotate the ClickHouse credential they write - with: a new user or password for the cell's servers and CLIs, the old one dropped. No - CLI sends a heartbeat, so attach cannot see a job still configured with `LOOM_CELL` - unset; the rotation cuts off any the rollout missed before it can write an org's rows + with: a new user or password for the cell's servers and CLIs, the old one dropped. A + CLI publishes no instance or cell heartbeat for attach to read, so attach cannot see a + job still configured with `LOOM_CELL` unset; the rotation cuts off any the rollout + missed before it can write an org's rows past attach's last scan. Name the new user with `--ch-allow-user` ([attaching the first cell](./cell-attach.md)). diff --git a/server/cli/_ch.py b/server/cli/_ch.py index bca6dadbaa..65a4385e82 100644 --- a/server/cli/_ch.py +++ b/server/cli/_ch.py @@ -344,7 +344,7 @@ async def connect_verified(ch: CHClient) -> None: await ch.connect() try: await check_configured(ch) - except BaseException: + except BaseException: # Keep the pairing failure while closing its clients. with suppress(BaseException): await disconnect(ch) raise @@ -370,7 +370,7 @@ async def disconnect(ch: CHClient) -> None: pg.expect_deployment(None) try: await pg.close_pool() - except BaseException: + except BaseException: # Keep the pool close failure while closing ClickHouse. with suppress(BaseException): await ch.disconnect() raise @@ -402,7 +402,7 @@ async def _go() -> T: async with model_command(): await check_acceptance(orgs=named, org_free=org_free) result = await run(ch) - except BaseException: + except BaseException: # Keep the command failure while disconnecting its clients. with suppress(BaseException): await disconnect(ch) raise diff --git a/server/cli/cell_attach_beats.py b/server/cli/cell_attach_beats.py index c2a8ed2428..f49819a1e0 100644 --- a/server/cli/cell_attach_beats.py +++ b/server/cli/cell_attach_beats.py @@ -293,7 +293,7 @@ async def read_once(self) -> list[Beat]: try: async with asyncio.timeout(_READ_TIMEOUT_S): beats = await read_beats(self.ch, since=self.since) - except Exception: + except Exception: # A failed read still checks whether the heartbeat watch has a gap. self._check_gap() raise self._check_gap() diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index 730c36bd42..bf412e2d08 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -146,7 +146,7 @@ async def _open(self, what: str) -> None: try: await pg._check_deployment(self.conn) await pg.init_connection(self.conn) - except Exception as error: + except Exception as error: # Classify unavailable openings; keep other failures. pg._raise_if_unavailable(error) raise refused = await apg.direct_refusals(self.conn, what) diff --git a/server/core/schema/pg_migrate.py b/server/core/schema/pg_migrate.py index a62b467c0c..4a0fbdf6bf 100644 --- a/server/core/schema/pg_migrate.py +++ b/server/core/schema/pg_migrate.py @@ -119,7 +119,7 @@ async def migrate( result = await asyncio.shield(task) if isinstance(result, (SystemExit, KeyboardInterrupt)): raise result - except BaseException as error: + except BaseException as error: # Keep the migration failure through owned cleanup. end = time.monotonic() + pg._RELEASE_TIMEOUT_S await _failed_migration(cm, conn, task, error, end) raise diff --git a/server/insights/app/provider_command.py b/server/insights/app/provider_command.py index 3e16b03c48..dd16904ddb 100644 --- a/server/insights/app/provider_command.py +++ b/server/insights/app/provider_command.py @@ -117,7 +117,7 @@ async def participant_command() -> AsyncIterator[None]: try: async with retirement.get_gate().admit(): yield - except BaseException: + except BaseException: # Keep the body failure while attempting required retirement cleanup. with contextlib.suppress(BaseException): await _finish_participant() raise @@ -154,6 +154,6 @@ async def model_command() -> AsyncIterator[None]: else: if fatal is not None: raise fatal - except BaseException: + except BaseException: # Keep the body's first failure through keeper cleanup. if first is None: raise diff --git a/server/router/directory.py b/server/router/directory.py index bdf9f4bdd6..3e438cf6d9 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -1014,7 +1014,7 @@ async def release(self) -> None: self.exit_task = self.start(lambda: self.context.__aexit__(None, None, None)) try: await self.wait(self.exit_task, self.end) - except BaseException: + except BaseException: # An interrupted pool exit must not continue without its owner. self.exit_task.cancel() raise diff --git a/tests/integration/test_cell_retirement.py b/tests/integration/test_cell_retirement.py index 0652b4403f..a769a0728e 100644 --- a/tests/integration/test_cell_retirement.py +++ b/tests/integration/test_cell_retirement.py @@ -3,6 +3,8 @@ from __future__ import annotations import asyncio +import contextlib +import json import os import sys from uuid import UUID, uuid4 @@ -11,26 +13,74 @@ from server.core import provider_process, retirement from server.core.ch import CHClient, query_tag +from server.core.sql_literal import sql_string from server.insights.store import provider_participants as store +def _write_scratch_owner(tmp_path, owner, **changes): + owner.update(changes) + owner["cleanup_complete"] = owner["drop_confirmed"] and owner["disconnect_confirmed"] + path = tmp_path / "scratch-owner.json" + staging = tmp_path / "scratch-owner.next" + descriptor = os.open(staging, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + json.dump(owner, stream, indent=2) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + os.replace(staging, path) + descriptor = os.open(tmp_path, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _scratch_owner(tmp_path, request, database, table): + owner = { + "schema": "loom.native-scratch.v1", + "case_nodeid": request.node.nodeid, + "database": database, + "table": table, + "ownership_marker": "loom-native-retirement-" + uuid4().hex, + "create_intended": True, + "create_confirmed": False, + "table_created": False, + "drop_confirmed": False, + "disconnect_confirmed": False, + } + _write_scratch_owner(tmp_path, owner) + return owner + + @pytest.mark.asyncio async def test_certified_retirement_survives_old_beat_and_prunes_only_certified_history( monkeypatch, + tmp_path, + request, ): if not os.environ.get("LOOM_CH_HOST"): pytest.skip("Set LOOM_CH_HOST to the owned disposable ClickHouse") database = "retirement_" + uuid4().hex table = database + ".participants" + owner = _scratch_owner(tmp_path, request, database, table) client = CHClient() - await client.connect() + created = False with query_tag("test.cell_retirement"): - await client.execute("CREATE DATABASE {database:Identifier}", {"database": database}) try: + await client.connect() + await client.execute( + "CREATE DATABASE {database:Identifier} COMMENT " + + sql_string(owner["ownership_marker"]), + {"database": database}, + ) + created = True + _write_scratch_owner(tmp_path, owner, create_confirmed=True) await client.execute( "CREATE TABLE {database:Identifier}.participants AS loom._provider_participants", {"database": database}, ) + _write_scratch_owner(tmp_path, owner, table_created=True) old_table = store.TABLE for name in ( "TABLE", @@ -78,7 +128,8 @@ async def test_certified_retirement_survives_old_beat_and_prunes_only_certified_ ], ) rows = await client.query_all( - f"SELECT participant_id, transition_id, state, version, stale_after_s, host_proof FROM {table} FINAL" + "SELECT participant_id, transition_id, state, version, stale_after_s, host_proof " + f"FROM {table} FINAL" ) final = next(row for row in rows if str(row["participant_id"]) == str(stopped)) assert str(final["transition_id"]) == str(UUID(int=0)) @@ -99,5 +150,24 @@ async def test_certified_retirement_survives_old_beat_and_prunes_only_certified_ (str(legacy), "retired") ] finally: - await client.execute("DROP DATABASE {database:Identifier} SYNC", {"database": database}) - await client.disconnect() + # Keep the test's failure if required scratch cleanup also fails. + with ( + contextlib.suppress(BaseException) + if sys.exception() is not None + else contextlib.nullcontext() + ): + try: + if created: + await client.execute( + "DROP DATABASE {database:Identifier} SYNC", {"database": database} + ) + _write_scratch_owner(tmp_path, owner, drop_confirmed=True) + finally: + # Disconnect must not replace a failed drop or receipt write. + with ( + contextlib.suppress(BaseException) + if sys.exception() is not None + else contextlib.nullcontext() + ): + await client.disconnect() + _write_scratch_owner(tmp_path, owner, disconnect_confirmed=True) diff --git a/tests/unit/test_cell_retirement.py b/tests/unit/test_cell_retirement.py index a5a3c8dcce..178ba2bd1f 100644 --- a/tests/unit/test_cell_retirement.py +++ b/tests/unit/test_cell_retirement.py @@ -651,7 +651,7 @@ async def work(ch): monkeypatch.setattr(provider_command, "register_process", register) monkeypatch.setattr(provider_command, "drain_task_sets", AsyncMock(return_value=True)) monkeypatch.setattr(provider_command, "flush_registered_buffers", AsyncMock(return_value=True)) - monkeypatch.setattr(provider_command, "retire_process", AsyncMock(return_value=True)) + monkeypatch.setattr(provider_command, "_retire_until", AsyncMock(return_value=True)) monkeypatch.setattr(_ch, "check_orgs", acceptance) assert sql_access._connected(work, orgs=("example",)) == 7 assert retirement.get_gate().closed and retirement.get_gate().active == 0 From d182c26bd532d69dacc159ff022e40be706b8dc2 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 16:59:47 +0000 Subject: [PATCH 066/114] WIP: preserve attach registration and error regressions Signed-off-by: zlareb1 --- tests/unit/test_bench_cell_trigger_gate.py | 2 +- tests/unit/test_cell_attach.py | 7 +- .../unit/test_cell_attach_serving_identity.py | 134 ++++++++++++++++++ .../test_cell_trigger_gate_error_envelope.py | 86 +++++++++++ tests/unit/test_platform_admin_rollout.py | 46 +++++- 5 files changed, 272 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_cell_attach_serving_identity.py create mode 100644 tests/unit/test_cell_trigger_gate_error_envelope.py diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py index 9d9ce8f180..7543067fd6 100644 --- a/tests/unit/test_bench_cell_trigger_gate.py +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -1143,7 +1143,7 @@ async def test_review_failed_request_keeps_raw_status_elapsed_and_classification ): async def send(self, request, **kwargs): return httpx.Response( - status, json={"error": {"code": code, "message": "secret-token"}}, request=request + status, json={"detail": {"code": code, "message": "secret-token"}}, request=request ) monkeypatch.setattr(httpx.AsyncClient, "send", send) diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 5188641547..819f03d90f 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -964,10 +964,14 @@ async def test_a_serving_row_is_done_only_once_the_cell_is_enforced( from server.core import pg state = ["enforced"] + conn = FakeConn([("SELECT deployment_id FROM loom.authority", "dep-1")]) + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "document_s3_bucket", "") + monkeypatch.setattr(settings, "document_s3_prefix", "") @asynccontextmanager async def dedicated(**_kw: Any) -> Any: - yield None + yield conn async def read_mode(_conn: Any) -> tuple[str, Any]: return state[0], (True, "first") @@ -976,6 +980,7 @@ async def read_mode(_conn: Any) -> tuple[str, Any]: monkeypatch.setattr(apg, "read_mode", read_mode) run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] run.owner = _owner(FakeConn()) + run.row = {"deployment_id": "dep-1", "cell": "first", "bucket": "", "bucket_prefix": ""} assert await run.done() state[0] = "recording" with pytest.raises(Refused, match="attach lowers and repairs neither"): diff --git a/tests/unit/test_cell_attach_serving_identity.py b/tests/unit/test_cell_attach_serving_identity.py new file mode 100644 index 0000000000..469ed16ea3 --- /dev/null +++ b/tests/unit/test_cell_attach_serving_identity.py @@ -0,0 +1,134 @@ +"""A serving shortcut validates the registration without claiming or writing.""" + +from __future__ import annotations + +from contextlib import asynccontextmanager +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import UUID + +import pytest + +from server.cli import cell_attach, cell_control +from server.cli.cell_ch import Refused +from server.core import pg +from server.settings import settings +from tests.unit.test_cell_attach import FakeConn, _args + + +@pytest.fixture +def serving_run(monkeypatch): + def make( + *, + deployment="registered", + configured_cell="first", + bucket="bucket", + prefix="docs/", + first=True, + mode="enforced", + ): + row = { + "cell": "first", + "is_first": first, + "state": "serving", + "host": "first.example", + "deployment_id": "registered", + "bucket": "bucket", + "bucket_prefix": "docs/", + "probe_url": "", + "upstream_url": "", + "current_fp": None, + "next_fp": None, + "mutation_generation": str(UUID(int=1)), + } + monkeypatch.setattr(settings, "cell", configured_cell) + monkeypatch.setattr(settings, "document_s3_bucket", bucket) + monkeypatch.setattr(settings, "document_s3_prefix", prefix) + conn = FakeConn( + [ + ("SELECT deployment_id FROM loom.authority", deployment), + ( + "SELECT (SELECT is_first FROM loom.cell_first)", + { + "is_first": first, + "cell": "first", + "mode": mode, + "has_own": True, + "has_tenancy": True, + }, + ), + ( + "SELECT is_first, cell FROM loom.cell_first", + {"is_first": first, "cell": "first"}, + ), + ] + ) + sessions = [] + + @asynccontextmanager + async def dedicated(**kwargs): + sessions.append("opened") + try: + yield conn + finally: + sessions.append("closed") + + monkeypatch.setattr(pg, "dedicated", dedicated) + owner = AsyncMock(side_effect=AssertionError("serving must not start an owner")) + claim = AsyncMock(side_effect=AssertionError("serving must not claim")) + mutate = AsyncMock(side_effect=AssertionError("serving must not mutate")) + monkeypatch.setattr(cell_control, "run", owner) + monkeypatch.setattr(cell_control, "claim", claim) + monkeypatch.setattr(cell_control, "mutate", mutate) + router = SimpleNamespace(call=AsyncMock(return_value=row)) + run = cell_attach.Run(None, _args(), router, lambda text: None) + return run, conn, router, sessions, (owner, claim, mutate) + + return make + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "overrides", + [ + pytest.param({"deployment": "another-deployment"}, id="deployment"), + pytest.param({"deployment": None}, id="missing-authority"), + pytest.param({"configured_cell": "other"}, id="configured-cell"), + pytest.param({"bucket": "another-bucket"}, id="bucket"), + pytest.param({"prefix": "another-prefix/"}, id="bucket-prefix"), + ], +) +async def test_serving_run_refuses_another_registration_without_writes(serving_run, overrides): + run, conn, router, sessions, mutations = serving_run(**overrides) + with pytest.raises(Refused, match="do not match the directory registration"): + await run.run() + assert "done" not in run.report + assert sessions == ["opened", "closed"] + assert router.call.await_args_list[0].args == ("",) + assert router.call.await_count == 1 + assert conn.executed == [] + assert all(call.await_count == 0 for call in mutations) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("first", [True, False], ids=["first", "later"]) +async def test_matching_serving_registration_is_done_without_writes(serving_run, first): + run, conn, router, sessions, mutations = serving_run(first=first) + assert await run.run() == 0 + assert run.report == {"cell": "first", "state": "serving", "done": True} + assert sessions == ["opened", "closed"] + assert router.call.await_count == 1 + assert conn.executed == [] + assert all(call.await_count == 0 for call in mutations) + + +@pytest.mark.asyncio +async def test_matching_serving_registration_still_refuses_recording(serving_run): + run, conn, router, sessions, mutations = serving_run(mode="recording") + with pytest.raises(Refused, match="attach lowers and repairs neither"): + await run.run() + assert "done" not in run.report + assert sessions == ["opened", "closed"] + assert router.call.await_count == 1 + assert conn.executed == [] + assert all(call.await_count == 0 for call in mutations) diff --git a/tests/unit/test_cell_trigger_gate_error_envelope.py b/tests/unit/test_cell_trigger_gate_error_envelope.py new file mode 100644 index 0000000000..2bdf47074e --- /dev/null +++ b/tests/unit/test_cell_trigger_gate_error_envelope.py @@ -0,0 +1,86 @@ +"""Failed benchmark requests classify real wire errors without echoing payloads.""" + +import json + +import httpx +import pytest +from fastapi import FastAPI + +from bench import cell_trigger_gate as gate +from server.core.errors import register_exception_handler +from server.core.pg_lease import PublicationPending +from tests.unit.test_bench_cell_trigger_gate import args + + +@pytest.fixture(autouse=True) +def no_database(monkeypatch): + async def forbidden(*args, **kwargs): + raise AssertionError("error classification must not open a database") + + monkeypatch.setattr(gate.asyncpg, "connect", forbidden) + monkeypatch.setattr(gate.clickhouse_connect, "get_async_client", forbidden) + + +@pytest.mark.asyncio +async def test_publication_pending_from_the_server_handler_retains_its_code(monkeypatch): + app = FastAPI() + register_exception_handler(app) + + @app.post("/v1/memory.set") + async def pending(): + error = PublicationPending() + error.message = "secret-token" + raise error + + transport = httpx.ASGITransport(app=app) + envelopes = [] + + async def send(self, request, **kwargs): + response = await transport.handle_async_request(request) + await response.aread() + envelopes.append(response.json()) + return response + + monkeypatch.setattr(httpx.AsyncClient, "send", send) + events = [] + with pytest.raises(gate.Invalid, match="memory.set HTTP status"): + await gate._memory_set(args(), emit=lambda kind, **row: events.append((kind, row))) + assert len(envelopes) == 1 and "error" not in envelopes[0] + assert envelopes[0]["detail"]["code"] == "PUBLICATION_PENDING" + assert envelopes[0]["detail"]["committed"] is True + assert len(events) == 1 and events[0][0] == "request_error" + assert events[0][1]["status"] == 503 + assert events[0][1]["code"] == "PUBLICATION_PENDING" + assert events[0][1]["measured"] is False + assert "secret-token" not in json.dumps(events) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "candidate,expected", + [ + pytest.param("BAD_CONFIG", "BAD_CONFIG", id="valid-code"), + pytest.param("A" * 64, "A" * 64, id="maximum-length"), + pytest.param("secret-token", "", id="unsafe-text"), + pytest.param("A" * 65, "", id="overlong"), + pytest.param(42, "", id="number"), + pytest.param(None, "", id="null"), + pytest.param([], "", id="list"), + ], +) +async def test_detail_code_uses_the_existing_allowlist(monkeypatch, candidate, expected): + async def send(self, request, **kwargs): + return httpx.Response( + 503, + request=request, + json={"detail": {"code": candidate, "message": "secret-token"}}, + ) + + monkeypatch.setattr(httpx.AsyncClient, "send", send) + events = [] + with pytest.raises(gate.Invalid, match="memory.set HTTP status"): + await gate._memory_set(args(), emit=lambda kind, **row: events.append((kind, row))) + assert len(events) == 1 and events[0][0] == "request_error" + assert events[0][1]["status"] == 503 + assert events[0][1]["code"] == expected + assert "secret-token" not in json.dumps(events) diff --git a/tests/unit/test_platform_admin_rollout.py b/tests/unit/test_platform_admin_rollout.py index 93d3a5eae5..6bf52c711b 100644 --- a/tests/unit/test_platform_admin_rollout.py +++ b/tests/unit/test_platform_admin_rollout.py @@ -116,7 +116,7 @@ def mint(**kw: Any) -> str: assert minted == [] -def test_cli_admin_mint_refuses_an_unread_fleet(monkeypatch, capsys) -> None: +def test_cli_admin_mint_refuses_when_the_connection_scope_cannot_open(monkeypatch, capsys) -> None: from server.auth import jwt from server.cli import access @@ -144,6 +144,50 @@ def mint(**kw): assert code == 1 and out == "" and "enrolled scope" in err +def test_cli_admin_mint_refuses_an_unread_fleet(monkeypatch, capsys) -> None: + import asyncio + + from server.auth import jwt, keyring + from server.cli import access + + connected = object() + reads = [] + minted = [] + + async def unread(ch, *, all_deployments): + assert all_deployments + reads.append(ch) + raise ConnectionError("offline") + + def mint(**kwargs): + minted.append(kwargs) + return "secret-token" + + monkeypatch.setattr(instances, "read_live", unread) + monkeypatch.setattr(keyring, "load_for_connected_mint", AsyncMock()) + monkeypatch.setattr("server.cli._ch.check_acceptance", AsyncMock()) + monkeypatch.setattr(access, "run_connected", lambda run, **kwargs: asyncio.run(run(connected))) + monkeypatch.setattr(access, "_audit_platform_admin_mint", AsyncMock(return_value=True)) + monkeypatch.setattr(jwt, "mint_token", mint) + code = cli.main( + [ + "mint-token", + "--subject", + "sa:controlplane", + "--org", + "acme", + "--platform-admin", + "--ttl-days", + "7", + ] + ) + out, err = capsys.readouterr() + assert reads == [connected] + assert code == 1 and out == "" and "rollout" in err + assert "secret-token" not in out + err + assert minted == [] + + def test_admin_token_floor_matches_the_physical_migration() -> None: from pathlib import Path From 858c90db94b94dd6b8a77bcb7dfd49925da609b4 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 18:51:24 +0000 Subject: [PATCH 067/114] WIP: validate serving cell registration and retain wire error codes Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 2 +- docs/architecture/cells.md | 7 ++++--- docs/operations/cell-attach.md | 5 +++-- server/cli/cell_attach.py | 6 ++++-- server/cli/cell_control.py | 4 ++-- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 3753ab4f90..e4a436b2ce 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -957,7 +957,7 @@ async def _memory_set( if response is not None: try: envelope = response.json() - candidate = envelope.get("error", {}).get("code", "") + candidate = envelope.get("detail", {}).get("code", "") if isinstance(candidate, str) and re.fullmatch( r"[A-Z][A-Z0-9_]{0,63}", candidate ): diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 13a673223f..7038121a79 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1168,9 +1168,10 @@ before the wait is judged spent, so `--wait 0` is one try of each wait with no r `--scan-budget` (default 2 hours) bounds each full scan, which `--wait` does not count. `--dry-run` runs the refusals and the scan, prints what would be placed and skipped and the scan's time, and writes nothing; the router's checks at `attach/begin` do not run. On a -`serving` row attach reads the cell's mode: `enforced` is done (exit 0), and any other mode -refuses: the directory and the cell's Postgres disagree, as after a restore of one of them, -and attach lowers and repairs neither. +`serving` row attach first verifies that the connected authority, configured cell and +bucket match the directory registration. It then reads the cell's mode: `enforced` is +done (exit 0), and any other mode refuses: the directory and the cell's Postgres disagree, +as after a restore of one of them, and attach lowers and repairs neither. A later cell is attached only once the first is `serving`. It must hold no org data (attach refuses otherwise) and starts in `enforced`. Its attach runs, in order: the diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index 20b541aec3..53e176d1cd 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -99,8 +99,9 @@ does not establish that server locks or an unacknowledged commit have disappeare Retry the command to resolve an unknown commit through its durable state checks. A refusal names what to fix. Rerun attach after fixing it: every step is idempotent, and a -rerun resumes by the directory row and the cell's mode. On a `serving` row it prints that it -is done. +rerun resumes by the directory row and the cell's mode. On a `serving` row it verifies +that the connected authority, configured cell and bucket match the directory registration, +then requires `enforced` mode before printing that it is done. ## The point of no return diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 821a77448b..ea20827a35 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -6,7 +6,8 @@ goes, each step idempotent, so a rerun resumes by the directory row and the cell's mode. Steps 1 to 4 are the design's, as cells.md numbers them: -- the directory row: on `serving` it is done (exit 0) once the cell reads `enforced`; +- the directory row: on `serving` it is done (exit 0) once the connected authority and + configured cell and bucket match the registration and the cell reads `enforced`; - the refusals, before any write: the stores are the registered ones, the session is direct and marked, the cell's processes (their heartbeats), the roles, settings and subscriptions that would let an unchecked writer through, the ClickHouse writers, the @@ -560,10 +561,11 @@ async def serving(self) -> dict[str, Any]: # -- the run ----------------------------------------------------------------------- async def done(self) -> bool: - """On a `serving` row: True once the cell reads `enforced`; Refused otherwise.""" + """Require matching Postgres and cell settings, then `enforced` mode.""" from server.core import pg async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: + await cell_control.paired(conn, self.row) state, _ = await apg.read_mode(conn) if state != "enforced": raise _refused( diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index bf412e2d08..93cc7e2e6b 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -58,7 +58,7 @@ async def check_lock(conn: Any) -> None: async def paired(conn: Any, row: dict[str, Any]) -> None: - """The locked authority and configured bucket name the observed registration.""" + """The connected authority and configured bucket name the observed registration.""" recorded = await conn.fetchval("SELECT deployment_id FROM loom.authority") prefix = settings.document_s3_prefix if settings.document_s3_bucket else "" if ( @@ -68,7 +68,7 @@ async def paired(conn: Any, row: dict[str, Any]) -> None: or (settings.document_s3_bucket, prefix) != (row.get("bucket"), row.get("bucket_prefix")) ): raise _refused( - "this cell configuration and locked Postgres do not match the directory registration" + "this cell configuration and connected Postgres do not match the directory registration" ) From 2a379d0d34cb2a9010944af869f09c6fd4981263 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 20:04:15 +0000 Subject: [PATCH 068/114] Native command cleanup has a configured deadline Keep the existing drain allowance as a positive finite setting. Exercise current command and shutdown boundaries and isolate permanent retirement state between tests. Signed-off-by: zlareb1 --- docs/reference/configuration.md | 1 + loom.example.yaml | 4 ++ schemas/loom-settings.schema.json | 6 +++ server/insights/app/provider_command.py | 4 +- server/settings/catalog.py | 3 ++ tests/integration/test_cell_control_fence.py | 2 +- tests/unit/test_cell_retirement.py | 1 + tests/unit/test_cli_native_completion.py | 2 +- tests/unit/test_cli_native_first_error.py | 4 +- tests/unit/test_provider_command.py | 10 ++++ tests/unit/test_readiness.py | 48 +++++++++++++++++--- tests/unit/test_settings_overrides.py | 7 ++- 12 files changed, 77 insertions(+), 15 deletions(-) diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 2a78730ac4..81d73f8c9e 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -1738,6 +1738,7 @@ Process shape, deployment, event stream and cost budgets. | `LOOM_KEEPER_SWEEP_INTERVAL_SECONDS` | `float` | `3600.0` | | Seconds between passes of the always-on sweep that deletes the KeeperMap keys (leases, claims, graph slots) a stopped process left behind. | | `LOOM_WORKERS_PROCESS` | `'child' \| 'inline'` | `'child'` | | auto mode: where the lease holder runs the workers. child = a separate process it starts and stops with the lease, so the passes do not share the event loop that serves requests; inline = in the serving process. | | `LOOM_SHUTDOWN_READY_DELAY_S` | `float` | `15.0` | | Seconds a process keeps serving after SIGTERM while /v1/health/ready answers 503, so the load balancer stops routing to it before its port closes. Size it to the LB's probe interval × failure threshold. 0 = stop at once. | +| `LOOM_NATIVE_COMMAND_DRAIN_TIMEOUT_S` | `float` | `15.0` | | Seconds a native CLI command allows its producers, deferred writes and participant retirement to finish. Expiry leaves the participant unretired. | | `LOOM_BOOT_WARMUP_TIMEOUT_S` | `float` | `30.0` | | Seconds boot waits for the first load of the tracker registry and the runtime overrides, and the first publish of the dashboard bundle, before it serves; readiness stays false until the first two load. | | `LOOM_READY_REQUIRES_MIRRORS` | `bool` | `False` | | True: /v1/health/ready also waits until the mirror warm-up has queued its clones and while a clone of this host's git mirrors is running or queued (a failed clone does not hold it). Needs LOOM_CODE_MIRROR_WARMUP: without the warm-up nothing is cloned at boot and readiness does not wait for mirrors. Off by default: history ops answer `no_git_mirror` meanwhile. | | `LOOM_RUNTIME_OVERRIDES_REFRESH_SECONDS` | `float` | `10.0` | | Seconds between re-reads of loom.runtime_overrides (the admin rate-limit, latency-budget and live-setting overrides) in every process. | diff --git a/loom.example.yaml b/loom.example.yaml index 44d36aece7..17df27d780 100644 --- a/loom.example.yaml +++ b/loom.example.yaml @@ -4891,6 +4891,10 @@ settings: # probe interval × failure threshold. 0 = stop at once. [restart] # shutdown_ready_delay_s: 15.0 + # Seconds a native CLI command allows its producers, deferred writes and participant + # retirement to finish. Expiry leaves the participant unretired. [restart] + # native_command_drain_timeout_s: 15.0 + # Seconds boot waits for the first load of the tracker registry and the runtime # overrides, and the first publish of the dashboard bundle, before it serves; # readiness stays false until the first two load. [restart] diff --git a/schemas/loom-settings.schema.json b/schemas/loom-settings.schema.json index fdbd311ef2..7e4b5d50a0 100644 --- a/schemas/loom-settings.schema.json +++ b/schemas/loom-settings.schema.json @@ -5324,6 +5324,12 @@ "type": "number", "description": "loom.namespace_write_marks is an in-process max per (org, namespace, plane), flushed as one INSERT this often." }, + "native_command_drain_timeout_s": { + "default": 15.0, + "exclusiveMinimum": 0, + "type": "number", + "description": "Seconds a native CLI command allows its producers, deferred writes and participant retirement to finish. Expiry leaves the participant unretired." + }, "oauth_access_token_ttl_seconds": { "default": 3600, "maximum": 86400, diff --git a/server/insights/app/provider_command.py b/server/insights/app/provider_command.py index dd16904ddb..6c91b93f0a 100644 --- a/server/insights/app/provider_command.py +++ b/server/insights/app/provider_command.py @@ -17,8 +17,6 @@ log = logging.getLogger(__name__) -COMMAND_DRAIN_S = 15.0 # bounded cleanup; timeout leaves the participant unretired - async def _adopt_overrides(*, failing: bool = False) -> bool: """Whether the stored overrides loaded. A failure is logged when a run of them @@ -64,7 +62,7 @@ async def _finish_participant() -> None: from server.memory.app import embedder_calibration loop = asyncio.get_running_loop() - end = loop.time() + COMMAND_DRAIN_S + end = loop.time() + settings.native_command_drain_timeout_s first = gate.producer_failure stopped = False try: diff --git a/server/settings/catalog.py b/server/settings/catalog.py index 8d9117b934..ed30d3a68f 100644 --- a/server/settings/catalog.py +++ b/server/settings/catalog.py @@ -5145,6 +5145,9 @@ class RuntimeSettings(BaseModel): # answers 503, so the load balancer stops routing to it before its port # closes. Size it to the LB's probe interval × failure threshold. 0 = stop at once. shutdown_ready_delay_s: float = Field(default=15.0, ge=0) + # Seconds a native CLI command allows its producers, deferred writes and + # participant retirement to finish. Expiry leaves the participant unretired. + native_command_drain_timeout_s: float = Field(default=15.0, gt=0, allow_inf_nan=False) # Seconds boot waits for the first load of the tracker registry and the # runtime overrides, and the first publish of the dashboard bundle, before # it serves; readiness stays false until the first two load. diff --git a/tests/integration/test_cell_control_fence.py b/tests/integration/test_cell_control_fence.py index e14eab1220..2e45540fb4 100644 --- a/tests/integration/test_cell_control_fence.py +++ b/tests/integration/test_cell_control_fence.py @@ -47,7 +47,7 @@ async def control_stores(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[Any]: transport = httpx.ASGITransport(app=app.build_app()) async with httpx.AsyncClient( transport=transport, - base_url="http://router", + base_url="http://test", headers={"Authorization": f"Bearer {TOKEN}"}, ) as client: yield client, conn, own diff --git a/tests/unit/test_cell_retirement.py b/tests/unit/test_cell_retirement.py index 178ba2bd1f..f3e7593fd3 100644 --- a/tests/unit/test_cell_retirement.py +++ b/tests/unit/test_cell_retirement.py @@ -62,6 +62,7 @@ async def write(proof): entered.set() await finish.wait() + monkeypatch.setattr(barrier, "_OWN", barrier._Own()) monkeypatch.setattr(barrier, "record_retirement", write) monkeypatch.setattr(barrier, "drain_task_sets", AsyncMock(return_value=True)) monkeypatch.setattr(barrier, "flush_registered_buffers", AsyncMock(return_value=True)) diff --git a/tests/unit/test_cli_native_completion.py b/tests/unit/test_cli_native_completion.py index 6b27052577..a2ff54b234 100644 --- a/tests/unit/test_cli_native_completion.py +++ b/tests/unit/test_cli_native_completion.py @@ -56,7 +56,7 @@ async def work(_): def test_permanent_heartbeat_respects_same_absolute_native_allowance( native, monkeypatch, first_class ): - monkeypatch.setattr(command, "COMMAND_DRAIN_S", 0.02) + monkeypatch.setattr(command.settings, "native_command_drain_timeout_s", 0.02) monkeypatch.setattr( barrier, "_HEARTBEAT", SimpleNamespace(signal_stop=lambda: None, running=True) ) diff --git a/tests/unit/test_cli_native_first_error.py b/tests/unit/test_cli_native_first_error.py index c77d29ba83..3fad11815a 100644 --- a/tests/unit/test_cli_native_first_error.py +++ b/tests/unit/test_cli_native_first_error.py @@ -353,7 +353,7 @@ async def work(_): def test_unfinished_role_producer_prevents_success_and_certificate(native, monkeypatch): producers = [] - monkeypatch.setattr(command, "COMMAND_DRAIN_S", 0) + monkeypatch.setattr(command.settings, "native_command_drain_timeout_s", 0) async def work(_): async def producer(): @@ -399,7 +399,7 @@ async def producer(error): def test_actual_retirement_refusal_stays_registered_and_refuses_success(native, monkeypatch): - monkeypatch.setattr(command, "COMMAND_DRAIN_S", 0.02) + monkeypatch.setattr(command.settings, "native_command_drain_timeout_s", 0.02) stopped = [] monkeypatch.setattr( barrier, diff --git a/tests/unit/test_provider_command.py b/tests/unit/test_provider_command.py index 9a70d03b37..645b0fcb3e 100644 --- a/tests/unit/test_provider_command.py +++ b/tests/unit/test_provider_command.py @@ -8,6 +8,16 @@ from server.insights.app import provider_command as command +@pytest.mark.parametrize("value", [0, -1, float("inf"), float("nan")]) +def test_native_drain_budget_is_finite_and_positive(value): + from pydantic import ValidationError + + from server.settings.catalog import RuntimeSettings + + with pytest.raises(ValidationError): + RuntimeSettings(native_command_drain_timeout_s=value) + + @pytest.fixture def setup(monkeypatch): from server.core import retirement diff --git a/tests/unit/test_readiness.py b/tests/unit/test_readiness.py index b7adc50935..49e153bc68 100644 --- a/tests/unit/test_readiness.py +++ b/tests/unit/test_readiness.py @@ -441,16 +441,50 @@ async def insert(self, table, rows, *, settings=None): usage._reset_for_tests() -def test_shutdown_drains_share_a_budget_before_retirement() -> None: +@pytest.mark.asyncio +async def test_shutdown_drains_share_a_budget_before_retirement(monkeypatch) -> None: import inspect + from types import SimpleNamespace from server import main - - src = inspect.getsource(main._shut_down) - budget = src.index("_shutdown_drains(lease, supervisor, emitter), timeout=left") - assert ( - src.index("await _instances.stop(ch)") < budget < src.index("await boot.leave_deployment(") - ) + from server.core import instances + from server.insights.app.provider_barrier import SHUTDOWN_RETIRE_S + + clock = [100.0] + calls = [] + wait_timeouts = [] + ch, lease, supervisor, emitter = object(), object(), object(), object() + + async def stop(client): + assert client is ch + calls.append("stop") + clock[0] += 3.0 + + async def drains(actual_lease, actual_supervisor, actual_emitter): + assert (actual_lease, actual_supervisor, actual_emitter) == (lease, supervisor, emitter) + calls.append("drains") + clock[0] += 2.0 + + async def wait(tasks, **kwargs): + wait_timeouts.append(kwargs["timeout"]) + return await asyncio.wait(tasks, **kwargs) + + async def leave(client, *, budget_s): + assert client is ch + calls.append("retire") + assert budget_s == SHUTDOWN_RETIRE_S + 5.0 + + monkeypatch.setattr(main, "time", SimpleNamespace(monotonic=lambda: clock[0])) + monkeypatch.setattr(main, "asyncio", SimpleNamespace(wait=wait)) + monkeypatch.setattr(instances, "stop", stop) + monkeypatch.setattr(main, "_shutdown_drains", drains) + monkeypatch.setattr(boot, "leave_deployment", leave) + monkeypatch.setattr(boot, "SHUTDOWN_DRAIN_BUDGET_S", 10.0) + + await main._shut_down(ch, lease, supervisor, emitter) + + assert calls == ["stop", "drains", "retire"] + assert wait_timeouts == [7.0] life = inspect.getsource(main.lifespan) assert "await _shut_down(" in life and "ch.disconnect()" not in life diff --git a/tests/unit/test_settings_overrides.py b/tests/unit/test_settings_overrides.py index e8eabb6761..e14c88c9c2 100644 --- a/tests/unit/test_settings_overrides.py +++ b/tests/unit/test_settings_overrides.py @@ -453,13 +453,18 @@ async def test_a_native_command_chunks_as_the_http_op_does_under_an_override( ) -> None: """`loom rechunk` and knowledge.upsert share `_chunk_text`: inside a native command it reads the stored override, not the startup default.""" + from server.core import retirement from server.insights.app import provider_command as pc from server.knowledge.app import writes async def _noop(*_a: Any, **_k: Any) -> bool: return True - for name in ("register_process", "load_active_configuration", "retire_process"): + async def _register() -> None: + retirement.get_gate().enrolled = True + + monkeypatch.setattr(pc, "register_process", _register) + for name in ("load_active_configuration", "_retire_until"): monkeypatch.setattr(pc, name, _noop) monkeypatch.setattr(pc, "drain_task_sets", _noop) monkeypatch.setattr(pc, "flush_registered_buffers", _noop) From 300c66578e9af5dfd4fa2fc801553c5d3708ec02 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 20:14:40 +0000 Subject: [PATCH 069/114] Check repeated cancellation against the original clock deadline Signed-off-by: zlareb1 --- tests/unit/test_cell_place_owner.py | 54 +++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/tests/unit/test_cell_place_owner.py b/tests/unit/test_cell_place_owner.py index 2cce1d25cf..b8dac073fb 100644 --- a/tests/unit/test_cell_place_owner.py +++ b/tests/unit/test_cell_place_owner.py @@ -4,12 +4,14 @@ import asyncio import time +from types import SimpleNamespace from unittest.mock import AsyncMock import pytest from server.cli import cell_control from server.core import pg +from tests.unit.clock_fake import ManualClock from tests.unit.test_cell_attach import FakeConn pytestmark = pytest.mark.asyncio @@ -163,7 +165,11 @@ def is_closed(self): await asyncio.gather(task, return_exceptions=True) -async def test_repeated_signals_cannot_extend_an_expired_public_close(): +async def test_repeated_signals_cannot_extend_an_expired_public_close(monkeypatch): + clock = ManualClock(t=100.0) + local_time = SimpleNamespace(monotonic=clock.monotonic) + monkeypatch.setattr(cell_control, "time", local_time) + monkeypatch.setattr(pg, "time", local_time) closing = asyncio.Event() conn = FakeConn() @@ -174,28 +180,46 @@ async def stalled(**kwargs): conn.close = stalled owner = cell_control.Owner(exit_s=0.04) owner.conn = conn - started = time.monotonic() - task = asyncio.create_task(owner.close()) - await closing.wait() - end = owner.close_end - - async def signals(): - while not task.done(): - task.cancel("first signal") - await asyncio.sleep(0.001) + waits = [] - signal_task = asyncio.create_task(signals()) + async def signalled_wait(tasks, **kwargs): + assert tasks == {owner._close_task} + waits.append(kwargs["timeout"]) + assert len(waits) <= 2, "signals cannot grant another cleanup allowance" + await closing.wait() + if len(waits) == 1: + clock.advance(0.01) + else: + clock.t = 100.04 + asyncio.current_task().cancel("first signal" if len(waits) == 1 else "second signal") + await asyncio.sleep(0) + raise AssertionError("cancellation must reach the public close") + + # Replace only this module's wait seam; native close and the event loop stay real. + monkeypatch.setattr( + cell_control, + "asyncio", + SimpleNamespace( + create_task=asyncio.create_task, + CancelledError=asyncio.CancelledError, + wait=signalled_wait, + ), + ) + task = asyncio.create_task(owner.close()) try: done, _ = await asyncio.wait({task}, timeout=0.2) assert done, "repeated signals cannot restart the original cleanup allowance" with pytest.raises(asyncio.CancelledError, match="first signal"): await task - assert time.monotonic() - started < 0.2 - assert owner.close_end == end + assert waits == pytest.approx([0.04, 0.03]) + assert clock.monotonic() - 100.0 == pytest.approx(0.04) + assert owner.close_end == 100.04 assert "TERMINATE" in conn.executed finally: - signal_task.cancel() - await asyncio.gather(signal_task, task, return_exceptions=True) + task.cancel() + await asyncio.gather(task, return_exceptions=True) + if owner._close_task is not None: + await asyncio.gather(owner._close_task, return_exceptions=True) async def test_closed_transport_cannot_acknowledge_pending_public_cancellation(): From 8444e8d82a792907e4dcaab919ceb14edcc15514 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 20:27:56 +0000 Subject: [PATCH 070/114] Check that native participant censuses exclude certified history Signed-off-by: zlareb1 --- .../test_provider_coordination_live_ch.py | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/tests/integration/test_provider_coordination_live_ch.py b/tests/integration/test_provider_coordination_live_ch.py index 16c9704fa8..8e74b88368 100644 --- a/tests/integration/test_provider_coordination_live_ch.py +++ b/tests/integration/test_provider_coordination_live_ch.py @@ -1,6 +1,8 @@ """Real participant rows and Keeper recovery on an opt-in disposable database.""" +import contextlib import os +import sys from uuid import uuid4 import pytest @@ -58,6 +60,78 @@ async def test_participant_ack_is_full_row_and_expiry_is_on_the_server_clock(mon await ch.disconnect() +@pytest.mark.asyncio +async def test_participant_census_excludes_certified_history_before_returning_rows(): + _live() + from server.core.ch import query_tag, sequential_reads + from server.core.provider_process import certified, has_certificate + from server.insights.store import provider_participants as store + + legacy = "test|boot||pidns|7|99||" + proof = certified(legacy) + invalid_pid = proof.replace("|7|99|", "|0|99|") + oversized_pid = proof.replace("|7|99|", "|" + "5" * 5000 + "|99|") + assert has_certificate(proof) + assert not has_certificate(invalid_pid) + assert not has_certificate(oversized_pid) + blockers = { + uuid4(): (state, identity) + for state, identity in ( + ("running", proof), + ("drained", proof), + ("retired", ""), + ("retired", legacy), + ("retired", legacy + "|unknown"), + ("retired", proof + "|unknown"), + ("retired", invalid_pid), + ("retired", oversized_pid), + ) + } + history = {uuid4(): ("retired", proof) for _ in range(256)} + created = {**history, **blockers} + expected = {str(identity): value for identity, value in blockers.items()} + ids = [str(identity) for identity in created] + ch = _default_client() + with query_tag("test.provider_participant_census"): + await ch.connect() + try: + await ch.insert( + store.TABLE, + [ + {"participant_id": identity, "state": state, "version": 1, "host_proof": value} + for identity, (state, value) in created.items() + ], + ) + with sequential_reads(): + raw = await ch.query_all( + "SELECT participant_id, state, host_proof FROM (" + + store.PARTICIPANTS_SQL + + ") WHERE participant_id IN {ids:Array(UUID)}", + {"ids": ids}, + ) + assert { + str(row["participant_id"]): (row["state"], row["host_proof"]) for row in raw + } == expected, "ClickHouse must exclude certified history before returning census rows" + kept = await store.participants(ch=ch) + assert { + str(row["participant_id"]): (row["state"], row["host_proof"]) + for row in kept + if str(row["participant_id"]) in ids + } == expected + finally: + with ( + contextlib.suppress(BaseException) if sys.exception() else contextlib.nullcontext() + ): + try: + await ch.execute( + f"ALTER TABLE {store.TABLE} DELETE WHERE participant_id IN {{ids:Array(UUID)}} " + "SETTINGS mutations_sync=2", + {"ids": ids}, + ) + finally: + await ch.disconnect() + + @pytest.mark.asyncio async def test_keeper_never_steals_from_a_beating_process_even_with_expired_timestamp(): _live() From 2397af549f22e054098da06184d293a5c4246d23 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 20:41:21 +0000 Subject: [PATCH 071/114] Describe the fixture owner's recorded ownership digest Signed-off-by: zlareb1 --- docs/benchmarks/results.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/benchmarks/results.md b/docs/benchmarks/results.md index 0f9bd094b1..00d4ca919b 100644 --- a/docs/benchmarks/results.md +++ b/docs/benchmarks/results.md @@ -563,7 +563,7 @@ diff <(jq -S . /tmp/summary.json) <(jq -S . bench/code_ask_cache_threshold/summa ``` Embedding APIs are not bit-stable across provider revisions: expect cosines to agree to a -few thousandths. A threshold is robust only when its margin exceeds that drift. +few thousandths. A threshold's margin must exceed that drift. **With the guard on** (`sweep_with_guard`: paraphrase hits of 42 and false hits of 65 different pairs): @@ -1260,9 +1260,11 @@ PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py \ The fixture owner supplies the exact source, launch, configuration and ownership receipts, excludes competing controllers and workers, and stops its processes before deleting its -exact scratch databases. The output path must not exist. The driver verifies those -receipts and the live fixture before native setup and every arm, restores the org triggers -within its cleanup bound and emits a final verdict only after cleanup and artifact sync. +exact scratch databases. The output path must not exist. The driver verifies source, +launch and configuration identities and the live fixture before native setup and every +arm, restores the org triggers within its cleanup bound and emits a final verdict only +after cleanup and artifact sync. It records the ownership-receipt digest supplied by +the fixture owner. Source tests and the diagnostic figures above are not evidence that this workload passes. ## Reference computations From 9f84a2b39306b8b71dbd9c8ff3918a55144c4389 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 20:50:41 +0000 Subject: [PATCH 072/114] Exclude certified retirement history from participant reads Signed-off-by: zlareb1 --- .../insights/store/provider_participants.py | 22 +++++++++++++------ tests/unit/snapshots/settings_defaults.json | 1 + 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/server/insights/store/provider_participants.py b/server/insights/store/provider_participants.py index faa8bc1d54..567aa5e676 100644 --- a/server/insights/store/provider_participants.py +++ b/server/insights/store/provider_participants.py @@ -20,12 +20,24 @@ # Heartbeat age is diagnostic; it never establishes permanent retirement. STALE_AFTER_S = 60 +# This conservative certificate subset never hides unknown retirement evidence. +# Bounded ASCII PIDs also exclude proofs the Python integer parser must refuse. +_CERTIFIED_RETIRED = ( + "state = 'retired' AND isValidUTF8(host_proof) " + "AND length(splitByChar('|', host_proof)) = 9 " + "AND arrayElement(splitByChar('|', host_proof), 9) = 'cell-retirement-v1' " + "AND arrayAll(i -> notEmpty(arrayElement(splitByChar('|', host_proof), i)), [1,2,4,5,6]) " + "AND length(arrayElement(splitByChar('|', host_proof), 5)) <= 20 " + "AND match(arrayElement(splitByChar('|', host_proof), 5), '^[1-9][0-9]*$') " + "AND match(arrayElement(splitByChar('|', host_proof), 6), '^[0-9]+$')" +) + # `expired` is decided on the server's clock, like the leases. PARTICIPANTS_SQL = ( "SELECT participant_id, transition_id, state, version, heartbeat_at, " "host_proof, " "heartbeat_at + toIntervalSecond(stale_after_s) < now64(3) AS expired " - f"FROM {TABLE} FINAL" + f"FROM {TABLE} FINAL WHERE NOT ({_CERTIFIED_RETIRED})" ) ONE_SQL = ( "SELECT state, version, " @@ -64,12 +76,8 @@ def running_sql(*, inserts: bool = False) -> str: # Retired participants whose last row is older than this are deleted. PRUNE_AFTER_H = 24 _PRUNE_IDS_SQL = ( - f"SELECT participant_id, state, version, host_proof FROM {TABLE} FINAL WHERE state = 'retired' " - "AND length(splitByChar('|', host_proof)) = 9 " - "AND arrayElement(splitByChar('|', host_proof), 9) = 'cell-retirement-v1' " - "AND arrayAll(i -> notEmpty(arrayElement(splitByChar('|', host_proof), i)), [1,2,4,5,6]) " - "AND match(arrayElement(splitByChar('|', host_proof), 5), '^[1-9][0-9]*$') " - "AND match(arrayElement(splitByChar('|', host_proof), 6), '^[0-9]+$') " + f"SELECT participant_id, state, version, host_proof FROM {TABLE} FINAL " + f"WHERE {_CERTIFIED_RETIRED} " "AND heartbeat_at < now64(3) - toIntervalHour({hours:UInt32}) LIMIT 1000" ) _PRUNE_SQL = f"ALTER TABLE {TABLE} DELETE WHERE participant_id IN {{ids:Array(UUID)}}" diff --git a/tests/unit/snapshots/settings_defaults.json b/tests/unit/snapshots/settings_defaults.json index 3c2309cc47..5872922964 100644 --- a/tests/unit/snapshots/settings_defaults.json +++ b/tests/unit/snapshots/settings_defaults.json @@ -939,6 +939,7 @@ "namespace_taxonomy_ttl_s": 300.0, "namespace_tune_document_sample": 50, "namespace_write_marks_flush_interval_s": 5.0, + "native_command_drain_timeout_s": 15.0, "oauth_access_token_ttl_seconds": 3600, "oauth_enabled": false, "oauth_issuer": "", From 17f8d534fdd0cebf2977a7e447b10c9aac0b045f Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 20:55:54 +0000 Subject: [PATCH 073/114] Compare declaration work after parser initialization Signed-off-by: zlareb1 --- tests/unit/test_code_definition_family.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/test_code_definition_family.py b/tests/unit/test_code_definition_family.py index 5db75f4efb..1ee5950224 100644 --- a/tests/unit/test_code_definition_family.py +++ b/tests/unit/test_code_definition_family.py @@ -223,6 +223,9 @@ def work(n: int) -> int: def test_reading_many_declarators_of_one_binding_is_linear() -> None: from tests.unit.step_count import steps + # Load grammars and start the parser child before comparing per-file work. + assert file_definitions("warm.js", b"const warm = () => 0;\n") + def work(n: int) -> int: src = "const " + ",\n ".join(f"v{i} = () => {i} // c" for i in range(n)) + ";\n" return steps(lambda: file_definitions("a.js", src.encode())) From 925479048218828eac3eb9c9bd96ad8f29d73eda Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 21:05:11 +0000 Subject: [PATCH 074/114] Prove worker crash recovery within the fixture process namespace Signed-off-by: zlareb1 --- scripts/gates/docker_provider_data.py | 153 ++++++++++++++++++ scripts/gates/docker_provider_smoke.py | 27 +++- tests/unit/test_docker_provider_smoke.py | 195 +++++++++++++++++++++++ 3 files changed, 370 insertions(+), 5 deletions(-) diff --git a/scripts/gates/docker_provider_data.py b/scripts/gates/docker_provider_data.py index 63b6097679..3b7265de77 100644 --- a/scripts/gates/docker_provider_data.py +++ b/scripts/gates/docker_provider_data.py @@ -1,12 +1,157 @@ """Seed/check nonempty rebuild surfaces inside the disposable Docker gate.""" import asyncio +import json +import os +import signal import sys +import time +from pathlib import Path +from server.core import provider_process from server.core.ch import get_client from server.insights.app.provider_command import model_command from server.settings import settings +SUPERVISOR = [ + "-m", + "uvicorn", + "docker_provider_app:app", + "--host", + "0.0.0.0", + "--port", + "7777", + "--workers", + "2", + "--timeout-graceful-shutdown", + "10", +] + + +def _process(pid: int) -> dict: + root = Path(f"/proc/{pid}") + stat = (root / "stat").read_text().rsplit(")", 1)[1].split() + return { + "pid": pid, + "parent": int(stat[1]), + "start": stat[19], + "command": (root / "cmdline").read_bytes().decode().rstrip("\0").split("\0"), + "namespace": os.stat(root / "ns/pid").st_ino, + "executable": os.readlink(root / "exe"), + } + + +def _supervisor() -> dict: + parent, me = _process(1), _process(os.getpid()) + assert parent["command"][1:] == SUPERVISOR, "Unexpected fixture supervisor" + assert parent["executable"] == me["executable"] + assert parent["namespace"] == me["namespace"] + return parent + + +def _worker_process(raw: str, parent: dict) -> dict: + proof = raw.split("|") + assert len(proof) == 9 and provider_process.has_certificate(raw) + assert proof[:4] == provider_process.host_proof().split("|")[:4] + process = _process(int(proof[4])) + assert process["pid"] > 1 + assert process["parent"] == 1 and process["start"] == proof[5] + assert process["namespace"] == parent["namespace"] + assert process["executable"] == parent["executable"] + command = process["command"] + assert len(command) == 4 and command[1] == "-c" + assert command[2].startswith("from multiprocessing.spawn import spawn_main; spawn_main(") + assert command[3] == "--multiprocessing-fork" + return process + + +def _workers(rows: list[dict], parent: dict) -> list[dict]: + here = provider_process.host_proof().split("|") + workers = [] + for row in rows: + proof = str(row["host_proof"]).split("|") + if proof[:4] != here[:4] or row["state"] == "retired": + continue + _worker_process(row["host_proof"], parent) + workers.append({key: row[key] for key in ("participant_id", "host_proof", "version")}) + assert len(workers) <= 2 + assert len({row["participant_id"] for row in workers}) == len(workers) + assert len({row["host_proof"].split("|")[4] for row in workers}) == len(workers) + return sorted(workers, key=lambda row: row["host_proof"]) + + +async def _worker_rows(client) -> list[dict]: + return await client.query_all( + "SELECT toString(participant_id) AS participant_id, state, version, host_proof " + "FROM loom._provider_participants FINAL WHERE state != 'retired'" + ) + + +async def workers(client) -> list[dict]: + parent = _supervisor() + result = _workers(await _worker_rows(client), parent) + assert len(result) == 2 + assert _supervisor() == parent + return result + + +def _kill_worker(worker: dict, parent: dict) -> None: + proof = worker["host_proof"].split("|") + pid = int(proof[4]) + assert pid > 1 + captured = _worker_process(worker["host_proof"], parent) + fd = os.pidfd_open(pid) + try: + # A reused PID or changed supervisor refuses before the process-bound signal. + assert _process(pid) == captured and _supervisor() == parent + signal.pidfd_send_signal(fd, signal.SIGKILL) + finally: + os.close(fd) + + +async def retired(client, captured: list[dict]) -> bool: + from server.insights.store.provider_participants import running_queries + + rows = await client.query_all( + "SELECT toString(participant_id) AS participant_id, state, version, host_proof " + "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)}", + {"ids": [row["participant_id"] for row in captured]}, + ) + if len(rows) != len(captured): + return False + expected = {row["participant_id"]: row for row in captured} + for row in rows: + old = expected[row["participant_id"]] + if ( + row["state"] != "retired" + or int(row["version"]) <= int(old["version"]) + or row["host_proof"] != old["host_proof"] + or not provider_process.has_certificate(row["host_proof"]) + or await running_queries(row["participant_id"], ch=client) != 0 + ): + return False + return True + + +async def crash_worker(client) -> None: + parent = _supervisor() + before = await workers(client) + victim = before[0] + _kill_worker(victim, parent) + deadline = time.monotonic() + 240 + while time.monotonic() < deadline: + assert _supervisor() == parent + if provider_process.proves_stopped(victim["host_proof"]) and await retired( + client, [victim] + ): + after = _workers(await _worker_rows(client), parent) + old = {row["participant_id"] for row in before} + new = {row["participant_id"] for row in after} + if len(new) == 2 and len(old & new) == 1 and victim["participant_id"] not in new: + return + await asyncio.sleep(1) + raise AssertionError("Crashed worker retirement or supervised replacement was not proved") + async def seed() -> None: from server import boot @@ -75,6 +220,14 @@ async def main() -> None: try: if sys.argv[1] == "seed": await seed() + elif sys.argv[1] == "workers": + print(json.dumps(await workers(client))) + elif sys.argv[1] == "crash-worker": + await crash_worker(client) + elif sys.argv[1] == "retired": + assert await retired(client, json.loads(sys.argv[2])), ( + "Recreated workers did not retire" + ) else: await verify(sys.argv[1], int(sys.argv[2])) finally: diff --git a/scripts/gates/docker_provider_smoke.py b/scripts/gates/docker_provider_smoke.py index a71ec4cf9f..b5445e145e 100644 --- a/scripts/gates/docker_provider_smoke.py +++ b/scripts/gates/docker_provider_smoke.py @@ -59,9 +59,19 @@ def control(compose: list[str], failed_model: str) -> None: run(*compose, "exec", "-T", "provider-smoke", "python", "-c", script) -def data_check(compose: list[str], *args: str) -> None: +def data_check(compose: list[str], *args: str) -> list[dict] | None: source = (ROOT / "scripts/gates/docker_provider_data.py").read_text() + if args[0] == "workers": + result = subprocess.run( + [*compose, "exec", "-T", "loom", "python", "-", *args], + input=source, + text=True, + check=True, + capture_output=True, + ) + return json.loads(result.stdout) run(*compose, "exec", "-T", "loom", "python", "-", *args, input=source, text=True) + return None def dashboard_action(base: str, action: str, *args: str) -> dict: @@ -105,6 +115,7 @@ def wait_admission(base: str) -> None: def smoke_providers(base: str, compose: list[str]) -> None: override = Path(compose[-1]) configuration = json.loads(override.read_text()) + # Match serve.sh's finite request grace so the lifespan fits Compose's stop grace. configuration["services"]["loom"]["entrypoint"] = [ "python", "-m", @@ -116,6 +127,8 @@ def smoke_providers(base: str, compose: list[str]) -> None: "7777", "--workers", "2", + "--timeout-graceful-shutdown", + "10", ] configuration["services"]["loom"]["volumes"] = [ f"{ROOT / 'tests/fixtures/docker_provider_app.py'}:/app/docker_provider_app.py:ro" @@ -171,15 +184,19 @@ def smoke_providers(base: str, compose: list[str]) -> None: control(compose, "") dashboard_action(base, "retry") - # SIGKILL exercises persistent process proofs and ambiguous-effect recovery, - # not just an orderly shutdown. Restart only this generated stack's Loom. - run(*compose, "kill", "-s", "SIGKILL", "loom") - run(*compose, "up", "-d", "--wait", "--wait-timeout", "180", "loom") + # Keep the supervisor's PID namespace: a different namespace cannot certify + # death. A live worker retires the exact crashed worker after its queries end. + data_check(compose, "crash-worker") status = wait_status(base, lambda row: row["pending"] is None) assert status["embedding_model"] == "docker-smoke-16" and status["embedding_dimensions"] == 16 wait_admission(base) data_check(compose, "docker-smoke-16", "16") + workers = data_check(compose, "workers") + assert workers is not None and len(workers) == 2 run(*compose, "up", "-d", "--force-recreate", "--wait", "--wait-timeout", "180", "loom") + # Recreation is graceful; prove the captured workers actually retired instead + # of treating Docker's container replacement as process-death evidence. + data_check(compose, "retired", json.dumps(workers)) status = request(base, "/v1/admin/models") assert status["source"] == "dashboard" and status["embedding_model"] == "docker-smoke-16" wait_admission(base) diff --git a/tests/unit/test_docker_provider_smoke.py b/tests/unit/test_docker_provider_smoke.py index a561016e08..7e51d06878 100644 --- a/tests/unit/test_docker_provider_smoke.py +++ b/tests/unit/test_docker_provider_smoke.py @@ -1,5 +1,6 @@ """The smoke fixture is deterministic and isolates recovery from paid providers.""" +import io import json import threading import urllib.error @@ -138,3 +139,197 @@ async def test_data_check_requires_no_retained_rebuild_copies(monkeypatch): monkeypatch.setattr(data, "get_client", lambda: client) with pytest.raises(AssertionError): await data.verify("model", 8) + + +def test_crash_keeps_the_supervisor_and_verifies_retirement_before_reset(monkeypatch, tmp_path): + override = tmp_path / "override.json" + override.write_text(json.dumps({"services": {"loom": {}}})) + compose = ["docker", "compose", "-f", str(override)] + events = [] + environment = { + "source": "environment", + "embedding_model": "environment", + "embedding_dimensions": 8, + "llm_model": "fake", + } + active = { + "source": "dashboard", + "pending": None, + "embedding_model": "docker-smoke-16", + "embedding_dimensions": 16, + } + statuses = iter( + [ + { + **active, + "embedding_model": "docker-smoke-8", + "configured_keys": {"compatible": True}, + }, + { + "embedding_model": "docker-smoke-8", + "pending": {"state": "failed", "error_code": "rebuild_rebuild_failed"}, + }, + active, + {**environment, "pending": None}, + ] + ) + model_reads = iter([environment, active]) + + def request(_base, path, _payload=None): + if path == "/v1/admin/models": + return next(model_reads) + raise urllib.error.HTTPError("http://fixture", 503, "closed", {}, None) + + def configure(*_args): + raise urllib.error.HTTPError( + "http://fixture", 422, "width", {}, io.BytesIO(b"unsupported_dimensions") + ) + + applications = iter(["queued", "active", "queued"]) + + def dashboard(_base, action, *_args): + events.append(action) + return {"state": next(applications) if action == "apply" else "queued"} + + def check(_compose, *args): + events.append(args[0]) + return ( + [{"participant_id": "first"}, {"participant_id": "second"}] + if args[0] == "workers" + else None + ) + + def status(_base, predicate): + value = next(statuses) + assert predicate(value) + return value + + monkeypatch.setattr(smoke, "request", request) + monkeypatch.setattr(smoke, "configure", configure) + monkeypatch.setattr(smoke, "dashboard_action", dashboard) + monkeypatch.setattr(smoke, "data_check", check) + monkeypatch.setattr(smoke, "wait_status", status) + monkeypatch.setattr(smoke, "wait_admission", lambda *_: None) + monkeypatch.setattr(smoke, "control", lambda *_: None) + monkeypatch.setattr(smoke, "run", lambda *args: events.append(tuple(args))) + smoke.smoke_providers("http://fixture", compose) + assert not any(isinstance(event, tuple) and "SIGKILL" in event for event in events) + assert events.index("retry") < events.index("crash-worker") < events.index("workers") + recreate = [ + i + for i, event in enumerate(events) + if isinstance(event, tuple) and "--force-recreate" in event + ] + assert len(recreate) == 2 + assert events.index("workers") < recreate[-1] < events.index("retired") < events.index("reset") + + +def _process_fixture(monkeypatch): + parent = { + "pid": 1, + "parent": 0, + "start": "10", + "namespace": 1, + "executable": "/python", + "command": ["python", *data.SUPERVISOR], + } + processes = { + pid: { + "pid": pid, + "parent": 1, + "start": str(pid * 10), + "namespace": 1, + "executable": "/python", + "command": [ + "python", + "-c", + "from multiprocessing.spawn import spawn_main; spawn_main(tracker_fd=4, pipe_handle=8)", + "--multiprocessing-fork", + ], + } + for pid in (8, 9) + } + rows = [ + { + "participant_id": str(pid), + "version": 5, + "state": "running", + "host_proof": f"h|b|c|1|{pid}|{pid * 10}|||cell-retirement-v1", + } + for pid in (8, 9) + ] + monkeypatch.setattr(data.provider_process, "host_proof", lambda: "h|b|c|1|99|990||") + monkeypatch.setattr(data, "_process", lambda pid: processes[pid]) + return parent, processes, rows + + +def test_worker_selection_requires_registered_supervised_processes(monkeypatch): + parent, processes, rows = _process_fixture(monkeypatch) + foreign = {**rows[0], "host_proof": "h|b|c|2|8|80|||cell-retirement-v1"} + assert data._workers([*rows, foreign], parent) == [ + {k: row[k] for k in ("participant_id", "host_proof", "version")} for row in rows + ] + processes[8] = {**processes[8], "parent": 2} + with pytest.raises(AssertionError): + data._workers(rows, parent) + processes[8] = { + **processes[8], + "parent": 1, + "command": ["python", "-c", "from multiprocessing.resource_tracker import main; main(4)"], + } + with pytest.raises(AssertionError): + data._workers(rows, parent) + + +@pytest.mark.parametrize("fault", ["none", "pid-reused", "supervisor-changed"]) +def test_pidfd_crash_refuses_changed_identity_before_signalling(monkeypatch, fault): + parent, processes, rows = _process_fixture(monkeypatch) + events = [] + observed = iter( + [processes[8], {**processes[8], "start": "81"} if fault == "pid-reused" else processes[8]] + ) + monkeypatch.setattr(data, "_process", lambda _pid: next(observed)) + monkeypatch.setattr( + data, + "_supervisor", + lambda: {**parent, "start": "11"} if fault == "supervisor-changed" else parent, + ) + monkeypatch.setattr(data.os, "pidfd_open", lambda pid: events.append(("open", pid)) or 73) + monkeypatch.setattr(data.os, "close", lambda fd: events.append(("close", fd))) + monkeypatch.setattr( + data.signal, "pidfd_send_signal", lambda fd, sig: events.append(("signal", fd, sig)) + ) + if fault == "none": + data._kill_worker(rows[0], parent) + assert events == [("open", 8), ("signal", 73, data.signal.SIGKILL), ("close", 73)] + else: + with pytest.raises(AssertionError): + data._kill_worker(rows[0], parent) + assert events == [("open", 8), ("close", 73)] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", ["none", "drained", "old-version", "different-proof", "busy", "unknown"] +) +async def test_retirement_requires_exact_newer_certificate_and_known_zero_queries( + monkeypatch, fault +): + from unittest.mock import AsyncMock + + from server.insights.store import provider_participants + + _parent, _processes, rows = _process_fixture(monkeypatch) + observed = {**rows[0], "state": "retired", "version": 6} + if fault == "drained": + observed["state"] = "drained" + elif fault == "old-version": + observed["version"] = 5 + elif fault == "different-proof": + observed["host_proof"] = rows[1]["host_proof"] + client = AsyncMock() + client.query_all.return_value = [observed] + count = 1 if fault == "busy" else None if fault == "unknown" else 0 + monkeypatch.setattr(provider_participants, "running_queries", AsyncMock(return_value=count)) + assert await data.retired(client, [rows[0]]) is (fault == "none") + assert client.query_all.await_args.args[1] == {"ids": [rows[0]["participant_id"]]} From 724299bef7a674ccbeef8647e2b601c8c87b4bbe Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 21:30:27 +0000 Subject: [PATCH 075/114] Quickstart matches live workers to their registrations Select supervisor spawn children before joining participant history, and validate their exact inherited interpreter flags. Keep process identity and retirement checks before any signal. Signed-off-by: zlareb1 --- scripts/gates/docker_provider_data.py | 99 ++++++++++++++++++------ tests/unit/test_docker_provider_smoke.py | 77 +++++++++++++++++- 2 files changed, 148 insertions(+), 28 deletions(-) diff --git a/scripts/gates/docker_provider_data.py b/scripts/gates/docker_provider_data.py index 3b7265de77..726582244e 100644 --- a/scripts/gates/docker_provider_data.py +++ b/scripts/gates/docker_provider_data.py @@ -3,9 +3,11 @@ import asyncio import json import os +import re import signal import sys import time +from multiprocessing import spawn from pathlib import Path from server.core import provider_process @@ -44,39 +46,87 @@ def _process(pid: int) -> dict: def _supervisor() -> dict: parent, me = _process(1), _process(os.getpid()) assert parent["command"][1:] == SUPERVISOR, "Unexpected fixture supervisor" - assert parent["executable"] == me["executable"] - assert parent["namespace"] == me["namespace"] + assert parent["executable"] == me["executable"], "Fixture supervisor executable mismatch" + assert parent["namespace"] == me["namespace"], "Fixture supervisor PID namespace mismatch" return parent def _worker_process(raw: str, parent: dict) -> dict: proof = raw.split("|") - assert len(proof) == 9 and provider_process.has_certificate(raw) - assert proof[:4] == provider_process.host_proof().split("|")[:4] + assert len(proof) == 9 and provider_process.has_certificate(raw), "Worker certificate invalid" + assert proof[:4] == provider_process.host_proof().split("|")[:4], "Worker proof scope mismatch" process = _process(int(proof[4])) - assert process["pid"] > 1 - assert process["parent"] == 1 and process["start"] == proof[5] - assert process["namespace"] == parent["namespace"] - assert process["executable"] == parent["executable"] - command = process["command"] - assert len(command) == 4 and command[1] == "-c" - assert command[2].startswith("from multiprocessing.spawn import spawn_main; spawn_main(") - assert command[3] == "--multiprocessing-fork" + pid = process["pid"] + assert pid > 1, "Worker identity names the supervisor" + assert process["parent"] == 1, f"Worker PID {pid} parent mismatch: {process['parent']} != 1" + assert process["start"] == proof[5], f"Worker PID {pid} process-start proof mismatch" + assert process["namespace"] == parent["namespace"], f"Worker PID {pid} namespace mismatch" + assert process["executable"] == parent["executable"], f"Worker PID {pid} executable mismatch" + _spawn_command(process["command"], pid) return process +def _spawn_command(command: list[str], pid: int) -> None: + # The fixed supervisor and exec helper use the same executable and container + # startup flags. Let stdlib reproduce those flags, including inherited -B. + expected = spawn.get_command_line(tracker_fd=0, pipe_handle=0) + assert len(command) == len(expected) and command[1:-2] == expected[1:-2], ( + f"Worker PID {pid} interpreter flags or -c mismatch" + ) + assert command[-1] == "--multiprocessing-fork", f"Worker PID {pid} fork argument mismatch" + assert re.fullmatch( + r"from multiprocessing\.spawn import spawn_main; " + r"spawn_main\(tracker_fd=[0-9]+, pipe_handle=[0-9]+\)", + command[-2], + ), f"Worker PID {pid} spawn command mismatch" + + +def _kernel_workers() -> dict[int, dict]: + """The supervisor's actual spawn children, before consulting participant history.""" + found = {} + children = Path("/proc/1/task/1/children").read_text().split() + for value in children: + pid = int(value) + process = _process(pid) + command = process["command"] + if not any( + value.startswith("from multiprocessing.spawn import spawn_main;") for value in command + ): + continue # The multiprocessing resource tracker is not an application worker. + _spawn_command(command, pid) + assert process["parent"] == 1, f"Kernel child PID {pid} parent changed during census" + assert pid > 1 and pid not in found, "Invalid supervisor child PID census" + found[pid] = process + assert len(found) == 2, f"Expected two live supervisor spawn children; found {len(found)}" + return found + + def _workers(rows: list[dict], parent: dict) -> list[dict]: here = provider_process.host_proof().split("|") + children = _kernel_workers() workers = [] - for row in rows: - proof = str(row["host_proof"]).split("|") - if proof[:4] != here[:4] or row["state"] == "retired": - continue - _worker_process(row["host_proof"], parent) + for pid, process in children.items(): + matches = [] + for row in rows: + proof = str(row["host_proof"]).split("|") + if ( + len(proof) >= 6 + and row["state"] != "retired" + and proof[:4] == here[:4] + and proof[4] == str(pid) + and proof[5] == process["start"] + ): + matches.append(row) + assert len(matches) == 1, ( + f"Worker PID {pid} exact registrations: {len(matches)}; expected 1" + ) + row = matches[0] + assert _worker_process(row["host_proof"], parent) == process, ( + f"Worker PID {pid} changed during proof join" + ) workers.append({key: row[key] for key in ("participant_id", "host_proof", "version")}) - assert len(workers) <= 2 - assert len({row["participant_id"] for row in workers}) == len(workers) - assert len({row["host_proof"].split("|")[4] for row in workers}) == len(workers) + assert len({row["participant_id"] for row in workers}) == 2, "Worker participant IDs collide" + assert _kernel_workers() == children, "Supervisor worker identities changed during census" return sorted(workers, key=lambda row: row["host_proof"]) @@ -90,20 +140,21 @@ async def _worker_rows(client) -> list[dict]: async def workers(client) -> list[dict]: parent = _supervisor() result = _workers(await _worker_rows(client), parent) - assert len(result) == 2 - assert _supervisor() == parent + assert len(result) == 2, "Worker registration census is incomplete" + assert _supervisor() == parent, "Fixture supervisor changed during registration census" return result def _kill_worker(worker: dict, parent: dict) -> None: proof = worker["host_proof"].split("|") pid = int(proof[4]) - assert pid > 1 + assert pid > 1, "Refusing to signal the supervisor" captured = _worker_process(worker["host_proof"], parent) fd = os.pidfd_open(pid) try: # A reused PID or changed supervisor refuses before the process-bound signal. - assert _process(pid) == captured and _supervisor() == parent + assert _process(pid) == captured, f"Worker PID {pid} changed after pidfd open" + assert _supervisor() == parent, "Fixture supervisor changed before worker signal" signal.pidfd_send_signal(fd, signal.SIGKILL) finally: os.close(fd) diff --git a/tests/unit/test_docker_provider_smoke.py b/tests/unit/test_docker_provider_smoke.py index 7e51d06878..b1a472c394 100644 --- a/tests/unit/test_docker_provider_smoke.py +++ b/tests/unit/test_docker_provider_smoke.py @@ -225,6 +225,9 @@ def status(_base, predicate): def _process_fixture(monkeypatch): + from multiprocessing import spawn + from types import SimpleNamespace + parent = { "pid": 1, "parent": 0, @@ -241,10 +244,8 @@ def _process_fixture(monkeypatch): "namespace": 1, "executable": "/python", "command": [ - "python", - "-c", - "from multiprocessing.spawn import spawn_main; spawn_main(tracker_fd=4, pipe_handle=8)", - "--multiprocessing-fork", + str(value, "utf-8") if isinstance(value, bytes) else value + for value in spawn.get_command_line(tracker_fd=4, pipe_handle=8) ], } for pid in (8, 9) @@ -260,9 +261,77 @@ def _process_fixture(monkeypatch): ] monkeypatch.setattr(data.provider_process, "host_proof", lambda: "h|b|c|1|99|990||") monkeypatch.setattr(data, "_process", lambda pid: processes[pid]) + monkeypatch.setattr(data, "Path", lambda _path: SimpleNamespace(read_text=lambda: "8 9")) return parent, processes, rows +def test_worker_selection_joins_kernel_children_without_treating_other_participants_as_workers( + monkeypatch, +): + parent, processes, rows = _process_fixture(monkeypatch) + processes[44] = { + **processes[8], + "pid": 44, + "parent": 0, + "start": "440", + "command": ["python", "-", "seed"], + } + mixed = [ + { + **rows[0], + "participant_id": "native", + "host_proof": "h|b|c|1|44|440|||cell-retirement-v1", + }, + { + **rows[0], + "participant_id": "reused-pid", + "host_proof": "h|b|c|1|8|79|||cell-retirement-v1", + }, + {**rows[0], "participant_id": "gone", "host_proof": "h|b|c|1|66|660|||cell-retirement-v1"}, + *rows, + ] + assert data._workers(mixed, parent) == [ + {k: row[k] for k in ("participant_id", "host_proof", "version")} for row in rows + ] + assert [row["participant_id"] for row in mixed] == ["native", "reused-pid", "gone", "8", "9"] + + +def test_worker_command_uses_exact_inherited_interpreter_flags(monkeypatch): + from multiprocessing import spawn, util + + parent, processes, rows = _process_fixture(monkeypatch) + monkeypatch.setattr(util, "_args_from_interpreter_flags", lambda: ["-B"]) + command = [ + str(value, "utf-8") if isinstance(value, bytes) else value + for value in spawn.get_command_line(tracker_fd=4, pipe_handle=8) + ] + processes[8] = {**processes[8], "command": command} + assert data._worker_process(rows[0]["host_proof"], parent) == processes[8] + for flags in (["-E"], ["-B", "-X", "unknown"]): + processes[8] = {**processes[8], "command": [command[0], *flags, *command[-3:]]} + with pytest.raises(AssertionError, match="interpreter flags"): + data._worker_process(rows[0]["host_proof"], parent) + + +def test_kernel_workers_never_ignore_missing_or_mismatched_current_proofs(monkeypatch): + parent, processes, rows = _process_fixture(monkeypatch) + for proof in ( + None, + "h|b|c|1|8|79|||cell-retirement-v1", + "h|b|c|2|8|80|||cell-retirement-v1", + "h|b|c|1|8|80|||unknown", + ): + changed = rows[1:] if proof is None else [{**rows[0], "host_proof": proof}, rows[1]] + with pytest.raises(AssertionError, match="exact registrations|certificate invalid"): + data._workers(changed, parent) + for field, value in (("namespace", 2), ("executable", "/other-python")): + original = processes[8] + processes[8] = {**original, field: value} + with pytest.raises(AssertionError, match=f"{field} mismatch"): + data._workers(rows, parent) + processes[8] = original + + def test_worker_selection_requires_registered_supervised_processes(monkeypatch): parent, processes, rows = _process_fixture(monkeypatch) foreign = {**rows[0], "host_proof": "h|b|c|2|8|80|||cell-retirement-v1"} From 727e60166cf83e02c4ab76bc4d2de9216605a4a9 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 21:46:16 +0000 Subject: [PATCH 076/114] Trigger acceptance runs on isolated CI stores Create a capped ClickHouse and synchronous Postgres pair for the fixed API and delivery experiment. Bind the actual checkout, interpreter and workload bytes, preserve retirement before owned store cleanup, and publish sanitized evidence. Signed-off-by: zlareb1 --- .github/workflows/docker-smoke.yml | 54 + scripts/gates/cell_trigger_acceptance.py | 883 ++++++ .../testsupport/cell_trigger/__init__.py | 1 + .../gates/testsupport/cell_trigger/capture.py | 1024 +++++++ .../gates/testsupport/cell_trigger/fixture.py | 1250 +++++++++ .../gates/testsupport/cell_trigger/native.py | 2391 +++++++++++++++++ .../cell_trigger/owned_children.py | 145 + .../testsupport/cell_trigger/protocol.json | 235 ++ .../testsupport/cell_trigger/tls_proxy.py | 168 ++ tests/unit/test_cell_trigger_ci_creator.py | 165 ++ 10 files changed, 6316 insertions(+) create mode 100644 scripts/gates/cell_trigger_acceptance.py create mode 100644 scripts/gates/testsupport/cell_trigger/__init__.py create mode 100644 scripts/gates/testsupport/cell_trigger/capture.py create mode 100644 scripts/gates/testsupport/cell_trigger/fixture.py create mode 100644 scripts/gates/testsupport/cell_trigger/native.py create mode 100644 scripts/gates/testsupport/cell_trigger/owned_children.py create mode 100644 scripts/gates/testsupport/cell_trigger/protocol.json create mode 100644 scripts/gates/testsupport/cell_trigger/tls_proxy.py create mode 100644 tests/unit/test_cell_trigger_ci_creator.py diff --git a/.github/workflows/docker-smoke.yml b/.github/workflows/docker-smoke.yml index 3088015c40..05f3564698 100644 --- a/.github/workflows/docker-smoke.yml +++ b/.github/workflows/docker-smoke.yml @@ -12,6 +12,10 @@ on: "server/**", "schemas/**", "scripts/gates/docker*", + "scripts/gates/cell_trigger_acceptance.py", + "scripts/gates/testsupport/cell_trigger/**", + "bench/cell_trigger_gate.py", + "tests/unit/test_cell_trigger_ci_creator.py", "frontend/**", "tests/fixtures/docker_provider_app.py", "scripts/install-code-toolchains.sh", @@ -45,6 +49,10 @@ on: "server/**", "schemas/**", "scripts/gates/docker*", + "scripts/gates/cell_trigger_acceptance.py", + "scripts/gates/testsupport/cell_trigger/**", + "bench/cell_trigger_gate.py", + "tests/unit/test_cell_trigger_ci_creator.py", "frontend/**", "tests/fixtures/docker_provider_app.py", "scripts/install-code-toolchains.sh", @@ -255,3 +263,49 @@ jobs: -v "$RUNNER_TEMP/pylib:/pylib:ro" -e PYTHONPATH=/pylib -e PYTHONDONTWRITEBYTECODE=1 \ -e LOOM_ENV_FILE= -e LOOM_TEST_REQUIRE_OWNER_CHECK=1 --entrypoint python loom:sandbox-ci \ -m pytest -q -p no:cacheprovider tests/unit/test_scip_checkout_fetch_ref.py + + cell-trigger-acceptance: + name: Cell trigger end-to-end acceptance + if: github.event_name == 'pull_request' || github.event_name == 'push' + needs: clickhouse + runs-on: ubuntu-24.04-arm + timeout-minutes: 150 + env: + PYTHONDONTWRITEBYTECODE: "1" + LOOM_ENV_FILE: "" + LOOM_SETTINGS_FILE: "" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + id: python + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + version: "0.12.19" + enable-cache: true + cache-dependency-glob: uv.lock + - name: Install the locked native dependencies + run: uv sync --locked --python "${{ steps.python.outputs.python-path }}" --extra dev --extra code + - name: Check native tools + run: | + command -v docker + command -v openssl + docker info >/dev/null + - name: Check triggers against real API writes and delivery passes + env: + CLICKHOUSE_IMAGE: ${{ needs.clickhouse.outputs.image }} + PYTHONPYCACHEPREFIX: ${{ runner.temp }}/trigger-bytecode + run: .venv/bin/python -B scripts/gates/cell_trigger_acceptance.py --clickhouse-image "$CLICKHOUSE_IMAGE" + - name: Remove only this attempt's created stores + if: always() + run: .venv/bin/python -B scripts/gates/cell_trigger_acceptance.py --cleanup-only + - name: Retain sanitized trigger acceptance evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: cell-trigger-acceptance + path: ${{ runner.temp }}/cell-trigger-artifacts + retention-days: 7 diff --git a/scripts/gates/cell_trigger_acceptance.py b/scripts/gates/cell_trigger_acceptance.py new file mode 100644 index 0000000000..97eedfeeab --- /dev/null +++ b/scripts/gates/cell_trigger_acceptance.py @@ -0,0 +1,883 @@ +"""Run the fixed trigger acceptance workload on fresh, job-owned stores.""" + +from __future__ import annotations + +import argparse +import datetime +import hashlib +import json +import os +import re +import shutil +import signal +import socket +import subprocess +import sys +import time +import uuid +from pathlib import Path + +GIB = 1024**3 +SUPPORT = Path(__file__).resolve().parent / "testsupport/cell_trigger" + + +def require(ok: object, message: str) -> None: + if not ok: + raise RuntimeError(message) + + +def sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def failure_site(error: BaseException | None) -> dict | None: + if error is None: + return None + site = {"file": Path(__file__).name, "line": 0, "stage": "unknown"} + trace = error.__traceback__ + while trace is not None: + if ( + Path(trace.tb_frame.f_code.co_filename).resolve() == Path(__file__).resolve() + and trace.tb_frame.f_code.co_name != "require" + ): + site.update(line=trace.tb_lineno, stage=trace.tb_frame.f_code.co_name[:64]) + trace = trace.tb_next + return site + + +def source_hashes(root: Path) -> dict: + paths = { + name: SUPPORT / name + for name in ( + "fixture.py", + "native.py", + "tls_proxy.py", + "capture.py", + "owned_children.py", + "protocol.json", + ) + } + paths.update( + driver=root / "bench/cell_trigger_gate.py", + entrypoint=Path(__file__).resolve(), + workflow=root / ".github/workflows/docker-smoke.yml", + ) + return {name: sha(path) for name, path in paths.items()} + + +def write(path: Path, value: object, *, replace: bool = False) -> None: + target = path.with_name(path.name + ".tmp") if replace else path + with target.open("x", encoding="utf-8") as stream: + os.chmod(target, 0o600) + json.dump(value, stream, indent=2) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + if replace: + os.replace(target, path) + + +def source(root: Path) -> dict: + def git(*args: str) -> str: + return subprocess.check_output(["git", *args], cwd=root, text=True, timeout=10).strip() + + require(not git("status", "--porcelain"), "Acceptance requires a clean checkout") + return {"head": git("rev-parse", "HEAD"), "tree": git("rev-parse", "HEAD^{tree}")} + + +def resources(path: Path, available_gib: int = 10) -> None: + available = next( + int(line.split()[1]) + for line in Path("/proc/meminfo").read_text().splitlines() + if line.startswith("MemAvailable:") + ) + require(available >= available_gib * 1024**2, "Acceptance available RAM floor") + require(shutil.disk_usage(path).free >= 16 * GIB, "Acceptance requires 16 GiB free disk") + + +def ports(count: int) -> list[int]: + sockets = [] + try: + for _ in range(count): + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + sockets.append(sock) + return [sock.getsockname()[1] for sock in sockets] + finally: + for sock in sockets: + sock.close() + + +def container_matches(actual: dict, expected: dict, label: str) -> bool: + return ( + actual.get("Id") == expected["id"] + and actual.get("Created") == expected["created"] + and actual.get("Config", {}).get("Labels", {}).get("loom.ci.trigger") == label + ) + + +def native_cleanup_complete(folder: Path, fixture_id: str) -> None: + fixture = folder / "fixture" + if not fixture.exists(): + return # Native validation never began; only the creator's stores exist. + receipt = fixture / "outer-result.json" + require( + not receipt.is_symlink() + and receipt.stat().st_uid == os.geteuid() + and receipt.stat().st_size <= 65536, + "Private native cleanup receipt", + ) + value = json.loads(receipt.read_text()) + require( + value.get("schema") == 2 + and value.get("fixture_id") == fixture_id + and value.get("native_cleanup_complete") is True, + "Native retirement, scratch cleanup or child reap remains incomplete", + ) + + +class Stores: + def __init__(self, root: Path, folder: Path, fixture_id: str, run: dict) -> None: + self.root, self.folder, self.fixture_id, self.run = root, folder, fixture_id, run + self.items: list[dict] = [] + self.network = None + self.prefix = "loom-trigger-" + fixture_id[:12] + self.cancelled = 0 + self.terminal = False + self.creation_deadline = time.monotonic() + 900 + self.env = { + "PATH": os.environ["PATH"], + "HOME": str(folder), + "LANG": "C.UTF-8", + "DOCKER_CONFIG": str(folder / "docker"), + } + + def save(self) -> None: + write( + self.folder / "stores.json", + { + "fixture_id": self.fixture_id, + "ci_run": self.run, + "items": self.items, + "network": self.network, + }, + replace=True, + ) + + def interrupted(self, signum: int, _frame: object) -> None: + self.cancelled = self.cancelled or signum + if self.terminal: + raise SystemExit(128 + self.cancelled) + + def command(self, *argv: str, timeout: int = 120, cleanup: bool = False) -> str: + if not cleanup: + require(not self.cancelled, "Acceptance creation cancelled") + resources(self.folder, 4) + remaining = self.creation_deadline - time.monotonic() + require(remaining > 0, "Owned store creation deadline") + timeout = min(timeout, max(1, int(remaining))) + result = subprocess.run(argv, env=self.env, capture_output=True, text=True, timeout=timeout) + require(len(result.stdout) + len(result.stderr) <= 8 * 1024**2, "Creator output bound") + require(result.returncode == 0, "Owned store command failed: " + argv[0]) + return result.stdout.strip() + + def inspect(self, identity: str, *, cleanup: bool = False) -> dict: + rows = json.loads(self.command("docker", "inspect", identity, cleanup=cleanup)) + require(len(rows) == 1, "Exactly one owned container") + return rows[0] + + def create(self, role: str, args: list[str]) -> dict: + name = self.prefix + "-" + role + cidfile = self.folder / (role + ".cid") + # The intent precedes Docker. A failed create is resolved by its unique label. + intent = {"role": role, "name": name, "id": None, "created": None} + self.items.append(intent) + self.save() + self.command( + "docker", + "create", + "--name", + name, + "--cidfile", + str(cidfile), + "--label", + "loom.ci.trigger=" + self.fixture_id, + "--label", + "loom.ci.run=" + self.run["run_id"], + *args, + ) + identity = cidfile.read_text().strip() + require(re.fullmatch(r"[a-f0-9]{64}", identity), "Actual created container ID") + actual = self.inspect(identity) + require( + actual["Name"] == "/" + name + and actual["Config"]["Labels"]["loom.ci.trigger"] == self.fixture_id, + "Created container identity", + ) + intent.update(id=identity, created=actual["Created"]) + self.save() + write( + self.folder / (role + "-creator.json"), + { + "task": "cell-trigger-acceptance", + "ci_run": self.run, + "container_id": identity, + "created_at": actual["Created"], + }, + ) + resources(self.folder) + self.command("docker", "start", identity) + return intent + + def wait(self, probe, seconds: int = 120) -> None: + end = time.monotonic() + seconds + while True: + require(not self.cancelled, "Owned store readiness cancelled") + try: + if probe(): + return + except (RuntimeError, subprocess.SubprocessError, ValueError): + pass + require(time.monotonic() < end, "Owned store readiness deadline") + time.sleep(1) + + def start(self, ch_image: str) -> tuple[dict, dict, dict, list[int]]: + selected = ports(8) + ch_port, native_port, pg_port, standby_port = selected[:4] + self.command("docker", "pull", ch_image, timeout=300) + self.command("docker", "pull", "postgres:16", timeout=300) + network_name = self.prefix + "-net" + self.network = {"name": network_name, "id": None} + self.save() + self.network["id"] = self.command( + "docker", + "network", + "create", + "--label", + "loom.ci.trigger=" + self.fixture_id, + network_name, + ) + self.save() + memory_config = self.folder / "memory.xml" + memory_config.write_text( + "4831838208\n" + ) + memory_config.chmod(0o644) + ch_data = self.folder / "clickhouse-data" + ch_data.mkdir(mode=0o777) + ch_data.chmod(0o777) + mounts = [ + (ch_data, "/var/lib/clickhouse", False), + (self.root / "deploy/lemmagen3", "/var/lib/clickhouse/lemmagen3", True), + (self.folder / "memory.xml", "/etc/clickhouse-server/config.d/memory.xml", True), + ] + for filename, section in ( + ("lemmatizers.xml", "config.d"), + ("nlp_user_settings.xml", "users.d"), + ("keeper.xml", "config.d"), + ("access_control.xml", "config.d"), + ): + mounts.append( + ( + self.root / "deploy/clickhouse-config" / filename, + "/etc/clickhouse-server/" + section + "/" + filename, + True, + ) + ) + for filename in ("log_queries_readonly.xml", "custom_settings_prefixes.xml"): + mounts.append( + ( + self.root / "dev/clickhouse-ci" / filename, + "/etc/clickhouse-server/users.d/" + filename, + True, + ) + ) + ch_args = [ + "--memory", + str(5 * GIB), + "--memory-swap", + str(5 * GIB), + "-p", + f"127.0.0.1:{ch_port}:8123", + "-p", + f"127.0.0.1:{native_port}:9000", + "-e", + "CLICKHOUSE_USER=loom", + "-e", + "CLICKHOUSE_PASSWORD=loomdev", + "-e", + "CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1", + "--ulimit", + "nofile=262144:262144", + ] + for host, target, readonly in mounts: + ch_args += ["-v", str(host) + ":" + target + (":ro" if readonly else "")] + ch = self.create("clickhouse", [*ch_args, ch_image]) + pg_data = self.folder / "pg-data" + pg_data.mkdir() + pg = self.create( + "pg-primary", + [ + "--network", + network_name, + "--network-alias", + "primary", + "--memory", + str(GIB), + "--memory-swap", + str(GIB), + "-p", + f"127.0.0.1:{pg_port}:5432", + "-v", + str(pg_data) + ":/var/lib/postgresql/data", + "-e", + "POSTGRES_USER=loom", + "-e", + "POSTGRES_PASSWORD=loomdev", + "-e", + "POSTGRES_DB=loom", + "postgres:16", + "-c", + "wal_level=logical", + "-c", + "max_wal_senders=10", + "-c", + "max_replication_slots=10", + "-c", + "synchronous_commit=on", + "-c", + "shared_buffers=128MB", + ], + ) + psql = ["docker", "exec", pg["id"], "psql", "-U", "loom", "-d", "loom", "-Atc"] + self.wait(lambda: self.command(*psql, "SELECT 1") == "1") + self.command(*psql, "CREATE ROLE replicator WITH REPLICATION LOGIN PASSWORD 'replicator'") + self.command( + "docker", + "exec", + pg["id"], + "bash", + "-c", + "echo 'host replication replicator all scram-sha-256' >> \"$PGDATA/pg_hba.conf\"", + ) + self.command(*psql, "ALTER SYSTEM SET synchronous_standby_names = 'FIRST 1 (loom_standby)'") + self.command(*psql, "SELECT pg_reload_conf()") + standby_data = self.folder / "standby-data" + standby_data.mkdir(mode=0o777) + standby_data.chmod(0o777) + standby_script = self.folder / "standby.sh" + standby_script.write_text("""#!/usr/bin/env bash +set -euo pipefail +export PGDATA=/var/lib/postgresql/data +if [ "$(id -u)" = 0 ]; then + chown postgres:postgres "$PGDATA" + exec gosu postgres bash "$0" +fi +pg_basebackup -h primary -U replicator -D "$PGDATA" -R -X stream +connection="host=primary user=replicator password=replicator" +echo "primary_conninfo = '$connection application_name=loom_standby'" \ + >> "$PGDATA/postgresql.auto.conf" +chmod 700 "$PGDATA" +exec postgres -D "$PGDATA" -c shared_buffers=128MB +""") + standby_script.chmod(0o644) + standby = self.create( + "pg-standby", + [ + "--network", + network_name, + "--memory", + str(GIB), + "--memory-swap", + str(GIB), + "-p", + f"127.0.0.1:{standby_port}:5432", + "-v", + str(standby_data) + ":/var/lib/postgresql/data", + "-e", + "PGPASSWORD=replicator", + "-v", + str(standby_script) + ":/standby.sh:ro", + "--entrypoint", + "bash", + "postgres:16", + "/standby.sh", + ], + ) + self.wait( + lambda: ( + self.command( + *psql, + ( + "SELECT sync_state FROM pg_stat_replication WHERE " + "application_name='loom_standby'" + ), + ) + == "sync" + ) + ) + self.wait( + lambda: ( + self.command( + "docker", + "exec", + ch["id"], + "clickhouse-client", + "--user", + "loom", + "--password", + "loomdev", + "--query", + "SELECT 1", + ) + == "1" + ) + ) + return ch, pg, standby, selected + + def observation(self, item: dict, role: str) -> dict: + actual = self.inspect(item["id"]) + require( + container_matches(actual, item, self.fixture_id) and actual["State"]["Running"], + "Actual job-owned running store", + ) + pid = actual["State"]["Pid"] + fields = Path("/proc", str(pid), "stat").read_text().rsplit(")", 1)[1].split() + creator = self.folder / (item["role"] + "-creator.json") + mappings = [ + { + "host": value["HostIp"], + "host_port": int(value["HostPort"]), + "internal_port": int(key.split("/")[0]), + "protocol": key.split("/")[1], + } + for key, values in actual["NetworkSettings"]["Ports"].items() + for value in values or [] + ] + disk_path = ( + self.folder + / { + "clickhouse": "clickhouse-data", + "pg_primary": "pg-data", + "pg_standby": "standby-data", + }[role] + ) + return { + "role": role, + "container_id": item["id"], + "container_name": actual["Name"].lstrip("/"), + "created_at": actual["Created"], + "owner_evidence": {"path": str(creator), "sha256": sha(creator)}, + "host_pid": pid, + "start_ticks": fields[19], + "boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + "uid": Path("/proc", str(pid)).stat().st_uid, + "ports": mappings, + "memory_limit_bytes": actual["HostConfig"]["Memory"], + "server_limit_bytes": 9 * GIB // 2 if role == "clickhouse" else 0, + "shared_buffers_bytes": 128 * 1024**2 if role != "clickhouse" else 0, + "retained": False, + "disk_path": str(disk_path), + } + + def cleanup(self) -> list[str]: + errors = [] + for item in reversed(self.items): + try: + if item["id"] is not None and not self.command( + "docker", + "ps", + "-a", + "--no-trunc", + "--filter", + "id=" + item["id"], + "--format", + "{{.ID}}", + cleanup=True, + ): + item["removed"] = True + self.save() + continue + if item.get("removed"): + require( + not self.command( + "docker", + "ps", + "-a", + "--no-trunc", + "--filter", + "id=" + item["id"], + "--format", + "{{.ID}}", + cleanup=True, + ), + "Previously removed container reappeared", + ) + continue + # Unique name plus exact random label resolves a partial create only. + actual = self.inspect(item["id"] or item["name"], cleanup=True) + if item["id"] is None: + require( + actual["Name"] == "/" + item["name"] + and actual["Config"]["Labels"].get("loom.ci.trigger") == self.fixture_id, + "Partial create belongs to this job", + ) + item.update(id=actual["Id"], created=actual["Created"]) + require( + container_matches(actual, item, self.fixture_id), "Cleanup ownership changed" + ) + self.command("docker", "rm", "--force", "--volumes", item["id"], cleanup=True) + remaining = self.command( + "docker", + "ps", + "-a", + "--no-trunc", + "--filter", + "id=" + item["id"], + "--format", + "{{.ID}}", + cleanup=True, + ) + require(not remaining, "Owned container remains after cleanup") + item["removed"] = True + self.save() + except ( + OSError, + RuntimeError, + subprocess.SubprocessError, + ValueError, + KeyError, + ) as error: + errors.append(type(error).__name__ + ":" + item["role"]) + if self.network is not None: + try: + if self.network["id"] is not None and not self.command( + "docker", + "network", + "ls", + "--no-trunc", + "--filter", + "id=" + self.network["id"], + "--format", + "{{.ID}}", + cleanup=True, + ): + self.network["removed"] = True + self.save() + if self.network.get("removed"): + require( + not self.command( + "docker", + "network", + "ls", + "--no-trunc", + "--filter", + "id=" + self.network["id"], + "--format", + "{{.ID}}", + cleanup=True, + ), + "Previously removed network reappeared", + ) + else: + rows = json.loads( + self.command( + "docker", + "network", + "inspect", + self.network["id"] or self.network["name"], + cleanup=True, + ) + ) + require( + len(rows) == 1 + and rows[0]["Labels"].get("loom.ci.trigger") == self.fixture_id + and (self.network["id"] is None or rows[0]["Id"] == self.network["id"]), + "Network cleanup ownership changed", + ) + self.network["id"] = rows[0]["Id"] + self.command("docker", "network", "rm", rows[0]["Id"], cleanup=True) + self.network["removed"] = True + self.save() + except ( + OSError, + RuntimeError, + subprocess.SubprocessError, + ValueError, + KeyError, + ) as error: + errors.append(type(error).__name__ + ":network") + write( + self.folder / "store-cleanup.json", + { + "removed": not errors, + "errors": errors, + "containers": [{"id": item["id"], "role": item["role"]} for item in self.items], + }, + replace=True, + ) + return errors + + +def dependency_receipt(root: Path, output: Path) -> dict: + # The CI job uses the same ARM64 CPython ABI as the reviewed codec contract. + sys.path.insert(0, str(SUPPORT)) + from fixture import CODEC_FILES, CODEC_VERSIONS + + value = { + "schema": 2, + "root": str(root), + "python": str(root / ".venv/bin/python"), + "python_sha256": sha(root / ".venv/bin/python"), + "lockfile_sha256": sha(root / "uv.lock"), + "versions": CODEC_VERSIONS, + "codec_sources": { + name: {"path": str(root / ".venv" / name), "sha256": sha(root / ".venv" / name)} + for name in CODEC_FILES + }, + } + write(output, value) + return {"path": str(output), "sha256": sha(output)} + + +def publish(folder: Path, destination: Path, error: BaseException | None) -> None: + destination.mkdir(mode=0o700) + # Raw native logs/settings contain bearer and bootstrap credentials: never upload them. + fixture = folder / "fixture" + for name in ( + "outer-result.json", + "driver.jsonl", + "provenance.json", + "final-retirement.json", + "teardown.json", + ): + path = fixture / name + if path.is_file(): + shutil.copyfile(path, destination / name) + for name in ("source-before.json", "source-after.json"): + if (folder / name).is_file(): + shutil.copyfile(folder / name, destination / name) + intent_output = destination / "delivery-intent" + for arm in range(10): + for suffix in (".jsonl", "-receipt.json"): + path = fixture / "delivery-intent" / (f"arm-{arm}" + suffix) + if path.is_file() and not path.is_symlink(): + intent_output.mkdir(mode=0o700, exist_ok=True) + shutil.copyfile(path, intent_output / path.name) + if (folder / "store-cleanup.json").exists(): + shutil.copyfile(folder / "store-cleanup.json", destination / "store-cleanup.json") + write( + destination / "result.json", + { + "accepted": error is None, + "error_type": type(error).__name__ if error is not None else "", + "failure": failure_site(error), + "utc": datetime.datetime.now(datetime.UTC).isoformat(), + }, + ) + + +def cleanup_only(root: Path, run: dict) -> int: + state = Path(os.environ["RUNNER_TEMP"]) / "cell-trigger-state.json" + if not state.exists(): + return 0 # The creator never published an intent and could not create a store. + require( + not state.is_symlink() + and state.stat().st_uid == os.geteuid() + and state.stat().st_size < 65536, + "Owned CI cleanup pointer", + ) + pointer = json.loads(state.read_text()) + fixture_id = pointer["fixture_id"] + require(str(uuid.UUID(fixture_id)) == fixture_id, "Cleanup fixture UUID") + folder = Path(os.environ["RUNNER_TEMP"]) / ("cell-trigger-" + fixture_id) + require( + folder.resolve() == folder + and pointer == {"fixture_id": fixture_id, "folder": str(folder), "ci_run": run}, + "Cleanup pointer belongs to this exact CI attempt", + ) + ledger = folder / "stores.json" + require( + not ledger.is_symlink() + and ledger.stat().st_uid == os.geteuid() + and ledger.stat().st_size < 65536, + "Owned creator ledger", + ) + data = json.loads(ledger.read_text()) + require(data["fixture_id"] == fixture_id and data["ci_run"] == run, "Current creator run") + stores = Stores(root, folder, fixture_id, run) + roles = {"clickhouse", "pg-primary", "pg-standby"} + require( + len(data["items"]) <= 3 + and len({item["role"] for item in data["items"]}) == len(data["items"]), + "Bounded exact creator roles", + ) + for item in data["items"]: + require( + item["role"] in roles + and item["name"] == stores.prefix + "-" + item["role"] + and (item["id"] is None or re.fullmatch(r"[a-f0-9]{64}", item["id"])), + "Exact recorded container identity", + ) + network = data["network"] + require(network is None or network["name"] == stores.prefix + "-net", "Exact recorded network") + stores.items, stores.network = data["items"], network + native_cleanup_complete(folder, fixture_id) + errors = stores.cleanup() + destination = Path(os.environ["RUNNER_TEMP"]) / "cell-trigger-artifacts" + destination.mkdir(mode=0o700, exist_ok=True) + shutil.copyfile(folder / "store-cleanup.json", destination / "store-cleanup.json") + return int(bool(errors)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--clickhouse-image") + parser.add_argument("--cleanup-only", action="store_true") + args = parser.parse_args() + require( + os.environ.get("GITHUB_ACTIONS") == os.environ.get("CI") == "true", "GitHub Actions only" + ) + root = Path(os.environ["GITHUB_WORKSPACE"]).resolve() + run = { + "run_id": os.environ["GITHUB_RUN_ID"], + "attempt": os.environ["GITHUB_RUN_ATTEMPT"], + "repository": os.environ["GITHUB_REPOSITORY"], + } + if args.cleanup_only: + return cleanup_only(root, run) + require(args.clickhouse_image, "Resolved ClickHouse release image") + require( + Path(sys.executable) == root / ".venv/bin/python" and sys.version_info[:2] == (3, 12), + "Use the locked checkout Python 3.12", + ) + before = source(root) + resources(Path(os.environ["RUNNER_TEMP"])) + fixture_id = str(uuid.uuid4()) + folder = Path(os.environ["RUNNER_TEMP"]) / ("cell-trigger-" + fixture_id) + folder.mkdir(mode=0o700) + os.umask(0o077) + hashes_before = source_hashes(root) + write(folder / "source-before.json", {**before, "source_hashes": hashes_before, "ci_run": run}) + stores = Stores(root, folder, fixture_id, run) + stores.save() + write( + Path(os.environ["RUNNER_TEMP"]) / "cell-trigger-state.json", + {"fixture_id": fixture_id, "folder": str(folder), "ci_run": run}, + ) + for signum in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP): + signal.signal(signum, stores.interrupted) + error = None + cleanup_complete = False + try: + ch, primary, standby, selected = stores.start(args.clickhouse_image) + resources(folder) + dependencies = dependency_receipt(root, folder / "dependencies.json") + from fixture import runtime + + spec = { + "schema": 2, + "fixture_id": fixture_id, + "root": str(root), + **before, + "python": str(root / ".venv/bin/python"), + "python_sha256": sha(root / ".venv/bin/python"), + "lockfile_sha256": sha(root / "uv.lock"), + "helper_path": str(SUPPORT / "owned_children.py"), + "source_hashes": hashes_before, + "workdir": str(folder / "fixture"), + "ports": dict(zip(("api", "router", "probe", "tls"), selected[4:], strict=True)), + "databases": { + name: "trigger_" + fixture_id.replace("-", "") + "_" + str(index) + for index, name in enumerate( + ("pg_cell", "pg_directory", "ch_schema", "ch_telemetry") + ) + }, + "stores": { + role: stores.observation(item, role) + for item, role in ( + (ch, "clickhouse"), + (primary, "pg_primary"), + (standby, "pg_standby"), + ) + }, + "pg_admin_dsn": f"postgresql://loom:loomdev@127.0.0.1:{selected[2]}/postgres", + "pg_standby_dsn": f"postgresql://loom:loomdev@127.0.0.1:{selected[3]}/postgres", + "ch": { + "host": "127.0.0.1", + "port": selected[0], + "username": "loom", + "password": "loomdev", + "keeper_prefix": "/keeper_map_tables", + }, + "dependency_receipt": dependencies, + "ci_run": run, + } + write(folder / "spec.json", spec) + require(not stores.cancelled, "Cancelled before native acceptance") + require(runtime(spec) == 0, "Trigger acceptance did not pass") + require(source(root) == before, "Acceptance source changed") + except BaseException as caught: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + error = caught + finally: + # Native ownership closes first; exact created containers are removed even on failure. + for signum in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP): + signal.signal(signum, stores.interrupted) + try: + native_cleanup_complete(folder, fixture_id) + cleanup_errors = stores.cleanup() + cleanup_complete = not cleanup_errors + if cleanup_errors and error is None: + error = RuntimeError("Owned store cleanup incomplete") + except BaseException as caught: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + error = caught if error is None else error + if stores.cancelled and error is None: + error = InterruptedError("Acceptance interrupted") + try: + require(source(root) == before, "Final acceptance source changed") + hashes_after = source_hashes(root) + write( + folder / "source-after.json", + {**before, "source_hashes": hashes_after, "ci_run": run}, + ) + require( + hashes_after == hashes_before, "Final driver, support or workflow bytes changed" + ) + except BaseException as caught: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + error = caught if error is None else error + try: + publish(folder, Path(os.environ["RUNNER_TEMP"]) / "cell-trigger-artifacts", error) + except BaseException as caught: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + error = caught if error is None else error + stores.terminal = cleanup_complete + if stores.cancelled and error is None: + error = InterruptedError("Acceptance interrupted during publication") + print( + json.dumps( + { + "accepted": error is None, + "error_type": type(error).__name__ if error is not None else "", + "failure": failure_site(error), + } + ) + ) + return int(error is not None or bool(stores.cancelled)) + + +if __name__ == "__main__": + try: + code = main() + except BaseException as error: # noqa: BLE001 — preflight failures must emit only nonsecret diagnostics. + result = { + "accepted": False, + "error_type": type(error).__name__, + "failure": failure_site(error), + } + print(json.dumps(result), flush=True) + if os.environ.get("GITHUB_ACTIONS") == "true" and os.environ.get("RUNNER_TEMP"): + output = Path(os.environ["RUNNER_TEMP"]) / "cell-trigger-artifacts" + output.mkdir(mode=0o700, exist_ok=True) + if not (output / "result.json").exists(): + write(output / "result.json", result) + code = 1 + raise SystemExit(code) diff --git a/scripts/gates/testsupport/cell_trigger/__init__.py b/scripts/gates/testsupport/cell_trigger/__init__.py new file mode 100644 index 0000000000..e5dad64539 --- /dev/null +++ b/scripts/gates/testsupport/cell_trigger/__init__.py @@ -0,0 +1 @@ +"""Isolated native trigger acceptance support.""" diff --git a/scripts/gates/testsupport/cell_trigger/capture.py b/scripts/gates/testsupport/cell_trigger/capture.py new file mode 100644 index 0000000000..1d398f00da --- /dev/null +++ b/scripts/gates/testsupport/cell_trigger/capture.py @@ -0,0 +1,1024 @@ +"""Capture unchanged delivery intent and independently verify persisted rows.""" + +from __future__ import annotations + +import argparse +import datetime +import hashlib +import importlib.util +import json +import math +import os +import re +import socket +import struct +import sys +import uuid +from pathlib import Path + +from fixture import ( + FILES, + HERE, + PROTOCOL, + PROTOCOL_SHA, + SPEC_FIELDS, + digest, + json_object, + need, + private_stream, + read, + ref, + sha, + source, + validate, + write, +) + +LINE = 65536 +ARM_BYTES = 32 * 1024 * 1024 +TOTAL_BYTES = 320 * 1024 * 1024 +ORDER = (True, False, False, True, True, False, False, True, True, False) +SUBJECT = "process:benchmark" + + +def protocol() -> dict: + return read(HERE / PROTOCOL, expected_sha256=PROTOCOL_SHA) + + +def integer(value, low=0, high=2**32 - 1): + need(type(value) is int and low <= value <= high, "typed bounded integer") + return value + + +def identifier(value): + if isinstance(value, uuid.UUID): + value = str(value) + need( + type(value) is str and str(uuid.UUID(value)) == value and uuid.UUID(value).int != 0, + "canonical UUID", + ) + return value + + +def equal(left, right): + need( + json.dumps(left, sort_keys=True, allow_nan=False, separators=(",", ":")) + == json.dumps(right, sort_keys=True, allow_nan=False, separators=(",", ":")), + "exact typed equality", + ) + + +def instant(value, *, audit=False): + if isinstance(value, str): + need( + re.fullmatch( + r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d{1,6})?(?:Z|[+-]\d\d:\d\d)?", value + ), + "exact ISO datetime", + ) + value = datetime.datetime.fromisoformat(value) + need(type(value) is datetime.datetime, "typed datetime") + value = ( + value.replace(tzinfo=datetime.UTC) + if value.tzinfo is None + else value.astimezone(datetime.UTC) + ) + if audit: + need(value.microsecond % 1000 == 0, "audit millisecond precision") + return value.isoformat(timespec="microseconds") + need(value.microsecond == 0 and 0 <= value.timestamp() < 2**32, "memory second precision/range") + return value.replace(tzinfo=None).isoformat(timespec="seconds") + + +def hex64(value, *, native=False): + if native and isinstance(value, (bytes, bytearray)): + value = bytes(value).decode("ascii") + need(type(value) is str and re.fullmatch(r"[0-9a-f]{64}", value), "typed hex64") + return value + + +def payload(key): + prefix = f"Cell trigger measurement {key}: " + need( + type(key) is str and key.isascii() and len(prefix) < 4096 and len(key) <= 128, "payload key" + ) + return prefix + ("abcdefghijklmnopqrstuvwxyz " * 4096)[: 4096 - len(prefix)] + + +def content_hash(key): + return hashlib.sha256( + json.dumps(payload(key), sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + + +class Codec: + def __init__(self, specification): + self.p = specification + need( + len(self.p["memory_fields"]) == 61 + and len(set(self.p["memory_fields"])) == 61 + and set(self.p["memory_types"]) == set(self.p["memory_fields"]) + and len(self.p["sparse_defaults"]) == 43 + and len(self.p["audit_fields"]) == 13, + "closed protocol", + ) + + def row(self, original, *, origin="capture", comparison=False): + need( + type(original) is dict and set(original) <= set(self.p["memory_fields"]), + "memory row fields", + ) + value = dict(original) + if origin == "pg": + for key, default in self.p["sparse_defaults"].items(): + if key not in value: + value[key] = list(default) if isinstance(default, list) else default + need(set(value) == set(self.p["memory_fields"]), "complete memory row") + result = {} + for key in self.p["memory_fields"]: + item = value[key] + kind = self.p["memory_types"][key] + if kind.startswith("nullable_") and item is None: + result[key] = None + continue + kind = kind.removeprefix("nullable_") + if kind == "string": + need(type(item) is str and len(item.encode()) <= 8192, "memory string") + elif kind == "json_object_field": + if origin == "ch": + need(type(item) is dict, "native CH JSON object") + parsed = json_object(json.dumps(item, allow_nan=False)) + else: + need(type(item) is str, "encoded object JSON") + parsed = json_object(item) + need(parsed == {}, "primitive workload JSON object") + if comparison: + item = parsed + elif kind == "utc_second": + item = instant(item) + elif kind == "uuid": + item = identifier(item) + elif kind in {"array_uuid", "array_string"}: + need(type(item) is list and len(item) <= 64, "typed array") + if kind == "array_uuid": + item = [identifier(member) for member in item] + else: + need( + all(type(member) is str and len(member.encode()) <= 512 for member in item), + "string array", + ) + elif kind == "hex64_fixedstring": + item = hex64(item, native=origin == "ch") + elif kind == "finite_exact_float32_not_bool": + need(type(item) in {int, float} and math.isfinite(item), "finite Float32") + rounded = struct.unpack("!f", struct.pack("!f", item))[0] + need(rounded == item, "exact stored Float32") + item = rounded + elif kind == "positive_int63_not_bool": + item = integer(item, 1, 2**63 - 1) + elif kind == "uint8_0_or_1_not_bool": + item = integer(item, 0, 1) + elif kind == "uint32_not_bool": + item = integer(item) + else: + need(False, "unknown protocol type") + result[key] = item + return result + + def audit(self, value, *, origin="capture"): + need( + type(value) is dict and set(value) == set(self.p["audit_fields"]), + "complete audit fields", + ) + result = {} + for key in self.p["audit_fields"]: + item = value[key] + if key == "ts": + item = instant(item, audit=True) + elif key == "row_hash": + item = hex64(item, native=origin == "ch") + else: + need(type(item) is str and len(item.encode()) <= 8192, "audit string") + result[key] = item + timestamp = ( + datetime.datetime.fromisoformat(result["ts"]) + .astimezone(datetime.UTC) + .replace(tzinfo=None) + .isoformat() + ) + canonical = { + key: result[key] + for key in ("event_type", "org", "namespace", "actor", "subject", "request_id") + } + canonical["ts"] = timestamp + actual = hashlib.sha256( + json.dumps(canonical, sort_keys=True, ensure_ascii=True).encode() + ).hexdigest() + need(actual == result["row_hash"], "canonical raw-subject audit hash") + return result + + def expected(self, row, audit, context, *, api, metadata, driver): + key = row["memory_key"] + equal( + [ + row[k] + for k in ( + "org", + "namespace", + "agent", + "user_id", + "session_id", + "actor", + "source_class", + ) + ], + [context["org"], context["namespace"], "bench", "-", "-", SUBJECT, "asserted"], + ) + need( + row["memory_type"] == "semantic" and row["status"] == "active" and row["pinned"] == 0, + "workload semantic/active", + ) + need(row["confidence"] == 1.0, "workload confidence") + for field, default in self.p["sparse_defaults"].items(): + equal(row[field], default) + need( + row["value"] == json.dumps(payload(key)) and row["content_sha256"] == content_hash(key), + "exact stored payload/hash", + ) + expected_subject = dict( + memory_key=key, + memory_id=row["memory_id"], + version=row["version"], + content_sha256=row["content_sha256"], + source_class="asserted", + status="active", + pinned=False, + tombstoned=False, + ) + equal(json_object(audit["subject"]), expected_subject) + need( + audit["subject"] == json.dumps(expected_subject, sort_keys=True), + "raw native subject spelling", + ) + need( + audit["event_type"] == "memory.revision" + and audit["org"] == context["org"] + and audit["namespace"] == context["namespace"] + and audit["actor"] == SUBJECT, + "audit event/scope/actor", + ) + api_launch = context["_api_server"] if api else None + instance = ( + api_launch["health_instance"] if api else driver["host"] + ":" + str(driver["pid"]) + ) + equal( + { + k: audit[k] + for k in ("build_commit", "instance", "instance_label", "consumer", "token_id") + }, + dict( + build_commit=context["head"], + instance=instance, + instance_label=("iso:trigger:" + context["fixture_id"][:12])[:64], + consumer="service-account" if api else "", + token_id=metadata["summary"]["jti"] if api else "", + ), + ) + need( + ( + type(audit["request_id"]) is str + and audit["request_id"].isascii() + and 0 < len(audit["request_id"]) <= 128 + ) + if api + else audit["request_id"] == "", + "audit request identity", + ) + + +def birth(): + pid = os.getpid() + return dict( + pid=pid, + start_ticks=Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()[19], + boot_id=Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + uid=os.geteuid(), + host=socket.gethostname(), + environment_sha256=digest(dict(os.environ)), + ) + + +def line(value): + raw = ( + json.dumps(value, sort_keys=True, allow_nan=False, separators=(",", ":")) + "\n" + ).encode() + need(len(raw) <= LINE, "JSONL line bound") + return raw + + +class Capture: + def __init__(self, context, directory, driver_birth, codec): + self.s = context + self.directory = directory + self.birth = driver_birth + self.codec = codec + self.calls = 0 + self.total = 0 + need( + directory == Path(context["workdir"]) / "delivery-intent" and not directory.exists(), + "new exact intent directory", + ) + directory.mkdir(mode=0o700) + + def header(self, arm): + w = Path(self.s["workdir"]) + return dict( + schema=1, + kind="header", + fixture_id=self.s["fixture_id"], + arm=arm, + on=ORDER[arm], + batch_rows=500, + expected_rows=2500, + head=self.s["head"], + tree=self.s["tree"], + driver_sha256=self.s["source_hashes"]["driver"], + capture_sha256=self.s["source_hashes"][FILES[3]], + protocol_sha256=PROTOCOL_SHA, + context_sha256=self.s["capture_context_sha256"], + fixture_sha256=sha(w / "fixture.json"), + launch_sha256=sha(w / "serving-cell/launch.json"), + mint_metadata_sha256=self.s["mint_metadata"]["sha256"], + birth=self.birth, + ) + + def delegate(self, original): + async def build(args, batch): + arm = self.calls + need(arm < 10 and type(batch) is int and batch == 500, "exact build count/batch") + self.calls += 1 + block = await original(args, batch) + need(len(block.rows) == len(block.audits) == 2500, "exact delivery intent count") + path = self.directory / f"arm-{arm}.jsonl" + full = hashlib.sha256() + prefix = hashlib.sha256() + size = 0 + header = self.header(arm) + with private_stream(path, output=True) as stream: + for index in range(-1, 2501): + if index == -1: + value = header + elif index == 2500: + value = dict( + schema=1, + kind="trailer", + arm=arm, + count=2500, + prefix_sha256=prefix.hexdigest(), + ) + else: + value = dict( + schema=1, + kind="delivery", + arm=arm, + index=index, + row=self.codec.row(block.rows[index]), + audit=self.codec.audit(block.audits[index]), + ) + raw = line(value) + size += len(raw) + self.total += len(raw) + need(size <= ARM_BYTES and self.total <= TOTAL_BYTES, "intent byte bounds") + need(stream.write(raw) == len(raw), "complete intent write") + full.update(raw) + if index < 2500: + prefix.update(raw) + write( + self.directory / f"arm-{arm}-receipt.json", + dict( + schema=1, + arm=arm, + bytes=size, + records=2502, + sha256=full.hexdigest(), + header=header, + ), + ) + return block + + return build + + +def mint_metadata(context, minted): + w = Path(context["workdir"]) + with private_stream(w / "mint-summary.stdout") as stream: + raw = stream.read(4097) + need(len(raw) <= 4096 and raw.endswith(b"\n") and raw.count(b"\n") == 1, "mint stdout shape") + summary = json_object(raw) + need(set(summary) == set(protocol()["mint_summary_fields"]), "mint summary fields") + need(type(summary["jti"]) is str and re.fullmatch(r"[a-f0-9]{32}", summary["jti"]), "mint jti") + equal( + {k: v for k, v in summary.items() if k != "jti"}, + dict( + subject="benchmark", + org=context["org"], + namespaces=[context["namespace"]], + roles=["writer"], + ttl_days=1, + is_service_account=True, + read_scope="namespace", + read_user_id="", + traffic_class="", + consumer="", + token_path=str(w / "writer.jwt"), + ), + ) + with private_stream(w / "writer.jwt") as stream: + token = stream.read(16385) + need( + 0 < len(token) <= 16384 and token.endswith(b"\n") and token.count(b"\n") == 1, + "private native bearer file", + ) + born = ref(minted["launch_receipt"]) + terminal = ref(minted["terminal_receipt"]) + need( + type(terminal["returncode"]) is int and terminal["returncode"] == 0, "mint terminal success" + ) + equal(minted["source_sha256"], digest(context["source_hashes"])) + equal(minted["cwd"], context["root"]) + expected = [ + context["python"], + "-m", + "server.cli", + "mint-token", + "--subject", + "benchmark", + "--org", + context["org"], + "--namespaces", + context["namespace"], + "--roles", + "writer", + "--service-account", + "--ttl-days", + "1", + "--save", + str(w / "writer.jwt"), + "--json", + ] + equal(minted["redacted_argv"], expected) + for key in ("pid", "start_ticks", "boot_id", "uid", "configuration_sha256"): + equal(minted[key], born[key]) + retirement = read(w / "mint-proof.json") + equal(retirement["birth"], born) + need(retirement["state"] == "retired", "mint retirement receipt") + return dict( + schema=1, + summary=summary, + stdout=dict( + path=str(w / "mint-summary.stdout"), + sha256=hashlib.sha256(raw).hexdigest(), + bytes=len(raw), + ), + token=dict( + path=str(w / "writer.jwt"), sha256=hashlib.sha256(token).hexdigest(), bytes=len(token) + ), + process=minted, + retirement=dict(path=str(w / "mint-proof.json"), sha256=sha(w / "mint-proof.json")), + ), token.decode().strip() + + +def verify_token(metadata): + expected = metadata["token"] + with private_stream(Path(expected["path"])) as stream: + raw = stream.read(16385) + need( + len(raw) == expected["bytes"] and hashlib.sha256(raw).hexdigest() == expected["sha256"], + "same actual native bearer bytes", + ) + return raw.decode().strip() + + +def closed(value, fields): + need(type(value) is dict and set(value) == set(fields.split()), "closed artifact record") + + +def record_stream(path, *, maximum): + full = hashlib.sha256() + size = 0 + count = 0 + with private_stream(path) as stream: + while True: + raw = stream.readline(LINE + 1) + if not raw: + break + count += 1 + size += len(raw) + need( + len(raw) <= LINE and raw.endswith(b"\n") and size <= maximum and count <= 101000, + "bounded complete JSONL", + ) + full.update(raw) + yield json_object(raw), raw + return dict(path=str(path), sha256=full.hexdigest(), bytes=size, records=count) + + +class Workload: + def __init__(self, context, codec): + self.s = context + self.w = Path(context["workdir"]) + self.codec = codec + self.metadata = ref(context["mint_metadata"]) + verified, _ = mint_metadata(context, self.metadata["process"]) + equal(self.metadata, verified) + verify_token(self.metadata) + self.driver = read(self.w / "driver-birth.json") + self.driver_birth = ref(self.driver["launch_receipt"]) + self.finished = read(self.w / "driver-finished.json") + terminal = ref(self.finished["terminal_receipt"]) + need( + type(terminal["returncode"]) is int and terminal["returncode"] in (0, 1), + "completed actual driver status", + ) + for key in ( + "pid", + "start_ticks", + "boot_id", + "uid", + "configuration_sha256", + "launch_receipt", + ): + equal(self.driver[key], self.finished[key]) + for key in ("pid", "start_ticks", "boot_id", "uid"): + equal(self.driver[key], self.driver_birth[key]) + equal(self.driver["source_sha256"], digest(context["source_hashes"])) + equal(self.driver["cwd"], context["root"]) + self.launch = read(self.w / "serving-cell/launch.json") + self.configuration = ref(self.launch["configuration_artifact"]) + fixture = read(self.w / "fixture.json") + ownership = read( + self.w / "ownership.json", expected_sha256=fixture["ownership_receipt_sha256"] + ) + candidates = [ + item for item in ownership["processes"] if item["pid"] == self.launch["server"]["pid"] + ] + need(len(candidates) == 1, "one owner-recorded actual API birth") + actual = ref(candidates[0]["launch_receipt"]) + for key in ("pid", "start_ticks", "boot_id", "uid"): + equal(self.launch["server"][key], actual[key]) + equal(self.launch["source"], dict(head=context["head"], tree=context["tree"])) + equal(self.launch["server"]["instance_id"], context["api_instance_id"]) + equal(self.launch["server"]["cwd"], context["root"]) + equal( + self.launch["server"]["health_instance"], + socket.gethostname() + ":" + str(self.launch["server"]["pid"]), + ) + self.ids = set() + self.keys = set() + self.total = 0 + self.counts = {"api": 0, "delivery": 0} + self.refs = [ + dict(path=str(self.w / name), sha256=sha(self.w / name)) + for name in ( + "fixture.json", + "attached-context.json", + "ownership.json", + "serving-cell/launch.json", + "driver-birth.json", + "driver-finished.json", + ) + ] + self.refs.extend( + [ + self.launch["configuration_artifact"], + context["mint_metadata"], + self.metadata["stdout"], + self.metadata["token"], + self.metadata["process"]["launch_receipt"], + self.metadata["process"]["terminal_receipt"], + self.metadata["retirement"], + ] + ) + self.driver_header = None + self.iterator = None + + def __enter__(self): + self.iterator = self.inputs() + return self + + def __exit__(self, kind, error, traceback): + try: + if self.iterator is not None: + self.iterator.close() + except BaseException: + if error is None: + raise + return False + + def unique(self, item): + identifier(item["memory_id"]) + key = item["key"] + need( + item["memory_id"] not in self.ids and key not in self.keys, + "unique independent workload identities", + ) + self.ids.add(item["memory_id"]) + self.keys.add(key) + need(len(self.ids) <= 45500 and len(self.keys) <= 45500, "workload set bound") + self.counts[item["kind"]] += 1 + return item + + def delivery(self, arm): + path = self.w / "delivery-intent" / f"arm-{arm}.jsonl" + receipt = read(path.parent / f"arm-{arm}-receipt.json") + closed(receipt, "schema arm bytes records sha256 header") + equal([receipt[key] for key in ("schema", "arm", "records")], [1, arm, 2502]) + integer(receipt["bytes"], 1, ARM_BYTES) + hex64(receipt["sha256"]) + prefix = hashlib.sha256() + full = hashlib.sha256() + size = 0 + records = record_stream(path, maximum=ARM_BYTES) + try: + header, raw = next(records) + full.update(raw) + prefix.update(raw) + size += len(raw) + closed( + header, + ( + "schema kind fixture_id arm on batch_rows expected_rows head " + "tree driver_sha256 capture_sha256 protocol_sha256 " + "context_sha256 fixture_sha256 launch_sha256 " + "mint_metadata_sha256 birth" + ), + ) + equal(header, receipt["header"]) + for key, expected in dict( + schema=1, + kind="header", + fixture_id=self.s["fixture_id"], + arm=arm, + on=ORDER[arm], + batch_rows=500, + expected_rows=2500, + head=self.s["head"], + tree=self.s["tree"], + driver_sha256=self.s["source_hashes"]["driver"], + capture_sha256=self.s["source_hashes"][FILES[3]], + protocol_sha256=PROTOCOL_SHA, + context_sha256=sha(self.w / "attached-context.json"), + fixture_sha256=sha(self.w / "fixture.json"), + launch_sha256=sha(self.w / "serving-cell/launch.json"), + mint_metadata_sha256=self.s["mint_metadata"]["sha256"], + ).items(): + equal(header[key], expected) + born = header["birth"] + closed(born, "pid start_ticks boot_id uid host environment_sha256") + for key in ("pid", "start_ticks", "boot_id", "uid"): + equal(born[key], self.driver[key]) + need( + born["host"] == socket.gethostname() + and born["environment_sha256"] == self.driver["configuration_sha256"], + "actual driver capture birth/config", + ) + if self.driver_header is None: + self.driver_header = born + else: + equal(born, self.driver_header) + pattern = None + for index in range(2500): + value, raw = next(records) + size += len(raw) + full.update(raw) + prefix.update(raw) + closed(value, "schema kind arm index row audit") + equal( + [value[k] for k in ("schema", "kind", "arm", "index")], + [1, "delivery", arm, index], + ) + row = self.codec.row(value["row"]) + audit = self.codec.audit(value["audit"]) + match = re.fullmatch(r"(gate-delivery-[a-f0-9]{32})-(\d+)", row["memory_key"]) + need(match is not None and match[2] == str(index), "delivery exact key counter") + if pattern is None: + pattern = match[1] + need(pattern == match[1], "one original delivery prefix") + yield self.unique( + dict( + kind="delivery", + key=row["memory_key"], + memory_id=row["memory_id"], + version=row["version"], + content_sha256=row["content_sha256"], + row=row, + audit=audit, + ) + ) + trailer, raw = next(records) + size += len(raw) + full.update(raw) + equal( + trailer, + dict( + schema=1, kind="trailer", arm=arm, count=2500, prefix_sha256=prefix.hexdigest() + ), + ) + try: + next(records) + except StopIteration: + pass + else: + need(False, "extra intent record") + need( + receipt["bytes"] == size and receipt["sha256"] == full.hexdigest(), + "complete intent bytes", + ) + self.total += size + need(self.total <= TOTAL_BYTES, "intent aggregate") + self.refs.append(dict(path=str(path), sha256=full.hexdigest(), bytes=size)) + self.refs.append( + dict( + path=str(path.parent / f"arm-{arm}-receipt.json"), + sha256=sha(path.parent / f"arm-{arm}-receipt.json"), + ) + ) + finally: + pending = sys.exception() + try: + records.close() + except BaseException: + if pending is None: + raise + + def inputs(self): + records = record_stream(self.w / "driver.jsonl", maximum=64 * 1024 * 1024) + + def take(kind, fields=None): + value, _ = next(records) + need(value.get("kind") == kind, "original artifact event order") + if fields is not None: + closed(value, fields) + return value + + try: + config = take( + "configuration", + ( + "kind manifest shared_configuration_excludes " + "credential_posture api_configuration_attested source launch " + "manifest_sha256 calls warmup rounds batches batch_rows " + "logical_keys value_bytes order driver_sha256" + ), + ) + equal(config["manifest"], read(self.w / "fixture.json")) + need(config["manifest_sha256"] == digest(config["manifest"]), "artifact fixture digest") + equal(config["source"], dict(head=self.s["head"], tree=self.s["tree"], clean=True)) + equal( + config["launch"], + dict( + health_instance=self.launch["server"]["health_instance"], + fingerprints=self.configuration["fingerprints"], + launch_sha256=sha(self.w / "serving-cell/launch.json"), + configuration_artifact_sha256=self.launch["configuration_artifact"]["sha256"], + birth_sha256=digest(self.launch["server"]), + ), + ) + equal(config["credential_posture"], self.configuration["credential_posture"]) + for key in ( + "api_configuration_sha256", + "cli_configuration_sha256", + "shared_configuration_sha256", + ): + equal(config["manifest"][key], self.configuration[key]) + for key, expected in dict( + calls=2000, + warmup=50, + rounds=5, + batches=5, + batch_rows=500, + logical_keys=45500, + value_bytes=4096, + order=list(ORDER), + driver_sha256=self.s["source_hashes"]["driver"], + shared_configuration_excludes=["cell_router_token"], + api_configuration_attested=True, + ).items(): + equal(config[key], expected) + for arm in range(10): + proof = take( + "arm_proof", + ( + "kind arm on deployment_id epoch instance_id " + "fingerprint_sha256 api_configuration_sha256 " + "cli_configuration_sha256 shared_configuration_sha256 " + "shared_configuration_excludes credential_posture " + "api_configuration_attested source launch tables " + "invariant_sha256 org_triggers_sha256" + ), + ) + equal([proof["arm"], proof["on"]], [arm, ORDER[arm]]) + for key in ( + "source", + "launch", + "credential_posture", + "shared_configuration_excludes", + "api_configuration_attested", + ): + equal(proof[key], config[key]) + for key in ( + "deployment_id", + "epoch", + "instance_id", + "api_configuration_sha256", + "cli_configuration_sha256", + "shared_configuration_sha256", + ): + equal(proof[key], config["manifest"][key]) + pattern = None + for index in range(2050): + value = take( + "request", + ( + "kind arm index status key memory_id version content_sha256 " + "elapsed_ms measured admission" + ), + ) + equal( + [value[k] for k in ("arm", "index", "status", "measured", "admission")], + [arm, index, 200, index >= 50, "derived_ready"], + ) + need( + type(value["elapsed_ms"]) in {int, float} + and math.isfinite(value["elapsed_ms"]) + and value["elapsed_ms"] >= 0, + "finite recorded latency", + ) + integer(value["version"], 1, 2**63 - 1) + match = re.fullmatch( + r"(gate-[a-f0-9]{32}-" + str(arm) + r")-(\d+)", value["key"] + ) + need(match is not None and match[2] == str(index), "API exact key counter") + if pattern is None: + pattern = match[1] + need( + match[1] == pattern + and value["content_sha256"] == content_hash(value["key"]), + "API original prefix/hash", + ) + yield self.unique( + dict( + kind="api", + **{ + key: value[key] + for key in ("key", "memory_id", "version", "content_sha256") + }, + ) + ) + for index in range(5): + value = take( + "worker", + ( + "kind arm index claimed published failed expired_keys " + "expired_evidence swept_settled published_audits" + ), + ) + equal( + [value[k] for k in ("arm", "index", "claimed", "published", "failed")], + [arm, index, 500, 500, 0], + ) + for key in ( + "expired_keys", + "expired_evidence", + "swept_settled", + "published_audits", + ): + integer(value[key]) + value = take("delivery_block", "kind arm elapsed_seconds rows rps") + equal([value["arm"], value["rows"]], [arm, 2500]) + need( + type(value["elapsed_seconds"]) in {int, float} + and math.isfinite(value["elapsed_seconds"]) + and value["elapsed_seconds"] > 0 + and type(value["rps"]) in {int, float} + and math.isfinite(value["rps"]) + and value["rps"] > 0, + "finite worker record", + ) + value = take("settled", "kind arm api_rows delivery_rows physical") + equal([value["arm"], value["api_rows"], value["delivery_rows"]], [arm, 2050, 2500]) + value = take("arm", "kind arm on p50_ms p99_ms delivery_rps") + equal([value["arm"], value["on"]], [arm, ORDER[arm]]) + yield from self.delivery(arm) + take( + "provisional_statistics", + ( + "kind on off p99_delta_ms throughput_loss " + "latency_within_limit throughput_within_limit " + "paired_p99_differences_ms" + ), + ) + equal( + take("triggers_restored", "kind verified"), + dict(kind="triggers_restored", verified=True), + ) + equal( + take("driver_cleanup", "kind native_returned outer_database_cleanup_required"), + dict( + kind="driver_cleanup", + native_returned=True, + outer_database_cleanup_required=True, + ), + ) + try: + next(records) + except StopIteration as end: + self.refs.append(end.value) + else: + need(False, "extra driver event") + equal(self.counts, dict(api=20500, delivery=25000)) + verify_token(self.metadata) + finally: + pending = sys.exception() + try: + records.close() + except BaseException: + if pending is None: + raise + + def pages(self): + page = [] + for value in self.iterator: + page.append(value) + if len(page) == 100: + yield page + page = [] + if page: + yield page + + +def entry(context, directory, argv): + need( + set(context) + == SPEC_FIELDS + | { + "cell", + "org", + "namespace", + "api_instance_id", + "mint_metadata", + "capture_context_sha256", + }, + "capture context fields", + ) + validate({key: context[key] for key in SPEC_FIELDS}) + source(context) + need( + len(argv) == 16 + and len(set(argv[::2])) == 8 + and set(argv[::2]) + == { + "--base-url", + "--pg-dsn", + "--token", + "--org", + "--namespace", + "--fixture-manifest", + "--launch-manifest", + "--output", + }, + "original driver arguments", + ) + # There are eight required original pairs; no private option reaches its parser. + driver_path = Path(context["root"]) / "bench/cell_trigger_gate.py" + name = "_fixture_measured_driver" + need(name not in sys.modules, "single actual driver module") + spec = importlib.util.spec_from_file_location(name, driver_path) + need(spec is not None and spec.loader is not None, "exact driver module") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + capture = Capture(context, directory, birth(), Codec(protocol())) + module._build_block = capture.delegate(module._build_block) + sys.argv = [str(driver_path), *argv] + result = module.main() + return 1 if result == 0 and capture.calls != 10 else result + + +def main(): + try: + divider = sys.argv.index("--") + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--context", type=Path, required=True) + parser.add_argument("--capture-directory", type=Path, required=True) + args = parser.parse_args(sys.argv[1:divider]) + context = read(args.context) + context["capture_context_sha256"] = sha(args.context) + return entry(context, args.capture_directory, sys.argv[divider + 1 :]) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + print(type(error).__name__, file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gates/testsupport/cell_trigger/fixture.py b/scripts/gates/testsupport/cell_trigger/fixture.py new file mode 100644 index 0000000000..42526c9e43 --- /dev/null +++ b/scripts/gates/testsupport/cell_trigger/fixture.py @@ -0,0 +1,1250 @@ +"""Own the isolated CI trigger acceptance fixture and its cleanup.""" + +from __future__ import annotations + +import argparse +import ast +import datetime +import fcntl +import hashlib +import json +import math +import os +import re +import shutil +import signal +import socket +import stat +import subprocess +import sys +import threading +import time +import uuid +from contextlib import contextmanager +from pathlib import Path +from urllib.parse import urlsplit, urlunsplit + +GIB = 2**30 + +POLICY = dict( + admit_mem_bytes=10 * GIB, + admit_disk_bytes=16 * GIB, + stop_mem_bytes=4 * GIB, + stop_disk_bytes=16 * GIB, + stop_process_rss_bytes=7 * GIB // 2, + sample_seconds=1, + wall_seconds=7200, + drain_reserve_seconds=180, +) +HERE = Path(__file__).resolve().parent +FILES = ("fixture.py", "native.py", "tls_proxy.py", "capture.py", "owned_children.py") +PROTOCOL = "protocol.json" +PROTOCOL_SHA = "7223a2018c518cfc8a58977c84aa88dec5784aa741cb34245cb092abbf848812" +SPEC_FIELDS = { + "schema", + "fixture_id", + "root", + "head", + "tree", + "python", + "python_sha256", + "lockfile_sha256", + "helper_path", + "source_hashes", + "workdir", + "ports", + "databases", + "stores", + "pg_admin_dsn", + "pg_standby_dsn", + "ch", + "dependency_receipt", + "ci_run", +} +CODEC_FILES = ( + "pyvenv.cfg", + "lib/python3.12/site-packages/clickhouse_connect/_version.py", + "lib/python3.12/site-packages/clickhouse_connect/datatypes/dynamic.py", + "lib/python3.12/site-packages/clickhouse_connect/datatypes/format.py", + "lib/python3.12/site-packages/asyncpg/_version.py", + "lib/python3.12/site-packages/asyncpg/connection.py", + "lib/python3.12/site-packages/asyncpg/protocol/protocol.cpython-312-aarch64-linux-gnu.so", + "lib/python3.12/site-packages/asyncpg/protocol/record.cpython-312-aarch64-linux-gnu.so", +) +CODEC_VERSIONS = {"clickhouse-connect": "1.9.0", "asyncpg": "0.32.0"} + + +class Refused(RuntimeError): + pass + + +def need(ok: object, field: str) -> None: + if not ok: + raise Refused(field) + + +def digest(value: object) -> str: + return hashlib.sha256( + json.dumps(value, sort_keys=True, default=str, separators=(",", ":")).encode() + ).hexdigest() + + +def sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def utc() -> str: + return datetime.datetime.now(datetime.UTC).isoformat() + + +def pairs(items: list[tuple]) -> dict: + out = {} + for key, value in items: + need(key not in out, "duplicate JSON field") + out[key] = value + return out + + +def invalid_constant(_value: str): + raise Refused("nonfinite JSON number") + + +def finite_float(value: str) -> float: + result = float(value) + need(math.isfinite(result), "nonfinite JSON number") + return result + + +def json_object(raw: bytes | str) -> dict: + value = json.loads( + raw, object_pairs_hook=pairs, parse_constant=invalid_constant, parse_float=finite_float + ) + + def depth(item, level=0): + need(level <= 8, "JSON nesting") + if isinstance(item, dict): + for member in item.values(): + depth(member, level + 1) + elif isinstance(item, list): + for member in item: + depth(member, level + 1) + + depth(value) + need(isinstance(value, dict), "JSON object") + return value + + +@contextmanager +def private_stream(path: Path, *, output: bool = False): + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL if output else os.O_RDONLY | os.O_NONBLOCK + fd = os.open(path, flags | os.O_NOFOLLOW, 0o600) + stream = None + first = None + try: + info = os.fstat(fd) + need( + stat.S_ISREG(info.st_mode) + and info.st_uid == os.geteuid() + and info.st_mode & 0o077 == 0, + "owned private stream", + ) + stream = os.fdopen(fd, "wb" if output else "rb", closefd=False) + yield stream + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error + finally: + if stream is not None: + if output: + try: + stream.flush() + os.fsync(fd) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + try: + stream.close() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + try: + os.close(fd) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + if first is not None: + raise first + + +def read(path: Path, *, limit: int = 65536, expected_sha256: str | None = None) -> dict: + with private_stream(path) as stream: + raw = stream.read(limit + 1) + need(len(raw) <= limit, "input size") + if expected_sha256 is not None: + need(hashlib.sha256(raw).hexdigest() == expected_sha256, "receipt bytes") + return json_object(raw) + + +def write(path: Path, value: dict, *, limit: int = 65536) -> dict: + raw = (json.dumps(value, sort_keys=True, allow_nan=False) + "\n").encode() + need(len(raw) <= limit, "receipt size") + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + first = None + stream = None + try: + stream = os.fdopen(fd, "wb", closefd=False) + need(stream.write(raw) == len(raw), "complete receipt write") + stream.flush() + os.fsync(fd) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error + finally: + if stream is not None: + try: + stream.close() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + try: + os.close(fd) + except BaseException: + if first is None: + raise + if first is not None: + raise first + return {"path": str(path), "sha256": hashlib.sha256(raw).hexdigest()} + + +def ref(value: dict) -> dict: + need(set(value) == {"path", "sha256"}, "receipt reference fields") + path = Path(value["path"]) + need(path.is_absolute(), "absolute receipt") + need( + isinstance(value["sha256"], str) and re.fullmatch(r"[a-f0-9]{64}", value["sha256"]), + "receipt digest", + ) + return read(path, expected_sha256=value["sha256"]) + + +def source(spec: dict) -> None: + need( + sys.version_info[:2] == (3, 12) and Path(sys.executable) == Path(spec["python"]), + "actual admitted interpreter", + ) + dependencies(spec, verify=True) + root = Path(spec["root"]) + for command, expected in ( + (["rev-parse", "HEAD"], spec["head"]), + (["rev-parse", "HEAD^{tree}"], spec["tree"]), + (["status", "--porcelain"], ""), + ): + result = subprocess.run( + ["git", *command], cwd=root, capture_output=True, text=True, check=False, timeout=5 + ) + need(result.returncode == 0 and result.stdout.strip() == expected, "frozen source") + for name, expected in spec["source_hashes"].items(): + path = ( + root / "bench/cell_trigger_gate.py" + if name == "driver" + else root / "scripts/gates/cell_trigger_acceptance.py" + if name == "entrypoint" + else root / ".github/workflows/docker-smoke.yml" + if name == "workflow" + else HERE / name + ) + need(sha(path) == expected, "private source bytes") + need( + Path(spec["helper_path"]) == HERE / "owned_children.py" + and sha(Path(spec["helper_path"])) == spec["source_hashes"]["owned_children.py"], + "owned child implementation", + ) + need(sha(Path(spec["python"])) == spec["python_sha256"], "private interpreter") + need(sha(root / "uv.lock") == spec["lockfile_sha256"], "locked dependency source") + + +def dependencies(spec: dict, *, verify: bool = False) -> None: + value = ref(spec["dependency_receipt"]) + need( + set(value) + == { + "schema", + "root", + "python", + "python_sha256", + "lockfile_sha256", + "versions", + "codec_sources", + } + and type(value["schema"]) is int + and value["schema"] == 2, + "closed dependency receipt", + ) + for key in ("root", "python", "python_sha256", "lockfile_sha256"): + need(value[key] == spec[key], "dependency source identity") + environment = Path(spec["root"]) / ".venv" + need( + Path(spec["python"]) == environment / "bin/python" + and value["versions"] == CODEC_VERSIONS + and set(value["codec_sources"]) == set(CODEC_FILES), + "fixed interpreter/codec versions and file set", + ) + for name in CODEC_FILES: + item = value["codec_sources"][name] + path = environment / name + need( + set(item) == {"path", "sha256"} + and item["path"] == str(path) + and type(item["sha256"]) is str + and re.fullmatch(r"[a-f0-9]{64}", item["sha256"]), + "exact fixed codec path/hash", + ) + if verify: + info = path.lstat() + need( + stat.S_ISREG(info.st_mode) + and info.st_size <= 32 * 1024 * 1024 + and sha(path) == item["sha256"], + "regular frozen codec bytes", + ) + if verify: + cfg = dict( + line.split(" = ", 1) + for line in (environment / "pyvenv.cfg").read_text().splitlines() + if " = " in line + ) + need( + cfg.get("implementation") == "CPython" + and sys.version_info[:2] == (3, 12) + and cfg.get("version_info") in {"3.12", ".".join(map(str, sys.version_info[:3]))} + and cfg.get("include-system-site-packages") == "false", + "fixed isolated CPython environment", + ) + for package, field, name in ( + ("clickhouse-connect", "version", CODEC_FILES[1]), + ("asyncpg", "__version__", CODEC_FILES[4]), + ): + module = ast.parse((environment / name).read_bytes()) + assignments = [ + node.value + for node in module.body + if ( + isinstance(node, ast.Assign) + and any( + isinstance(target, ast.Name) and target.id == field + for target in node.targets + ) + ) + or ( + isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) + and node.target.id == field + ) + ] + need( + len(assignments) == 1 + and ast.literal_eval(assignments[0]) == CODEC_VERSIONS[package], + "frozen codec source version", + ) + + +def database(name: str) -> str: + need( + isinstance(name, str) and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,62}", name), + "database identifier", + ) + need( + name.lower() + not in { + "loom", + "loom_router", + "loom_telemetry", + "default", + "system", + "postgres", + "template0", + "template1", + "information_schema", + }, + "scratch database", + ) + return name + + +def pg_dsn(admin: str, db: str) -> str: + value = urlsplit(admin) + need( + value.scheme in {"postgres", "postgresql"} + and value.hostname == "127.0.0.1" + and value.port + and not value.query + and not value.fragment, + "numeric PG DSN", + ) + return urlunsplit((value.scheme, value.netloc, "/" + database(db), "", "")) + + +def validate(spec: dict) -> None: + fields = { + "schema", + "fixture_id", + "root", + "head", + "tree", + "python", + "python_sha256", + "lockfile_sha256", + "helper_path", + "source_hashes", + "workdir", + "ports", + "databases", + "stores", + "pg_admin_dsn", + "pg_standby_dsn", + "ch", + "dependency_receipt", + "ci_run", + } + need(set(spec) == fields and type(spec["schema"]) is int and spec["schema"] == 2, "spec fields") + need(str(uuid.UUID(spec["fixture_id"])) == spec["fixture_id"], "fixture UUID") + for field in ("root", "python", "helper_path", "workdir"): + need(Path(spec[field]).is_absolute(), "absolute fixture path") + need( + all( + isinstance(spec[field], str) and re.fullmatch(r"[a-f0-9]{40}", spec[field]) + for field in ("head", "tree") + ), + "full source identity", + ) + need( + set(spec["source_hashes"]) == set(FILES) | {"driver", "entrypoint", "workflow", PROTOCOL}, + "source hash set", + ) + need(spec["source_hashes"][PROTOCOL] == PROTOCOL_SHA, "fixed capture protocol") + need( + all( + isinstance(value, str) and re.fullmatch(r"[a-f0-9]{64}", value) + for value in spec["source_hashes"].values() + ), + "source digest values", + ) + need(set(spec["ports"]) == {"api", "router", "probe", "tls"}, "port roles") + need( + len(set(spec["ports"].values())) == 4 + and all(type(p) is int and 0 < p < 65536 for p in spec["ports"].values()), + "distinct ports", + ) + need( + set(spec["databases"]) == {"pg_cell", "pg_directory", "ch_schema", "ch_telemetry"}, + "DB roles", + ) + need(len(set(spec["databases"].values())) == 4, "distinct databases") + for name in spec["databases"].values(): + database(name) + pg_dsn(spec["pg_admin_dsn"], spec["databases"]["pg_cell"]) + need( + set(spec["ch"]) == {"host", "port", "username", "password", "keeper_prefix"}, + "CH input fields", + ) + need( + spec["ch"]["host"] == "127.0.0.1" + and type(spec["ch"]["port"]) is int + and 0 < spec["ch"]["port"] < 65536, + "numeric CH endpoint", + ) + need( + os.environ.get("GITHUB_ACTIONS") == "true" and os.environ.get("CI") == "true", + "GitHub CI only", + ) + need( + spec["ci_run"] + == { + "run_id": os.environ.get("GITHUB_RUN_ID"), + "attempt": os.environ.get("GITHUB_RUN_ATTEMPT"), + "repository": os.environ.get("GITHUB_REPOSITORY"), + } + and str(spec["ci_run"]["run_id"]).isdigit(), + "actual CI run binding", + ) + dependencies(spec) + need(set(spec["stores"]) == {"clickhouse", "pg_primary", "pg_standby"}, "three owned stores") + for role, expected in spec["stores"].items(): + store_fields = { + "role", + "container_id", + "container_name", + "created_at", + "owner_evidence", + "host_pid", + "start_ticks", + "boot_id", + "uid", + "ports", + "memory_limit_bytes", + "server_limit_bytes", + "shared_buffers_bytes", + "retained", + "disk_path", + } + need(set(expected) == store_fields, "store input fields") + need(expected["role"] == role and expected["retained"] is False, "job-created store") + need( + type(expected["host_pid"]) is int + and expected["host_pid"] > 1 + and str(expected["start_ticks"]).isdigit(), + "store birth input", + ) + for mapping in expected["ports"]: + need( + set(mapping) == {"host", "host_port", "internal_port", "protocol"} + and mapping["host"] == "127.0.0.1" + and mapping["protocol"] == "tcp", + "store loopback mapping", + ) + if role == "clickhouse": + port = spec["ch"]["port"] + need( + expected["memory_limit_bytes"] == 5 * GIB + and expected["server_limit_bytes"] == 9 * GIB // 2, + "fixed CH caps", + ) + else: + target = urlsplit( + spec["pg_admin_dsn"] if role == "pg_primary" else spec["pg_standby_dsn"] + ) + need( + target.scheme in {"postgres", "postgresql"} + and target.hostname == "127.0.0.1" + and target.port + and not target.query + and not target.fragment, + "numeric owned PG endpoint", + ) + port = target.port + need( + type(expected["memory_limit_bytes"]) is int + and expected["memory_limit_bytes"] > 0 + and type(expected["shared_buffers_bytes"]) is int + and expected["shared_buffers_bytes"] > 0, + "observed PG caps", + ) + need(any(m["host_port"] == port for m in expected["ports"]), "store endpoint binding") + proof = ref(expected["owner_evidence"]) + need( + proof.get("container_id") == expected["container_id"] + and proof.get("created_at") == expected["created_at"] + and proof.get("task") == "cell-trigger-acceptance" + and proof.get("ci_run") == spec["ci_run"], + "positive creator evidence", + ) + + +def child_env(spec: dict, settings: dict, *, ca: bool = False) -> dict: + env = { + "PATH": "/usr/local/bin:/usr/bin:/bin", + "LANG": "C.UTF-8", + "TZ": "UTC", + "PYTHONPATH": spec["root"], + "LOOM_ENV_FILE": "", + "LOOM_SETTINGS_FILE": "", + "HOME": str(Path(spec["workdir"]) / "home"), + "TMPDIR": str(Path(spec["workdir"]) / "tmp"), + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONPYCACHEPREFIX": str(Path(spec["workdir"]) / "bytecode"), + "GITHUB_ACTIONS": os.environ.get("GITHUB_ACTIONS", ""), + "CI": os.environ.get("CI", ""), + **{ + "GITHUB_" + k: os.environ.get("GITHUB_" + k, "") + for k in ("RUN_ID", "RUN_ATTEMPT", "REPOSITORY") + }, + "LOOM_INSTANCE_LABEL": "iso:trigger:" + spec["fixture_id"][:12], + } + for key, value in settings.items(): + need(re.fullmatch(r"[a-z][a-z0-9_]*", key), "setting name") + env["LOOM_" + key.upper()] = value if isinstance(value, str) else json.dumps(value) + if ca: + env["SSL_CERT_FILE"] = str(Path(spec["workdir"]) / "ca.crt") + return env + + +def load_owned(path: Path): + from owned_children import OwnedChildren + + return OwnedChildren() + + +class Owner: + def __init__(self, spec: dict, owned, note) -> None: + self.spec, self.owned, self.note = spec, owned, note + self.leaders: list[dict] = [] + self.failures: list[list] = [] + self.phase = "owner" + self.failure_overflow = False + self.failure_attempted = 0 + self.failure_written = 0 + self.diagnostic_failures = 0 + self.interruption = Refused("interrupted") + self.stopping = False + self.terminal_reporting = False + self.blocked = False + self.started = time.monotonic() + self.serial = 0 + self.sampled = 0.0 + self.guard_stop = threading.Event() + self.guard = None + + def select(self, error: BaseException, *, phase: str | None = None) -> None: + # Select before formatting evidence. No IO in a signal handler. + if len(self.failures) >= 64: + self.failure_overflow = True + return + entry = [error, {"phase": phase or self.phase}] + self.failures.append(entry) + if len(self.failures) > 64: + self.failure_overflow = True + del self.failures[64:] + selected_phase = entry[1]["phase"] + unknown = {"file": "unknown", "line": 0, "function": "unknown"} + try: + site = unknown + frame = sys._getframe(1) + line = frame.f_lineno + trace = error.__traceback__ + for _ in range(64): + if trace is None: + break + candidate = trace.tb_frame + if Path( + candidate.f_code.co_filename + ).name in FILES and candidate.f_code.co_name not in { + "need", + "closed", + "equal", + "integer", + "identifier", + "hex64", + }: + frame = candidate + line = trace.tb_lineno + trace = trace.tb_next + while frame is not None and frame.f_code.co_name in {"select", "block_teardown"}: + frame = frame.f_back + if frame is not None: + line = frame.f_lineno + if frame is not None and Path(frame.f_code.co_filename).name in FILES: + site = { + "file": Path(frame.f_code.co_filename).name, + "line": line, + "function": frame.f_code.co_name[:64], + } + entry[1] = { + "phase": selected_phase, + "error_type": type(error).__name__[:64], + "site": site, + "outcome": "refused" + if isinstance(error, Refused) + else "error" + if isinstance(error, Exception) + else "cancelled", + } + except BaseException as secondary: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + entry[1] = { + "phase": selected_phase, + "error_type": type(error).__name__[:64], + "site": unknown, + "outcome": "evidence-error", + } + if len(self.failures) < 64: + self.failures.append( + [ + secondary, + { + "phase": "diagnostic", + "error_type": type(secondary).__name__[:64], + "site": unknown, + "outcome": "error", + }, + ] + ) + else: + self.failure_overflow = True + if len(self.failures) > 64: + self.failure_overflow = True + del self.failures[64:] + + def failure_summary(self) -> dict: + records = [] + snapshot = self.failures[:64] + for index, (_, record) in enumerate(snapshot): + if "outcome" not in record: + break + records.append(dict(index=index, primary=index == 0, **record)) + return { + "records": records, + "pending_records": len(snapshot) - len(records), + "overflow": self.failure_overflow or len(self.failures) > 64, + "event_attempts": self.failure_attempted, + "event_records_written": self.failure_written, + "diagnostic_write_failures": self.diagnostic_failures, + } + + def flush_failures(self) -> None: + # Snapshot once: failed evidence writes cannot recursively create more writes here. + records = self.failure_summary()["records"] + for index in range(self.failure_attempted, len(records)): + self.failure_attempted = index + 1 + try: + self.note("failure", **records[index]) + self.failure_written += 1 + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.diagnostic_failures += 1 + self.select(error, phase="diagnostic") + + @property + def first(self) -> BaseException | None: + return self.failures[0][0] if self.failures else None + + def record(self, event: str, **fields) -> None: + try: + self.note(event, **fields) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error, phase="diagnostic") + + def interrupt(self, _signum, _frame) -> None: + if not self.stopping: + self.stopping = True + self.select(self.interruption, phase="interrupt") + if self.terminal_reporting: + # Closed ownership permits cancellation to escape the final return boundary. + first = self.first + raise self.interruption if first is None else first + + def sample(self, *, admit: bool = False) -> None: + try: + now = time.monotonic() + available = next( + int(line.split()[1]) * 1024 + for line in Path("/proc/meminfo").read_text().splitlines() + if line.startswith("MemAvailable:") + ) + need(available >= POLICY["admit_mem_bytes" if admit else "stop_mem_bytes"], "RAM floor") + for path in [ + Path(self.spec["workdir"]), + *[Path(s["disk_path"]) for s in self.spec["stores"].values()], + ]: + need(shutil.disk_usage(path).free >= 16 * GIB, "disk floor") + boot = Path("/proc/sys/kernel/random/boot_id").read_text().strip() + for store in self.spec["stores"].values(): + identity = self.owned.identity(store["host_pid"]) + need( + identity is not None + and identity[1] == store["start_ticks"] + and boot == store["boot_id"] + and os.stat(f"/proc/{store['host_pid']}").st_uid == store["uid"], + "resource store birth changed", + ) + pending = [self.owned.pid] + [s["host_pid"] for s in self.spec["stores"].values()] + seen = set() + while pending: + pid = pending.pop() + if pid in seen: + continue + seen.add(pid) + before = self.owned.identity(pid) + need( + before is not None, "enumerated resource process disappeared; sample incomplete" + ) + need(self.owned.rss(pid) * 1024 <= POLICY["stop_process_rss_bytes"], "process RSS") + pending.extend(self.owned.children(pid)) + after = self.owned.identity(pid) + need( + after is not None and after[:2] == before[:2], + "resource process disappeared or changed; sample incomplete", + ) + for item in tuple(self.leaders): + need(item["log"].stat().st_size <= 16 * 1024 * 1024, "child log bound") + driver_artifact = Path(self.spec["workdir"]) / "driver.jsonl" + if driver_artifact.exists(): + need(driver_artifact.stat().st_size <= 64 * 1024 * 1024, "driver artifact bound") + summary = Path(self.spec["workdir"]) / "mint-summary.stdout" + if summary.exists(): + need(summary.stat().st_size <= 4096, "mint stdout bound") + total = 0 + for arm in range(10): + path = Path(self.spec["workdir"]) / "delivery-intent" / f"arm-{arm}.jsonl" + if path.exists(): + size = path.stat().st_size + need(size <= 32 * 1024 * 1024, "intent file bound") + total += size + need(total <= 320 * 1024 * 1024, "intent aggregate bound") + self.sampled = now + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error, phase="resource") + + def start_guard(self) -> None: + def watch() -> None: + try: + while not self.guard_stop.wait(1): + self.sample() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error) + + self.sample(admit=True) + if self.first is not None: + raise self.first + self.guard = threading.Thread(target=watch, name="trigger-resource-readonly", daemon=False) + self.guard.start() + + def monitor(self, *, admit: bool = False, cleanup: bool = False) -> None: + self.flush_failures() + if admit: + self.sample(admit=True) + if time.monotonic() - self.started >= 7200: + self.select(Refused("global wall overrun; cleanup retains exclusion")) + if not cleanup: + try: + need( + not self.stopping and time.monotonic() - self.started < 7020, + "global work deadline", + ) + need(time.monotonic() - self.sampled < 3, "resource sampler stale") + for item in self.leaders: + if item["persistent"]: + need(item["process"].poll() is None, "fixture leader ended") + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error) + if self.first is not None and not cleanup: + raise self.first + + def start( + self, + role: str, + argv: list[str], + env: dict, + *, + persistent: bool = False, + cleanup: bool = False, + ) -> dict: + self.monitor(cleanup=cleanup) + source(self.spec) + argv = list(argv) + executable = shutil.which(argv[0], path=env.get("PATH")) + need(executable is not None and Path(executable).is_absolute(), "known child executable") + argv[0] = executable + self.serial += 1 + label = role + "-" + str(self.serial) + log_path = Path(self.spec["workdir"]) / (label + ".log") + fd = os.open(log_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + item = None + stdout_fd = None + try: + if role == "mint-token": + stdout_fd = os.open( + Path(self.spec["workdir"]) / "mint-summary.stdout", + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, + ) + process = subprocess.Popen( + argv, + cwd=self.spec["root"], + env=env, + stdout=stdout_fd if stdout_fd is not None else fd, + stderr=fd if stdout_fd is not None else subprocess.STDOUT, + start_new_session=True, + ) + # Register Popen before any observation/callback can fail. + item = { + "role": label, + "process": process, + "pidfd": None, + "identity": None, + "persistent": persistent, + "log": log_path, + "terminal": None, + "argv": list(argv), + "configuration_sha256": digest(env), + "launch_receipt": None, + "terminal_receipt": None, + } + item["signals"] = [] + self.leaders.append(item) + identity = self.owned.identity(process.pid) + need(identity and identity[0] == self.owned.pid, "direct child birth") + need(os.stat(f"/proc/{process.pid}").st_uid == os.geteuid(), "child UID") + item["identity"] = identity + item["pidfd"] = os.pidfd_open(process.pid) + after = self.owned.identity(process.pid) + need(after is not None and after[:2] == identity[:2], "stable pinned child") + item["launch_receipt"] = write( + Path(self.spec["workdir"]) / (label + "-birth.json"), + { + "pid": process.pid, + "start_ticks": identity[1], + "ppid": identity[0], + "uid": os.geteuid(), + "boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + "cwd": self.spec["root"], + "configuration_sha256": digest(env), + "utc": utc(), + }, + ) + self.record("birth", role=label, pid=process.pid, start_ticks=identity[1]) + if self.first is not None and not cleanup: + raise self.first + return item + except BaseException as error: + self.select(error) + raise + finally: + if stdout_fd is not None: + try: + os.close(stdout_fd) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error) + try: + os.close(fd) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error) + + def poll(self, item: dict) -> int | None: + result = item["process"].poll() + if result is not None and item["terminal"] is None: + item["terminal"] = {"returncode": result, "utc": utc()} + item["persistent"] = False + try: + item["terminal_receipt"] = write( + Path(self.spec["workdir"]) / (item["role"] + "-terminal.json"), item["terminal"] + ) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error) + self.record("terminal", role=item["role"], returncode=result) + return result + + def command( + self, role: str, argv: list[str], env: dict, *, seconds: float = 120, cleanup: bool = False + ) -> dict: + item = self.start(role, argv, env, cleanup=cleanup) + end = ( + time.monotonic() + seconds + if cleanup + else min(self.started + 7020, time.monotonic() + seconds) + ) + while self.poll(item) is None: + self.monitor(cleanup=cleanup) + need(time.monotonic() < end, "serial command deadline") + time.sleep(0.05) + need(item["process"].returncode == 0, "native command refused") + if self.first is not None and not cleanup: + raise self.first + return item + + def send(self, item: dict, sig: int) -> None: + if self.poll(item) is not None: + return + before = self.owned.identity(item["process"].pid) + if ( + before is None + or before[0] != self.owned.pid + or (item["identity"] is not None and before[:2] != item["identity"][:2]) + ): + self.blocked = True + raise Refused("child identity mismatch") + # No generic reaper runs while Popen owns this direct child, so an unreaped + # Popen leader cannot have its PID reused during an interrupted birth capture. + if item["identity"] is None: + item["identity"] = before + if item["pidfd"] is None: + item["pidfd"] = os.pidfd_open(item["process"].pid) + need(os.stat(f"/proc/{item['process'].pid}").st_uid == os.geteuid(), "signal UID") + after = self.owned.identity(item["process"].pid) + need(after is not None and after[:2] == before[:2], "signal birth recheck") + try: + signal.pidfd_send_signal(item["pidfd"], sig) + item.setdefault("signals", []).append(sig) + except ProcessLookupError: + need(self.poll(item) is not None, "lost pinned child") + + def stop_one(self, item: dict, *, driver: bool = False, api: bool = False) -> None: + if self.poll(item) is not None: + self.terminal_outcome(item, driver=driver, api=api) + return + phases = ( + [(signal.SIGINT, 60), (signal.SIGTERM, 15)] + if driver + else [(signal.SIGTERM, 90 if api else 15)] + ) + for sig, seconds in phases + [(signal.SIGKILL, 5)]: + if self.poll(item) is not None: + break + if sig == signal.SIGKILL or (driver and sig == signal.SIGTERM): + self.select(Refused("cleanup escalation")) + self.send(item, sig) + end = time.monotonic() + seconds + while self.poll(item) is None and time.monotonic() < end: + self.monitor(cleanup=True) + time.sleep(0.05) + need(self.poll(item) is not None, "child not reaped") + item["process"].wait() + self.terminal_outcome(item, driver=driver, api=api) + + def terminal_outcome(self, item: dict, *, driver: bool, api: bool) -> None: + role = item["role"].split("-", 1)[0] + result = item["process"].returncode + sent = item.get("signals", []) + if driver: + ok = result in (0, 1) and not sent + elif api or role in {"router", "proxy"}: + # Uvicorn re-raises its captured TERM after graceful shutdown; the + # private proxy consumes TERM. Only an owner-requested stop qualifies. + ok = signal.SIGTERM in sent and result in ( + (0, -signal.SIGTERM) if role != "proxy" else (0,) + ) + else: + return + if not ok: + self.select(Refused("unexpected fixture terminal outcome")) + + def cleanup(self) -> None: + # Each Popen owns its status until all leaders are reaped. + self.flush_failures() + if self.owned is None: + need(not self.leaders, "uninitialized ownership with registered children") + return + for prefix in ("driver", "api", "router", "proxy", ""): + for item in self.leaders: + if item.get("cleaned") or (prefix and not item["role"].startswith(prefix)): + continue + try: + self.stop_one( + item, + driver=item["role"].startswith("driver"), + api=item["role"].startswith("api"), + ) + item["cleaned"] = True + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error) + if self.blocked or any(self.poll(item) is None for item in self.leaders): + raise Refused("owned leader cleanup unresolved; exclusion retained") + try: + self.owned.stop(None, lambda *args, **kwargs: self.record("adopted-cleanup", **kwargs)) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error) + need(not self.owned.children(self.owned.pid), "kernel children remain") + for item in self.leaders: + if item["pidfd"] is not None: + try: + os.close(item["pidfd"]) + item["pidfd"] = None + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error) + + +def runtime(spec: dict) -> int: + """The separately admitted path; never called by inert controls.""" + from native import Fixture + + validate(spec) + source(spec) + for port in spec["ports"].values(): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as check: + check.bind(("127.0.0.1", port)) + workdir = Path(spec["workdir"]) + need(workdir.is_absolute() and not workdir.exists(), "exclusive work directory") + os.mkdir(workdir, 0o700) + for name in ("home", "tmp", "bytecode"): + (workdir / name).mkdir(mode=0o700) + events = os.open( + workdir / "events.jsonl", os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600 + ) + + def note(event: str, **fields) -> None: + raw = (json.dumps(dict(utc=utc(), event=event, **fields), allow_nan=False) + "\n").encode() + need( + len(raw) <= 65536 and os.lseek(events, 0, os.SEEK_CUR) + len(raw) <= 16 * 1024 * 1024, + "event bound", + ) + need(os.write(events, raw) == len(raw), "complete event write") + os.fsync(events) + + owner = None + lock = None + fixture = None + release = False + verdict = "INVALID" + first = None + events_closed = False + lock_closed = False + + def selected() -> BaseException | None: + retained = owner.first if owner is not None else None + return retained if retained is not None else first + + try: + owner = Owner(spec, None, note) + owner.phase = "ownership" + owner.owned = load_owned(Path(spec["helper_path"])) + for signum in (signal.SIGINT, signal.SIGTERM): + signal.signal(signum, owner.interrupt) + owner.start_guard() + owner.monitor(admit=True) + lock = os.open( + "/tmp/loom-full-unit-suite.lock", os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600 + ) + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + # This isolated job owns the canonical slot through final native cleanup. + fixture = Fixture(spec, owner) + owner.phase = "setup" + fixture.setup() + owner.monitor(admit=True) + owner.phase = "measure" + result = fixture.measure() + owner.phase = "provenance" + fixture.provenance() + verdict = result + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error + if owner is not None: + owner.select(error) + finally: + if owner is not None: + try: + owner.phase = "cleanup" + owner.cleanup() + if fixture is not None: + owner.phase = "teardown" + fixture.teardown() + owner.phase = "cleanup" + owner.cleanup() + release = True + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + owner.select(error) + else: + release = True + if not release: + # Never exit over unreaped children or uncertain task writes. + while True: + if owner is not None: + try: + owner.phase = "cleanup" + owner.cleanup() + if fixture is not None: + owner.phase = "teardown" + fixture.teardown() + owner.phase = "cleanup" + owner.cleanup() + release = True + break + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + owner.select(error) + try: + time.sleep(1) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + if owner is not None: + owner.select(error) + if owner is not None: + try: + owner.phase = "monitor-stop" + owner.monitor(cleanup=True) + owner.guard_stop.set() + if owner.guard is not None: + owner.guard.join() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + owner.select(error) + owner.flush_failures() + first = owner.first if owner.first is not None else first + try: + os.fsync(events) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + if owner is not None: + owner.select(error, phase="events-close") + first = error if first is None else first + finally: + try: + os.close(events) + events_closed = True + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + if owner is not None: + owner.select(error, phase="events-close") + first = error if first is None else first + first = selected() + if first is not None: + verdict = "INVALID" + try: + write( + workdir / "outer-result.json", + { + "schema": 2, + "fixture_id": spec["fixture_id"], + "candidate_verdict": verdict, + "native_cleanup_complete": release + and (fixture is None or not fixture.teardown_incomplete), + "error_type": type(first).__name__ if first is not None else "", + "failure_evidence": owner.failure_summary() if owner is not None else None, + "utc": utc(), + }, + ) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + if owner is not None: + owner.select(error, phase="result") + first = error if first is None else first + verdict = "INVALID" + if lock is not None and release: + try: + os.close(lock) + lock_closed = True + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + if owner is not None: + owner.select(error, phase="unlock") + first = error if first is None else first + verdict = "INVALID" + if owner is not None and release and events_closed and (lock is None or lock_closed): + try: + owner.terminal_reporting = True + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + owner.select(error, phase="reporting-mode") + try: + first = selected() + if first is not None: + verdict = "INVALID" + # Candidate bytes require successful terminal status; they grant no PASS alone. + print( + json.dumps( + { + "candidate_verdict": verdict, + "error_type": type(first).__name__ if first is not None else "", + "failure_evidence": owner.failure_summary() if owner is not None else None, + } + ), + flush=True, + ) + return 0 if verdict == "PASS" and selected() is None else 1 + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + if owner is not None: + owner.select(error, phase="stdout") + first = selected() + first = error if first is None else first + verdict = "INVALID" + try: + write( + workdir / "reporting-failure.json", + { + "schema": 2, + "verdict": "INVALID", + "error_type": type(first).__name__, + "failure_evidence": owner.failure_summary() if owner is not None else None, + }, + ) + except BaseException as secondary: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + if owner is not None: + owner.select(secondary, phase="reporting") + return 1 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--spec", required=True, type=Path) + args = parser.parse_args() + try: + return runtime(read(args.spec)) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + print(json.dumps({"verdict": "INVALID", "error_type": type(error).__name__}), flush=True) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gates/testsupport/cell_trigger/native.py b/scripts/gates/testsupport/cell_trigger/native.py new file mode 100644 index 0000000000..a4c3661fb5 --- /dev/null +++ b/scripts/gates/testsupport/cell_trigger/native.py @@ -0,0 +1,2391 @@ +"""Native operations for the isolated trigger acceptance fixture.""" + +from __future__ import annotations + +import argparse +import asyncio +import datetime +import hashlib +import json +import os +import re +import socket +import sys +import time +import uuid +from contextlib import asynccontextmanager +from pathlib import Path + +from capture import ( + Codec, + Workload, + closed, + equal, + hex64, + identifier, + integer, + json_object, + mint_metadata, + protocol, + verify_token, +) +from fixture import ( + FILES, + GIB, + HERE, + POLICY, + PROTOCOL_SHA, + Refused, + child_env, + database, + digest, + need, + pg_dsn, + read, + ref, + sha, + utc, + write, +) + +COUNTS = ("head_global", "head_scope", "revision_global", "revision_scope") +DEBT = ( + "delivery_global", + "delivery_scope", + "audit_global", + "audit_scope", + "error_global", + "error_scope", +) +PG_FIELDS = ( + tuple( + "c_" + k + for k in ( + "org", + "namespace", + "agent", + "user_id", + "session_id", + "memory_key", + "memory_id", + "version", + "seq", + "content_sha256", + ) + ) + + ("head",) + + tuple( + "r_" + k + for k in ( + "org", + "namespace", + "seq", + "user_id", + "memory_id", + "version", + "content_sha256", + "actor", + "source_class", + "request_id", + ) + ) + + ("revision", "audit", "targets", "attempts", "last_error") +) + + +def pg_census_sql(): + # One statement snapshot supplies all ten fixed counts and server observations. + counts = [] + for label, table in ( + ("head", "memory_current"), + ("revision", "memory_revisions"), + ("delivery", "memory_deliveries"), + ("audit", "audit_outbox"), + ("error", "memory_deliveries"), + ): + predicate = {"delivery": "targets<>0", "error": "(attempts<>0 OR last_error<>'')"}.get( + label, "TRUE" + ) + limit = 45501 if label in {"head", "revision"} else 1 + for scope in ("global", "scope"): + where = predicate + (" AND org=$1 AND namespace=$2" if scope == "scope" else "") + counts.append( + f"(SELECT count(*) FROM (SELECT 1 FROM loom.{table} WHERE " + f"{where} LIMIT {limit}) bounded) AS {label}_{scope}" + ) + return ( + "SELECT " + + ",".join(counts) + + ( + ",CASE WHEN octet_length(pg_current_snapshot()::text)<=16384 " + "THEN pg_current_snapshot()::text ELSE NULL END AS " + "snapshot,current_setting('transaction_isolation') AS " + "isolation,current_setting('transaction_read_only') AS " + "read_only,current_database() AS database,pg_backend_pid() " + "AS backend_pid" + ) + ) + + +def pg_record(value, fields): + need( + value is not None and list(value.keys()) == list(fields), "closed ordered PG result columns" + ) + return dict(value) + + +async def begin_observation(conn, context, isolation, *, backend=None): + initial = conn.is_in_transaction() + need(initial is False, "idle PG snapshot boundary") + statement = "BEGIN ISOLATION LEVEL " + isolation.upper() + " READ ONLY" + status = await conn.execute(statement) + active = conn.is_in_transaction() + need(status == "BEGIN" and active is True, "PG read-only BEGIN acknowledged") + value = pg_record( + await conn.fetchrow(pg_census_sql(), context["org"], context["namespace"]), + ( + *COUNTS[:2], + *COUNTS[2:], + *DEBT[:2], + *DEBT[2:4], + *DEBT[4:], + "snapshot", + "isolation", + "read_only", + "database", + "backend_pid", + ), + ) + for key in COUNTS: + need(integer(value[key]) == 45500, "exact global/scoped PG census") + for key in DEBT: + need(integer(value[key], 0, 1) == 0, "settled global/scoped PG debt") + snapshot = value.pop("snapshot") + need( + type(snapshot) is str + and len(snapshot.encode()) <= 16384 + and re.fullmatch(r"[0-9]+:[0-9]+:(?:[0-9]+(?:,[0-9]+)*)?", snapshot), + "actual bounded PG snapshot", + ) + need( + value["isolation"] == isolation + and value["read_only"] == "on" + and value["database"] == context["databases"]["pg_cell"], + "actual PG snapshot settings/database", + ) + integer(value["backend_pid"], 1, 2**31 - 1) + need(backend is None or backend == value["backend_pid"], "same owned PG backend") + return dict( + snapshot=snapshot, + snapshot_bytes=len(snapshot.encode()), + snapshot_sha256=hashlib.sha256(snapshot.encode()).hexdigest(), + isolation=value["isolation"], + read_only=value["read_only"], + database=value["database"], + backend_pid=value["backend_pid"], + begin_status=status, + initial_transaction_state=initial, + active_transaction_state=active, + counts={key: value[key] for key in (*COUNTS, *DEBT)}, + ) + + +async def end_observation(conn, observation): + status = await conn.execute("COMMIT") + ended = conn.is_in_transaction() + need(status == "COMMIT" and ended is False, "PG snapshot end acknowledged") + observation.update(end_status=status, ended_transaction_state=ended) + + +async def ch_result(ch, query, parameters, fields, limit): + result = await ch.query( + query, + parameters=parameters, + settings={ + "max_result_rows": limit, + "max_result_bytes": 20 * 1024 * 1024, + "result_overflow_mode": "throw", + "max_execution_time": 30, + }, + ) + need(list(result.column_names) == list(fields), "closed ordered CH result columns") + data = result.result_rows + need( + type(data) in {list, tuple} + and len(data) <= limit + and all(type(row) in {list, tuple} and len(row) == len(fields) for row in data), + "bounded CH tuples before maps", + ) + return data + + +async def ch_census(ch, context): + db = database(context["databases"]["ch_schema"]) + observed = {} + for label, table in (("memory", "memory"), ("audit", "audit_log")): + for scope in ("global", "scope"): + where = "event_type='memory.revision'" if label == "audit" else "1" + if scope == "scope": + where += " AND org={org:String} AND namespace={ns:String}" + result = await ch_result( + ch, + f"SELECT count() AS n FROM (SELECT 1 FROM `{db}`.{table}" + + (" FINAL" if label == "memory" else "") + + f" WHERE {where} LIMIT 45501)", + {"org": context["org"], "ns": context["namespace"]}, + ("n",), + 1, + ) + need( + len(result) == 1 and integer(result[0][0]) == 45500, + "exact CH global/scoped workload census", + ) + observed[label + "_" + scope] = result[0][0] + return observed + + +def pg_page_sql(): + c = ( + "org", + "namespace", + "agent", + "user_id", + "session_id", + "memory_key", + "memory_id", + "version", + "seq", + "content_sha256", + ) + r = ( + "org", + "namespace", + "seq", + "user_id", + "memory_id", + "version", + "content_sha256", + "actor", + "source_class", + "request_id", + ) + fields = ( + [f"c.{k} AS c_{k}" for k in c] + + ["c.row::text AS head"] + + [f"r.{k} AS r_{k}" for k in r] + + [ + "r.row::text AS revision", + "r.audit::text AS audit", + "d.targets", + "d.attempts", + "d.last_error", + ] + ) + text = ["c." + k for k in c if k not in {"memory_id", "version", "seq"}] + [ + "r." + k for k in r if k not in {"memory_id", "version", "seq"} + ] + sizes = " AND ".join(f"octet_length({k})<=512" for k in text) + return ( + "SELECT " + + ",".join(fields) + + ( + " FROM loom.memory_current c JOIN loom.memory_revisions r ON " + "(r.org,r.namespace,r.seq)=(c.org,c.namespace,c.seq) LEFT " + "JOIN loom.memory_deliveries d ON " + "(d.org,d.namespace,d.seq)=(c.org,c.namespace,c.seq) WHERE " + "c.org=$1 AND c.namespace=$2 AND c.memory_id=ANY($3::uuid[]) " + "AND octet_length(c.row::text)<=65536 AND " + "octet_length(r.row::text)<=65536 AND " + "octet_length(r.audit::text)<=16384 AND " + ) + + sizes + + ( + " AND (d.last_error IS NULL OR " + "octet_length(d.last_error)<=512) ORDER BY " + "c.memory_id,c.agent,c.user_id,c.session_id,c.memory_key " + "LIMIT $4" + ) + ) + + +def complete_ids(values, expected): + # Inspect multiplicity BEFORE constructing lookup maps. + need( + len(values) == len(expected) + and len(set(values)) == len(values) + and set(values) == set(expected), + "complete unique result multiset before map", + ) + + +def typed_revision(stored, row, audit, item, seen): + for prefix, names in ( + ( + "c_", + ( + "org", + "namespace", + "agent", + "user_id", + "session_id", + "memory_key", + "memory_id", + "version", + "content_sha256", + ), + ), + ( + "r_", + ( + "org", + "namespace", + "user_id", + "memory_id", + "version", + "content_sha256", + "actor", + "source_class", + ), + ), + ): + for key in names: + value = stored[prefix + key] + if key == "memory_id": + value = identifier(value) + elif key == "version": + value = integer(value, 1, 2**63 - 1) + else: + need(type(value) is str and len(value.encode()) <= 512, "typed PG identity text") + equal(value, row[key]) + for key, expected in ( + ("memory_key", item["key"]), + ("memory_id", item["memory_id"]), + ("version", item["version"]), + ("content_sha256", item["content_sha256"]), + ): + equal(row[key], expected) + seq = integer(stored["c_seq"], 1, 2**63 - 1) + need( + integer(stored["r_seq"], 1, 2**63 - 1) == seq and seq not in seen, + "unique exact PG revision link", + ) + seen.add(seq) + need(len(seen) <= 45500, "PG seq bound") + equal(stored["r_request_id"], audit["request_id"]) + delivery = [stored[key] for key in ("targets", "attempts", "last_error")] + if delivery != [None, None, None]: + need( + integer(delivery[0], 0, 0) == 0 + and integer(delivery[1], 0, 0) == 0 + and type(delivery[2]) is str + and delivery[2] == "", + "settled typed PG delivery", + ) + return seq + + +def chain_update(chain, index, value): + raw = json.dumps( + dict(index=index, value=value), sort_keys=True, allow_nan=False, separators=(",", ":") + ).encode() + chain.update(len(raw).to_bytes(8, "big")) + chain.update(raw) + + +async def verify_pages(conn, ch, context, workload, codec, native_hash): + db = database(context["databases"]["ch_schema"]) + seen_seq = set() + seen_hash = set() + count = 0 + expected_chain = hashlib.sha256() + observed_chain = hashlib.sha256() + authority_chain = hashlib.sha256() + raw_keys = hashlib.sha256() + for page in workload.pages(): + need(0 < len(page) <= 100, "fixed proof page") + ids = [item["memory_id"] for item in page] + complete_ids(ids, ids) + fetched = await conn.fetch( + pg_page_sql(), + context["org"], + context["namespace"], + [uuid.UUID(value) for value in ids], + len(page) + 1, + ) + need(len(fetched) == len(page), "PG page exact cardinality/overflow") + values = [pg_record(row, PG_FIELDS) for row in fetched] + complete_ids([identifier(row["c_memory_id"]) for row in values], ids) + heads = {identifier(row["c_memory_id"]): row for row in values} + checked = {} + audits = {} + hashes = [] + for item in page: + stored = heads[item["memory_id"]] + head = json_object(stored["head"]) + revision = json_object(stored["revision"]) + row = codec.row(head, origin="pg", comparison=True) + other = codec.row(revision, origin="pg", comparison=True) + equal(row, other) + audit = codec.audit(json_object(stored["audit"]), origin="pg") + native = dict(audit, ts=datetime.datetime.fromisoformat(audit["ts"])) + need(native_hash(native), "actual decoded native audit hash") + seq = typed_revision(stored, row, audit, item, seen_seq) + codec.expected( + row, + audit, + context, + api=item["kind"] == "api", + metadata=workload.metadata, + driver=workload.driver_header + or dict(host=socket.gethostname(), pid=workload.driver["pid"]), + ) + if item["kind"] == "delivery": + equal(row, codec.row(item["row"], comparison=True)) + equal(audit, item["audit"]) + need(audit["row_hash"] not in seen_hash, "unique workload audit identity") + seen_hash.add(audit["row_hash"]) + hashes.append(audit["row_hash"]) + checked[item["memory_id"]] = row + audits[audit["row_hash"]] = audit + intent = { + key: item[key] for key in ("kind", "key", "memory_id", "version", "content_sha256") + } + observed = dict( + intent, + key=row["memory_key"], + memory_id=row["memory_id"], + version=row["version"], + content_sha256=row["content_sha256"], + ) + if item["kind"] == "delivery": + intent.update(row=codec.row(item["row"], comparison=True), audit=item["audit"]) + observed.update(row=row, audit=audit) + chain_update(expected_chain, count, intent) + chain_update(observed_chain, count, observed) + chain_update(authority_chain, count, dict(row=row, audit=audit, seq=seq)) + chain_update(raw_keys, count, dict(head=sorted(head), revision=sorted(revision))) + count += 1 + fields = codec.p["memory_fields"] + data = await ch_result( + ch, + "SELECT " + + ",".join("`" + key + "`" for key in fields) + + ( + f" FROM `{db}`.memory FINAL WHERE org={{org:String}} AND " + f"namespace={{ns:String}} AND memory_id IN " + f"{{ids:Array(UUID)}} ORDER BY " + f"memory_id,agent,user_id,session_id,memory_key LIMIT " + f"{len(page) + 1}" + ), + dict( + org=context["org"], ns=context["namespace"], ids=[uuid.UUID(value) for value in ids] + ), + fields, + len(page) + 1, + ) + complete_ids([identifier(row[fields.index("memory_id")]) for row in data], ids) + for value in data: + row = codec.row(dict(zip(fields, value, strict=True)), origin="ch", comparison=True) + equal(row, checked[row["memory_id"]]) + fields = codec.p["audit_fields"] + data = await ch_result( + ch, + "SELECT " + + ",".join("`" + key + "`" for key in fields) + + ( + f" FROM `{db}`.audit_log WHERE org={{org:String}} AND " + f"namespace={{ns:String}} AND event_type='memory.revision' " + f"AND row_hash IN {{hashes:Array(String)}} ORDER BY row_hash " + f"LIMIT {len(page) + 1}" + ), + dict(org=context["org"], ns=context["namespace"], hashes=hashes), + fields, + len(page) + 1, + ) + complete_ids([hex64(row[fields.index("row_hash")], native=True) for row in data], hashes) + for value in data: + audit = codec.audit(dict(zip(fields, value, strict=True)), origin="ch") + equal(audit, audits[audit["row_hash"]]) + need(count == len(seen_seq) == len(seen_hash) == 45500, "complete exact workload identities") + need( + expected_chain.hexdigest() == observed_chain.hexdigest(), + "streamed intended/observed proof chain", + ) + return dict( + rows=count, + api=workload.counts["api"], + delivery=workload.counts["delivery"], + expected_sha256=expected_chain.hexdigest(), + observed_sha256=observed_chain.hexdigest(), + authority_sha256=authority_chain.hexdigest(), + pg_raw_key_sets_sha256=raw_keys.hexdigest(), + ) + + +def codec_contract(): + from server.memory.store.pg_memory import DATETIME_COLS + from server.memory.store.rows import FIRST_VERSION_DEFAULTS, MEMORY_PERSISTED_COLS + + specification = protocol() + codec = Codec(specification) + equal(list(MEMORY_PERSISTED_COLS), specification["memory_fields"]) + equal(FIRST_VERSION_DEFAULTS, specification["sparse_defaults"]) + equal( + sorted(DATETIME_COLS), + sorted( + key + for key, value in specification["memory_types"].items() + if value.endswith("utc_second") + ), + ) + return codec + + +async def independent_provenance(context, output): + from server.observability.audit import row_matches_hash + + codec = codec_contract() + context = dict( + context, _api_server=read(Path(context["workdir"]) / "serving-cell/launch.json")["server"] + ) + first = None + summary = None + try: + with Workload(context, codec) as workload: + async with connections(context) as (conn, ch): + try: + page = await begin_observation(conn, context, "repeatable read") + initial_ch = await ch_census(ch, context) + checked = await verify_pages( + conn, ch, context, workload, codec, row_matches_hash + ) + await end_observation(conn, page) + final = await begin_observation( + conn, context, "read committed", backend=page["backend_pid"] + ) + await end_observation(conn, final) + need( + conn.is_in_transaction() is False, + "final CH checks outside both PG snapshots", + ) + final_ch = await ch_census(ch, context) + summary = dict( + schema=2, + fixture_id=context["fixture_id"], + verified=True, + provenance=True, + **checked, + pg_page_observation=page, + pg_final_observation=final, + ch_initial_census=initial_ch, + ch_final_census=final_ch, + inputs=workload.refs, + source=dict( + head=context["head"], + tree=context["tree"], + hashes=context["source_hashes"], + ), + driver_birth=workload.driver, + api_birth=context["_api_server"], + mint_metadata=context["mint_metadata"], + hash_and_independent_stamps=True, + utc=utc(), + ) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error + finally: + try: + if conn.is_in_transaction(): + await conn.execute("ROLLBACK") + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + if first is not None: + raise first + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + if first is not None: + raise first + need(summary is not None, "complete proof before closure") + write(output, summary) + + +def api(context: dict, output: Path) -> None: + import uvicorn + + from bench.cell_trigger_gate import _workload + from server.core import deployment_config, instances + from server.core.provider_process import _start_ticks + from server.settings import settings + + actual = settings.model_dump(mode="json") + need( + actual["router_operator_token"] == "" + and re.fullmatch(r"lcr_[0-9a-f]{128}", actual["cell_router_token"]), + "API credential posture", + ) + candidate = dict(actual, cell_router_token="") + common = {k: v for k, v in actual.items() if k != "cell_router_token"} + folder = output.parent + artifact = write( + folder / "configuration.json", + { + "schema": 2, + "api_configuration_sha256": digest(actual), + "cli_configuration_sha256": digest(candidate), + "shared_configuration_sha256": digest(common), + "workload": _workload(settings), + "fingerprints": deployment_config.fingerprints(settings), + "credential_posture": { + "api_cell_router_token_present": True, + "cli_cell_router_token_present": False, + "api_router_operator_token_present": False, + "cli_router_operator_token_present": False, + }, + }, + ) + write(folder / "private-settings.json", actual, limit=1024 * 1024) + write( + output, + { + "schema": 2, + "server": { + "pid": os.getpid(), + "start_ticks": _start_ticks(os.getpid()), + "boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + "uid": os.geteuid(), + "cwd": os.getcwd(), + "health_instance": f"{socket.gethostname()}:{os.getpid()}", + "instance_id": instances.instance_id(), + }, + "source": {"head": context["head"], "tree": context["tree"]}, + "configuration_artifact": artifact, + }, + ) + uvicorn.run( + "server.main:app", + app_dir=context["root"], + host="127.0.0.1", + port=context["ports"]["api"], + loop="asyncio", + workers=1, + reload=False, + ) + + +@asynccontextmanager +async def connections(context: dict, *, admin: bool = False): + import asyncpg + import clickhouse_connect + + from server.core import pg + + conn = ch = None + first = None + try: + dsn = ( + context["pg_admin_dsn"] + if admin + else pg_dsn(context["pg_admin_dsn"], context["databases"]["pg_cell"]) + ) + conn = await asyncpg.connect( + dsn, + timeout=10, + command_timeout=30, + server_settings={ + "application_name": "trigger-fixture:" + context["fixture_id"], + "statement_timeout": "30000", + "idle_in_transaction_session_timeout": "30000", + }, + ) + values = {k: context["ch"][k] for k in ("host", "port", "username", "password")} + ch = await clickhouse_connect.get_async_client( + **values, + database="", + query_retries=0, + connect_timeout=10, + send_receive_timeout=30, + settings={"log_comment": "fixture.private", "async_insert": 0}, + ) + yield conn, ch + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error + finally: + if ch is not None: + try: + async with asyncio.timeout(15): + await ch.close() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + if conn is not None: + try: + await pg.close_owned(conn, end=time.monotonic() + 15) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + if first is not None: + raise first + + +async def rows(ch, query: str, parameters: dict | None = None) -> list[dict]: + result = await ch.query(query, parameters=parameters or {}) + return list(result.named_results()) + + +async def stores(context: dict, role: str, output: Path, *, drop: bool = False) -> None: + spec = context + db = database(spec["databases"][role]) + pg_role = role.startswith("pg_") + async with connections(spec, admin=True) as (conn, ch): + if drop: + creation = read(Path(spec["workdir"]) / (role + "-created.json")) + need( + creation["database"] == db and creation["fixture_id"] == spec["fixture_id"], + "exact created database", + ) + expected_id = spec["stores"]["pg_primary" if pg_role else "clickhouse"]["container_id"] + need(creation["store_container_id"] == expected_id, "created store identity") + if pg_role: + need( + await conn.fetchval( + "SELECT count(*) FROM pg_stat_activity WHERE datname=$1", db + ) + == 0, + "scratch PG sessions", + ) + exists = await conn.fetchval( + "SELECT EXISTS(SELECT FROM pg_database WHERE datname=$1)", db + ) + if exists: + need( + str(await conn.fetchval("SELECT oid FROM pg_database WHERE datname=$1", db)) + == creation["database_id"], + "same created PG database", + ) + await conn.execute(f'DROP DATABASE "{db}"') + else: + # Refuse asynchronous or running writes; do not flush a global queue. + aliases = [spec["databases"]["ch_schema"], spec["databases"]["ch_telemetry"]] + need( + ( + await rows( + ch, + ( + "SELECT count() AS n FROM system.asynchronous_inserts WHERE " + "database IN {dbs:Array(String)}" + ), + {"dbs": aliases}, + ) + ) + == [{"n": 0}], + "CH async writers", + ) + live = await rows( + ch, + ( + "SELECT count() AS n FROM system.processes WHERE query_id != " + "currentQueryID() AND (current_database IN " + "{dbs:Array(String)} OR arrayExists(d -> position(query, d) " + "> 0, {dbs:Array(String)}))" + ), + {"dbs": aliases}, + ) + need(live == [{"n": 0}], "CH running writers") + current = await rows( + ch, + "SELECT toString(uuid) AS id FROM system.databases WHERE name={db:String}", + {"db": db}, + ) + need( + not current or current == [{"id": creation["database_id"]}], + "same created CH database", + ) + await ch.command(f"DROP DATABASE IF EXISTS `{db}` SYNC") + write( + output, + { + "schema": 2, + "fixture_id": spec["fixture_id"], + "database": db, + "dropped": True, + "utc": utc(), + }, + ) + return + need(await conn.fetchval("SELECT NOT pg_is_in_recovery()"), "PG primary") + need( + 160000 <= int(await conn.fetchval("SHOW server_version_num")) < 170000, + "owned PG16 primary", + ) + need(await conn.fetchval("SHOW max_prepared_transactions") == "0", "prepared transactions") + need(await conn.fetchval("SHOW synchronous_standby_names"), "synchronous standby names") + need( + await conn.fetchval( + "SELECT EXISTS(SELECT FROM pg_stat_replication WHERE " + "state='streaming' AND sync_state='sync' AND flush_lsn IS " + "NOT NULL)" + ), + "synchronous standby", + ) + need( + await conn.fetchval("SELECT pg_size_bytes(current_setting('shared_buffers'))") + == spec["stores"]["pg_primary"]["shared_buffers_bytes"], + "primary buffers", + ) + import asyncpg + + standby = await asyncpg.connect( + spec["pg_standby_dsn"], + timeout=10, + command_timeout=10, + server_settings={ + "default_transaction_read_only": "on", + "statement_timeout": "10000", + "application_name": "trigger-fixture:" + context["fixture_id"], + }, + ) + standby_error = None + try: + need(await standby.fetchval("SELECT pg_is_in_recovery()"), "standby recovery") + need( + 160000 <= int(await standby.fetchval("SHOW server_version_num")) < 170000, + "owned PG16 standby", + ) + need( + await standby.fetchval("SELECT pg_size_bytes(current_setting('shared_buffers'))") + == spec["stores"]["pg_standby"]["shared_buffers_bytes"], + "standby buffers", + ) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + standby_error = error + try: + from server.core import pg + + await pg.close_owned(standby, end=time.monotonic() + 15) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + standby_error = error if standby_error is None else standby_error + if standby_error is not None: + raise standby_error + limits = await rows( + ch, "SELECT value FROM system.server_settings WHERE name='max_server_memory_usage'" + ) + need( + len(limits) == 1 + and int(limits[0]["value"]) + == spec["stores"]["clickhouse"]["server_limit_bytes"] + == 9 * GIB // 2, + "CH server memory cap", + ) + if pg_role: + need( + not await conn.fetchval( + "SELECT EXISTS(SELECT FROM pg_database WHERE datname=$1)", db + ), + "new PG database", + ) + write( + Path(spec["workdir"]) / (role + "-intent.json"), + { + "schema": 2, + "fixture_id": spec["fixture_id"], + "database": db, + "absent": True, + "utc": utc(), + }, + ) + await conn.execute(f"CREATE DATABASE \"{db}\" ENCODING 'UTF8' TEMPLATE template0") + need( + await conn.fetchval( + "SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname=$1", db + ) + == "UTF8", + "created UTF8", + ) + else: + need( + await rows( + ch, + "SELECT count() AS n FROM system.databases WHERE name={db:String}", + {"db": db}, + ) + == [{"n": 0}], + "new CH database", + ) + prefix = await rows( + ch, "SELECT value FROM system.server_settings WHERE name='keeper_map_path_prefix'" + ) + need( + len(prefix) == 1 + and prefix[0]["value"].rstrip("/") == spec["ch"]["keeper_prefix"].rstrip("/"), + "actual Keeper prefix", + ) + existing = await rows( + ch, + "SELECT name FROM system.zookeeper WHERE path={path:String}", + {"path": spec["ch"]["keeper_prefix"]}, + ) + need( + spec["databases"]["ch_schema"] not in {r["name"] for r in existing}, + "new private Keeper root", + ) + write( + Path(spec["workdir"]) / (role + "-intent.json"), + { + "schema": 2, + "fixture_id": spec["fixture_id"], + "database": db, + "absent": True, + "keeper_root_absent": True, + "utc": utc(), + }, + ) + await ch.command(f"CREATE DATABASE `{db}` ENGINE=Atomic") + identity = ( + str(await conn.fetchval("SELECT oid FROM pg_database WHERE datname=$1", db)) + if pg_role + else ( + await rows( + ch, + "SELECT toString(uuid) AS id FROM system.databases WHERE name={db:String}", + {"db": db}, + ) + )[0]["id"] + ) + write( + output, + { + "schema": 2, + "fixture_id": spec["fixture_id"], + "role": role, + "store_container_id": spec["stores"]["pg_primary" if pg_role else "clickhouse"][ + "container_id" + ], + "database": db, + "database_id": identity, + "created_at": utc(), + "created": True, + }, + ) + + +async def resolve_creation(context: dict, role: str, output: Path) -> None: + name = database(context["databases"][role]) + async with connections(context, admin=True) as (conn, ch): + if role.startswith("pg_"): + present = bool( + await conn.fetchval("SELECT EXISTS(SELECT FROM pg_database WHERE datname=$1)", name) + ) + else: + present = ( + await rows( + ch, + "SELECT count() AS n FROM system.databases WHERE name={db:String}", + {"db": name}, + ) + ) != [{"n": 0}] + # Presence after an unacknowledged create is not adoption authority. + write( + output, + { + "schema": 2, + "fixture_id": context["fixture_id"], + "role": role, + "database": name, + "present": present, + "utc": utc(), + }, + ) + + +def mint_retirement(group: list[dict], birth: dict, certified) -> dict: + matching = [] + for row in group: + parts = str(row["host_proof"]).split("|") + if len(parts) == 9 and (parts[0], parts[1], parts[4], parts[5]) == ( + socket.gethostname(), + birth["boot_id"], + str(birth["pid"]), + str(birth["start_ticks"]), + ): + matching.append(row) + need( + len(matching) == 1 + and matching[0]["state"] == "retired" + and certified(matching[0]["host_proof"]), + "mint native positive retirement", + ) + return { + "participant_id": str(matching[0]["participant_id"]), + "state": "retired", + "version": int(matching[0]["version"]), + "birth": birth, + } + + +async def final_retirement(context: dict, output: Path) -> None: + from server.core.provider_process import has_certificate + + targets = read(Path(context["workdir"]) / "final-participant-births.json")["processes"] + need(0 < len(targets) <= 16, "final participant target bound") + db = database(context["databases"]["ch_schema"]) + async with connections(context) as (_, ch): + group = await rows( + ch, + ( + f"SELECT participant_id,state,version,host_proof FROM " + f"`{db}`._provider_participants FINAL LIMIT 65" + ), + ) + need(len(group) <= 64, "final participant census bound") + observed = [] + for target in targets: + birth = ref(target["birth"]) + terminal = ref(target["terminal"]) + need(terminal["returncode"] == target["returncode"], "final terminal receipt") + observed.append( + dict( + role=target["role"], + terminal=target["terminal"], + **mint_retirement(group, birth, has_certificate), + ) + ) + write( + output, + { + "schema": 2, + "fixture_id": context["fixture_id"], + "retired": observed, + "verified": True, + "utc": utc(), + }, + ) + + +async def mint_proof(context: dict, output: Path) -> None: + from server.core.provider_process import has_certificate + + birth = ref(read(Path(context["workdir"]) / "mint-process.json")["launch_receipt"]) + db = database(context["databases"]["ch_schema"]) + async with connections(context) as (_, ch): + group = await rows( + ch, + ( + f"SELECT participant_id,state,version,host_proof FROM " + f"`{db}`._provider_participants FINAL LIMIT 65" + ), + ) + need(len(group) <= 64, "bounded native participant census") + observed = mint_retirement(group, birth, has_certificate) + write(output, {"schema": 2, "fixture_id": context["fixture_id"], **observed, "utc": utc()}) + + +async def proof( + context: dict, + output: Path, + *, + bootstrap: bool = False, + quiescent: bool = False, + keeper: bool = False, +) -> None: + from server.cli.cell_attach_beats import widened_window_s + from server.core import cell, cell_triggers + from server.settings import settings + + db = database(context["databases"]["ch_schema"]) + org, ns = context["org"], context["namespace"] + async with connections(context, admin=quiescent or keeper) as (conn, ch): + if keeper: + original = read(Path(context["workdir"]) / "cleanup-keeper-paths.json") + prefix = context["ch"]["keeper_prefix"].rstrip("/") + for entry in original["keeper_paths"]: + need(entry["path"].startswith(prefix + "/"), "recorded Keeper prefix") + parent = prefix + absent = False + for segment in entry["path"][len(prefix) + 1 :].split("/"): + children = await rows( + ch, + "SELECT name FROM system.zookeeper WHERE path={path:String}", + {"path": parent}, + ) + if segment not in {r["name"] for r in children}: + absent = True + break + parent += "/" + segment + need(absent, "owned Keeper path remains") + write( + output, + { + "schema": 2, + "fixture_id": context["fixture_id"], + "keeper_paths_absent": True, + "utc": utc(), + }, + ) + return + if quiescent: + names = [context["databases"]["pg_cell"], context["databases"]["pg_directory"]] + need( + await conn.fetchval( + "SELECT count(*) FROM pg_stat_activity WHERE datname=ANY($1::text[])", names + ) + == 0, + "all scratch PG sessions closed", + ) + need( + await conn.fetchval( + ( + "SELECT count(*) FROM pg_stat_activity WHERE " + "pid<>pg_backend_pid() AND application_name=$1" + ), + "trigger-fixture:" + context["fixture_id"], + ) + == 0, + "cut-off fixture admin sessions", + ) + aliases = [db, context["databases"]["ch_telemetry"]] + need( + await rows( + ch, + ( + "SELECT count() AS n FROM system.asynchronous_inserts WHERE " + "database IN {dbs:Array(String)}" + ), + {"dbs": aliases}, + ) + == [{"n": 0}], + "quiescent CH queue", + ) + need( + await rows( + ch, + ( + "SELECT count() AS n FROM system.processes WHERE query_id != " + "currentQueryID() AND (current_database IN " + "{dbs:Array(String)} OR arrayExists(d -> position(query, d) " + "> 0, {dbs:Array(String)}))" + ), + {"dbs": aliases}, + ) + == [{"n": 0}], + "quiescent CH writers", + ) + keeper_paths = [] + for row in await rows( + ch, + ( + "SELECT database,name,create_table_query FROM system.tables " + "WHERE database IN {dbs:Array(String)} AND engine='KeeperMap'" + ), + {"dbs": aliases}, + ): + match = re.search(r"KeeperMap\('([^']+)'", row["create_table_query"]) + need(match and match[1].startswith("/" + db + "/"), "quiescent private Keeper path") + keeper_paths.append( + { + "database": row["database"], + "table": row["name"], + "path": context["ch"]["keeper_prefix"].rstrip("/") + match[1], + } + ) + need(len(keeper_paths) <= 64, "Keeper census bound") + write( + output, + { + "schema": 2, + "fixture_id": context["fixture_id"], + "pg_sessions": 0, + "ch_writers": 0, + "keeper_paths": keeper_paths, + "utc": utc(), + }, + ) + return + authority = dict( + await conn.fetchrow("SELECT authority,deployment_id,epoch FROM loom.authority") + ) + authority["deployment_id"] = str(authority["deployment_id"]) + if bootstrap: + need( + authority["authority"] == "postgres" and authority["epoch"] > 0, + "native boot authority", + ) + pair = await rows( + ch, f"SELECT authority,deployment_id,epoch FROM `{db}`.storage_authority FINAL" + ) + need( + len(pair) == 1 and all(str(pair[0][k]) == str(v) for k, v in authority.items()), + "native boot pair", + ) + params = {"org": org, "ns": ns} + need( + await rows( + ch, + ( + f"SELECT owner_email FROM `{db}`.namespaces FINAL WHERE " + f"org={{org:String}} AND namespace={{ns:String}}" + ), + params, + ) + == [{"owner_email": "benchmark"}], + "native registered namespace", + ) + need( + await rows( + ch, + ( + f"SELECT subject,role FROM `{db}`.namespace_grants FINAL " + f"WHERE org={{org:String}} AND namespace={{ns:String}} AND " + f"subject='benchmark' AND revoked_at IS NULL" + ), + params, + ) + == [{"subject": "benchmark", "role": "admin"}], + "native registration grant", + ) + need( + await conn.fetchval("SELECT count(*) FROM loom.memory_current") == 0, + "no bootstrap memories", + ) + write( + output, + { + "schema": 2, + "fixture_id": context["fixture_id"], + **authority, + "namespace_registered": True, + "admin_grant": True, + "memory_rows": 0, + "utc": utc(), + }, + ) + return + first = await conn.fetchrow("SELECT is_first,cell FROM loom.cell_first") + need( + first and first["is_first"] is True and first["cell"] == context["cell"], + "first cell tuple", + ) + need(await conn.fetchval("SELECT mode FROM loom.tenancy_mode") == "enforced", "enforced") + need( + await conn.fetchval("SELECT EXISTS(SELECT FROM loom.tenancy WHERE org=$1)", org), + "held fixture org", + ) + own = await conn.fetchrow("SELECT current_fp,next_fp FROM loom.cell_self") + accepted = ( + json.loads(own["current_fp"]) + if isinstance(own["current_fp"], str) + else own["current_fp"] + ) + need( + own["next_fp"] is None + and cell.matches(cell.fingerprint(str(authority["deployment_id"])), accepted), + "accepted stable fingerprint", + ) + need(not await cell_triggers.missing(conn), "canonical triggers") + paired = await rows( + ch, f"SELECT authority,deployment_id,epoch FROM `{db}`.storage_authority FINAL" + ) + need( + len(paired) == 1 and all(str(paired[0][k]) == str(v) for k, v in authority.items()), + "paired authority", + ) + for table in ("provider_configuration", "runtime_overrides"): + need( + await rows(ch, f"SELECT count() AS n FROM `{db}`.{table} FINAL") == [{"n": 0}], + "empty saved configuration", + ) + need( + await conn.fetchval( + "SELECT (SELECT count(*) FROM loom.memory_deliveries WHERE " + "targets<>0)+(SELECT count(*) FROM loom.audit_outbox)" + ) + == 0, + "settled setup", + ) + instance = context["api_instance_id"] + from types import SimpleNamespace + + from server.core import deployment_config, instances + from server.core.ch_names import NamedClient + + named = NamedClient(ch) + adapter = SimpleNamespace(query_all=lambda sql, params=None: rows(named, sql, params)) + live = await instances.read_live(adapter, all_deployments=True) + need( + len(live) == 1 + and live[0]["instance_id"] == instance + and live[0]["serving"] == 1 + and live[0]["build_commit"] == context["head"] + and live[0]["host"] == socket.gethostname() + and not live[0]["workers_lease"], + "single serving API source", + ) + need( + all(live[0].get(k) == v for k, v in deployment_config.fingerprints(settings).items()), + "live API fingerprints", + ) + from server.core.provider_process import has_certificate + + launch = read(Path(context["workdir"]) / "serving-cell/launch.json")["server"] + participants = await rows( + ch, + ( + f"SELECT participant_id,state,host_proof FROM " + f"`{db}`._provider_participants FINAL LIMIT 65" + ), + ) + need(0 < len(participants) <= 64, "bounded native participant evidence") + serving = 0 + for participant in participants: + participant["participant_id"] = str(participant["participant_id"]) + proof_value = participant["host_proof"] + need(has_certificate(proof_value), "native participant capability") + parts = proof_value.split("|") + own = (parts[0], parts[1], parts[4], parts[5]) == ( + socket.gethostname(), + launch["boot_id"], + str(launch["pid"]), + str(launch["start_ticks"]), + ) + if own: + need(participant["state"] == "running", "serving native participant") + serving += 1 + else: + need( + participant["state"] == "retired", + "prior native participant not positively retired", + ) + need(serving == 1, "single native serving participant") + beat = await rows( + ch, + ( + f"SELECT fingerprint,cell,tenancy_mode FROM " + f"`{db}`.cell_heartbeats WHERE instance_id={{id:String}}" + ), + {"id": instance}, + ) + need(len(beat) == 1, "API heartbeat") + value = json.loads(beat[0]["fingerprint"]) + need( + value.get("serves") is True + and value.get("marked") is True + and value.get("router") is True, + "native API router state", + ) + namespace = await rows( + ch, + ( + f"SELECT owner_email FROM `{db}`.namespaces FINAL WHERE " + f"org={{org:String}} AND namespace={{ns:String}}" + ), + {"org": org, "ns": ns}, + ) + need(namespace == [{"owner_email": "benchmark"}], "native namespace owner") + keeper = [] + for row in await rows( + ch, + ( + "SELECT database,name,create_table_query FROM system.tables " + "WHERE database IN {dbs:Array(String)} AND engine='KeeperMap'" + ), + {"dbs": [db, context["databases"]["ch_telemetry"]]}, + ): + match = re.search(r"KeeperMap\('([^']+)'", row["create_table_query"]) + need(match and match[1].startswith("/" + db + "/"), "private actual Keeper path") + keeper.append( + { + "database": row["database"], + "table": row["name"], + "path": context["ch"]["keeper_prefix"].rstrip("/") + match[1], + } + ) + write( + output, + { + "schema": 2, + "fixture_id": context["fixture_id"], + **authority, + "first": [True, context["cell"]], + "mode": "enforced", + "held": True, + "keeper_paths": keeper, + "native_participants": participants, + "widened_window_s": widened_window_s(), + "utc": utc(), + }, + ) + + +class Fixture: + def __init__(self, spec: dict, owner) -> None: + self.s, self.o = dict(spec), owner + self.w = Path(spec["workdir"]) + self.receipts = [] + self.created = [] + self.teardown_incomplete = False + self.retirement_attempted = False + self.retirement_receipt = None + self.settings: dict = {} + self.api = None + self.cell = "gate-" + spec["fixture_id"][:12] + self.org, self.namespace = ( + "trigger-" + spec["fixture_id"], + "trigger-" + spec["fixture_id"][:12], + ) + self.s.update(cell=self.cell, org=self.org, namespace=self.namespace) + self.context = self.w / "context.json" + write(self.context, self.s) + + def env(self, settings: dict | None = None, *, ca: bool = False) -> dict: + return child_env(self.s, self.settings if settings is None else settings, ca=ca) + + def cli(self, *args: str, settings: dict | None = None, seconds: float = 120): + return self.o.command( + args[0], + [self.s["python"], "-m", "server.cli", *args], + self.env(settings, ca=True), + seconds=seconds, + ) + + def receipt(self, operation: str, value: dict) -> None: + reference = write( + self.w / (operation + "-receipt.json"), {"schema": 2, "operation": operation, **value} + ) + self.receipts.append({"operation": operation, "receipt": reference}) + + def command_receipt(self, operation: str, item: dict, **observations) -> None: + self.receipt( + operation, + { + "returncode": item["process"].returncode, + **observations, + "process": self.process_receipt(item), + "log": { + "path": str(item["log"]), + "sha256": sha(item["log"]), + "bytes": item["log"].stat().st_size, + }, + "utc": utc(), + }, + ) + + def action( + self, task: str, *, role: str = "", settings: dict | None = None, cleanup: bool = False + ) -> dict: + target = self.w / ( + role + "-created.json" + if task == "create" + else task + ("-" + role if role else "") + ".json" + ) + if task == "quiescent": + target = self.w / ("quiescent-" + str(self.o.serial + 1) + ".json") + argv = [ + self.s["python"], + str(HERE / FILES[1]), + "--task", + task, + "--context", + str(self.context), + "--output", + str(target), + ] + if role: + argv += ["--role", role] + try: + self.o.command( + task, + argv, + self.env(settings), + seconds=600 if task == "provenance" else 120, + cleanup=cleanup, + ) + except BaseException as error: + # A terminal cleanup refusal is recorded, never retried as another mutation. + if cleanup and self.o.leaders and self.o.leaders[-1]["terminal"] is not None: + self.o.select(error) + return {"incomplete": True, "task": task, "error_type": type(error).__name__} + raise + return read(target) + + def http( + self, path: str, *, body: dict | None = None, token: str = "", tls: bool = False + ) -> dict: + import ssl + + import httpx + + port = self.s["ports"]["tls" if tls else "api"] + verify = ssl.create_default_context(cafile=str(self.w / "ca.crt")) if tls else True + context = httpx.Client(timeout=10, follow_redirects=False, trust_env=False, verify=verify) + client = context.__enter__() + try: + response = client.request( + "POST" if body is not None else "GET", + f"{'https' if tls else 'http'}://127.0.0.1:{port}{path}", + json=body, + headers={"Authorization": "Bearer " + token} if token else {}, + ) + need(response.status_code == 200, "native HTTP refused") + value = response.json() + need(isinstance(value, dict), "native HTTP shape") + except BaseException as error: + try: + context.__exit__(type(error), error, error.__traceback__) + except BaseException as secondary: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.o.select(error) + self.o.select(secondary) + raise + else: + context.__exit__(None, None, None) + self.o.monitor() + return value + + def wait_api(self) -> None: + import httpx + + end = min(time.monotonic() + 120, self.o.started + 7020) + while True: + self.o.monitor() + try: + value = self.http("/v1/health") + ready = value.get("readiness", {}) + checks = ready.get("checks", {}) + required = { + "draining", + "clickhouse", + "migrations", + "revocation", + "keyring", + "tracker_registry", + "runtime_overrides", + "calibration", + "config_drift", + "cell", + "sandbox", + } + if self.settings["cell"]: + required.add("cell_marker") + need( + value.get("ch_ok") is True + and value.get("pg_ok") is True + and value.get("storage_authority") == "postgres" + and value.get("pending_migrations") == [] + and ready.get("ready") is True + and ready.get("failing") == [] + and required <= set(checks) + and all( + checks[name].get("required", True) is True + and checks[name].get("ok") is True + for name in required + ) + and all( + isinstance(check, dict) + and type(check.get("ok")) is bool + and (check.get("required", True) is False or check["ok"] is True) + for check in checks.values() + ), + "healthy ready API stores", + ) + return + except (OSError, httpx.TransportError, Refused): + need(time.monotonic() < end, "API boot deadline") + time.sleep(1) + + def start_api(self, stage: str) -> None: + folder = self.w / stage + folder.mkdir(mode=0o700) + self.api = self.o.start( + "api-" + stage, + [ + self.s["python"], + str(HERE / FILES[1]), + "--task", + "api", + "--context", + str(self.context), + "--output", + str(folder / "launch.json"), + ], + self.env(), + persistent=True, + ) + self.wait_api() + self.launch = read(folder / "launch.json") + self.launch_path = folder / "launch.json" + + def setup(self) -> None: + import secrets + + # docker inspect is read-only and the input names exact positive IDs. + self.inspect_stores() + for role in ("pg_cell", "pg_directory", "ch_schema", "ch_telemetry"): + output = self.w / (role + "-created.json") + self.action("create", role=role) + # Native action writes the authoritative creation receipt, not this observation. + need(read(output)["created"] is True, "actual database creation receipt") + self.created.append(role) + cell = self.cli("cell", "credential", "new", settings={}) + operator = self.cli("cell", "credential", "new", "--operator", settings={}) + + def credential(item, prefix): + text = item["log"].read_text() + match = re.search(r"^credential: (" + prefix + r"[0-9a-f]{128})$", text, re.M) + need(match, "native credential output") + return match[1] + + cell_token = credential(cell, "lcr_") + self.operator = credential(operator, "lco_") + p, db = self.s["ports"], self.s["databases"] + self.settings = dict( + ch_host="127.0.0.1", + ch_port=self.s["ch"]["port"], + ch_user=self.s["ch"]["username"], + ch_password=self.s["ch"]["password"], + ch_database=db["ch_schema"], + ch_telemetry_database=db["ch_telemetry"], + pg_dsn=pg_dsn(self.s["pg_admin_dsn"], db["pg_cell"]), + storage_authority="postgres", + pg_durability="synchronous", + embedding_provider="fake", + embedding_model="fake-ngram-1024", + embedding_dim=1024, + llm_provider="fake", + llm_model="fake-echo-1", + require_real_providers=False, + background_workers="off", + pg_delivery_batch_rows=500, + auth_required=True, + identity_provider="gateway", + self_signup="off", + mcp_auth_mode="required", + jwt_secret=secrets.token_hex(32), + scim_token="", + cell="", + cell_probe_port=p["probe"], + cell_router_url=f"http://127.0.0.1:{p['router']}", + cell_router_token=cell_token, + router_operator_token="", + public_base_url=f"https://127.0.0.1:{p['tls']}", + build_commit=self.s["head"], + bind_host="127.0.0.1", + document_s3_bucket="", + ) + router = dict( + self.settings, + cell_router_token="", + router_pg_dsn=pg_dsn(self.s["pg_admin_dsn"], db["pg_directory"]), + router_operator_sha256=__import__("hashlib").sha256(self.operator.encode()).hexdigest(), + router_shared_host=f"127.0.0.1:{p['tls']}", + router_upstream_url=f"http://127.0.0.1:{p['api']}", + router_upstream_probe_url=f"http://127.0.0.1:{p['probe']}", + ) + self.command_receipt("directory_migrate", self.cli("router", "migrate", settings=router)) + self.o.start( + "router", + [ + self.s["python"], + "-m", + "server.cli", + "router", + "serve", + "--host", + "127.0.0.1", + "--port", + str(p["router"]), + ], + self.env(router), + persistent=True, + ) + self.certificate() + self.start_api("before-cell") + self.o.start( + "proxy", + [ + self.s["python"], + str(HERE / FILES[2]), + "--listen-port", + str(p["tls"]), + "--upstream-port", + str(p["api"]), + "--certificate", + str(self.w / "server.crt"), + "--key", + str(self.w / "server.key"), + ], + self.env({}), + persistent=True, + ) + minted = self.cli( + "mint-token", + "--subject", + "benchmark", + "--org", + self.org, + "--namespaces", + self.namespace, + "--roles", + "writer", + "--service-account", + "--ttl-days", + "1", + "--save", + str(self.w / "writer.jwt"), + "--json", + ) + write(self.w / "mint-process.json", self.process_receipt(minted)) + retirement = self.action("mint-proof") + metadata, self.token = mint_metadata(self.s, self.process_receipt(minted)) + self.s["mint_metadata"] = write(self.w / "mint-metadata.json", metadata) + self.command_receipt( + "mint", minted, retirement=retirement, metadata=self.s["mint_metadata"] + ) + result = self.http( + "/v1/namespace.register", + token=self.token, + body={"org": self.org, "namespace": self.namespace, "owner_email": "benchmark"}, + ) + need( + result.get("registered") is True and result.get("namespace") == self.namespace, + "native namespace registration", + ) + write(self.w / "namespace-register.json", result) + self.receipt("namespace_register", result) + self.receipt("authority_boot", self.action("bootstrap-proof")) + import httpx + + end = min(time.monotonic() + 30, self.o.started + 7020) + while True: + self.o.monitor() + try: + self.http("/v1/health/deployment", tls=True) + break + except httpx.TransportError: + need(time.monotonic() < end, "TLS listener startup") + time.sleep(0.1) + operator_settings = dict( + self.settings, cell=self.cell, router_operator_token=self.operator, cell_router_token="" + ) + self.command_receipt( + "prepare", + self.cli( + "cell", + "prepare", + "--cell", + self.cell, + "--host", + f"127.0.0.1:{p['tls']}", + "--credential-sha256", + __import__("hashlib").sha256(cell_token.encode()).hexdigest(), + settings=operator_settings, + seconds=600, + ), + ) + self.o.stop_one(self.api, api=True) + self.settings["cell"] = self.cell + self.start_api("serving-cell") + # Read the actual source's widened interval in a bounded metadata child. + interval = self.action("window")["seconds"] + end = time.monotonic() + interval + while time.monotonic() < end: + self.o.monitor() + time.sleep(1) + self.command_receipt( + "attach", + self.cli( + "cell", + "attach", + "--cell", + self.cell, + "--wait", + "600", + "--scan-budget", + "600", + "--ch-allow-user", + self.s["ch"]["username"], + settings=operator_settings, + seconds=650, + ), + ) + self.s["api_instance_id"] = self.launch["server"]["instance_id"] + self.context = self.w / "attached-context.json" + write(self.context, self.s) + self.cli_settings = dict(self.settings, cell_router_token="") + self.attached = self.action("proof", settings=self.cli_settings) + self.receipt("post_attach_proof", self.attached) + + def inspect_stores(self, *, cleanup: bool = False) -> None: + for expected in self.s["stores"].values(): + command = self.o.command( + "inspect", + ["docker", "inspect", expected["container_id"]], + self.env({}), + cleanup=cleanup, + ) + raw = json.loads(command["log"].read_text()) + need(len(raw) == 1, "one inspected container") + actual = raw[0] + need( + actual["Id"] == expected["container_id"] + and actual["Created"] == expected["created_at"] + and actual["State"]["Running"] is True + and actual["State"]["Pid"] == expected["host_pid"] + and actual["HostConfig"]["Memory"] == expected["memory_limit_bytes"], + "current owned container", + ) + observed = [] + for internal, bindings in actual["NetworkSettings"]["Ports"].items(): + for binding in bindings or []: + observed.append( + { + "host": binding["HostIp"], + "host_port": int(binding["HostPort"]), + "internal_port": int(internal.split("/")[0]), + "protocol": internal.split("/")[1], + } + ) + need( + sorted(observed, key=digest) == sorted(expected["ports"], key=digest), + "actual loopback bindings", + ) + need( + expected["disk_path"] in {m["Source"] for m in actual["Mounts"]}, + "owned store filesystem", + ) + identity = self.o.owned.identity(expected["host_pid"]) + need( + identity + and identity[1] == expected["start_ticks"] + and os.stat(f"/proc/{expected['host_pid']}").st_uid == expected["uid"] + and Path("/proc/sys/kernel/random/boot_id").read_text().strip() + == expected["boot_id"], + "store process identity", + ) + + def certificate(self) -> None: + def openssl(*args): + self.o.command("openssl", ["openssl", *args], self.env({})) + + previous = os.umask(0o077) + try: + openssl( + "req", + "-x509", + "-newkey", + "rsa:2048", + "-noenc", + "-keyout", + str(self.w / "ca.key"), + "-out", + str(self.w / "ca.crt"), + "-days", + "1", + "-subj", + "/CN=local-trigger-ca", + ) + openssl( + "req", + "-newkey", + "rsa:2048", + "-noenc", + "-keyout", + str(self.w / "server.key"), + "-out", + str(self.w / "server.csr"), + "-subj", + "/CN=127.0.0.1", + ) + extension = self.w / "server.ext" + extension.write_text("subjectAltName=IP:127.0.0.1\nextendedKeyUsage=serverAuth\n") + openssl( + "x509", + "-req", + "-in", + str(self.w / "server.csr"), + "-CA", + str(self.w / "ca.crt"), + "-CAkey", + str(self.w / "ca.key"), + "-CAcreateserial", + "-out", + str(self.w / "server.crt"), + "-days", + "1", + "-extfile", + str(extension), + ) + finally: + os.umask(previous) + openssl("verify", "-CAfile", str(self.w / "ca.crt"), str(self.w / "server.crt")) + dates = self.o.command( + "certificate-dates", + ["openssl", "x509", "-in", str(self.w / "server.crt"), "-noout", "-dates"], + self.env({}), + ) + values = dict(line.split("=", 1) for line in dates["log"].read_text().splitlines()) + parsed = { + key: datetime.datetime.strptime(value, "%b %d %H:%M:%S %Y %Z") + .replace(tzinfo=datetime.UTC) + .isoformat() + for key, value in values.items() + } + private = write( + self.w / "tls-files.json", + { + "schema": 2, + "files": [ + { + "path": str(self.w / name), + "uid": (self.w / name).stat().st_uid, + "mode": (self.w / name).stat().st_mode & 0o777, + "bytes": (self.w / name).stat().st_size, + "sha256": sha(self.w / name), + } + for name in ("ca.key", "ca.crt", "server.key", "server.crt") + ], + }, + ) + self.tls = { + "ca_certificate_sha256": sha(self.w / "ca.crt"), + "server_certificate_sha256": sha(self.w / "server.crt"), + "server_ip_san": "127.0.0.1", + "not_before": parsed["notBefore"], + "not_after": parsed["notAfter"], + "listener": f"https://127.0.0.1:{self.s['ports']['tls']}", + "upstream": f"http://127.0.0.1:{self.s['ports']['api']}", + "private_files_receipt": private, + } + + def measure(self) -> str: + # Receipt construction is explicit; no old single-digest schema is accepted. + config = ref(self.launch["configuration_artifact"]) + ownership = self.ownership() + ownership_ref = write(self.w / "ownership.json", ownership) + p, db = self.s["ports"], self.s["databases"] + fixture = dict( + schema=2, + base_url=f"http://127.0.0.1:{p['api']}", + pg_host="127.0.0.1", + pg_port=url_port(self.s["pg_admin_dsn"]), + pg_database=db["pg_cell"], + pg_internal_port=5432, + ch_host="127.0.0.1", + ch_port=self.s["ch"]["port"], + ch_database=db["ch_schema"], + ch_telemetry_database=db["ch_telemetry"], + cell=self.cell, + org=self.org, + namespace=self.namespace, + deployment_id=str(self.attached["deployment_id"]), + epoch=self.attached["epoch"], + instance_id=self.launch["server"]["instance_id"], + build_commit=self.s["head"], + source_tree=self.s["tree"], + launch_manifest_sha256=sha(self.launch_path), + ownership_receipt_sha256=ownership_ref["sha256"], + background_workers="off", + controller_excluded=True, + **{ + k: config[k] + for k in ( + "api_configuration_sha256", + "cli_configuration_sha256", + "shared_configuration_sha256", + ) + }, + ) + write(self.w / "fixture.json", fixture) + self.action("resolve", settings=self.cli_settings) + metadata = ref(self.s["mint_metadata"]) + need(verify_token(metadata) == self.token, "native bearer unchanged before driver") + item = self.o.start( + "driver", + [ + self.s["python"], + str(HERE / FILES[3]), + "--context", + str(self.context), + "--capture-directory", + str(self.w / "delivery-intent"), + "--", + "--base-url", + fixture["base_url"], + "--pg-dsn", + self.cli_settings["pg_dsn"], + "--token", + self.token, + "--org", + self.org, + "--namespace", + self.namespace, + "--fixture-manifest", + str(self.w / "fixture.json"), + "--launch-manifest", + str(self.launch_path), + "--output", + str(self.w / "driver.jsonl"), + ], + self.env(self.cli_settings), + ) + write( + self.w / "driver-birth.json", + dict(self.process_receipt(item), schema=2, fixture_id=self.s["fixture_id"]), + ) + while self.o.poll(item) is None: + self.o.monitor() + time.sleep(0.05) + write( + self.w / "driver-finished.json", + dict(self.process_receipt(item), schema=2, fixture_id=self.s["fixture_id"]), + ) + need(verify_token(metadata) == self.token, "native bearer unchanged after driver") + lines = item["log"].read_bytes().splitlines() + need(lines and len(lines[-1]) <= 65536, "driver final result") + result = json.loads(lines[-1]) + need( + result.get("verdict") in {"PASS", "FAIL"} + and item["process"].returncode == (0 if result["verdict"] == "PASS" else 1), + "completed driver result", + ) + return result["verdict"] + + def process_receipt(self, item: dict) -> dict: + identity = item["identity"] + args = list(item["argv"]) + for index, value in enumerate(args[:-1]): + if value in {"--token", "--pg-dsn"}: + args[index + 1] = "" + return { + "role": item["role"], + "pid": item["process"].pid, + "start_ticks": identity[1], + "boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + "uid": os.geteuid(), + "cwd": self.s["root"], + "executable": item["argv"][0], + "executable_sha256": sha(Path(item["argv"][0])) + if Path(item["argv"][0]).is_absolute() + else sha(Path(__import__("shutil").which(item["argv"][0]))), + "source_sha256": digest(self.s["source_hashes"]), + "configuration_sha256": item["configuration_sha256"], + "redacted_argv": args, + "launch_receipt": item["launch_receipt"], + "terminal_receipt": item["terminal_receipt"], + } + + def ownership(self) -> dict: + # All entries are observations from completed setup, never a future driver PID. + need( + len(self.o.leaders) <= 64 and len(self.attached["keeper_paths"]) <= 64, + "ownership entry bounds", + ) + return { + "schema": 2, + "fixture_id": self.s["fixture_id"], + "source": { + "root": self.s["root"], + "head": self.s["head"], + "tree": self.s["tree"], + "source_hashes": self.s["source_hashes"], + "driver_sha256": self.s["source_hashes"]["driver"], + "launcher_sha256": self.s["source_hashes"][FILES[0]], + "proxy_sha256": self.s["source_hashes"][FILES[2]], + "owned_children_sha256": self.s["source_hashes"]["owned_children.py"], + "python_executable": self.s["python"], + "python_sha256": self.s["python_sha256"], + "lockfile_sha256": self.s["lockfile_sha256"], + "dependency_receipt": self.s["dependency_receipt"], + }, + "stores": [ + {k: v for k, v in s.items() if k != "disk_path"} for s in self.s["stores"].values() + ], + "created_databases": [ + { + "role": role, + "store_container_id": read(self.w / (role + "-created.json"))[ + "store_container_id" + ], + "database": self.s["databases"][role], + "created_at": read(self.w / (role + "-created.json"))["created_at"], + "creation_receipt": { + "path": str(self.w / (role + "-created.json")), + "sha256": sha(self.w / (role + "-created.json")), + }, + "drop_required": True, + } + for role in self.created + ], + "keeper_paths": [ + dict( + row, + store_container_id=self.s["stores"]["clickhouse"]["container_id"], + creation_receipt={ + "path": str(self.w / "ch_schema-created.json"), + "sha256": sha(self.w / "ch_schema-created.json"), + }, + drop_required=True, + ) + for row in self.attached["keeper_paths"] + ], + "processes": [self.process_receipt(item) for item in self.o.leaders], + "tls": self.tls, + "native_receipts": self.receipts, + "controller_excluded": True, + "resource_policy": POLICY, + } + + def provenance(self) -> None: + self.action("provenance", settings=self.cli_settings) + + def block_teardown(self, error: BaseException) -> None: + # Retain the mutation barrier even if the negative receipt cannot be created. + self.teardown_incomplete = True + self.o.select(error) + try: + if not (self.w / "cleanup-incomplete.json").exists(): + write( + self.w / "cleanup-incomplete.json", + { + "schema": 2, + "fixture_id": self.s["fixture_id"], + "reason": "required cleanup evidence incomplete; stores retained", + "utc": utc(), + }, + ) + except BaseException as secondary: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.o.select(secondary) + + def retirement_evidence(self, retired: dict, final: list[dict]) -> None: + closed(retired, "schema fixture_id retired verified utc") + need( + type(retired["schema"]) is int + and retired["schema"] == 2 + and retired["fixture_id"] == self.s["fixture_id"] + and retired["verified"] is True + and type(retired["retired"]) is list + and len(retired["retired"]) == len(final), + "acknowledged complete final retirement", + ) + for value, item in zip(retired["retired"], final, strict=True): + closed(value, "role terminal participant_id state version birth") + equal(value["role"], item["role"]) + equal(value["terminal"], item["terminal_receipt"]) + equal(value["birth"], ref(item["launch_receipt"])) + terminal = ref(item["terminal_receipt"]) + need( + type(terminal["returncode"]) is int + and terminal["returncode"] == item["process"].returncode, + "final retirement terminal binding", + ) + need(value["state"] == "retired", "positive native retirement state") + identifier(value["participant_id"]) + integer(value["version"], 1, 2**63 - 1) + + def teardown(self) -> None: + # Only creation receipts authorize deletion, even after partial setup. + if not any( + (self.w / (role + suffix)).exists() + for role in self.s["databases"] + for suffix in ("-intent.json", "-created.json") + ): + return + self.inspect_stores(cleanup=True) + for port in self.s["ports"].values(): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.settimeout(1) + need(probe.connect_ex(("127.0.0.1", port)) != 0, "fixture port remains open") + quiescent = self.action("quiescent", settings={}, cleanup=True) + need( + quiescent.get("ch_writers") == quiescent.get("pg_sessions") == 0, + "server writer settlement unknown", + ) + if self.teardown_incomplete or (self.w / "cleanup-incomplete.json").exists(): + self.teardown_incomplete = True + self.o.select(Refused("known incomplete teardown; no mutation retry")) + return + if not (self.w / "cleanup-keeper-paths.json").exists(): + write(self.w / "cleanup-keeper-paths.json", {"keeper_paths": quiescent["keeper_paths"]}) + final = [item for item in self.o.leaders if item["role"].startswith(("api-", "driver-"))] + if final: + try: + if self.retirement_receipt is None: + need( + not self.retirement_attempted + and not (self.w / "final-retirement.json").exists(), + "unacknowledged final retirement artifact; no retry", + ) + self.retirement_attempted = True + need( + all( + item["terminal_receipt"] is not None + and item["launch_receipt"] is not None + for item in final + ), + "final native process receipts", + ) + write( + self.w / "final-participant-births.json", + { + "processes": [ + { + "role": item["role"], + "birth": item["launch_receipt"], + "terminal": item["terminal_receipt"], + "returncode": item["process"].returncode, + } + for item in final + ] + }, + ) + retired = self.action("final-retirement", settings={}, cleanup=True) + self.retirement_evidence(retired, final) + receipt = { + "path": str(self.w / "final-retirement.json"), + "sha256": sha(self.w / "final-retirement.json"), + } + equal(ref(receipt), retired) + self.retirement_receipt = receipt + self.retirement_evidence(ref(self.retirement_receipt), final) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.block_teardown(error) + return + for role in self.s["databases"]: + if (self.w / (role + "-intent.json")).exists() and not ( + self.w / (role + "-created.json") + ).exists(): + state = self.action("resolve-creation", role=role, settings={}, cleanup=True) + if state.get("present") is not False: + self.block_teardown(Refused("uncertain database ownership; leave untouched")) + return + for role in ("ch_telemetry", "ch_schema", "pg_directory", "pg_cell"): + created = self.w / (role + "-created.json") + if created.exists() and not (self.w / ("drop-" + role + ".json")).exists(): + result = self.action("drop", role=role, settings={}, cleanup=True) + if result.get("dropped") is not True: + self.block_teardown(Refused("native scratch cleanup incomplete")) + # A failed DROP may still run on the server; prove settlement afresh. + settled = self.action("quiescent", settings={}, cleanup=True) + need( + settled.get("ch_writers") == settled.get("pg_sessions") == 0, + "drop outcome still running", + ) + return + if not (self.w / "keeper-proof.json").exists(): + result = self.action("keeper-proof", settings={}, cleanup=True) + if result.get("keeper_paths_absent") is not True: + self.block_teardown(Refused("owned Keeper cleanup incomplete")) + return + if not (self.w / "teardown.json").exists(): + write( + self.w / "teardown.json", + { + "schema": 2, + "fixture_id": self.s["fixture_id"], + "dropped": [ + role + for role in self.s["databases"] + if (self.w / ("drop-" + role + ".json")).exists() + ], + "owned_ports_closed": True, + "pg_sessions_closed": True, + "job_container_cleanup_required": True, + "utc": utc(), + }, + ) + + +def url_port(dsn: str) -> int: + from urllib.parse import urlsplit + + port = urlsplit(dsn).port + need(port is not None, "explicit PG port") + return port + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--task", + required=True, + choices=[ + "api", + "create", + "drop", + "resolve-creation", + "mint-proof", + "final-retirement", + "proof", + "provenance", + "bootstrap-proof", + "quiescent", + "keeper-proof", + "window", + "resolve", + ], + ) + parser.add_argument("--context", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--role", default="") + args = parser.parse_args() + try: + context = read(args.context) + sys.path.insert(0, context["root"]) + if args.task == "api": + api(context, args.output) + elif args.task in {"create", "drop"}: + need(args.role in context["databases"], "fixed database role") + asyncio.run(stores(context, args.role, args.output, drop=args.task == "drop")) + elif args.task == "resolve-creation": + need(args.role in context["databases"], "fixed database role") + asyncio.run(resolve_creation(context, args.role, args.output)) + elif args.task == "window": + from server.cli.cell_attach_beats import widened_window_s + + write(args.output, {"seconds": widened_window_s()}) + elif args.task == "mint-proof": + asyncio.run(mint_proof(context, args.output)) + elif args.task == "provenance": + asyncio.run(independent_provenance(context, args.output)) + elif args.task == "final-retirement": + asyncio.run(final_retirement(context, args.output)) + elif args.task == "resolve": + from server.settings import settings + + codec_contract() + config = ref( + read(args.output.parent / "serving-cell/launch.json")["configuration_artifact"] + ) + actual = settings.model_dump(mode="json") + need( + actual["cell_router_token"] == actual["router_operator_token"] == "", + "CLI credentials", + ) + need( + digest(actual) == config["cli_configuration_sha256"] + and digest({k: v for k, v in actual.items() if k != "cell_router_token"}) + == config["shared_configuration_sha256"], + "actual CLI configuration", + ) + write( + args.output, + { + "schema": 2, + "cli_configuration_sha256": digest(actual), + "codec_protocol_sha256": PROTOCOL_SHA, + "verified": True, + }, + ) + else: + asyncio.run( + proof( + context, + args.output, + bootstrap=args.task == "bootstrap-proof", + quiescent=args.task == "quiescent", + keeper=args.task == "keeper-proof", + ) + ) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + print(type(error).__name__, file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gates/testsupport/cell_trigger/owned_children.py b/scripts/gates/testsupport/cell_trigger/owned_children.py new file mode 100644 index 0000000000..b0c5d950f8 --- /dev/null +++ b/scripts/gates/testsupport/cell_trigger/owned_children.py @@ -0,0 +1,145 @@ +"""Reap only exact direct and adopted children of the fixture subreaper.""" + +from __future__ import annotations + +import ctypes +import os +import signal +import subprocess +import time +from pathlib import Path + + +class OwnedChildren: + """Signal only unreaped kernel children of this private subreaper.""" + + def __init__(self) -> None: + if not hasattr(os, "pidfd_open") or not hasattr(signal, "pidfd_send_signal"): + raise RuntimeError("exact owned-process signaling needs Linux pidfds") + if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0: + error = ctypes.get_errno() + raise OSError(error, "PR_SET_CHILD_SUBREAPER: " + os.strerror(error)) + self.pid = os.getpid() + + @staticmethod + def identity(pid: int) -> tuple[int, str, str] | None: + try: + fields = Path("/proc", str(pid), "stat").read_text().rsplit(")", 1)[1].split() + return int(fields[1]), fields[19], fields[0] + except (FileNotFoundError, ProcessLookupError): + return None + + @staticmethod + def children(pid: int) -> list[int]: + return [ + int(value) + for value in Path("/proc", str(pid), "task", str(pid), "children").read_text().split() + ] + + @staticmethod + def rss(pid: int) -> int: + try: + status = Path("/proc", str(pid), "status").read_text() + except FileNotFoundError: + return 0 # Vanished sample, never proof that the process used no memory. + return next( + (int(line.split()[1]) for line in status.splitlines() if line.startswith("VmRSS:")), 0 + ) + + def stop(self, process: subprocess.Popen | None, note: object) -> None: + started = None + last_note = None + diagnostic_error = None + # Keep each verified pidfd pinned, including across numeric PID reuse. + pinned: dict[int, tuple[int, set[int]]] = {} + + def cleanup_note(**details: object) -> None: + nonlocal diagnostic_error + try: + note("cleanup-wait", **details) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + diagnostic_error = error + + try: + while True: + try: + children = self.children(self.pid) + for pid in children: + row = self.identity(pid) + if row and row[0] == self.pid and row[2] == "Z": + if ( + process is not None + and process.returncode is None + and pid == process.pid + ): + process.poll() # Preserve the real leader exit status. + else: + os.waitpid(pid, os.WNOHANG) + children = self.children(self.pid) + if not children: + break # Kernel list is empty AFTER zombie reaping. + if started is None: + started = last_note = time.monotonic() + elapsed = time.monotonic() - started + sig = ( + signal.SIGINT + if elapsed < 30 + else (signal.SIGTERM if elapsed < 40 else signal.SIGKILL) + ) + for pid in children: + row = self.identity(pid) + if row is None or row[0] != self.pid or row[2] == "Z": + continue + if pid in pinned: + descriptor, sent = pinned[pid] + try: + signal.pidfd_send_signal(descriptor, 0) + except ProcessLookupError: + os.close(descriptor) + del pinned[pid] + if pid not in pinned: + try: + descriptor = os.pidfd_open(pid) + except ProcessLookupError: + continue + try: + exact = self.identity(pid) + if exact is None or exact[0] != self.pid or exact[1] != row[1]: + os.close(descriptor) + continue + except BaseException: + os.close(descriptor) + raise + pinned[pid] = (descriptor, set()) + descriptor, sent = pinned[pid] + if int(sig) not in sent: + try: + signal.pidfd_send_signal(descriptor, sig) + sent.add(int(sig)) + except ProcessLookupError: + pass + if time.monotonic() - last_note >= 30: + cleanup_note(owned_direct_pids=children) + last_note = time.monotonic() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + # Unknown census/signal state retains exclusion until the kernel + # proves that every direct/adopted child has been reaped. + if diagnostic_error is None: + cleanup_note( + reason="owned-child cleanup failed", error=type(error).__name__ + ) + diagnostic_error = error + try: + time.sleep(0.1) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + # Even a sleep/clock failure cannot escape over an unreaped child. + diagnostic_error = error + if process is not None: + process.wait() + finally: + for descriptor, _sent in pinned.values(): + os.close(descriptor) + if diagnostic_error is not None: + raise RuntimeError( + "cleanup failed; kernel children are now empty and reaped" + ) from diagnostic_error diff --git a/scripts/gates/testsupport/cell_trigger/protocol.json b/scripts/gates/testsupport/cell_trigger/protocol.json new file mode 100644 index 0000000000..3d3670f64d --- /dev/null +++ b/scripts/gates/testsupport/cell_trigger/protocol.json @@ -0,0 +1,235 @@ +{ + "schema": 1, + "status": "r5 design only; no runtime or entry implementation", + "memory_fields": [ + "org", + "namespace", + "agent", + "user_id", + "session_id", + "memory_key", + "memory_type", + "value", + "value_json", + "confidence", + "source_trace", + "version", + "tombstoned_at", + "created_at", + "expires_at", + "pinned", + "run_id", + "entities", + "memory_id", + "redacted_at", + "redaction_reason", + "status", + "conflict_with", + "duplicate_of", + "evidence_count", + "recall_count", + "last_recalled_at", + "cross_promoted_from_memory_id", + "cross_promoted_from_namespace", + "tags", + "superseded_by", + "rejection_reason", + "temporal_anchor", + "temporal_confidence", + "updated_at", + "valid_at", + "valid_to", + "invalid_at", + "char_limit", + "reinforcement_count", + "last_reinforced_at", + "mentioned_at", + "source_resource_id", + "content_sha256", + "files", + "concepts", + "verified_by", + "verified_at", + "verified_sha256", + "grounding", + "evidence", + "sensitivity", + "actor", + "source_class", + "lock_mode", + "lock_reason", + "lock_until", + "locked_by", + "locked_at", + "on_expire", + "expire_reason" + ], + "memory_types": { + "org": "string", + "namespace": "string", + "agent": "string", + "user_id": "string", + "session_id": "string", + "memory_key": "string", + "memory_type": "string", + "value": "string", + "source_trace": "string", + "run_id": "string", + "redaction_reason": "string", + "status": "string", + "duplicate_of": "string", + "cross_promoted_from_namespace": "string", + "superseded_by": "string", + "rejection_reason": "string", + "source_resource_id": "string", + "verified_by": "string", + "verified_sha256": "string", + "grounding": "string", + "evidence": "string", + "sensitivity": "string", + "actor": "string", + "source_class": "string", + "lock_mode": "string", + "lock_reason": "string", + "locked_by": "string", + "on_expire": "string", + "expire_reason": "string", + "tombstoned_at": "nullable_utc_second", + "created_at": "utc_second", + "expires_at": "nullable_utc_second", + "redacted_at": "nullable_utc_second", + "last_recalled_at": "nullable_utc_second", + "temporal_anchor": "nullable_utc_second", + "updated_at": "utc_second", + "valid_at": "nullable_utc_second", + "valid_to": "nullable_utc_second", + "invalid_at": "nullable_utc_second", + "last_reinforced_at": "utc_second", + "mentioned_at": "nullable_utc_second", + "verified_at": "nullable_utc_second", + "lock_until": "nullable_utc_second", + "locked_at": "nullable_utc_second", + "entities": "array_string", + "tags": "array_string", + "files": "array_string", + "concepts": "array_string", + "evidence_count": "uint32_not_bool", + "recall_count": "uint32_not_bool", + "reinforcement_count": "uint32_not_bool", + "confidence": "finite_exact_float32_not_bool", + "temporal_confidence": "finite_exact_float32_not_bool", + "value_json": "json_object_field", + "memory_id": "uuid", + "cross_promoted_from_memory_id": "nullable_uuid", + "conflict_with": "array_uuid", + "version": "positive_int63_not_bool", + "pinned": "uint8_0_or_1_not_bool", + "char_limit": "nullable_uint32_not_bool", + "content_sha256": "hex64_fixedstring" + }, + "sparse_defaults": { + "status": "active", + "conflict_with": [], + "duplicate_of": "", + "superseded_by": "", + "rejection_reason": "", + "invalid_at": null, + "redacted_at": null, + "redaction_reason": "", + "grounding": "", + "evidence": "", + "temporal_anchor": null, + "temporal_confidence": 0.0, + "source_resource_id": "", + "verified_by": "", + "verified_at": null, + "verified_sha256": "", + "cross_promoted_from_memory_id": null, + "cross_promoted_from_namespace": "", + "evidence_count": 1, + "recall_count": 0, + "last_recalled_at": null, + "reinforcement_count": 1, + "mentioned_at": null, + "source_trace": "", + "tombstoned_at": null, + "expires_at": null, + "pinned": 0, + "run_id": "-", + "entities": [], + "tags": [], + "files": [], + "concepts": [], + "valid_at": null, + "valid_to": null, + "char_limit": null, + "sensitivity": "", + "lock_mode": "", + "lock_reason": "", + "lock_until": null, + "locked_by": "", + "locked_at": null, + "on_expire": "", + "expire_reason": "" + }, + "audit_fields": [ + "ts", + "event_type", + "org", + "namespace", + "actor", + "subject", + "request_id", + "row_hash", + "build_commit", + "instance", + "instance_label", + "consumer", + "token_id" + ], + "audit_types": { + "ts": "utc_millisecond", + "event_type": "string", + "org": "string", + "namespace": "string", + "actor": "string", + "subject": "raw_subject_string", + "request_id": "string", + "row_hash": "hex64_fixedstring", + "build_commit": "string", + "instance": "string", + "instance_label": "string", + "consumer": "string", + "token_id": "string" + }, + "hash_byte_fields": { + "memory": [ + "content_sha256" + ], + "audit": [ + "row_hash" + ] + }, + "mint_summary_fields": [ + "jti", + "subject", + "org", + "namespaces", + "roles", + "ttl_days", + "is_service_account", + "read_scope", + "read_user_id", + "traffic_class", + "consumer", + "token_path" + ], + "page_rows": 100, + "query_limit_extra_rows": 1, + "memory_row_bytes": 65536, + "audit_row_bytes": 16384, + "query_result_bytes": 20971520, + "total_expected_rows": 45500, + "api_expected_rows": 20500, + "delivery_expected_rows": 25000 +} diff --git a/scripts/gates/testsupport/cell_trigger/tls_proxy.py b/scripts/gates/testsupport/cell_trigger/tls_proxy.py new file mode 100644 index 0000000000..e579fa688d --- /dev/null +++ b/scripts/gates/testsupport/cell_trigger/tls_proxy.py @@ -0,0 +1,168 @@ +"""Private fixed-target TLS bridge; run only after separate runtime admission.""" + +from __future__ import annotations + +import argparse +import asyncio +import signal +import ssl +import sys + + +class Bridge: + def __init__(self, upstream_port: int) -> None: + self.port = upstream_port + self.tasks: set[asyncio.Task] = set() + self.stopping = asyncio.Event() + self.failure: BaseException | None = None + + def fail(self, error: BaseException) -> None: + if self.failure is None: + self.failure = error + self.stopping.set() + + async def copy(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + while data := await asyncio.wait_for(reader.read(65536), 30): + writer.write(data) + await asyncio.wait_for(writer.drain(), 30) + + async def finish(self, pumps: list[asyncio.Task], endpoints: tuple) -> BaseException | None: + first = None + for pump in pumps: + if not pump.done(): + pump.cancel() + try: + outcomes = await asyncio.gather(*pumps, return_exceptions=True) + for outcome in outcomes: + if isinstance(outcome, BaseException) and not isinstance( + outcome, asyncio.CancelledError + ): + first = outcome if first is None else first + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + for endpoint in endpoints: + if endpoint is None: + continue + try: + endpoint.close() + await asyncio.wait_for(endpoint.wait_closed(), 5) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + return first + + async def drain(self, pumps: list[asyncio.Task], endpoints: tuple, first=None): + closing = asyncio.create_task(self.finish(pumps, endpoints)) + while not closing.done(): + try: + await asyncio.shield(closing) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + try: + outcome = closing.result() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + outcome = error + return outcome if first is None else first + + async def connection(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + task = asyncio.current_task() + assert task is not None + if len(self.tasks) >= 8 or self.stopping.is_set(): + error = await self.drain([], (writer,)) + if error is not None: + self.fail(error) + return + self.tasks.add(task) + upstream = None + pumps: list[asyncio.Task] = [] + first = None + try: + async with asyncio.timeout(120): + source, upstream = await asyncio.wait_for( + asyncio.open_connection("127.0.0.1", self.port, limit=65536), 5 + ) + pumps = [ + asyncio.create_task(self.copy(reader, upstream)), + asyncio.create_task(self.copy(source, writer)), + ] + done, _ = await asyncio.wait(pumps, return_when=asyncio.FIRST_COMPLETED) + for pump in done: + pump.result() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error + finally: + first = await self.drain(pumps, (upstream, writer), first) + self.tasks.discard(task) + if first is not None and not ( + self.stopping.is_set() and isinstance(first, asyncio.CancelledError) + ): + self.fail(first) + + async def serve(self, port: int, certificate: str, key: str) -> None: + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.minimum_version = ssl.TLSVersion.TLSv1_2 + context.set_alpn_protocols(["http/1.1"]) + context.load_cert_chain(certificate, key) + loop = asyncio.get_running_loop() + for signum in (signal.SIGINT, signal.SIGTERM): + loop.add_signal_handler(signum, self.stopping.set) + server = await asyncio.start_server( + self.connection, + "127.0.0.1", + port, + ssl=context, + limit=65536, + ssl_handshake_timeout=5, + ssl_shutdown_timeout=5, + ) + try: + await self.stopping.wait() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.fail(error) + finally: + closing = asyncio.create_task(self.shutdown(server)) + while not closing.done(): + try: + await asyncio.shield(closing) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.fail(error) + error = closing.result() + if error is not None: + self.fail(error) + if self.failure is not None: + raise self.failure + + async def shutdown(self, server) -> BaseException | None: + first = None + try: + server.close() + await server.wait_closed() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error + for task in tuple(self.tasks): + task.cancel() + result = await self.finish(list(self.tasks), ()) + return result if first is None else first + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--listen-port", required=True, type=int) + parser.add_argument("--upstream-port", required=True, type=int) + parser.add_argument("--certificate", required=True) + parser.add_argument("--key", required=True) + args = parser.parse_args() + if any(not 0 < port < 65536 for port in (args.listen_port, args.upstream_port)): + return 2 + if args.listen_port == args.upstream_port: + return 2 + try: + asyncio.run(Bridge(args.upstream_port).serve(args.listen_port, args.certificate, args.key)) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + # Error text may contain request bytes or private file names. + print(type(error).__name__, file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/unit/test_cell_trigger_ci_creator.py b/tests/unit/test_cell_trigger_ci_creator.py new file mode 100644 index 0000000000..3650641cd8 --- /dev/null +++ b/tests/unit/test_cell_trigger_ci_creator.py @@ -0,0 +1,165 @@ +"""A failed fixture may remove only the containers its creator recorded.""" + +import signal +from copy import deepcopy +from pathlib import Path + +import pytest + +from scripts.gates.cell_trigger_acceptance import Stores, native_cleanup_complete, write + + +class FakeStores(Stores): + def __init__(self, folder: Path, rows: dict): + super().__init__(folder, folder, "fixture", {"run_id": "123"}) + self.rows = deepcopy(rows) + self.removed = [] + + def inspect(self, identity, *, cleanup=False): + return self.rows[identity] + + def command(self, *argv, timeout=120, cleanup=False): + assert cleanup + if argv[:3] == ("docker", "rm", "--force"): + self.removed.append(argv[-1]) + return "" + assert argv[:3] == ("docker", "ps", "-a") + identity = next(part.removeprefix("id=") for part in argv if part.startswith("id=")) + return identity if identity in self.rows and identity not in self.removed else "" + + +def container(identity="own", created="birth", label="fixture"): + return { + "Id": identity, + "Created": created, + "Name": "/intent", + "Config": {"Labels": {"loom.ci.trigger": label}}, + } + + +@pytest.mark.parametrize( + "change", + [ + {"Id": "reused"}, + {"Created": "new-birth"}, + {"Config": {"Labels": {"loom.ci.trigger": "foreign"}}}, + ], +) +def test_cleanup_refuses_changed_container_identity(tmp_path, change): + actual = {**container(), **change} + stores = FakeStores(tmp_path, {"own": actual}) + stores.items = [{"role": "pg-primary", "name": "intent", "id": "own", "created": "birth"}] + assert stores.cleanup() + assert stores.removed == [] + + +def test_cleanup_continues_exact_owned_containers_after_a_refusal(tmp_path): + stores = FakeStores(tmp_path, {"own": container(), "foreign": container(label="other")}) + stores.items = [ + {"role": "pg-primary", "name": "intent", "id": "own", "created": "birth"}, + {"role": "pg-standby", "name": "other", "id": "foreign", "created": "birth"}, + ] + assert stores.cleanup() == ["RuntimeError:pg-standby"] + assert stores.removed == ["own"] + + +def test_partial_create_requires_its_exact_name_and_random_label(tmp_path): + stores = FakeStores(tmp_path, {"intent": container()}) + stores.items = [{"role": "clickhouse", "name": "intent", "id": None, "created": None}] + assert stores.cleanup() == [] + assert stores.removed == ["own"] + + +def test_partial_create_never_adopts_a_foreign_matching_name(tmp_path): + stores = FakeStores(tmp_path, {"intent": container(label="other")}) + stores.items = [{"role": "clickhouse", "name": "intent", "id": None, "created": None}] + assert stores.cleanup() == ["RuntimeError:clickhouse"] + assert stores.removed == [] + + +def test_cancellation_is_deferred_until_owned_cleanup_finishes(tmp_path): + stores = FakeStores(tmp_path, {}) + stores.interrupted(signal.SIGTERM, None) + assert stores.cancelled == signal.SIGTERM + stores.terminal = True + with pytest.raises(SystemExit) as caught: + stores.interrupted(signal.SIGINT, None) + assert caught.value.code == 128 + signal.SIGTERM + + +def test_store_removal_requires_acknowledged_native_cleanup(tmp_path): + (tmp_path / "fixture").mkdir() + with pytest.raises(FileNotFoundError): + native_cleanup_complete(tmp_path, "fixture") + write( + tmp_path / "fixture/outer-result.json", + {"schema": 2, "fixture_id": "fixture", "native_cleanup_complete": False}, + ) + with pytest.raises(RuntimeError, match="Native retirement"): + native_cleanup_complete(tmp_path, "fixture") + + +def test_failed_measurement_can_close_only_after_real_native_cleanup(tmp_path): + (tmp_path / "fixture").mkdir() + write( + tmp_path / "fixture/outer-result.json", + { + "schema": 2, + "fixture_id": "fixture", + "candidate_verdict": "FAIL", + "native_cleanup_complete": True, + }, + ) + native_cleanup_complete(tmp_path, "fixture") + + +def test_dependency_metadata_accepts_only_the_actual_python_version(tmp_path): + import hashlib + import json + import sys + + from scripts.gates.testsupport.cell_trigger import fixture + + environment = tmp_path / ".venv" + for name in fixture.CODEC_FILES: + path = environment / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("fixture codec bytes") + (environment / fixture.CODEC_FILES[1]).write_text('version = "1.9.0"\n') + (environment / fixture.CODEC_FILES[4]).write_text('__version__ = "0.32.0"\n') + spec = { + "root": str(tmp_path), + "python": str(environment / "bin/python"), + "python_sha256": "0" * 64, + "lockfile_sha256": "1" * 64, + } + + def check(version): + (environment / "pyvenv.cfg").write_text( + f"implementation = CPython\nversion_info = {version}\n" + "include-system-site-packages = false\n" + ) + receipt = { + "schema": 2, + **spec, + "versions": fixture.CODEC_VERSIONS, + "codec_sources": { + name: {"path": str(environment / name), "sha256": fixture.sha(environment / name)} + for name in fixture.CODEC_FILES + }, + } + receipt.pop("dependency_receipt", None) + path = tmp_path / "receipt.json" + path.write_text(json.dumps(receipt)) + path.chmod(0o600) + spec["dependency_receipt"] = { + "path": str(path), + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + } + fixture.dependencies(spec, verify=True) + + check(".".join(map(str, sys.version_info[:3]))) + check("3.12") + for changed in ("3.13.0", "3.12.invalid", "3.12.999"): + with pytest.raises(fixture.Refused, match="fixed isolated CPython"): + check(changed) From 1e71631ac673e087b011ad8ecee75bbd6e0cd84d Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 22:06:40 +0000 Subject: [PATCH 077/114] Trigger acceptance waits for TCP and reports owned startup state Read the fixed public protocol with its own file constraints, preserve private receipt checks, select dependency inputs in CI and regenerate operation reachability. Signed-off-by: zlareb1 --- .github/workflows/docker-smoke.yml | 6 + scripts/gates/cell_trigger_acceptance.py | 131 +++++++++++++- .../gates/testsupport/cell_trigger/capture.py | 18 +- server/operations/reach_manifest.py | 1 + tests/unit/test_cell_trigger_ci_creator.py | 163 ++++++++++++++++++ 5 files changed, 310 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-smoke.yml b/.github/workflows/docker-smoke.yml index 05f3564698..8a0a64defa 100644 --- a/.github/workflows/docker-smoke.yml +++ b/.github/workflows/docker-smoke.yml @@ -37,6 +37,9 @@ on: "tests/unit/test_scip_checkout_fetch_ref.py", "install.sh", "pyproject.toml", + "uv.lock", + "dev/clickhouse-ci/log_queries_readonly.xml", + "dev/clickhouse-ci/custom_settings_prefixes.xml", ".github/workflows/docker-smoke.yml", ] pull_request: @@ -74,6 +77,9 @@ on: "tests/unit/test_scip_checkout_fetch_ref.py", "install.sh", "pyproject.toml", + "uv.lock", + "dev/clickhouse-ci/log_queries_readonly.xml", + "dev/clickhouse-ci/custom_settings_prefixes.xml", ".github/workflows/docker-smoke.yml", ] # Weekly, so a critical CVE fixed upstream fails here before a release's scan. diff --git a/scripts/gates/cell_trigger_acceptance.py b/scripts/gates/cell_trigger_acceptance.py index 97eedfeeab..587c0b5c4f 100644 --- a/scripts/gates/cell_trigger_acceptance.py +++ b/scripts/gates/cell_trigger_acceptance.py @@ -144,6 +144,7 @@ def __init__(self, root: Path, folder: Path, fixture_id: str, run: dict) -> None self.prefix = "loom-trigger-" + fixture_id[:12] self.cancelled = 0 self.terminal = False + self.phase = "store_creation" self.creation_deadline = time.monotonic() + 900 self.env = { "PATH": os.environ["PATH"], @@ -229,7 +230,11 @@ def create(self, role: str, args: list[str]) -> dict: self.command("docker", "start", identity) return intent - def wait(self, probe, seconds: int = 120) -> None: + def wait(self, phase: str, probe, seconds: int = 120) -> None: + require( + phase in {"primary_tcp", "standby_sync", "clickhouse_query"}, "Known readiness phase" + ) + self.phase = phase end = time.monotonic() + seconds while True: require(not self.cancelled, "Owned store readiness cancelled") @@ -241,6 +246,65 @@ def wait(self, probe, seconds: int = 120) -> None: require(time.monotonic() < end, "Owned store readiness deadline") time.sleep(1) + def failure_diagnostics(self) -> dict: + observations = [] + statuses = {"created", "running", "paused", "restarting", "removing", "exited", "dead"} + for item in self.items: + row = {"role": item["role"], "available": False} + observations.append(row) + if item["id"] is None: + continue + try: + records = json.loads( + self.command("docker", "inspect", item["id"], timeout=5, cleanup=True) + ) + require( + len(records) == 1 and container_matches(records[0], item, self.fixture_id), + "Diagnostic container ownership changed", + ) + state = records[0]["State"] + row.update( + available=True, + status=state["Status"] if state["Status"] in statuses else "unknown", + exit_code=state["ExitCode"] if type(state["ExitCode"]) is int else None, + oom_killed=state["OOMKilled"] if type(state["OOMKilled"]) is bool else None, + ) + if item["role"] in {"pg-primary", "pg-standby"} and state["Running"] is True: + value = self.command( + "docker", + "exec", + item["id"], + "psql", + "-U", + "loom", + "-d", + "loom", + "-Atc", + "SELECT CASE WHEN pg_is_in_recovery() THEN 'standby' ELSE 'primary' END " + "|| ':' || COALESCE((SELECT sync_state FROM pg_stat_replication " + "WHERE application_name='loom_standby'), 'absent')", + timeout=5, + cleanup=True, + ) + recovery, sync = value.split(":") + if recovery in {"primary", "standby"} and sync in { + "sync", + "async", + "potential", + "quorum", + "absent", + }: + row["pg_state"] = {"recovery": recovery == "standby", "standby_sync": sync} + except ( + OSError, + RuntimeError, + subprocess.SubprocessError, + ValueError, + KeyError, + ) as error: + row["diagnostic_error_type"] = type(error).__name__ + return {"phase": self.phase, "containers": observations} + def start(self, ch_image: str) -> tuple[dict, dict, dict, list[int]]: selected = ports(8) ch_port, native_port, pg_port, standby_port = selected[:4] @@ -349,8 +413,33 @@ def start(self, ch_image: str) -> tuple[dict, dict, dict, list[int]]: "shared_buffers=128MB", ], ) - psql = ["docker", "exec", pg["id"], "psql", "-U", "loom", "-d", "loom", "-Atc"] - self.wait(lambda: self.command(*psql, "SELECT 1") == "1") + psql = [ + "docker", + "exec", + pg["id"], + "psql", + "-U", + "loom", + "-d", + "loom", + "-Atc", + ] + self.wait( + "primary_tcp", + lambda: self.command( + "docker", + "exec", + pg["id"], + "pg_isready", + "-U", + "loom", + "-d", + "loom", + "-h", + "127.0.0.1", + ), + ) + self.phase = "replication_configuration" self.command(*psql, "CREATE ROLE replicator WITH REPLICATION LOGIN PASSWORD 'replicator'") self.command( "docker", @@ -405,6 +494,7 @@ def start(self, ch_image: str) -> tuple[dict, dict, dict, list[int]]: ], ) self.wait( + "standby_sync", lambda: ( self.command( *psql, @@ -414,9 +504,10 @@ def start(self, ch_image: str) -> tuple[dict, dict, dict, list[int]]: ), ) == "sync" - ) + ), ) self.wait( + "clickhouse_query", lambda: ( self.command( "docker", @@ -431,7 +522,7 @@ def start(self, ch_image: str) -> tuple[dict, dict, dict, list[int]]: "SELECT 1", ) == "1" - ) + ), ) return ch, pg, standby, selected @@ -640,7 +731,14 @@ def dependency_receipt(root: Path, output: Path) -> dict: return {"path": str(output), "sha256": sha(output)} -def publish(folder: Path, destination: Path, error: BaseException | None) -> None: +def publish( + folder: Path, + destination: Path, + error: BaseException | None, + *, + phase: str, + diagnostic_error_type: str | None, +) -> None: destination.mkdir(mode=0o700) # Raw native logs/settings contain bearer and bootstrap credentials: never upload them. fixture = folder / "fixture" @@ -654,7 +752,7 @@ def publish(folder: Path, destination: Path, error: BaseException | None) -> Non path = fixture / name if path.is_file(): shutil.copyfile(path, destination / name) - for name in ("source-before.json", "source-after.json"): + for name in ("source-before.json", "source-after.json", "startup-diagnostics.json"): if (folder / name).is_file(): shutil.copyfile(folder / name, destination / name) intent_output = destination / "delivery-intent" @@ -672,6 +770,8 @@ def publish(folder: Path, destination: Path, error: BaseException | None) -> Non "accepted": error is None, "error_type": type(error).__name__ if error is not None else "", "failure": failure_site(error), + "phase": phase, + "diagnostic_error_type": diagnostic_error_type, "utc": datetime.datetime.now(datetime.UTC).isoformat(), }, ) @@ -768,9 +868,11 @@ def main() -> int: for signum in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP): signal.signal(signum, stores.interrupted) error = None + diagnostic_error_type = None cleanup_complete = False try: ch, primary, standby, selected = stores.start(args.clickhouse_image) + stores.phase = "dependency_metadata" resources(folder) dependencies = dependency_receipt(root, folder / "dependencies.json") from fixture import runtime @@ -815,10 +917,15 @@ def main() -> int: } write(folder / "spec.json", spec) require(not stores.cancelled, "Cancelled before native acceptance") + stores.phase = "native" require(runtime(spec) == 0, "Trigger acceptance did not pass") require(source(root) == before, "Acceptance source changed") except BaseException as caught: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. error = caught + try: + write(folder / "startup-diagnostics.json", stores.failure_diagnostics()) + except BaseException as diagnostic_error: # noqa: BLE001 — diagnostic failure cannot replace the selected first error. + diagnostic_error_type = type(diagnostic_error).__name__ finally: # Native ownership closes first; exact created containers are removed even on failure. for signum in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP): @@ -846,7 +953,13 @@ def main() -> int: except BaseException as caught: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. error = caught if error is None else error try: - publish(folder, Path(os.environ["RUNNER_TEMP"]) / "cell-trigger-artifacts", error) + publish( + folder, + Path(os.environ["RUNNER_TEMP"]) / "cell-trigger-artifacts", + error, + phase=stores.phase, + diagnostic_error_type=diagnostic_error_type, + ) except BaseException as caught: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. error = caught if error is None else error stores.terminal = cleanup_complete @@ -858,6 +971,8 @@ def main() -> int: "accepted": error is None, "error_type": type(error).__name__ if error is not None else "", "failure": failure_site(error), + "phase": stores.phase, + "diagnostic_error_type": diagnostic_error_type, } ) ) diff --git a/scripts/gates/testsupport/cell_trigger/capture.py b/scripts/gates/testsupport/cell_trigger/capture.py index 1d398f00da..3f1c78429a 100644 --- a/scripts/gates/testsupport/cell_trigger/capture.py +++ b/scripts/gates/testsupport/cell_trigger/capture.py @@ -11,6 +11,7 @@ import os import re import socket +import stat import struct import sys import uuid @@ -42,7 +43,22 @@ def protocol() -> dict: - return read(HERE / PROTOCOL, expected_sha256=PROTOCOL_SHA) + descriptor = os.open(HERE / PROTOCOL, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(descriptor, "rb") as stream: + info = os.fstat(stream.fileno()) + need( + stat.S_ISREG(info.st_mode) + and info.st_uid == os.geteuid() + and info.st_mode & 0o022 == 0 + and info.st_size <= 65536, + "owned fixed protocol constant", + ) + raw = stream.read(65537) + need( + len(raw) <= 65536 and hashlib.sha256(raw).hexdigest() == PROTOCOL_SHA, + "fixed protocol bytes", + ) + return json_object(raw) def integer(value, low=0, high=2**32 - 1): diff --git a/server/operations/reach_manifest.py b/server/operations/reach_manifest.py index 38db700e6f..71dc49e22b 100644 --- a/server/operations/reach_manifest.py +++ b/server/operations/reach_manifest.py @@ -577,6 +577,7 @@ "namespace.mark_seen": ("dashboard",), "namespace.register": ( "sdk_method", + "sdk_example", "dashboard", ), "namespace.register_category": ( diff --git a/tests/unit/test_cell_trigger_ci_creator.py b/tests/unit/test_cell_trigger_ci_creator.py index 3650641cd8..8cb90a343c 100644 --- a/tests/unit/test_cell_trigger_ci_creator.py +++ b/tests/unit/test_cell_trigger_ci_creator.py @@ -163,3 +163,166 @@ def check(version): for changed in ("3.13.0", "3.12.invalid", "3.12.999"): with pytest.raises(fixture.Refused, match="fixed isolated CPython"): check(changed) + + +@pytest.fixture +def protocol_reader(tmp_path, monkeypatch): + import sys + + from scripts.gates.testsupport.cell_trigger import fixture + + monkeypatch.setitem(sys.modules, "fixture", fixture) + from scripts.gates.testsupport.cell_trigger import capture + + data = (Path(fixture.__file__).parent / fixture.PROTOCOL).read_bytes() + path = tmp_path / fixture.PROTOCOL + path.write_bytes(data) + path.chmod(0o644) + monkeypatch.setattr(capture, "HERE", tmp_path) + return capture, fixture, path, data + + +def test_committed_public_protocol_is_readable_without_private_receipt_mode(protocol_reader): + capture, _, _, _ = protocol_reader + assert capture.protocol()["schema"] == 1 + + +def test_public_protocol_keeps_hash_and_symlink_guards(protocol_reader): + capture, fixture, path, data = protocol_reader + path.write_bytes(data + b"\n") + with pytest.raises(fixture.Refused): + capture.protocol() + path.unlink() + target = path.with_name("target.json") + target.write_bytes(data) + target.chmod(0o644) + path.symlink_to(target) + with pytest.raises((OSError, fixture.Refused)): + capture.protocol() + + +def test_private_receipt_still_refuses_public_read_permissions(protocol_reader): + _, fixture, path, _ = protocol_reader + with pytest.raises(fixture.Refused, match="owned private stream"): + fixture.read(path) + + +def test_replication_waits_for_primary_tcp_after_temporary_socket_ready(tmp_path, monkeypatch): + from scripts.gates import cell_trigger_acceptance as creator + + monkeypatch.setattr(creator, "ports", lambda count: list(range(18000, 18000 + count))) + monkeypatch.setattr(creator.time, "sleep", lambda _: None) + + class InitializingStores(Stores): + def __init__(self): + super().__init__(tmp_path, tmp_path, "fixture", {"run_id": "123"}) + self.tcp_attempts = 0 + self.tcp_ready = False + + def create(self, role, args): + return {"id": role} + + def command(self, *argv, timeout=120, cleanup=False): + if "pg_isready" in argv: + assert argv[argv.index("-h") + 1] == "127.0.0.1" + self.tcp_attempts += 1 + if self.tcp_attempts == 1: + raise RuntimeError("simulated TCP startup refusal") + self.tcp_ready = True + return "127.0.0.1:5432 - accepting connections" + if "psql" in argv: + query = argv[-1] + if query == "SELECT 1": + return "1" # The temporary init server accepts Unix sockets only. + assert self.tcp_ready, "Replication configuration raced the temporary init server" + return "sync" if "SELECT sync_state" in query else "" + return "1" if "clickhouse-client" in argv else "network-id" + + stores = InitializingStores() + stores.start("test-image") + assert stores.tcp_attempts == 2 and stores.tcp_ready + + +def test_failure_diagnostics_expose_only_allowlisted_owned_state(tmp_path): + import json + + class DiagnosticStores(Stores): + def command(self, *argv, timeout=120, cleanup=False): + assert cleanup and timeout == 5 + if argv[:2] == ("docker", "inspect"): + value = container() + value["Config"]["Env"] = ["TOKEN=private-value"] + value["State"] = { + "Status": "running", + "Running": True, + "ExitCode": 0, + "OOMKilled": False, + "Error": "private-log", + } + return json.dumps([value]) + return "primary:sync" + + stores = DiagnosticStores(tmp_path, tmp_path, "fixture", {"run_id": "123"}) + stores.phase = "standby_sync" + stores.items = [{"role": "pg-primary", "name": "intent", "id": "own", "created": "birth"}] + result = stores.failure_diagnostics() + assert result == { + "phase": "standby_sync", + "containers": [ + { + "role": "pg-primary", + "available": True, + "status": "running", + "exit_code": 0, + "oom_killed": False, + "pg_state": {"recovery": False, "standby_sync": "sync"}, + } + ], + } + assert "private" not in json.dumps(result) + + +def test_diagnostic_failure_preserves_primary_result_and_runs_cleanup( + tmp_path, monkeypatch, capsys +): + import json + import sys + + from scripts.gates import cell_trigger_acceptance as creator + + cleaned = [] + + class FailingStores(Stores): + def start(self, image): + raise ValueError("private primary failure") + + def failure_diagnostics(self): + raise OSError("private diagnostic failure") + + def cleanup(self): + cleaned.append(True) + return [] + + root = Path(sys.executable).parent.parent.parent + monkeypatch.setenv("GITHUB_ACTIONS", "true") + monkeypatch.setenv("CI", "true") + monkeypatch.setenv("GITHUB_WORKSPACE", str(root)) + monkeypatch.setenv("RUNNER_TEMP", str(tmp_path)) + for name in ("GITHUB_RUN_ID", "GITHUB_RUN_ATTEMPT", "GITHUB_REPOSITORY"): + monkeypatch.setenv(name, "123") + monkeypatch.setattr(sys, "argv", ["creator", "--clickhouse-image", "test-image"]) + monkeypatch.setattr(creator, "Stores", FailingStores) + monkeypatch.setattr(creator, "source", lambda _: {"head": "a" * 40, "tree": "b" * 40}) + monkeypatch.setattr(creator, "source_hashes", lambda _: {}) + monkeypatch.setattr(creator, "resources", lambda _: None) + monkeypatch.setattr(creator.os, "umask", lambda _: 0) + monkeypatch.setattr(creator.signal, "signal", lambda *_: None) + assert creator.main() == 1 + result = json.loads(capsys.readouterr().out) + assert cleaned == [True] + assert result["error_type"] == "ValueError" and result["diagnostic_error_type"] == "OSError" + assert "private" not in json.dumps(result) + published = json.loads((tmp_path / "cell-trigger-artifacts/result.json").read_text()) + assert ( + published["error_type"] == "ValueError" and published["diagnostic_error_type"] == "OSError" + ) From 79cd5467e4ebec8d3fd63791535a3a0c4ab1bb47 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 22:15:22 +0000 Subject: [PATCH 078/114] Trigger stores use the CI server configuration Mount the global custom-setting prefix under config.d and create the configured database, matching the existing ClickHouse CI helper. Signed-off-by: zlareb1 --- scripts/gates/cell_trigger_acceptance.py | 9 +++-- tests/unit/test_cell_trigger_ci_creator.py | 41 ++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/scripts/gates/cell_trigger_acceptance.py b/scripts/gates/cell_trigger_acceptance.py index 587c0b5c4f..43e9691ed3 100644 --- a/scripts/gates/cell_trigger_acceptance.py +++ b/scripts/gates/cell_trigger_acceptance.py @@ -348,11 +348,14 @@ def start(self, ch_image: str) -> tuple[dict, dict, dict, list[int]]: True, ) ) - for filename in ("log_queries_readonly.xml", "custom_settings_prefixes.xml"): + for filename, section in ( + ("log_queries_readonly.xml", "users.d"), + ("custom_settings_prefixes.xml", "config.d"), + ): mounts.append( ( self.root / "dev/clickhouse-ci" / filename, - "/etc/clickhouse-server/users.d/" + filename, + "/etc/clickhouse-server/" + section + "/" + filename, True, ) ) @@ -366,6 +369,8 @@ def start(self, ch_image: str) -> tuple[dict, dict, dict, list[int]]: "-p", f"127.0.0.1:{native_port}:9000", "-e", + "CLICKHOUSE_DB=loom", + "-e", "CLICKHOUSE_USER=loom", "-e", "CLICKHOUSE_PASSWORD=loomdev", diff --git a/tests/unit/test_cell_trigger_ci_creator.py b/tests/unit/test_cell_trigger_ci_creator.py index 8cb90a343c..a591c2bd95 100644 --- a/tests/unit/test_cell_trigger_ci_creator.py +++ b/tests/unit/test_cell_trigger_ci_creator.py @@ -243,6 +243,47 @@ def command(self, *argv, timeout=120, cleanup=False): assert stores.tcp_attempts == 2 and stores.tcp_ready +@pytest.mark.parametrize("setting", ["config_mount", "database"]) +def test_clickhouse_creation_uses_ci_server_configuration(tmp_path, monkeypatch, setting): + from scripts.gates import cell_trigger_acceptance as creator + + monkeypatch.setattr(creator, "ports", lambda count: list(range(18000, 18000 + count))) + + class CapturedCreation(Exception): + pass + + class CapturingStores(Stores): + def command(self, *argv, timeout=120, cleanup=False): + return "network-id" + + def create(self, role, args): + assert role == "clickhouse" + self.creation_args = args + raise CapturedCreation + + stores = CapturingStores(tmp_path, tmp_path, "fixture", {"run_id": "123"}) + with pytest.raises(CapturedCreation): + stores.start("test-image") + args = stores.creation_args + if setting == "config_mount": + mounts = [args[index + 1] for index, value in enumerate(args) if value == "-v"] + for filename, section in ( + ("custom_settings_prefixes.xml", "config.d"), + ("log_queries_readonly.xml", "users.d"), + ): + assert ( + str(tmp_path / "dev/clickhouse-ci" / filename) + + ":/etc/clickhouse-server/" + + section + + "/" + + filename + + ":ro" + ) in mounts + else: + environment = [args[index + 1] for index, value in enumerate(args) if value == "-e"] + assert "CLICKHOUSE_DB=loom" in environment + + def test_failure_diagnostics_expose_only_allowlisted_owned_state(tmp_path): import json From c9758437b7c1415397fc3e404daf15965db80d79 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 22:24:02 +0000 Subject: [PATCH 079/114] Trigger ClickHouse runs as its data-directory owner Match the supported container UID and GID to the owned bind directory so startup keeps the nested dictionary mount read-only. Signed-off-by: zlareb1 --- scripts/gates/cell_trigger_acceptance.py | 6 ++++++ tests/unit/test_cell_trigger_ci_creator.py | 13 +++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/scripts/gates/cell_trigger_acceptance.py b/scripts/gates/cell_trigger_acceptance.py index 43e9691ed3..52701f103b 100644 --- a/scripts/gates/cell_trigger_acceptance.py +++ b/scripts/gates/cell_trigger_acceptance.py @@ -330,6 +330,8 @@ def start(self, ch_image: str) -> tuple[dict, dict, dict, list[int]]: ch_data = self.folder / "clickhouse-data" ch_data.mkdir(mode=0o777) ch_data.chmod(0o777) + # Matching ownership avoids recursive chown through the read-only dictionary mount. + ch_owner = ch_data.stat() mounts = [ (ch_data, "/var/lib/clickhouse", False), (self.root / "deploy/lemmagen3", "/var/lib/clickhouse/lemmagen3", True), @@ -371,6 +373,10 @@ def start(self, ch_image: str) -> tuple[dict, dict, dict, list[int]]: "-e", "CLICKHOUSE_DB=loom", "-e", + f"CLICKHOUSE_UID={ch_owner.st_uid}", + "-e", + f"CLICKHOUSE_GID={ch_owner.st_gid}", + "-e", "CLICKHOUSE_USER=loom", "-e", "CLICKHOUSE_PASSWORD=loomdev", diff --git a/tests/unit/test_cell_trigger_ci_creator.py b/tests/unit/test_cell_trigger_ci_creator.py index a591c2bd95..728132f87d 100644 --- a/tests/unit/test_cell_trigger_ci_creator.py +++ b/tests/unit/test_cell_trigger_ci_creator.py @@ -243,7 +243,7 @@ def command(self, *argv, timeout=120, cleanup=False): assert stores.tcp_attempts == 2 and stores.tcp_ready -@pytest.mark.parametrize("setting", ["config_mount", "database"]) +@pytest.mark.parametrize("setting", ["config_mount", "database", "data_owner"]) def test_clickhouse_creation_uses_ci_server_configuration(tmp_path, monkeypatch, setting): from scripts.gates import cell_trigger_acceptance as creator @@ -279,9 +279,18 @@ def create(self, role, args): + filename + ":ro" ) in mounts - else: + elif setting == "database": environment = [args[index + 1] for index, value in enumerate(args) if value == "-e"] assert "CLICKHOUSE_DB=loom" in environment + else: + environment = [args[index + 1] for index, value in enumerate(args) if value == "-e"] + owner = (tmp_path / "clickhouse-data").stat() + assert f"CLICKHOUSE_UID={owner.st_uid}" in environment + assert f"CLICKHOUSE_GID={owner.st_gid}" in environment + assert not any( + value.startswith(("CLICKHOUSE_RUN_AS_ROOT=", "CLICKHOUSE_DO_NOT_CHOWN=")) + for value in environment + ) def test_failure_diagnostics_expose_only_allowlisted_owned_state(tmp_path): From bc6bcd63cd92b9f8df551aa0d9febba867006c10 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 22:34:52 +0000 Subject: [PATCH 080/114] Trigger setup verifies the owned Keeper configuration Read the effective prefix from the exact verified container because system.server_settings omits this configuration key. Keep mismatched-prefix and changed-identity refusals. Signed-off-by: zlareb1 --- .../gates/testsupport/cell_trigger/native.py | 28 ++- tests/unit/test_cell_trigger_ci_creator.py | 168 ++++++++++++++++++ 2 files changed, 188 insertions(+), 8 deletions(-) diff --git a/scripts/gates/testsupport/cell_trigger/native.py b/scripts/gates/testsupport/cell_trigger/native.py index a4c3661fb5..5f970cd051 100644 --- a/scripts/gates/testsupport/cell_trigger/native.py +++ b/scripts/gates/testsupport/cell_trigger/native.py @@ -909,14 +909,6 @@ async def stores(context: dict, role: str, output: Path, *, drop: bool = False) == [{"n": 0}], "new CH database", ) - prefix = await rows( - ch, "SELECT value FROM system.server_settings WHERE name='keeper_map_path_prefix'" - ) - need( - len(prefix) == 1 - and prefix[0]["value"].rstrip("/") == spec["ch"]["keeper_prefix"].rstrip("/"), - "actual Keeper prefix", - ) existing = await rows( ch, "SELECT name FROM system.zookeeper WHERE path={path:String}", @@ -1857,6 +1849,26 @@ def inspect_stores(self, *, cleanup: bool = False) -> None: == expected["boot_id"], "store process identity", ) + prefix = self.o.command( + "keeper-prefix", + [ + "docker", + "exec", + self.s["stores"]["clickhouse"]["container_id"], + "clickhouse", + "extract-from-config", + "--config-file", + "/etc/clickhouse-server/config.xml", + "--key", + "keeper_map_path_prefix", + ], + self.env({}), + cleanup=cleanup, + ) + need( + prefix["log"].read_text().strip() == self.s["ch"]["keeper_prefix"], + "actual Keeper prefix", + ) def certificate(self) -> None: def openssl(*args): diff --git a/tests/unit/test_cell_trigger_ci_creator.py b/tests/unit/test_cell_trigger_ci_creator.py index 728132f87d..b78cb245b8 100644 --- a/tests/unit/test_cell_trigger_ci_creator.py +++ b/tests/unit/test_cell_trigger_ci_creator.py @@ -376,3 +376,171 @@ def cleanup(self): assert ( published["error_type"] == "ValueError" and published["diagnostic_error_type"] == "OSError" ) + + +@pytest.fixture +def keeper_config_case(tmp_path, monkeypatch): + import json + import sys + from types import SimpleNamespace + + from scripts.gates.testsupport.cell_trigger import fixture + + monkeypatch.setitem(sys.modules, "fixture", fixture) + from scripts.gates.testsupport.cell_trigger import capture + + monkeypatch.setitem(sys.modules, "capture", capture) + from scripts.gates.testsupport.cell_trigger import native + + expected = { + "container_id": "owned-ch", + "created_at": "birth", + "host_pid": 123, + "memory_limit_bytes": 5 * fixture.GIB, + "server_limit_bytes": 9 * fixture.GIB // 2, + "shared_buffers_bytes": 128 * 1024**2, + "ports": [], + "disk_path": str(tmp_path), + "start_ticks": "start", + "uid": 1000, + "boot_id": "boot", + } + actual = { + "Id": "owned-ch", + "Created": "birth", + "State": {"Running": True, "Pid": 123}, + "HostConfig": {"Memory": 5 * fixture.GIB}, + "NetworkSettings": {"Ports": {}}, + "Mounts": [{"Source": str(tmp_path)}], + } + calls = [] + owner = SimpleNamespace(owned=SimpleNamespace(identity=lambda _: (1, "start"))) + owner.prefix = "/keeper_map_tables\n" + + def command(role, argv, env, *, seconds=120, cleanup=False): + calls.append((argv, cleanup)) + log = tmp_path / (role + ".log") + if argv == ["docker", "inspect", "owned-ch"]: + log.write_text(json.dumps([actual])) + else: + assert argv == [ + "docker", + "exec", + "owned-ch", + "clickhouse", + "extract-from-config", + "--config-file", + "/etc/clickhouse-server/config.xml", + "--key", + "keeper_map_path_prefix", + ] + log.write_text(owner.prefix) + return {"log": log} + + owner.command = command + original_stat, original_read = native.os.stat, Path.read_text + + def stat(path, *args, **kwargs): + if str(path) == "/proc/123": + return SimpleNamespace(st_uid=1000) + return original_stat(path, *args, **kwargs) + + def read_text(path, *args, **kwargs): + if str(path) == "/proc/sys/kernel/random/boot_id": + return "boot\n" + return original_read(path, *args, **kwargs) + + monkeypatch.setattr(native.os, "stat", stat) + monkeypatch.setattr(Path, "read_text", read_text) + spec = { + "workdir": str(tmp_path), + "fixture_id": "fixture", + "stores": {"clickhouse": expected}, + "ch": {"keeper_prefix": "/keeper_map_tables"}, + "databases": {"ch_schema": "bench_schema"}, + "pg_standby_dsn": "fake", + } + instance = native.Fixture(spec, owner) + monkeypatch.setattr(instance, "env", lambda _: {}) + return native, instance, owner, actual, calls + + +@pytest.mark.asyncio +async def test_keeper_config_allows_creation_without_sql_server_setting( + keeper_config_case, monkeypatch +): + from contextlib import asynccontextmanager + + import asyncpg + + native, instance, _, _, calls = keeper_config_case + instance.inspect_stores() + commands = [] + queries = [] + + class Postgres: + async def fetchval(self, query, *args): + return { + "SELECT NOT pg_is_in_recovery()": True, + "SELECT pg_is_in_recovery()": True, + "SHOW server_version_num": "160000", + "SHOW max_prepared_transactions": "0", + "SHOW synchronous_standby_names": "FIRST 1 (loom_standby)", + "SELECT pg_size_bytes(current_setting('shared_buffers'))": 128 * 1024**2, + "SELECT EXISTS(SELECT FROM pg_stat_replication WHERE state='streaming' " + "AND sync_state='sync' AND flush_lsn IS NOT NULL)": True, + }[query] + + async def close(self, **kwargs): + assert kwargs["timeout"] > 0 + + class ClickHouse: + async def command(self, query): + commands.append(query) + + async def connect(*args, **kwargs): + return Postgres() + + @asynccontextmanager + async def connections(*args, **kwargs): + yield Postgres(), ClickHouse() + + async def rows(ch, query, parameters=None): + queries.append(query) + if "keeper_map_path_prefix" in query: + return [] # The real config key is absent from declared ServerSettings. + if "max_server_memory_usage" in query: + return [{"value": str(9 * native.GIB // 2)}] + if "count()" in query: + return [{"n": 0}] + if "toString(uuid)" in query: + return [{"id": "database-uuid"}] + assert "system.zookeeper" in query + return [] + + monkeypatch.setattr(asyncpg, "connect", connect) + monkeypatch.setattr(native, "connections", connections) + monkeypatch.setattr(native, "rows", rows) + instance.s["stores"]["pg_primary"] = {"shared_buffers_bytes": 128 * 1024**2} + instance.s["stores"]["pg_standby"] = {"shared_buffers_bytes": 128 * 1024**2} + await native.stores(instance.s, "ch_schema", instance.w / "created.json") + assert commands == ["CREATE DATABASE `bench_schema` ENGINE=Atomic"] + assert len(calls) == 2 and calls[-1][0][-1] == "keeper_map_path_prefix" + assert not any("keeper_map_path_prefix" in query for query in queries) + + +@pytest.mark.parametrize("cleanup", [False, True]) +def test_keeper_config_refuses_wrong_actual_prefix(keeper_config_case, cleanup): + native, instance, owner, _, calls = keeper_config_case + owner.prefix = "/different-prefix\n" + with pytest.raises(native.Refused, match="actual Keeper prefix"): + instance.inspect_stores(cleanup=cleanup) + assert calls[-1][1] is cleanup + + +def test_keeper_config_probe_follows_store_ownership_validation(keeper_config_case): + native, instance, _, actual, calls = keeper_config_case + actual["Created"] = "foreign-birth" + with pytest.raises(native.Refused, match="current owned container"): + instance.inspect_stores() + assert [argv for argv, _ in calls] == [["docker", "inspect", "owned-ch"]] From 30d57c3fcf0ce595959d09b91231e60c1f0786bf Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 22:57:11 +0000 Subject: [PATCH 081/114] Worker crash checks report the unproved predicate Keep physical death, retirement and replacement checks unchanged while reporting bounded failure observations. Signed-off-by: zlareb1 --- scripts/gates/docker_provider_data.py | 47 +++++++++++++++------- tests/unit/test_docker_provider_smoke.py | 50 ++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 15 deletions(-) diff --git a/scripts/gates/docker_provider_data.py b/scripts/gates/docker_provider_data.py index 726582244e..bdf79baaf2 100644 --- a/scripts/gates/docker_provider_data.py +++ b/scripts/gates/docker_provider_data.py @@ -160,28 +160,36 @@ def _kill_worker(worker: dict, parent: dict) -> None: os.close(fd) -async def retired(client, captured: list[dict]) -> bool: +async def retired(client, captured: list[dict], *, observation: dict | None = None) -> bool: from server.insights.store.provider_participants import running_queries + def result(reason: str) -> bool: + if observation is not None: + observation["retirement"] = reason + return reason == "proved" + rows = await client.query_all( "SELECT toString(participant_id) AS participant_id, state, version, host_proof " "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)}", {"ids": [row["participant_id"] for row in captured]}, ) if len(rows) != len(captured): - return False + return result("row_count") expected = {row["participant_id"]: row for row in captured} for row in rows: old = expected[row["participant_id"]] - if ( - row["state"] != "retired" - or int(row["version"]) <= int(old["version"]) - or row["host_proof"] != old["host_proof"] - or not provider_process.has_certificate(row["host_proof"]) - or await running_queries(row["participant_id"], ch=client) != 0 - ): - return False - return True + if row["state"] != "retired": + return result("state") + if int(row["version"]) <= int(old["version"]): + return result("version") + if row["host_proof"] != old["host_proof"]: + return result("host_proof") + if not provider_process.has_certificate(row["host_proof"]): + return result("certificate") + running = await running_queries(row["participant_id"], ch=client) + if running != 0: + return result("queries_unknown" if running is None else "queries_running") + return result("proved") async def crash_worker(client) -> None: @@ -190,18 +198,27 @@ async def crash_worker(client) -> None: victim = before[0] _kill_worker(victim, parent) deadline = time.monotonic() + 240 + observation: dict = {"death": "not_checked", "retirement": "not_checked"} while time.monotonic() < deadline: assert _supervisor() == parent - if provider_process.proves_stopped(victim["host_proof"]) and await retired( - client, [victim] - ): + stopped = provider_process.proves_stopped(victim["host_proof"]) + observation = {"death": "proved" if stopped else "unproved", "retirement": "not_checked"} + if stopped and await retired(client, [victim], observation=observation): after = _workers(await _worker_rows(client), parent) old = {row["participant_id"] for row in before} new = {row["participant_id"] for row in after} + observation.update( + replacement_count=len(new), + survivors=len(old & new), + victim_absent=victim["participant_id"] not in new, + ) if len(new) == 2 and len(old & new) == 1 and victim["participant_id"] not in new: return await asyncio.sleep(1) - raise AssertionError("Crashed worker retirement or supervised replacement was not proved") + raise AssertionError( + "Crashed worker retirement or supervised replacement was not proved: " + + json.dumps(observation, sort_keys=True) + ) async def seed() -> None: diff --git a/tests/unit/test_docker_provider_smoke.py b/tests/unit/test_docker_provider_smoke.py index b1a472c394..8b29bc9077 100644 --- a/tests/unit/test_docker_provider_smoke.py +++ b/tests/unit/test_docker_provider_smoke.py @@ -402,3 +402,53 @@ async def test_retirement_requires_exact_newer_certificate_and_known_zero_querie monkeypatch.setattr(provider_participants, "running_queries", AsyncMock(return_value=count)) assert await data.retired(client, [rows[0]]) is (fault == "none") assert client.query_all.await_args.args[1] == {"ids": [rows[0]["participant_id"]]} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault,reason", + [ + ("death_unknown", '"death": "unproved"'), + ("missing_rows", '"retirement": "row_count"'), + ("queries_unknown", '"retirement": "queries_unknown"'), + ("replacement_mismatch", '"victim_absent": false'), + ("none", None), + ], +) +async def test_crash_observation_distinguishes_unproved_predicates(monkeypatch, fault, reason): + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from server.insights.store import provider_participants + + parent, _processes, rows = _process_fixture(monkeypatch) + monkeypatch.setattr(data, "_supervisor", lambda: parent) + monkeypatch.setattr(data, "workers", AsyncMock(return_value=rows)) + signals = [] + monkeypatch.setattr(data, "_kill_worker", lambda victim, _: signals.append(victim)) + monkeypatch.setattr(data.provider_process, "proves_stopped", lambda _: fault != "death_unknown") + victim = rows[0] + replacement = {**victim, "participant_id": "replacement"} + after = rows if fault == "replacement_mismatch" else [rows[1], replacement] + monkeypatch.setattr(data, "_workers", lambda *_: after) + client = AsyncMock() + client.query_all.return_value = ( + [] if fault == "missing_rows" else [{**victim, "state": "retired", "version": 6}] + ) + monkeypatch.setattr( + provider_participants, + "running_queries", + AsyncMock(return_value=None if fault == "queries_unknown" else 0), + ) + ticks = iter([0, 0, 241]) + monkeypatch.setattr(data, "time", SimpleNamespace(monotonic=lambda: next(ticks))) + monkeypatch.setattr(data.asyncio, "sleep", AsyncMock()) + if reason is None: + await data.crash_worker(client) + else: + with pytest.raises(AssertionError) as failure: + await data.crash_worker(client) + assert reason in str(failure.value) + assert victim["host_proof"] not in str(failure.value) + assert victim["participant_id"] not in str(failure.value) + assert signals == [victim] From f32c6f18f1dc716eb0d768d67c57d2ec043ca9db Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 23:15:36 +0000 Subject: [PATCH 082/114] Attach fixtures isolate authority records and preserve failure causes Migrate a private ClickHouse database pair before attach tests so replacing authority rows cannot erase the integration server's deployment. Exercise the current remove command, roll back the expected SQL failure, and give the doctor a deterministic two-table inventory. Trigger verification names the first failed authority field without changing its comparisons. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 12 +-- tests/integration/test_cell_attach.py | 97 +++++++++++++++++-- .../integration/test_cell_org_triggers_pg.py | 3 +- tests/integration/test_cell_unset_gate_pg.py | 52 +++++++++- tests/unit/test_bench_cell_trigger_gate.py | 78 +++++++++++++++ tests/unit/test_integration_suite_targets.py | 72 ++++++++++++++ 6 files changed, 297 insertions(+), 17 deletions(-) diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index e4a436b2ce..68512d0e9d 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -1316,14 +1316,14 @@ async def _verify( ) else: key = str(expected["key"]) + _require(head["memory_key"] == key, "authority API memory_key") + _require(json.loads(head["value"]) == _payload(key), "authority API value") + _require(head["version"] == expected["version"], "authority API version") _require( - head["memory_key"] == key - and json.loads(head["value"]) == _payload(key) - and head["version"] == expected["version"] - and head["content_sha256"] == expected["content_sha256"] - and head["status"] == "active", - "authority API payload/version", + head["content_sha256"] == expected["content_sha256"], + "authority API content_sha256", ) + _require(head["status"] == "active", "authority API status") _compare(head, projections[id], tuple(head), "full memory projection") key = str(head["memory_key"]) _require(key in documents, "search document identity") diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index a84474806e..15094d202a 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -15,11 +15,13 @@ import argparse import asyncio +import contextlib import hashlib import json import os +import sys import uuid -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Iterator from typing import Any import httpx @@ -29,12 +31,13 @@ from server.cli import cell_attach, cell_attach_pg, cell_prepare from server.cli.cell_ch import Refused from server.core import cell, cell_probe, cell_triggers, instances, pg -from server.core.ch import get_client +from server.core import ch as ch_core from server.core.schema import pg_migrate from server.router import app as router_app from server.router import directory from server.settings import settings from tests.integration._pg_database import create_database, drop_database +from tests.integration._resources import ch_client_from_env from tests.unit.cell_control_owner import owner as _owner OPERATOR = "lco_" + "2" * 128 @@ -71,8 +74,67 @@ async def _admin(sql: str, *args: Any) -> Any: await conn.close() +@pytest.fixture(scope="module") +def cell_clickhouse_databases() -> Iterator[tuple[str, str]]: + """Migrate one owned pair; no authority write can replace the suite server's row.""" + from server.core.schema import migrate + + for name in ("LOOM_PG_DSN", "LOOM_ROUTER_PG_DSN", "LOOM_CH_HOST"): + os.environ.get(name) or pytest.skip(f"{name} not set") + suffix = uuid.uuid4().hex + names = (f"loom_it_attach_{suffix}", f"loom_it_attacht_{suffix}") + created: list[str] = [] + admin = ch_client_from_env(database="") + error: BaseException | None = None + try: + for name in names: + admin.command(f"CREATE DATABASE {name}") + created.append(name) + with pytest.MonkeyPatch.context() as patch: + patch.setattr(settings, "ch_database", names[0]) + patch.setattr(settings, "ch_telemetry_database", names[1]) + normal_client = migrate.client + bootstrap = normal_client() + patch.setattr( + migrate, "client", lambda **kwargs: normal_client(**kwargs) if kwargs else bootstrap + ) + try: + assert migrate.main([]) == 0 + finally: + if sys.exception() is None: + bootstrap.close() + else: + with contextlib.suppress(Exception): + bootstrap.close() + admin.close() + admin = None + yield names + except BaseException as caught: # noqa: BLE001 — exact owned cleanup retains the first failure. + error = caught + finally: + if admin is None: + try: + admin = ch_client_from_env(database="") + except BaseException as caught: # noqa: BLE001 — cleanup cannot replace the first failure. + error = error or caught + if admin is not None: + for name in reversed(created): + try: + admin.command(f"DROP DATABASE {name} SYNC") + except BaseException as caught: # noqa: BLE001 — attempt every exact owned cleanup. + error = error or caught + try: + admin.close() + except BaseException as caught: # noqa: BLE001 — close cannot replace the first failure. + error = error or caught + if error is not None: + raise error + + @pytest.fixture -async def cell_stores(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[dict[str, Any]]: +async def cell_stores( + monkeypatch: pytest.MonkeyPatch, cell_clickhouse_databases: tuple[str, str] +) -> AsyncIterator[dict[str, Any]]: cell_dsn = os.environ.get("LOOM_PG_DSN") or pytest.skip("LOOM_PG_DSN not set") router_dsn = os.environ.get("LOOM_ROUTER_PG_DSN") or pytest.skip("LOOM_ROUTER_PG_DSN not set") os.environ.get("LOOM_CH_HOST") or pytest.skip("LOOM_CH_HOST not set") @@ -92,6 +154,8 @@ async def cell_stores(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[dict[str "router_operator_sha256": hashlib.sha256(OPERATOR.encode()).hexdigest(), "router_operator_token": OPERATOR, "ch_cluster": "", + "ch_database": cell_clickhouse_databases[0], + "ch_telemetry_database": cell_clickhouse_databases[1], "public_base_url": f"https://{HOST}", "oauth_issuer": "", }.items(): @@ -100,7 +164,8 @@ async def cell_stores(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[dict[str monkeypatch.setattr(directory, "_pool", None) # A step's poll interval and the waits stay short. monkeypatch.setattr(cell_attach_pg, "POLL_S", 0.2) - ch = get_client() + ch = ch_core.CHClient() + monkeypatch.setattr(ch_core, "_ch", ch) await ch.connect() ids: list[str] = [] real_live, real_recent = instances.read_live, instances.read_recent @@ -115,7 +180,7 @@ async def read_recent( rows = await real_recent(ch, window_s=window_s, since=since) return [r for r in rows if r["instance_id"] in ids] - # The integration server shares ClickHouse and reports its own process heartbeats. + # These assertions inspect only the heartbeat identities created by this fixture. monkeypatch.setattr(instances, "read_live", read_live) monkeypatch.setattr(instances, "read_recent", read_recent) try: @@ -763,6 +828,23 @@ async def test_doctor_reads_smallest_first_and_leaves_the_largest_unchecked( "INSERT INTO loom.it_large SELECT $1 || g FROM generate_series(1, 20000) g", tag + "l" ) await pg.execute("ANALYZE loom.it_small, loom.it_large") + # This ordering check owns two Postgres tables; shared ClickHouse logs keep growing. + monkeypatch.setattr( + cell_triggers, + "ORG_TABLES", + "SELECT relname FROM pg_class WHERE oid IN " + "('loom.it_small'::regclass, 'loom.it_large'::regclass)", + ) + + async def clickhouse_inventory(*_args: Any) -> tuple[list[Any], list[Any]]: + return [], [] + + monkeypatch.setattr(cell_scan, "list_clickhouse", clickhouse_inventory) + clock = [0.0] + real_budget = cell_scan.Budget + monkeypatch.setattr( + cell_scan, "Budget", lambda seconds: real_budget(seconds, clock=lambda: clock[0]) + ) # The budget runs out on the largest table, as it does on a real cell when that table's # read is what is too slow: every smaller one was read first. real, read = cell_scan.pg_orgs, [] @@ -770,12 +852,13 @@ async def test_doctor_reads_smallest_first_and_leaves_the_largest_unchecked( async def pg_orgs(table: str, budget: Any) -> list[str]: read.append(table) if table == "loom.it_large": - await asyncio.sleep(budget.left() + 0.1) + clock[0] += budget.left() + 0.1 + budget.require(table) return await real(table, budget) monkeypatch.setattr(cell_scan, "pg_orgs", pg_orgs) lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 3)} - assert read[-1] == "loom.it_large" and "loom.it_small" in read + assert read == ["loom.it_small", "loom.it_large"] assert tag + "small" in lines["cell_stray_orgs"][1] grade, detail = lines["cell_stray_orgs_unchecked"] assert grade == "WARN" and detail.endswith(": loom.it_large") diff --git a/tests/integration/test_cell_org_triggers_pg.py b/tests/integration/test_cell_org_triggers_pg.py index c219d05777..e8c797522e 100644 --- a/tests/integration/test_cell_org_triggers_pg.py +++ b/tests/integration/test_cell_org_triggers_pg.py @@ -672,7 +672,8 @@ async def test_a_dropped_guard_function_is_named_in_any_state_and_fails_an_enfor await conn.execute(f"DROP FUNCTION {cell_triggers.TENANCY_FUNCTION}()") if state == "enforced": with pytest.raises(asyncpg.exceptions.UndefinedFunctionError): - await cell_triggers.ensure(conn) + async with conn.transaction(): + await cell_triggers.ensure(conn) defects = {(d.kind, d.text) for d in await cell_triggers.missing(conn)} assert ("function", "loom.tenancy_refuse_unmarked() is absent") in defects if state == "enforced": # the failed boot created no guard diff --git a/tests/integration/test_cell_unset_gate_pg.py b/tests/integration/test_cell_unset_gate_pg.py index e94d5adb3f..37e80a7057 100644 --- a/tests/integration/test_cell_unset_gate_pg.py +++ b/tests/integration/test_cell_unset_gate_pg.py @@ -245,22 +245,68 @@ async def test_loom_cell_removes_check_reads_the_first_cells_postgres_alone( ) -> None: """`loom cell remove`'s check of a first cell: one session from LOOM_PG_DSN, open and recording the row's deployment id, or it refuses.""" + import argparse + import asyncpg + import httpx from server.cli import cell as cell_cli + from server.cli.cell_ch import Refused async with own_database() as dsn: monkeypatch.setattr(settings, "pg_dsn", dsn) + monkeypatch.setattr(settings, "cell", "eu-1") + monkeypatch.setattr(settings, "document_s3_bucket", "") monkeypatch.setattr(pg, "_expected", None) + row = { + "cell": "eu-1", + "is_first": True, + "host": "cell.example.test", + "state": "attaching", + "placed_orgs": 0, + "deployment_id": "dep-1", + "bucket": "", + "bucket_prefix": "", + "probe_url": "", + "upstream_url": "", + "current_fp": None, + "next_fp": None, + "mutation_generation": "00000000-0000-0000-0000-000000000001", + } + calls = [] + + def router(request: httpx.Request) -> httpx.Response: + calls.append(request.method) + if request.method == "GET" or request.url.path.endswith("/claim"): + return httpx.Response(200, json=row) + assert request.method == "DELETE" + return httpx.Response(200, json={"cell": "eu-1", "removed": True}) + + args = argparse.Namespace(cell="eu-1") + url = httpx.URL("https://router.example.test") + transport = httpx.MockTransport(router) conn = await asyncpg.connect(dsn) try: await conn.execute( "INSERT INTO loom.authority (authority, epoch, deployment_id) " "VALUES ('postgres', 1, 'dep-1')" ) - assert await cell_cli._first_cell_refusal("dep-1") == "" - assert "records deployment id 'dep-1'" in await cell_cli._first_cell_refusal("dep-2") + assert await cell_cli._remove_locked(args, url, transport) == { + "cell": "eu-1", + "removed": True, + } + assert calls == ["GET", "POST", "DELETE"] + calls.clear() + row["deployment_id"] = "dep-2" + with pytest.raises(Refused, match="do not match the directory registration"): + await cell_cli._remove_locked(args, url, transport) + assert calls == ["GET"] + assert await conn.fetchval("SELECT deployment_id FROM loom.authority") == "dep-1" + calls.clear() + row["deployment_id"] = "dep-1" await _write_state(conn, {"tenancy"}) - assert "reads `partial`" in await cell_cli._first_cell_refusal("dep-1") + with pytest.raises(Refused, match="reads partial"): + await cell_cli._remove_locked(args, url, transport) + assert calls == ["GET"] finally: await conn.close() diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py index 7543067fd6..d7d59688e5 100644 --- a/tests/unit/test_bench_cell_trigger_gate.py +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -957,6 +957,84 @@ async def query(sql, params): ) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", ["", "memory_key", "value", "version", "content_sha256", "status"] +) +async def test_api_authority_failure_names_only_the_failed_field(monkeypatch, fault): + import logging + + import server.embedding + from server.core.polled_copy import PolledCopy + from server.embedding.fake import FakeNGramEmbedder + from server.llm import customer + from server.memory.store import pg_memory + + monkeypatch.setattr( + customer, + "_switch", + PolledCopy("switch", AsyncMock(return_value=frozenset()), logging.getLogger(__name__)), + ) + monkeypatch.setattr(server.embedding, "get_embedder", lambda: FakeNGramEmbedder(dim=16)) + block = await gate._build_block(args(), 1) + row, audit = block.rows[0], block.audits[0] + expected = gate.RequestRow( + str(row["memory_key"]), + str(row["memory_id"]), + row["version"], + row["content_sha256"], + 1.0, + True, + "derived_ready", + ) + head = dict(row) + if fault: + head[fault] = { + "memory_key": "changed-key", + "value": json.dumps("changed-value"), + "version": row["version"] + 1, + "content_sha256": "0" * 64, + "status": "conflicted", + }[fault] + stored = json.dumps(pg_memory.encode_row(head)) + conn = SimpleNamespace( + fetch=AsyncMock( + return_value=[ + { + "head": stored, + "revision": stored, + "audit": json.dumps(pg_memory.encode_audit(audit)), + "seq": 1, + "targets": 0, + "attempts": 0, + "last_error": "", + } + ] + ), + fetchrow=AsyncMock(return_value={"pending": 0, "audits": 0, "errors": 0}), + ) + document = await gate._expected_document(args(), row) + for field in ("embedding", "memory_key_embedding"): + document[field] = stored_bfloat16(document[field]) + + async def query(sql, params): + if "FROM loom.memory FINAL" in sql: + return [row] + if "FROM loom.memory_embeddings FINAL" in sql: + return [document] + if "FROM loom.audit_log" in sql: + return [audit] + raise AssertionError(sql) + + ch = SimpleNamespace(query_all=query) + if fault: + with pytest.raises(gate.Invalid) as raised: + await gate._verify(conn, ch, args(), [expected]) + assert str(raised.value) == "authority API " + fault + else: + await gate._verify(conn, ch, args(), [expected]) + + @pytest.mark.asyncio async def test_real_customer_policy_can_refuse_expected_vectors_without_bypass(monkeypatch): import logging diff --git a/tests/unit/test_integration_suite_targets.py b/tests/unit/test_integration_suite_targets.py index df7a98561a..1665edcd26 100644 --- a/tests/unit/test_integration_suite_targets.py +++ b/tests/unit/test_integration_suite_targets.py @@ -114,6 +114,78 @@ def test_the_gate_catches_each_implicit_target(tmp_path: Path) -> None: ], found +@pytest.mark.parametrize("fault", ["none", "second_create", "migration", "first_drop"]) +def test_cell_attach_owns_only_its_created_clickhouse_databases(monkeypatch, fault): + from server.core.schema import migrate + from server.settings import settings + from tests.integration import test_cell_attach as attach + + for name in ("LOOM_PG_DSN", "LOOM_ROUTER_PG_DSN", "LOOM_CH_HOST"): + monkeypatch.setenv(name, "owned-test-target") + initial = settings.ch_database, settings.ch_telemetry_database + created, dropped, connections, migrations = [], [], [], [] + + class Driver: + closed = False + + def command(self, sql): + assert not self.closed + action, _, name, *_ = sql.split() + if action == "CREATE": + if fault == "second_create" and created: + raise RuntimeError("second create refused") + created.append(name) + else: + assert action == "DROP" and name in created + dropped.append(name) + if fault == "first_drop" and len(dropped) == 1: + raise RuntimeError("first drop refused") + + def close(self): + self.closed = True + + def admin(**kwargs): + assert kwargs == {"database": ""} + driver = Driver() + connections.append(driver) + return driver + + bootstrap = Driver() + monkeypatch.setattr(attach, "ch_client_from_env", admin) + monkeypatch.setattr(migrate, "client", lambda **kwargs: bootstrap) + + def apply(args): + assert args == [] + assert migrate.client() is bootstrap + migrations.append((settings.ch_database, settings.ch_telemetry_database)) + assert migrations[-1] == tuple(created) + assert all(name.startswith("loom_it_attach") for name in created) + return 1 if fault == "migration" else 0 + + monkeypatch.setattr(migrate, "main", apply) + fixture = attach.cell_clickhouse_databases.__wrapped__() + if fault == "second_create": + with pytest.raises(RuntimeError, match="second create refused"): + next(fixture) + assert migrations == [] + elif fault == "migration": + with pytest.raises(AssertionError): + next(fixture) + else: + assert next(fixture) == tuple(created) + assert bootstrap.closed and all(driver.closed for driver in connections) + if fault == "first_drop": + with pytest.raises(RuntimeError, match="first drop refused"): + next(fixture) + else: + with pytest.raises(StopIteration): + next(fixture) + assert dropped == list(reversed(created)) + assert all(driver.closed for driver in connections) + assert (settings.ch_database, settings.ch_telemetry_database) == initial + assert len(migrations) == (0 if fault == "second_create" else 1) + + def test_the_gate_catches_a_clickhouse_cli_without_the_target_fixture(tmp_path: Path) -> None: probe = tmp_path / "test_probe.py" probe.write_text('CLI = ROOT / "dev" / "backup_loom.py"\n') From 7ad45ce88cf5af644750453ca138ebe67eff7dc8 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 00:04:01 +0000 Subject: [PATCH 083/114] Attach acceptance verifies two genuine API replicas Join HTTP identities to live process and participant evidence, and finish owned startup and cleanup before store teardown. Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 1 + tests/integration/test_cell_attach.py | 87 +++- .../test_cell_attach_two_replicas.py | 492 ++++++++++++++++++ .../test_cell_attach_replica_ownership.py | 132 +++++ 4 files changed, 693 insertions(+), 19 deletions(-) create mode 100644 tests/integration/test_cell_attach_two_replicas.py create mode 100644 tests/unit/test_cell_attach_replica_ownership.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f3ed6e347..a96e01ebe4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -910,6 +910,7 @@ jobs: tests/integration/test_cell_unset_gate_pg.py tests/integration/test_cell_prepare.py tests/integration/test_cell_attach.py + tests/integration/test_cell_attach_two_replicas.py tests/integration/test_cell_attach_cleanup.py tests/integration/test_cell_control_cleanup.py tests/integration/test_cell_control_fence.py diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 15094d202a..71a2b41b71 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -131,15 +131,30 @@ def cell_clickhouse_databases() -> Iterator[tuple[str, str]]: raise error +@pytest.fixture +def genuine_cell_processes() -> bool: + """Synthetic cases own their rows; the live-replica case uses the real readers.""" + return False + + @pytest.fixture async def cell_stores( - monkeypatch: pytest.MonkeyPatch, cell_clickhouse_databases: tuple[str, str] + monkeypatch: pytest.MonkeyPatch, + cell_clickhouse_databases: tuple[str, str], + genuine_cell_processes: bool, ) -> AsyncIterator[dict[str, Any]]: cell_dsn = os.environ.get("LOOM_PG_DSN") or pytest.skip("LOOM_PG_DSN not set") router_dsn = os.environ.get("LOOM_ROUTER_PG_DSN") or pytest.skip("LOOM_ROUTER_PG_DSN not set") os.environ.get("LOOM_CH_HOST") or pytest.skip("LOOM_CH_HOST not set") cell_db, own_cell = await create_database(cell_dsn) - router_db, own_router = await create_database(router_dsn) + try: + router_db, own_router = await create_database(router_dsn) + except BaseException as first: # Release only the database already created by this fixture. + try: + await drop_database(cell_dsn, cell_db) + except BaseException as error: # noqa: BLE001 — cleanup retains the creation failure. + first.add_note(f"cell database cleanup also failed: {type(error).__name__}") + raise deployment = f"dep-{uuid.uuid4().hex[:8]}" tag = f"it-{uuid.uuid4().hex[:8]}-" for name, value in { @@ -160,13 +175,11 @@ async def cell_stores( "oauth_issuer": "", }.items(): monkeypatch.setattr(settings, name, value) - await pg.close_pool() monkeypatch.setattr(directory, "_pool", None) # A step's poll interval and the waits stay short. monkeypatch.setattr(cell_attach_pg, "POLL_S", 0.2) ch = ch_core.CHClient() monkeypatch.setattr(ch_core, "_ch", ch) - await ch.connect() ids: list[str] = [] real_live, real_recent = instances.read_live, instances.read_recent @@ -181,9 +194,12 @@ async def read_recent( return [r for r in rows if r["instance_id"] in ids] # These assertions inspect only the heartbeat identities created by this fixture. - monkeypatch.setattr(instances, "read_live", read_live) - monkeypatch.setattr(instances, "read_recent", read_recent) + if not genuine_cell_processes: + monkeypatch.setattr(instances, "read_live", read_live) + monkeypatch.setattr(instances, "read_recent", read_recent) try: + await pg.close_pool() + await ch.connect() await pg_migrate.migrate() await pg.execute( "INSERT INTO loom.authority (authority, epoch, deployment_id) " @@ -193,30 +209,63 @@ async def read_recent( await directory.migrate() await ch.insert( "loom.storage_authority", - [{"authority": "postgres", "epoch": 10**12, "deployment_id": deployment}], + [ + { + "authority": "postgres", + "epoch": 1 if genuine_cell_processes else 10**12, + "deployment_id": deployment, + } + ], ) cell.set_deployment_id(deployment) yield {"ch": ch, "ids": ids, "tag": tag, "deployment": deployment, "db": cell_db} finally: cell.set_deployment_id("") - await ch.execute( + await _close_cell_stores( + ch, deployment, ids, tag, (cell_dsn, cell_db), (router_dsn, router_db), sys.exception() + ) + + +async def _close_cell_stores( + ch: Any, + deployment: str, + ids: list[str], + tag: str, + cell_database: tuple[str, str], + router_database: tuple[str, str], + first: BaseException | None, +) -> None: + # Every operation owns an exact database, deployment, heartbeat id or org prefix. + cleanup = [ + ch.execute( "ALTER TABLE loom.storage_authority DELETE WHERE deployment_id = {d:String} " "SETTINGS mutations_sync = 2", {"d": deployment}, - ) - for table in (instances.TABLE, instances.CELL_TABLE): - await ch.execute( + ), + *( + ch.execute( f"ALTER TABLE {table} DELETE WHERE has({{ids:Array(String)}}, instance_id)", {"ids": ids}, ) - await ch.execute( - "DELETE FROM loom.namespaces WHERE startsWith(org, {p:String})", {"p": tag} - ) - await ch.disconnect() - await pg.close_pool() - await directory.close() - await drop_database(cell_dsn, cell_db) - await drop_database(router_dsn, router_db) + for table in (instances.TABLE, instances.CELL_TABLE) + ), + ch.execute("DELETE FROM loom.namespaces WHERE startsWith(org, {p:String})", {"p": tag}), + ch.disconnect(), + pg.close_pool(), + directory.close(), + drop_database(*cell_database), + drop_database(*router_database), + ] + for operation in cleanup: + try: + await operation + except BaseException as error: # noqa: BLE001 — attempt every owned cleanup; keep first failure. + if first is None: + first = error + else: + first.add_note(f"store cleanup also failed: {type(error).__name__}") + if first is not None: + raise first def _value(**keys: Any) -> str: diff --git a/tests/integration/test_cell_attach_two_replicas.py b/tests/integration/test_cell_attach_two_replicas.py new file mode 100644 index 0000000000..09a28fba88 --- /dev/null +++ b/tests/integration/test_cell_attach_two_replicas.py @@ -0,0 +1,492 @@ +"""Attach a first cell while two real API processes share its private stores. + +The router uses real HTTP upstreams and a test-owned, verified TLS listener. No +heartbeat row or health response is supplied by the test. Requires the same +synchronous Postgres and ClickHouse services as test_cell_attach. +""" + +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import hashlib +import ipaddress +import json +import os +import socket +import ssl +import subprocess +import sys +from collections.abc import AsyncIterator, Generator +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any +from uuid import UUID + +import httpx +import pytest +import uvicorn +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID + +from server.cli import cell_attach, cell_prepare +from server.core import cell_triggers, instances, pg, provider_process +from server.core.ch import query_tag +from server.insights.store import provider_participants +from server.router import app as router_app +from server.settings import settings +from tests.integration import test_cell_attach as attach_fixtures +from tests.integration._ports import free_port +from tests.integration._redact import redact +from tests.integration._resources import unique_org +from tests.integration._spawn import ROOT, loom_server +from tests.integration.test_cell_attach import ( + _args, + _mode, + _placed, + _row, +) + +_CREDENTIAL = "lcr_" + "3" * 128 +_WAIT_S = 45 + +cell_clickhouse_databases = attach_fixtures.cell_clickhouse_databases +cell_stores = attach_fixtures.cell_stores + + +@pytest.fixture +def genuine_cell_processes() -> bool: + return True + + +def _tls_files(root: Path) -> tuple[Path, Path, Path]: + """An ephemeral CA and IP-address leaf, trusted only by this test's clients.""" + now = datetime.now(UTC) + ca_key = ec.generate_private_key(ec.SECP256R1()) + ca_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "attach test CA")]) + ca = ( + x509.CertificateBuilder() + .subject_name(ca_name) + .issuer_name(ca_name) + .public_key(ca_key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - timedelta(minutes=1)) + .not_valid_after(now + timedelta(days=1)) + .add_extension(x509.BasicConstraints(ca=True, path_length=0), critical=True) + .sign(ca_key, hashes.SHA256()) + ) + key = ec.generate_private_key(ec.SECP256R1()) + leaf = ( + x509.CertificateBuilder() + .subject_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "localhost")])) + .issuer_name(ca_name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - timedelta(minutes=1)) + .not_valid_after(now + timedelta(days=1)) + .add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True) + .add_extension( + x509.SubjectAlternativeName([x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]), + critical=False, + ) + .add_extension(x509.ExtendedKeyUsage([ExtendedKeyUsageOID.SERVER_AUTH]), critical=False) + .sign(ca_key, hashes.SHA256()) + ) + ca_path, cert_path, key_path = (root / name for name in ("ca.pem", "server.pem", "key.pem")) + ca_path.write_bytes(ca.public_bytes(serialization.Encoding.PEM)) + cert_path.write_bytes(leaf.public_bytes(serialization.Encoding.PEM)) + key_path.write_bytes( + key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ) + ) + key_path.chmod(0o600) + return ca_path, cert_path, key_path + + +class _RouterServer(uvicorn.Server): + @contextlib.contextmanager + def capture_signals(self) -> Generator[None]: + # pytest owns the main thread's signals. + yield + + +@contextlib.asynccontextmanager +async def _router(port: int, cert: Path, key: Path) -> AsyncIterator[None]: + router = router_app.Router() + server = _RouterServer( + uvicorn.Config( + router_app.build_app(router), + host="127.0.0.1", + port=port, + ssl_certfile=str(cert), + ssl_keyfile=str(key), + log_level="warning", + timeout_graceful_shutdown=5, + ) + ) + task = asyncio.create_task(server.serve(), name="attach-test-router") + try: + async with asyncio.timeout(10): + while not server.started: + if task.done(): + await task + raise AssertionError("router stopped before listening") + await asyncio.sleep(0.05) + yield + finally: + first = sys.exception() + server.should_exit = True + try: + await asyncio.wait_for(task, timeout=10) + except BaseException as error: # noqa: BLE001 — finish all owned cleanup; keep first failure. + first = first or error + finally: + # A timed-out uvicorn task must not leave its listener or requests running. + for listener in getattr(server, "servers", ()): + try: + listener.close() + await listener.wait_closed() + except BaseException as error: # noqa: BLE001 — attempt remaining owned cleanup. + first = first or error + requests = tuple(server.server_state.tasks) + for request in requests: + request.cancel() + for operation in ( + asyncio.wait_for(asyncio.gather(*requests, return_exceptions=True), timeout=5), + router.close(), + ): + try: + await operation + except BaseException as error: # noqa: BLE001 — keep first; finish router cleanup. + first = first or error + if first is not None: + raise first + + +@contextlib.asynccontextmanager +async def _api(name: str, env: dict[str, str], port: int) -> AsyncIterator[str]: + """Keep the router loop available and retain each worker until its context settles.""" + owned = loom_server(name, {**env, "LOOM_INSTANCE_LABEL": name}, port=port) + first: BaseException | None = None + + async def finish[T](task: asyncio.Task[T]) -> T: + nonlocal first + while not task.done(): + try: + await asyncio.wait({task}) + except asyncio.CancelledError as error: + if first is None: + first = error + return task.result() + + entry = asyncio.create_task(asyncio.to_thread(owned.__enter__)) + try: + yield await asyncio.shield(entry) + except BaseException as error: # noqa: BLE001 — cleanup completes before the first error escapes. + first = error + finally: + try: + await finish(entry) + except BaseException as error: # noqa: BLE001 — a failed entry already ran its own cleanup. + first = first or error + else: + try: + await finish( + asyncio.create_task(asyncio.to_thread(owned.__exit__, None, None, None)) + ) + except BaseException as error: # noqa: BLE001 — retain the body error after owned cleanup. + if first is None: + first = error + else: + first.add_note(f"{name} process cleanup also failed: {type(error).__name__}") + if first is not None: + raise first + + +def _configure(monkeypatch: pytest.MonkeyPatch, host: str, api: int, probe: int) -> dict[str, str]: + values = { + "cell": "first", + "storage_authority": "postgres", + "pg_durability": "synchronous", + "bind_host": "127.0.0.1", + "serves_traffic": True, + "pg_dsn": settings.pg_dsn, + "router_pg_dsn": settings.router_pg_dsn, + "ch_database": settings.ch_database, + "ch_telemetry_database": settings.ch_telemetry_database, + "ch_cluster": "", + "document_s3_bucket": "", + "public_base_url": f"https://{host}", + "oauth_issuer": "", + "cell_router_url": f"https://{host}", + "cell_router_token": _CREDENTIAL, + "cell_probe_port": probe, + "router_shared_host": host, + "router_upstream_url": f"http://127.0.0.1:{api}", + "router_upstream_probe_url": f"http://127.0.0.1:{probe}", + "instance_heartbeat_seconds": 1, + "tenancy_mode_cache_s": 1, + "router_directory_cache_s": 1, + "background_workers": "off", + "embedding_provider": "fake", + "embedding_model": "fake-ngram-1024", + "embedding_dim": 1024, + "llm_provider": "fake", + "llm_model": "fake-echo-1", + "require_real_providers": False, + "boot_migrate": False, + "auth_required": True, + "identity_provider": "gateway", + "self_signup": "off", + "mcp_auth_mode": "required", + } + for key, value in values.items(): + monkeypatch.setattr(settings, key, value) + env = { + f"LOOM_{key.upper()}": str(value).lower() if isinstance(value, bool) else str(value) + for key, value in values.items() + } + # API processes get the cell credential, never directory/operator authority. + env.update(LOOM_ROUTER_OPERATOR_TOKEN="", LOOM_ROUTER_OPERATOR_SHA256="", LOOM_ROUTER_PG_DSN="") + return env + + +async def _identities(client: httpx.AsyncClient, bases: list[str]) -> list[dict[str, Any]]: + identities = [] + for base in bases: + response = await client.get(f"{base}/v1/health") + assert response.status_code == 200, response.text + health = response.json() + host, pid = health["instance"].rsplit(":", 1) + assert host == socket.gethostname() + text = await asyncio.to_thread(Path(f"/proc/{int(pid)}/stat").read_text) + stat = text.rsplit(")", 1)[1].split() + assert int(stat[1]) == os.getpid(), "the API must be this test's child" + assert stat[0] not in {"Z", "X"} + identities.append( + { + "pid": int(pid), + "start": stat[19], + "participant": health["participant_id"], + "instance": health["deployment_config"]["instance_id"], + "label": health["instance_label"], + } + ) + assert len({i["pid"] for i in identities}) == 2 + assert len({i["participant"] for i in identities}) == 2 + assert len({i["instance"] for i in identities}) == 2 + assert {i["label"] for i in identities} == {"attach-two-a", "attach-two-b"} + return identities + + +async def _live(ch: Any, identities: list[dict[str, Any]], mode: str) -> None: + """Match unfiltered live and cell rows to the exact HTTP and kernel witnesses.""" + expected = {i["instance"] for i in identities} + async with asyncio.timeout(_WAIT_S): + while True: + rows = await instances.read_live(ch, all_deployments=True) + # KeeperMap PRIMARY KEY instance_id: one current row, no merge ordering. + cells = await ch.query_all(f"SELECT * FROM {instances.CELL_TABLE}") + by_id = {r["instance_id"]: r for r in cells} + if {r["instance_id"] for r in rows} == expected and all( + i["instance"] in by_id + and by_id[i["instance"]]["tenancy_mode"] == mode + and ( + mode != "enforced" + or next(r for r in rows if r["instance_id"] == i["instance"])["serving"] + ) + for i in identities + ): + break + await asyncio.sleep(0.2) + processes = await ch.query_all( + f"SELECT instance_id, pid FROM {instances.TABLE} " + "WHERE instance_id IN {ids:Array(String)}", + {"ids": sorted(expected)}, + ) + pids = {r["instance_id"]: int(r["pid"]) for r in processes} + boot = await asyncio.to_thread(Path("/proc/sys/kernel/random/boot_id").read_text) + participants = await provider_participants.participants(ch=ch) + by_participant = {str(r["participant_id"]): r for r in participants} + for identity in identities: + assert pids[identity["instance"]] == identity["pid"] + raw = by_id[identity["instance"]] + fingerprint = json.loads(raw["fingerprint"]) + assert raw["cell"] == "first" + assert fingerprint["participant_id"] == identity["participant"] + participant = by_participant[identity["participant"]] + assert participant["state"] == "running" + proof = participant["host_proof"] + assert provider_process.has_certificate(proof) + parts = proof.split("|") + assert parts[0] == socket.gethostname() + assert parts[1] == boot.strip() + assert parts[3] == str(os.stat("/proc/self/ns/pid").st_ino) + assert (parts[4], parts[5]) == (str(identity["pid"]), identity["start"]) + assert not provider_process.proves_stopped(proof) + if mode == "enforced": + assert fingerprint["router"] is True + assert fingerprint["marked"] is True + assert next(r for r in rows if r["instance_id"] == identity["instance"])["serving"] + identity["proof"] = proof + + +async def _retired(ch: Any, identities: list[dict[str, Any]]) -> None: + """A helper's TERM/KILL/reap alone does not establish guarded retirement.""" + rows = await ch.query_all( + f"SELECT participant_id, state, host_proof FROM {provider_participants.TABLE} FINAL " + "WHERE participant_id IN {ids:Array(UUID)}", + {"ids": [UUID(i["participant"]) for i in identities]}, + ) + by_id = {str(row["participant_id"]): row for row in rows} + for identity in identities: + assert provider_process.proves_stopped(identity["proof"]) + row = by_id[identity["participant"]] + assert row["state"] == "retired", "stopping or killing is not retirement proof" + assert row["host_proof"] == identity["proof"] + assert provider_process.has_certificate(row["host_proof"]) + + +async def _mint(env: dict[str, str], org: str) -> str: + # A fresh native command reads this cell's keyring, not pytest's signing cache. + result = await asyncio.to_thread( + subprocess.run, + [ + sys.executable, + "-m", + "server.cli", + "mint-token", + "--subject", + "attach-two", + "--org", + org, + "--namespaces", + "n", + "--roles", + "writer", + "--ttl-days", + "1", + "--json", + ], + cwd=ROOT, + env={**os.environ, **env, "LOOM_CELL_ROUTER_TOKEN": ""}, + capture_output=True, + text=True, + timeout=_WAIT_S, + check=False, + ) + assert result.returncode == 0, redact(result.stderr) + token = json.loads(result.stdout)["token"] + assert isinstance(token, str) and token + return token + + +async def _requests(client: httpx.AsyncClient, bases: list[str], org: str, token: str) -> None: + headers = {"Authorization": f"Bearer {token}"} + key = {"org": org, "namespace": "n", "agent": "attach-test", "memory_key": "shared"} + for index, base in enumerate(bases): + unauthenticated = await client.post(f"{base}/v1/memory.get", json=key) + assert unauthenticated.status_code == 401, unauthenticated.text + value = {"writer": index} + written = await client.post( + f"{base}/v1/memory.set", + headers=headers, + json={**key, "memory_type": "semantic", "value": value}, + ) + assert written.status_code == 200, written.text + for reader in bases: + read = await client.post(f"{reader}/v1/memory.get", headers=headers, json=key) + assert read.status_code == 200, read.text + assert read.json()["value"] == value + + +@pytest.mark.asyncio +async def test_attach_with_two_live_api_replicas( + cell_stores: dict[str, Any], + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + stores = cell_stores + api_a, api_b, probe, router_port = (free_port() for _ in range(4)) + host = f"127.0.0.1:{router_port}" + env = _configure(monkeypatch, host, api_a, probe) + ca, certificate, key = _tls_files(tmp_path) + env["SSL_CERT_FILE"] = str(ca) + trust = ssl.create_default_context(cafile=str(ca)) + router = httpx.URL(f"https://{host}") + org = unique_org("attach-two") + identities: list[dict[str, Any]] = [] + with query_tag("test.cell_attach_two_replicas"): + async with _router(router_port, certificate, key): + try: + async with ( + _api("attach-two-a", env, api_a) as a, + _api("attach-two-b", env, api_b) as b, + httpx.AsyncClient(verify=trust, timeout=30) as client, + ): + identities = await _identities(client, [a, b]) + stores["ids"].extend(i["instance"] for i in identities) + await _live(stores["ch"], identities, "open") + args = argparse.Namespace( + cell="first", + host=host, + credential_sha256=hashlib.sha256(_CREDENTIAL.encode()).hexdigest(), + probe_url="", + wait=20, + router=str(router), + ) + assert ( + await cell_prepare.prepare( + stores["ch"], args, router, httpx.AsyncHTTPTransport(verify=trust) + ) + == 0 + ) + await _live(stores["ch"], identities, "recording") + await pg.execute("INSERT INTO loom.tenancy (org) VALUES ($1)", org) + await stores["ch"].insert( + "loom.namespaces", [{"org": org, "namespace": "n", "owner_email": ""}] + ) + capsys.readouterr() + assert ( + await cell_attach.attach( + stores["ch"], + _args(router=str(router), wait=_WAIT_S), + router, + httpx.AsyncHTTPTransport(verify=trust), + ) + == 0 + ) + assert json.loads(capsys.readouterr().out)["state"] == "serving" + assert await _mode() == "enforced" + row = await _row() + assert (row.state, row.mode) == ("serving", "enforced") + assert org in await _placed() + async with pg.dedicated() as conn: + assert await cell_triggers.missing(conn) == [] + await _live(stores["ch"], identities, "enforced") + token = await _mint(env, org) + await _requests(client, [a, b, str(router)], org, token) + assert await _identities(client, [a, b]) == [ + {k: v for k, v in identity.items() if k != "proof"} + for identity in identities + ] + finally: + first = sys.exception() + if identities: + try: + await _retired(stores["ch"], identities) + except ( + BaseException + ) as error: # Retirement failure cannot hide the test failure. + if first is None: + raise + first.add_note( + f"retirement proof also failed: {type(error).__name__}: {error}" + ) diff --git a/tests/unit/test_cell_attach_replica_ownership.py b/tests/unit/test_cell_attach_replica_ownership.py new file mode 100644 index 0000000000..f2317d31a9 --- /dev/null +++ b/tests/unit/test_cell_attach_replica_ownership.py @@ -0,0 +1,132 @@ +"""The threaded API fixture owns entry and exit until cleanup has finished.""" + +import ast +import asyncio +import contextlib +import sys +import threading +from collections.abc import AsyncIterator +from pathlib import Path + +import pytest + +SOURCE = Path(__file__).resolve().parents[1] / "integration/test_cell_attach_two_replicas.py" + + +def _fixture(context): + # Execute the actual helper alone: no server imports, services or subprocesses. + tree = ast.parse(SOURCE.read_text()) + helper = next(n for n in tree.body if isinstance(n, ast.AsyncFunctionDef) and n.name == "_api") + + def spawn(name, env, *, port): + assert (name, env, port) == ("owned", {"LOOM_INSTANCE_LABEL": "owned"}, 1) + return context + + namespace = { + "asyncio": asyncio, + "contextlib": contextlib, + "sys": sys, + "AsyncIterator": AsyncIterator, + "loom_server": spawn, + } + exec(compile(ast.Module(body=[helper], type_ignores=[]), str(SOURCE), "exec"), namespace) + return namespace["_api"] + + +@pytest.mark.parametrize("phase", ["entry", "exit", "body-error"]) +def test_cancellation_waits_for_owned_context_before_fixture_unwinds(phase): + async def scenario(): + observed = asyncio.Event() + entry_finished = threading.Event() + release = threading.Event() + loop = asyncio.get_running_loop() + unwound = [] + first = ValueError("body failed") if phase == "body-error" else None + + class Context: + alive = False + exits = 0 + + def __enter__(self): + self.alive = True + try: + if phase == "entry": + loop.call_soon_threadsafe(observed.set) + assert release.wait(5), "test did not release entry" + return "http://owned" + finally: + entry_finished.set() + + def __exit__(self, *_): + self.exits += 1 + if phase != "entry": + loop.call_soon_threadsafe(observed.set) + assert release.wait(5), "test did not release exit" + self.alive = False + + context = Context() + api = _fixture(context) + + async def owner(): + try: + async with api("owned", {}, 1) as base: + assert base == "http://owned" + if first is not None: + raise first + finally: + unwound.append(context.alive) + + task = asyncio.create_task(owner()) + try: + await asyncio.wait_for(observed.wait(), timeout=2) + for message in ("first cancel", "second cancel", "third cancel"): + task.cancel(message) + await asyncio.sleep(0) + assert not task.done(), "owner unwound while its context was still live" + assert unwound == [] and context.alive + release.set() + with pytest.raises( + ValueError if first is not None else asyncio.CancelledError + ) as caught: + await asyncio.wait_for(task, timeout=2) + if first is not None: + assert caught.value is first + else: + assert caught.value.args == ("first cancel",) + assert context.exits == 1 and not context.alive + assert unwound == [False] + finally: + release.set() + with contextlib.suppress(BaseException): + await task + assert await asyncio.to_thread(entry_finished.wait, 2) + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("startup_fails", [False, True]) +def test_entry_outcome_selects_exactly_one_exit_or_its_existing_failure_cleanup(startup_fails): + first = ValueError("startup failed after its own cleanup") + exits = [] + + class Context: + def __enter__(self): + if startup_fails: + raise first + return "http://owned" + + def __exit__(self, *_): + exits.append(True) + + async def scenario(): + async with _fixture(Context())("owned", {}, 1) as base: + assert base == "http://owned" + + if startup_fails: + with pytest.raises(ValueError) as caught: + asyncio.run(scenario()) + assert caught.value is first + assert exits == [] + else: + asyncio.run(scenario()) + assert exits == [True] From 69b6efe7c03e8f00c75f73b0b756d443ca27d014 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 00:04:01 +0000 Subject: [PATCH 084/114] Worker crash checks read captured identities before signalling Refuse an unreadable worker before signalling and retain finite native-reader evidence when the original retirement proof fails. Signed-off-by: zlareb1 --- scripts/gates/docker_provider_data.py | 44 +++++++++++++-- tests/unit/test_docker_provider_smoke.py | 70 +++++++++++++++++++++++- 2 files changed, 106 insertions(+), 8 deletions(-) diff --git a/scripts/gates/docker_provider_data.py b/scripts/gates/docker_provider_data.py index bdf79baaf2..c85f60c5bf 100644 --- a/scripts/gates/docker_provider_data.py +++ b/scripts/gates/docker_provider_data.py @@ -160,6 +160,30 @@ def _kill_worker(worker: dict, parent: dict) -> None: os.close(fd) +async def _retirement_rows(client, captured: list[dict]) -> list[dict]: + return await client.query_all( + "SELECT toString(participant_id) AS participant_id, state, version, host_proof " + "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)}", + {"ids": [row["participant_id"] for row in captured]}, + ) + + +async def _native_retirement_observation(client, captured: list[dict]) -> dict: + try: + rows = await client.query_all( + "SELECT participant_id " + "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)}", + {"ids": [row["participant_id"] for row in captured]}, + ) + return { + "reference_count": len(rows), + "reference_identity": {str(row["participant_id"]) for row in rows} + == {row["participant_id"] for row in captured}, + } + except Exception: # noqa: BLE001 — diagnostic failure cannot replace the unproved retirement + return {"reference_read": "query_error"} + + async def retired(client, captured: list[dict], *, observation: dict | None = None) -> bool: from server.insights.store.provider_participants import running_queries @@ -168,11 +192,7 @@ def result(reason: str) -> bool: observation["retirement"] = reason return reason == "proved" - rows = await client.query_all( - "SELECT toString(participant_id) AS participant_id, state, version, host_proof " - "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)}", - {"ids": [row["participant_id"] for row in captured]}, - ) + rows = await _retirement_rows(client, captured) if len(rows) != len(captured): return result("row_count") expected = {row["participant_id"]: row for row in captured} @@ -196,6 +216,18 @@ async def crash_worker(client) -> None: parent = _supervisor() before = await workers(client) victim = before[0] + for worker in before: + rows = await _retirement_rows(client, [worker]) + identity_matches = len(rows) == 1 and rows[0]["participant_id"] == worker["participant_id"] + if not identity_matches: + readback = { + "reader_count": len(rows), + "reader_identity": identity_matches, + **await _native_retirement_observation(client, [worker]), + } + raise AssertionError( + "Captured worker readback failed: " + json.dumps(readback, sort_keys=True) + ) _kill_worker(victim, parent) deadline = time.monotonic() + 240 observation: dict = {"death": "not_checked", "retirement": "not_checked"} @@ -215,6 +247,8 @@ async def crash_worker(client) -> None: if len(new) == 2 and len(old & new) == 1 and victim["participant_id"] not in new: return await asyncio.sleep(1) + if observation["retirement"] == "row_count": + observation.update(await _native_retirement_observation(client, [victim])) raise AssertionError( "Crashed worker retirement or supervised replacement was not proved: " + json.dumps(observation, sort_keys=True) diff --git a/tests/unit/test_docker_provider_smoke.py b/tests/unit/test_docker_provider_smoke.py index 8b29bc9077..0122b28ed7 100644 --- a/tests/unit/test_docker_provider_smoke.py +++ b/tests/unit/test_docker_provider_smoke.py @@ -432,9 +432,12 @@ async def test_crash_observation_distinguishes_unproved_predicates(monkeypatch, after = rows if fault == "replacement_mismatch" else [rows[1], replacement] monkeypatch.setattr(data, "_workers", lambda *_: after) client = AsyncMock() - client.query_all.return_value = ( - [] if fault == "missing_rows" else [{**victim, "state": "retired", "version": 6}] - ) + client.query_all.side_effect = [ + [rows[0]], + [rows[1]], + [] if fault == "missing_rows" else [{**victim, "state": "retired", "version": 6}], + [victim], + ] monkeypatch.setattr( provider_participants, "running_queries", @@ -452,3 +455,64 @@ async def test_crash_observation_distinguishes_unproved_predicates(monkeypatch, assert victim["host_proof"] not in str(failure.value) assert victim["participant_id"] not in str(failure.value) assert signals == [victim] + reads = client.query_all.await_args_list + assert [call.args[1] for call in reads[:2]] == [ + {"ids": [row["participant_id"]]} for row in rows + ] + assert all( + "SELECT toString(participant_id) AS participant_id" in call.args[0] for call in reads[:2] + ) + if fault == "missing_rows": + assert '"reference_count": 1' in str(failure.value) + assert '"reference_identity": true' in str(failure.value) + assert "SELECT participant_id FROM" in reads[-1].args[0] + else: + assert all("SELECT participant_id FROM" not in call.args[0] for call in reads) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", ["missing-first", "missing-second", "wrong-identity", "reference-error"] +) +async def test_crash_refuses_unreadable_captured_worker_before_signalling(monkeypatch, fault): + from types import SimpleNamespace + from unittest.mock import AsyncMock + + parent, _processes, rows = _process_fixture(monkeypatch) + monkeypatch.setattr(data, "_supervisor", lambda: parent) + monkeypatch.setattr(data, "workers", AsyncMock(return_value=rows)) + signals = [] + monkeypatch.setattr(data, "_kill_worker", lambda victim, _: signals.append(victim)) + monkeypatch.setattr(data.provider_process, "proves_stopped", lambda _: True) + client = AsyncMock() + + async def query(sql, parameters): + selected = next(row for row in rows if [row["participant_id"]] == parameters["ids"]) + if sql.startswith("SELECT participant_id FROM"): + if fault == "reference-error": + raise RuntimeError("private diagnostic failure") + return [selected] + if fault == "missing-second" and selected == rows[0]: + return [selected] + return [rows[1]] if fault == "wrong-identity" else [] + + client.query_all.side_effect = query + ticks = iter([0, 0, 241]) + monkeypatch.setattr(data, "time", SimpleNamespace(monotonic=lambda: next(ticks))) + monkeypatch.setattr(data.asyncio, "sleep", AsyncMock()) + with pytest.raises(AssertionError, match="Captured worker readback failed") as failure: + await data.crash_worker(client) + assert signals == [] + message = str(failure.value) + assert '"reader_identity": false' in message + assert f'"reader_count": {int(fault == "wrong-identity")}' in message + if fault == "reference-error": + assert '"reference_read": "query_error"' in message + assert "private diagnostic failure" not in message + else: + assert '"reference_count": 1' in message + assert '"reference_identity": true' in message + assert all(row["host_proof"] not in message for row in rows) + assert [call.args[1] for call in client.query_all.await_args_list] == ( + [{"ids": [rows[0]["participant_id"]]}] if fault == "missing-second" else [] + ) + [{"ids": [rows[int(fault == "missing-second")]["participant_id"]]}] * 2 From eadf2bf13038f7f8885cbb849fafa721c40f7f56 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 00:04:01 +0000 Subject: [PATCH 085/114] Standby drill checks ledger visibility after interrupted commit A commit waiting for its standby remains in the transaction snapshot. Keep the exact ledger assertion after cancellation releases the backend, and prove the index is ready beforehand. Signed-off-by: zlareb1 --- .../test_pg_online_index_durability.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_pg_online_index_durability.py b/tests/integration/test_pg_online_index_durability.py index f38453e747..5acb835065 100644 --- a/tests/integration/test_pg_online_index_durability.py +++ b/tests/integration/test_pg_online_index_durability.py @@ -131,14 +131,14 @@ async def pause_before_ledger(*args, **kwargs): first = asyncio.create_task(migrate()) tasks.append(first) pid = await _waiting(observer, first) - # The index is acknowledged already; its ledger COMMIT alone is unflushed. + # The index is acknowledged; the pending ledger transaction is not visible yet. async with connect() as conn: assert ( await conn.fetchval( - "SELECT count(*) FROM loom_router.pg_migrations " - "WHERE filename = '004_placements_by_cell_org.sql'" + "SELECT indisvalid AND indisready FROM pg_index " + "WHERE indexrelid = 'loom_router.placements_by_cell_org'::regclass" ) - == 1 + is True ) first.cancel() with pytest.raises(asyncio.CancelledError): @@ -152,6 +152,15 @@ async def pause_before_ledger(*args, **kwargs): await asyncio.sleep(0.01) else: pytest.fail("cancelled migration retained backend locks") + # Cancellation releases the commit wait before other sessions can see the ledger. + async with connect() as conn: + assert ( + await conn.fetchval( + "SELECT count(*) FROM loom_router.pg_migrations " + "WHERE filename = '004_placements_by_cell_org.sql'" + ) + == 1 + ) monkeypatch.setattr(pg_migrate, "_record_online", record) retry = asyncio.create_task(migrate()) tasks.append(retry) From bd11af3730fdc49d6f4e76eb3bd3d6b08eff720c Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 00:04:01 +0000 Subject: [PATCH 086/114] Trigger acceptance keeps independent write heads stable Declare and verify the existing append-only policy in both arms while retaining exact row proofs and workload limits. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 7 ++ docs/benchmarks/results.md | 6 +- .../gates/testsupport/cell_trigger/native.py | 12 ++- tests/unit/test_bench_cell_trigger_gate.py | 12 +++ tests/unit/test_cell_trigger_ci_creator.py | 95 +++++++++++++++++++ 5 files changed, 128 insertions(+), 4 deletions(-) diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 68512d0e9d..103eb8754d 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -3,6 +3,7 @@ Requires an independently owned, attached scratch fixture and its source/ownership manifest. The outer fixture owner stops its processes and drops its exact databases. This driver restores triggers but never deletes fixture rows or changes providers. +The namespace uses append_only so independent writes keep their exact heads. """ from __future__ import annotations @@ -634,6 +635,12 @@ async def _proof( "SELECT authority, epoch, deployment_id FROM loom.storage_authority FINAL" ) _require(len(records) == 1 and records[0] == dict(authority), "paired CH authority") + namespace = await ch.query_all( + "SELECT default_conflict_policy FROM loom.namespaces FINAL WHERE " + "org={org:String} AND namespace={ns:String} LIMIT 2", + {"org": args.org, "ns": args.namespace}, + ) + _require(namespace == [{"default_conflict_policy": "append_only"}], "independent write policy") for table in ("provider_configuration", "runtime_overrides"): rows = await ch.query_all(f"SELECT count() AS n FROM loom.{table}") _require(rows == [{"n": 0}], f"fresh {table}") diff --git a/docs/benchmarks/results.md b/docs/benchmarks/results.md index 00d4ca919b..fd940bf8ab 100644 --- a/docs/benchmarks/results.md +++ b/docs/benchmarks/results.md @@ -1244,7 +1244,11 @@ end-to-end gate is established by these measurements. `bench/cell_trigger_gate.py` measures the acceptance workload on an independently owned, attached scratch fixture with explicit fake providers, enforced mode and an already held -org. Each API run has 50 unmeasured warmups and 2,000 sequential measured requests with +org. Its namespace uses `append_only` for independent writes: sibling values do not +rewrite one another's heads. The fixture verifies the stored policy before every arm, +and the row proof requires each exact returned version and active status. This measures +trigger cost for independent writes, not semantic conflict resolution. Each API run has +50 unmeasured warmups and 2,000 sequential measured requests with 4,096-byte values, with five runs per arm in counterbalanced order. Each delivery block times five complete `pg_delivery.run_pass` calls at the unchanged configured batch size, with five blocks per arm. The latency gate is `median(p99_on) - median(p99_off) <= 10 ms`; diff --git a/scripts/gates/testsupport/cell_trigger/native.py b/scripts/gates/testsupport/cell_trigger/native.py index 5f970cd051..c984661506 100644 --- a/scripts/gates/testsupport/cell_trigger/native.py +++ b/scripts/gates/testsupport/cell_trigger/native.py @@ -1211,12 +1211,13 @@ async def proof( await rows( ch, ( - f"SELECT owner_email FROM `{db}`.namespaces FINAL WHERE " + "SELECT owner_email,default_conflict_policy " + f"FROM `{db}`.namespaces FINAL WHERE " f"org={{org:String}} AND namespace={{ns:String}}" ), params, ) - == [{"owner_email": "benchmark"}], + == [{"owner_email": "benchmark", "default_conflict_policy": "append_only"}], "native registered namespace", ) need( @@ -1731,7 +1732,12 @@ def credential(item, prefix): result = self.http( "/v1/namespace.register", token=self.token, - body={"org": self.org, "namespace": self.namespace, "owner_email": "benchmark"}, + body={ + "org": self.org, + "namespace": self.namespace, + "owner_email": "benchmark", + "default_conflict_policy": "append_only", + }, ) need( result.get("registered") is True and result.get("namespace") == self.namespace, diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py index d7d59688e5..0cbd369201 100644 --- a/tests/unit/test_bench_cell_trigger_gate.py +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -627,6 +627,8 @@ async def query_all(self, sql, parameters=None): "epoch": 1, } ] + if "FROM loom.namespaces" in sql: + return [{"default_conflict_policy": "append_only"}] if "FROM loom.provider_configuration" in sql: return [{"n": int(self.fault == "saved_provider")}] if "FROM loom.runtime_overrides" in sql: @@ -1253,6 +1255,7 @@ async def send(self, request, **kwargs): "host", "instance", "uuid", + "namespace_policy", ], ) async def test_review_composed_proof_matches_actual_server_fingerprints_and_health( @@ -1276,6 +1279,15 @@ async def fetchrow(self, sql, *params): class CH(ProofCH): async def query_all(self, sql, params=None): + if "FROM loom.namespaces" in sql: + assert params == {"org": args().org, "ns": args().namespace} + return [ + { + "default_conflict_policy": "auto_reinforce" + if fault == "namespace_policy" + else "append_only" + } + ] if "FROM loom.instances" in sql: return [ { diff --git a/tests/unit/test_cell_trigger_ci_creator.py b/tests/unit/test_cell_trigger_ci_creator.py index b78cb245b8..0d412fb0d7 100644 --- a/tests/unit/test_cell_trigger_ci_creator.py +++ b/tests/unit/test_cell_trigger_ci_creator.py @@ -544,3 +544,98 @@ def test_keeper_config_probe_follows_store_ownership_validation(keeper_config_ca with pytest.raises(native.Refused, match="current owned container"): instance.inspect_stores() assert [argv for argv, _ in calls] == [["docker", "inspect", "owned-ch"]] + + +def test_native_setup_registers_independent_append_only_writes(keeper_config_case, monkeypatch): + native, instance, owner, _, _ = keeper_config_case + instance.s.update( + ports={"probe": 7790, "router": 7791, "tls": 7792, "api": 7793}, + pg_admin_dsn="postgresql://u:p@127.0.0.1:15432/postgres", + databases={ + "pg_cell": "bench_cell", + "pg_directory": "bench_directory", + "ch_schema": "bench_schema", + "ch_telemetry": "bench_telemetry", + }, + ch={"port": 18123, "username": "fixture", "password": "test-only"}, + head="a" * 40, + python="unused", + ) + log = instance.w / "credentials.log" + log.write_text("credential: lcr_" + "1" * 128 + "\ncredential: lco_" + "2" * 128 + "\n") + + def action(task, *, role="", **kwargs): + if task == "create": + native.write(instance.w / (role + "-created.json"), {"created": True}) + return {} + + class Registered(Exception): + pass + + def register(path, *, token, body): + assert path == "/v1/namespace.register" + assert token == "fixture-token" + assert body == { + "org": instance.org, + "namespace": instance.namespace, + "owner_email": "benchmark", + "default_conflict_policy": "append_only", + } + raise Registered + + monkeypatch.setattr(instance, "inspect_stores", lambda: None) + monkeypatch.setattr(instance, "action", action) + monkeypatch.setattr(instance, "cli", lambda *args, **kwargs: {"log": log}) + monkeypatch.setattr(instance, "command_receipt", lambda *args, **kwargs: None) + monkeypatch.setattr(instance, "certificate", lambda: None) + monkeypatch.setattr(instance, "start_api", lambda *args: None) + monkeypatch.setattr(instance, "process_receipt", lambda *args: {}) + monkeypatch.setattr(owner, "start", lambda *args, **kwargs: {}, raising=False) + monkeypatch.setattr(native, "mint_metadata", lambda *args: ({}, "fixture-token")) + monkeypatch.setattr(instance, "http", register) + with pytest.raises(Registered): + instance.setup() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy", ["auto_reinforce", "append_only"]) +async def test_native_bootstrap_checks_persisted_independent_write_policy( + keeper_config_case, monkeypatch, policy +): + from contextlib import asynccontextmanager + from types import SimpleNamespace + from unittest.mock import AsyncMock + + native, instance, _, _, _ = keeper_config_case + authority = {"authority": "postgres", "deployment_id": "deployment", "epoch": 1} + conn = SimpleNamespace( + fetchrow=AsyncMock(return_value=authority), fetchval=AsyncMock(return_value=0) + ) + + @asynccontextmanager + async def connected(*args, **kwargs): + yield conn, object() + + async def rows(ch, sql, params=None): + if ".storage_authority" in sql: + return [authority] + if ".namespaces " in sql: + assert params == {"org": instance.org, "ns": instance.namespace} + row = {"owner_email": "benchmark"} + if "default_conflict_policy" in sql: + row["default_conflict_policy"] = policy + return [row] + if ".namespace_grants " in sql: + return [{"subject": "benchmark", "role": "admin"}] + raise AssertionError(sql) + + monkeypatch.setattr(native, "connections", connected) + monkeypatch.setattr(native, "rows", rows) + output = instance.w / "bootstrap.json" + if policy != "append_only": + with pytest.raises(native.Refused, match="native registered namespace"): + await native.proof(instance.s, output, bootstrap=True) + assert not output.exists() + else: + await native.proof(instance.s, output, bootstrap=True) + assert native.read(output)["namespace_registered"] is True From 449769eb4ebe32f697d0d2beb36318f04a59d2df Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 00:19:34 +0000 Subject: [PATCH 087/114] Retirement readers filter the native participant UUID A same-name string projection masks the UUID filter in the native query. Read the native UUID and normalize its returned identity while retaining every retirement predicate. Signed-off-by: zlareb1 --- scripts/gates/docker_provider_data.py | 5 +++-- tests/unit/test_docker_provider_smoke.py | 26 +++++++++++++++++++++++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/scripts/gates/docker_provider_data.py b/scripts/gates/docker_provider_data.py index c85f60c5bf..8c0104dd16 100644 --- a/scripts/gates/docker_provider_data.py +++ b/scripts/gates/docker_provider_data.py @@ -161,11 +161,12 @@ def _kill_worker(worker: dict, parent: dict) -> None: async def _retirement_rows(client, captured: list[dict]) -> list[dict]: - return await client.query_all( - "SELECT toString(participant_id) AS participant_id, state, version, host_proof " + rows = await client.query_all( + "SELECT participant_id, state, version, host_proof " "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)}", {"ids": [row["participant_id"] for row in captured]}, ) + return [{**row, "participant_id": str(row["participant_id"])} for row in rows] async def _native_retirement_observation(client, captured: list[dict]) -> dict: diff --git a/tests/unit/test_docker_provider_smoke.py b/tests/unit/test_docker_provider_smoke.py index 0122b28ed7..53f02680b9 100644 --- a/tests/unit/test_docker_provider_smoke.py +++ b/tests/unit/test_docker_provider_smoke.py @@ -460,7 +460,7 @@ async def test_crash_observation_distinguishes_unproved_predicates(monkeypatch, {"ids": [row["participant_id"]]} for row in rows ] assert all( - "SELECT toString(participant_id) AS participant_id" in call.args[0] for call in reads[:2] + "SELECT participant_id, state, version, host_proof" in call.args[0] for call in reads[:2] ) if fault == "missing_rows": assert '"reference_count": 1' in str(failure.value) @@ -516,3 +516,27 @@ async def query(sql, parameters): assert [call.args[1] for call in client.query_all.await_args_list] == ( [{"ids": [rows[0]["participant_id"]]}] if fault == "missing-second" else [] ) + [{"ids": [rows[int(fault == "missing-second")]["participant_id"]]}] * 2 + + +@pytest.mark.asyncio +async def test_retirement_reader_preserves_string_identity_from_native_uuid(): + from unittest.mock import AsyncMock + from uuid import UUID + + participant_id = UUID("00112233-4455-6677-8899-aabbccddeeff") + row = { + "participant_id": participant_id, + "state": "retired", + "version": 6, + "host_proof": "proof", + } + client = AsyncMock() + client.query_all.return_value = [row] + captured = [{**row, "participant_id": str(participant_id)}] + assert await data._retirement_rows(client, captured) == captured + assert row["participant_id"] is participant_id + assert client.query_all.await_args.args == ( + "SELECT participant_id, state, version, host_proof " + "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)}", + {"ids": [str(participant_id)]}, + ) From 644f6531c9e742487caedae9b3ee1ff42357d90f Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 00:39:25 +0000 Subject: [PATCH 088/114] The worker census skips exited supervisor children Signed-off-by: zlareb1 --- scripts/gates/docker_provider_data.py | 12 ++++++- tests/unit/test_docker_provider_smoke.py | 40 ++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/scripts/gates/docker_provider_data.py b/scripts/gates/docker_provider_data.py index 8c0104dd16..bc476ee25b 100644 --- a/scripts/gates/docker_provider_data.py +++ b/scripts/gates/docker_provider_data.py @@ -87,7 +87,17 @@ def _kernel_workers() -> dict[int, dict]: children = Path("/proc/1/task/1/children").read_text().split() for value in children: pid = int(value) - process = _process(pid) + try: + process = _process(pid) + except FileNotFoundError: + # An exited child can remain in the supervisor's census until reaped. + try: + state = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()[0] + except FileNotFoundError: + continue + if state in {"Z", "X"}: + continue + raise command = process["command"] if not any( value.startswith("from multiprocessing.spawn import spawn_main;") for value in command diff --git a/tests/unit/test_docker_provider_smoke.py b/tests/unit/test_docker_provider_smoke.py index 53f02680b9..1c7ffe3f18 100644 --- a/tests/unit/test_docker_provider_smoke.py +++ b/tests/unit/test_docker_provider_smoke.py @@ -540,3 +540,43 @@ async def test_retirement_reader_preserves_string_identity_from_native_uuid(): "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)}", {"ids": [str(participant_id)]}, ) + + +@pytest.mark.parametrize( + "fault", ["gone", "zombie", "dead", "live", "permission", "unreadable", "malformed", "one-live"] +) +def test_worker_census_omits_only_proved_exited_children(monkeypatch, fault): + from types import SimpleNamespace + + parent, _processes, rows = _process_fixture(monkeypatch) + process = data._process + + def observed_process(pid): + if pid == 7: + if fault == "permission": + raise PermissionError("unreadable child executable") + raise FileNotFoundError("disappeared child executable") + return process(pid) + + def read(path): + if path == "/proc/1/task/1/children": + return "7 8" if fault == "one-live" else "7 8 9" + assert path == "/proc/7/stat" + if fault == "gone": + raise FileNotFoundError("reaped child") + if fault == "unreadable": + raise PermissionError("unreadable child stat") + if fault == "malformed": + return "7 (incomplete child)" + state = "X" if fault == "dead" else "R" if fault == "live" else "Z" + return f"7 (exited child) {state} 1" + + monkeypatch.setattr(data, "_process", observed_process) + monkeypatch.setattr(data, "Path", lambda path: SimpleNamespace(read_text=lambda: read(path))) + if fault in {"gone", "zombie", "dead"}: + assert data._workers(rows, parent) == [ + {key: row[key] for key in ("participant_id", "host_proof", "version")} for row in rows + ] + else: + with pytest.raises((OSError, IndexError, AssertionError)): + data._workers(rows, parent) From 88165daa343b43ad352fe4bbea0e5727aa855096 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 00:39:25 +0000 Subject: [PATCH 089/114] Trigger setup refusals retain bounded child diagnostics Signed-off-by: zlareb1 --- .../gates/testsupport/cell_trigger/fixture.py | 100 ++++++++------- .../gates/testsupport/cell_trigger/native.py | 29 +++++ tests/unit/test_cell_trigger_ci_creator.py | 117 ++++++++++++++++++ 3 files changed, 203 insertions(+), 43 deletions(-) diff --git a/scripts/gates/testsupport/cell_trigger/fixture.py b/scripts/gates/testsupport/cell_trigger/fixture.py index 42526c9e43..b659c4b807 100644 --- a/scripts/gates/testsupport/cell_trigger/fixture.py +++ b/scripts/gates/testsupport/cell_trigger/fixture.py @@ -75,7 +75,8 @@ class Refused(RuntimeError): - pass + command: dict | None = None + native: dict | None = None def need(ok: object, field: str) -> None: @@ -568,6 +569,47 @@ def load_owned(path: Path): return OwnedChildren() +def error_metadata(error: BaseException) -> dict: + site = {"file": "unknown", "line": 0, "function": "unknown"} + frame = sys._getframe(1) + line = frame.f_lineno + trace = error.__traceback__ + for _ in range(64): + if trace is None: + break + candidate = trace.tb_frame + if Path(candidate.f_code.co_filename).name in FILES and candidate.f_code.co_name not in { + "need", + "closed", + "equal", + "integer", + "identifier", + "hex64", + }: + frame = candidate + line = trace.tb_lineno + trace = trace.tb_next + while frame is not None and frame.f_code.co_name in {"select", "block_teardown"}: + frame = frame.f_back + if frame is not None: + line = frame.f_lineno + if frame is not None and Path(frame.f_code.co_filename).name in FILES: + site = { + "file": Path(frame.f_code.co_filename).name, + "line": line, + "function": frame.f_code.co_name[:64], + } + return { + "error_type": type(error).__name__[:64], + "site": site, + "outcome": "refused" + if isinstance(error, Refused) + else "error" + if isinstance(error, Exception) + else "cancelled", + } + + class Owner: def __init__(self, spec: dict, owned, note) -> None: self.spec, self.owned, self.note = spec, owned, note @@ -601,47 +643,12 @@ def select(self, error: BaseException, *, phase: str | None = None) -> None: selected_phase = entry[1]["phase"] unknown = {"file": "unknown", "line": 0, "function": "unknown"} try: - site = unknown - frame = sys._getframe(1) - line = frame.f_lineno - trace = error.__traceback__ - for _ in range(64): - if trace is None: - break - candidate = trace.tb_frame - if Path( - candidate.f_code.co_filename - ).name in FILES and candidate.f_code.co_name not in { - "need", - "closed", - "equal", - "integer", - "identifier", - "hex64", - }: - frame = candidate - line = trace.tb_lineno - trace = trace.tb_next - while frame is not None and frame.f_code.co_name in {"select", "block_teardown"}: - frame = frame.f_back - if frame is not None: - line = frame.f_lineno - if frame is not None and Path(frame.f_code.co_filename).name in FILES: - site = { - "file": Path(frame.f_code.co_filename).name, - "line": line, - "function": frame.f_code.co_name[:64], - } - entry[1] = { - "phase": selected_phase, - "error_type": type(error).__name__[:64], - "site": site, - "outcome": "refused" - if isinstance(error, Refused) - else "error" - if isinstance(error, Exception) - else "cancelled", - } + entry[1] = {"phase": selected_phase, **error_metadata(error)} + if isinstance(error, Refused): + if error.command is not None: + entry[1]["command"] = error.command + if error.native is not None: + entry[1]["native"] = error.native except BaseException as secondary: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. entry[1] = { "phase": selected_phase, @@ -928,7 +935,14 @@ def command( self.monitor(cleanup=cleanup) need(time.monotonic() < end, "serial command deadline") time.sleep(0.05) - need(item["process"].returncode == 0, "native command refused") + if item["process"].returncode != 0: + error = Refused("native command refused") + error.command = { + "role": role if role in {"inspect", "keeper-prefix", "create"} else "other", + "serial": self.serial, + "returncode": item["process"].returncode, + } + raise error if self.first is not None and not cleanup: raise self.first return item diff --git a/scripts/gates/testsupport/cell_trigger/native.py b/scripts/gates/testsupport/cell_trigger/native.py index c984661506..e4fb7ed810 100644 --- a/scripts/gates/testsupport/cell_trigger/native.py +++ b/scripts/gates/testsupport/cell_trigger/native.py @@ -39,6 +39,7 @@ child_env, database, digest, + error_metadata, need, pg_dsn, read, @@ -1490,6 +1491,26 @@ def action( cleanup=cleanup, ) except BaseException as error: + if isinstance(error, Refused): + try: + failure = read(target.with_name(target.name + ".failure.json"), limit=4096) + closed(failure, "error_type site outcome") + site = failure["site"] + closed(site, "file line function") + need( + isinstance(failure["error_type"], str) + and re.fullmatch(r"[A-Za-z_][A-Za-z_0-9]{0,63}", failure["error_type"]) + and failure["outcome"] in {"refused", "error", "cancelled"} + and site["file"] in (*FILES, "unknown") + and type(site["line"]) is int + and 0 <= site["line"] < 2**31 + and isinstance(site["function"], str) + and re.fullmatch(r"[A-Za-z_][A-Za-z_0-9]{0,63}", site["function"]), + "native failure metadata", + ) + error.native = failure + except (OSError, ValueError, Refused, KeyError, TypeError): + pass # Missing or invalid diagnostics cannot replace the command failure. # A terminal cleanup refusal is recorded, never retried as another mutation. if cleanup and self.o.leaders and self.o.leaders[-1]["terminal"] is not None: self.o.select(error) @@ -2400,6 +2421,14 @@ def main() -> int: ) ) except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + try: + write( + args.output.with_name(args.output.name + ".failure.json"), + error_metadata(error), + limit=4096, + ) + except BaseException: # noqa: BLE001 — diagnostic output must not replace the native failure. + pass print(type(error).__name__, file=sys.stderr) return 1 return 0 diff --git a/tests/unit/test_cell_trigger_ci_creator.py b/tests/unit/test_cell_trigger_ci_creator.py index 0d412fb0d7..1b6c0aa88f 100644 --- a/tests/unit/test_cell_trigger_ci_creator.py +++ b/tests/unit/test_cell_trigger_ci_creator.py @@ -639,3 +639,120 @@ async def rows(ch, sql, params=None): else: await native.proof(instance.s, output, bootstrap=True) assert native.read(output)["namespace_registered"] is True + + +@pytest.mark.parametrize("role", ["inspect", "keeper-prefix", "create", "private-role"]) +def test_nonzero_native_command_retains_only_structural_metadata(monkeypatch, role): + import json + from types import SimpleNamespace + + from scripts.gates.testsupport.cell_trigger import fixture + + owner = fixture.Owner({}, None, lambda *args, **kwargs: None) + owner.phase, owner.serial = "setup", 5 + item = {"process": SimpleNamespace(returncode=7)} + monkeypatch.setattr(owner, "start", lambda *args, **kwargs: item) + monkeypatch.setattr(owner, "poll", lambda _: 7) + with pytest.raises(fixture.Refused) as caught: + owner.command(role, ["private-argv"], {"TOKEN": "private-token"}) + owner.select(caught.value) + record = owner.failure_summary()["records"][0] + assert owner.first is caught.value + assert record["command"] == { + "role": role if role != "private-role" else "other", + "serial": 5, + "returncode": 7, + } + assert record["site"]["file"] == "fixture.py" + assert record["site"]["function"] == "command" + assert "private-" not in json.dumps(record) + + +def test_native_failure_site_reaches_published_evidence_without_private_logs( + keeper_config_case, monkeypatch, tmp_path, capsys +): + import json + import sys + + from scripts.gates.cell_trigger_acceptance import publish + from scripts.gates.testsupport.cell_trigger import fixture + + native, instance, owner, _, _ = keeper_config_case + context, output = tmp_path / "failure-context.json", tmp_path / "pg_cell-created.json" + instance.context = context + instance.s["databases"]["pg_cell"] = "bench_cell" + instance.s["python"] = "unused" + native.write(context, instance.s | {"root": str(tmp_path)}) + + async def failed(*args, **kwargs): + raise ValueError("private-native-secret") + + monkeypatch.setattr(native, "stores", failed) + monkeypatch.setattr(sys, "path", list(sys.path)) + monkeypatch.setattr( + sys, + "argv", + [ + "native.py", + "--task", + "create", + "--role", + "pg_cell", + "--context", + str(context), + "--output", + str(output), + ], + ) + assert native.main() == 1 + assert not output.exists() + metadata = native.read(output.with_name(output.name + ".failure.json"), limit=4096) + assert metadata["error_type"] == "ValueError" + assert metadata["site"]["file"] == "native.py" + assert metadata["site"]["function"] == "main" + assert metadata["site"]["line"] > 0 + assert "private-native-secret" not in json.dumps(metadata) + capsys.readouterr().err + original = fixture.Refused("private-command-secret") + original.command = {"role": "create", "serial": 5, "returncode": 1} + + def refused(*args, **kwargs): + raise original + + owner.command = refused + with pytest.raises(fixture.Refused) as caught: + instance.action("create", role="pg_cell") + assert caught.value is original + observer = fixture.Owner({}, None, lambda *args, **kwargs: None) + observer.select(caught.value, phase="setup") + evidence = observer.failure_summary() + assert evidence["records"][0]["native"] == metadata + assert evidence["records"][0]["command"] == original.command + folder = tmp_path / "publication" + (folder / "fixture").mkdir(parents=True) + native.write(folder / "fixture/outer-result.json", {"failure_evidence": evidence}) + (folder / "fixture/private.log").write_text("private-log-secret") + (folder / "fixture/private-settings.json").write_text('{"token":"private-settings-secret"}') + destination = tmp_path / "published" + publish(folder, destination, original, phase="native", diagnostic_error_type=None) + assert {path.name for path in destination.iterdir()} == {"outer-result.json", "result.json"} + published = json.loads((destination / "outer-result.json").read_text()) + assert published["failure_evidence"] == evidence + assert "private-" not in json.dumps(published) + + +@pytest.mark.parametrize("contents", ["not-json", '{"error_type":"private-extra"}', "x" * 4097]) +def test_invalid_native_failure_sidecar_preserves_command_refusal(keeper_config_case, contents): + native, instance, owner, _, _ = keeper_config_case + instance.s["python"] = "unused" + path = instance.w / "pg_cell-created.json.failure.json" + path.write_text(contents) + path.chmod(0o600) + original = native.Refused("original") + + def refused(*args, **kwargs): + raise original + + owner.command = refused + with pytest.raises(native.Refused) as caught: + instance.action("create", role="pg_cell") + assert caught.value is original and original.native is None From 1a1cee2314c5ca206d54cb532e3bf0ba285099d4 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 00:45:26 +0000 Subject: [PATCH 090/114] Diagnostic read failures preserve the original command refusal Signed-off-by: zlareb1 --- .../gates/testsupport/cell_trigger/native.py | 4 +-- tests/unit/test_cell_trigger_ci_creator.py | 36 ++++++++++++++++++- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/scripts/gates/testsupport/cell_trigger/native.py b/scripts/gates/testsupport/cell_trigger/native.py index e4fb7ed810..9928315e56 100644 --- a/scripts/gates/testsupport/cell_trigger/native.py +++ b/scripts/gates/testsupport/cell_trigger/native.py @@ -1509,8 +1509,8 @@ def action( "native failure metadata", ) error.native = failure - except (OSError, ValueError, Refused, KeyError, TypeError): - pass # Missing or invalid diagnostics cannot replace the command failure. + except BaseException: # noqa: BLE001 — diagnostics must not replace the original command failure. + pass # A terminal cleanup refusal is recorded, never retried as another mutation. if cleanup and self.o.leaders and self.o.leaders[-1]["terminal"] is not None: self.o.select(error) diff --git a/tests/unit/test_cell_trigger_ci_creator.py b/tests/unit/test_cell_trigger_ci_creator.py index 1b6c0aa88f..e1e77569c5 100644 --- a/tests/unit/test_cell_trigger_ci_creator.py +++ b/tests/unit/test_cell_trigger_ci_creator.py @@ -740,7 +740,15 @@ def refused(*args, **kwargs): assert "private-" not in json.dumps(published) -@pytest.mark.parametrize("contents", ["not-json", '{"error_type":"private-extra"}', "x" * 4097]) +@pytest.mark.parametrize( + "contents", + [ + "not-json", + '{"error_type":"private-extra"}', + "x" * 4097, + pytest.param("[" * 1500 + "0" + "]" * 1500, id="deep-nesting"), + ], +) def test_invalid_native_failure_sidecar_preserves_command_refusal(keeper_config_case, contents): native, instance, owner, _, _ = keeper_config_case instance.s["python"] = "unused" @@ -756,3 +764,29 @@ def refused(*args, **kwargs): with pytest.raises(native.Refused) as caught: instance.action("create", role="pg_cell") assert caught.value is original and original.native is None + + +@pytest.mark.parametrize("kind", [KeyboardInterrupt, SystemExit]) +@pytest.mark.parametrize("site", ["read", "closed"]) +def test_fatal_native_diagnostic_preserves_original_command_refusal( + keeper_config_case, monkeypatch, kind, site +): + native, instance, owner, _, _ = keeper_config_case + instance.s["python"] = "unused" + original = native.Refused("original") + + def refused(*args, **kwargs): + raise original + + def diagnostic(*args, **kwargs): + raise kind("private-diagnostic-secret") + + owner.command = refused + monkeypatch.setattr(native, "read", lambda *args, **kwargs: {}) + monkeypatch.setattr(native, site, diagnostic) + try: + instance.action("create", role="pg_cell") + except BaseException as error: # noqa: BLE001 — assert the actual retained failure, including fatal diagnostics. + assert error is original and original.native is None + else: + pytest.fail("command refusal was swallowed") From 1fcd50a4e08ef8814f5551d5ce0838c3a7e11521 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 00:58:16 +0000 Subject: [PATCH 091/114] The replica fixture matches its migrated vector width Signed-off-by: zlareb1 --- tests/integration/test_cell_attach_two_replicas.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_cell_attach_two_replicas.py b/tests/integration/test_cell_attach_two_replicas.py index 09a28fba88..d9cb647db4 100644 --- a/tests/integration/test_cell_attach_two_replicas.py +++ b/tests/integration/test_cell_attach_two_replicas.py @@ -235,8 +235,8 @@ def _configure(monkeypatch: pytest.MonkeyPatch, host: str, api: int, probe: int) "router_directory_cache_s": 1, "background_workers": "off", "embedding_provider": "fake", - "embedding_model": "fake-ngram-1024", - "embedding_dim": 1024, + "embedding_model": "fake-ngram-3072", + "embedding_dim": 3072, "llm_provider": "fake", "llm_model": "fake-echo-1", "require_real_providers": False, From 6521d1b7a9dba8234495c7766b838d663b858100 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 01:16:46 +0000 Subject: [PATCH 092/114] Wait for supervised replacement enrollment after retirement Signed-off-by: zlareb1 --- scripts/gates/docker_provider_data.py | 15 +++++- tests/unit/test_docker_provider_smoke.py | 64 ++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/scripts/gates/docker_provider_data.py b/scripts/gates/docker_provider_data.py index bc476ee25b..9704e7c06f 100644 --- a/scripts/gates/docker_provider_data.py +++ b/scripts/gates/docker_provider_data.py @@ -30,6 +30,10 @@ ] +class _WorkersPending(AssertionError): + """A supervised replacement has not spawned or enrolled yet.""" + + def _process(pid: int) -> dict: root = Path(f"/proc/{pid}") stat = (root / "stat").read_text().rsplit(")", 1)[1].split() @@ -107,6 +111,8 @@ def _kernel_workers() -> dict[int, dict]: assert process["parent"] == 1, f"Kernel child PID {pid} parent changed during census" assert pid > 1 and pid not in found, "Invalid supervisor child PID census" found[pid] = process + if len(found) < 2: + raise _WorkersPending("Expected two live supervisor spawn children") assert len(found) == 2, f"Expected two live supervisor spawn children; found {len(found)}" return found @@ -127,6 +133,8 @@ def _workers(rows: list[dict], parent: dict) -> list[dict]: and proof[5] == process["start"] ): matches.append(row) + if not matches: + raise _WorkersPending(f"Worker PID {pid} exact registrations: 0; expected 1") assert len(matches) == 1, ( f"Worker PID {pid} exact registrations: {len(matches)}; expected 1" ) @@ -247,7 +255,12 @@ async def crash_worker(client) -> None: stopped = provider_process.proves_stopped(victim["host_proof"]) observation = {"death": "proved" if stopped else "unproved", "retirement": "not_checked"} if stopped and await retired(client, [victim], observation=observation): - after = _workers(await _worker_rows(client), parent) + try: + after = _workers(await _worker_rows(client), parent) + except _WorkersPending: + observation["replacement"] = "pending" + await asyncio.sleep(1) + continue old = {row["participant_id"] for row in before} new = {row["participant_id"] for row in after} observation.update( diff --git a/tests/unit/test_docker_provider_smoke.py b/tests/unit/test_docker_provider_smoke.py index 1c7ffe3f18..64119f0817 100644 --- a/tests/unit/test_docker_provider_smoke.py +++ b/tests/unit/test_docker_provider_smoke.py @@ -580,3 +580,67 @@ def read(path): else: with pytest.raises((OSError, IndexError, AssertionError)): data._workers(rows, parent) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", ["spawn", "registration", "timeout", "duplicate", "certificate", "namespace"] +) +async def test_crash_waits_only_for_incomplete_replacement_census(monkeypatch, fault): + from types import SimpleNamespace + from unittest.mock import AsyncMock + + parent, processes, rows = _process_fixture(monkeypatch) + replacement = { + **rows[0], + "participant_id": "replacement", + "host_proof": "h|b|c|1|10|100|||cell-retirement-v1", + } + processes[10] = {**processes[8], "pid": 10, "start": "100"} + if fault == "certificate": + replacement["host_proof"] = "h|b|c|1|10|100|||unknown" + if fault == "namespace": + processes[10]["namespace"] = 2 + stage = 0 + + def children(_path): + pending = fault in {"spawn", "timeout"} and stage == 0 + return SimpleNamespace(read_text=lambda: "9" if pending else "9 10") + + async def registered(_client): + if fault == "registration" and stage == 0: + return [rows[1]] + if fault == "duplicate": + return [rows[1], replacement, {**replacement, "participant_id": "ambiguous"}] + return [rows[1], replacement] + + async def pause(_seconds): + nonlocal stage + stage += 1 + + monkeypatch.setattr(data, "Path", children) + monkeypatch.setattr(data, "_supervisor", lambda: parent) + monkeypatch.setattr(data, "workers", AsyncMock(return_value=rows)) + monkeypatch.setattr(data, "_worker_rows", registered) + monkeypatch.setattr(data, "_retirement_rows", AsyncMock(side_effect=[[rows[0]], [rows[1]]])) + monkeypatch.setattr(data, "retired", AsyncMock(return_value=True)) + monkeypatch.setattr(data.provider_process, "proves_stopped", lambda _proof: True) + signals = [] + monkeypatch.setattr(data, "_kill_worker", lambda victim, _parent: signals.append(victim)) + ticks = iter([0, 0, 241] if fault == "timeout" else [0, 0, 1, 241]) + monkeypatch.setattr(data, "time", SimpleNamespace(monotonic=lambda: next(ticks))) + monkeypatch.setattr(data.asyncio, "sleep", pause) + if fault in {"spawn", "registration"}: + await data.crash_worker(object()) + assert stage == 1 + else: + pattern = { + "timeout": "supervised replacement was not proved", + "duplicate": "exact registrations: 2", + "certificate": "certificate invalid", + "namespace": "namespace mismatch", + }[fault] + with pytest.raises(AssertionError, match=pattern): + await data.crash_worker(object()) + assert stage == int(fault == "timeout") + assert signals == [rows[0]] From 3e8225721ed90cbd7d9d2b1f6136746329e2ce45 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 01:16:46 +0000 Subject: [PATCH 093/114] Attach deadline tests use an owned cleanup clock Signed-off-by: zlareb1 --- tests/unit/test_cell_attach.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 819f03d90f..ce341a619e 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -24,6 +24,7 @@ from server.router.limits import ATTACH_BATCH, MAX_BODY_BYTES from server.settings import settings from tests.unit.cell_control_owner import owner as _owner +from tests.unit.clock_fake import ManualClock FIELDS = {**cell.DEFAULTS, "deployment_id": "dep-1"} CURRENT = cell.as_value(FIELDS) @@ -531,6 +532,10 @@ def is_closed(self) -> bool: def _short_phases(monkeypatch: pytest.MonkeyPatch, bounds: list[float] | None = None) -> None: + # The work timer stays real; scheduling cannot consume the fake close budget. + phase_clock = ManualClock() + monkeypatch.setattr(cell_control, "time", phase_clock) + monkeypatch.setattr(cell_control.pg, "time", phase_clock) supervise = cell_control.Owner.call async def bounded(self: Any, work: Any, *, total_s: float) -> Any: From d5e14113b019b93e0ed7ab859a3bb4e7fd050bcb Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 01:37:40 +0000 Subject: [PATCH 094/114] Resource samples require a complete process forest Restart an incomplete census only after a nonroot process is proven absent. Bound retries by the original freshness interval and retain identity, ownership and resource refusals. Signed-off-by: zlareb1 --- .../gates/testsupport/cell_trigger/fixture.py | 89 +++++-- .../unit/test_cell_trigger_resource_census.py | 242 ++++++++++++++++++ 2 files changed, 315 insertions(+), 16 deletions(-) create mode 100644 tests/unit/test_cell_trigger_resource_census.py diff --git a/scripts/gates/testsupport/cell_trigger/fixture.py b/scripts/gates/testsupport/cell_trigger/fixture.py index b659c4b807..a279c450cb 100644 --- a/scripts/gates/testsupport/cell_trigger/fixture.py +++ b/scripts/gates/testsupport/cell_trigger/fixture.py @@ -745,24 +745,81 @@ def sample(self, *, admit: bool = False) -> None: and os.stat(f"/proc/{store['host_pid']}").st_uid == store["uid"], "resource store birth changed", ) - pending = [self.owned.pid] + [s["host_pid"] for s in self.spec["stores"].values()] - seen = set() - while pending: - pid = pending.pop() - if pid in seen: - continue - seen.add(pid) - before = self.owned.identity(pid) - need( - before is not None, "enumerated resource process disappeared; sample incomplete" - ) - need(self.owned.rss(pid) * 1024 <= POLICY["stop_process_rss_bytes"], "process RSS") - pending.extend(self.owned.children(pid)) - after = self.owned.identity(pid) + roots = [self.owned.pid] + [s["host_pid"] for s in self.spec["stores"].values()] + births = {pid: self.owned.identity(pid) for pid in roots} + need(all(births.values()), "resource root disappeared") + observed = dict(births) + disappeared = set() + + def complete_forest() -> bool: + pending = list(roots) + seen = set() + while pending: + need(time.monotonic() - now < 3, "resource census stale") + pid = pending.pop() + if pid in seen: + continue + seen.add(pid) + before = self.owned.identity(pid) + if before is None: + need(pid not in births, "resource root disappeared") + disappeared.add(pid) + return False + need( + pid not in disappeared + and (pid not in observed or observed[pid][:2] == before[:2]), + "resource process birth changed", + ) + observed[pid] = before + need( + self.owned.rss(pid) * 1024 <= POLICY["stop_process_rss_bytes"], + "process RSS", + ) + try: + children = self.owned.children(pid) + except (FileNotFoundError, ProcessLookupError): + need( + pid not in births and self.owned.identity(pid) is None, + "resource child census unknown", + ) + disappeared.add(pid) + return False + after = self.owned.identity(pid) + if after is None: + need(pid not in births, "resource root disappeared") + disappeared.add(pid) + return False + need(after[:2] == before[:2], "resource process birth changed") + pending.extend(children) + return True + + # Discard an incomplete traversal; only a fresh whole forest can renew the sample. + for _attempt in range(3): + need(time.monotonic() - now < 3, "resource census stale") + for pid, birth in births.items(): + current = self.owned.identity(pid) + need( + current is not None and current[:2] == birth[:2], + "resource root birth changed", + ) need( - after is not None and after[:2] == before[:2], - "resource process disappeared or changed; sample incomplete", + Path("/proc/sys/kernel/random/boot_id").read_text().strip() == boot + and os.stat(f"/proc/{self.owned.pid}").st_uid == os.geteuid(), + "resource owner birth changed", ) + for store in self.spec["stores"].values(): + need( + births[store["host_pid"]][1] == store["start_ticks"] + and Path("/proc/sys/kernel/random/boot_id").read_text().strip() + == store["boot_id"] + and os.stat(f"/proc/{store['host_pid']}").st_uid == store["uid"], + "resource store birth changed", + ) + if complete_forest(): + need(time.monotonic() - now < 3, "resource census stale") + break + else: + raise Refused("resource process census remained incomplete") for item in tuple(self.leaders): need(item["log"].stat().st_size <= 16 * 1024 * 1024, "child log bound") driver_artifact = Path(self.spec["workdir"]) / "driver.jsonl" diff --git a/tests/unit/test_cell_trigger_resource_census.py b/tests/unit/test_cell_trigger_resource_census.py new file mode 100644 index 0000000000..31a10c22cd --- /dev/null +++ b/tests/unit/test_cell_trigger_resource_census.py @@ -0,0 +1,242 @@ +"""A vanished descendant requires a fresh complete resource census.""" + +from collections import Counter +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from scripts.gates.testsupport.cell_trigger import fixture + + +@pytest.fixture +def census(monkeypatch, tmp_path): + class Owned: + pid = 100 + + def __init__(self): + self.round = 0 + self.reads = Counter() + self.sampled_pids = [] + self.rows = {100: (1, "self", "S"), 200: (1, "store", "S")} + self.trees = [[300], [400]] + self.identify = lambda pid: self.rows.get(pid, (100, str(pid), "S")) + self.enumerate = lambda pid: ( + self.trees[min(self.round - 1, len(self.trees) - 1)] if pid == 100 else [] + ) + self.measure = lambda pid: 1 + + def identity(self, pid): + self.reads[pid] += 1 + return self.identify(pid) + + def children(self, pid): + if pid == self.pid: + self.round += 1 + return self.enumerate(pid) + + def rss(self, pid): + self.sampled_pids.append(pid) + return self.measure(pid) + + owned = Owned() + clock = SimpleNamespace(now=10.0) + original_read = Path.read_text + + def read(path, *args, **kwargs): + if str(path) == "/proc/meminfo": + return f"MemAvailable: {20 * 1024**2} kB\n" + if str(path) == "/proc/sys/kernel/random/boot_id": + return "boot" + return original_read(path, *args, **kwargs) + + monkeypatch.setattr(fixture, "time", SimpleNamespace(monotonic=lambda: clock.now)) + monkeypatch.setattr(Path, "read_text", read) + monkeypatch.setattr( + fixture.shutil, "disk_usage", lambda _: SimpleNamespace(free=32 * fixture.GIB) + ) + original_stat = fixture.os.stat + monkeypatch.setattr( + fixture.os, + "stat", + lambda path, *args, **kwargs: ( + SimpleNamespace(st_uid=fixture.os.geteuid()) + if str(path) in {"/proc/100", "/proc/200"} + else original_stat(path, *args, **kwargs) + ), + ) + spec = { + "workdir": str(tmp_path), + "stores": { + "pg": { + "host_pid": 200, + "start_ticks": "store", + "boot_id": "boot", + "uid": fixture.os.geteuid(), + "disk_path": str(tmp_path), + } + }, + } + owner = fixture.Owner(spec, owned, lambda *args, **kwargs: None) + owner.sampled = 7.0 + return owner, owned, clock + + +@pytest.mark.parametrize("stage", ["before", "after", "children"]) +def test_a_vanished_descendant_restarts_the_whole_forest(census, stage): + owner, owned, _ = census + original_identity, original_children = owned.identify, owned.enumerate + + def identity(pid): + if pid == 300 and (stage != "after" or owned.reads[pid] > 1): + return None + return original_identity(pid) + + def children(pid): + if pid == 300 and stage == "children": + raise FileNotFoundError("gone") + return original_children(pid) + + owned.identify, owned.enumerate = identity, children + if stage == "children": + owned.identify = lambda pid: ( + None if pid == 300 and owned.reads[pid] > 1 else original_identity(pid) + ) + owner.sample() + assert owner.first is None + assert owner.sampled == 10.0 + assert owned.round == 2 + assert owned.sampled_pids.count(200) == 2 + assert 400 in owned.sampled_pids + + +def test_a_complete_first_forest_needs_no_retry(census): + owner, owned, _ = census + owner.sample() + assert owner.first is None and owner.sampled == 10.0 + assert owned.round == 1 and set(owned.sampled_pids) == {100, 200, 300} + + +@pytest.mark.parametrize( + "fault", + [ + "churn", + "root-gone", + "store-gone", + "store-birth", + "pid-reuse", + "identity-change", + "permission", + "malformed", + "rss", + "stale", + ], +) +def test_incomplete_or_unsafe_census_never_refreshes_the_sample(census, fault): + owner, owned, clock = census + original = owned.identify + + def identify(pid): + if fault == "root-gone" and pid == 100: + return None + if fault == "store-gone" and pid == 200: + return None + if fault == "store-birth" and pid == 200: + return (1, "changed", "S") + if pid >= 300: + if fault in {"churn", "stale"}: + if fault == "stale": + clock.now = 13.0 + return None + if fault == "pid-reuse" and owned.round == 1: + return None + if fault == "identity-change" and owned.reads[pid] > 1: + return (100, "new-birth", "S") + if fault == "permission": + raise PermissionError("unknown") + if fault == "malformed": + raise ValueError("unknown") + return original(pid) + + owned.identify = identify + if fault == "churn": + owned.trees = [[300], [400], [500], [600]] + if fault == "pid-reuse": + owned.trees = [[300], [300]] + if fault == "rss": + owned.measure = lambda pid: fixture.POLICY["stop_process_rss_bytes"] // 1024 + 1 + owner.sample() + assert owner.first is not None + assert owner.sampled == 7.0 + assert owned.round <= 3 + if fault not in {"churn", "pid-reuse", "stale"}: + assert owned.round <= 1 + + +@pytest.mark.parametrize( + "fault", ["root-birth", "store-birth", "new-child-rss", "live-child-census"] +) +def test_restarted_forest_retains_root_and_descendant_guards(census, fault): + owner, owned, _ = census + original_identity, original_children = owned.identify, owned.enumerate + + def identity(pid): + if pid == 300: + return original_identity(pid) if fault == "live-child-census" else None + if owned.round and ( + (pid == 100 and fault == "root-birth") or (pid == 200 and fault == "store-birth") + ): + return (1, "reused", "S") + return original_identity(pid) + + def children(pid): + if pid == 300 and fault == "live-child-census": + raise FileNotFoundError("unverified disappearance") + return original_children(pid) + + owned.identify, owned.enumerate = identity, children + if fault == "new-child-rss": + owned.measure = lambda pid: ( + fixture.POLICY["stop_process_rss_bytes"] // 1024 + 1 if pid == 400 else 1 + ) + owner.sample() + assert owner.first is not None + assert owner.sampled == 7.0 + if fault == "new-child-rss": + assert owned.round == 2 and 400 in owned.sampled_pids + else: + assert owned.round == 1 + + +def test_census_deadline_stops_before_the_next_descendant(census): + owner, owned, clock = census + original_children = owned.enumerate + owned.enumerate = lambda pid: [301] if pid == 300 else original_children(pid) + + def measure(pid): + if pid == 300: + clock.now = 13.0 + return 1 + + owned.measure = measure + owner.sample() + assert isinstance(owner.first, fixture.Refused) + assert owner.sampled == 7.0 + assert 300 in owned.sampled_pids and 301 not in owned.sampled_pids + + +def test_restarted_forest_refuses_store_uid_drift(census, monkeypatch): + owner, owned, _ = census + original_identity, original_stat = owned.identify, fixture.os.stat + owned.identify = lambda pid: None if pid == 300 else original_identity(pid) + + def stat(path, *args, **kwargs): + if str(path) == "/proc/200" and owned.round: + return SimpleNamespace(st_uid=fixture.os.geteuid() + 1) + return original_stat(path, *args, **kwargs) + + monkeypatch.setattr(fixture.os, "stat", stat) + owner.sample() + assert isinstance(owner.first, fixture.Refused) + assert owner.sampled == 7.0 + assert owned.round == 1 From 3f69214963eb90835890e321d159809f3f25e415 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 01:46:13 +0000 Subject: [PATCH 095/114] Trigger protocol metadata describes its implemented runtime Keep every workload field and acceptance oracle unchanged while updating the fixed descriptor checksum. Signed-off-by: zlareb1 --- scripts/gates/testsupport/cell_trigger/fixture.py | 2 +- scripts/gates/testsupport/cell_trigger/protocol.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/gates/testsupport/cell_trigger/fixture.py b/scripts/gates/testsupport/cell_trigger/fixture.py index a279c450cb..4862bf892e 100644 --- a/scripts/gates/testsupport/cell_trigger/fixture.py +++ b/scripts/gates/testsupport/cell_trigger/fixture.py @@ -39,7 +39,7 @@ HERE = Path(__file__).resolve().parent FILES = ("fixture.py", "native.py", "tls_proxy.py", "capture.py", "owned_children.py") PROTOCOL = "protocol.json" -PROTOCOL_SHA = "7223a2018c518cfc8a58977c84aa88dec5784aa741cb34245cb092abbf848812" +PROTOCOL_SHA = "690f1a06d0a523600a3ef4dcfa5cb58cc48052812ecad4e6e9e591c9e4582924" SPEC_FIELDS = { "schema", "fixture_id", diff --git a/scripts/gates/testsupport/cell_trigger/protocol.json b/scripts/gates/testsupport/cell_trigger/protocol.json index 3d3670f64d..850a6e3a27 100644 --- a/scripts/gates/testsupport/cell_trigger/protocol.json +++ b/scripts/gates/testsupport/cell_trigger/protocol.json @@ -1,6 +1,6 @@ { "schema": 1, - "status": "r5 design only; no runtime or entry implementation", + "status": "Runtime and entry implementation; native acceptance required", "memory_fields": [ "org", "namespace", From c9685ba94382169626258a852b7afdce1058c322 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 01:46:13 +0000 Subject: [PATCH 096/114] Phone citation geometry waits for the selected Ask option The option can be visible before its keyboard selection effect runs. Require that selection before the existing Enter action and retain every geometry assertion. Signed-off-by: zlareb1 --- frontend/e2e/assistant-launch.spec.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/e2e/assistant-launch.spec.ts b/frontend/e2e/assistant-launch.spec.ts index f62187437d..ee9f2ad1df 100644 --- a/frontend/e2e/assistant-launch.spec.ts +++ b/frontend/e2e/assistant-launch.spec.ts @@ -369,7 +369,9 @@ test("on a phone the card's citation number keeps its glyph's width, and each so await page.getByRole("button", { name: "Search", exact: true }).tap(); const search = page.getByRole("combobox", { name: "Search Loom" }); await search.pressSequentially("how often are invoices sent?"); - await expect(page.getByRole("option", { name: 'Ask AI about "how often are invoices sent?"' })).toBeVisible(); + const ask = page.getByRole("option", { name: 'Ask AI about "how often are invoices sent?"' }); + await expect(ask).toBeVisible(); + await expect(ask).toHaveAttribute("aria-selected", "true"); await page.keyboard.press("Enter"); const card = page.getByTestId("search-answer"); await expect(card.getByTestId("assistant-support")).toBeVisible(); From a400ba33a5e7d05bc000f7b55b6de5f228f4e9ea Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 01:52:12 +0000 Subject: [PATCH 097/114] Native trigger failures retain bounded driver diagnostics Keep numeric error codes and bounded cause classes without exception text or connection details. Validate the sidecar before attaching it to the original failure; execution and cleanup guards remain unchanged. Signed-off-by: zlareb1 --- .../gates/testsupport/cell_trigger/native.py | 63 ++++++++- ...est_cell_trigger_native_failure_details.py | 129 ++++++++++++++++++ 2 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_cell_trigger_native_failure_details.py diff --git a/scripts/gates/testsupport/cell_trigger/native.py b/scripts/gates/testsupport/cell_trigger/native.py index 9928315e56..979d548483 100644 --- a/scripts/gates/testsupport/cell_trigger/native.py +++ b/scripts/gates/testsupport/cell_trigger/native.py @@ -94,6 +94,33 @@ ) +def native_failure_metadata(error: BaseException) -> dict: + metadata = error_metadata(error) + code = getattr(error, "code", None) + code = code if type(code) is int and 0 <= code < 2**31 else None + causes = [] + seen = {id(error)} + cause = error.__cause__ if error.__cause__ is not None else error.__context__ + for _ in range(4): + if cause is None or id(cause) in seen: + break + seen.add(id(cause)) + name = type(cause).__name__ + number = getattr(cause, "errno", None) + causes.append( + { + "error_type": name + if re.fullmatch(r"[A-Za-z_][A-Za-z_0-9]{0,63}", name) + else "unknown", + "errno": number if type(number) is int and -(2**31) <= number < 2**31 else None, + } + ) + cause = cause.__cause__ if cause.__cause__ is not None else cause.__context__ + if code is not None or causes: + metadata["driver"] = {"code": code, "causes": causes} + return metadata + + def pg_census_sql(): # One statement snapshot supplies all ten fixed counts and server observations. counts = [] @@ -1494,7 +1521,39 @@ def action( if isinstance(error, Refused): try: failure = read(target.with_name(target.name + ".failure.json"), limit=4096) - closed(failure, "error_type site outcome") + closed( + failure, + "error_type site outcome driver" + if "driver" in failure + else "error_type site outcome", + ) + if "driver" in failure: + driver = failure["driver"] + closed(driver, "code causes") + need( + ( + driver["code"] is None + or type(driver["code"]) is int + and 0 <= driver["code"] < 2**31 + ) + and type(driver["causes"]) is list + and len(driver["causes"]) <= 4, + "native driver failure metadata", + ) + for cause in driver["causes"]: + closed(cause, "error_type errno") + need( + isinstance(cause["error_type"], str) + and re.fullmatch( + r"[A-Za-z_][A-Za-z_0-9]{0,63}", cause["error_type"] + ) + and ( + cause["errno"] is None + or type(cause["errno"]) is int + and -(2**31) <= cause["errno"] < 2**31 + ), + "native driver failure cause", + ) site = failure["site"] closed(site, "file line function") need( @@ -2424,7 +2483,7 @@ def main() -> int: try: write( args.output.with_name(args.output.name + ".failure.json"), - error_metadata(error), + native_failure_metadata(error), limit=4096, ) except BaseException: # noqa: BLE001 — diagnostic output must not replace the native failure. diff --git a/tests/unit/test_cell_trigger_native_failure_details.py b/tests/unit/test_cell_trigger_native_failure_details.py new file mode 100644 index 0000000000..8f661ac541 --- /dev/null +++ b/tests/unit/test_cell_trigger_native_failure_details.py @@ -0,0 +1,129 @@ +"""Native failure details expose bounded numbers and classes without payloads.""" + +import json +import sys +from copy import deepcopy + +import pytest +from clickhouse_connect.driver.exceptions import OperationalError + +from tests.unit.test_cell_trigger_ci_creator import keeper_config_case as keeper_config_case + + +def detail(): + return { + "error_type": "OperationalError", + "site": {"file": "native.py", "line": 702, "function": "connections"}, + "outcome": "error", + "driver": { + "code": 159, + "causes": [{"error_type": "ConnectionRefusedError", "errno": 111}], + }, + } + + +@pytest.mark.parametrize("shape", ["transport", "bounded-cycle"]) +def test_native_sidecar_keeps_only_numeric_code_and_bounded_causes( + keeper_config_case, monkeypatch, tmp_path, shape +): + native, instance, _, _, _ = keeper_config_case + context, output = tmp_path / "failure-context.json", tmp_path / "pg_cell-created.json" + native.write( + context, + instance.s | {"root": str(tmp_path), "databases": {"pg_cell": "bench_cell"}}, + ) + monkeypatch.setattr(sys, "path", list(sys.path)) + primary = OperationalError("private-SQL-URL-settings-token", code=159) + cause = ConnectionRefusedError(111, "private-network-address") + primary.__cause__ = cause + expected = detail()["driver"] + if shape == "bounded-cycle": + previous = cause + for _ in range(6): + child = OSError(5, "private-payload") + previous.__cause__ = child + previous = child + previous.__cause__ = cause + expected["causes"] += [{"error_type": "OSError", "errno": 5}] * 3 + + async def failed(*args, **kwargs): + raise primary + + monkeypatch.setattr(native, "stores", failed) + monkeypatch.setattr( + sys, + "argv", + [ + "native.py", + "--task", + "create", + "--role", + "pg_cell", + "--context", + str(context), + "--output", + str(output), + ], + ) + assert native.main() == 1 + assert not output.exists() + raw = output.with_name(output.name + ".failure.json").read_bytes() + metadata = native.read(output.with_name(output.name + ".failure.json"), limit=4096) + assert metadata.get("driver") == expected + assert metadata["error_type"] == "OperationalError" + assert len(raw) <= 4096 and b"private-" not in raw + + +def test_valid_driver_detail_reaches_the_original_command_failure(keeper_config_case): + native, instance, owner, _, _ = keeper_config_case + instance.s["python"] = "unused" + metadata = detail() + native.write(instance.w / "pg_cell-created.json.failure.json", metadata) + original = native.Refused("private-command-failure") + + def refused(*args, **kwargs): + raise original + + owner.command = refused + with pytest.raises(native.Refused) as caught: + instance.action("create", role="pg_cell") + assert caught.value is original and original.native == metadata + assert "private-" not in json.dumps(original.native) + + +@pytest.mark.parametrize( + "field,value", + [ + ("code", True), + ("code", -1), + ("code", 2**31), + ("code", "private-code"), + ("causes", [{"error_type": "OSError", "errno": 5}] * 5), + ("class", "private-class-name"), + ("class", "A" * 65), + ("errno", True), + ("errno", 2**31), + ("extra", "private-message"), + ], +) +def test_unsafe_driver_detail_cannot_replace_the_command_failure(keeper_config_case, field, value): + native, instance, owner, _, _ = keeper_config_case + instance.s["python"] = "unused" + metadata = deepcopy(detail()) + driver = metadata["driver"] + if field in {"code", "causes"}: + driver[field] = value + elif field in {"class", "errno"}: + driver["causes"][0]["error_type" if field == "class" else "errno"] = value + else: + driver["message"] = value + native.write(instance.w / "pg_cell-created.json.failure.json", metadata) + original = native.Refused("original") + + def refused(*args, **kwargs): + raise original + + owner.command = refused + with pytest.raises(native.Refused) as caught: + instance.action("create", role="pg_cell") + assert caught.value is original and original.native is None From 24a159f743f89fcdfd36acc27b7bb5e42c6fea63 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 02:22:00 +0000 Subject: [PATCH 098/114] Provider smoke retains bounded reset failure evidence Collect read-only participant and configuration-lease counts before the failed fixture stack is removed. Keep the original activation failure and deadline when diagnostics fail or time out. Signed-off-by: zlareb1 --- scripts/gates/docker_provider_data.py | 69 +++++- scripts/gates/docker_provider_smoke.py | 23 +- .../test_docker_provider_reset_diagnostics.py | 220 ++++++++++++++++++ 3 files changed, 308 insertions(+), 4 deletions(-) create mode 100644 tests/unit/test_docker_provider_reset_diagnostics.py diff --git a/scripts/gates/docker_provider_data.py b/scripts/gates/docker_provider_data.py index 9704e7c06f..7f8611f850 100644 --- a/scripts/gates/docker_provider_data.py +++ b/scripts/gates/docker_provider_data.py @@ -9,6 +9,7 @@ import time from multiprocessing import spawn from pathlib import Path +from uuid import UUID from server.core import provider_process from server.core.ch import get_client @@ -340,11 +341,77 @@ async def verify(model: str, dimensions: int) -> None: print(f"PASS provider data: {model}/{dimensions}, memory + code metadata, no retained copies") +async def diagnostic(client, transition: str) -> None: + """Read capped aggregate evidence; never acknowledge or retire a participant.""" + from server.insights.store.provider_participants import PARTICIPANTS_SQL + + scope = provider_process.host_proof().split("|")[:4] + if len(scope) != 4 or not all(scope): + raise ValueError("Unavailable provider diagnostic scope") + sql = ( + "WITH retained AS (" + PARTICIPANTS_SQL + "), " + "lease AS (SELECT holder, renewed_at + toIntervalSecond(ttl_s) < now64(3) AS expired " + "FROM loom._migration_lock WHERE name = {name:String} LIMIT 2) " + "SELECT count() AS retained, " + "countIf(p.state = 'running') AS running, " + "countIf(p.state = 'drained' AND p.transition_id = {transition:UUID}) AS matching_drained, " + "countIf(p.state = 'drained' AND p.transition_id != {transition:UUID}) AS other_drained, " + "countIf(p.state = 'retired') AS retained_retired, " + "countIf(p.expired) AS expired, " + "countIf(arraySlice(splitByChar('|', p.host_proof), 1, 4) = {scope:Array(String)}) " + "AS same_scope, " + "(SELECT count() FROM lease) AS lease_rows, " + "(SELECT countIf(expired) FROM lease) AS expired_leases, " + "countIf(l.holder != '') AS retained_holders, " + "countIf(l.holder != '' AND p.state = 'drained' " + "AND p.transition_id = {transition:UUID}) AS matching_drained_holders " + "FROM retained AS p ANY LEFT JOIN lease AS l " + "ON toString(p.participant_id) = arrayElement(splitByChar('/', l.holder), 1) " + "SETTINGS max_execution_time = 3, max_rows_to_read = 10000, " + "max_bytes_to_read = 10485760, read_overflow_mode = 'throw', " + "max_result_rows = 1, max_result_bytes = 4096, result_overflow_mode = 'throw', " + "max_memory_usage = 134217728, max_threads = 1, select_sequential_consistency = 1" + ) + row = await asyncio.wait_for( + client.query_one( + sql, + { + "name": "provider_configuration", + "transition": str(UUID(transition)), + "scope": scope, + }, + ), + timeout=5, + ) + keys = { + "retained", + "running", + "matching_drained", + "other_drained", + "retained_retired", + "expired", + "same_scope", + "lease_rows", + "expired_leases", + "retained_holders", + "matching_drained_holders", + } + if ( + row is None + or row.keys() != keys + or any(type(value) is not int or not 0 <= value <= 10000 for value in row.values()) + ): + raise ValueError("Invalid provider diagnostic aggregate") + print("Provider reset diagnostic: " + json.dumps(row, sort_keys=True)) + + async def main() -> None: client = get_client() await client.connect() try: - if sys.argv[1] == "seed": + if sys.argv[1] == "diagnostic": + await diagnostic(client, sys.argv[2]) + elif sys.argv[1] == "seed": await seed() elif sys.argv[1] == "workers": print(json.dumps(await workers(client))) diff --git a/scripts/gates/docker_provider_smoke.py b/scripts/gates/docker_provider_smoke.py index b5445e145e..036da0dbc0 100644 --- a/scripts/gates/docker_provider_smoke.py +++ b/scripts/gates/docker_provider_smoke.py @@ -61,6 +61,16 @@ def control(compose: list[str], failed_model: str) -> None: def data_check(compose: list[str], *args: str) -> list[dict] | None: source = (ROOT / "scripts/gates/docker_provider_data.py").read_text() + if args[0] == "diagnostic": + subprocess.run( + [*compose, "exec", "-T", "loom", "python", "-", *args], + input=source, + text=True, + check=True, + timeout=10, + stderr=subprocess.DEVNULL, + ) + return None if args[0] == "workers": result = subprocess.run( [*compose, "exec", "-T", "loom", "python", "-", *args], @@ -203,9 +213,16 @@ def smoke_providers(base: str, compose: list[str]) -> None: data_check(compose, "docker-smoke-16", "16") reset = dashboard_action(base, "reset") assert reset["state"] == "queued", reset - status = wait_status( - base, lambda row: row["pending"] is None and row["source"] == "environment" - ) + try: + status = wait_status( + base, lambda row: row["pending"] is None and row["source"] == "environment" + ) + except BaseException: + try: + data_check(compose, "diagnostic", reset["config_id"]) + except BaseException: # noqa: BLE001 — diagnostic failure must preserve activation failure + print("Provider reset diagnostic unavailable") + raise assert status["embedding_model"] == environment["embedding_model"], status assert status["llm_model"] == environment["llm_model"], status wait_admission(base) diff --git a/tests/unit/test_docker_provider_reset_diagnostics.py b/tests/unit/test_docker_provider_reset_diagnostics.py new file mode 100644 index 0000000000..36223ff9b7 --- /dev/null +++ b/tests/unit/test_docker_provider_reset_diagnostics.py @@ -0,0 +1,220 @@ +"""Failure evidence preserves the provider smoke's original reset refusal.""" + +import io +import json +import subprocess +import urllib.error +from unittest.mock import AsyncMock +from uuid import UUID + +import pytest + +import scripts.gates.docker_provider_data as data +import scripts.gates.docker_provider_smoke as smoke + +TRANSITION = str(UUID(int=17)) + + +@pytest.mark.parametrize("diagnostic_failure", [None, RuntimeError, KeyboardInterrupt]) +def test_reset_failure_collects_diagnostic_and_preserves_first_error( + monkeypatch, tmp_path, diagnostic_failure +): + events, failure = _provider_flow(monkeypatch, tmp_path, diagnostic_failure=diagnostic_failure) + with pytest.raises(AssertionError) as caught: + smoke.smoke_providers("http://fixture", events["compose"]) + assert caught.value is failure + assert events["checks"][-1] == ("diagnostic", TRANSITION) + assert events["waits"] == 4 + assert events["resets"] == 1 + assert events["checks"].count(("crash-worker",)) == 1 + assert ( + "retired", + json.dumps([{"participant_id": "first"}, {"participant_id": "second"}]), + ) in events["checks"] + assert not any(args[0] == "environment" for args in events["checks"]) + + +def test_healthy_reset_does_not_collect_diagnostic(monkeypatch, tmp_path): + events, _ = _provider_flow(monkeypatch, tmp_path, healthy=True) + smoke.smoke_providers("http://fixture", events["compose"]) + assert not any(args[0] == "diagnostic" for args in events["checks"]) + assert events["checks"][-1] == ("environment", "8") + assert events["waits"] == 4 + assert events["resets"] == 1 + + +def _provider_flow(monkeypatch, tmp_path, *, diagnostic_failure=None, healthy=False): + override = tmp_path / "override.json" + override.write_text(json.dumps({"services": {"loom": {}}})) + events = { + "compose": ["docker", "compose", "-f", str(override)], + "checks": [], + "waits": 0, + "resets": 0, + } + failure = AssertionError("Timed out waiting for provider activation") + environment = { + "source": "environment", + "embedding_model": "environment", + "embedding_dimensions": 8, + "llm_model": "fake", + "pending": None, + } + active = { + "source": "dashboard", + "pending": None, + "embedding_model": "docker-smoke-16", + "embedding_dimensions": 16, + } + statuses = iter( + [ + { + **active, + "embedding_model": "docker-smoke-8", + "configured_keys": {"compatible": True}, + }, + { + "embedding_model": "docker-smoke-8", + "pending": {"state": "failed", "error_code": "rebuild_rebuild_failed"}, + }, + active, + environment, + ] + ) + model_reads = iter([environment, active]) + applications = iter(["queued", "active", "queued"]) + + def request(_base, path, _payload=None): + if path == "/v1/admin/models": + return next(model_reads) + raise urllib.error.HTTPError("http://fixture", 503, "closed", {}, None) + + def configure(*_args): + raise urllib.error.HTTPError( + "http://fixture", 422, "width", {}, io.BytesIO(b"unsupported_dimensions") + ) + + def dashboard(_base, action, *_args): + events["resets"] += action == "reset" + return { + "state": next(applications) if action == "apply" else "queued", + "config_id": TRANSITION, + } + + def check(_compose, *args): + events["checks"].append(args) + if args[0] == "diagnostic" and diagnostic_failure is not None: + raise diagnostic_failure("diagnostic secret must not replace the original") + return ( + [{"participant_id": "first"}, {"participant_id": "second"}] + if args[0] == "workers" + else None + ) + + def status(_base, predicate): + events["waits"] += 1 + value = next(statuses) + assert predicate(value) + if events["waits"] == 4 and not healthy: + raise failure + return value + + monkeypatch.setattr(smoke, "request", request) + monkeypatch.setattr(smoke, "configure", configure) + monkeypatch.setattr(smoke, "dashboard_action", dashboard) + monkeypatch.setattr(smoke, "data_check", check) + monkeypatch.setattr(smoke, "wait_status", status) + monkeypatch.setattr(smoke, "wait_admission", lambda *_: None) + monkeypatch.setattr(smoke, "control", lambda *_: None) + monkeypatch.setattr(smoke, "run", lambda *_: None) + return events, failure + + +def test_diagnostic_exec_is_bounded_and_discards_stderr(monkeypatch): + calls = [] + monkeypatch.setattr( + smoke.subprocess, "run", lambda *args, **kwargs: calls.append((args, kwargs)) + ) + smoke.data_check(["docker", "compose", "-f", "owned.json"], "diagnostic", TRANSITION) + args, kwargs = calls[0] + assert args[0][-2:] == ["diagnostic", TRANSITION] + assert kwargs["timeout"] == 10 + assert kwargs["check"] is True + assert kwargs["stderr"] == subprocess.DEVNULL + assert "capture_output" not in kwargs + + +@pytest.mark.asyncio +async def test_diagnostic_emits_only_closed_bounded_counts(monkeypatch, capsys): + from types import SimpleNamespace + + row = _counts() + query = AsyncMock(return_value=row) + monkeypatch.setattr( + data.provider_process, "host_proof", lambda: "host|boot|container|namespace|55|90" + ) + await data.diagnostic(SimpleNamespace(query_one=query), TRANSITION) + sql, params = query.await_args.args + assert sql.lstrip().startswith("WITH retained AS (SELECT") + assert "ANY LEFT JOIN" in sql and "LIMIT 2" in sql + for guard in ( + "max_execution_time = 3", + "max_rows_to_read = 10000", + "max_bytes_to_read = 10485760", + "max_result_rows = 1", + "max_result_bytes = 4096", + "max_memory_usage = 134217728", + ): + assert guard in sql + assert params == { + "name": "provider_configuration", + "transition": TRANSITION, + "scope": ["host", "boot", "container", "namespace"], + } + output = capsys.readouterr().out + assert json.loads(output.split(": ", 1)[1]) == row + assert len(output) < 1024 + assert "host|boot" not in output and TRANSITION not in output + assert query.await_count == 1 + + +@pytest.mark.parametrize("fault", ["extra", "bool", "large", "negative", "missing", "none"]) +@pytest.mark.asyncio +async def test_diagnostic_refuses_invalid_aggregate_without_output(monkeypatch, capsys, fault): + from types import SimpleNamespace + + row = _counts() + if fault == "extra": + row["payload"] = "secret" + elif fault == "bool": + row["retained"] = True + elif fault == "large": + row["retained"] = 10001 + elif fault == "negative": + row["retained"] = -1 + elif fault == "missing": + del row["retained"] + else: + row = None + monkeypatch.setattr( + data.provider_process, "host_proof", lambda: "host|boot|container|namespace|55|90" + ) + with pytest.raises(ValueError, match="Invalid provider diagnostic aggregate"): + await data.diagnostic(SimpleNamespace(query_one=AsyncMock(return_value=row)), TRANSITION) + assert capsys.readouterr().out == "" + + +def _counts(): + return { + "retained": 4, + "running": 1, + "matching_drained": 3, + "other_drained": 0, + "retained_retired": 0, + "expired": 1, + "same_scope": 3, + "lease_rows": 1, + "expired_leases": 0, + "retained_holders": 1, + "matching_drained_holders": 1, + } From 2460fa2ff8298e957d7a5a0f852d385e811077e8 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 03:04:54 +0000 Subject: [PATCH 099/114] Trigger acceptance retains bounded Postgres failure evidence Preserve the wrapped cause, error identifiers and commit uncertainty when staging refuses, without changing workload, retry or acceptance rules. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 15 +- .../gates/testsupport/cell_trigger/fixture.py | 54 +++- .../gates/testsupport/cell_trigger/native.py | 74 +++--- ...est_cell_trigger_driver_failure_details.py | 242 ++++++++++++++++++ 4 files changed, 349 insertions(+), 36 deletions(-) create mode 100644 tests/unit/test_cell_trigger_driver_failure_details.py diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 103eb8754d..18237c300c 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -34,6 +34,7 @@ import clickhouse_connect import httpx +from scripts.gates.testsupport.cell_trigger.fixture import FILES, native_failure_metadata from server.core import cell_triggers from server.core.hashing import hash_text, sha256_value @@ -1603,16 +1604,28 @@ async def run(ch: Any) -> dict[str, Any]: except BaseException as error: # noqa: BLE001 — every failed run must emit INVALID # Never serialize exception text: clients may include credentials or values. reason = str(error) if isinstance(error, Invalid) else "operation refused" + failure = None + try: + failure = native_failure_metadata( + error, files=(*FILES, "cell_trigger_gate.py", "pg.py", "pg_memory.py") + ) + except BaseException: # noqa: BLE001 — diagnostic failure must preserve INVALID + pass verdict = { "verdict": "INVALID", "phase": phase, "reason": reason, "error_type": type(error).__name__, + **({"failure": failure} if failure is not None else {}), } if artifact is not None: try: artifact.emit( - "invalid", phase=phase, reason=reason, error_type=type(error).__name__ + "invalid", + phase=phase, + reason=reason, + error_type=type(error).__name__, + **({"failure": failure} if failure is not None else {}), ) except BaseException: # noqa: BLE001 — secondary output cannot replace INVALID. pass diff --git a/scripts/gates/testsupport/cell_trigger/fixture.py b/scripts/gates/testsupport/cell_trigger/fixture.py index 4862bf892e..27c280092c 100644 --- a/scripts/gates/testsupport/cell_trigger/fixture.py +++ b/scripts/gates/testsupport/cell_trigger/fixture.py @@ -569,16 +569,18 @@ def load_owned(path: Path): return OwnedChildren() -def error_metadata(error: BaseException) -> dict: +def error_metadata( + error: BaseException, *, files: tuple[str, ...] = FILES, fallback: bool = True +) -> dict: site = {"file": "unknown", "line": 0, "function": "unknown"} - frame = sys._getframe(1) - line = frame.f_lineno + frame = sys._getframe(1) if fallback else None + line = frame.f_lineno if frame is not None else 0 trace = error.__traceback__ for _ in range(64): if trace is None: break candidate = trace.tb_frame - if Path(candidate.f_code.co_filename).name in FILES and candidate.f_code.co_name not in { + if Path(candidate.f_code.co_filename).name in files and candidate.f_code.co_name not in { "need", "closed", "equal", @@ -593,7 +595,7 @@ def error_metadata(error: BaseException) -> dict: frame = frame.f_back if frame is not None: line = frame.f_lineno - if frame is not None and Path(frame.f_code.co_filename).name in FILES: + if frame is not None and Path(frame.f_code.co_filename).name in files: site = { "file": Path(frame.f_code.co_filename).name, "line": line, @@ -610,6 +612,48 @@ def error_metadata(error: BaseException) -> dict: } +def native_failure_metadata(error: BaseException, *, files: tuple[str, ...] = FILES) -> dict: + metadata = error_metadata(error, files=files) + if not re.fullmatch(r"[A-Za-z_][A-Za-z_0-9]{0,63}", metadata["error_type"]): + metadata["error_type"] = "unknown" + code = getattr(error, "code", None) + code = code if type(code) is int and 0 <= code < 2**31 else None + causes = [] + seen = {id(error)} + cause = error.__cause__ if error.__cause__ is not None else error.__context__ + for _ in range(4): + if cause is None or id(cause) in seen: + break + seen.add(id(cause)) + name = type(cause).__name__ + number = getattr(cause, "errno", None) + causes.append( + { + "error_type": name + if re.fullmatch(r"[A-Za-z_][A-Za-z_0-9]{0,63}", name) + else "unknown", + "errno": number if type(number) is int and -(2**31) <= number < 2**31 else None, + } + ) + cause_site = error_metadata(cause, files=files, fallback=False)["site"] + if cause_site["file"] != "unknown": + causes[-1]["site"] = cause_site + sqlstate = getattr(cause, "sqlstate", None) + if type(sqlstate) is str and re.fullmatch(r"[0-9A-Z]{5}", sqlstate): + causes[-1]["sqlstate"] = sqlstate + cause = cause.__cause__ if cause.__cause__ is not None else cause.__context__ + driver = {"code": code, "causes": causes} + sqlstate = getattr(error, "sqlstate", None) + if type(sqlstate) is str and re.fullmatch(r"[0-9A-Z]{5}", sqlstate): + driver["sqlstate"] = sqlstate + extra = getattr(error, "extra", None) + if type(extra) is dict and type(extra.get("outcome_unknown")) is bool: + driver["outcome_unknown"] = extra["outcome_unknown"] + if code is not None or causes or len(driver) > 2: + metadata["driver"] = driver + return metadata + + class Owner: def __init__(self, spec: dict, owned, note) -> None: self.spec, self.owned, self.note = spec, owned, note diff --git a/scripts/gates/testsupport/cell_trigger/native.py b/scripts/gates/testsupport/cell_trigger/native.py index 979d548483..8d68c386db 100644 --- a/scripts/gates/testsupport/cell_trigger/native.py +++ b/scripts/gates/testsupport/cell_trigger/native.py @@ -39,7 +39,7 @@ child_env, database, digest, - error_metadata, + native_failure_metadata, need, pg_dsn, read, @@ -94,33 +94,6 @@ ) -def native_failure_metadata(error: BaseException) -> dict: - metadata = error_metadata(error) - code = getattr(error, "code", None) - code = code if type(code) is int and 0 <= code < 2**31 else None - causes = [] - seen = {id(error)} - cause = error.__cause__ if error.__cause__ is not None else error.__context__ - for _ in range(4): - if cause is None or id(cause) in seen: - break - seen.add(id(cause)) - name = type(cause).__name__ - number = getattr(cause, "errno", None) - causes.append( - { - "error_type": name - if re.fullmatch(r"[A-Za-z_][A-Za-z_0-9]{0,63}", name) - else "unknown", - "errno": number if type(number) is int and -(2**31) <= number < 2**31 else None, - } - ) - cause = cause.__cause__ if cause.__cause__ is not None else cause.__context__ - if code is not None or causes: - metadata["driver"] = {"code": code, "causes": causes} - return metadata - - def pg_census_sql(): # One statement snapshot supplies all ten fixed counts and server observations. counts = [] @@ -1529,7 +1502,24 @@ def action( ) if "driver" in failure: driver = failure["driver"] - closed(driver, "code causes") + closed( + driver, + "code causes" + + (" sqlstate" if "sqlstate" in driver else "") + + (" outcome_unknown" if "outcome_unknown" in driver else ""), + ) + need( + ( + "sqlstate" not in driver + or type(driver["sqlstate"]) is str + and re.fullmatch(r"[0-9A-Z]{5}", driver["sqlstate"]) + ) + and ( + "outcome_unknown" not in driver + or type(driver["outcome_unknown"]) is bool + ), + "native Postgres failure metadata", + ) need( ( driver["code"] is None @@ -1541,7 +1531,31 @@ def action( "native driver failure metadata", ) for cause in driver["causes"]: - closed(cause, "error_type errno") + closed( + cause, + "error_type errno" + + (" sqlstate" if "sqlstate" in cause else "") + + (" site" if "site" in cause else ""), + ) + need( + "sqlstate" not in cause + or type(cause["sqlstate"]) is str + and re.fullmatch(r"[0-9A-Z]{5}", cause["sqlstate"]), + "native Postgres failure cause", + ) + if "site" in cause: + cause_site = cause["site"] + closed(cause_site, "file line function") + need( + cause_site["file"] in (*FILES, "unknown") + and type(cause_site["line"]) is int + and 0 <= cause_site["line"] < 2**31 + and isinstance(cause_site["function"], str) + and re.fullmatch( + r"[A-Za-z_][A-Za-z_0-9]{0,63}", cause_site["function"] + ), + "native failure cause site", + ) need( isinstance(cause["error_type"], str) and re.fullmatch( diff --git a/tests/unit/test_cell_trigger_driver_failure_details.py b/tests/unit/test_cell_trigger_driver_failure_details.py new file mode 100644 index 0000000000..ff41e5e20e --- /dev/null +++ b/tests/unit/test_cell_trigger_driver_failure_details.py @@ -0,0 +1,242 @@ +"""The native driver retains finite Postgres failure evidence without payloads.""" + +import asyncio +import dataclasses +import json +import sys +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import asyncpg +import pytest + +import bench.cell_trigger_gate as gate +from tests.unit.test_bench_cell_trigger_gate import args, fixture_config +from tests.unit.test_cell_trigger_ci_creator import keeper_config_case as keeper_config_case +from tests.unit.test_cell_trigger_native_failure_details import detail + + +@pytest.mark.parametrize("fault", ["query_cancel", "connection", "unknown_commit"]) +def test_actual_stage_failure_retains_pg_cause_in_invalid_result( + monkeypatch, tmp_path, capsys, fault +): + output, calls = _main_case(monkeypatch, tmp_path, fault) + assert gate.main() == 1 + verdict = json.loads(capsys.readouterr().out) + assert verdict["verdict"] == "INVALID" + assert verdict["phase"] == "native_driver" + assert verdict["reason"] == "operation refused" + assert verdict["error_type"] == "PostgresUnavailableError" + assert calls == ["wire", "stage", "native_returned"] + assert "secret-trigger-diagnostic-payload" not in json.dumps(verdict) + assert "failure" in verdict + metadata = verdict["failure"] + assert metadata["error_type"] == "PostgresUnavailableError" + assert metadata["site"]["file"] == "pg.py" + assert metadata["site"]["function"] == "_raise_if_unavailable" + assert metadata["site"]["line"] > 0 + assert metadata["driver"]["outcome_unknown"] is (fault == "unknown_commit") + expected = ( + {"error_type": "ConnectionResetError", "errno": 104} + if fault == "connection" + else {"error_type": "QueryCanceledError", "errno": None, "sqlstate": "57014"} + ) + assert metadata["driver"]["causes"] == [expected] + invalid = json.loads(output.read_text().splitlines()[-1]) + assert invalid["kind"] == "invalid" and invalid["failure"] == metadata + + +def test_healthy_native_driver_keeps_pass_without_failure_metadata(monkeypatch, tmp_path, capsys): + output, calls = _main_case(monkeypatch, tmp_path, None) + assert gate.main() == 0 + verdict = json.loads(capsys.readouterr().out) + assert verdict["verdict"] == "PASS" and "failure" not in verdict + assert calls == ["wire", "native_returned"] + assert json.loads(output.read_text().splitlines()[-1])["kind"] == "driver_cleanup" + + +def _main_case(monkeypatch, tmp_path, fault): + import server.settings + import server.wiring + from server.cli import _ch + from server.core import pg + from server.memory.store import pg_memory + + fixture, config = fixture_config() + manifest = tmp_path / "manifest" + manifest.write_text(json.dumps(dataclasses.asdict(fixture))) + output = tmp_path / "driver.jsonl" + argv = ["driver"] + for name, value in { + **vars(args()), + "fixture_manifest": str(manifest), + "output": str(output), + }.items(): + argv += ["--" + name.replace("_", "-"), str(value)] + calls = [] + monkeypatch.setattr(sys, "argv", argv) + monkeypatch.setattr(server.settings, "settings", config) + monkeypatch.setattr(server.wiring, "wire", lambda: calls.append("wire")) + monkeypatch.setattr(gate, "_preflight", AsyncMock()) + monkeypatch.setattr(gate, "_source", lambda _: {"clean": True}) + monkeypatch.setattr(gate, "_launch", lambda *_: {}) + monkeypatch.setattr(gate, "_debt", AsyncMock()) + monkeypatch.setattr(gate, "_absent", AsyncMock()) + + async def commit(*_args, **_kwargs): + calls.append("stage") + cause = ( + ConnectionResetError(104, "secret-trigger-diagnostic-payload") + if fault == "connection" + else asyncpg.QueryCanceledError("secret-trigger-diagnostic-payload") + ) + pg._raise_if_unavailable(cause, outcome_unknown=fault == "unknown_commit") + raise AssertionError("the actual PG wrapper must refuse") + + monkeypatch.setattr(pg_memory, "commit_versions", commit) + + def native(*_args, **_kwargs): + try: + if fault: + block = gate.Block( + rows=[ + { + "org": "owned", + "namespace": "owned", + "memory_id": "00000000-0000-0000-0000-000000000011", + "memory_key": "owned", + } + ], + audits=[], + documents=[], + ) + asyncio.run( + gate._stage( + SimpleNamespace(fetchval=AsyncMock(return_value=0)), object(), block + ) + ) + return { + "latency_within_limit": True, + "throughput_within_limit": True, + "p99_delta_ms": 0.0, + "throughput_loss": 0.0, + } + finally: + calls.append("native_returned") + + monkeypatch.setattr(_ch, "run_connected", native) + return output, calls + + +@pytest.mark.parametrize("cause_state", ["57014", "08006", "secret-query-state", True]) +def test_shared_sanitizer_keeps_only_closed_pg_values(cause_state): + from scripts.gates.testsupport.cell_trigger.fixture import native_failure_metadata + from server.core.pg import PostgresUnavailableError + + cause = OSError(104, "secret-trigger-diagnostic-payload") + cause.sqlstate = cause_state + error = PostgresUnavailableError("secret-trigger-diagnostic-payload", outcome_unknown=True) + error.__cause__ = cause + metadata = native_failure_metadata(error) + assert metadata["driver"]["outcome_unknown"] is True + entry = metadata["driver"]["causes"][0] + assert entry["error_type"] == "ConnectionResetError" and entry["errno"] == 104 + if type(cause_state) is str and cause_state in {"57014", "08006"}: + assert entry["sqlstate"] == cause_state + else: + assert "sqlstate" not in entry + assert "secret-trigger-diagnostic-payload" not in json.dumps(metadata) + assert len(json.dumps(metadata)) < 4096 + + +def test_shared_sanitizer_preserves_existing_transport_shape_and_cycle_bound(): + from clickhouse_connect.driver.exceptions import OperationalError + + from scripts.gates.testsupport.cell_trigger.fixture import native_failure_metadata + + error = OperationalError("secret-trigger-diagnostic-payload", code=159) + cause = ConnectionRefusedError(111, "secret-trigger-diagnostic-payload") + error.__cause__ = cause + cause.__cause__ = error + metadata = native_failure_metadata(error) + assert metadata["driver"] == { + "code": 159, + "causes": [{"error_type": "ConnectionRefusedError", "errno": 111}], + } + assert "secret-trigger-diagnostic-payload" not in json.dumps(metadata) + + +def test_diagnostic_error_cannot_replace_actual_stage_invalid(monkeypatch, tmp_path, capsys): + output, calls = _main_case(monkeypatch, tmp_path, "query_cancel") + + def failed(*_args, **_kwargs): + raise RuntimeError("secret-trigger-diagnostic-payload") + + monkeypatch.setattr(gate, "native_failure_metadata", failed) + assert gate.main() == 1 + verdict = json.loads(capsys.readouterr().out) + assert verdict["verdict"] == "INVALID" and verdict["error_type"] == "PostgresUnavailableError" + assert verdict["phase"] == "native_driver" and verdict["reason"] == "operation refused" + assert "failure" not in verdict + assert calls == ["wire", "stage", "native_returned"] + invalid = json.loads(output.read_text().splitlines()[-1]) + assert invalid["kind"] == "invalid" and "failure" not in invalid + assert "secret-trigger-diagnostic-payload" not in json.dumps(verdict) + + +@pytest.mark.parametrize( + "fault", + [ + "none", + "root_sqlstate", + "cause_sqlstate", + "boolean_sqlstate", + "outcome_unknown", + "site_file", + "site_line", + "site_function", + "site_extra", + "driver_extra", + ], +) +def test_native_reader_validates_new_pg_fields_without_replacing_first_error( + keeper_config_case, fault +): + native, instance, owner, _, _ = keeper_config_case + instance.s["python"] = "unused" + metadata = detail() + driver = metadata["driver"] + driver.update(sqlstate="57014", outcome_unknown=True) + cause = driver["causes"][0] + cause.update( + sqlstate="08006", site={"file": "native.py", "line": 702, "function": "connections"} + ) + if fault == "root_sqlstate": + driver["sqlstate"] = "secret-query" + elif fault == "cause_sqlstate": + cause["sqlstate"] = "secret-query" + elif fault == "boolean_sqlstate": + driver["sqlstate"] = True + elif fault == "outcome_unknown": + driver["outcome_unknown"] = 1 + elif fault == "site_file": + cause["site"]["file"] = "/secret/path/native.py" + elif fault == "site_line": + cause["site"]["line"] = True + elif fault == "site_function": + cause["site"]["function"] = "secret-payload" + elif fault == "site_extra": + cause["site"]["message"] = "secret-payload" + elif fault == "driver_extra": + driver["message"] = "secret-payload" + native.write(instance.w / "pg_cell-created.json.failure.json", metadata) + original = native.Refused("original-command-failure") + + def refused(*_args, **_kwargs): + raise original + + owner.command = refused + with pytest.raises(native.Refused) as caught: + instance.action("create", role="pg_cell") + assert caught.value is original + assert original.native == (metadata if fault == "none" else None) From e9bd5b10c291a55872c395a41dfcf3b0a61db47e Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 03:16:38 +0000 Subject: [PATCH 100/114] Retirement census distinguishes method calls from local cleanup Signed-off-by: zlareb1 --- tests/unit/test_cell_retirement.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_cell_retirement.py b/tests/unit/test_cell_retirement.py index f3e7593fd3..48517fb11e 100644 --- a/tests/unit/test_cell_retirement.py +++ b/tests/unit/test_cell_retirement.py @@ -420,7 +420,7 @@ def test_retirement_writers_and_accepting_command_roots_are_an_explicit_census() commands.add(relative) if name == "participant_command": plain.add(relative) - if name == "stopped": + if name == "stopped" and isinstance(func, ast.Attribute): stopped.add(relative) assert constructors == {"core/retirement.py"} assert writers == {"insights/app/provider_barrier.py"} From 79943b75065eeabd7c03f15b08b9feeba3544081 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 03:50:30 +0000 Subject: [PATCH 101/114] Worker parents retire verified stopped children A child can exit before completing its own retirement. The parent records retirement only after its captured child identity is physically gone and every tagged query has ended, within the existing stop or restart deadline. Signed-off-by: zlareb1 --- server/insights/app/provider_barrier.py | 14 +- .../insights/store/provider_participants.py | 12 +- server/workers/child.py | 74 +++++- tests/unit/test_workers_child_retirement.py | 233 ++++++++++++++++++ 4 files changed, 326 insertions(+), 7 deletions(-) create mode 100644 tests/unit/test_workers_child_retirement.py diff --git a/server/insights/app/provider_barrier.py b/server/insights/app/provider_barrier.py index 794b457649..26009456e3 100644 --- a/server/insights/app/provider_barrier.py +++ b/server/insights/app/provider_barrier.py @@ -319,19 +319,25 @@ async def wrapped(app: T) -> AsyncIterator[None]: return wrapped -async def retire_participant(participant: UUID, *, ch: Any = None) -> str: +async def retire_participant( + participant: UUID, *, ch: Any = None, expected_proof: str | None = None +) -> str: """`retired`, `absent` (no row), or `live` (not proved stopped). Retirement needs the recorded identity proved absent in the same host, boot and PID namespace, with tagged queries known zero. Unknown evidence blocks; heartbeat expiry alone proves nothing. """ - row = next( - (r for r in await participants(ch=ch) if str(r["participant_id"]) == str(participant)), - None, + rows = ( + await participants(ch=ch) + if expected_proof is None + else await participants(ch=ch, host_proof=expected_proof) ) + row = next((r for r in rows if str(r["participant_id"]) == str(participant)), None) if row is None: return "absent" + if expected_proof is not None and (len(rows) != 1 or row.get("host_proof") != expected_proof): + return "live" if str(participant) == str(process_identity()): return "live" if not proves_stopped(str(row.get("host_proof") or "")): diff --git a/server/insights/store/provider_participants.py b/server/insights/store/provider_participants.py index 567aa5e676..e03c5b22bf 100644 --- a/server/insights/store/provider_participants.py +++ b/server/insights/store/provider_participants.py @@ -83,9 +83,17 @@ def running_sql(*, inserts: bool = False) -> str: _PRUNE_SQL = f"ALTER TABLE {TABLE} DELETE WHERE participant_id IN {{ids:Array(UUID)}}" -async def participants(*, ch: _CHClient | None = None) -> list[dict[str, Any]]: +async def participants( + *, ch: _CHClient | None = None, host_proof: str | None = None +) -> list[dict[str, Any]]: with sequential_reads(): - rows = await (ch or _default_client()).query_all(PARTICIPANTS_SQL) + if host_proof is None: + rows = await (ch or _default_client()).query_all(PARTICIPANTS_SQL) + else: + rows = await (ch or _default_client()).query_all( + PARTICIPANTS_SQL + " AND host_proof = {proof:String} LIMIT 2", + {"proof": host_proof}, + ) return [ r for r in rows diff --git a/server/workers/child.py b/server/workers/child.py index 0d782ca1c4..730ab2478d 100644 --- a/server/workers/child.py +++ b/server/workers/child.py @@ -28,6 +28,8 @@ import sys import time from collections.abc import Callable, Mapping +from pathlib import Path +from uuid import UUID from server.core.errors import error_kind from server.settings import settings @@ -43,6 +45,8 @@ # The longest a stop waits for the child before killing it, whatever the # deadline: the lifespan's shutdown drains share 40 s. STOP_TIMEOUT_S = 20.0 +# Not a setting: proof work fits within the existing stop or restart deadline. +_REAP_RETIRE_S = 1.0 RESTART_DELAY_S = 5.0 # A child that exits sooner than this after it started failed to start; after # MAX_FAILED_STARTS in a row this process gives the lease up. @@ -61,6 +65,8 @@ class ChildWorkers: def __init__(self, lease: leader.SupervisorLease) -> None: self._lease = lease self._proc: asyncio.subprocess.Process | None = None + self._child_proof: str | None = None + self._child_parent_proof: str | None = None self._watch: asyncio.Task[None] | None = None self._publish: asyncio.Task[None] | None = None self._write_fd: int | None = None @@ -120,10 +126,13 @@ async def _keep_running(self) -> None: # The outcome reaches `/v1/health` (no token) as the lease's reason. log.warning("workers child could not start: %s: %s", type(e).__name__, e) outcome = f"could not start ({error_kind(e)})" + restart_at = time.monotonic() + RESTART_DELAY_S else: assert self._proc is not None self.last_exit = await self._proc.wait() + restart_at = time.monotonic() + RESTART_DELAY_S self._close_pipe() + await self._retire_child(self._proc, restart_at) outcome = f"exited (code {self.last_exit})" if not self._lease.fresh(): return # the lease loop stops the workers, and starts them once it is @@ -139,7 +148,7 @@ async def _keep_running(self) -> None: return log.error("workers child %s; restarting in %.0f s", outcome, RESTART_DELAY_S) self.restarts += 1 - await asyncio.sleep(RESTART_DELAY_S) + await asyncio.sleep(max(0.0, restart_at - time.monotonic())) async def _spawn(self) -> None: """Start the child. On an OSError the child's pipe ends are closed here and @@ -163,12 +172,72 @@ async def _spawn(self) -> None: finally: for fd in child_fds: os.close(fd) + self._capture_child_proof() asyncio.get_running_loop().add_reader(self._alive_fd, self._read_beats) log.info("workers child started: pid %s", self._proc.pid) self._publish = asyncio.get_running_loop().create_task( self._publish_loop(), name="loom-workers-child-lease" ) + def _capture_child_proof(self) -> None: + """Capture the owned child's birth while its PID still names our direct child.""" + from server.core import provider_process + + self._child_proof = None + self._child_parent_proof = None + proc = self._proc + if proc is None or proc.returncode is not None or proc.pid <= 1 or proc.pid == os.getpid(): + return + try: + parent_proof = provider_process.host_proof() + parent = parent_proof.split("|") + root = Path(f"/proc/{proc.pid}") + stat = (root / "stat").read_text().rsplit(")", 1)[1].split() + if ( + proc.returncode is not None + or len(parent) != 8 + or parent[4] != str(os.getpid()) + or int(stat[1]) != os.getpid() + or str(os.stat(root / "ns/pid").st_ino) != parent[3] + ): + return + parent[4:6] = [str(proc.pid), stat[19]] + self._child_proof = provider_process.certified("|".join(parent)) + self._child_parent_proof = parent_proof + except (OSError, ValueError, IndexError): + return # An unknown birth never authorizes post-reap retirement. + + async def _retire_child(self, proc: asyncio.subprocess.Process, end: float) -> None: + from server.core import provider_process + from server.insights.app.provider_barrier import retire_participant + from server.insights.store.provider_participants import participants + + proof = self._child_proof + if proc is not self._proc or proc.returncode is None or proof is None: + return + captured = proof.split("|") + parent_proof = provider_process.host_proof() + here = parent_proof.split("|") + if ( + parent_proof != self._child_parent_proof + or len(captured) != 9 + or not provider_process.has_certificate(proof) + or len(here) != 8 + or captured[:4] != here[:4] + or captured[4] != str(proc.pid) + or proc.pid == os.getpid() + or end <= time.monotonic() + ): + return + try: + async with asyncio.timeout(min(_REAP_RETIRE_S, max(0.0, end - time.monotonic()))): + rows = await participants(host_proof=proof) + if len(rows) != 1 or rows[0].get("host_proof") != proof: + return + await retire_participant(UUID(str(rows[0]["participant_id"])), expected_proof=proof) + except Exception: # noqa: BLE001 — unproved retirement must not replace child cleanup + log.warning("reaped workers child retirement remains unproved") + def _send_deadline(self) -> None: """Send the lease deadline; it moves only while the lease is fresh.""" if self._lease.fresh(): @@ -225,6 +294,9 @@ async def _reap(self, kill_at: float) -> None: with contextlib.suppress(ProcessLookupError): proc.kill() await proc.wait() + if proc is not None: + await proc.wait() + await self._retire_child(proc, kill_at) finally: if proc is not None and proc.returncode is None: with contextlib.suppress(ProcessLookupError): diff --git a/tests/unit/test_workers_child_retirement.py b/tests/unit/test_workers_child_retirement.py new file mode 100644 index 0000000000..08fc6c8bb3 --- /dev/null +++ b/tests/unit/test_workers_child_retirement.py @@ -0,0 +1,233 @@ +"""The reaped worker child's original process identity gates retirement.""" + +import asyncio +import os +import signal +import time +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import UUID + +import pytest + +from server.core import provider_process +from server.insights.app import provider_barrier as barrier +from server.insights.store import provider_participants as store +from server.workers import child + + +@pytest.mark.asyncio +async def test_actual_reap_certifies_its_exact_uncertified_child(monkeypatch): + workers, proc, proof, record, publish = await _owned(monkeypatch) + try: + await workers._reap(time.monotonic() + 10) + assert proc.returncode == -signal.SIGTERM and proc.waits >= 1 + assert record.await_count == 1 + evidence = record.await_args.args[0] + assert evidence.participant == UUID(int=17) + assert evidence.host_proof == proof and evidence.after == 9 + barrier.running_queries.assert_awaited_once_with(UUID(int=17), ch=None) + assert proc.signals == [signal.SIGTERM] + finally: + workers._close_pipe() + publish.cancel() + await asyncio.gather(publish, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_actual_reap_does_not_rewrite_an_already_certified_child(monkeypatch): + workers, proc, _, record, publish = await _owned(monkeypatch, fault="already_retired") + try: + await workers._reap(time.monotonic() + 10) + assert proc.returncode == -signal.SIGTERM + assert not record.await_count + finally: + workers._close_pipe() + publish.cancel() + await asyncio.gather(publish, return_exceptions=True) + + +@pytest.mark.parametrize( + "fault", + [ + "missing", + "duplicate", + "wrong_pid", + "reused_pid_proof", + "different_namespace", + "own_uuid", + "alive", + "queries_running", + "queries_unknown", + "query_error", + "birth_unknown", + "birth_other_parent", + "birth_wrong_namespace", + "parent_changed", + ], +) +@pytest.mark.asyncio +async def test_reap_retirement_refuses_unknown_or_unsettled_identity(monkeypatch, fault): + workers, proc, _, record, publish = await _owned(monkeypatch, fault=fault) + try: + await workers._reap(time.monotonic() + 10) + assert proc.returncode == -signal.SIGTERM + assert not record.await_count + finally: + workers._close_pipe() + publish.cancel() + await asyncio.gather(publish, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_a_live_owned_child_cannot_be_certified_before_its_wait(monkeypatch): + workers, proc, _, record, publish = await _owned(monkeypatch) + try: + await workers._retire_child(proc, time.monotonic() + 10) + assert proc.returncode is None and proc.waits == 0 + assert not record.await_count + finally: + workers._close_pipe() + publish.cancel() + await asyncio.gather(publish, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_exhausted_original_stop_budget_refuses_retirement(monkeypatch): + workers, proc, _, record, publish = await _owned(monkeypatch) + try: + await workers._reap(time.monotonic() - 1) + assert proc.returncode is not None + assert not record.await_count + finally: + workers._close_pipe() + publish.cancel() + await asyncio.gather(publish, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_exact_proof_participant_read_is_bounded_and_parameterized(monkeypatch): + client = SimpleNamespace(query_all=AsyncMock(return_value=[])) + proof = "host|boot|container|77|123|901|||cell-retirement-v1" + assert await store.participants(ch=client, host_proof=proof) == [] + sql, params = client.query_all.await_args.args + assert sql == store.PARTICIPANTS_SQL + " AND host_proof = {proof:String} LIMIT 2" + assert params == {"proof": proof} + assert proof not in sql + + +async def _owned(monkeypatch, *, fault=None): + parent = os.getpid() + pid = parent + 100000 + here = f"host|boot|container|77|{parent}|900||" + proof = f"host|boot|container|77|{pid}|901|||cell-retirement-v1" + rows = [{"participant_id": UUID(int=17), "host_proof": proof, "state": "drained", "version": 9}] + if fault in {"missing", "already_retired"}: + rows = [] + elif fault == "duplicate": + rows *= 2 + elif fault in {"wrong_pid", "reused_pid_proof", "different_namespace"}: + parts = proof.split("|") + parts[{"wrong_pid": 4, "reused_pid_proof": 5, "different_namespace": 3}[fault]] = "999" + rows[0]["host_proof"] = "|".join(parts) + elif fault == "own_uuid": + rows[0]["participant_id"] = UUID(int=18) + participants = AsyncMock(return_value=rows) + monkeypatch.setattr(store, "participants", participants) + monkeypatch.setattr(barrier, "participants", participants) + monkeypatch.setattr(barrier, "process_identity", lambda: UUID(int=18)) + monkeypatch.setattr(barrier, "proves_stopped", lambda _: fault != "alive") + monkeypatch.setattr(provider_process, "proves_stopped", lambda _: fault != "alive") + monkeypatch.setattr(provider_process, "host_proof", lambda: here) + queries = AsyncMock( + return_value=None if fault == "queries_unknown" else int(fault == "queries_running") + ) + if fault == "query_error": + queries.side_effect = RuntimeError("unavailable") + monkeypatch.setattr(barrier, "running_queries", queries) + record = AsyncMock() + monkeypatch.setattr(barrier, "record_retirement", record) + original_read, original_stat = Path.read_text, os.stat + fields = ["S", str(parent)] + ["0"] * 18 + fields[19] = "901" + if fault == "birth_other_parent": + fields[1] = str(parent + 1) + + def read(path, *args, **kwargs): + if path == Path(f"/proc/{pid}/stat"): + if fault == "birth_unknown": + raise PermissionError("uninspectable") + return f"{pid} (python) " + " ".join(fields) + return original_read(path, *args, **kwargs) + + def stat(path, *args, **kwargs): + if str(path) == f"/proc/{pid}/ns/pid": + return SimpleNamespace(st_ino=78 if fault == "birth_wrong_namespace" else 77) + return original_stat(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", read) + monkeypatch.setattr(os, "stat", stat) + + class Process: + returncode = None + waits = 0 + + def __init__(self): + self.pid = pid + self.signals = [] + + def send_signal(self, sig): + self.signals.append(sig) + + def kill(self): + self.returncode = -signal.SIGKILL + + async def wait(self): + self.waits += 1 + self.returncode = -signal.SIGTERM if self.returncode is None else self.returncode + return self.returncode + + proc = Process() + monkeypatch.setattr(child.asyncio, "create_subprocess_exec", AsyncMock(return_value=proc)) + lease = SimpleNamespace(fresh=lambda: True, remaining_s=lambda: 10) + workers = child.ChildWorkers(lease) + await workers._spawn() + if fault == "parent_changed": + monkeypatch.setattr( + provider_process, "host_proof", lambda: f"host|boot|container|77|{parent + 1}|902||" + ) + return workers, proc, proof, record, workers._publish + + +@pytest.mark.asyncio +async def test_naturally_exited_owned_child_is_retired_within_restart_delay(monkeypatch): + workers, _, proof, record, publish = await _owned(monkeypatch) + clock = [time.monotonic()] + sleeps = [] + + class StopLoop(Exception): + pass + + async def record_with_elapsed_proof(_, *, ch=None): + clock[0] += 0.4 + + async def stop_after_restart_delay(delay): + sleeps.append(delay) + raise StopLoop + + record.side_effect = record_with_elapsed_proof + monkeypatch.setattr(workers, "_spawn", AsyncMock()) + monkeypatch.setattr(child, "time", SimpleNamespace(monotonic=lambda: clock[0])) + monkeypatch.setattr(child.asyncio, "sleep", stop_after_restart_delay) + try: + with pytest.raises(StopLoop): + await workers._keep_running() + assert record.await_count == 1 + assert record.await_args.args[0].host_proof == proof + assert sleeps == [pytest.approx(child.RESTART_DELAY_S - 0.4)] + assert workers.restarts == 1 + finally: + workers._close_pipe() + publish.cancel() + await asyncio.gather(publish, return_exceptions=True) From ba5579054a1e2f9ff8c82f122192f11f21252c3d Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 04:18:12 +0000 Subject: [PATCH 102/114] Worker child retirement follows the application boundary Wire exact-proof retirement from the serving composition root and expose its bounded proof timeout through settings. Keep the original child identity, query-settlement and cleanup deadlines. Signed-off-by: zlareb1 --- docs/reference/configuration.md | 1 + loom.example.yaml | 4 +++ schemas/loom-settings.schema.json | 6 +++++ server/insights/app/provider_barrier.py | 8 ++++++ server/main.py | 4 +-- server/settings/catalog.py | 3 +++ server/workers/child.py | 27 ++++++++++----------- tests/unit/snapshots/settings_defaults.json | 1 + tests/unit/test_workers_child.py | 25 ++++++++++--------- tests/unit/test_workers_child_retirement.py | 2 +- 10 files changed, 53 insertions(+), 28 deletions(-) diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 2c5ba04af8..e4b0045384 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -1756,6 +1756,7 @@ Process shape, deployment, event stream and cost budgets. | `LOOM_SUPERVISOR_LEASE_TTL_S` | `int` | `30` | | auto mode: seconds a lease holder that stops renewing keeps the workers before another process takes them over (failover time). At least 24: below that a renewal blocked for its 2 s timeout leaves no retry before the holder must give up (server/workers/leader.py has the timings). | | `LOOM_KEEPER_SWEEP_INTERVAL_SECONDS` | `float` | `3600.0` | | Seconds between passes of the always-on sweep that deletes the KeeperMap keys (leases, claims, graph slots) a stopped process left behind. | | `LOOM_WORKERS_PROCESS` | `'child' \| 'inline'` | `'child'` | | auto mode: where the lease holder runs the workers. child = a separate process it starts and stops with the lease, so the passes do not share the event loop that serves requests; inline = in the serving process. | +| `LOOM_WORKERS_CHILD_RETIREMENT_TIMEOUT_S` | `float` | `1.0` | | At most this many seconds to certify a reaped worker child; also bounded by the remaining stop or restart delay. Expiry leaves its participant unretired. | | `LOOM_SHUTDOWN_READY_DELAY_S` | `float` | `15.0` | | Seconds a process keeps serving after SIGTERM while /v1/health/ready answers 503, so the load balancer stops routing to it before its port closes. Size it to the LB's probe interval × failure threshold. 0 = stop at once. | | `LOOM_NATIVE_COMMAND_DRAIN_TIMEOUT_S` | `float` | `15.0` | | Seconds a native CLI command allows its producers, deferred writes and participant retirement to finish. Expiry leaves the participant unretired. | | `LOOM_BOOT_WARMUP_TIMEOUT_S` | `float` | `30.0` | | Seconds boot waits for the first load of the tracker registry and the runtime overrides, and the first publish of the dashboard bundle, before it serves; readiness stays false until the first two load. | diff --git a/loom.example.yaml b/loom.example.yaml index 26b894641c..d3670a9b4f 100644 --- a/loom.example.yaml +++ b/loom.example.yaml @@ -4938,6 +4938,10 @@ settings: # serves requests; inline = in the serving process. [restart] # workers_process: "child" + # At most this many seconds to certify a reaped worker child; also bounded by the + # remaining stop or restart delay. Expiry leaves its participant unretired. [restart] + # workers_child_retirement_timeout_s: 1.0 + # Seconds a process keeps serving after SIGTERM while /v1/health/ready answers 503, so # the load balancer stops routing to it before its port closes. Size it to the LB's # probe interval × failure threshold. 0 = stop at once. [restart] diff --git a/schemas/loom-settings.schema.json b/schemas/loom-settings.schema.json index 8f40e96e0d..33ec1f54d0 100644 --- a/schemas/loom-settings.schema.json +++ b/schemas/loom-settings.schema.json @@ -7078,6 +7078,12 @@ "type": "number", "description": "A pass with a failed leg on a worker whose interval is longer than this retries after this long, doubling with each further such pass, up to the interval: a weekly pass must not leave one namespace's transient failure standing for a week." }, + "workers_child_retirement_timeout_s": { + "default": 1.0, + "exclusiveMinimum": 0, + "type": "number", + "description": "At most this many seconds to certify a reaped worker child; also bounded by the remaining stop or restart delay. Expiry leaves its participant unretired." + }, "workers_process": { "default": "child", "enum": [ diff --git a/server/insights/app/provider_barrier.py b/server/insights/app/provider_barrier.py index 26009456e3..8399587f0c 100644 --- a/server/insights/app/provider_barrier.py +++ b/server/insights/app/provider_barrier.py @@ -319,6 +319,14 @@ async def wrapped(app: T) -> AsyncIterator[None]: return wrapped +async def retire_reaped_process(host_proof: str) -> None: + """Retire only the one recorded process matching its owner's captured birth.""" + rows = await participants(host_proof=host_proof) + if len(rows) != 1 or rows[0].get("host_proof") != host_proof: + return + await retire_participant(UUID(str(rows[0]["participant_id"])), expected_proof=host_proof) + + async def retire_participant( participant: UUID, *, ch: Any = None, expected_proof: str | None = None ) -> str: diff --git a/server/main.py b/server/main.py index 79f7da23ee..a522035bae 100644 --- a/server/main.py +++ b/server/main.py @@ -228,7 +228,7 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: await boot.require_storage_authority(ch) # The deployment's cell state, before the first heartbeat reports this process # serving and before the lease is decided on it. - from server.insights.app.provider_barrier import register_process + from server.insights.app.provider_barrier import register_process, retire_reaped_process await register_process() for task in await boot.start_cell_gate(): @@ -273,7 +273,7 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: # the background workers, the holder of the ClickHouse lease, in a child # process of its own unless LOOM_WORKERS_PROCESS=inline. lease = await _leader.decide_supervisor(settings.workers_mode()) - supervisor = _workers_child.worker_runner(lease) + supervisor = _workers_child.worker_runner(lease, retire_child=retire_reaped_process) _leader.set_current(lease) # Apply per-table retention overrides from LOOM_RETENTION_*_DAYS, if set — # from the deployment's leader only, and only where the TTL differs. diff --git a/server/settings/catalog.py b/server/settings/catalog.py index a9aceefc9f..b5da3d7607 100644 --- a/server/settings/catalog.py +++ b/server/settings/catalog.py @@ -5176,6 +5176,9 @@ class RuntimeSettings(BaseModel): # process it starts and stops with the lease, so the passes do not share # the event loop that serves requests; inline = in the serving process. workers_process: Literal["child", "inline"] = "child" + # At most this many seconds to certify a reaped worker child; also bounded by + # the remaining stop or restart delay. Expiry leaves its participant unretired. + workers_child_retirement_timeout_s: float = Field(default=1.0, gt=0, allow_inf_nan=False) # Seconds a process keeps serving after SIGTERM while /v1/health/ready # answers 503, so the load balancer stops routing to it before its port # closes. Size it to the LB's probe interval × failure threshold. 0 = stop at once. diff --git a/server/workers/child.py b/server/workers/child.py index 730ab2478d..6a24b8510f 100644 --- a/server/workers/child.py +++ b/server/workers/child.py @@ -27,9 +27,8 @@ import signal import sys import time -from collections.abc import Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping from pathlib import Path -from uuid import UUID from server.core.errors import error_kind from server.settings import settings @@ -45,8 +44,6 @@ # The longest a stop waits for the child before killing it, whatever the # deadline: the lifespan's shutdown drains share 40 s. STOP_TIMEOUT_S = 20.0 -# Not a setting: proof work fits within the existing stop or restart deadline. -_REAP_RETIRE_S = 1.0 RESTART_DELAY_S = 5.0 # A child that exits sooner than this after it started failed to start; after # MAX_FAILED_STARTS in a row this process gives the lease up. @@ -62,8 +59,11 @@ class ChildWorkers: """The supervisor's start/stop/status surface, backed by a child process.""" - def __init__(self, lease: leader.SupervisorLease) -> None: + def __init__( + self, lease: leader.SupervisorLease, *, retire_child: Callable[[str], Awaitable[None]] + ) -> None: self._lease = lease + self._retire_callback = retire_child self._proc: asyncio.subprocess.Process | None = None self._child_proof: str | None = None self._child_parent_proof: str | None = None @@ -209,8 +209,6 @@ def _capture_child_proof(self) -> None: async def _retire_child(self, proc: asyncio.subprocess.Process, end: float) -> None: from server.core import provider_process - from server.insights.app.provider_barrier import retire_participant - from server.insights.store.provider_participants import participants proof = self._child_proof if proc is not self._proc or proc.returncode is None or proof is None: @@ -230,11 +228,10 @@ async def _retire_child(self, proc: asyncio.subprocess.Process, end: float) -> N ): return try: - async with asyncio.timeout(min(_REAP_RETIRE_S, max(0.0, end - time.monotonic()))): - rows = await participants(host_proof=proof) - if len(rows) != 1 or rows[0].get("host_proof") != proof: - return - await retire_participant(UUID(str(rows[0]["participant_id"])), expected_proof=proof) + async with asyncio.timeout( + min(settings.workers_child_retirement_timeout_s, max(0.0, end - time.monotonic())) + ): + await self._retire_callback(proof) except Exception: # noqa: BLE001 — unproved retirement must not replace child cleanup log.warning("reaped workers child retirement remains unproved") @@ -330,12 +327,14 @@ def child_env(parent: Mapping[str, str]) -> dict[str, str]: _RUNNER: WorkerSupervisor | ChildWorkers | None = None -def worker_runner(lease: leader.SupervisorLease) -> WorkerSupervisor | ChildWorkers: +def worker_runner( + lease: leader.SupervisorLease, *, retire_child: Callable[[str], Awaitable[None]] +) -> WorkerSupervisor | ChildWorkers: """What runs the workers here: a child process for an `auto` lease holder (`LOOM_WORKERS_PROCESS=child`), else the in-process supervisor.""" global _RUNNER if lease.mode == "auto" and settings.workers_process == "child": - _RUNNER = ChildWorkers(lease) + _RUNNER = ChildWorkers(lease, retire_child=retire_child) else: _RUNNER = get_supervisor() return _RUNNER diff --git a/tests/unit/snapshots/settings_defaults.json b/tests/unit/snapshots/settings_defaults.json index 804775722c..fd7142a9bf 100644 --- a/tests/unit/snapshots/settings_defaults.json +++ b/tests/unit/snapshots/settings_defaults.json @@ -1258,6 +1258,7 @@ "worker_backoff_cap_s": 60.0, "worker_ledger_rows": true, "worker_partial_retry_s": 3600.0, + "workers_child_retirement_timeout_s": 1.0, "workers_process": "child", "workload_exchanges_per_agent": 300, "workload_exchanges_per_trust": 50000 diff --git a/tests/unit/test_workers_child.py b/tests/unit/test_workers_child.py index 1f3c1854a6..57269fd3b2 100644 --- a/tests/unit/test_workers_child.py +++ b/tests/unit/test_workers_child.py @@ -12,6 +12,7 @@ import threading import time from types import SimpleNamespace +from unittest.mock import AsyncMock import pytest @@ -83,7 +84,7 @@ async def test_parent_sends_the_deadline_and_stops_the_child(monkeypatch, tmp_pa monkeypatch.setattr( child, "child_argv", lambda fd, _alive_fd: [sys.executable, "-c", _STUB, str(fd), str(out)] ) - workers = child.ChildWorkers(_Lease()) + workers = child.ChildWorkers(_Lease(), retire_child=AsyncMock()) workers.start_all() await _until(lambda: out.exists() and bool(out.read_text())) deadline = float(out.read_text()) @@ -96,7 +97,7 @@ async def test_parent_sends_the_deadline_and_stops_the_child(monkeypatch, tmp_pa def test_the_deadline_stops_moving_once_the_lease_is_not_fresh() -> None: lease = _Lease(remaining=12.0) - workers = child.ChildWorkers(lease) + workers = child.ChildWorkers(lease, retire_child=AsyncMock()) workers._send_deadline() sent = workers._deadline lease.remaining = 0.0 # lost @@ -111,7 +112,7 @@ async def test_a_child_that_ignores_sigterm_is_killed_at_the_deadline_plus_the_d ) -> None: ready = _stubborn(monkeypatch, tmp_path) lease = _Lease(remaining=0.3) - workers = child.ChildWorkers(lease) + workers = child.ChildWorkers(lease, retire_child=AsyncMock()) workers.start_all() await _until(ready.exists) lease.remaining = 0.0 # the lease is lost: the deadline stays where it was @@ -124,7 +125,7 @@ async def test_a_child_that_ignores_sigterm_is_killed_at_the_deadline_plus_the_d @pytest.mark.asyncio async def test_a_cancelled_stop_kills_the_child(monkeypatch, tmp_path) -> None: ready = _stubborn(monkeypatch, tmp_path) - workers = child.ChildWorkers(_Lease(remaining=30.0)) + workers = child.ChildWorkers(_Lease(remaining=30.0), retire_child=AsyncMock()) workers.start_all() await _until(ready.exists) proc = workers._proc @@ -139,7 +140,7 @@ async def test_a_cancelled_stop_kills_the_child(monkeypatch, tmp_path) -> None: async def test_a_child_that_exits_unasked_is_started_again(monkeypatch) -> None: monkeypatch.setattr(child, "RESTART_DELAY_S", 0.01) monkeypatch.setattr(child, "child_argv", lambda fd, _alive_fd: [sys.executable, "-c", "pass"]) - workers = child.ChildWorkers(_Lease()) + workers = child.ChildWorkers(_Lease(), retire_child=AsyncMock()) workers.start_all() await _until(lambda: workers.restarts >= 2) await workers.stop_all() @@ -153,7 +154,7 @@ async def test_a_child_that_keeps_failing_to_start_gives_the_lease_up(monkeypatc child, "child_argv", lambda fd, _alive_fd: [sys.executable, "-c", "exit(3)"] ) lease = _Lease() - workers = child.ChildWorkers(lease) + workers = child.ChildWorkers(lease, retire_child=AsyncMock()) workers.start_all() await _until(lambda: bool(lease.given_up)) assert workers.failed_starts == 3 and workers.restarts == 2 @@ -173,7 +174,7 @@ async def test_a_child_that_cannot_be_created_gives_the_lease_up_and_leaks_no_fd monkeypatch.setattr(child, "child_argv", lambda fd, _alive_fd: [missing]) fds = _open_fds() lease = _Lease() - workers = child.ChildWorkers(lease) + workers = child.ChildWorkers(lease, retire_child=AsyncMock()) workers.start_all() await _until(lambda: bool(lease.given_up)) assert workers.failed_starts == 3 and "could not start" in lease.given_up[0] @@ -204,7 +205,7 @@ async def test_a_child_whose_loop_hangs_is_not_alive(monkeypatch) -> None: monkeypatch.setattr( child, "child_argv", lambda _fd, alive_fd: [sys.executable, "-c", _HANGS, str(alive_fd)] ) - workers = child.ChildWorkers(_Lease()) + workers = child.ChildWorkers(_Lease(), retire_child=AsyncMock()) workers.start_all() await _until(lambda: workers._proc is not None) started = workers._beat_at @@ -304,12 +305,14 @@ def test_an_auto_holder_runs_its_workers_in_a_child(monkeypatch) -> None: lease = SupervisorLease("auto") monkeypatch.setattr(settings, "workers_process", "child") - assert isinstance(child.worker_runner(lease), child.ChildWorkers) + assert isinstance(child.worker_runner(lease, retire_child=AsyncMock()), child.ChildWorkers) assert child.current_runner() is child._RUNNER monkeypatch.setattr(settings, "workers_process", "inline") - assert isinstance(child.worker_runner(lease), WorkerSupervisor) + assert isinstance(child.worker_runner(lease, retire_child=AsyncMock()), WorkerSupervisor) monkeypatch.setattr(settings, "workers_process", "child") - assert isinstance(child.worker_runner(SupervisorLease("on")), WorkerSupervisor) + assert isinstance( + child.worker_runner(SupervisorLease("on"), retire_child=AsyncMock()), WorkerSupervisor + ) def test_the_child_runs_its_own_entry_point_not_the_serving_app() -> None: diff --git a/tests/unit/test_workers_child_retirement.py b/tests/unit/test_workers_child_retirement.py index 08fc6c8bb3..d41972c7dd 100644 --- a/tests/unit/test_workers_child_retirement.py +++ b/tests/unit/test_workers_child_retirement.py @@ -191,7 +191,7 @@ async def wait(self): proc = Process() monkeypatch.setattr(child.asyncio, "create_subprocess_exec", AsyncMock(return_value=proc)) lease = SimpleNamespace(fresh=lambda: True, remaining_s=lambda: 10) - workers = child.ChildWorkers(lease) + workers = child.ChildWorkers(lease, retire_child=barrier.retire_reaped_process) await workers._spawn() if fault == "parent_changed": monkeypatch.setattr( From 23bd673479d4db9525e682d0df56374c7d24815d Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 04:45:57 +0000 Subject: [PATCH 103/114] Verify captured worker subtrees in native recovery checks Capture exact background-child births before controlled crash and recreation. Keep certified retirement and settled queries mandatory for every captured participant, and report bounded role counts on failure. Signed-off-by: zlareb1 --- scripts/gates/docker_provider_data.py | 126 +++++++++++- .../test_docker_provider_owned_subtree.py | 187 ++++++++++++++++++ tests/unit/test_docker_provider_smoke.py | 3 +- 3 files changed, 306 insertions(+), 10 deletions(-) create mode 100644 tests/unit/test_docker_provider_owned_subtree.py diff --git a/scripts/gates/docker_provider_data.py b/scripts/gates/docker_provider_data.py index 7f8611f850..059e40e3d8 100644 --- a/scripts/gates/docker_provider_data.py +++ b/scripts/gates/docker_provider_data.py @@ -9,12 +9,17 @@ import time from multiprocessing import spawn from pathlib import Path +from typing import TYPE_CHECKING, Any from uuid import UUID from server.core import provider_process from server.core.ch import get_client from server.insights.app.provider_command import model_command from server.settings import settings +from server.workers.child import child_argv + +if TYPE_CHECKING: + from server.core.repo_base import _CHClient SUPERVISOR = [ "-m", @@ -149,21 +154,93 @@ def _workers(rows: list[dict], parent: dict) -> list[dict]: return sorted(workers, key=lambda row: row["host_proof"]) -async def _worker_rows(client) -> list[dict]: - return await client.query_all( +def _proof(process: dict[str, Any]) -> str: + parts = provider_process.host_proof().split("|") + assert len(parts) == 8 and all(parts[:4]), "Incomplete fixture process scope" + parts[4:6] = [str(process["pid"]), process["start"]] + return provider_process.certified("|".join(parts)) + + +async def _proof_rows(client: "_CHClient", proofs: list[str]) -> list[dict[str, Any]]: + assert 1 <= len(proofs) <= 4 and len(set(proofs)) == len(proofs) + rows = await client.query_all( "SELECT toString(participant_id) AS participant_id, state, version, host_proof " - "FROM loom._provider_participants FINAL WHERE state != 'retired'" + "FROM loom._provider_participants FINAL WHERE state != 'retired' " + "AND host_proof IN {proofs:Array(String)} LIMIT 5", + {"proofs": proofs}, ) + assert len(rows) <= len(proofs), "Duplicate captured process registrations" + return rows + + +async def _worker_rows(client) -> list[dict]: + return await _proof_rows(client, [_proof(p) for p in _kernel_workers().values()]) + + +def _background_children( + roots: list[dict[str, Any]], parent: dict[str, Any] +) -> dict[str, list[dict[str, Any]]]: + found = {} + for root in roots: + web = _worker_process(root["host_proof"], parent) + children = [] + for value in Path(f"/proc/{web['pid']}/task/{web['pid']}/children").read_text().split(): + process = _process(int(value)) + command = process["command"] + if "server.worker_main" not in command: + continue + assert process["parent"] == web["pid"], "Background child parent changed" + assert process["namespace"] == web["namespace"], "Background child namespace changed" + assert process["executable"] == web["executable"], "Background child executable changed" + assert len(command) == 7, "Background child command mismatch" + fds = [int(command[4]), int(command[6])] + assert min(fds) >= 0 and command == child_argv(*fds), ( + "Background child command mismatch" + ) + children.append(process) + assert len(children) <= 1, "More than one background child per web worker" + assert _worker_process(root["host_proof"], parent) == web, ( + "Web worker changed during child census" + ) + found[root["participant_id"]] = children + return found async def workers(client) -> list[dict]: parent = _supervisor() result = _workers(await _worker_rows(client), parent) assert len(result) == 2, "Worker registration census is incomplete" + children = _background_children(result, parent) + proofs = [_proof(p) for values in children.values() for p in values] + rows = await _proof_rows(client, proofs) if proofs else [] + for root in result: + root["children"] = [] + for process in children[root["participant_id"]]: + matches = [row for row in rows if row["host_proof"] == _proof(process)] + if not matches: + raise _WorkersPending("Background child exact registration is pending") + assert len(matches) == 1, "Background child registration is ambiguous" + row = matches[0] + assert provider_process.has_certificate(row["host_proof"]), ( + "Background child certificate invalid" + ) + root["children"].append( + {key: row[key] for key in ("participant_id", "host_proof", "version")} + ) + assert _background_children(result, parent) == children, ( + "Background child identities changed during census" + ) + assert len({row["participant_id"] for row in _captured(result)}) == 2 + len(proofs) assert _supervisor() == parent, "Fixture supervisor changed during registration census" return result +def _captured(roots: list[dict[str, Any]]) -> list[dict[str, Any]]: + assert 1 <= len(roots) <= 2 + assert all(type(root["children"]) is list and len(root["children"]) <= 1 for root in roots) + return [row for root in roots for row in (root, *root["children"])] + + def _kill_worker(worker: dict, parent: dict) -> None: proof = worker["host_proof"].split("|") pid = int(proof[4]) @@ -180,9 +257,10 @@ def _kill_worker(worker: dict, parent: dict) -> None: async def _retirement_rows(client, captured: list[dict]) -> list[dict]: + assert 1 <= len(captured) <= 4 rows = await client.query_all( "SELECT participant_id, state, version, host_proof " - "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)}", + "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)} LIMIT 5", {"ids": [row["participant_id"] for row in captured]}, ) return [{**row, "participant_id": str(row["participant_id"])} for row in rows] @@ -232,11 +310,38 @@ def result(reason: str) -> bool: return result("proved") +async def _subtree_diagnostic( + client: "_CHClient", roots: list[dict[str, Any]], *, proved: bool +) -> None: + """Print closed role counts, preserving the original retirement failure.""" + try: + roles = {"web": roots, "background": [c for r in roots for c in r["children"]]} + summary = {} + for role, captured in roles.items(): + counts = { + "captured": len(captured), + "retirement_proved": 0, + "death_proved": 0, + "query_zero_proved": 0, + "unproved": 0, + } + for row in captured: + counts["death_proved"] += int(provider_process.proves_stopped(row["host_proof"])) + accepted = proved or (bool(roles["background"]) and await retired(client, [row])) + counts["retirement_proved"] += int(accepted) + counts["query_zero_proved"] += int(accepted) + counts["unproved"] += int(not accepted) + summary[role] = counts + print("Provider subtree retirement diagnostic: " + json.dumps(summary, sort_keys=True)) + except BaseException: # noqa: BLE001 — diagnostics must preserve the original retirement failure + print("Provider subtree retirement diagnostic unavailable") + + async def crash_worker(client) -> None: parent = _supervisor() before = await workers(client) victim = before[0] - for worker in before: + for worker in _captured(before): rows = await _retirement_rows(client, [worker]) identity_matches = len(rows) == 1 and rows[0]["participant_id"] == worker["participant_id"] if not identity_matches: @@ -255,7 +360,7 @@ async def crash_worker(client) -> None: assert _supervisor() == parent stopped = provider_process.proves_stopped(victim["host_proof"]) observation = {"death": "proved" if stopped else "unproved", "retirement": "not_checked"} - if stopped and await retired(client, [victim], observation=observation): + if stopped and await retired(client, _captured([victim]), observation=observation): try: after = _workers(await _worker_rows(client), parent) except _WorkersPending: @@ -270,10 +375,12 @@ async def crash_worker(client) -> None: victim_absent=victim["participant_id"] not in new, ) if len(new) == 2 and len(old & new) == 1 and victim["participant_id"] not in new: + await _subtree_diagnostic(client, [victim], proved=True) return await asyncio.sleep(1) if observation["retirement"] == "row_count": observation.update(await _native_retirement_observation(client, [victim])) + await _subtree_diagnostic(client, [victim], proved=False) raise AssertionError( "Crashed worker retirement or supervised replacement was not proved: " + json.dumps(observation, sort_keys=True) @@ -418,9 +525,10 @@ async def main() -> None: elif sys.argv[1] == "crash-worker": await crash_worker(client) elif sys.argv[1] == "retired": - assert await retired(client, json.loads(sys.argv[2])), ( - "Recreated workers did not retire" - ) + captured = json.loads(sys.argv[2]) + accepted = await retired(client, _captured(captured)) + await _subtree_diagnostic(client, captured, proved=accepted) + assert accepted, "Recreated workers did not retire" else: await verify(sys.argv[1], int(sys.argv[2])) finally: diff --git a/tests/unit/test_docker_provider_owned_subtree.py b/tests/unit/test_docker_provider_owned_subtree.py new file mode 100644 index 0000000000..1eefcbe195 --- /dev/null +++ b/tests/unit/test_docker_provider_owned_subtree.py @@ -0,0 +1,187 @@ +"""The native fixture captures and checks its exact controlled process subtree.""" + +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +import scripts.gates.docker_provider_data as data +from server.insights.store import provider_participants +from server.workers.child import child_argv +from tests.unit.test_docker_provider_smoke import _process_fixture + + +def _fixture(monkeypatch, *, background=True, fault=None): + parent, processes, roots = _process_fixture(monkeypatch) + children = {1: "8 9", 8: "18" if background else "", 9: ""} + descendant = { + **processes[8], + "pid": 18, + "parent": 8, + "start": "180", + "command": child_argv(4, 8), + } + processes[18] = descendant + child = { + "participant_id": "background-participant", + "version": 5, + "state": "running", + "host_proof": "h|b|c|1|18|180|||cell-retirement-v1", + } + registrations = [*roots, *([child] if background else [])] + if fault == "root-reused": + processes[8] = {**processes[8], "start": "81"} + elif fault == "root-parent": + processes[8] = {**processes[8], "parent": 2} + elif fault == "child-reused": + processes[18] = {**descendant, "start": "181"} + elif fault == "child-parent": + processes[18] = {**descendant, "parent": 9} + elif fault == "child-namespace": + processes[18] = {**descendant, "namespace": 2} + elif fault == "child-command": + processes[18] = {**descendant, "command": [*descendant["command"], "--unknown"]} + elif fault == "two-background": + processes[19] = {**descendant, "pid": 19, "start": "190"} + children[8] = "18 19" + elif fault == "missing-child": + registrations = roots + elif fault == "duplicate-child": + registrations.append({**child, "participant_id": "ambiguous-background"}) + + def read(path): + parts = str(path).split("/") + assert parts[1] == "proc" and parts[3] == "task" and parts[2] == parts[4] + assert parts[5] == "children" + return children[int(parts[2])] + + monkeypatch.setattr(data, "Path", lambda path: SimpleNamespace(read_text=lambda: read(path))) + monkeypatch.setattr(data, "_supervisor", lambda: parent) + + async def query(sql, parameters=None): + selected = registrations + if parameters and "proofs" in parameters: + selected = [r for r in selected if r["host_proof"] in parameters["proofs"]] + if ( + fault == "child-changed-after-query" + and parameters + and child["host_proof"] in parameters.get("proofs", []) + ): + processes[18] = {**descendant, "start": "181"} + return selected + + client = SimpleNamespace(query_all=AsyncMock(side_effect=query)) + return roots, child, client + + +@pytest.mark.asyncio +async def test_actual_workers_capture_the_exact_background_child(monkeypatch): + roots, child, client = _fixture(monkeypatch) + captured = await data.workers(client) + assert [r["participant_id"] for r in captured] == [r["participant_id"] for r in roots] + assert [c["participant_id"] for r in captured for c in r.get("children", [])] == [ + child["participant_id"] + ] + assert len(captured) == 2 + + +@pytest.mark.asyncio +async def test_actual_workers_keep_the_same_two_web_roots_without_a_background_child(monkeypatch): + roots, _, client = _fixture(monkeypatch, background=False) + captured = await data.workers(client) + assert [r["participant_id"] for r in captured] == [r["participant_id"] for r in roots] + assert not any(r.get("children", []) for r in captured) + assert len(captured) == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fault", ["root-reused", "root-parent"]) +async def test_actual_workers_refuse_changed_web_identity(monkeypatch, fault): + _, _, client = _fixture(monkeypatch, fault=fault) + with pytest.raises(AssertionError): + await data.workers(client) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", + [ + "child-reused", + "child-parent", + "child-namespace", + "child-command", + "two-background", + "missing-child", + "duplicate-child", + "child-changed-after-query", + ], +) +async def test_actual_workers_refuse_uncertain_background_identity(monkeypatch, fault): + _, _, client = _fixture(monkeypatch, fault=fault) + with pytest.raises(AssertionError): + await data.workers(client) + + +@pytest.mark.asyncio +async def test_actual_recreated_check_refuses_an_unretired_background_child(monkeypatch): + roots, child, _ = _fixture(monkeypatch) + captured = [{**r, "children": [child] if i == 0 else []} for i, r in enumerate(roots)] + observed = [{**r, "state": "retired", "version": 6} for r in roots] + [ + {**child, "state": "drained", "version": 6} + ] + + async def query(_sql, parameters): + return [r for r in observed if r["participant_id"] in parameters["ids"]] + + client = SimpleNamespace( + connect=AsyncMock(), disconnect=AsyncMock(), query_all=AsyncMock(side_effect=query) + ) + monkeypatch.setattr(data, "get_client", lambda: client) + monkeypatch.setattr(data.sys, "argv", ["fixture", "retired", json.dumps(captured)]) + monkeypatch.setattr(provider_participants, "running_queries", AsyncMock(return_value=0)) + monkeypatch.setattr(data.provider_process, "proves_stopped", lambda _: False) + with pytest.raises(AssertionError, match="Recreated workers did not retire"): + await data.main() + client.disconnect.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_actual_recreated_check_reports_only_closed_role_proof_counts(monkeypatch, capsys): + roots, child, _ = _fixture(monkeypatch) + captured = [{**r, "children": [child] if i == 0 else []} for i, r in enumerate(roots)] + observed = [{**r, "state": "retired", "version": 6} for r in [*roots, child]] + + async def query(_sql, parameters): + return [r for r in observed if r["participant_id"] in parameters["ids"]] + + client = SimpleNamespace( + connect=AsyncMock(), disconnect=AsyncMock(), query_all=AsyncMock(side_effect=query) + ) + monkeypatch.setattr(data, "get_client", lambda: client) + monkeypatch.setattr(data.sys, "argv", ["fixture", "retired", json.dumps(captured)]) + monkeypatch.setattr(provider_participants, "running_queries", AsyncMock(return_value=0)) + monkeypatch.setattr(data.provider_process, "proves_stopped", lambda _: False) + await data.main() + prefix = "Provider subtree retirement diagnostic: " + output = capsys.readouterr().out + assert output.startswith(prefix) + summary = json.loads(output.removeprefix(prefix)) + assert summary == { + "web": { + "captured": 2, + "retirement_proved": 2, + "death_proved": 0, + "query_zero_proved": 2, + "unproved": 0, + }, + "background": { + "captured": 1, + "retirement_proved": 1, + "death_proved": 0, + "query_zero_proved": 1, + "unproved": 0, + }, + } + assert all(row["host_proof"] not in output for row in [*roots, child]) + assert child["participant_id"] not in output diff --git a/tests/unit/test_docker_provider_smoke.py b/tests/unit/test_docker_provider_smoke.py index 64119f0817..a7c59cf446 100644 --- a/tests/unit/test_docker_provider_smoke.py +++ b/tests/unit/test_docker_provider_smoke.py @@ -253,6 +253,7 @@ def _process_fixture(monkeypatch): rows = [ { "participant_id": str(pid), + "children": [], "version": 5, "state": "running", "host_proof": f"h|b|c|1|{pid}|{pid * 10}|||cell-retirement-v1", @@ -537,7 +538,7 @@ async def test_retirement_reader_preserves_string_identity_from_native_uuid(): assert row["participant_id"] is participant_id assert client.query_all.await_args.args == ( "SELECT participant_id, state, version, host_proof " - "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)}", + "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)} LIMIT 5", {"ids": [str(participant_id)]}, ) From 74d59a42e8b762441937e9c326fea71e4a889b48 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 05:05:08 +0000 Subject: [PATCH 104/114] Test child retirement limits and clarify native token minting Cover finite positive retirement budgets and the original remaining allowance. Distinguish an already issued scoped token from native minting after enforcement. Signed-off-by: zlareb1 --- docs/operations/cell-attach.md | 7 ++-- tests/unit/test_workers_child_retirement.py | 40 +++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index 53e176d1cd..451de55f3e 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -115,9 +115,10 @@ and `attaching`, and the cell goes forward: fix the cause and rerun attach. Unse `LOOM_CELL` then gates every process, an outage, not a rollback. A row with `placed > 0` always goes forward. -From `enforced` on, a `--platform-admin` mint that declares a new org and the admin account -routes cannot bring in a new org (`WRONG_CELL`); an ordinary mint scoped to its org places -the org on its first request. A direct call of the router's operator place route, or a live +From `enforced` on, native minting requires the org to be held by this cell, with or +without `--platform-admin`. Admin account routes cannot bring in a new org (`WRONG_CELL`). +An already issued ordinary token scoped to its org places that org on its first request. +A direct call of the router's operator place route, or a live placement with the cell's credential, cannot read the cell's Postgres and is accepted once the directory mode is `enforced`; the cell's credential is the servers' alone. diff --git a/tests/unit/test_workers_child_retirement.py b/tests/unit/test_workers_child_retirement.py index d41972c7dd..bfa1829226 100644 --- a/tests/unit/test_workers_child_retirement.py +++ b/tests/unit/test_workers_child_retirement.py @@ -14,7 +14,9 @@ from server.core import provider_process from server.insights.app import provider_barrier as barrier from server.insights.store import provider_participants as store +from server.settings import Settings from server.workers import child +from tests.unit.clock_fake import ManualClock @pytest.mark.asyncio @@ -231,3 +233,41 @@ async def stop_after_restart_delay(delay): workers._close_pipe() publish.cancel() await asyncio.gather(publish, return_exceptions=True) + + +@pytest.mark.parametrize("value", [0.0, -1.0, float("nan"), float("inf"), -float("inf")]) +def test_child_retirement_proof_timeout_requires_a_positive_finite_setting(value): + with pytest.raises(ValueError): + Settings(_env_file=None, workers_child_retirement_timeout_s=value) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("configured", "remaining", "expected"), + [(0.125, 2.0, 0.125), (1.0, 0.25, 0.25), (2.0, 0.0625, 0.0625)], +) +async def test_child_retirement_setting_caps_only_the_original_remaining_allowance( + monkeypatch, configured, remaining, expected +): + workers, proc, _, record, publish = await _owned(monkeypatch) + clock = ManualClock(t=100.0) + allowances = [] + original_timeout = asyncio.timeout + + def observed_timeout(delay): + allowances.append(delay) + return original_timeout(delay) + + monkeypatch.setattr(child.settings, "workers_child_retirement_timeout_s", configured) + monkeypatch.setattr(child, "time", clock) + monkeypatch.setattr(child.asyncio, "timeout", observed_timeout) + try: + await workers._reap(clock.monotonic() + remaining) + assert proc.returncode == -signal.SIGTERM and proc.waits >= 1 + assert record.await_count == 1 + assert allowances == [expected] + assert clock.monotonic() == 100.0 + finally: + workers._close_pipe() + publish.cancel() + await asyncio.gather(publish, return_exceptions=True) From 9d6d5861f1554917a1b3628427da4025b3ead016 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 05:29:24 +0000 Subject: [PATCH 105/114] Prove exact exit of single-thread zombie participants Signed-off-by: zlareb1 --- docs/architecture/cells.md | 7 ++ scripts/gates/docker_provider_data.py | 27 +++++ server/core/provider_process.py | 16 ++- tests/unit/test_provider_host_proof.py | 62 ++++++++++ tests/unit/test_provider_zombie_exit.py | 152 ++++++++++++++++++++++++ 5 files changed, 259 insertions(+), 5 deletions(-) create mode 100644 tests/unit/test_provider_zombie_exit.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 7038121a79..45b319c1d8 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -36,6 +36,13 @@ setup; its optional analyst-row work opens a separate verified, enrolled scope and obtains fresh acceptance. That bootstrap exception admits no tenant or token work. +Local physical-death evidence requires the same host, kernel boot and PID namespace. +It accepts an absent PID, a different process birth, or the captured birth in a +single fresh kernel process record showing a zombie with exactly one thread. A +zombie leader with other threads is unproved. Known-zero tagged queries remain a +separate requirement before retirement; kernel process exit does not settle a +query still executing in a store. + The capability identifies installed code; it does not prove that every older process or unsafe retirement client has disappeared. diff --git a/scripts/gates/docker_provider_data.py b/scripts/gates/docker_provider_data.py index 059e40e3d8..175b4eac3c 100644 --- a/scripts/gates/docker_provider_data.py +++ b/scripts/gates/docker_provider_data.py @@ -324,8 +324,35 @@ async def _subtree_diagnostic( "death_proved": 0, "query_zero_proved": 0, "unproved": 0, + "kernel_zombie_single_thread": 0, + "kernel_zombie_multi_thread": 0, + "kernel_present_non_zombie": 0, + "kernel_missing": 0, + "kernel_uninspectable": 0, } for row in captured: + kernel = "kernel_uninspectable" + proof = row["host_proof"].split("|") + here = provider_process.host_proof().split("|") + if len(proof) in {8, 9} and proof[:4] == here[:4] and proof[4].isdecimal(): + try: + raw = await asyncio.to_thread(Path(f"/proc/{int(proof[4])}/stat").read_text) + stat = raw.rsplit(")", 1)[1].split() + if stat[19] == proof[5]: + threads = int(stat[17]) + if stat[0] == "Z" and threads >= 1: + kernel = ( + "kernel_zombie_single_thread" + if threads == 1 + else "kernel_zombie_multi_thread" + ) + elif threads >= 1: + kernel = "kernel_present_non_zombie" + except FileNotFoundError: + kernel = "kernel_missing" + except (OSError, IndexError, ValueError): + pass # An unreadable identity does not establish a kernel state. + counts[kernel] += 1 counts["death_proved"] += int(provider_process.proves_stopped(row["host_proof"])) accepted = proved or (bool(roles["background"]) and await retired(client, [row])) counts["retirement_proved"] += int(accepted) diff --git a/server/core/provider_process.py b/server/core/provider_process.py index a2c6ad6328..aa88ffe6fc 100644 --- a/server/core/provider_process.py +++ b/server/core/provider_process.py @@ -96,8 +96,8 @@ def host_proof() -> str: def proves_stopped(proof: str) -> bool: """Whether the process `proof` names has stopped, judged from this one: same host - name and kernel boot, and either its pid is free or runs another process than - the one that started then (same pid namespace). Another namespace or API + name and kernel boot, and its pid is free, runs a different birth, or is a + single-thread zombie of that birth (same pid namespace). Another namespace or API object proves nothing. Sharing a host name or container identity alone does not prove death; an uninspectable process is not proven stopped.""" parts = proof.split("|") @@ -116,7 +116,7 @@ def proves_stopped(proof: str) -> bool: return False if pidns != my_pidns: return False # namespace identity alone says nothing about old processes - # Only positive evidence: no such process, or another one under its pid. + # Only positive evidence: absence, a different birth, or a fully exited group. # A pid this process may not inspect (hidepid, a permission error) proves nothing. try: os.kill(int(pid), 0) @@ -125,8 +125,14 @@ def proves_stopped(proof: str) -> bool: except (OSError, ValueError): return False try: - return _start_ticks(pid) != start - except (OSError, IndexError): + if _start_ticks(pid) != start: + return True + with open(f"/proc/{pid}/stat", encoding="ascii") as f: + stat = f.read().rsplit(")", 1)[1].split() + # A zombie leader can still have live siblings. Bind all three fields to + # one fresh record: only a single-thread zombie cannot resume this birth. + return stat[19] == start and stat[0] == "Z" and stat[17] == "1" + except (OSError, IndexError, ValueError): return False diff --git a/tests/unit/test_provider_host_proof.py b/tests/unit/test_provider_host_proof.py index 0e70602dd7..295f8dc03f 100644 --- a/tests/unit/test_provider_host_proof.py +++ b/tests/unit/test_provider_host_proof.py @@ -160,3 +160,65 @@ async def test_this_host_retires_its_dead_participants_once_their_queries_ended( assert proof.participant == dead and proof.after == 5 assert proof.host_proof == pp.certified(gone) assert await barrier.retire_dead_here(now=100.0 + barrier.HERE_S / 2) == 0 # throttled + + +def _fresh_process_stat( + monkeypatch: pytest.MonkeyPatch, text: str, error: type[Exception] | None = None +) -> str: + """Hold the first birth check steady and supply the subsequent kernel record.""" + from io import StringIO + + monkeypatch.setattr(pp.socket, "gethostname", lambda: "host") + monkeypatch.setattr(pp, "_boot_and_pidns", lambda: ("boot", "77")) + monkeypatch.setattr(pp.os, "kill", lambda pid, sig: None) + monkeypatch.setattr(pp, "_start_ticks", lambda pid: "900") + + def fresh_open(path: str, *args: Any, **kwargs: Any) -> StringIO: + assert path == "/proc/424242/stat" + if error is not None: + raise error("uninspectable fresh process record") + return StringIO(text) + + monkeypatch.setattr(pp, "open", fresh_open, raising=False) + return "host|boot|container|77|424242|900||" + + +def _process_stat_record(state: str = "Z", threads: str = "1", start: str = "900") -> str: + fields = ["0"] * 20 + fields[0], fields[17], fields[19] = state, threads, start + return "424242 (owned child) " + " ".join(fields) + + +def test_an_exact_single_thread_zombie_record_proves_exit(monkeypatch: pytest.MonkeyPatch) -> None: + proof = _fresh_process_stat(monkeypatch, _process_stat_record()) + assert pp.proves_stopped(proof) + + +@pytest.mark.parametrize( + "record", + [ + _process_stat_record(threads="0"), + _process_stat_record(threads="2"), + _process_stat_record(threads="-1"), + _process_stat_record(threads="unknown"), + _process_stat_record(threads="1.0"), + *[_process_stat_record(state=state) for state in ("S", "D", "R", "T", "t", "X")], + _process_stat_record(start="901"), + _process_stat_record(start="unknown"), + "424242 (owned child) Z", + "unparseable kernel record", + ], +) +def test_fresh_kernel_record_must_prove_the_same_single_thread_exit( + monkeypatch: pytest.MonkeyPatch, record: str +) -> None: + proof = _fresh_process_stat(monkeypatch, record) + assert not pp.proves_stopped(proof) + + +@pytest.mark.parametrize("error", [PermissionError, OSError, IndexError]) +def test_unreadable_fresh_kernel_record_never_proves_exit( + monkeypatch: pytest.MonkeyPatch, error: type[Exception] +) -> None: + proof = _fresh_process_stat(monkeypatch, _process_stat_record(), error) + assert not pp.proves_stopped(proof) diff --git a/tests/unit/test_provider_zombie_exit.py b/tests/unit/test_provider_zombie_exit.py new file mode 100644 index 0000000000..501f3f56a1 --- /dev/null +++ b/tests/unit/test_provider_zombie_exit.py @@ -0,0 +1,152 @@ +"""Exact single-thread process exit is provable before its parent collects it.""" + +import ctypes +import os +import select +import signal +import threading +import time +from pathlib import Path + +import pytest + +from server.core import provider_process + +pytestmark = pytest.mark.skipif(not provider_process.host_proof(), reason="needs /proc") + + +def _stat(pid: int) -> tuple[str, int, str, int]: + parts = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split() + return parts[0], int(parts[1]), parts[19], int(parts[17]) + + +def _wait_owned(pid: int, start: str) -> int: + """Collect only this fork's child, within a finite cleanup allowance.""" + for force in (False, True): + end = time.monotonic() + 2.0 + if force: + try: + _, parent, now_start, _ = _stat(pid) + except FileNotFoundError: + parent, now_start = None, None + if parent == os.getpid() and now_start == start: + os.kill(pid, signal.SIGKILL) + while time.monotonic() < end: + found, status = os.waitpid(pid, os.WNOHANG) + if found == pid: + return status + time.sleep(0.005) + raise AssertionError("Owned witness child did not settle within cleanup allowance") + + +def test_an_exact_single_thread_zombie_is_proven_stopped_before_wait() -> None: + """A captured child has exited, even while its parent retains the wait status.""" + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: + os.close(write_fd) + os.read(read_fd, 1) + os._exit(0) + os.close(read_fd) + start = None + released = False + try: + state, parent, start, threads = _stat(pid) + assert state != "Z" and parent == os.getpid() and threads == 1 + proof = provider_process.host_proof().split("|") + assert len(proof) == 8 and all(proof[i] for i in (0, 1, 3)) + proof[4:6] = [str(pid), start] + exact = "|".join(proof) + assert not provider_process.proves_stopped(exact), "Live child must refuse" + wrong_namespace = proof.copy() + wrong_namespace[3] = str(int(proof[3]) + 1) + assert not provider_process.proves_stopped("|".join(wrong_namespace)) + os.write(write_fd, b"x") + released = True + end = time.monotonic() + 2.0 + while time.monotonic() < end: + state, parent, observed_start, threads = _stat(pid) + assert parent == os.getpid() and observed_start == start + if state == "Z": + break + time.sleep(0.005) + assert state == "Z" and threads == 1, ( + "Controlled child must reach single-thread Z before any waitpid" + ) + assert provider_process.proves_stopped(exact) + finally: + if not released: + os.write(write_fd, b"x") + os.close(write_fd) + if start is None: + _, parent, start, _ = _stat(pid) + assert parent == os.getpid() + status = _wait_owned(pid, start) + assert os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0 + assert provider_process.proves_stopped(exact), "After waitpid, absence proves death" + + +def test_a_zombie_leader_is_refused_until_its_last_live_thread_exits() -> None: + """Z alone is insufficient while another thread of that exact process runs.""" + read_fd, write_fd = os.pipe() + ready_read, ready_write = os.pipe() + pid = os.fork() + if pid == 0: + os.close(write_fd) + os.close(ready_read) + + def live_thread() -> None: + os.write(ready_write, b"r") + os.read(read_fd, 1) + os._exit(0) + + threading.Thread(target=live_thread).start() + libc = ctypes.CDLL(None) + libc.pthread_exit.argtypes = [ctypes.c_void_p] + libc.pthread_exit.restype = None + libc.pthread_exit(None) + os._exit(99) # pthread_exit must not return. + os.close(read_fd) + os.close(ready_write) + start = None + released = False + try: + _, parent, start, _ = _stat(pid) + assert parent == os.getpid() + proof = provider_process.host_proof().split("|") + assert len(proof) == 8 and all(proof[i] for i in (0, 1, 3)) + proof[4:6] = [str(pid), start] + exact = "|".join(proof) + assert select.select([ready_read], [], [], 2.0)[0] == [ready_read] + assert os.read(ready_read, 1) == b"r" + end = time.monotonic() + 2.0 + while time.monotonic() < end: + state, parent, observed_start, threads = _stat(pid) + assert parent == os.getpid() and observed_start == start + if state == "Z" and threads > 1: + break + time.sleep(0.005) + assert state == "Z" and threads > 1, "Live-thread leader-Z witness was not established" + assert not provider_process.proves_stopped(exact) + os.write(write_fd, b"x") + released = True + end = time.monotonic() + 2.0 + while time.monotonic() < end: + state, parent, observed_start, threads = _stat(pid) + assert parent == os.getpid() and observed_start == start + if state == "Z" and threads == 1: + break + time.sleep(0.005) + assert state == "Z" and threads == 1, "Controlled group exit must reach Z1 before waitpid" + assert provider_process.proves_stopped(exact) + finally: + if not released: + os.write(write_fd, b"x") + os.close(write_fd) + os.close(ready_read) + if start is None: + _, parent, start, _ = _stat(pid) + assert parent == os.getpid() + status = _wait_owned(pid, start) + assert os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0 + assert provider_process.proves_stopped(exact) From da95f2de6c79b0622f13040f8f605b0537b5bac4 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 05:52:35 +0000 Subject: [PATCH 106/114] Match subtree diagnostic controls to kernel observations Model the captured process stat records and assert all closed kernel counts. Keep identity and secrecy controls and cover missing, unreadable and reused process observations. Signed-off-by: zlareb1 --- .../test_docker_provider_owned_subtree.py | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/tests/unit/test_docker_provider_owned_subtree.py b/tests/unit/test_docker_provider_owned_subtree.py index 1eefcbe195..05b3fcc7ef 100644 --- a/tests/unit/test_docker_provider_owned_subtree.py +++ b/tests/unit/test_docker_provider_owned_subtree.py @@ -52,6 +52,11 @@ def _fixture(monkeypatch, *, background=True, fault=None): def read(path): parts = str(path).split("/") + if len(parts) == 4 and parts[1] == "proc" and parts[3] == "stat": + process = processes[int(parts[2])] + fields = ["0"] * 20 + fields[0], fields[17], fields[19] = "S", "1", process["start"] + return f"{process['pid']} (fixture worker) " + " ".join(fields) assert parts[1] == "proc" and parts[3] == "task" and parts[2] == parts[4] assert parts[5] == "children" return children[int(parts[2])] @@ -174,6 +179,11 @@ async def query(_sql, parameters): "death_proved": 0, "query_zero_proved": 2, "unproved": 0, + "kernel_zombie_single_thread": 0, + "kernel_zombie_multi_thread": 0, + "kernel_present_non_zombie": 2, + "kernel_missing": 0, + "kernel_uninspectable": 0, }, "background": { "captured": 1, @@ -181,7 +191,71 @@ async def query(_sql, parameters): "death_proved": 0, "query_zero_proved": 1, "unproved": 0, + "kernel_zombie_single_thread": 0, + "kernel_zombie_multi_thread": 0, + "kernel_present_non_zombie": 1, + "kernel_missing": 0, + "kernel_uninspectable": 0, }, } assert all(row["host_proof"] not in output for row in [*roots, child]) assert child["participant_id"] not in output + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("state", "threads", "start", "error", "expected"), + [ + ("Z", "1", "180", None, "kernel_zombie_single_thread"), + ("Z", "2", "180", None, "kernel_zombie_multi_thread"), + ("S", "1", "180", None, "kernel_present_non_zombie"), + ("S", "1", "180", FileNotFoundError, "kernel_missing"), + ("S", "1", "180", PermissionError, "kernel_uninspectable"), + ("Z", "1", "181", None, "kernel_uninspectable"), + ], +) +async def test_subtree_kernel_counts_observe_only_the_captured_identity( + monkeypatch, capsys, state, threads, start, error, expected +): + roots, child, _ = _fixture(monkeypatch) + captured = [{**roots[0], "children": [child]}] + fixture_path = data.Path + + def read(path): + if path != "/proc/18/stat": + return fixture_path(path).read_text() + if error is not None: + raise error("kernel record unavailable") + fields = ["0"] * 20 + fields[0], fields[17], fields[19] = state, threads, start + return "18 (fixture child) " + " ".join(fields) + + monkeypatch.setattr(data, "Path", lambda path: SimpleNamespace(read_text=lambda: read(path))) + monkeypatch.setattr(data.provider_process, "proves_stopped", lambda _: False) + client = SimpleNamespace(query_all=AsyncMock(side_effect=AssertionError("no query expected"))) + await data._subtree_diagnostic(client, captured, proved=True) + prefix = "Provider subtree retirement diagnostic: " + output = capsys.readouterr().out + assert output.startswith(prefix) + summary = json.loads(output.removeprefix(prefix)) + common = { + "captured": 1, + "retirement_proved": 1, + "death_proved": 0, + "query_zero_proved": 1, + "unproved": 0, + } + kernel_keys = { + "kernel_zombie_single_thread", + "kernel_zombie_multi_thread", + "kernel_present_non_zombie", + "kernel_missing", + "kernel_uninspectable", + } + assert summary == { + "web": {**common, **{key: int(key == "kernel_present_non_zombie") for key in kernel_keys}}, + "background": {**common, **{key: int(key == expected) for key in kernel_keys}}, + } + assert all(row["host_proof"] not in output for row in [*roots, child]) + assert child["participant_id"] not in output + client.query_all.assert_not_awaited() From a1ebaad9602c9868e1995c18ec880742c59125e0 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 06:12:05 +0000 Subject: [PATCH 107/114] Distinguish store guard failures and reap interrupted test children Keep the existing store checks and failure envelope while giving each refused observation its own source site. Protect the native test child with the same kill and wait ownership as neighboring tests. Signed-off-by: zlareb1 --- .../gates/testsupport/cell_trigger/fixture.py | 8 +-- .../test_directory_control_cleanup.py | 9 +++- .../unit/test_cell_trigger_resource_census.py | 34 +++++++++++++ tests/unit/test_directory_control_cleanup.py | 50 +++++++++++++++++++ 4 files changed, 95 insertions(+), 6 deletions(-) diff --git a/scripts/gates/testsupport/cell_trigger/fixture.py b/scripts/gates/testsupport/cell_trigger/fixture.py index 27c280092c..4634ea5081 100644 --- a/scripts/gates/testsupport/cell_trigger/fixture.py +++ b/scripts/gates/testsupport/cell_trigger/fixture.py @@ -782,11 +782,11 @@ def sample(self, *, admit: bool = False) -> None: boot = Path("/proc/sys/kernel/random/boot_id").read_text().strip() for store in self.spec["stores"].values(): identity = self.owned.identity(store["host_pid"]) + need(identity is not None, "resource store birth changed") + need(identity[1] == store["start_ticks"], "resource store birth changed") + need(boot == store["boot_id"], "resource store birth changed") need( - identity is not None - and identity[1] == store["start_ticks"] - and boot == store["boot_id"] - and os.stat(f"/proc/{store['host_pid']}").st_uid == store["uid"], + os.stat(f"/proc/{store['host_pid']}").st_uid == store["uid"], "resource store birth changed", ) roots = [self.owned.pid] + [s["host_pid"] for s in self.spec["stores"].values()] diff --git a/tests/integration/test_directory_control_cleanup.py b/tests/integration/test_directory_control_cleanup.py index 2df016414a..c07ce79ac8 100644 --- a/tests/integration/test_directory_control_cleanup.py +++ b/tests/integration/test_directory_control_cleanup.py @@ -300,8 +300,13 @@ async def work(conn): stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) - stdout, stderr = await asyncio.wait_for(process.communicate(), 10) - assert process.returncode == 0, (stdout.decode(), stderr.decode()) + try: + stdout, stderr = await asyncio.wait_for(process.communicate(), 10) + assert process.returncode == 0, (stdout.decode(), stderr.decode()) + finally: + if process.returncode is None: + process.kill() + await process.wait() @pytest.mark.parametrize("kind", ["ordinary", "cancel", "exit", "keyboard"]) diff --git a/tests/unit/test_cell_trigger_resource_census.py b/tests/unit/test_cell_trigger_resource_census.py index 31a10c22cd..fb3cf02058 100644 --- a/tests/unit/test_cell_trigger_resource_census.py +++ b/tests/unit/test_cell_trigger_resource_census.py @@ -240,3 +240,37 @@ def stat(path, *args, **kwargs): assert isinstance(owner.first, fixture.Refused) assert owner.sampled == 7.0 assert owned.round == 1 + + +def test_store_identity_refusals_keep_distinct_failure_sites(census, monkeypatch): + owner, owned, _ = census + original_identity, original_stat = owned.identify, fixture.os.stat + sites = [] + for fault in ("missing", "start", "boot", "uid"): + owner.failures.clear() + owner.sampled = 7.0 + owner.spec["stores"]["pg"]["boot_id"] = "changed" if fault == "boot" else "boot" + owned.identify = lambda pid, fault=fault: ( + None + if pid == 200 and fault == "missing" + else (1, "changed", "S") + if pid == 200 and fault == "start" + else original_identity(pid) + ) + + def stat(path, *args, fault=fault, **kwargs): + if str(path) == "/proc/200" and fault == "uid": + return SimpleNamespace(st_uid=fixture.os.geteuid() + 1) + return original_stat(path, *args, **kwargs) + + monkeypatch.setattr(fixture.os, "stat", stat) + owner.sample() + assert isinstance(owner.first, fixture.Refused) + assert str(owner.first) == "resource store birth changed" + assert owner.sampled == 7.0 + record = owner.failure_summary()["records"][0] + assert record["primary"] is True and record["phase"] == "resource" + assert record["site"]["file"] == "fixture.py" + assert record["site"]["function"] == "sample" + sites.append(record["site"]["line"]) + assert len(set(sites)) == 4 diff --git a/tests/unit/test_directory_control_cleanup.py b/tests/unit/test_directory_control_cleanup.py index be6fee8329..c8dbe8ed0e 100644 --- a/tests/unit/test_directory_control_cleanup.py +++ b/tests/unit/test_directory_control_cleanup.py @@ -309,3 +309,53 @@ async def observed(): assert result is first assert began.is_set() conn.terminate.assert_called_once_with() + + +@pytest.mark.parametrize("ending", ["timeout", "cancel", "healthy"]) +async def test_integration_fatal_child_reaps_its_process_after_interruption(monkeypatch, ending): + from tests.integration import test_directory_control_cleanup as native + + original = { + "timeout": TimeoutError("owned child communication timed out"), + "cancel": asyncio.CancelledError("owned child communication cancelled"), + "healthy": None, + }[ending] + observer = Mock(fetchval=AsyncMock(return_value="owned_fake")) + process = Mock(returncode=None) + + async def communicate(): + if original is not None: + raise original + process.returncode = 0 + return b"", b"" + + async def wait(): + process.returncode = -9 + return process.returncode + + process.communicate = AsyncMock(side_effect=communicate) + process.wait = AsyncMock(side_effect=wait) + create = AsyncMock(return_value=process) + monkeypatch.setenv("LOOM_PG_DSN", "postgresql://u:test-only@127.0.0.1:15432/admin") + monkeypatch.setattr(native.asyncio, "create_subprocess_exec", create) + if original is None: + await native.test_directory_fatal_child_is_inert_until_owned_cleanup( + observer, "work", "exit" + ) + else: + with pytest.raises(type(original)) as caught: + await native.test_directory_fatal_child_is_inert_until_owned_cleanup( + observer, "work", "exit" + ) + assert caught.value is original + observer.fetchval.assert_awaited_once_with("SELECT current_database()") + create.assert_awaited_once() + process.communicate.assert_awaited_once() + if original is None: + assert process.returncode == 0 + process.kill.assert_not_called() + process.wait.assert_not_awaited() + else: + process.kill.assert_called_once_with() + process.wait.assert_awaited_once_with() + assert process.returncode == -9 From 3d86ef4534de27a094c269027a41af1f1fae6668 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 06:30:04 +0000 Subject: [PATCH 108/114] Wait for the published ClickHouse HTTP listener The initialization server can answer native queries before its final HTTP listener starts. Probe the published endpoint through the existing bounded creator command. Signed-off-by: zlareb1 --- scripts/gates/cell_trigger_acceptance.py | 45 ++++++----- tests/unit/test_cell_trigger_ci_creator.py | 38 +++++++++ ...test_cell_trigger_ci_creator_http_ready.py | 79 +++++++++++++++++++ 3 files changed, 144 insertions(+), 18 deletions(-) create mode 100644 tests/unit/test_cell_trigger_ci_creator_http_ready.py diff --git a/scripts/gates/cell_trigger_acceptance.py b/scripts/gates/cell_trigger_acceptance.py index 52701f103b..d43d81d4f0 100644 --- a/scripts/gates/cell_trigger_acceptance.py +++ b/scripts/gates/cell_trigger_acceptance.py @@ -20,6 +20,26 @@ GIB = 1024**3 SUPPORT = Path(__file__).resolve().parent / "testsupport/cell_trigger" +_CLICKHOUSE_READY = """import http.client +import sys + +connection = http.client.HTTPConnection("127.0.0.1", int(sys.argv[1]), timeout=2) +ready = False +try: + connection.request( + "POST", "/", body="SELECT 1", + headers={"X-ClickHouse-User": "loom", "X-ClickHouse-Key": "loomdev"}, + ) + response = connection.getresponse() + ready = response.status == 200 and response.read(3) == b"1\\n" +except (OSError, http.client.HTTPException): + pass +finally: + connection.close() +if ready: + print("1") +""" + def require(ok: object, message: str) -> None: if not ok: @@ -246,6 +266,12 @@ def wait(self, phase: str, probe, seconds: int = 120) -> None: require(time.monotonic() < end, "Owned store readiness deadline") time.sleep(1) + def _clickhouse_ready(self, port: int) -> bool: + return ( + self.command(sys.executable, "-I", "-S", "-c", _CLICKHOUSE_READY, str(port), timeout=2) + == "1" + ) + def failure_diagnostics(self) -> dict: observations = [] statuses = {"created", "running", "paused", "restarting", "removing", "exited", "dead"} @@ -517,24 +543,7 @@ def start(self, ch_image: str) -> tuple[dict, dict, dict, list[int]]: == "sync" ), ) - self.wait( - "clickhouse_query", - lambda: ( - self.command( - "docker", - "exec", - ch["id"], - "clickhouse-client", - "--user", - "loom", - "--password", - "loomdev", - "--query", - "SELECT 1", - ) - == "1" - ), - ) + self.wait("clickhouse_query", lambda: self._clickhouse_ready(ch_port)) return ch, pg, standby, selected def observation(self, item: dict, role: str) -> dict: diff --git a/tests/unit/test_cell_trigger_ci_creator.py b/tests/unit/test_cell_trigger_ci_creator.py index e1e77569c5..730f265968 100644 --- a/tests/unit/test_cell_trigger_ci_creator.py +++ b/tests/unit/test_cell_trigger_ci_creator.py @@ -219,6 +219,9 @@ def __init__(self): self.tcp_attempts = 0 self.tcp_ready = False + def _clickhouse_ready(self, port): + return True + def create(self, role, args): return {"id": role} @@ -790,3 +793,38 @@ def diagnostic(*args, **kwargs): assert error is original and original.native is None else: pytest.fail("command refusal was swallowed") + + +def test_clickhouse_waits_for_published_http_after_temporary_native_ready(tmp_path, monkeypatch): + from scripts.gates import cell_trigger_acceptance as creator + + monkeypatch.setattr(creator, "ports", lambda count: list(range(18000, 18000 + count))) + monkeypatch.setattr(creator.time, "sleep", lambda _: None) + + class InitializingClickHouse(Stores): + def __init__(self): + super().__init__(tmp_path, tmp_path, "fixture", {"run_id": "123"}) + self.published_attempts = 0 + self.native_attempts = 0 + + def create(self, role, args): + return {"id": role} + + def command(self, *argv, timeout=120, cleanup=False): + if "clickhouse-client" in argv: + self.native_attempts += 1 + return "1" # The temporary loopback-only server already accepts native queries. + if "pg_isready" in argv: + return "127.0.0.1:5432 - accepting connections" + return "sync" if "SELECT sync_state" in argv[-1] else "network-id" + + def _clickhouse_ready(self, port): + assert port == 18000 + self.published_attempts += 1 + return self.published_attempts == 2 + + stores = InitializingClickHouse() + stores.start("test-image") + assert stores.published_attempts == 2 + assert stores.native_attempts == 0 + assert stores.phase == "clickhouse_query" diff --git a/tests/unit/test_cell_trigger_ci_creator_http_ready.py b/tests/unit/test_cell_trigger_ci_creator_http_ready.py new file mode 100644 index 0000000000..8bad85a598 --- /dev/null +++ b/tests/unit/test_cell_trigger_ci_creator_http_ready.py @@ -0,0 +1,79 @@ +"""The fixed creator probe accepts only the published HTTP query response.""" + +import http.client +import sys + +import pytest + +from scripts.gates import cell_trigger_acceptance as creator + + +@pytest.mark.parametrize( + ("status", "body", "failure", "expected"), + [ + (200, b"1\n", None, True), + (200, b"1\nx", None, False), + (200, b"1", None, False), + (200, b"", None, False), + (500, b"1\n", None, False), + (200, b"1\n", "disconnect", False), + (200, b"1\n", "socket", False), + ], +) +def test_fixed_http_probe_checks_response_and_closes_connection( + monkeypatch, capsys, status, body, failure, expected +): + calls = [] + + class Response: + def __init__(self): + self.status = status + + def read(self, amount): + assert amount == 3 + calls.append("read") + return body + + class Connection: + def request(self, method, path, *, body, headers): + assert (method, path, body) == ("POST", "/", "SELECT 1") + assert headers == {"X-ClickHouse-User": "loom", "X-ClickHouse-Key": "loomdev"} + calls.append("request") + if failure == "socket": + raise OSError("not listening") + + def getresponse(self): + calls.append("response") + if failure == "disconnect": + raise http.client.RemoteDisconnected("bootstrap stopped") + return Response() + + def close(self): + calls.append("close") + + def connect(host, port, *, timeout): + assert (host, port, timeout) == ("127.0.0.1", 18000, 2) + return Connection() + + monkeypatch.setattr(http.client, "HTTPConnection", connect) + monkeypatch.setattr(sys, "argv", ["probe", "18000"]) + exec(creator._CLICKHOUSE_READY, {}) + assert capsys.readouterr().out == ("1\n" if expected else "") + assert calls[-1] == "close" + assert calls.count("close") == 1 + assert calls.count("read") == (1 if status == 200 and failure is None else 0) + + +def test_readiness_uses_existing_bounded_creator_command(tmp_path): + class RecordingStores(creator.Stores): + def command(self, *argv, timeout=120, cleanup=False): + self.recorded = (argv, timeout, cleanup) + return "1" + + stores = RecordingStores(tmp_path, tmp_path, "fixture", {"run_id": "123"}) + assert stores._clickhouse_ready(18000) + assert stores.recorded == ( + (sys.executable, "-I", "-S", "-c", creator._CLICKHOUSE_READY, "18000"), + 2, + False, + ) From 00ceb04854511d41425fe9fbd6bb545f8be46726 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 06:50:17 +0000 Subject: [PATCH 109/114] Wait for the complete single-thread exit witness A zombie leader can still report multiple threads. Keep polling within the existing allowance until the complete asserted exit state is observed. Signed-off-by: zlareb1 --- tests/unit/test_provider_zombie_exit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_provider_zombie_exit.py b/tests/unit/test_provider_zombie_exit.py index 501f3f56a1..db6bd2868f 100644 --- a/tests/unit/test_provider_zombie_exit.py +++ b/tests/unit/test_provider_zombie_exit.py @@ -67,7 +67,7 @@ def test_an_exact_single_thread_zombie_is_proven_stopped_before_wait() -> None: while time.monotonic() < end: state, parent, observed_start, threads = _stat(pid) assert parent == os.getpid() and observed_start == start - if state == "Z": + if state == "Z" and threads == 1: break time.sleep(0.005) assert state == "Z" and threads == 1, ( From 71121962969eb18c46c44a80cfb35c8ca9f42ae4 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 07:00:16 +0000 Subject: [PATCH 110/114] Reap the retirement fixture child after interrupted waits The fixture owns the child through timeout and cancellation. Keep its existing wait budget and always collect that exact process before store cleanup. Signed-off-by: zlareb1 --- tests/integration/test_cell_retirement.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_cell_retirement.py b/tests/integration/test_cell_retirement.py index a769a0728e..8032268818 100644 --- a/tests/integration/test_cell_retirement.py +++ b/tests/integration/test_cell_retirement.py @@ -95,7 +95,12 @@ async def test_certified_retirement_survives_old_beat_and_prunes_only_certified_ ) stopped, legacy = uuid4(), uuid4() process = await asyncio.create_subprocess_exec(sys.executable, "-c", "pass") - await asyncio.wait_for(process.wait(), timeout=10) + try: + await asyncio.wait_for(process.wait(), timeout=10) + finally: + if process.returncode is None: + process.kill() + await process.wait() fields = provider_process.host_proof().split("|") fields[4:6] = [str(process.pid), "1"] identity = "|".join(fields) From 1d752a74c1c7f06addafd428cde6802986348a48 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 07:08:55 +0000 Subject: [PATCH 111/114] Collect the attach lock fixture's delayed rollback task Cancel and join the owned task before dedicated sessions close on an interrupted control operation. Preserve the contention checks and wait allowances. Signed-off-by: zlareb1 --- tests/integration/test_cell_attach.py | 31 ++++++++++++++++----------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 71a2b41b71..72f6604578 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -647,19 +647,24 @@ async def release() -> None: await tx.rollback() task = asyncio.create_task(release()) - await cell_attach_pg.take_lock(conn, "mode test") - own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") - await cell_attach_pg.commit_enforced( - _owner(conn), - ok, - ok, - cell_attach_pg.Wait(20), - logged.append, - lambda _m: None, - expected_cell="first", - expected_fp=(own["current_fp"], own["next_fp"]), - ) - await task + try: + await cell_attach_pg.take_lock(conn, "mode test") + own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") + await cell_attach_pg.commit_enforced( + _owner(conn), + ok, + ok, + cell_attach_pg.Wait(20), + logged.append, + lambda _m: None, + expected_cell="first", + expected_fp=(own["current_fp"], own["next_fp"]), + ) + await task + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) assert await _mode() == "enforced" assert logged == ["the mode's lock is busy; retrying"] From efcfc407712f23b35894092730377390a5308d07 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 07:30:05 +0000 Subject: [PATCH 112/114] Isolate retirement gates between integration scenarios Give each in-process test its own real gate while keeping shutdown permanent within the scenario. A private patch context preserves the fixture through explicit test monkeypatch cleanup. Signed-off-by: zlareb1 --- tests/integration/conftest.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index e84ef1b5a1..acf7a1b199 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -250,6 +250,16 @@ async def asend(self: httpx.AsyncClient, request: httpx.Request, **kw: Any) -> h mp.undo() +@pytest.fixture(autouse=True) +def isolated_retirement_gate() -> Iterator[None]: + """Each test owns a fresh process gate; shutdown stays permanent within it.""" + from server.core import retirement + + with pytest.MonkeyPatch.context() as patch: + patch.setattr(retirement, "_gate", retirement.RetirementGate()) + yield + + @pytest.fixture(autouse=True) def _code_registry_read() -> Iterator[None]: """In-process code ops answer only once the self-service registry has been read From 96bd15ee4b19036d66301b8d5b5c6ac6dbfb9e8f Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 08:56:14 +0000 Subject: [PATCH 113/114] Trigger acceptance retains bounded staging failure evidence A canceled head-lock query leaves its cancellation mechanism and query plan unknown. Record closed reasons and a bounded fresh planning observation while retaining the original failure and cleanup contract. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 110 ++++++- .../gates/testsupport/cell_trigger/fixture.py | 11 + .../gates/testsupport/cell_trigger/native.py | 10 + ...est_cell_trigger_driver_failure_details.py | 7 +- .../test_cell_trigger_stage_cancellation.py | 291 ++++++++++++++++++ 5 files changed, 426 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_cell_trigger_stage_cancellation.py diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 18237c300c..9cf9f5b206 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -1453,8 +1453,13 @@ async def _run( async with asyncio.timeout(SETTLE_S): await _settle_api(conn, ch, args, api) block = await _build_block(args, batch) - async with asyncio.timeout(SETTLE_S): - await _stage(conn, ch, block) + stage_started = time.perf_counter_ns() + try: + async with asyncio.timeout(SETTLE_S): + await _stage(conn, ch, block) + except BaseException as error: + await _stage_failure(conn, block, artifact, arm, stage_started, error) + raise rps = await _delivery(args, batch, emit=artifact.emit, arm=arm) async with asyncio.timeout(SETTLE_S): await _verify(conn, ch, args, block) @@ -1512,6 +1517,107 @@ async def _run( raise asyncio.CancelledError +def _stage_plan(value: Any) -> list[dict[str, Any]]: + if type(value) is str: + _require(len(value) <= 65_536, "bounded stage plan bytes") + value = json.loads(value) + _require(type(value) is list and len(value) == 1, "one stage plan") + pending = [(value[0]["Plan"], 0)] + nodes = [] + node_types = { + "Nested Loop", + "Hash Join", + "Merge Join", + "Seq Scan", + "Index Scan", + "Index Only Scan", + "Bitmap Heap Scan", + "Bitmap Index Scan", + "Sort", + "Incremental Sort", + "Function Scan", + "LockRows", + "Hash", + "Materialize", + "Memoize", + "Gather", + "Gather Merge", + "Result", + "Append", + "Merge Append", + "Unique", + "Aggregate", + "Limit", + } + join_types = {"Inner", "Left", "Right", "Full", "Semi", "Anti", "Right Anti"} + while pending: + node, depth = pending.pop() + _require(type(node) is dict and len(nodes) < 32 and depth <= 8, "bounded stage plan") + kind = node.get("Node Type") + out: dict[str, Any] = { + "depth": depth, + "node_type": kind if type(kind) is str and kind in node_types else "other", + } + if "Join Type" in node: + join = node["Join Type"] + out["join_type"] = join if type(join) is str and join in join_types else "other" + for source, target in ( + ("Plan Rows", "plan_rows"), + ("Startup Cost", "startup_cost"), + ("Total Cost", "total_cost"), + ): + number = node.get(source) + _require( + type(number) in {int, float} and math.isfinite(number) and 0 <= number <= 1e18, + "bounded stage plan estimate", + ) + out[target] = number + nodes.append(out) + children = node.get("Plans", []) + _require(type(children) is list and len(children) <= 32, "bounded stage plan children") + pending.extend((child, depth + 1) for child in reversed(children)) + return nodes + + +async def _stage_failure( + conn: Any, block: Block, artifact: Artifact, arm: int, started: int, error: BaseException +) -> None: + try: + artifact.emit( + "stage_error", + arm=arm, + key_count=len(block.rows), + elapsed_ms=max(0, time.perf_counter_ns() - started) / 1_000_000, + ) + except BaseException: # noqa: BLE001 — stage diagnostics cannot replace the work failure. + pass + try: + causes = native_failure_metadata(error).get("driver", {}).get("causes", []) + if not any(cause.get("sqlstate") == "57014" for cause in causes): + return + from server.memory.store import pg_memory + + async with asyncio.timeout(IO_S): + _require(not conn.is_in_transaction(), "idle stage plan observer") + keys = sorted({pg_memory.identity_key(row) for row in block.rows}) + plan = await conn.fetchval( + "EXPLAIN (FORMAT JSON) " + pg_memory._LOCK_HEADS, + *pg_memory._key_arrays(keys), + ) + # This independent session plans afresh after rollback; it cannot expose + # the failed pooled backend's cached plan or its former wait state. + artifact.emit( + "freshplan_after_failure", + arm=arm, + key_count=len(keys), + observer="independent_connection", + analyze=False, + nodes=_stage_plan(plan), + ) + except BaseException: # noqa: BLE001 — a failed observation preserves the stage's first error. + pass + + async def _cleanup(triggers: Triggers | None, conn: Any, artifact: Artifact) -> None: end = time.monotonic() + RESTORE_S try: diff --git a/scripts/gates/testsupport/cell_trigger/fixture.py b/scripts/gates/testsupport/cell_trigger/fixture.py index 4634ea5081..93c7a6ff44 100644 --- a/scripts/gates/testsupport/cell_trigger/fixture.py +++ b/scripts/gates/testsupport/cell_trigger/fixture.py @@ -641,6 +641,17 @@ def native_failure_metadata(error: BaseException, *, files: tuple[str, ...] = FI sqlstate = getattr(cause, "sqlstate", None) if type(sqlstate) is str and re.fullmatch(r"[0-9A-Z]{5}", sqlstate): causes[-1]["sqlstate"] = sqlstate + if name == "QueryCanceledError" and sqlstate == "57014": + # Compare fixed server messages; exception text never enters the receipt. + message = getattr(cause, "message", None) + causes[-1]["cancel_reason"] = ( + { + "canceling statement due to statement timeout": "statement_timeout", + "canceling statement due to user request": "user_request", + }.get(message, "unknown") + if type(message) is str + else "unknown" + ) cause = cause.__cause__ if cause.__cause__ is not None else cause.__context__ driver = {"code": code, "causes": causes} sqlstate = getattr(error, "sqlstate", None) diff --git a/scripts/gates/testsupport/cell_trigger/native.py b/scripts/gates/testsupport/cell_trigger/native.py index 8d68c386db..79e58d48a9 100644 --- a/scripts/gates/testsupport/cell_trigger/native.py +++ b/scripts/gates/testsupport/cell_trigger/native.py @@ -1535,6 +1535,7 @@ def action( cause, "error_type errno" + (" sqlstate" if "sqlstate" in cause else "") + + (" cancel_reason" if "cancel_reason" in cause else "") + (" site" if "site" in cause else ""), ) need( @@ -1543,6 +1544,15 @@ def action( and re.fullmatch(r"[0-9A-Z]{5}", cause["sqlstate"]), "native Postgres failure cause", ) + if "cancel_reason" in cause: + need( + cause["error_type"] == "QueryCanceledError" + and cause.get("sqlstate") == "57014" + and type(cause["cancel_reason"]) is str + and cause["cancel_reason"] + in {"statement_timeout", "user_request", "unknown"}, + "native Postgres cancellation reason", + ) if "site" in cause: cause_site = cause["site"] closed(cause_site, "file line function") diff --git a/tests/unit/test_cell_trigger_driver_failure_details.py b/tests/unit/test_cell_trigger_driver_failure_details.py index ff41e5e20e..15db973026 100644 --- a/tests/unit/test_cell_trigger_driver_failure_details.py +++ b/tests/unit/test_cell_trigger_driver_failure_details.py @@ -39,7 +39,12 @@ def test_actual_stage_failure_retains_pg_cause_in_invalid_result( expected = ( {"error_type": "ConnectionResetError", "errno": 104} if fault == "connection" - else {"error_type": "QueryCanceledError", "errno": None, "sqlstate": "57014"} + else { + "error_type": "QueryCanceledError", + "errno": None, + "sqlstate": "57014", + "cancel_reason": "unknown", + } ) assert metadata["driver"]["causes"] == [expected] invalid = json.loads(output.read_text().splitlines()[-1]) diff --git a/tests/unit/test_cell_trigger_stage_cancellation.py b/tests/unit/test_cell_trigger_stage_cancellation.py new file mode 100644 index 0000000000..4fcc55148b --- /dev/null +++ b/tests/unit/test_cell_trigger_stage_cancellation.py @@ -0,0 +1,291 @@ +"""Actual staging failures retain bounded cancellation and fresh-plan evidence.""" + +import asyncio +import json +from copy import deepcopy +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import asyncpg +import pytest + +from bench import cell_trigger_gate as gate +from scripts.gates.testsupport.cell_trigger.fixture import native_failure_metadata +from server.core import pg +from server.memory.store import pg_memory +from tests.unit.test_cell_trigger_ci_creator import keeper_config_case as keeper_config_case +from tests.unit.test_cell_trigger_driver_failure_details import _main_case +from tests.unit.test_cell_trigger_gate_diagnostics import run_driver as run_driver + +SECRET = "secret-stage-SQL-value-condition-credential" +MESSAGES = [ + ("canceling statement due to statement timeout", "statement_timeout"), + ("canceling statement due to user request", "user_request"), + (SECRET, "unknown"), +] +SUMMARY = { + "latency_within_limit": True, + "throughput_within_limit": True, + "p99_delta_ms": 0.0, + "throughput_loss": 0.0, +} + + +def canceled(message): + try: + raise asyncpg.PostgresError.new({"C": "57014", "M": message}) + except asyncpg.QueryCanceledError as cause: + try: + pg._raise_if_unavailable(cause) + except pg.PostgresUnavailableError as error: + return error + raise AssertionError("the real PostgreSQL wrapper must refuse") + + +def plan_node(kind="Hash Join"): + return { + "Node Type": kind, + "Join Type": "Inner", + "Plan Rows": 0, + "Startup Cost": 1.0, + "Total Cost": 2.0, + "Relation Name": SECRET, + "Index Name": SECRET, + "Hash Cond": SECRET, + "Index Cond": SECRET, + "Output": [SECRET], + "SQL": SECRET, + } + + +@pytest.fixture +def stage_case(monkeypatch, tmp_path, run_driver): + def configure(*, fault=None, message=MESSAGES[0][0], plan_fault=None): + artifact_error = OSError(SECRET) if fault == "artifact" else None + cleanup_error = RuntimeError(SECRET) if fault == "explain" else None + run, events = run_driver( + prior=None, cleanup=cleanup_error, output=artifact_error, event="stage_error" + ) + rows = [ + { + "org": "owned", + "namespace": "scratch", + "agent": "bench", + "user_id": "-", + "session_id": "-", + "memory_key": key, + "memory_id": f"00000000-0000-0000-0000-{index:012d}", + } + for index, key in enumerate(("b", "a"), 1) + ] + block = gate.Block(rows=rows, audits=[{}, {}], documents=[]) + first = ( + None + if fault == "healthy" + else ValueError(SECRET) + if fault == "noncancel" + else KeyboardInterrupt(SECRET) + if fault == "fatal" + else canceled(message) + ) + state = SimpleNamespace(first=first, explain=[], commits=[], block=block) + conn = gate.asyncpg.connect.return_value + monkeypatch.setattr(conn, "is_in_transaction", lambda: False, raising=False) + + async def fetchval(sql, *values): + if sql.startswith("EXPLAIN"): + state.explain.append((sql, values)) + assert sql == "EXPLAIN (FORMAT JSON) " + pg_memory._LOCK_HEADS + assert values == ( + ["owned", "owned"], + ["scratch", "scratch"], + ["bench", "bench"], + ["-", "-"], + ["-", "-"], + ["a", "b"], + ) + if fault == "explain": + raise OSError(SECRET) + if plan_fault == "oversized": + return " " * 65_537 + node = plan_node() + if plan_fault == "deep": + for _ in range(9): + node = {**plan_node("Materialize"), "Plans": [node]} + elif plan_fault == "nonfinite": + node["Total Cost"] = float("nan") + else: + node["Plans"] = [plan_node("Index Scan"), plan_node("Function Scan")] + return json.dumps([{"Plan": node, "Private": SECRET}]) + if "fresh" in sql: + raise AssertionError("freshness is an invariant label, never SQL") + if "memory_id=ANY" in sql: + return 0 + assert "targets=7" in sql + return len(rows) + + async def commit(actual_rows, actual_audits, **kwargs): + state.commits.append((actual_rows, actual_audits, kwargs)) + assert actual_rows is block.rows and actual_audits is block.audits + assert kwargs == {"targets": 7, "publisher_lease": False} + if first is not None: + raise first + return SimpleNamespace( + seqs=[("owned", "scratch", index) for index in (1, 2)], + targets=[7, 7], + rows=rows, + audits=block.audits, + ) + + monkeypatch.setattr(conn, "fetchval", fetchval, raising=False) + monkeypatch.setattr(gate, "_arm_order", lambda: (True,)) + monkeypatch.setattr( + gate, + "_memory_set", + AsyncMock(return_value=[SimpleNamespace(measured=True, elapsed_ms=1.0)]), + ) + monkeypatch.setattr(gate, "_settle_api", AsyncMock()) + monkeypatch.setattr(gate, "_build_block", AsyncMock(return_value=block)) + monkeypatch.setattr(gate, "_delivery", AsyncMock(return_value=100.0)) + monkeypatch.setattr(gate, "_verify", AsyncMock()) + monkeypatch.setattr(gate, "_physical", AsyncMock(return_value={})) + monkeypatch.setattr(gate, "_debt", AsyncMock()) + monkeypatch.setattr(gate, "_absent", AsyncMock()) + monkeypatch.setattr(gate, "_statistics", lambda samples: SUMMARY.copy()) + monkeypatch.setattr(pg_memory, "commit_versions", commit) + + async def observe(): + try: + result = await run() + except BaseException as error: # noqa: BLE001 — keep native fatals inside their owner. + result = error + records = [ + json.loads(line) for line in (tmp_path / "raw.jsonl").read_text().splitlines() + ] + assert SECRET not in json.dumps(records) + return result, records + + return run, observe, events, state + + return configure + + +@pytest.mark.asyncio +@pytest.mark.parametrize("message,reason", MESSAGES) +async def test_actual_stage_records_cancellation_before_preserving_original( + stage_case, message, reason +): + _, observe, events, state = stage_case(message=message) + result, records = await observe() + stage = [record for record in records if record["kind"] == "stage_error"] + assert len(stage) == 1 # Baseline fails here after the real _run -> _stage refusal. + assert stage[0]["arm"] == 0 and stage[0]["key_count"] == 2 + assert type(stage[0]["elapsed_ms"]) in {int, float} and stage[0]["elapsed_ms"] >= 0 + assert result is state.first and len(state.commits) == len(state.explain) == 1 + metadata = native_failure_metadata(result) + cause = metadata["driver"]["causes"][0] + assert cause["cancel_reason"] == reason and cause["sqlstate"] == "57014" + assert metadata["driver"]["outcome_unknown"] is False + assert SECRET not in json.dumps(metadata) + fresh = [record for record in records if record["kind"] == "freshplan_after_failure"] + assert len(fresh) == 1 + assert fresh[0]["observer"] == "independent_connection" and fresh[0]["analyze"] is False + assert fresh[0]["key_count"] == 2 + assert fresh[0]["nodes"] == [ + { + "depth": depth, + "node_type": kind, + "join_type": "Inner", + "plan_rows": 0, + "startup_cost": 1.0, + "total_cost": 2.0, + } + for depth, kind in ((0, "Hash Join"), (1, "Index Scan"), (1, "Function Scan")) + ] + assert events.index("stage_error") < events.index("restoration") + assert "triggers_restored" in events and "public close" in events + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fault", ["noncancel", "fatal", "explain", "artifact"]) +async def test_actual_stage_diagnostics_cannot_replace_work_or_cleanup(stage_case, fault): + _, observe, events, state = stage_case(fault=fault) + result, records = await observe() + assert events.count("stage_error") == 1 # Causal baseline failure, not a new helper lookup. + assert result is state.first and len(state.commits) == 1 + assert len(state.explain) == (0 if fault in {"noncancel", "fatal"} else 1) + assert "restoration" in events and "public close" in events + assert ("cleanup_error" in events) is (fault == "explain") + assert any(record["kind"] == "freshplan_after_failure" for record in records) is ( + fault == "artifact" + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("plan_fault", ["oversized", "deep", "nonfinite"]) +async def test_actual_stage_rejects_unsafe_plan_and_retains_cancellation(stage_case, plan_fault): + _, observe, events, state = stage_case(plan_fault=plan_fault) + result, records = await observe() + assert any(record["kind"] == "stage_error" for record in records) + assert result is state.first and len(state.explain) == 1 + assert not any(record["kind"] == "freshplan_after_failure" for record in records) + assert "triggers_restored" in events and "public close" in events + + +@pytest.mark.parametrize("fault", ["healthy", None]) +def test_actual_stage_reaches_main_verdict_and_retains_cleanup( + monkeypatch, tmp_path, capsys, stage_case, fault +): + from server.cli import _ch + + output, _ = _main_case(monkeypatch, tmp_path, None) + run, _, events, state = stage_case(fault=fault) + monkeypatch.setattr(_ch, "run_connected", lambda *_args, **_kwargs: asyncio.run(run())) + assert gate.main() == (0 if fault == "healthy" else 1) + verdict = json.loads(capsys.readouterr().out) + assert verdict["verdict"] == ("PASS" if fault == "healthy" else "INVALID") + assert len(state.commits) == 1 and "triggers_restored" in events and "public close" in events + if fault == "healthy": + assert not state.explain and "settled" in events and "arm" in events + assert not any(kind in events for kind in ("stage_error", "freshplan_after_failure")) + assert json.loads(output.read_text().splitlines()[-1])["kind"] == "driver_cleanup" + else: + assert "stage_error" in events and len(state.explain) == 1 + assert verdict["error_type"] == "PostgresUnavailableError" + assert verdict["failure"]["driver"]["outcome_unknown"] is False + assert json.loads(output.read_text().splitlines()[-1])["kind"] == "invalid" + assert SECRET not in json.dumps(verdict) + + +@pytest.mark.parametrize( + "message,reason,fault", + [(message, reason, None) for message, reason in MESSAGES] + + [(MESSAGES[0][0], "statement_timeout", fault) for fault in ("enum", "sqlstate", "extra")], +) +def test_native_action_reads_only_closed_cancellation_evidence( + keeper_config_case, message, reason, fault +): + native, instance, owner, _, _ = keeper_config_case + instance.s["python"] = "unused" + metadata = native_failure_metadata(canceled(message)) + cause = metadata["driver"]["causes"][0] + assert cause.get("cancel_reason") == reason # Baseline misses the actual server reason. + expected = deepcopy(metadata) + if fault == "enum": + cause["cancel_reason"] = SECRET + elif fault == "sqlstate": + cause["sqlstate"] = "08006" + elif fault == "extra": + cause["message"] = SECRET + native.write(instance.w / "pg_cell-created.json.failure.json", metadata) + original = native.Refused(SECRET) + + def refused(*_args, **_kwargs): + raise original + + owner.command = refused + with pytest.raises(native.Refused) as caught: + instance.action("create", role="pg_cell") + assert caught.value is original + assert original.native == (expected if fault is None else None) + assert SECRET not in json.dumps(original.native) From 184125564937483f7f056ad4f7f296657dbcfee0 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 10:01:57 +0000 Subject: [PATCH 114/114] Exercise absent head locks at the native staging boundary Keep the production query and statement budget while reproducing the shared-identity workload in an isolated temporary table. Exact-row, ordering and identity controls distinguish a timeout from an incorrect match. Signed-off-by: zlareb1 --- tests/integration/test_pg_authority.py | 109 +++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/tests/integration/test_pg_authority.py b/tests/integration/test_pg_authority.py index fdeb0889c4..eee1707490 100644 --- a/tests/integration/test_pg_authority.py +++ b/tests/integration/test_pg_authority.py @@ -5750,3 +5750,112 @@ async def offered(after: Any = None) -> list[pg_memory.Clearable]: } assert a["value"] == "[REDACTED]" and a["redaction_reason"].startswith("leak") assert a["redacted_at"] and a["content_sha256"] != b["content_sha256"] + + +async def test_exact_head_locks_bound_a_large_absent_batch(pg_conn: Any, org: str) -> None: + """Absent keys sharing an identity do not exhaust the commit's statement budget.""" + from server.core.hashing import sha256_value + from server.memory.store import pg_memory + from server.memory.store.rows import ( + IDENTITY_COLUMNS, + MemoryRow, + build_first_version_row, + build_memory_version_row, + written_now, + ) + + namespace = "head-locks" + prefix = "gate-delivery-" + "1" * 32 + first_key = prefix + "-0" + template = pg_memory.encode_row( + build_memory_version_row( + MemoryRow.from_snapshot( + build_first_version_row( + { + "org": org, + "namespace": namespace, + "agent": "bench", + "memory_key": first_key, + "memory_type": "semantic", + "value": json.dumps(""), + "version": 1, + "content_sha256": sha256_value(""), + "actor": "process:head-lock-test", + "source_class": "asserted", + }, + now=written_now(), + ) + ) + ) + ) + assert pg_memory._LOCK_HEADS.count("loom.memory_current") == 1 + query = pg_memory._LOCK_HEADS.replace("loom.memory_current", "pg_temp.head_lock_candidates") + async with pg_conn.transaction(): + await pg_conn.execute( + "CREATE TEMP TABLE head_lock_candidates " + "(LIKE loom.memory_current INCLUDING ALL) WITH (fillfactor = 90) ON COMMIT DROP" + ) + # One statement builds full persisted rows without per-row network calls. + inserted = await pg_conn.execute( + "WITH keys AS (" + " SELECT i, $2::text || '-' || i::text AS key, " + "lpad(to_hex(i + 1), 32, '0')::uuid AS id FROM generate_series(0, 15699) AS i" + "), prefixes AS (" + " SELECT *, 'Cell trigger measurement ' || key || ': ' AS prefix FROM keys" + "), payloads AS (" + " SELECT *, prefix || left(repeat('abcdefghijklmnopqrstuvwxyz ', 4096), " + "4096 - length(prefix)) AS payload FROM prefixes" + "), stored AS (" + " SELECT *, to_json(payload)::text AS value, " + "encode(sha256(convert_to(to_json(payload)::text, 'UTF8')), 'hex') AS hash " + "FROM payloads) " + "INSERT INTO pg_temp.head_lock_candidates " + "(org, namespace, agent, user_id, session_id, memory_key, memory_id, " + "version, seq, content_sha256, row) " + "SELECT $1::jsonb->>'org', $1::jsonb->>'namespace', $1::jsonb->>'agent', " + "$1::jsonb->>'user_id', $1::jsonb->>'session_id', key, id, 1, i + 1, hash, " + "$1::jsonb || jsonb_build_object('memory_key', key, 'memory_id', id::text, " + "'value', value, 'content_sha256', hash) FROM stored", + template, + prefix, + ) + assert inserted == "INSERT 0 15700" + await pg_conn.execute("ANALYZE pg_temp.head_lock_candidates") + await pg_conn.execute("SET LOCAL statement_timeout = 5000") + await pg_conn.execute("SET LOCAL lock_timeout = 2000") + expected = {} + for index in (2, 10, 1): + key = f"{prefix}-{index}" + start = f"Cell trigger measurement {key}: " + payload = start + ("abcdefghijklmnopqrstuvwxyz " * 4096)[: 4096 - len(start)] + row = { + **template, + "memory_key": key, + "memory_id": str(uuid.UUID(int=index + 1)), + "value": json.dumps(payload), + "content_sha256": sha256_value(payload), + } + identity = pg_memory.identity_key(row) + expected[identity] = {**dict(zip(IDENTITY_COLUMNS, identity, strict=True)), "row": row} + matched = await pg_conn.fetch(query, *pg_memory._key_arrays(list(expected))) + assert [dict(row) for row in matched] == [expected[key] for key in sorted(expected)] + identity = next(iter(expected)) + changed = [ + tuple( + "different-" + value if position == axis else value + for position, value in enumerate(identity) + ) + for axis in range(6) + ] + assert await pg_conn.fetch(query, *pg_memory._key_arrays(sorted(changed))) == [] + absent_prefix = "gate-delivery-" + "2" * 32 + absent = sorted( + (org, namespace, "bench", "-", "-", f"{absent_prefix}-{index}") for index in range(2500) + ) + arguments = pg_memory._key_arrays(absent) + plan = await pg_conn.fetchval("EXPLAIN (FORMAT JSON) " + query, *arguments) + try: + assert await pg_conn.fetch(query, *arguments) == [] + except Exception as error: # The synthetic plan must not replace the native failure. + error.add_note("Synthetic head-lock plan before query: " + json.dumps(plan)) + raise