From a8eff507a2420437307aab457f92b0d9c9104bda Mon Sep 17 00:00:00 2001 From: Lushenwar Date: Sun, 11 Oct 2026 16:39:31 -0400 Subject: [PATCH 1/3] fix(core): no more incidents stuck in triage; dedup survives restarts - Open-incident dedup moves from an in-memory dict to a partial unique index on incidents.signature (status triggered/triaging), so it holds across restarts and workers. Rows from before the column existed stay NULL and never dedup, so no backfill is needed. - A triage that throws now ends `degraded` with the reason recorded. It used to revert to `triggered` and stay open with nothing in it. - At startup, open incidents with no diagnostics and no update for SENTINEL_STUCK_AFTER_S (default 300s) are re-triaged in a background thread. - Drops the unused app_logs table from the schema. Existing tables are left alone. - CI runs a Postgres service so the new SQL is tested against a real database (test_db_integration, opt-in via SENTINEL_DB_TESTS=1). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 13 ++++- core/db.py | 79 ++++++++++++++++++++----------- core/main.py | 3 ++ core/orchestrator.py | 64 ++++++++++++------------- core/tests/test_db_integration.py | 62 ++++++++++++++++++++++++ core/tests/test_orchestrator.py | 79 ++++++++++++++++++------------- 6 files changed, 208 insertions(+), 92 deletions(-) create mode 100644 core/tests/test_db_integration.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 654e846..09c73dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,9 +8,20 @@ jobs: backend: runs-on: ubuntu-latest env: - # tests mock all external boundaries; these only satisfy import-time env lookups + # unit tests mock all external boundaries; test_db_integration runs its SQL against the service below DATABASE_URL: postgresql://ci:ci@localhost:5432/ci + SENTINEL_DB_TESTS: "1" ANTHROPIC_API_KEY: ci-dummy + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: ci + POSTGRES_PASSWORD: ci + POSTGRES_DB: ci + ports: ["5432:5432"] + options: >- + --health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 10 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 diff --git a/core/db.py b/core/db.py index 2ff7323..837574a 100644 --- a/core/db.py +++ b/core/db.py @@ -27,39 +27,64 @@ def init_schema(): created_at TIMESTAMPTZ DEFAULT NOW(), updated_at TIMESTAMPTZ DEFAULT NOW() ); - CREATE TABLE IF NOT EXISTS app_logs ( - id SERIAL PRIMARY KEY, - incident_id TEXT REFERENCES incidents(id), - ts TIMESTAMPTZ DEFAULT NOW(), - level TEXT, - message TEXT, - endpoint TEXT, - raw JSONB - ); + -- dedup key. Rows created before this column existed stay NULL and never dedup. + ALTER TABLE incidents ADD COLUMN IF NOT EXISTS signature TEXT; + -- at most one open incident per error signature, across workers and restarts + CREATE UNIQUE INDEX IF NOT EXISTS incidents_one_open_per_signature + ON incidents (signature) WHERE status IN ('triggered', 'triaging'); """) conn.close() -def save_incident(incident: dict): +_OPEN = "status IN ('triggered', 'triaging')" # must match the partial index predicate + + +def open_incident(incident: dict, signature: str) -> tuple[str, bool]: + """Insert the incident unless one is already open for this signature. + Returns (incident_id, created); when not created, the id is the open incident's.""" conn = _conn() - with conn: - with conn.cursor() as cur: - cur.execute( - """INSERT INTO incidents (id, status, trigger_data, diagnostics, postmortem) - VALUES (%s, %s, %s, %s, %s) - ON CONFLICT (id) DO UPDATE SET - status = EXCLUDED.status, - diagnostics = EXCLUDED.diagnostics, - updated_at = NOW()""", - ( - incident["incident_id"], - incident["status"], - json.dumps(incident["trigger"]), - json.dumps(incident.get("diagnostics")), - incident.get("postmortem_draft_url"), - ), - ) + try: + for _ in range(3): # the open incident can close between the conflict and the lookup + with conn: + with conn.cursor() as cur: + cur.execute( + f"""INSERT INTO incidents (id, status, trigger_data, signature) + VALUES (%s, %s, %s, %s) + ON CONFLICT (signature) WHERE {_OPEN} DO NOTHING + RETURNING id""", + ( + incident["incident_id"], + incident["status"], + json.dumps(incident["trigger"]), + signature, + ), + ) + if cur.fetchone(): + return incident["incident_id"], True + cur.execute(f"SELECT id FROM incidents WHERE signature = %s AND {_OPEN}", (signature,)) + row = cur.fetchone() + if row: + return row[0], False + raise RuntimeError(f"could not open or find an incident for signature {signature!r}") + finally: + conn.close() + + +def stuck_incidents(older_than_s: float) -> list[dict]: + """Open incidents with no diagnostics whose last update is older than older_than_s: + the process running their triage died.""" + conn = _conn() + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute( + f"""SELECT id, trigger_data FROM incidents + WHERE {_OPEN} AND (diagnostics IS NULL OR diagnostics = 'null'::jsonb) + AND updated_at < NOW() - make_interval(secs => %s) + ORDER BY created_at""", + (older_than_s,), + ) + rows = cur.fetchall() conn.close() + return [dict(r) for r in rows] def get_incident(incident_id: str) -> dict | None: diff --git a/core/main.py b/core/main.py index 10a46ef..125c944 100644 --- a/core/main.py +++ b/core/main.py @@ -1,3 +1,4 @@ +import threading from contextlib import asynccontextmanager from fastapi import FastAPI, HTTPException, BackgroundTasks from fastapi.middleware.cors import CORSMiddleware @@ -10,6 +11,8 @@ async def lifespan(app: FastAPI): db.init_schema() n = vector_store.ingest_runbooks(orchestrator.RUNBOOKS_DIR) print(f"[core] runbooks ingested: {n}") + # off the startup path: re-triage can take ~15s per orphaned incident + threading.Thread(target=orchestrator.recover_stuck_incidents, daemon=True).start() yield diff --git a/core/orchestrator.py b/core/orchestrator.py index 2d5714c..5e43cf2 100644 --- a/core/orchestrator.py +++ b/core/orchestrator.py @@ -8,42 +8,28 @@ REPO_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # Sentinel root RUNBOOKS_DIR = os.path.join(REPO_PATH, "sandbox", "runbooks") -# ponytail: in-process sliding-window dedup; per-worker, resets on restart. -# Move to Postgres/Redis if you run multiple workers or need durability. +# Open-incident dedup lives in Postgres (a partial unique index on signature). Only the optional burst +# threshold is in memory. ponytail: per-worker, resets on restart; it is off at the default threshold of 1. _DEDUP_WINDOW_S = float(os.getenv("SENTINEL_DEDUP_WINDOW_S", "300")) _DEDUP_THRESHOLD = int(os.getenv("SENTINEL_DEDUP_THRESHOLD", "1")) # Nth hit in window fires _recent_alerts: dict[str, list[float]] = defaultdict(list) # signature -> hit times # Pass matched runbook text into commit ranking. Off: Phase 9 A/B found no measurable benefit (ADR-7). _RAG_RANKING = os.getenv("SENTINEL_RAG_RANKING", "0") == "1" -_open_signatures: dict[str, str] = {} # signature -> live incident_id +# Triage takes ~15s; an open incident with no diagnostics this old was orphaned by a dead process. +STUCK_AFTER_S = float(os.getenv("SENTINEL_STUCK_AFTER_S", "300")) -def _dedup_gate(signature: str) -> tuple[str, str | None]: - """'fire' (open a new incident), 'duplicate' (fold into an active one), - or 'suppress' (below the burst threshold).""" +def _below_burst_threshold(signature: str) -> bool: now = time.monotonic() hits = _recent_alerts[signature] hits[:] = [t for t in hits if t >= now - _DEDUP_WINDOW_S] hits.append(now) - - if signature in _open_signatures: - return "duplicate", _open_signatures[signature] - if len(hits) < _DEDUP_THRESHOLD: - return "suppress", None - return "fire", None - - -def _clear_signature(signature: str): - _open_signatures.pop(signature, None) + return len(hits) < _DEDUP_THRESHOLD def handle_alert(alert_data: dict) -> dict: signature = alert_data.get("error_signature", "") - decision, existing_id = _dedup_gate(signature) - if decision == "duplicate": - print(f"[dedup] alert folded into active incident {existing_id}") - return {"incident_id": existing_id, "status": "duplicate", "suppressed": True} - if decision == "suppress": + if _below_burst_threshold(signature): seen = len(_recent_alerts[signature]) print(f"[dedup] alert suppressed ({seen}/{_DEDUP_THRESHOLD} in {int(_DEDUP_WINDOW_S)}s window)") return {"incident_id": None, "status": "suppressed", "suppressed": True} @@ -56,12 +42,23 @@ def handle_alert(alert_data: dict) -> dict: "diagnostics": None, "postmortem_draft_url": None, } - db.save_incident(incident) - _open_signatures[signature] = incident["incident_id"] - print(f"[orchestrator] {incident['incident_id']} created -> triggered") + incident_id, created = db.open_incident(incident, signature) + if not created: + print(f"[dedup] alert folded into active incident {incident_id}") + return {"incident_id": incident_id, "status": "duplicate", "suppressed": True} + print(f"[orchestrator] {incident_id} created -> triggered") return incident +def recover_stuck_incidents() -> list[str]: + """Re-run triage for incidents orphaned mid-triage (crash, restart). Call once at startup.""" + stuck = db.stuck_incidents(STUCK_AFTER_S) + for inc in stuck: + print(f"[orchestrator] {inc['id']} orphaned mid-triage -> re-running diagnostics") + run_diagnostics(inc["id"], inc["trigger_data"]) + return [inc["id"] for inc in stuck] + + def run_diagnostics(incident_id: str, alert_data: dict): """Runs in a background task after the alert response is sent.""" print(f"[orchestrator] {incident_id} -> triaging") @@ -110,8 +107,7 @@ def run_diagnostics(incident_id: str, alert_data: dict): db.update_diagnostics(incident_id, diagnostics) if degraded_reason: - db.update_status(incident_id, "degraded") - _clear_signature(alert_data.get("error_signature", "")) # terminal: stop folding new alerts + db.update_status(incident_id, "degraded") # terminal: leaves the open set, new alerts open anew print(f"[orchestrator] {incident_id} -> degraded (terminal)") else: alert_dt = datetime.fromisoformat(alert_data["timestamp"].replace("Z", "+00:00")) @@ -126,8 +122,15 @@ def run_diagnostics(incident_id: str, alert_data: dict): except Exception as e: print(f"[orchestrator] {incident_id} diagnostics failed: {e}") - db.update_status(incident_id, "triggered") # revert so it can be retried - _clear_signature(alert_data.get("error_signature", "")) # allow a retry alert through + try: # terminal and visible, rather than an incident left open with nothing in it + reason = f"Diagnostics failed — manual investigation required ({e})" + db.update_diagnostics( + incident_id, + {"suspect_commits": [], "matched_runbooks": [], "degraded": True, "degraded_reason": reason}, + ) + db.update_status(incident_id, "degraded") + except Exception as e2: # db down too: the row stays open and empty; startup recovery retries it + print(f"[orchestrator] {incident_id} could not record the failure ({e2}); left for recovery") def resolve_incident(incident_id: str) -> dict: @@ -143,10 +146,7 @@ def resolve_incident(incident_id: str) -> dict: "trigger and diagnostics data remain on the incident record." ) print(f"[orchestrator] {incident_id} -- postmortem LLM unavailable, degrading: {e}") - db.update_postmortem(incident_id, draft) - _clear_signature( - (incident.get("trigger_data") or {}).get("error_signature", "") - ) # resolved: reopen to new alerts + db.update_postmortem(incident_id, draft) # status -> resolved: leaves the open set print(f"[orchestrator] {incident_id} -> resolved, postmortem generated") print(f"[metrics] {incident_id} resolve_to_postmortem_s={time.monotonic() - t0:.1f}") return db.get_incident(incident_id) diff --git a/core/tests/test_db_integration.py b/core/tests/test_db_integration.py new file mode 100644 index 0000000..1c2e43e --- /dev/null +++ b/core/tests/test_db_integration.py @@ -0,0 +1,62 @@ +# Runs the dedup/recovery SQL against a real Postgres. Opt-in, because DATABASE_URL in a dev .env is +# usually the shared Neon database: CI sets SENTINEL_DB_TESTS=1 with a throwaway Postgres service. +import os +import uuid + +import pytest + +pytestmark = pytest.mark.skipif(os.getenv("SENTINEL_DB_TESTS") != "1", reason="needs SENTINEL_DB_TESTS=1") + +from core import db # noqa: E402 + + +def _inc(): + return {"incident_id": f"inc_test_{uuid.uuid4().hex[:8]}", "status": "triggered", "trigger": {"x": 1}} + + +@pytest.fixture(scope="module", autouse=True) +def schema(): + db.init_schema() + db.init_schema() # idempotent + + +def test_one_open_incident_per_signature(): + sig = f"sig-{uuid.uuid4().hex}" + first, second, other = _inc(), _inc(), _inc() + assert db.open_incident(first, sig) == (first["incident_id"], True) + assert db.open_incident(second, sig) == (first["incident_id"], False) + assert db.get_incident(second["incident_id"]) is None + assert db.open_incident(other, sig + "-other")[1] is True + + db.update_status(first["incident_id"], "degraded") # terminal: leaves the open set + assert db.open_incident(second, sig) == (second["incident_id"], True) + + +def test_triaged_incident_is_still_open(): + sig = f"sig-{uuid.uuid4().hex}" + first = _inc() + db.open_incident(first, sig) + db.update_diagnostics(first["incident_id"], {"suspect_commits": []}) # status triaging, awaiting resolve + assert db.open_incident(_inc(), sig) == (first["incident_id"], False) + db.update_postmortem(first["incident_id"], "# pm") # resolved + assert db.open_incident(_inc(), sig)[1] is True + + +def test_stuck_means_open_without_diagnostics_and_old(): + old, fresh, done = _inc(), _inc(), _inc() + for inc in (old, fresh, done): + db.open_incident(inc, f"sig-{uuid.uuid4().hex}") + db.update_diagnostics(done["incident_id"], {"suspect_commits": []}) + conn = db._conn() + with conn, conn.cursor() as cur: + cur.execute( + "UPDATE incidents SET updated_at = NOW() - INTERVAL '1 hour' WHERE id IN (%s, %s)", + (old["incident_id"], done["incident_id"]), + ) + conn.close() + stuck = {r["id"] for r in db.stuck_incidents(300)} + assert old["incident_id"] in stuck + assert fresh["incident_id"] not in stuck and done["incident_id"] not in stuck + assert next(r for r in db.stuck_incidents(300) if r["id"] == old["incident_id"])["trigger_data"] == { + "x": 1 + } diff --git a/core/tests/test_orchestrator.py b/core/tests/test_orchestrator.py index 2f16d97..46a8605 100644 --- a/core/tests/test_orchestrator.py +++ b/core/tests/test_orchestrator.py @@ -1,4 +1,4 @@ -# ponytail: mocks db so test runs without a live postgres connection +# ponytail: mocks db so test runs without a live postgres connection (the SQL itself: test_db_integration.py) from unittest.mock import patch import core.orchestrator as orch from core.orchestrator import handle_alert @@ -11,56 +11,71 @@ } -def _reset_dedup(): - orch._recent_alerts.clear() - orch._open_signatures.clear() +def _created(incident, _signature): + return incident["incident_id"], True def test_handle_alert(): - _reset_dedup() - with patch("core.orchestrator.db.save_incident") as mock_save: + orch._recent_alerts.clear() + with patch("core.orchestrator.db.open_incident", side_effect=_created) as mock_open: result = handle_alert(_ALERT) assert result["status"] == "triggered" assert result["incident_id"].startswith("inc_") assert result["trigger"] == _ALERT assert result["diagnostics"] is None - mock_save.assert_called_once() - + mock_open.assert_called_once_with(result, _ALERT["error_signature"]) -def test_dedup_folds_repeat_alerts(): - """Second alert for a still-open signature must not open a new incident.""" - _reset_dedup() - with patch("core.orchestrator.db.save_incident") as mock_save: - first = handle_alert(_ALERT) - second = handle_alert(_ALERT) - assert first["status"] == "triggered" - assert second["status"] == "duplicate" - assert second["incident_id"] == first["incident_id"] - mock_save.assert_called_once() # only the first alert hit the DB - # once the incident is cleared (resolved/degraded/failed), a new alert fires again - orch._clear_signature(_ALERT["error_signature"]) - with patch("core.orchestrator.db.save_incident"): - third = handle_alert(_ALERT) - assert third["status"] == "triggered" - assert third["incident_id"] != first["incident_id"] +def test_open_incident_conflict_folds_alert_into_it(): + orch._recent_alerts.clear() + with patch("core.orchestrator.db.open_incident", return_value=("inc_existing", False)): + result = handle_alert(_ALERT) + assert result == {"incident_id": "inc_existing", "status": "duplicate", "suppressed": True} def test_dedup_threshold_suppresses_below_burst(): """With a threshold >1, alerts below the burst count are suppressed, not opened.""" - _reset_dedup() - with patch.object(orch, "_DEDUP_THRESHOLD", 3), patch("core.orchestrator.db.save_incident") as mock_save: + orch._recent_alerts.clear() + with ( + patch.object(orch, "_DEDUP_THRESHOLD", 3), + patch("core.orchestrator.db.open_incident", side_effect=_created) as mock_open, + ): r1 = handle_alert(_ALERT) r2 = handle_alert(_ALERT) r3 = handle_alert(_ALERT) assert r1["status"] == "suppressed" assert r2["status"] == "suppressed" assert r3["status"] == "triggered" # 3rd hit crosses the threshold - mock_save.assert_called_once() + mock_open.assert_called_once() + + +def test_failed_diagnostics_end_degraded_not_open(): + with ( + patch("core.orchestrator.db") as db, + patch("core.orchestrator.git_client.get_recent_diffs", side_effect=OSError("git missing")), + ): + orch.run_diagnostics("inc_x", _ALERT) + diag = db.update_diagnostics.call_args.args[1] + assert diag["degraded"] and "git missing" in diag["degraded_reason"] + assert db.update_status.call_args_list[-1].args == ("inc_x", "degraded") + + +def test_failure_with_db_down_leaves_row_for_recovery(): + with ( + patch("core.orchestrator.db") as db, + patch("core.orchestrator.git_client.get_recent_diffs", side_effect=OSError("git missing")), + ): + db.update_diagnostics.side_effect = ConnectionError("db down") + orch.run_diagnostics("inc_x", _ALERT) # must not raise + assert ("inc_x", "degraded") not in [c.args for c in db.update_status.call_args_list] -if __name__ == "__main__": - test_handle_alert() - test_dedup_folds_repeat_alerts() - test_dedup_threshold_suppresses_below_burst() - print("[ok] orchestrator + dedup tests passed") +def test_recover_reruns_triage_for_each_stuck_incident(): + stuck = [{"id": "inc_a", "trigger_data": _ALERT}, {"id": "inc_b", "trigger_data": _ALERT}] + with ( + patch("core.orchestrator.db.stuck_incidents", return_value=stuck) as q, + patch.object(orch, "run_diagnostics") as run, + ): + assert orch.recover_stuck_incidents() == ["inc_a", "inc_b"] + q.assert_called_once_with(orch.STUCK_AFTER_S) + assert [c.args for c in run.call_args_list] == [("inc_a", _ALERT), ("inc_b", _ALERT)] From 14485b4f4285c33161bd7379e4467bc8f01f7ad2 Mon Sep 17 00:00:00 2001 From: Lushenwar Date: Sun, 11 Oct 2026 16:41:50 -0400 Subject: [PATCH 2/3] feat(core): HMAC-signed alerts; CORS origins from env With SENTINEL_ALERT_SECRET set, /alert requires X-Sentinel-Signature: sha256=, compared in constant time; anything else gets a 401. Unset, unsigned alerts are still accepted, with a startup warning, so the local demo works without setup. chaos_cli and the load generator sign through sandbox/alert_signing.py. /alert now reads the raw body (the signature covers exact bytes) and rejects non-object JSON with a 400. The handler runs in the threadpool so its DB call doesn't block the event loop. CORS origins come from SENTINEL_CORS_ORIGINS. An empty value falls back to http://localhost:3000 rather than allowing no origins. Co-Authored-By: Claude Opus 5.5 --- .env.example | 5 +++ core/main.py | 35 ++++++++++++++++--- core/tests/test_alert_auth.py | 65 +++++++++++++++++++++++++++++++++++ docker-compose.yml | 1 + sandbox/alert_signing.py | 22 ++++++++++++ sandbox/chaos_cli.py | 5 ++- sandbox/load_generator.py | 5 ++- 7 files changed, 131 insertions(+), 7 deletions(-) create mode 100644 core/tests/test_alert_auth.py create mode 100644 sandbox/alert_signing.py diff --git a/.env.example b/.env.example index 5305a55..16b9e8a 100644 --- a/.env.example +++ b/.env.example @@ -7,3 +7,8 @@ SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... # Optional — docker-compose Postgres password (defaults to "sentinel") POSTGRES_PASSWORD=sentinel ANTHROPIC_WORKSPACE_ID= # only needed if the key is not workspace-scoped +# Recommended — shared secret: core rejects /alert requests without a matching HMAC signature, and the +# sandbox senders sign with it. Unset = unsigned alerts accepted (local dev). e.g. `openssl rand -hex 32` +SENTINEL_ALERT_SECRET= +# Optional — comma-separated dashboard origins allowed by CORS (default http://localhost:3000) +SENTINEL_CORS_ORIGINS= diff --git a/core/main.py b/core/main.py index 125c944..841cdcd 100644 --- a/core/main.py +++ b/core/main.py @@ -1,16 +1,26 @@ +import hashlib +import hmac +import json +import os import threading from contextlib import asynccontextmanager -from fastapi import FastAPI, HTTPException, BackgroundTasks +from fastapi import FastAPI, HTTPException, BackgroundTasks, Request +from starlette.concurrency import run_in_threadpool from fastapi.middleware.cors import CORSMiddleware from . import db, orchestrator from .services import vector_store, git_client +# Shared secret for alert senders. Unset = unsigned alerts accepted, with a startup warning (local dev). +_ALERT_SECRET = os.getenv("SENTINEL_ALERT_SECRET", "") + @asynccontextmanager async def lifespan(app: FastAPI): db.init_schema() n = vector_store.ingest_runbooks(orchestrator.RUNBOOKS_DIR) print(f"[core] runbooks ingested: {n}") + if not _ALERT_SECRET: + print("[core] WARNING: SENTINEL_ALERT_SECRET unset, /alert accepts unsigned alerts (dev only)") # off the startup path: re-triage can take ~15s per orphaned incident threading.Thread(target=orchestrator.recover_stuck_incidents, daemon=True).start() yield @@ -18,18 +28,33 @@ async def lifespan(app: FastAPI): app = FastAPI(title="Sentinel Core", lifespan=lifespan) -# ponytail: dev-only origin; lock down to the deployed dashboard origin before shipping app.add_middleware( CORSMiddleware, - allow_origins=["http://localhost:3000"], + # set-but-empty (as in .env.example) must not mean "no origins" + allow_origins=(os.getenv("SENTINEL_CORS_ORIGINS") or "http://localhost:3000").split(","), allow_methods=["GET", "POST", "PATCH"], allow_headers=["*"], ) +def signature_ok(body: bytes, header: str | None, secret: str) -> bool: + """X-Sentinel-Signature: sha256=, the scheme GitHub and Sentry use.""" + expected = "sha256=" + hmac.new(secret.encode(), body, hashlib.sha256).hexdigest() + return hmac.compare_digest(expected, header or "") + + @app.post("/alert") -def receive_alert(payload: dict, background_tasks: BackgroundTasks) -> dict: - incident = orchestrator.handle_alert(payload) +async def receive_alert(request: Request, background_tasks: BackgroundTasks) -> dict: + body = await request.body() + if _ALERT_SECRET and not signature_ok(body, request.headers.get("x-sentinel-signature"), _ALERT_SECRET): + raise HTTPException(401, "bad or missing X-Sentinel-Signature") + try: + payload = json.loads(body) + except ValueError: + raise HTTPException(400, "body must be JSON") + if not isinstance(payload, dict): + raise HTTPException(400, "body must be a JSON object") + incident = await run_in_threadpool(orchestrator.handle_alert, payload) # blocking DB call if not incident.get("suppressed"): background_tasks.add_task(orchestrator.run_diagnostics, incident["incident_id"], payload) return incident diff --git a/core/tests/test_alert_auth.py b/core/tests/test_alert_auth.py new file mode 100644 index 0000000..0fb462a --- /dev/null +++ b/core/tests/test_alert_auth.py @@ -0,0 +1,65 @@ +# ponytail: /alert signature check end to end with the sandbox's signer; orchestrator mocked +from unittest.mock import patch + +import pytest +from fastapi.testclient import TestClient + +from core import main +from sandbox import alert_signing + +_ALERT = {"source": "t", "alert_name": "a", "timestamp": "2026-09-01T12:00:00Z", "error_signature": "E"} + + +@pytest.fixture +def client(): + with ( + patch.object( + main.orchestrator, "handle_alert", return_value={"incident_id": "inc_1", "status": "triggered"} + ), + patch.object(main.orchestrator, "run_diagnostics") as run, + ): + yield TestClient(main.app), run # no `with`: skips lifespan (db, chroma) + + +def _post(client, body: bytes, headers: dict | None = None): + return client.post( + "/alert", content=body, headers={"Content-Type": "application/json", **(headers or {})} + ) + + +def test_secret_set_requires_valid_signature(client, monkeypatch): + c, run = client + monkeypatch.setattr(main, "_ALERT_SECRET", "s3cret") + monkeypatch.setenv("SENTINEL_ALERT_SECRET", "s3cret") + body, headers = alert_signing.signed(_ALERT) + + assert _post(c, body).status_code == 401 + assert _post(c, body, {"X-Sentinel-Signature": "sha256=" + "0" * 64}).status_code == 401 + assert _post(c, body + b" ", headers).status_code == 401 # signature covers the exact bytes + run.assert_not_called() + + r = _post(c, body, headers) + assert r.status_code == 200 and r.json()["incident_id"] == "inc_1" + run.assert_called_once_with("inc_1", _ALERT) + + +def test_wrong_secret_is_rejected(client, monkeypatch): + c, _ = client + monkeypatch.setattr(main, "_ALERT_SECRET", "server-secret") + monkeypatch.setenv("SENTINEL_ALERT_SECRET", "sender-secret") + body, headers = alert_signing.signed(_ALERT) + assert _post(c, body, headers).status_code == 401 + + +def test_secret_unset_accepts_unsigned(client, monkeypatch): + c, _ = client + monkeypatch.setattr(main, "_ALERT_SECRET", "") + assert _post(c, b'{"error_signature": "E", "timestamp": "2026-09-01T12:00:00Z"}').status_code == 200 + + +@pytest.mark.parametrize("body", [b"not json", b"[1, 2]"]) +def test_body_must_be_a_json_object(client, monkeypatch, body): + c, run = client + monkeypatch.setattr(main, "_ALERT_SECRET", "") + assert _post(c, body).status_code == 400 + run.assert_not_called() diff --git a/docker-compose.yml b/docker-compose.yml index 6ee4d96..af77378 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,7 @@ services: DATABASE_URL: postgresql://postgres:${POSTGRES_PASSWORD:-sentinel}@postgres:5432/postgres ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY} SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL:-} + SENTINEL_ALERT_SECRET: ${SENTINEL_ALERT_SECRET:-} ports: - "8000:8000" depends_on: diff --git a/sandbox/alert_signing.py b/sandbox/alert_signing.py new file mode 100644 index 0000000..bb5aed3 --- /dev/null +++ b/sandbox/alert_signing.py @@ -0,0 +1,22 @@ +"""Signs alerts for core's /alert (see core.main.signature_ok) when SENTINEL_ALERT_SECRET is set.""" + +import hashlib +import hmac +import json +import os + +from dotenv import load_dotenv + +load_dotenv() + + +def signed(alert: dict) -> tuple[bytes, dict]: + """(body, headers) to POST. The signature covers these exact bytes, so send body as-is.""" + body = json.dumps(alert).encode() + headers = {"Content-Type": "application/json"} + secret = os.getenv("SENTINEL_ALERT_SECRET") + if secret: + headers["X-Sentinel-Signature"] = ( + "sha256=" + hmac.new(secret.encode(), body, hashlib.sha256).hexdigest() + ) + return body, headers diff --git a/sandbox/chaos_cli.py b/sandbox/chaos_cli.py index 2ae4f09..4a4f0f8 100644 --- a/sandbox/chaos_cli.py +++ b/sandbox/chaos_cli.py @@ -13,6 +13,8 @@ import click import httpx +from sandbox.alert_signing import signed + REPO_ROOT = Path(__file__).parent.parent CONFIG = REPO_ROOT / "sandbox" / "app" / "config.json" CORE_ALERT = "http://localhost:8000/alert" @@ -73,7 +75,8 @@ def trigger_bug(bug_type: str): alert["traceback"] = tb click.echo(f"[chaos] traceback: {'captured from sandbox app' if tb else 'none (app not running?)'}") try: - r = httpx.post(CORE_ALERT, json=alert, timeout=5.0) + body, headers = signed(alert) + r = httpx.post(CORE_ALERT, content=body, headers=headers, timeout=5.0) data = r.json() click.echo(f"[chaos] incident created: {data['incident_id']} (status={data['status']})") except Exception as e: diff --git a/sandbox/load_generator.py b/sandbox/load_generator.py index a7e7041..d31e705 100644 --- a/sandbox/load_generator.py +++ b/sandbox/load_generator.py @@ -10,6 +10,8 @@ import httpx from datetime import datetime, timezone +from sandbox.alert_signing import signed + APP = "http://localhost:8001" CORE = "http://localhost:8000/alert" ALERT_THRESHOLD_PCT = 20 @@ -53,7 +55,8 @@ async def run(tps: int, duration: int): if last_error.get("traceback"): alert["traceback"] = last_error["traceback"] try: - await client.post(CORE, json=alert, timeout=5.0) + body, headers = signed(alert) + await client.post(CORE, content=body, headers=headers, timeout=5.0) print("\n[load] alert fired -> sentinel core") except Exception as e: print(f"\n[load] core unreachable: {e}") From 9a6b01edc483941bd73fbaea0e5d44f7b0be3155 Mon Sep 17 00:00:00 2001 From: Lushenwar Date: Sun, 11 Oct 2026 16:44:09 -0400 Subject: [PATCH 3/3] refactor: drop estimated_affected_users, which was always null Nothing upstream reports user ids, so the field could never be filled honestly. It is removed from the orchestrator, the contract schema, the dashboard type and the CLAUDE.md contract example. Old rows that still carry it validate, because the schema allows extra properties. CLAUDE.md's pipeline step 2 now says Sentinel reads the alert's stack trace; the Postgres log table it described never existed in practice. Co-Authored-By: Claude Opus 5.5 --- claude.md | 5 ++--- contract/pipeline_schema.json | 3 +-- core/orchestrator.py | 7 +++---- core/tests/test_phase3.py | 2 +- dashboard/src/lib/api.ts | 2 +- 5 files changed, 8 insertions(+), 11 deletions(-) diff --git a/claude.md b/claude.md index ef8d836..5551ebf 100644 --- a/claude.md +++ b/claude.md @@ -46,7 +46,7 @@ Instead of connecting to high-risk production infrastructure, Sentinel operates ### How It Works: 1. **Trigger:** A mock deployment script injects a faulty commit into the sandbox repository, spiking the traffic generator's error rates and triggering a simulated critical alert webhook. -2. **Diagnose:** Sentinel intercepts the alert, queries local Postgres logs, pulls recent git diffs, ranks suspect commits using Claude Sonnet structured JSON outputs, and fetches matching runbooks using vector similarity. +2. **Diagnose:** Sentinel intercepts the alert, reads its stack trace, pulls recent git diffs, ranks suspect commits using Claude Sonnet structured JSON outputs, and fetches matching runbooks using vector similarity. 3. **Communicate:** The system constructs an incredibly concise operational summary and dispatches an asynchronous structured diagnostic card to an active Slack channel via an Incoming Webhook. 4. **Document:** Upon simulated incident resolution, the orchestration loop synthesizes the entire timeline into an perfectly structured postmortem markdown document ready for human export to Notion/Confluence. @@ -154,8 +154,7 @@ Communication across state machines utilizes strict, structured schemas. The sta } ], "impact_assessment": { - "error_rate_delta_pct": 340.5, - "estimated_affected_users": 142 + "error_rate_delta_pct": 340.5 } }, "postmortem_draft_url": null diff --git a/contract/pipeline_schema.json b/contract/pipeline_schema.json index cc7f361..915bfee 100644 --- a/contract/pipeline_schema.json +++ b/contract/pipeline_schema.json @@ -48,8 +48,7 @@ "impact_assessment": { "type": "object", "properties": { - "error_rate_delta_pct": { "type": ["number", "null"] }, - "estimated_affected_users": { "type": ["integer", "null"] } + "error_rate_delta_pct": { "type": ["number", "null"] } } }, "degraded": { "type": "boolean" }, diff --git a/core/orchestrator.py b/core/orchestrator.py index 5e43cf2..d795b77 100644 --- a/core/orchestrator.py +++ b/core/orchestrator.py @@ -92,10 +92,9 @@ def run_diagnostics(incident_id: str, alert_data: dict): "suspect_commits": ranked, # runbook text is prompt-only; never persisted or sent to Slack "matched_runbooks": [{k: v for k, v in r.items() if k != "content"} for r in runbooks], - "impact_assessment": { - "error_rate_delta_pct": alert_data.get("error_rate_pct"), - "estimated_affected_users": None, - }, + # no affected-user estimate: nothing upstream reports user ids, and the dashboard + # renders no fabricated figures + "impact_assessment": {"error_rate_delta_pct": alert_data.get("error_rate_pct")}, } if degraded_reason: diagnostics["degraded"] = True diff --git a/core/tests/test_phase3.py b/core/tests/test_phase3.py index 0052ddf..d92f298 100644 --- a/core/tests/test_phase3.py +++ b/core/tests/test_phase3.py @@ -19,7 +19,7 @@ "matched_runbooks": [ {"title": "Database Connection Pool Exhaustion", "primary_action": "Check pool limits."} ], - "impact_assessment": {"error_rate_delta_pct": 340.5, "estimated_affected_users": 142}, + "impact_assessment": {"error_rate_delta_pct": 340.5}, }, } diff --git a/dashboard/src/lib/api.ts b/dashboard/src/lib/api.ts index 2004182..85e933d 100644 --- a/dashboard/src/lib/api.ts +++ b/dashboard/src/lib/api.ts @@ -18,7 +18,7 @@ export type MatchedRunbook = { export type Diagnostics = { suspect_commits: SuspectCommit[]; matched_runbooks: MatchedRunbook[]; - impact_assessment: { error_rate_delta_pct: number | null; estimated_affected_users: number | null }; + impact_assessment: { error_rate_delta_pct: number | null }; }; export type Trigger = {