diff --git a/.env.example b/.env.example index 5305a55..16b9e8a 100644 --- a/.env.example +++ b/.env.example @@ -7,3 +7,8 @@ SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... # Optional — docker-compose Postgres password (defaults to "sentinel") POSTGRES_PASSWORD=sentinel ANTHROPIC_WORKSPACE_ID= # only needed if the key is not workspace-scoped +# Recommended — shared secret: core rejects /alert requests without a matching HMAC signature, and the +# sandbox senders sign with it. Unset = unsigned alerts accepted (local dev). e.g. `openssl rand -hex 32` +SENTINEL_ALERT_SECRET= +# Optional — comma-separated dashboard origins allowed by CORS (default http://localhost:3000) +SENTINEL_CORS_ORIGINS= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 654e846..09c73dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,9 +8,20 @@ jobs: backend: runs-on: ubuntu-latest env: - # tests mock all external boundaries; these only satisfy import-time env lookups + # unit tests mock all external boundaries; test_db_integration runs its SQL against the service below DATABASE_URL: postgresql://ci:ci@localhost:5432/ci + SENTINEL_DB_TESTS: "1" ANTHROPIC_API_KEY: ci-dummy + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: ci + POSTGRES_PASSWORD: ci + POSTGRES_DB: ci + ports: ["5432:5432"] + options: >- + --health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 10 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 diff --git a/claude.md b/claude.md index ef8d836..5551ebf 100644 --- a/claude.md +++ b/claude.md @@ -46,7 +46,7 @@ Instead of connecting to high-risk production infrastructure, Sentinel operates ### How It Works: 1. **Trigger:** A mock deployment script injects a faulty commit into the sandbox repository, spiking the traffic generator's error rates and triggering a simulated critical alert webhook. -2. **Diagnose:** Sentinel intercepts the alert, queries local Postgres logs, pulls recent git diffs, ranks suspect commits using Claude Sonnet structured JSON outputs, and fetches matching runbooks using vector similarity. +2. **Diagnose:** Sentinel intercepts the alert, reads its stack trace, pulls recent git diffs, ranks suspect commits using Claude Sonnet structured JSON outputs, and fetches matching runbooks using vector similarity. 3. **Communicate:** The system constructs an incredibly concise operational summary and dispatches an asynchronous structured diagnostic card to an active Slack channel via an Incoming Webhook. 4. **Document:** Upon simulated incident resolution, the orchestration loop synthesizes the entire timeline into an perfectly structured postmortem markdown document ready for human export to Notion/Confluence. @@ -154,8 +154,7 @@ Communication across state machines utilizes strict, structured schemas. The sta } ], "impact_assessment": { - "error_rate_delta_pct": 340.5, - "estimated_affected_users": 142 + "error_rate_delta_pct": 340.5 } }, "postmortem_draft_url": null diff --git a/contract/pipeline_schema.json b/contract/pipeline_schema.json index cc7f361..915bfee 100644 --- a/contract/pipeline_schema.json +++ b/contract/pipeline_schema.json @@ -48,8 +48,7 @@ "impact_assessment": { "type": "object", "properties": { - "error_rate_delta_pct": { "type": ["number", "null"] }, - "estimated_affected_users": { "type": ["integer", "null"] } + "error_rate_delta_pct": { "type": ["number", "null"] } } }, "degraded": { "type": "boolean" }, diff --git a/core/db.py b/core/db.py index 2ff7323..837574a 100644 --- a/core/db.py +++ b/core/db.py @@ -27,39 +27,64 @@ def init_schema(): created_at TIMESTAMPTZ DEFAULT NOW(), updated_at TIMESTAMPTZ DEFAULT NOW() ); - CREATE TABLE IF NOT EXISTS app_logs ( - id SERIAL PRIMARY KEY, - incident_id TEXT REFERENCES incidents(id), - ts TIMESTAMPTZ DEFAULT NOW(), - level TEXT, - message TEXT, - endpoint TEXT, - raw JSONB - ); + -- dedup key. Rows created before this column existed stay NULL and never dedup. + ALTER TABLE incidents ADD COLUMN IF NOT EXISTS signature TEXT; + -- at most one open incident per error signature, across workers and restarts + CREATE UNIQUE INDEX IF NOT EXISTS incidents_one_open_per_signature + ON incidents (signature) WHERE status IN ('triggered', 'triaging'); """) conn.close() -def save_incident(incident: dict): +_OPEN = "status IN ('triggered', 'triaging')" # must match the partial index predicate + + +def open_incident(incident: dict, signature: str) -> tuple[str, bool]: + """Insert the incident unless one is already open for this signature. + Returns (incident_id, created); when not created, the id is the open incident's.""" conn = _conn() - with conn: - with conn.cursor() as cur: - cur.execute( - """INSERT INTO incidents (id, status, trigger_data, diagnostics, postmortem) - VALUES (%s, %s, %s, %s, %s) - ON CONFLICT (id) DO UPDATE SET - status = EXCLUDED.status, - diagnostics = EXCLUDED.diagnostics, - updated_at = NOW()""", - ( - incident["incident_id"], - incident["status"], - json.dumps(incident["trigger"]), - json.dumps(incident.get("diagnostics")), - incident.get("postmortem_draft_url"), - ), - ) + try: + for _ in range(3): # the open incident can close between the conflict and the lookup + with conn: + with conn.cursor() as cur: + cur.execute( + f"""INSERT INTO incidents (id, status, trigger_data, signature) + VALUES (%s, %s, %s, %s) + ON CONFLICT (signature) WHERE {_OPEN} DO NOTHING + RETURNING id""", + ( + incident["incident_id"], + incident["status"], + json.dumps(incident["trigger"]), + signature, + ), + ) + if cur.fetchone(): + return incident["incident_id"], True + cur.execute(f"SELECT id FROM incidents WHERE signature = %s AND {_OPEN}", (signature,)) + row = cur.fetchone() + if row: + return row[0], False + raise RuntimeError(f"could not open or find an incident for signature {signature!r}") + finally: + conn.close() + + +def stuck_incidents(older_than_s: float) -> list[dict]: + """Open incidents with no diagnostics whose last update is older than older_than_s: + the process running their triage died.""" + conn = _conn() + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute( + f"""SELECT id, trigger_data FROM incidents + WHERE {_OPEN} AND (diagnostics IS NULL OR diagnostics = 'null'::jsonb) + AND updated_at < NOW() - make_interval(secs => %s) + ORDER BY created_at""", + (older_than_s,), + ) + rows = cur.fetchall() conn.close() + return [dict(r) for r in rows] def get_incident(incident_id: str) -> dict | None: diff --git a/core/main.py b/core/main.py index 10a46ef..841cdcd 100644 --- a/core/main.py +++ b/core/main.py @@ -1,32 +1,60 @@ +import hashlib +import hmac +import json +import os +import threading from contextlib import asynccontextmanager -from fastapi import FastAPI, HTTPException, BackgroundTasks +from fastapi import FastAPI, HTTPException, BackgroundTasks, Request +from starlette.concurrency import run_in_threadpool from fastapi.middleware.cors import CORSMiddleware from . import db, orchestrator from .services import vector_store, git_client +# Shared secret for alert senders. Unset = unsigned alerts accepted, with a startup warning (local dev). +_ALERT_SECRET = os.getenv("SENTINEL_ALERT_SECRET", "") + @asynccontextmanager async def lifespan(app: FastAPI): db.init_schema() n = vector_store.ingest_runbooks(orchestrator.RUNBOOKS_DIR) print(f"[core] runbooks ingested: {n}") + if not _ALERT_SECRET: + print("[core] WARNING: SENTINEL_ALERT_SECRET unset, /alert accepts unsigned alerts (dev only)") + # off the startup path: re-triage can take ~15s per orphaned incident + threading.Thread(target=orchestrator.recover_stuck_incidents, daemon=True).start() yield app = FastAPI(title="Sentinel Core", lifespan=lifespan) -# ponytail: dev-only origin; lock down to the deployed dashboard origin before shipping app.add_middleware( CORSMiddleware, - allow_origins=["http://localhost:3000"], + # set-but-empty (as in .env.example) must not mean "no origins" + allow_origins=(os.getenv("SENTINEL_CORS_ORIGINS") or "http://localhost:3000").split(","), allow_methods=["GET", "POST", "PATCH"], allow_headers=["*"], ) +def signature_ok(body: bytes, header: str | None, secret: str) -> bool: + """X-Sentinel-Signature: sha256=, the scheme GitHub and Sentry use.""" + expected = "sha256=" + hmac.new(secret.encode(), body, hashlib.sha256).hexdigest() + return hmac.compare_digest(expected, header or "") + + @app.post("/alert") -def receive_alert(payload: dict, background_tasks: BackgroundTasks) -> dict: - incident = orchestrator.handle_alert(payload) +async def receive_alert(request: Request, background_tasks: BackgroundTasks) -> dict: + body = await request.body() + if _ALERT_SECRET and not signature_ok(body, request.headers.get("x-sentinel-signature"), _ALERT_SECRET): + raise HTTPException(401, "bad or missing X-Sentinel-Signature") + try: + payload = json.loads(body) + except ValueError: + raise HTTPException(400, "body must be JSON") + if not isinstance(payload, dict): + raise HTTPException(400, "body must be a JSON object") + incident = await run_in_threadpool(orchestrator.handle_alert, payload) # blocking DB call if not incident.get("suppressed"): background_tasks.add_task(orchestrator.run_diagnostics, incident["incident_id"], payload) return incident diff --git a/core/orchestrator.py b/core/orchestrator.py index 2d5714c..d795b77 100644 --- a/core/orchestrator.py +++ b/core/orchestrator.py @@ -8,42 +8,28 @@ REPO_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # Sentinel root RUNBOOKS_DIR = os.path.join(REPO_PATH, "sandbox", "runbooks") -# ponytail: in-process sliding-window dedup; per-worker, resets on restart. -# Move to Postgres/Redis if you run multiple workers or need durability. +# Open-incident dedup lives in Postgres (a partial unique index on signature). Only the optional burst +# threshold is in memory. ponytail: per-worker, resets on restart; it is off at the default threshold of 1. _DEDUP_WINDOW_S = float(os.getenv("SENTINEL_DEDUP_WINDOW_S", "300")) _DEDUP_THRESHOLD = int(os.getenv("SENTINEL_DEDUP_THRESHOLD", "1")) # Nth hit in window fires _recent_alerts: dict[str, list[float]] = defaultdict(list) # signature -> hit times # Pass matched runbook text into commit ranking. Off: Phase 9 A/B found no measurable benefit (ADR-7). _RAG_RANKING = os.getenv("SENTINEL_RAG_RANKING", "0") == "1" -_open_signatures: dict[str, str] = {} # signature -> live incident_id +# Triage takes ~15s; an open incident with no diagnostics this old was orphaned by a dead process. +STUCK_AFTER_S = float(os.getenv("SENTINEL_STUCK_AFTER_S", "300")) -def _dedup_gate(signature: str) -> tuple[str, str | None]: - """'fire' (open a new incident), 'duplicate' (fold into an active one), - or 'suppress' (below the burst threshold).""" +def _below_burst_threshold(signature: str) -> bool: now = time.monotonic() hits = _recent_alerts[signature] hits[:] = [t for t in hits if t >= now - _DEDUP_WINDOW_S] hits.append(now) - - if signature in _open_signatures: - return "duplicate", _open_signatures[signature] - if len(hits) < _DEDUP_THRESHOLD: - return "suppress", None - return "fire", None - - -def _clear_signature(signature: str): - _open_signatures.pop(signature, None) + return len(hits) < _DEDUP_THRESHOLD def handle_alert(alert_data: dict) -> dict: signature = alert_data.get("error_signature", "") - decision, existing_id = _dedup_gate(signature) - if decision == "duplicate": - print(f"[dedup] alert folded into active incident {existing_id}") - return {"incident_id": existing_id, "status": "duplicate", "suppressed": True} - if decision == "suppress": + if _below_burst_threshold(signature): seen = len(_recent_alerts[signature]) print(f"[dedup] alert suppressed ({seen}/{_DEDUP_THRESHOLD} in {int(_DEDUP_WINDOW_S)}s window)") return {"incident_id": None, "status": "suppressed", "suppressed": True} @@ -56,12 +42,23 @@ def handle_alert(alert_data: dict) -> dict: "diagnostics": None, "postmortem_draft_url": None, } - db.save_incident(incident) - _open_signatures[signature] = incident["incident_id"] - print(f"[orchestrator] {incident['incident_id']} created -> triggered") + incident_id, created = db.open_incident(incident, signature) + if not created: + print(f"[dedup] alert folded into active incident {incident_id}") + return {"incident_id": incident_id, "status": "duplicate", "suppressed": True} + print(f"[orchestrator] {incident_id} created -> triggered") return incident +def recover_stuck_incidents() -> list[str]: + """Re-run triage for incidents orphaned mid-triage (crash, restart). Call once at startup.""" + stuck = db.stuck_incidents(STUCK_AFTER_S) + for inc in stuck: + print(f"[orchestrator] {inc['id']} orphaned mid-triage -> re-running diagnostics") + run_diagnostics(inc["id"], inc["trigger_data"]) + return [inc["id"] for inc in stuck] + + def run_diagnostics(incident_id: str, alert_data: dict): """Runs in a background task after the alert response is sent.""" print(f"[orchestrator] {incident_id} -> triaging") @@ -95,10 +92,9 @@ def run_diagnostics(incident_id: str, alert_data: dict): "suspect_commits": ranked, # runbook text is prompt-only; never persisted or sent to Slack "matched_runbooks": [{k: v for k, v in r.items() if k != "content"} for r in runbooks], - "impact_assessment": { - "error_rate_delta_pct": alert_data.get("error_rate_pct"), - "estimated_affected_users": None, - }, + # no affected-user estimate: nothing upstream reports user ids, and the dashboard + # renders no fabricated figures + "impact_assessment": {"error_rate_delta_pct": alert_data.get("error_rate_pct")}, } if degraded_reason: diagnostics["degraded"] = True @@ -110,8 +106,7 @@ def run_diagnostics(incident_id: str, alert_data: dict): db.update_diagnostics(incident_id, diagnostics) if degraded_reason: - db.update_status(incident_id, "degraded") - _clear_signature(alert_data.get("error_signature", "")) # terminal: stop folding new alerts + db.update_status(incident_id, "degraded") # terminal: leaves the open set, new alerts open anew print(f"[orchestrator] {incident_id} -> degraded (terminal)") else: alert_dt = datetime.fromisoformat(alert_data["timestamp"].replace("Z", "+00:00")) @@ -126,8 +121,15 @@ def run_diagnostics(incident_id: str, alert_data: dict): except Exception as e: print(f"[orchestrator] {incident_id} diagnostics failed: {e}") - db.update_status(incident_id, "triggered") # revert so it can be retried - _clear_signature(alert_data.get("error_signature", "")) # allow a retry alert through + try: # terminal and visible, rather than an incident left open with nothing in it + reason = f"Diagnostics failed — manual investigation required ({e})" + db.update_diagnostics( + incident_id, + {"suspect_commits": [], "matched_runbooks": [], "degraded": True, "degraded_reason": reason}, + ) + db.update_status(incident_id, "degraded") + except Exception as e2: # db down too: the row stays open and empty; startup recovery retries it + print(f"[orchestrator] {incident_id} could not record the failure ({e2}); left for recovery") def resolve_incident(incident_id: str) -> dict: @@ -143,10 +145,7 @@ def resolve_incident(incident_id: str) -> dict: "trigger and diagnostics data remain on the incident record." ) print(f"[orchestrator] {incident_id} -- postmortem LLM unavailable, degrading: {e}") - db.update_postmortem(incident_id, draft) - _clear_signature( - (incident.get("trigger_data") or {}).get("error_signature", "") - ) # resolved: reopen to new alerts + db.update_postmortem(incident_id, draft) # status -> resolved: leaves the open set print(f"[orchestrator] {incident_id} -> resolved, postmortem generated") print(f"[metrics] {incident_id} resolve_to_postmortem_s={time.monotonic() - t0:.1f}") return db.get_incident(incident_id) diff --git a/core/tests/test_alert_auth.py b/core/tests/test_alert_auth.py new file mode 100644 index 0000000..0fb462a --- /dev/null +++ b/core/tests/test_alert_auth.py @@ -0,0 +1,65 @@ +# ponytail: /alert signature check end to end with the sandbox's signer; orchestrator mocked +from unittest.mock import patch + +import pytest +from fastapi.testclient import TestClient + +from core import main +from sandbox import alert_signing + +_ALERT = {"source": "t", "alert_name": "a", "timestamp": "2026-09-01T12:00:00Z", "error_signature": "E"} + + +@pytest.fixture +def client(): + with ( + patch.object( + main.orchestrator, "handle_alert", return_value={"incident_id": "inc_1", "status": "triggered"} + ), + patch.object(main.orchestrator, "run_diagnostics") as run, + ): + yield TestClient(main.app), run # no `with`: skips lifespan (db, chroma) + + +def _post(client, body: bytes, headers: dict | None = None): + return client.post( + "/alert", content=body, headers={"Content-Type": "application/json", **(headers or {})} + ) + + +def test_secret_set_requires_valid_signature(client, monkeypatch): + c, run = client + monkeypatch.setattr(main, "_ALERT_SECRET", "s3cret") + monkeypatch.setenv("SENTINEL_ALERT_SECRET", "s3cret") + body, headers = alert_signing.signed(_ALERT) + + assert _post(c, body).status_code == 401 + assert _post(c, body, {"X-Sentinel-Signature": "sha256=" + "0" * 64}).status_code == 401 + assert _post(c, body + b" ", headers).status_code == 401 # signature covers the exact bytes + run.assert_not_called() + + r = _post(c, body, headers) + assert r.status_code == 200 and r.json()["incident_id"] == "inc_1" + run.assert_called_once_with("inc_1", _ALERT) + + +def test_wrong_secret_is_rejected(client, monkeypatch): + c, _ = client + monkeypatch.setattr(main, "_ALERT_SECRET", "server-secret") + monkeypatch.setenv("SENTINEL_ALERT_SECRET", "sender-secret") + body, headers = alert_signing.signed(_ALERT) + assert _post(c, body, headers).status_code == 401 + + +def test_secret_unset_accepts_unsigned(client, monkeypatch): + c, _ = client + monkeypatch.setattr(main, "_ALERT_SECRET", "") + assert _post(c, b'{"error_signature": "E", "timestamp": "2026-09-01T12:00:00Z"}').status_code == 200 + + +@pytest.mark.parametrize("body", [b"not json", b"[1, 2]"]) +def test_body_must_be_a_json_object(client, monkeypatch, body): + c, run = client + monkeypatch.setattr(main, "_ALERT_SECRET", "") + assert _post(c, body).status_code == 400 + run.assert_not_called() diff --git a/core/tests/test_db_integration.py b/core/tests/test_db_integration.py new file mode 100644 index 0000000..1c2e43e --- /dev/null +++ b/core/tests/test_db_integration.py @@ -0,0 +1,62 @@ +# Runs the dedup/recovery SQL against a real Postgres. Opt-in, because DATABASE_URL in a dev .env is +# usually the shared Neon database: CI sets SENTINEL_DB_TESTS=1 with a throwaway Postgres service. +import os +import uuid + +import pytest + +pytestmark = pytest.mark.skipif(os.getenv("SENTINEL_DB_TESTS") != "1", reason="needs SENTINEL_DB_TESTS=1") + +from core import db # noqa: E402 + + +def _inc(): + return {"incident_id": f"inc_test_{uuid.uuid4().hex[:8]}", "status": "triggered", "trigger": {"x": 1}} + + +@pytest.fixture(scope="module", autouse=True) +def schema(): + db.init_schema() + db.init_schema() # idempotent + + +def test_one_open_incident_per_signature(): + sig = f"sig-{uuid.uuid4().hex}" + first, second, other = _inc(), _inc(), _inc() + assert db.open_incident(first, sig) == (first["incident_id"], True) + assert db.open_incident(second, sig) == (first["incident_id"], False) + assert db.get_incident(second["incident_id"]) is None + assert db.open_incident(other, sig + "-other")[1] is True + + db.update_status(first["incident_id"], "degraded") # terminal: leaves the open set + assert db.open_incident(second, sig) == (second["incident_id"], True) + + +def test_triaged_incident_is_still_open(): + sig = f"sig-{uuid.uuid4().hex}" + first = _inc() + db.open_incident(first, sig) + db.update_diagnostics(first["incident_id"], {"suspect_commits": []}) # status triaging, awaiting resolve + assert db.open_incident(_inc(), sig) == (first["incident_id"], False) + db.update_postmortem(first["incident_id"], "# pm") # resolved + assert db.open_incident(_inc(), sig)[1] is True + + +def test_stuck_means_open_without_diagnostics_and_old(): + old, fresh, done = _inc(), _inc(), _inc() + for inc in (old, fresh, done): + db.open_incident(inc, f"sig-{uuid.uuid4().hex}") + db.update_diagnostics(done["incident_id"], {"suspect_commits": []}) + conn = db._conn() + with conn, conn.cursor() as cur: + cur.execute( + "UPDATE incidents SET updated_at = NOW() - INTERVAL '1 hour' WHERE id IN (%s, %s)", + (old["incident_id"], done["incident_id"]), + ) + conn.close() + stuck = {r["id"] for r in db.stuck_incidents(300)} + assert old["incident_id"] in stuck + assert fresh["incident_id"] not in stuck and done["incident_id"] not in stuck + assert next(r for r in db.stuck_incidents(300) if r["id"] == old["incident_id"])["trigger_data"] == { + "x": 1 + } diff --git a/core/tests/test_orchestrator.py b/core/tests/test_orchestrator.py index 2f16d97..46a8605 100644 --- a/core/tests/test_orchestrator.py +++ b/core/tests/test_orchestrator.py @@ -1,4 +1,4 @@ -# ponytail: mocks db so test runs without a live postgres connection +# ponytail: mocks db so test runs without a live postgres connection (the SQL itself: test_db_integration.py) from unittest.mock import patch import core.orchestrator as orch from core.orchestrator import handle_alert @@ -11,56 +11,71 @@ } -def _reset_dedup(): - orch._recent_alerts.clear() - orch._open_signatures.clear() +def _created(incident, _signature): + return incident["incident_id"], True def test_handle_alert(): - _reset_dedup() - with patch("core.orchestrator.db.save_incident") as mock_save: + orch._recent_alerts.clear() + with patch("core.orchestrator.db.open_incident", side_effect=_created) as mock_open: result = handle_alert(_ALERT) assert result["status"] == "triggered" assert result["incident_id"].startswith("inc_") assert result["trigger"] == _ALERT assert result["diagnostics"] is None - mock_save.assert_called_once() - + mock_open.assert_called_once_with(result, _ALERT["error_signature"]) -def test_dedup_folds_repeat_alerts(): - """Second alert for a still-open signature must not open a new incident.""" - _reset_dedup() - with patch("core.orchestrator.db.save_incident") as mock_save: - first = handle_alert(_ALERT) - second = handle_alert(_ALERT) - assert first["status"] == "triggered" - assert second["status"] == "duplicate" - assert second["incident_id"] == first["incident_id"] - mock_save.assert_called_once() # only the first alert hit the DB - # once the incident is cleared (resolved/degraded/failed), a new alert fires again - orch._clear_signature(_ALERT["error_signature"]) - with patch("core.orchestrator.db.save_incident"): - third = handle_alert(_ALERT) - assert third["status"] == "triggered" - assert third["incident_id"] != first["incident_id"] +def test_open_incident_conflict_folds_alert_into_it(): + orch._recent_alerts.clear() + with patch("core.orchestrator.db.open_incident", return_value=("inc_existing", False)): + result = handle_alert(_ALERT) + assert result == {"incident_id": "inc_existing", "status": "duplicate", "suppressed": True} def test_dedup_threshold_suppresses_below_burst(): """With a threshold >1, alerts below the burst count are suppressed, not opened.""" - _reset_dedup() - with patch.object(orch, "_DEDUP_THRESHOLD", 3), patch("core.orchestrator.db.save_incident") as mock_save: + orch._recent_alerts.clear() + with ( + patch.object(orch, "_DEDUP_THRESHOLD", 3), + patch("core.orchestrator.db.open_incident", side_effect=_created) as mock_open, + ): r1 = handle_alert(_ALERT) r2 = handle_alert(_ALERT) r3 = handle_alert(_ALERT) assert r1["status"] == "suppressed" assert r2["status"] == "suppressed" assert r3["status"] == "triggered" # 3rd hit crosses the threshold - mock_save.assert_called_once() + mock_open.assert_called_once() + + +def test_failed_diagnostics_end_degraded_not_open(): + with ( + patch("core.orchestrator.db") as db, + patch("core.orchestrator.git_client.get_recent_diffs", side_effect=OSError("git missing")), + ): + orch.run_diagnostics("inc_x", _ALERT) + diag = db.update_diagnostics.call_args.args[1] + assert diag["degraded"] and "git missing" in diag["degraded_reason"] + assert db.update_status.call_args_list[-1].args == ("inc_x", "degraded") + + +def test_failure_with_db_down_leaves_row_for_recovery(): + with ( + patch("core.orchestrator.db") as db, + patch("core.orchestrator.git_client.get_recent_diffs", side_effect=OSError("git missing")), + ): + db.update_diagnostics.side_effect = ConnectionError("db down") + orch.run_diagnostics("inc_x", _ALERT) # must not raise + assert ("inc_x", "degraded") not in [c.args for c in db.update_status.call_args_list] -if __name__ == "__main__": - test_handle_alert() - test_dedup_folds_repeat_alerts() - test_dedup_threshold_suppresses_below_burst() - print("[ok] orchestrator + dedup tests passed") +def test_recover_reruns_triage_for_each_stuck_incident(): + stuck = [{"id": "inc_a", "trigger_data": _ALERT}, {"id": "inc_b", "trigger_data": _ALERT}] + with ( + patch("core.orchestrator.db.stuck_incidents", return_value=stuck) as q, + patch.object(orch, "run_diagnostics") as run, + ): + assert orch.recover_stuck_incidents() == ["inc_a", "inc_b"] + q.assert_called_once_with(orch.STUCK_AFTER_S) + assert [c.args for c in run.call_args_list] == [("inc_a", _ALERT), ("inc_b", _ALERT)] diff --git a/core/tests/test_phase3.py b/core/tests/test_phase3.py index 0052ddf..d92f298 100644 --- a/core/tests/test_phase3.py +++ b/core/tests/test_phase3.py @@ -19,7 +19,7 @@ "matched_runbooks": [ {"title": "Database Connection Pool Exhaustion", "primary_action": "Check pool limits."} ], - "impact_assessment": {"error_rate_delta_pct": 340.5, "estimated_affected_users": 142}, + "impact_assessment": {"error_rate_delta_pct": 340.5}, }, } diff --git a/dashboard/src/lib/api.ts b/dashboard/src/lib/api.ts index 2004182..85e933d 100644 --- a/dashboard/src/lib/api.ts +++ b/dashboard/src/lib/api.ts @@ -18,7 +18,7 @@ export type MatchedRunbook = { export type Diagnostics = { suspect_commits: SuspectCommit[]; matched_runbooks: MatchedRunbook[]; - impact_assessment: { error_rate_delta_pct: number | null; estimated_affected_users: number | null }; + impact_assessment: { error_rate_delta_pct: number | null }; }; export type Trigger = { diff --git a/docker-compose.yml b/docker-compose.yml index 6ee4d96..af77378 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,7 @@ services: DATABASE_URL: postgresql://postgres:${POSTGRES_PASSWORD:-sentinel}@postgres:5432/postgres ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY} SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL:-} + SENTINEL_ALERT_SECRET: ${SENTINEL_ALERT_SECRET:-} ports: - "8000:8000" depends_on: diff --git a/sandbox/alert_signing.py b/sandbox/alert_signing.py new file mode 100644 index 0000000..bb5aed3 --- /dev/null +++ b/sandbox/alert_signing.py @@ -0,0 +1,22 @@ +"""Signs alerts for core's /alert (see core.main.signature_ok) when SENTINEL_ALERT_SECRET is set.""" + +import hashlib +import hmac +import json +import os + +from dotenv import load_dotenv + +load_dotenv() + + +def signed(alert: dict) -> tuple[bytes, dict]: + """(body, headers) to POST. The signature covers these exact bytes, so send body as-is.""" + body = json.dumps(alert).encode() + headers = {"Content-Type": "application/json"} + secret = os.getenv("SENTINEL_ALERT_SECRET") + if secret: + headers["X-Sentinel-Signature"] = ( + "sha256=" + hmac.new(secret.encode(), body, hashlib.sha256).hexdigest() + ) + return body, headers diff --git a/sandbox/chaos_cli.py b/sandbox/chaos_cli.py index 2ae4f09..4a4f0f8 100644 --- a/sandbox/chaos_cli.py +++ b/sandbox/chaos_cli.py @@ -13,6 +13,8 @@ import click import httpx +from sandbox.alert_signing import signed + REPO_ROOT = Path(__file__).parent.parent CONFIG = REPO_ROOT / "sandbox" / "app" / "config.json" CORE_ALERT = "http://localhost:8000/alert" @@ -73,7 +75,8 @@ def trigger_bug(bug_type: str): alert["traceback"] = tb click.echo(f"[chaos] traceback: {'captured from sandbox app' if tb else 'none (app not running?)'}") try: - r = httpx.post(CORE_ALERT, json=alert, timeout=5.0) + body, headers = signed(alert) + r = httpx.post(CORE_ALERT, content=body, headers=headers, timeout=5.0) data = r.json() click.echo(f"[chaos] incident created: {data['incident_id']} (status={data['status']})") except Exception as e: diff --git a/sandbox/load_generator.py b/sandbox/load_generator.py index a7e7041..d31e705 100644 --- a/sandbox/load_generator.py +++ b/sandbox/load_generator.py @@ -10,6 +10,8 @@ import httpx from datetime import datetime, timezone +from sandbox.alert_signing import signed + APP = "http://localhost:8001" CORE = "http://localhost:8000/alert" ALERT_THRESHOLD_PCT = 20 @@ -53,7 +55,8 @@ async def run(tps: int, duration: int): if last_error.get("traceback"): alert["traceback"] = last_error["traceback"] try: - await client.post(CORE, json=alert, timeout=5.0) + body, headers = signed(alert) + await client.post(CORE, content=body, headers=headers, timeout=5.0) print("\n[load] alert fired -> sentinel core") except Exception as e: print(f"\n[load] core unreachable: {e}")