feat(observability)!: retire kube-prometheus-stack

.github/manifest-naming.yaml

@@ -13,7 +13,6 @@ filename-exempt-directories:
   - k8s/bases/infrastructure/controllers/testkube/custom-resource-definitions
 cr-directories:
   - k8s/bases/bootstrap/priority-classes
-  - k8s/bases/infrastructure/alerts
   - k8s/bases/infrastructure/cluster-policies
   - k8s/bases/infrastructure/cluster-role-bindings
   - k8s/bases/infrastructure/cluster-roles

.github/workflows/bootstrap.yaml

@@ -24,7 +24,7 @@ name: 🌱 Bootstrap
 #     ADMIN_EMAIL, HETZNER_LOCATION, SSO_GITHUB_APP_CLIENT_ID, R2_BUCKET (optional).
 #   * GitHub *Secrets*: HCLOUD_TOKEN, GHCR_TOKEN, CLOUDFLARE_API_TOKEN,
 #     R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, ALERTMANAGER_WEBHOOK_URL,
-#     ALERTMANAGER_HEARTBEAT_URL, SSO_GITHUB_APP_CLIENT_SECRET, APP_PRIVATE_KEY.
+#     CLUSTER_HEARTBEAT_URL, SSO_GITHUB_APP_CLIENT_SECRET, APP_PRIVATE_KEY.
 #   GitHub reserves the GITHUB_ prefix for Variables and Secrets, so the SSO
 #   App's credentials carry an SSO_ prefix.
 
@@ -97,7 +97,7 @@ jobs:
           SSO_GITHUB_APP_CLIENT_SECRET: ${{ secrets.SSO_GITHUB_APP_CLIENT_SECRET }}
           APP_PRIVATE_KEY: ${{ secrets.APP_PRIVATE_KEY }}
           HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
-          ALERTMANAGER_HEARTBEAT_URL: ${{ secrets.ALERTMANAGER_HEARTBEAT_URL }}
+          CLUSTER_HEARTBEAT_URL: ${{ secrets.CLUSTER_HEARTBEAT_URL }}
           HETZNER_LOCATION: ${{ vars.HETZNER_LOCATION }}
         run: |
           set -euo pipefail
@@ -108,7 +108,7 @@ jobs:
           )
           if [ "${BOOTSTRAP_ENVIRONMENT}" = "prod" ]; then
             # Without a location, prod would silently keep the template's default datacenter.
-            required+=(HCLOUD_TOKEN ALERTMANAGER_HEARTBEAT_URL HETZNER_LOCATION)
+            required+=(HCLOUD_TOKEN CLUSTER_HEARTBEAT_URL HETZNER_LOCATION)
           fi
           missing=()
           for name in "${required[@]}"; do
@@ -236,7 +236,6 @@ jobs:
           R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
           R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
           ALERTMANAGER_WEBHOOK_URL: ${{ secrets.ALERTMANAGER_WEBHOOK_URL }}
-          ALERTMANAGER_HEARTBEAT_URL: ${{ secrets.ALERTMANAGER_HEARTBEAT_URL }}
           CLUSTER_HEARTBEAT_URL: ${{ secrets.CLUSTER_HEARTBEAT_URL }}
           SSO_GITHUB_APP_CLIENT_SECRET: ${{ secrets.SSO_GITHUB_APP_CLIENT_SECRET }}
           HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
@@ -250,34 +249,19 @@ jobs:
           OAUTH2_PROXY_COOKIE_SECRET="$(openssl rand -base64 32)"
           export DEX_CLIENT_SECRET FLUX_WEB_CLIENT_SECRET OAUTH2_PROXY_COOKIE_SECRET
 
-          # The dedicated Coroot heartbeat has an exact hc-ping.com egress
-          # policy. Reject a configured URL that the cluster cannot reach rather
-          # than encrypting a silently dead monitor. Empty stays valid so default
-          # profiles and operators who have not opted into Coroot remain inert.
-          heartbeat_url_pattern='^https://hc-ping\.com/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$'
-          heartbeat_host_pattern='^https?://([^/?#]*@)?hc-ping\.com([:/?#]|$)'
-          cluster_heartbeat_check_id=""
-          alertmanager_heartbeat_check_id=""
-          alertmanager_heartbeat_url_lower="$(printf '%s' "${ALERTMANAGER_HEARTBEAT_URL}" | tr '[:upper:]' '[:lower:]')"
-          if [ -n "${CLUSTER_HEARTBEAT_URL}" ]; then
-            if [[ ! "${CLUSTER_HEARTBEAT_URL}" =~ ${heartbeat_url_pattern} ]]; then
-              echo "::error::CLUSTER_HEARTBEAT_URL must be a canonical https://hc-ping.com/<lowercase-uuid> URL or be unset"
-              exit 1
-            fi
-            cluster_heartbeat_check_id="${BASH_REMATCH[1]}"
-          fi
-          if [[ "${alertmanager_heartbeat_url_lower}" =~ ${heartbeat_host_pattern} ]]; then
-            if [[ ! "${ALERTMANAGER_HEARTBEAT_URL}" =~ ${heartbeat_url_pattern} ]]; then
-              echo "::error::ALERTMANAGER_HEARTBEAT_URL must be canonical when it uses hc-ping.com"
-              exit 1
-            fi
-            alertmanager_heartbeat_check_id="${BASH_REMATCH[1]}"
-          fi
-          if [ -n "${cluster_heartbeat_check_id}" ] && \
-             [ "${cluster_heartbeat_check_id}" = "${alertmanager_heartbeat_check_id}" ]; then
-            echo "::error::CLUSTER_HEARTBEAT_URL must use a different external check than ALERTMANAGER_HEARTBEAT_URL"
+          # The cluster heartbeat is the one off-cluster dead-man signal and has
+          # an exact hc-ping.com egress policy. Reject a URL the cluster cannot
+          # reach rather than encrypting a silently dead monitor. Empty stays
+          # valid here so a local bootstrap needs none; the configuration check
+          # above already requires it for prod.
+          heartbeat_url_pattern='^https://hc-ping\.com/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
+          if [ -n "${CLUSTER_HEARTBEAT_URL}" ] && \
+             [[ ! "${CLUSTER_HEARTBEAT_URL}" =~ ${heartbeat_url_pattern} ]]; then
+            echo "::error::CLUSTER_HEARTBEAT_URL must be a canonical https://hc-ping.com/<lowercase-uuid> URL or be unset"
             exit 1
           fi
+          # End of the heartbeat preflight. scripts/validate-observability-option.sh
+          # runs exactly the lines from the comment above down to this marker.
 
           fill_base() { # variables-base-secret
             yq -i '
@@ -288,10 +272,11 @@ jobs:
           }
           fill_base k8s/bases/bootstrap/variables-base-secret.enc.yaml
 
-          # prod cluster secret
+          # prod cluster secret. The placeholder still lists the retired Alertmanager
+          # heartbeat key, which nothing reads; drop it so it is never encrypted.
           yq -i '
+            del(.stringData.alertmanager_heartbeat_url) |
             .stringData.alertmanager_webhook_url = strenv(ALERTMANAGER_WEBHOOK_URL) |
-            .stringData.alertmanager_heartbeat_url = strenv(ALERTMANAGER_HEARTBEAT_URL) |
             .stringData.cluster_heartbeat_url = strenv(CLUSTER_HEARTBEAT_URL) |
             .stringData.dex_client_secret = strenv(DEX_CLIENT_SECRET) |
             .stringData.flux_web_client_secret = strenv(FLUX_WEB_CLIENT_SECRET) |

.github/workflows/validate-scaffold.yaml

@@ -57,11 +57,23 @@ jobs:
       - name: 🧪 Validate heartbeat image contract
         run: ./scripts/tests/test-heartbeat-image-contract.sh
 
-      - name: 🔭 Validate default-off observability option
-        # Coroot stays absent from the default provider paths. The validator also
-        # renders the explicit local/prod profiles selected through KSail.
+      - name: 🔭 Validate the observability stack
+        # Coroot is the default on both providers and kube-prometheus-stack is
+        # retired: the validator renders every provider payload and both cluster
+        # aliases, and fails on anything that would bring the retired stacks back.
         run: ./scripts/validate-observability-option.sh
 
+      - name: 🔔 Validate Flux alert coverage
+        # The production Alert is the only notification for a failed HelmRelease,
+        # and it has no namespace wildcard: every namespace holding a release or a
+        # Flux Kustomization must be listed.
+        run: ./scripts/validate-alert-coverage.sh
+
+      - name: 🧪 Validate the alert coverage check
+        # Proves the check above fails on an unwatched namespace, a source without
+        # a namespace, a named source and a layer that does not render.
+        run: ./scripts/tests/test-alert-coverage.sh
+
       - name: 🔐 Validate SSO deep-link redirects
         # Every host behind oauth2-proxy finishes its login on its own host and
         # returns the user to the page they asked for: Dex must accept exactly

AGENTS.md

@@ -263,7 +263,10 @@ kustomize k8s/clusters/local/` and `kubectl kustomize k8s/clusters/prod/`. Per f
 `kubectl apply --dry-run=client -f <file>`. Changes to Coroot, the
 audit-log-forwarder or the cluster heartbeat also run
 `./scripts/validate-observability-option.sh` to cover the default layers and their
-aliases. **Never run a cluster** (no `ksail cluster create`/`update`/
+aliases. A HelmRelease or Flux Kustomization in a new namespace, or a change to
+the production Flux `Alert`, also runs `./scripts/validate-alert-coverage.sh`: the
+Alert is the only notification for a failed release and lists every namespace by
+name. **Never run a cluster** (no `ksail cluster create`/`update`/
 `delete`, no mutating `~/.kube/config`). **Protected — never modify**
 in this repo: `*.enc.yaml`, `.sops.yaml`. Manifest layering (which change belongs
 in a base and which in an overlay patch) and Flux dependency order are stated

CONTRIBUTING.md

@@ -73,6 +73,7 @@ kubectl kustomize k8s/clusters/prod/ > /dev/null
 |---|---|
 | Anything under `k8s/`, `talos*/`, or a `ksail*.yaml` file | The five commands above |
 | The observability stack (Coroot, audit-log-forwarder, the cluster heartbeat) | `./scripts/validate-observability-option.sh` |
+| A HelmRelease or Flux Kustomization in a new namespace, or the production Flux `Alert` | `./scripts/validate-alert-coverage.sh` and `./scripts/tests/test-alert-coverage.sh` |
 | Single sign-on hosts or routes (oauth2-proxy, Dex) | `./scripts/tests/test-oauth2-proxy-sso-redirects.sh` |
 | Cilium default-deny network policies | `./scripts/tests/test-default-deny-shape.sh` |
 | `.github/workflows/bootstrap.yaml` or its documented inputs | `./scripts/validate-bootstrap-workflow.sh` (needs [actionlint](https://github.com/rhysd/actionlint)) |

README.md

@@ -18,7 +18,7 @@ It is the reusable, automatically bootstrapped form of
   Cloud in production.
 - **The services an app needs, already running** — Cilium networking and Gateway API, cert-manager
   certificates, OpenBao secrets, Dex single sign-on, Kyverno policies, Longhorn storage,
-  CloudNativePG databases, Coroot observability with Prometheus alerting, and Velero backups.
+  CloudNativePG databases, Coroot observability with Slack alerting, and Velero backups.
 - **Changes tested before they deploy** — CI validates every pull request and system-tests it on
   a throwaway cluster before the merge queue deploys it to production.
 - **Your apps from their own repositories** — run an application as a *tenant* that deploys from

docs/ARCHITECTURE.md

@@ -23,8 +23,8 @@ controllers to save resources (noted inline).
 - **Storage** — Longhorn (replicated block / RWX, prod via Hetzner CSI), CloudNativePG (PostgreSQL operator); see [`rwx-storage.md`](rwx-storage.md)
 - **Autoscaling** — Cluster Autoscaler (nodes), Vertical Pod Autoscaler, KEDA + KEDA HTTP add-on; see [`node-autoscaling.md`](node-autoscaling.md)
 - **Observability** — [Coroot](TEMPLATING.md#observability-coroot-is-the-default) (service health,
-  logs, traces and incidents), with kube-prometheus-stack (Prometheus and Alertmanager,
-  no web UIs) kept for the metric-backed alerts until that migration is delivered
+  logs, traces and incidents), Flux notifications for failed reconciliations and an
+  off-cluster heartbeat; see [`dr/alerting.md`](dr/alerting.md)
 - **Backup / DR** — Velero with CloudNativePG backups to S3-compatible storage (Cloudflare R2 in prod); see [`dr/`](dr)
 - **Virtualization** — KubeVirt + CDI _(local/CI only; disabled on the Hetzner/prod overlay)_
 - **Testing** — Testkube _(local/CI only; not deployed to prod)_

docs/BOOTSTRAP.md

@@ -37,7 +37,7 @@ it some GitHub configuration and one GitHub App; it does the rest:
 | **GitHub App for cluster SSO** | Dex’s GitHub OIDC connector — lets you log in to platform UIs and `kubectl` with GitHub | A *second*, separate App from the bootstrap App below (this one is the cluster’s identity provider). |
 | **GitHub App for the bootstrap** | Lets the workflow write `prod` environment secrets back | See [Install the bootstrap GitHub App](#1-install-the-bootstrap-github-app). |
 | **GHCR (GitHub Container Registry)** | Hosts the published OCI manifest artifact | A token with `packages: read/write`. |
-| **An external heartbeat monitor** (e.g. [healthchecks.io](https://healthchecks.io)) | Dead-man’s-switch so you’re paged if the whole cluster goes dark | Optional; see [`docs/dr/alerting.md`](dr/alerting.md). |
+| **An external heartbeat monitor** ([healthchecks.io](https://healthchecks.io)) | Dead-man’s-switch so you’re notified if the whole cluster goes dark | One check with a five-minute period. Required for a `prod` bootstrap; see [`docs/dr/alerting.md`](dr/alerting.md). |
 
 > **You do not need to install SOPS, Age, KSail, or Talos locally to bootstrap.**
 > The workflow installs its own tooling on the runner. You only need them for
@@ -131,9 +131,8 @@ Set these in your **new instance repository** under *Settings → Secrets and va
 | `CLOUDFLARE_API_TOKEN` | Cloudflare API token — DNS edit for the zone (+ Origin CA if used) | Cloudflare → **My Profile → API Tokens**. Scope to the zone’s DNS. |
 | `R2_ACCESS_KEY_ID` | R2 (S3) access key ID for backups | Cloudflare R2 → **Manage R2 API Tokens**. |
 | `R2_SECRET_ACCESS_KEY` | R2 (S3) secret access key for backups | Same place as the access key ID. |
-| `ALERTMANAGER_WEBHOOK_URL` | Incoming webhook for alert notifications (e.g. Slack) | Your chat platform’s incoming-webhook setup. |
-| `ALERTMANAGER_HEARTBEAT_URL` | Ping URL that proves the Prometheus → Alertmanager pipeline is alive | healthchecks.io (or similar) check ping URL. |
-| `CLUSTER_HEARTBEAT_URL` | Optional cluster-liveness ping URL for the cluster heartbeat | A **separate**, canonical `https://hc-ping.com/<lowercase-uuid>` URL; never reuse the Alertmanager check under another URL spelling. |
+| `ALERTMANAGER_WEBHOOK_URL` | Slack incoming webhook for Coroot incident and Flux failure notifications | Slack’s incoming-webhook setup for your alerts channel. |
+| `CLUSTER_HEARTBEAT_URL` | Ping URL for the cluster heartbeat, the one off-cluster dead-man signal (required for a `prod` bootstrap) | A canonical `https://hc-ping.com/<lowercase-uuid>` check ping URL from healthchecks.io. |
 | `SSO_GITHUB_APP_CLIENT_SECRET` | **Client secret** of the *cluster SSO* GitHub App (Dex’s GitHub connector) | The SSO App’s settings page (generate a client secret). |
 | `APP_PRIVATE_KEY` | Private key (`.pem`) of the **bootstrap** GitHub App, also used by the weekly template sync | Generated when you created the bootstrap App. |