From abd6a928a37bd0a8ae8f39738c3b4daa9c274037 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 10 Nov 2026 17:58:59 +0100 Subject: [PATCH 1/3] feat(observability)!: retire kube-prometheus-stack Coroot is the observability stack; kube-prometheus-stack only remained for a handful of alert rules and the Watchdog heartbeat. This removes it, like the reference platform did: - Delete the kube-prometheus-stack release, its production storage patch, the platform-critical PrometheusRule, the Alertmanager webhook secret and its seed, and the node-exporter Talos firewall patches. - Watch every HelmRelease directly in the production Flux Alert, which replaces FluxHelmReleaseNotReady. notification-controller has no namespace wildcard, so the Alert lists every namespace holding a release, and scripts/validate-alert-coverage.sh fails CI when one is missing. - Make the cluster heartbeat the one off-cluster dead-man signal: a production bootstrap requires CLUSTER_HEARTBEAT_URL, and ALERTMANAGER_HEARTBEAT_URL is no longer read. - Switch off the Tetragon ServiceMonitors and drop every allowance for the monitoring namespace from the network policies, cluster policies, security exceptions and the oauth2-proxy reference grant. - Rewrite the observability validator to prove the stack is absent from every payload, and document what is no longer alerted and how an existing instance migrates. BREAKING CHANGE: Flux deletes the monitoring namespace with its Prometheus and Alertmanager data on the next sync, and a production bootstrap now requires CLUSTER_HEARTBEAT_URL. docs/TEMPLATING.md lists the steps an existing instance takes first. Part of #387 Co-Authored-By: Claude Fable 5.2 --- .github/manifest-naming.yaml | 1 - .github/workflows/bootstrap.yaml | 47 +-- .github/workflows/validate-scaffold.yaml | 18 +- AGENTS.md | 5 +- CONTRIBUTING.md | 1 + README.md | 2 +- docs/ARCHITECTURE.md | 4 +- docs/BOOTSTRAP.md | 7 +- docs/TEMPLATING.md | 82 +++-- docs/dr/alerting.md | 316 +++++++--------- docs/secret-rotation.md | 2 +- k8s/bases/bootstrap/kustomization.yaml | 2 +- .../priority-classes/platform-critical.yaml | 6 +- .../infrastructure/alerts/kustomization.yaml | 4 - .../alerts/platform-critical.yaml | 161 -------- .../best-practices/add-security-context.yaml | 2 - .../validate-host-restrictions.yaml | 2 - .../best-practices/validate-pod-security.yaml | 3 - .../controller-rbac.yaml | 3 +- .../health-probes.yaml | 1 - .../infrastructure-privileged.yaml | 4 +- .../service-account-tokens.yaml | 1 - .../cdi/cilium-network-policy.yaml | 8 - .../cert-manager/cilium-network-policy.yaml | 8 - .../cloudnative-pg/cilium-network-policy.yaml | 8 - .../dex/cilium-network-policy.yaml | 11 - .../flux-operator/cilium-network-policy.yaml | 4 - .../keda/cilium-network-policy.yaml | 12 - .../cilium-network-policy.yaml | 56 --- .../kube-prometheus-stack/helm-release.yaml | 312 ---------------- .../helm-repository.yaml | 8 - .../kube-prometheus-stack/kustomization.yaml | 10 - .../kube-prometheus-stack/namespace.yaml | 9 - .../kubevirt/cilium-network-policy.yaml | 8 - .../controllers/kustomization.yaml | 1 - .../kyverno/cilium-network-policy.yaml | 8 - .../oauth2-proxy/reference-grant.yaml | 3 - .../testkube/cilium-network-policy.yaml | 4 - .../controllers/tetragon/helm-release.yaml | 13 +- .../velero/cilium-network-policy.yaml | 9 - .../alertmanager-webhook.yaml | 23 -- .../external-secrets/kustomization.yaml | 1 - k8s/bases/infrastructure/kustomization.yaml | 1 - .../vault-seed/kustomization.yaml | 1 - .../push-secret-seed-alertmanager.yaml | 20 - .../base/flux-kustomization-bootstrap.yaml | 2 +- ...tomization-infrastructure-controllers.yaml | 4 +- .../variables-cluster-config-map.yaml | 9 - k8s/clusters/prod/kustomization.yaml | 3 +- .../external-dns/cilium-network-policy.yaml | 9 - .../controllers/flux-notifications/alert.yaml | 81 +++++ .../patches/store-tsdb-on-hcloud.yaml | 51 --- .../cilium-network-policy.yaml | 9 - .../controllers/kustomization.yaml | 2 - .../longhorn/cilium-network-policy.yaml | 4 - .../hetzner/infrastructure/kustomization.yaml | 7 - .../patches/retain-helm-release-alert.yaml | 24 -- scripts/tests/test-alert-coverage.sh | 90 +++++ scripts/tests/test-drain-safety-policies.sh | 6 +- scripts/validate-alert-coverage.sh | 127 +++++++ scripts/validate-bootstrap-workflow.sh | 15 +- scripts/validate-observability-option.sh | 344 ++++++++---------- .../allow-node-exporter-ingress.yaml | 15 - .../workers/allow-node-exporter-ingress.yaml | 15 - 64 files changed, 697 insertions(+), 1332 deletions(-) delete mode 100644 k8s/bases/infrastructure/alerts/kustomization.yaml delete mode 100644 k8s/bases/infrastructure/alerts/platform-critical.yaml delete mode 100644 k8s/bases/infrastructure/controllers/kube-prometheus-stack/cilium-network-policy.yaml delete mode 100644 k8s/bases/infrastructure/controllers/kube-prometheus-stack/helm-release.yaml delete mode 100644 k8s/bases/infrastructure/controllers/kube-prometheus-stack/helm-repository.yaml delete mode 100644 k8s/bases/infrastructure/controllers/kube-prometheus-stack/kustomization.yaml delete mode 100644 k8s/bases/infrastructure/controllers/kube-prometheus-stack/namespace.yaml delete mode 100644 k8s/bases/infrastructure/external-secrets/alertmanager-webhook.yaml delete mode 100644 k8s/bases/infrastructure/vault-seed/push-secret-seed-alertmanager.yaml delete mode 100644 k8s/providers/hetzner/infrastructure/controllers/kube-prometheus-stack/patches/store-tsdb-on-hcloud.yaml delete mode 100644 k8s/providers/hetzner/infrastructure/patches/retain-helm-release-alert.yaml create mode 100755 scripts/tests/test-alert-coverage.sh create mode 100755 scripts/validate-alert-coverage.sh delete mode 100644 talos/control-planes/allow-node-exporter-ingress.yaml delete mode 100644 talos/workers/allow-node-exporter-ingress.yaml diff --git a/.github/manifest-naming.yaml b/.github/manifest-naming.yaml index 83c8808..8e2190e 100644 --- a/.github/manifest-naming.yaml +++ b/.github/manifest-naming.yaml @@ -13,7 +13,6 @@ filename-exempt-directories: - k8s/bases/infrastructure/controllers/testkube/custom-resource-definitions cr-directories: - k8s/bases/bootstrap/priority-classes - - k8s/bases/infrastructure/alerts - k8s/bases/infrastructure/cluster-policies - k8s/bases/infrastructure/cluster-role-bindings - k8s/bases/infrastructure/cluster-roles diff --git a/.github/workflows/bootstrap.yaml b/.github/workflows/bootstrap.yaml index 5775f04..2784301 100644 --- a/.github/workflows/bootstrap.yaml +++ b/.github/workflows/bootstrap.yaml @@ -24,7 +24,7 @@ name: 🌱 Bootstrap # ADMIN_EMAIL, HETZNER_LOCATION, SSO_GITHUB_APP_CLIENT_ID, R2_BUCKET (optional). # * GitHub *Secrets*: HCLOUD_TOKEN, GHCR_TOKEN, CLOUDFLARE_API_TOKEN, # R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, ALERTMANAGER_WEBHOOK_URL, -# ALERTMANAGER_HEARTBEAT_URL, SSO_GITHUB_APP_CLIENT_SECRET, APP_PRIVATE_KEY. +# CLUSTER_HEARTBEAT_URL, SSO_GITHUB_APP_CLIENT_SECRET, APP_PRIVATE_KEY. # GitHub reserves the GITHUB_ prefix for Variables and Secrets, so the SSO # App's credentials carry an SSO_ prefix. @@ -97,7 +97,7 @@ jobs: SSO_GITHUB_APP_CLIENT_SECRET: ${{ secrets.SSO_GITHUB_APP_CLIENT_SECRET }} APP_PRIVATE_KEY: ${{ secrets.APP_PRIVATE_KEY }} HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} - ALERTMANAGER_HEARTBEAT_URL: ${{ secrets.ALERTMANAGER_HEARTBEAT_URL }} + CLUSTER_HEARTBEAT_URL: ${{ secrets.CLUSTER_HEARTBEAT_URL }} HETZNER_LOCATION: ${{ vars.HETZNER_LOCATION }} run: | set -euo pipefail @@ -108,7 +108,7 @@ jobs: ) if [ "${BOOTSTRAP_ENVIRONMENT}" = "prod" ]; then # Without a location, prod would silently keep the template's default datacenter. - required+=(HCLOUD_TOKEN ALERTMANAGER_HEARTBEAT_URL HETZNER_LOCATION) + required+=(HCLOUD_TOKEN CLUSTER_HEARTBEAT_URL HETZNER_LOCATION) fi missing=() for name in "${required[@]}"; do @@ -236,7 +236,6 @@ jobs: R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} ALERTMANAGER_WEBHOOK_URL: ${{ secrets.ALERTMANAGER_WEBHOOK_URL }} - ALERTMANAGER_HEARTBEAT_URL: ${{ secrets.ALERTMANAGER_HEARTBEAT_URL }} CLUSTER_HEARTBEAT_URL: ${{ secrets.CLUSTER_HEARTBEAT_URL }} SSO_GITHUB_APP_CLIENT_SECRET: ${{ secrets.SSO_GITHUB_APP_CLIENT_SECRET }} HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} @@ -250,32 +249,15 @@ jobs: OAUTH2_PROXY_COOKIE_SECRET="$(openssl rand -base64 32)" export DEX_CLIENT_SECRET FLUX_WEB_CLIENT_SECRET OAUTH2_PROXY_COOKIE_SECRET - # The dedicated Coroot heartbeat has an exact hc-ping.com egress - # policy. Reject a configured URL that the cluster cannot reach rather - # than encrypting a silently dead monitor. Empty stays valid so default - # profiles and operators who have not opted into Coroot remain inert. - heartbeat_url_pattern='^https://hc-ping\.com/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$' - heartbeat_host_pattern='^https?://([^/?#]*@)?hc-ping\.com([:/?#]|$)' - cluster_heartbeat_check_id="" - alertmanager_heartbeat_check_id="" - alertmanager_heartbeat_url_lower="$(printf '%s' "${ALERTMANAGER_HEARTBEAT_URL}" | tr '[:upper:]' '[:lower:]')" - if [ -n "${CLUSTER_HEARTBEAT_URL}" ]; then - if [[ ! "${CLUSTER_HEARTBEAT_URL}" =~ ${heartbeat_url_pattern} ]]; then - echo "::error::CLUSTER_HEARTBEAT_URL must be a canonical https://hc-ping.com/ URL or be unset" - exit 1 - fi - cluster_heartbeat_check_id="${BASH_REMATCH[1]}" - fi - if [[ "${alertmanager_heartbeat_url_lower}" =~ ${heartbeat_host_pattern} ]]; then - if [[ ! "${ALERTMANAGER_HEARTBEAT_URL}" =~ ${heartbeat_url_pattern} ]]; then - echo "::error::ALERTMANAGER_HEARTBEAT_URL must be canonical when it uses hc-ping.com" - exit 1 - fi - alertmanager_heartbeat_check_id="${BASH_REMATCH[1]}" - fi - if [ -n "${cluster_heartbeat_check_id}" ] && \ - [ "${cluster_heartbeat_check_id}" = "${alertmanager_heartbeat_check_id}" ]; then - echo "::error::CLUSTER_HEARTBEAT_URL must use a different external check than ALERTMANAGER_HEARTBEAT_URL" + # The cluster heartbeat is the one off-cluster dead-man signal and has + # an exact hc-ping.com egress policy. Reject a URL the cluster cannot + # reach rather than encrypting a silently dead monitor. Empty stays + # valid here so a local bootstrap needs none; the configuration check + # above already requires it for prod. + heartbeat_url_pattern='^https://hc-ping\.com/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + if [ -n "${CLUSTER_HEARTBEAT_URL}" ] && \ + [[ ! "${CLUSTER_HEARTBEAT_URL}" =~ ${heartbeat_url_pattern} ]]; then + echo "::error::CLUSTER_HEARTBEAT_URL must be a canonical https://hc-ping.com/ URL or be unset" exit 1 fi @@ -288,10 +270,11 @@ jobs: } fill_base k8s/bases/bootstrap/variables-base-secret.enc.yaml - # prod cluster secret + # prod cluster secret. The placeholder still lists the retired Alertmanager + # heartbeat key, which nothing reads; drop it so it is never encrypted. yq -i ' + del(.stringData.alertmanager_heartbeat_url) | .stringData.alertmanager_webhook_url = strenv(ALERTMANAGER_WEBHOOK_URL) | - .stringData.alertmanager_heartbeat_url = strenv(ALERTMANAGER_HEARTBEAT_URL) | .stringData.cluster_heartbeat_url = strenv(CLUSTER_HEARTBEAT_URL) | .stringData.dex_client_secret = strenv(DEX_CLIENT_SECRET) | .stringData.flux_web_client_secret = strenv(FLUX_WEB_CLIENT_SECRET) | diff --git a/.github/workflows/validate-scaffold.yaml b/.github/workflows/validate-scaffold.yaml index 9937429..3cc7f3a 100644 --- a/.github/workflows/validate-scaffold.yaml +++ b/.github/workflows/validate-scaffold.yaml @@ -57,11 +57,23 @@ jobs: - name: πŸ§ͺ Validate heartbeat image contract run: ./scripts/tests/test-heartbeat-image-contract.sh - - name: πŸ”­ Validate default-off observability option - # Coroot stays absent from the default provider paths. The validator also - # renders the explicit local/prod profiles selected through KSail. + - name: πŸ”­ Validate the observability stack + # Coroot is the default on both providers and kube-prometheus-stack is + # retired: the validator renders every provider payload and both cluster + # aliases, and fails on anything that would bring the retired stacks back. run: ./scripts/validate-observability-option.sh + - name: πŸ”” Validate Flux alert coverage + # The production Alert is the only notification for a failed HelmRelease, + # and it has no namespace wildcard: every namespace holding a release or a + # Flux Kustomization must be listed. + run: ./scripts/validate-alert-coverage.sh + + - name: πŸ§ͺ Validate the alert coverage check + # Proves the check above fails on an unwatched namespace, a source without + # a namespace, a named source and a layer that does not render. + run: ./scripts/tests/test-alert-coverage.sh + - name: πŸ” Validate SSO deep-link redirects # Every host behind oauth2-proxy finishes its login on its own host and # returns the user to the page they asked for: Dex must accept exactly diff --git a/AGENTS.md b/AGENTS.md index 29d3b06..58ba2a6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -263,7 +263,10 @@ kustomize k8s/clusters/local/` and `kubectl kustomize k8s/clusters/prod/`. Per f `kubectl apply --dry-run=client -f `. Changes to Coroot, the audit-log-forwarder or the cluster heartbeat also run `./scripts/validate-observability-option.sh` to cover the default layers and their -aliases. **Never run a cluster** (no `ksail cluster create`/`update`/ +aliases. A HelmRelease or Flux Kustomization in a new namespace, or a change to +the production Flux `Alert`, also runs `./scripts/validate-alert-coverage.sh`: the +Alert is the only notification for a failed release and lists every namespace by +name. **Never run a cluster** (no `ksail cluster create`/`update`/ `delete`, no mutating `~/.kube/config`). **Protected β€” never modify** in this repo: `*.enc.yaml`, `.sops.yaml`. Manifest layering (which change belongs in a base and which in an overlay patch) and Flux dependency order are stated diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2f54545..5e15ce3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -73,6 +73,7 @@ kubectl kustomize k8s/clusters/prod/ > /dev/null |---|---| | Anything under `k8s/`, `talos*/`, or a `ksail*.yaml` file | The five commands above | | The observability stack (Coroot, audit-log-forwarder, the cluster heartbeat) | `./scripts/validate-observability-option.sh` | +| A HelmRelease or Flux Kustomization in a new namespace, or the production Flux `Alert` | `./scripts/validate-alert-coverage.sh` and `./scripts/tests/test-alert-coverage.sh` | | Single sign-on hosts or routes (oauth2-proxy, Dex) | `./scripts/tests/test-oauth2-proxy-sso-redirects.sh` | | Cilium default-deny network policies | `./scripts/tests/test-default-deny-shape.sh` | | `.github/workflows/bootstrap.yaml` or its documented inputs | `./scripts/validate-bootstrap-workflow.sh` (needs [actionlint](https://github.com/rhysd/actionlint)) | diff --git a/README.md b/README.md index 8d091c5..e17b09e 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ It is the reusable, automatically bootstrapped form of Cloud in production. - **The services an app needs, already running** β€” Cilium networking and Gateway API, cert-manager certificates, OpenBao secrets, Dex single sign-on, Kyverno policies, Longhorn storage, - CloudNativePG databases, Coroot observability with Prometheus alerting, and Velero backups. + CloudNativePG databases, Coroot observability with Slack alerting, and Velero backups. - **Changes tested before they deploy** β€” CI validates every pull request and system-tests it on a throwaway cluster before the merge queue deploys it to production. - **Your apps from their own repositories** β€” run an application as a *tenant* that deploys from diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 8a09302..5d7f2b2 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -23,8 +23,8 @@ controllers to save resources (noted inline). - **Storage** β€” Longhorn (replicated block / RWX, prod via Hetzner CSI), CloudNativePG (PostgreSQL operator); see [`rwx-storage.md`](rwx-storage.md) - **Autoscaling** β€” Cluster Autoscaler (nodes), Vertical Pod Autoscaler, KEDA + KEDA HTTP add-on; see [`node-autoscaling.md`](node-autoscaling.md) - **Observability** β€” [Coroot](TEMPLATING.md#observability-coroot-is-the-default) (service health, - logs, traces and incidents), with kube-prometheus-stack (Prometheus and Alertmanager, - no web UIs) kept for the metric-backed alerts until that migration is delivered + logs, traces and incidents), Flux notifications for failed reconciliations and an + off-cluster heartbeat; see [`dr/alerting.md`](dr/alerting.md) - **Backup / DR** β€” Velero with CloudNativePG backups to S3-compatible storage (Cloudflare R2 in prod); see [`dr/`](dr) - **Virtualization** β€” KubeVirt + CDI _(local/CI only; disabled on the Hetzner/prod overlay)_ - **Testing** β€” Testkube _(local/CI only; not deployed to prod)_ diff --git a/docs/BOOTSTRAP.md b/docs/BOOTSTRAP.md index 2aa369b..9c69163 100644 --- a/docs/BOOTSTRAP.md +++ b/docs/BOOTSTRAP.md @@ -37,7 +37,7 @@ it some GitHub configuration and one GitHub App; it does the rest: | **GitHub App for cluster SSO** | Dex’s GitHub OIDC connector β€” lets you log in to platform UIs and `kubectl` with GitHub | A *second*, separate App from the bootstrap App below (this one is the cluster’s identity provider). | | **GitHub App for the bootstrap** | Lets the workflow write `prod` environment secrets back | See [Install the bootstrap GitHub App](#1-install-the-bootstrap-github-app). | | **GHCR (GitHub Container Registry)** | Hosts the published OCI manifest artifact | A token with `packages: read/write`. | -| **An external heartbeat monitor** (e.g. [healthchecks.io](https://healthchecks.io)) | Dead-man’s-switch so you’re paged if the whole cluster goes dark | Optional; see [`docs/dr/alerting.md`](dr/alerting.md). | +| **An external heartbeat monitor** ([healthchecks.io](https://healthchecks.io)) | Dead-man’s-switch so you’re notified if the whole cluster goes dark | One check with a five-minute period. Required for a `prod` bootstrap; see [`docs/dr/alerting.md`](dr/alerting.md). | > **You do not need to install SOPS, Age, KSail, or Talos locally to bootstrap.** > The workflow installs its own tooling on the runner. You only need them for @@ -131,9 +131,8 @@ Set these in your **new instance repository** under *Settings β†’ Secrets and va | `CLOUDFLARE_API_TOKEN` | Cloudflare API token β€” DNS edit for the zone (+ Origin CA if used) | Cloudflare β†’ **My Profile β†’ API Tokens**. Scope to the zone’s DNS. | | `R2_ACCESS_KEY_ID` | R2 (S3) access key ID for backups | Cloudflare R2 β†’ **Manage R2 API Tokens**. | | `R2_SECRET_ACCESS_KEY` | R2 (S3) secret access key for backups | Same place as the access key ID. | -| `ALERTMANAGER_WEBHOOK_URL` | Incoming webhook for alert notifications (e.g. Slack) | Your chat platform’s incoming-webhook setup. | -| `ALERTMANAGER_HEARTBEAT_URL` | Ping URL that proves the Prometheus β†’ Alertmanager pipeline is alive | healthchecks.io (or similar) check ping URL. | -| `CLUSTER_HEARTBEAT_URL` | Optional cluster-liveness ping URL for the cluster heartbeat | A **separate**, canonical `https://hc-ping.com/` URL; never reuse the Alertmanager check under another URL spelling. | +| `ALERTMANAGER_WEBHOOK_URL` | Slack incoming webhook for Coroot incident and Flux failure notifications | Slack’s incoming-webhook setup for your alerts channel. | +| `CLUSTER_HEARTBEAT_URL` | Ping URL for the cluster heartbeat, the one off-cluster dead-man signal (required for a `prod` bootstrap) | A canonical `https://hc-ping.com/` check ping URL from healthchecks.io. | | `SSO_GITHUB_APP_CLIENT_SECRET` | **Client secret** of the *cluster SSO* GitHub App (Dex’s GitHub connector) | The SSO App’s settings page (generate a client secret). | | `APP_PRIVATE_KEY` | Private key (`.pem`) of the **bootstrap** GitHub App, also used by the weekly template sync | Generated when you created the bootstrap App. | diff --git a/docs/TEMPLATING.md b/docs/TEMPLATING.md index e239ee3..b61918f 100644 --- a/docs/TEMPLATING.md +++ b/docs/TEMPLATING.md @@ -74,21 +74,35 @@ ksail --config ksail.prod.yaml cluster update ksail --config ksail.prod.yaml workload reconcile ``` -**An instance on the previous default migrates with its next template sync.** -That instance ran Loki, Alloy and OpenCost. When the sync merges and Flux -reconciles it, Flux removes the Loki, Alloy and OpenCost releases and the -`opencost` namespace, and installs Coroot. No KSail config changes. Two things -do not carry over: logs held in Loki and cost data held in OpenCost are not -migrated, and Loki's log volume is a StatefulSet claim that an uninstall leaves -behind. Delete that leftover PersistentVolumeClaim in the `monitoring` namespace -once the logs are no longer needed. - -The stack is transitional in one respect. It runs Coroot, has no OpenCost, -has no Loki or Alloy log path, and keeps kube-prometheus-stack -until the metrics and alerting migration is proven separately. Cost allocation is therefore unavailable. -Coroot is the only observability UI: the Grafana, Prometheus and Alertmanager -web UIs, their routes, SSO entries and Grafana sign-in secret are not published, while -Prometheus and Alertmanager keep running for alerting. Docker runs Coroot +**An instance on an earlier stack migrates with its next template sync.** +An earlier instance ran Loki, Alloy and OpenCost, and every instance ran +kube-prometheus-stack. When the sync merges and Flux reconciles it, Flux removes +the Loki, Alloy and OpenCost releases, the kube-prometheus-stack release and the +`opencost` and `monitoring` namespaces, and installs Coroot. No KSail config +changes. Four things need your attention: + +- Nothing is migrated, and the old data is deleted. Flux removes the + `monitoring` namespace with everything in it: the Prometheus and Alertmanager + volumes, their silences and metric history, and a Loki volume an earlier + uninstall left behind. Take what you need from them before the sync merges. +- Helm does not delete CustomResourceDefinitions. The Prometheus operator's + `monitoring.coreos.com` definitions stay until you delete them, and a chart + that creates a ServiceMonitor when they exist keeps doing so until then. +- Set the `CLUSTER_HEARTBEAT_URL` secret (and `cluster_heartbeat_url` in the + production cluster secret) before syncing. It is now the only off-cluster + dead-man signal, and `ALERTMANAGER_HEARTBEAT_URL` is no longer read. You can + reuse the old check: nothing else pings it once Alertmanager is gone. Give it + a five-minute period and a ten-minute grace. +- `talos/` belongs to the instance, so the sync does not touch it. Delete + `talos/control-planes/allow-node-exporter-ingress.yaml` and + `talos/workers/allow-node-exporter-ingress.yaml`: nothing listens on that + port any more. + +The stack runs Coroot and nothing beside it. It has no OpenCost, +has no Loki or Alloy log path, and has no kube-prometheus-stack: no Prometheus +operator, Alertmanager, node-exporter or kube-state-metrics, and no +`monitoring.coreos.com` resource in any payload. Cost allocation is therefore unavailable. +Coroot is the only observability UI. Docker runs Coroot without the audit log forwarder, while Hetzner includes the forwarder for host audit logs. Because Coroot's eBPF agent and the production audit collector need host access, the `observability` namespace is exempt from the @@ -98,24 +112,27 @@ Production / Hetzner reuses the existing encrypted webhook variable for Coroot incident and resolution notifications. Its matching provider policy permits only `hooks.slack.com:443`, so keep this value a Slack incoming-webhook URL. Flux notification-controller reuses the same `alertmanager_webhook_url` in -production to report failed reconciliations. The +production to report failed reconciliations; the variable keeps its earlier +name so an existing instance needs no secret change. The controller layer applies both Flux alerts before it waits for every controller -to become ready: one forwards all `Kustomization` error events, while a +to become ready: one forwards the error events of every `Kustomization` and +every `HelmRelease`, while a narrow info-severity path for dependency waits covers a prerequisite that is -not ready. The legacy Prometheus rule keeps only HelmRelease coverage, so a -Kustomization failure reaches Slack once. There -is no fourth secret or polling workload to configure. +not ready. There is no polling workload to configure. +notification-controller has no namespace wildcard, so the error alert lists +every namespace that holds a `HelmRelease`, and `scripts/validate-alert-coverage.sh` +fails CI when one is missing. An instance that adds a release in a new namespace +adds that namespace to the alert. Per-alert notifications remain visible only in the Coroot UI; this avoids sending every noisy alert to the shared webhook. Local / Docker Coroot stays notification-free. The apps layer also stages one hardened `cluster-heartbeat` CronJob for the cluster -dead-man signal. It uses a separate optional `CLUSTER_HEARTBEAT_URL` secret so -the existing Alertmanager Watchdog check keeps proving the Prometheus-to-Alertmanager -pipeline. Never point both inputs at the same monitor. The CronJob runs at +dead-man signal. It is the only one, so a production bootstrap requires the +`CLUSTER_HEARTBEAT_URL` secret. The CronJob runs at platform-critical priority, and its dedicated policy permits only `hc-ping.com:443` plus that hostname's exact DNS query. Bootstrap accepts only a canonical -`https://hc-ping.com/` URL and compares check identities, so -fragments, query strings, or trailing slashes cannot make both inputs target the -same external check under different spellings. +`https://hc-ping.com/` URL, so +fragments, query strings, or trailing slashes cannot encrypt a monitor the +cluster cannot reach. The substituted bearer URL lives in a Kubernetes Secret and reaches the container through a Secret-backed environment variable. A shell builtin feeds it to curl as stdin configuration, so neither the PodSpec nor curl's runtime argument vector @@ -124,13 +141,12 @@ The infrastructure layer narrows the generated namespace policies around the CronJob's purpose-specific label. The dependent apps layer stages the heartbeat only after that infrastructure reconciliation is ready, preventing an existing namespace-wide deny policy from blocking the new workload during an upgrade. The invalid URL -fallback keeps local and unconfigured instances inert. Both checks -keep Watchdog unchanged on the distinct Alertmanager pipeline monitor, -which preserves a dead-man signal until the apps layer starts the direct check. +fallback keeps local and unconfigured instances inert. Flux notification-controller now owns reconciliation errors in production, including dependency waits. -Kube-prometheus-stack keeps owning its remaining metric-backed alert rules and -the Watchdog pipeline until those migrations are delivered separately. +Some Prometheus alert rules have no replacement: certificate expiry, backup +freshness and autoscaler failures are not alerted today. +[`docs/dr/alerting.md`](dr/alerting.md) lists what is and is not alerted. Open the Coroot UI at `https://observability.`. Community Edition does not provide native OIDC, so every UI request goes through the @@ -200,7 +216,7 @@ resources Flux reads that are genuinely per-cluster: replica counts, Hetzner LB location/type, Longhorn settings, etc.). Bootstrap fills `domain`, `domain_regex`, `github_app_client_id`, and `admin_email` from your Variables; the rest are sensible defaults you can tune. -- `variables-cluster-secret.enc.yaml` β€” SOPS-encrypted secrets (Alertmanager URLs, +- `variables-cluster-secret.enc.yaml` β€” SOPS-encrypted secrets (the alert webhook and heartbeat URLs, the OIDC/cookie secrets, the GitHub SSO client secret, the Hetzner/R2 tokens). Bootstrap fills these from your Secrets and `sops -e` encrypts them. @@ -241,7 +257,7 @@ plus which values the bootstrap auto-generates β€” live in `APP_CLIENT_ID`. - **Secrets you set:** `HCLOUD_TOKEN`, `GHCR_TOKEN`, `CLOUDFLARE_API_TOKEN`, `R2_ACCESS_KEY_ID`, `R2_SECRET_ACCESS_KEY`, `ALERTMANAGER_WEBHOOK_URL`, - `ALERTMANAGER_HEARTBEAT_URL`, `SSO_GITHUB_APP_CLIENT_SECRET`, `APP_PRIVATE_KEY`. + `CLUSTER_HEARTBEAT_URL`, `SSO_GITHUB_APP_CLIENT_SECRET`, `APP_PRIVATE_KEY`. - **Auto-generated by bootstrap:** `SOPS_AGE_KEY`, and the `prod`-environment `KUBE_CONFIG` / `TALOS_CONFIG`. diff --git a/docs/dr/alerting.md b/docs/dr/alerting.md index 3ac907b..f309cc8 100644 --- a/docs/dr/alerting.md +++ b/docs/dr/alerting.md @@ -2,224 +2,170 @@ **Coroot** is the observability stack: service health, logs, traces, profiles and incidents, collected by an eBPF node agent with no application changes. -`kube-prometheus-stack` (Prometheus + Alertmanager + node-exporter + -kube-state-metrics) still runs beside it for the metric-backed alert rules and -the `Watchdog` dead-man signal; its Grafana, Prometheus and Alertmanager web UIs -are not published. Self-hosted, no SaaS metrics tier. The stack is -production-hardened along four axes: - -1. **Alerts go to Slack** via Alertmanager's native `slack_configs`, Coroot's - incident notifications and Flux's own reconciliation alerts. -2. **Two independent dead-man checks** separate the Prometheus β†’ Alertmanager - pipeline from direct cluster liveness, so one sender cannot hide the other - path's failure. -3. **State is persistent** β€” Prometheus and Alertmanager use Hetzner - Cloud block volumes in prod, and Velero ships the `monitoring` - namespace to R2 daily (24 h RPO). -4. **One UI, behind SSO**: Coroot at `https://observability.${domain}`, reached - only through oauth2-proxy. - -The earlier stack (Loki, Alloy, OpenCost and the Grafana UI) is gone. There is -no cost allocation today. [`TEMPLATING.md`](../TEMPLATING.md#observability-coroot-is-the-default) -says how an instance that ran the earlier stack migrates. +Self-hosted, no SaaS metrics tier. There is no kube-prometheus-stack: no +Prometheus operator, Alertmanager, node-exporter or kube-state-metrics, and no +`monitoring` namespace. Alerting has three parts: + +1. **Coroot incidents go to Slack.** Coroot inspects every workload and node + and posts incident and resolution notifications in production. +2. **Flux reports its own failures.** Flux notification-controller posts + reconciliation errors for every `Kustomization` and every `HelmRelease`. +3. **One off-cluster dead-man check.** The `cluster-heartbeat` CronJob pings an + external monitor, so a cluster that is down still gets noticed. + +There is one UI, behind SSO: Coroot at `https://observability.${domain}`, +reached only through oauth2-proxy. + +The earlier stacks (Loki, Alloy, OpenCost, Grafana and kube-prometheus-stack) +are gone. There is no cost allocation today. +[`TEMPLATING.md`](../TEMPLATING.md#observability-coroot-is-the-default) +says how an instance that ran them migrates. ## Components -| Component | Role | Persistence (prod) | -| ------------------ | --------------------------------------------------- | ------------------------- | -| Coroot | UI, incidents, service map, alert inspections | PVC, 2 Gi | -| Coroot Prometheus | Coroot's own metric store | PVC, 10 Gi | -| Coroot ClickHouse | Logs, traces and profiles, 3 d retention | PVC, 10 Gi | -| Coroot node agent | eBPF telemetry from every node (DaemonSet) | n/a | -| Audit log forwarder | Ships Talos host audit logs to Coroot (prod only) | n/a | -| Prometheus | Metrics + alert evaluation, 14 d / 5 GiB retention | `hcloud` PVC, 20 Gi | -| Alertmanager | Routing, grouping, silences (2 replicas, gossiped) | `hcloud` PVC, 2 Gi | -| node-exporter | Node metrics (DaemonSet) | n/a | -| kube-state-metrics | Kubernetes object metrics | n/a | -| cluster-heartbeat | Direct dead-man ping every five minutes (CronJob) | n/a | - -Local/CI runs the same stack without the audit log forwarder, with Prometheus -and Alertmanager on emptyDir β€” losing those on a restart is fine there. The -`hcloud` PVC overrides for Prometheus and Alertmanager live in the hetzner -overlay (`k8s/providers/hetzner/.../*/patches/`), the same way OpenBao gets -block storage. +| Component | Role | Persistence (prod) | +| ------------------- | -------------------------------------------------- | ------------------ | +| Coroot | UI, incidents, service map, alert inspections | PVC, 2 Gi | +| Coroot Prometheus | Coroot's own metric store | PVC, 10 Gi | +| Coroot ClickHouse | Logs, traces and profiles, 3 d retention | PVC, 10 Gi | +| Coroot node agent | eBPF telemetry from every node (DaemonSet) | n/a | +| Audit log forwarder | Ships Talos host audit logs to Coroot (prod only) | n/a | +| Flux notifications | `Provider` and two `Alert` objects (prod only) | n/a | +| cluster-heartbeat | Dead-man ping every five minutes (CronJob) | n/a | + +Local/CI runs the same stack without the audit log forwarder and without the +Flux notification path. Coroot's bundled Prometheus is its own metric store; it +is not a Prometheus operator and installs no `monitoring.coreos.com` resources. ## Alert routing β†’ Slack -Alertmanager sends to a Slack incoming webhook (`slack_configs` with -`api_url_file`). The URL is per-cluster and SOPS-encrypted, so local -clusters get an invalid URL and stay quiet by design. +In production, Coroot posts its incident and resolution notifications to a +Slack incoming webhook. The URL is per-cluster and SOPS-encrypted. Coroot's +per-alert notifications stay in the Coroot UI, so a noisy alert does not flood +the channel. Local / Docker renders no webhook and stays quiet by design. -- `critical` β†’ Slack immediately, repeat every 12 h. -- `warning` β†’ Slack, repeat every 24 h. -- `critical` inhibits matching `warning` (same alertname/cluster/namespace). +Flux notification-controller now owns reconciliation errors in +production. Two event-driven `Alert` objects post through the same webhook: -In production, Coroot posts its incident and resolution notifications to the -same webhook. Its per-alert notifications stay in the Coroot UI, so a noisy -alert does not flood the channel. +- `platform-reconciliation` forwards `error` events for failed builds, applies + and health checks. It watches every `Kustomization` and every `HelmRelease`. +- `platform-dependency-wait` forwards only the `info` events of a + `Kustomization` whose prerequisite is not ready. + +The controller layer applies them before it waits for every controller, so the +notification path already exists if that layer cannot become ready. +Local / Docker renders no Slack `Provider`, `Alert`, or webhook `Secret`. + +### Why HelmRelease is watched directly + +A failing `HelmRelease` can be invisible to its parent `Kustomization`. With +Helm remediation on, a failed upgrade is rolled back to the last good release. +The release is then Ready again, the parent stays healthy, and the release +silently stops advancing. The Prometheus rule `FluxHelmReleaseNotReady` used to +report that; the `Alert` reports the failed upgrade itself. + +notification-controller has no namespace wildcard, so the `Alert` lists every +namespace that holds a `HelmRelease`. `scripts/validate-alert-coverage.sh` +fails CI when a release appears in a namespace that is not listed. **An +instance that adds a release in a new namespace adds that namespace to +`k8s/providers/hetzner/infrastructure/controllers/flux-notifications/alert.yaml`.** ## Dead-man's-switch (off-cluster heartbeat) -In-cluster Alertmanager cannot tell you the cluster is down β€” it's down -too. The `Watchdog` alert is routed to a -dedicated `heartbeat` receiver that POSTs to an **external** monitor on a -tight cadence (`repeat_interval: 50s`). If the cluster β€” or the Prometheus β†’ -Alertmanager pipeline β€” dies, the monitor stops receiving pings and notifies -Slack out-of-band. - -A dedicated `cluster-heartbeat` CronJob pings a -**second external check** every five minutes independently of Coroot. It never -reuses `alertmanager_heartbeat_url`: doing so would let the CronJob keep that -monitor green while the Prometheus-to-Alertmanager pipeline was down. -Watchdog stays enabled on its existing check. -The CronJob's separate `cluster_heartbeat_url` accepts only a canonical -`https://hc-ping.com/` URL; its network policy allows only -`hc-ping.com:443` plus that hostname's exact DNS query. Bootstrap compares -canonical check identities, so URL fragments, query strings, and trailing slashes -cannot disguise reuse of the Alertmanager check. The infrastructure layer -narrows the namespace deny and DNS policies around this purpose-labelled -workload. The dependent apps layer -stages the CronJob only after that exclusion is ready, while Watchdog preserves the -original dead-man path until the apps reconciliation completes. An invalid fallback -keeps an unconfigured CronJob quiet. +Nothing inside the cluster can tell you the cluster is down. The +`cluster-heartbeat` CronJob pings an **external** monitor every five minutes. +If the cluster dies, the monitor stops receiving pings and notifies Slack +out-of-band. It is the platform's one dead-man signal, so a production +bootstrap refuses to start without it. + +`cluster_heartbeat_url` accepts only a canonical +`https://hc-ping.com/` URL; the CronJob's network policy allows +only `hc-ping.com:443` plus that hostname's exact DNS query. Bootstrap rejects +any other spelling, because a URL the cluster cannot reach is a monitor that +is silently dead. The infrastructure layer narrows the namespace deny and DNS +policies around this purpose-labelled workload, and the dependent apps layer +stages the CronJob only after that exclusion is ready. An invalid fallback +keeps an unconfigured CronJob quiet in local/CI. The bearer URL reaches the CronJob through a Secret-backed environment variable. The shell feeds it to curl as stdin configuration, so the credential is absent from both the PodSpec and curl's runtime argument vector. -Flux notification-controller now owns reconciliation errors in -production. Two event-driven `Alert` objects watch every `Kustomization`: -one forwards `error` events for failed builds, applies, and health checks; the -other forwards only `info` dependency waits whose prerequisite is not ready. -The controller layer applies them before it waits for every controller, so the -notification path already exists if that layer cannot become ready. Both post -through the existing `alertmanager_webhook_url`. -Local / Docker renders no Slack `Provider`, `Alert`, or -webhook `Secret`. -`kube-prometheus-stack` remains transitional for Watchdog and the remaining -metric-backed alert rules until those paths are migrated separately. -Its Grafana, Prometheus and Alertmanager web UIs are -not published; the alerts themselves still run. Recommended monitor: [healthchecks.io](https://healthchecks.io) (free, -open-source, native Slack integration). Keep two distinct checks: the existing -tight-cadence Watchdog URL in `alertmanager_heartbeat_url`, and a ~5 min period / -~10 min grace direct-cluster check in `cluster_heartbeat_url`. Connect both to -Slack, but never give them the same ping URL. A self-hosted alternative for the -Alertmanager check is supported; the CronJob deliberately accepts only -`hc-ping.com` because its egress policy is fail closed. Another alternative is a -scheduled GitHub Actions workflow that probes the public Gateway and posts -to Slack β€” fully under your control, no third-party monitor. - -Both URLs are injected by Flux substitution. An unset `cluster_heartbeat_url` -defaults to an invalid URL, so local/CI and instances without the optional -second check stay inert while Watchdog continues unchanged. - -## Off-cluster metric backup - -There is no remote-write or SaaS mirror. Instead, the persistent -Prometheus and Alertmanager volumes live in the `monitoring` -namespace, which Velero's `daily-full` schedule backs up to R2 every day -(`includedNamespaces: ["*"]`, Kopia fs-backup). Restore is the standard -Velero flow in [runbook.md](./runbook.md). Backups are filesystem-level -and crash-consistent (Prometheus recovers via its WAL on restore); -fine for a 24 h RPO. Coroot's stores live in the `observability` namespace and are -covered by the same schedule. - -## What gets alerted - -Two sources: - -1. **Curated chart default rules** (`defaultRules.create: true`). We keep - the well-tested groups β€” `general` (incl. `Watchdog`), `alertmanager`, - `prometheus`, `prometheusOperator`, `kubernetesApps` - (CrashLooping/ReplicasMismatch/…), `kubernetesStorage`, `node`, - `kubeStateMetrics` β€” and disable the groups for control-plane - components we don't scrape (etcd, kube-apiserver/-scheduler/-controller- - manager, kube-proxy, windows). `KubeCPUOvercommit` / `KubeMemoryOvercommit` - are disabled β€” guaranteed noise on a cluster that runs hot on purpose. - -2. **Platform-specific rules** in - `k8s/bases/infrastructure/alerts/platform-critical.yaml` (not in the - chart): Velero/CNPG backups, Flux reconciliation, cert-manager expiry, - cluster-autoscaler and resource-pressure. - -| Alert | Severity | Why | -| --------------------------- | -------- | ----------------------------------------- | -| `Watchdog` | none | Always firing β†’ external heartbeat | -| `NodeNotReady` | critical | Single node loss | -| `PersistentVolumeFillingUp` | critical | >90% PVC | -| `CertificateExpiringSoon` | warning | <14 d to expiry, cert-manager not renewing | -| `FluxKustomizationNotReady` | critical | Reconciliation broken >15 min | -| `VeleroNoRecentBackup` | critical | RPO breach β€” no successful backup in 30h | -| `CNPGClusterDegraded` | critical | Primary alone, no streaming replica | - -(plus the chart's workload/storage/self-monitoring alerts.) - -In production, the event-driven Flux notifications replace -the Kustomization half of `FluxKustomizationNotReady`. A production patch renames -the remaining metric rule to `FluxHelmReleaseNotReady` and narrows it to -HelmRelease readiness, so the two paths do not send the same failure twice. The -other table entries still come from `kube-prometheus-stack` while it is -transitional. +open-source, native Slack integration). Create one check with a ~5 min period +and ~10 min grace and connect it to Slack. The CronJob deliberately accepts only +`hc-ping.com` because its egress policy is fail closed. + +## What is alerted, and what is not + +| Signal | Where it shows | +| ------------------------------------------- | ------------------------------------------- | +| A workload misses its objectives | Coroot incident β†’ Slack | +| Node, pod, memory, disk and restart health | Coroot inspections, in the Coroot UI | +| A `Kustomization` fails to build or apply | Flux `Alert` β†’ Slack | +| A `HelmRelease` fails to install or upgrade | Flux `Alert` β†’ Slack | +| A prerequisite layer is not ready | Flux `Alert` β†’ Slack (dependency wait) | +| The cluster stops answering | External monitor β†’ Slack | + +**Not alerted today.** The Prometheus rules that left with kube-prometheus-stack +had no direct equivalent in Coroot's model, and these are not replaced: + +- certificate expiry (`CertificateExpiringSoon`); +- a failed or stale Velero backup, and a CloudNativePG cluster without a recent + backup or a streaming replica; +- cluster-autoscaler scale-up failures. + +Check backups with the restore drill in [restore-drill.md](./restore-drill.md) +and certificates with `kubectl get certificates -A` until an instance adds its +own check for them. + +## Off-cluster backup + +There is no remote-write or SaaS mirror. Coroot's stores live in the +`observability` namespace, which Velero's `daily-full` schedule backs up to R2 +every day (`includedNamespaces: ["*"]`, Kopia fs-backup). Restore is the +standard Velero flow in [runbook.md](./runbook.md). Backups are +filesystem-level and crash-consistent; fine for a 24 h RPO. ## Per-environment setup (manual SOPS steps) -The three secret inputs β€” `alertmanager_webhook_url`, -`alertmanager_heartbeat_url`, and `cluster_heartbeat_url` β€” live in the -per-cluster `variables-cluster-secret.enc.yaml` (under `bootstrap/`) and must -be set by hand. All three are read from the `Secret` `variables-cluster`, which -is a Flux `substituteFrom` source. +The two secret inputs β€” `alertmanager_webhook_url` and +`cluster_heartbeat_url` β€” live in the per-cluster +`variables-cluster-secret.enc.yaml` (under `bootstrap/`). Both are read from +the `Secret` `variables-cluster`, which is a Flux `substituteFrom` source. The +Bootstrap workflow writes them from the `ALERTMANAGER_WEBHOOK_URL` and +`CLUSTER_HEARTBEAT_URL` repository secrets; to change one later: ```bash -# 1. Slack incoming webhook for Alertmanager, Coroot, and Flux notifications. +# 1. Slack incoming webhook for Coroot and Flux notifications. sops --set '["stringData"]["alertmanager_webhook_url"] "https://hooks.slack.com/services/XXX/YYY/ZZZ"' \ k8s/clusters/prod/bootstrap/variables-cluster-secret.enc.yaml -# 2. Prometheus β†’ Alertmanager pipeline check (healthchecks.io or similar). -sops --set '["stringData"]["alertmanager_heartbeat_url"] "https://hc-ping.com/"' \ - k8s/clusters/prod/bootstrap/variables-cluster-secret.enc.yaml - -# 3. Separate direct-cluster check for the cluster heartbeat (hc-ping.com only). -sops --set '["stringData"]["cluster_heartbeat_url"] "https://hc-ping.com/"' \ +# 2. The cluster heartbeat's external check (hc-ping.com only). +sops --set '["stringData"]["cluster_heartbeat_url"] "https://hc-ping.com/"' \ k8s/clusters/prod/bootstrap/variables-cluster-secret.enc.yaml ``` -Slack side: create an incoming webhook for your alerts channel (e.g. -`#platform-alerts`) β€” the channel in the config is cosmetic, since an incoming -webhook posts to the channel it was created for. On healthchecks.io, create two -distinct checks, connect each check's Slack integration, and copy their different -ping URLs into `alertmanager_heartbeat_url` and `cluster_heartbeat_url` respectively. +The webhook variable keeps its earlier name so an existing instance needs no +secret change. Slack side: create an incoming webhook for your alerts channel +(e.g. `#platform-alerts`). On healthchecks.io, create one check, connect its +Slack integration, and copy its ping URL into `cluster_heartbeat_url`. -| Env | `alertmanager_webhook_url` | `alertmanager_heartbeat_url` | `cluster_heartbeat_url` | -| ----- | ------------------------------- | ---------------------------------- | --------------------------------------- | -| local | invalid URL (alerts stay local) | unset β†’ invalid | unset β†’ invalid | -| prod | Slack alerts-channel webhook | pipeline-monitor ping URL | distinct cluster-monitor ping URL | +| Env | `alertmanager_webhook_url` | `cluster_heartbeat_url` | +| ----- | ------------------------------- | ------------------------ | +| local | invalid URL (alerts stay local) | unset β†’ invalid | +| prod | Slack alerts-channel webhook | cluster-monitor ping URL | -## On-call: silence and inspect +## On-call: inspect - **Investigate** in Coroot at `https://observability.${domain}`: service health, logs, traces and incidents. It is behind GitHub SSO (oauth2-proxy, restricted to the operator). -- **Silence or inspect a firing alert**: Alertmanager still sends the metric-backed alerts, - but its web UI is not published, so reach it through the cluster API with your own - kubeconfig. Run - `kubectl -n monitoring port-forward svc/kube-prometheus-stack-alertmanager 9093`, open - `http://localhost:9093` and use Silences β†’ New. For Prometheus, use - `svc/kube-prometheus-stack-prometheus 9090` the same way. - -## Resource footprint (prod) - -| Component | Requests | Limits | -| -------------- | ----------------- | ----------- | -| Prometheus | 50m / 256 Mi | β€” / 1.5 Gi | -| Alertmanager | 50m / 64 Mi (Γ—2) | β€” / 128 Mi | -| Operator | 50m / 128 Mi | β€” / 256 Mi | - -VPA right-sizes the requests at runtime (RequestsOnly), so the limits are -the real ceilings. The always-on tier (Coroot and -persistent Prometheus + Alertmanager) is what motivates the planned bump to a 4th -static worker (`ksail.prod.yaml`); it's held at 3 today while prod is -right-sized, and the hetzner overlay opts out of kubevirt/cdi to free -~1.5 GiB so the tier still fits. +- **A Flux alert** names the failing object. Read it with + `kubectl -n describe helmrelease ` or + `kubectl -n flux-system describe kustomization `. +- **There is nothing to silence.** Flux notifications are deduplicated and + rate-limited by notification-controller, and Coroot resolves an incident + itself when the workload recovers. ## Related diff --git a/docs/secret-rotation.md b/docs/secret-rotation.md index 99fe564..8693f14 100644 --- a/docs/secret-rotation.md +++ b/docs/secret-rotation.md @@ -78,7 +78,7 @@ preferring OpenBao-native mechanisms. [`k8s/bases/infrastructure/vault-seed/`](../k8s/bases/infrastructure/vault-seed)) seed OpenBao KV once; `ExternalSecret`s sync to consumer namespaces (1 h refresh). Used by app DB/cache credentials, OIDC client secrets for the demo apps, the - Cloudflare token, the R2 credentials, and the Alertmanager URLs. + Cloudflare token and the R2 credentials. - **SOPS β†’ Flux postBuild substitution**: `${dex_client_secret}`, `${flux_web_client_secret}`, `${oauth2_proxy_cookie_secret}` etc. are still read from `k8s/clusters/*/bootstrap/variables-cluster-secret.enc.yaml`. **Dex (the OIDC diff --git a/k8s/bases/bootstrap/kustomization.yaml b/k8s/bases/bootstrap/kustomization.yaml index ccf3bd0..2a55f1b 100644 --- a/k8s/bases/bootstrap/kustomization.yaml +++ b/k8s/bases/bootstrap/kustomization.yaml @@ -6,5 +6,5 @@ resources: - variables-base-secret.enc.yaml # Cluster-scoped PriorityClasses (provider- and cluster-agnostic) live in the # bootstrap layer so they exist before infrastructure-controllers reconciles β€” - # e.g. kube-prometheus-stack references platform-critical. + # e.g. Coroot and the cluster heartbeat reference platform-critical. - priority-classes/ diff --git a/k8s/bases/bootstrap/priority-classes/platform-critical.yaml b/k8s/bases/bootstrap/priority-classes/platform-critical.yaml index 8d35c33..64e17d9 100644 --- a/k8s/bases/bootstrap/priority-classes/platform-critical.yaml +++ b/k8s/bases/bootstrap/priority-classes/platform-critical.yaml @@ -1,6 +1,6 @@ # Priority tier for important platform add-ons that are NOT core Kubernetes / # node infrastructure but must still survive node memory pressure β€” currently -# the monitoring & alerting stack (kube-prometheus-stack). +# Coroot and the cluster heartbeat. # # Sits ABOVE normal workloads (priority 0) so the kubelet evicts workload pods # first, but BELOW the built-in system-* classes (2000000000) so true cluster @@ -10,8 +10,8 @@ # "important platform add-on" tier. # # Deliberately a custom class rather than system-cluster-critical: putting the -# monitoring stack at the very top would rank it alongside core infra, so a -# memory-hungry Prometheus would be reclaimed only after everything else β€” +# observability stack at the very top would rank it alongside core infra, so a +# memory-hungry metric store would be reclaimed only after everything else β€” # pushing the node toward the kernel OOM-killer. platform-critical keeps it # "evicted after normal workloads" while still leaving it reclaimable before # true cluster/node infrastructure. diff --git a/k8s/bases/infrastructure/alerts/kustomization.yaml b/k8s/bases/infrastructure/alerts/kustomization.yaml deleted file mode 100644 index ff72bfc..0000000 --- a/k8s/bases/infrastructure/alerts/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - platform-critical.yaml diff --git a/k8s/bases/infrastructure/alerts/platform-critical.yaml b/k8s/bases/infrastructure/alerts/platform-critical.yaml deleted file mode 100644 index 7731219..0000000 --- a/k8s/bases/infrastructure/alerts/platform-critical.yaml +++ /dev/null @@ -1,161 +0,0 @@ -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: platform-critical - namespace: monitoring - labels: - prometheus: kube-prometheus - role: alert-rules -spec: - groups: - - name: nodes - rules: - - alert: NodeNotReady - expr: kube_node_status_condition{condition="Ready",status="true"} == 0 - for: 10m - labels: - severity: critical - annotations: - summary: "Node {{ $labels.node }} not Ready" - description: "Node {{ $labels.node }} has been NotReady for more than 10 minutes." - - - alert: NodeDiskFillingUp - expr: | - (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|squashfs"} - / node_filesystem_size_bytes{fstype!~"tmpfs|squashfs"})) * 100 > 90 - for: 30m - labels: - severity: warning - annotations: - summary: "Node {{ $labels.instance }} disk >90 % full" - description: "Filesystem {{ $labels.mountpoint }} is {{ $value | humanize }}% full." - - - name: storage - rules: - - alert: PersistentVolumeFillingUp - expr: | - (kubelet_volume_stats_used_bytes - / kubelet_volume_stats_capacity_bytes) * 100 > 90 - for: 30m - labels: - severity: critical - annotations: - summary: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} >90 % full" - description: "PV {{ $labels.persistentvolumeclaim }} usage at {{ $value | humanize }}%." - - - name: certificates - rules: - - alert: CertificateExpiringSoon - expr: | - (certmanager_certificate_expiration_timestamp_seconds - time()) / 86400 < 14 - for: 1h - labels: - severity: warning - annotations: - summary: "Certificate {{ $labels.namespace }}/{{ $labels.name }} expires in <14d" - description: "Renewal hasn't happened. cert-manager logs: kubectl logs -n cert-manager -l app=cert-manager" - - - name: flux - rules: - - alert: FluxKustomizationNotReady - expr: | - gotk_reconcile_condition{type="Ready",status="False", - kind=~"Kustomization|HelmRelease"} == 1 - for: 15m - labels: - severity: critical - annotations: - summary: "Flux {{ $labels.kind }} {{ $labels.namespace }}/{{ $labels.name }} not Ready" - description: "kubectl describe {{ $labels.kind }} -n {{ $labels.namespace }} {{ $labels.name }}" - - - name: backups - rules: - - alert: VeleroBackupFailed - expr: increase(velero_backup_failure_total[1h]) > 0 - labels: - severity: critical - annotations: - summary: "Velero backup failure(s) in the last hour" - description: "{{ $value }} Velero backup(s) failed. kubectl -n velero get backups | grep -v Completed" - - - alert: VeleroNoRecentBackup - expr: | - time() - max(velero_backup_last_successful_timestamp) > 60 * 60 * 30 - labels: - severity: critical - annotations: - summary: "No successful Velero backup in >30h" - description: "RPO breached. Latest backup is older than 30h; daily schedule should run every 24h." - - - alert: CNPGNoRecentBackup - expr: | - time() - cnpg_collector_last_failed_backup_timestamp == 0 - unless on(cluster) - time() - cnpg_collector_last_available_backup_timestamp < 60 * 60 * 30 - labels: - severity: critical - annotations: - summary: "CNPG cluster {{ $labels.cluster }} has no successful backup in >30h" - description: "kubectl cnpg status {{ $labels.cluster }}" - - - alert: CNPGClusterDegraded - expr: cnpg_pg_replication_in_recovery == 0 and on(namespace, cluster) cnpg_pg_replication_streaming_replicas < 1 - for: 15m - labels: - severity: critical - annotations: - summary: "CNPG cluster {{ $labels.cluster }} has no streaming replica" - description: "Single instance left -- a primary failure would lose data up to the last WAL archive." - - - name: resource-pressure - rules: - - alert: ClusterCPUUtilizationHigh - expr: | - (1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m]))) * 100 > 80 - for: 15m - labels: - severity: warning - annotations: - summary: "Cluster-wide CPU utilization >80 %" - description: "Average CPU across all nodes is {{ $value | humanize }}%. Cluster Autoscaler should add capacity; check for stuck scale-up." - - - alert: ClusterMemoryUtilizationHigh - expr: | - (1 - sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes)) * 100 > 85 - for: 15m - labels: - severity: warning - annotations: - summary: "Cluster-wide memory utilization >85 %" - description: "Aggregate memory usage is {{ $value | humanize }}%. Verify Cluster Autoscaler is scaling and VPA is adjusting pod requests." - - - alert: PodsFailedScheduling - expr: | - increase(scheduler_pending_pods{queue="unschedulable"}[10m]) > 0 - or count(kube_pod_status_phase{phase="Pending"} == 1) > 3 - for: 10m - labels: - severity: critical - annotations: - summary: "Pods stuck in Pending for >10 m" - description: "Pods cannot be scheduled β€” likely insufficient node resources. kubectl get pods -A --field-selector=status.phase=Pending" - - - alert: ClusterAutoscalerScaleUpFailed - expr: | - increase(cluster_autoscaler_failed_scale_ups_total[15m]) > 0 - for: 5m - labels: - severity: critical - annotations: - summary: "Cluster Autoscaler failed to scale up" - description: "{{ $value }} scale-up attempts failed. Check autoscaler logs: kubectl -n kube-system logs -l app.kubernetes.io/name=cluster-autoscaler --tail=100" - - - alert: ClusterAutoscalerUnneededNodes - expr: | - cluster_autoscaler_unneeded_nodes_count > 0 - for: 30m - labels: - severity: info - annotations: - summary: "{{ $value }} node(s) underutilized for >30 m" - description: "Autoscaler should scale down soon. If cost is a concern, verify scale-down-unneeded-time." diff --git a/k8s/bases/infrastructure/cluster-policies/best-practices/add-security-context.yaml b/k8s/bases/infrastructure/cluster-policies/best-practices/add-security-context.yaml index 82d6cf7..eafd862 100644 --- a/k8s/bases/infrastructure/cluster-policies/best-practices/add-security-context.yaml +++ b/k8s/bases/infrastructure/cluster-policies/best-practices/add-security-context.yaml @@ -44,7 +44,6 @@ spec: - kubescape - kubevirt - cdi - - monitoring - velero - observability mutate: @@ -73,7 +72,6 @@ spec: - kubescape - kubevirt - cdi - - monitoring - velero - observability mutate: diff --git a/k8s/bases/infrastructure/cluster-policies/best-practices/validate-host-restrictions.yaml b/k8s/bases/infrastructure/cluster-policies/best-practices/validate-host-restrictions.yaml index 0408f27..41f41fb 100644 --- a/k8s/bases/infrastructure/cluster-policies/best-practices/validate-host-restrictions.yaml +++ b/k8s/bases/infrastructure/cluster-policies/best-practices/validate-host-restrictions.yaml @@ -47,7 +47,6 @@ spec: - kubescape - kubevirt - cdi - - monitoring - velero - observability - resources: @@ -79,7 +78,6 @@ spec: - kubescape - kubevirt - cdi - - monitoring - velero - observability - resources: diff --git a/k8s/bases/infrastructure/cluster-policies/best-practices/validate-pod-security.yaml b/k8s/bases/infrastructure/cluster-policies/best-practices/validate-pod-security.yaml index 9d516a4..8c88596 100644 --- a/k8s/bases/infrastructure/cluster-policies/best-practices/validate-pod-security.yaml +++ b/k8s/bases/infrastructure/cluster-policies/best-practices/validate-pod-security.yaml @@ -51,7 +51,6 @@ spec: - kubescape - kubevirt - cdi - - monitoring - velero - observability - resources: @@ -108,7 +107,6 @@ spec: - kubescape - kubevirt - cdi - - monitoring - velero - observability - resources: @@ -141,7 +139,6 @@ spec: - kubescape - kubevirt - cdi - - monitoring - velero - observability - resources: diff --git a/k8s/bases/infrastructure/cluster-security-exceptions/controller-rbac.yaml b/k8s/bases/infrastructure/cluster-security-exceptions/controller-rbac.yaml index e80e364..69e973b 100644 --- a/k8s/bases/infrastructure/cluster-security-exceptions/controller-rbac.yaml +++ b/k8s/bases/infrastructure/cluster-security-exceptions/controller-rbac.yaml @@ -15,7 +15,7 @@ spec: to manage their target resources cluster-wide. This includes GitOps (Flux), policy engines (Kyverno), certificate managers (cert-manager), database operators (CNPG), backup controllers (Velero), autoscalers - (KEDA), monitoring (Prometheus/Coroot), virtualisation (KubeVirt, + (KEDA), monitoring (Coroot), virtualisation (KubeVirt, CDI), storage (Longhorn), service discovery (homepage), identity (dex), auth proxies (oauth2-proxy), and config watchers (reloader). posture: @@ -51,7 +51,6 @@ spec: - cnpg-system - velero - keda - - monitoring - kubescape - vertical-pod-autoscaler - kubevirt diff --git a/k8s/bases/infrastructure/cluster-security-exceptions/health-probes.yaml b/k8s/bases/infrastructure/cluster-security-exceptions/health-probes.yaml index e15f3eb..8aa63da 100644 --- a/k8s/bases/infrastructure/cluster-security-exceptions/health-probes.yaml +++ b/k8s/bases/infrastructure/cluster-security-exceptions/health-probes.yaml @@ -33,7 +33,6 @@ spec: - cnpg-system - velero - keda - - monitoring - kubescape - vertical-pod-autoscaler - longhorn-system diff --git a/k8s/bases/infrastructure/cluster-security-exceptions/infrastructure-privileged.yaml b/k8s/bases/infrastructure/cluster-security-exceptions/infrastructure-privileged.yaml index 94427bb..cfd948d 100644 --- a/k8s/bases/infrastructure/cluster-security-exceptions/infrastructure-privileged.yaml +++ b/k8s/bases/infrastructure/cluster-security-exceptions/infrastructure-privileged.yaml @@ -3,7 +3,6 @@ # Cilium (CNI) needs: privileged, hostPath, hostNetwork, hostPID, NET_ADMIN, SYS_ADMIN # Longhorn (storage) needs: privileged, hostPath, hostPID # Velero node-agent (backup) needs: hostPath, privileged -# Prometheus node-exporter (monitoring) needs: hostPath, hostNetwork, hostPID # SPIRE agent (identity) needs: hostPath, hostPID # KubeVirt (VM hosting) needs: privileged, hostPath, hostPID, NET_ADMIN # CDI (container disk images) needs: privileged for disk operations @@ -14,7 +13,7 @@ metadata: spec: reason: >- CNI (Cilium), distributed storage (Longhorn), node monitoring - (Prometheus node-exporter and Coroot node agent), identity (SPIRE + (Coroot node agent), identity (SPIRE agent), backup (Velero node-agent), virtualisation (KubeVirt, CDI), and opt-in host audit collection require host-level access by design. posture: @@ -48,7 +47,6 @@ spec: values: - kube-system - longhorn-system - - monitoring - velero - kubevirt - cdi diff --git a/k8s/bases/infrastructure/cluster-security-exceptions/service-account-tokens.yaml b/k8s/bases/infrastructure/cluster-security-exceptions/service-account-tokens.yaml index 4c25a9c..eeb6393 100644 --- a/k8s/bases/infrastructure/cluster-security-exceptions/service-account-tokens.yaml +++ b/k8s/bases/infrastructure/cluster-security-exceptions/service-account-tokens.yaml @@ -43,7 +43,6 @@ spec: - cnpg-system - velero - keda - - monitoring - kubescape - vertical-pod-autoscaler - kubevirt diff --git a/k8s/bases/infrastructure/controllers/cdi/cilium-network-policy.yaml b/k8s/bases/infrastructure/controllers/cdi/cilium-network-policy.yaml index 8b58dc1..0b1b430 100644 --- a/k8s/bases/infrastructure/controllers/cdi/cilium-network-policy.yaml +++ b/k8s/bases/infrastructure/controllers/cdi/cilium-network-policy.yaml @@ -23,14 +23,6 @@ spec: - fromEndpoints: - matchLabels: k8s:io.kubernetes.pod.namespace: cdi - # Metrics scraping from monitoring - - fromEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: monitoring - toPorts: - - ports: - - port: "8443" - protocol: TCP egress: # Kube API for operator and controller management - toEntities: diff --git a/k8s/bases/infrastructure/controllers/cert-manager/cilium-network-policy.yaml b/k8s/bases/infrastructure/controllers/cert-manager/cilium-network-policy.yaml index 94db8f4..4146536 100644 --- a/k8s/bases/infrastructure/controllers/cert-manager/cilium-network-policy.yaml +++ b/k8s/bases/infrastructure/controllers/cert-manager/cilium-network-policy.yaml @@ -17,14 +17,6 @@ spec: protocol: TCP - port: "6443" protocol: TCP - # Metrics scraping from monitoring - - fromEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: monitoring - toPorts: - - ports: - - port: "9402" - protocol: TCP egress: # ACME challenges / CA communication - toEntities: diff --git a/k8s/bases/infrastructure/controllers/cloudnative-pg/cilium-network-policy.yaml b/k8s/bases/infrastructure/controllers/cloudnative-pg/cilium-network-policy.yaml index d9f31b4..131e130 100644 --- a/k8s/bases/infrastructure/controllers/cloudnative-pg/cilium-network-policy.yaml +++ b/k8s/bases/infrastructure/controllers/cloudnative-pg/cilium-network-policy.yaml @@ -15,14 +15,6 @@ spec: - ports: - port: "9443" protocol: TCP - # Metrics scraping from Prometheus - - fromEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: monitoring - toPorts: - - ports: - - port: "8080" - protocol: TCP egress: # Kube API for managing PG clusters - toEntities: diff --git a/k8s/bases/infrastructure/controllers/dex/cilium-network-policy.yaml b/k8s/bases/infrastructure/controllers/dex/cilium-network-policy.yaml index 25d4101..045f976 100644 --- a/k8s/bases/infrastructure/controllers/dex/cilium-network-policy.yaml +++ b/k8s/bases/infrastructure/controllers/dex/cilium-network-policy.yaml @@ -23,17 +23,6 @@ spec: protocol: TCP - port: "5558" protocol: TCP - # OIDC token + userinfo exchange from Grafana (native auth.generic_oauth). - # Same in-cluster path as oauth2-proxy: Grafana's server-side - # calls hit the Dex Service directly and arrive with the monitoring pod - # identity, so they must be allowed explicitly here. - - fromEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: monitoring - toPorts: - - ports: - - port: "5556" - protocol: TCP egress: # Kube API for reading secrets - toEntities: diff --git a/k8s/bases/infrastructure/controllers/flux-operator/cilium-network-policy.yaml b/k8s/bases/infrastructure/controllers/flux-operator/cilium-network-policy.yaml index d547e58..65e7b48 100644 --- a/k8s/bases/infrastructure/controllers/flux-operator/cilium-network-policy.yaml +++ b/k8s/bases/infrastructure/controllers/flux-operator/cilium-network-policy.yaml @@ -24,10 +24,6 @@ spec: - fromEndpoints: - matchLabels: k8s:io.kubernetes.pod.namespace: flux-system - # Metrics scraping - - fromEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: monitoring egress: # Intra-namespace (controller-to-controller communication) - toEndpoints: diff --git a/k8s/bases/infrastructure/controllers/keda/cilium-network-policy.yaml b/k8s/bases/infrastructure/controllers/keda/cilium-network-policy.yaml index 9c2be31..216d3bc 100644 --- a/k8s/bases/infrastructure/controllers/keda/cilium-network-policy.yaml +++ b/k8s/bases/infrastructure/controllers/keda/cilium-network-policy.yaml @@ -28,10 +28,6 @@ spec: - fromEndpoints: - matchLabels: k8s:io.kubernetes.pod.namespace: keda - # Metrics scraping - - fromEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: monitoring egress: # Kube API for watching scalers - toEntities: @@ -55,14 +51,6 @@ spec: - ports: - port: "8081" protocol: TCP - # Prometheus for metrics-based scaling - - toEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: monitoring - toPorts: - - ports: - - port: "9090" - protocol: TCP # DNS resolution - toEndpoints: - matchLabels: diff --git a/k8s/bases/infrastructure/controllers/kube-prometheus-stack/cilium-network-policy.yaml b/k8s/bases/infrastructure/controllers/kube-prometheus-stack/cilium-network-policy.yaml deleted file mode 100644 index b3f245f..0000000 --- a/k8s/bases/infrastructure/controllers/kube-prometheus-stack/cilium-network-policy.yaml +++ /dev/null @@ -1,56 +0,0 @@ -apiVersion: cilium.io/v2 -kind: CiliumNetworkPolicy -metadata: - name: allow-monitoring - namespace: monitoring -spec: - endpointSelector: {} - ingress: - # Intra-namespace (prometheus β†’ alertmanager, etc) - - fromEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: monitoring - # Webhook from kube-apiserver (hostNetwork on control plane nodes) - - fromEntities: - - kube-apiserver - - remote-node - - host - toPorts: - - ports: - - port: "10250" - protocol: TCP - egress: - # Kube API for service discovery - - toEntities: - - kube-apiserver - # Scrape targets in all namespaces - - toEndpoints: - - matchExpressions: - - key: k8s:io.kubernetes.pod.namespace - operator: Exists - # Scrape node-exporter on nodes - - toEntities: - - host - - remote-node - # Alertmanager webhooks (Slack, etc) - - toEntities: - - world - toPorts: - - ports: - - port: "443" - protocol: TCP - # Intra-namespace - - toEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: monitoring - # DNS resolution - - toEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: kube-system - k8s-app: kube-dns - toPorts: - - ports: - - port: "53" - protocol: UDP - - port: "53" - protocol: TCP diff --git a/k8s/bases/infrastructure/controllers/kube-prometheus-stack/helm-release.yaml b/k8s/bases/infrastructure/controllers/kube-prometheus-stack/helm-release.yaml deleted file mode 100644 index bf255c2..0000000 --- a/k8s/bases/infrastructure/controllers/kube-prometheus-stack/helm-release.yaml +++ /dev/null @@ -1,312 +0,0 @@ -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: kube-prometheus-stack - namespace: monitoring - labels: - helm.toolkit.fluxcd.io/crds: enabled - helm.toolkit.fluxcd.io/remediation: enabled -spec: - interval: 5m - chart: - spec: - chart: kube-prometheus-stack - version: 92.17.1 - sourceRef: - kind: HelmRepository - name: prometheus-community - install: - crds: CreateReplace - upgrade: - crds: CreateReplace - timeout: 15m - # https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/values.yaml - # - # Prometheus + Alertmanager + node-exporter + kube-state-metrics + - # Grafana. Production-ready posture (see docs/dr/alerting.md): - # - Alerts ship to Slack via native slack_configs. - # - An always-firing Watchdog is pushed to an external heartbeat - # monitor (dead-man's-switch) so a full-cluster outage still pages. - # - Prometheus & Alertmanager get persistent volumes in the hetzner - # overlay; Velero backs the monitoring namespace up to R2 daily. - # - Grafana is exposed behind oauth2-proxy SSO with Prometheus + Loki - # datasources for dashboards and log exploration. - values: - crds: - enabled: true - - # Grafana authenticates users directly against Dex via native OIDC - # (auth.generic_oauth) -- it is NOT fronted by oauth2-proxy (its - # HTTPRoute backends straight to the Grafana Service). Every user who - # clears the Dex gate is mapped to Admin (single-operator homelab) via - # the JMESPath literal in role_attribute_path. auto_login + - # disable_login_form send users straight to Dex with no Grafana login - # form (break-glass: https://grafana.${domain}/login?disableAutoLogin=true). - # Grafana is switched off: Coroot is the observability UI, and no route, sign-in - # secret or Dex callback is published for it. The settings below are kept only - # until kube-prometheus-stack itself is retired. - grafana: - enabled: false - grafana.ini: - server: - root_url: https://grafana.${domain} - auth: - disable_login_form: true - auth.basic: - enabled: false - auth.generic_oauth: - enabled: true - name: Dex - auto_login: true - client_id: public-client - scopes: openid profile email groups - # Split the OIDC endpoints so this works on every cluster (mirrors - # oauth2-proxy): the browser-facing authorize endpoint stays the - # public Dex URL (the browser resolves dex.${domain} via the host - # /etc/hosts locally, public DNS in prod), while the server-side - # token + userinfo calls use the in-cluster Dex Service the Grafana - # pod can always resolve. dex.${domain} does NOT resolve via - # in-cluster CoreDNS on the docker/local cluster, so targeting the - # public URL here fails the back-channel token exchange with - # "Failed to get token from provider". The dex NetworkPolicy allows - # the matching monitoring->5556 ingress. - auth_url: https://dex.${domain}/auth - token_url: http://dex.dex.svc.cluster.local:5556/token - api_url: http://dex.dex.svc.cluster.local:5556/userinfo - # JMESPath string literal -> every authenticated user gets Admin. - role_attribute_path: "'Admin'" - # Client secret (Dex 'public-client') from the grafana-oidc - # ExternalSecret. GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET overrides the - # grafana.ini value. optional:true so a bootstrap race fails login - # closed instead of crash-looping the pod; reloader restarts Grafana - # when ESO lands the secret. - envValueFrom: - GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: - secretKeyRef: - name: grafana-oidc - key: client-secret - optional: true - annotations: - secret.reloader.stakater.com/reload: grafana-oidc - # Dashboards are provisioned as code, so the pod can stay ephemeral. - persistence: - enabled: false - defaultDashboardsEnabled: true - sidecar: - dashboards: - enabled: true - searchNamespace: ALL - datasources: - enabled: true - resources: - requests: - cpu: 50m - memory: 128Mi - limits: - memory: 256Mi - - # We don't host an external Prometheus and don't expose this one. - prometheus: - prometheusSpec: - replicas: 1 - # Monitoring is critical platform infrastructure β€” keep it alive under - # node memory pressure (evicted only after normal workloads). See the - # platform-critical PriorityClass. - priorityClassName: platform-critical - retention: 14d - retentionSize: "5GiB" - resources: - requests: - cpu: 50m - memory: 256Mi - limits: - memory: 768Mi - # Discover ServiceMonitors / PodMonitors / PrometheusRules from - # any namespace -- charts in this repo (Velero, CNPG, etc.) drop - # their own. - serviceMonitorSelectorNilUsesHelmValues: false - podMonitorSelectorNilUsesHelmValues: false - ruleSelectorNilUsesHelmValues: false - probeSelectorNilUsesHelmValues: false - scrapeConfigSelectorNilUsesHelmValues: false - # Discover targets through EndpointSlices rather than the default - # core/v1 Endpoints API. ServiceMonitor discovery inherently lists - # *every* endpoint in a monitor's target namespace and keeps only the - # matching Service, dropping the rest -- so a large "dropped" count on - # Prometheus' Service Discovery page is expected, not a fault (each of - # the ~14 monitors in this namespace scans ~50 endpoints). EndpointSlice - # is the scalable, operator-recommended SD role: it trims the apiserver - # watch + relabel churn behind those drops and future-proofs against the - # core/v1 Endpoints deprecation (k8s 1.33). It does not (and need not) - # lower the dropped count -- that is just how ServiceMonitors select. - serviceDiscoveryRole: EndpointSlice - # External labels so the webhook payload identifies the source - # cluster. - externalLabels: - domain: ${domain} - # No persistence -- 14d/5GiB fits in memory + emptyDir for a - # homelab. PR #4 already backs up persistent state separately; - # losing the metrics history on a restart is acceptable. - storageSpec: {} - - alertmanager: - alertmanagerSpec: - replicas: ${alertmanager_replicas:=2} - # Alerting must survive memory pressure (it is how a degrading node is - # noticed in the first place). - priorityClassName: platform-critical - podDisruptionBudget: - enabled: true - minAvailable: 1 - topologySpreadConstraints: - - maxSkew: 1 - topologyKey: kubernetes.io/hostname - whenUnsatisfiable: ScheduleAnyway - labelSelector: - matchLabels: - app.kubernetes.io/name: alertmanager - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - memory: 128Mi - externalUrl: "" - config: - global: - resolve_timeout: 5m - route: - # Group on the labels Prometheus actually sets externally - # (externalLabels.domain = ${domain}) -- `cluster` is not a - # configured label here, so grouping by it would dedup nothing. - group_by: ["alertname", "domain", "severity"] - group_wait: 30s - group_interval: 5m - repeat_interval: 12h - receiver: slack - routes: - # Dead-man's-switch. The Watchdog alert always fires (from the - # chart's general rules), so we push it to an external heartbeat - # monitor (e.g. healthchecks.io) on a tight cadence. If the - # cluster or the Prometheus->Alertmanager pipeline dies, the - # monitor stops receiving pings and notifies Slack out-of-band - # -- the one failure in-cluster alerting can never cover itself. - - matchers: - - alertname = "Watchdog" - receiver: heartbeat - # Alertmanager re-notifies only when both group_interval AND - # repeat_interval have elapsed since the last send. Keeping - # group_interval (30s) below repeat_interval (50s) lets the - # 50s cadence actually take effect; with group_interval >= - # repeat_interval the heartbeat would be silently throttled - # to the longer of the two. - group_wait: 0s - group_interval: 30s - repeat_interval: 50s - continue: false - - matchers: - - severity = "critical" - receiver: slack - continue: false - - matchers: - - severity = "warning" - receiver: slack - repeat_interval: 24h - receivers: - # Normal alerts -> Slack via a native incoming webhook. The URL - # is per-cluster and SOPS-encrypted (alertmanager_webhook_url), - # so local clusters get an invalid URL and stay quiet by design. - - name: slack - slack_configs: - - api_url_file: /etc/alertmanager/secrets/alertmanager-webhook/webhook_url - channel: "#platform-alerts" - send_resolved: true - title: '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }}' - text: >- - {{ range .Alerts }}*{{ .Labels.severity | toUpper }}* β€” {{ .Annotations.summary }} - {{ .Annotations.description }} - {{ end }} - # Heartbeat receiver: just POSTs to the external monitor URL. No - # resolved notifications -- a missing ping is the signal. The URL - # is injected by Flux substitution from the per-cluster secret; - # if unset (local/CI) it defaults to an invalid URL so the - # heartbeat simply never reaches anywhere -- harmless, and the - # same "fail closed locally" pattern as the Slack webhook. - - name: heartbeat - webhook_configs: - - url: ${alertmanager_heartbeat_url:=https://example.invalid/no-heartbeat-configured} - send_resolved: false - inhibit_rules: - # Equal labels must exist on both source and target alerts; - # `cluster` isn't configured here (we set `domain` via - # externalLabels), so use that for the cluster-scoped match. - - source_matchers: [severity = "critical"] - target_matchers: [severity = "warning"] - equal: [alertname, domain, namespace] - secrets: - - alertmanager-webhook - - # Don't try to scrape the API server / kubelet / etcd through the - # default ServiceMonitors here -- Talos exposes some of these - # differently and we don't need k8s-internal histograms for alerting. - # Targeted alerts in PrometheusRule cover what we actually care about. - kubeApiServer: - enabled: false - kubeControllerManager: - enabled: false - kubeScheduler: - enabled: false - kubeProxy: - enabled: false - kubeEtcd: - enabled: false - kubelet: - enabled: true - - nodeExporter: - enabled: true - # node-exporter subchart passthrough (alias `prometheus-node-exporter`). - prometheus-node-exporter: - priorityClassName: platform-critical - kube-state-metrics: - enabled: true - priorityClassName: platform-critical - - # Enable the chart's well-tested default rules, but prune the groups - # for control-plane components we deliberately don't scrape (their - # ServiceMonitors are disabled above, so their alerts would only fire - # on permanently-absent targets). This gives us: - # - Watchdog (general.rules) -> the dead-man's-switch heartbeat. - # - Self-monitoring: PrometheusOperator/Prometheus/Alertmanager down, - # config-reload failures, TSDB problems. - # - Workload health: KubePodCrashLooping, KubeDeploymentReplicasMismatch, - # KubeContainerWaiting, KubePersistentVolume* etc. - # Platform-specific alerts (Velero/CNPG/Flux/cert-manager/autoscaler) - # live in k8s/bases/infrastructure/alerts/platform-critical.yaml. - defaultRules: - create: true - rules: - etcd: false - kubeApiserverAvailability: false - kubeApiserverBurnrate: false - kubeApiserverHistogram: false - kubeApiserverSlos: false - kubeControllerManager: false - kubeProxy: false - kubeSchedulerAlerting: false - kubeSchedulerRecording: false - windows: false - # Overcommit alerts are guaranteed noise on a cluster that runs hot - # on purpose (VPA right-sizes requests; the autoscaler adds nodes). - disabled: - KubeCPUOvercommit: true - KubeMemoryOvercommit: true - - prometheusOperator: - priorityClassName: platform-critical - resources: - requests: - cpu: 50m - memory: 128Mi - limits: - memory: 256Mi diff --git a/k8s/bases/infrastructure/controllers/kube-prometheus-stack/helm-repository.yaml b/k8s/bases/infrastructure/controllers/kube-prometheus-stack/helm-repository.yaml deleted file mode 100644 index 96e0994..0000000 --- a/k8s/bases/infrastructure/controllers/kube-prometheus-stack/helm-repository.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: source.toolkit.fluxcd.io/v1 -kind: HelmRepository -metadata: - name: prometheus-community - namespace: monitoring -spec: - interval: 24h - url: https://prometheus-community.github.io/helm-charts diff --git a/k8s/bases/infrastructure/controllers/kube-prometheus-stack/kustomization.yaml b/k8s/bases/infrastructure/controllers/kube-prometheus-stack/kustomization.yaml deleted file mode 100644 index 13926e5..0000000 --- a/k8s/bases/infrastructure/controllers/kube-prometheus-stack/kustomization.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - namespace.yaml - - helm-repository.yaml - - helm-release.yaml - - cilium-network-policy.yaml -components: - - ../../../../components/helmrelease-flux-defaults/ - - ../../../../components/helmrelease-drift-detection/ diff --git a/k8s/bases/infrastructure/controllers/kube-prometheus-stack/namespace.yaml b/k8s/bases/infrastructure/controllers/kube-prometheus-stack/namespace.yaml deleted file mode 100644 index 63c702a..0000000 --- a/k8s/bases/infrastructure/controllers/kube-prometheus-stack/namespace.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: monitoring - labels: - # prometheus-node-exporter DaemonSet uses hostNetwork/hostPID/hostPath/ - # hostPort to scrape node-level metrics β€” requires privileged PSS. - pod-security.kubernetes.io/enforce: privileged - pod-security.kubernetes.io/enforce-version: latest diff --git a/k8s/bases/infrastructure/controllers/kubevirt/cilium-network-policy.yaml b/k8s/bases/infrastructure/controllers/kubevirt/cilium-network-policy.yaml index 038ebf0..c36cec9 100644 --- a/k8s/bases/infrastructure/controllers/kubevirt/cilium-network-policy.yaml +++ b/k8s/bases/infrastructure/controllers/kubevirt/cilium-network-policy.yaml @@ -23,14 +23,6 @@ spec: - fromEndpoints: - matchLabels: k8s:io.kubernetes.pod.namespace: kubevirt - # Metrics scraping from monitoring - - fromEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: monitoring - toPorts: - - ports: - - port: "8443" - protocol: TCP egress: # Kube API for operator and controller management - toEntities: diff --git a/k8s/bases/infrastructure/controllers/kustomization.yaml b/k8s/bases/infrastructure/controllers/kustomization.yaml index 26cd379..d7ab690 100644 --- a/k8s/bases/infrastructure/controllers/kustomization.yaml +++ b/k8s/bases/infrastructure/controllers/kustomization.yaml @@ -13,7 +13,6 @@ resources: - flux-operator/ - keda/ - keda-http-add-on/ - - kube-prometheus-stack/ - kubevirt/ - kubescape/ - kyverno/ diff --git a/k8s/bases/infrastructure/controllers/kyverno/cilium-network-policy.yaml b/k8s/bases/infrastructure/controllers/kyverno/cilium-network-policy.yaml index ba6bf03..f457839 100644 --- a/k8s/bases/infrastructure/controllers/kyverno/cilium-network-policy.yaml +++ b/k8s/bases/infrastructure/controllers/kyverno/cilium-network-policy.yaml @@ -15,14 +15,6 @@ spec: - ports: - port: "9443" protocol: TCP - # Metrics scraping - - fromEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: monitoring - toPorts: - - ports: - - port: "8000" - protocol: TCP egress: # Kube API for policy enforcement - toEntities: diff --git a/k8s/bases/infrastructure/controllers/oauth2-proxy/reference-grant.yaml b/k8s/bases/infrastructure/controllers/oauth2-proxy/reference-grant.yaml index 8243c6b..6cc2201 100644 --- a/k8s/bases/infrastructure/controllers/oauth2-proxy/reference-grant.yaml +++ b/k8s/bases/infrastructure/controllers/oauth2-proxy/reference-grant.yaml @@ -15,9 +15,6 @@ spec: - group: gateway.networking.k8s.io kind: HTTPRoute namespace: kube-system - - group: gateway.networking.k8s.io - kind: HTTPRoute - namespace: monitoring - group: gateway.networking.k8s.io kind: HTTPRoute namespace: observability diff --git a/k8s/bases/infrastructure/controllers/testkube/cilium-network-policy.yaml b/k8s/bases/infrastructure/controllers/testkube/cilium-network-policy.yaml index 6be32c7..2adfaee 100644 --- a/k8s/bases/infrastructure/controllers/testkube/cilium-network-policy.yaml +++ b/k8s/bases/infrastructure/controllers/testkube/cilium-network-policy.yaml @@ -19,10 +19,6 @@ spec: - fromEndpoints: - matchLabels: k8s:io.kubernetes.pod.namespace: testkube - # Metrics scraping - - fromEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: monitoring egress: # Kube API for test execution and CRD management - toEntities: diff --git a/k8s/bases/infrastructure/controllers/tetragon/helm-release.yaml b/k8s/bases/infrastructure/controllers/tetragon/helm-release.yaml index a32d48f..2be3937 100644 --- a/k8s/bases/infrastructure/controllers/tetragon/helm-release.yaml +++ b/k8s/bases/infrastructure/controllers/tetragon/helm-release.yaml @@ -28,7 +28,7 @@ spec: # + host PID, which the validate-host-restrictions Kyverno policy blocks # in every namespace except the infrastructure set (kube-system among # them). kube-system is also exempt from the add-default-deny CNP, so no - # NetworkPolicy is required for Prometheus to scrape the metrics ports. + # NetworkPolicy is required to reach the metrics ports. # # Observability-only for now: no TracingPolicies are shipped yet, so the # agent emits its built-in process_exec / process_exit events. Runtime @@ -85,18 +85,17 @@ spec: enableProcessNs: true prometheus: enabled: true - # Prometheus discovers ServiceMonitors cluster-wide - # (serviceMonitorSelectorNilUsesHelmValues: false), so enabling - # this is all that's needed for the agent metrics (:2112) to be - # scraped. + # The prometheus-operator ServiceMonitor CRD left with + # kube-prometheus-stack. The agent metrics endpoint (:2112) stays up; + # Coroot's node agent observes the pod instead of a ServiceMonitor. serviceMonitor: - enabled: true + enabled: false tetragonOperator: # Operator metrics (:2113); operator already ships resource requests # and a non-root securityContext by default. prometheus: serviceMonitor: - enabled: true + enabled: false # The export-stdout sidecar is on by default (export.mode: stdout), so # Tetragon's JSON events go to the pod's stdout -- where Coroot's node # agent collects them with the other container logs, with no extra diff --git a/k8s/bases/infrastructure/controllers/velero/cilium-network-policy.yaml b/k8s/bases/infrastructure/controllers/velero/cilium-network-policy.yaml index cfc446a..0a0fa5e 100644 --- a/k8s/bases/infrastructure/controllers/velero/cilium-network-policy.yaml +++ b/k8s/bases/infrastructure/controllers/velero/cilium-network-policy.yaml @@ -5,15 +5,6 @@ metadata: namespace: velero spec: endpointSelector: {} - ingress: - # Metrics scraping - - fromEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: monitoring - toPorts: - - ports: - - port: "8085" - protocol: TCP egress: # Kube API for backup operations - toEntities: diff --git a/k8s/bases/infrastructure/external-secrets/alertmanager-webhook.yaml b/k8s/bases/infrastructure/external-secrets/alertmanager-webhook.yaml deleted file mode 100644 index b84a43c..0000000 --- a/k8s/bases/infrastructure/external-secrets/alertmanager-webhook.yaml +++ /dev/null @@ -1,23 +0,0 @@ ---- -apiVersion: external-secrets.io/v1 -kind: ExternalSecret -metadata: - name: alertmanager-webhook - namespace: monitoring -spec: - refreshInterval: 1h - secretStoreRef: - name: openbao - kind: ClusterSecretStore - target: - name: alertmanager-webhook - creationPolicy: Owner - data: - # Slack incoming-webhook URL for normal alert notifications. The - # Watchdog heartbeat URL is injected via Flux substitution - # (${alertmanager_heartbeat_url}) rather than here, so an unset value - # degrades to an invalid URL instead of failing this ExternalSecret. - - secretKey: webhook_url - remoteRef: - key: infrastructure/monitoring/alertmanager - property: webhook_url diff --git a/k8s/bases/infrastructure/external-secrets/kustomization.yaml b/k8s/bases/infrastructure/external-secrets/kustomization.yaml index 213740f..ead6375 100644 --- a/k8s/bases/infrastructure/external-secrets/kustomization.yaml +++ b/k8s/bases/infrastructure/external-secrets/kustomization.yaml @@ -3,4 +3,3 @@ kind: Kustomization resources: - velero-r2-credentials.yaml - cnpg-r2-credentials.yaml - - alertmanager-webhook.yaml diff --git a/k8s/bases/infrastructure/kustomization.yaml b/k8s/bases/infrastructure/kustomization.yaml index fe1f4cf..6305394 100644 --- a/k8s/bases/infrastructure/kustomization.yaml +++ b/k8s/bases/infrastructure/kustomization.yaml @@ -2,7 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - alerts/ - certificates/ - cluster-policies/ - cluster-role-bindings/ diff --git a/k8s/bases/infrastructure/vault-seed/kustomization.yaml b/k8s/bases/infrastructure/vault-seed/kustomization.yaml index 47dd461..8d3392c 100644 --- a/k8s/bases/infrastructure/vault-seed/kustomization.yaml +++ b/k8s/bases/infrastructure/vault-seed/kustomization.yaml @@ -9,6 +9,5 @@ resources: - push-secret-oauth2-proxy-cookie-secret.yaml - push-secret-seed-r2-credentials.yaml - push-secret-seed-cloudflare.yaml - - push-secret-seed-alertmanager.yaml - push-secret-seed-github-app.yaml - push-secret-seed-velero-repo-credentials.yaml diff --git a/k8s/bases/infrastructure/vault-seed/push-secret-seed-alertmanager.yaml b/k8s/bases/infrastructure/vault-seed/push-secret-seed-alertmanager.yaml deleted file mode 100644 index 2b4b617..0000000 --- a/k8s/bases/infrastructure/vault-seed/push-secret-seed-alertmanager.yaml +++ /dev/null @@ -1,20 +0,0 @@ ---- -apiVersion: external-secrets.io/v1alpha1 -kind: PushSecret -metadata: - name: seed-alertmanager - namespace: flux-system -spec: - refreshInterval: "0" - secretStoreRefs: - - name: openbao - kind: ClusterSecretStore - selector: - secret: - name: variables-cluster - data: - - match: - secretKey: alertmanager_webhook_url - remoteRef: - remoteKey: infrastructure/monitoring/alertmanager - property: webhook_url diff --git a/k8s/clusters/base/flux-kustomization-bootstrap.yaml b/k8s/clusters/base/flux-kustomization-bootstrap.yaml index ac80095..5b393fd 100644 --- a/k8s/clusters/base/flux-kustomization-bootstrap.yaml +++ b/k8s/clusters/base/flux-kustomization-bootstrap.yaml @@ -5,7 +5,7 @@ metadata: # The bootstrap layer owns cluster-foundational resources that must exist # BEFORE the infrastructure controllers reconcile: substitution variables # (variables-base / variables-cluster) and cluster-scoped PriorityClasses - # (e.g. platform-critical, which kube-prometheus-stack references). Everything + # (e.g. platform-critical, which Coroot and the cluster heartbeat reference). Everything # else dependsOn this (directly or transitively), so its contents are applied # and ready first β€” which is what keeps helm-controller from rolling pods that # reference a PriorityClass before that class exists. diff --git a/k8s/clusters/base/flux-kustomization-infrastructure-controllers.yaml b/k8s/clusters/base/flux-kustomization-infrastructure-controllers.yaml index b79ebb4..5d80266 100644 --- a/k8s/clusters/base/flux-kustomization-infrastructure-controllers.yaml +++ b/k8s/clusters/base/flux-kustomization-infrastructure-controllers.yaml @@ -15,8 +15,8 @@ spec: dependsOn: # The bootstrap layer applies cluster PriorityClasses (e.g. platform-critical) # and substitution variables; it must be ready before these controllers - # reconcile, since e.g. kube-prometheus-stack references platform-critical and - # helm-controller would otherwise roll its pods before the class exists. + # reconcile, so a workload that names a class (Coroot, the cluster heartbeat) + # never rolls its pods before the class exists. - name: bootstrap decryption: provider: sops diff --git a/k8s/clusters/prod/bootstrap/variables-cluster-config-map.yaml b/k8s/clusters/prod/bootstrap/variables-cluster-config-map.yaml index 701ae62..ee1d655 100644 --- a/k8s/clusters/prod/bootstrap/variables-cluster-config-map.yaml +++ b/k8s/clusters/prod/bootstrap/variables-cluster-config-map.yaml @@ -88,15 +88,6 @@ data: kyverno_background_replicas: "1" kyverno_reports_replicas: "1" kyverno_cleanup_replicas: "1" - # --- Alertmanager --- - # 2 replicas for HA: Alertmanager gossips notification/silence state, so - # a second replica means a rolling restart or single-node loss never - # drops an alert notification. Restored 1 -> 2 (was trimmed to 1 during - # the #1585 stabilization) now that prod has regained headroom via the - # lowered VPA floor (#1585), cert-manager/ESO restore (#1601), and the - # kubevirt/cdi opt-out (#1671); the planned 4th-worker bump is tracked - # separately in ksail.prod.yaml. - alertmanager_replicas: "2" # --- OpenBao secrets vault --- openbao_replicas: "1" openbao_storage_size: 10Gi diff --git a/k8s/clusters/prod/kustomization.yaml b/k8s/clusters/prod/kustomization.yaml index f1b898f..e306250 100644 --- a/k8s/clusters/prod/kustomization.yaml +++ b/k8s/clusters/prod/kustomization.yaml @@ -17,8 +17,7 @@ patches: cluster_name: prod provider: hetzner # Prod has heavier components (Longhorn, etc.) that need more time to - # reconcile than the 3m base timeout allows. kube-prometheus-stack has a - # 15m HelmRelease timeout itself; adding margin gives the full chain + # reconcile than the 3m base timeout allows. The margin gives the full chain # (infrastructure-controllers β†’ infrastructure β†’ apps) room to complete # within the ksail connection.timeout (30m). - patch: | diff --git a/k8s/providers/hetzner/infrastructure/controllers/external-dns/cilium-network-policy.yaml b/k8s/providers/hetzner/infrastructure/controllers/external-dns/cilium-network-policy.yaml index f80251c..14210e0 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/external-dns/cilium-network-policy.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/external-dns/cilium-network-policy.yaml @@ -5,15 +5,6 @@ metadata: namespace: external-dns spec: endpointSelector: {} - ingress: - # Metrics scraping - - fromEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: monitoring - toPorts: - - ports: - - port: "7979" - protocol: TCP egress: # Kube API for watching services/HTTPRoutes - toEntities: diff --git a/k8s/providers/hetzner/infrastructure/controllers/flux-notifications/alert.yaml b/k8s/providers/hetzner/infrastructure/controllers/flux-notifications/alert.yaml index 921f745..e4b9713 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/flux-notifications/alert.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/flux-notifications/alert.yaml @@ -1,6 +1,29 @@ # Coroot sees workload health, but it cannot observe failed Flux builds, # applies, or health checks. Error events cover those failure classes without # introducing another polling workload. +# +# HelmRelease is watched DIRECTLY rather than left to its parent Kustomization, +# because a failing HelmRelease can be invisible to it. With Helm remediation +# enabled, a failed upgrade is rolled back to the last good release. The +# HelmRelease is then Ready again, its parent Kustomization stays healthy, and +# the release silently stops advancing. This Alert replaces the Prometheus rule +# that used to report it. +# +# WHY THE NAMESPACES ARE ENUMERATED +# There is NO namespace wildcard. notification-controller defaults an omitted +# eventSources[].namespace to the Alert's own namespace and then matches on +# strict equality; `name: "*"` is a wildcard over names only. A bare +# `kind: HelmRelease, name: "*"` here would watch only flux-system and miss +# every other release. So every namespace holding a HelmRelease or a Flux +# Kustomization is listed. +# +# Enumeration drifts, and a namespace nobody watches is the very gap this Alert +# closes, so scripts/validate-alert-coverage.sh fails CI when a HelmRelease or +# a Kustomization appears in a namespace that is not listed below. An instance +# that adds a release in a new namespace adds its entry here. +# +# notification-controller dedupes and rate-limits, so a resource that keeps +# failing notifies once, not on every reconcile. apiVersion: notification.toolkit.fluxcd.io/v1beta3 kind: Alert metadata: @@ -13,5 +36,63 @@ spec: eventSources: - kind: Kustomization name: "*" + namespace: flux-system + - kind: HelmRelease + name: "*" + namespace: cert-manager + - kind: HelmRelease + name: "*" + namespace: cnpg-system + - kind: HelmRelease + name: "*" + namespace: dex + - kind: HelmRelease + name: "*" + namespace: external-dns + - kind: HelmRelease + name: "*" + namespace: external-secrets + - kind: HelmRelease + name: "*" + namespace: flux-system + - kind: HelmRelease + name: "*" + namespace: homepage + - kind: HelmRelease + name: "*" + namespace: keda + - kind: HelmRelease + name: "*" + namespace: kube-system + - kind: HelmRelease + name: "*" + namespace: kubescape + - kind: HelmRelease + name: "*" + namespace: kyverno + - kind: HelmRelease + name: "*" + namespace: longhorn-system + - kind: HelmRelease + name: "*" + namespace: oauth2-proxy + - kind: HelmRelease + name: "*" + namespace: observability + - kind: HelmRelease + name: "*" + namespace: openbao + - kind: HelmRelease + name: "*" + namespace: reloader + - kind: HelmRelease + name: "*" + namespace: velero + - kind: HelmRelease + name: "*" + namespace: vertical-pod-autoscaler + - kind: HelmRelease + name: "*" + namespace: whoami eventMetadata: summary: "Flux reconciliation error β€” production platform" diff --git a/k8s/providers/hetzner/infrastructure/controllers/kube-prometheus-stack/patches/store-tsdb-on-hcloud.yaml b/k8s/providers/hetzner/infrastructure/controllers/kube-prometheus-stack/patches/store-tsdb-on-hcloud.yaml deleted file mode 100644 index a834dbc..0000000 --- a/k8s/providers/hetzner/infrastructure/controllers/kube-prometheus-stack/patches/store-tsdb-on-hcloud.yaml +++ /dev/null @@ -1,51 +0,0 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: kube-prometheus-stack - namespace: monitoring -spec: - values: - prometheus: - prometheusSpec: - # Persist the TSDB on Hetzner Cloud block storage. The base uses - # an emptyDir (fine for local/CI), but in prod a VPA eviction, - # node drain, or autoscaler scale-down would otherwise wipe all - # metrics history. hcloud block storage (not Longhorn) for the - # same reason OpenBao uses it: the ~36 GiB Talos rootfs is too - # small to host Longhorn replicas this large. - # - # 20Gi gives headroom over the 14d / 5GiB retention so a backlog - # of unshipped blocks never wedges the WAL. Velero's daily backup - # of the monitoring namespace ships this volume to R2 (24h RPO). - storageSpec: - volumeClaimTemplate: - spec: - storageClassName: hcloud - accessModes: ["ReadWriteOnce"] - resources: - requests: - storage: 20Gi - # Raise the memory ceiling: VPA only manages requests - # (controlledValues: RequestsOnly), so this hard limit is the - # real OOM guard under cardinality spikes. 768Mi (base) is tight - # once defaultRules + Grafana scrape targets are active. - resources: - requests: - cpu: 50m - memory: 256Mi - limits: - memory: 1536Mi - alertmanager: - alertmanagerSpec: - # Persist silences and the notification log so an operator's - # silence survives a restart and the gossiped dedup state has a - # durable backing store. - storage: - volumeClaimTemplate: - spec: - storageClassName: hcloud - accessModes: ["ReadWriteOnce"] - resources: - requests: - storage: 2Gi diff --git a/k8s/providers/hetzner/infrastructure/controllers/kubelet-serving-cert-approver/cilium-network-policy.yaml b/k8s/providers/hetzner/infrastructure/controllers/kubelet-serving-cert-approver/cilium-network-policy.yaml index 1b9e414..0f606fb 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/kubelet-serving-cert-approver/cilium-network-policy.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/kubelet-serving-cert-approver/cilium-network-policy.yaml @@ -5,15 +5,6 @@ metadata: namespace: kubelet-serving-cert-approver spec: endpointSelector: {} - ingress: - # Metrics scraping - - fromEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: monitoring - toPorts: - - ports: - - port: "9090" - protocol: TCP egress: # Kube API for approving CSRs - toEntities: diff --git a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml index f64e0d0..7194e05 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml @@ -20,7 +20,6 @@ resources: - ../../../../bases/infrastructure/controllers/flux-operator/ - ../../../../bases/infrastructure/controllers/keda/ - ../../../../bases/infrastructure/controllers/keda-http-add-on/ - - ../../../../bases/infrastructure/controllers/kube-prometheus-stack/ - ../../../../bases/infrastructure/controllers/kubescape/ - ../../../../bases/infrastructure/controllers/kyverno/ - ../../../../bases/infrastructure/controllers/metrics-server/ @@ -44,7 +43,6 @@ resources: - origin-ca-issuer/ patches: - path: openbao/patches/store-data-on-hcloud.yaml - - path: kube-prometheus-stack/patches/store-tsdb-on-hcloud.yaml - path: coroot/patches/allow-slack-webhook-egress.yaml target: group: cilium.io diff --git a/k8s/providers/hetzner/infrastructure/controllers/longhorn/cilium-network-policy.yaml b/k8s/providers/hetzner/infrastructure/controllers/longhorn/cilium-network-policy.yaml index d6a0d99..96aefab 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/longhorn/cilium-network-policy.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/longhorn/cilium-network-policy.yaml @@ -19,10 +19,6 @@ spec: - fromEndpoints: - matchLabels: k8s:io.kubernetes.pod.namespace: longhorn-system - # Metrics scraping - - fromEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: monitoring egress: # Kube API - toEntities: diff --git a/k8s/providers/hetzner/infrastructure/kustomization.yaml b/k8s/providers/hetzner/infrastructure/kustomization.yaml index a798d63..01747d6 100644 --- a/k8s/providers/hetzner/infrastructure/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/kustomization.yaml @@ -18,10 +18,3 @@ patches: kind: Coroot name: coroot namespace: observability - - path: patches/retain-helm-release-alert.yaml - target: - group: monitoring.coreos.com - version: v1 - kind: PrometheusRule - name: platform-critical - namespace: monitoring diff --git a/k8s/providers/hetzner/infrastructure/patches/retain-helm-release-alert.yaml b/k8s/providers/hetzner/infrastructure/patches/retain-helm-release-alert.yaml deleted file mode 100644 index 8c3407d..0000000 --- a/k8s/providers/hetzner/infrastructure/patches/retain-helm-release-alert.yaml +++ /dev/null @@ -1,24 +0,0 @@ -- op: test - path: /spec/groups/3/name - value: flux -- op: test - path: /spec/groups/3/rules/0/alert - value: FluxKustomizationNotReady -- op: test - path: /spec/groups/3/rules/0/expr - value: | - gotk_reconcile_condition{type="Ready",status="False", - kind=~"Kustomization|HelmRelease"} == 1 -- op: replace - path: /spec/groups/3/rules/0/alert - value: FluxHelmReleaseNotReady -- op: replace - path: /spec/groups/3/rules/0/expr - value: | - gotk_reconcile_condition{type="Ready",status="False",kind="HelmRelease"} == 1 -- op: replace - path: /spec/groups/3/rules/0/annotations/summary - value: "Flux HelmRelease {{ $labels.namespace }}/{{ $labels.name }} not Ready" -- op: replace - path: /spec/groups/3/rules/0/annotations/description - value: "kubectl describe HelmRelease -n {{ $labels.namespace }} {{ $labels.name }}" diff --git a/scripts/tests/test-alert-coverage.sh b/scripts/tests/test-alert-coverage.sh new file mode 100755 index 0000000..bd03f55 --- /dev/null +++ b/scripts/tests/test-alert-coverage.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Proves scripts/validate-alert-coverage.sh fails when it should (platform-template#387). +# The production reconciliation Alert is the only notification for a failed +# HelmRelease, so a check that passes on a namespace nobody watches would be worse +# than no check. Each case edits a private copy of the manifests; the repository +# is never touched and no cluster is contacted. + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd "${script_dir}/../.." && pwd)" +readonly repo_root +readonly alert="k8s/providers/hetzner/infrastructure/controllers/flux-notifications/alert.yaml" + +fail() { + printf 'FAIL: %s\n' "$1" >&2 + exit 1 +} + +tmp_dir="$(mktemp -d)" +trap 'rm -rf "${tmp_dir}"' EXIT + +# A private copy of everything the validator reads. +fresh_copy() { + rm -rf "${tmp_dir}/repo" + mkdir -p "${tmp_dir}/repo/scripts" + cp -R "${repo_root}/k8s" "${tmp_dir}/repo/k8s" + cp "${repo_root}/scripts/validate-alert-coverage.sh" "${tmp_dir}/repo/scripts/" +} + +# run_validator : the validator's exit status, run from the copy's root. +run_validator() { + (cd "${tmp_dir}/repo" && bash scripts/validate-alert-coverage.sh) >"$1" 2>&1 +} + +# expect_failure +expect_failure() { + local name="$1" want="$2" out="${tmp_dir}/out.txt" + if run_validator "${out}"; then + fail "${name}: the validator passed" + fi + grep -Fq -- "${want}" "${out}" || { + cat "${out}" >&2 + fail "${name}: failed for another reason than '${want}'" + } + printf 'ok %s\n' "${name}" +} + +# --- the repository as committed passes ------------------------------------------------ +fresh_copy +run_validator "${tmp_dir}/out.txt" || { + cat "${tmp_dir}/out.txt" >&2 + fail "the committed manifests do not pass" +} +grep -Fq 'namespaces holding a HelmRelease' "${tmp_dir}/out.txt" || + fail "the validator passed without reporting HelmRelease coverage" +grep -Fq 'namespaces holding a Kustomization' "${tmp_dir}/out.txt" || + fail "the validator passed without reporting Kustomization coverage" +printf 'ok %s\n' "the committed manifests pass and report both kinds" + +# --- a namespace holding a HelmRelease is not watched ---------------------------------- +fresh_copy +yq -i 'del(.spec.eventSources[] | select(.kind == "HelmRelease" and .namespace == "velero"))' \ + "${tmp_dir}/repo/${alert}" +expect_failure "an unwatched HelmRelease namespace fails" \ + 'does not watch every namespace holding a HelmRelease' +grep -Fq 'namespace: velero' "${tmp_dir}/out.txt" || + fail "the failure did not name the unwatched namespace" + +# --- the Kustomization source relies on the default namespace --------------------------- +fresh_copy +yq -i 'del(.spec.eventSources[] | select(.kind == "Kustomization") | .namespace)' \ + "${tmp_dir}/repo/${alert}" +expect_failure "a Kustomization source without a namespace fails" \ + 'does not watch every namespace holding a Kustomization' + +# --- a named source is not a whole-namespace watch -------------------------------------- +fresh_copy +yq -i '(.spec.eventSources[] | select(.kind == "HelmRelease" and .namespace == "dex") | .name) = "dex"' \ + "${tmp_dir}/repo/${alert}" +expect_failure "a source that names one release does not cover its namespace" \ + 'does not watch every namespace holding a HelmRelease' + +# --- a layer that does not render is not a pass ----------------------------------------- +fresh_copy +printf 'resources:\n - does-not-exist.yaml\n' >"${tmp_dir}/repo/k8s/providers/hetzner/apps/kustomization.yaml" +expect_failure "a layer that fails to render fails closed" \ + 'refusing to validate Alert coverage on a partial render' + +printf 'alert-coverage: all cases passed\n' diff --git a/scripts/tests/test-drain-safety-policies.sh b/scripts/tests/test-drain-safety-policies.sh index c6b3897..3216eed 100755 --- a/scripts/tests/test-drain-safety-policies.sh +++ b/scripts/tests/test-drain-safety-policies.sh @@ -16,16 +16,14 @@ readonly catalog_source='^ - https://raw\.githubusercontent\.com/devantler-tech # The only chart PDB values allowed to set minAvailable, as # ":": the PDBs KSail installs with chart defaults # before Flux (cert-manager, metrics-server, the Kyverno admission controller, -# the Hetzner CSI controller), which the consumer PolicyException exempts, and -# the kube-prometheus-stack Alertmanager that platform-template#65 retires. +# the Hetzner CSI controller), which the consumer PolicyException exempts. readonly allowed_min_available='[ "cert-manager:podDisruptionBudget", "cert-manager:cainjector.podDisruptionBudget", "cert-manager:webhook.podDisruptionBudget", "metrics-server:podDisruptionBudget", "kyverno:admissionController.podDisruptionBudget", - "hcloud-csi:controller.podDisruptionBudget", - "kube-prometheus-stack:alertmanager.alertmanagerSpec.podDisruptionBudget" + "hcloud-csi:controller.podDisruptionBudget" ]' fail() { diff --git a/scripts/validate-alert-coverage.sh b/scripts/validate-alert-coverage.sh new file mode 100755 index 0000000..fdcc5a9 --- /dev/null +++ b/scripts/validate-alert-coverage.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# Fail if a Flux HelmRelease or Kustomization lives in a namespace the production +# reconciliation Alert does not watch. +# +# notification-controller has NO namespace wildcard: it defaults an omitted +# eventSources[].namespace to the Alert's own namespace and then matches on strict +# equality. `name: "*"` wildcards the NAME only. So every namespace holding a watched +# resource has to be listed explicitly in the Alert, and that list drifts the moment +# someone adds a release in a new namespace. +# +# An uncovered namespace is invisible. With Helm remediation on, a failed upgrade is +# rolled back to the last good release; the HelmRelease reports Ready again, its parent +# Kustomization stays healthy, and no alert ever fires while the release silently stops +# advancing. The Alert is the only production signal for that, so drift is a CI +# failure, not a comment. +# +# Run from the repository root. Renders the production layers; never contacts a cluster. + +set -euo pipefail + +readonly ALERT_FILE="k8s/providers/hetzner/infrastructure/controllers/flux-notifications/alert.yaml" +readonly LAYERS=( + # The cluster overlay declares the Flux Kustomizations that drive every other layer, + # so leaving it out would make flux-system look like it holds no Kustomization. + "k8s/clusters/prod" + "k8s/providers/hetzner/bootstrap" + "k8s/providers/hetzner/infrastructure/controllers" + "k8s/providers/hetzner/infrastructure" + "k8s/providers/hetzner/apps" +) + +for tool in kubectl yq; do + command -v "${tool}" >/dev/null 2>&1 || { + echo "::error::${tool} is required to validate Alert coverage." + exit 64 + } +done +[[ -f "${ALERT_FILE}" ]] || { + echo "::error::${ALERT_FILE} not found (run from the repository root)." + exit 64 +} + +work_dir="$(mktemp -d)" +trap 'rm -rf "${work_dir}"' EXIT +rendered="${work_dir}/rendered.yaml" + +# Render every layer once; both kinds are then read from the same output. +# +# Fail CLOSED on a render error. Swallowing a `kubectl kustomize` failure would drop +# every namespace declared only in that layer, so an uncovered namespace would never +# be compared against the Alert. A partial render is not a pass. +for layer in "${LAYERS[@]}"; do + if ! kubectl kustomize "${layer}"; then + echo "::error::kubectl kustomize failed to render ${layer}; refusing to validate Alert coverage on a partial render." >&2 + exit 1 + fi + echo '---' +done >"${rendered}" + +for kind in HelmRelease Kustomization; do + declared="${work_dir}/declared-${kind}.txt" + missing_namespace="${work_dir}/missing-namespace-${kind}.txt" + watched="${work_dir}/watched-${kind}.txt" + + # Only the Flux kinds count. A `kind: Kustomization` in the kustomize.config.k8s.io + # API group is a build input, not a Flux resource. + case "${kind}" in + HelmRelease) group="helm.toolkit.fluxcd.io" ;; + Kustomization) group="kustomize.toolkit.fluxcd.io" ;; + *) + echo "::error::Unknown watched kind ${kind}." + exit 64 + ;; + esac + + # These Flux APIs are namespaced and this render path has no targetNamespace + # fallback. Dropping a resource without a namespace would make the coverage set + # look complete although the resource itself is malformed. + yq ea -r " + select(.kind == \"${kind}\" and (.apiVersion | test(\"^${group}/\"))) + | select((.metadata.namespace // \"\") == \"\") + | (.kind + \"/\" + (.metadata.name // \"\")) + " "${rendered}" | awk 'NF && $0 != "---"' >"${missing_namespace}" + if [[ -s "${missing_namespace}" ]]; then + echo "::error::Rendered ${kind} resources are missing metadata.namespace; Alert coverage cannot be proven." + while read -r resource; do + [[ -n "${resource}" ]] && echo " ${resource}" + done <"${missing_namespace}" + exit 1 + fi + + yq ea -r " + select(.kind == \"${kind}\" and (.apiVersion | test(\"^${group}/\"))) + | .metadata.namespace + | select(. != null and . != \"\") + " "${rendered}" | awk 'NF && $0 != "---"' | LC_ALL=C sort -u >"${declared}" + + if [[ ! -s "${declared}" ]]; then + echo "::error::Rendered no ${kind}s at all, so coverage cannot be proven. Failing closed." + exit 1 + fi + + # yq treats `== "*"` as a wildcard comparison, so anchor a regex to require the + # literal whole-resource wildcard instead of accepting any named source. + yq e ".spec.eventSources[] | select(.kind == \"${kind}\" and (.name | test(\"^\\\\*$\"))) | .namespace | select(. != null and . != \"\")" "${ALERT_FILE}" | + LC_ALL=C sort -u >"${watched}" + + uncovered="$(comm -23 "${declared}" "${watched}")" + if [[ -n "${uncovered}" ]]; then + echo "::error::The reconciliation Alert does not watch every namespace holding a ${kind}." + echo "A ${kind} in an unwatched namespace can fail, roll back and report Ready, with no alert." + echo "Add a ${kind} eventSource for each namespace below to ${ALERT_FILE}:" + while read -r ns; do + [[ -n "${ns}" ]] && printf ' - kind: %s\n name: "*"\n namespace: %s\n' "${kind}" "${ns}" + done <<<"${uncovered}" + exit 1 + fi + + # A listed namespace that holds nothing never matches anything: harmless, but it is + # dead configuration, so say so without failing the build. + stale="$(comm -13 "${declared}" "${watched}" || true)" + if [[ -n "${stale}" ]]; then + echo "::warning::The Alert watches namespaces that hold no ${kind} (dead entries): $(echo "${stale}" | tr '\n' ' ')" + fi + + echo "βœ… Alert covers all $(wc -l <"${declared}" | tr -d ' ') namespaces holding a ${kind}." +done diff --git a/scripts/validate-bootstrap-workflow.sh b/scripts/validate-bootstrap-workflow.sh index e769f00..cbe305f 100755 --- a/scripts/validate-bootstrap-workflow.sh +++ b/scripts/validate-bootstrap-workflow.sh @@ -96,6 +96,19 @@ else if ! run_check local "HETZNER_LOCATION"; then fail "the configuration check required the prod-only HETZNER_LOCATION for a local bootstrap" fi + # The cluster heartbeat is production's one off-cluster dead-man signal, so a + # prod bootstrap without it would come up with nothing watching the cluster. + if run_check prod "CLUSTER_HEARTBEAT_URL"; then + fail "the configuration check accepted an empty CLUSTER_HEARTBEAT_URL for a prod bootstrap" + elif ! grep -q "CLUSTER_HEARTBEAT_URL" "$test_root/check.out"; then + fail "the configuration check did not name the missing CLUSTER_HEARTBEAT_URL" + fi + if ! run_check local "CLUSTER_HEARTBEAT_URL"; then + fail "the configuration check required the prod-only CLUSTER_HEARTBEAT_URL for a local bootstrap" + fi + if grep -q 'ALERTMANAGER_HEARTBEAT_URL' "$workflow"; then + fail "the bootstrap workflow still reads the retired ALERTMANAGER_HEARTBEAT_URL" + fi fi # --- A failed secret generator stops the encryption step --- @@ -114,7 +127,7 @@ else printf '#!/usr/bin/env bash\n[ %s -eq 0 ] || exit %s\necho generated\n' "$1" "$1" >"$stubs/openssl" chmod +x "$stubs/openssl" rm -f "$test_root/mutations" - env -i PATH="$stubs:$PATH" ALERTMANAGER_HEARTBEAT_URL= CLUSTER_HEARTBEAT_URL= \ + env -i PATH="$stubs:$PATH" CLUSTER_HEARTBEAT_URL= \ bash -euo pipefail -c "$encrypt_script" >"$test_root/encrypt.out" 2>&1 } diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh index 4477508..0ee2bb3 100755 --- a/scripts/validate-observability-option.sh +++ b/scripts/validate-observability-option.sh @@ -48,15 +48,39 @@ opencost_reference_count() { jq -s '[.[] | select(type == "object" and (tojson | test("opencost"; "i")))] | length' } -# Counts Grafana data sources that target the retired Loki service. -loki_datasource_count() { +# Counts what would bring the retired kube-prometheus-stack back into a render: +# a resource of a Prometheus-operator kind, anything in (or the) `monitoring` +# namespace, a rule or grant that names that namespace, the chart's release or +# repository, and a chart value that switches a ServiceMonitor, PodMonitor or +# PrometheusRule on. Vendored operators keep RBAC rules that merely name the +# `monitoring.coreos.com` API group; those grant nothing here and are not counted. +prometheus_stack_trace_count() { local rendered_path="$1" yq eval-all -o=json '.' "${rendered_path}" | - jq -s '[.[] | - select(type == "object" and .kind == "HelmRelease" and .metadata.namespace == "monitoring" and .metadata.name == "kube-prometheus-stack") | - .spec.values.grafana.additionalDataSources[]? | - select(.type == "loki" or ((.url // "") | test("loki\\.monitoring\\.svc")))] | length' + jq -s '[.[] | select(type == "object") | + select( + ((.apiVersion // "") | startswith("monitoring.coreos.com/")) or + (.kind == "Namespace" and .metadata.name == "monitoring") or + ((.metadata.namespace // "") == "monitoring") or + (.kind == "HelmRelease" and .metadata.name == "kube-prometheus-stack") or + (.kind == "HelmRepository" and .metadata.name == "prometheus-community") or + (.kind == "CiliumNetworkPolicy" and + ([.. | objects | .["k8s:io.kubernetes.pod.namespace"]? | select(. == "monitoring")] | length > 0)) or + (.kind == "ReferenceGrant" and any(.spec.from[]?; .namespace == "monitoring")) or + (.kind == "HelmRelease" and + ([.spec.values | .. | objects | to_entries[] | + select(.key | test("^(serviceMonitor|podMonitor|prometheusRule)$"; "i")) | + select((.value | type) == "object" and .value.enabled == true)] | length > 0)) + )] | length' +} + +# Counts HelmReleases in a render, to prove a zero above came from real input. +helm_release_count() { + local rendered_path="$1" + + yq eval-all -o=json '.' "${rendered_path}" | + jq -s '[.[] | select(type == "object" and .kind == "HelmRelease")] | length' } # Counts rendered resources whose serialized content still references Loki. @@ -145,32 +169,35 @@ assert_heartbeat_staged_after_policy_exclusion() { assert_resource_count "${apps_rendered_path}" CiliumNetworkPolicy observability allow-cluster-heartbeat 1 } -# Executes the workflow's real heartbeat preflight with bounded fixture URLs. +# Executes the workflow's real heartbeat preflight with one bounded fixture URL. run_bootstrap_heartbeat_preflight() { local bootstrap_workflow="$1" local cluster_heartbeat_url="$2" - local alertmanager_heartbeat_url="$3" local validation_script validation_script="$( - sed -n '/# The dedicated Coroot heartbeat has an exact hc-ping.com egress/,/^[[:space:]]*fill_base()/p' "${bootstrap_workflow}" | + sed -n '/# The cluster heartbeat is the one off-cluster dead-man signal and has/,/^[[:space:]]*fill_base()/p' "${bootstrap_workflow}" | sed '$d' )" + # An empty extraction would run nothing and "accept" every URL. + if [[ "${validation_script}" != *'CLUSTER_HEARTBEAT_URL'* ]]; then + echo "could not extract the heartbeat preflight from ${bootstrap_workflow}" >&2 + return 2 + fi env \ CLUSTER_HEARTBEAT_URL="${cluster_heartbeat_url}" \ - ALERTMANAGER_HEARTBEAT_URL="${alertmanager_heartbeat_url}" \ bash -c "set -euo pipefail ${validation_script}" >/dev/null 2>&1 } -# Proves that bootstrap accepts two canonical checks while rejecting aliases, -# alternate healthchecks.io operations, and noncanonical check identifiers. +# Proves that bootstrap accepts one canonical check and an unset one, while +# rejecting every other spelling: a URL the cluster's egress policy cannot reach +# would encrypt a monitor that is silently dead. assert_bootstrap_heartbeat_identity_contract() { local bootstrap_workflow="$1" local cluster_url='https://hc-ping.com/11111111-1111-4111-8111-111111111111' - local alertmanager_url='https://hc-ping.com/22222222-2222-4222-8222-222222222222' local cluster_alias - local alertmanager_alias + local status local -a cluster_aliases=( "${cluster_url}#cluster" "${cluster_url}?source=cluster" @@ -181,41 +208,26 @@ assert_bootstrap_heartbeat_identity_contract() { "${cluster_url}/1" 'https://HC-PING.com/11111111-1111-4111-8111-111111111111' 'https://hc-ping.com:443/11111111-1111-4111-8111-111111111111' - 'https://hc-ping.com/project-slug/check-slug' - 'https://hc-ping.com/AAAAAAAA-AAAA-4AAA-8AAA-AAAAAAAAAAAA' - ) - local -a alertmanager_aliases=( - "${cluster_url}" - "${cluster_url}#alertmanager" - "${cluster_url}?source=alertmanager" - "${cluster_url}/" - "${cluster_url}/start" - "${cluster_url}/fail" - "${cluster_url}/log" - "${cluster_url}/1" - 'https://HC-PING.com/11111111-1111-4111-8111-111111111111' - 'https://hc-ping.com:443/11111111-1111-4111-8111-111111111111' 'https://user@hc-ping.com/11111111-1111-4111-8111-111111111111' + 'http://hc-ping.com/11111111-1111-4111-8111-111111111111' 'https://hc-ping.com/project-slug/check-slug' + 'https://hc-ping.com/AAAAAAAA-AAAA-4AAA-8AAA-AAAAAAAAAAAA' + 'https://example.com/11111111-1111-4111-8111-111111111111' ) - if ! run_bootstrap_heartbeat_preflight "${bootstrap_workflow}" "${cluster_url}" "${alertmanager_url}"; then - echo "bootstrap must accept two distinct canonical healthchecks.io checks" >&2 + if ! run_bootstrap_heartbeat_preflight "${bootstrap_workflow}" "${cluster_url}"; then + echo "bootstrap must accept a canonical healthchecks.io check" >&2 return 1 fi - if ! run_bootstrap_heartbeat_preflight "${bootstrap_workflow}" "" "${alertmanager_url}"; then - echo "bootstrap must keep an unset direct-cluster check valid" >&2 + if ! run_bootstrap_heartbeat_preflight "${bootstrap_workflow}" ""; then + echo "bootstrap must keep an unset cluster check valid for a local bootstrap" >&2 return 1 fi for cluster_alias in "${cluster_aliases[@]}"; do - if run_bootstrap_heartbeat_preflight "${bootstrap_workflow}" "${cluster_alias}" "${alertmanager_url}"; then - echo "bootstrap must reject every noncanonical direct-cluster check URL" >&2 - return 1 - fi - done - for alertmanager_alias in "${alertmanager_aliases[@]}"; do - if run_bootstrap_heartbeat_preflight "${bootstrap_workflow}" "${cluster_url}" "${alertmanager_alias}"; then - echo "bootstrap must reject every alias of the direct-cluster check" >&2 + status=0 + run_bootstrap_heartbeat_preflight "${bootstrap_workflow}" "${cluster_alias}" || status=$? + if [[ "${status}" != "1" ]]; then + echo "bootstrap must reject every noncanonical cluster check URL (${cluster_alias}: status ${status})" >&2 return 1 fi done @@ -420,29 +432,7 @@ assert_coroot_heartbeat_substitution() { fi } -# Validates that the default profile still produces the Watchdog rule. -assert_watchdog_enabled() { - local rendered_path="$1" - local matches - - matches="$( - yq eval-all -o=json '.' "${rendered_path}" | - jq -s '[.[] | - select(.kind == "HelmRelease" and .metadata.namespace == "monitoring" and .metadata.name == "kube-prometheus-stack") | - select(.spec.values.defaultRules.create == true) | - select((.spec.values.defaultRules.rules.general // true) == true) | - select((.spec.values.defaultRules.disabled.Watchdog // false) == false) | - select([.spec.values.alertmanager.config.receivers[]? | - select(.name == "heartbeat") | - select(.webhook_configs[0].url == "${alertmanager_heartbeat_url:=https://example.invalid/no-heartbeat-configured}")] | length == 1)] | length' - )" - if [[ "${matches}" != "1" ]]; then - echo "kube-prometheus-stack must keep Watchdog on the distinct Alertmanager pipeline monitor in ${rendered_path}" >&2 - return 1 - fi -} - -# Counts the Dex public-client redirect that exists only for Grafana's native OIDC login. +# Counts the Dex public-client redirect that existed only for Grafana's native OIDC login. grafana_dex_redirect_count() { local rendered_path="$1" @@ -453,55 +443,20 @@ grafana_dex_redirect_count() { select(. == "https://grafana.${domain}/login/generic_oauth")] | length' } -# Reads whether the kube-prometheus-stack release still deploys Grafana. -grafana_enabled() { - local rendered_path="$1" - - yq eval-all -o=json '.' "${rendered_path}" | - jq -sr '[.[] | - select(type == "object" and .kind == "HelmRelease" and .metadata.namespace == "monitoring" and .metadata.name == "kube-prometheus-stack") | - .spec.values.grafana.enabled] | map(tostring) | join(",")' -} - -# Counts the network rules that exist only to serve the UIs: the auth proxy's egress to -# the monitoring namespace, and the monitoring namespace's ingress from the auth proxy -# and from the Gateway to Grafana's port. -kps_ui_network_rule_count() { +# Verifies that a rendered payload carries nothing of the retired +# kube-prometheus-stack. Controller payloads must also hold releases at all, so +# an empty or truncated render cannot pass as a clean one. +assert_prometheus_stack_absent() { local rendered_path="$1" + local traces - yq eval-all -o=json '.' "${rendered_path}" | - jq -s ' - ([.[] | select(type == "object" and .kind == "CiliumNetworkPolicy" and .metadata.namespace == "oauth2-proxy" and .metadata.name == "allow-auth-proxy") | - .spec.egress[]? | select(any(.toEndpoints[]?; .matchLabels["k8s:io.kubernetes.pod.namespace"] == "monitoring"))] | length) + - ([.[] | select(type == "object" and .kind == "CiliumNetworkPolicy" and .metadata.namespace == "monitoring" and .metadata.name == "allow-monitoring") | - .spec.ingress[]? | - select(any(.fromEndpoints[]?; .matchLabels["k8s:io.kubernetes.pod.namespace"] == "oauth2-proxy") or - any(.toPorts[]?.ports[]?; .port == "3000"))] | length)' -} - -# Verifies the Grafana, Prometheus and Alertmanager UI surfaces for one profile. -# expected_routes is 1 when the UIs are published (default) and 0 when retired (Coroot). -assert_kps_ui_surfaces() { - local controllers_path="$1" - local infrastructure_path="$2" - local expected_routes="$3" - local expected_grafana="$4" - local route - - for route in grafana prometheus alertmanager; do - assert_resource_count "${controllers_path}" HTTPRoute monitoring "${route}" "${expected_routes}" - done - assert_resource_count "${infrastructure_path}" ExternalSecret monitoring grafana-oidc "${expected_routes}" - if [[ "$(grafana_dex_redirect_count "${controllers_path}")" != "${expected_routes}" ]]; then - echo "expected ${expected_routes} Dex redirect(s) for Grafana in ${controllers_path}" >&2 + traces="$(prometheus_stack_trace_count "${rendered_path}")" + if [[ "${traces}" != "0" ]]; then + echo "expected no kube-prometheus-stack resource, monitoring namespace reference or enabled Prometheus-operator monitor in ${rendered_path}, found ${traces}" >&2 return 1 fi - if [[ "$(kps_ui_network_rule_count "${controllers_path}")" != "$((expected_routes * 3))" ]]; then - echo "expected $((expected_routes * 3)) UI-only network rule(s) for the kube-prometheus-stack UIs in ${controllers_path}" >&2 - return 1 - fi - if [[ "$(grafana_enabled "${controllers_path}")" != "${expected_grafana}" ]]; then - echo "expected kube-prometheus-stack grafana.enabled=${expected_grafana} in ${controllers_path}" >&2 + if [[ "$(grafana_dex_redirect_count "${rendered_path}")" != "0" ]]; then + echo "expected no Dex redirect for Grafana in ${rendered_path}" >&2 return 1 fi } @@ -526,12 +481,20 @@ assert_flux_notification_contract() { jq -s '[.[] | select(.apiVersion == "notification.toolkit.fluxcd.io/v1beta3") | select(.kind == "Alert" and .metadata.namespace == "flux-system" and .metadata.name == "platform-reconciliation") | - select(.spec == { + select((.spec | del(.eventSources)) == { "providerRef":{"name":"slack"}, "eventSeverity":"error", - "eventSources":[{"kind":"Kustomization","name":"*"}], "eventMetadata":{"summary":"Flux reconciliation error β€” production platform"} - })] | length' + }) | + # The cluster Kustomizations first, then one whole-namespace HelmRelease + # source per namespace. Which namespaces must be listed is proven against + # the renders by scripts/validate-alert-coverage.sh. + select(.spec.eventSources[0] == {"kind":"Kustomization","name":"*","namespace":"flux-system"}) | + select((.spec.eventSources[1:] | length) > 0) | + select(all(.spec.eventSources[1:][]; + (keys == ["kind","name","namespace"]) and .kind == "HelmRelease" and .name == "*" and + (.namespace | type == "string" and length > 0))) | + select(([.spec.eventSources[1:][].namespace] | unique | length) == (.spec.eventSources[1:] | length))] | length' )" dependency_alert_contract="$( yq eval-all -o=json '.' "${rendered_path}" | @@ -564,37 +527,6 @@ assert_flux_notification_contract() { fi } -# Ensures the production Coroot profile does not send Kustomization failures -# through both Flux notification-controller and the legacy Prometheus rule. -assert_legacy_flux_alert_deduplicated() { - local rendered_path="$1" - local matches - - matches="$( - yq eval-all -o=json '.' "${rendered_path}" | - jq -s '[.[] | - select(.apiVersion == "monitoring.coreos.com/v1") | - select(.kind == "PrometheusRule" and .metadata.namespace == "monitoring" and .metadata.name == "platform-critical") | - .spec.groups[] | - select(.name == "flux") | - select(.rules == [{ - "alert":"FluxHelmReleaseNotReady", - "expr":"gotk_reconcile_condition{type=\"Ready\",status=\"False\",kind=\"HelmRelease\"} == 1\n", - "for":"15m", - "labels":{"severity":"critical"}, - "annotations":{ - "summary":"Flux HelmRelease {{ $labels.namespace }}/{{ $labels.name }} not Ready", - "description":"kubectl describe HelmRelease -n {{ $labels.namespace }} {{ $labels.name }}" - } - }])] | length' - )" - - if [[ "${matches}" != "1" ]]; then - echo "production Coroot must retain only HelmRelease coverage in the legacy Flux Prometheus rule" >&2 - return 1 - fi -} - # Validates that only the observability namespace's generated policies defer # the dedicated heartbeat label to its narrower workload policy. assert_heartbeat_policy_exclusion() { @@ -924,7 +856,13 @@ for retired_path in \ k8s/providers/docker/apps-coroot \ k8s/providers/hetzner/infrastructure-controllers-coroot \ k8s/providers/hetzner/infrastructure-coroot \ - k8s/providers/hetzner/apps-coroot; do + k8s/providers/hetzner/apps-coroot \ + k8s/bases/infrastructure/controllers/kube-prometheus-stack \ + k8s/bases/infrastructure/alerts \ + k8s/bases/infrastructure/external-secrets/alertmanager-webhook.yaml \ + k8s/bases/infrastructure/vault-seed/push-secret-seed-alertmanager.yaml \ + k8s/providers/hetzner/infrastructure/controllers/kube-prometheus-stack \ + k8s/providers/hetzner/infrastructure/patches/retain-helm-release-alert.yaml; do if [[ -e "${repo_root}/${retired_path}" ]]; then echo "retired observability source must stay deleted: ${retired_path}" >&2 exit 1 @@ -948,10 +886,6 @@ for rendered_path in "${docker_controllers}" "${hetzner_controllers}"; do assert_resource_count "${rendered_path}" HelmRelease monitoring alloy 0 assert_resource_count "${rendered_path}" HelmRelease monitoring loki 0 assert_resource_count "${rendered_path}" HelmRepository monitoring grafana 0 - if [[ "$(loki_datasource_count "${rendered_path}")" != "0" ]]; then - echo "the Grafana Loki datasource must stay removed with Loki" >&2 - exit 1 - fi done # A leftover Loki address can sit in any payload (a Coroot, homepage or # datasource setting), so every render is searched, not only the controllers. @@ -975,23 +909,31 @@ assert_heartbeat_staged_after_policy_exclusion "${hetzner_controllers}" "${hetzn assert_auth_proxy_with_coroot "${docker_controllers}" assert_auth_proxy_with_coroot "${hetzner_controllers}" -# Coroot is the observability UI, so the Grafana, Prometheus and Alertmanager -# UIs, their routes, SSO redirect and OIDC secret are not published. -assert_kps_ui_surfaces "${docker_controllers}" "${docker_infrastructure}" 0 false -assert_kps_ui_surfaces "${hetzner_controllers}" "${hetzner_infrastructure}" 0 false +# kube-prometheus-stack is retired: no payload carries a Prometheus-operator +# resource, the monitoring namespace, a rule or grant that names it, or a chart +# value that would need the operator's definitions. The controller payloads must +# hold releases at all, so a truncated render cannot pass as a clean one. +for rendered_path in "${docker_controllers}" "${hetzner_controllers}"; do + if [[ "$(helm_release_count "${rendered_path}")" -lt 10 ]]; then + echo "expected the controller payload ${rendered_path} to hold its HelmReleases" >&2 + exit 1 + fi +done +for rendered_path in "${docker_controllers}" "${docker_infrastructure}" "${docker_apps}" \ + "${hetzner_controllers}" "${hetzner_infrastructure}" "${hetzner_apps}" "${coroot_base}"; do + assert_prometheus_stack_absent "${rendered_path}" +done for rendered_path in "${docker_controllers}" "${hetzner_controllers}"; do assert_resource_count "${rendered_path}" HelmRelease observability coroot-operator 1 assert_resource_count "${rendered_path}" Coroot observability coroot 0 assert_resource_count "${rendered_path}" HelmRelease observability audit-log-forwarder 0 - assert_resource_count "${rendered_path}" HelmRelease monitoring kube-prometheus-stack 1 assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-coroot 1 assert_resource_count "${rendered_path}" CronJob observability cluster-heartbeat 0 assert_resource_count "${rendered_path}" Secret observability cluster-heartbeat 0 assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-cluster-heartbeat 0 assert_opencost_absent "${rendered_path}" assert_coroot_sso_controller_contract "${rendered_path}" - assert_watchdog_enabled "${rendered_path}" done # The demo apps prove each apps check examined a real render rather than an @@ -1032,7 +974,6 @@ assert_resource_count "${hetzner_infrastructure}" Provider flux-system slack 0 assert_resource_count "${hetzner_infrastructure}" Alert flux-system platform-reconciliation 0 assert_resource_count "${hetzner_infrastructure}" Alert flux-system platform-dependency-wait 0 assert_resource_count "${hetzner_infrastructure}" Secret flux-system slack-webhook 0 -assert_legacy_flux_alert_deduplicated "${hetzner_infrastructure}" assert_opencost_absent "${hetzner_infrastructure}" assert_opencost_absent "${docker_apps}" assert_opencost_absent "${hetzner_apps}" @@ -1174,24 +1115,18 @@ for config_path in ksail.yaml ksail.prod.yaml; do fi done for documented_boundary in \ - "The stack is transitional in one respect." \ - "It runs Coroot, has no OpenCost," \ + "The stack runs Coroot and nothing beside it." \ + "It has no OpenCost," \ "has no Loki or Alloy log path" \ + "has no kube-prometheus-stack" \ "stages one hardened \`cluster-heartbeat\` CronJob" \ - "uses a separate optional \`CLUSTER_HEARTBEAT_URL\` secret" \ - "Never point both inputs at the same monitor" \ + "a production bootstrap requires the" \ "permits only \`hc-ping.com:443\`" \ "Bootstrap accepts only a canonical" \ - "compares check identities" \ "dependent apps layer stages the heartbeat only after" \ "Secret-backed environment variable" \ "stdin configuration" \ "platform-critical priority" \ - "keep Watchdog unchanged" \ - "preserves a dead-man signal" \ - "keeps kube-prometheus-stack" \ - "the Grafana, Prometheus and Alertmanager" \ - "Prometheus and Alertmanager keep running for alerting" \ "Cost allocation is therefore unavailable" \ "reuses the same \`alertmanager_webhook_url\`" \ "production to report failed reconciliations" \ @@ -1202,55 +1137,58 @@ for documented_boundary in \ "Local / Docker Coroot stays notification-free" \ "Flux notification-controller now owns reconciliation errors" \ "controller layer applies both Flux alerts" \ + "every \`HelmRelease\`" \ "narrow info-severity path for dependency waits" \ - "legacy Prometheus rule keeps only HelmRelease coverage" \ - "Kube-prometheus-stack keeps owning its remaining metric-backed" \ + "scripts/validate-alert-coverage.sh" \ + "adds that namespace to the alert" \ + "are not alerted today" \ "https://observability." \ "Dex-backed oauth2-proxy" \ "no direct Gateway route to the Coroot service" \ "the Admin role only behind that complete" \ "render exactly what the default paths render" \ "migrates with its next template sync" \ - "Flux removes the Loki, Alloy and OpenCost releases" \ - "Delete that leftover PersistentVolumeClaim"; do + "Flux removes" \ + "the old data is deleted" \ + "Helm does not delete CustomResourceDefinitions" \ + "\`ALERTMANAGER_HEARTBEAT_URL\` is no longer read" \ + "allow-node-exporter-ingress.yaml"; do if ! grep -Fq "${documented_boundary}" "${templating_guide}"; then echo "templating guide does not retain Coroot boundary: ${documented_boundary}" >&2 exit 1 fi done -if grep -R -n 'alertmanager_heartbeat_host' \ - "${repo_root}/.github/workflows/bootstrap.yaml" \ - "${repo_root}/docs" \ - "${repo_root}/k8s"; then - echo "Coroot heartbeat must not add an instance-owned host variable" >&2 - exit 1 -fi - alerting_guide="${repo_root}/docs/dr/alerting.md" for documented_heartbeat_boundary in \ - "Two independent dead-man checks" \ - "never give them the same ping URL" \ - "Watchdog stays enabled" \ + "There is no kube-prometheus-stack" \ + "One off-cluster dead-man check" \ "cluster_heartbeat_url" \ - "The three secret inputs" \ + "The two secret inputs" \ "\`hc-ping.com:443\`" \ - "Bootstrap compares" \ - "canonical check identities" \ + "Bootstrap rejects" \ "dependent apps layer" \ - "while Watchdog preserves" \ "curl as stdin configuration" \ "Flux notification-controller now owns reconciliation errors" \ "Two event-driven \`Alert\` objects" \ "The controller layer applies them before it waits" \ - "\`FluxHelmReleaseNotReady\`" \ - "watch every \`Kustomization\`" \ + "every \`Kustomization\` and every \`HelmRelease\`" \ + "Why HelmRelease is watched directly" \ + "scripts/validate-alert-coverage.sh" \ "Local / Docker renders no Slack" \ - "\`kube-prometheus-stack\` remains transitional for Watchdog" \ - "Alertmanager web UIs are" \ - "port-forward svc/kube-prometheus-stack-alertmanager 9093"; do + "Not alerted today" \ + "certificate expiry"; do if ! grep -Fq "${documented_heartbeat_boundary}" "${alerting_guide}"; then - echo "alerting guide does not retain heartbeat boundary: ${documented_heartbeat_boundary}" >&2 + echo "alerting guide does not retain alerting boundary: ${documented_heartbeat_boundary}" >&2 + exit 1 + fi +done + +# Operator documentation must not send anyone to a service that no longer exists. +for stale_doc in "${templating_guide}" "${alerting_guide}" \ + "${repo_root}/docs/BOOTSTRAP.md" "${repo_root}/docs/ARCHITECTURE.md" "${repo_root}/README.md"; do + if grep -n -E 'svc/kube-prometheus-stack|Watchdog|alertmanager_heartbeat_url|grafana\.\$\{domain\}' "${stale_doc}"; then + echo "documentation still describes the retired kube-prometheus-stack: ${stale_doc}" >&2 exit 1 fi done @@ -1258,15 +1196,9 @@ done bootstrap_workflow="${repo_root}/.github/workflows/bootstrap.yaml" for bootstrap_boundary in \ "CLUSTER_HEARTBEAT_URL: \${{ secrets.CLUSTER_HEARTBEAT_URL }}" \ - "heartbeat_url_pattern='^https://hc-ping\\.com/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$'" \ - "heartbeat_host_pattern='^https?://([^/?#]*@)?hc-ping\\.com([:/?#]|$)'" \ - "alertmanager_heartbeat_url_lower=\"\$(printf '%s' \"\${ALERTMANAGER_HEARTBEAT_URL}\" | tr '[:upper:]' '[:lower:]')\"" \ + "heartbeat_url_pattern='^https://hc-ping\\.com/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'" \ 'CLUSTER_HEARTBEAT_URL must be a canonical https://hc-ping.com/ URL or be unset' \ - 'ALERTMANAGER_HEARTBEAT_URL must be canonical when it uses hc-ping.com' \ - "cluster_heartbeat_check_id=\"\${BASH_REMATCH[1]}\"" \ - "alertmanager_heartbeat_check_id=\"\${BASH_REMATCH[1]}\"" \ - "[ \"\${cluster_heartbeat_check_id}\" = \"\${alertmanager_heartbeat_check_id}\" ]; then" \ - 'CLUSTER_HEARTBEAT_URL must use a different external check than ALERTMANAGER_HEARTBEAT_URL' \ + 'required+=(HCLOUD_TOKEN CLUSTER_HEARTBEAT_URL HETZNER_LOCATION)' \ '.stringData.cluster_heartbeat_url = strenv(CLUSTER_HEARTBEAT_URL)'; do if ! grep -Fq "${bootstrap_boundary}" "${bootstrap_workflow}"; then echo "bootstrap workflow does not retain cluster-heartbeat boundary: ${bootstrap_boundary}" >&2 @@ -1274,8 +1206,22 @@ for bootstrap_boundary in \ fi done -if grep -Fq 'alertmanager_heartbeat_url' "${repo_root}/k8s/bases/apps/cluster-heartbeat/cron-job.yaml"; then - echo "Coroot CronJob must not share the Alertmanager pipeline monitor" >&2 +# The Alertmanager heartbeat is retired with its stack: nothing may read it again. +# The bootstrap workflow may only delete the old key, and the protected placeholder +# secrets (`*.enc.yaml`, which automation never edits) are the one place it may +# still be listed until the workflow drops it. +if grep -n 'ALERTMANAGER_HEARTBEAT_URL' "${bootstrap_workflow}" || + grep -n 'alertmanager_heartbeat' "${bootstrap_workflow}" | + grep -v -F 'del(.stringData.alertmanager_heartbeat_url) |'; then + echo "the bootstrap workflow must not read the retired Alertmanager heartbeat" >&2 + exit 1 +fi +if ! grep -Fq 'del(.stringData.alertmanager_heartbeat_url) |' "${bootstrap_workflow}"; then + echo "the bootstrap workflow must drop the retired Alertmanager heartbeat key from the prod secret" >&2 + exit 1 +fi +if grep -R -n -i --exclude='*.enc.yaml' 'alertmanager_heartbeat' "${repo_root}/k8s"; then + echo "no manifest may read the retired Alertmanager heartbeat" >&2 exit 1 fi diff --git a/talos/control-planes/allow-node-exporter-ingress.yaml b/talos/control-planes/allow-node-exporter-ingress.yaml deleted file mode 100644 index cc838cd..0000000 --- a/talos/control-planes/allow-node-exporter-ingress.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1alpha1 -kind: NetworkRuleConfig -name: node-exporter-ingress -portSelector: - ports: - - 9100 - protocol: tcp -ingress: - # Node CIDR covers cross-node scrapes: Cilium masquerades pod->node-IP - # traffic to the source node IP. Same-node scrapes (the Prometheus pod - # hitting the hostNetwork node-exporter on its own node) are NOT - # masqueraded, so node-exporter sees the raw pod IP and the pod CIDR must - # be allowed too. - - subnet: 10.0.0.0/16 - - subnet: 10.244.0.0/16 diff --git a/talos/workers/allow-node-exporter-ingress.yaml b/talos/workers/allow-node-exporter-ingress.yaml deleted file mode 100644 index cc838cd..0000000 --- a/talos/workers/allow-node-exporter-ingress.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1alpha1 -kind: NetworkRuleConfig -name: node-exporter-ingress -portSelector: - ports: - - 9100 - protocol: tcp -ingress: - # Node CIDR covers cross-node scrapes: Cilium masquerades pod->node-IP - # traffic to the source node IP. Same-node scrapes (the Prometheus pod - # hitting the hostNetwork node-exporter on its own node) are NOT - # masqueraded, so node-exporter sees the raw pod IP and the pod CIDR must - # be allowed too. - - subnet: 10.0.0.0/16 - - subnet: 10.244.0.0/16 From aff02be623bba7f53ab0e50cebebcf78d4521cd4 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 10 Nov 2026 18:58:08 +0100 Subject: [PATCH 2/3] fix(observability): bound the heartbeat preflight extraction and tidy two texts The validator ran the bootstrap workflow's heartbeat preflight by extracting it up to the next function. If that function were renamed, the extraction would run on to the end of the workflow and execute its secret writes against the checkout. The preflight now ends at an explicit marker, and the validator refuses an extraction without it or with a secret-writing command in it. Also say what the platform-critical priority class is for now, and reword one migration note. Part of #387 Co-Authored-By: Claude Fable 5.2 --- .github/workflows/bootstrap.yaml | 2 ++ docs/TEMPLATING.md | 2 +- .../priority-classes/platform-critical.yaml | 7 ++++--- scripts/validate-observability-option.sh | 13 ++++++++----- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/bootstrap.yaml b/.github/workflows/bootstrap.yaml index 2784301..b35c818 100644 --- a/.github/workflows/bootstrap.yaml +++ b/.github/workflows/bootstrap.yaml @@ -260,6 +260,8 @@ jobs: echo "::error::CLUSTER_HEARTBEAT_URL must be a canonical https://hc-ping.com/ URL or be unset" exit 1 fi + # End of the heartbeat preflight. scripts/validate-observability-option.sh + # runs exactly the lines from the comment above down to this marker. fill_base() { # variables-base-secret yq -i ' diff --git a/docs/TEMPLATING.md b/docs/TEMPLATING.md index b61918f..7b5b0c6 100644 --- a/docs/TEMPLATING.md +++ b/docs/TEMPLATING.md @@ -96,7 +96,7 @@ changes. Four things need your attention: - `talos/` belongs to the instance, so the sync does not touch it. Delete `talos/control-planes/allow-node-exporter-ingress.yaml` and `talos/workers/allow-node-exporter-ingress.yaml`: nothing listens on that - port any more. + port now. The stack runs Coroot and nothing beside it. It has no OpenCost, has no Loki or Alloy log path, and has no kube-prometheus-stack: no Prometheus diff --git a/k8s/bases/bootstrap/priority-classes/platform-critical.yaml b/k8s/bases/bootstrap/priority-classes/platform-critical.yaml index 64e17d9..d825d0a 100644 --- a/k8s/bases/bootstrap/priority-classes/platform-critical.yaml +++ b/k8s/bases/bootstrap/priority-classes/platform-critical.yaml @@ -23,6 +23,7 @@ metadata: value: 1000000000 globalDefault: false description: >- - Important platform add-ons (monitoring/alerting) that must survive node - memory pressure. Ranked far above normal workloads but below the system-* - classes reserved for core cluster/node infrastructure. Not eviction-exempt. + Important platform add-ons (observability and the cluster heartbeat) that + must survive node memory pressure. Ranked far above normal workloads but + below the system-* classes reserved for core cluster/node infrastructure. + Not eviction-exempt. diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh index 0ee2bb3..161948e 100755 --- a/scripts/validate-observability-option.sh +++ b/scripts/validate-observability-option.sh @@ -176,12 +176,15 @@ run_bootstrap_heartbeat_preflight() { local validation_script validation_script="$( - sed -n '/# The cluster heartbeat is the one off-cluster dead-man signal and has/,/^[[:space:]]*fill_base()/p' "${bootstrap_workflow}" | - sed '$d' + sed -n '/# The cluster heartbeat is the one off-cluster dead-man signal and has/,/# End of the heartbeat preflight\./p' "${bootstrap_workflow}" )" - # An empty extraction would run nothing and "accept" every URL. - if [[ "${validation_script}" != *'CLUSTER_HEARTBEAT_URL'* ]]; then - echo "could not extract the heartbeat preflight from ${bootstrap_workflow}" >&2 + # The extraction must be exactly the preflight. An empty one would run nothing + # and "accept" every URL. One without its end marker ran on to the end of the + # workflow, and executing that would run the secret writes against this checkout. + if [[ "${validation_script}" != *'CLUSTER_HEARTBEAT_URL'* || + "${validation_script}" != *'# End of the heartbeat preflight.'* ]] || + grep -Eq '(^|[^[:alnum:]_.-])(yq|sops)([^[:alnum:]_.-]|$)' <<<"${validation_script}"; then + echo "could not extract exactly the heartbeat preflight from ${bootstrap_workflow}" >&2 return 2 fi env \ From fcdc61a0515afc70d45701f59a0ee152c621a52e Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 10 Nov 2026 18:59:57 +0100 Subject: [PATCH 3/3] test(observability): tighten two checks the review flagged - The alert-coverage check read a source without a name through a yq test on null, which stopped the script with an unclear error. A missing name is now simply not a whole-namespace watch, so the usual message names the namespace. The test covers it. - The retired-heartbeat check accepted any workflow line that contained the deletion expression. It now accepts only that exact line, so a line that deletes the key and sets it again is refused. Part of #387 Co-Authored-By: Claude Fable 5.2 --- scripts/tests/test-alert-coverage.sh | 9 +++++++++ scripts/validate-alert-coverage.sh | 2 +- scripts/validate-observability-option.sh | 6 +++--- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/scripts/tests/test-alert-coverage.sh b/scripts/tests/test-alert-coverage.sh index bd03f55..2f4a25f 100755 --- a/scripts/tests/test-alert-coverage.sh +++ b/scripts/tests/test-alert-coverage.sh @@ -81,6 +81,15 @@ yq -i '(.spec.eventSources[] | select(.kind == "HelmRelease" and .namespace == " expect_failure "a source that names one release does not cover its namespace" \ 'does not watch every namespace holding a HelmRelease' +# --- a source without a name is not a whole-namespace watch, and is reported plainly ----- +fresh_copy +yq -i 'del(.spec.eventSources[] | select(.kind == "HelmRelease" and .namespace == "keda") | .name)' \ + "${tmp_dir}/repo/${alert}" +expect_failure "a source without a name does not cover its namespace" \ + 'does not watch every namespace holding a HelmRelease' +grep -Fq 'namespace: keda' "${tmp_dir}/out.txt" || + fail "the failure did not name the namespace whose source has no name" + # --- a layer that does not render is not a pass ----------------------------------------- fresh_copy printf 'resources:\n - does-not-exist.yaml\n' >"${tmp_dir}/repo/k8s/providers/hetzner/apps/kustomization.yaml" diff --git a/scripts/validate-alert-coverage.sh b/scripts/validate-alert-coverage.sh index fdcc5a9..cfa32b5 100755 --- a/scripts/validate-alert-coverage.sh +++ b/scripts/validate-alert-coverage.sh @@ -102,7 +102,7 @@ for kind in HelmRelease Kustomization; do # yq treats `== "*"` as a wildcard comparison, so anchor a regex to require the # literal whole-resource wildcard instead of accepting any named source. - yq e ".spec.eventSources[] | select(.kind == \"${kind}\" and (.name | test(\"^\\\\*$\"))) | .namespace | select(. != null and . != \"\")" "${ALERT_FILE}" | + yq e ".spec.eventSources[] | select(.kind == \"${kind}\" and ((.name // \"\") | test(\"^\\\\*$\"))) | .namespace | select(. != null and . != \"\")" "${ALERT_FILE}" | LC_ALL=C sort -u >"${watched}" uncovered="$(comm -23 "${declared}" "${watched}")" diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh index 161948e..1141a1d 100755 --- a/scripts/validate-observability-option.sh +++ b/scripts/validate-observability-option.sh @@ -1214,12 +1214,12 @@ done # secrets (`*.enc.yaml`, which automation never edits) are the one place it may # still be listed until the workflow drops it. if grep -n 'ALERTMANAGER_HEARTBEAT_URL' "${bootstrap_workflow}" || - grep -n 'alertmanager_heartbeat' "${bootstrap_workflow}" | - grep -v -F 'del(.stringData.alertmanager_heartbeat_url) |'; then + grep 'alertmanager_heartbeat' "${bootstrap_workflow}" | + grep -v -x -E '[[:space:]]*del\(\.stringData\.alertmanager_heartbeat_url\) \|'; then echo "the bootstrap workflow must not read the retired Alertmanager heartbeat" >&2 exit 1 fi -if ! grep -Fq 'del(.stringData.alertmanager_heartbeat_url) |' "${bootstrap_workflow}"; then +if ! grep -q -x -E '[[:space:]]*del\(\.stringData\.alertmanager_heartbeat_url\) \|' "${bootstrap_workflow}"; then echo "the bootstrap workflow must drop the retired Alertmanager heartbeat key from the prod secret" >&2 exit 1 fi