From 315ae316abdb3c58a93162b6e1720d84f1d58990 Mon Sep 17 00:00:00 2001 From: Alejandro Acevedo Date: Thu, 5 Nov 2026 15:28:39 +0100 Subject: [PATCH 1/2] Copy product StackPacks from contrib-stackpacks Copy exactly open-telemetry, otel-k8s-crd and suse-observability from contrib-stackpacks at 9730d52293a9b9292c2ef8622d1ea44cab199783. All 180 content files match byte for byte, including existing formatting; package names and versions are unchanged. The legacy StackPacks repository and SDK are untouched. Validation: all three packages build with CLI 3.9.5 and pass the pinned platform validator. The negative mapping fixture is rejected as expected. Tracking: https://github.com/StackVista/stackstate/issues/892 --- stackpacks/open-telemetry/README.md | 0 stackpacks/open-telemetry/icons/databases.svg | 13 + .../open-telemetry/icons/open-telemetry.svg | 11 + stackpacks/open-telemetry/icons/queues.svg | 14 + .../icons/service-instances.svg | 16 + .../icons/service-namespaces.svg | 14 + stackpacks/open-telemetry/icons/services.svg | 16 + .../database-span-duration.md.hbs | 12 + .../service-instance-span-duration.md.hbs | 48 ++ .../service-instance-span-error-ratio.md.hbs | 54 ++ .../service-span-duration.md.hbs | 55 ++ .../service-span-error-ratio.md.hbs | 53 ++ .../open-telemetry/resources/RELEASE.md | 2 + .../open-telemetry/resources/configuration.md | 22 + .../resources/deprovisioning.md | 0 .../resources/detailed-overview.md | 3 + .../open-telemetry/resources/enabled.md | 19 + stackpacks/open-telemetry/resources/error.md | 0 stackpacks/open-telemetry/resources/logo.png | Bin 0 -> 29271 bytes .../open-telemetry/resources/overview.md | 9 + .../open-telemetry/resources/provisioning.md | 3 + .../resources/waitingfordata.md | 9 + .../settings/component-mappings/ecs-tasks.sty | 34 + .../settings/component-mappings/functions.sty | 32 + .../settings/component-mappings/hosts.sty | 31 + .../settings/component-mappings/k8s-pods.sty | 46 ++ .../component-mappings/namespaces.sty | 38 + .../component-mappings/service-instances.sty | 58 ++ .../settings/component-mappings/services.sty | 59 ++ .../open-telemetry/settings/main-menu.sty | 7 + .../monitor.sty | 25 + .../monitor.sty | 21 + .../service-span-duration/monitor.sty | 25 + .../service-span-error-ratio/monitor.sty | 21 + .../settings/presentations/common_filters.sty | 44 + .../settings/presentations/component.sty | 61 ++ .../settings/presentations/k8s-hosts.sty | 19 + .../settings/presentations/k8s-pods.sty | 19 + .../presentations/service-instances.sty | 451 +++++++++++ .../presentations/service-namespaces.sty | 62 ++ .../settings/presentations/services.sty | 608 ++++++++++++++ .../asynchronous-consumer-peer.sty | 38 + .../asynchronous-producer-consumer.sty | 40 + .../asynchronous-producer-peer.sty | 37 + .../relation-mappings/executes-ecs-task.sty | 36 + .../relation-mappings/executes-function.sty | 36 + .../relation-mappings/executes-host.sty | 36 + .../relation-mappings/k8s-pod-to-otel.sty | 46 ++ .../relation-mappings/provided-by.sty | 34 + .../relation-mappings/synchronous.sty | 49 ++ stackpacks/open-telemetry/stackpack.yaml | 20 + stackpacks/otel-k8s-crd/README.md | 0 stackpacks/otel-k8s-crd/icons/cr.svg | 4 + stackpacks/otel-k8s-crd/icons/crd.svg | 4 + .../otel-k8s-crd/resources/configuration.md | 43 + .../resources/detailed-overview.md | 18 + stackpacks/otel-k8s-crd/resources/enabled.md | 38 + stackpacks/otel-k8s-crd/resources/error.md | 0 stackpacks/otel-k8s-crd/resources/logo.svg | 16 + stackpacks/otel-k8s-crd/resources/overview.md | 5 + .../otel-k8s-crd/resources/provisioning.md | 3 + .../otel-k8s-crd/resources/waitingfordata.md | 0 .../custom-resource-definitions.sty | 49 ++ .../component-mappings/custom-resources.sty | 51 ++ .../otel-k8s-crd/settings/main-menu.sty | 7 + .../settings/presentations/common-filters.sty | 68 ++ .../custom-resource-definitions.sty | 114 +++ .../presentations/custom-resources.sty | 129 +++ .../relation-mappings/cr-instance-of-crd.sty | 30 + stackpacks/otel-k8s-crd/stackpack.yaml | 17 + stackpacks/suse-observability/README.md | 0 .../icons/suse-observability.svg | 5 + .../sts-apps-overrun-cpu-request.md.hbs | 26 + .../includes/generic/sts-java-long-gc.md.hbs | 22 + .../checks/monitors-execution-time-pod.md.hbs | 9 + .../monitors-failure-percentage-pod.md.hbs | 10 + ...ouse-replicated-part-failed-fetches.md.hbs | 5 + ...ta-after-merge-differs-from-replica.md.hbs | 14 + ...lickhouse-delayed-insert-throttling.md.hbs | 2 + ...se-distributed-files-to-insert-high.md.hbs | 16 + ...ted-sync-insertion-timeout-exceeded.md.hbs | 4 + ...criptor-buffer-read-or-write-failed.md.hbs | 4 + ...khouse-max-part-count-for-partition.md.hbs | 5 + .../clickhouse-readonly-replica.md.hbs | 10 + .../clickhouse-rejected-insert.md.hbs | 5 + .../clickhouse-replicated-data-loss.md.hbs | 6 + ...house-replicated-part-checks-failed.md.hbs | 1 + .../clickhouse/clickhouse-slow-read.md.hbs | 4 + .../clickhouse-too-many-connections.md.hbs | 5 + ...clickhouse-too-many-running-queries.md.hbs | 9 + ...house-zookeeper-hardware-exceptions.md.hbs | 2 + .../clickhouse-zookeeper-session.md.hbs | 3 + .../services/correlate/latency.md.hbs | 6 + .../includes/services/hbase/slow-gets.md.hbs | 6 + .../services/health-sync/latency.md.hbs | 51 ++ .../services/kafka-to-es/latency.md.hbs | 51 ++ .../metric-store-unreachable.md.hbs | 1 + ...tel-collector-metric-export-failure.md.hbs | 17 + ...el-collector-metric-processor-drops.md.hbs | 18 + ...l-collector-metric-refused-receiver.md.hbs | 19 + .../otel-collector-span-export-failure.md.hbs | 17 + ...otel-collector-span-processor-drops.md.hbs | 18 + ...tel-collector-span-refused-receiver.md.hbs | 18 + .../receiver/budget-saturation.md.hbs | 35 + .../receiver/hourly-budget-saturation.md.hbs | 31 + .../receiver/http-growing-requests.md.hbs | 11 + .../services/receiver/memory-limiter.md.hbs | 37 + .../services/receiver/rejected-logs.md.hbs | 50 ++ .../services/topo-sync/blocked.md.hbs | 10 + .../topo-sync/internal-latency.md.hbs | 56 ++ .../services/topo-sync/latency.md.hbs | 56 ++ .../services/topo-sync/multiple-hosts.md.hbs | 6 + .../topo-sync/snapshot-already-open.md.hbs | 6 + .../services/victoria-metrics/errors.md.hbs | 1 + .../victoria-metrics/freshness.md.hbs | 10 + .../insert-concurrency.md.hbs | 5 + .../victoria-metrics/labels-limit.md.hbs | 38 + .../services/victoria-metrics/rejected.md.hbs | 1 + .../select-concurrency.md.hbs | 5 + .../victoria-metrics/slow-insert-rate.md.hbs | 34 + .../victoria-metrics/storage-read-only.md.hbs | 8 + .../persistentqueue-dropping-data.md.hbs | 29 + .../remotewrite-samples-dropped.md.hbs | 9 + .../suse-observability/resources/RELEASE.md | 6 + .../resources/configuration.md | 3 + .../resources/deprovisioning.md | 0 .../resources/detailed-overview.md | 4 + .../suse-observability/resources/enabled.md | 11 + .../suse-observability/resources/error.md | 0 .../suse-observability/resources/logo.png | Bin 0 -> 17408 bytes .../suse-observability/resources/overview.md | 3 + .../resources/provisioning.md | 3 + .../resources/waitingfordata.md | 1 + .../settings/dashboards/data-ingestion.sty | 323 ++++++++ .../settings/dashboards/logs-processing.sty | 318 ++++++++ .../dashboards/metrics-processing.sty | 755 ++++++++++++++++++ .../dashboards/topology-processing.sty | 642 +++++++++++++++ .../settings/dashboards/traces-processing.sty | 272 +++++++ .../monitors/sts-apps-overrun-cpu-request.sty | 24 + .../generic/monitors/sts-java-long-gc.sty | 80 ++ .../generic/presentations/deployments.sty | 342 ++++++++ .../generic/presentations/overview.sty | 63 ++ .../settings/generic/presentations/pods.sty | 302 +++++++ .../generic/presentations/stackgraph-pods.sty | 95 +++ .../presentations/stackgraph-services.sty | 85 ++ .../settings/main-menu/main-menu.sty | 7 + .../monitors/k8s-resource-collector.sty | 66 ++ .../settings/services/aad/presentation.sty | 66 ++ .../settings/services/api/presentation.sty | 92 +++ .../checks/deployment-presentation.sty | 64 ++ .../settings/services/checks/monitors.sty | 50 ++ .../services/checks/pod-presentation.sty | 61 ++ .../settings/services/clickhouse/monitors.sty | 358 +++++++++ .../services/clickhouse/pod-presentation.sty | 83 ++ .../clickhouse/statefulset-presentation.sty | 84 ++ .../correlate/deployment-presentation.sty | 277 +++++++ .../settings/services/correlate/monitors.sty | 43 + .../services/correlate/pod-presentation.sty | 209 +++++ .../settings/services/es/presentation.sty | 71 ++ .../settings/services/hbase/monitors.sty | 23 + .../services/health-sync/monitors.sty | 22 + .../services/health-sync/presentation.sty | 92 +++ .../services/kafka-to-es/monitors.sty | 45 ++ .../services/kafka-to-es/presentation.sty | 80 ++ .../metrics-store-clients/monitors.sty | 23 + .../metrics-store-clients/presentations.sty | 130 +++ .../services/otel-collector/monitors.sty | 126 +++ .../services/otel-collector/presentation.sty | 240 ++++++ .../receiver/deployment-presentation.sty | 295 +++++++ .../settings/services/receiver/monitors.sty | 181 +++++ .../services/receiver/pod-presentation.sty | 257 ++++++ .../settings/services/server/presentation.sty | 41 + .../settings/services/state/presentation.sty | 134 ++++ .../settings/services/topo-sync/monitors.sty | 115 +++ .../services/topo-sync/presentation.sty | 160 ++++ .../services/victoria-metrics/monitors.sty | 330 ++++++++ .../victoria-metrics/presentation.sty | 194 +++++ .../settings/services/vmagent/monitors.sty | 165 ++++ .../services/vmagent/presentation.sty | 121 +++ stackpacks/suse-observability/stackpack.yaml | 17 + 180 files changed, 11483 insertions(+) create mode 100644 stackpacks/open-telemetry/README.md create mode 100644 stackpacks/open-telemetry/icons/databases.svg create mode 100644 stackpacks/open-telemetry/icons/open-telemetry.svg create mode 100644 stackpacks/open-telemetry/icons/queues.svg create mode 100644 stackpacks/open-telemetry/icons/service-instances.svg create mode 100644 stackpacks/open-telemetry/icons/service-namespaces.svg create mode 100644 stackpacks/open-telemetry/icons/services.svg create mode 100644 stackpacks/open-telemetry/includes/remediation-hints/database-span-duration.md.hbs create mode 100644 stackpacks/open-telemetry/includes/remediation-hints/service-instance-span-duration.md.hbs create mode 100644 stackpacks/open-telemetry/includes/remediation-hints/service-instance-span-error-ratio.md.hbs create mode 100644 stackpacks/open-telemetry/includes/remediation-hints/service-span-duration.md.hbs create mode 100644 stackpacks/open-telemetry/includes/remediation-hints/service-span-error-ratio.md.hbs create mode 100644 stackpacks/open-telemetry/resources/RELEASE.md create mode 100644 stackpacks/open-telemetry/resources/configuration.md create mode 100644 stackpacks/open-telemetry/resources/deprovisioning.md create mode 100644 stackpacks/open-telemetry/resources/detailed-overview.md create mode 100644 stackpacks/open-telemetry/resources/enabled.md create mode 100644 stackpacks/open-telemetry/resources/error.md create mode 100644 stackpacks/open-telemetry/resources/logo.png create mode 100644 stackpacks/open-telemetry/resources/overview.md create mode 100644 stackpacks/open-telemetry/resources/provisioning.md create mode 100644 stackpacks/open-telemetry/resources/waitingfordata.md create mode 100644 stackpacks/open-telemetry/settings/component-mappings/ecs-tasks.sty create mode 100644 stackpacks/open-telemetry/settings/component-mappings/functions.sty create mode 100644 stackpacks/open-telemetry/settings/component-mappings/hosts.sty create mode 100644 stackpacks/open-telemetry/settings/component-mappings/k8s-pods.sty create mode 100644 stackpacks/open-telemetry/settings/component-mappings/namespaces.sty create mode 100644 stackpacks/open-telemetry/settings/component-mappings/service-instances.sty create mode 100644 stackpacks/open-telemetry/settings/component-mappings/services.sty create mode 100644 stackpacks/open-telemetry/settings/main-menu.sty create mode 100644 stackpacks/open-telemetry/settings/monitors/service-instance-span-duration/monitor.sty create mode 100644 stackpacks/open-telemetry/settings/monitors/service-instance-span-error-ratio/monitor.sty create mode 100644 stackpacks/open-telemetry/settings/monitors/service-span-duration/monitor.sty create mode 100644 stackpacks/open-telemetry/settings/monitors/service-span-error-ratio/monitor.sty create mode 100644 stackpacks/open-telemetry/settings/presentations/common_filters.sty create mode 100644 stackpacks/open-telemetry/settings/presentations/component.sty create mode 100644 stackpacks/open-telemetry/settings/presentations/k8s-hosts.sty create mode 100644 stackpacks/open-telemetry/settings/presentations/k8s-pods.sty create mode 100644 stackpacks/open-telemetry/settings/presentations/service-instances.sty create mode 100644 stackpacks/open-telemetry/settings/presentations/service-namespaces.sty create mode 100644 stackpacks/open-telemetry/settings/presentations/services.sty create mode 100644 stackpacks/open-telemetry/settings/relation-mappings/asynchronous-consumer-peer.sty create mode 100644 stackpacks/open-telemetry/settings/relation-mappings/asynchronous-producer-consumer.sty create mode 100644 stackpacks/open-telemetry/settings/relation-mappings/asynchronous-producer-peer.sty create mode 100644 stackpacks/open-telemetry/settings/relation-mappings/executes-ecs-task.sty create mode 100644 stackpacks/open-telemetry/settings/relation-mappings/executes-function.sty create mode 100644 stackpacks/open-telemetry/settings/relation-mappings/executes-host.sty create mode 100644 stackpacks/open-telemetry/settings/relation-mappings/k8s-pod-to-otel.sty create mode 100644 stackpacks/open-telemetry/settings/relation-mappings/provided-by.sty create mode 100644 stackpacks/open-telemetry/settings/relation-mappings/synchronous.sty create mode 100644 stackpacks/open-telemetry/stackpack.yaml create mode 100644 stackpacks/otel-k8s-crd/README.md create mode 100644 stackpacks/otel-k8s-crd/icons/cr.svg create mode 100644 stackpacks/otel-k8s-crd/icons/crd.svg create mode 100644 stackpacks/otel-k8s-crd/resources/configuration.md create mode 100644 stackpacks/otel-k8s-crd/resources/detailed-overview.md create mode 100644 stackpacks/otel-k8s-crd/resources/enabled.md create mode 100644 stackpacks/otel-k8s-crd/resources/error.md create mode 100644 stackpacks/otel-k8s-crd/resources/logo.svg create mode 100644 stackpacks/otel-k8s-crd/resources/overview.md create mode 100644 stackpacks/otel-k8s-crd/resources/provisioning.md create mode 100644 stackpacks/otel-k8s-crd/resources/waitingfordata.md create mode 100644 stackpacks/otel-k8s-crd/settings/component-mappings/custom-resource-definitions.sty create mode 100644 stackpacks/otel-k8s-crd/settings/component-mappings/custom-resources.sty create mode 100644 stackpacks/otel-k8s-crd/settings/main-menu.sty create mode 100644 stackpacks/otel-k8s-crd/settings/presentations/common-filters.sty create mode 100644 stackpacks/otel-k8s-crd/settings/presentations/custom-resource-definitions.sty create mode 100644 stackpacks/otel-k8s-crd/settings/presentations/custom-resources.sty create mode 100644 stackpacks/otel-k8s-crd/settings/relation-mappings/cr-instance-of-crd.sty create mode 100644 stackpacks/otel-k8s-crd/stackpack.yaml create mode 100644 stackpacks/suse-observability/README.md create mode 100644 stackpacks/suse-observability/icons/suse-observability.svg create mode 100644 stackpacks/suse-observability/includes/generic/sts-apps-overrun-cpu-request.md.hbs create mode 100644 stackpacks/suse-observability/includes/generic/sts-java-long-gc.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/checks/monitors-execution-time-pod.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/checks/monitors-failure-percentage-pod.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/clickhouse/clickhouse-clickhouse-replicated-part-failed-fetches.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/clickhouse/clickhouse-data-after-merge-differs-from-replica.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/clickhouse/clickhouse-delayed-insert-throttling.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/clickhouse/clickhouse-distributed-files-to-insert-high.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/clickhouse/clickhouse-distributed-sync-insertion-timeout-exceeded.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/clickhouse/clickhouse-file-descriptor-buffer-read-or-write-failed.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/clickhouse/clickhouse-max-part-count-for-partition.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/clickhouse/clickhouse-readonly-replica.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/clickhouse/clickhouse-rejected-insert.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/clickhouse/clickhouse-replicated-data-loss.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/clickhouse/clickhouse-replicated-part-checks-failed.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/clickhouse/clickhouse-slow-read.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/clickhouse/clickhouse-too-many-connections.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/clickhouse/clickhouse-too-many-running-queries.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/clickhouse/clickhouse-zookeeper-hardware-exceptions.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/clickhouse/clickhouse-zookeeper-session.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/correlate/latency.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/hbase/slow-gets.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/health-sync/latency.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/kafka-to-es/latency.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/metrics-store-clients/metric-store-unreachable.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/otel-collector/otel-collector-metric-export-failure.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/otel-collector/otel-collector-metric-processor-drops.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/otel-collector/otel-collector-metric-refused-receiver.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/otel-collector/otel-collector-span-export-failure.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/otel-collector/otel-collector-span-processor-drops.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/otel-collector/otel-collector-span-refused-receiver.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/receiver/budget-saturation.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/receiver/hourly-budget-saturation.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/receiver/http-growing-requests.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/receiver/memory-limiter.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/receiver/rejected-logs.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/topo-sync/blocked.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/topo-sync/internal-latency.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/topo-sync/latency.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/topo-sync/multiple-hosts.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/topo-sync/snapshot-already-open.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/victoria-metrics/errors.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/victoria-metrics/freshness.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/victoria-metrics/insert-concurrency.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/victoria-metrics/labels-limit.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/victoria-metrics/rejected.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/victoria-metrics/select-concurrency.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/victoria-metrics/slow-insert-rate.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/victoria-metrics/storage-read-only.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/vmagent/persistentqueue-dropping-data.md.hbs create mode 100644 stackpacks/suse-observability/includes/services/vmagent/remotewrite-samples-dropped.md.hbs create mode 100644 stackpacks/suse-observability/resources/RELEASE.md create mode 100644 stackpacks/suse-observability/resources/configuration.md create mode 100644 stackpacks/suse-observability/resources/deprovisioning.md create mode 100644 stackpacks/suse-observability/resources/detailed-overview.md create mode 100644 stackpacks/suse-observability/resources/enabled.md create mode 100644 stackpacks/suse-observability/resources/error.md create mode 100644 stackpacks/suse-observability/resources/logo.png create mode 100644 stackpacks/suse-observability/resources/overview.md create mode 100644 stackpacks/suse-observability/resources/provisioning.md create mode 100644 stackpacks/suse-observability/resources/waitingfordata.md create mode 100644 stackpacks/suse-observability/settings/dashboards/data-ingestion.sty create mode 100644 stackpacks/suse-observability/settings/dashboards/logs-processing.sty create mode 100644 stackpacks/suse-observability/settings/dashboards/metrics-processing.sty create mode 100644 stackpacks/suse-observability/settings/dashboards/topology-processing.sty create mode 100644 stackpacks/suse-observability/settings/dashboards/traces-processing.sty create mode 100644 stackpacks/suse-observability/settings/generic/monitors/sts-apps-overrun-cpu-request.sty create mode 100644 stackpacks/suse-observability/settings/generic/monitors/sts-java-long-gc.sty create mode 100644 stackpacks/suse-observability/settings/generic/presentations/deployments.sty create mode 100644 stackpacks/suse-observability/settings/generic/presentations/overview.sty create mode 100644 stackpacks/suse-observability/settings/generic/presentations/pods.sty create mode 100644 stackpacks/suse-observability/settings/generic/presentations/stackgraph-pods.sty create mode 100644 stackpacks/suse-observability/settings/generic/presentations/stackgraph-services.sty create mode 100644 stackpacks/suse-observability/settings/main-menu/main-menu.sty create mode 100644 stackpacks/suse-observability/settings/monitors/k8s-resource-collector.sty create mode 100644 stackpacks/suse-observability/settings/services/aad/presentation.sty create mode 100644 stackpacks/suse-observability/settings/services/api/presentation.sty create mode 100644 stackpacks/suse-observability/settings/services/checks/deployment-presentation.sty create mode 100644 stackpacks/suse-observability/settings/services/checks/monitors.sty create mode 100644 stackpacks/suse-observability/settings/services/checks/pod-presentation.sty create mode 100644 stackpacks/suse-observability/settings/services/clickhouse/monitors.sty create mode 100644 stackpacks/suse-observability/settings/services/clickhouse/pod-presentation.sty create mode 100644 stackpacks/suse-observability/settings/services/clickhouse/statefulset-presentation.sty create mode 100644 stackpacks/suse-observability/settings/services/correlate/deployment-presentation.sty create mode 100644 stackpacks/suse-observability/settings/services/correlate/monitors.sty create mode 100644 stackpacks/suse-observability/settings/services/correlate/pod-presentation.sty create mode 100644 stackpacks/suse-observability/settings/services/es/presentation.sty create mode 100644 stackpacks/suse-observability/settings/services/hbase/monitors.sty create mode 100644 stackpacks/suse-observability/settings/services/health-sync/monitors.sty create mode 100644 stackpacks/suse-observability/settings/services/health-sync/presentation.sty create mode 100644 stackpacks/suse-observability/settings/services/kafka-to-es/monitors.sty create mode 100644 stackpacks/suse-observability/settings/services/kafka-to-es/presentation.sty create mode 100644 stackpacks/suse-observability/settings/services/metrics-store-clients/monitors.sty create mode 100644 stackpacks/suse-observability/settings/services/metrics-store-clients/presentations.sty create mode 100644 stackpacks/suse-observability/settings/services/otel-collector/monitors.sty create mode 100644 stackpacks/suse-observability/settings/services/otel-collector/presentation.sty create mode 100644 stackpacks/suse-observability/settings/services/receiver/deployment-presentation.sty create mode 100644 stackpacks/suse-observability/settings/services/receiver/monitors.sty create mode 100644 stackpacks/suse-observability/settings/services/receiver/pod-presentation.sty create mode 100644 stackpacks/suse-observability/settings/services/server/presentation.sty create mode 100644 stackpacks/suse-observability/settings/services/state/presentation.sty create mode 100644 stackpacks/suse-observability/settings/services/topo-sync/monitors.sty create mode 100644 stackpacks/suse-observability/settings/services/topo-sync/presentation.sty create mode 100644 stackpacks/suse-observability/settings/services/victoria-metrics/monitors.sty create mode 100644 stackpacks/suse-observability/settings/services/victoria-metrics/presentation.sty create mode 100644 stackpacks/suse-observability/settings/services/vmagent/monitors.sty create mode 100644 stackpacks/suse-observability/settings/services/vmagent/presentation.sty create mode 100644 stackpacks/suse-observability/stackpack.yaml diff --git a/stackpacks/open-telemetry/README.md b/stackpacks/open-telemetry/README.md new file mode 100644 index 0000000..e69de29 diff --git a/stackpacks/open-telemetry/icons/databases.svg b/stackpacks/open-telemetry/icons/databases.svg new file mode 100644 index 0000000..49e2227 --- /dev/null +++ b/stackpacks/open-telemetry/icons/databases.svg @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/stackpacks/open-telemetry/icons/open-telemetry.svg b/stackpacks/open-telemetry/icons/open-telemetry.svg new file mode 100644 index 0000000..923736b --- /dev/null +++ b/stackpacks/open-telemetry/icons/open-telemetry.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/stackpacks/open-telemetry/icons/queues.svg b/stackpacks/open-telemetry/icons/queues.svg new file mode 100644 index 0000000..8c580ba --- /dev/null +++ b/stackpacks/open-telemetry/icons/queues.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/stackpacks/open-telemetry/icons/service-instances.svg b/stackpacks/open-telemetry/icons/service-instances.svg new file mode 100644 index 0000000..b4ff7f8 --- /dev/null +++ b/stackpacks/open-telemetry/icons/service-instances.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/stackpacks/open-telemetry/icons/service-namespaces.svg b/stackpacks/open-telemetry/icons/service-namespaces.svg new file mode 100644 index 0000000..4db9548 --- /dev/null +++ b/stackpacks/open-telemetry/icons/service-namespaces.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/stackpacks/open-telemetry/icons/services.svg b/stackpacks/open-telemetry/icons/services.svg new file mode 100644 index 0000000..d6f1259 --- /dev/null +++ b/stackpacks/open-telemetry/icons/services.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/stackpacks/open-telemetry/includes/remediation-hints/database-span-duration.md.hbs b/stackpacks/open-telemetry/includes/remediation-hints/database-span-duration.md.hbs new file mode 100644 index 0000000..b5be314 --- /dev/null +++ b/stackpacks/open-telemetry/includes/remediation-hints/database-span-duration.md.hbs @@ -0,0 +1,12 @@ +SUSE Observability has detected that this service has a 95th percentile span duration above {{threshold}}s for spans. +This signals that a significant number of users are experiencing a slow service. + +Take the following steps to diagnose the problem: + +## Possible causes + +- Load on the database has increased + +### Load on the database has increased + +Check if the number of requests to the database has increased by looking at the "Span rate" chart on the [database highlight page](/#/components/{{ componentUrnForUrl }}). If so, a quick fix can be to give the database more resources like CPU and memory. diff --git a/stackpacks/open-telemetry/includes/remediation-hints/service-instance-span-duration.md.hbs b/stackpacks/open-telemetry/includes/remediation-hints/service-instance-span-duration.md.hbs new file mode 100644 index 0000000..38456ac --- /dev/null +++ b/stackpacks/open-telemetry/includes/remediation-hints/service-instance-span-duration.md.hbs @@ -0,0 +1,48 @@ +SUSE Observability has detected that this service instance has a 95th percentile span duration above {{threshold}}ms for spans. +This signals that a significant number of users are experiencing a slow service. + +Take the following steps to diagnose the problem: + +## Possible causes + +- Slow dependency or dependency serving errors +- Recent start of the instance +- Load on the service instance has increased +- Environment issues (e.g. certain nodes, database or services that the service depends on) + +### Slow dependency or dependency serving errors + +When troubleshooting slow dependencies or dependencies that are serving errors, their are two ways to find the root +cause: + +1. Analyze traces: begin by examining the traces with an error status in the [traces +perspective](/#/components/{{componentUrnForUrl}}/traces?filterBy=duration--{{ multiply threshold 1000000 }}__parentType--External%2CRoot). Expanding a span will show +the full trace for that span. Spans with an error status are marked and slow services will be displayed as long spans in +the trace. If you find an erroneous or slow span you can follow the link to the component to continue the investigation. + +2. Review Related Health Violations: Check, in the related health violations of this monitor if there are any health +violations on one of the services or instances that this instance depends on (focus on the lowest dependency). If you find a +violation to click on the component to see the related health violations. You can than continue your troubleshooting by +following those health violations. The related health violations can be found in the expanded version if you read this +in the pinned minimised version. + +### Recent start of the service instance + +Check if the instance was recently started: + +- If the trace metrics charts on the [highlight page](/#/components/{{ componentUrnForUrl }}) have no data at all, or only for very recent time period, the instance has usually just started. +- If the instance has just started, it might be that it is still warming-up. Compare the span duration metrics +for the current instance with the previous instances using the span duration metrics on the service highlights page for this service. Quickly navigate to the service via the link in the "About" section. +- Check if the instance is using more resources than before, consider scaling it up or giving it more resources. +- If increased latency is crucial, consider rolling back the service to the previous version: + - if that helps, then the issue is likely with new deployment + - if that does not help, then the issue may be in the environment (e.g. network issues or issues with the underlying infrastructure, e. g. database). [Traces](/#/components/{{componentUrnForUrl}}/traces?filterBy=duration--{{ multiply threshold 1000000 }}__parentType--External%2CRoot) can be used to look deeper. + +### Load on the service instance has increased + +Check if the amount of requests to the instance has increased by looking at the "Span rate" chart on the [service instance highlight page](/#/components/{{ +componentUrnForUrl }}). If so, a quick fix can be to start more instances for the service or giving the instances more resources like CPU and memory. + +### Environment issues + +Check the span duration of the different instances of the service. If only certain instances are having problems, it might be an issue with the instance itself or the server the is running on. Check if other service instances running in the same location show similar issues. If that's the case consider replacing the entire server. \ No newline at end of file diff --git a/stackpacks/open-telemetry/includes/remediation-hints/service-instance-span-error-ratio.md.hbs b/stackpacks/open-telemetry/includes/remediation-hints/service-instance-span-error-ratio.md.hbs new file mode 100644 index 0000000..bef737c --- /dev/null +++ b/stackpacks/open-telemetry/includes/remediation-hints/service-instance-span-error-ratio.md.hbs @@ -0,0 +1,54 @@ +SUSE Observability has detected that more than {{threshold}}% spans for your service instance have an Error +status, this may lead to noticeable disruptions or degraded performance for end-users. + +Take the following steps to diagnose the problem: + +## Possible causes + +- Slow dependency or dependency serving errors +- Recent update of the application +- Load on the application has increased +- Environment issues (e.g. certain nodes, database or services that the service depends on) + +### Slow dependency or dependency serving errors +When troubleshooting slow dependencies or dependencies that are serving errors, their are two ways to find the root +cause: + +1. Analyze traces: begin by examining the traces with an error status in the [traces perspective](/#/components/{{ +componentUrnForUrl}}/traces?filterBy=status--Error__parentType--External%2CRoot). Expanding a span will show +the full trace for that span. Spans with an error status are marked and slow services will be displayed as long spans in +the trace. If you find an erroneous or slow span you can follow the link to the component to continue the investigation. + +2. Review Related Health Violations: Check, in the related health violations of this monitor if there are any health +violations on one of the services or instances that this instance depends on (focus on the lowest dependency). If you +find a +violation to click on the component to see the related health violations. You can than continue your troubleshooting by +following those health violations. The related health violations can be found in the expanded version if you read this +in the pinned minimised version. + +### Recent update of the service + +Check if the instance was recently started: + +- If the trace metrics charts on the [highlight page](/#/components/{{ componentUrnForUrl }}) have no data at all, or +only for very recent time period, the instance has usually just started. +- If the instance has just started, it might be that it is still warming-up. Compare the span duration metrics +for the current instance with the previous instances using the span duration metrics on the service highlights page for +this service. Quickly navigate to the service via the link in the "About" section. +- Check if the instance is using more resources than before, consider scaling it up or giving it more resources. +- If increased latency is crucial, consider rolling back the service to the previous version: + - if that helps, then the issue is likely with new deployment + - if that does not help, then the issue may be in the environment (e.g. network issues or issues with the underlying infrastructure, e. g. database). [Traces](/#/components/{{componentUrnForUrl}}/traces?filterBy=duration--{{ multiply threshold 1000000 }}__parentType--External%2CRoot) can be used to look deeper. + +### Load on the service has increased + +Check if the amount of requests to the instance has increased by looking at the "Span rate" chart on the [service +instance highlight page](/#/components/{{ +componentUrnForUrl }}). If so, a quick fix can be to start more instances for the service or giving the instances more +resources like CPU and memory. + +### Environment issues + +Check the span duration of the different instances of the service. If only certain instances are having problems, it +might be an issue with the instance itself or the server the is running on. Check if other service instances running in +the same location show similar issues. If that's the case consider replacing the entire server. \ No newline at end of file diff --git a/stackpacks/open-telemetry/includes/remediation-hints/service-span-duration.md.hbs b/stackpacks/open-telemetry/includes/remediation-hints/service-span-duration.md.hbs new file mode 100644 index 0000000..1227b4e --- /dev/null +++ b/stackpacks/open-telemetry/includes/remediation-hints/service-span-duration.md.hbs @@ -0,0 +1,55 @@ +SUSE Observability has detected that this service has a 95th percentile span duration above {{threshold}}ms for spans. +This signals that a significant number of users are experiencing a slow service. + +Take the following steps to diagnose the problem: + +## Possible causes + +- Slow dependency or dependency serving errors +- Recent update of the service +- Load on the service has increased +- Environment issues (e.g. certain nodes, database or services that the service depends on) + +### Slow dependency or dependency serving errors + +When troubleshooting slow dependencies or dependencies that are serving errors, their are two ways to find the root +cause: + +1. Analyze traces: begin by examining the traces with an error status in the [traces +perspective](/#/components/{{componentUrnForUrl}}/traces?filterBy=duration--{{ multiply threshold 1000000 +}}__parentType--External%2CRoot). Expanding a span will show +the full trace for that span. Spans with an error status are marked and slow services will be displayed as long spans in +the trace. If you find an erroneous or slow span you can follow the link to the component to continue the investigation. + +2. Review Related Health Violations: Check, in the related health violations of this monitor if there are any health +violations on one of the services or service instances that this service depends on (focus on the lowest dependency). If you find a +violation to click on the component to see the related health violations. You can than continue your troubleshooting by +following those health violations. The related health violations can be found in the expanded version if you read this +in the pinned minimised version. + +### Recent update of the service + +Check if new instances for service were recently deployed: + +- Check if the instances are recently updated / redeployed. The "Service instances" metric chart on +the [service highlight page](/#/components/{{ componentUrnForUrl }}) will show a change in instance count, if an instance was restarted or replaced it shows a short spike in the chart. +- If application has just started, it might be that the service has not warmed up yet. Compare the response time metrics +for the current deployment with the previous deployment by checking the response time metric chart with a time interval +including both. +- Check if application is using more resources than before, consider scaling it up or giving it more resources. +- If increased latency is crucial, consider rolling back the service to the previous version: + - if that helps, then the issue is likely with new deployment + - if that does not help, then the issue may be in the environment (e.g. network issues or issues with the underlying infrastructure, e. g. database). [Traces](/#/components/{{componentUrnForUrl}}/traces?filterBy=duration--{{ multiply threshold 1000000 }}__parentType--External%2CRoot) can be used to look deeper. +- Repeat these steps for related services if needed + +### Load on the service has increased + +Check if the amount of requests to the service has increased by looking at the "Span rate" chart on the [service +highlight page](/#/components/{{ componentUrnForUrl }}). If so, a quick fix can be to start more instances for the service or giving the instances more +resources like CPU and memory. + +### Environment issues + +Check the span duration of the different instances of the service. If only certain instances are having problems, it +might be an issue with the instance itself or the server the is running on. Check if other service instances running in +the same location show similar issues. If that's the case consider replacing the entire server. diff --git a/stackpacks/open-telemetry/includes/remediation-hints/service-span-error-ratio.md.hbs b/stackpacks/open-telemetry/includes/remediation-hints/service-span-error-ratio.md.hbs new file mode 100644 index 0000000..46ef041 --- /dev/null +++ b/stackpacks/open-telemetry/includes/remediation-hints/service-span-error-ratio.md.hbs @@ -0,0 +1,53 @@ +SUSE Observability has detected that more than {{threshold}}% spans for your service have an Error +status, this may lead to noticeable disruptions or degraded performance for end-users. + +Take the following steps to diagnose the problem: + +## Possible causes + +- Slow dependency or dependency serving errors +- Recent update of the application +- Load on the application has increased +- Environment issues (e.g. certain nodes, database or services that the service depends on) + +### Slow dependency or dependency serving errors +When troubleshooting slow dependencies or dependencies that are serving errors, their are two ways to find the root +cause: + +1. Analyze traces: begin by examining the traces with an error status in the [traces perspective](/#/components/{{ +componentUrnForUrl}}/traces?filterBy=status--Error__parentType--External%2CRoot). Expanding a span will show +the full trace for that span. Spans with an error status are marked and slow services will be displayed as long spans in +the trace. If you find an erroneous or slow span you can follow the link to the component to continue the investigation. + +2. Review Related Health Violations: Check, in the related health violations of this monitor if there are any health +violations on one of the services or service instances that this service depends on (focus on the lowest dependency). If you find a +violation to click on the component to see the related health violations. You can than continue your troubleshooting by +following those health violations. The related health violations can be found in the expanded version if you read this +in the pinned minimised version. + +### Recent update of the service + +Check if the service was recently updated: + +- Go to the "Events" section in the middle of the [service highlight page](/#/components/{{ componentUrnForUrl }}) +- Check if there was a recent deployment by clicking on the Deployment section. +- Check if any of the pods are recently updated by clicking on "Pods of this service" in "Related resource" section of +the [service highlight page](/#/components/{{ componentUrnForUrl }}) and look if their Age is recent. +- If application has just started, it might be that the service has not warmed up yet. Compare the response time metrics +for the current deployment with the previous deployment by checking the response time metric chart with a time interval +including both. +- Check if application is using more resources than before, consider scaling it up or giving it more resources. +- If increased latency is crucial, consider rolling back the service to the previous version: + - if that helps, then the issue is likely with new deployment + - if that does not help, then the issue may be in the environment (e.g. network issues or issues with the underlying + infrastructure, e. g. database). [Traces](/#/components/{{componentUrnForUrl}}/traces?filterBy=duration--{{ multiply + threshold 1000000 }}__parentType--External%2CRoot) can be used to look deeper. +- Repeat these steps for related services if needed + +Check if the amount of requests to the service has increased by looking at the "Span rate" chart on the [service highlight page](/#/components/{{ componentUrnForUrl }}). If so, a quick fix can be to start more instances for the service or giving the instances more resources like CPU and memory. + +### Environment issues + +Check the span duration of the different instances of the service. If only certain instances are having problems, it +might be an issue with the instance itself or the server the is running on. Check if other service instances running in +the same location show similar issues. If that's the case consider replacing the entire server. \ No newline at end of file diff --git a/stackpacks/open-telemetry/resources/RELEASE.md b/stackpacks/open-telemetry/resources/RELEASE.md new file mode 100644 index 0000000..595de92 --- /dev/null +++ b/stackpacks/open-telemetry/resources/RELEASE.md @@ -0,0 +1,2 @@ +# Open Telemetry + diff --git a/stackpacks/open-telemetry/resources/configuration.md b/stackpacks/open-telemetry/resources/configuration.md new file mode 100644 index 0000000..91dc4f8 --- /dev/null +++ b/stackpacks/open-telemetry/resources/configuration.md @@ -0,0 +1,22 @@ +## Installation + +You can click on `Install` to install the Open Telemetry StackPack. Then follow the instructions here and in [the documentation](https://l.stackstate.com/open-telemetry-setup) to finish the Open Telemetry setup. + +The SUSE Observability Agent's Open Telemetry components are disabled by default. To receive traces and metrics pushed over OTLP from application SDKs, enable the agent's telemetry gateway when installing or upgrading the agent with Helm: + +```yaml +otel: + enabled: true + telemetryGateway: + enabled: true +``` + +or + +```bash +--set otel.enabled=true --set otel.telemetryGateway.enabled=true +``` + +Then point your SDKs at the telemetry gateway service, as described in the [telemetry gateway documentation](https://documentation.suse.com/cloudnative/suse-observability/latest/en/setup/otel/telemetry-gateway.html). + +On Rancher-managed clusters, also set `--set otel.integrations.rancherAgent=true` to enrich emitted logs with Rancher Manager URL and Harvester cluster ID metadata. Keep the default, `false`, on non-Rancher clusters. diff --git a/stackpacks/open-telemetry/resources/deprovisioning.md b/stackpacks/open-telemetry/resources/deprovisioning.md new file mode 100644 index 0000000..e69de29 diff --git a/stackpacks/open-telemetry/resources/detailed-overview.md b/stackpacks/open-telemetry/resources/detailed-overview.md new file mode 100644 index 0000000..dd44b19 --- /dev/null +++ b/stackpacks/open-telemetry/resources/detailed-overview.md @@ -0,0 +1,3 @@ +## Prerequisites + +The Open Telemetry StackPack expects traces and metrics to be provided via the Open Telemetry protocol. The easiest way to do this is by instrumenting your applications with one of the Open Telemetry SDKs and/or installing the Open Telemetry collector to send observability data to SUSE Observability. See the [SUSE Observability Open Telemetry documentation](https://l.stackstate.com/open-telemetry-setup). diff --git a/stackpacks/open-telemetry/resources/enabled.md b/stackpacks/open-telemetry/resources/enabled.md new file mode 100644 index 0000000..c5153f7 --- /dev/null +++ b/stackpacks/open-telemetry/resources/enabled.md @@ -0,0 +1,19 @@ +## The Open Telemetry StackPack is installed + +### What's next + +Instrument one or more applications with Open Telemetry SDKs to generate traces and metrics and install and configure the Open Telemetry collector to send data to SUSE Observability. See the [SUSE Observability Open Telemetry documentation](https://l.stackstate.com/open-telemetry-setup). + +To send SDK telemetry through the SUSE Observability Agent, enable its telemetry gateway when installing or upgrading the agent with Helm. The agent's Open Telemetry components are disabled by default: + +```bash +--set otel.enabled=true --set otel.telemetryGateway.enabled=true +``` + +Then point your SDKs at the telemetry gateway service, as described in the [telemetry gateway documentation](https://documentation.suse.com/cloudnative/suse-observability/latest/en/setup/otel/telemetry-gateway.html). + +On Rancher-managed clusters, also set `--set otel.integrations.rancherAgent=true`. Keep the default, `false`, on non-Rancher clusters. + +To send data to SUSE Observability a service token is needed. + + diff --git a/stackpacks/open-telemetry/resources/error.md b/stackpacks/open-telemetry/resources/error.md new file mode 100644 index 0000000..e69de29 diff --git a/stackpacks/open-telemetry/resources/logo.png b/stackpacks/open-telemetry/resources/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..ac6f4bd7e2f2c4543d5782fd6deee848b12393f0 GIT binary patch literal 29271 zcmY&A2RN1O`v+x}?3BG{k;;+1vO{*+B>UKV@4X|+EG1->gKU}ELXN#hGLF5+|2g>T z_jg@gpXYh+=U&gf-`5xRR8blammC)Yf#Au?JXV1~u;35~Ivg7l>;#yjCxids*vdS2 zfIx0Cq5h%CsxYoYAhZzK$6{)(sVn13))d2j_0x;A6a{m95W2-HgQ@l1>ApM=w7XrY znTf>DywEvV%-m%~MaLj}cQa@=ap)z;Vl{Ovo_1$RbR0^2eU`tscm#>^Ab+r`q8=wX zSTy0$V{K!D0Vtz>J}f9jV_*CVVIzX0UHmH3mqB$CBGci??@B0c=B?L?=TQUq)7)5O z4z#Us@${NrKL>ouZfS-gioVeQ-4NT5Qv4Av9c6xnWL*e>FjT~&6~P0&}FiCqP9G@<*zh-usqm>i|;5&vh0_hDnaO{1;;9b@n>xB?88iE>8oIkSy z&x$3smZsc@-%4QDtxVaRj68W!>*9IxlN+T)2(7d~6teOI#VVq5%|dNg!h2kxR@f=^ zoW&nenRpO^dl@%wNMMuE1)iku?u7u#UtZv$&Wb9Y&Y7W!mJ@{zN0o0a8O%IA*Gnw1 z7zKnsL8eoWp!5PuIKV3f5byoBsA?0n-ZtrFr~ zRS!^I4{U$2>z{M3r21}f1?$+{M_8991W+h2npQl*E|jqn35f0yl+Z9;a8*nU*c_uX zDSCXtcBTVZ-ytkP&|f_7WdmH(b9*pc;!6V@h;~}bUqYi!{~|6a7r$5=EMI{^@|aC* z5--`&0PLvAcdi9qt_K0XYOt_&p`}FV~tNv zdw5AM5s)X5?(F_gd#?cbaC#F{$qR9&lmYo>f|3%(3-ar&Q8K|zyoSFv9} zi_QaPzS5a!%3d&&GgLA6p|}ITvE{} zbM7dUkx7Em{4d9~GMlY5EyY&D2z?mUTmp61DI5Onpe~X=!-QB@280QGChQp4E zT|9-#G|z?KL<#@&8pFlK-e&+f^LnDMQ~ie&7*cGWymWB%rEu4Q zDSh~p#URmoxzPlqWi&_bA2J87xBYzg;!>gpltj!Azh1hu===2Xi3xrvgwax%N+*cm z1|^;IRO7_P&QSDu7?BAf1bEj52#us8FYG`j66`SA0O&vhJwBC{(6|fR54$H$;Y*qN zNNy$eza12h69Css>}g~)iNK{5Gu9`}LW~(OTgszv2^b&Rp)ALj`Zl;~sL`|Isn9PH zQWxgFEd!Lz@g6F4sRVbF5~!3@s4sOO1n~ce&r!g?l$Z*5*N@l#vj#cp$J_r|gV^yS z{eRZrJgfh|bu`cziNrwmW$Y3S0jdh8x9G>X@YYNQf8dRTrAvMnR@RG#^2V@{ZP5$G z4BZE7qyJgcgZ2CWSrdWv%>S&>!J0kv?|Cy>5Io5LiK_+J%L)I-9v)y%P5Li;^_YOY zJjTE587Khuei8j;uNV`c&-+L2SsVx!JC9-Rm)@Q)+KUNMPVv4}rV^k=`|tERHi)~g z|1sz43*xrRzxB6}^Em#Onk{OLPw;nL3__8w^gsAez^v;(U1z$Bf#<*eJ!0>95Z+4WuX3J}0`|53v7?5{!LR=ZJ&@Mm z+P`%(ik$qvwIpgC@o(J>Ov#|*-&zu^GeMB}kG&b-eZ7SL*i#1V{f_;|-YrxeMWq1Y z=b_9H!Kb5~I1sQldG+(zKEEzNCI=RQ++ z#uI4!WuJ}DKH{&jd~~?NM|V$C^u@7F=>prV?pLy{{{E-iuOJ8a z%3UA+MD=*2h}$Y7@n)IiUo&_PBpDjA^_&P?aiKzfH)+}h=MSIt#r?AdW9eQ@1DKtf zGjX#1nl19=1@v}Wf+PH&Hl72=de&j<>3vK5n^}e8Znkg;fnL(Xs}+A0Kuh6cp96WO z-oq`>L^w?b_})^vp>;iihG>Z7&jwUj<~#_-3IdZ&WF$b9J!VQ5UWF=_);|Oqp(>2k zu=7wOx(T#PooIlnWZnr~IH$~8ptS&^XjG{JXSs+UZ9uednwY4PD0uhrc}k&`1$H9R z#)_(Wdeblbm=TCKg?0P9s2RBk)xICVeR?L@{XYu9fCBH;CRFY79~dJ5OCb4QcHMy; zs}roxn2Glkjr)k@pElBfN#nAZh{*g^My9GiRP=k(3d_H9 zj#9mttO?0A_y5VL2w1&JRDxjsD^Rf$IA_?j67}z#RU)8Az+J`j?_2>e3S2r9o2b9{ zRTo1F$!7-BWKfp7p?`)-Dfs&`3Ge10?^BDT1g}Pd)=tQD;5;Hi~r6I z0*^43>RkIDdRjm)k)T>5{(J7(8*mPXTj2ITy>OuD(VDPG|7B_l7>Vdi(~9vw`2>O1 zWs;NA1pmp01tL-yQAr8iU%l9J0b#_9oSpwMwikX2TTQq4iEohY9G|y zGJ(mu*Z(260YFzmhe5Xa&%i;zI%GB>lmE+QPHHAzF9Ua#>t9hso588{(BWVI+=3mr z{wQ0rTpKysl1I{v?g}@&!J?tqV0FrL9q6ZvVzTbUwa^CjMdpLV| zEZ@U$eA2l|gGaD3cpf+uAmA>aervf5&N zR8+_mL_HAgG*Li3__RxdLXyyi4fDNATw?xR1~xRNpa1c^+MMSWr!a_UveLS*LIHI# znnWEJbpxE-B`g^xQl@}97}x#VBPbaLWYA;QvW zbHF_S5e22CFHh+3_LG6&RyP||ZROhl+<}9rL>4qB_(w*QC`_Wy0T^lMFr(B!@R9B? z7i#b6O)-io93yDhWx>gso{0xqtdyp(k@c)=N9WC5JFqiB(N1RK_t4`cpkHw8oKK-9 ztNRQVjXt1|MxjDUvqn(TmXM>cTDwA}Pw{X2b?C6M;y|_uvGlou+d&2T*a#X#P?`%| z8+G<7h^9?e?F<&GDfrnGSYXHZKF)~fPLmZblZEOlRq69mydCtV>ck~%sW}RGr-qK_ z0Wd+n7c-B^L={IlCEkmh{{oH~z!8Bwww(9>oG2papbRFeY|1Hp$1qU5HYi_3 zQThrfsW(}%vskExqdE>^X<`(3JBYTRJ)5RbP8stGZKXu*<$y{>O{_DRCUj&@uJc^c z?bsurOSAfxl%rtXWOWV{n~6uF&>1WpHZmuE0VD@Vd}*?J7M7#%=MpFx0LgX+mxhjf z_zx2RvTL#$3e8clxBvoGR6D(i1SnH;2})3U_XWP2dCLfD6_leec>yyeAV$IJpNyx? zWYJH?eJ=mLUnZWhQfIJY=*Y_3)CK8_>!<;40d4j9~_Z=oGF=TlRt7JAg zziE6TxZjOmt(%V7lFmK!k9zTm-t$sZbv z$4wDWYtjGuf?)m=K#b{VFHjW#rQ0`Ifdait_u^dd2&#khAe51cG)$rk z#E`!Fb(=46-5q*+Ji&e+w0Vqy>rg+1wy;Z_L4+fs7R*La<1QK4r4W{YncJXxP{y8) zVn7Embor8T?U(I#v^d%1A@py&t2rH_dTr~p(F|eoJkkt9YM%fv7KWT-FUpZ8G5c(QIL;mui8*v&GFCCy>W4o zLpWj-zkk_!o{kXu=7o+N5MSI0m{a)G;*30{Q5Mj@@{(Cie<&mtU0FcrQm4SQATmKYX_xGr z4|P5~9$3q`puh+U-$UnlY*g)ja%nH1xDlP?q(Ymbe!shRNyZ)+t&Kl)oj}>c<$@Be z?0L<4J5v+rTLIf}QW-+M{@-gZ5k;GT}R_-{}g>q0;*$3bu z>GT#hlSP++{6JyjR`)IZcgdvDUB!t@d*nuBxmnM=T76a#1PznB(M3uw$9v}!1=K)A zg)SfR*;RltgC-IF2|P@ZIYUqh4Q;Un+I|BK^zipdREU85wXOzGh?0swvVG{w@{l1? zzmqvDrO$c5li!SbjSq)z#pXa{ylGRF?%WN3L;)L4T*xO((aL8zAC6PnM@e4Cn& zgrga=+cR93s*nbTBzrr$CuM4LP#H~QJ!#n3QGGM|trAuUU7$(YD#8wLL;YcYxPBmk z@EVoA{_fE^b_jgS^ax+($pg6~+bCO%f#ruS6GqLnCtkbZx4vbxW;E^>am{3Cf_Ns0 z<&BadQy36x1BCEjWDf6=e-(XR+s&M{7|n)yC|Z&K4v*0kdRw<;$*GQ?zn)HzmHf21Y6I zrXRqr#3iC901+s0U7H*PP(dD&iQ2#e8@{0LNBsnW2$Uas@jGOP0PeT)-@*wj?p)O9 z5Xd|++|Md(MAXsnJncggKzsZ3GC@KTmWNkY3mUH(+dL8$iOhPqN2QR{VRheqA@)4T zq0)OMwv>fLsj$HIInjAQ)F^c1Dbup)IRps^IG{I|aALl}hXOcP4oTbFk*7giC2hZh zBrdeT35=2+%%apEIV7WkGo+o_45OK-^qrZ$0`Cpf##3Pso54V=nLh<+<;Ho&6&(he zR{IW+CG-Gfy>#aW#+m_ag*0*?_9wKX4?T$1v80$yv+W=R3H`_LMo%ORaR; zG$ya&ex64S2q`E_a-!o#VwooH8` zGA}&k6QjrhwV5o@d1!%vN)Yryxfjk1*#{MSC;$fO1;s(V3#d*&spEbb7uPXB@o)pI z=`N!3Iv7_2zlW|QT*B9Yf&(jbq?q`^J0T#ypizUG_R|G35NeQu1VIC(co702g5VmT zfeDEAC4piv)i98J57ONw!uIn*XQcNsGt{GsaGtP{V!{jIeM>>jL_tt$c=wVZg&U|U zEqlugZo)<&jR6dW_tS{w+1DsZ}SCx+(j!$Y&#(3|#7S4R}sM%()EoZn; zWicrT=)xF&8IjDm5hMC3v!p)ZxI;?M`HNCmkM6G)C`4as z2n?a?M`lKvSm*+?2pYa)o`p%W3UP2-n^Kq7HNAMicJMvGsfn#UW~L{_v#grzQcv38 zA~+)GjvfZCJ4bjF+^+HBPl@D|<0pC}2_y87oY~E~)J7R-C(=n6^d8V3>4tbLOKiE2 zeB*RzS3Z(RML&IkJ{q64@7t>E3A4JdDkgh*29*_K!j+xBW;q3>?aPJ*2 zit4<1S&UiU_ZHv1R5vk5PXXtHfEsE#0cAVfvl8@%vP+$I1D(3ypnCWtb8%R_`<=V+l1P!R^$clybQVYluM2)+J2=&STTmihh5WRr{JT2m)3)}#Jzb%e_ zPk`JmD7C6jG&z2G6Ey_zclNiNSAs^eb`d623m_ScY-E3t*bp=uM0h7od6Qi@>TmEq ziL~RM3D*F5sYtW(@L(IKnty-nrJ&n#y}@!XDND#K!bkL9{5TzI%>MGij(l@r%)5W9 z_CFM900ldk6ATcP`owS_Ww$D1VWD(>ebg&*@a6>_RL)Oi12c^T@=^*GrW5OYuIRzj zZ?mPP&F94~B%aPpLJ~J}_!^VcDuw0C5(31acU=J0W%!9D5u;#Nw(ZnoWw&V5-4F{g zf$xe*3rJsF94()Zy-cPjY12xJ&Bk6JeHU{UnEV^+XnKua+D^Q2Ae5vqKDnk-1{PjE z&-UX#s!V1Q-`N4x_t0jF=pOx6wjX=i$Ay9Yh>~Xv2qIg_148Mw^Qe!N%c#fDY7X; z{6mNL?w>qT4~;u)B>sC!1Dpy>K4Ar?nx37MV|i;{hNTtkk*5%D52p7}^-$H7$*7S_ zKEZs9&cSF`>oF|Pmc`=o?u^gVR~FVAq7vTS@>|E5X5dbifgMb~%IG%`{Cml%aBLKkg_dOLP_MU3AgjCZ?cd|DnWGZ@k|M*_u`j~faOBb-h zd@q*u)zY)d)^S^&l*mF#(9;+UXRPA=2uL2!HTmF7P+G;CapP^uOhn3T$Oc!nN8`#b zA&O0&z+%sU9g`)WI3i&zQ|5 zFu{<8ptRgdn5sIIR}w$TdzbfEdly;p?v(lTfb)0}X?eQe7x^bNt_o*@4Ar%Tt(=z; z5Y+IQQL%G^O}Q%QR&k6`llM5iUb!8CPi-smz;T@kb8TTUtNuJgK@JGubcROGT#r|I zIzI)N4D37#UI}>2TQXc&zOh&ip!ovth1rcBdZO|f(2N)b5tGTD+<^CEX2K(GE(vU8 z3XR=Gebone_CJ69CJ*Y-ty{%PMg*MA_G70tVWWUOnrnwAC>*FT<@UgJK7Ucj@jeKe z9ut_fhJazIPDQyLWr4y7FCuB7XpXE$!x!C{xO{`bZe6d&j)b4qmBKqYD z?Dw1Gw8YOuKDoEdi)IF91oLr5k&FFe#AW)UN+_`)y)`1Et-Und0vu;r=yyRKc=0Vj zvVs&D(}_5E4}LGO_X*eOeZQ_Y*^-uB8nClR(BKN%>g-*r^ZFX13u+U0@Wydo9(x-8 zK|kj?F@NAX7k5=6F9=0hkv``oAVlUZ;YcqAw}8Oxu(4Vbt4`L}^1?U4#~D(V7sBvH z_6AcCmz1b>s?FLJSY>^!`f{rP27{e*IRxH=fbmkdBB5$j4|sJ5xOjm*?S-DA`t{^n znX__#{{@naE3hF#ZUK3g3MC%uPq*qq8Ni!%moAJeT>)2i&*w7`ID=W^4eHG8k36d|<8w}=DC5XuG^toMmK>c#wV{jF~`z7lJu1J}-(z2T@ zE(8Ch9IQQ`fIz0W{h@G16EeE0pTpcH0?5rW8u4$*N?2SLlZ5B$UndZw028c;xL?0l z9_8)b@l$wL!))P`s)W_EDt`{K4~C)?L?sa$Jf8p#Pwz;&g!|Rk>5R+01`&?5Ne3$ zgC_I%L4Zs&;-d?WRbvtOS6R3@q61D^(3v%oH?7Qg0)yq{J65q9Um>yX?kkXW$Oq%X zl1E%~-jgW7_Vl1Q~c0{unWnaN0prOGZm0%YpKbQ^BuhH zH)c)qs$T1-L`8fQvAO@YIjrGs+GNCX*Nl(;ubHvh$i=cAcGA$ruFb{7WN!@H+!tCWR})j+!iUSSa+6NbeYJZ>4JXWA`(5KiE){ z*JQ4Y%=MwJWhCoq#jD$rpwC=D3)~A^2?NcQSz6BvS(G|<*P(H+W6-u$y04r~frf5% z4T^Q2-h^zm(dxdOFiAIv?s*&F)^Se`KFgHpavN|7ySi*lAE_Suk5QIi%UbCth&h@` zrg7fCdgf4#^X<{;dzpo|!}n)AT9mOCyFR%VdO8+bA;8P2qolE6k=<<}Kv&r$x^1sImvqTLi8I-db13i%P@-i~C-F*#%r1=NdthzllIb>3Y>2kYHix z%n_ZfRYzBTQ@m4%A8|5sq=yweyOp&sJT!k~$hza6(O^2=+cbE?nxgoTe^{qzRj&b$T z#FgP+uO_%=d&;+fk9+D~=^*d++vi{2^83;O5DX%^_aRL)E=!wZ&#U#TQxu=UH zLxvRYS^aTC&zdXWtk<$){8U9Ab`rZ#Njz!ea&WqyY=^VAI=k~AD49xQmB$|nKIo;dcpF(p7GXe056nAfojqkC|A)U3| ziM$t;09^|^gCYM{$IezSpXu;jWGY5oYq9Ph%R|YQ{?WVL8U0k_*K3w(g|WB5=V$vQ zlN{B1svkf83}(Sm5IQ*F;it^-(GBOP7B##ZmOdE)lmHC5hkO4u`Jzhb^@8nNWH)B6 z+jj*J(9C!k*i2d)K@_GP&SrKTZhnFZeTHeoC;;)JiQRRno4%bu^^hVhFaEeIsn3M2 z$kOkr-(<$Qx|$`9`Cf>R-d(!3kUtAdp5>cX*yrx?AKiOb%2Br3(P!42nEj&mfy!zK zS0}gdouY9SqbcDuB`m4ELb~WeLHTv3?bQf7$}l%}4VY4@!W82>E?)ct=SJ`~MryNz z<5ySLkkh8U$o}R|T%3I&5cyVjlAz)c(-NM$SC;-`K|X_&Icc8yj0( z7kjl|A~+5kR2rYOBdA+ZuXsIGc@vFwgR2G04!)LKJg(^3^~wH)4b@QuA*bEkDcFf| zy^+RrnEo>raTdu?*4j-dm#F-^s;7J()b|-J>x*x@@n^6<^tm3VYp|82}R6XOAu(-H%_~ybdYqgjRzgAO~_9QMD7Ga z%jT`KixFzkJMD;e5vMAuSS;=;@-`)2YU$griC(09T6oJ3DjEe zPU{;z=4zIu6#Iek_6;7WCtpSeRzwcy2d;kJA2F5eIc>uOoHToX&l3mg6=>*cr`fo|>Y`TK$KWGPA68SguM!IaHyB$YhF z9f!)FTgklKX5_<8G5b^EVZ0$pt6Y-ubRRG_{QM}>1$|^3PI(tYC`^sn6kjH5e;95u zBG;2|QDemX{5deYVWqw5*6+@H1AZU(+H`0JPwom2{FI}-o4h5_(Xl(*sQ62V`0duW z+s&!N*DX?9NonS6YgpvDO%`|SW#v>)P2Gy1!TV~MVb~+ZxZG88xkU$wTUb+IN>JC5 z+w6fq>;eLqeoO9;GivGB#ox4rJ94Q;f)l^l?pm+4@;Iwz>(d#Le|EBPb;0C6*nz{W6A)aHoR z>aVnkts-euFI2sELM{GiteJcm26*Mw&wpEd!nXtS^34rBqbT9RJ@Y@-C_Z$~Z{2v7 zbnpHP-6OjwCwyKwiK)Yn9q?`NZjE$o#3S^n_x{SC@!eiNw5*u1s|kOwI`BK`mYt`D zIa{<0=aTTRQK#9}zMc1Ti2jiyCNg=|d4zjSk-NLwCJ~Kb+vbs7bzMgLo%{+#f%JLD zS+U-1DI@1nm6(Ex;C6jb^2Ml7CZ5Eaz?~yU>ONHj#kk)_coz(+bQT(JcNmeFB(@4- z{+MJZU)qjsZp5=Dv7ucnk3dJPf&l|x=wywB*FE!LZ_2Je(@p+Ov(FpMM#|nI_7a_p zd6W7NKU%t1{ZZ?b=dKD~ZXWF`#NwV=5L7=cQ{$}>JP6C^aqwAZ{CL~w>)YpC-HdS! zkawVSH&OvpRq$v>$f^2x}E#JEH zM`zdEc_?cA=cSJRc+|{}*PY`Qtu1Op5)s*OUI(G%?W{eXC%oDL{up!FwL-?E$cp0N zROz;Y-{Yzb8nbR&MeT)TNcr1s6tkWR@|^ka!8 zp*2$_=B~0XX=5GEq>|QOc<8i=xK&beW;>C8KqbB}jh^x~I*Ytf@*}s$uno`S)sLk= zEq%V82}{iuvPExl!O~8Ss>ilFXdRg*n!HT0CO;1?nOG7a_I9SmPipTb(OVQ1#wy!? zP8Nih+0O6%p&sL^ShwxT(m~3~=};T1^m>)5HF#S0M}OMn123;az!wKc3HvvGtX@0& z7MNTEzH0P}e?BnLv%XZP0m5hm_&iwrr*c(%>CaK3Qcbm~x5G+q4>E2%dHIBBrc3(^ z($3o6lyw3d8g$zKc$HMmIm6+24BBoO|Xa>DGeSBMIGD#_tn8QHZp&P94{Lh#PikUTXDGviwZFo&Dat zmBpI!^g~Y0T@xE)!qS2iPidW&tic}d=P{dfKGxPYb1)r|q`!;%qod_c z-W?UDwwNDTUO$ZCi$XqlZd)@6-Z4+gom9VRUuI7-Z@nb^q|>BLO7+=_45yOq3HY*I zx3Q~i!E06?ee_E>xs*AtQeK^`fq;smMgz8Pe_J`d#o3CaQGfK#w0h)Wv8uIo`DR31 z{t{7%b`=;Xs%)*O9&PLu@%$Ds7E!#KDu_7Jm{@p3o^jl~RrsTa8`RYF?~cC7`lKtF zRc=hq{yvPt9szH}P1=&6Ht_xCNI#RBvA1B}a zww5RsuLyuo-ZCmH#@_`Kz z9#@~5Y}7xwmV3Qk?nD^Ijua|b(*$ItI~SsjrA=E2)%ZN)C|Y~9gFr>Rkb1xqF!}4p z%iRYwQHMYAY6{F=)2@FI>9`@MW0~==8L5Y)OfT4HJ^a|*NUlALCiqO@YsM0&0+|P@ zl?+tnrrdit)6~FQPxifLl?-RWps77HqZuUpFCpTygufWKW_Tg{9-$qo39fQ_Y(6T zRZccWG9RU96imBSGE{E6mu37;D199^QmZ-GZ-QlUE3GEBeX&eUr1jHRU4Q5m?vz$t zrfMG*nlx9HezjSb=C~!AzLC6)P|`cbcKtkTE#hc9Y>llmKazStx?>!YviPY-(z+v`E2oP#A zmyUn+Jtvn76#v%H4G6lDs(q*&!t7!MRg!VTn;Qsl?oy5N zDF@*v@sJg!+WvLUSeT0OD9|O6nlDa&Wo7k2}8WTkL*SL&IQ4M|8j!g4dOw`HLg8+FwJ1z+FaERNi+X zPP4ZUSMQouJw?=`?T}f@?AzKRjLmslsd>xjfgk$Ip53{*yg0V#rZ~s`VSLK6xj$`H z@HUgqFL}v=7Mo2K(wI^)Iec`1?=(01N7}HorCk=$%7Qj`uS9=0rvnM}(^Df3BnZ!9IX0dhc%<`M&a5wC!!u#M5sw8hdg_f@$H5twIePUk9dRJ@0#=&z!8E2J(^8 zSUoA!iyCp?mJmkj3UNQ%>?#rSR`L+TT2mbz-k6{3t#QNJyd^O(@A;aq_KZS zUq*Fe?&Ap!t9*QmL}ObgK|5V~Rl_c>CuGr557~;=>a15a(b)3Df2=18qri~b@QJ~T zMP5JXFG7=k;yA=h2jY_gxY8*pEbD%vKu6V&lJb6xwNGOzr%Lur8*2?cdO-zm6&dbY z#v&Y}2j^+kdDHSoqv&Y_mXM0}--LXgh24L4><*=havG(jdbK2Q%QpU+Q@BQrdq(+^ z;D-DuOX4d_;t-P2uG$d}c+!jo-B)RYBk>EBaVN~NEp`C0^)$XjF z*53H+J{@O1dG?Ba2Dlt`Hg6XeDg6hhU^n8jLLZ8zVoG>F$ym;Am~g!FahBahJ`}!C z3r+8Q9+E`X5s(_U#@FZMwrFqf`t|$wNwt%=6N{7Rx?B2-rUoOV<|BZzk{-^M)p8u{MVt2jbgMPw61rC?=fT74Mn7g#;r?Rdky~l;%}MJZ z;;>!z=VY&k-lxj=A(p~spYFSeKXz%FvGj^6$jqAhE(%KiMZnLpMZ34H9&pFyjgEQTvn~P|e;W2pOf=fn*csq;C{KwqW z99#qoPICG z$;oMJlqL7qX7Ap2;+2%4H3!R$08a6-fq234hy$1jqK4nkQ; zn`(%1_>UUr=hibjW(CUWM6io>+RJcPZN|9ZX*5L--Z0@&e~RT4DXQ+ND{+oEns-e@ z+r-+HH)dL!q<bd4v<02f`725kQ-*I$fLfw=X9rm8sq+eCJF(?MDj4pr5 z@T?c(^LB4l3VJrD{=*QhCCrTs-P$!=Ig#tquD8yj2@y=-hFsUZD*u|~*S#EUL@A!KZ z{GZSd+<5TOx7G95hdG9|1T-zrgbb1s2cL4wYJZ zd^=V=D)rLhFm1^N>rqSr!)lU7Ax1^J2bXPm8Ygm6PSdmp(HU)9!=+ne6OYeR+4d?% zcn;YsA(Ga%7_S(iN3Cr9gmY3eac#i)XA9pr`A8sRrwInVCWSPW+Ti;WHw4GWi0!S9}N!f{3q9^Km1vPw(k=(`de zv?U6+Y_^WYa%YbpWt^5;j>@m)6zL^YIW@y7Ra%BrFqfV z`sU9GD>qj@Z%#4^yZ=1vz~-Yx+YU39S5e=~kKWz|$U;rA%QiOfW}Iy3w0#~&9={eXqY7gd#-RV|mzmMrBsT4wdf-~F9ot8;Je*f0~; zY9pV@VV^?2KpC58cOD2A2qxlLg| z(jnG)w!jIS#Edxj9z{DYIYd%*G)kMs>-Kh=pR$bG_=3!GzthP*9Y;~)<=>pf0)w8> z>IVY_MvLKs6(3ui*ry{-Hii|a3&rEKVd*5-{jb~FNG+Y*I@srQqv368)0AZ2ad<{% z7<)6C9YE4a~?jmGhwVIC;v*})EQJcU6vgP_J%T-@t153 zTmGUc-oz#t#UEacjR;iEir!tW^uEREwnga$`xzfVFL6zb(>_N>rLFgxUfvwv?2D#9 zzbXXR_1Ll9(;Bh#vgbBV=-T)Y=V;43|f^&cS`uj^U~nc1EA42!o<1{6{fkI`Y_=ZwUFBp=6ta~k_u zJPUr+ODH9p|MoHGnDsGKvo$0!CPD=xNg`!l-}Bym&G`_#`KjAXMTR_TehFhz>LXlx z65kQg!Uu)ihFK3JV}cqC$5rW9lIWapI7gj8X7jCtPgC()f-Fu(hi{iQgWb?4$ zBc`~xre)VW?r4NKmyPeyRotR}Pr?d*4^AH}rv!5ZEVjMeY;iP3td>Fj$@_^|6=Kgd zj*5w-)F}gW3SGI@1HB+iV-XG)3AS;uFP{sq)mZaRV8hvDlkw_th1L2jjxCiN@(VHR z?4{BqiVmBFS8sk#m@IPz5)FSQV)`J*K^=}9S@5cyxlOGx{uUlHtd$X!ltp`sFWLLa}eJ9uNwUHqh5)~jR)MG z(oMZ(CdvEdnHPBg!6E!_RgWtk-DN&^$Qo^@{w*0<~ zqtI>mO7O>_dh<;7?;lCWOfbfl*-9scF)cq<-)VIbpif_>r_3NkCLRjfRnHAcMInw< zTE&-kY?nR-$RXvS?;O9#S_n)?<>%KAYIh)q;OZJnJ47u@u6U+YK=Mu-Y% z5+qp*OzgUS8E7mM_28+g6~@%F@xg|1EQP3a5RvG=oxLij2DM(4yRM=7hy2+xPkl|h z?c;A5(fdM64pXZ6RT|A=T5%qUCmU}Z=u@`ewa9*;a3nXNIJ{-%0#%*@X`7&tu`@dp z;pcLy-fN)t>?m;9k%Vs@xz%NsF)`{kjeKs2-$0e$bCM9yP*hi_fO;CYXjQ6xk zWlx}Jx7Cy-aji<_`=)gd5^eP9A7QKVnc1p9Jx#_Nf-leoib{+FZjWEHcB)Fic|7Vm<>M8U%BOba*uRn{Ie5m zt~H^M-^`)D#yD}P%D+fAOSpU^Qm{8mJVh&kKcnuahQmvC`0~c!-1M&x;Z4=S+`Z8% zh*&kx%~w5+Ow7e!*gj!c^VFKD?AP9L(&GPN>24#w^k*f4Glt`Ct9j796m0bJ=?KA` zyKpa^Ij??1<_AqAN4;?zGyvVwZSaN1Z-Gn#46Fh1936bb1S#NHaB)B~V%$2>TQS#d zI83V|c%bnL6gn*|Q?-iSy>-fJ{)*0V$zx0r!9-?OV%K1W9_AiE?e?^4JHIg2r2psV zAL^t+pN;eeTJs`n%O3B>>c$N_8pqy(gm%_2!8;fc?#Z6@%sJiNo8CiePTQ4GzEeH> zJ7?Yji3{?CW+fo{y6UaJp0K$>g&$72z2^q?pc%C=Pm{UQy;aW(?gR75^7|QA9`wG- zM>J^1-8&Hqzv1C!=pR4-s)JMcEOE8fWzAjr*d4>+2c=->WX}A7|2^*W!x#)JiHC7} zVNTDNRkpX}mgma1KjZPE5A^u5c`fK&t-k`j^&Z~xWhVS8C@>yET~u8kK$Qj6Dr)iF$Hk+IaLrWtv+cE-rity|wfu0)OZuF_zy%Lye^b_#pinQ z`9~z8ck)UyG%Bf%Y>RpZPl<-x* za%vIaQ9X7qQ2)yp$eqfXR7&r!(&_5%$1Z8?tMR((5rrL`y(UZzR_u(u-fw*7HB_FWWXA3UU?=IAg5wNJDh}K`jfhWa&9+Hp#QaIr3`D3+(!VlU(WTN^dZ7N02 z{pQepy`^!D^k6U#WZ`^&U zl0?ogBssH>xqsq*mU03KFi16#byN5~>Yq|9m#O>pX}0V46wwqGoRXyg5W0uTno*-K zQaq^>ODI)h>ea!#VaEi{b`iC&6=dLvge!K`grOnTG{VG`;JOjofYL8KuZ4X~Uouu* zS?zc6hN~*)cOwwgN_usqi3!Xi-fOxIs+k|j(;#?&A+jj+S6?e;>mZk_+R#%sTih9z zw{R%S#rjyeU+vqPbNI37$wFR4&$SV!7iSu?v%M}5OFN~yO7fBDB9*opb237HPi(kb z?7fuqFzWijn~Le6nTaRsjcPaCRU(CLp7^4#{+7qg&)crZ>2g-Zq(VFV9!Tacn@Cij z;p!dW?J!yL9`xRy8i-|H-?ro$<|}n|a%SjVplh&lXLU?81-aKgEm|GMtFVO>hST)G zS}bu(UTyioDbC)TQlZJ!B|&2-@Z}*V1_d?l*bEw){SU}QQv~bKJUOgJHyT!3kX!t- z%_mF^QUoD@NB1oH6F~*xhMPGgFnY*ePPertoYp%_uTGUnsDKAGZacJnXKip zAU`^0j*nzYzBT4fi5@xJ>8UV#y8s`E&}@g?Tu+YQ>+m7#D{Ptz`1R3ix3YeD;$~q^ zZkz|HnP!{ikC;M34m+DBe$UoHTRy&2;7W=P+?qI8N{;h)hVSxpK7)&;4kc!Q|G0b9{E!NQLgk8F@1rt-^@z72z#K>k}*zGI`aPHVO>XZaGBlY}5*7o)3H z95&;ExnBb@lmAJuWN-i5Q!%CR+W6-W1Sk9rl>v}9%R<{Jq8?u6o%=LFdPljv?(2Te zS@Vw^j%G{7(F)qy?*-{?@r8V+LAPq>S4rG*!YgV5r~(ny>FFLMmVx%QvgudsMoXoDYkkYzOG3AhK+ z^)5XAY=_BsjOSpDqvMgdJ7!cYv}8mzH|+QjIO?Jh?$<@dE&gQ*4a>w`yVzwb&0qcE z1ipu)R8!pn(B9Yat>vN=p&#ZB)^$0*qGPj!d|SL)zC0D@gVg$F{SJKt&dDN=gH?QKb_Env*6Q(9WX31nW$-iW9#Qrn?5x)9r2xa{liGk|d z8(1826YDVyh12*?QyO4z$`zizf60WATK+TXJ9_Xe813dx@sC8w3X9q*k3=Wo!jYgE zXKLN2X6&Vg^4_?vjHfv`XZs)iz zW-FscZrZi5DLrJ!YZYqu9&jP@VV3bn*&T-VCoM;wdQ0MMVn%e>^$Q7<)sHkjDrj#8 zM#ZBk(!5SaKWyBgo{5HhATm zWLKlFIV;$d^QHavzwI6b+ z=_R~-vQ@bozD3RMMRO2C&XKwW<(fd&{$Ev39th>y{Y3UeS-LlS!cD~eVr1Xi=voRX zF}U5tFk~Cc7|XS^-J%pS*}9P>OBh1-N)a>J4P%sLvW$^o7|ZxQ!;J6y&42Se?|Yte z&U4;#&U4Or&ujOLzwbD@7%ih~5cWqyNmM{R={KIx)Ar%rHUeCT7JvU6pJ0bzv448i zn5~daqnui`=Jn!U+o##;d!`Y^-~BwXS@RP2?OnohKZcZ;tMsJ0UMUDlRj4Vpao{RII^|ER^%{;+eZ2DO&w#R-hARx_bC^4SY zETSD85$YCz8$lY$33?3BJ~k+(zZ36C(#wZUv@a#?CA3bO#^w|iSYDg`>24u`HGjTY zT_O%YBz%JA)OMhCZv9dW(kS6VR&_%h-&b=={AW>PgP(2P{<6$^NWHkiAIxLnO0yB0 zUrK9zBGid08*R0%zfVA3*$aKHp&VA$n;2i8d2}}<(64ehzVHlxGO14#P6Y->?%Bx6 zROWhV$%?o4^5{wQ2_va~&6h`F@l=Ow;eCB|P98~QOJX`#Xn#mi=Z4cI(NLEcAA)@O zHmT9>$N<>eZeuh4MdjNB1-^URreOkA@m?q*rab(K;*SHVh5lP7F%PZl z_&0#WjQZ%UE&Y*e>6OdJ#Kp$R(`@y>yw4Uddv$L%&x?uu?uxd9FY;=7Ui^Eaa_UxE z)yEpRHU;BKLw{0@EGWRjU4bW7%2hdpkSt`h7-UV2uSYz%i=jo2#`)yiJXu(FX`0>q zEGYKq(B?#VV@ekE_ea0mn4RxHZKTA`IZy}2?vD(S|Ae3vHyDZM~ zjEv_}R!P9qo#_fD^!Y!EO<~lx=RKtM-gN`{xF5QFGyac{UsW80UCH~GKD#MGn^fX* zrQWx`6ZL*}q)*LiwYD?4ShoClWgenHVg;md8rs?jz&F?Y9yVrU?j`O(nQs=uYY8CB#%+pDOj?f;Lz|n zc0{$h{q9@wKARdn|L&>uUdC$XMADe6x-^MEF*_5`+-6m#4)QalG`&y7&KmmfMsLqm zIyAqp9bjAu{c$eM%5g2i#_j{kF2~S$K*Ljx2fYZzCB4q;3oO*KatMp@=qztQfkgh) z@}Jl6z%|$!_yA?<8%{@xs+w4T3pcu1TswMYc&eGs@`02|?_BH4(v4-zQ4DhSzkIXe zaH_8i@r?98d~1`ZN?%EJ4oBEz6@Kb5L_4jlsFns-myh%Z=w!96=+)|kCbxFfA68Xd zQ>M^B;2Edkqd-cajJQm>l>Bj_jn24`=@KYf8Cu%?;pe6=Y)=W$ir-FYypia{80=3e ztkPK2-hvZnr+|2N(7s{fEh%>f){PvqT9lvjzH=QBkAdd6@W6lN%gvaO8iY?o=O2JO zsvG_`_Y_7>gBO;hF8}T+G~*9zR4vr0QS~w!`gVQZy)o=^Q;XcK7RW*l>dniMjeZY%GGArj@-}c&nqGV(`SMDJu2P)g+bP(|xyRH~pKGL`zYp&T>g5 zD}Uj=R_I}O0$yV*2ILCD!z^m2N9y*D3Ne$K#~;QIRmx8sj<2aW{l5KG+ZQ=&_>2eb zM0F3TXL*n3SN}|Yrwhy02r`W_a(THeI(DQ!W#FD1AH{4TNr&+&5B0RNejuznfh>QA z^0xJ%9maNi*~lSXr@3S$YcFBpwS`EdN}oZ<6?0Vf;y?HA=TP}Rog6MI?1DWh`f2oj z=t#=ab-b$644q~11I`RsG%XLVOzZ zb1-wsRTo*E{>=h(*j?N%A{}o z;teMmbGcB}!?-%8?Mg29(GpBhgIy>nX7A;%_cc-0`7XllP=sztPG^vB#TE{N=?)pL zxpXaGxbn=8fd?U8NvNoZJlffFi?nae@}oXi0c1189m6gzUEwa@HF?#deqOD9ch(Ns9> zNWs}K^i8paT7P&8?|Uphd}O%+?HB7zwQ`HTIf43M6yfN*h$=`h!Px02?fXNcbxt=2 zd8NgN5OB>)%J1ana7by(K2JN@jk2!5^~e%0m%4_x&!y9&2Of+!7-7s^CuZx(_vmQ; zfcg;Jo{nkr9M9nEeO#274w3yT<|L7QXG>e>YNX*?zg+j`)!?xt2~~J~H5D%Y*qBls zW|L1W^!NR_AD{j%vueGI{<0a7%qv*)Tsn_jm>3&DBZg+-6ijZ_NML8ts1u9YDj>$I z_DX~#5d&TRd@|AS<=2o{{t9wH^je6_FZb30z6d<={Fe?|G8zQ6k0m%JNtYqpH6Mn%-TDYX3{p9wi&LA7`ExfI3a`X^jcZM#oD zU$S<#welN;bMH?}Tyoj-^v2^3z95LBAt~Fv@3$VE*nI%iE0q`zmk)z4OhX%kgQ^{` zOZ|vG;hZYFq@baW7c!$fIOxIsfgh{mfyZea5WwnKrU^goAbpEklXg{GTy7BY<~E;Q z4A`eF`IyLewIb>U7lN5|?cOi8c{claC#xEOq2`l4<&@f$5jCt=+oR_f=$zWn2Xghk z3cXO<^5Tjo&g5tJJS}|Fq3Ryj+HsM9DC)BSuPt$UpSjB4 zPb0!$z#0pl&Az?399i0(Qm15pvIf}I)nf?1XsZ}}0MoYTQkIVHR?%Y<44(kDAWd`q z*BTzi8#v_<#_>kcy*ZPTNV%EdX|h7xP#{BE`!aFmbs|IBo)XKC_hUQ@P{}F{9ZXg7 z4sg(y_?$IF`sbO!G}IV_g8ePqF|bZ-=gKHSPZv6To5-|FmT)_!O`NqyyDHAj~@uW3;W z7Q$==*X%kjUg)=1jY29T4Cao=k(5W9ArrHkF+X$ph!ZsTU>EG->WnFi#Ko1ml+SWp zh_ws(R%hadyxYCKsh;M{*FmxyWMDe5T7#R&3OxO(OYGhB;Z)*q}(=TRmWvRVMeM#~WylUadj#V?dP_ROzv z%l%@}_Mt>T+oZTV@W$a@nb3vQlIu~}%v)O%qBn1jsfUmqhBBYIHGy6HA%^ZgHMxSW z871c`iZ?g)!1*hSG}1xfTs#LFBT%998&7-~#i^N|6+0(ZjK0o*XG}IYW2XA5+a6Z? zO~NVl8_z_%WJ<*&BdBj7Ykw-NN_E$J)cS7btJaYuo#vScHzGgSiUq!her~?s!i`_b zFB!Q~QPmSIpc>^$zHnm4ipitWR_fcL5g4PL)&n{e0mnd{O3Urx|V9MvgkaL%a}_kq6n-0*MQiskFN%I8&!U-X-hRfJM#l%#2nQMKr2B!%CYY!ESb@`N#29HGIr3Gk zG5uY)JoeK%3Z}SjpX8Dl@x0%uW?g}ANZIDKyyK57wda^Mtt-!!Z*F+jN4oJ}QZl!R zr3}Z%P{eR*txwDzh|{M~PmRQy+B#i-HWxEonW6HqE|oDe5|o6qGPYksb~kmMz9YQvqVHFo zQMoJ`av;M$cjei!%IExyD^C({TgaZXXj`eX3`rW73VjqmyIIs;y5gla9h6UQeJ(sV zg`>WLV3wX}F$2<#HO4;$DP$aC9`LatE#3HaZw%D?0NgG~v#xApjef!Hiq~89_w#S0 zZ)E~xdm@}O_v%6YXtBMgwdqr=k~#}6a>cZ#N)&y1WJ!6|1BW~JP#}6FJ;>rNuTMSp+P;-QYFV6gB{f5}&yIUBXoulr1QeiLJQ=1=^c3rs0`;UfX*E!yVJ+hDZ?)NO$UHxTlROmlf zr^h#(eQhxgljNbWK6+97E&cYBy;coT!4cfW9pq;kv-wul044Q=^IjJMlb&>m)Cn0QvQ)Y|XJ18FTk4cLa`^-Z zku?y{1Fy~0OY*&^9#Xb-7I@PUKK8D1r7x_2A{ad!aUwk{u~LlhN^6T-NHTX5qb7oh zgHnG#MAc{&#e(e{&*zSdRP| z@guzPMQP!DooZ-}%j%b|;DZWDOD?NQpQfCbrk#Nf{SZe8GD&mEkv_WV#XIPumu{T& zWou{xGknHRF0`7)cfW=t$y`Z|K5zT}KU0VTa+DU|^XV%iS5GO+>G3uqlM`EQ803!F z>U+b3k}5i1Ca=4Zi?WorsAMiG!yi9!eJt^Mi>^Sn4CC3D%>XUll*(X4{Df76eSh8% zD(fd%a)1_C+w7x<6`k5pNUSzkl3s@2Z9*9-{ChPtA$_aor*Pa;oJ~}Cn7z;SEI+<0UUu%OsmvG&465#pT0o_;~m_Ws=^s za1hjfuzuOf2U3oTd$tC~5qzRGciw`}@AUshssF6&Qp#WH6>p0<#0Cm+7?R7PXj#$h zz9HsaSow=QD>FpD{7#1s2-G(!0$FZsr0?mE%j8?_-AgD_&4C{0+$>dXYq&`T2dqBk zs*hiCDUiksri+-CtdZLd9%@_&CzW0gP$;p3mAZ;h;|iijs}TeQOF+(TJhcI~zk3i%S(S-`~IKIe`zyIWUI+ZJ5XkE)3$d=c1b z#S#sbF{ag32gWbgf#drcNiHyD`Pi14Y^=hEec!!FfGSusLm;v!3SR+LiQD5p zzA#tSZdm!D8rt~mSTW|iXY+~2IqP|w=Xc5S4JLH)R|=fVCkk=K?LhJ+BhM@H2LrYR zvO*i;Zu)wPc;%xx$iBFpen2JyXjjLzNO_4LQAhB~YB|WKoC$J3Fug95${1Jh(Bwjx ziNPJu?6ih~Jn3F(pG*XqSc_=NkLMyUOaljdo66kf8saJrK&FAYe@rliVD12W?hHtYd^9ZS zVx4s8XvD%zfy#o1w)%V7n8b>(8*~agHkXhNtgXL*FT!=!Y z-V-eX;@hj;dFiM7NNwsk3C-Nl?d5$_I}}totkr4Vqg5xeFV$!J{^{ zGBmJ!;KePj_!y8S_Ugu$@{pEUg56LhwA~j2>c$FW4?k?2ZDwxZwL0IZP2u(uB+{+U z0cqF$WNfU)*xx<(geJZLq9ImmRW@-1P@J+uxBIgEAd?IvLaPZGr-(241cF?>hklPA zo)+@24GtQ>L|@7$9%fT&8!s43Nvu&_*K4YZJBlwcDfwgsgzc5EA40Io26%emCx^}R z>p_vkHkH|Wx`VRhtbsk%ThPfMz}wN_%cEK`$Xj9PIa*4ir`J^5Ta8&CA*{|>_Ws)Y z0p(Fv_S|oJ<9t&6kr<>K2w8Xj!pQws{f|BZ3aG?-&zthSk-u}E2Yn!B_0m4Cq8g@u zY!auq`tSCCFB0?Q>+?@DRO!v*?4WrWP%|CO`Kendm92NgrfW9DM|L+Sp47Mqym397 zEn}O*I+z>!un~7#2vAxX85LU}w_{qA+)%@);P>a z*gh$@YcREH|G^GZLlF*hMIV~z-!XBgKet7lVhc9}@NK`m8w=LFF$Wfi0}Ba(@=gKl z`UN&8OgyaI)N zW?dJ)J9f{c&fIVt-ZFE{deE{Pq|JzwB^u-M-(p$9CZz+DlOXWt-j;HXPdg_$zgMnpk|PljiWC{s?|l z-kmNNM<8<8hu7z#2p{l>GS4T<>|S;hKMHZD{~Jf3>|S<_i*o3ohw+7cqU`Qv%J@id zn(U4vU_y3Lp*E~52@#K7^iOk*a4u) z&qa~ouh5yz1}67)H-13IqTDI3(2A{zu3Vd`;VUWQ}hgSj&!MN4AXOO9#8TG(Z;9c2K~ zsQzk#0=0kbh?GRRg41YA!5SO8;FjJkl zEFFw?Ac?l!SFOC+bJ0m|y%r?T!Z48`kq^4~EAIbTItjXBv3@gU$n1<*k}D=}$oFSP z9%<9tQT-DrzMtE_gl3~}bLBX`%b~}Gw-MAv!G$XB6MkJlv2l=n6OdhB=7Ah=62gMJ z1lV6WiuQXAV$WVRovU_GS#ZlvI~mBIPRHAba)W5FhRIx1~M83q~c%{wX?~)JQ zCa_fnBziDqe`O$KLl0kfRlir#<949-5^&76e-YkK1u3!7>wc5A*V1IV{mi!;SA(gh z>J&jh?pL7cBl&B$QuYIs++9S_Y+NBErz$g)#%co!3Lv&&gM(g8ZuhZ)Ko%a{;HD4- zk;H!iTKbhBHp*UW%pF~T&ss-}m{{2JF*y{hdxmkkGF3m@2pv^jn@+IyY*ku^8ma$B-=J zDAl-3s zq=1}c#J2nQXssA=H5_59I|$SxLy#g4cYQT>DZdl=m^*AJyXehBcwmb$=IA1bR5dc2 z$d6@YaAd~w!+kEw*?hwwj=*GVYwWL-%AwnGRCDxY#8JF&9KoEUMxW8N z)-6uEFka{huz(^s{5A2+cu+?pg zl+XYCGm`?vX#br%9PvN@%+^@M8a?xY%21$Ifqhz?$UDM>=Z$_c#Eg~6PB!#bc|6n7 z#++gw){uJqD|?Z`?8#Nfa6+U(JS`bA%pNS$ilK9dABHyq*BX?YvPpJ3=Je1w=QgFy z&ZMyd*yC%_2=?`%j|C|zOJ$ezd1xSJ%f5WuxK_%=xJR*In&?}wdR22q1o;nDI zP0GS=5(Mxy1J3?*ih25-zmY>n{6FksRRyu7{|`H<%NF@)2{-nZLo$KxGUdskhyCwu zbxJYj6xJIpSzJDBv_a>3g1X>gcos;-?;srw&QaGS)q1V_hWc^ru|j$Lyh*n0`rCzB zHgDCevWbr%Ido$hhadLe>Tp@c>o4`RfKP+Tvp_Uf>qiW+yrJ>$ezH5T-FQ?h^)F{% zy2kd{KyUVJmnO4(bkAZD$H;?E4^;|X<;Q9`5&n)6Hx6>?dCz=gh%U=WJ~ne>kKp%v0vXk^ z3QW58m}($;i?r>S1vUV28pf}%*@xHQ;>d#8nw*>vdw~NS_XKk16{sB(p{Y0=xD$+I ReJK)RamM;|iJ9Bq{|9*oL=gZ0 literal 0 HcmV?d00001 diff --git a/stackpacks/open-telemetry/resources/overview.md b/stackpacks/open-telemetry/resources/overview.md new file mode 100644 index 0000000..d276aca --- /dev/null +++ b/stackpacks/open-telemetry/resources/overview.md @@ -0,0 +1,9 @@ +## Open Telemetry StackPack + +The Open Telemetry StackPack provides out-of-the-box support for traces and metrics ingested via the Open Telemetry protocol. Based on the traces a topology is created from services and service instances and their relations. Components are also created for other resources, like hosts, databases, and Kubernetes resources, that can be derived from the Open Telemetry data. + +Next to this also metric bindings and monitors are provided for traces and an increasing number of the languages supported by Open Telemetry like: +* Java +* .NET + +You can add custom metric bindings and monitors to support your application-specific use cases via Open Telemetry as well. diff --git a/stackpacks/open-telemetry/resources/provisioning.md b/stackpacks/open-telemetry/resources/provisioning.md new file mode 100644 index 0000000..a9522b1 --- /dev/null +++ b/stackpacks/open-telemetry/resources/provisioning.md @@ -0,0 +1,3 @@ +## Provisioning... + +Provisioning Open Telemetry monitoring. Please wait a moment. You can already start configuring your application for [Open Telemetry observability](https://l.stackstate.com/open-telemetry-setup). diff --git a/stackpacks/open-telemetry/resources/waitingfordata.md b/stackpacks/open-telemetry/resources/waitingfordata.md new file mode 100644 index 0000000..8317e75 --- /dev/null +++ b/stackpacks/open-telemetry/resources/waitingfordata.md @@ -0,0 +1,9 @@ +## The Open Telemetry StackPack is installed + +### What's next + +Instrument one or more applications with Open Telemetry SDKs to generate traces and metrics and install and configure the Open Telemetry collector to send data to SUSE Observability. See the [SUSE Observability Open Telemetry documentation](https://l.stackstate.com/open-telemetry-setup). + +To send data to SUSE Observability a service token is needed. + + \ No newline at end of file diff --git a/stackpacks/open-telemetry/settings/component-mappings/ecs-tasks.sty b/stackpacks/open-telemetry/settings/component-mappings/ecs-tasks.sty new file mode 100644 index 0000000..83c04ab --- /dev/null +++ b/stackpacks/open-telemetry/settings/component-mappings/ecs-tasks.sty @@ -0,0 +1,34 @@ +nodes: + - _type: "OtelComponentMapping" + name: "Task Component" + description: "Represents an ECS task" + identifier: "urn:stackpack:open-telemetry:otel-component-mapping:task" + input: + signal: + - "TRACES" + - "METRICS" + resource: + action: "'CREATE'" + condition: "'aws.ecs.task.id' in resource.attributes" + vars: + - name: "awsEcsTaskId" + value: "resource.attributes['aws.ecs.task.id']" + output: + identifier: "'urn:opentelemetry:task/' + vars.awsEcsTaskId" + name: "vars.awsEcsTaskId" + typeName: "'task'" + required: + tags: + - source: "'open-telemetry'" + target: "stackpack" + optional: + tags: + - source: "resource.attributes" + pattern: "^aws.ecs\\.(.*)" + target: "ecs.${1}" + - source: "resource.attributes" + pattern: "^cloud\\.(.*)" + target: "cloud.${1}" + expireAfter: 900000 + rank: + specificity: 110 diff --git a/stackpacks/open-telemetry/settings/component-mappings/functions.sty b/stackpacks/open-telemetry/settings/component-mappings/functions.sty new file mode 100644 index 0000000..cc42202 --- /dev/null +++ b/stackpacks/open-telemetry/settings/component-mappings/functions.sty @@ -0,0 +1,32 @@ +nodes: + - _type: "OtelComponentMapping" + name: "Function Component" + description: "Represents a serverless function (faas)." + identifier: "urn:stackpack:open-telemetry:otel-component-mapping:function" + input: + signal: + - "TRACES" + - "METRICS" + resource: + action: "'CREATE'" + condition: "'faas.id' in resource.attributes" + vars: + - name: "faasId" + value: "resource.attributes['faas.id']" + output: + identifier: "'urn:opentelemetry:function/' + vars.faasId" + name: "vars.faasId" + typeName: "'function'" + required: + version: "resource.attributes['faas.version']" + tags: + - source: "'open-telemetry'" + target: "stackpack" + optional: + tags: + - source: "resource.attributes" + pattern: "^(faas|cloud)\\.(.*)" + target: "${1}.${2}" + expireAfter: 900000 + rank: + specificity: 200 diff --git a/stackpacks/open-telemetry/settings/component-mappings/hosts.sty b/stackpacks/open-telemetry/settings/component-mappings/hosts.sty new file mode 100644 index 0000000..e804bb8 --- /dev/null +++ b/stackpacks/open-telemetry/settings/component-mappings/hosts.sty @@ -0,0 +1,31 @@ +nodes: + - _type: "OtelComponentMapping" + name: "Host Component" + description: "Represents a machine host" + identifier: "urn:stackpack:open-telemetry:otel-component-mapping:host" + input: + signal: + - "TRACES" + - "METRICS" + resource: + action: "'CREATE'" + condition: "'host.id' in resource.attributes" + vars: + - name: "hostId" + value: "resource.attributes['host.id']" + output: + identifier: "'urn:opentelemetry:host/' + vars.hostId" + name: "vars.hostId" + typeName: "'host'" + required: + tags: + - source: "'open-telemetry'" + target: "stackpack" + optional: + tags: + - source: "resource.attributes" + pattern: "^(os|host|cloud|azure|gcp)\\.(.*)" + target: "${1}.${2}" + expireAfter: 900000 + rank: + specificity: 50 diff --git a/stackpacks/open-telemetry/settings/component-mappings/k8s-pods.sty b/stackpacks/open-telemetry/settings/component-mappings/k8s-pods.sty new file mode 100644 index 0000000..f2f0b01 --- /dev/null +++ b/stackpacks/open-telemetry/settings/component-mappings/k8s-pods.sty @@ -0,0 +1,46 @@ +nodes: + - _type: "OtelComponentMapping" + name: "Kubernetes Pod" + description: "Represents a Kubernetes Pod" + identifier: "urn:stackpack:open-telemetry:otel-component-mapping:pod" + input: + signal: + - "TRACES" + - "METRICS" + resource: + action: "'CREATE'" + condition: >- + 'k8s.cluster.name' in resource.attributes && + 'k8s.namespace.name' in resource.attributes && + 'k8s.pod.name' in resource.attributes + vars: + - name: "clusterName" + value: "resource.attributes['k8s.cluster.name']" + - name: "namespaceName" + value: "resource.attributes['k8s.namespace.name']" + - name: "podName" + value: "resource.attributes['k8s.pod.name']" + output: + identifier: >- + 'urn:opentelemetry:kubernetes:/' + vars.clusterName + ':' + vars.namespaceName + ':pod/' + vars.podName + name: "vars.podName" + typeName: "'pod'" + required: + additionalIdentifiers: + - "'urn:kubernetes:/' + vars.clusterName + ':' + vars.namespaceName + ':pod/' + vars.podName" + tags: + - source: "'open-telemetry'" + target: "stackpack" + - source: "vars.clusterName" + target: "cluster-name" + - source: "vars.namespaceName" + target: "namespace" + - source: "vars.clusterName + '/' + vars.namespaceName" + target: "k8s-scope" + optional: + tags: + - source: "resource.attributes['deployment.environment']" + target: "deployment-environment" + expireAfter: 900000 + rank: + specificity: 120 diff --git a/stackpacks/open-telemetry/settings/component-mappings/namespaces.sty b/stackpacks/open-telemetry/settings/component-mappings/namespaces.sty new file mode 100644 index 0000000..eebe21b --- /dev/null +++ b/stackpacks/open-telemetry/settings/component-mappings/namespaces.sty @@ -0,0 +1,38 @@ +nodes: + - _type: "OtelComponentMapping" + name: "Kubernetes Namespace" + description: "Represents a logical service namespace in Kubernetes" + identifier: "urn:stackpack:open-telemetry:otel-component-mapping:namespace" + input: + signal: + - "TRACES" + - "METRICS" + resource: + action: "'CREATE'" + condition: "'service.namespace' in resource.attributes" + vars: + - name: "namespace" + value: "resource.attributes['service.namespace']" + - name: "clusterName" + value: "resource.attributes['k8s.cluster.name']" + output: + identifier: "'urn:opentelemetry:namespace/' + vars.namespace" + name: "vars.namespace" + typeName: "'service namespace (otel)'" + required: + tags: + - source: "'open-telemetry'" + target: "stackpack" + - source: "vars.clusterName" + target: "cluster-name" + optional: + tags: + - source: "resource.attributes['deployment.environment']" + target: "deployment-environment" + - source: "vars.clusterName + '/' + resource.attributes['k8s.namespace.name']" + target: "k8s-scope" + - source: "resource.attributes['k8s.namespace.name']" + target: "namespace" + expireAfter: 900000 + rank: + specificity: 100 diff --git a/stackpacks/open-telemetry/settings/component-mappings/service-instances.sty b/stackpacks/open-telemetry/settings/component-mappings/service-instances.sty new file mode 100644 index 0000000..07a4bc0 --- /dev/null +++ b/stackpacks/open-telemetry/settings/component-mappings/service-instances.sty @@ -0,0 +1,58 @@ +nodes: + - _type: "OtelComponentMapping" + name: "OTel Service Instance" + description: "Represents an instance of an OTel service" + identifier: "urn:stackpack:open-telemetry:otel-component-mapping:service-instance" + input: + signal: + - "TRACES" + - "METRICS" + resource: + action: "'CREATE'" + condition: "'service.instance.id' in resource.attributes" + vars: + - name: "namespace" + value: "resource.attributes['service.namespace']" + - name: "service" + value: "resource.attributes['service.name']" + - name: "version" + value: >- + 'service.version' in resource.attributes ? + resource.attributes['service.version'] : + 'undefined' + - name: "instanceId" + value: "resource.attributes['service.instance.id']" + - name: "instanceName" + value: "resource.attributes['service.name'] + ' - ' + resource.attributes['service.instance.id']" + - name: "clusterName" + value: "resource.attributes['k8s.cluster.name']" + output: + identifier: >- + 'urn:opentelemetry:namespace/' + vars.namespace + ':service/' + vars.service + ':serviceInstance/' + vars.instanceId + name: "vars.instanceName" + typeName: "'otel service instance'" + required: + version: "vars.version" + tags: + - source: "'open-telemetry'" + target: "stackpack" + - source: "vars.service" + target: "service.name" + - source: "vars.namespace" + target: "service.namespace" + - source: "vars.clusterName" + target: "cluster-name" + - source: "resource.attributes" + pattern: "(.*)" + target: "${1}" + optional: + tags: + - source: "resource.attributes['deployment.environment']" + target: "deployment-environment" + - source: "vars.clusterName + '/' + resource.attributes['k8s.namespace.name']" + target: "k8s-scope" + - source: "resource.attributes['k8s.namespace.name']" + target: "namespace" + expireAfter: 900000 + rank: + specificity: 210 diff --git a/stackpacks/open-telemetry/settings/component-mappings/services.sty b/stackpacks/open-telemetry/settings/component-mappings/services.sty new file mode 100644 index 0000000..f1a1485 --- /dev/null +++ b/stackpacks/open-telemetry/settings/component-mappings/services.sty @@ -0,0 +1,59 @@ +nodes: + - _type: "OtelComponentMapping" + name: "OTel Service Component" + description: "Represents an OTel service based on service.name and service.namespace" + identifier: "urn:stackpack:open-telemetry:otel-component-mapping:service" + input: + signal: + - "TRACES" + - "METRICS" + resource: + action: "'CREATE'" + condition: "'service.name' in resource.attributes" + vars: + - name: "namespace" + value: >- + resource.attributes['service.namespace'] != '' ? + resource.attributes['service.namespace'] : + 'default' + - name: "service" + value: "resource.attributes['service.name']" + - name: "version" + value: >- + 'service.version' in resource.attributes ? + resource.attributes['service.version'] : + 'undefined' + - name: "clusterName" + value: "resource.attributes['k8s.cluster.name']" + output: + identifier: >- + 'urn:opentelemetry:namespace/' + vars.namespace + ':service/' + vars.service + name: "vars.service" + typeName: "'otel service'" + required: + version: "vars.version" + tags: + - source: "'open-telemetry'" + target: "stackpack" + - source: "vars.service" + target: "service.name" + - source: "vars.namespace" + target: "service.namespace" + - source: "vars.version" + target: "service.version" + - source: "vars.clusterName" + target: "cluster-name" + - source: "resource.attributes" + pattern: "telemetry.sdk\\.(.*)" + target: "telemetry.sdk.${1}" + optional: + tags: + - source: "resource.attributes['deployment.environment']" + target: "deployment-environment" + - source: "vars.clusterName + '/' + resource.attributes['k8s.namespace.name']" + target: "k8s-scope" + - source: "resource.attributes['k8s.namespace.name']" + target: "namespace" + expireAfter: 900000 + rank: + specificity: 200 diff --git a/stackpacks/open-telemetry/settings/main-menu.sty b/stackpacks/open-telemetry/settings/main-menu.sty new file mode 100644 index 0000000..b5ad538 --- /dev/null +++ b/stackpacks/open-telemetry/settings/main-menu.sty @@ -0,0 +1,7 @@ +nodes: + - _type: MainMenuGroup + name: Open Telemetry + iconbase64: !icon open-telemetry.svg + defaultOpen: true + items: [] + identifier: "urn:stackpack:open-telemetry:main-menu-group:open-telemetry" \ No newline at end of file diff --git a/stackpacks/open-telemetry/settings/monitors/service-instance-span-duration/monitor.sty b/stackpacks/open-telemetry/settings/monitors/service-instance-span-duration/monitor.sty new file mode 100644 index 0000000..d25ed8b --- /dev/null +++ b/stackpacks/open-telemetry/settings/monitors/service-instance-span-duration/monitor.sty @@ -0,0 +1,25 @@ +nodes: + - _type: Monitor + arguments: + comparator: GT + failureState: DEVIATING + metric: + # _seconds_ = telemetry gateway (target/new spanmetrics default); _milliseconds_ = standalone collector (current spanmetrics default, transitioning to s per OTel semantic conventions) + query: |- + (histogram_quantile(0.95, sum by(service_namespace, service_name, service_instance_id, le)(rate(otel_span_duration_seconds_bucket{span_kind=~"SPAN_KIND_SERVER|SPAN_KIND_CONSUMER"}[5m])))) + or + (histogram_quantile(0.95, sum by(service_namespace, service_name, service_instance_id, le)(rate(otel_span_duration_milliseconds_bucket{span_kind=~"SPAN_KIND_SERVER|SPAN_KIND_CONSUMER"}[5m]))) / 1000) + aliasTemplate: Span duration 95th percentile + unit: s + threshold: 5 + titleTemplate: Span duration + urnTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}:serviceInstance/${service_instance_id}" + description: | + Monitors the duration of the server and consumer spans. When the 95th percentile of the duration is greater than the threshold (default 5s) the monitor has a Deviating state. + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:open-telemetry:monitor:otel-service-instance-span-duration + intervalSeconds: 30 + name: Span duration for Open Telemetry service instances + status: ENABLED + tags: ["service", "traces"] + remediationHint: !include remediation-hints/service-instance-span-duration.md.hbs \ No newline at end of file diff --git a/stackpacks/open-telemetry/settings/monitors/service-instance-span-error-ratio/monitor.sty b/stackpacks/open-telemetry/settings/monitors/service-instance-span-error-ratio/monitor.sty new file mode 100644 index 0000000..60e27e4 --- /dev/null +++ b/stackpacks/open-telemetry/settings/monitors/service-instance-span-error-ratio/monitor.sty @@ -0,0 +1,21 @@ +nodes: + - _type: Monitor + arguments: + comparator: GT + failureState: DEVIATING + metric: + query: 'sum by(service_namespace, service_name, service_instance_id)(rate(otel_span_calls_total{status_code="STATUS_CODE_ERROR", span_kind=~"SPAN_KIND_SERVER|SPAN_KIND_CONSUMER"}[5m]) or (present_over_time(otel_span_calls_total{span_kind=~"SPAN_KIND_SERVER|SPAN_KIND_CONSUMER"}[5m]) -1)) / sum by(service_namespace, service_name, service_instance_id)(rate(otel_span_calls_total{span_kind=~"SPAN_KIND_SERVER|SPAN_KIND_CONSUMER"}[5m])) * 100' + aliasTemplate: Error ratio (errors/total) + unit: percent + threshold: 5 + titleTemplate: Span error ratio + urnTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}:serviceInstance/${service_instance_id}" + description: | + Percentage of server and consumer spans that are in an error state for the service instance. + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:open-telemetry:monitor:otel-service-instance-span-error-ratio + intervalSeconds: 30 + name: Span error ratio for Open Telemetry service instances + status: ENABLED + tags: ["service", "traces"] + remediationHint: !include remediation-hints/service-instance-span-error-ratio.md.hbs \ No newline at end of file diff --git a/stackpacks/open-telemetry/settings/monitors/service-span-duration/monitor.sty b/stackpacks/open-telemetry/settings/monitors/service-span-duration/monitor.sty new file mode 100644 index 0000000..021e761 --- /dev/null +++ b/stackpacks/open-telemetry/settings/monitors/service-span-duration/monitor.sty @@ -0,0 +1,25 @@ +nodes: + - _type: Monitor + arguments: + comparator: GT + failureState: DEVIATING + metric: + # _seconds_ = telemetry gateway (target/new spanmetrics default); _milliseconds_ = standalone collector (current spanmetrics default, transitioning to s per OTel semantic conventions) + query: |- + (histogram_quantile(0.95, sum by(service_namespace, service_name, le)(rate(otel_span_duration_seconds_bucket{span_kind=~"SPAN_KIND_SERVER|SPAN_KIND_CONSUMER"}[5m])))) + or + (histogram_quantile(0.95, sum by(service_namespace, service_name, le)(rate(otel_span_duration_milliseconds_bucket{span_kind=~"SPAN_KIND_SERVER|SPAN_KIND_CONSUMER"}[5m]))) / 1000) + aliasTemplate: Span duration 95th percentile + unit: s + threshold: 5 + titleTemplate: Span duration + urnTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + description: | + Monitors the duration of the server and consumer spans. When the 95th percentile of the duration is greater than the threshold (default 5s) the monitor has a Deviating state. + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:open-telemetry:monitor:otel-service-span-duration + intervalSeconds: 30 + name: Span duration for Open Telemetry services + status: ENABLED + tags: ["service", "traces"] + remediationHint: !include remediation-hints/service-span-duration.md.hbs \ No newline at end of file diff --git a/stackpacks/open-telemetry/settings/monitors/service-span-error-ratio/monitor.sty b/stackpacks/open-telemetry/settings/monitors/service-span-error-ratio/monitor.sty new file mode 100644 index 0000000..e3a2ede --- /dev/null +++ b/stackpacks/open-telemetry/settings/monitors/service-span-error-ratio/monitor.sty @@ -0,0 +1,21 @@ +nodes: + - _type: Monitor + arguments: + comparator: GT + failureState: DEVIATING + metric: + query: 'sum by(service_namespace, service_name)(rate(otel_span_calls_total{status_code="STATUS_CODE_ERROR", span_kind=~"SPAN_KIND_SERVER|SPAN_KIND_CONSUMER"}[5m]) or (present_over_time(otel_span_calls_total{span_kind=~"SPAN_KIND_SERVER|SPAN_KIND_CONSUMER"}[5m]) -1)) / sum by(service_namespace, service_name)(rate(otel_span_calls_total{span_kind=~"SPAN_KIND_SERVER|SPAN_KIND_CONSUMER"}[5m])) * 100' + aliasTemplate: Error ratio (errors/total) + unit: percent + threshold: 5 + titleTemplate: Span error ratio + urnTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + description: | + Percentage of server and consumer spans that are in an error state for the service. + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:open-telemetry:monitor:otel-service-span-error-ratio + intervalSeconds: 30 + name: Span error ratio for Open Telemetry services + status: ENABLED + tags: ["service", "traces"] + remediationHint: !include remediation-hints/service-span-error-ratio.md.hbs diff --git a/stackpacks/open-telemetry/settings/presentations/common_filters.sty b/stackpacks/open-telemetry/settings/presentations/common_filters.sty new file mode 100644 index 0000000..9ceeb3f --- /dev/null +++ b/stackpacks/open-telemetry/settings/presentations/common_filters.sty @@ -0,0 +1,44 @@ +nodes: + - _type: ComponentPresentation + name: "Otel common columns" + description: "ComponentPresentation common for Otel components" + identifier: urn:stackpack:open-telemetry:component-presentation:common_filters + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("stackpack:open-telemetry") + rank: + specificity: 2 + presentation: + filters: + - filterId: "otel_namespace" + displayName: + singular: "OTEL namespace" + plural: "OTEL namespaces" + filter: + _type: "TagFilter" + tagKey: "service.namespace" + moreTab: Otel + - filterId: "otel_service" + displayName: + singular: "OTEL service" + plural: "OTEL services" + filter: + _type: "TagFilter" + tagKey: "service.name" + moreTab: Otel + - filterId: "k8s_namespace" + displayName: + singular: "K8s namespace" + plural: "K8s namespaces" + filter: + _type: "TagFilter" + tagKey: "namespace" + moreTab: K8s + - filterId: "k8s_cluster" + displayName: + singular: "K8s cluster" + plural: "K8s clusters" + filter: + _type: "TagFilter" + tagKey: "cluster-name" + moreTab: K8s \ No newline at end of file diff --git a/stackpacks/open-telemetry/settings/presentations/component.sty b/stackpacks/open-telemetry/settings/presentations/component.sty new file mode 100644 index 0000000..6d3814e --- /dev/null +++ b/stackpacks/open-telemetry/settings/presentations/component.sty @@ -0,0 +1,61 @@ +nodes: + - _type: ComponentPresentation + name: "Otel component" + description: "ComponentPresentation for Otel components" + identifier: urn:stackpack:open-telemetry:component-presentation:component + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("stackpack:open-telemetry") + rank: + specificity: 2 + presentation: + overview: + name: + singular: "otel component" + plural: "otel components" + title: "Otel component" + columns: + - columnId: "health" + title: "Health" + projection: + _type: "HealthProjection" + value: "healthState" + - columnId: "otel_name" + title: "Otel component" + projection: + _type: "ComponentLinkProjection" + name: "name" + identifier: "identifiers[0]" + - columnId: "otel_namespace" + title: "Namespace" + projection: + _type: "ComponentLinkProjection" + name: "tags.singleValue('service.namespace')" + identifier: "'urn:opentelemetry:namespace/' + tags.singleValue('service.namespace')" + - columnId: otel_sdk_language + title: "Language" + projection: + _type: "TextProjection" + value: "tags.singleValue('telemetry.sdk.language')" + highlight: + title: "Otel Component" + fields: + - fieldId: "type" + title: "Type" + order: 100.0 + projection: + _type: "TextProjection" + value: "typeName" + - fieldId: "health" + title: "Health" + order: 90.0 + projection: + _type: "HealthProjection" + value: "healthState" + - fieldId: "labels" + title: "Labels" + order: 80.0 + projection: + _type: "TagProjection" + events: + showEvents: true diff --git a/stackpacks/open-telemetry/settings/presentations/k8s-hosts.sty b/stackpacks/open-telemetry/settings/presentations/k8s-hosts.sty new file mode 100644 index 0000000..19549f3 --- /dev/null +++ b/stackpacks/open-telemetry/settings/presentations/k8s-hosts.sty @@ -0,0 +1,19 @@ +nodes: + - _type: ComponentPresentation + name: "Otel Host Extension" + description: "Extends the Host presentation for OTel hosts with related service instances" + identifier: urn:stackpack:open-telemetry:component-presentation:host + binding: + _type: ComponentPresentationQueryBinding + query: (label = "source-type:host") + rank: + specificity: 10 + presentation: + highlight: + relatedResources: + - resourceId: "otel_host_service_instances" + title: "Related service instances" + order: 1.0 + topologyQuery: >- + type = "otel service instance" AND label = "k8s.node.name:${name}" + presentationIdentifier: "urn:stackpack:open-telemetry:component-presentation:service-instance" diff --git a/stackpacks/open-telemetry/settings/presentations/k8s-pods.sty b/stackpacks/open-telemetry/settings/presentations/k8s-pods.sty new file mode 100644 index 0000000..233433e --- /dev/null +++ b/stackpacks/open-telemetry/settings/presentations/k8s-pods.sty @@ -0,0 +1,19 @@ +nodes: + - _type: ComponentPresentation + name: "Otel Pod Extension" + description: "Extends the Pod presentation for OTel pods with related service instances" + identifier: urn:stackpack:open-telemetry:component-presentation:pod + binding: + _type: ComponentPresentationQueryBinding + query: (label = "stackpack:open-telemetry" and type = "pod") + rank: + specificity: 10 + presentation: + highlight: + relatedResources: + - resourceId: "otel_pod_service_instances" + title: "Related service instances" + order: 1.0 + topologyQuery: >- + type = "otel service instance" AND label = "k8s.pod.name:${name}" + presentationIdentifier: "urn:stackpack:open-telemetry:component-presentation:service-instance" diff --git a/stackpacks/open-telemetry/settings/presentations/service-instances.sty b/stackpacks/open-telemetry/settings/presentations/service-instances.sty new file mode 100644 index 0000000..197b168 --- /dev/null +++ b/stackpacks/open-telemetry/settings/presentations/service-instances.sty @@ -0,0 +1,451 @@ +nodes: + - _type: ComponentPresentation + name: "Otel Service Instances presentation" + description: "ComponentPresentation for Otel Service Instances" + identifier: urn:stackpack:open-telemetry:component-presentation:service-instance + binding: + _type: ComponentPresentationQueryBinding + query: (label = "stackpack:open-telemetry" and type = "otel service instance") + rank: + specificity: 10 + presentation: + icon: !icon service-instances.svg + filters: + - filterId: "otel_namespace" + - filterId: "otel_service" + - filterId: "k8s_cluster" + - filterId: "k8s_namespace" + topology: + groupingEnabled: true + showIndirectRelations: false + minimumGroupSize: 8 + groupedByLayers: false + groupedByDomains: true + autoGrouping: true + connectedComponents: true + overview: + name: + singular: "service instance" + plural: "service instances" + title: "Service instances" + mainMenu: + group: "Open Telemetry" + order: 99 + columns: + - columnId: health + - columnId: otel_name + title: Service Instance + - columnId: otel_namespace + title: OTel Namespace + - columnId: otel_service + title: "Service" + projection: + _type: "ComponentLinkProjection" + name: "tags.singleValue('service.name')" + identifier: "'urn:opentelemetry:namespace/' + tags.singleValue('service.namespace') + ':service/' + tags.singleValue('service.name')" + - columnId: otel_k8s_cluster + title: "Kubernetes Cluster" + projection: + _type: ComponentLinkProjection + name: "tags.singleValue('cluster-name')" + identifier: "'urn:cluster:/kubernetes:' + tags.singleValue('cluster-name')" + - columnId: otel_k8s_namespace + title: "Kubernetes Namespace" + projection: + _type: ComponentLinkProjection + name: "tags.singleValue('namespace')" + identifier: "'urn:kubernetes:/' + tags.singleValue('cluster-name') + ':namespace/' + tags.singleValue('namespace')" + - columnId: otel_service_instance_span_error_percent_summary + title: Span error % + projection: + _type: MetricProjection + sparkline: true + decimalPlaces: 2 + unit: percentunit + query: |- + (sum by(service_namespace, service_name, service_instance_id)(rate(otel_span_calls_total{ + status_code="STATUS_CODE_ERROR", service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}" + }[5m])) or 0) / + sum by(service_namespace, service_name, service_instance_id)(rate(otel_span_calls_total{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}" + }[5m])) + metricId: service-instance-spans-error-rate + - columnId: otel_service_instance_span_duration_summary + title: Span duration + projection: + _type: MetricProjection + sparkline: true + decimalPlaces: 2 + unit: s + # _seconds_ = telemetry gateway (target/new spanmetrics default); _milliseconds_ = standalone collector (current spanmetrics default, transitioning to s per OTel semantic conventions) + query: |- + (histogram_quantile(0.95, sum(rate(otel_span_duration_seconds_bucket{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}" + }[${__rate_interval}])) by (service_namespace, service_name, service_instance_id, le))) + or + (histogram_quantile(0.95, sum(rate(otel_span_duration_milliseconds_bucket{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}" + }[${__rate_interval}])) by (service_namespace, service_name, service_instance_id, le)) / 1000) + metricId: service-instance-spans-duration + - columnId: otel_service_instance_span_rate_summary + title: Span rate + projection: + _type: MetricProjection + sparkline: true + decimalPlaces: 2 + unit: cps + query: |- + sum by(service_namespace, service_name, service_instance_id)(rate(otel_span_calls_total{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}" + }[${__rate_interval}])) + metricId: service-instance-spans-rate + - columnId: otel_sdk_language + sort: + - columnId: health + - columnId: otel_name + highlight: + title: "OTel service" + fields: + - fieldId: "namespace" + title: "OTel Namespace" + order: 85.0 + projection: + _type: ComponentLinkProjection + name: "tags.singleValue('service.namespace')" + identifier: "'urn:opentelemetry:namespace/' + tags.singleValue('service.namespace')" + - fieldId: "service" + title: "Service" + order: 82.5 + projection: + _type: ComponentLinkProjection + name: "tags.singleValue('service.name')" + identifier: "'urn:opentelemetry:namespace/' + tags.singleValue('service.namespace') + ':service/' + tags.singleValue('service.name')" + - fieldId: "language" + title: "Language" + order: 81.0 + projection: + _type: TextProjection + value: "tags.singleValue('telemetry.sdk.language')" + - fieldId: "k8s_cluster" + title: "K8s Cluster" + order: 77.5 + projection: + _type: ComponentLinkProjection + name: "tags.singleValue('cluster-name')" + identifier: "'urn:cluster:/kubernetes:' + tags.singleValue('cluster-name')" + - fieldId: "k8s_namespace" + title: "K8s Namespace" + order: 75.0 + projection: + _type: ComponentLinkProjection + name: "tags.singleValue('namespace')" + identifier: "'urn:kubernetes:/' + tags.singleValue('cluster-name') + ':namespace/' + tags.singleValue('namespace')" + provisioning: + topologySourceSelector: "source.mapping.identifier == 'urn:stackpack:open-telemetry:otel-component-mapping:service-instance'" + showConfiguration: false + showStatus: false + relatedResources: + - resourceId: "otel_service_instance_service_instances" + title: "Related service instances" + order: 2.0 + topologyQuery: '(withNeighborsOf(direction = "both", components = (identifier = "${identifiers[0]}"), levels = "1")) and type = "otel service instance"' + presentationIdentifier: "urn:stackpack:open-telemetry:component-presentation:service-instance" + - resourceId: "otel_service_instance_services" + title: "Related services" + order: 1.0 + topologyQuery: '(withNeighborsOf(direction = "up", components = (withNeighborsOf(direction = "both", components = (identifier = "${identifiers[0]}"), levels = "1") and type = "otel service instance") , levels = "1") and type = "otel service")' + presentationIdentifier: "urn:stackpack:open-telemetry:component-presentation:service" + - resourceId: "otel_service_instance_pods" + title: "Related pods" + order: 3.0 + topologyQuery: >- + label = "source-name:${tags.singleValue('k8s.pod.name')}" + presentationIdentifier: "urn:stackpack:open-telemetry:view-type:pod-link" + - resourceId: "otel_service_instance_hosts" + title: "Related hosts" + order: 4.0 + topologyQuery: >- + label = "source-type:host" AND label = "source-name:${tags.singleValue('k8s.node.name')}" + presentationIdentifier: "urn:stackpack:open-telemetry:view-type:host-link" + events: + showEvents: true + relatedResourcesQuery: | + ((withNeighborsOf(direction = "both", components = (identifier = "${identifiers[0]}"), levels = "1")) and type = "otel service instance") + OR + (withNeighborsOf(direction = "up", components = (withNeighborsOf(direction = "both", components = (identifier = "${identifiers[0]}"), levels = "1") and type = "otel service instance") , levels = "1") and type = "otel service") + OR + (identifier = "${identifiers[0]}") + metrics: + - sectionId: traces + title: Traces + description: "Rate, Errors and Duration metrics for the trace spans" + order: 100 + metrics: + - metricId: service-instance-spans-rate + order: 100 + - metricId: service-instance-spans-error-rate + order: 90 + - metricId: service-instance-spans-duration + order: 80 + + summary: + metrics: + - summaryMetricId: service-instance-spans-rate + name: Span rate + description: The number of spans for the service instance per second + order: 100 + unit: cps + query: sum(rate(otel_span_calls_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"}[${__rate_interval}])) + metricId: service-instance-spans-rate + - + summaryMetricId: service-instance-spans-error-rate + name: Span error % + description: The percentage of errors per second + order: 90 + unit: percentunit + query: | + (sum(rate(otel_span_calls_total{status_code="STATUS_CODE_ERROR", service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"}[5m])) or 0) / + sum(rate(otel_span_calls_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"}[5m])) + metricId: service-instance-spans-error-rate + - + summaryMetricId: service-instance-spans-duration + name: Span duration + description: + order: 80 + unit: s + # _seconds_ = telemetry gateway (target/new spanmetrics default); _milliseconds_ = standalone collector (current spanmetrics default, transitioning to s per OTel semantic conventions) + query: |- + (histogram_quantile(0.95, sum(rate(otel_span_duration_seconds_bucket{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}" + }[${__rate_interval}])) by (le))) + or + (histogram_quantile(0.95, sum(rate(otel_span_duration_milliseconds_bucket{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}" + }[${__rate_interval}])) by (le)) / 1000) + metricId: service-instance-spans-duration + + tracesPerspective: + showTraces: true + spanFilter: + attributes: + service.namespace: ["${tags.singleValue('service.namespace')}"] + service.name: ["${tags.singleValue('service.name')}"] + service.instance.id: ["${tags.singleValue('service.instance.id')}"] + metricPerspective: + tabs: + - + tabId: dotnet + title: .NET + order: 80 + sections: + - + sectionId: gc + title: GC + order: 30 + metrics: + - metricId: dotnet-gc-collections + - metricId: dotnet-gc-duration + - metricId: dotnet-gc-allocated + - metricId: dotnet-gc-objects-and-committed-memory + - metricId: dotnet-gc-heap-size + - metricId: dotnet-gc-heap-fragmentation + - + tabId: jvm + title: JVM + order: 80 + sections: + - + sectionId: gc + title: GC + order: 20 + metrics: + - metricId: jvm_gc_durations_95th_quantile + - metricId: jvm_gc_per_second + - metricId: jvm_mem_after_last_gc + - + sectionId: mem + title: Memory + order: 30 + metrics: + - metricId: jvm_heap_memory + - metricId: jvm_non_heap_memory + - + tabId: traces + title: Traces Performance + order: 10 + sections: + - + sectionId: spans + title: Spans + order: 10 + metrics: + - metricId: service-instance-spans-rate + - metricId: service-instance-spans-error-rate + - metricId: service-instance-spans-duration + metrics: + - + metricId: dotnet-gc-collections + description: 'GC Collections per Second' + name: .NET GC Collections + metricQueries: + - expression: |- + rate(process_runtime_dotnet_gc_collections_count_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"}[${__rate_interval}]) + alias: Generation - ${generation} + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: dotnet-gc-duration + description: 'GC Duration per Collection' + name: .NET GC Duration + metricQueries: + - expression: |- + rate(process_runtime_dotnet_gc_duration_nanoseconds_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"}[${__rate_interval}]) + alias: duration(ns) + chart: + _type: TimeSeriesChart + unit: ns + - + metricId: dotnet-gc-allocated + description: 'Bytes allocated to GC Heap' + name: .NET GC Allocated + metricQueries: + - expression: |- + rate(process_runtime_dotnet_gc_allocations_size_bytes_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"}[${__rate_interval}]) + alias: allocated + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: dotnet-gc-objects-and-committed-memory + description: 'Size of Uncollected Objects and Committed Vmem size for GC Heap' + name: .NET GC Objects and Committed Memory + metricQueries: + - expression: |- + process_runtime_dotnet_gc_objects_size_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"} + alias: uncollected objects + - expression: |- + process_runtime_dotnet_gc_committed_memory_size_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"} + alias: committed vmem heap size + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: dotnet-gc-heap-size + description: 'GC Heap size (including fragmentation)' + name: .NET GC Heap Size Bytes + metricQueries: + - expression: |- + process_runtime_dotnet_gc_heap_size_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"} + alias: Generation - ${generation} + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: dotnet-gc-heap-fragmentation + description: 'Heap Fragmentation size in Bytes' + name: .NET GC Heap Fragmentation Bytes + metricQueries: + - expression: |- + process_runtime_dotnet_gc_heap_fragmentation_size_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"} + alias: Generation - ${generation} + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: jvm_gc_durations_95th_quantile + description: 'GC operations that are slower than 95% of total gc operations.' + name: JVM GC Durations 95th Quantile + metricQueries: + - expression: |- + histogram_quantile(0.95, (rate(jvm_gc_duration_seconds_bucket{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"}[${__rate_interval}]))) + alias: GC Name - ${jvm_gc_name} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: jvm_gc_per_second + description: 'Garbage collections per second.' + name: JVM GC Per Second + metricQueries: + - expression: |- + rate(jvm_gc_duration_seconds_count{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"}[${__rate_interval}]) + alias: GC Name - ${jvm_gc_name} + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: jvm_mem_after_last_gc + description: 'JVM memory size after last garbage collection.' + name: JVM Mem After Last GC + metricQueries: + - expression: |- + jvm_memory_used_after_last_gc_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"} + alias: Memory Pool - ${jvm_memory_pool_name} + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: jvm_heap_memory + description: 'JVM heap memory.' + name: JVM Heap Memory + metricQueries: + - expression: |- + jvm_memory_used_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}",jvm_memory_type="heap"} + alias: Memory Pool - ${jvm_memory_pool_name} + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: jvm_non_heap_memory + description: 'JVM Non-Heap memory.' + name: JVM Non-Heap Memory + metricQueries: + - expression: |- + jvm_memory_used_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}",jvm_memory_type="non_heap"} + alias: Memory Pool - ${jvm_memory_pool_name} + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: service-instance-spans-rate + description: The number of spans for the service instance per second + name: Span rate + metricQueries: + - expression: sum(rate(otel_span_calls_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"}[${__rate_interval}])) by (span_kind) + alias: 'All ${span_kind}' + - expression: sum(rate(otel_span_calls_total{status_code="STATUS_CODE_ERROR", service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"}[${__rate_interval}])) by (span_kind) + alias: 'Errors ${span_kind}' + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: service-instance-spans-error-rate + description: The percentage of errors per second + name: Span error rate + metricQueries: + - expression: |- + ((sum(rate(otel_span_calls_total{status_code="STATUS_CODE_ERROR", service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"}[5m])) by (span_kind)) or 0) + / sum(rate(otel_span_calls_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"}[5m])) by (span_kind) + alias: '${span_kind}' + chart: + _type: TimeSeriesChart + unit: percentunit + - + metricId: service-instance-spans-duration + description: + name: Span duration + metricQueries: + # _seconds_ = telemetry gateway (target/new spanmetrics default); _milliseconds_ = standalone collector (current spanmetrics default, transitioning to s per OTel semantic conventions) + - expression: |- + (histogram_quantile(0.95, sum(rate(otel_span_duration_seconds_bucket{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}" + }[${__rate_interval}])) by (k8s_cluster_name, k8s_namespace_name, k8s_pod_name, span_kind, le))) + or + (histogram_quantile(0.95, sum(rate(otel_span_duration_milliseconds_bucket{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}" + }[${__rate_interval}])) by (k8s_cluster_name, k8s_namespace_name, k8s_pod_name, span_kind, le)) / 1000) + alias: '95th percentile ${span_kind}' + chart: + _type: TimeSeriesChart + unit: s \ No newline at end of file diff --git a/stackpacks/open-telemetry/settings/presentations/service-namespaces.sty b/stackpacks/open-telemetry/settings/presentations/service-namespaces.sty new file mode 100644 index 0000000..2bc80fe --- /dev/null +++ b/stackpacks/open-telemetry/settings/presentations/service-namespaces.sty @@ -0,0 +1,62 @@ +nodes: + - _type: ComponentPresentation + name: "Otel Service Namespaces presentation" + description: "ComponentPresentation for Otel Service Namespaces" + identifier: urn:stackpack:open-telemetry:component-presentation:service-namespace + binding: + _type: ComponentPresentationQueryBinding + query: (label IN ("stackpack:open-telemetry") AND type IN ("service namespace (otel)")) + rank: + specificity: 10 + presentation: + icon: !icon service-namespaces.svg + filters: + - filterId: "k8s_cluster" + - filterId: "k8s_namespace" + topology: + groupingEnabled: true + showIndirectRelations: false + minimumGroupSize: 8 + groupedByLayers: false + groupedByDomains: true + autoGrouping: true + connectedComponents: true + overview: + name: + singular: "service namespace" + plural: "service namespaces" + title: "Service Namespaces" + mainMenu: + group: "Open Telemetry" + order: 97 + columns: + - columnId: health + - columnId: "otel_name" + title: "Namespace" + provisioning: + topologySourceSelector: "source.mapping.identifier == 'urn:stackpack:open-telemetry:otel-component-mapping:namespace'" + showConfiguration: false + showStatus: false + sort: + - columnId: health + - columnId: otel_name + highlight: + title: "Otel namespace" + fields: [] + relatedResources: + - resourceId: "otel_namespace_service_instances" + title: "Service instances" + order: 1.0 + topologyQuery: 'type = "otel service instance" AND label in ("service.namespace:${name}")' + presentationIdentifier: "urn:stackpack:open-telemetry:component-presentation:service-instance" + - resourceId: "otel_namespace_services" + title: "Services" + order: 2.0 + topologyQuery: 'type = "otel service" AND label in ("service.namespace:${name}")' + presentationIdentifier: "urn:stackpack:open-telemetry:component-presentation:service" + events: + showEvents: true + relatedResourcesQuery: | + type IN ("otel service", "otel service instance") AND label IN ("service.namespace:${name}") + OR + identifier = "${identifiers[0]}" diff --git a/stackpacks/open-telemetry/settings/presentations/services.sty b/stackpacks/open-telemetry/settings/presentations/services.sty new file mode 100644 index 0000000..72cdcd6 --- /dev/null +++ b/stackpacks/open-telemetry/settings/presentations/services.sty @@ -0,0 +1,608 @@ +nodes: + - _type: ComponentPresentation + name: "Otel Services presentation" + description: "ComponentPresentation for Otel Services" + identifier: urn:stackpack:open-telemetry:component-presentation:service + binding: + _type: ComponentPresentationQueryBinding + query: (label = "stackpack:open-telemetry" and type = "otel service") + rank: + specificity: 10 + presentation: + icon: !icon services.svg + filters: + - filterId: "otel_namespace" + - filterId: "k8s_cluster" + - filterId: "k8s_namespace" + topology: + groupingEnabled: true + showIndirectRelations: false + minimumGroupSize: 8 + groupedByLayers: false + groupedByDomains: true + autoGrouping: true + connectedComponents: true + overview: + name: + singular: "service" + plural: "services" + title: Services + mainMenu: + group: "Open Telemetry" + order: 100 + columns: + - columnId: health + - columnId: otel_name + title: Service + - columnId: otel_namespace + title: "OTel Namespace" + - columnId: otel_k8s_cluster + title: "Kubernetes Cluster" + projection: + _type: ComponentLinkProjection + name: "tags.singleValue('cluster-name')" + identifier: "'urn:cluster:/kubernetes:' + tags.singleValue('cluster-name')" + - columnId: otel_k8s_namespace + title: "Kubernetes Namespace" + projection: + _type: ComponentLinkProjection + name: "tags.singleValue('namespace')" + identifier: "'urn:kubernetes:/' + tags.singleValue('cluster-name') + ':namespace/' + tags.singleValue('namespace')" + - columnId: otel_service_span_error_percent_summary + title: Span error % + projection: + _type: MetricProjection + sparkline: true + decimalPlaces: 2 + unit: percentunit + query: |- + (sum by (service_namespace, service_name) (rate(otel_span_calls_total{ + status_code="STATUS_CODE_ERROR", service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}" + }[5m])) or 0) / + sum by (service_namespace, service_name) (rate(otel_span_calls_total{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}" + }[5m])) + metricId: service-spans-error-rate + - columnId: otel_service_span_duration_summary + title: Span duration + projection: + _type: MetricProjection + sparkline: true + decimalPlaces: 2 + unit: s + # _seconds_ = telemetry gateway (target/new spanmetrics default); _milliseconds_ = standalone collector (current spanmetrics default, transitioning to s per OTel semantic conventions) + query: |- + (histogram_quantile(0.95, sum(rate(otel_span_duration_seconds_bucket{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}" + }[${__rate_interval}])) by (service_namespace, service_name, le))) + or + (histogram_quantile(0.95, sum(rate(otel_span_duration_milliseconds_bucket{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}" + }[${__rate_interval}])) by (service_namespace, service_name, le)) / 1000) + metricId: service-spans-duration + - columnId: otel_service_span_rate_summary + title: "Span rate" + projection: + _type: "MetricProjection" + sparkline: true + decimalPlaces: 2 + unit: "cps" + query: |- + sum by (service_namespace, service_name)(rate(otel_span_calls_total{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}" + }[${__rate_interval}])) + metricId: service-spans-rate + - columnId: otel_sdk_language + sort: + - columnId: health + - columnId: otel_name + highlight: + title: "OTel service" + fields: + - fieldId: "namespace" + title: "OTel Namespace" + order: 85.0 + projection: + _type: ComponentLinkProjection + name: tags.singleValue('service.namespace') + identifier: "'urn:opentelemetry:namespace/' + tags.singleValue('service.namespace')" + - fieldId: "language" + title: "Language" + order: 82.5 + projection: + _type: TextProjection + value: "tags.singleValue('telemetry.sdk.language')" + - fieldId: "k8s_cluster" + title: "K8s Cluster" + order: 77.5 + projection: + _type: ComponentLinkProjection + name: "tags.singleValue('cluster-name')" + identifier: "'urn:cluster:/kubernetes:' + tags.singleValue('cluster-name')" + - fieldId: "k8s_namespace" + title: "K8s Namespace" + order: 75.0 + projection: + _type: ComponentLinkProjection + name: "tags.singleValue('namespace')" + identifier: "'urn:kubernetes:/' + tags.singleValue('cluster-name') + ':namespace/' + tags.singleValue('namespace')" + provisioning: + topologySourceSelector: "source.mapping.identifier == 'urn:stackpack:open-telemetry:otel-component-mapping:service'" + showConfiguration: false + showStatus: false + relatedResources: + - resourceId: "otel_service_service_instances" + title: "Instances of this service" + order: 2.0 + topologyQuery: '(withNeighborsOf(direction = "down", components = (identifier = "${identifiers[0]}"), levels = "1")) and type = "otel service instance"' + presentationIdentifier: "urn:stackpack:open-telemetry:component-presentation:service-instance" + - resourceId: "otel_service_services" + title: "Related services" + order: 1.0 + topologyQuery: '(withNeighborsOf(direction = "up", components = (withNeighborsOf(direction = "both", components = (withNeighborsOf(direction = "down", components = (identifier = "${identifiers[0]}"), levels = "1") and type = "otel service instance"), levels = "1") and type = "otel service instance") , levels = "1") and type = "otel service")' + presentationIdentifier: "urn:stackpack:open-telemetry:component-presentation:service" + - resourceId: "otel_service_pods" + title: "Related pods" + order: 3.0 + topologyQuery: '(withNeighborsOf(direction = "both", components = (withNeighborsOf(direction = "down", components = (identifier = "${identifiers[0]}"), levels = "1") and type = "otel service instance"), levels = "1")) and type = "pod"' + presentationIdentifier: "urn:stackpack:open-telemetry:view-type:pod-link" + events: + showEvents: true + relatedResourcesQuery: | + (withNeighborsOf(direction = "up", components = (withNeighborsOf(direction = "both", components = (identifier = "${identifiers[0]}"), levels = "1") and type = "otel service instance"), levels = "1") and type = "otel service") + OR + ((withNeighborsOf(direction = "down", components = (identifier = "${identifiers[0]}"), levels = "1")) and type = "otel service instance") + OR + identifier = "${identifiers[0]}" + metrics: + - sectionId: traces + title: Traces + description: "Rate, Errors and Duration metrics for the trace spans" + order: 100 + metrics: + - metricId: service-spans-rate + order: 100 + - metricId: service-spans-error-rate + order: 90 + - metricId: service-spans-duration + order: 80 + - sectionId: service + title: Service + description: "Open Telemetry service metrics" + order: 90 + metrics: + - metricId: service-instance-count + order: 100 + summary: + metrics: + - summaryMetricId: otel_service_span_error_percent_summary + name: Span error % + order: 100 + decimalPlaces: 2 + unit: percentunit + query: |- + (sum by (service_namespace, service_name) (rate(otel_span_calls_total{ + status_code="STATUS_CODE_ERROR", service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}" + }[5m])) or 0) / + sum by (service_namespace, service_name) (rate(otel_span_calls_total{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}" + }[5m])) + metricId: service-spans-error-rate + - summaryMetricId: otel_service_span_duration_summary + name: Span duration + order: 90 + decimalPlaces: 2 + unit: s + # _seconds_ = telemetry gateway (target/new spanmetrics default); _milliseconds_ = standalone collector (current spanmetrics default, transitioning to s per OTel semantic conventions) + query: |- + (histogram_quantile(0.95, sum(rate(otel_span_duration_seconds_bucket{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}" + }[${__rate_interval}])) by (service_namespace, service_name, le))) + or + (histogram_quantile(0.95, sum(rate(otel_span_duration_milliseconds_bucket{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}" + }[${__rate_interval}])) by (service_namespace, service_name, le)) / 1000) + metricId: service-spans-duration + - summaryMetricId: otel_service_span_rate_summary + name: Span rate + order: 80 + decimalPlaces: 2 + unit: cps + query: |- + sum by (service_namespace, service_name)(rate(otel_span_calls_total{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}" + }[${__rate_interval}])) + metricId: service-spans-rate + tracesPerspective: + showTraces: true + spanFilter: + attributes: + service.namespace: ["${tags.singleValue('service.namespace')}"] + service.name: ["${name}"] + metricPerspective: + tabs: + - + tabId: dotnet + title: .NET + order: 80 + sections: + - + sectionId: gc + title: GC + order: 3 + metrics: + - metricId: dotnet-gc-collections + - metricId: dotnet-gc-duration + - metricId: dotnet-gc-allocated + - metricId: dotnet-gc-objects-and-committed-memory + - metricId: dotnet-gc-heap-size + - metricId: dotnet-gc-heap-fragmentation + - + tabId: jvm + title: JVM + order: 80 + sections: + - + sectionId: gc + title: GC + order: 3 + metrics: + - metricId: jvm_gc_durations_95th_quantile + - metricId: jvm_gc_per_second + - metricId: jvm_mem_after_last_gc + - + sectionId: mem + title: Memory + order: 3 + metrics: + - metricId: jvm_heap_memory + - metricId: jvm_non_heap_memory + - + tabId: service + title: Service + order: 20 + sections: + - + sectionId: instances + title: Instances + order: 10 + metrics: + - metricId: service-instance-count + - + tabId: traces-performance + title: Traces Performance + order: 10 + sections: + - + sectionId: spans + title: Spans + order: 10 + metrics: + - metricId: service-spans-rate + - metricId: service-spans-error-rate + - metricId: service-spans-duration + - + sectionId: spans-per-instance + title: Spans per instance + order: 20 + metrics: + - metricId: service-spans-rate-per-instance + - metricId: service-spans-error-rate-per-instance + - metricId: service-spans-duration-per-instance + - + sectionId: server-spans + title: Server Spans + order: 30 + metrics: + - metricId: service-server-spans-failed-percentage + - metricId: service-server-spans-rate + - metricId: service-server-spans-duration + - + sectionId: client-spans + title: Client Spans + order: 40 + metrics: + - metricId: service-client-spans-failed-percentage + - metricId: service-client-spans-rate + - metricId: service-client-spans-duration + metrics: + - + metricId: dotnet-gc-collections + description: 'GC Collections per Second' + name: .NET GC Collections + metricQueries: + - expression: |- + rate(process_runtime_dotnet_gc_collections_count_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}"}[${__rate_interval}]) + alias: Generation - ${generation} + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: dotnet-gc-duration + description: 'GC Duration per Collection' + name: .NET GC Duration + metricQueries: + - expression: |- + rate(process_runtime_dotnet_gc_duration_nanoseconds_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}"}[${__rate_interval}]) + alias: duration(ns) + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + chart: + _type: TimeSeriesChart + unit: ns + - + metricId: dotnet-gc-allocated + description: 'Bytes allocated to GC Heap' + name: .NET GC Allocated + metricQueries: + - expression: |- + rate(process_runtime_dotnet_gc_allocations_size_bytes_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}"}[${__rate_interval}]) + alias: allocated + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: dotnet-gc-objects-and-committed-memory + description: 'Size of Uncollected Objects and Committed Vmem size for GC Heap' + name: .NET GC Objects and Committed Memory + metricQueries: + - expression: |- + process_runtime_dotnet_gc_objects_size_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}"} + alias: uncollected objects + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + - expression: |- + process_runtime_dotnet_gc_committed_memory_size_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}"} + alias: committed vmvem heap size + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: dotnet-gc-heap-size + description: 'GC Heap size (including fragmentation)' + name: .NET GC Heap Size Bytes + metricQueries: + - expression: |- + process_runtime_dotnet_gc_heap_size_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}"} + alias: Generation - ${generation} + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: dotnet-gc-heap-fragmentation + description: 'Heap Fragmentation size in Bytes' + name: .NET GC Heap Fragmentation Bytes + metricQueries: + - expression: |- + process_runtime_dotnet_gc_heap_fragmentation_size_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}"} + alias: Generation - ${generation} + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: jvm_gc_durations_95th_quantile + description: 'GC operations that are slower than 95% of total gc operations.' + name: JVM GC Durations 95th Quantile + metricQueries: + - expression: |- + histogram_quantile(0.95, (rate(jvm_gc_duration_seconds_bucket{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}"}[${__rate_interval}]))) + alias: GC Name - ${jvm_gc_name} + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + chart: + _type: TimeSeriesChart + unit: s + - + metricId: jvm_gc_per_second + description: 'Garbage collections per second.' + name: JVM GC Per Second + metricQueries: + - expression: |- + rate(jvm_gc_duration_seconds_count{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}"}[${__rate_interval}]) + alias: GC Name - ${jvm_gc_name} + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: jvm_mem_after_last_gc + description: 'JVM memory size after last garbage collection.' + name: JVM Mem After Last GC + metricQueries: + - expression: |- + jvm_memory_used_after_last_gc_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}"} + alias: Memory Pool - ${jvm_memory_pool_name} + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: jvm_heap_memory + description: 'JVM heap memory.' + name: JVM Heap Memory + metricQueries: + - expression: |- + jvm_memory_used_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}",jvm_memory_type="heap"} + alias: Memory Pool - ${jvm_memory_pool_name} + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: jvm_non_heap_memory + description: 'JVM Non-Heap memory.' + name: JVM Non-Heap Memory + metricQueries: + - expression: |- + jvm_memory_used_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}",jvm_memory_type="non_heap"} + alias: Memory Pool - ${jvm_memory_pool_name} + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: service-spans-rate + description: The number of spans for the otel service per second + name: Span rate + metricQueries: + - expression: sum(rate(otel_span_calls_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}"}[${__rate_interval}])) by (span_kind) + alias: 'All ${span_kind}' + - expression: sum(rate(otel_span_calls_total{status_code="STATUS_CODE_ERROR", service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}"}[${__rate_interval}])) by (span_kind) + alias: 'Errors ${span_kind}' + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: service-spans-error-rate + description: The percentage of errors per second + name: Span error rate + metricQueries: + - expression: |- + ((sum(rate(otel_span_calls_total{status_code="STATUS_CODE_ERROR", service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}"}[5m])) by (span_kind)) or 0) + / sum(rate(otel_span_calls_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}"}[5m])) by (span_kind) + alias: '${span_kind}' + chart: + _type: TimeSeriesChart + unit: percentunit + - + metricId: service-spans-duration + description: + name: Span duration + metricQueries: + # _seconds_ = telemetry gateway (target/new spanmetrics default); _milliseconds_ = standalone collector (current spanmetrics default, transitioning to s per OTel semantic conventions) + - expression: |- + (histogram_quantile(0.95, sum(rate(otel_span_duration_seconds_bucket{service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}"}[${__rate_interval}])) by (span_kind, le))) + or + (histogram_quantile(0.95, sum(rate(otel_span_duration_milliseconds_bucket{service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}"}[${__rate_interval}])) by (span_kind, le)) / 1000) + alias: '95th percentile ${span_kind}' + - expression: |- + (histogram_quantile(0.99, sum(rate(otel_span_duration_seconds_bucket{service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}"}[${__rate_interval}])) by (span_kind, le))) + or + (histogram_quantile(0.99, sum(rate(otel_span_duration_milliseconds_bucket{service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}"}[${__rate_interval}])) by (span_kind, le)) / 1000) + alias: '99th percentile ${span_kind}' + chart: + _type: TimeSeriesChart + unit: s + - + metricId: service-spans-rate-per-instance + description: The number of spans for the otel service per second + name: Span rate + metricQueries: + - expression: sum(rate(otel_span_calls_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}"}[${__rate_interval}])) by (service_instance_id, service_name, service_namespace) + alias: '${service_instance_id}' + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}:serviceInstance/${service_instance_id}" + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: service-spans-error-rate-per-instance + description: The percentage of errors per second + name: Span error rate + metricQueries: + - expression: |- + ((sum(rate(otel_span_calls_total{status_code="STATUS_CODE_ERROR", service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}"}[5m])) by (service_instance_id, service_name, service_namespace)) or 0) + / sum(rate(otel_span_calls_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}"}[5m])) by (service_instance_id, service_name, service_namespace) + alias: '${service_instance_id}' + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}:serviceInstance/${service_instance_id}" + chart: + _type: TimeSeriesChart + unit: percentunit + - + metricId: service-spans-duration-per-instance + description: + name: Span duration + metricQueries: + # _seconds_ = telemetry gateway (target/new spanmetrics default); _milliseconds_ = standalone collector (current spanmetrics default, transitioning to s per OTel semantic conventions) + - expression: |- + (histogram_quantile(0.95, sum(rate(otel_span_duration_seconds_bucket{service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}"}[${__rate_interval}])) by (service_instance_id, service_name, service_namespace, le))) + or + (histogram_quantile(0.95, sum(rate(otel_span_duration_milliseconds_bucket{service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}"}[${__rate_interval}])) by (service_instance_id, service_name, service_namespace, le)) / 1000) + alias: '95th percentile ${service_instance_id}' + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}:serviceInstance/${service_instance_id}" + chart: + _type: TimeSeriesChart + unit: s + - + metricId: service-instance-count + description: + name: Service instance count + metricQueries: + - expression: count(count(otel_span_calls_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}"}[${__rate_interval}]) by (service_instance_id)) + alias: Service instance count + chart: + _type: TimeSeriesChart + unit: short + - + metricId: service-server-spans-failed-percentage + name: Span error rate + metricQueries: + - expression: | + ( + ((sum by (client_service_namespace, client) (rate(traces_service_graph_request_failed_total{server_service_namespace="${tags.singleValue('service.namespace')}", server="${name}"}[${__rate_interval}]))) or 0) + / + sum by (client_service_namespace, client) (rate(traces_service_graph_request_total{server_service_namespace="${tags.singleValue('service.namespace')}", server="${name}"}[${__rate_interval}])) + ) * 100 + componentIdentifierTemplate: "urn:opentelemetry:namespace/${client_service_namespace}:service/${client}" + alias: "Failed call rate ${client}" + chart: + _type: TimeSeriesChart + unit: percent + - + metricId: service-server-spans-rate + name: Span rate + metricQueries: + - expression: sum by (client_service_namespace, client) (rate(traces_service_graph_request_total{server_service_namespace="${tags.singleValue('service.namespace')}", server="${name}"}[${__rate_interval}])) + componentIdentifierTemplate: "urn:opentelemetry:namespace/${client_service_namespace}:service/${client}" + alias: "Client ${client}" + chart: + _type: TimeSeriesChart + unit: reqps + - + metricId: service-server-spans-duration + description: + name: Span duration + metricQueries: + - expression: histogram_quantile(0.95, sum(rate(traces_service_graph_request_server_seconds_bucket{server_service_namespace="${tags.singleValue('service.namespace')}", server="${name}"}[${__rate_interval}])) by (client, client_service_namespace, le)) + alias: '95th percentile ${client}' + componentIdentifierTemplate: "urn:opentelemetry:namespace/${client_service_namespace}:service/${client}" + chart: + _type: TimeSeriesChart + unit: s + - + metricId: service-client-spans-failed-percentage + name: Span error rate + metricQueries: + - expression: | + ( + ((sum by (server_service_namespace, server) (rate(traces_service_graph_request_failed_total{client_service_namespace="${tags.singleValue('service.namespace')}", client="${name}"}[${__rate_interval}]))) or 0) + / + sum by (server_service_namespace, server) (rate(traces_service_graph_request_total{client_service_namespace="${tags.singleValue('service.namespace')}", client="${name}"}[${__rate_interval}])) + ) * 100 + componentIdentifierTemplate: "urn:opentelemetry:namespace/${server_service_namespace}:service/${server}" + alias: "${server}" + chart: + _type: TimeSeriesChart + unit: percent + - + metricId: service-client-spans-rate + name: Span rate + metricQueries: + - expression: sum by (server_service_namespace, server) (rate(traces_service_graph_request_total{client_service_namespace="${tags.singleValue('service.namespace')}", client="${name}"}[${__rate_interval}])) + componentIdentifierTemplate: "urn:opentelemetry:namespace/${server_service_namespace}:service/${server}" + alias: "Server ${server}" + chart: + _type: TimeSeriesChart + unit: reqps + - + metricId: service-client-spans-duration + description: + name: Span duration + metricQueries: + - expression: histogram_quantile(0.95, sum(rate(traces_service_graph_request_client_seconds_bucket{client_service_namespace="${tags.singleValue('service.namespace')}", client="${name}"}[${__rate_interval}])) by (server, server_service_namespace, le)) + alias: '95th percentile ${server}' + componentIdentifierTemplate: "urn:opentelemetry:namespace/${server_service_namespace}:service/${server}" + chart: + _type: TimeSeriesChart + unit: s diff --git a/stackpacks/open-telemetry/settings/relation-mappings/asynchronous-consumer-peer.sty b/stackpacks/open-telemetry/settings/relation-mappings/asynchronous-consumer-peer.sty new file mode 100644 index 0000000..d5579e4 --- /dev/null +++ b/stackpacks/open-telemetry/settings/relation-mappings/asynchronous-consumer-peer.sty @@ -0,0 +1,38 @@ +nodes: + - _type: "OtelRelationMapping" + name: "OTel Async Relation (Consumer -> Peer)" + description: "Creates a relation from the consumer service instance to the message broker (peer service)" + identifier: "urn:stackpack:open-telemetry:otel-relation-mapping:asynchronous-consumer-peer" + input: + signal: + - "METRICS" + resource: + scope: + condition: "scope.name == 'traces_service_graph'" + metric: + condition: "metric.name == 'traces_service_graph_request_total'" + datapoint: + action: "'CREATE'" + condition: >- + 'client' in datapoint.attributes && + 'server' in datapoint.attributes && + 'client_service.namespace' in datapoint.attributes && + 'server_service.namespace' in datapoint.attributes && + 'connection_type' in datapoint.attributes && + datapoint.attributes['connection_type'] == 'messaging_system' && + 'client_peer.service' in datapoint.attributes + vars: + - name: "sourceId" + value: >- + 'server_service.instance.id' in datapoint.attributes ? + "urn:opentelemetry:namespace/" + datapoint.attributes['server_service.namespace'] + ":service/" + datapoint.attributes['server'] + ":serviceInstance/" + datapoint.attributes['server_service.instance.id'] : + "urn:opentelemetry:namespace/" + datapoint.attributes['server_service.namespace'] + ":service/" + datapoint.attributes['server'] + ":serviceInstance/" + datapoint.attributes['server'] + - name: "targetId" + value: >- + 'urn:opentelemetry:namespace/' + datapoint.attributes['client_service.namespace'] + ':service/' + datapoint.attributes['client_peer.service'] + output: + sourceId: "vars.sourceId" + targetId: "vars.targetId" + typeName: "'asynchronous'" + dependencyType: "'CONNECTION'" + expireAfter: 900000 diff --git a/stackpacks/open-telemetry/settings/relation-mappings/asynchronous-producer-consumer.sty b/stackpacks/open-telemetry/settings/relation-mappings/asynchronous-producer-consumer.sty new file mode 100644 index 0000000..adb84d7 --- /dev/null +++ b/stackpacks/open-telemetry/settings/relation-mappings/asynchronous-producer-consumer.sty @@ -0,0 +1,40 @@ +nodes: + - _type: "OtelRelationMapping" + name: "OTel Async Relation (Producer -> Consumer)" + description: "Creates a direct asynchronous relation from producer to consumer" + identifier: "urn:stackpack:open-telemetry:otel-relation-mapping:asynchronous-producer-consumer" + input: + signal: + - "METRICS" + resource: + scope: + condition: "scope.name == 'traces_service_graph'" + metric: + condition: "metric.name == 'traces_service_graph_request_total'" + datapoint: + action: "'CREATE'" + condition: >- + 'client' in datapoint.attributes && + 'server' in datapoint.attributes && + 'client_service.namespace' in datapoint.attributes && + 'server_service.namespace' in datapoint.attributes && + 'connection_type' in datapoint.attributes && + datapoint.attributes['connection_type'] == 'messaging_system' && + !('client_peer.service' in datapoint.attributes) + vars: + - name: "sourceId" + value: >- + 'client_service.instance.id' in datapoint.attributes ? + "urn:opentelemetry:namespace/" + datapoint.attributes['client_service.namespace'] + ":service/" + datapoint.attributes['client'] + ":serviceInstance/" + datapoint.attributes['client_service.instance.id'] : + "urn:opentelemetry:namespace/" + datapoint.attributes['client_service.namespace'] + ":service/" + datapoint.attributes['client'] + ":serviceInstance/" + datapoint.attributes['client'] + - name: "targetId" + value: >- + 'server_service.instance.id' in datapoint.attributes ? + "urn:opentelemetry:namespace/" + datapoint.attributes['server_service.namespace'] + ":service/" + datapoint.attributes['server'] + ":serviceInstance/" + datapoint.attributes['server_service.instance.id'] : + "urn:opentelemetry:namespace/" + datapoint.attributes['server_service.namespace'] + ":service/" + datapoint.attributes['server'] + ":serviceInstance/" + datapoint.attributes['server'] + output: + sourceId: "vars.sourceId" + targetId: "vars.targetId" + typeName: "'asynchronous'" + dependencyType: "'CONNECTION'" + expireAfter: 900000 diff --git a/stackpacks/open-telemetry/settings/relation-mappings/asynchronous-producer-peer.sty b/stackpacks/open-telemetry/settings/relation-mappings/asynchronous-producer-peer.sty new file mode 100644 index 0000000..e65c035 --- /dev/null +++ b/stackpacks/open-telemetry/settings/relation-mappings/asynchronous-producer-peer.sty @@ -0,0 +1,37 @@ +nodes: + - _type: "OtelRelationMapping" + name: "OTel Async Relation (Producer -> Peer)" + description: "Creates a relation from the producer to the message broker (peer service)" + identifier: "urn:stackpack:open-telemetry:otel-relation-mapping:asynchronous-producer-peer" + input: + signal: + - "METRICS" + resource: + scope: + condition: "scope.name == 'traces_service_graph'" + metric: + condition: "metric.name == 'traces_service_graph_request_total'" + datapoint: + action: "'CREATE'" + condition: >- + 'client' in datapoint.attributes && + 'server' in datapoint.attributes && + 'client_service.namespace' in datapoint.attributes && + 'connection_type' in datapoint.attributes && + datapoint.attributes['connection_type'] == 'messaging_system' && + 'client_peer.service' in datapoint.attributes + vars: + - name: "sourceId" + value: >- + 'client_service.instance.id' in datapoint.attributes ? + "urn:opentelemetry:namespace/" + datapoint.attributes['client_service.namespace'] + ":service/" + datapoint.attributes['client'] + ":serviceInstance/" + datapoint.attributes['client_service.instance.id'] : + "urn:opentelemetry:namespace/" + datapoint.attributes['client_service.namespace'] + ":service/" + datapoint.attributes['client'] + ":serviceInstance/" + datapoint.attributes['client'] + - name: "targetId" + value: >- + 'urn:opentelemetry:namespace/' + datapoint.attributes['client_service.namespace'] + ':service/' + datapoint.attributes['client_peer.service'] + output: + sourceId: "vars.sourceId" + targetId: "vars.targetId" + typeName: "'asynchronous'" + dependencyType: "'CONNECTION'" + expireAfter: 900000 diff --git a/stackpacks/open-telemetry/settings/relation-mappings/executes-ecs-task.sty b/stackpacks/open-telemetry/settings/relation-mappings/executes-ecs-task.sty new file mode 100644 index 0000000..b48aad8 --- /dev/null +++ b/stackpacks/open-telemetry/settings/relation-mappings/executes-ecs-task.sty @@ -0,0 +1,36 @@ +nodes: + - _type: "OtelRelationMapping" + name: "Executes Relation (Task)" + description: "Tasks executes a service instance" + identifier: "urn:stackpack:open-telemetry:otel-relation-mapping:executes-task" + input: + signal: + - "TRACES" + - "METRICS" + resource: + action: "'CREATE'" + condition: "'aws.ecs.task.id' in resource.attributes" + vars: + - name: "awsEcsTaskId" + value: "resource.attributes['aws.ecs.task.id']" + - name: "namespace" + value: >- + 'service.namespace' in resource.attributes && + resource.attributes['service.namespace'] != '' ? + resource.attributes['service.namespace'] : + 'default' + - name: "service" + value: "resource.attributes['service.name']" + - name: "instanceId" + value: >- + 'service.instance.id' in resource.attributes && + resource.attributes['service.instance.id'] != '' ? + resource.attributes['service.instance.id'] : + resource.attributes['service.name'] + output: + sourceId: "'urn:opentelemetry:task/' + vars.awsEcsTaskId" + targetId: >- + 'urn:opentelemetry:namespace/' + vars.namespace + ':service/' + vars.service + ':serviceInstance/' + vars.instanceId + typeName: "'executes'" + dependencyType: "'UNCLASSIFIED'" + expireAfter: 900000 diff --git a/stackpacks/open-telemetry/settings/relation-mappings/executes-function.sty b/stackpacks/open-telemetry/settings/relation-mappings/executes-function.sty new file mode 100644 index 0000000..dc35c52 --- /dev/null +++ b/stackpacks/open-telemetry/settings/relation-mappings/executes-function.sty @@ -0,0 +1,36 @@ +nodes: + - _type: "OtelRelationMapping" + name: "Executes Relation (Function)" + description: "Functions executes a service instance" + identifier: "urn:stackpack:open-telemetry:otel-relation-mapping:executes-function" + input: + signal: + - "TRACES" + - "METRICS" + resource: + action: "'CREATE'" + condition: "'faas.id' in resource.attributes" + vars: + - name: "faasId" + value: "resource.attributes['faas.id']" + - name: "namespace" + value: >- + 'service.namespace' in resource.attributes && + resource.attributes['service.namespace'] != '' ? + resource.attributes['service.namespace'] : + 'default' + - name: "service" + value: "resource.attributes['service.name']" + - name: "instanceId" + value: >- + 'service.instance.id' in resource.attributes && + resource.attributes['service.instance.id'] != '' ? + resource.attributes['service.instance.id'] : + resource.attributes['service.name'] + output: + sourceId: "'urn:opentelemetry:function/' + vars.faasId" + targetId: >- + 'urn:opentelemetry:namespace/' + vars.namespace + ':service/' + vars.service + ':serviceInstance/' + vars.instanceId + typeName: "'executes'" + dependencyType: "'UNCLASSIFIED'" + expireAfter: 900000 diff --git a/stackpacks/open-telemetry/settings/relation-mappings/executes-host.sty b/stackpacks/open-telemetry/settings/relation-mappings/executes-host.sty new file mode 100644 index 0000000..b91cef5 --- /dev/null +++ b/stackpacks/open-telemetry/settings/relation-mappings/executes-host.sty @@ -0,0 +1,36 @@ +nodes: + - _type: "OtelRelationMapping" + name: "Executes Relation (Host)" + description: "Host executes a service instance" + identifier: "urn:stackpack:open-telemetry:otel-relation-mapping:executes-host" + input: + signal: + - "TRACES" + - "METRICS" + resource: + action: "'CREATE'" + condition: "'host.id' in resource.attributes" + vars: + - name: "hostId" + value: "resource.attributes['host.id']" + - name: "namespace" + value: >- + 'service.namespace' in resource.attributes && + resource.attributes['service.namespace'] != '' ? + resource.attributes['service.namespace'] : + 'default' + - name: "service" + value: "resource.attributes['service.name']" + - name: "instanceId" + value: >- + 'service.instance.id' in resource.attributes && + resource.attributes['service.instance.id'] != '' ? + resource.attributes['service.instance.id'] : + resource.attributes['service.name'] + output: + sourceId: "'urn:opentelemetry:host/' + vars.hostId" + targetId: >- + 'urn:opentelemetry:namespace/' + vars.namespace + ':service/' + vars.service + ':serviceInstance/' + vars.instanceId + typeName: "'executes'" + dependencyType: "'UNCLASSIFIED'" + expireAfter: 900000 diff --git a/stackpacks/open-telemetry/settings/relation-mappings/k8s-pod-to-otel.sty b/stackpacks/open-telemetry/settings/relation-mappings/k8s-pod-to-otel.sty new file mode 100644 index 0000000..da973a0 --- /dev/null +++ b/stackpacks/open-telemetry/settings/relation-mappings/k8s-pod-to-otel.sty @@ -0,0 +1,46 @@ +nodes: + - _type: "OtelRelationMapping" + name: "Kubernetes to OTEL Relation" + description: "Relates a Kubernetes pod to a service instance" + identifier: "urn:stackpack:open-telemetry:otel-relation-mapping:kubernetes-to-otel" + input: + signal: + - "TRACES" + - "METRICS" + resource: + action: "'CREATE'" + condition: >- + 'k8s.cluster.name' in resource.attributes && + 'k8s.namespace.name' in resource.attributes && + 'k8s.pod.name' in resource.attributes + vars: + - name: "clusterName" + value: "resource.attributes['k8s.cluster.name']" + - name: "namespaceName" + value: "resource.attributes['k8s.namespace.name']" + - name: "podName" + value: "resource.attributes['k8s.pod.name']" + - name: "serviceNamespace" + value: >- + 'service.namespace' in resource.attributes && + resource.attributes['service.namespace'] != '' ? + resource.attributes['service.namespace'] : + 'default' + - name: "service" + value: >- + 'service.name' in resource.attributes ? + resource.attributes['service.name'] : + '' + - name: "instanceId" + value: >- + 'service.instance.id' in resource.attributes ? + resource.attributes['service.instance.id'] : + resource.attributes['service.name'] + output: + sourceId: >- + 'urn:opentelemetry:kubernetes:/' + vars.clusterName + ':' + vars.namespaceName + ':pod/' + vars.podName + targetId: >- + 'urn:opentelemetry:namespace/' + vars.serviceNamespace + ':service/' + vars.service + ':serviceInstance/' + vars.instanceId + typeName: "'kubernetes-to-otel'" + dependencyType: "'UNCLASSIFIED'" + expireAfter: 900000 diff --git a/stackpacks/open-telemetry/settings/relation-mappings/provided-by.sty b/stackpacks/open-telemetry/settings/relation-mappings/provided-by.sty new file mode 100644 index 0000000..1162cdf --- /dev/null +++ b/stackpacks/open-telemetry/settings/relation-mappings/provided-by.sty @@ -0,0 +1,34 @@ +nodes: + - _type: "OtelRelationMapping" + name: "OTel service instance provided-by service relation" + description: "Links a service to its instance" + identifier: "urn:stackpack:open-telemetry:otel-relation-mapping:provided-by" + input: + signal: + - "TRACES" + - "METRICS" + resource: + action: "'CREATE'" + condition: "'service.name' in resource.attributes" + vars: + - name: "namespace" + value: >- + 'service.namespace' in resource.attributes && + resource.attributes['service.namespace'] != '' ? + resource.attributes['service.namespace'] : + 'default' + - name: "service" + value: "resource.attributes['service.name']" + - name: "instanceId" + value: >- + 'service.instance.id' in resource.attributes ? + resource.attributes['service.instance.id'] : + resource.attributes['service.name'] + output: + sourceId: >- + 'urn:opentelemetry:namespace/' + vars.namespace + ':service/' + vars.service + targetId: >- + 'urn:opentelemetry:namespace/' + vars.namespace + ':service/' + vars.service + ':serviceInstance/' + vars.instanceId + typeName: "'provided-by'" + dependencyType: "'HIERARCHICAL'" + expireAfter: 900000 diff --git a/stackpacks/open-telemetry/settings/relation-mappings/synchronous.sty b/stackpacks/open-telemetry/settings/relation-mappings/synchronous.sty new file mode 100644 index 0000000..6ee708b --- /dev/null +++ b/stackpacks/open-telemetry/settings/relation-mappings/synchronous.sty @@ -0,0 +1,49 @@ +nodes: + - _type: "OtelRelationMapping" + name: "OTel Synchronous Relation" + description: "Adds a synchronous connection to a service" + identifier: "urn:stackpack:open-telemetry:otel-relation-mapping:synchronous" + input: + signal: + - "METRICS" + resource: + scope: + condition: "scope.name == 'traces_service_graph'" + metric: + condition: "metric.name == 'traces_service_graph_request_total'" + datapoint: + action: "'CREATE'" + condition: >- + 'client' in datapoint.attributes && + 'server' in datapoint.attributes && + 'client_service.namespace' in datapoint.attributes && + 'server_service.namespace' in datapoint.attributes && + 'connection_type' in datapoint.attributes && + datapoint.attributes['connection_type'] == '' + vars: + - name: "clientServiceNamespace" + value: "datapoint.attributes['client_service.namespace']" + - name: "serverServiceNamespace" + value: "datapoint.attributes['server_service.namespace']" + - name: "client" + value: "datapoint.attributes['client']" + - name: "server" + value: "datapoint.attributes['server']" + - name: "clientServiceInstanceId" + value: >- + 'client_service.instance.id' in datapoint.attributes ? + datapoint.attributes['client_service.instance.id'] : + datapoint.attributes['client'] + - name: "serverServiceInstanceId" + value: >- + 'server_service.instance.id' in datapoint.attributes ? + datapoint.attributes['server_service.instance.id'] : + datapoint.attributes['server'] + output: + sourceId: >- + 'urn:opentelemetry:namespace/' + vars.clientServiceNamespace + ':service/' + vars.client + ':serviceInstance/' + vars.clientServiceInstanceId + targetId: >- + 'urn:opentelemetry:namespace/' + vars.serverServiceNamespace + ':service/' + vars.server + ':serviceInstance/' + vars.serverServiceInstanceId + typeName: "'synchronous'" + dependencyType: "'CONNECTION'" + expireAfter: 900000 diff --git a/stackpacks/open-telemetry/stackpack.yaml b/stackpacks/open-telemetry/stackpack.yaml new file mode 100644 index 0000000..43859e5 --- /dev/null +++ b/stackpacks/open-telemetry/stackpack.yaml @@ -0,0 +1,20 @@ +name: "open-telemetry" +version: "0.1.5" +schemaVersion: "2.0" +displayName: "Open Telemetry" +categories: [ "Open Telemetry" ] +logoUrl: !resource "logo.png" +# information that will go in the overview section +overviewUrl: !resource "overview.md" +# information that will go to the detailed-overview section +detailedOverviewUrl: !resource "detailed-overview.md" +# information that will go in the configuration section +configurationUrls: + NOT_INSTALLED: !resource "configuration.md" + PROVISIONING: !resource "provisioning.md" + WAITING_FOR_DATA: !resource "waitingfordata.md" + INSTALLED: !resource "enabled.md" + DEPROVISIONING: !resource "configuration.md" + ERROR: !resource "configuration.md" +provision: + companionStackPacks: [] diff --git a/stackpacks/otel-k8s-crd/README.md b/stackpacks/otel-k8s-crd/README.md new file mode 100644 index 0000000..e69de29 diff --git a/stackpacks/otel-k8s-crd/icons/cr.svg b/stackpacks/otel-k8s-crd/icons/cr.svg new file mode 100644 index 0000000..efefc1c --- /dev/null +++ b/stackpacks/otel-k8s-crd/icons/cr.svg @@ -0,0 +1,4 @@ + + + + diff --git a/stackpacks/otel-k8s-crd/icons/crd.svg b/stackpacks/otel-k8s-crd/icons/crd.svg new file mode 100644 index 0000000..ff6700b --- /dev/null +++ b/stackpacks/otel-k8s-crd/icons/crd.svg @@ -0,0 +1,4 @@ + + + + diff --git a/stackpacks/otel-k8s-crd/resources/configuration.md b/stackpacks/otel-k8s-crd/resources/configuration.md new file mode 100644 index 0000000..27cc6a3 --- /dev/null +++ b/stackpacks/otel-k8s-crd/resources/configuration.md @@ -0,0 +1,43 @@ +## Prerequisites + +- SUSE Observability Agent installed with Helm +- Network connectivity from the agent to the platform OTLP ingest endpoint + +## Enable the k8s resource collector + +The k8s resource collector is part of the agent's OpenTelemetry components, which are disabled by default. Enable them with `otel.enabled`; the collector itself is enabled by default once OpenTelemetry is on: + +```yaml +otel: + enabled: true +``` + +Or, when installing or upgrading the agent with Helm: + +```bash +--set otel.enabled=true +``` + +## Select Custom Resource API groups + +CRDs are always collected. Custom Resource instances are only collected for selected API groups. Enabled integration presets add common SUSE-related API groups, such as Kubewarden, SUSE Runtime Enforcer and SUSE Virtualization. Add more with `otel.k8sResourceCollector.crDiscovery.apiGroups.include`: + +```yaml +otel: + enabled: true + k8sResourceCollector: + crDiscovery: + discoveryMode: api_groups + apiGroups: + include: + "policies.kubewarden.io": true + "kubevirt.io": true + exclude: + "internal.example.com": true +``` + +Set an included API group to `false` to disable an integration-provided default. Set `discoveryMode: all` to collect CRs for every CRD API group; `apiGroups` filters are then ignored. + +When `otel.k8sResourceCollector.rbac.useWildcard: false`, the truthy `crDiscovery.apiGroups.include` entries are also used for restricted RBAC. Kubernetes RBAC only supports exact API groups or `"*"`, so wildcard patterns like `"*.example.com"` require `rbac.useWildcard: true`. + +For the full configuration guide, see the [k8s resource collector documentation](https://documentation.suse.com/cloudnative/suse-observability/latest/en/setup/otel/k8s-resource-collector.html). diff --git a/stackpacks/otel-k8s-crd/resources/detailed-overview.md b/stackpacks/otel-k8s-crd/resources/detailed-overview.md new file mode 100644 index 0000000..9de123d --- /dev/null +++ b/stackpacks/otel-k8s-crd/resources/detailed-overview.md @@ -0,0 +1,18 @@ +## How it works + +The StackPack defines SUSE Observability settings (component and relation mappings) that the platform's OTel collector applies to telemetry produced by the `k8scrdreceiver` OpenTelemetry receiver, which runs in the SUSE Observability Agent's k8s CRD collector. The receiver discovers CRDs and their CR instances using watch mode (real-time informer events) plus periodic snapshots (full-state emission from the informer cache, controlled by `snapshotInterval`). + +### Components created + +- **Custom Resource Definition** - Each CRD installed in the cluster (e.g., `helmcharts.helm.cattle.io`) +- **Custom Resource** - Each CR instance (e.g., a specific HelmChart resource) + +### Relations created + +- **instance of** - Links each Custom Resource to its defining Custom Resource Definition + +### Data sources + +- CRD and CR discovery via the Kubernetes API server +- Watch mode for real-time change detection +- Periodic snapshots from the informer cache for state reconciliation diff --git a/stackpacks/otel-k8s-crd/resources/enabled.md b/stackpacks/otel-k8s-crd/resources/enabled.md new file mode 100644 index 0000000..2edd0e7 --- /dev/null +++ b/stackpacks/otel-k8s-crd/resources/enabled.md @@ -0,0 +1,38 @@ +## The Open Telemetry Kubernetes CRD StackPack is installed + +### What's next + +Explore the discovered Custom Resources and their relationships in SUSE Observability. + +### No data + +Make sure the k8s resource collector is enabled in the SUSE Observability Agent. It is part of the agent's OpenTelemetry components, which are disabled by default: + +```yaml +otel: + enabled: true +``` + +Or, when installing or upgrading the agent with Helm: + +```bash +--set otel.enabled=true +``` + +CRDs are always collected. Custom Resource instances are only collected for selected API groups. Add more with `otel.k8sResourceCollector.crDiscovery.apiGroups.include`, or set `discoveryMode: all` to collect CRs for every CRD API group: + +```yaml +otel: + enabled: true + k8sResourceCollector: + crDiscovery: + discoveryMode: api_groups + apiGroups: + include: + "policies.kubewarden.io": true + "kubevirt.io": true +``` + +When `otel.k8sResourceCollector.rbac.useWildcard: false`, the truthy `crDiscovery.apiGroups.include` entries are also used for restricted RBAC. Kubernetes RBAC only supports exact API groups or `"*"`, so wildcard patterns like `"*.example.com"` require `rbac.useWildcard: true`. + +For the full configuration guide, see the [k8s resource collector documentation](https://documentation.suse.com/cloudnative/suse-observability/latest/en/setup/otel/k8s-resource-collector.html). diff --git a/stackpacks/otel-k8s-crd/resources/error.md b/stackpacks/otel-k8s-crd/resources/error.md new file mode 100644 index 0000000..e69de29 diff --git a/stackpacks/otel-k8s-crd/resources/logo.svg b/stackpacks/otel-k8s-crd/resources/logo.svg new file mode 100644 index 0000000..7f5d1f3 --- /dev/null +++ b/stackpacks/otel-k8s-crd/resources/logo.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/stackpacks/otel-k8s-crd/resources/overview.md b/stackpacks/otel-k8s-crd/resources/overview.md new file mode 100644 index 0000000..2cbd870 --- /dev/null +++ b/stackpacks/otel-k8s-crd/resources/overview.md @@ -0,0 +1,5 @@ +## Open Telemetry Kubernetes CRD StackPack + +The **Kubernetes Custom Resources** StackPack discovers Custom Resource Definitions (CRDs) and their Custom Resource (CR) instances in your Kubernetes clusters, creating topology components and relations for each. + +This provides visibility into the custom extensions installed in your clusters, including operators, controllers, and platform add-ons. diff --git a/stackpacks/otel-k8s-crd/resources/provisioning.md b/stackpacks/otel-k8s-crd/resources/provisioning.md new file mode 100644 index 0000000..0935880 --- /dev/null +++ b/stackpacks/otel-k8s-crd/resources/provisioning.md @@ -0,0 +1,3 @@ +## Provisioning... + +Provisioning Kubernetes Custom Resource discovery. Please wait a moment. diff --git a/stackpacks/otel-k8s-crd/resources/waitingfordata.md b/stackpacks/otel-k8s-crd/resources/waitingfordata.md new file mode 100644 index 0000000..e69de29 diff --git a/stackpacks/otel-k8s-crd/settings/component-mappings/custom-resource-definitions.sty b/stackpacks/otel-k8s-crd/settings/component-mappings/custom-resource-definitions.sty new file mode 100644 index 0000000..00c8454 --- /dev/null +++ b/stackpacks/otel-k8s-crd/settings/component-mappings/custom-resource-definitions.sty @@ -0,0 +1,49 @@ +nodes: + - _type: "OtelComponentMapping" + name: "Kubernetes Custom Resource Definition" + description: "Represents a CRD installed in a Kubernetes cluster" + identifier: "urn:stackpack:otel-k8s-crd:otel-component-mapping:custom-resource-definition" + input: + signal: + - "LOGS" + resource: + scope: + log: + action: "log.body['type'] in ['ADDED', 'MODIFIED'] ? 'CREATE' : log.body['type'] == 'DELETED' ? 'DELETE' : 'CONTINUE'" + condition: log.eventName == 'KubernetesCustomResourceDefinitionEvent' + vars: + - name: "clusterName" + value: "resource.attributes['k8s.cluster.name']" + - name: "name" + value: "log.attributes['k8s.object.name']" + - name: "group" + value: "log.attributes['k8s.resource.group']" + - name: "kind" + value: "log.attributes['k8s.resource.kind']" + - name: "customResourcesWatched" + value: "'k8s.crd.custom_resources_watched' in log.attributes && log.attributes['k8s.crd.custom_resources_watched'] ? 'Yes' : 'No'" + output: + identifier: "'urn:k8s:crd:/cluster/' + vars.clusterName + ':group/' + vars.group + ':kind/' + vars.kind" + name: "vars.name" + typeName: "'custom resource definition'" + required: + configuration: "log.body['object']['spec']" + tags: + - source: "'otel-k8s-crd'" + target: "stackpack" + - source: "vars.group" + target: "k8s.crd.group" + - source: "vars.kind" + target: "k8s.crd.kind" + - source: "vars.clusterName" + target: "cluster-name" + - source: "vars.customResourcesWatched" + target: "k8s.crd.custom_resources_watched" + optional: + status: "log.body['object']['status']" + tags: + - source: "log.body['object']['metadata']['creationTimestamp']" + target: "k8s.creationTimestamp" + expireAfter: 900000 + rank: + specificity: 100 diff --git a/stackpacks/otel-k8s-crd/settings/component-mappings/custom-resources.sty b/stackpacks/otel-k8s-crd/settings/component-mappings/custom-resources.sty new file mode 100644 index 0000000..077b0e0 --- /dev/null +++ b/stackpacks/otel-k8s-crd/settings/component-mappings/custom-resources.sty @@ -0,0 +1,51 @@ +nodes: + - _type: "OtelComponentMapping" + name: "Kubernetes Custom Resource" + description: "Represents a CR instance of a CRD in a Kubernetes cluster" + identifier: "urn:stackpack:otel-k8s-crd:otel-component-mapping:custom-resource" + input: + signal: + - "LOGS" + resource: + scope: + log: + action: "log.body['type'] in ['ADDED', 'MODIFIED'] ? 'CREATE' : log.body['type'] == 'DELETED' ? 'DELETE' : 'CONTINUE'" + condition: log.eventName == 'KubernetesCustomResourceEvent' + vars: + - name: "clusterName" + value: "resource.attributes['k8s.cluster.name']" + - name: "kind" + value: "log.attributes['k8s.resource.kind']" + - name: "group" + value: "log.attributes['k8s.resource.group']" + - name: "name" + value: "log.attributes['k8s.object.name']" + - name: "namespace" + value: "'k8s.namespace.name' in log.attributes ? log.attributes['k8s.namespace.name'] : ''" + output: + identifier: "'urn:k8s:cr:/cluster/' + vars.clusterName + ':namespace/' + vars.namespace + ':group/' + vars.group + ':kind/' + vars.kind + ':name/' + vars.name" + name: "vars.name" + typeName: "'custom resource'" + required: + configuration: "omit(log.body['object'], ['status', 'metadata'])" + tags: + - source: "'otel-k8s-crd'" + target: "stackpack" + - source: "vars.kind" + target: "k8s.resource.kind" + - source: "vars.group" + target: "k8s.resource.group" + - source: "vars.name" + target: "k8s.resource.name" + - source: "vars.clusterName" + target: "cluster-name" + optional: + status: "log.body['object']['status']" + tags: + - source: "vars.namespace" + target: "namespace" + - source: "log.body['object']['metadata']['creationTimestamp']" + target: "k8s.creationTimestamp" + expireAfter: 900000 + rank: + specificity: 100 diff --git a/stackpacks/otel-k8s-crd/settings/main-menu.sty b/stackpacks/otel-k8s-crd/settings/main-menu.sty new file mode 100644 index 0000000..4966499 --- /dev/null +++ b/stackpacks/otel-k8s-crd/settings/main-menu.sty @@ -0,0 +1,7 @@ +nodes: + - _type: MainMenuGroup + name: Kubernetes + iconbase64: "" + defaultOpen: true + items: [] + identifier: "urn:stackpack:otel-k8s-crd:main-menu-group:kubernetes" diff --git a/stackpacks/otel-k8s-crd/settings/presentations/common-filters.sty b/stackpacks/otel-k8s-crd/settings/presentations/common-filters.sty new file mode 100644 index 0000000..521906c --- /dev/null +++ b/stackpacks/otel-k8s-crd/settings/presentations/common-filters.sty @@ -0,0 +1,68 @@ +nodes: + - _type: ComponentPresentation + name: "K8s CRD Common Filters" + description: "Common filters for Kubernetes Custom Resource components" + identifier: "urn:stackpack:otel-k8s-crd:component-presentation:common-filters" + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("stackpack:otel-k8s-crd") + rank: + specificity: 1 + presentation: + filters: + - filterId: "crd_cluster" + displayName: + singular: "Cluster" + plural: "Clusters" + filter: + _type: "TagFilter" + tagKey: "cluster-name" + moreTab: "Kubernetes" + - filterId: "crd_api_group" + displayName: + singular: "API Group" + plural: "API Groups" + filter: + _type: "TagFilter" + tagKey: "k8s.crd.group" + moreTab: "Custom Resources" + - filterId: "crd_kind" + displayName: + singular: "Kind" + plural: "Kinds" + filter: + _type: "TagFilter" + tagKey: "k8s.crd.kind" + moreTab: "Custom Resources" + - filterId: "crd_custom_resources_watched" + displayName: + singular: "CRs collected" + plural: "CRs collected" + filter: + _type: "TagFilter" + tagKey: "k8s.crd.custom_resources_watched" + moreTab: "Custom Resources" + - filterId: "cr_api_group" + displayName: + singular: "API Group" + plural: "API Groups" + filter: + _type: "TagFilter" + tagKey: "k8s.resource.group" + moreTab: "Custom Resources" + - filterId: "cr_kind" + displayName: + singular: "Kind" + plural: "Kinds" + filter: + _type: "TagFilter" + tagKey: "k8s.resource.kind" + moreTab: "Custom Resources" + - filterId: "cr_namespace" + displayName: + singular: "Namespace" + plural: "Namespaces" + filter: + _type: "TagFilter" + tagKey: "namespace" + moreTab: "Kubernetes" diff --git a/stackpacks/otel-k8s-crd/settings/presentations/custom-resource-definitions.sty b/stackpacks/otel-k8s-crd/settings/presentations/custom-resource-definitions.sty new file mode 100644 index 0000000..e527fe4 --- /dev/null +++ b/stackpacks/otel-k8s-crd/settings/presentations/custom-resource-definitions.sty @@ -0,0 +1,114 @@ +nodes: + - _type: ComponentPresentation + name: "Custom Resource Definition" + description: "Presentation for Kubernetes Custom Resource Definitions" + identifier: "urn:stackpack:otel-k8s-crd:component-presentation:custom-resource-definition" + binding: + _type: ComponentPresentationQueryBinding + query: (label = "stackpack:otel-k8s-crd" and label = "source-type:custom resource definition") + rank: + specificity: 10 + presentation: + icon: !icon crd.svg + filters: + - filterId: "crd_cluster" + - filterId: "crd_api_group" + - filterId: "crd_kind" + - filterId: "crd_custom_resources_watched" + overview: + name: + singular: "custom resource definition" + plural: "custom resource definitions" + title: "Custom Resource Definitions" + mainMenu: + group: "Kubernetes" + order: 0 + columns: + - columnId: crd_name + title: "Name" + projection: + _type: "ComponentLinkProjection" + name: "name" + identifier: "identifiers[0]" + - columnId: crd_crs_collected + title: "CRs collected" + projection: + _type: "TextProjection" + value: "tags.singleValue('k8s.crd.custom_resources_watched')" + - columnId: crd_api_group + title: "API Group" + projection: + _type: "TextProjection" + value: "tags.singleValue('k8s.crd.group')" + - columnId: crd_kind + title: "Kind" + projection: + _type: "TextProjection" + value: "tags.singleValue('k8s.crd.kind')" + - columnId: crd_cluster + title: "Cluster" + projection: + _type: "TextProjection" + value: "tags.singleValue('cluster-name')" + - columnId: crd_age + title: "Age" + projection: + _type: "DurationProjection" + startTime: "tags.singleValue('k8s.creationTimestamp')" + highlight: + title: "Custom Resource Definition" + fields: + - fieldId: "type" + title: "Type" + order: 100.0 + projection: + _type: "TextProjection" + value: "typeName" + - fieldId: "api_group" + title: "API Group" + order: 90.0 + projection: + _type: "TextProjection" + value: "tags.singleValue('k8s.crd.group')" + - fieldId: "kind" + title: "Kind" + order: 80.0 + projection: + _type: "TextProjection" + value: "tags.singleValue('k8s.crd.kind')" + - fieldId: "cluster" + title: "Cluster" + order: 70.0 + projection: + _type: "TextProjection" + value: "tags.singleValue('cluster-name')" + - fieldId: "age" + title: "Age" + order: 60.0 + projection: + _type: "DurationProjection" + startTime: "tags.singleValue('k8s.creationTimestamp')" + - fieldId: "labels" + title: "Labels" + order: 10.0 + projection: + _type: "TagProjection" + provisioning: + topologySourceSelector: "source.identifier == identifiers[0]" + showConfiguration: true + showStatus: true + relatedResources: + - resourceId: "crd_instances" + title: "Custom Resource instances" + order: 1.0 + topologyQuery: >- + (withNeighborsOf(direction = "both", + components = (identifier = "${identifiers[0]}"), + levels = "1")) and label = "source-type:custom resource" + presentationIdentifier: "urn:stackpack:otel-k8s-crd:component-presentation:custom-resource" + events: + showEvents: true + relatedResourcesQuery: | + identifier = "${identifiers[0]}" + OR + (withNeighborsOf(direction = "both", components = (identifier = "${identifiers[0]}"), levels = "1") and label = "source-type:custom resource") diff --git a/stackpacks/otel-k8s-crd/settings/presentations/custom-resources.sty b/stackpacks/otel-k8s-crd/settings/presentations/custom-resources.sty new file mode 100644 index 0000000..c4d3162 --- /dev/null +++ b/stackpacks/otel-k8s-crd/settings/presentations/custom-resources.sty @@ -0,0 +1,129 @@ +nodes: + - _type: ComponentPresentation + name: "Custom Resource" + description: "Presentation for Kubernetes Custom Resource instances" + identifier: "urn:stackpack:otel-k8s-crd:component-presentation:custom-resource" + binding: + _type: ComponentPresentationQueryBinding + query: (label = "stackpack:otel-k8s-crd" and label = "source-type:custom resource") + rank: + specificity: 10 + presentation: + icon: !icon cr.svg + filters: + - filterId: "crd_cluster" + - filterId: "cr_api_group" + - filterId: "cr_kind" + - filterId: "cr_namespace" + topology: + groupingEnabled: true + showIndirectRelations: false + minimumGroupSize: 8 + groupedByLayers: false + groupedByDomains: true + autoGrouping: true + connectedComponents: true + overview: + name: + singular: "custom resource" + plural: "custom resources" + title: "Custom Resources" + mainMenu: + group: "Kubernetes" + order: 0 + columns: + - columnId: cr_name + title: "Name" + projection: + _type: "ComponentLinkProjection" + name: "name" + identifier: "identifiers[0]" + - columnId: cr_kind + title: "Kind" + projection: + _type: "TextProjection" + value: "tags.singleValue('k8s.resource.kind')" + - columnId: cr_api_group + title: "API Group" + projection: + _type: "TextProjection" + value: "tags.singleValue('k8s.resource.group')" + - columnId: cr_namespace + title: "Namespace" + projection: + _type: "TextProjection" + value: "tags.singleValue('namespace')" + - columnId: cr_cluster + title: "Cluster" + projection: + _type: "TextProjection" + value: "tags.singleValue('cluster-name')" + - columnId: cr_age + title: "Age" + projection: + _type: "DurationProjection" + startTime: "tags.singleValue('k8s.creationTimestamp')" + highlight: + title: "Custom Resource" + fields: + - fieldId: "type" + title: "Type" + order: 100.0 + projection: + _type: "TextProjection" + value: "typeName" + - fieldId: "kind" + title: "Kind" + order: 90.0 + projection: + _type: "ComponentLinkProjection" + name: "tags.singleValue('k8s.resource.kind')" + identifier: "'urn:k8s:crd:/cluster/' + tags.singleValue('cluster-name') + ':group/' + tags.singleValue('k8s.resource.group') + ':kind/' + tags.singleValue('k8s.resource.kind')" + - fieldId: "api_group" + title: "API Group" + order: 80.0 + projection: + _type: "TextProjection" + value: "tags.singleValue('k8s.resource.group')" + - fieldId: "namespace" + title: "Namespace" + order: 60.0 + projection: + _type: "TextProjection" + value: "tags.singleValue('namespace')" + - fieldId: "cluster" + title: "Cluster" + order: 50.0 + projection: + _type: "TextProjection" + value: "tags.singleValue('cluster-name')" + - fieldId: "age" + title: "Age" + order: 40.0 + projection: + _type: "DurationProjection" + startTime: "tags.singleValue('k8s.creationTimestamp')" + - fieldId: "labels" + title: "Labels" + order: 10.0 + projection: + _type: "TagProjection" + provisioning: + topologySourceSelector: "source.identifier == identifiers[0]" + showConfiguration: true + showStatus: true + relatedResources: + - resourceId: "cr_definition" + title: "Custom Resource Definition" + order: 1.0 + topologyQuery: >- + (withNeighborsOf(direction = "both", + components = (identifier = "${identifiers[0]}"), + levels = "1")) and label = "source-type:custom resource definition" + presentationIdentifier: "urn:stackpack:otel-k8s-crd:component-presentation:custom-resource-definition" + events: + showEvents: true + relatedResourcesQuery: | + identifier = "${identifiers[0]}" + OR + (withNeighborsOf(direction = "both", components = (identifier = "${identifiers[0]}"), levels = "1") and label = "source-type:custom resource definition") diff --git a/stackpacks/otel-k8s-crd/settings/relation-mappings/cr-instance-of-crd.sty b/stackpacks/otel-k8s-crd/settings/relation-mappings/cr-instance-of-crd.sty new file mode 100644 index 0000000..b33c813 --- /dev/null +++ b/stackpacks/otel-k8s-crd/settings/relation-mappings/cr-instance-of-crd.sty @@ -0,0 +1,30 @@ +nodes: + - _type: "OtelRelationMapping" + name: "Custom Resource instance of CRD" + description: "Links a Custom Resource to the Custom Resource Definition it is an instance of" + identifier: "urn:stackpack:otel-k8s-crd:otel-relation-mapping:cr-instance-of-crd" + input: + signal: + - "LOGS" + resource: + scope: + log: + action: "log.body['type'] in ['ADDED', 'MODIFIED'] ? 'CREATE' : log.body['type'] == 'DELETED' ? 'DELETE' : 'CONTINUE'" + condition: log.eventName == 'KubernetesCustomResourceEvent' + vars: + - name: "clusterName" + value: "resource.attributes['k8s.cluster.name']" + - name: "kind" + value: "log.attributes['k8s.resource.kind']" + - name: "group" + value: "log.attributes['k8s.resource.group']" + - name: "name" + value: "log.attributes['k8s.object.name']" + - name: "namespace" + value: "'k8s.namespace.name' in log.attributes ? log.attributes['k8s.namespace.name'] : ''" + output: + sourceId: "'urn:k8s:cr:/cluster/' + vars.clusterName + ':namespace/' + vars.namespace + ':group/' + vars.group + ':kind/' + vars.kind + ':name/' + vars.name" + targetId: "'urn:k8s:crd:/cluster/' + vars.clusterName + ':group/' + vars.group + ':kind/' + vars.kind" + typeName: "'instance of'" + dependencyType: "'UNCLASSIFIED'" + expireAfter: 900000 diff --git a/stackpacks/otel-k8s-crd/stackpack.yaml b/stackpacks/otel-k8s-crd/stackpack.yaml new file mode 100644 index 0000000..e82813d --- /dev/null +++ b/stackpacks/otel-k8s-crd/stackpack.yaml @@ -0,0 +1,17 @@ +name: "otel-k8s-crd" +version: "0.0.9" +schemaVersion: "2.0" +displayName: "Kubernetes Custom Resources" +categories: [ "Kubernetes" ] +logoUrl: !resource "logo.svg" +overviewUrl: !resource "overview.md" +detailedOverviewUrl: !resource "detailed-overview.md" +configurationUrls: + NOT_INSTALLED: !resource "configuration.md" + PROVISIONING: !resource "provisioning.md" + WAITING_FOR_DATA: !resource "waitingfordata.md" + INSTALLED: !resource "enabled.md" + DEPROVISIONING: !resource "configuration.md" + ERROR: !resource "error.md" +provision: + companionStackPacks: [] diff --git a/stackpacks/suse-observability/README.md b/stackpacks/suse-observability/README.md new file mode 100644 index 0000000..e69de29 diff --git a/stackpacks/suse-observability/icons/suse-observability.svg b/stackpacks/suse-observability/icons/suse-observability.svg new file mode 100644 index 0000000..0c19ca3 --- /dev/null +++ b/stackpacks/suse-observability/icons/suse-observability.svg @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/generic/sts-apps-overrun-cpu-request.md.hbs b/stackpacks/suse-observability/includes/generic/sts-apps-overrun-cpu-request.md.hbs new file mode 100644 index 0000000..46e2919 --- /dev/null +++ b/stackpacks/suse-observability/includes/generic/sts-apps-overrun-cpu-request.md.hbs @@ -0,0 +1,26 @@ +The Average CPU usage for container (`{{ tags.pod_name }}/{{ tags.container }}`) exceeds CPU requests by more than 100%. +This might not be an issue if there are enough resources in the node where the container is running, but is that's not the case performance degradation can be expected in the service. + +### Mitigation +1. Verify with the metrics chart if the average CPU usage has increased for a long time or if the monitor was triggered just due to a spike. +2. In the latter case you can increase the CPU request of the service related to the container `{{ tags.pod_name }}/{{ tags.container }}` +3. Inspect the [Configuration](/#/components/{{ componentUrnForUrl }}#configuration) of this pod and look for the amount of requested CPU. +4. Based on the pod locate the relevant section of the `values.yaml` and update the CPU requests to give more resources, for example if the pod with the issue if kafka then we would need to update + +``` +kafka: + resources: + requests: + cpu: 1000m +``` + +5. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously + +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/generic/sts-java-long-gc.md.hbs b/stackpacks/suse-observability/includes/generic/sts-java-long-gc.md.hbs new file mode 100644 index 0000000..b4edc6c --- /dev/null +++ b/stackpacks/suse-observability/includes/generic/sts-java-long-gc.md.hbs @@ -0,0 +1,22 @@ +GC collection times have been excessive for `{{ tags.pod_name }}`. This means that the CPU has not been able to do any kind of work but GC during those pauses. + +### Mitigation +1. Inspect the [Configuration](/#/components/{{ componentUrnForUrl }}#configuration) of this pod and look for the amount of requested Memory. +2. Based on the pod locate the relevant section of the `values.yaml` and increase the Memory requests to give more resources, for example if the pod with the issue if kafka then we would need to update + +``` +kafka: + resources: + requests: + memory: 3Gi +``` + +3. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/checks/monitors-execution-time-pod.md.hbs b/stackpacks/suse-observability/includes/services/checks/monitors-execution-time-pod.md.hbs new file mode 100644 index 0000000..7e5e8eb --- /dev/null +++ b/stackpacks/suse-observability/includes/services/checks/monitors-execution-time-pod.md.hbs @@ -0,0 +1,9 @@ +This monitor takes longer on average to evaluate than a one minute interval, so runs cannot reliably complete before the next one is due, and results arrive late or are skipped. + +Check: + +- the monitor's own query - long ranges over many series are the usual cause +- `Select concurrency saturated` on VictoriaMetrics, since a saturated query path slows every monitor at once +- whether the `checks` pod (or `server`, in non-HA) is CPU starved, on the *Metrics processing* dashboard + +Either simplify the query or lengthen the monitor's interval so it matches how long it actually takes. diff --git a/stackpacks/suse-observability/includes/services/checks/monitors-failure-percentage-pod.md.hbs b/stackpacks/suse-observability/includes/services/checks/monitors-failure-percentage-pod.md.hbs new file mode 100644 index 0000000..2be6e53 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/checks/monitors-failure-percentage-pod.md.hbs @@ -0,0 +1,10 @@ +The monitor failures monitor triggers when a monitor running in SUSE Observability is failing more than expected. This will impact the +monitor results by being late, incorrect or incomplete. + +## Possible solutions + +- Look into the logs of the pod running the monitor to understand what is causing the monitor to fail. +- It could be SUSE observability is under provisioned. Investigate the CPU usage and throttling of the server or checks pod + running the monitor. If the pod is being throttled, increase CPU requests and limits. It could also be an upstream like VictoriaMetrics is throttled + slowing down execution of a monitor. +- If this does not solve the issue, please contact SUSE Observability [support](https://www.stackstate.com/company/contact-us/). diff --git a/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-clickhouse-replicated-part-failed-fetches.md.hbs b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-clickhouse-replicated-part-failed-fetches.md.hbs new file mode 100644 index 0000000..593e2a5 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-clickhouse-replicated-part-failed-fetches.md.hbs @@ -0,0 +1,5 @@ +ClickHouse increase `ReplicatedPartFailedFetches` in `system.events` table. +It mean server was failed to download data part from replica of a ReplicatedMergeTree table. +Please check following things: +- connections between ClickHouse pod and his replicas (see remote_server section in /etc/clickhouse-server/) +- [logs](/#/components/{{ componentUrnForUrl }}#logs) of the pod \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-data-after-merge-differs-from-replica.md.hbs b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-data-after-merge-differs-from-replica.md.hbs new file mode 100644 index 0000000..b98f3df --- /dev/null +++ b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-data-after-merge-differs-from-replica.md.hbs @@ -0,0 +1,14 @@ +ClickHouse increase DataAfterMergeDiffersFromReplica in `system.events` table. +It mean Data after merge is not byte-identical to data on another replicas. +There could be several reasons: +- Using newer version of compression library after server update. +- Using another compression method. +- Non-deterministic compression algorithm (highly unlikely). +- Non-deterministic merge algorithm due to logical error in code. +- Data corruption in memory due to bug in code. +- Data corruption in memory due to hardware issue. +- Manual modification of source data after server startup. +- Manual modification of checksums stored in ZooKeeper. +- Part format related settings like 'enable_mixed_granularity_parts' are different on different replicas. + +Server will download merged part from replica to force byte-identical result. \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-delayed-insert-throttling.md.hbs b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-delayed-insert-throttling.md.hbs new file mode 100644 index 0000000..a7bb60d --- /dev/null +++ b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-delayed-insert-throttling.md.hbs @@ -0,0 +1,2 @@ +ClickHouse has INSERT statements that are throttled due to high number of active data parts for partition in a MergeTree, +please decrease INSERT frequency https://clickhouse.com/docs/en/development/architecture/#merge-tree \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-distributed-files-to-insert-high.md.hbs b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-distributed-files-to-insert-high.md.hbs new file mode 100644 index 0000000..c3dcc03 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-distributed-files-to-insert-high.md.hbs @@ -0,0 +1,16 @@ +ClickHouse has too much files which not insert to `*MergeTree` tables via `Distributed` table engine +Check not synced .bin files via + +```kubectl exec -n {{labels.namespace}} pod/{{labels.pod_name}} -- bash -c "ls -la /var/lib/clickhouse/data/*/*/*/*.bin"``` + +Also, check documentation: +https://clickhouse.com/docs/en/engines/table-engines/special/distributed/ +When you insert data to `Distributed` table. +Data is written to target `*MergreTree` tables asynchronously. +When inserted in the table, the data block is just written to the local file system. +The data is sent to the remote servers in the background as soon as possible. +The period for sending data is managed by the `distributed_directory_monitor_sleep_time_ms` and `distributed_directory_monitor_max_sleep_time_ms` settings. +The Distributed engine sends each file with inserted data separately, but you can enable batch sending of files with the `distributed_directory_monitor_batch_inserts` setting + +Also, you can manage distributed tables: +https://clickhouse.com/docs/en/sql-reference/statements/system/#managing-distributed-tables \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-distributed-sync-insertion-timeout-exceeded.md.hbs b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-distributed-sync-insertion-timeout-exceeded.md.hbs new file mode 100644 index 0000000..e81007c --- /dev/null +++ b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-distributed-sync-insertion-timeout-exceeded.md.hbs @@ -0,0 +1,4 @@ +ClickHouse increase DistributedSyncInsertionTimeoutExceeded in `system.events` table. +It mean Synchronous distributed insert timeout exceeded after successfully distributed connection. +Please check documentation https://clickhouse.com/docs/en/operations/settings/settings/#insert_distributed_sync +And check connection between the Pod and all nodes in shards from remote_servers config section diff --git a/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-file-descriptor-buffer-read-or-write-failed.md.hbs b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-file-descriptor-buffer-read-or-write-failed.md.hbs new file mode 100644 index 0000000..41fc116 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-file-descriptor-buffer-read-or-write-failed.md.hbs @@ -0,0 +1,4 @@ +ClickHouse increase ReadBufferFromFileDescriptorReadFailed or ReadBufferFromFileDescriptorReadFailed in `system.events` table. +It mean the read (read/pread) or writes (write/pwrite) to a file descriptor. Does not include sockets. +System can't read or write to some files. +Please check [logs](/#/components/{{ componentUrnForUrl }}#logs) of the pod. \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-max-part-count-for-partition.md.hbs b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-max-part-count-for-partition.md.hbs new file mode 100644 index 0000000..f09357f --- /dev/null +++ b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-max-part-count-for-partition.md.hbs @@ -0,0 +1,5 @@ +ClickHouse has too many parts in one partition. +ClickHouse MergeTree table engine split each INSERT statement to partitions (PARTITION BY expression) +and add one or more PARTS per INSERT inside each partition, after that background merge process run, +and when you have too much unmerged parts inside partition, +SELECT statements performance can significant degrade, so clickhouse tries to delay or reject INSERT \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-readonly-replica.md.hbs b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-readonly-replica.md.hbs new file mode 100644 index 0000000..a361aaf --- /dev/null +++ b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-readonly-replica.md.hbs @@ -0,0 +1,10 @@ +ClickHouse has ReplicatedMergeTree tables that are currently in readonly state due to re-initialization after ZooKeeper session loss or due to startup without ZooKeeper configured. +Please check following things: +- kubernetes nodes have free enough RAM and Disk via `kubectl top node` +- status of ClickHouse Pod +- connection between ClickHouse pods and ZooKeeper ```kubectl exec -n {{labels.namespace}} pod/{{labels.pod_name}} -- bash -c "clickhouse-client -u admin --password CHANGE_ME -q \"SELECT * FROM system.zookeeper WHERE path='/' FORMAT Vertical\""``` +- connection between ClickHouse pods via kubernetes services ```kubectl exec -n {{labels.namespace}} pod/{{labels.pod_name}} -- bash -c "clickhouse-client -u admin --password CHANGE_ME -q \"SELECT host_name, errors_count FROM system.clusters WHERE errors_count > 0 FORMAT PrettyCompactMonoBlock\""``` +- status of PersistentVolumeClaims for the Pod + +Also read documentation: +https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/#recovery-after-failures diff --git a/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-rejected-insert.md.hbs b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-rejected-insert.md.hbs new file mode 100644 index 0000000..48345fd --- /dev/null +++ b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-rejected-insert.md.hbs @@ -0,0 +1,5 @@ +ClickHouse has INSERT statements that are rejected due to high number of active data parts for partition in a MergeTree, +please decrease INSERT frequency: +- [MergeTreeArchitecture](https://clickhouse.com/docs/en/development/architecture/#merge-tree) +- [system.part_log](https://clickhouse.com/docs/en/operations/system-tables/part_log) +- [system.merge_tree_settings](https://clickhouse.com/docs/en/operations/system-tables/merge_tree_settings) \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-replicated-data-loss.md.hbs b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-replicated-data-loss.md.hbs new file mode 100644 index 0000000..158fcfb --- /dev/null +++ b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-replicated-data-loss.md.hbs @@ -0,0 +1,6 @@ +ClickHouse increase ReplicatedDataLoss in `system.events` table. +It mean data part that server wanted doesn't exist on any replica (even on replicas that are offline right now). +That data parts are definitely lost. This is normal due to asynchronous replication (if quorum inserts were not enabled), +when the replica on which the data part was written was failed and when it became online after fail it doesn't contain that data part. + +Please check [logs](/#/components/{{ componentUrnForUrl }}#logs) of the pod. \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-replicated-part-checks-failed.md.hbs b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-replicated-part-checks-failed.md.hbs new file mode 100644 index 0000000..80ecc2b --- /dev/null +++ b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-replicated-part-checks-failed.md.hbs @@ -0,0 +1 @@ +ClickHouse increase ReplicatedPartCheckFailed in `system.events` table. Please check [logs](/#/components/{{ componentUrnForUrl }}#logs) of the pod. \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-slow-read.md.hbs b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-slow-read.md.hbs new file mode 100644 index 0000000..1a39f83 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-slow-read.md.hbs @@ -0,0 +1,4 @@ +ClickHouse increase SlowRead in `system.events` table. +It mean reads from a files that were slow. This indicate system overload. Thresholds are controlled by `SELECT * FROM system.settings WHERE name LIKE 'read_backoff_%'`. +System will reduce the number of threads which used for processing queries. +Check you disks utilization and hardware failures. \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-too-many-connections.md.hbs b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-too-many-connections.md.hbs new file mode 100644 index 0000000..7399cda --- /dev/null +++ b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-too-many-connections.md.hbs @@ -0,0 +1,5 @@ +ClickHouse has too many open connections. +The ClickHouse is adapted to run not a very large number of parallel SQL requests, not every HTTP/TCP(Native)/MySQL protocol connection means a running SQL request, but a large number of open connections can cause a spike in sudden SQL requests, resulting in performance degradation. + +Also read documentation: +- https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings/#max-concurrent-queries diff --git a/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-too-many-running-queries.md.hbs b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-too-many-running-queries.md.hbs new file mode 100644 index 0000000..6218428 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-too-many-running-queries.md.hbs @@ -0,0 +1,9 @@ +Please analyze your workload. +Each concurrent SELECT statements use memory in JOINs use CPU for running aggregation function and can read lot of data from disk when scan parts in partitions and utilize disk I/O. +Each concurrent INSERT statements, allocate around 1MB per each column in an inserted table and utilize disk I/O. + +Look at following documentation parts: +- https://clickhouse.com/docs/en/operations/settings/query-complexity/ +- https://clickhouse.com/docs/en/operations/quotas/ +- https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings/#max-concurrent-queries +- https://clickhouse.com/docs/en/operations/system-tables/query_log/ \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-zookeeper-hardware-exceptions.md.hbs b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-zookeeper-hardware-exceptions.md.hbs new file mode 100644 index 0000000..08dc982 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-zookeeper-hardware-exceptions.md.hbs @@ -0,0 +1,2 @@ +ClickHouse has unexpected Network errors and similar with communication with Zookeeper. +ClickHouse should reinitialize ZooKeeper session in case of these errors. \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-zookeeper-session.md.hbs b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-zookeeper-session.md.hbs new file mode 100644 index 0000000..3d7c883 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/clickhouse/clickhouse-zookeeper-session.md.hbs @@ -0,0 +1,3 @@ +Number of sessions (connections) from ClickHouse to ZooKeeper shall be no more than one, +because using more than one connection to ZooKeeper may lead to bugs due to lack of linearizability (stale reads) +that ZooKeeper consistency model allows. \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/correlate/latency.md.hbs b/stackpacks/suse-observability/includes/services/correlate/latency.md.hbs new file mode 100644 index 0000000..18c2215 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/correlate/latency.md.hbs @@ -0,0 +1,6 @@ +The Correlator pod consumes observations made from our agent and transforms them into topology elements and telemetry. +When the latency of the transformation process is high we might have components and/or relations missing in the topology, and metrics +not present or arriving late. + +### Investigate +There are a few potential causes of the high latency. To identify the root cause, please contact SUSE Observability [support](https://www.stackstate.com/company/contact-us/). \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/hbase/slow-gets.md.hbs b/stackpacks/suse-observability/includes/services/hbase/slow-gets.md.hbs new file mode 100644 index 0000000..0373818 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/hbase/slow-gets.md.hbs @@ -0,0 +1,6 @@ +HBase is slow to response to get requests. To proceed take the following steps: + +### Remediation +1. Check the resource usage (CPU/Memory) of the pod slow to repsond to get requests. A common issue is memory pressure +2. Look in the logs for signs of memory pressure +3. If this does not resolve it, contact SUSE Observability support. \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/health-sync/latency.md.hbs b/stackpacks/suse-observability/includes/services/health-sync/latency.md.hbs new file mode 100644 index 0000000..df11f62 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/health-sync/latency.md.hbs @@ -0,0 +1,51 @@ +The Health Synchronization has high latency. + +The Health Synchronization consumes all results emmited by the monitors, persists them and links them to the corresponding topology elements. +When the latency of the process is high we might have components missing health states or with health states not shown timely in the components. + +### Investigate +Evaluate with the monitor metric chart if the deviation was caused by a burst of incoming data or if the load has consistently increased. If the high latency +is present regularly you can consider: +1. Add more resources to the Health Synchronization. + +### Add more resources to the Health Synchronization. +1. Inspect the [Configuration](/#/components/{{ componentUrnForUrl }}#configuration) of this pod and look for the amount of requested CPU and memory. +``` +resources: + limits: + cpu: 1000m + ephemeral-storage: 1Mi + memory: 3Gi + requests: + cpu: 750m + ephemeral-storage: 1Mi + memory: 2Gi +``` + +* Verify with the memory, JVM heap, JVM Gc metric charts if the pod needs more memory. + + +2. Update your helm `values.yaml` to add more cpu cores. +``` +stackstate: + components: + healthSync: + resources: + limits: + memory: "4500Mi" + cpu: "3000m" + requests: + memory: "4500Mi" + cpu: "2000m" +``` + + +3. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/kafka-to-es/latency.md.hbs b/stackpacks/suse-observability/includes/services/kafka-to-es/latency.md.hbs new file mode 100644 index 0000000..87609d0 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/kafka-to-es/latency.md.hbs @@ -0,0 +1,51 @@ +Data ingestion for `{{ labels.data_type }}` has high latency. + +The Kafka-to-es pod consumes `{{ labels.data_type }}`(s) sent by an agent and persists them, so we can visualize them on topology elements. +When the latency of the process is high, components might miss events (they will show up later). + +### Investigate +Evaluate with the monitor metric chart if the deviation was caused by a burst of incoming data or if the load has consistently increased. If the high latency +is present regularly you can consider: +1. Add more resources to the Kafka-to-es pod. + +### Add more resources to the Kafka-to-es pod. +1. Inspect the [Configuration](/#/components/{{ componentUrnForUrl }}#configuration) of this pod and look for the amount of requested CPU and memory. +``` +resources: + limits: + cpu: 1000m + ephemeral-storage: 1Mi + memory: 3Gi + requests: + cpu: 750m + ephemeral-storage: 1Mi + memory: 2Gi +``` + +* Verify with the memory, JVM heap, JVM Gc metric charts if the pod needs more memory. + + +2. Update your helm `values.yaml` to add more cpu cores. +``` +stackstate: + components: + e2es: + resources: + limits: + memory: "4500Mi" + cpu: "3000m" + requests: + memory: "4500Mi" + cpu: "2000m" +``` + + +3. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` diff --git a/stackpacks/suse-observability/includes/services/metrics-store-clients/metric-store-unreachable.md.hbs b/stackpacks/suse-observability/includes/services/metrics-store-clients/metric-store-unreachable.md.hbs new file mode 100644 index 0000000..e5530b2 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/metrics-store-clients/metric-store-unreachable.md.hbs @@ -0,0 +1 @@ +An unreachable metric store can be caused by either networking issues but more likely the metric store, Victoria Metrics, is down or restarting. If this happens outside of an upgrade continue troubleshooting on the Victoria Metrics pod(s), look for monitors in critical or deviating state or for errors in the log. \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/otel-collector/otel-collector-metric-export-failure.md.hbs b/stackpacks/suse-observability/includes/services/otel-collector/otel-collector-metric-export-failure.md.hbs new file mode 100644 index 0000000..e464b31 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/otel-collector/otel-collector-metric-export-failure.md.hbs @@ -0,0 +1,17 @@ +`stackstate_otelcol_exporter_send_failed_metric_points` metric in your OpenTelemetry collector indicates that metric points +are failing to be sent to the receiving endpoint (e.g., Prometheus server, Victoria Metrics). + +Potential Reasons: +- Network Issues: + - **Connectivity**: Verify the collector's ability to connect to the receiving endpoint. Use tools like ping or network diagnostics to confirm connectivity. + - **Firewall Rules**: Ensure firewall rules don't block communication between the collector and the endpoint. Adjust rules as necessary. + - **DNS Resolution**: Check if the collector can resolve the hostname of the endpoint correctly. +- Receiving Endpoint Overload: + - **Monitor Resources**: Track resource usage (CPU, memory) of the receiving endpoint (open Metrics of the receiver component(s)). If overloaded, consider scaling it up. + - **Endpoint Configuration**: Ensure the endpoint is configured to accept metrics from the collector correctly. Refer to the endpoint's documentation for specific configuration details. +- Collector Configuration: + - **Exporter Configuration**: Double-check your exporter configuration for typos, incorrect endpoint URLs, or invalid authentication credentials. + - **Batch Size and Interval**: Adjust batch size and sending interval settings in your exporter configuration to manage data flow better. Smaller batches and shorter intervals might help with overloaded endpoints. + - **Queue Management**: Monitor metrics like `stackstate_otelcol_exporter_queue_size` and `stackstate_otelcol_exporter_queue_capacity`. If the queue is often full, consider increasing its capacity or adjusting batch sizes/intervals. + +You may find more details by checking [logs](/#/components/{{ componentUrnForUrl }}#logs). \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/otel-collector/otel-collector-metric-processor-drops.md.hbs b/stackpacks/suse-observability/includes/services/otel-collector/otel-collector-metric-processor-drops.md.hbs new file mode 100644 index 0000000..12e7a34 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/otel-collector/otel-collector-metric-processor-drops.md.hbs @@ -0,0 +1,18 @@ +`stackstate_otelcol_processor_dropped_metric_points` metric in your OpenTelemetry collector indicates that metric points +are being dropped by processors within your OpenTelemetry collector before they can be exported. + +Potential Reasons: +- Processor Configuration: + - **Incorrect configuration**: Double-check your processor configuration for errors like typos, invalid parameters, or incorrect filtering rules. + - **Disabled processor**: Ensure the relevant processor is enabled and not accidentally commented out. + - **Memory limits**: Some processors have memory limits for storing processed data. If exceeded, metric points might be dropped. +- Resource Limitations: + - **Processor overload**: If the processor is overloaded with processing tasks, it might drop metric points to avoid resource exhaustion. Monitor CPU and memory usage of the collector by checking `Metrics` tab. + - **Queue overflows**: Check related metrics like `stackstate_otelcol_processor_queue_size` and `stackstate_otelcol_processor_queue_capacity`. If the queue is constantly full, metric points might be dropped due to lack of space. +- Metric Point Data Issues: + - **Invalid format**: Ensure the format of your metric points (e.g., Prometheus format) adheres to the processor's requirements. + - **Data type mismatches**: Check if the data types of your metric points (e.g., gauge, counter) match what the processor expects. +- Other Potential Causes: + - **Bugs in the processor**: While less likely, consider checking for known issues or bugs in the specific processor you're using. + +You may find more details by checking [logs](/#/components/{{ componentUrnForUrl }}#logs). \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/otel-collector/otel-collector-metric-refused-receiver.md.hbs b/stackpacks/suse-observability/includes/services/otel-collector/otel-collector-metric-refused-receiver.md.hbs new file mode 100644 index 0000000..bc00015 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/otel-collector/otel-collector-metric-refused-receiver.md.hbs @@ -0,0 +1,19 @@ +`stackstate_otelcol_receiver_refused_metric_points` metric in your OpenTelemetry collector indicates that metric points are being rejected by the +collector. This could be happening for several reasons: + +Common Reasons: +- Receiver configuration: + - **Disabled receiver**: Ensure the receiver you're trying to send metrics to (e.g., Prometheus exporter) is actually enabled and configured correctly in your collector config. + - **Invalid configuration**: Double-check your receiver configuration for typos, incorrect values, or any other errors that might prevent it from accepting metrics. + - **Metric name conflicts**: If you're using the Prometheus exporter, verify that metric names don't clash with existing metrics collected by Prometheus itself. +- Metric data format: + - **Invalid or unsupported format**: Prometheus expects metrics in a specific format. Ensure your metric points adhere to this format (e.g., timestamp, name, value, labels). + - **Data type mismatches**: Check if the data types of your metric values (e.g., gauge, counter) match what the receiver expects. +- Resource limitations: + - **Processor or memory overload**: If the collector is overloaded, it might refuse metrics to avoid further resource exhaustion. Monitor your collector's resource usage by checking `Metrics` tab and consider scaling if necessary + - **Queue overflows**: Check related metrics like `stackstate_otelcol_exporter_queue_size` and `stackstate_otelcol_exporter_queue_capacity`. If the queue is consistently full, you might need to adjust batch sizes, sending intervals, or increase queue capacity. +- Network issues: + - **Connectivity problems**: Ensure the collector can establish and maintain a connection to the receiving endpoint (e.g., backend). Connectivity issues might prevent metrics delivery. + - **Firewall restrictions**: Check for any firewall rules that might block communication between the collector and the receiving endpoint. + +You may find more details by checking [logs](/#/components/{{ componentUrnForUrl }}#logs). \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/otel-collector/otel-collector-span-export-failure.md.hbs b/stackpacks/suse-observability/includes/services/otel-collector/otel-collector-span-export-failure.md.hbs new file mode 100644 index 0000000..884f881 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/otel-collector/otel-collector-span-export-failure.md.hbs @@ -0,0 +1,17 @@ +`stackstate_otelcol_exporter_send_failed_spans` metric in your OpenTelemetry collector indicates that spans are failing +to be sent to the receiving backend. + +Potential Reasons: +- Network Issues: + - **Connectivity**: Verify that the collector can connect to the receiving backend/database. You can use tools like ping or network diagnostics to check connectivity. + - **Firewall Rules**: Ensure firewall rules do not block communication between the collector and the backend. Adjust rules as needed. + - **DNS Resolution**: Check if the collector can resolve the hostname of the backend correctly. +- Backend Overload: + - **Monitor Backend Resources**: Check the resource usage (CPU, memory) of the receiving backend (open Metrics of the receiver component(s)). If overloaded, consider scaling it up. + - **Backend Configuration**: Ensure the backend is configured to accept spans from the collector correctly. Refer to the backend's documentation for specific configuration details. +- Collector Configuration: + - **Exporter Configuration**: Double-check your exporter configuration for typos, incorrect endpoint URLs, or invalid authentication credentials. + - **Batch Size and Interval**: Adjust batch size and sending interval settings in your exporter configuration to manage data flow better. Smaller batches and shorter intervals might help with overloaded backends. + - **Queue Management**: Monitor metrics like `stackstate_otelcol_exporter_queue_size` and `stackstate_otelcol_exporter_queue_capacity`. If the queue is often full, consider increasing its capacity or adjusting batch sizes/intervals. + +You may find more details by checking [logs](/#/components/{{ componentUrnForUrl }}#logs). \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/otel-collector/otel-collector-span-processor-drops.md.hbs b/stackpacks/suse-observability/includes/services/otel-collector/otel-collector-span-processor-drops.md.hbs new file mode 100644 index 0000000..86cac12 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/otel-collector/otel-collector-span-processor-drops.md.hbs @@ -0,0 +1,18 @@ +`stackstate_otelcol_processor_dropped_spans` metric in your OpenTelemetry collector indicates that spans are being dropped +by processors within your OpenTelemetry collector before they can be exported. + +Potential Reasons: +- Processor Configuration: + - **Incorrect configuration**: Double-check your processor configuration for errors like typos, invalid parameters, or incorrect sampling rules. + - **Disabled processor**: Ensure the relevant processor is enabled and not accidentally commented out. + - **Memory limits**: Some processors have memory limits for storing processed data. If exceeded, spans might be dropped. +- Resource Limitations: + - **Processor overload**: If the processor is overloaded with processing tasks, it might drop spans to avoid resource exhaustion. Monitor CPU and memory usage of the collector by checking `Metrics` tab. + - **Queue overflows**: Check related metrics like `stackstate_otelcol_processor_queue_size` and `stackstate_otelcol_processor_queue_capacity`. If the queue is constantly full, spans might be dropped due to lack of space. +- Span Data Issues: + - **Invalid format**: Ensure the format of your spans (e.g., Protobuf) adheres to the processor's requirements. + - **Size limits**: Some processors have size limits for individual spans or batches. If your spans exceed those limits, they might be dropped. +- Other Potential Causes: + - **Bugs in the processor**: While less likely, consider checking for known issues or bugs in the specific processor you're using. + +You may find more details by checking [logs](/#/components/{{ componentUrnForUrl }}#logs). \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/otel-collector/otel-collector-span-refused-receiver.md.hbs b/stackpacks/suse-observability/includes/services/otel-collector/otel-collector-span-refused-receiver.md.hbs new file mode 100644 index 0000000..b4e3ba9 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/otel-collector/otel-collector-span-refused-receiver.md.hbs @@ -0,0 +1,18 @@ +`stackstate_otelcol_receiver_refused_spans` metric in your OpenTelemetry collector indicates that spans are being rejected by the +collector. This could be happening for several reasons: + +Common Reasons: +- Receiver configuration: + - **Disabled receiver**: Ensure the receiver you're trying to send spans to (e.g., OTLP) is actually enabled and configured correctly in your collector config. + - **Invalid configuration**: Double-check your receiver configuration for typos, incorrect values, or any other errors that might prevent it from accepting spans. +- Span data format: + - **Invalid or unsupported format**: Verify that the format of your spans (e.g., Protobuf) is valid and supported by the chosen receiver. + - **Size limits**: Some receivers might have size limits for individual spans or batches. Ensure your spans are within those limits. +- Resource limitations: + - **Processor or memory overload**: If the collector is overloaded, it might refuse spans to avoid further resource exhaustion. Monitor your collector's resource usage by checking `Metrics` tab and consider scaling if necessary. + - **Queue overflows**: Check related metrics like `stackstate_otelcol_exporter_queue_size` and `stackstate_otelcol_exporter_queue_capacity`. If the queue is consistently full, you might need to adjust batch sizes, sending intervals, or increase queue capacity. +- Network issues: + - **Connectivity problems**: Ensure the collector can establish and maintain a connection to the receiving endpoint (e.g., backend). Connectivity issues might prevent span delivery. + - **Firewall restrictions**: Check for any firewall rules that might block communication between the collector and the receiving endpoint. + +You may find more details by checking [logs](/#/components/{{ componentUrnForUrl }}#logs). \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/receiver/budget-saturation.md.hbs b/stackpacks/suse-observability/includes/services/receiver/budget-saturation.md.hbs new file mode 100644 index 0000000..d3ba362 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/receiver/budget-saturation.md.hbs @@ -0,0 +1,35 @@ +Element budget **nearly saturated for `{{ labels.element_type }}`**, has high unique element budget saturation level. When the budget is saturated data starts being dropped which translates in missing topology or metrics. + +### Increase the budget of elements allowed +1. We can control the budget by setting extra environment variables for the receiver pod. +Here are the default settings: +``` + receiver: + extraEnv: + open: + CONFIG_FORCE_stackstate_receiver_processAgent_maxAgents: "2000" + CONFIG_FORCE_stackstate_receiver_processAgent_maxComponents: "5000" + CONFIG_FORCE_stackstate_receiver_processAgent_maxConnections: "150000" + CONFIG_FORCE_stackstate_receiver_processAgent_maxRelations: "15000" +``` + +2. Identify which budget got saturated and increment the right environment variable to allow more elements to be passed: + +| element type | environment variable | +|--------------|--------------------------------------------------------------| +| agent | CONFIG_FORCE_stackstate_receiver_processAgent_maxAgents | +| component | CONFIG_FORCE_stackstate_receiver_processAgent_maxComponents | +| connection | CONFIG_FORCE_stackstate_receiver_processAgent_maxConnections | +| relation | CONFIG_FORCE_stackstate_receiver_processAgent_maxRelations | + +Keep in mind that allowing more elements to be processed will have an impact on downstream services such as the Correlator and the Topology Sync. + +3. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/receiver/hourly-budget-saturation.md.hbs b/stackpacks/suse-observability/includes/services/receiver/hourly-budget-saturation.md.hbs new file mode 100644 index 0000000..d38f1a4 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/receiver/hourly-budget-saturation.md.hbs @@ -0,0 +1,31 @@ +Element budget **nearly saturated for `{{ labels.element_type }}`**, has high element hourly budget saturation level. When the budget is saturated data starts being dropped which translates in missing topology or metrics. + +### Increase the budget of elements allowed +1. We can control the budget by setting extra environment variables for the receiver pod. +Here are the default settings: +``` + receiver: + extraEnv: + open: + CONFIG_FORCE_stackstate_receiver_processAgent_maxNewComponentsHourly: "2000" + CONFIG_FORCE_stackstate_receiver_processAgent_maxNewRelationsHourly: "5000" +``` + +2. Identify which budget got saturated and increment the right environment variable to allow more elements to be passed: + +| element type | environment variable | +|--------------|--------------------------------------------------------------| +| component | CONFIG_FORCE_stackstate_receiver_processAgent_maxNewComponentsHourly | +| relation | CONFIG_FORCE_stackstate_receiver_processAgent_maxNewRelationsHourly | + +Keep in mind that allowing more elements to be processed will have an impact on downstream services such as the Correlator and the Topology Sync. + +3. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/receiver/http-growing-requests.md.hbs b/stackpacks/suse-observability/includes/services/receiver/http-growing-requests.md.hbs new file mode 100644 index 0000000..20ce4f5 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/receiver/http-growing-requests.md.hbs @@ -0,0 +1,11 @@ +The # of active requests for SUSE Observability receiver has only been growing for the last 5 minutes. +This either means that there is a huge and sudden increase in load but much more likely it means that the receiver is “stuck” and not able to write data to Kafka, eventually the requests will all timeout and will be retried later. The end result will be that data doesn't receive any more data and metrics, health states etc. go stale, components disappear. + +### Troubleshoot steps +1. Go to the [metrics](/#/components/{{ componentUrnForUrl }}/metrics +2. Verify the `Active HTTP Requests` chart, normal behavior is that it fluctuates but is pretty much at a constant level even quite low at about 0 to 5 active requests. If this alert triggers it is expected to be a continuously increasing value. +3. After confirming the increasing value restart the receiver pod using +``` +kubectl delete pod {{ labels.pod_name }} -n {{ labels.namespace }} +``` +4. After restart check the same charts again for the next 10 minutes to make sure issue is fixed. \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/receiver/memory-limiter.md.hbs b/stackpacks/suse-observability/includes/services/receiver/memory-limiter.md.hbs new file mode 100644 index 0000000..e3bf7e1 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/receiver/memory-limiter.md.hbs @@ -0,0 +1,37 @@ +SUSE Observability requests are rejected is the memory budget is exhausted and that can cause data loss. + +### Increase the memory +1. Check if this monitor triggers for a long time, in that case add more memory to the pod +2. Inspect the [Configuration](/#/components/{{ componentUrnForUrl }}#configuration) of this pod and look for the amount of memory. +``` +resources: + limits: + cpu: 1000m + ephemeral-storage: 1Mi + memory: 3Gi + requests: + cpu: 750m + ephemeral-storage: 1Mi + memory: 2Gi +``` +2. Update your helm `values.yaml` to add more memory. +``` + stackstate: + components: + receiver: + resources: + limits: + memory: "4500Mi" + requests: + memory: "4500Mi" +``` + +3. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/receiver/rejected-logs.md.hbs b/stackpacks/suse-observability/includes/services/receiver/rejected-logs.md.hbs new file mode 100644 index 0000000..801e7ab --- /dev/null +++ b/stackpacks/suse-observability/includes/services/receiver/rejected-logs.md.hbs @@ -0,0 +1,50 @@ +SUSE Observability logs are rejected when one or more log sources (for example pods) are generating more logs than their budget based on the log retention and disk space available. Solutions: + +### Troubleshoot steps +- The cause can be an increase in log volume for 1 or more sources, for example because debug logging was enabled for an application. The solution in that case is to reduce the log volume for these pods. +- If the log volume is legitimate and not expected to change either the available disk space can be increased or the retention can be decreased. + +### Increase Disk Space for ElasticSearch +1. Verify the current ElasticSearch disk space and inspect its configuration to check the current size. + * Verify that the used storage class can be resized. For that please follow our [resizing documentation] https://docs.stackstate.com/self-hosted-setup/data-management/data_retention#resizing-storage +2. Update the values.yaml adding more +``` +elasticsearch: + volumeClaimTemplate: + resources: + requests: + storage: 250Gi +``` + +3. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` + +For more detailed information visit our [data-retention](https://docs.stackstate.com/self-hosted-setup/data-management/data_retention) documentation. + +### Decrease retention for logs +1. By default, SUSE Observability sets a 7-day retention for logs +2. Update the values.yaml with a lower retention +``` +stackstate: + components: + receiver: + # Number of days to keep the logs data on Es + retention: 7 +``` + +3. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/topo-sync/blocked.md.hbs b/stackpacks/suse-observability/includes/services/topo-sync/blocked.md.hbs new file mode 100644 index 0000000..8675478 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/topo-sync/blocked.md.hbs @@ -0,0 +1,10 @@ +The Topology Synchronization pod uses several workers and one of them if blocked and can't resume work. + +### Remediation +1. Restart the pod so the synchronization of topology data resumes. +``` +kubectl delete pod {{ labels.pod_name }} -n {{ labels.namespace }} +``` +`Restarting the pod can help by either unblocking the issue or making it more evident in the logs` + +2. Inspect the [Logs](/#/components/{{ componentUrnForUrl }}#logs) of the pod in order to find the root cause of the blockage. \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/topo-sync/internal-latency.md.hbs b/stackpacks/suse-observability/includes/services/topo-sync/internal-latency.md.hbs new file mode 100644 index 0000000..bdbaff8 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/topo-sync/internal-latency.md.hbs @@ -0,0 +1,56 @@ +The Topology Synchronization for `{{ labels.integration_type }}` has high latency. + +The Topology Synchronization pod consumes data sent by an agent and transforms them into topology elements. +When the latency of the transformation process is high we might have components and/or relations missing in the topology. + +### Investigate +Evaluate with the monitor metric chart if the deviation was caused by a burst of incoming data or if the load has consistently increased. If the high latency +is present regularly you can consider: +* Add more resources to the Topology Synchronization. +* Reducing the frequency of topology updates for {{ labels.integration_type }} + +### Add more resources to the Topology Synchronization. +1. Inspect the [Configuration](/#/components/{{ componentUrnForUrl }}#configuration) of this pod and look for the amount of requested CPU and memory. More CPU's translate into more +parallel work achieved by the pod. +``` +resources: + limits: + cpu: 1000m + ephemeral-storage: 1Mi + memory: 3Gi + requests: + cpu: 750m + ephemeral-storage: 1Mi + memory: 2Gi +``` + +* Verify with the memory, JVM heap, JVM Gc metric charts if the pod needs more memory. + + +2. Update your helm `values.yaml` to add more cpu cores. +``` +stackstate: + components: + sync: + resources: + limits: + memory: "4500Mi" + cpu: "3000m" + requests: + memory: "4500Mi" + cpu: "2000m" +``` + +3. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` + +### Reduce the frequency of the topology updates. +Update the settings of the agent collection of data for integration: `{{ labels.integration_type }}`, +Reducing the frequency of updates will remove pressure on the Topology Synchronization but as well means that the topology data will be less up-to-date as we don't refresh it that often. \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/topo-sync/latency.md.hbs b/stackpacks/suse-observability/includes/services/topo-sync/latency.md.hbs new file mode 100644 index 0000000..bdbaff8 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/topo-sync/latency.md.hbs @@ -0,0 +1,56 @@ +The Topology Synchronization for `{{ labels.integration_type }}` has high latency. + +The Topology Synchronization pod consumes data sent by an agent and transforms them into topology elements. +When the latency of the transformation process is high we might have components and/or relations missing in the topology. + +### Investigate +Evaluate with the monitor metric chart if the deviation was caused by a burst of incoming data or if the load has consistently increased. If the high latency +is present regularly you can consider: +* Add more resources to the Topology Synchronization. +* Reducing the frequency of topology updates for {{ labels.integration_type }} + +### Add more resources to the Topology Synchronization. +1. Inspect the [Configuration](/#/components/{{ componentUrnForUrl }}#configuration) of this pod and look for the amount of requested CPU and memory. More CPU's translate into more +parallel work achieved by the pod. +``` +resources: + limits: + cpu: 1000m + ephemeral-storage: 1Mi + memory: 3Gi + requests: + cpu: 750m + ephemeral-storage: 1Mi + memory: 2Gi +``` + +* Verify with the memory, JVM heap, JVM Gc metric charts if the pod needs more memory. + + +2. Update your helm `values.yaml` to add more cpu cores. +``` +stackstate: + components: + sync: + resources: + limits: + memory: "4500Mi" + cpu: "3000m" + requests: + memory: "4500Mi" + cpu: "2000m" +``` + +3. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` + +### Reduce the frequency of the topology updates. +Update the settings of the agent collection of data for integration: `{{ labels.integration_type }}`, +Reducing the frequency of updates will remove pressure on the Topology Synchronization but as well means that the topology data will be less up-to-date as we don't refresh it that often. \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/topo-sync/multiple-hosts.md.hbs b/stackpacks/suse-observability/includes/services/topo-sync/multiple-hosts.md.hbs new file mode 100644 index 0000000..96f264d --- /dev/null +++ b/stackpacks/suse-observability/includes/services/topo-sync/multiple-hosts.md.hbs @@ -0,0 +1,6 @@ +The Topology Synchronization gets data from the agent. Snapshots of data should come from a single agent. If multiple agents +send snapshot data to the same synchronization, this is regarded a violation, because there are multiple sources of truth. + +### Remediation +1. Investigate the error in details through the stackstate cli (`sts-cli topology-sync`) +2. Remediate by disabling the agent sending duplicate data. \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/topo-sync/snapshot-already-open.md.hbs b/stackpacks/suse-observability/includes/services/topo-sync/snapshot-already-open.md.hbs new file mode 100644 index 0000000..99bb290 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/topo-sync/snapshot-already-open.md.hbs @@ -0,0 +1,6 @@ +The Topology Synchronization snapshot model requires that snapshots are properly closed in order to perform the cleanup of obsolete topology items. +Observing subsequent starts of a snapshot is regarded as an error as it will lead to inconsistent topology. + +### Remediation +1. Investigate the error in details through the stackstate cli (`sts-cli topology-sync`) +2. Remediate by inspecting the agent or integration and finding the cause of the truncated snapshots. \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/victoria-metrics/errors.md.hbs b/stackpacks/suse-observability/includes/services/victoria-metrics/errors.md.hbs new file mode 100644 index 0000000..404ae1f --- /dev/null +++ b/stackpacks/suse-observability/includes/services/victoria-metrics/errors.md.hbs @@ -0,0 +1 @@ +Requests to path victoria metrics are receiving errors. Please verify if clients are sending correct requests by inspecting the [logs](/#/components/{{ componentUrnForUrl }}#logs). diff --git a/stackpacks/suse-observability/includes/services/victoria-metrics/freshness.md.hbs b/stackpacks/suse-observability/includes/services/victoria-metrics/freshness.md.hbs new file mode 100644 index 0000000..3db970d --- /dev/null +++ b/stackpacks/suse-observability/includes/services/victoria-metrics/freshness.md.hbs @@ -0,0 +1,10 @@ +Metrics are taking longer than the freshness objective to become queryable after they are scraped, so dashboards and monitors are evaluating late data. + +The value is the smallest sample age seen over the last minute, which removes the normal rise between scrapes and leaves the delay through the pipeline. Check in order: + +- is vmagent holding data back? see the `vmagent - Write queue` panel +- is the collector queue backing up? see the metrics export queue occupancy panel +- is the receiver rejecting? see `Receiver - Memory limiter saturation` +- is VictoriaMetrics slow to persist? see `VictoriaMetrics - Pending rows` and `Insert concurrency saturated` + +If scraping has stopped entirely this monitor fires only after about 90 seconds, and it goes quiet once the series expire. `Receiver - No data received` covers a full stop. diff --git a/stackpacks/suse-observability/includes/services/victoria-metrics/insert-concurrency.md.hbs b/stackpacks/suse-observability/includes/services/victoria-metrics/insert-concurrency.md.hbs new file mode 100644 index 0000000..abc0865 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/victoria-metrics/insert-concurrency.md.hbs @@ -0,0 +1,5 @@ +Concurrent inserts are at the limit VictoriaMetrics derives from the CPU it detects, so new writes queue behind in-flight ones. Sustained saturation shows up downstream as vmagent queueing and then as lost data. + +Check `stackstate_vm_concurrent_insert_limit_reached_total` to confirm writes are actually being held, and the `CPU Usage VictoriaMetrics` and `CPU Pressure` panels to see whether the pod is CPU starved. + +The limit scales with available CPU, so raising the CPU request or limit raises it. Reducing ingestion rate or series churn lowers demand for it. diff --git a/stackpacks/suse-observability/includes/services/victoria-metrics/labels-limit.md.hbs b/stackpacks/suse-observability/includes/services/victoria-metrics/labels-limit.md.hbs new file mode 100644 index 0000000..ba0936a --- /dev/null +++ b/stackpacks/suse-observability/includes/services/victoria-metrics/labels-limit.md.hbs @@ -0,0 +1,38 @@ +VictoriaMetrics limits the number of labels per each metric with '-maxLabelsPerTimeseries' command-line flag.\n +This prevents from ingesting metrics with too many labels. Please verify that '-maxLabelsPerTimeseries' is configured +correctly or that clients which send these metrics aren't misbehaving. + +### Troubleshooting steps +1. Inspect the [Configuration](/#/components/{{ componentUrnForUrl }}#configuration) of this pod and look for the `-maxLabelsPerTimeseries` argument. +``` +spec: + automountServiceAccountToken: true + containers: + - args: + - --dedup.minScrapeInterval=1ms + - --envflag.enable=true + - --envflag.prefix=VM_ + - --loggerFormat=json + - --maxLabelsPerTimeseries=35 + - --retentionPeriod=15d + - --storageDataPath=/storage +``` + +Please verify that '-maxLabelsPerTimeseries' is configured correctly or that clients which send these metrics aren't misbehaving. + +2. Update your helm `values.yaml` to allow more labels just in case you really need them and can't remove any. +``` + victoria-metrics-0: + extraArgs: + maxLabelsPerTimeseries: 35 +``` + +3. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/victoria-metrics/rejected.md.hbs b/stackpacks/suse-observability/includes/services/victoria-metrics/rejected.md.hbs new file mode 100644 index 0000000..b3c8ef1 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/victoria-metrics/rejected.md.hbs @@ -0,0 +1 @@ +VM is rejecting to ingest rows due to the following reason: {{ labels.reason }} diff --git a/stackpacks/suse-observability/includes/services/victoria-metrics/select-concurrency.md.hbs b/stackpacks/suse-observability/includes/services/victoria-metrics/select-concurrency.md.hbs new file mode 100644 index 0000000..321f55f --- /dev/null +++ b/stackpacks/suse-observability/includes/services/victoria-metrics/select-concurrency.md.hbs @@ -0,0 +1,5 @@ +Concurrent queries are at the limit VictoriaMetrics derives from the CPU it detects. Further queries wait, and can time out. + +Check `stackstate_vm_concurrent_select_limit_reached_total` to confirm queries are being held, and `stackstate_vm_slow_queries_total` for expensive queries driving it. + +Usual causes are dashboards or monitors querying long ranges over many series. Narrow the heaviest queries, or raise the CPU allocation - the limit scales with detected CPU. diff --git a/stackpacks/suse-observability/includes/services/victoria-metrics/slow-insert-rate.md.hbs b/stackpacks/suse-observability/includes/services/victoria-metrics/slow-insert-rate.md.hbs new file mode 100644 index 0000000..95d037d --- /dev/null +++ b/stackpacks/suse-observability/includes/services/victoria-metrics/slow-insert-rate.md.hbs @@ -0,0 +1,34 @@ +High rate of slow inserts may be a sign of resource exhaustion for the current load. It is likely more RAM is needed for optimal handling of the current number of active time series. + +### Troubleshooting steps +1. Inspect the [Configuration](/#/components/{{ componentUrnForUrl }}#configuration) of this pod and look for the amount of memory. +``` +resources: + limits: + cpu: 1000m + memory: 3Gi + requests: + cpu: 750m + memory: 2Gi +``` +Please verify that `-maxLabelsPerTimeseries` is configured correctly or that clients which send these metrics aren't misbehaving. + +2. Update your helm `values.yaml` to add more memory +``` + victoria-metrics-0: + resources: + limits: + memory: "4500Mi" + requests: + memory: "4500Mi" +``` + +3. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/victoria-metrics/storage-read-only.md.hbs b/stackpacks/suse-observability/includes/services/victoria-metrics/storage-read-only.md.hbs new file mode 100644 index 0000000..25ff973 --- /dev/null +++ b/stackpacks/suse-observability/includes/services/victoria-metrics/storage-read-only.md.hbs @@ -0,0 +1,8 @@ +VictoriaMetrics has switched to read-only mode and is no longer accepting writes. Metrics sent while this lasts are lost once the upstream queues fill. + +This is almost always free disk space falling below `-storage.minFreeDiskSpaceBytes`. Check: + +- the `Volume usage` panel on the *Metrics processing* dashboard, and the `Disk runs out of space` monitors +- `stackstate_vm_free_disk_space_bytes` against `stackstate_vm_free_disk_space_limit_bytes` for this pod + +To recover, free space or grow the volume. Reducing retention releases space only after the next merge, so it is not an immediate fix. diff --git a/stackpacks/suse-observability/includes/services/vmagent/persistentqueue-dropping-data.md.hbs b/stackpacks/suse-observability/includes/services/vmagent/persistentqueue-dropping-data.md.hbs new file mode 100644 index 0000000..eb6356a --- /dev/null +++ b/stackpacks/suse-observability/includes/services/vmagent/persistentqueue-dropping-data.md.hbs @@ -0,0 +1,29 @@ +Vmagent dropped data from persistent queue. Vmagent receives data and tries to push it to victoria metrics, the persistent queue helps as a buffer while writting to victoria metrics. + +### Troubleshooting steps +1. Inspect the [Logs](/#/components/{{ componentUrnForUrl }}#logs) to verify if there's in issue while writing to victoria metrics. + 1.1 Continue the troubleshooting process if you find errors in the logs by inspecting victoria metrics telemetry. +2. If there are no issues regarding writing the problem could be related to a producer faster than what we can write to victoria metrics. + 2a. Consider giving Victoria metrics more resources in the values.yaml. You can check the current settings on the victoria metrics pod's configuration. + ``` + victoria-metrics-0: + resources: + requests: + cpu: 300m + memory: 3584Mi + limits: + memory: 4Gi + cpu: 1 + ``` + + Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` + + 2b. Consider to limit the ingestion rate. \ No newline at end of file diff --git a/stackpacks/suse-observability/includes/services/vmagent/remotewrite-samples-dropped.md.hbs b/stackpacks/suse-observability/includes/services/vmagent/remotewrite-samples-dropped.md.hbs new file mode 100644 index 0000000..c7f548f --- /dev/null +++ b/stackpacks/suse-observability/includes/services/vmagent/remotewrite-samples-dropped.md.hbs @@ -0,0 +1,9 @@ +vmagent is discarding samples rather than delivering them. This is unrecoverable data loss: unlike a full queue, dropped samples are never retried. + +Check: + +- `stackstate_vmagent_remotewrite_pending_data_bytes` and the persistent queue - drops usually follow a queue that has been full for some time +- whether VictoriaMetrics is rejecting writes, via the `Rows rejected` and `Storage is read only` monitors +- `stackstate_vmagent_remotewrite_rate_limit_reached_total` if a remote-write rate limit is configured + +Fix the downstream cause rather than growing the queue; a larger queue only delays the same loss. diff --git a/stackpacks/suse-observability/resources/RELEASE.md b/stackpacks/suse-observability/resources/RELEASE.md new file mode 100644 index 0000000..73bd0eb --- /dev/null +++ b/stackpacks/suse-observability/resources/RELEASE.md @@ -0,0 +1,6 @@ +# SUSE Observability v2 + +## 0.0.1 + +**Features** +First pre-release version \ No newline at end of file diff --git a/stackpacks/suse-observability/resources/configuration.md b/stackpacks/suse-observability/resources/configuration.md new file mode 100644 index 0000000..0bf48e1 --- /dev/null +++ b/stackpacks/suse-observability/resources/configuration.md @@ -0,0 +1,3 @@ +## Installation + +You can click on `Install` for installing the SUSE Observability StackPack. The StackPack only supports the Kubernetes installation of SUSE Observability for adding metrics and monitors. Metrics and monitors will be automatically added to the SUSE Observability related Kubernetes components. diff --git a/stackpacks/suse-observability/resources/deprovisioning.md b/stackpacks/suse-observability/resources/deprovisioning.md new file mode 100644 index 0000000..e69de29 diff --git a/stackpacks/suse-observability/resources/detailed-overview.md b/stackpacks/suse-observability/resources/detailed-overview.md new file mode 100644 index 0000000..52c405a --- /dev/null +++ b/stackpacks/suse-observability/resources/detailed-overview.md @@ -0,0 +1,4 @@ +## Prerequisites + +* The [Kubernetes StackPack](#/stackpacks/kubernetes-v2) is installed for the cluster where SUSE Observability is installed +* The suse-observability-agent must be installed on the Observability platform cluster. Check the [matching instructions](#/stackpacks/kubernetes-v2) for your cluster. diff --git a/stackpacks/suse-observability/resources/enabled.md b/stackpacks/suse-observability/resources/enabled.md new file mode 100644 index 0000000..6279a33 --- /dev/null +++ b/stackpacks/suse-observability/resources/enabled.md @@ -0,0 +1,11 @@ +## The SUSE Observability StackPack is installed + +Congratulations! The SUSE Observability StackPack is configured correctly. + +### What's next + +The topology that is created by the Kubernetes StackPack will be enriched with specific SUSE Observability monitors and metric bindings. See the SUSE Observability view that has been created to discover SUSE Observability's own topology. + +### No data + +Make sure the suse-observability-agent is installed on the Observability platform cluster. Check the [matching instructions](#/stackpacks/kubernetes-v2) for your cluster in the Kubernetes stackpack. diff --git a/stackpacks/suse-observability/resources/error.md b/stackpacks/suse-observability/resources/error.md new file mode 100644 index 0000000..e69de29 diff --git a/stackpacks/suse-observability/resources/logo.png b/stackpacks/suse-observability/resources/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..5fa03e23fc656ca39f581339c93c0cb30e22afdf GIT binary patch literal 17408 zcma%B1y@^5*G)ojcXxM(;!>=*LxC2T7I%jPcPZN9@Zc184HS2GiWO^dcfP#;;9HTK zOm1e~HTRso&)IvTHPsa{(a6vM008C(B{^*X04NRrT!VrHe~-IC{s2FKZDrJC0D!tg z^k;KK_;(shC2chTz>f(4fP?`6kMJSLApqdX0|5Lr0{}!b0RUo`>~<}2_zy@HDvEM| zxBorzyGm2wBd9J)AKd`}4we7CfW;E&A_ z)sr0WY)a22SdZ;#$*EdVxVS zNJ&jr+SlXT2IdMNBrFN6jq=~IQ~k<@*ujfyP=nar=}E^($#eHvgQVNp`oqMt&Hng9 z4x0nryjmWJ>}BjFa#+($%*?=yJGBevR(b_-8AyW2fE$sI0l-7a4@04pCZCR}M>B-h zf(_&9am;aRNem&AXM|YD8-N?gjZlLyruixI zI!C%HOLC7MMkx(SxvT+tgIYjpr>%Yx2}8TAiScZJeMx6s z3a7`^6i5dY$T(<+MD+{2J>FV-6EeMfBqkIEI3@~!)FQZmT(C+`F#nyUoW9owO_?Xe zA6fMwGK5rfyNBy-G_s~ZgK4bS_|_n$yfXJVqi!|0e&T_Od1x?-cxQo$-7-xvDNSzSQ9GUzJAAEh!xe^NG^+cZ5z87y0CQJ<`F&T^LBXf9h9U+Me@k~^O z*8a;|1K9#hVPI3uJttvRuk(@65VnUtLD_jGoRKuwBA-kXR%mKuJgMkDqxgwiYo`I? zz65^Q!iNb%10SEPgj`=k2Oa~Vm{;F?2=8gnz4PbLW#GX^h%T-2Q3po^&`Qv7ccT|E zn68f05Pds%T}#q{z%?+A;SWHz_yV1PVMFX8SlnQV+?~`Hnnj6AN|g7t%UwDUY6Q#2 zCvY+n-983BPEdAn@lQ1%;mDoPS`ja}8n3kEQQhVso35&XoB zi4ikkpK?DZBk-=xJv}{{wa5|yyz$WDQ)-4gWb_MAVJk?^0QFzu*Zfg>85cg{8-Er6 z|EP}q%CKM{->&n2)MQX$^e;i`_X~gy;95Rf4rr9Z40n{5#sB8+gO7L?+d~Gd`Rgj> zJ?44$tK{&{Oko+Z*Z?V#d#rRwN8%&I9|Q8YYMk+J^!@Q^T*|VLDo#8s#UEAv4$gM zH!p#gS>k$yY@s~#uNSb7 zD3Lp{?0+sr&poTE(tX2!Zb)Y{=zrPQHj`Tv9`%h+DSd6~(dgSx-yVJSNX z2cX{b^)}`af8bs8a!2cPMQpZ5BFgWK$1&snxly2$hr2gBIHD6gzsY=f;qY}H zCrp+kTlS&vOk`^;YEpLK$LqDsMw=bAI=e{>;avdMp0ZUXO9vwgJ(O8Kr$vz(+nC6toj4c?}>$3GV0T=`u2F|uJD^c^&cFqiy;0igzP08 z$dd{lHoV&zb7T!j=)EME4xxB}36wF2by>3TRraUxv;K69rV9FN%W zj=q6PY5XW_g$^WkE_^KJX&C(jI@Mh{El|#_FKx-^5rp%_;|Zk?Vd-{a1r$ixs1&Cl zKYH*z0};B<7dUvIe0-=$Db*G}F;;{HO$}tFG@0gR1IdJ*&S{5r4Y!ZoAUMJrU&^>Ai=_}+o!Tm$33WWJ@NwB}~?tiBx((x%J@E}$0*rRm^ zX}ba=<6{XqU)ax$QwI!>e=CJiLN+wL+84bp9>m}?ktoQyDb`hopN@3DnMBJM4BBXz0yBq9~pRkj#J5NzR>8}kzVq*K=@hPXOU^xys` zOkVQe9J)1fB7}Rk>iI+>+~92TzLC3Sg91|1Tn%9FYF8`+o$g+VG}UYRJIo@2|h2G?;AclX?kiuDz&ee zhVN>(cB$zx*e_EWAU#FmX=_Wrb}z1}fh7`MMiw2va1utCgUFcG5N8Y^Dw@B?AN&kP zooh&G5a%u<3gwBCZzpsNdm-8w((ee%%Ju2(vjx%yM+`zlAVKKQB%G^zP6FpWD7%{Y z`JRbBj_{J)8Ki3fNK9XLVmT)a!}Sf1Kx4r+Bc;Fw#cJ9Vn7j|-b|rGkD}scgIp7gJ z1@2zx0F5+XO=NN8v!hoq@IB{Bao#Gy2MZ_QYMFcE=m~5@Cz*HnByxw~Z47P{iQHGZ zS{i}VVvpHpqzzAa=VW z`cXORKUJE~v+utA-HDl!Cq)>STdoVqs|J(+f}OGj z1go~Z;iXtczs_>5bZdFsW*o3u@sFKuWxeIVu_LM~`6E|&RzVMOWE-gts(WH{8xcI5v+G5{5H>rckRLe^Yzt1BO~ol$G}l#4$%ykr z<&4!&&^fJZv$pRflOYC<$xyCsn;V@q148+-?w3=p-DA-{}ehv z=m!T2B-)yniIqZqgtz!he}fsH-GX({GwASoH4h#WngoJN9eXaEL_7h}SL;si#u5D4 z^(B12d!zY?=VE@~#dc_K_3y0q2ClTql!mPkm6#jWgxQ&(>sreooM37;_;V! zh5w|byA}`NP|Ky~T~X%mquh+BMVJINw9del{%K4X$>NAI!BQz>M5D_tQpLX?1I|{j+%Xrd=zn^hJB4p&B zSj1}<8vQSPnkg>PVCIojab6Mn+wxY8C_@O$08f}*)reNSJ)p|ac-+R!yrk>JcS~jq zRo;MUj`cq#nJag9G^#MLsPr_q`3c{Z7B$;aN=>g1Ziuuu!V8;b)`2~u6CjtfDGwKa zC>fQ%jE%0oAu0gp9%oUy`!JyKr@jB=#83*05H-rgjk^j}Fkep0AO)uj0o|^&V)V84 z3qKL42fTRO+by0u@r?@m8z(uU**h1kb3`)^-Fd18OdqONtX5<@Xy2*^Uk(|DIDo>~ zEZCbipwZe(r{r6TN{P-cG}xT6{nGWbO5mKS$}#=p6t0t`r0HIOIB`~@{&eBknUyG_ zEacLBt0S}aXXBv*?`J^5mvU#LmB^=SA0hVoMeSd+macuy&RdDF8lImuvxtXgRlrAT z=szfeo*!I#v3TEPU(VMg6c?=DYPy~t$25FI%K(Vb~zcTGp3Gs zdv@k~?$d15F7Sf~12tp|qw1eAFM%Nv2l6LuZS-TXgU%n~8<+;K@v@DK;)zGyPh+{YH3j7`21VaKQA?DVZ7l3`|%PY!~J>o*;l? zA9gL1szGOkyvf=CohrAp%nu~!uH$85DkBYVdZWsvI{fS%2S-iEfe7f@V!r@E5qtWh$X;5k;Zg*A1)s=Yb7-0_Q@CmF1as`oT>#J!i z@HiY|>i2Ot`WzJrsrmCgaRJ7+upEGLV9uS+v2DP^q^XxwxVl0PV}w=eo(&sDUQPOo z8?y@nMlM(((T3v28mYxFZ^C_IKbH;3eFS1l>_djFoe8abOqE3WnZ7G?naNH0*l|Ts z#8G>tr2qE6zq2V63EE2JzVvp73VAfmkpltO;Z zFLZWx>f;uoej<4224+SQBse;hJ6rU(x?16VZ(@;p->R;+$;J9SYWa!!1dgQ@yg(Kd zp875QQL_A^RGL_-VJp!EEQjho>?XOjWO<=4NTMpJ`3vqGPmChJ_lvN@(|xuNF0OMg zcUHr?Ah;tQU#&&3f7`sd5V3yU5?atUj&J~B2g<+WRD*Uv+_iF#I)y zCvOX~%@05*hVIDA$JVi*^KU)DwkA#r62tNf69I6ASMduRhmIqxM*?i!v#}FxRf;0n z>)?M~aw&L*XSH>J}Nwzc3E@L zh$Zf;MU{~ZA@$O7PC6a}`in0Zsn~Rk|CUa$MZaqmRTrZH4dWlwW`@7dFndf4-B&;O zQ2KmI>S1qpcaThAx0J*J51{*F^@BfH zhv<-K?ojw|5dIWgTy?v@4@G5!pAPu!b>-@noOg!E$1oL;D&8=`+dK=G?+*1O_1s@f z-0?DFZm`brIm8&G`gp=*e7jMhH;BQd4vG4UnyezUh(tl!hu#G@=Z9j80-e$-f%s4) znktv8y4G`U50XQe+uhj(yoaR%)d~gjG*$O~Xfr3PW84lf zquKi-n%GU*gMe}_`S?dHul3Z>k3nKTF=}{;MW4h7uxF0@^}Wao@s_pdyDu^4janHe zYb?o9@%QH+pQH9>h%6M%zE%8~_z52IK*J?P%r>-Y5X=;AIt-%d2tEF(D@1+T852Gt zV0qga^FjTZH;hWd`KUf|LH!+xyF)$k;*kS6Zkv)`UJTWvg%@+n1ePI-676pQ>pYn| zRSD{TtQI+DQWf#4 zj$U5n-xx@KNlQeTBwnlD&mKyd`M~e=>)NMT=qoI}%ML~TK3eb`*!n>(2xT3qYV2-n z2rQ3zPKqXVjwT*^>tWS6FPyAeN>=;YE=GL+0?ed&BswRk_)fK&$rcr-ecb@_W~EBG z6Iks1iCp(`8ry}D;Q7`qA4YA)o9xIGMN2i5!B3r6>{63vCs`3av_?!KqhMgLx1(SA zaajz#cBtWDR{(}{Eb|hmdJjeI-!!NEU5#K<`BmlQo_9I4)Q(9Guh*`s^jw%V+qxE< z`LRFkLUhseZdX3x&SeX0E67Y#zrSf9zDCp*D*EwKFhLSyL}=(iP^&U}Y{S<<(hm$PM{@O?2xcWKTz1C+w259GW(U z!;!knP_?+hVFo-DO)_V&lHx$O;27|XS%VrF<#M$;I1DKZD^cFmq*lHu=Btv*@ols299Q=D=i90H+=Mj|bL*1+ttlW-tqZR-1}4K_6+Tg&;qBsJwy<^e*@)-(fBYM zhE8G$ng$3hIbE2~^#}t&`Ar3#F0s90C*X6gK95wJeqZs*Qp!0NZlpG16qTDJN7MRj z8L_jdnC|WUB824{;;R|5ti)1Ldn4srG^%%gkyxn*xL&`+w>ED3yL~uORG>Z0+#i20 z{tJXl&8#24MzP-!49pZm-Eq}nZayk$zG!u@owr}dKK!mKt;$#WMVjBN-pLwS-N>L& z4`uV50l$td%cF20=MFTMMQZBPrctU0P)kF2fO{rkBZO0tZuotwsh*mji>}S_2Ma3z z^jfdz+p1<#O&(dV9MVsm_lZ>E!-H z@&G9HQJQ;jz@Nz)9W`vF9utZl6i~4CU_cql&n~-ED(aTi@kEmVrl=vE-V|LnWG~A! zTV<+mF^GwOh>^&54Xd~;x@S&wOshx#-%VrQ+*5W%ysDcln=>gO&v$!8(}Qqe*w;vC zz`we$I^0-7uxmqx77FHKbzo@}pD9h@G$gI@X?xHh@6tB*?JZRJpihsg;q39>&5%^R90iaqPh#`dr`haNSKnMjX)A! zva3Zr#H)`CpRtzize-&z-t$63jid}cPc;z<&)9-OI<_xc@9Ppx&(5N+9cEB?l8^mQ z_k!Lf%q+)B{r++;vX!>ngFP-X8L+Ou-703@<-D^y`L|T$P-hemGUBns=S}0nLis(> zg7VL8L|;6YB~ywX%mlZMxYbzifO(EpE^HrdOs8eTz1w+pEOQ4=kf^tmcn{G>|+NpjOI z0focJx>Hn3J7pGkS+NUBl~S^UIr|$MqR7mG>%$G2ES+5m1k>C1{z~IMV7LgMe_&br3 zG9cP3v6M%4%2vq!%MW(LZ%4YO9G^nPchk|DqDi8@!wPqY0qiSseJ*||(#+4Gt=P;7 zZiD+S43lU#;lr;CU%vtAKFaTi@y4Hj(F-(^bCPZYY&XEpmtVbr)mR~BtUs(f;63RI z2Xa^9v8`LL0?D1D@F}cXBkaNY)l1xklV|wliF3$@md#^uY`1 z+*8jH&)!`>o$X)P-{4xqiUVjKsAEuQ1-X@CyXV`{yjWI@-|>Zm(IJHcBcUz5Pf-)< zy3_qx5?6cn;tpctHr}UFVfpCt`+lE<*T;#Tu4m6Z#E_V zK2Jjvo&LJals&e`lAD&HhGN`VscqpdO5r%p`S(Hv2VyFKc9#r($lZa%9i2;EAvyx4 zol{x$fTha&6x0=IDw=r}sOii}t_I)$!F#D>sB_;DN~Iyh`Esp#z0^t`$^m4dD}re3 z1$1NG*Trb7u!DyOXdab&G~}?)`}Z4n-&$$$q1f042hR_A&6s-4#!5q*xHTQ z9=~;4KWlwfGJ19G2qvd&@N&++x_aKVTavsAtT}yKTNzMPr~Zh zs!;=vDUhcFZEKKapMa=$|E+Q%ej|CoNo;SW=?2|R7BDIEBB|!@O3BGU)KqY>iv41;@(z1*bq~uQgHq$vgUm_JKjVs!y#p){ZnSb5%ESa) z(qc(TuQ09_S0Cu7cCd9ZM-&eg9!j!o4#gdyeCz#omlx}7ob^&j&UqfB$JF+(wm~e9 z>=^nLAsxY8?hiHWSH34ew{*#@%{=_tXaB_=gf5T7vEa zIa0)D9IH~g$Xcp<)N}0x=46#&lrZXck?~*$J24l;B>(eJt#0JYTsxKH?<6E^Hey># zEcWi+?1vbt55QQ)SIJSyjZJ>+YqTBLggt(WJ#DkZWn1Tda6T3S*2Ke1C5nb}d9eXu z)t3M$T+TpUN64nk`j3_-vJ#1(ZPmOCuvTaS4#;&tR_UiX9;CqR#J=Zn!FOpYj2(-9 zuUevGr;seo1jsO-xkW&C_#FEsJhI)=PhYMY@pLFmX~q>I6P1k^fIy=7+B9vZO1_Sc zpOlB2JO+J&<621^asa{Tky3_28_6D8SSXa*Tx7FJq~{AT0?4uo!XW-LzI;cIn}gJ- z?}jSIpTmF`od`G>kjza)tvPP1(_6%f3IL&3lDyxBz0$&1IM9zgwm#staiI8Ac?0ih zY1S;aE%JT|Oa(CTLfsg=rZ!NXwP#vlP_}7VnCXu{y7&}RwkC8FK$!Z~W!u3TlA?9% zTP9y2*o72E46aTrUU~bLf*YV(%(;BayEF>)b%QJ_{z^P*sDC;Wh&U++@107335Q*b00Ugm_^?nk<4FB5+QSk8ETdo zup5aNex6eExz2HcX((sl4n}Wds6FqtF;0h>CAmpR5!Ikj%ci=z{f89k5bB^#mV8 zZrxZ~>a{R!{JVKE9-CbrMrz~+oD?k`dNFT*3E(NO>)@8MV5_$g9yGY)^LPZ z2RHKP0eG(e8Kzn`cDMfiv`lqxdVE4P0~-GYYS|nBFHpgHSvn3>x3yq5bZ)s7M7qcd zsuYS$;||y+(VPprmJ(a9`<7#~47JoQ0;9Z9#@Z}zkpr^(1E^I6G*VzLh7Pb+Rv!g) zB|ohB$eSNMB6z*e|Y%_=jAETjmOd$dBS(Z z+`{p5RPW!Bf8sijjQRFYEje5euBWvL8g5>Y@qcg%WgKHs$a&cOprDT<%??O=ci}s{D|Fo*)%w2s z6@^1~nd^2NCuzoGlRqclna($M#OurBqy`VN4@E}?ojT#4AAbtU_c*80x^3vONEMk< zNaKrt4}M4Lq#%YQzf}0sZN+|!V@}!CIoo%?_plMgO->o+ifaSiGo1^k-njNwR3t_Z=haG2 zPg#tOGS10dW$U0oVYUI z*^0I7EzH3x^D8&d>m-6EbU$+T<-l+^I%e>prWiRc)5i9;8P;QJX56){gAW4JNe9ar zmLXht5JyAh{S(8OeL6G8GIJ6zG#^7=1P8w9NW=qq=QO<7FX(rU4r4&rslSti&1f5u z9ZvFsIf=sEI*pk|Y6Ijdxe5$<4=IP0!fT3(>?;$XpU}#WlOPynW~lWqnkw*%)op7% z_Gpes2q*Qw+BC%gy=zH_CW5Z)40D-<*sEvBA*(`1nie4QOyC3-IAKw}t1EkM#EQ?1 z?5$g5cOB?4$n4FqQeiE&^7kM@7TV%2NscDk7!rul!q!xew9U757tq?glPjZK!_Z=VBbsQj7}6Qc&Ha|tgkCrX`X5}mQreYqSCz~JX5j{x+0!q z>p3P&InqE^IPwY2iAuCu5KN;^PQc?rhUGcT_1~*OL4ap1^PkgunbA)9H?W@fBhT+> zc=0=0u%b^67`BdSaGw*7{kcL>W0${-@vNMOWBSF#YODC(X(&xLR1L3v3;di@#v#+_ z!Iq7}l4#*Nyt`hCz}vCam3lWz7S&a?Q-A8awUYi=Qibh~`9PX| zg8q!ZCBEa}OS0KrBP+Ln3jBp0XV2(w8J9hyYRSKkv3_ga8x06QvQP-BB|pUo6XT)P#KDva+jXDxm*k;O%GDzH zDtnp-BF-O24CK8VNK$coewU<*emg6M+TOXiW{Gt96=TCOGd&Pf)^3UU&1qfYPN46> z5I4DuM+wsPUgSu+h0^eAI)F#@0iD={mjGNDlL#3l^0ZH6cyx&hRjAfGCEQpP5 z{wGbPTdv-;E*lJ?xS2Wnjb~d-;*?T!BJ;b-hruXgs-ed|d9NHk%fu1uPdGuPb|uB# z<@x6b9c{YOGi{l-IKk3)$T`^~&X1}|InulvMZ79ji9uOKVl5GuT0P%tbjd z)ztq@cZ>vO-T@NIrfGQ*_G~mAe>xX7b(d$mB&$!04SPjU6d%eoF%A-$voMwI-tC5> z#9nrB82!t_In-U~Vec$cKPSw~6PdZI0mff!AAKw?GdTsm0>XrX%t=>gr@^3#Rmfi7 zP*Vmk>>^7cFjWm}pl@L(86XBImD}4!$Bm%PjG*?`&vs^Og1*)l<*X^QoQZo572BM66 ze0x2XhvUkVCK>bqfm222tdFy2iDo^KRGD7e5Vrpztmyq$S> zi0K)~mqVunuJV$Q8@R-EbL1=5UwB$R!PG=2xq?0G7; zY2^{Hw*}R;-A*E*RnO{mCh;!!(jVGgnv|93lSJOah;G5lyB4$1-z?X)1mI155VrMI@7mda_o)QL-l=iGThb@?`Jt z!?(zDVl#zI)T;wtquF~{wj%#FZLdk$ z7hM*20(LYb8j5}t z->;$6(PN)E^m|g6auhb?i5}Xxo(M6)%8t9#;b85Ye%~Z6?Dw)dHr~&ao$C?RPPAbl#`aR|L{9UZUs|)?K@lfx$X~;e>jX-beX#Xb**5Cr+JW)Fu=IgdVMh7PkWxJ={N{2GZNM znclS-N;Rh<;S!8uCpF=hI#MmlM*j}9veYFwiDU^-TWMjWUyB<>e@!OQ;QkqWl>m;9 z+*wS-iy6hVWzioXXPUqI!&cx{yu7n}D7Wa`(@nZ;CoxHScBT=&wX-fHDje4T%lUV|KSYJDq7t_a0bM>;)h7kK zA|@hv6T|+GDDmZAzrOar%M}cwGtf+6=H5uJ(Rh0FzyO1E_6ZJ3sMxZB+Ce$gN6k5+W?c6$kVES80Edzm<8IVP>7ozZsgF6k$s0`>{V zI^Im3FwaLlk-MP`eZhklCFXtCc(Fhx%gHD}L#!7!Ff9@iZf)`;G3@n2kbWJbk$4p8 zOU}l8+Dj;BL*8NN6*+Qe&FnE|i^VeL)%j34^V6#T<<=|>HA4qvv>FzfJ*<-QN$Zet zD_K?x@nc@|yfE$eaZuHuk@2LLX&+fJwn1ZIRpg&AU2jjzIj))k@$owTU3rekrf(QY zQM9n|5C@%8*EWpQvddAKx*AbPWRvuzcj`r*`=^<2K?1_#@#!PV8V9q}hM5WuXm))d z2l}rN?$}``?+s6b5u@<7q2)Zdd023sYQ#Pi)m3{M>~9us)gk09^vkV(1SPv={$a#O z^EVeWT7FYEE1UX-K6SPQrweXhRLrS}f4by*ug6bfHXZARxqh5?pZ~_zDL;46nJi9$ zR8bT%`TILxN|Pv`yt#+#1#ccK!~CNLc#;9eNN<7%l~BWr=}i!F+1I5|eI@$1Ztuw+ zXjA~nTtvd;iihAi0^y4Wce)=npc@fGy+p?bgA+oT5^_m9u1=U2sgSCOY>l+{Ldleh zRE@f*OJ}99|J;!lkj9~#RW~bhks(cX&&$L1<0|Ap(A;6E&019K=@ZIm*|@Sp8|lV1 z>al`E?D5v!0pwt;^VZ%q_mja1GwQMCWE)l%4^f6wv&u;P*sf32=v2N1cg~-F-8%_e z?8A(a1XyPLz*pz76h9led}UXA5@%VuCf_W}2WmoHg!KpWfq z`M^ENr?ijtV#SV%UNVXhCSv6DaQ@$$@u*SQ1Kp2co1$lGy+FpNPK%XTuN>s);ppfe zKM*ZDs1w6)UMK&pkxg6&)XWW&H(LNf(BUm15QSym@Aql}~QL zBFLcd8LyzB-OR=jAs6f`b?fzjG z#kVJu!g|M)v{jP{^V8E?VjN18NiGvJ+;Vg2QEXP}CT%4@YFl@`>K^?x{Bm@gglzau z8}23yWTT_=OchwHJ~U_Ey?UsQk%WY!Ju`^BZh3NIrj?@Jrf2g1(Q04I!y5a37Y_SH z=+nO0$EP-#7YET6xAQ8!qa=^1n~;x_aJ(J%Jzi z^)o$5Ki*LwBjo*O+iSoR-_yeVW>Gc7<~}4?q*JS)(=vVH8#;~(|^Pkym29VOp+8=Rh<3l%LsMD7ZM|z2Rr5* zr7fEB-vR9ZkuET^|C3S3kL#sM@`)Eie2u#dMhx$gjd0PI4T*(p zCch}GZN@)fs@B$cT}cw)L$sczqEsMD(r>bFh$~)ZX+NeII(B^e-@`)Cs)L!mx)O`U zW;aEndOy(dVWRhaJ{_zkwCC`^*biIb5+YXrI}EFBXut!KScI!(n6VcE{6{Ed?j%}} z{Am^MOu9imcLmQ7G`3M)sKw`L_oHrNWQ)5?5fAkIO*8J_-IOLx@^Y^sUpA6IeX%{3 zZ;UwJyD+UIS(ydXj0$l4vV0Ra7685}g9rYZ5D-9q-ni5MJW$)BOFe+p(E1{~N9yXc zX^7n-_0aV~O8I7IVyjI)?N$#`-QsjiBdi~E%rp?c=Spt5G>+k-(lm5pFBu7d9M7tW zSzxD+iKX`KvIv{Y_G}rC3|QhtY$M3v1fcCHGYpxh7RwXPP^6aPkUcfcTLQwwp@^&p zUX%`=#fxLN#c*cewOWU|<45<)8}eLt*CjKx_h4vkT67HUhO|&Ns8A^y5QJbY9x9E! zCkQmq?LGx9Pr2?g@L=DZG6rn|pDFj&VBauVo(uH`R19}ZOp_^nwNzU8G>wnC$9}NP z9iR0RM8+QK7lP*!b{WPtYS`iJCXsr&2Ga)PaLhtOY?ffm{&)CQXyGpc8Y>Q)U>{E7 z1v5tdS*ixbKjjTN2x-`q8Qn9wy^2B8T;AA54Yc;P9v!JHDjt^V_U8M@z8Wq}8`}Bn zUE`+CX}zwcZL!&6?d+O_)fUcUZfbAJ;ec5lSxQ9IJmo>7wEZC~-17I_^5mnLvG=jH z8sw*jaoKR?C|BHcN_NyM$b;I94qAv^6hJZOp$k(1V>gfMYK^ymD;9kfj~*(rOyNh*Rpq0}p$hUrz%6PM`8#TB>v z`Y=@oR%Lb|Ezw?WHEchvyZpW2_K&Q_T7p&s){`UFj}TWqOZsWO*(X85F4ZI{v*xZM zo{?^Ho>@Xo%vzg8d)|U;GX2muQy4|+;_#~WM3x5U?Y6qB=VfinQNUe^z23}b^e94R z(T*XjRsu}cztCnqdimX(*TG9! z1s7h}iPvUAx}WmS>yWMZEz%g)5ik~#LSiXrjO;PMFPC~y=~YV8~6 z`%L9Kyq}nJk54#y9L_Bf4*a9LT3;@eqdvv1u;j_UBgp)ZN2|S}Hz~h^^#L^g+aq|a zqQrM3x!jS5+|VjEAbw{xk8VApYZxXN?zZaJ5c6u3^mN#La62UaHY0H-1eb+X^*(Vh zD&G@tIdTLExdE&NOW8zCcXbM4B>%)Lsskt^SNiD-uB?(jRY~8AY>*itr@qK0R4*CH z*Wmwg;s`vE9(TYGt{Z3b&Z~Qw><#<6KKOv5`GLOn@{3w8WrFF?HFrkNx3U?M&ZuGei;$}$J|#I7(UmK5Dh-XY+XZ;? zig_1LiFqGcQnS_U=G+--*OZSqFr9q$AndG+!*$nKf4klDw3PM=|G4nCdU}?4(zBMU zKaVZG^zqK`@S}OJc%Sz6WIf+$^YC5qYMb6Q$M?7tH{bnJdAc{X^wR7E{f|E^*8>9?mA!qt~?KX{qz$s_{0z3^vwZt`|BqgyV z)hf9t6-Y4{85o)98d&NYn1&ddSeckx8Chr>7+4t?bj8VAplHa=PsvQHMA2YmWo&L` XUUftDnm{r-UW|<(A}( literal 0 HcmV?d00001 diff --git a/stackpacks/suse-observability/resources/overview.md b/stackpacks/suse-observability/resources/overview.md new file mode 100644 index 0000000..c46ae3a --- /dev/null +++ b/stackpacks/suse-observability/resources/overview.md @@ -0,0 +1,3 @@ +## SUSE Observability StackPack + +This StackPack adds SUSE Observability self-monitoring and other SUSE Observability instances to the standard Kubernetes StackPack via monitors and metric bindings. diff --git a/stackpacks/suse-observability/resources/provisioning.md b/stackpacks/suse-observability/resources/provisioning.md new file mode 100644 index 0000000..55f8623 --- /dev/null +++ b/stackpacks/suse-observability/resources/provisioning.md @@ -0,0 +1,3 @@ +## Provisioning... + +Provisioning SUSE Observability monitoring. Please wait a moment. diff --git a/stackpacks/suse-observability/resources/waitingfordata.md b/stackpacks/suse-observability/resources/waitingfordata.md new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/stackpacks/suse-observability/resources/waitingfordata.md @@ -0,0 +1 @@ + diff --git a/stackpacks/suse-observability/settings/dashboards/data-ingestion.sty b/stackpacks/suse-observability/settings/dashboards/data-ingestion.sty new file mode 100644 index 0000000..10dd19e --- /dev/null +++ b/stackpacks/suse-observability/settings/dashboards/data-ingestion.sty @@ -0,0 +1,323 @@ +nodes: + - _type: Dashboard + identifier: urn:stackpack:suse-observability:dashboard:data-ingestion + name: SUSE Observability - Data ingestion + description: "" + dashboard: + metadata: + project: '{"saveVariables":false}' + spec: + panels: + YHvrlowP7KLrAuLWFrWBd: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: binBps + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (local_pod_name)(rate(pod_received_bytes_count{local_pod_name=~".*-(receiver|collector)-.*", + cluster_name='${cluster}', direction="incoming", + intra_pod!='true', + local_pod_ns='${namespace}'}[${__rate_interval}])) + alias: ${local_pod_name} + display: + name: Ingested data / second + description: Volume of ingested data per pod. The 2 main entry points for data + in the platform are the receiver pod(s) and the OTel collector + pod(s) that are part of the platform. + KP9lrGvp3jTeJbIDkdzP8: + spec: + plugin: + kind: StatChart + spec: + calculation: last + format: + unit: binBps + decimalPlaces: 0 + sparkline: true + thresholds: + defaultColor: "#33A02C" + steps: [] + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum(rate(pod_received_bytes_count{local_pod_name=~".*-(receiver|collector)-.*", + cluster_name='${cluster}', direction="incoming", + intra_pod!='true', + local_pod_ns='${namespace}'}[${__rate_interval}])) + alias: Total Data Volume + display: + name: Total data volume + description: The total amount of data received on the SUSE Observability API, + this includes topology, metrics, logs, traces, events, and health. + qpExljGT-9X2iu50hKfx_: + spec: + plugin: + kind: StatChart + spec: + calculation: last + format: + unit: short + decimalPlaces: 0 + sparkline: true + thresholds: + defaultColor: "#33A02C" + steps: [] + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by + (element_type)(max_over_time(stackstate_stackstate_receiver_unique_element_passed_count{cluster_name="${cluster}", + namespace="${namespace}", + element_type='agent'}[${__interval}])) + alias: node + display: + name: Incoming data + description: The number of standard nodes (4 vCPU and 16GiB of memory) from + which data is received. The real nodes can be a different size, so + the real node count can be significantly lower. + P_IBNZlmdKtK-uXWJWE8z: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: percent + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: 100 * sum by (kube_app_component) + (container_cpu_throttled_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-base"}) / sum by + (kube_app_component) + (container_cpu_elapsed_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-base"}) + alias: ${kube_app_component} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: 100 * sum by (kube_app_name) + (container_cpu_throttled_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_name=~"opentelemetry-collector"}) / sum by + (kube_app_name) + (container_cpu_elapsed_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_name=~"opentelemetry-collector"}) + alias: ${kube_app_name} + display: + name: CPU Throttling + description: CPU throttling for the main pods in the data ingestion pipeline. + High throttling can lead to delays in data processing, a profile + upgrade or CPU limit increase can resolve the problem. + IFl4huEAwoMRTyV0l5IeP: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + label: something + range: + min: + max: + format: + unit: short + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (cluster_name, namespace, pod_name, + kube_app_component)(max_over_time(container_cpu_usage{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-base"}[${__interval}])) + / 1000000000 + alias: ${kube_app_component} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (cluster_name, namespace, + kube_app_name)(max_over_time(container_cpu_usage{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_name="opentelemetry-collector"}[${__interval}])) / + 1000000000 + alias: ${kube_app_name} + display: + name: CPU Usage Processing + description: "For the receiver and opentelemetry collector: the 2 APIs that + accept telemetry data." + L9E1jtO844cup6_P4Po1g: + spec: + plugin: + kind: GaugeChart + spec: + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 0.9 + - color: "#FF7F00" + value: 0.8 + mode: absolute + max: 1 + format: + unit: percentunit + decimalPlaces: 0 + calculation: last + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by + (storage)(label_replace(max_over_time(kubernetes_kubelet_volume_stats_used_bytes{cluster_name="${cluster}", + namespace="${namespace}", + persistentvolumeclaim=~".*(kafka-|hbase-|victoria-metrics|clickhouse|vmagent|zookeeper|sync).*"}[5m]) + / + max_over_time(kubernetes_kubelet_volume_stats_capacity_bytes{cluster_name="${cluster}", + namespace="${namespace}", + persistentvolumeclaim=~".*(kafka-|hbase-|victoria-metrics|clickhouse|vmagent|zookeeper|sync).*"}[5m]), + "storage","$1", "persistentvolumeclaim", + ".*-(kafka-.*|hbase-.*|victoria-metrics-.*|clickhouse-.*|vmagent-.*|zookeeper-.*|sync-tmp)")) + alias: ${storage} + display: + name: Volume usage + description: "Percentage of persistent volume capacity used by data ingestion components." + layouts: + - kind: Grid + spec: + items: + - x: 6 + y: 0 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/YHvrlowP7KLrAuLWFrWBd" + - x: 3 + y: 0 + width: 3 + height: 2 + content: + $ref: "#/spec/panels/KP9lrGvp3jTeJbIDkdzP8" + - x: 0 + y: 0 + width: 3 + height: 2 + content: + $ref: "#/spec/panels/qpExljGT-9X2iu50hKfx_" + - x: 0 + y: 2 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/P_IBNZlmdKtK-uXWJWE8z" + - x: 8 + y: 2 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/IFl4huEAwoMRTyV0l5IeP" + - x: 0 + y: 4 + width: 24 + height: 2 + content: + $ref: "#/spec/panels/L9E1jtO844cup6_P4Po1g" + variables: + - kind: ListVariable + spec: + plugin: + kind: MetricLabelValues + spec: + labelName: cluster_name + matchers: + - stackstate_stackstate_receiver_element_create_passed_count + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: cluster + display: + name: Cluster + description: "" + - kind: ListVariable + spec: + plugin: + kind: MetricPromQL + spec: + labelName: namespace + expr: stackstate_stackstate_receiver_element_create_passed_count{cluster_name="${cluster}"} + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: namespace + display: + name: Namespace + description: "" diff --git a/stackpacks/suse-observability/settings/dashboards/logs-processing.sty b/stackpacks/suse-observability/settings/dashboards/logs-processing.sty new file mode 100644 index 0000000..83077f8 --- /dev/null +++ b/stackpacks/suse-observability/settings/dashboards/logs-processing.sty @@ -0,0 +1,318 @@ +nodes: + - _type: Dashboard + identifier: urn:stackpack:suse-observability:dashboard:logs-processing + name: SUSE Observability - Logs processing + description: "" + dashboard: + metadata: + project: '{"saveVariables":false}' + spec: + panels: + HxCR4e7TCojuRrKLl9T-y: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + format: + unit: bytes/sec(IEC) + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: rate(stackstate_stackstate_receiver_events_accepted_size{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-logs", + element_type="k8sLogs"}[${__rate_interval}]) + alias: Accepted + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: rate(stackstate_stackstate_receiver_events_rejected_size{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-logs", + element_type="k8sLogs"}[${__rate_interval}]) + alias: Rejected + display: + name: SUSE Observability Receiver - Log volume + description: "Rate of log data accepted and rejected by the receiver." + HILv2hOoS5lHs6NjEv72p: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: bytes/sec(IEC) + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max(stackstate_stackstate_receiver_logs_budget_per_pod{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-logs"}) + alias: Logs budget per pod per time window + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: topk(10, + rate(stackstate_stackstate_receiver_logs_rejected_size{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-logs"}[${__rate_interval}] + @ end())) + alias: Logs from ${limited_pod_name} rejected + display: + name: SUSE Observability Receiver - Rejected Log volume per pod + description: "Log volume rejected per pod due to budget limits. The budget is determined based on available storage and the number of active log streams." + qX3IWlWiFY_lnvFHOqc8W: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: percent + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: 100 * sum by (kube_app_component) + (container_cpu_throttled_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-logs"}) / sum by + (kube_app_component) + (container_cpu_elapsed_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-logs"}) + alias: ${kube_app_component} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: 100 * sum by (kube_app_name) + (container_cpu_throttled_periods{cluster_name="${cluster}", + namespace="${namespace}", kube_app_name=~"elasticsearch"}) / + sum by (kube_app_name) + (container_cpu_elapsed_periods{cluster_name="${cluster}", + namespace="${namespace}", kube_app_name=~"elasticsearch"}) + alias: ${kube_app_name} + display: + name: CPU Throttling + description: CPU throttling for the main pods in the logs ingestion pipeline. + mvjWBYx0ldBjryYE8MEwg: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (cluster_name, namespace, pod_name, + kube_app_component)(max_over_time(container_cpu_usage{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-logs"}[${__interval}])) + / 1000000000 + alias: ${kube_app_component} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (cluster_name, namespace, + kube_app_name)(max_over_time(container_cpu_usage{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_name="elasticsearch"}[${__interval}])) / 1000000000 + alias: ${kube_app_name} + display: + name: CPU Usage Processing + description: "CPU usage of the receiver and Elasticsearch pods." + hTa7uB5XaIDsbVQluJF4d: + spec: + plugin: + kind: GaugeChart + spec: + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 0.9 + - color: "#FF7F00" + value: 0.8 + mode: absolute + max: 1 + format: + unit: percentunit + decimalPlaces: 0 + calculation: last + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by + (storage)(label_replace(max_over_time(kubernetes_kubelet_volume_stats_used_bytes{cluster_name="${cluster}", + namespace="${namespace}", + persistentvolumeclaim=~"data-.*-(elasticsearch-master-\\d)"}[5m]) + / + max_over_time(kubernetes_kubelet_volume_stats_capacity_bytes{cluster_name="${cluster}", + namespace="${namespace}", + persistentvolumeclaim=~"data-.*-(elasticsearch-master-\\d)"}[5m]), + "storage","$1", "persistentvolumeclaim", + "data-.*-(elasticsearch-.*)")) + alias: ${storage} + display: + name: Volume usage + description: "Percentage of persistent volume capacity used by Elasticsearch." + mt7i8AHw8Ke4R4EJ269Rs: + spec: + plugin: + kind: Markdown + spec: + text: >- + # More details + + These are the pods and workloads involved in the logs pipeline: + + + * + [Pods](/#/views/urn:stackpack:kubernetes-v2:shared:query-view:pods/overview?query=label%20IN%20%28"app.kubernetes.io%2Fcomponent%3Areceiver"%2C%20"app.kubernetes.io%2Fcomponent%3Areceiver-logs"%2C%20"app.kubernetes.io%2Fname%3Aelasticsearch"%29%20AND%20label%20%3D%20"cluster-name%3A${cluster}"%20AND%20label%20%3D%20"namespace%3A${namespace}"%20AND%20type%20%3D%20"pod") + + * + [Statefulsets](/#/views/urn:stackpack:kubernetes-v2:shared:query-view:statefulsets/overview?query=label%20IN%20%28"app.kubernetes.io%2Fname%3Aelasticsearch"%29%20AND%20label%20%3D%20"cluster-name%3A${cluster}"%20AND%20label%20%3D%20"namespace%3A${namespace}"%20AND%20type%20IN%28"statefulset"%29) + + * + [Deployments](/#/views/urn:stackpack:kubernetes-v2:shared:query-view:deployments/overview?query=label%20IN%20%28"app.kubernetes.io%2Fcomponent%3Areceiver"%2C"app.kubernetes.io%2Fcomponent%3Areceiver-logs"%29%20AND%20label%20%3D%20"cluster-name%3A${cluster}"%20AND%20label%20%3D%20"namespace%3A${namespace}"%20AND%20type%20IN%28"deployment"%29) + links: [] + queries: [] + display: + name: "-" + description: "Links to related Pods, StatefulSets, and Deployments." + layouts: + - kind: Grid + spec: + items: + - x: 0 + y: 2 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/HxCR4e7TCojuRrKLl9T-y" + - x: 8 + y: 2 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/HILv2hOoS5lHs6NjEv72p" + - x: 0 + y: 4 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/qX3IWlWiFY_lnvFHOqc8W" + - x: 8 + y: 4 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/mvjWBYx0ldBjryYE8MEwg" + - x: 0 + y: 0 + width: 6 + height: 2 + content: + $ref: "#/spec/panels/hTa7uB5XaIDsbVQluJF4d" + - x: 6 + y: 0 + width: 4 + height: 2 + content: + $ref: "#/spec/panels/mt7i8AHw8Ke4R4EJ269Rs" + variables: + - kind: ListVariable + spec: + plugin: + kind: MetricLabelValues + spec: + labelName: cluster_name + matchers: + - stackstate_stackstate_receiver_element_create_passed_count + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: cluster + display: + name: Cluster + description: "" + - kind: ListVariable + spec: + plugin: + kind: MetricPromQL + spec: + labelName: namespace + expr: stackstate_stackstate_receiver_element_create_passed_count{cluster_name="${cluster}"} + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: namespace + display: + name: Namespace + description: "" diff --git a/stackpacks/suse-observability/settings/dashboards/metrics-processing.sty b/stackpacks/suse-observability/settings/dashboards/metrics-processing.sty new file mode 100644 index 0000000..ba72408 --- /dev/null +++ b/stackpacks/suse-observability/settings/dashboards/metrics-processing.sty @@ -0,0 +1,755 @@ +nodes: + - _type: Dashboard + identifier: urn:stackpack:suse-observability:dashboard:metrics-processing + name: SUSE Observability - Metrics processing + description: "" + dashboard: + metadata: + project: '{"saveVariables":false}' + spec: + panels: + eIFw6RjqVknqJoWc309aa: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: percentunit + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: + - color: "#E31A1C" + value: 0.02 + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (pod_name)(rate(stackstate_vm_slow_row_inserts_total{cluster_name="${cluster}", + namespace="${namespace}", + pod_name=~"suse-observability-victoria-metrics-.*"}[${__rate_interval}])) + / sum by + (pod_name)(rate(stackstate_vm_rows_added_to_storage_total{cluster_name="${cluster}", + namespace="${namespace}", + pod_name=~"suse-observability-victoria-metrics-.*"}[${__rate_interval}])) + alias: ${pod_name} + display: + name: VictoriaMetrics - Slow inserts + description: "Percentage of inserts into VictoriaMetrics that are considered slow." + 6RpPvPOqS5SdnjYcbMZtB: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (pod_name)(rate(stackstate_vm_slow_queries_total{cluster_name="${cluster}", + namespace="${namespace}", + pod_name=~"suse-observability-victoria-metrics-.*"}[${__rate_interval}])) + alias: ${pod_name} + display: + name: VictoriaMetrics - Slow queries rate + description: "Rate of slow queries executed against VictoriaMetrics." + b2OMg07XzmxsM0C6TiFoP: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (pod_name)(increase(stackstate_vm_new_timeseries_created_total{cluster_name="${cluster}", + namespace="${namespace}", + pod_name=~"suse-observability-victoria-metrics-.*"}[24h])) + alias: ${pod_name} over 24 hrs + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (pod_name)(rate(stackstate_vm_new_timeseries_created_total{cluster_name="${cluster}", + namespace="${namespace}", + pod_name=~"suse-observability-victoria-metrics-.*"}[${__rate_interval}])) + alias: ${pod_name} / second + display: + name: VictoriaMetrics - Churn rate + description: "Rate of new time series creation in VictoriaMetrics." + tSVthJSBDQ55Ag0zWATTh: + spec: + plugin: + kind: StatChart + spec: + calculation: last + format: + unit: short + decimalPlaces: 0 + sparkline: true + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 10 + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (statefulset)(increase(stackstate_vm_rows_ignored_total{pod_name=~"suse-observability-vmagent-.*", + cluster_name="${cluster}", namespace="${namespace}"}[1h])) + alias: vmagent + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (statefulset)(increase(stackstate_vm_rows_ignored_total{pod_name=~"suse-observability-victoria-metrics-.*", + cluster_name="${cluster}", namespace="${namespace}"}[1h])) + alias: VictoriaMetrics + display: + name: Dropped timeseries in the past hour + description: >- + VictoriaMetrics limits the number of labels per timeseries with + `-maxLabelsPerTimeseries` command-line flag. + + This prevents from ingesting metrics with too many labels. The value + of `maxLabelsPerTimeseries` must be adjusted for your workload. + + When limit is exceeded (graph is > 0) - the timeseries with too many + labels are dropped, which results in missing metric data. + H5gtvG8QRQgcH2_9kvYLJ: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: percent + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: 100 * sum by (container) + (container_cpu_throttled_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_instance="suse-observability", container=~"receiver|receiver-base|vmagent|opentelemetry-collector|victoria-metrics-\\d-server"}) + / (sum by (container) + (container_cpu_elapsed_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_instance="suse-observability", container=~"receiver|receiver-base|vmagent|opentelemetry-collector|victoria-metrics-\\d-server"}) + > 0) + alias: ${container} + display: + name: CPU Throttling + description: CPU throttling for the main pods in the data ingestion pipeline. + High throttling can lead to delays in data processing, a profile + upgrade or CPU limit increase can resolve the problem. A container + with no CPU limit is never throttled and reports nothing here, so + read this alongside CPU Pressure. + 1LYJbBBKSUMFsD1dbNDvi: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (cluster_name, namespace, pod_name, + kube_app_component)(max_over_time(container_cpu_usage{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-base|vmagent"}[${__interval}])) + / 1000000000 + alias: ${kube_app_component} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (cluster_name, namespace, + kube_app_name)(max_over_time(container_cpu_usage{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_name="opentelemetry-collector"}[${__interval}])) / + 1000000000 + alias: ${kube_app_name} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (container)(kubernetes_state_container_cpu_requested{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_instance="suse-observability", container=~"receiver|receiver-base|vmagent|opentelemetry-collector"}) + alias: ${container} request + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (container)(kubernetes_state_container_cpu_limit{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_instance="suse-observability", container=~"receiver|receiver-base|vmagent|opentelemetry-collector"}) + alias: ${container} limit + display: + name: CPU Usage Processing + description: "CPU usage of receiver, vmagent, and OpenTelemetry collector + pods, against their configured request and limit. A missing limit line + means no CPU limit is set." + jGL_6sXlItqnyvKC2NU_D: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (cluster_name, namespace, + pod_name)(max_over_time(container_cpu_usage{cluster_name="${cluster}", + namespace="${namespace}", + pod_name=~"suse-observability-victoria-metrics-.*"}[${__interval}])) + / 1000000000 + alias: ${pod_name} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (pod_name)(kubernetes_state_container_cpu_requested{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_instance="suse-observability", container=~"victoria-metrics-\\d-server"}) + alias: ${pod_name} request + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (pod_name)(kubernetes_state_container_cpu_limit{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_instance="suse-observability", container=~"victoria-metrics-\\d-server"}) + alias: ${pod_name} limit + display: + name: CPU Usage VictoriaMetrics + description: "CPU usage of VictoriaMetrics pods, against their configured + request and limit. A missing limit line means no CPU limit is set." + LENPxpypcWV-l3jYWsgPA: + spec: + plugin: + kind: GaugeChart + spec: + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 0.9 + - color: "#FF7F00" + value: 0.8 + mode: absolute + max: 1 + format: + unit: percentunit + decimalPlaces: 0 + calculation: last + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by + (storage)(label_replace(max_over_time(kubernetes_kubelet_volume_stats_used_bytes{cluster_name="${cluster}", + namespace="${namespace}", + persistentvolumeclaim=~"server-volume-.*-(victoria-metrics-.+)"}[5m]) + / + max_over_time(kubernetes_kubelet_volume_stats_capacity_bytes{cluster_name="${cluster}", + namespace="${namespace}", + persistentvolumeclaim=~"server-volume-.*-(victoria-metrics-.+)"}[5m]), + "storage","$1", "persistentvolumeclaim", + "server-volume-.*-(victoria-metrics-.+)")) + alias: ${storage} + display: + name: Volume usage + description: "Percentage of persistent volume capacity used by VictoriaMetrics." + cpuPressurePanelMetricsPath: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: s + decimalPlaces: 3 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (container)(max_over_time(container_cpu_partial_stall{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_instance="suse-observability", container=~"receiver|receiver-base|vmagent|opentelemetry-collector|victoria-metrics-\\d-server"}[${__interval}])) + / 1000000000 + alias: ${container} + display: + name: CPU Pressure + description: >- + Time these containers spent stalled waiting for CPU, from kernel + pressure stall information. + + Unlike CPU Throttling this reports contention even when no CPU limit + is set, so it is the signal to use when comparing CPU requests. Read + it as a trend: rising stall alongside flat throughput means the + workload is CPU starved. + vmPendingRowsPanelMetricsPath: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (pod_name, + type)(stackstate_vm_pending_rows{cluster_name="${cluster}", + namespace="${namespace}", + pod_name=~"suse-observability-victoria-metrics-.*"}) + alias: ${pod_name} ${type} + display: + name: VictoriaMetrics - Pending rows + description: >- + Rows buffered in memory before VictoriaMetrics has written them to + storage, split by storage and indexdb. + + A steady baseline is normal. Sustained growth means ingestion is + arriving faster than it can be persisted, and is the first place + backpressure appears in the metrics path. + vmagentQueueWritePanelMetricsPath: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 3 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (pod_name)(rate(stackstate_vm_persistentqueue_write_duration_seconds_total{cluster_name="${cluster}", + namespace="${namespace}", + pod_name=~"suse-observability-vmagent-.*"}[${__rate_interval}])) + alias: ${pod_name} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (pod_name)(stackstate_vmagent_remotewrite_pending_data_bytes{cluster_name="${cluster}", + namespace="${namespace}", + pod_name=~"suse-observability-vmagent-.*"}) / 1000000 + alias: ${pod_name} pending MB + display: + name: vmagent - Write queue + description: >- + Seconds per second that vmagent spent writing to its on-disk + persistent queue, alongside the megabytes currently pending. + + Both sitting at zero means remote write is keeping up. Writing time + above zero means vmagent is spilling to disk because the metric store + is not accepting data fast enough. + collectorQueueOccupancyPanelMetricsPath: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: percentunit + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: + - color: "#FF7F00" + value: 0.8 + - color: "#E31A1C" + value: 0.9 + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (pod_name, + exporter)(stackstate_otelcol_exporter_queue_size{cluster_name="${cluster}", + namespace="${namespace}", data_type="metrics"}) / sum by + (pod_name, + exporter)(stackstate_otelcol_exporter_queue_capacity{cluster_name="${cluster}", + namespace="${namespace}", data_type="metrics"}) + alias: ${exporter} + display: + name: OTel Collector - Metrics export queue occupancy + description: >- + How full the collector's metric export queue is, as a fraction of its + configured capacity. + + Occupancy is the useful form rather than raw depth, because capacity + differs per exporter and per profile. A queue at capacity drops or + refuses data rather than buffering it further. + receiverBackpressurePanelMetricsPath: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: percentunit + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: + - color: "#FF7F00" + value: 0.8 + - color: "#E31A1C" + value: 0.95 + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (pod_name)(stackstate_stackstate_receiver_memory_limiter_claimed_size{cluster_name="${cluster}", + namespace="${namespace}"}) / sum by + (pod_name)(stackstate_stackstate_receiver_memory_limiter_limit{cluster_name="${cluster}", + namespace="${namespace}"}) + alias: ${pod_name} + display: + name: Receiver - Memory limiter saturation + description: >- + Share of the receiver's memory limiter budget currently claimed. + + The receiver has no queue to measure; this is its backpressure + equivalent, because reaching the limit is what makes it start + rejecting incoming data. + Ao0rRQFkI3j5Va7erPJP3: + spec: + plugin: + kind: Markdown + spec: + text: >- + # More details + + These are the pods and workloads involved in the metrics pipeline: + + + * + [Pods](/#/views/urn:stackpack:kubernetes-v2:shared:query-view:pods/overview?query=label%20IN%20%28"app.kubernetes.io%2Fcomponent%3Areceiver"%2C%20"app.kubernetes.io%2Fcomponent%3Areceiver-base"%2C%20"app.kubernetes.io%2Fname%3Avictoria-metrics-0"%2C%20"app.kubernetes.io%2Fname%3Avictoria-metrics-1"%2C%20"app.kubernetes.io%2Fname%3Aopentelemetry-collector"%29%20AND%20label%20%3D%20"cluster-name%3A${cluster}"%20AND%20label%20%3D%20"namespace%3A${namespace}"%20AND%20type%20%3D%20"pod") + + * + [Statefulsets](/#/views/urn:stackpack:kubernetes-v2:shared:query-view:statefulsets/overview?query=label%20IN%20%28"app.kubernetes.io%2Fname%3Avictoria-metrics-0"%2C%20"app.kubernetes.io%2Fname%3Avictoria-metrics-1"%2C%20"app.kubernetes.io%2Fname%3Aopentelemetry-collector"%29%20AND%20label%20%3D%20"cluster-name%3A${cluster}"%20AND%20label%20%3D%20"namespace%3A${namespace}"%20AND%20type%20IN%28"statefulset"%29) + + * + [Deployments](/#/views/urn:stackpack:kubernetes-v2:shared:query-view:deployments/overview?query=label%20IN%20%28"app.kubernetes.io%2Fcomponent%3Areceiver"%2C"app.kubernetes.io%2Fcomponent%3Areceiver-base"%29%20AND%20label%20%3D%20"cluster-name%3A${cluster}"%20AND%20label%20%3D%20"namespace%3A${namespace}"%20AND%20type%20IN%28"deployment"%29) + links: [] + queries: [] + display: + name: "-" + description: "Links to related Pods, StatefulSets, and Deployments." + layouts: + - kind: Grid + spec: + items: + - x: 0 + y: 2 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/eIFw6RjqVknqJoWc309aa" + - x: 16 + y: 2 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/6RpPvPOqS5SdnjYcbMZtB" + - x: 8 + y: 2 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/b2OMg07XzmxsM0C6TiFoP" + - x: 0 + y: 0 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/tSVthJSBDQ55Ag0zWATTh" + - x: 0 + y: 4 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/H5gtvG8QRQgcH2_9kvYLJ" + - x: 8 + y: 4 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/1LYJbBBKSUMFsD1dbNDvi" + - x: 16 + y: 4 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/jGL_6sXlItqnyvKC2NU_D" + - x: 0 + y: 6 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/cpuPressurePanelMetricsPath" + - x: 8 + y: 6 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/vmPendingRowsPanelMetricsPath" + - x: 16 + y: 6 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/vmagentQueueWritePanelMetricsPath" + - x: 0 + y: 8 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/collectorQueueOccupancyPanelMetricsPath" + - x: 8 + y: 8 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/receiverBackpressurePanelMetricsPath" + - x: 8 + y: 0 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/LENPxpypcWV-l3jYWsgPA" + - x: 16 + y: 0 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/Ao0rRQFkI3j5Va7erPJP3" + variables: + - kind: ListVariable + spec: + plugin: + kind: MetricLabelValues + spec: + labelName: cluster_name + matchers: + - stackstate_stackstate_receiver_element_create_passed_count + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: cluster + display: + name: Cluster + description: "" + - kind: ListVariable + spec: + plugin: + kind: MetricPromQL + spec: + labelName: namespace + expr: stackstate_stackstate_receiver_element_create_passed_count{cluster_name="${cluster}"} + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: namespace + display: + name: Namespace + description: "" diff --git a/stackpacks/suse-observability/settings/dashboards/topology-processing.sty b/stackpacks/suse-observability/settings/dashboards/topology-processing.sty new file mode 100644 index 0000000..c440f2e --- /dev/null +++ b/stackpacks/suse-observability/settings/dashboards/topology-processing.sty @@ -0,0 +1,642 @@ +nodes: + - _type: Dashboard + identifier: urn:stackpack:suse-observability:dashboard:topology-processing + name: SUSE Observability - Topology processing + description: "" + dashboard: + metadata: + project: '{"saveVariables":false}' + spec: + panels: + dBMublgJcGkzpzDGSKUvq: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: s + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (integration_type) + (stackstate_stackstate_sync_stage_latency_seconds_quantile{start_stage="enterReceiver", + end_stage="commit", quantile="1.0", + cluster_name="${cluster}", namespace="${namespace}", + kube_app_component=~"sync|server", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability"}) + alias: ${integration_type} + display: + name: Topology Sync Internal Latency + description: "Internal latency within the topology synchronization process." + zv3RIlz8UQBwAoCBrrpRC: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: s + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (pod_name, integration_type) + (stackstate_stackstate_sync_stage_latency_seconds_quantile{start_stage="source", + end_stage="commit", quantile="1.0", + cluster_name="${cluster}", namespace="${namespace}", + kube_app_component=~"sync|server", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability"}) + alias: ${integration_type} + display: + name: Topology Sync Latency + description: "Total latency of the topology synchronization process from source to commit." + 3Py1h-ak88rlmOZpULtQB: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: cps + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: topk(5, sum by (pod_name, topic) + (stackstate_kafka_consumer_consumer_fetch_manager_metrics_records_lag{cluster_name="${cluster}", + kube_app_component=~"sync|server", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability"})) + alias: ${topic} + display: + name: Top 5 Topology Sync Topic lag + description: "Lag in processing topology synchronization topics for the top 5 consumers." + iiJ3ngNvJpch-G_1kggvS: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (element_type)(stackstate_stackstate_sync_topo_total_counts{cluster_name="${cluster}", + kube_app_component=~"sync|server", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability"}) + alias: ${element_type} + display: + name: Topology Sync Elements count + description: "Total count of topology elements synchronized." + bLeT4qQjqsZBvePCyyctd: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: percentunit + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (element_type) + (avg_over_time(stackstate_stackstate_receiver_unique_element_passed_count{cluster_name="${cluster}", + namespace="${namespace}"}[${__interval}])) / sum by + (element_type) + (avg_over_time(stackstate_stackstate_receiver_unique_element_passed_max{cluster_name="${cluster}", + namespace="${namespace}"}[${__interval}])) + alias: ${element_type} + display: + name: Profile budget saturation + description: "Ratio of processed elements to the maximum allowed budget." + zf9-XhnHykeb7b_PZPALv: + spec: + plugin: + kind: GaugeChart + spec: + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 0.9 + - color: "#FF7F00" + value: 0.8 + mode: absolute + format: + unit: percentunit + decimalPlaces: 0 + calculation: last + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (element_type) + (avg_over_time(stackstate_stackstate_receiver_unique_element_passed_count{cluster_name="${cluster}", + namespace="${namespace}"}[${__interval}])) / sum by + (element_type) + (avg_over_time(stackstate_stackstate_receiver_unique_element_passed_max{cluster_name="${cluster}", + namespace="${namespace}"}[${__interval}])) + alias: ${element_type} + display: + name: Profile budget saturation + description: "Ratio of processed elements to the maximum allowed budget." + hExuFO5EwYoceUpJ9gHv4: + spec: + plugin: + kind: GaugeChart + spec: + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 50 + - color: "#FF7F00" + value: 30 + mode: absolute + max: 60 + format: + unit: s + decimalPlaces: 1 + calculation: last + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max + (stackstate_stackstate_sync_stage_latency_seconds_quantile{start_stage="enterReceiver", + end_stage="commit", quantile="1.0", + cluster_name="${cluster}", namespace="${namespace}", + kube_app_component=~"sync|server", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability"}) + alias: Latency internal + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max + (stackstate_stackstate_sync_stage_latency_seconds_quantile{start_stage="source", + end_stage="commit", quantile="1.0", + cluster_name="${cluster}", namespace="${namespace}", + kube_app_component=~"sync|server", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability"}) + alias: Latency total + display: + name: Topology Sync Latency + description: "Current internal and total latency for topology synchronization." + bIYe-LnD5LgOMa47Cou8W: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: table + values: + - max + - min + - mean + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by + (element_type)(max_over_time(stackstate_stackstate_receiver_unique_element_passed_count{cluster_name="${cluster}", + namespace="${namespace}", + element_type='agent'}[${__interval}])) + alias: node + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by + (element_type)(max_over_time(stackstate_stackstate_receiver_unique_element_passed_count{cluster_name="${cluster}", + namespace="${namespace}", + element_type!='agent'}[${__interval}])) + alias: ${element_type} + display: + name: Process agent traffic + description: "Traffic volume of unique elements passed by the process agent vs other agents." + GpHAhwMv6nirtbgom2vA5: + spec: + plugin: + kind: Markdown + spec: + text: >- + # Overview + + Topology input at a quick glance. How close to the profile limits + are you and how up-to-date is the topology? + + + These are the pods and workloads involved in the topology + pipeline: + + + * + [Pods](/#/views/urn:stackpack:kubernetes-v2:shared:query-view:pods/overview?query=label%20IN%20%28"app.kubernetes.io%2Fcomponent%3Areceiver"%2C%20"app.kubernetes.io%2Fcomponent%3Areceiver-base"%2C%20"app.kubernetes.io%2Fcomponent%3Areceiver-process-agent"%2C%20"app.kubernetes.io%2Fcomponent%3Async"%2C%20"app.kubernetes.io%2Fcomponent%3Aserver"%2C%20"app.kubernetes.io%2Fname%3Ahbase"%2C%20"app.kubernetes.io%2Fname%3Akafka"%29%20AND%20label%20%3D%20"cluster-name%3A${cluster}"%20AND%20label%20%3D%20"namespace%3A${namespace}"%20AND%20type%20%3D%20"pod") + + * + [Statefulsets](/#/views/urn:stackpack:kubernetes-v2:shared:query-view:statefulsets/overview?query=label%20IN%20%28"app.kubernetes.io%2Fcomponent%3Areceiver"%2C%20"app.kubernetes.io%2Fcomponent%3Areceiver-base"%2C%20"app.kubernetes.io%2Fcomponent%3Areceiver-process-agent"%2C%20"app.kubernetes.io%2Fcomponent%3Async"%2C%20"app.kubernetes.io%2Fcomponent%3Aserver"%2C%20"app.kubernetes.io%2Fname%3Ahbase"%2C%20"app.kubernetes.io%2Fname%3Akafka"%29%20AND%20label%20%3D%20"cluster-name%3A${cluster}"%20AND%20label%20%3D%20"namespace%3A${namespace}"%20AND%20type%20IN%28"statefulset"%29) + + * + [Deployments](/#/views/urn:stackpack:kubernetes-v2:shared:query-view:deployments/overview?query=label%20IN%20%28"app.kubernetes.io%2Fcomponent%3Areceiver"%2C%20"app.kubernetes.io%2Fcomponent%3Areceiver-base"%2C%20"app.kubernetes.io%2Fcomponent%3Areceiver-process-agent"%2C%20"app.kubernetes.io%2Fcomponent%3Async"%2C%20"app.kubernetes.io%2Fcomponent%3Aserver"%2C%20"app.kubernetes.io%2Fname%3Ahbase"%2C%20"app.kubernetes.io%2Fname%3Akafka"%29%20AND%20label%20%3D%20"cluster-name%3A${cluster}"%20AND%20label%20%3D%20"namespace%3A${namespace}"%20AND%20type%20IN%28"deployment"%29) + links: [] + queries: [] + display: + name: "-" + description: "Overview of topology input and links to related workloads." + 4GEucAw6zR3hmI1J-lO4H: + spec: + plugin: + kind: Markdown + spec: + text: >- + # Detailed status + + A more in-depth overview over time of the topology input in volume + and latency. + links: [] + queries: [] + display: + name: "-" + description: "Detailed status of topology input volume and latency over time." + q5ZUHAngpZY0H0SwYKSdt: + spec: + plugin: + kind: StatChart + spec: + calculation: last + format: + unit: short + decimalPlaces: 0 + sparkline: true + thresholds: + defaultColor: "#33A02C" + steps: [] + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (element_type)(stackstate_stackstate_sync_topo_total_counts{cluster_name="${cluster}", + kube_app_component=~"sync|server", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability"}) + alias: ${element_type} + display: + name: Topology Sync Elements count + description: "Total count of topology elements processed." + Q_uoMywjEMcIh5Tp3bAd8: + spec: + plugin: + kind: StatChart + spec: + calculation: last + format: + unit: short + decimalPlaces: 0 + sparkline: true + thresholds: + defaultColor: "#33A02C" + steps: [] + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by + (element_type)(max_over_time(stackstate_stackstate_receiver_unique_element_passed_count{cluster_name="${cluster}", + namespace="${namespace}", + element_type='agent'}[${__interval}])) + alias: node + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by + (element_type)(max_over_time(stackstate_stackstate_receiver_unique_element_passed_count{cluster_name="${cluster}", + namespace="${namespace}", + element_type!='agent'}[${__interval}])) + alias: ${element_type} + display: + name: Incoming data + description: "Maximum count of unique elements received from agents." + d7kCYa0DT6HuQXfjirCoe: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: percent + decimalPlaces: 0 + legend: + show: true + position: right + mode: table + values: + - mean + - last + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: 100 * sum by (kube_app_component) + (container_cpu_throttled_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-base|kafka|sync"}) / + sum by (kube_app_component) + (container_cpu_elapsed_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-base|kafka|sync"}) + alias: ${kube_app_component} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: 100 * sum by (kube_app_name) + (container_cpu_throttled_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_name=~"opentelemetry-collector"}) / sum by + (kube_app_name) + (container_cpu_elapsed_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_name=~"opentelemetry-collector"}) + alias: ${kube_app_name} + display: + name: CPU Throttling + description: CPU throttling for the services that are part of the topology + processing pipeline. + K_Av99mNhjRFhJOD-Gfud: + spec: + plugin: + kind: GaugeChart + spec: + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 0.9 + - color: "#FF7F00" + value: 0.8 + mode: absolute + format: + unit: percentunit + decimalPlaces: 0 + calculation: last + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by + (storage)(label_replace(max_over_time(kubernetes_kubelet_volume_stats_used_bytes{cluster_name="${cluster}", + namespace="${namespace}", + persistentvolumeclaim=~".*-(kafka-\\d|hbase-.*|sync-tmp)"}[5m]) + / + max_over_time(kubernetes_kubelet_volume_stats_capacity_bytes{cluster_name="${cluster}", + namespace="${namespace}", + persistentvolumeclaim=~".*-(kafka-\\d|hbase-.*|sync-tmp)"}[5m]), + "storage","$1", "persistentvolumeclaim", + ".*-(kafka-.*|hbase-.*|sync-tmp)")) + alias: ${storage} + display: + name: Volume usage + description: "Percentage of persistent volume capacity used by Kafka, HBase, and Sync." + layouts: + - kind: Grid + spec: + items: + - x: 0 + y: 9 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/dBMublgJcGkzpzDGSKUvq" + - x: 8 + y: 9 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/zv3RIlz8UQBwAoCBrrpRC" + - x: 16 + y: 9 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/3Py1h-ak88rlmOZpULtQB" + - x: 8 + y: 7 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/iiJ3ngNvJpch-G_1kggvS" + - x: 16 + y: 7 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/bLeT4qQjqsZBvePCyyctd" + - x: 8 + y: 2 + width: 6 + height: 2 + content: + $ref: "#/spec/panels/zf9-XhnHykeb7b_PZPALv" + - x: 19 + y: 2 + width: 5 + height: 2 + content: + $ref: "#/spec/panels/hExuFO5EwYoceUpJ9gHv4" + - x: 0 + y: 7 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/bIYe-LnD5LgOMa47Cou8W" + - x: 0 + y: 0 + width: 24 + height: 2 + content: + $ref: "#/spec/panels/GpHAhwMv6nirtbgom2vA5" + - x: 0 + y: 6 + width: 24 + height: 1 + content: + $ref: "#/spec/panels/4GEucAw6zR3hmI1J-lO4H" + - x: 14 + y: 2 + width: 5 + height: 2 + content: + $ref: "#/spec/panels/q5ZUHAngpZY0H0SwYKSdt" + - x: 0 + y: 2 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/Q_uoMywjEMcIh5Tp3bAd8" + - x: 0 + y: 11 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/d7kCYa0DT6HuQXfjirCoe" + - x: 0 + y: 4 + width: 24 + height: 2 + content: + $ref: "#/spec/panels/K_Av99mNhjRFhJOD-Gfud" + variables: + - kind: ListVariable + spec: + plugin: + kind: MetricLabelValues + spec: + labelName: cluster_name + matchers: + - stackstate_stackstate_receiver_element_create_passed_count + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: cluster + display: + name: Cluster + description: "" + - kind: ListVariable + spec: + plugin: + kind: MetricPromQL + spec: + labelName: namespace + expr: stackstate_stackstate_receiver_element_create_passed_count{cluster_name="${cluster}"} + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: namespace + display: + name: Namespace + description: "" diff --git a/stackpacks/suse-observability/settings/dashboards/traces-processing.sty b/stackpacks/suse-observability/settings/dashboards/traces-processing.sty new file mode 100644 index 0000000..0cfad2a --- /dev/null +++ b/stackpacks/suse-observability/settings/dashboards/traces-processing.sty @@ -0,0 +1,272 @@ +nodes: + - _type: Dashboard + identifier: urn:stackpack:suse-observability:dashboard:traces-processing + name: SUSE Observability - Traces processing + description: "" + dashboard: + metadata: + project: '{"saveVariables":false}' + spec: + panels: + ExAp1uCSS2aYmub4u_Hpt: + spec: + plugin: + kind: StatChart + spec: + calculation: last + format: + unit: "" + decimalPlaces: 0 + sparkline: true + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 5 + - color: "#FF7F00" + value: 1 + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum(increase(stackstate_ClickHouseProfileEvents_DelayedInserts{cluster_name="${cluster}", + namespace="${namespace}"}[1h])) by (cluster_name, namespace) + alias: Delayed inserts (last hour) + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum(increase(stackstate_ClickHouseProfileEvents_RejectedInserts{cluster_name="${cluster}", + namespace="${namespace}"}[1h])) by (cluster_name, namespace) + alias: Rejected inserts (last hour) + display: + name: ClickHouse + description: Traces are stored in ClickHouse, if data is delayed or rejected it + means some traces are later or not at all available. + Rv6ehkvi0i3ws1Z7KA9wf: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: percent + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: 100 * sum by (kube_app_component) + (container_cpu_throttled_periods{cluster_name="${cluster}", + namespace="${namespace}", kube_app_component=~"clickhouse"}) + / sum by (kube_app_component) + (container_cpu_elapsed_periods{cluster_name="${cluster}", + namespace="${namespace}", kube_app_component=~"clickhouse"}) + alias: ${kube_app_component} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: 100 * sum by (kube_app_name) + (container_cpu_throttled_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_name=~"opentelemetry-collector"}) / sum by + (kube_app_name) + (container_cpu_elapsed_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_name=~"opentelemetry-collector"}) + alias: ${kube_app_name} + display: + name: CPU Throttling + description: CPU throttling for the main pods in the data ingestion pipeline. + High throttling can lead to delays in data processing, a profile + upgrade or CPU limit increase can resolve the problem. + wYSjcYqXSXLRvrKEeAgZQ: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (cluster_name, namespace, pod_name, + kube_app_component)(max_over_time(container_cpu_usage{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"clickhouse"}[${__interval}])) / + 1000000000 + alias: ${kube_app_component} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (cluster_name, namespace, + kube_app_name)(max_over_time(container_cpu_usage{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_name="opentelemetry-collector"}[${__interval}])) / + 1000000000 + alias: ${kube_app_name} + display: + name: CPU Usage Processing + description: "CPU usage of ClickHouse and OpenTelemetry collector pods." + 7nAaWHcI1m_yktoqpY5gH: + spec: + plugin: + kind: GaugeChart + spec: + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 0.9 + - color: "#FF7F00" + value: 0.8 + mode: absolute + max: 1 + format: + unit: percentunit + decimalPlaces: 0 + calculation: last + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by + (storage)(label_replace(max_over_time(kubernetes_kubelet_volume_stats_used_bytes{cluster_name="${cluster}", + namespace="${namespace}", + persistentvolumeclaim=~"data-.*-(clickhouse-.*)"}[5m]) / + max_over_time(kubernetes_kubelet_volume_stats_capacity_bytes{cluster_name="${cluster}", + namespace="${namespace}", + persistentvolumeclaim=~"data-.*-(clickhouse-.*)"}[5m]), + "storage","$1", "persistentvolumeclaim", + "data-.*-(clickhouse-.*)")) + alias: ${storage} + display: + name: Volume usage + description: "Percentage of persistent volume capacity used by ClickHouse." + HgyHHI7knLfl4MHuvyvbF: + spec: + plugin: + kind: Markdown + spec: + text: >- + # More details + + These are the pods and workloads involved in the metrics pipeline: + + + * + [Pods](/#/views/urn:stackpack:kubernetes-v2:shared:query-view:pods/overview?query=label%20IN%20%28"app.kubernetes.io%2Fname%3Aclickhouse"%2C%20"app.kubernetes.io%2Fname%3Aopentelemetry-collector"%29%20AND%20label%20%3D%20"cluster-name%3A${cluster}"%20AND%20label%20%3D%20"namespace%3A${namespace}"%20AND%20type%20%3D%20"pod") + + * + [Statefulsets](/#/views/urn:stackpack:kubernetes-v2:shared:query-view:statefulsets/overview?query=label%20IN%20%28"app.kubernetes.io%2Fname%3Aclickhouse"%2C%20"app.kubernetes.io%2Fname%3Aopentelemetry-collector"%29%20AND%20label%20%3D%20"cluster-name%3A${cluster}"%20AND%20label%20%3D%20"namespace%3A${namespace}"%20AND%20type%20IN%28"statefulset"%29) + links: [] + queries: [] + display: + name: "-" + description: "Links to related Pods and StatefulSets." + layouts: + - kind: Grid + spec: + items: + - x: 0 + y: 0 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/ExAp1uCSS2aYmub4u_Hpt" + - x: 0 + y: 2 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/Rv6ehkvi0i3ws1Z7KA9wf" + - x: 8 + y: 2 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/wYSjcYqXSXLRvrKEeAgZQ" + - x: 8 + y: 0 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/7nAaWHcI1m_yktoqpY5gH" + - x: 16 + y: 0 + width: 5 + height: 2 + content: + $ref: "#/spec/panels/HgyHHI7knLfl4MHuvyvbF" + variables: + - kind: ListVariable + spec: + plugin: + kind: MetricLabelValues + spec: + labelName: cluster_name + matchers: + - stackstate_stackstate_receiver_element_create_passed_count + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: cluster + display: + name: Cluster + description: "" + - kind: ListVariable + spec: + plugin: + kind: MetricPromQL + spec: + labelName: namespace + expr: stackstate_stackstate_receiver_element_create_passed_count{cluster_name="${cluster}"} + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: namespace + display: + name: Namespace + description: "" diff --git a/stackpacks/suse-observability/settings/generic/monitors/sts-apps-overrun-cpu-request.sty b/stackpacks/suse-observability/settings/generic/monitors/sts-apps-overrun-cpu-request.sty new file mode 100644 index 0000000..7ec3fb2 --- /dev/null +++ b/stackpacks/suse-observability/settings/generic/monitors/sts-apps-overrun-cpu-request.sty @@ -0,0 +1,24 @@ +nodes: + - _type: "Monitor" + name: "Overrunning requested CPU" + tags: + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:sts-apps-overrun-cpu-request + status: "ENABLED" + description: "Critical state when average CPU usage is consistently more than 100% higher than requested CPU." + arguments: + titleTemplate: "Overrunning requested CPU ${container}" + comparator: "GT" + failureState: "DEVIATING" + threshold: 2 + metric: + query: |- + (sum(avg_over_time(container_cpu_usage{kube_app_name=~"stackstate|stackstate-k8s|suse-observability"}[30m])) by (pod_name, namespace, cluster_name, container) / 1000000000) / sum(avg_over_time(kubernetes_cpu_requests{kube_app_name=~"stackstate|stackstate-k8s|suse-observability"}[30m])) by (pod_name, namespace, cluster_name, container) + or + (sum(avg_over_time(container_cpu_usage{kube_app_part_of=~"stackstate|stackstate-k8s|suse-observability"}[30m])) by (pod_name, namespace, cluster_name, container) / 1000000000) / sum(avg_over_time(kubernetes_cpu_requests{kube_app_part_of=~"stackstate|stackstate-k8s|suse-observability"}[30m])) by (pod_name, namespace, cluster_name, container) + unit: percentunit + aliasTemplate: "CPU usage as percentage of requested" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include generic/sts-apps-overrun-cpu-request.md.hbs diff --git a/stackpacks/suse-observability/settings/generic/monitors/sts-java-long-gc.sty b/stackpacks/suse-observability/settings/generic/monitors/sts-java-long-gc.sty new file mode 100644 index 0000000..f2bf67b --- /dev/null +++ b/stackpacks/suse-observability/settings/generic/monitors/sts-java-long-gc.sty @@ -0,0 +1,80 @@ +nodes: + - _type: "Monitor" + name: "Long GCs" + tags: + - jvm + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:sts-java-long-gc + status: "ENABLED" + description: "Warn when the time of old-gen garbage collects gets above a certain threshold." + arguments: + comparator: "GT" + titleTemplate: "Java long GCs" + failureState: "DEVIATING" + threshold: 4 + metric: + query: |- + sum by (pod_name, namespace, cluster_name) ( + (rate(stackstate_jvm_gc_collection_seconds_sum{gc="G1 Old Generation"}[5m]) + / + rate(stackstate_jvm_gc_collection_seconds_count{gc="G1 Old Generation"}[5m])) + or (present_over_time(stackstate_jvm_gc_collection_seconds_count{gc="G1 Old Generation"}[5m]) - 1)) + or + sum by (pod_name, namespace, cluster_name) ( + (rate(stackstate_jvm_gc_collection_seconds_sum{gc="G1 Old Generation"}[5m]) + / + rate(stackstate_jvm_gc_collection_seconds_count{gc="G1 Old Generation"}[5m]) + ) or (present_over_time(stackstate_jvm_gc_collection_seconds_count{gc="G1 Old Generation"}[5m]) - 1) + ) + aliasTemplate: "${gc}" + unit: s + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include generic/sts-java-long-gc.md.hbs + - _type: "Monitor" + name: "Total time spend in Old Gen GC" + tags: + - jvm + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:sts-java-gc-time-spent + status: "ENABLED" + description: "Deviating when the amount of time spent on old generation garbagecollect exceeds a threshold." + arguments: + comparator: "GT" + titleTemplate: "Java GC time spent" + failureState: "DEVIATING" + threshold: 0.2 + metric: + query: |- + sum by (pod_name, namespace, cluster_name) (rate(stackstate_jvm_gc_collection_seconds_sum{gc="G1 Old Generation"}[5m])) + or + sum by (pod_name, namespace, cluster_name) (rate(stackstate_jvm_gc_collection_seconds_sum{gc="G1 Old Generation"}[5m])) + aliasTemplate: "${gc}" + unit: s + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include generic/sts-java-long-gc.md.hbs + - _type: "Monitor" + name: "Java threads deadlocked" + tags: + - jvm + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:sts-java-threads-deadlocked + status: "ENABLED" + description: "Critical state when there are deadlocked threads in the JVM." + arguments: + comparator: "GT" + titleTemplate: "Java threads deadlocked" + failureState: "CRITICAL" + threshold: 1.5 + metric: + query: |- + sum by (pod_name, namespace, cluster_name) (max_over_time(stackstate_jvm_threads_deadlocked{}[5m])) + aliasTemplate: "# threads deadlocked" + unit: short + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: "Deadlocked threads indicate a problem within the application that needs to be resolved via a code change. A short-term, temporary, mitigation is to restart the application but the deadlock may return. Depending on the likelihood of the deadlock it may return immediately or only after a long period of time." diff --git a/stackpacks/suse-observability/settings/generic/presentations/deployments.sty b/stackpacks/suse-observability/settings/generic/presentations/deployments.sty new file mode 100644 index 0000000..abc35ee --- /dev/null +++ b/stackpacks/suse-observability/settings/generic/presentations/deployments.sty @@ -0,0 +1,342 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability deployments" + description: "ComponentPresentation for SUSE Observability Deployments" + identifier: urn:stackpack:suse-observability:component-presentation:deployments + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/part-of:stackstate", "app.kubernetes.io/part-of:stackstate-k8s", "app.kubernetes.io/name:suse-observability", "app.kubernetes.io/part-of:suse-observability") and type in ("deployment", "statefulset") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: jvm + title: JVM + order: 100 + sections: + - + sectionId: gc + title: GC + order: 100 + metrics: + - metricId: sts-java-gc-count-deployments + - metricId: java-gc-duration-deployments + - metricId: java-gc-total-deployments + - metricId: agent-otel-jvm-gc-durations-95th-quantile-deployments + - metricId: agent-otel-jvm-gc-per-second-deployments + - metricId: agent-otel-jvm-mem-after-last-gc-deployments + - + sectionId: memory + title: Memory + order: 90 + metrics: + - metricId: sts-java-heap-memory-deployments + - metricId: sts-java-nonheap-memory-deployments + - metricId: agent-otel-jvm-heap-memory-deployments + - metricId: agent-otel-jvm-non-heap-memory-deployments + - + sectionId: threads + title: Threads + order: 80 + metrics: + - metricId: sts-java-threads-deployments + metrics: + - + metricId: java-gc-duration-deployments + description: 'Java GC duration' + name: Java GC Duration + metricQueries: + - expression: |- + max by (gc, pod_name, cluster_name, namespace) (rate(stackstate_jvm_gc_collection_seconds_sum{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}]) / rate(stackstate_jvm_gc_collection_seconds_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: ${gc} - ${pod_name} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: java-gc-total-deployments + description: 'Java total GC' + name: Java GC Total + metricQueries: + - expression: |- + max by (gc, pod_name, cluster_name, namespace) (rate(stackstate_jvm_gc_collection_seconds_sum{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: ${gc} - ${pod_name} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-java-gc-count-deployments + description: 'Java GCs/s' + name: Java GC count + metricQueries: + - expression: |- + max by (gc, pod_name, cluster_name, namespace) (rate(stackstate_jvm_gc_collection_seconds_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: ${gc} - ${pod_name} + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: sts-java-heap-memory-deployments + description: 'Java Heap Memory' + name: Java Heap Memory + metricQueries: + - expression: |- + max by (pod_name, cluster_name, namespace) (max_over_time(stackstate_jvm_memory_used_bytes{ + area="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) or max by (pod_name, cluster_name, namespace) (max_over_time(stackstate_jvm_memory_bytes_used{ + area="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Used - ${pod_name} + - expression: |- + max by (pod_name, cluster_name, namespace) (max_over_time(stackstate_jvm_memory_max_bytes{ + area="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) or max by (pod_name, cluster_name, namespace) (max_over_time(stackstate_jvm_memory_bytes_max{ + area="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Max - ${pod_name} + - expression: |- + max by (pod_name, cluster_name, namespace) (max_over_time(stackstate_jvm_memory_committed_bytes{ + area="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) or max by (pod_name, cluster_name, namespace) (max_over_time(stackstate_jvm_memory_bytes_committed{ + area="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Committed - ${pod_name} + chart: + _type: TimeSeriesChart + unit: bytes + - + metricId: sts-java-nonheap-memory-deployments + description: 'Java Non-Heap Memory' + name: Java Non-Heap Memory + metricQueries: + - expression: |- + max by (pod_name, cluster_name, namespace) (max_over_time(stackstate_jvm_memory_used_bytes{ + area="nonheap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) or max by (pod_name, cluster_name, namespace) (max_over_time(stackstate_jvm_memory_bytes_used{ + area="nonheap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Used - ${pod_name} + - expression: |- + max by (pod_name, cluster_name, namespace) (max_over_time(stackstate_jvm_memory_max_bytes{ + area="nonheap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) or max by (pod_name, cluster_name, namespace) (max_over_time(stackstate_jvm_memory_bytes_max{ + area="nonheap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Max - ${pod_name} + - expression: |- + max by (pod_name, cluster_name, namespace) (max_over_time(stackstate_jvm_memory_committed_bytes{ + area="nonheap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) or max by (pod_name, cluster_name, namespace) (max_over_time(stackstate_jvm_memory_bytes_committed{ + area="nonheap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Committed - ${pod_name} + chart: + _type: TimeSeriesChart + unit: bytes + - + metricId: sts-java-threads-deployments + description: 'Java Threads' + name: Java Threads + metricQueries: + - expression: |- + sum by (pod_name, namespace, cluster_name) (max_over_time(stackstate_jvm_threads_current{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Current - ${pod_name} + - expression: |- + sum by (pod_name, namespace, cluster_name) (max_over_time(stackstate_jvm_threads_current{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Peak - ${pod_name} + - expression: |- + sum by (pod_name, namespace, cluster_name) (max_over_time(stackstate_jvm_threads_deadlocked{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Deadlocked - ${pod_name} + chart: + _type: TimeSeriesChart + unit: short + - + metricId: agent-otel-jvm-gc-durations-95th-quantile-deployments + description: 'GC operations that are slower than 95% of total gc operations.' + name: JVM GC Durations 95th Quantile + metricQueries: + - expression: |- + histogram_quantile(0.95, (rate(stackstate_jvm_gc_duration_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}]))) + alias: GC Name - ${jvm_gc_name} - pod - ${pod_name} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: agent-otel-jvm-gc-per-second-deployments + description: 'Garbage collections per second.' + name: JVM GC Per Second + metricQueries: + - expression: |- + rate(stackstate_jvm_gc_duration_seconds_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}]) + alias: GC Name - ${jvm_gc_name} - pod - ${pod_name} + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: agent-otel-jvm-mem-after-last-gc-deployments + description: 'JVM memory size after last garbage collection.' + name: JVM Mem After Last GC + metricQueries: + - expression: |- + stackstate_jvm_memory_used_after_last_gc_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + } + alias: Memory Pool - ${jvm_memory_pool_name} - pod - ${pod_name} + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: agent-otel-jvm-heap-memory-deployments + description: 'JVM heap memory.' + name: JVM Heap Memory + metricQueries: + - expression: |- + stackstate_jvm_memory_used_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + jvm_memory_type="heap" + } + alias: Memory Pool - ${jvm_memory_pool_name} - pod - ${pod_name} + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: agent-otel-jvm-non-heap-memory-deployments + description: 'JVM Non-Heap memory.' + name: JVM Non-Heap Memory + metricQueries: + - expression: |- + stackstate_jvm_memory_used_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + jvm_memory_type="non_heap" + } + alias: Memory Pool - ${jvm_memory_pool_name} - pod - ${pod_name} + chart: + _type: TimeSeriesChart + unit: bytes(IEC) diff --git a/stackpacks/suse-observability/settings/generic/presentations/overview.sty b/stackpacks/suse-observability/settings/generic/presentations/overview.sty new file mode 100644 index 0000000..9bcbe2d --- /dev/null +++ b/stackpacks/suse-observability/settings/generic/presentations/overview.sty @@ -0,0 +1,63 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Stackgraph overview" + description: "ComponentPresentation for SUSE Observability overview" + identifier: urn:stackpack:suse-observability:component-presentation:overview + binding: + _type: ComponentPresentationQueryBinding + query: label in ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/part-of:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/part-of:stackstate-k8s", "app.kubernetes.io/name:suse-observability", "app.kubernetes.io/part-of:suse-observability") AND type in ("pod", "service", "deployment", "statefulset") + rank: + specificity: 10 + presentation: + icon: !icon suse-observability.svg + overview: + name: + singular: "SUSE Observability component" + plural: "SUSE Observability components" + title: "SUSE Observability" + mainMenu: + group: "SUSE Observability" + order: 1 + columns: + - columnId: "health" + title: "Health" + projection: + _type: "HealthProjection" + value: "healthState" + - columnId: "name" + title: "Name" + projection: + _type: "ComponentLinkProjection" + name: "name" + identifier: "identifiers[0]" + - columnId: "type" + title: "Type" + projection: + _type: "TextProjection" + value: "typeName" + topology: + showIndirectRelations: false + connectedComponents: true + groupedByDomains: true + groupedByLayers: true + groupedByRelations: true + groupingEnabled: true + autoGrouping: true + minimumGroupSize: 2 + filters: + - filterId: "namespace" + displayName: + singular: "namespace" + plural: "namespaces" + filter: + _type: "TagFilter" + tagKey: "namespace" + moreTab: K8s + - filterId: "cluster" + displayName: + singular: "cluster" + plural: "clusters" + filter: + _type: "TagFilter" + tagKey: "cluster-name" + moreTab: K8s diff --git a/stackpacks/suse-observability/settings/generic/presentations/pods.sty b/stackpacks/suse-observability/settings/generic/presentations/pods.sty new file mode 100644 index 0000000..801b637 --- /dev/null +++ b/stackpacks/suse-observability/settings/generic/presentations/pods.sty @@ -0,0 +1,302 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Otel pods" + description: "ComponentPresentation for SUSE Observability Pods" + identifier: urn:stackpack:suse-observability:component-presentation:pods + binding: + _type: ComponentPresentationQueryBinding + query: |- + label IN ( + "app.kubernetes.io/name:stackstate", + "app.kubernetes.io/name:stackstate-k8s", + "app.kubernetes.io/part-of:stackstate", + "app.kubernetes.io/part-of:stackstate-k8s", + "app.kubernetes.io/name:suse-observability", + "app.kubernetes.io/part-of:suse-observability" + ) and type in ("pod") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: jvm + title: JVM + order: 100 + sections: + - + sectionId: gc + title: GC + order: 100 + metrics: + - metricId: sts-java-gc-count-pod + - metricId: java-gc-duration-pod + - metricId: java-gc-total-pod + - metricId: agent-otel-jvm-gc-durations-95th-quantile-pod + - metricId: agent-otel-jvm-mem-after-last-gc-pod + - metricId: agent-otel-jvm-gc-per-second-pod + - + sectionId: memory + title: Memory + order: 90 + metrics: + - metricId: sts-java-heap-memory-pod + - metricId: sts-java-nonheap-memory-pod + - metricId: agent-otel-jvm-heap-memory-pod + - metricId: agent-otel-jvm-non-heap-memory-pod + - + sectionId: threads + title: Threads + order: 80 + metrics: + - metricId: sts-java-threads-pod + metrics: + - + metricId: java-gc-duration-pod + description: 'Java GC duration' + name: Java GC Duration + metricQueries: + - expression: |- + rate(stackstate_jvm_gc_collection_seconds_sum{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) / rate(stackstate_jvm_gc_collection_seconds_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: "${gc}" + chart: + _type: TimeSeriesChart + unit: s + - + metricId: java-gc-total-pod + description: 'Java total GC' + name: Java GC Total + metricQueries: + - expression: |- + rate(stackstate_jvm_gc_collection_seconds_sum{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: "${gc}" + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-java-gc-count-pod + description: 'Java GCs/s' + name: Java GC count + metricQueries: + - expression: |- + rate(stackstate_jvm_gc_collection_seconds_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: "${gc}" + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: sts-java-heap-memory-pod + description: 'Java Heap Memory' + name: Java Heap Memory + metricQueries: + - expression: |- + max_over_time(stackstate_jvm_memory_used_bytes{ + area="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) or max_over_time(stackstate_jvm_memory_bytes_used{ + area="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: Used + - expression: |- + max_over_time(stackstate_jvm_memory_max_bytes{ + area="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) or max_over_time(stackstate_jvm_memory_bytes_max{ + area="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: Max + - expression: |- + max_over_time(stackstate_jvm_memory_committed_bytes{ + area="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) or max_over_time(stackstate_jvm_memory_bytes_committed{ + area="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: Committed + chart: + _type: TimeSeriesChart + unit: bytes + - + metricId: sts-java-nonheap-memory-pod + description: 'Java Non-Heap Memory' + name: Java Non-Heap Memory + metricQueries: + - expression: |- + max_over_time(stackstate_jvm_memory_used_bytes{ + area="nonheap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) or max_over_time(stackstate_jvm_memory_bytes_used{ + area="nonheap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: Used + - expression: |- + max_over_time(stackstate_jvm_memory_max_bytes{ + area="nonheap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) or max_over_time(stackstate_jvm_memory_bytes_max{ + area="nonheap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: Max + - expression: |- + max_over_time(stackstate_jvm_memory_committed_bytes{ + area="nonheap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) or max_over_time(stackstate_jvm_memory_bytes_committed{ + area="nonheap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: Committed + chart: + _type: TimeSeriesChart + unit: bytes + - + metricId: sts-java-threads-pod + description: 'Java Threads' + name: Java Threads + metricQueries: + - expression: |- + sum by (pod_name, namespace, cluster_name) (max_over_time(stackstate_jvm_threads_current{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Current + - expression: |- + sum by (pod_name, namespace, cluster_name) (max_over_time(stackstate_jvm_threads_current{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Peak + - expression: |- + sum by (pod_name, namespace, cluster_name) (max_over_time(stackstate_jvm_threads_deadlocked{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Deadlocked + chart: + _type: TimeSeriesChart + unit: short + - + metricId: agent-otel-jvm-gc-durations-95th-quantile-pod + description: 'GC operations that are slower than 95% of total gc operations.' + name: JVM GC Durations 95th Quantile + metricQueries: + - expression: |- + histogram_quantile(0.95, (rate(stackstate_jvm_gc_duration_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]))) + alias: GC Name - ${jvm_gc_name} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: agent-otel-jvm-gc-per-second-pod + description: 'Garbage collections per second.' + name: JVM GC Per Second + metricQueries: + - expression: |- + rate(stackstate_jvm_gc_duration_seconds_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: GC Name - ${jvm_gc_name} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: agent-otel-jvm-mem-after-last-gc-pod + description: 'JVM memory size after last garbage collection.' + name: JVM Mem After Last GC + metricQueries: + - expression: |- + stackstate_jvm_memory_used_after_last_gc_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + } + alias: Memory Pool - ${jvm_memory_pool_name} + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: agent-otel-jvm-heap-memory-pod + description: 'JVM heap memory.' + name: JVM Heap Memory + metricQueries: + - expression: |- + stackstate_jvm_memory_used_bytes{ + jvm_memory_type="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + } + alias: Memory Pool - ${jvm_memory_pool_name} + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: agent-otel-jvm-non-heap-memory-pod + description: 'JVM Non-Heap memory.' + name: JVM Non-Heap Memory + metricQueries: + - expression: |- + stackstate_jvm_memory_used_bytes{ + jvm_memory_type="non_heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + } + alias: Memory Pool - ${jvm_memory_pool_name} + chart: + _type: TimeSeriesChart + unit: bytes(IEC) diff --git a/stackpacks/suse-observability/settings/generic/presentations/stackgraph-pods.sty b/stackpacks/suse-observability/settings/generic/presentations/stackgraph-pods.sty new file mode 100644 index 0000000..83a61d3 --- /dev/null +++ b/stackpacks/suse-observability/settings/generic/presentations/stackgraph-pods.sty @@ -0,0 +1,95 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Stackgraph pods" + title: "SUSE Observability Stackgraph pods" + description: "ComponentPresentation for SUSE Observability Stackgraph pods" + identifier: urn:stackpack:suse-observability:component-presentation:stackgraph-pods + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("stackstate.com/connects-to-stackgraph:true") and type in ("pod") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: stackgraph + title: StackGraph + order: 100 + metrics: + - metricId: sts-stackgraph-pod-read-cache-utilization + - metricId: sts-stackgraph-pod-read-cache-survival-time + - metricId: sts-stackgraph-pod-transaction-duration + metrics: + - + metricId: sts-stackgraph-pod-read-cache-utilization + description: 'ReadCache space utilization' + name: ReadCache space utilization + metricQueries: + - expression: |- + sum(stackstate_stackgraph_readcache_size_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }) + alias: Used + - expression: |- + sum(stackstate_stackgraph_readcache_max_size_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }) + alias: Max + chart: + _type: TimeSeriesChart + unit: bytes + - + metricId: sts-stackgraph-pod-read-cache-survival-time + description: 'ReadCache entry survival time' + name: ReadCache entry survival time + metricQueries: + - expression: |- + histogram_quantile(0.50, (sum by (le) (rate(stackstate_stackgraph_readcache_retention_time_ms{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[20m])) or sum by (le) (rate(stackstate_stackgraph_readcache_retention_time_ms_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[20m])))) + alias: 0.5 quantile + - expression: |- + histogram_quantile(0.95, (sum by (le) (rate(stackstate_stackgraph_readcache_retention_time_ms{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[20m])) or sum by (le) (rate(stackstate_stackgraph_readcache_retention_time_ms_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[20m])))) + alias: 0.95 quantile + chart: + _type: TimeSeriesChart + unit: ms + - + metricId: sts-stackgraph-pod-transaction-duration + description: 'StackGraph transaction duration' + name: StackGraph transaction duration + metricQueries: + - expression: |- + histogram_quantile(0.98, rate(stackstate_stackgraph_transaction_duration_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + })) + alias: Used - ${name} + chart: + _type: TimeSeriesChart + unit: s \ No newline at end of file diff --git a/stackpacks/suse-observability/settings/generic/presentations/stackgraph-services.sty b/stackpacks/suse-observability/settings/generic/presentations/stackgraph-services.sty new file mode 100644 index 0000000..c9eb8e1 --- /dev/null +++ b/stackpacks/suse-observability/settings/generic/presentations/stackgraph-services.sty @@ -0,0 +1,85 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Stackgraph services" + title: "SUSE Observability Stackgraph services" + description: "ComponentPresentation for SUSE Observability Stackgraph services" + identifier: urn:stackpack:suse-observability:component-presentation:stackgraph-services + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("stackstate.com/connects-to-stackgraph:true") and type in ("deployment", "service") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: stackgraph + title: StackGraph + order: 100 + metrics: + - metricId: sts-stackgraph-deployment-read-cache-utilization + - metricId: sts-stackgraph-deployment-transaction-duration + - metricId: sts-stackgraph-deployment-read-cache-survival-time + metrics: + - + metricId: sts-stackgraph-deployment-read-cache-utilization + description: 'ReadCache space utilization' + name: ReadCache space utilization + metricQueries: + - expression: |- + sum by (pod_name) (stackstate_stackgraph_readcache_size_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }) + alias: Used ${pod_name} + chart: + _type: TimeSeriesChart + unit: bytes + - + metricId: sts-stackgraph-deployment-transaction-duration + description: 'StackGraph transaction duration' + name: StackGraph transaction duration + metricQueries: + - expression: |- + histogram_quantile(0.98, rate(stackstate_stackgraph_transaction_duration_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + })) + alias: Used ${pod_name} - ${name} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-stackgraph-deployment-read-cache-survival-time + description: 'ReadCache entry survival time' + name: ReadCache entry survival time + metricQueries: + - expression: |- + histogram_quantile(0.50, (sum by (pod_name, le) (rate(stackstate_stackgraph_readcache_retention_time_ms{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[20m])) or sum by (pod_name, le) (rate(stackstate_stackgraph_readcache_retention_time_ms_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[20m])))) + alias: .5 quantile - ${pod_name} + chart: + _type: TimeSeriesChart + unit: ms \ No newline at end of file diff --git a/stackpacks/suse-observability/settings/main-menu/main-menu.sty b/stackpacks/suse-observability/settings/main-menu/main-menu.sty new file mode 100644 index 0000000..c1b3a41 --- /dev/null +++ b/stackpacks/suse-observability/settings/main-menu/main-menu.sty @@ -0,0 +1,7 @@ +nodes: + - _type: MainMenuGroup + name: SUSE Observability + iconbase64: !icon suse-observability.svg + defaultOpen: true + items: [] + identifier: "urn:stackpack:suse-observability:main-menu-group:overview" \ No newline at end of file diff --git a/stackpacks/suse-observability/settings/monitors/k8s-resource-collector.sty b/stackpacks/suse-observability/settings/monitors/k8s-resource-collector.sty new file mode 100644 index 0000000..c463ac6 --- /dev/null +++ b/stackpacks/suse-observability/settings/monitors/k8s-resource-collector.sty @@ -0,0 +1,66 @@ +nodes: + - _type: "Monitor" + name: "K8s resource collector - no records emitted" + tags: + - k8s-resource-collector + - otel-k8s-crd + function: urn:stackpack:common:monitor-function:threshold + identifier: "urn:stackpack:suse-observability:monitor:k8s-resource-collector-no-records-emitted" + status: "ENABLED" + description: "Monitors whether the k8s resource collector is emitting CRD or CR log records." + arguments: + metric: + query: sum by (k8s_cluster_name, k8s_namespace_name, k8s_pod_name) (increase(receiver_k8sresource_records_emitted_total[15m])) + unit: "short" + aliasTemplate: "Records emitted in 15m" + comparator: LT + threshold: 1 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${k8s_cluster_name}:${k8s_namespace_name}:pod/${k8s_pod_name}" + titleTemplate: "K8s resource collector - no records emitted" + intervalSeconds: 300 + remediationHint: "Check that the suse-observability-agent k8s-resource-collector pod is running, has Kubernetes API/RBAC access, and can export OTLP telemetry to SUSE Observability." + + - _type: "Monitor" + name: "K8s resource collector - payloads dropped by budget" + tags: + - k8s-resource-collector + - otel-k8s-crd + function: urn:stackpack:common:monitor-function:threshold + identifier: "urn:stackpack:suse-observability:monitor:k8s-resource-collector-payloads-dropped" + status: "ENABLED" + description: "Monitors CR or Kubernetes object payloads dropped because they did not fit the configured total payload budget." + arguments: + metric: + query: sum by (k8s_cluster_name, k8s_namespace_name, k8s_pod_name, source, api_group, kind) (increase(receiver_k8sresource_payloads_dropped_total[5m])) + unit: "short" + aliasTemplate: "Payloads dropped in 5m" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${k8s_cluster_name}:${k8s_namespace_name}:pod/${k8s_pod_name}" + titleTemplate: "K8s resource collector - payloads dropped by budget" + intervalSeconds: 300 + remediationHint: "Review which CRs and Kubernetes objects are being collected. Use the [Metrics Explorer](/#/metrics?metrics=%5B%5B%5B%22histogram_quantile%28%5Cn%20%200.95%2C%5Cn%20%20sum%20by%20%28le%2C%20source%2C%20api_group%2C%20kind%2C%20outcome%29%20%28%5Cn%20%20%20%20rate%28receiver_k8sresource_payload_size_bytes_bucket%7Boutcome%3D%5C%22dropped%5C%22%7D%5B5m%5D%29%5Cn%20%20%29%5Cn%29%22%5D%5D%2C%5B0%2C%5B0%2C%22bytes%22%2C6%2C%5Btrue%2C%22%22%5D%5D%5D%5D) to see the 95th percentile size of dropped payloads by source, API group, and kind. Increase `otel.k8sResourceCollector.dataLimits.maxCrTotalDataSizeBytes` or `otel.k8sResourceCollector.dataLimits.maxObjectTotalDataSizeBytes` only when larger payloads are expected and safe to forward." + + - _type: "Monitor" + name: "K8s resource collector - informer reconcile failures" + tags: + - k8s-resource-collector + - otel-k8s-crd + function: urn:stackpack:common:monitor-function:threshold + identifier: "urn:stackpack:suse-observability:monitor:k8s-resource-collector-informer-reconcile-failures" + status: "ENABLED" + description: "Monitors failed informer reconciliation attempts for CR and static Kubernetes resource watches." + arguments: + metric: + query: sum by (k8s_cluster_name, k8s_namespace_name, k8s_pod_name, kind) (increase(receiver_k8sresource_informer_reconciles_total{outcome="failed"}[5m])) + unit: "short" + aliasTemplate: "Informer reconcile failures in 5m" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${k8s_cluster_name}:${k8s_namespace_name}:pod/${k8s_pod_name}" + titleTemplate: "K8s resource collector - informer reconcile failures" + intervalSeconds: 300 + remediationHint: "Check k8s-resource-collector logs and RBAC. Failed CR informer reconciliation usually means the collector cannot list/watch a discovered resource." diff --git a/stackpacks/suse-observability/settings/services/aad/presentation.sty b/stackpacks/suse-observability/settings/services/aad/presentation.sty new file mode 100644 index 0000000..81147bc --- /dev/null +++ b/stackpacks/suse-observability/settings/services/aad/presentation.sty @@ -0,0 +1,66 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability AAD" + description: "ComponentPresentation for SUSE Observability AAD" + identifier: urn:stackpack:suse-observability:component-presentation:service-aad + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:spotlight") and type = "pod" + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: aad + title: AAD + order: 100 + sections: + - + sectionId: streams + title: Streams + order: 100 + metrics: + - metricId: aad-checked + - metricId: aad-duration + - metricId: aad-throughput + metrics: + - + metricId: aad-checked + description: 'Number of streams checked' + name: AAD - Streams checked + metricQueries: + - expression: |- + sum by (cluster_name, namespace, pod_name)(stackstate_spotlight_streams_checked{ + cluster_name="${tags.singleValue('cluster-name')}", namespace="${tags.singleValue('namespace')}", pod_name="${name}" + }[${__rate_interval}]) + alias: Streams checked + chart: + _type: TimeSeriesChart + unit: short + - + metricId: aad-duration + description: 'Time taken by AAD steps' + name: AAD - 95% Step duration + metricQueries: + - expression: |- + histogram_quantile(0.95, sum by (cluster_name, namespace, step, le)(rate(stackstate_spotlight_step_timings_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", namespace="${tags.singleValue('namespace')}", pod_name="${name}" + }[5m]))) + alias: Step ${step} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: aad-throughput + description: 'Number of AAD steps' + name: AAD - Throughput + metricQueries: + - expression: |- + sum by (cluster_name, namespace, step)(rate(stackstate_spotlight_step_timings_seconds_count{ + cluster_name="${tags.singleValue('cluster-name')}", namespace="${tags.singleValue('namespace')}", pod_name="${name}" + }[5m])) + alias: Step ${step} + chart: + _type: TimeSeriesChart + unit: ops \ No newline at end of file diff --git a/stackpacks/suse-observability/settings/services/api/presentation.sty b/stackpacks/suse-observability/settings/services/api/presentation.sty new file mode 100644 index 0000000..00cd709 --- /dev/null +++ b/stackpacks/suse-observability/settings/services/api/presentation.sty @@ -0,0 +1,92 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability API" + description: "ComponentPresentation for SUSE Observability API" + identifier: urn:stackpack:suse-observability:component-presentation:service-api + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("app.kubernetes.io/component:api", "app.kubernetes.io/component:server", "app.kubernetes.io/component:view-health") and type in ("pod", "deployment") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: topology-api + title: Topology API + order: 100 + metrics: + - metricId: sts-topology-query-duration + - metricId: sts-topology-query-latency + - metricId: sts-topology-query-execution-rate + - metricId: sts-topology-query-timeout-rate + metrics: + - + metricId: sts-topology-query-duration + description: 'Topology Api - Duration of a query execution on the topology query service' + name: Topology Api Query Execution duration + metricQueries: + - expression: |- + histogram_quantile(0.98, rate(stackstate_stackstate_topology_query_execute_duration_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + })) + alias: ${pod_name} - ${query} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-topology-query-latency + description: 'Topology Api - Latency of a query execution on the topology query service' + name: Topology Api Query Execution latency + metricQueries: + - expression: |- + histogram_quantile(0.98, rate(stackstate_stackstate_topology_query_execute_latency_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + })) + alias: ${pod_name} - ${query} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-topology-query-execution-rate + description: 'Topology Api - Number of times we execute the query on the topology query service' + name: Topology Api Query execution rate + metricQueries: + - expression: |- + rate(stackstate_stackstate_topology_query_execute{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }[${__interval}]) + alias: ${pod_name} - ${query} + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-topology-query-timeout-rate + description: 'Topology Api - Number of queries that got rejected due to the queue being full on the topology query service' + name: Topology Api Query timeout rate + metricQueries: + - expression: |- + rate(stackstate_stackstate_topology_query_timeout{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }[${__interval}]) + alias: ${pod_name} - ${query} + chart: + _type: TimeSeriesChart + unit: short \ No newline at end of file diff --git a/stackpacks/suse-observability/settings/services/checks/deployment-presentation.sty b/stackpacks/suse-observability/settings/services/checks/deployment-presentation.sty new file mode 100644 index 0000000..1ff438f --- /dev/null +++ b/stackpacks/suse-observability/settings/services/checks/deployment-presentation.sty @@ -0,0 +1,64 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Checks" + description: "ComponentPresentation for SUSE Observability Checks" + identifier: urn:stackpack:suse-observability:component-presentation:service-checks-service + binding: + _type: ComponentPresentationQueryBinding + query: |- + label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") + and label IN ("app.kubernetes.io/component:server", "app.kubernetes.io/component:checks") + and type in ("deployment", "service") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: monitors + title: Monitors + order: 100 + metrics: + - metricId: sts-checks-deployment-active-monitors-count + - metricId: sts-checks-deployment-failing-monitors + metrics: + - + metricId: sts-checks-deployment-active-monitors-count + description: 'Active monitors' + name: Active monitors count + metricQueries: + - expression: | + count(count without (monitorResult) (stackstate_stackstate_monitor_last_run_timestamp { + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + })) + alias: Active monitors + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-checks-deployment-failing-monitors + description: 'Failing monitors' + name: Failing monitors + metricQueries: + - expression: | + sum by (monitorName) (count_over_time(stackstate_stackstate_monitor_last_run_timestamp { + monitorResult = "FAILURE", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}]) > 0) + alias: "${monitorName} fail count" + chart: + _type: TimeSeriesChart + unit: short \ No newline at end of file diff --git a/stackpacks/suse-observability/settings/services/checks/monitors.sty b/stackpacks/suse-observability/settings/services/checks/monitors.sty new file mode 100644 index 0000000..fb0a935 --- /dev/null +++ b/stackpacks/suse-observability/settings/services/checks/monitors.sty @@ -0,0 +1,50 @@ +nodes: + - _type: "Monitor" + name: "Failure percentage" + tags: + - monitors + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:monitors-failure-percentage-pod + arguments: + metric: + query: |- + ( + (sum by(cluster_name, namespace, pod_name, monitorName) (changes(stackstate_stackstate_monitor_last_run_timestamp{monitorResult="FAILURE"}[10m])) + / (sum by(cluster_name, namespace, pod_name, monitorName) (changes(stackstate_stackstate_monitor_last_run_timestamp{}[10m])))) + and (sum by(cluster_name, namespace, pod_name, monitorName) (changes(stackstate_stackstate_monitor_last_run_timestamp{}[10m]) > 5)) + ) + unit: "percentunit" + aliasTemplate: "Failed run %" + comparator: GT + threshold: 0.3 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Monitor ${monitorName} execution" + intervalSeconds: 30 + remediationHint: !include services/checks/monitors-failure-percentage-pod.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "Monitor execution time" + description: "Monitor that triggers when a monitor takes so long to evaluate that it cannot keep up with a one minute interval" + tags: + - monitors + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:monitors-execution-time-pod + arguments: + metric: + query: |- + sum by(cluster_name, namespace, pod_name, monitorName) (rate(stackstate_stackstate_monitor_latency_seconds_sum[10m])) + / sum by(cluster_name, namespace, pod_name, monitorName) (rate(stackstate_stackstate_monitor_latency_seconds_count[10m])) + unit: "s" + aliasTemplate: "Mean run duration" + comparator: GT + threshold: 60 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Monitor ${monitorName} execution time" + intervalSeconds: 60 + remediationHint: !include services/checks/monitors-execution-time-pod.md.hbs + status: "ENABLED" diff --git a/stackpacks/suse-observability/settings/services/checks/pod-presentation.sty b/stackpacks/suse-observability/settings/services/checks/pod-presentation.sty new file mode 100644 index 0000000..22d525c --- /dev/null +++ b/stackpacks/suse-observability/settings/services/checks/pod-presentation.sty @@ -0,0 +1,61 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Checks" + title: "SUSE Observability Checks" + description: "ComponentPresentation for SUSE Observability Checks" + identifier: urn:stackpack:suse-observability:component-presentation:service-checks-pods + binding: + _type: ComponentPresentationQueryBinding + query: |- + label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") + and label IN ("app.kubernetes.io/component:server", "app.kubernetes.io/component:checks") + and type in ("pod") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: monitors + title: Monitors + order: 100 + metrics: + - metricId: sts-checks-pod-active-monitors-count + - metricId: sts-checks-pod-failing-monitors + metrics: + - + metricId: sts-checks-pod-active-monitors-count + description: 'Active monitors' + name: Active monitors count + metricQueries: + - expression: | + count(count without (monitorResult) (stackstate_stackstate_monitor_last_run_timestamp { + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + })) + alias: Active monitors + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-checks-pod-failing-monitors + description: 'Failing monitors' + name: Failing monitors + metricQueries: + - expression: | + sum by (monitorName) (count_over_time(stackstate_stackstate_monitor_last_run_timestamp { + monitorResult = "FAILURE", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) > 0) + alias: "${monitorName} fail count" + chart: + _type: TimeSeriesChart + unit: short \ No newline at end of file diff --git a/stackpacks/suse-observability/settings/services/clickhouse/monitors.sty b/stackpacks/suse-observability/settings/services/clickhouse/monitors.sty new file mode 100644 index 0000000..85f8e4f --- /dev/null +++ b/stackpacks/suse-observability/settings/services/clickhouse/monitors.sty @@ -0,0 +1,358 @@ +nodes: + - _type: "Monitor" + name: "ClickHouse - backups performed" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:clickhouse-backups-not-executed + arguments: + metric: + query: |- + sum(increase(stackstate_clickhouse_backup_successful_uploads{kube_app_name="clickhouse"}[12h])) by (cluster_name, namespace, statefulset) or sum(stackstate_clickhouse_backup_number_backups_remote_expected{kube_app_name="clickhouse"}) by (cluster_name, namespace, statefulset) * 0 + unit: "short" + aliasTemplate: "Number of backups execution in 12h window" + comparator: LTE + threshold: 0 + failureState: "CRITICAL" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:statefulset/${statefulset}" + titleTemplate: "Backup performed in the last 12 hours" + intervalSeconds: 3600 + remediationHint: "Potential reasons why backups haven't been executed:\n- wrong configuration of MinIO (like secrets, bucket name and so on)\n- interval between backups is longer than 12h (you have to change the configuration of the Monitor)\n\nTo investigate the problem you should check:\n- logs of `backup` container\n- logs/status of `MinIO` pods" + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - too many Distributed Files to insert" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:clickhouse-distributed-files-to-insert-high + arguments: + metric: + query: sum(stackstate_ClickHouseMetrics_DistributedFilesToInsert) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of pending files to process for asynchronous insertion into Distributed tables" + comparator: GT + threshold: 50 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - has too many Distributed Files to insert" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-distributed-files-to-insert-high.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - rejected insert" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:clickhouse-rejected-insert + arguments: + metric: + query: sum(increase(stackstate_ClickHouseProfileEvents_RejectedInserts[1m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of times the INSERT of a block to a MergeTree table was rejected" + comparator: GT + threshold: 0 + failureState: "CRITICAL" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - rejected INSERT statements occurred" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-rejected-insert.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - delayed insert" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:clickhouse-delayed-insert-throttling + arguments: + metric: + query: sum(increase(stackstate_ClickHouseProfileEvents_DelayedInserts[1m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of times the INSERT of a block to a MergeTree table was throttled due to high number of active data parts for partition" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - delayed INSERT statements occurred" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-delayed-insert-throttling.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - too many parts per partition" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:clickhouse-max-part-count-for-partition + arguments: + metric: + query: sum (stackstate_ClickHouseAsyncMetrics_MaxPartCountForPartition) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Maximum number of parts per partition across all partitions of all tables of MergeTree family" + comparator: GT + threshold: 300 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - too many parts per a partition" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-max-part-count-for-partition.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - readonly replica" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:clickhouse-readonly-replica + arguments: + metric: + query: sum (stackstate_ClickHouseMetrics_ReadonlyReplica) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of Replicated tables that are currently in readonly state" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - readOnly replica occurred" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-readonly-replica.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - too many connections" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:clickhouse-too-many-connections + arguments: + metric: + query: sum (stackstate_ClickHouseMetrics_HTTPConnection + stackstate_ClickHouseMetrics_TCPConnection + stackstate_ClickHouseMetrics_MySQLConnection) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of active connections" + comparator: GT + threshold: 100 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - too many connections to ClickHouse" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-too-many-connections.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - too many queries" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:clickhouse-too-many-running-queries + arguments: + metric: + query: sum ((stackstate_ClickHouseMetrics_Query - stackstate_ClickHouseMetrics_PendingAsyncInsert) or stackstate_ClickHouseMetrics_Query) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of executing queries" + comparator: GT + threshold: 80 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - too many running queries" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-too-many-running-queries.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - ZooKeeper hardware exceptions" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:clickhouse-zookeeper-hardware-exceptions + arguments: + metric: + query: sum(increase(stackstate_ClickHouseProfileEvents_ZooKeeperHardwareExceptions[1m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of exceptions while working with ZooKeeper" + comparator: GT + threshold: 0 + failureState: "CRITICAL" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - too many exceptions of ZooKeeper" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-zookeeper-hardware-exceptions.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - too many ZooKeeper session" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:clickhouse-zookeeper-session + arguments: + metric: + query: max(stackstate_ClickHouseMetrics_ZooKeeperSession) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of sessions (connections) to ZooKeeper" + comparator: GT + threshold: 1 + failureState: "CRITICAL" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - too many ZooKeeper sessions" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-zookeeper-session.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - replicated part checks failed" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:clickhouse-replicated-part-checks-failed + arguments: + metric: + query: sum(increase(stackstate_ClickHouseProfileEvents_ReplicatedPartChecksFailed[1m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of times the advanced search for a data part on replicas did not give result or when unexpected part has been found and moved away" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - advanced search for a data part on replicas did not give result or when unexpected part has been found and moved away occurred" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-replicated-part-checks-failed.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - replicated part fetches failed" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:clickhouse-replicated-part-failed-fetches + arguments: + metric: + query: sum(increase(stackstate_ClickHouseProfileEvents_ReplicatedPartFailedFetches[1m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of times a data part was failed to download from replica of a ReplicatedMergeTree table" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - data part was failed to download from replica of a ReplicatedMergeTree table" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-clickhouse-replicated-part-failed-fetches.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - replicated data loss" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:clickhouse-replicated-data-loss + arguments: + metric: + query: sum(increase(stackstate_ClickHouseProfileEvents_ReplicatedDataLoss[1m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of times a data part that we wanted doesn't exist on any replica" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - data part that we wanted doesn't exist on any replica" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-replicated-data-loss.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - data after merge differs from replica" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:clickhouse-data-after-merge-differs-from-replica + arguments: + metric: + query: sum(increase(stackstate_ClickHouseProfileEvents_DataAfterMergeDiffersFromReplica[1m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of times data after merge is not byte-identical to the data on another replicas" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - data after merge is not byte-identical to the data on another replicas" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-data-after-merge-differs-from-replica.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - distributed sync insertion timeout" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:clickhouse-distributed-sync-insertion-timeout-exceeded + arguments: + metric: + query: sum(increase(stackstate_ClickHouseProfileEvents_DistributedSyncInsertionTimeoutExceeded[1m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of times a timeout has exceeded while waiting for shards during synchronous insertion into a Distributed table" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - a timeout has exceeded while waiting for shards during synchronous insertion into a Distributed table" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-distributed-sync-insertion-timeout-exceeded.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - file descriptor buffer read or write failed" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:clickhouse-file-descriptor-buffer-read-or-write-failed + arguments: + metric: + query: sum(increase(stackstate_ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadFailed[1m]) + increase(stackstate_ClickHouseProfileEvents_WriteBufferFromFileDescriptorWriteFailed[1m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of times the write or read to a file descriptor have failed" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - write or read operation to a file descriptor have failed occurred" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-file-descriptor-buffer-read-or-write-failed.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - slow reads" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:clickhouse-slow-read + arguments: + metric: + query: sum (increase(stackstate_ClickHouseProfileEvents_SlowRead[1m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of reads from a file that were slow" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - slow reads occurred" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-slow-read.md.hbs + status: "ENABLED" \ No newline at end of file diff --git a/stackpacks/suse-observability/settings/services/clickhouse/pod-presentation.sty b/stackpacks/suse-observability/settings/services/clickhouse/pod-presentation.sty new file mode 100644 index 0000000..fb7bd79 --- /dev/null +++ b/stackpacks/suse-observability/settings/services/clickhouse/pod-presentation.sty @@ -0,0 +1,83 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability ClickHouse" + description: "ComponentPresentation for SUSE Observability ClickHouse" + identifier: urn:stackpack:suse-observability:component-presentation:service-clickhouse-pod + binding: + _type: ComponentPresentationQueryBinding + query: |- + label IN ("statefulset.kubernetes.io/pod-name:stackstate-clickhouse-shard0-0", "statefulset.kubernetes.io/pod-name:suse-observability-clickhouse-shard0-0") + AND type = "pod" + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: backup + title: Backup + order: 100 + metrics: + - metricId: clickhouse-backup-duration-pod + - + sectionId: storage + title: Storage + order: 100 + metrics: + - metricId: clickhouse-storage + metrics: + - + metricId: clickhouse-backup-duration-pod + name: ClickHouse - Backup duration + description: Duration of ClickHouse backups + metricQueries: + - expression: |- + (stackstate_clickhouse_backup_last_create_duration{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + } + stackstate_clickhouse_backup_last_upload_duration{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }) / 1000000000 + alias: "${kube_app_name}" + chart: + _type: TimeSeriesChart + unit: s + - + metricId: clickhouse-storage + name: Volume usage + description: "Percentage of persistent volume capacity used by ClickHouse." + metricQueries: + - expression: |- + max by (storage)(label_replace( + max_over_time(kubernetes_kubelet_volume_stats_used_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + persistentvolumeclaim=~"data-.*-(clickhouse-.*)" + }[5m]) / max_over_time(kubernetes_kubelet_volume_stats_capacity_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + persistentvolumeclaim=~"data-.*-(clickhouse-.*)" + }[5m]), "storage"," $1", "persistentvolumeclaim", "data-.*-(clickhouse-.*)")) + alias: ${storage} + chart: + _type: GaugeChart + unit: percentunit + decimals: 0 + max: 1 + calculation: last + thresholds: + defaultColor: "#33A02C" + mode: absolute + steps: + - color: "#E31A1C" + value: 0.9 + - color: "#FF7F00" + value: 0.8 \ No newline at end of file diff --git a/stackpacks/suse-observability/settings/services/clickhouse/statefulset-presentation.sty b/stackpacks/suse-observability/settings/services/clickhouse/statefulset-presentation.sty new file mode 100644 index 0000000..e9d7897 --- /dev/null +++ b/stackpacks/suse-observability/settings/services/clickhouse/statefulset-presentation.sty @@ -0,0 +1,84 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability ClickHouse" + title: "SUSE Observability ClickHouse" + description: "ComponentPresentation for SUSE Observability ClickHouse" + identifier: urn:stackpack:suse-observability:component-presentation:service-clickhouse-statefulset + binding: + _type: ComponentPresentationQueryBinding + query: |- + label IN ("statefulset:stackstate-clickhouse-shard0", "statefulset:suse-observability-clickhouse-shard0") + AND type = "statefulset" + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: backup + title: Backup + order: 100 + metrics: + - metricId: clickhouse-backup-duration-statefulset + - + sectionId: storage + title: Storage + order: 100 + metrics: + - metricId: clickhouse-storage + metrics: + - + metricId: clickhouse-backup-duration-statefulset + name: ClickHouse - Backup duration + description: Duration of ClickHouse backups + metricQueries: + - expression: |- + (stackstate_clickhouse_backup_last_create_duration{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + statefulset="${tags.singleValue('statefulset')}" + } + stackstate_clickhouse_backup_last_upload_duration{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + statefulset="${tags.singleValue('statefulset')}" + }) / 1000000000 + alias: "${kube_app_name}" + chart: + _type: TimeSeriesChart + unit: s + - + metricId: clickhouse-storage + name: Volume usage + description: "Percentage of persistent volume capacity used by ClickHouse." + metricQueries: + - expression: |- + max by (storage)(label_replace( + max_over_time(kubernetes_kubelet_volume_stats_used_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + persistentvolumeclaim=~"data-.*-(clickhouse-.*)" + }[5m]) / max_over_time(kubernetes_kubelet_volume_stats_capacity_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + persistentvolumeclaim=~"data-.*-(clickhouse-.*)" + }[5m]), "storage"," $1", "persistentvolumeclaim", "data-.*-(clickhouse-.*)")) + alias: ${storage} + chart: + _type: GaugeChart + unit: percentunit + decimals: 0 + max: 1 + calculation: last + thresholds: + defaultColor: "#33A02C" + mode: absolute + steps: + - color: "#E31A1C" + value: 0.9 + - color: "#FF7F00" + value: 0.8 diff --git a/stackpacks/suse-observability/settings/services/correlate/deployment-presentation.sty b/stackpacks/suse-observability/settings/services/correlate/deployment-presentation.sty new file mode 100644 index 0000000..7e191ca --- /dev/null +++ b/stackpacks/suse-observability/settings/services/correlate/deployment-presentation.sty @@ -0,0 +1,277 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Correlate" + description: "ComponentPresentation for SUSE Observability Correlate" + identifier: urn:stackpack:suse-observability:component-presentation:service-correlate-deployment + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("app.kubernetes.io/component:correlate") and type in ("service", "deployment") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: backup + title: Correlate Performance + order: 100 + metrics: + - metricId: sts-correlate-deployment-total-observations + - metricId: sts-correlate-deployment-internal-latency + - + sectionId: backup + title: Correlate Produced + order: 90 + metrics: + - metricId: sts-correlate-deployment-approximate-count-of-elements-produced + - metricId: sts-correlate-deployment-approximate-count-of-elements-produced-hour + - + sectionId: backup + title: Correlate Details + order: 80 + metrics: + - metricId: sts-correlate-deployment-buffers-occupancy + - metricId: sts-correlate-deployment-http-trace-observations + - metricId: sts-correlate-deployment-connection-observations + - metricId: sts-correlate-deployment-http-trace-observations-cluster + - metricId: sts-correlate-deployment-connection-observations-cluster + metrics: + - + metricId: sts-correlate-deployment-total-observations + name: SUSE Observability Correlate - total observations + description: Total observations at the correlator + metricQueries: + - expression: |- + sum(rate(stackstate_stackstate_correlate_http_observations_added_to_state{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Http + - expression: |- + sum(rate(stackstate_stackstate_correlate_endpoints_added_to_state{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Connections + chart: + _type: TimeSeriesChart + unit: ops + - + metricId: sts-correlate-deployment-internal-latency + description: 'Correlate Internal Latency per data type measured from reception time to correlation finished' + name: Stackstate Correlate - Internal Latency + metricQueries: + - expression: |- + histogram_quantile(0.98, sum by (data_type, le) (rate(stackstate_stackstate_correlate_received_data_latency_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }))) + alias: Latency to ${data_type} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-correlate-deployment-approximate-count-of-elements-produced + name: SUSE Observability Correlate - Approximate count of elements produced + description: Amount of elements produced in the correlator + metricQueries: + - expression: |- + sum by (element_type) (avg_over_time(stackstate_stackstate_correlate_unique_active_element_approx_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__interval}])) + alias: "${element_type}" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-correlate-deployment-approximate-count-of-elements-produced-hour + name: SUSE Observability Correlate - Approximate count of created elements in the last hour + description: Approximate count of new elements create in the past hour + metricQueries: + - expression: |- + sum by (element_type) (avg_over_time(stackstate_stackstate_correlate_element_create_approx_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__interval}])) + alias: "${element_type}" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-correlate-deployment-buffers-occupancy + name: SUSE Observability Correlate - buffers occupancy + description: Amount of elements produced in the correlator + metricQueries: + - expression: |- + sum by (correlation_type) (avg_over_time(stackstate_stackstate_correlate_buffer_occupancy{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__interval}])) + alias: "${correlation_type}" + chart: + _type: TimeSeriesChart + unit: percentunit + - + metricId: sts-correlate-deployment-http-trace-observations + name: SUSE Observability HTTP Correlate - bservation correlation + description: HTTP Trace observations received and processing results + metricQueries: + - expression: |- + sum(rate(stackstate_stackstate_correlate_http_observations_added_to_state{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Total + - expression: |- + sum(rate(stackstate_stackstate_correlate_http_observations_correlated{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Correlated + - expression: |- + sum(rate(stackstate_stackstate_correlate_http_observations_uncorrelated{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Uncorrelated + - expression: |- + sum(rate(stackstate_stackstate_correlate_http_observations_subsumed{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Subsumed + - expression: |- + sum(rate(stackstate_stackstate_correlate_http_observations_data_dropped{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Dropped + chart: + _type: TimeSeriesChart + unit: ops + - + metricId: sts-correlate-deployment-connection-observations + name: SUSE Observability Connection Correlate - observation correlation + description: Layer 4 connection observations received and processing results + metricQueries: + - expression: |- + sum(rate(stackstate_stackstate_correlate_endpoints_added_to_state{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Total + - expression: |- + sum(rate(stackstate_stackstate_correlate_endpoints_correlated{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Correlated + - expression: |- + sum(rate(stackstate_stackstate_correlate_endpoints_uncorrelated{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Uncorrelated + - expression: |- + sum(rate(stackstate_stackstate_correlate_endpoints_data_dropped{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Dropped + - expression: |- + sum(rate(stackstate_stackstate_correlate_ambiguous_endpoints{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Ambiguous + chart: + _type: TimeSeriesChart + unit: ops + - + metricId: sts-correlate-deployment-http-trace-observations-cluster + name: SUSE Observability HTTP Correlations - per cluster + description: HTTP Trace observations received and processing results + metricQueries: + - expression: |- + sum by (agent_cluster) (rate(stackstate_stackstate_correlate_http_observations_added_to_state{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: "${agent_cluster}" + chart: + _type: TimeSeriesChart + unit: ops + - + metricId: sts-correlate-deployment-connection-observations-cluster + name: SUSE Observability Connection Correlate - per cluster + description: Layer 4 connection observations received and processing results + metricQueries: + - expression: |- + sum by (agent_cluster) (rate(stackstate_stackstate_correlate_endpoints_added_to_state{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: "${agent_cluster}" + chart: + _type: TimeSeriesChart + unit: ops \ No newline at end of file diff --git a/stackpacks/suse-observability/settings/services/correlate/monitors.sty b/stackpacks/suse-observability/settings/services/correlate/monitors.sty new file mode 100644 index 0000000..a6fec24 --- /dev/null +++ b/stackpacks/suse-observability/settings/services/correlate/monitors.sty @@ -0,0 +1,43 @@ +nodes: + - _type: "Monitor" + name: "Correlate - Processing latency" + tags: + - correlate + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:sts-pod-correlation-latency + status: "ENABLED" + description: "Critical state when the internal correlation latency is too high. The latency is measured from the time the data is received until a metric is produced." + arguments: + comparator: GT + failureState: "DEVIATING" + threshold: 300.0 + metric: + query: |- + histogram_quantile(0.98, sum by (cluster_name, namespace, pod_name, le)(rate(stackstate_stackstate_correlate_received_data_latency_seconds_bucket{data_type="prometheus_earliest"}[5m]))) + aliasTemplate: "Latency" + unit: s + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/correlate/latency.md.hbs + - _type: "Monitor" + name: "Correlate - Processing latency" + tags: + - correlate + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:sts-correlation-deployment-latency + status: "ENABLED" + description: "Critical state when the internal correlation latency is too high. The latency is measured from the time the data is received until a metric is produced." + arguments: + comparator: GT + failureState: "DEVIATING" + threshold: 300.0 + metric: + query: |- + histogram_quantile(0.98, sum by (cluster_name, namespace, deployment, le)(rate(stackstate_stackstate_correlate_received_data_latency_seconds_bucket{data_type="prometheus_earliest"}[5m]))) + aliasTemplate: "Latency" + unit: s + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:deployment/${deployment}" + intervalSeconds: 30 + remediationHint: !include services/correlate/latency.md.hbs diff --git a/stackpacks/suse-observability/settings/services/correlate/pod-presentation.sty b/stackpacks/suse-observability/settings/services/correlate/pod-presentation.sty new file mode 100644 index 0000000..2911f6e --- /dev/null +++ b/stackpacks/suse-observability/settings/services/correlate/pod-presentation.sty @@ -0,0 +1,209 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Correlate" + description: "ComponentPresentation for SUSE Observability Correlate" + identifier: urn:stackpack:suse-observability:component-presentation:service-correlate-pod + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("app.kubernetes.io/component:correlate") and type in ("pod") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: backup + title: Correlate Performance + order: 100 + metrics: + - metricId: sts-correlate-pod-total-observations + - metricId: sts-correlate-pod-internal-latency + - + sectionId: backup + title: Correlate Produced + order: 90 + metrics: + - metricId: sts-correlate-pod-approximate-count-of-elements-produced + - metricId: sts-correlate-pod-approximate-count-of-elements-produced-hour + - + sectionId: backup + title: Correlate Details + order: 80 + metrics: + - metricId: sts-correlate-pod-buffers-occupancy + - metricId: sts-correlate-pod-http-trace-observations + - metricId: sts-correlate-pod-connection-observations + metrics: + - + metricId: sts-correlate-pod-total-observations + name: SUSE Observability Correlate - total observations + description: Total observations at the correlator + metricQueries: + - expression: |- + sum(rate(stackstate_stackstate_correlate_http_observations_added_to_state{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Http + - expression: |- + sum(rate(stackstate_stackstate_correlate_endpoints_added_to_state{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Connections + chart: + _type: TimeSeriesChart + unit: ops + - + metricId: sts-correlate-pod-internal-latency + description: 'Correlate Internal Latency per data type measured from reception time to correlation finished' + name: Stackstate Correlate - Internal Latency + metricQueries: + - expression: |- + histogram_quantile(0.98, rate(stackstate_stackstate_correlate_received_data_latency_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + })) + alias: Latency to ${data_type} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-correlate-pod-approximate-count-of-elements-produced + name: SUSE Observability Correlate - Approximate count of elements produced + description: Amount of elements produced in the correlator + metricQueries: + - expression: |- + avg_over_time(stackstate_stackstate_correlate_unique_active_element_approx_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__interval}]) + alias: "${element_type}" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-correlate-pod-approximate-count-of-elements-produced-hour + name: SUSE Observability Correlate - Approximate count of created elements in the last hour + description: Approximate count of new elements create in the past hour + metricQueries: + - expression: |- + avg_over_time(stackstate_stackstate_correlate_element_create_approx_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__interval}]) + alias: "${element_type}" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-correlate-pod-buffers-occupancy + name: SUSE Observability Correlate - buffers occupancy + description: Amount of elements produced in the correlator + metricQueries: + - expression: |- + avg_over_time(stackstate_stackstate_correlate_buffer_occupancy{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__interval}]) + alias: "${correlation_type}" + chart: + _type: TimeSeriesChart + unit: percentunit + - + metricId: sts-correlate-pod-http-trace-observations + name: SUSE Observability HTTP Correlate - observation correlation + description: HTTP Trace observations received and processing results + metricQueries: + - expression: |- + sum(rate(stackstate_stackstate_correlate_http_observations_added_to_state{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Total + - expression: |- + sum(rate(stackstate_stackstate_correlate_http_observations_correlated{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Correlated + - expression: |- + sum(rate(stackstate_stackstate_correlate_http_observations_uncorrelated{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Uncorrelated + - expression: |- + sum(rate(stackstate_stackstate_correlate_http_observations_subsumed{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Subsumed + - expression: |- + sum(rate(stackstate_stackstate_correlate_http_observations_data_dropped{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Dropped + chart: + _type: TimeSeriesChart + unit: ops + - + metricId: sts-correlate-pod-connection-observations + name: SUSE Observability Connection Correlate - observation correlation + description: Layer 4 connection observations received and processing results + metricQueries: + - expression: |- + sum(rate(stackstate_stackstate_correlate_endpoints_added_to_state{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Total + - expression: |- + sum(rate(stackstate_stackstate_correlate_endpoints_correlated{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Correlated + - expression: |- + sum(rate(stackstate_stackstate_correlate_endpoints_uncorrelated{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Uncorrelated + - expression: |- + sum(rate(stackstate_stackstate_correlate_endpoints_data_dropped{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Dropped + - expression: |- + sum(rate(stackstate_stackstate_correlate_ambiguous_endpoints{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Ambiguous + chart: + _type: TimeSeriesChart + unit: ops \ No newline at end of file diff --git a/stackpacks/suse-observability/settings/services/es/presentation.sty b/stackpacks/suse-observability/settings/services/es/presentation.sty new file mode 100644 index 0000000..62a1d91 --- /dev/null +++ b/stackpacks/suse-observability/settings/services/es/presentation.sty @@ -0,0 +1,71 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Elasticsearch" + description: "ComponentPresentation for SUSE Observability Elasticsearch" + identifier: urn:stackpack:suse-observability:component-presentation:service-elasticsearch-pod + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:elasticsearch") AND label IN("app.kubernetes.io/part-of:stackstate-k8s", "app.kubernetes.io/part-of:suse-observability") and type in ("service", "pod") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: elasticsearch + title: Elasticsearch + order: 100 + metrics: + - metricId: sts-es-current-index-size + - metricId: sts-es-storage + metrics: + - + metricId: sts-es-current-index-size + name: SUSE Observability ElasticSearch - Current index disk space + description: Amount of data stored on ElasticSearch + metricQueries: + - expression: |- + sum(label_replace(stackstate_elasticsearch_indices_store_size_bytes_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}" + }, "sts_index","$1","index", ".*(sts_k8s_logs|sts_multi_metrics|sts_internal_events|sts_topology_events|sts_generic_events|sts_trace_events|sts_annotations).*")) by (sts_index) / 1000000000 + alias: "${sts_index}" + chart: + _type: TimeSeriesChart + unit: gbytes + - + metricId: sts-es-storage + name: Volume usage + description: "Percentage of persistent volume capacity used by Elasticsearch." + metricQueries: + - expression: |- + max by (storage)(label_replace( + max_over_time(kubernetes_kubelet_volume_stats_used_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + persistentvolumeclaim=~"data-.*-(elasticsearch-master-\\d)" + }[5m]) / max_over_time(kubernetes_kubelet_volume_stats_capacity_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + persistentvolumeclaim=~"data-.*-(elasticsearch-master-\\d)" + }[5m]), "storage", "$1", "persistentvolumeclaim", "data-.*-(elasticsearch-.*)")) + alias: ${storage} + chart: + _type: GaugeChart + unit: percentunit + decimals: 0 + max: 1 + calculation: last + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 0.9 + - color: "#FF7F00" + value: 0.8 + mode: absolute \ No newline at end of file diff --git a/stackpacks/suse-observability/settings/services/hbase/monitors.sty b/stackpacks/suse-observability/settings/services/hbase/monitors.sty new file mode 100644 index 0000000..6d0baf0 --- /dev/null +++ b/stackpacks/suse-observability/settings/services/hbase/monitors.sty @@ -0,0 +1,23 @@ +nodes: + - _type: "Monitor" + name: "HBase slow gets" + tags: + - suse-observability + - hbase + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:sts-hbase-slow-get-count + status: "ENABLED" + description: "Monitors the amount of slow get counts." + arguments: + comparator: GT + titleTemplate: "HBase - slow get count" + failureState: "DEVIATING" + threshold: 0 + metric: + query: |- + sum by (cluster_name, namespace, pod_name)(rate(stackstate_hbase_slowgetcount{}[5m])) + aliasTemplate: "slow get count" + unit: ops/sec (ops) + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/hbase/slow-gets.md.hbs \ No newline at end of file diff --git a/stackpacks/suse-observability/settings/services/health-sync/monitors.sty b/stackpacks/suse-observability/settings/services/health-sync/monitors.sty new file mode 100644 index 0000000..006a284 --- /dev/null +++ b/stackpacks/suse-observability/settings/services/health-sync/monitors.sty @@ -0,0 +1,22 @@ +nodes: + - _type: "Monitor" + name: "Health sync data processing latency" + tags: + - health-sync + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:sts-health-sync-data-processing-latency + status: "ENABLED" + description: "Critical state when the processing latency is too high. The latency is measured from the time the data is received until it is stored as a health state." + arguments: + comparator: GT + failureState: "DEVIATING" + threshold: 30.0 + metric: + query: |- + histogram_quantile(0.98, sum by (cluster_name, namespace, pod_name, le)(rate(stackstate_stackstate_health_sync_latency_seconds_bucket{}[5m]))) + aliasTemplate: "Latency" + unit: s + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/health-sync/latency.md.hbs diff --git a/stackpacks/suse-observability/settings/services/health-sync/presentation.sty b/stackpacks/suse-observability/settings/services/health-sync/presentation.sty new file mode 100644 index 0000000..a13ce15 --- /dev/null +++ b/stackpacks/suse-observability/settings/services/health-sync/presentation.sty @@ -0,0 +1,92 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Elasticsearch" + description: "ComponentPresentation for SUSE Observability Elasticsearch" + identifier: urn:stackpack:suse-observability:component-presentation:service-health-sync-pod + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("app.kubernetes.io/component:health-sync", "app.kubernetes.io/component:server") and type in ("pod", "deployment") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: health-sync + title: Health Sync + order: 100 + metrics: + - metricId: sts-health-sync-latency + - metricId: sts-health-sync-messages-rate + - metricId: sts-health-sync-operations-rate + - metricId: sts-health-sync-kafka-lag + metrics: + - + description: 'Health Sync latency' + name: Health Sync Latency + metricId: sts-health-sync-latency + metricQueries: + - expression: |- + histogram_quantile(0.98, rate(stackstate_stackstate_health_sync_latency_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + })) + alias: ${pod_name} - ${partition} + chart: + _type: TimeSeriesChart + unit: s + - + description: 'Rate of all received check state messages that were received for a stream and substreams' + name: Health Sync Received messages rate + metricId: sts-health-sync-messages-rate + metricQueries: + - expression: |- + stackstate_stackstate_health_sync_all_check_state_message_rate{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + } + alias: ${pod_name} - ${stream} + chart: + _type: TimeSeriesChart + unit: cps + - + description: 'Rate of all check state operations that were performed for the stream and substreams' + name: Health Sync Received operations rate + metricId: sts-health-sync-operations-rate + metricQueries: + - expression: |- + stackstate_health_sync_all_check_state_operation_rate{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + } > 0 + alias: ${pod_name} - ${stream} - ${operation} + chart: + _type: TimeSeriesChart + unit: cps + - + description: 'Kafka consumer lag per topic' + name: Health Sync Topic lag + metricId: sts-health-sync-kafka-lag + metricQueries: + - expression: |- + sum by (pod_name, topic) (stackstate_kafka_consumer_consumer_fetch_manager_metrics_records_lag{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }) + alias: ${pod_name} - ${topic} + chart: + _type: TimeSeriesChart + unit: cps \ No newline at end of file diff --git a/stackpacks/suse-observability/settings/services/kafka-to-es/monitors.sty b/stackpacks/suse-observability/settings/services/kafka-to-es/monitors.sty new file mode 100644 index 0000000..fdcd2c6 --- /dev/null +++ b/stackpacks/suse-observability/settings/services/kafka-to-es/monitors.sty @@ -0,0 +1,45 @@ +nodes: + - _type: "Monitor" + name: "Elasticsearch data ingestion latency" + tags: + - kafka-to-es + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:sts-es-data-ingestion-latency + status: "ENABLED" + description: "Critical state when the ingestion latency is too high. The latency includes the time taken from producing the data to it being received on the SUSE Observability api." + arguments: + comparator: GT + failureState: "DEVIATING" + titleTemplate: "Elasticsearch ingestion latency for ${data_type}" + threshold: 120.0 + metric: + query: |- + histogram_quantile(0.98, sum by (cluster_name, namespace, pod_name, data_type, le)(rate(stackstate_stackstate_kafka2es_data_latency_seconds_bucket{}[5m]))) + aliasTemplate: "${data_type} latency" + unit: s + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/kafka-to-es/latency.md.hbs + - _type: "Monitor" + name: "Elasticsearch data processing latency" + tags: + - kafka-to-es + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:java-es-data-processing-latency + status: "ENABLED" + description: "Critical state when the processing latency is too high. The latency is measured from the time the data is received until it is stored in Elasticsearch." + arguments: + comparator: GT + titleTemplate: "Elasticsearch processing latency for ${data_type}" + failureState: "DEVIATING" + threshold: 30.0 + metric: + query: |- + histogram_quantile(0.98, sum by (cluster_name, namespace, pod_name, data_type, le)(rate(stackstate_stackstate_kafka2es_received_data_latency_seconds_bucket{}[5m]))) + aliasTemplate: "${data_type} latency" + unit: s + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/kafka-to-es/latency.md.hbs \ No newline at end of file diff --git a/stackpacks/suse-observability/settings/services/kafka-to-es/presentation.sty b/stackpacks/suse-observability/settings/services/kafka-to-es/presentation.sty new file mode 100644 index 0000000..259ec79 --- /dev/null +++ b/stackpacks/suse-observability/settings/services/kafka-to-es/presentation.sty @@ -0,0 +1,80 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Kafka2ES" + description: "ComponentPresentation for SUSE Observability Kafka2ES" + identifier: urn:stackpack:suse-observability:component-presentation:service-kafka-to-es-pod + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("app.kubernetes.io/component:trace2es", "app.kubernetes.io/component:mm2es", "app.kubernetes.io/component:e2es") and type in ("pod", "deployment") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: kafka2es + title: Kafka2ES + order: 100 + metrics: + - metricId: sts-topo-k2es-latency + - metricId: sts-topo-k2es-internal-latency + - metricId: sts-topo-k2es-catch-up + metrics: + - + metricId: sts-topo-k2es-latency + description: 'Kafka2Es Latency per data type measured from collection time to database' + name: Kafka2Es Latency + metricQueries: + - expression: |- + histogram_quantile(0.98, rate(stackstate_stackstate_kafka2es_data_latency_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + })) + alias: ${pod_name} - ${data_type} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-topo-k2es-internal-latency + description: 'Kafka2Es Internal Latency per data type measured from reception time to database' + name: Kafka2Es Internal Latency + metricQueries: + - expression: |- + histogram_quantile(0.98, rate(stackstate_stackstate_kafka2es_received_data_latency_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + })) + alias: ${pod_name} - ${data_type} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-topo-k2es-catch-up + description: 'Kafka2Es time to catch up' + name: Kafka2Es Time to catch up + metricQueries: + - expression: |- + histogram_quantile(0.98, sum by(data_type, pod_name, le)(rate(stackstate_stackstate_kafka2es_received_data_latency_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }))) / sum by(data_type, pod_name)(rate(stackstate_stackstate_kafka2es_received_data_latency_seconds_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }[${__interval}]) > 0.05) + alias: ${pod_name} - ${data_type} + chart: + _type: TimeSeriesChart + unit: s \ No newline at end of file diff --git a/stackpacks/suse-observability/settings/services/metrics-store-clients/monitors.sty b/stackpacks/suse-observability/settings/services/metrics-store-clients/monitors.sty new file mode 100644 index 0000000..7fb251b --- /dev/null +++ b/stackpacks/suse-observability/settings/services/metrics-store-clients/monitors.sty @@ -0,0 +1,23 @@ +nodes: + - _type: "Monitor" + name: "Metric store unreachable" + tags: + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:sts-metric-store-unreachable + status: "ENABLED" + description: | + Critical state when the service cannot connect to the metric store. + A circuit breaker can have 3 states, closed (0), open (1) or half-open (2). When closed, metrics queries are working normally, when open there were problems running recent queries. While open queries will be failed immediately without querying the metric store. To test connectivity the circuit breaker will switch regularly to a half-open state where 1 query request is send to the store as a test. If that passes the circuit breaker will close again. + arguments: + comparator: GT + failureState: "CRITICAL" + threshold: 0 + metric: + query: |- + max by (cluster_name, namespace, pod_name)(min_over_time(stackstate_stackstate_metrics_circuit_breaker_status[2m])) + unit: short + aliasTemplate: "Connection status (0 = connected, > 0 = unreachable)" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/metrics-store-clients/metric-store-unreachable.md.hbs diff --git a/stackpacks/suse-observability/settings/services/metrics-store-clients/presentations.sty b/stackpacks/suse-observability/settings/services/metrics-store-clients/presentations.sty new file mode 100644 index 0000000..1f37dcf --- /dev/null +++ b/stackpacks/suse-observability/settings/services/metrics-store-clients/presentations.sty @@ -0,0 +1,130 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Metrics Store Clients - Circuit Breaker" + description: "ComponentPresentation for SUSE Observability Metrics Store Clients - Circuit Breaker" + identifier: urn:stackpack:suse-observability:component-presentation:service-metrics-store-clients-circuit-breaker + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("app.kubernetes.io/component:server", "app.kubernetes.io/component:api", "app.kubernetes.io/component:checks", "app.kubernetes.io/component:sync", "app.kubernetes.io/component:health-sync") and type in ("pod", "deployment") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - sectionId: metrics + title: Metrics + order: 100 + metrics: + - metricId: sts-metric-store-circuit-breaker + metrics: + - + metricId: sts-metric-store-circuit-breaker + description: 'States: closed (0), open (1) or half-open (2)' + name: Metrics client - Circuit Breaker + metricQueries: + - expression: |- + sum by (pod_name)(max_over_time(stackstate_stackstate_metrics_circuit_breaker_status{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }[${__interval}])) + alias: "${pod_name}" + chart: + _type: TimeSeriesChart + unit: short + + - _type: ComponentPresentation + name: "SUSE Observability Metrics Store Clients - PromQL Hit" + title: "SUSE Observability Metrics Store Clients - PromQL Hit" + description: "ComponentPresentation for SUSE Observability Metrics Store Clients - PromQL Hit" + identifier: urn:stackpack:suse-observability:component-presentation:service-metrics-store-clients-promql-hit-deployment + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("app.kubernetes.io/component:server", "app.kubernetes.io/component:api", "app.kubernetes.io/component:checks") and type in ("deployment", "service") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - sectionId: metrics + title: Metrics + order: 100 + metrics: + - metricId: sts-checks-deployment-promql-hit-pct + metrics: + - + metricId: sts-checks-deployment-promql-hit-pct + description: 'PromQL cache hit %' + name: PromQL cache hit % + metricQueries: + - expression: | + sum by (cluster_name, namespace, pod_name) (rate(stackstate_caffeine_cache_hit{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + } )) / sum by (cluster_name, namespace, pod_name) (rate(stackstate_caffeine_cache_requests{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + } )) + alias: hit % - ${pod_name} + componentIdentifierTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + chart: + _type: TimeSeriesChart + unit: percentunit + + - _type: ComponentPresentation + name: "SUSE Observability Metrics Store Clients - PromQL Hit" + title: "SUSE Observability Metrics Store Clients - PromQL Hit" + description: "ComponentPresentation for SUSE Observability Metrics Store Clients - PromQL Hit" + identifier: urn:stackpack:suse-observability:component-presentation:service-metrics-store-clients-promql-hit-pod + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("app.kubernetes.io/component:server", "app.kubernetes.io/component:api", "app.kubernetes.io/component:checks") and type in ("pod") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - sectionId: metrics + title: Metrics + order: 100 + metrics: + - metricId: sts-checks-pod-promql-hit-pct + metrics: + - + metricId: sts-checks-pod-promql-hit-pct + description: 'PromQL cache hit %' + name: PromQL cache hit % + metricQueries: + - expression: | + sum(rate(stackstate_caffeine_cache_hit{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + } )) / sum(rate(stackstate_caffeine_cache_requests{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + } )) + alias: hit % + chart: + _type: TimeSeriesChart + unit: percentunit diff --git a/stackpacks/suse-observability/settings/services/otel-collector/monitors.sty b/stackpacks/suse-observability/settings/services/otel-collector/monitors.sty new file mode 100644 index 0000000..d06fdfe --- /dev/null +++ b/stackpacks/suse-observability/settings/services/otel-collector/monitors.sty @@ -0,0 +1,126 @@ +nodes: + - _type: "Monitor" + name: "OTel Collector - spans refused by a receiver" + tags: + - otel-collector + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:otel-collector-span-refused-receiver + arguments: + metric: + query: sum by (cluster_name, namespace, pod_name) (rate(stackstate_otelcol_receiver_refused_spans[1m])) + unit: "short" + aliasTemplate: "Number of spans that could not be pushed into the pipeline" + comparator: GT + threshold: 1 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "OTel Collector - spans refused by a receiver" + intervalSeconds: 60 + remediationHint: !include services/otel-collector/otel-collector-span-refused-receiver.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "OTel Collector - metric points refused by a receiver" + tags: + - otel-collector + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:otel-collector-metric-refused-receiver + arguments: + metric: + query: sum by (cluster_name, namespace, pod_name) (rate(stackstate_otelcol_receiver_refused_metric_points[1m])) + unit: "short" + aliasTemplate: "Number of metrics that could not be pushed into the pipeline" + comparator: GT + threshold: 1 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "OTel Collector - metric points refused by a receiver" + intervalSeconds: 60 + remediationHint: !include services/otel-collector/otel-collector-metric-refused-receiver.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "OTel Collector - spans export failures" + tags: + - otel-collector + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:otel-collector-span-export-failure + arguments: + metric: + query: sum by (cluster_name, namespace, pod_name) (rate(stackstate_otelcol_exporter_send_failed_spans[1m])) + unit: "short" + aliasTemplate: "Number of spans in failed attempts to send to destination" + comparator: GT + threshold: 1 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "OTel Collector - spans export failures" + intervalSeconds: 60 + remediationHint: !include services/otel-collector/otel-collector-span-export-failure.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "OTel Collector - metric points export failures" + tags: + - otel-collector + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:otel-collector-metric-export-failure + arguments: + metric: + query: sum by (cluster_name, namespace, pod_name) (rate(stackstate_otelcol_exporter_send_failed_metric_points[1m])) + unit: "short" + aliasTemplate: "Number of metric points in failed attempts to send to destination" + comparator: GT + threshold: 1 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "OTel Collector - metric points export failures" + intervalSeconds: 60 + remediationHint: !include services/otel-collector/otel-collector-metric-export-failure.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "OTel Collector - a processor drops spans" + tags: + - otel-collector + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:otel-collector-span-processor-drops + arguments: + metric: + query: sum by (cluster_name, namespace, pod_name) (rate(stackstate_otelcol_processor_dropped_spans[1m])) + unit: "short" + aliasTemplate: "Number of spans that were dropped" + comparator: GT + threshold: 1 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "OTel Collector - a processor drops spans" + intervalSeconds: 60 + remediationHint: !include services/otel-collector/otel-collector-span-processor-drops.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "OTel Collector - a processor drops metric points" + tags: + - otel-collector + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:otel-collector-metric-processor-drops + arguments: + metric: + query: sum by (cluster_name, namespace, pod_name) (rate(stackstate_otelcol_processor_dropped_metric_points[1m])) + unit: "short" + aliasTemplate: "Number of metric points that were dropped" + comparator: GT + threshold: 1 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "OTel Collector - a processor drops metric points" + intervalSeconds: 60 + remediationHint: !include services/otel-collector/otel-collector-metric-processor-drops.md.hbs + status: "ENABLED" \ No newline at end of file diff --git a/stackpacks/suse-observability/settings/services/otel-collector/presentation.sty b/stackpacks/suse-observability/settings/services/otel-collector/presentation.sty new file mode 100644 index 0000000..35206c0 --- /dev/null +++ b/stackpacks/suse-observability/settings/services/otel-collector/presentation.sty @@ -0,0 +1,240 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability OTel Collector" + description: "ComponentPresentation for SUSE Observability OTel Collector" + identifier: urn:stackpack:suse-observability:component-presentation:service-otel-collector-pod + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:opentelemetry-collector") and type = "pod" + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: otel-collector + title: OTel Collector + order: 100 + metrics: + - metricId: otel-collector-accepted-metrics + - metricId: otel-collector-accepted-spans + - + sectionId: topology-connector + title: Topology connector + order: 90 + metrics: + - metricId: otel-collector-connector-topology-topology-produced + - metricId: otel-collector-connector-topology-mappings-removed + - metricId: otel-collector-connector-topology-mapping-errors + - metricId: otel-collector-connector-topology-mapping-duration + - metricId: otel-collector-connector-topology-signal-processing-duration + - + sectionId: settings-provider-extension + title: Settings provider extension + order: 80 + metrics: + - metricId: otel-collector-extension-incomplete-snapshots + - metricId: otel-collector-extension-successful-snapshot-ends + - metricId: otel-collector-extension-settings-count + - metricId: otel-collector-extension-settings-size + metrics: + - + metricId: otel-collector-accepted-metrics + description: 'Number of metric points successfully pushed into the pipeline' + name: OTel Collector - accepted metric points + metricQueries: + - expression: |- + sum by (cluster_name, namespace, pod_name, le)(increase(stackstate_otelcol_receiver_accepted_metric_points{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__rate_interval}])) + alias: Metric points + chart: + _type: TimeSeriesChart + unit: short + - + metricId: otel-collector-accepted-spans + description: 'Number of spans successfully pushed into the pipeline' + name: OTel Collector - accepted spans + metricQueries: + - expression: |- + sum by (cluster_name, namespace, pod_name, le)(increase(stackstate_otelcol_receiver_accepted_spans{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__rate_interval}])) + alias: Spans + chart: + _type: TimeSeriesChart + unit: s + - + metricId: otel-collector-connector-topology-topology-produced + description: 'Total number of topology produced by the topology mapping connector' + name: OTel topology connector - components produced + metricQueries: + - expression: |- + sum by (cluster_name, namespace, pod_name, type, signal)( + rate(stackstate_otelcol_connector_topology_topology_produced_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__rate_interval}]) + ) + alias: Components produced + chart: + _type: TimeSeriesChart + unit: s + - + metricId: otel-collector-connector-topology-mappings-removed + description: 'Total number of mappings removed by the topology mapping connector' + name: OTel topology connector - mappings removed + metricQueries: + - expression: |- + sum by (cluster_name, namespace, pod_name, type)( + rate(stackstate_otelcol_connector_topology_mappings_removed_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__rate_interval}]) + ) + alias: Components produced + chart: + _type: TimeSeriesChart + unit: s + - + metricId: otel-collector-connector-topology-mapping-errors + description: 'Number of mapping errors observed by the topology connector' + name: OTel topology connector - mapping errors + metricQueries: + - expression: |- + sum by (cluster_name, namespace, pod_name, type, signal)( + rate(stackstate_otelcol_connector_topology_mapping_errors_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__rate_interval}]) + ) + alias: Mapping errors + chart: + _type: TimeSeriesChart + unit: s + - + metricId: otel-collector-connector-topology-mapping-duration + description: '95th percentile of mapping input to topology duration in seconds' + name: OTel topology connector - input mapping duration (p95) + metricQueries: + - expression: |- + histogram_quantile(0.95, + sum by (le, cluster_name, namespace, pod_name, signal)( + rate(stackstate_otelcol_connector_topology_mapping_duration_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__rate_interval}]) + ) + ) + alias: P95 Mapping duration + chart: + _type: TimeSeriesChart + unit: s + - + metricId: otel-collector-connector-topology-signal-processing-duration + description: '95th percentile of processing a single input signal request in seconds' + name: OTel topology connector - single input signal processing duration (p95) + metricQueries: + - expression: |- + histogram_quantile(0.95, + sum by (le, cluster_name, namespace, pod_name, signal)( + rate(stackstate_otelcol_connector_topology_mapping_duration_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__rate_interval}]) + ) + ) + alias: P95 Mapping duration + chart: + _type: TimeSeriesChart + unit: s + - + metricId: otel-collector-extension-incomplete-snapshots + description: 'Total number of incomplete setting snapshots (replaced by new snapshot starts messages)' + name: OTel settings provider extension - incomplete snapshots + metricQueries: + - expression: |- + sum by (cluster_name, namespace, pod_name)( + rate(stackstate_otelcol_extension_sts_settings_provider_settings_snapshot_incomplete{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__rate_interval}]) + ) + alias: Incomplete snapshots + chart: + _type: TimeSeriesChart + unit: short + - + metricId: otel-collector-extension-successful-snapshot-ends + description: 'Total number of setting snapshots processed completely' + name: OTel settings provider extension - successful snapshots + metricQueries: + - expression: |- + sum by (cluster_name, namespace, pod_name)( + rate(stackstate_otelcol_extension_sts_settings_provider_settings_snapshot_complete{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__rate_interval}]) + ) + alias: Successful snapshots + chart: + _type: TimeSeriesChart + unit: short + - + metricId: otel-collector-extension-settings-count + description: 'Settings count per snapshot by type' + name: OTel settings provider extension - settings count + metricQueries: + - expression: | + sum by (cluster_name, namespace, pod_name, setting_type)( + rate(stackstate_otelcol_extension_sts_settings_provider_settings_count_by_type_sum{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__rate_interval}]) + ) + alias: Settings count by type + chart: + _type: TimeSeriesChart + unit: settings + - + metricId: otel-collector-extension-settings-size + description: 'Maximum average size of completed settings snapshots (in bytes)' + name: OTel settings provider extension - settings size + metricQueries: + - expression: | + sum by (cluster_name, namespace, setting_type)( + max_over_time(stackstate_otelcol_extension_sts_settings_provider_settings_size_by_type_bytes_sum{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__interval}]) + ) + / + sum by (cluster_name, namespace, setting_type)( + max_over_time(stackstate_otelcol_extension_sts_settings_provider_settings_size_by_type_bytes_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__interval}]) + ) + alias: Max average snapshot size (bytes) + chart: + _type: TimeSeriesChart + unit: By \ No newline at end of file diff --git a/stackpacks/suse-observability/settings/services/receiver/deployment-presentation.sty b/stackpacks/suse-observability/settings/services/receiver/deployment-presentation.sty new file mode 100644 index 0000000..090bfdb --- /dev/null +++ b/stackpacks/suse-observability/settings/services/receiver/deployment-presentation.sty @@ -0,0 +1,295 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Receiver" + description: "ComponentPresentation for SUSE Observability Receiver" + identifier: urn:stackpack:suse-observability:component-presentation:service-receiver-deployment + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("app.kubernetes.io/component:receiver-base", "app.kubernetes.io/component:receiver-process-agent", "app.kubernetes.io/component:receiver") and type in ("deployment", "service") + rank: + specificity: 10 + presentation: + highlight: + title: "SUSE Observability Receiver" + fields: [] + links: + - linkId: data-ingestion-dashboard + title: "Data ingestion dashboard" + target: "#/dashboards/urn:stackpack:suse-observability:dashboard:data-ingestion" + tooltip: "Open the bundled SUSE Observability data ingestion dashboard" + order: 100.0 + - linkId: topology-processing-dashboard + title: "Topology processing dashboard" + target: "#/dashboards/urn:stackpack:suse-observability:dashboard:topology-processing" + tooltip: "Open the bundled SUSE Observability topology processing dashboard" + order: 90.0 + - linkId: traces-processing-dashboard + title: "Traces processing dashboard" + target: "#/dashboards/urn:stackpack:suse-observability:dashboard:traces-processing" + tooltip: "Open the bundled SUSE Observability traces processing dashboard" + order: 80.0 + - linkId: logs-processing-dashboard + title: "Logs processing dashboard" + target: "#/dashboards/urn:stackpack:suse-observability:dashboard:logs-processing" + tooltip: "Open the bundled SUSE Observability logs processing dashboard" + order: 70.0 + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: receiver + title: Receiver + order: 100 + metrics: + - metricId: sts-receiver-incoming-data + - metricId: sts-receiver-deployment-current-incoming-element-count + - metricId: sts-receiver-deployment-current-incoming-element-created + - metricId: sts-receiver-deployment-budget-saturation + - metricId: sts-receiver-deployment-created-budget-saturation + - metricId: sts-receiver-deployment-agent-count + - metricId: sts-receiver-deployment-created-elements + - metricId: sts-receiver-deployment-active-http-requests + - metricId: sts-receiver-deployment-log-volume + - metricId: sts-receiver-deployment-rejected-log-volume + metrics: + - + metricId: sts-receiver-deployment-current-incoming-element-count + name: SUSE Observability Receiver - Current incoming element (approximate) count + description: Amount of data being offered to the receiver at this moment + metricQueries: + - expression: |- + sum by (element_type) (avg_over_time(stackstate_stackstate_receiver_unique_element_request_approx_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__interval}])) + alias: "${element_type}" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-receiver-deployment-current-incoming-element-created + name: SUSE Observability Receiver - Created elements in the last hour (approximate) count + description: Amount of elements expected to result in new element creates in SUSE Observability + metricQueries: + - expression: |- + sum by (element_type) (avg_over_time(stackstate_stackstate_receiver_element_create_request_approx_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__interval}])) + alias: "${element_type}" + - expression: |- + sum by (element_type) (avg_over_time(stackstate_stackstate_receiver_element_create_passed_max{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__interval}])) + alias: Budget ${element_type} + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-receiver-deployment-budget-saturation + name: SUSE Observability Receiver - Element budget saturation + description: Percentage of agent element processing budget filled up + metricQueries: + - expression: |- + sum by (element_type, cluster_name, namespace, pod_name) (avg_over_time(stackstate_stackstate_receiver_unique_element_passed_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__interval}])) / sum by (element_type, cluster_name, namespace, pod_name) (avg_over_time(stackstate_stackstate_receiver_unique_element_passed_max{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__interval}])) + alias: "${element_type} - ${pod_name}" + componentIdentifierTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + chart: + _type: GaugeChart + unit: percentunit + decimals: 0 + calculation: last + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 0.9 + - color: "#FF7F00" + value: 0.8 + mode: absolute + - + metricId: sts-receiver-deployment-created-budget-saturation + name: SUSE Observability Receiver - Created elements hourly budget saturation + description: Percentage of newly created elements budget filled up + metricQueries: + - expression: |- + sum by (element_type, cluster_name, namespace, pod_name) (avg_over_time(stackstate_stackstate_receiver_element_create_passed_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__interval}])) / sum by (element_type, cluster_name, namespace, pod_name) (avg_over_time(stackstate_stackstate_receiver_element_create_passed_max{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__interval}])) + alias: "${element_type} - ${pod_name}" + componentIdentifierTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + chart: + _type: TimeSeriesChart + unit: percentunit + - + metricId: sts-receiver-deployment-agent-count + name: SUSE Observability Receiver - Connected agent count + description: Number of currently connected agents + metricQueries: + - expression: |- + max_over_time(stackstate_stackstate_receiver_unique_element_passed_count{ + element_type="agent", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__interval}]) + alias: Agent count - ${pod_name} + componentIdentifierTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-receiver-deployment-created-elements + name: SUSE Observability Receiver - Created elements hourly total + description: Total elements created per hour + metricQueries: + - expression: |- + sum by (element_type) (stackstate_stackstate_receiver_element_create_passed_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }) + alias: "${element_type}" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-receiver-deployment-active-http-requests + name: SUSE Observability Receiver - Active HTTP Requests + description: Currently active http requests, this should not be an ever increasing line (see monitor) + metricQueries: + - expression: |- + sum(delta(stackstate_akka_http_requests_active{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Active requests + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-receiver-deployment-log-volume + name: SUSE Observability Receiver - Log volume + description: The volume of logs received by Stackstate, accepted logs are stored, rejected logs are dropped because they are over the allocated disk-space + metricQueries: + - expression: |- + sum(rate(stackstate_stackstate_receiver_events_accepted_size{ + element_type="k8sLogs", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Accepted + - expression: |- + sum(rate(stackstate_stackstate_receiver_events_rejected_size{ + element_type="k8sLogs", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Rejected + chart: + _type: TimeSeriesChart + unit: bytes/sec(IEC) + - + metricId: sts-receiver-deployment-rejected-log-volume + name: SUSE Observability Receiver - Rejected Log volume per pod + description: The volume of logs rejected by Stackstate per pod + metricQueries: + - expression: |- + topk(10, rate(stackstate_stackstate_receiver_logs_rejected_size{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) @ end() + alias: ${limited_pod_name} Rejected by - ${cluster_name}/${namespace}/${pod_name} + componentIdentifierTemplate: "urn:kubernetes:/${cluster_name}/pod/${limited_pod_uid}" + - expression: |- + max(stackstate_stackstate_receiver_logs_budget_per_pod { + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }) + alias: Logs budget per pod per time window + chart: + _type: TimeSeriesChart + unit: bytes/sec(IEC) + - + metricId: sts-receiver-incoming-data + name: Incoming data + description: "Maximum count of unique elements received from agents." + metricQueries: + - expression: |- + max by (element_type)(max_over_time(stackstate_stackstate_receiver_unique_element_passed_count{ + cluster_name="${tags.singleValue('cluster')}", + namespace="${tags.singleValue('namespace')}", + element_type='agent' + }[${__interval}])) + alias: node + - expression: |- + max by (element_type)(max_over_time(stackstate_stackstate_receiver_unique_element_passed_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + element_type!='agent' + }[${__interval}])) + alias: ${element_type} + chart: + _type: StatChart + unit: short + decimals: 0 + calculation: last + sparkline: true + thresholds: + defaultColor: "#33A02C" + steps: [] diff --git a/stackpacks/suse-observability/settings/services/receiver/monitors.sty b/stackpacks/suse-observability/settings/services/receiver/monitors.sty new file mode 100644 index 0000000..9e68733 --- /dev/null +++ b/stackpacks/suse-observability/settings/services/receiver/monitors.sty @@ -0,0 +1,181 @@ +nodes: + - _type: "Monitor" + name: "Receiver - unique element budget saturation" + tags: + - receiver + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:sts-receiver-budget-saturation + status: "ENABLED" + description: "The SUSE Observability receiver has a budget for the number of unique elements (agents, connections, processes, etc...) it will accept data for within a certain time. If this budget is exceeded, the receiver will stop accepting new elements. This monitor will alert when the budget is exceeded." + arguments: + comparator: GT + failureState: "DEVIATING" + titleTemplate: "Receiver - unique element budget saturation - ${element_type}" + threshold: 0.95 + metric: + query: |- + sum by (namespace, cluster_name, pod_name, element_type) (avg_over_time(stackstate_stackstate_receiver_unique_element_passed_count{kube_app_component=~"receiver.*"}[5m]) / stackstate_stackstate_receiver_unique_element_passed_max{kube_app_component=~"receiver.*"}) + aliasTemplate: "${element_type} saturation" + unit: percentunit + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/receiver/budget-saturation.md.hbs + - _type: "Monitor" + name: "Receiver - hourly element budget saturation" + tags: + - receiver + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:sts-receiver-hourly-budget-saturation + status: "ENABLED" + description: "The SUSE Observability receiver has an budget for the number of elements (agents, connections, processes, etc..) it will accept data for within an hour. If this budget is exceeded, the receiver will stop accepting new elements. This monitor will alert when the budget is exceeded." + arguments: + comparator: GT + titleTemplate: "Receiver - hourly element budget saturation - ${element_type}" + failureState: "DEVIATING" + threshold: 0.95 + metric: + query: |- + sum by (namespace, cluster_name, pod_name, element_type) (avg_over_time(stackstate_stackstate_receiver_element_create_passed_count{kube_app_component=~"receiver.*"}[5m]) / stackstate_stackstate_receiver_element_create_passed_max{kube_app_component=~"receiver.*"}) + aliasTemplate: "${element_type} hourly saturation" + unit: percentunit + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/receiver/hourly-budget-saturation.md.hbs + - _type: "Monitor" + name: "Receiver - Maximum number of elements exceeded" + tags: + - receiver + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:sts-receiver-max-elements-exceeded + status: "ENABLED" + description: "The SUSE Observability receiver has a limit on the maximum number of unqiue elements (agents, connections, processes, etc..) it will accept data for. If this limit is exceeded, the receiver will stop accepting new elements. This monitor will alert when the limit is exceeded." + arguments: + comparator: GTE + titleTemplate: "Receiver - Maximum number of elements exceeded - ${element_type}" + failureState: "DEVIATING" + threshold: 1.0 + metric: + query: |- + sum by (namespace, cluster_name, pod_name, element_type) (avg_over_time(stackstate_stackstate_receiver_unique_element_passed_count{kube_app_component=~"receiver.*"}[5m])) + / sum by (namespace, cluster_name, pod_name, element_type) (stackstate_stackstate_receiver_unique_element_passed_max{kube_app_component=~"receiver.*"}) + aliasTemplate: "# of ${element_type} elements" + unit: short + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/receiver/budget-saturation.md.hbs + - _type: "Monitor" + name: "Receiver - Maximum number of hourly elements exceeded" + tags: + - receiver + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:sts-receiver-hourly-max-elements-exceeded + status: "ENABLED" + description: "The SUSE Observability receiver has a limit on the maximum number of elements (agents, connections, processes, etc..) it will accept data for within an hour. If this hourly limit is exceeded, the receiver will stop accepting new elements for the rest of the hour. This monitor will alert when the limit is exceeded." + arguments: + comparator: GTE + titleTemplate: "Receiver - Maximum number of hourly elements exceeded - ${element_type}" + failureState: "DEVIATING" + threshold: 1.0 + metric: + query: |- + sum by (namespace, cluster_name, pod_name, element_type) (avg_over_time(stackstate_stackstate_receiver_element_create_passed_count{kube_app_component=~"receiver.*"}[5m])) + / sum by (namespace, cluster_name, pod_name, element_type) (stackstate_stackstate_receiver_element_create_passed_max{kube_app_component=~"receiver.*"}) + aliasTemplate: "# of ${element_type} elements" + unit: short + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/receiver/hourly-budget-saturation.md.hbs + - _type: "Monitor" + name: "Receiver - Active number of http requests growing" + tags: + - receiver + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:sts-receiver-active-http-requests-growing + status: "ENABLED" + description: "The number of active http requests should not keep growing. This can happen for short periods of time (for example with a sudden spike of traffic), but if it keeps growing, it is a sign of a problem." + arguments: + comparator: GT + failureState: "CRITICAL" + threshold: 0.0 + metric: + query: |- + sum by(namespace, cluster_name, pod_name) (min_over_time(delta(stackstate_receiver_akka_http_requests_active{kube_app_component=~"receiver.*"}[1m])[5m:1m])) + aliasTemplate: "# of active requests" + unit: short + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/receiver/http-growing-requests.md.hbs + - _type: "Monitor" + name: "Receiver - No data received" + tags: + - receiver + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:sts-receiver-no-data + status: "ENABLED" + description: "When SUSE Observability is not receiving data this monitor will trigger." + arguments: + failureState: "DEVIATING" + threshold: 0.00000001 + comparator: LTE + metric: + query: |- + sum by (cluster_name, namespace, pod_name) (rate(stackstate_receiver_agent_request{valid_api_key="true"}[5m])) + aliasTemplate: "# of requests, valid_api_key=${valid_api_key}" + unit: short + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: |- + No more data flowing into SUSE Observability will result in most of the topology slowly disappearing based on timeouts. + Common causes are: + - An invalid api key is used by the agent, if this is the case there will be 2 lines in the chart, one for valid_api_key="true" and one for valid_api_key="false" + - SUSE Observability agents in the cluster are not (yet) running + - Network connectivity issues between the agents in the cluster and SUSE Observability, these can also be caused by, for example firewalls or proxy servers. + + In case of invalid api keys try [this query in the metrics explorer](/#/metrics?promql=rate(stackstate_receiver_agent_request{valid_api_key%3D"false"}[%24{__rate_interval}])) to see the network addresses for the agents with the invalid api key and redeploy them with a valid api key. + - _type: "Monitor" + name: "Receiver - Logs rejected" + tags: + - receiver + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:sts-receiver-logs-rejected + status: "ENABLED" + description: "SUSE Observability reserves disk space for logs such that the retention period (default 7 days) can be met for all logs. Based on retention and available disk space each log source is allocated a fixed amount of log space per hour, any excessive logs will be rejected (reported as truncated in the log) however short-lived spikes are accepted. When logs are rejected this monitor triggers." + arguments: + failureState: "DEVIATING" + threshold: 0 + comparator: GT + metric: + query: |- + sum by (cluster_name, namespace, pod_name) (rate(stackstate_stackstate_receiver_events_rejected_size{element_type="k8sLogs"}[5m])) + aliasTemplate: "Rejected logs (bytes) per second" + unit: bytes/sec(IEC) + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/receiver/rejected-logs.md.hbs + - _type: "Monitor" + name: "Receiver - Memory Limiter saturation" + tags: + - receiver + - memory + function: urn:stackpack:common:monitor-function:threshold + arguments: + metric: + query: "stackstate_stackstate_receiver_memory_limiter_claimed_size / stackstate_stackstate_receiver_memory_limiter_limit" + unit: "percentunit" + aliasTemplate: "Memory Limiter saturation % = ${namespace}" + threshold: 0.8 + comparator: GTE + failureState: "CRITICAL" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + identifier: urn:stackpack:suse-observability:monitor:sts-receiver-memory-limiter-saturation + status: "ENABLED" + description: "he SUSE Observability receiver rejects requests if the allocated memory is (almost) exhausted. This protects the receiver from sudden spikes in the incoming data volume. However if this happens permanently or for long periods data will be dropped and the receiver needs more memory." + intervalSeconds: 60 + remediationHint: !include services/receiver/memory-limiter.md.hbs diff --git a/stackpacks/suse-observability/settings/services/receiver/pod-presentation.sty b/stackpacks/suse-observability/settings/services/receiver/pod-presentation.sty new file mode 100644 index 0000000..fa61172 --- /dev/null +++ b/stackpacks/suse-observability/settings/services/receiver/pod-presentation.sty @@ -0,0 +1,257 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Receiver" + description: "ComponentPresentation for SUSE Observability Receiver" + identifier: urn:stackpack:suse-observability:component-presentation:service-receiver-pod + binding: + _type: ComponentPresentationQueryBinding + query: |- + label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") + and label IN ("app.kubernetes.io/component:receiver-base", "app.kubernetes.io/component:receiver-process-agent", "app.kubernetes.io/component:receiver") + and type in ("pod") + rank: + specificity: 10 + presentation: + highlight: + title: "SUSE Observability Receiver" + fields: [] + links: + - linkId: data-ingestion-dashboard + title: "Data ingestion dashboard" + target: "#/dashboards/urn:stackpack:suse-observability:dashboard:data-ingestion" + tooltip: "Open the bundled SUSE Observability data ingestion dashboard" + order: 100.0 + - linkId: topology-processing-dashboard + title: "Topology processing dashboard" + target: "#/dashboards/urn:stackpack:suse-observability:dashboard:topology-processing" + tooltip: "Open the bundled SUSE Observability topology processing dashboard" + order: 90.0 + - linkId: traces-processing-dashboard + title: "Traces processing dashboard" + target: "#/dashboards/urn:stackpack:suse-observability:dashboard:traces-processing" + tooltip: "Open the bundled SUSE Observability traces processing dashboard" + order: 80.0 + - linkId: logs-processing-dashboard + title: "Logs processing dashboard" + target: "#/dashboards/urn:stackpack:suse-observability:dashboard:logs-processing" + tooltip: "Open the bundled SUSE Observability logs processing dashboard" + order: 70.0 + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: receiver + title: Receiver + order: 100 + metrics: + - metricId: sts-receiver-incoming-data + - metricId: sts-receiver-pod-current-incoming-element-count + - metricId: sts-receiver-pod-current-incoming-element-created + - metricId: sts-receiver-pod-budget-saturation + - metricId: sts-receiver-pod-created-budget-saturation + - metricId: sts-receiver-pod-agent-count + - metricId: sts-receiver-pod-created-elements + - metricId: sts-receiver-pod-active-http-requests + - metricId: sts-receiver-pod-log-volume + - metricId: sts-receiver-pod-rejected-log-volume + metrics: + - + metricId: sts-receiver-pod-current-incoming-element-count + name: SUSE Observability Receiver - Current incoming element (approximate) count + description: Amount of data being offered to the receiver at this moment + metricQueries: + - expression: |- + sum by (element_type) (avg_over_time(stackstate_stackstate_receiver_unique_element_request_approx_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__interval}])) + alias: "${element_type}" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-receiver-pod-current-incoming-element-created + name: SUSE Observability Receiver - Created elements in the last hour (approximate) count + description: Amount of elements expected to result in new element creates in SUSE Observability + metricQueries: + - expression: |- + sum by (element_type) (avg_over_time(stackstate_stackstate_receiver_element_create_request_approx_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__interval}])) + alias: "${element_type}" + - expression: |- + sum by (element_type) (avg_over_time(stackstate_stackstate_receiver_element_create_passed_max{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__interval}])) + alias: Budget ${element_type} + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-receiver-pod-budget-saturation + name: SUSE Observability Receiver - Element budget saturation + description: Percentage of agent element processing budget filled up + metricQueries: + - expression: |- + sum by (element_type) (avg_over_time(stackstate_stackstate_receiver_unique_element_passed_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__interval}])) / sum by (element_type) (avg_over_time(stackstate_stackstate_receiver_unique_element_passed_max{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__interval}])) + alias: "${element_type}" + chart: + _type: TimeSeriesChart + unit: percentunit + - + metricId: sts-receiver-pod-created-budget-saturation + name: SUSE Observability Receiver - Created elements hourly budget saturation + description: Percentage of newly created elements budget filled up + metricQueries: + - expression: |- + sum by (element_type) (avg_over_time(stackstate_stackstate_receiver_element_create_passed_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__interval}])) / sum by (element_type) (avg_over_time(stackstate_stackstate_receiver_element_create_passed_max{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__interval}])) + alias: "${element_type}" + chart: + _type: TimeSeriesChart + unit: percentunit + - + metricId: sts-receiver-pod-agent-count + name: SUSE Observability Receiver - Connected agent count + description: Number of currently connected agents + metricQueries: + - expression: |- + max_over_time(stackstate_stackstate_receiver_unique_element_passed_count{ + element_type="agent", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__interval}]) + alias: Agent Count + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-receiver-pod-created-elements + name: SUSE Observability Receiver - Created elements hourly total + description: Total elements created per hour + metricQueries: + - expression: |- + sum by (element_type) (stackstate_stackstate_receiver_element_create_passed_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }) + alias: "${element_type}" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-receiver-pod-active-http-requests + name: SUSE Observability Receiver - Active HTTP Requests + description: Currently active http requests, this should not be an ever increasing line (see monitor) + metricQueries: + - expression: |- + delta(stackstate_akka_http_requests_active{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: Active Requests + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-receiver-pod-log-volume + name: SUSE Observability Receiver - Log volume + description: The volume of logs received by Stackstate, accepted logs are stored, rejected logs are dropped because they are over the allocated disk-space + metricQueries: + - expression: |- + rate(stackstate_stackstate_receiver_events_accepted_size{ + element_type="k8sLogs", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: Accepted + - expression: |- + rate(stackstate_stackstate_receiver_events_rejected_size{ + element_type="k8sLogs", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: Rejected + chart: + _type: TimeSeriesChart + unit: bytes/sec(IEC) + - + metricId: sts-receiver-pod-rejected-log-volume + name: SUSE Observability Receiver - Rejected Log volume per pod + description: The volume of logs rejected by Stackstate per pod + metricQueries: + - expression: |- + topk(10, rate(stackstate_stackstate_receiver_logs_rejected_size{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) @ end() + alias: ${limited_pod_name} Rejected by - ${cluster_name}/${namespace}/${pod_name} + componentIdentifierTemplate: "urn:kubernetes:/${cluster_name}/pod/${limited_pod_uid}" + - expression: |- + max(stackstate_stackstate_receiver_logs_budget_per_pod { + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }) + alias: Logs budget per pod per time window + chart: + _type: TimeSeriesChart + unit: bytes/sec(IEC) + - + metricId: sts-receiver-incoming-data + name: Incoming data + description: "Maximum count of unique elements received from agents." + metricQueries: + - expression: |- + max by (element_type)(max_over_time(stackstate_stackstate_receiver_unique_element_passed_count{ + cluster_name="${tags.singleValue('cluster')}", + namespace="${tags.singleValue('namespace')}", + element_type='agent' + }[${__interval}])) + alias: node + - expression: |- + max by (element_type)(max_over_time(stackstate_stackstate_receiver_unique_element_passed_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + element_type!='agent' + }[${__interval}])) + alias: ${element_type} + chart: + _type: StatChart + unit: short + decimals: 0 + calculation: last + sparkline: true + thresholds: + defaultColor: "#33A02C" + steps: [] diff --git a/stackpacks/suse-observability/settings/services/server/presentation.sty b/stackpacks/suse-observability/settings/services/server/presentation.sty new file mode 100644 index 0000000..727f438 --- /dev/null +++ b/stackpacks/suse-observability/settings/services/server/presentation.sty @@ -0,0 +1,41 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Server" + description: "ComponentPresentation for SUSE Observability Server" + identifier: urn:stackpack:suse-observability:component-presentation:service-server + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("app.kubernetes.io/component:server") and type in ("pod", "deployment") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: consumer-lag + title: Consumer lag + order: 100 + metrics: + - metricId: sts-server-kafka-lag + metrics: + - + metricId: sts-server-kafka-lag + description: 'Kafka consumer lag per topic' + name: Kafka Topic lag + metricQueries: + - expression: |- + sum by (pod_name, topic) (stackstate_kafka_consumer_consumer_fetch_manager_metrics_records_lag{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }) + alias: ${pod_name} - ${topic} + chart: + _type: TimeSeriesChart + unit: cps \ No newline at end of file diff --git a/stackpacks/suse-observability/settings/services/state/presentation.sty b/stackpacks/suse-observability/settings/services/state/presentation.sty new file mode 100644 index 0000000..ccf66ff --- /dev/null +++ b/stackpacks/suse-observability/settings/services/state/presentation.sty @@ -0,0 +1,134 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability State Propagation" + description: "ComponentPresentation for SUSE Observability State Propagation" + identifier: urn:stackpack:suse-observability:component-presentation:service-state + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("app.kubernetes.io/component:state", "app.kubernetes.io/component:server") and type in ("pod", "deployment") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: state-propagation + title: State Propagation + order: 100 + metrics: + - metricId: sts-topo-state-calc-duration + - metricId: sts-topo-state-calc-batches-rate + - metricId: sts-topo-state-calc-steps-duration + - metricId: sts-topo-state-persistence-duration + - metricId: sts-topo-state-queued-rate + - metricId: sts-topo-state-latency + metrics: + - + metricId: sts-topo-state-calc-duration + description: 'State Propagation - Calculation duration' + name: State Propagation - Calculation duration + metricQueries: + - expression: |- + histogram_quantile(0.98, rate(stackstate_stackstate_state_propagation_calculation_duration_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + })) + alias: "${pod_name}" + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-topo-state-calc-batches-rate + description: 'State Propagation - Calculation batches rate' + name: State Propagation - Calculation batches rate + metricQueries: + - expression: |- + rate(stackstate_stackstate_state_propagation_calculation_duration_seconds_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }[${__interval}]) + alias: "${pod_name}" + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: sts-topo-state-calc-steps-duration + description: 'State Propagation - Calculation steps duration' + name: State Propagation - Calculation steps duration + metricQueries: + - expression: |- + histogram_quantile(0.98, rate(stackstate_stackstate_state_propagation_calculation_steps_duration_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + })) + alias: ${pod_name} - ${step} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-topo-state-persistence-duration + description: 'State Propagation - Persistence duration' + name: State Propagation - Persistence duration + metricQueries: + - expression: |- + histogram_quantile(0.98, rate(stackstate_stackstate_state_propagation_persistence_duration_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + })) + alias: "${pod_name}" + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-topo-state-queued-rate + description: 'State Propagation - Dequeued/Enqueued rate' + name: State Propagation - Dequeued commands rate + metricQueries: + - expression: |- + rate(stackstate_stackstate_state_propagation_dequeued_commands{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }[${__interval}]) + alias: ${pod_name} - Dequeued + - expression: |- + rate(stackstate_stackstate_state_propagation_enqueued_commands{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }[${__interval}]) + alias: ${pod_name} - Enqueued + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: sts-topo-state-latency + description: 'State State Propagation - Latency' + name: State Propagation - Latency + metricQueries: + - expression: |- + histogram_quantile(0.98, rate(stackstate_stackstate_state_calculation_latency_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + })) + alias: "${pod_name}" + chart: + _type: TimeSeriesChart + unit: s \ No newline at end of file diff --git a/stackpacks/suse-observability/settings/services/topo-sync/monitors.sty b/stackpacks/suse-observability/settings/services/topo-sync/monitors.sty new file mode 100644 index 0000000..23d4224 --- /dev/null +++ b/stackpacks/suse-observability/settings/services/topo-sync/monitors.sty @@ -0,0 +1,115 @@ +nodes: + - _type: "Monitor" + name: "Topology Sync - latency too high" + tags: + - jvm + - suse-observability + - topo-sync + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:sts-topo-sync-latency-too-high + status: "ENABLED" + description: "Monitors the latency of the topology sync and goes to a DEVIATING state when it is too high. Latency is measured from time of collection to commit of the data" + arguments: + comparator: GT + titleTemplate: "Topology Sync - latency too high - ${integration_type}" + failureState: DEVIATING + threshold: 45.000000 + metric: + query: |- + max by (cluster_name, namespace, pod_name, integration_type)(stackstate_stackstate_sync_stage_latency_seconds_quantile{start_stage="source", end_stage="commit", quantile="1.0"}) + aliasTemplate: "${integration_type} latency" + unit: s + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/topo-sync/latency.md.hbs + + - _type: "Monitor" + name: "Topology Sync - high processing time" + tags: + - suse-observability + - topo-sync + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:sts-topo-sync-high-processing-time-per-message + status: "ENABLED" + description: "Monitors the average processing time and goes to a DEVIATING state when it is considered too high. Processing time is measured from receiver to commit of the data." + arguments: + comparator: GT + threshold: 30.0 + failureState: DEVIATING + metric: + query: max by (cluster_name, namespace, pod_name, integration_type)(stackstate_stackstate_sync_stage_latency_seconds_quantile{start_stage="enterReceiver", end_stage="commit", quantile="1.0"}) + aliasTemplate: "${integration_type} processing time" + unit: s + titleTemplate: "Topology Sync - high processing time - ${integration_type}" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/topo-sync/internal-latency.md.hbs + + - _type: "Monitor" + name: "Topology Sync - Sync Workers blocked" + tags: + - suse-observability + - topo-sync + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:sts-topo-sync-workers-blocked + status: "ENABLED" + description: "Monitors the total time for the sync workers have not been processing any messages due to an error." + arguments: + comparator: GT + titleTemplate: "Topology Sync - Workers blocked - ${worker_name}" + failureState: "CRITICAL" + threshold: 180 + metric: + query: |- + max by (worker_name, pod_name, namespace, cluster_name)(last_over_time(stackstate_stackstate_sync_blocked_duration_seconds{}[5m])) + aliasTemplate: "${worker_name} blockage" + unit: s + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/topo-sync/blocked.md.hbs + + - _type: "Monitor" + name: "Topology Sync - SnapshotDataFromMultipleHosts error" + tags: + - suse-observability + - topo-sync + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:sts-topo-sync-snapshot-data-multiple-hosts + status: "ENABLED" + description: "Monitors whether snapshot data for a single synchronization is coming from multiple hosts." + arguments: + comparator: GT + titleTemplate: "Topology Sync - SnapshotDataFromMultipleHosts - ${name}" + failureState: DEVIATING + threshold: 0 + metric: + query: |- + max by (name, pod_name, namespace, cluster_name)(rate(stackstate_stackstate_sync_extTopo_batch_errors{category="SnapshotDataFromMultipleHosts"}[5m])) + aliasTemplate: "${name} SnapshotDataFromMultipleHosts errors" + unit: ops/sec (ops) + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/topo-sync/multiple-hosts.md.hbs + + - _type: "Monitor" + name: "Topology Sync - SnapshotAlreadyOpen error" + tags: + - suse-observability + - topo-sync + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:sts-topo-sync-snapshot-already-open + status: "ENABLED" + description: "Monitors whether snapshots are properly closed." + arguments: + comparator: GT + titleTemplate: "Topology Sync - SnapshotAlreadyOpen - ${name}" + failureState: DEVIATING + threshold: 0 + metric: + query: |- + max by (name, pod_name, namespace, cluster_name)(stackstate_stackstate_sync_extTopo_batch_errors{category="SnapshotAlreadyOpen"}[5m]) > 0 + aliasTemplate: "${name} SnapshotAlreadyOpen errors" + unit: ops + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/topo-sync/snapshot-already-open.md.hbs \ No newline at end of file diff --git a/stackpacks/suse-observability/settings/services/topo-sync/presentation.sty b/stackpacks/suse-observability/settings/services/topo-sync/presentation.sty new file mode 100644 index 0000000..b0f7ce0 --- /dev/null +++ b/stackpacks/suse-observability/settings/services/topo-sync/presentation.sty @@ -0,0 +1,160 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Topology Sync" + description: "ComponentPresentation for SUSE Observability Topology Sync" + identifier: urn:stackpack:suse-observability:component-presentation:service-topology-sync + binding: + _type: ComponentPresentationQueryBinding + query: |- + label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") + and label IN ("app.kubernetes.io/component:sync", "app.kubernetes.io/component:server") + and type in ("pod", "deployment") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: topology-sync + title: Topology Sync + order: 100 + metrics: + - metricId: sts-topo-sync-elements-total + - metricId: sts-topo-sync-latency + - metricId: sts-topo-sync-internal-latency + - metricId: sts-topo-sync-changes-rate + - metricId: sts-sync-kafka-lag + - + sectionId: storage + title: Storage + order: 100 + metrics: + - metricId: sts-topo-sync-storage + metrics: + - + metricId: sts-topo-sync-elements-total + description: 'Topology Sync total number of elements' + name: Topology Sync Elements count + metricQueries: + - expression: |- + stackstate_stackstate_sync_topo_total_counts{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + } + alias: ${pod_name} - ${element_type}s + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-topo-sync-latency + description: 'Topology Sync latency per integration type measured from collection time to database' + name: Topology Sync Latency + metricQueries: + - expression: |- + max by (pod_name, integration_type)(stackstate_stackstate_sync_stage_latency_seconds_quantile{ + start_stage="source", + end_stage="commit", + quantile="1.0", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }) + alias: ${pod_name} - ${integration_type} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-topo-sync-internal-latency + description: 'Topology Sync internal latency per integration type measured from reception time to database' + name: Topology Sync Internal Latency + metricQueries: + - expression: |- + max by (pod_name, integration_type)(stackstate_stackstate_sync_stage_latency_seconds_quantile{ + start_stage="enterReceiver", + end_stage="commit", + quantile="1.0", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }) + alias: ${pod_name} - ${integration_type} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-topo-sync-changes-rate + description: 'Topology Sync changes rate per synchronization' + name: Topology Sync Changes rate + metricQueries: + - expression: |- + (sum by (node_id, operation_type, pod_name)(rate(stackstate_stackstate_sync_extTopo_changes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }[1m])) or sum by (node_id, operation_type, pod_name)(rate(stackstate_stackstate_sync_extTopo_changes_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }[1m]))) > 0 + alias: ${pod_name} - ${node_id} - ${operation_type} + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: sts-sync-kafka-lag + description: 'Kafka consumer lag per topic' + name: Topology Sync Topic lag + metricQueries: + - expression: |- + sum by (pod_name, topic) (stackstate_kafka_consumer_consumer_fetch_manager_metrics_records_lag{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }) + alias: ${pod_name} - ${topic} + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: sts-topo-sync-storage + name: Volume usage + description: "Percentage of persistent volume capacity used by Kafka, HBase, and Sync." + metricQueries: + - expression: |- + max by (storage)(label_replace( + max_over_time(kubernetes_kubelet_volume_stats_used_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + persistentvolumeclaim=~".*-(kafka-\\d|hbase-.*|sync-tmp)" + }[5m]) / max_over_time(kubernetes_kubelet_volume_stats_capacity_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + persistentvolumeclaim=~".*-(kafka-\\d|hbase-.*|sync-tmp)" + }[5m]), "storage", "$1", "persistentvolumeclaim", ".*-(kafka-.*|hbase-.*|sync-tmp)")) + alias: ${storage} + chart: + _type: GaugeChart + unit: percentunit + decimals: 0 + calculation: last + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 0.9 + - color: "#FF7F00" + value: 0.8 + mode: absolute + diff --git a/stackpacks/suse-observability/settings/services/victoria-metrics/monitors.sty b/stackpacks/suse-observability/settings/services/victoria-metrics/monitors.sty new file mode 100644 index 0000000..c7b6d5b --- /dev/null +++ b/stackpacks/suse-observability/settings/services/victoria-metrics/monitors.sty @@ -0,0 +1,330 @@ +nodes: + - _type: "Monitor" + name: "VictoriaMetrics - backups performed" + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:victoria-metrics-0-backups-not-executed + arguments: + metric: + query: |- + sum(increase(stackstate_supercronic_successful_executions{kube_app_name=~"victoria-metrics-.*"}[12h])) by (cluster_name, namespace, pod_name) or sum(kubernetes_state_container_running{pod_name=~"stackstate-victoria-metrics-.*", container="vmbackup"}) by (cluster_name, namespace, pod_name) * 0 + unit: "short" + aliasTemplate: "Number of backups execution in 12h window" + comparator: LTE + threshold: 0 + failureState: "CRITICAL" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Backup performed in the last 12 hours" + intervalSeconds: 3600 + remediationHint: "Potential reasons why backups haven't been executed:\n- wrong configuration of MinIO (like secrets, bucket name and so on)\n- interval between backups is longer than 12h (you have to change the configuration of the Monitor)\n\nTo investigate the problem you should check:\n- logs of `vmbackup` container\n- logs/status of `MinIO` pods" + status: "ENABLED" + + - _type: "Monitor" + name: "VictoriaMetrics - Disk runs out of space in 3 days" + description: "Monitor that triggers when free disk space will be enough only for less than 3 days" + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:victoria-metrics-disk-space-3days + arguments: + metric: + query: |- + sum ( + stackstate_vm_free_disk_space_bytes / ignoring(path) + ( + ( + rate(stackstate_vm_rows_added_to_storage_total[1d]) - + ignoring(type) rate(stackstate_vm_deduplicated_samples_total{type="merge"}[1d]) + ) + * scalar( + sum(stackstate_vm_data_size_bytes{type!~"indexdb.*"}) / + sum(stackstate_vm_rows{type!~"indexdb.*"}) + ) + ) + ) by (cluster_name, namespace, pod_name) + unit: "s" + aliasTemplate: "Time to fill the disk space" + comparator: LTE + threshold: 259200 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Time to fill the disk space" + intervalSeconds: 90 + remediationHint: "Taking into account current ingestion rate, free disk space will be enough only + for less than 3 days.\n + Consider to limit the ingestion rate, decrease retention or scale the disk space if possible." + status: "ENABLED" + + - _type: "Monitor" + name: "VictoriaMetrics - Disk runs out of space" + description: "Monitor that triggers when disk utilisation on the instance is more than 80%." + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:victoria-metrics-disk-space + arguments: + metric: + query: |- + sum(stackstate_vm_data_size_bytes) by (cluster_name, namespace, pod_name) / + ( + sum(stackstate_vm_free_disk_space_bytes) by (cluster_name, namespace, pod_name) + + sum(stackstate_vm_data_size_bytes) by (cluster_name, namespace, pod_name) + ) + unit: "percentunit" + aliasTemplate: "Disk space usage" + comparator: GT + threshold: 0.8 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Disk space usage" + intervalSeconds: 90 + remediationHint: "Disk utilisation on the instance is more than 80%.\n + Having less than 20% of free disk space could cripple merges processes and overall performance. + Consider to limit the ingestion rate, decrease retention or scale the disk space if possible." + status: "ENABLED" + + - _type: "Monitor" + name: "VictoriaMetrics - Requests errors to api" + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:victoria-metrics-errors + arguments: + metric: + query: |- + sum(increase(stackstate_vm_http_request_errors_total[5m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Errors" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Request errors to api" + intervalSeconds: 90 + remediationHint: !include services/victoria-metrics/errors.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "VictoriaMetrics - Rows rejected on ingestion" + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:victoria-metrics-rows-rejected + arguments: + metric: + query: |- + sum(rate(stackstate_vm_rows_ignored_total[5m])) by (cluster_name, namespace, pod_name, instance, reason) + unit: "short" + aliasTemplate: "Rows rejected" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Rows rejected due to: ${reason}" + intervalSeconds: 90 + remediationHint: !include services/victoria-metrics/rejected.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "VictoriaMetrics - Too high churn rate" + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:victoria-metrics-churn-rate + arguments: + metric: + query: |- + sum(rate(stackstate_vm_new_timeseries_created_total[5m])) by (cluster_name, namespace, pod_name) + / + sum(rate(stackstate_vm_rows_inserted_total[5m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Time series created" + comparator: GT + threshold: 0.1 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "High churn rate" + intervalSeconds: 90 + description: "VM constantly creates new time series.\n + This effect is known as Churn Rate.\n + High Churn Rate tightly connected with database performance and may + result in unexpected OOM's or slow queries." + remediationHint: "The [cardinality explorer](https://docs.victoriametrics.com/#cardinality-explorer) of Victoria Metrics can help determine which metrics have the highest cardinality. These are likely sources of churn. Only known solution is to investigate the source of the metric and remove labels that change frequently." + status: "ENABLED" + + - _type: "Monitor" + name: "VictoriaMetrics - Too high churn rate 24hrs" + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:victoria-metrics-churn-rate-24 + status: "ENABLED" + description: "The number of created new time series over last 24h is 4x times higher than + current number of active series.\n + This effect is known as Churn Rate.\n + High Churn Rate tightly connected with database performance and may + result in unexpected OOM's or slow queries." + arguments: + titleTemplate: "High churn rate 24hrs" + comparator: GT + failureState: "DEVIATING" + threshold: 1.0 + metric: + query: |- + sum(increase(stackstate_vm_new_timeseries_created_total[24h])) by (cluster_name, namespace, pod_name, container) + / sum(max_over_time(stackstate_vm_cache_entries{type="storage/hour_metric_ids"}[24h])*4) by (cluster_name, namespace, pod_name, container) + aliasTemplate: "Time series created" + unit: short + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 90 + remediationHint: "The [cardinality explorer](https://docs.victoriametrics.com/#cardinality-explorer) of Victoria Metrics can help determine which metrics have the highest cardinality. These are likely sources of churn. Only known solution is to investigate the source of the metric and remove labels that change frequently." + + - _type: "Monitor" + name: "VictoriaMetrics - Too high slow insert rate" + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:victoria-metrics-slow-insert-rate + arguments: + metric: + query: |- + sum(rate(stackstate_vm_slow_row_inserts_total[5m])) by (cluster_name, namespace, pod_name) + / + sum(rate(stackstate_vm_rows_inserted_total[5m])) by (cluster_name, namespace, pod_name) + unit: "percentunit" + aliasTemplate: "Percentage slow inserts" + comparator: GT + threshold: 0.05 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "High slow insert rate" + intervalSeconds: 90 + remediationHint: !include services/victoria-metrics/slow-insert-rate.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "VictoriaMetrics - Labels limit exceeded on ingestion" + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:victoria-metrics-labels-limit + arguments: + metric: + query: |- + sum(increase(stackstate_vm_metrics_with_dropped_labels_total[5m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Dropped labels" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Labels limit exceeded on ingestion" + intervalSeconds: 90 + remediationHint: !include services/victoria-metrics/labels-limit.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "VictoriaMetrics - Storage is read only" + description: "Monitor that triggers when VictoriaMetrics has stopped accepting writes" + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:victoria-metrics-storage-read-only + arguments: + metric: + query: |- + sum(stackstate_vm_storage_is_read_only) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Storage is read only" + comparator: GT + threshold: 0 + failureState: "CRITICAL" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Storage is read only" + intervalSeconds: 60 + remediationHint: !include services/victoria-metrics/storage-read-only.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "VictoriaMetrics - Insert concurrency saturated" + description: "Monitor that triggers when concurrent inserts reach the limit VictoriaMetrics derives from its CPU allocation" + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:victoria-metrics-insert-concurrency + arguments: + metric: + query: |- + sum(avg_over_time(stackstate_vm_concurrent_insert_current{kube_app_name=~"victoria-metrics-.*"}[1m])) by (cluster_name, namespace, pod_name) + / sum(stackstate_vm_concurrent_insert_capacity{kube_app_name=~"victoria-metrics-.*"}) by (cluster_name, namespace, pod_name) + unit: "percentunit" + aliasTemplate: "Insert concurrency in use" + comparator: GTE + threshold: 0.9 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Insert concurrency saturated" + intervalSeconds: 90 + remediationHint: !include services/victoria-metrics/insert-concurrency.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "VictoriaMetrics - Select concurrency saturated" + description: "Monitor that triggers when concurrent queries reach the limit VictoriaMetrics derives from its CPU allocation" + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:victoria-metrics-select-concurrency + arguments: + metric: + query: |- + sum(avg_over_time(stackstate_vm_concurrent_select_current{kube_app_name=~"victoria-metrics-.*"}[1m])) by (cluster_name, namespace, pod_name) + / sum(stackstate_vm_concurrent_select_capacity{kube_app_name=~"victoria-metrics-.*"}) by (cluster_name, namespace, pod_name) + unit: "percentunit" + aliasTemplate: "Select concurrency in use" + comparator: GTE + threshold: 0.9 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Select concurrency saturated" + intervalSeconds: 90 + remediationHint: !include services/victoria-metrics/select-concurrency.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "VictoriaMetrics - Metrics are not fresh" + description: "Monitor that triggers when metrics take longer than the freshness objective to become queryable after they are scraped" + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:victoria-metrics-freshness + arguments: + metric: + query: |- + max by (cluster_name, namespace, pod_name) ( + min_over_time((time() - timestamp(stackstate_vm_rows_inserted_total{type="promremotewrite"}))[1m:5s]) + ) + unit: "s" + aliasTemplate: "Scrape to queryable delay" + comparator: GT + threshold: 30 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Metrics are not fresh" + intervalSeconds: 60 + remediationHint: !include services/victoria-metrics/freshness.md.hbs + status: "ENABLED" diff --git a/stackpacks/suse-observability/settings/services/victoria-metrics/presentation.sty b/stackpacks/suse-observability/settings/services/victoria-metrics/presentation.sty new file mode 100644 index 0000000..454eca0 --- /dev/null +++ b/stackpacks/suse-observability/settings/services/victoria-metrics/presentation.sty @@ -0,0 +1,194 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Victoria Metrics" + description: "ComponentPresentation for SUSE Observability Victoria Metrics" + identifier: urn:stackpack:suse-observability:component-presentation:service-victoria-metrics + binding: + _type: ComponentPresentationQueryBinding + query: |- + label IN ("app.kubernetes.io/name:victoria-metrics-0", "app.kubernetes.io/name:victoria-metrics-1") + AND type IN ("statefulset") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: backup + title: Backup + order: 90 + metrics: + - metricId: victoria-metrics-backup-duration + - + sectionId: performance + title: Performance + order: 100 + metrics: + - metricId: victoria-metrics-storage + - metricId: victoria-metrics-churn-rate + - metricId: victoria-metrics-timeseries-dropped + - metricId: victoria-metrics-slow-inserts + - metricId: victoria-metrics-slow-queries + - + sectionId: limits + title: Limits + order: 80 + metrics: + - metricId: victoria-metrics-labels-limit + metrics: + - + metricId: victoria-metrics-backup-duration + name: VictoriaMetrics - Backup duration + description: Duration of VictoriaMetrics backups + metricQueries: + - expression: |- + increase(stackstate_supercronic_cron_execution_time_seconds_sum{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name=~".*-victoria-metrics-.*" + }[${__rate_interval}]) > 0 + alias: "${kube_app_name}" + chart: + _type: TimeSeriesChart + unit: s + - + metricId: victoria-metrics-churn-rate + name: VictoriaMetrics - Churn rate + description: "Shows the rate and total number of new series created over last 24h.\n\nHigh churn rate tightly connected with database performance and may result in unexpected OOM's or slow queries. It is recommended to always keep an eye on this metric to avoid unexpected cardinality \"explosions\".\n\nThe higher churn rate is, the more resources required to handle it. Consider to keep the churn rate as low as possible.\n\nGood references to read:\n* https://www.robustperception.io/cardinality-is-key\n* https://www.robustperception.io/using-tsdb-analyze-to-investigate-churn-and-cardinality" + metricQueries: + - expression: |- + sum(rate(stackstate_vm_new_timeseries_created_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name=~".*-victoria-metrics-.*" + }[${__rate_interval}])) + alias: "Time series created" + - expression: |- + sum(increase(stackstate_vm_new_timeseries_created_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name=~".*-victoria-metrics-.*" + }[24h])) + alias: "Time series created over 24 hrs" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: victoria-metrics-slow-inserts + name: VictoriaMetrics - Slow inserts + description: "The percentage of slow inserts comparing to total insertion rate during the last 5 minutes. + The less value is better. If percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of active time series. + In general, VictoriaMetrics requires ~1KB or RAM per active time series, so it should be easy calculating the required amounts of RAM for the current workload according to capacity planning docs. But the resulting number may be far from the real number because the required amounts of memory depends on may other factors such as the number of labels per time series and the length of label values." + metricQueries: + - expression: |- + max(rate(stackstate_vm_slow_row_inserts_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__rate_interval}]) / rate(stackstate_vm_rows_added_to_storage_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name=~".*-victoria-metrics-.*" + }[${__rate_interval}])) + alias: "Percentage of slow inserts" + chart: + _type: TimeSeriesChart + unit: percentunit + - + metricId: victoria-metrics-slow-queries + name: VictoriaMetrics - Slow queries rate + description: "Slow queries rate according to `search.logSlowQueryDuration` flag, which is `5s` by default." + metricQueries: + - expression: |- + sum(rate(stackstate_vm_slow_queries_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name=~".*-victoria-metrics-.*" + }[${__rate_interval}])) + alias: "Slow queries rate" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: victoria-metrics-labels-limit + name: VictoriaMetrics - Dropped timeseries + description: "VictoriaMetrics limits the number of labels per timeseries with `-maxLabelsPerTimeseries` command-line flag. + This prevents from ingesting metrics with too many labels. The value of `maxLabelsPerTimeseries` must be adjusted for your workload. + When limit is exceeded (graph is > 0) - the timeseries with too many labels are dropped, which results in missing metric data." + metricQueries: + - expression: |- + sum(increase(vm_rows_ignored_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name=~".*-victoria-metrics-.*" + }[${__rate_interval}])) + alias: "Slow queries rate" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: victoria-metrics-storage + name: Volume usage + description: "Percentage of persistent volume capacity used by VictoriaMetrics." + chart: + _type: GaugeChart + unit: percentunit + decimalPlaces: 0 + max: 1 + calculation: last + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 0.9 + - color: "#FF7F00" + value: 0.8 + mode: absolute + metricQueries: + - expression: |- + max by (storage)(label_replace(max_over_time(kubernetes_kubelet_volume_stats_used_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + persistentvolumeclaim=~"server-volume-.*-(victoria-metrics-.+)"}[5m] + ) / max_over_time(kubernetes_kubelet_volume_stats_capacity_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + persistentvolumeclaim=~"server-volume-.*-(victoria-metrics-.+)"}[5m] + ), "storage","$1", "persistentvolumeclaim", "server-volume-.*-(victoria-metrics-.+)")) + alias: ${storage} + - + metricId: victoria-metrics-timeseries-dropped + name: Dropped timeseries in the past hour + description: >- + VictoriaMetrics limits the number of labels per timeseries with + `-maxLabelsPerTimeseries` command-line flag. + + This prevents from ingesting metrics with too many labels. The value + of `maxLabelsPerTimeseries` must be adjusted for your workload. + + When limit is exceeded (graph is > 0) - the timeseries with too many + labels are dropped, which results in missing metric data. + metricQueries: + - expression: |- + sum by (statefulset)(increase(stackstate_vm_rows_ignored_total{ + pod_name=~".*-victoria-metrics-.*", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}" + }[1h])) + alias: VictoriaMetrics + chart: + _type: StatChart + unit: short + decimals: 0 + calculation: last + sparkline: true + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 10 diff --git a/stackpacks/suse-observability/settings/services/vmagent/monitors.sty b/stackpacks/suse-observability/settings/services/vmagent/monitors.sty new file mode 100644 index 0000000..58c7b7c --- /dev/null +++ b/stackpacks/suse-observability/settings/services/vmagent/monitors.sty @@ -0,0 +1,165 @@ +nodes: + - _type: "Monitor" + name: "Vmagent - Persistent queue is droppingData" + tags: + - vmagent + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:vmagent-queue-dropping-data + arguments: + metric: + query: |- + sum(increase(stackstate_vm_persistentqueue_bytes_dropped_total[5m])) by (cluster_name, namespace, pod_name) + unit: "bytes(IEC)" + aliasTemplate: "Bytes dropped" + comparator: GT + threshold: 0 + failureState: "CRITICAL" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Persistent queue is dropping data" + intervalSeconds: 90 + remediationHint: !include services/vmagent/persistentqueue-dropping-data.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "Vmagent - RemoteWrite packets dropped" + tags: + - vmagent + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:vmagent-remotewrite-packets-dropped + arguments: + metric: + query: |- + sum(increase(stackstate_vmagent_remotewrite_packets_dropped_total[5m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Packets dropped" + comparator: GT + threshold: 0 + failureState: "CRITICAL" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "RemoteWrite packets dropped" + intervalSeconds: 90 + remediationHint: "Vmagent drops the rejected by remote-write server data blocks. Check the logs to find the reason for rejects" + status: "ENABLED" + + - _type: "Monitor" + name: "Vmagent - Too many write errors" + tags: + - vmagent + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:vmagent-errors + arguments: + metric: + query: |- + (sum(increase(stackstate_vm_ingestserver_request_errors_total[5m])) by (cluster_name, namespace, pod_name) + + + sum(increase(stackstate_vmagent_http_request_errors_total[5m])) by (cluster_name, namespace, pod_name)) + unit: "short" + aliasTemplate: "Error count" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Write errors" + intervalSeconds: 90 + remediationHint: "Vmagent responds with errors to write requests. Check the logs to find the reason for the errors" + status: "ENABLED" + + - _type: "Monitor" + name: "Vmagent - Too many remote write errors" + tags: + - vmagent + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:vmagent-remotewrite-errors + arguments: + metric: + query: |- + sum(rate(stackstate_vmagent_remotewrite_retries_count_total[5m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Error count" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Remote write errors" + intervalSeconds: 90 + remediationHint: "Vmagent responds with errors to remote write requests. Check the logs to find the reason for the errors" + status: "ENABLED" + + - _type: "Monitor" + name: "Vmagent - Remote write connection is saturated" + tags: + - vmagent + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:vmagent-remotewrite-connection-saturated + arguments: + metric: + query: |- + sum( + rate(stackstate_vmagent_remotewrite_send_duration_seconds_total[5m]) + / + stackstate_vmagent_remotewrite_queues + ) by (cluster_name, namespace, pod_name) + unit: "percentunit" + aliasTemplate: "Saturation %" + comparator: GT + threshold: 0.9 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Remote write connection saturated" + intervalSeconds: 90 + remediationHint: "The remote write connection between vmagent and destination + is saturated by more than 90% and vmagent won't be able to keep up.\n + This usually means that `-remoteWrite.queues` command-line flag must be increased in order to increase + the number of connections per each remote storage." + status: "ENABLED" + + - _type: "Monitor" + name: "Vmagent - Persistent queue for writes is saturated" + tags: + - vmagent + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:vmagent-persistent-queue-saturated + arguments: + metric: + query: |- + sum(rate(stackstate_vm_persistentqueue_write_duration_seconds_total[5m])) by (cluster_name, namespace, pod_name) + unit: "percentunit" + aliasTemplate: "Saturation %" + comparator: GT + threshold: 0.9 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Persistent queue writes saturated" + intervalSeconds: 90 + remediationHint: "Persistent queue writes for vmagent + are saturated by more than 90% and vmagent won't be able to keep up with flushing data on disk. + In this case, consider to decrease load on the vmagent or improve the disk throughput." + status: "ENABLED" + - _type: "Monitor" + name: "Vmagent - Samples dropped on remote write" + description: "Monitor that triggers when vmagent discards samples instead of delivering them to the metric store" + tags: + - vmagent + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability:monitor:vmagent-remotewrite-samples-dropped + arguments: + metric: + query: |- + sum(increase(stackstate_vmagent_remotewrite_samples_dropped_total[5m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Samples dropped" + comparator: GT + threshold: 0 + failureState: "CRITICAL" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Samples dropped on remote write" + intervalSeconds: 90 + remediationHint: !include services/vmagent/remotewrite-samples-dropped.md.hbs + status: "ENABLED" diff --git a/stackpacks/suse-observability/settings/services/vmagent/presentation.sty b/stackpacks/suse-observability/settings/services/vmagent/presentation.sty new file mode 100644 index 0000000..76476f7 --- /dev/null +++ b/stackpacks/suse-observability/settings/services/vmagent/presentation.sty @@ -0,0 +1,121 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability VM Agent" + description: "ComponentPresentation for SUSE Observability VM Agent" + identifier: urn:stackpack:suse-observability:component-presentation:service-vmagent + binding: + _type: ComponentPresentationQueryBinding + query: |- + label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") + and label IN ("app.kubernetes.io/component:vmagent") + and type in ("statefulset") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - sectionId: vmagent + title: VMAgent + order: 100 + metrics: + - metricId: vmagent-write-duration + - metricId: vmagent-blocks-dropped + - metricId: vmagent-datapoints-rate + - metricId: vmagent-invalid-datapoints-rate + - metricId: vmagent-timeseries-dropped + metrics: + - + metricId: vmagent-write-duration + name: Persistent queue write saturation + description: "Shows saturation persistent queue for writes. If the threshold of 0.9sec is reached, then persistent is saturated by more than 90% and vmagent won't be able to keep up with flushing data on disk. In this case, consider to decrease load on the vmagent or improve the disk throughput." + metricQueries: + - expression: |- + max(rate(stackstate_vm_persistentqueue_write_duration_seconds_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name=~".*-vmagent-.*", + }[${__rate_interval}])) + alias: "Write duration" + chart: + _type: TimeSeriesChart + unit: s + - + metricId: vmagent-blocks-dropped + name: Data blocks dropped + description: "Shows the rate of dropped data blocks in cases when remote storage replies with `400 Bad Request` and `409 Conflict` HTTP responses. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149" + metricQueries: + - expression: |- + sum(rate(stackstate_vmagent_remotewrite_packets_dropped_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name=~".*-vmagent-.*", + }[${__rate_interval}])) by(url) + alias: "Data blocks dropped" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: vmagent-datapoints-rate + name: Data points rate + description: "Shows the rate of parsed data points from write or scrape requests." + metricQueries: + - expression: |- + sum(rate(stackstate_vm_protoparser_rows_read_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name=~".*-vmagent-.*", + }[${__rate_interval}])) by(type) + alias: "Data points rate" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: vmagent-invalid-datapoints-rate + name: Invalid data points rate + description: "Tracks the rate of dropped invalid rows because of errors while unmarshaling write requests. The exact errors messages will be printed in logs." + metricQueries: + - expression: |- + sum(rate(stackstate_vm_rows_invalid_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name=~".*-vmagent-.*", + }[${__rate_interval}])) by(type) + alias: "Invalid data points rate" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: vmagent-timeseries-dropped + name: Dropped timeseries in the past hour + description: >- + VictoriaMetrics limits the number of labels per timeseries with + `-maxLabelsPerTimeseries` command-line flag. + + This prevents from ingesting metrics with too many labels. The value + of `maxLabelsPerTimeseries` must be adjusted for your workload. + + When limit is exceeded (graph is > 0) - the timeseries with too many + labels are dropped, which results in missing metric data. + metricQueries: + - expression: |- + sum by (statefulset)(increase(stackstate_vm_rows_ignored_total{ + pod_name=~".*-vmagent-.*", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}" + }[1h])) + alias: vmagent + chart: + _type: StatChart + unit: short + decimals: 0 + calculation: last + sparkline: true + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 10 diff --git a/stackpacks/suse-observability/stackpack.yaml b/stackpacks/suse-observability/stackpack.yaml new file mode 100644 index 0000000..ac929b9 --- /dev/null +++ b/stackpacks/suse-observability/stackpack.yaml @@ -0,0 +1,17 @@ +name: suse-observability +version: "0.0.10" +schemaVersion: "2.0" +displayName: "SUSE Observability" +categories: [ "SUSE Observability" ] +logoUrl: !resource "logo.png" +overviewUrl: !resource "overview.md" +detailedOverviewUrl: !resource "detailed-overview.md" +configurationUrls: + NOT_INSTALLED: !resource "configuration.md" + PROVISIONING: !resource "provisioning.md" + WAITING_FOR_DATA: !resource "waitingfordata.md" + INSTALLED: !resource "enabled.md" + DEPROVISIONING: !resource "configuration.md" + ERROR: !resource "configuration.md" +provision: + companionStackPacks: [] From a62d0600bf9a1dd0923989f8986219d5f41674ea Mon Sep 17 00:00:00 2001 From: Alejandro Acevedo Date: Thu, 5 Nov 2026 15:28:54 +0100 Subject: [PATCH 2/2] Add validated core bundle delivery and ownership docs Validate and package every core StackPack, require version and workflow security checks before both image builds, and publish the exact scanned architecture images through the shared image pipeline. Use digest-pinned SUSE BCI and a checksum-verified CLI. Trusted branch PRs publish a bundle for downstream validation; forks receive no publishing credentials. New GHCR packages begin private. Require anonymous manifest access after publication; retried jobs use a fresh immutable tag so changing visibility can recover the initial run without overwriting a release. The chart transition remains separate until a real public image and product registry availability are verified. Validation: online Zizmor 1.31.1, actionlint 1.7.12, positive/negative platform validation, both architecture builds, amd64 non-root copy smoke and separate Trivy secret/HIGH-CRITICAL scans pass. New tooling passes diff whitespace checks; imported content preserves inherited source whitespace. Tracking: https://github.com/StackVista/stackstate/issues/892 --- .agents/skills/create-stackpack/SKILL.md | 115 ++++++ .claude/skills/create-stackpack | 1 + .github/CODEOWNERS | 1 + .github/workflows/stackpacks-ci.yml | 338 ++++++++++++++++++ .gitignore | 4 + .pre-commit-config.yaml | 21 ++ README.md | 90 ++++- docker/Dockerfile | 5 + scripts/ci/validate_stackpack.sh | 11 + scripts/ci/verify_versions_bumped.sh | 47 +++ .../resources/logo.svg | 1 + .../resources/overview.md | 1 + .../settings/mapping.sty | 25 ++ .../invalid-component-mapping/stackpack.yaml | 11 + 14 files changed, 669 insertions(+), 2 deletions(-) create mode 100644 .agents/skills/create-stackpack/SKILL.md create mode 120000 .claude/skills/create-stackpack create mode 100644 .github/CODEOWNERS create mode 100644 .github/workflows/stackpacks-ci.yml create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 docker/Dockerfile create mode 100755 scripts/ci/validate_stackpack.sh create mode 100755 scripts/ci/verify_versions_bumped.sh create mode 100644 tests/fixtures/invalid-component-mapping/resources/logo.svg create mode 100644 tests/fixtures/invalid-component-mapping/resources/overview.md create mode 100644 tests/fixtures/invalid-component-mapping/settings/mapping.sty create mode 100644 tests/fixtures/invalid-component-mapping/stackpack.yaml diff --git a/.agents/skills/create-stackpack/SKILL.md b/.agents/skills/create-stackpack/SKILL.md new file mode 100644 index 0000000..c6a72a4 --- /dev/null +++ b/.agents/skills/create-stackpack/SKILL.md @@ -0,0 +1,115 @@ +--- +name: create-stackpack +description: Create or extend SUSE Observability StackPacks for a product or technology using OpenTelemetry data, semantic conventions, mappings, presentations, and monitors. +--- + +# Create a StackPack + +Start with the product or technology the user wants to observe. +Use the current declarative scaffold and adapt it to that integration. + +## Sources + +Read the official [creation guide](https://documentation.suse.com/cloudnative/suse-observability/latest/en/setup/custom-integrations/create.html) before starting. +Follow its reference links for the settings being changed. +Use the [StackPack reference](https://documentation.suse.com/cloudnative/suse-observability/latest/en/setup/custom-integrations/stackpacks.html) for project structure, versioning, and CLI commands. +Use [contrib-stackpacks](https://github.com/StackVista/contrib-stackpacks/tree/main/stackpacks) for working examples: + +- `open-telemetry`: component and relation mappings, presentations, and monitors. +- `otel-k8s-crd`: topology from Kubernetes custom-resource events. +- `suse-observability`: presentations and monitors for existing platform components. + +Read only the relevant example files. Prefer current schemas over copying an entire integration. + +## Telemetry design and discovery + +Use OpenTelemetry as the base data model. Prefer collection through the OpenTelemetry Collector. +Find existing receivers or instrumentation for the target technology before proposing custom collection. +Identify the required signals, resource identity, and Collector configuration. +If collection is missing, provide the required configuration and ingestion instructions with the StackPack. + +Use the relevant [OpenTelemetry semantic conventions](https://github.com/open-telemetry/semantic-conventions) for resource attributes, metrics, spans, and logs. +Read the technology-specific conventions and check their stability and support in the selected receiver or instrumentation. +Prefer standard names and units. Document extensions only where no suitable convention exists. +Compare conventions with observed telemetry, including metric-name and label transformations during ingestion. +If they differ, make collection normalization or mapping compatibility explicit. + +Use the official [MCP connection guide](https://documentation.suse.com/cloudnative/suse-observability/latest/en/use/ai-assistant/mcp-server.html) to connect your assistant to the SUSE Observability MCP server. +When it is available, use its query tools to inspect data for the target technology. +Read the relevant MCP domain guides first and establish that MCP and CLI target the intended instance. +Scope queries to relevant resources and time ranges. +Inspect available metrics, labels, topology, traces, and logs to establish identifiers and usable signals. +Use those results to author mappings and queries, then check their behavior after deployment. +If MCP is unavailable, use CLI queries or supplied telemetry samples. +If data is absent, document the collection gap and leave runtime behavior unverified. + +## CLI and authentication + +Check whether `sts` is available. +If it is missing, ask whether the user will install it. +Direct them to the **CLI** main menu item in the SUSE Observability UI for installation. +Wait for installation before running CLI commands. Continue independent design work while waiting. +Use the latest stable Go `sts` CLI from [stackstate-cli releases](https://github.com/StackVista/stackstate-cli/releases/latest). +Use the official [CLI installation guide](https://documentation.suse.com/cloudnative/suse-observability/latest/en/setup/cli/cli-sts.html) for installation details. +Resolve the release on each task. Do not embed a fixed CLI version in this skill. +Compare the installed version with that release and ask the user to upgrade when necessary. +Use the matching operating-system and architecture artifact, and check its published checksum before execution. + +The user authenticates the CLI to the correct SUSE Observability instance. +Do not log in, change contexts, request tokens, or read credential files. +If authentication fails, ask the user to authenticate and continue local work. +Before instance operations, establish the intended target from the user's request without displaying credentials. +Inspect command help from the installed CLI when flags or server compatibility are unclear. + +## Create and customize + +1. Establish the target product or technology, integration name, and desired topology or monitoring behavior. + Use telemetry discovery to select signals and resource attributes. Ask for missing evidence rather than inventing it. +2. Scaffold into a new directory: + + ```bash + sts stackpack scaffold --name my-stackpack --display-name "My StackPack" + ``` + +3. Read the generated `README.md` and `stackpack.yaml`. Preserve the required settings and resource paths. +4. Replace demo content with the requested integration. Keep identifiers unique within the pack namespace. + Align component identifiers, relation endpoints, monitor bindings, and presentation selectors. + Use actual telemetry labels in queries and meaningful production thresholds in monitors. + Add only the mappings, presentations, monitors, and resources the integration needs. +5. Package from the StackPack directory: + + ```bash + sts stackpack package -d . + ``` + +Packaging checks local structure. It does not prove server schema validity or successful installation. + +## Validate and deploy + +For authorized instance validation, use: + +```bash +sts stackpack validate -d . +``` + +Validation sends the package to the configured server. +For CI without an instance, use the offline validator described in the StackPack reference, pinned to the target server release. +For an authorized development deployment, use: + +```bash +sts stackpack test-deploy -d . --yes +``` + +This command uploads and installs or upgrades the pack with a test version suffix. +It can upgrade an existing installation. It is not a dry run. +Repeat after edits within the authorized scope. +Check the resulting topology, relations, metric queries, monitor bindings, and presentations against matching telemetry. +Use MCP `topology_query` and `component_get` to check component identities, properties, and relation endpoints. +Use `metrics_query_instant` or `metrics_query_range` to run the actual PromQL queries, and `metrics_bound` to check component bindings. +Use `monitors_overview`, `monitor_config`, and `monitor_violations` to check installed monitors and their results. +Verify rendered presentations in the SUSE Observability UI; MCP queries do not establish their visual correctness. + +For delivery, package again and report the generated `.sts` path. +If upload is requested, run `sts stackpack upload --file `. +Upload makes a version available. Installation is a separate operation. +Report which local and instance checks passed, and which remain untested. diff --git a/.claude/skills/create-stackpack b/.claude/skills/create-stackpack new file mode 120000 index 0000000..bd49a91 --- /dev/null +++ b/.claude/skills/create-stackpack @@ -0,0 +1 @@ +../../.agents/skills/create-stackpack \ No newline at end of file diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..f70c26b --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @StackVista/dev-team diff --git a/.github/workflows/stackpacks-ci.yml b/.github/workflows/stackpacks-ci.yml new file mode 100644 index 0000000..688fc6c --- /dev/null +++ b/.github/workflows/stackpacks-ci.yml @@ -0,0 +1,338 @@ +name: Core StackPacks CI + +run-name: >- + core stackpacks - + ${{ github.event_name == 'pull_request' && format('PR #{0} {1}', github.event.number, github.event.pull_request.title) || + github.event_name == 'workflow_dispatch' && 'manual rebuild' || + github.event_name == 'push' && format('{0}: {1}', github.ref_name, github.event.head_commit.message) || + github.ref_name }} + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +defaults: + run: + shell: bash --noprofile --norc -euo pipefail {0} + +concurrency: + group: core-stackpacks-${{ github.ref }} + cancel-in-progress: false + +jobs: + zizmor: + name: Workflow security (Zizmor) + runs-on: ubuntu-24.04 + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Audit workflows and actions + uses: zizmorcore/zizmor-action@45d7276c9ca790aabd705e31ca99c6185c3e76dd # v0.8.1 + with: + advanced-security: false + collect: all + inputs: . + + discover-stackpacks: + name: Discover stackpacks + runs-on: ubuntu-24.04 + outputs: + matrix: ${{ steps.discover.outputs.matrix }} + has_stackpacks: ${{ steps.discover.outputs.has_stackpacks }} + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Discover stackpack directories + id: discover + run: | + stackpacks=() + if [[ -d stackpacks ]]; then + for dir in stackpacks/*; do + [[ -d "${dir}" && -f "${dir}/stackpack.yaml" ]] || continue + stackpacks+=("$(basename "${dir}")") + done + fi + + if [[ ${#stackpacks[@]} -eq 0 ]]; then + echo 'matrix={"stackpack":[]}' >> "$GITHUB_OUTPUT" + echo 'has_stackpacks=false' >> "$GITHUB_OUTPUT" + echo "No stackpacks discovered" + exit 0 + fi + + matrix=$(printf '%s\n' "${stackpacks[@]}" | jq -R . | jq -cs '{stackpack:.}') + echo "matrix=${matrix}" >> "$GITHUB_OUTPUT" + echo 'has_stackpacks=true' >> "$GITHUB_OUTPUT" + echo "Discovered stackpacks: ${stackpacks[*]}" + + validate: + needs: [discover-stackpacks] + if: ${{ needs.discover-stackpacks.outputs.has_stackpacks == 'true' }} + name: validate (${{ matrix.stackpack }}) + runs-on: ubuntu-24.04 + env: + VALIDATOR_IMAGE: quay.io/stackstate/stackstate-server:7.0.0-snapshot.20261012082244-master-05004e2-2.5@sha256:2b980e1e2948eff5892e5815ed0d4a17780b8312b1eb2831234fdf7cfacfab8c + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.discover-stackpacks.outputs.matrix) }} + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Validate ${{ matrix.stackpack }} + env: + STACKPACK: ${{ matrix.stackpack }} + run: bash scripts/ci/validate_stackpack.sh "stackpacks/${STACKPACK}" + + - name: Verify validator rejects an invalid mapping expression + run: | + output=$(mktemp) + if bash scripts/ci/validate_stackpack.sh tests/fixtures/invalid-component-mapping >"${output}" 2>&1; then + cat "${output}" + echo "Validator accepted an invalid mapping" >&2 + exit 1 + fi + cat "${output}" + grep -q "undeclared.*undeclaredValidationControl" "${output}" + + package: + name: package (${{ matrix.stackpack }}) + needs: [discover-stackpacks, validate, check-version-bump] + if: | + always() && + needs.validate.result == 'success' && + needs.check-version-bump.result == 'success' + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.discover-stackpacks.outputs.matrix) }} + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install yq + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends yq + + - name: Install StackState CLI + env: + CLI_VERSION: 3.9.5 + run: | + tmp=$(mktemp -d) + trap 'rm -rf "${tmp}"' EXIT + archive="stackstate-cli-${CLI_VERSION}.linux-x86_64.tar.gz" + base="https://dl.stackstate.com/stackstate-cli/v${CLI_VERSION}" + curl --fail --silent --show-error --location "${base}/${archive}" -o "${tmp}/${archive}" + curl --fail --silent --show-error --location \ + "${base}/stackstate-cli_${CLI_VERSION}_checksums.txt" -o "${tmp}/checksums.txt" + (cd "${tmp}" && awk -v archive="${archive}" '$2 == archive { print }' checksums.txt | sha256sum --check --strict) + tar xzf "${tmp}/${archive}" -C "${tmp}" + sudo install -m 0755 "${tmp}/sts" /usr/local/bin/sts + sts version + + - name: Package ${{ matrix.stackpack }} + env: + STACKPACK: ${{ matrix.stackpack }} + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + RUN_NUMBER: ${{ github.run_number }} + SHA: ${{ github.sha }} + EVENT_NAME: ${{ github.event_name }} + REF: ${{ github.ref }} + run: | + cd "stackpacks/${STACKPACK}" + base_version=$(yq -r '.version' stackpack.yaml) + echo "Version from stackpack.yaml: ${base_version}" + + if [[ "${EVENT_NAME}" == "push" && "${REF}" == "refs/heads/main" ]]; then + stackpack_version=${base_version} + else + major=$(echo "${base_version}" | cut -d. -f1) + minor=$(echo "${base_version}" | cut -d. -f2) + patch=$(echo "${base_version}" | cut -d. -f3) + new_patch=$((patch + 1)) + branch_sanitized=${BRANCH_NAME//[^a-zA-Z0-9]/-} + short_sha="${SHA:0:7}" + stackpack_version="${major}.${minor}.${new_patch}-${branch_sanitized}-${RUN_NUMBER}-${short_sha}-SNAPSHOT" + sed -i "s/^version: .*/version: \"${stackpack_version}\"/" stackpack.yaml + fi + + export STS_EXPERIMENTAL_STACKPACKS=1 + sts stackpack package + mkdir -p target + mv -- *.sts target/ + ls -lh target/*.sts + + - name: Upload ${{ matrix.stackpack }} sts artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: sts-${{ matrix.stackpack }} + path: stackpacks/${{ matrix.stackpack }}/target/*.sts + if-no-files-found: error + + check-version-bump: + needs: [discover-stackpacks] + if: ${{ needs.discover-stackpacks.outputs.has_stackpacks == 'true' }} + name: Check version bump + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.discover-stackpacks.outputs.matrix) }} + env: + TARGET_BRANCH: ${{ github.base_ref || 'main' }} + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Install yq + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends yq + + - name: Compare versions + env: + STACKPACK: ${{ matrix.stackpack }} + run: ./scripts/ci/verify_versions_bumped.sh "${STACKPACK}" + + build-image: + name: Bundle build, smoke test, Trivy secret and vulnerability gate (${{ matrix.arch }}) + needs: [package, zizmor] + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + arch: [amd64, arm64] + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Download packaged StackPacks + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: sts-* + path: docker/rootfs/stackpacks + merge-multiple: true + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + - name: Build bundle and verify packaged content + env: + ARCH: ${{ matrix.arch }} + run: | + local_ref="local/core-stackpacks:${ARCH}" + docker buildx build --platform "linux/${ARCH}" --load \ + --provenance=false --sbom=false \ + --label "org.opencontainers.image.source=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ + --label "org.opencontainers.image.revision=${GITHUB_SHA}" \ + --label 'org.opencontainers.image.title=SUSE Observability Core StackPacks' \ + --tag "${local_ref}" docker + actual_arch=$(docker image inspect "${local_ref}" --format '{{.Architecture}}') + [[ "${actual_arch}" == "${ARCH}" ]] + cid=$(docker create --platform "linux/${ARCH}" --entrypoint /bin/sh "${local_ref}") + tmp=$(mktemp -d) + trap 'docker rm -f "${cid}" >/dev/null; rm -rf "${tmp}"' EXIT + docker cp "${cid}:/stackpacks/." "${tmp}/" + diff <(cd docker/rootfs/stackpacks && sha256sum -- *.sts | sort) \ + <(cd "${tmp}" && sha256sum -- *.sts | sort) + [[ $(find "${tmp}" -maxdepth 1 -type f -name '*.sts' | wc -l) -gt 0 ]] + - name: Trivy secret and vulnerability gate + uses: StackVista/image-pipeline/.github/actions/scan-image@6869879307d976158fa07ee5bff27ceb8eb031af # v1.0.0 + with: + image: local/core-stackpacks:${{ matrix.arch }} + mode: gate + severity: HIGH,CRITICAL + with-grype: true + upload-sarif: false + sarif-category: core-stackpacks-${{ matrix.arch }} + - name: Save tested image + env: + ARCH: ${{ matrix.arch }} + run: | + mkdir -p target + docker save "local/core-stackpacks:${ARCH}" -o "target/image-${ARCH}.tar" + - name: Upload tested image + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: core-image-${{ matrix.arch }} + path: target/image-${{ matrix.arch }}.tar + if-no-files-found: error + retention-days: 3 + + publish-image: + name: Publish and sign tested core bundle + needs: [build-image] + if: >- + (github.event_name == 'push' && github.ref == 'refs/heads/main') || + (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) + runs-on: ubuntu-24.04 + permissions: + contents: read + packages: write + id-token: write + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Download tested images + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: core-image-* + path: images + merge-multiple: true + - name: Compute image identity + id: image + env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + run: | + branch="${BRANCH_NAME//[^a-zA-Z0-9_.-]/-}" + short_sha=$(git rev-parse --short=7 HEAD) + timestamp=$(TZ=UTC0 git log -1 --format=%cd --date=format-local:'%Y%m%d%H%M%S') + tag="${timestamp}-${branch}-${short_sha}" + if (( GITHUB_RUN_ATTEMPT > 1 )); then + tag="${tag}-retry-${GITHUB_RUN_ATTEMPT}" + fi + echo "tag=${tag}" >> "${GITHUB_OUTPUT}" + echo "ref=ghcr.io/${GITHUB_REPOSITORY,,}" >> "${GITHUB_OUTPUT}" + - name: Publish and sign the tested images + uses: StackVista/image-pipeline/.github/actions/scan-sign-push@6869879307d976158fa07ee5bff27ceb8eb031af # v1.0.0 + with: + image: ${{ steps.image.outputs.ref }} + tags: ${{ steps.image.outputs.tag }} + arches: amd64,arm64 + source-mode: tarball + tarball-dir: images + target-registry: ghcr.io + target-registry-user: ${{ github.actor }} + target-registry-password: ${{ github.token }} + severity: HIGH,CRITICAL + with-grype: true + upload-sarif: false + - name: Verify anonymous image access + env: + IMAGE_REF: ${{ steps.image.outputs.ref }}:${{ steps.image.outputs.tag }} + run: | + tmp=$(mktemp -d) + trap 'rm -rf "${tmp}"' EXIT + if ! DOCKER_CONFIG="${tmp}" docker buildx imagetools inspect "${IMAGE_REF}" >/dev/null; then + echo 'Set the core-stackpacks container package visibility to Public before using it in the chart.' >&2 + echo 'After changing visibility, rerun the failed publication job.' >&2 + exit 1 + fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3b19c6d --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.idea +*.sts +**/target/ +docker/rootfs/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..87ecdeb --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,21 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.4.0 + hooks: + - id: check-added-large-files + stages: [pre-commit] + exclude: "^stackstate-generated-api/.*$" + - id: check-case-conflict + stages: [pre-commit] + - id: check-merge-conflict + stages: [pre-commit] + - id: detect-private-key + stages: [pre-commit] + - id: detect-aws-credentials + stages: [pre-commit] + args: + - --allow-missing-credentials + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: v1.25.0 + hooks: + - id: zizmor diff --git a/README.md b/README.md index 13b9077..ebdcaf9 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,88 @@ -# core-stackpacks -SUSE Observability core StackPacks maintained by the Observability team and bundled with the product +# Core StackPacks + +The SUSE Observability team maintains these StackPacks: + +- `open-telemetry` +- `otel-k8s-crd` +- `suse-observability` + +The product includes the core bundle by default. + +## Repository ownership + +| Repository | Content | Maintainer | Delivery | +| --- | --- | --- | --- | +| [core-stackpacks](https://github.com/StackVista/core-stackpacks) | Product integrations | SUSE Observability team | Included with the product | +| [suse-stackpacks](https://github.com/StackVista/suse-stackpacks) | SUSE product integrations | Respective product teams | Product distribution | +| [contrib-stackpacks](https://github.com/StackVista/contrib-stackpacks) | Community integrations for external technologies | Named community maintainers | Customers add individual packages | + +The existing `stackpacks` repository and SDK keep their current roles. + +## Build and validate + +The CI workflow validates every StackPack with the platform validator. +It packages the content with the `sts` CLI. +Changes to an existing StackPack require a higher package version. +The migration keeps the existing package names, versions, and content. + +For a local package build, run: + +```sh +cd stackpacks/open-telemetry +STS_EXPERIMENTAL_STACKPACKS=1 sts stackpack package +``` + +Use the validator image declared in `.github/workflows/stackpacks-ci.yml`: + +```sh +VALIDATOR_IMAGE='' \ + bash scripts/ci/validate_stackpack.sh stackpacks/open-telemetry +``` + +Use the [Create a StackPack skill](.agents/skills/create-stackpack/SKILL.md) +to create or extend an integration. + +## Image delivery + +CI builds a bundle for `linux/amd64` and `linux/arm64`. +The image contains the packaged files in `/stackpacks`. +It uses SUSE BCI and a non-root user. +The Helm chart supplies the script that copies these files into the platform. + +The image builds compare the packaged files against the image contents. +Trivy secret scanning and vulnerability scanning gate publication. +The shared image pipeline publishes those tested images and signs their combined manifest. +Fork pull requests run validation and image checks without publication credentials. +Same-repository pull requests and `main` pushes can publish a bundle. + +The image reference uses this form: + +```text +ghcr.io/stackvista/core-stackpacks:-- +``` + +The timestamp and commit identify the workflow checkout. +The tag on a pull request can identify GitHub's test merge commit. +Retried publication jobs append `-retry-` to create a new immutable tag. +The content versions remain in each `stackpack.yaml`. + +## First publication + +GitHub creates new container packages with private visibility. +An organization member with package administration access must set this package to public. +The publication workflow fails its anonymous access check until this change is complete. + +1. Open the `core-stackpacks` package settings in the StackVista organization. +2. Change the package visibility to `Public`. +3. Rerun the failed publication job. +4. Make sure that the image permits pulls without authentication: + +```sh +image='ghcr.io/stackvista/core-stackpacks:' +temporary_docker_configuration=$(mktemp -d) +DOCKER_CONFIG="${temporary_docker_configuration}" docker buildx imagetools inspect "${image}" +rm -rf "${temporary_docker_configuration}" +``` + +Before the chart adopts a new bundle, make sure that this check succeeds. +The chart's existing bundle stays in place until the replacement image is available. diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..de5543e --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,5 @@ +FROM registry.suse.com/bci/bci-minimal:15.7@sha256:207e6eab7c25f517e30c18e0479bbbb366d1bb3380f8d71a83064eb119edc71d + +COPY rootfs/stackpacks /stackpacks + +USER 1001 diff --git a/scripts/ci/validate_stackpack.sh b/scripts/ci/validate_stackpack.sh new file mode 100755 index 0000000..579c09d --- /dev/null +++ b/scripts/ci/validate_stackpack.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +stackpack_dir=$(realpath "${1:?StackPack directory is required}") +: "${VALIDATOR_IMAGE:?VALIDATOR_IMAGE must be set}" + +docker run --rm --network none \ + --entrypoint /opt/docker/bin/stack-pack-validator \ + -v "${stackpack_dir}:/stackpack:ro,z" \ + "${VALIDATOR_IMAGE}" \ + -directory /stackpack diff --git a/scripts/ci/verify_versions_bumped.sh b/scripts/ci/verify_versions_bumped.sh new file mode 100755 index 0000000..21f539a --- /dev/null +++ b/scripts/ci/verify_versions_bumped.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Verifies if a version of a stackpack has been "bumped" if not then it returns an error. +# The script is executed when there are some changes in a stackpack directory. +# +# Required env: TARGET_BRANCH (the PR base branch). In GitHub Actions, set this +# from ${{ github.base_ref }} in the workflow step. + +stackpack="$1" +: "${TARGET_BRANCH:?TARGET_BRANCH must be set (e.g. from github.base_ref)}" + +git fetch origin "${TARGET_BRANCH}" + +# Gate: skip if THIS BRANCH didn't modify stackpacks//. We use the merge-base +# (not ${TARGET_BRANCH}'s tip) because if master moves forward after this +# branch was cut — e.g. someone else legitimately bumps a stackpack on master — the +# tip comparison would report our branch as "having changes" for stackpacks we +# never touched, then the version-bump check below would falsely demand a bump. +# Comparing to the merge-base reflects only what this branch introduced. +# +# Note: the VERSION comparison below intentionally still uses the tip — if we +# *did* modify the stackpack, we want to fail loudly if master has bumped past us +# in the meantime (concurrent bumps would otherwise silently downgrade on +# merge or force an awkward stackpack.yaml conflict). +base=$(git merge-base "origin/${TARGET_BRANCH}" HEAD 2>/dev/null || echo "") +if [ -z "$base" ]; then + echo "Cannot determine merge-base with origin/${TARGET_BRANCH}; running version check against tip" +elif git diff --quiet "$base" HEAD -- "stackpacks/${stackpack}/"; then + echo "stackpacks/${stackpack}/ has no changes on this branch (since merge-base ${base}); skipping version-bump check" + exit 0 +fi + +check_if_stackpack_exists=$(git ls-tree -r "origin/${TARGET_BRANCH}" -- "stackpacks/${stackpack}/stackpack.yaml") +if [ -z "$check_if_stackpack_exists" ]; then + echo "stackpack.yaml doesnt exist in the target branch, probably it is the first commit for the stackpack. Skipping version check!!!" + exit 0 +fi + +remote_version=$(git show "origin/${TARGET_BRANCH}:stackpacks/${stackpack}/stackpack.yaml" | yq ".version") +new_version=$(yq ".version" "stackpacks/${stackpack}/stackpack.yaml") + +if [ "$(printf '%s\n' "$remote_version" "$new_version" | sort -V | head -n1)" = "$new_version" ]; then + echo "Version of the stackpack should be updated" + exit 1 +fi diff --git a/tests/fixtures/invalid-component-mapping/resources/logo.svg b/tests/fixtures/invalid-component-mapping/resources/logo.svg new file mode 100644 index 0000000..3b4bcab --- /dev/null +++ b/tests/fixtures/invalid-component-mapping/resources/logo.svg @@ -0,0 +1 @@ + diff --git a/tests/fixtures/invalid-component-mapping/resources/overview.md b/tests/fixtures/invalid-component-mapping/resources/overview.md new file mode 100644 index 0000000..ff4101c --- /dev/null +++ b/tests/fixtures/invalid-component-mapping/resources/overview.md @@ -0,0 +1 @@ +This fixture must fail validation because its component mapping uses an undeclared CEL variable. diff --git a/tests/fixtures/invalid-component-mapping/settings/mapping.sty b/tests/fixtures/invalid-component-mapping/settings/mapping.sty new file mode 100644 index 0000000..164bca8 --- /dev/null +++ b/tests/fixtures/invalid-component-mapping/settings/mapping.sty @@ -0,0 +1,25 @@ +nodes: + - _type: OtelComponentMapping + name: Invalid component mapping + identifier: urn:stackpack:validator-test:otel-component-mapping:invalid + input: + signal: [LOGS] + resource: + scope: + log: + action: "'CREATE'" + condition: "true" + vars: [] + output: + identifier: "'urn:validator-test:component'" + name: "undeclaredValidationControl" + typeName: "'vm'" + required: + configuration: "{}" + tags: [] + optional: + status: "{}" + tags: [] + rank: + specificity: 10.0 + expireAfter: 900000 diff --git a/tests/fixtures/invalid-component-mapping/stackpack.yaml b/tests/fixtures/invalid-component-mapping/stackpack.yaml new file mode 100644 index 0000000..88add3a --- /dev/null +++ b/tests/fixtures/invalid-component-mapping/stackpack.yaml @@ -0,0 +1,11 @@ +name: invalid-component-mapping +version: "0.0.1" +schemaVersion: "2.0" +displayName: Invalid component mapping +logoUrl: !resource "logo.svg" +overviewUrl: !resource "overview.md" +detailedOverviewUrl: !resource "overview.md" +categories: [] +configurationUrls: {} +provision: + companionStackPacks: []