From 92c4e71a102efd30f11e835baf29e02406ddf1da Mon Sep 17 00:00:00 2001 From: Alejandro Bonilla Date: Mon, 19 Oct 2026 19:04:27 -0400 Subject: [PATCH 1/2] feat(portworx): add Portworx StackPack Adds a StackPack for Portworx Enterprise 3.x on Kubernetes. Metrics come from a ServiceMonitor the agent's OpenTelemetry scraper picks up; topology comes from the StorageCluster and StorageNode custom resources. Portworx nodes and volumes merge into the Kubernetes node and persistent volume components. Includes presentations, 20 monitors ported from the Portworx operator's alert rules plus pool and volume health checks, and three dashboards. --- stackpacks/portworx/README.md | 28 + stackpacks/portworx/icons/cluster.svg | 1 + stackpacks/portworx/icons/portworx.svg | 1 + stackpacks/portworx/icons/storage-node.svg | 1 + stackpacks/portworx/icons/volume.svg | 1 + .../kvdb-cluster-unhealthy.md.hbs | 9 + .../kvdb-node-unhealthy.md.hbs | 10 + .../remediation-hints/license-expiry.md.hbs | 5 + .../remediation-hints/license-invalid.md.hbs | 5 + .../remediation-hints/member-down.md.hbs | 10 + .../remediation-hints/node-status.md.hbs | 15 + .../remediation-hints/pool-degraded.md.hbs | 9 + .../remediation-hints/pool-nearly-full.md.hbs | 12 + .../remediation-hints/pool-status.md.hbs | 10 + .../remediation-hints/quorum-unhealthy.md.hbs | 9 + .../storage-node-down.md.hbs | 10 + .../storage-usage-critical.md.hbs | 15 + .../storage-usage-warning.md.hbs | 15 + .../storage-will-fill.md.hbs | 9 + .../remediation-hints/volume-down.md.hbs | 10 + .../volume-fs-needs-inspection.md.hbs | 9 + .../volume-fs-safe-to-fix.md.hbs | 5 + .../volume-replication.md.hbs | 18 + .../volume-usage-critical.md.hbs | 14 + .../volume-usage-warning.md.hbs | 14 + stackpacks/portworx/resources/RELEASE.md | 8 + .../portworx/resources/configuration.md | 76 ++ .../portworx/resources/deprovisioning.md | 1 + .../portworx/resources/detailed-overview.md | 22 + stackpacks/portworx/resources/enabled.md | 5 + stackpacks/portworx/resources/error.md | 1 + stackpacks/portworx/resources/logo.svg | 1 + stackpacks/portworx/resources/overview.md | 1 + stackpacks/portworx/resources/provisioning.md | 1 + .../portworx/resources/waitingfordata.md | 9 + .../settings/component-mappings/cluster.sty | 54 ++ .../component-mappings/storage-nodes.sty | 59 ++ .../settings/component-mappings/volumes.sty | 50 ++ .../settings/dashboards/cluster-overview.sty | 793 ++++++++++++++++++ .../portworx/settings/dashboards/nodes.sty | 508 +++++++++++ .../portworx/settings/dashboards/volumes.sty | 617 ++++++++++++++ stackpacks/portworx/settings/main-menu.sty | 8 + .../portworx/settings/monitors/clusters.sty | 128 +++ .../settings/monitors/storage-nodes.sty | 170 ++++ .../portworx/settings/monitors/volumes.sty | 128 +++ .../settings/presentations/cluster.sty | 382 +++++++++ .../settings/presentations/common-filters.sty | 29 + .../settings/presentations/storage-nodes.sty | 277 ++++++ .../settings/presentations/volumes.sty | 225 +++++ .../node-member-of-cluster.sty | 31 + .../volume-provided-by-cluster.sty | 29 + stackpacks/portworx/stackpack.yaml | 17 + 52 files changed, 3875 insertions(+) create mode 100644 stackpacks/portworx/README.md create mode 100644 stackpacks/portworx/icons/cluster.svg create mode 100644 stackpacks/portworx/icons/portworx.svg create mode 100644 stackpacks/portworx/icons/storage-node.svg create mode 100644 stackpacks/portworx/icons/volume.svg create mode 100644 stackpacks/portworx/includes/remediation-hints/kvdb-cluster-unhealthy.md.hbs create mode 100644 stackpacks/portworx/includes/remediation-hints/kvdb-node-unhealthy.md.hbs create mode 100644 stackpacks/portworx/includes/remediation-hints/license-expiry.md.hbs create mode 100644 stackpacks/portworx/includes/remediation-hints/license-invalid.md.hbs create mode 100644 stackpacks/portworx/includes/remediation-hints/member-down.md.hbs create mode 100644 stackpacks/portworx/includes/remediation-hints/node-status.md.hbs create mode 100644 stackpacks/portworx/includes/remediation-hints/pool-degraded.md.hbs create mode 100644 stackpacks/portworx/includes/remediation-hints/pool-nearly-full.md.hbs create mode 100644 stackpacks/portworx/includes/remediation-hints/pool-status.md.hbs create mode 100644 stackpacks/portworx/includes/remediation-hints/quorum-unhealthy.md.hbs create mode 100644 stackpacks/portworx/includes/remediation-hints/storage-node-down.md.hbs create mode 100644 stackpacks/portworx/includes/remediation-hints/storage-usage-critical.md.hbs create mode 100644 stackpacks/portworx/includes/remediation-hints/storage-usage-warning.md.hbs create mode 100644 stackpacks/portworx/includes/remediation-hints/storage-will-fill.md.hbs create mode 100644 stackpacks/portworx/includes/remediation-hints/volume-down.md.hbs create mode 100644 stackpacks/portworx/includes/remediation-hints/volume-fs-needs-inspection.md.hbs create mode 100644 stackpacks/portworx/includes/remediation-hints/volume-fs-safe-to-fix.md.hbs create mode 100644 stackpacks/portworx/includes/remediation-hints/volume-replication.md.hbs create mode 100644 stackpacks/portworx/includes/remediation-hints/volume-usage-critical.md.hbs create mode 100644 stackpacks/portworx/includes/remediation-hints/volume-usage-warning.md.hbs create mode 100644 stackpacks/portworx/resources/RELEASE.md create mode 100644 stackpacks/portworx/resources/configuration.md create mode 100644 stackpacks/portworx/resources/deprovisioning.md create mode 100644 stackpacks/portworx/resources/detailed-overview.md create mode 100644 stackpacks/portworx/resources/enabled.md create mode 100644 stackpacks/portworx/resources/error.md create mode 100644 stackpacks/portworx/resources/logo.svg create mode 100644 stackpacks/portworx/resources/overview.md create mode 100644 stackpacks/portworx/resources/provisioning.md create mode 100644 stackpacks/portworx/resources/waitingfordata.md create mode 100644 stackpacks/portworx/settings/component-mappings/cluster.sty create mode 100644 stackpacks/portworx/settings/component-mappings/storage-nodes.sty create mode 100644 stackpacks/portworx/settings/component-mappings/volumes.sty create mode 100644 stackpacks/portworx/settings/dashboards/cluster-overview.sty create mode 100644 stackpacks/portworx/settings/dashboards/nodes.sty create mode 100644 stackpacks/portworx/settings/dashboards/volumes.sty create mode 100644 stackpacks/portworx/settings/main-menu.sty create mode 100644 stackpacks/portworx/settings/monitors/clusters.sty create mode 100644 stackpacks/portworx/settings/monitors/storage-nodes.sty create mode 100644 stackpacks/portworx/settings/monitors/volumes.sty create mode 100644 stackpacks/portworx/settings/presentations/cluster.sty create mode 100644 stackpacks/portworx/settings/presentations/common-filters.sty create mode 100644 stackpacks/portworx/settings/presentations/storage-nodes.sty create mode 100644 stackpacks/portworx/settings/presentations/volumes.sty create mode 100644 stackpacks/portworx/settings/relation-mappings/node-member-of-cluster.sty create mode 100644 stackpacks/portworx/settings/relation-mappings/volume-provided-by-cluster.sty create mode 100644 stackpacks/portworx/stackpack.yaml diff --git a/stackpacks/portworx/README.md b/stackpacks/portworx/README.md new file mode 100644 index 0000000..637ce27 --- /dev/null +++ b/stackpacks/portworx/README.md @@ -0,0 +1,28 @@ +# Portworx StackPack + +Monitors Portworx Enterprise 3.x on Kubernetes. User-facing setup is in `resources/configuration.md`. + +## Data path + +- **Metrics**: the agent's OpenTelemetry metrics scraper reads a `ServiceMonitor` labelled `observability.suse.com/agent: scrape` on the Portworx service (port `px-api`, `/metrics`). Metric names stay as Portworx emits them (`px_*`), plus `k8s_cluster_name`. The ServiceMonitor keeps only Portworx's own labels, because `px_volume_*` series copy every PVC label. +- **Topology**: the agent's Kubernetes resource collector sends `StorageCluster` and `StorageNode` custom resources (API group `core.libopenstorage.org`) as `KubernetesCustomResourceEvent` logs. + +## Topology + +- `settings/component-mappings/cluster.sty`: one `portworx cluster` component per `StorageCluster` (`urn:portworx:/:cluster/`). +- `settings/component-mappings/storage-nodes.sty`: each `StorageNode` adds `urn:portworx:/:node/` to the Kubernetes node (via `additionalIdentifiers`). It ranks below the Kubernetes node, so the node keeps its type and gets the `source-type:portworx node` label. +- `settings/component-mappings/volumes.sty`: Portworx has no resource per volume, so `px_volume_replication_status` (the only volume metric reported for every volume, attached or not) adds `urn:portworx:/:volume/` to the Kubernetes persistent volume. The label `volumename` is the PV name. +- Relations: node `member of` cluster, volume `provided by` cluster. + +## Metrics, monitors and dashboards + +- Presentations add a **Portworx** metrics tab to clusters, storage nodes and volumes, plus a **Portworx** main menu group with the three lists. +- Monitors port the Portworx operator's `PrometheusRule` (`portworx`) alerts with the same thresholds, and add per-pool "nearly full" (Portworx marks a pool full below min(10%, 100 GiB) free), pool degraded, volume down/error and filesystem health. Pool status ignores maintenance (4) and background activity (6). Status codes are compared with `bool`, so healthy components report 0 instead of no data. +- Volume metrics are reported by every node that holds a replica, and some of those series carry only `volumeid`, so volume queries use `max by` and `volumename!=""`. +- `px_volume_reads_total` / `px_volume_writes_total` carry only `volumeid`. Per-volume IOPS use the `px_volume_read_iops` / `px_volume_write_iops` gauges. +- Volume performance metrics only exist while a volume is attached. +- A volume with no matching Kubernetes PV stays a standalone `portworx volume` component. On the test cluster, one Portworx node kept exporting series for two deleted volumes, so they showed up this way until the series stopped. + +## Tested with + +Portworx 3.7.1 (operator-installed, internal KVDB) on a 3-node RKE2 / SUSE Virtualization cluster, with agent chart 1.6.12. diff --git a/stackpacks/portworx/icons/cluster.svg b/stackpacks/portworx/icons/cluster.svg new file mode 100644 index 0000000..c17c99a --- /dev/null +++ b/stackpacks/portworx/icons/cluster.svg @@ -0,0 +1 @@ + diff --git a/stackpacks/portworx/icons/portworx.svg b/stackpacks/portworx/icons/portworx.svg new file mode 100644 index 0000000..508a05d --- /dev/null +++ b/stackpacks/portworx/icons/portworx.svg @@ -0,0 +1 @@ + diff --git a/stackpacks/portworx/icons/storage-node.svg b/stackpacks/portworx/icons/storage-node.svg new file mode 100644 index 0000000..ba5c83f --- /dev/null +++ b/stackpacks/portworx/icons/storage-node.svg @@ -0,0 +1 @@ + diff --git a/stackpacks/portworx/icons/volume.svg b/stackpacks/portworx/icons/volume.svg new file mode 100644 index 0000000..bc5754d --- /dev/null +++ b/stackpacks/portworx/icons/volume.svg @@ -0,0 +1 @@ + diff --git a/stackpacks/portworx/includes/remediation-hints/kvdb-cluster-unhealthy.md.hbs b/stackpacks/portworx/includes/remediation-hints/kvdb-cluster-unhealthy.md.hbs new file mode 100644 index 0000000..6eae38a --- /dev/null +++ b/stackpacks/portworx/includes/remediation-hints/kvdb-cluster-unhealthy.md.hbs @@ -0,0 +1,9 @@ +### KVDB unhealthy +The cluster sees the Portworx key-value database as unhealthy. Volume operations (create, attach, failover) fail until it recovers. + +#### Check first +1. [Portworx - Cluster overview](/#/dashboards/urn:stackpack:portworx:dashboard:cluster-overview) → **KVDB health per node** and **KVDB watch latency**. +2. `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl service kvdb members`: are all members healthy and is there a leader? + +#### Fix +Restore the failed KVDB member or its disk, or the network between members. diff --git a/stackpacks/portworx/includes/remediation-hints/kvdb-node-unhealthy.md.hbs b/stackpacks/portworx/includes/remediation-hints/kvdb-node-unhealthy.md.hbs new file mode 100644 index 0000000..e12d9cb --- /dev/null +++ b/stackpacks/portworx/includes/remediation-hints/kvdb-node-unhealthy.md.hbs @@ -0,0 +1,10 @@ +### KVDB unhealthy on node +For 5 minutes this node has seen the Portworx key-value database (internal KVDB or external etcd) as unhealthy. Without it, Portworx can't change volume state (create, attach, failover). + +#### Check first +1. [Portworx - Cluster overview](/#/dashboards/urn:stackpack:portworx:dashboard:cluster-overview) → **KVDB health per node**: one node or all? +2. `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl service kvdb members` and `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl service kvdb endpoints`. +3. The `portworx-kvdb` pods and the KVDB disk: slow fsync is the most common cause. + +#### Fix +Restore network between KVDB members, replace a slow or failed KVDB disk, or bring back the failed member. diff --git a/stackpacks/portworx/includes/remediation-hints/license-expiry.md.hbs b/stackpacks/portworx/includes/remediation-hints/license-expiry.md.hbs new file mode 100644 index 0000000..acbcee0 --- /dev/null +++ b/stackpacks/portworx/includes/remediation-hints/license-expiry.md.hbs @@ -0,0 +1,5 @@ +### License expires soon +The Portworx license expires in less than **{{ threshold }}** days. After it expires, Portworx can't create or attach new volumes. + +#### Fix +Renew or activate the license: `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl license activate `, or update the license server settings. Check with `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl license list`. diff --git a/stackpacks/portworx/includes/remediation-hints/license-invalid.md.hbs b/stackpacks/portworx/includes/remediation-hints/license-invalid.md.hbs new file mode 100644 index 0000000..851361e --- /dev/null +++ b/stackpacks/portworx/includes/remediation-hints/license-invalid.md.hbs @@ -0,0 +1,5 @@ +### License invalid +Portworx reports an invalid license. New volumes can't be created or attached. + +#### Fix +Check `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl license list` and activate a valid license, or fix the connection to the license server. diff --git a/stackpacks/portworx/includes/remediation-hints/member-down.md.hbs b/stackpacks/portworx/includes/remediation-hints/member-down.md.hbs new file mode 100644 index 0000000..554e6f6 --- /dev/null +++ b/stackpacks/portworx/includes/remediation-hints/member-down.md.hbs @@ -0,0 +1,10 @@ +### Cluster member down +For 5 minutes fewer nodes have been online than the cluster size. + +#### Check first +1. [Cluster metrics](/#/components/{{ componentUrnForUrl }}/metrics) → **Nodes**: online, offline, storage down. +2. `kubectl -n get storagenodes` and the `px-cluster` pods. + +#### Likely causes +- A node is powered off, cordoned and drained, or the `px-cluster` pod there isn't running. +- A node was removed from Kubernetes but not decommissioned in Portworx (`kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl cluster delete `). diff --git a/stackpacks/portworx/includes/remediation-hints/node-status.md.hbs b/stackpacks/portworx/includes/remediation-hints/node-status.md.hbs new file mode 100644 index 0000000..bc1c28d --- /dev/null +++ b/stackpacks/portworx/includes/remediation-hints/node-status.md.hbs @@ -0,0 +1,15 @@ +### Storage node not OK +For 5 minutes Portworx has reported this node in a state other than OK (2). Common codes: 3 offline, 5 not in quorum, 8 storage down. + +#### Check first +1. [Node metrics](/#/components/{{ componentUrnForUrl }}/metrics) → **Portworx** tab → **Health**: status code, pool status, KVDB health. +2. The `px-cluster` pod on this node: is it running and ready? Read its logs (container `portworx`). +3. `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl status` on the node, and `kubectl -n get storagenodes`. + +#### Likely causes +- The node lost network to the other Portworx nodes (ports 9001-9022). +- A pool disk failed, or the pool is full. +- The `px-cluster` pod can't start. On RKE2, an empty directory at `/run/containerd/containerd.sock` blocks it ("not a directory"). + +#### Verify +The status returns to 2 and the StorageNode phase is Online. diff --git a/stackpacks/portworx/includes/remediation-hints/pool-degraded.md.hbs b/stackpacks/portworx/includes/remediation-hints/pool-degraded.md.hbs new file mode 100644 index 0000000..baa459f --- /dev/null +++ b/stackpacks/portworx/includes/remediation-hints/pool-degraded.md.hbs @@ -0,0 +1,9 @@ +### Storage pool degraded +A storage pool on this node is degraded (status 5). It still serves I/O, but with reduced redundancy or performance. + +#### Check first +1. [Node metrics](/#/components/{{ componentUrnForUrl }}/metrics) → **Portworx** tab → **Health**, and **Disk latency** under **Performance**. +2. `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl service pool show` and `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl service drive show` on the node, and `dmesg` for disk errors. + +#### Fix +Replace the failing drive (`kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl service drive replace`) and let the pool recover. diff --git a/stackpacks/portworx/includes/remediation-hints/pool-nearly-full.md.hbs b/stackpacks/portworx/includes/remediation-hints/pool-nearly-full.md.hbs new file mode 100644 index 0000000..b397d34 --- /dev/null +++ b/stackpacks/portworx/includes/remediation-hints/pool-nearly-full.md.hbs @@ -0,0 +1,12 @@ +### Storage pool nearly full +One pool on this node is close to the point where Portworx marks it **full**: free space below 10% of the pool or 100 GiB, whichever is smaller. A full pool puts the node in StorageDown and its volumes become read-only. This can happen on one pool while the node's total capacity looks fine. + +#### Check first +1. [Node metrics](/#/components/{{ componentUrnForUrl }}/metrics) → **Portworx** tab → **Pool usage**. +2. `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl service pool show` on the node. +3. [Portworx - Cluster overview](/#/dashboards/urn:stackpack:portworx:dashboard:cluster-overview) → **Volumes in trashcan**: deleted volumes in the trashcan still use pool space. + +#### Fix +1. Expand the pool (`kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl service pool expand`) or add drives. +2. Move replicas to other pools or nodes (`kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl service pool rebalance`, or `volume ha-update`). +3. Empty the trashcan and delete unused volumes and snapshots. diff --git a/stackpacks/portworx/includes/remediation-hints/pool-status.md.hbs b/stackpacks/portworx/includes/remediation-hints/pool-status.md.hbs new file mode 100644 index 0000000..bccd289 --- /dev/null +++ b/stackpacks/portworx/includes/remediation-hints/pool-status.md.hbs @@ -0,0 +1,10 @@ +### Storage pool offline or full +A storage pool on this node is **offline** (0), **full** (2) or **not found** (3). The **Pool status** chart shows which pool and which code. A full or offline pool puts the node in StorageDown, and volumes with replicas on it become read-only or degraded. + +#### Check first +1. [Node metrics](/#/components/{{ componentUrnForUrl }}/metrics) → **Portworx** tab → **Health** and **Capacity**. +2. `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl service pool show` on the node, and `dmesg` for disk errors. + +#### Fix +- Full: expand the pool (`kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl service pool expand`), or move replicas off it (`kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl service pool rebalance`). +- Offline or not found: check the drive and its path, replace a failed drive (`kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl service drive replace`), then let Portworx resync. diff --git a/stackpacks/portworx/includes/remediation-hints/quorum-unhealthy.md.hbs b/stackpacks/portworx/includes/remediation-hints/quorum-unhealthy.md.hbs new file mode 100644 index 0000000..4a7a5df --- /dev/null +++ b/stackpacks/portworx/includes/remediation-hints/quorum-unhealthy.md.hbs @@ -0,0 +1,9 @@ +### Cluster quorum unhealthy +For 5 minutes Portworx has reported a quorum problem. Without quorum, Portworx stops I/O to protect data. + +#### Check first +1. [Cluster metrics](/#/components/{{ componentUrnForUrl }}/metrics) → **Health**: quorum and nodes online. +2. `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl status` and `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl cluster list`. + +#### Fix +Bring enough storage nodes back online to restore a majority. Check the network between nodes first. diff --git a/stackpacks/portworx/includes/remediation-hints/storage-node-down.md.hbs b/stackpacks/portworx/includes/remediation-hints/storage-node-down.md.hbs new file mode 100644 index 0000000..287b5cf --- /dev/null +++ b/stackpacks/portworx/includes/remediation-hints/storage-node-down.md.hbs @@ -0,0 +1,10 @@ +### Storage node down +For 5 minutes one or more storage nodes have reported their storage down. Volumes with replicas on those nodes run degraded or lose quorum. + +#### Check first +1. [Cluster metrics](/#/components/{{ componentUrnForUrl }}/metrics) → **Portworx** tab → **Health**. +2. **Storage nodes** (on this page): which node is not OK. +3. `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl status` from any healthy node. + +#### Fix +Fix the node (pod, network, disk) as described in *Storage node not OK*. diff --git a/stackpacks/portworx/includes/remediation-hints/storage-usage-critical.md.hbs b/stackpacks/portworx/includes/remediation-hints/storage-usage-critical.md.hbs new file mode 100644 index 0000000..a8b75d9 --- /dev/null +++ b/stackpacks/portworx/includes/remediation-hints/storage-usage-critical.md.hbs @@ -0,0 +1,15 @@ +### Storage node almost full +More than **{{ threshold }}%** of this node's Portworx pool capacity is used. Volumes with replicas here can't grow, and new replicas can't be placed on this node. + +#### Check first +1. [Node metrics](/#/components/{{ componentUrnForUrl }}/metrics) → **Portworx** tab: capacity, pool usage and replica count. +2. [Portworx - Nodes and pools](/#/dashboards/urn:stackpack:portworx:dashboard:nodes): compare nodes. One full node usually means the replicas are spread unevenly. +3. `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl status` and `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl service pool show` on the node. + +#### Fix +1. Add capacity: expand the pool (`kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl service pool expand`) or add drives to the StorageCluster. +2. Rebalance replicas to emptier nodes: `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl service pool rebalance` (Portworx 3.x) or `volume ha-update`. +3. Delete unused volumes and old snapshots. + +#### Verify +The used share drops below the threshold. diff --git a/stackpacks/portworx/includes/remediation-hints/storage-usage-warning.md.hbs b/stackpacks/portworx/includes/remediation-hints/storage-usage-warning.md.hbs new file mode 100644 index 0000000..630e504 --- /dev/null +++ b/stackpacks/portworx/includes/remediation-hints/storage-usage-warning.md.hbs @@ -0,0 +1,15 @@ +### Storage node filling up +More than **{{ threshold }}%** of this node's Portworx pool capacity is used. Plan more capacity before it reaches 80%. + +#### Check first +1. [Node metrics](/#/components/{{ componentUrnForUrl }}/metrics) → **Portworx** tab: capacity, pool usage and replica count. +2. [Portworx - Nodes and pools](/#/dashboards/urn:stackpack:portworx:dashboard:nodes): compare nodes. One full node usually means the replicas are spread unevenly. +3. `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl status` and `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl service pool show` on the node. + +#### Fix +1. Add capacity: expand the pool (`kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl service pool expand`) or add drives to the StorageCluster. +2. Rebalance replicas to emptier nodes: `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl service pool rebalance` (Portworx 3.x) or `volume ha-update`. +3. Delete unused volumes and old snapshots. + +#### Verify +The used share drops below the threshold. diff --git a/stackpacks/portworx/includes/remediation-hints/storage-will-fill.md.hbs b/stackpacks/portworx/includes/remediation-hints/storage-will-fill.md.hbs new file mode 100644 index 0000000..8970bfd --- /dev/null +++ b/stackpacks/portworx/includes/remediation-hints/storage-will-fill.md.hbs @@ -0,0 +1,9 @@ +### Storage node will fill within 2 weeks +Less than 30% of this node's pool capacity is free, and at the growth rate of the last hour it fills up within 14 days. + +#### Check first +1. [Node metrics](/#/components/{{ componentUrnForUrl }}/metrics) → **Portworx** tab → **Capacity**: is the growth steady or a one-off (a restore, a large copy)? +2. [Portworx - Volumes](/#/dashboards/urn:stackpack:portworx:dashboard:volumes): which volumes grow fastest? + +#### Fix +Add capacity, rebalance replicas, or clean up volumes and snapshots, as for *Storage node filling up*. diff --git a/stackpacks/portworx/includes/remediation-hints/volume-down.md.hbs b/stackpacks/portworx/includes/remediation-hints/volume-down.md.hbs new file mode 100644 index 0000000..498ca8f --- /dev/null +++ b/stackpacks/portworx/includes/remediation-hints/volume-down.md.hbs @@ -0,0 +1,10 @@ +### Volume down +For 5 minutes Portworx has reported the volume status as **down** or its state as **error**. The workload can't use the volume. + +#### Check first +1. [Volume metrics](/#/components/{{ componentUrnForUrl }}/metrics) → **Portworx** tab → **Replication**. +2. `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl volume inspect `: state, replica sets, and the nodes that hold them. +3. Portworx → **Storage nodes**: are the nodes holding its replicas online? + +#### Fix +Bring back the nodes or pools that hold the volume's replicas. If a replica is lost for good, add a new one with `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl volume ha-update --repl --node `. diff --git a/stackpacks/portworx/includes/remediation-hints/volume-fs-needs-inspection.md.hbs b/stackpacks/portworx/includes/remediation-hints/volume-fs-needs-inspection.md.hbs new file mode 100644 index 0000000..350b9c8 --- /dev/null +++ b/stackpacks/portworx/includes/remediation-hints/volume-fs-needs-inspection.md.hbs @@ -0,0 +1,9 @@ +### Volume filesystem needs inspection +Portworx's filesystem check found errors that it can't fix safely on its own (filesystem health 3). Mounting or resizing the volume can fail until they are fixed. + +#### Check first +`kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl volume check status `. + +#### Fix +1. Stop the workload that uses the volume. +2. Run `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl volume check start --mode check_health ` to see the errors, then `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl volume check start --mode fix_all ` if you accept the repairs. `fix_all` can discard damaged data, so take a snapshot first. diff --git a/stackpacks/portworx/includes/remediation-hints/volume-fs-safe-to-fix.md.hbs b/stackpacks/portworx/includes/remediation-hints/volume-fs-safe-to-fix.md.hbs new file mode 100644 index 0000000..5169d26 --- /dev/null +++ b/stackpacks/portworx/includes/remediation-hints/volume-fs-safe-to-fix.md.hbs @@ -0,0 +1,5 @@ +### Volume filesystem has fixable errors +Portworx's filesystem check found errors that are safe to fix (filesystem health 2). + +#### Fix +Stop the workload that uses the volume, then run `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl volume check start --mode fix_safe `. Check progress with `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl volume check status `. diff --git a/stackpacks/portworx/includes/remediation-hints/volume-replication.md.hbs b/stackpacks/portworx/includes/remediation-hints/volume-replication.md.hbs new file mode 100644 index 0000000..646b35c --- /dev/null +++ b/stackpacks/portworx/includes/remediation-hints/volume-replication.md.hbs @@ -0,0 +1,18 @@ +### Volume replication unhealthy +Portworx reports the volume as **not in quorum** (1), **resyncing** (2) or **degraded** (3). The **Replication status** chart shows which. + +- *Not in quorum*: too few replicas are reachable; I/O to the volume stops until quorum returns. +- *Resync*: a replica is catching up after a node came back. It runs on its own. +- *Degraded*: a replica is missing and the volume runs with fewer copies. + +#### Check first +1. [Volume metrics](/#/components/{{ componentUrnForUrl }}/metrics) → **Portworx** tab → **Replication**: status code, configured vs current replicas. +2. `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl volume inspect `: the replica sets and the nodes that hold them. +3. Portworx → **Storage nodes**: is a node that holds a replica offline or not OK? + +#### Fix +1. Bring the missing node back. Portworx resyncs the replica by itself. +2. If the node is gone for good, move the replica: `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl volume ha-update --repl --node `, then remove the dead node from the cluster. + +#### Verify +Status returns to 0 (up) and the current replicas match the configured ones. diff --git a/stackpacks/portworx/includes/remediation-hints/volume-usage-critical.md.hbs b/stackpacks/portworx/includes/remediation-hints/volume-usage-critical.md.hbs new file mode 100644 index 0000000..b1439a2 --- /dev/null +++ b/stackpacks/portworx/includes/remediation-hints/volume-usage-critical.md.hbs @@ -0,0 +1,14 @@ +### Volume almost full +More than **{{ threshold }}%** of the volume is used. When it fills up, writes from the workload fail. + +#### Check first +1. [Volume metrics](/#/components/{{ componentUrnForUrl }}/metrics) → **Portworx** tab: used bytes against the size, and how fast it grows. +2. [Portworx - Volumes](/#/dashboards/urn:stackpack:portworx:dashboard:volumes): is it only this volume or many? + +#### Fix +1. Expand the volume: raise the PVC's `spec.resources.requests.storage` (the StorageClass needs `allowVolumeExpansion: true`), or `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl volume update --size `. +2. Or free space inside the workload (old data, logs, snapshots kept by the application). +3. To expand automatically, add a Portworx Autopilot rule for this PVC. + +#### Verify +The used share drops below the threshold. diff --git a/stackpacks/portworx/includes/remediation-hints/volume-usage-warning.md.hbs b/stackpacks/portworx/includes/remediation-hints/volume-usage-warning.md.hbs new file mode 100644 index 0000000..a91ddb2 --- /dev/null +++ b/stackpacks/portworx/includes/remediation-hints/volume-usage-warning.md.hbs @@ -0,0 +1,14 @@ +### Volume filling up +More than **{{ threshold }}%** of the volume is used. Plan an expansion before it reaches 80%, where writes start to be at risk. + +#### Check first +1. [Volume metrics](/#/components/{{ componentUrnForUrl }}/metrics) → **Portworx** tab: used bytes against the size, and how fast it grows. +2. [Portworx - Volumes](/#/dashboards/urn:stackpack:portworx:dashboard:volumes): is it only this volume or many? + +#### Fix +1. Expand the volume: raise the PVC's `spec.resources.requests.storage` (the StorageClass needs `allowVolumeExpansion: true`), or `kubectl -n exec -c portworx -- /opt/pwx/bin/pxctl volume update --size `. +2. Or free space inside the workload (old data, logs, snapshots kept by the application). +3. To expand automatically, add a Portworx Autopilot rule for this PVC. + +#### Verify +The used share drops below the threshold. diff --git a/stackpacks/portworx/resources/RELEASE.md b/stackpacks/portworx/resources/RELEASE.md new file mode 100644 index 0000000..1f3e962 --- /dev/null +++ b/stackpacks/portworx/resources/RELEASE.md @@ -0,0 +1,8 @@ +# Release notes + +## 0.1.0 + +- Portworx clusters as components (from the `StorageCluster` resource), with Portworx state merged into Kubernetes nodes and persistent volumes. +- Metrics for clusters, storage nodes and volumes. +- 20 monitors: the Portworx operator's Prometheus alert rules, plus per-pool "nearly full", pool degraded, volume down/error and volume filesystem health. +- Dashboards: Cluster overview, Volumes, Nodes and pools. diff --git a/stackpacks/portworx/resources/configuration.md b/stackpacks/portworx/resources/configuration.md new file mode 100644 index 0000000..b1ba9b0 --- /dev/null +++ b/stackpacks/portworx/resources/configuration.md @@ -0,0 +1,76 @@ +## Prerequisites + +- Portworx Enterprise 3.x installed with the Portworx operator (`StorageCluster`). +- SUSE Observability agent (Helm chart `suse-observability-agent` 1.6 or later) with OpenTelemetry enabled (`otel.enabled: true`). +- The Prometheus Operator CRDs (`monitoring.coreos.com`) in the cluster. The agent reads `ServiceMonitor` objects; no Prometheus server is needed. + +## 1. Let Portworx export metrics + +Portworx serves metrics on every node at `:9001/metrics` (`:17001` on OpenShift). In the `StorageCluster`, make sure metrics export is on: + +```yaml +spec: + monitoring: + prometheus: + exportMetrics: true +``` + +## 2. Let the agent scrape Portworx + +Create this `ServiceMonitor` in the namespace where Portworx runs (`portworx` or `kube-system`). The `observability.suse.com/agent: scrape` label makes the agent pick it up. + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: so-agent-portworx + labels: + observability.suse.com/agent: scrape +spec: + selector: + matchLabels: + name: portworx + endpoints: + - port: px-api + path: /metrics + interval: 60s + # Keep the Portworx 'namespace' label (the PVC namespace) instead of renaming it to exported_namespace. + honorLabels: true + metricRelabelings: + - sourceLabels: [__name__] + regex: px_.* + action: keep + # px_volume_* series copy every label of the PVC; keep only the labels Portworx itself defines. + - regex: __name__|job|instance|cluster|clusterUUID|node|nodeID|nodeid|pool|poolid|disk|proc|coordinator|target|volumeid|volumename|namespace|pvc|repl|kube_datastore|le|quantile + action: labelkeep +``` + +The `labelkeep` rule matters: Portworx adds every label of a PVC to its volume metrics, which can create many series. If the Portworx `Service` with label `name: portworx` doesn't exist, point `selector` at the service that exposes port `px-api`. + +## 3. Let the agent read the Portworx resources + +Add the Portworx API group to the agent's Kubernetes resource collector, in the agent Helm values: + +```yaml +otel: + enabled: true + k8sResourceCollector: + crDiscovery: + apiGroups: + include: + core.libopenstorage.org: true +``` + +Then upgrade the agent, for example: + +```bash +helm upgrade suse-observability-agent suse-observability/suse-observability-agent \ + -n suse-observability --reuse-values \ + --set 'otel.k8sResourceCollector.crDiscovery.apiGroups.include.core\.libopenstorage\.org=true' +``` + +If a Rancher app, a Fleet bundle or a Harvester add-on manages the agent, change its values there instead, or the next sync reverts the change. + +## 4. Install the StackPack + +Install this StackPack. Portworx clusters, storage nodes and volumes appear under **Portworx** in the main menu within a few minutes. diff --git a/stackpacks/portworx/resources/deprovisioning.md b/stackpacks/portworx/resources/deprovisioning.md new file mode 100644 index 0000000..4db3932 --- /dev/null +++ b/stackpacks/portworx/resources/deprovisioning.md @@ -0,0 +1 @@ +The Portworx StackPack is being uninstalled. The ServiceMonitor and agent settings you added for it stay in place; remove them if you no longer need them. diff --git a/stackpacks/portworx/resources/detailed-overview.md b/stackpacks/portworx/resources/detailed-overview.md new file mode 100644 index 0000000..25ab1fb --- /dev/null +++ b/stackpacks/portworx/resources/detailed-overview.md @@ -0,0 +1,22 @@ +## Portworx StackPack + +The Portworx StackPack shows your Portworx clusters, storage nodes and volumes in SUSE Observability and tracks their health with monitors ported from the Portworx operator's alert rules. + +### What you get + +- **Portworx clusters** as components (from the `StorageCluster` resource), linked to their storage nodes and volumes. +- **Storage nodes**: Kubernetes nodes that run Portworx get Portworx state (phase, version, KVDB membership) and a **Portworx** metrics tab. +- **Volumes**: Kubernetes persistent volumes that Portworx provides get a **Portworx** metrics tab (usage, replication, latency, IOPS, throughput). +- **Monitors** with Portworx's own thresholds and alert conditions: + - volume usage above 70% / 80% + - volume replication unhealthy, volume down or in error, volume filesystem errors + - storage node usage above 70% / 80%, and storage that will fill within 2 weeks + - storage node not OK; storage pool offline, full, degraded or nearly full (per pool) + - storage node down, cluster member down, quorum lost + - KVDB unhealthy + - license expiring or invalid +- **Dashboards**: *Portworx - Cluster overview*, *Portworx - Volumes* and *Portworx - Nodes and pools*, modelled on the Portworx Grafana dashboards. + +### How it works + +The SUSE Observability agent scrapes the Portworx metrics endpoint through a `ServiceMonitor` and reads the Portworx custom resources with its Kubernetes resource collector. No Prometheus server is needed. diff --git a/stackpacks/portworx/resources/enabled.md b/stackpacks/portworx/resources/enabled.md new file mode 100644 index 0000000..353f41f --- /dev/null +++ b/stackpacks/portworx/resources/enabled.md @@ -0,0 +1,5 @@ +## The Portworx StackPack is installed + +Open **Portworx** in the main menu for the cluster, storage node and volume lists, or start with the *Portworx - Cluster overview* dashboard. + +To monitor another Kubernetes cluster that runs Portworx, repeat steps 1 to 3 of the configuration there. diff --git a/stackpacks/portworx/resources/error.md b/stackpacks/portworx/resources/error.md new file mode 100644 index 0000000..4ce9e5f --- /dev/null +++ b/stackpacks/portworx/resources/error.md @@ -0,0 +1 @@ +The Portworx StackPack could not be installed. Check the StackPack's settings for errors and try again, or contact your SUSE Observability administrator. diff --git a/stackpacks/portworx/resources/logo.svg b/stackpacks/portworx/resources/logo.svg new file mode 100644 index 0000000..d4c1c06 --- /dev/null +++ b/stackpacks/portworx/resources/logo.svg @@ -0,0 +1 @@ + diff --git a/stackpacks/portworx/resources/overview.md b/stackpacks/portworx/resources/overview.md new file mode 100644 index 0000000..97377a2 --- /dev/null +++ b/stackpacks/portworx/resources/overview.md @@ -0,0 +1 @@ +Monitor [Portworx Enterprise](https://docs.portworx.com/) storage clusters on Kubernetes: capacity, quorum, storage node and volume health, and I/O performance, with the alerts Portworx ships for Prometheus. diff --git a/stackpacks/portworx/resources/provisioning.md b/stackpacks/portworx/resources/provisioning.md new file mode 100644 index 0000000..feb3d5a --- /dev/null +++ b/stackpacks/portworx/resources/provisioning.md @@ -0,0 +1 @@ +The Portworx StackPack is being installed. diff --git a/stackpacks/portworx/resources/waitingfordata.md b/stackpacks/portworx/resources/waitingfordata.md new file mode 100644 index 0000000..03cb459 --- /dev/null +++ b/stackpacks/portworx/resources/waitingfordata.md @@ -0,0 +1,9 @@ +The Portworx StackPack is installed and waiting for data. + +If nothing appears after a few minutes, check that: + +- the `so-agent-portworx` ServiceMonitor exists and carries the label `observability.suse.com/agent: scrape`; +- the agent has OpenTelemetry enabled (`otel.enabled: true`) and `core.libopenstorage.org` in `otel.k8sResourceCollector.crDiscovery.apiGroups.include`; +- `curl http://:9001/metrics` on a Portworx node returns `px_*` metrics. + +See the configuration steps above. diff --git a/stackpacks/portworx/settings/component-mappings/cluster.sty b/stackpacks/portworx/settings/component-mappings/cluster.sty new file mode 100644 index 0000000..d90ca55 --- /dev/null +++ b/stackpacks/portworx/settings/component-mappings/cluster.sty @@ -0,0 +1,54 @@ +# Component mapping: Portworx StorageCluster custom resource -> portworx cluster. +# Needs the agent's k8s resource collector to watch the core.libopenstorage.org API group. +nodes: + - _type: "OtelComponentMapping" + name: "Portworx cluster" + description: "Turns a Portworx StorageCluster custom resource into a Portworx cluster component" + identifier: "urn:stackpack:portworx:otel-component-mapping:cluster" + input: + signal: + - "LOGS" + resource: + scope: + log: + action: "log.body['type'] in ['ADDED', 'MODIFIED'] ? 'CREATE' : log.body['type'] == 'DELETED' ? 'DELETE' : 'CONTINUE'" + condition: >- + log.eventName == 'KubernetesCustomResourceEvent' && + log.attributes['k8s.resource.group'] == 'core.libopenstorage.org' && + log.body['object']['kind'] == 'StorageCluster' + vars: + - name: "clusterName" + value: "resource.attributes['k8s.cluster.name']" + - name: "name" + value: "log.attributes['k8s.object.name']" + - name: "namespace" + value: "log.attributes['k8s.namespace.name']" + output: + identifier: "'urn:portworx:/' + vars.clusterName + ':cluster/' + vars.name" + name: "vars.name" + typeName: "'portworx cluster'" + required: + configuration: "log.body['object']['spec']" + tags: + - source: "'portworx'" + target: "stackpack" + - source: "vars.clusterName" + target: "cluster-name" + - source: "vars.namespace" + target: "namespace" + - source: "vars.name" + target: "portworx.cluster" + optional: + status: "log.body['object']['status']" + tags: + - source: "log.body['object']['status']['phase']" + target: "portworx.phase" + - source: "log.body['object']['status']['version']" + target: "portworx.version" + - source: "log.body['object']['status']['clusterUid']" + target: "portworx.cluster-uid" + - source: "log.body['object']['metadata']['creationTimestamp']" + target: "k8s.creationTimestamp" + expireAfter: 900000 + rank: + specificity: 100 diff --git a/stackpacks/portworx/settings/component-mappings/storage-nodes.sty b/stackpacks/portworx/settings/component-mappings/storage-nodes.sty new file mode 100644 index 0000000..6373b00 --- /dev/null +++ b/stackpacks/portworx/settings/component-mappings/storage-nodes.sty @@ -0,0 +1,59 @@ +# Component mapping: Portworx StorageNode custom resource -> portworx node, merged into the Kubernetes node. +# Ranked below the Kubernetes node so the merged component keeps its "node" type; +# ours survives as the "source-type:portworx node" label. +nodes: + - _type: "OtelComponentMapping" + name: "Portworx storage node" + description: "Adds Portworx state to the Kubernetes node that runs a Portworx StorageNode" + identifier: "urn:stackpack:portworx:otel-component-mapping:storage-node" + input: + signal: + - "LOGS" + resource: + scope: + log: + action: "log.body['type'] in ['ADDED', 'MODIFIED'] ? 'CREATE' : log.body['type'] == 'DELETED' ? 'DELETE' : 'CONTINUE'" + condition: >- + log.eventName == 'KubernetesCustomResourceEvent' && + log.attributes['k8s.resource.group'] == 'core.libopenstorage.org' && + log.body['object']['kind'] == 'StorageNode' + vars: + - name: "clusterName" + value: "resource.attributes['k8s.cluster.name']" + - name: "name" + value: "log.attributes['k8s.object.name']" + - name: "namespace" + value: "log.attributes['k8s.namespace.name']" + output: + identifier: "'urn:portworx:/' + vars.clusterName + ':node/' + vars.name" + name: "vars.name" + typeName: "'portworx node'" + required: + additionalIdentifiers: + - "'urn:kubernetes:/' + vars.clusterName + ':node/' + vars.name" + tags: + - source: "'portworx'" + target: "stackpack" + - source: "vars.clusterName" + target: "cluster-name" + - source: "vars.namespace" + target: "portworx.namespace" + optional: + tags: + - source: "log.body['object']['metadata']['ownerReferences'][0]['name']" + target: "portworx.cluster" + - source: "log.body['object']['status']['phase']" + target: "portworx.phase" + - source: "log.body['object']['spec']['version']" + target: "portworx.version" + - source: "log.body['object']['status']['nodeUid']" + target: "portworx.node-id" + - source: "log.body['object']['status']['network']['dataIP']" + target: "portworx.data-ip" + - source: "string(log.body['object']['status']['nodeAttributes']['storage'])" + target: "portworx.storage-node" + - source: "string(log.body['object']['status']['nodeAttributes']['kvdb'])" + target: "portworx.kvdb-member" + expireAfter: 900000 + rank: + specificity: 10 diff --git a/stackpacks/portworx/settings/component-mappings/volumes.sty b/stackpacks/portworx/settings/component-mappings/volumes.sty new file mode 100644 index 0000000..03d7331 --- /dev/null +++ b/stackpacks/portworx/settings/component-mappings/volumes.sty @@ -0,0 +1,50 @@ +# Component mapping: Portworx volume metrics -> portworx volume, merged into the Kubernetes persistent volume. +# Portworx has no custom resource per volume. px_volume_replication_status drives it: it is the only volume metric reported for every volume, attached or not. +# Ranked below the Kubernetes persistent volume so the merged component keeps its type. +nodes: + - _type: "OtelComponentMapping" + name: "Portworx volume" + description: "Marks the Kubernetes persistent volumes that Portworx provides" + identifier: "urn:stackpack:portworx:otel-component-mapping:volume" + input: + signal: + - "METRICS" + resource: + condition: "'k8s.cluster.name' in resource.attributes" + scope: + metric: + condition: "metric.name == 'px_volume_replication_status'" + datapoint: + action: "'CREATE'" + condition: >- + 'volumename' in datapoint.attributes && + 'cluster' in datapoint.attributes && + datapoint.attributes['volumename'] != '' + vars: + - name: "clusterName" + value: "resource.attributes['k8s.cluster.name']" + - name: "name" + value: "datapoint.attributes['volumename']" + output: + identifier: "'urn:portworx:/' + vars.clusterName + ':volume/' + vars.name" + name: "vars.name" + typeName: "'portworx volume'" + required: + additionalIdentifiers: + - "'urn:kubernetes:/' + vars.clusterName + ':persistent-volume/' + vars.name" + tags: + - source: "'portworx'" + target: "stackpack" + - source: "vars.clusterName" + target: "cluster-name" + - source: "datapoint.attributes['cluster']" + target: "portworx.cluster" + optional: + tags: + - source: "datapoint.attributes['volumeid']" + target: "portworx.volume-id" + - source: "datapoint.attributes['repl']" + target: "portworx.replicas" + expireAfter: 900000 + rank: + specificity: 10 diff --git a/stackpacks/portworx/settings/dashboards/cluster-overview.sty b/stackpacks/portworx/settings/dashboards/cluster-overview.sty new file mode 100644 index 0000000..c7687a3 --- /dev/null +++ b/stackpacks/portworx/settings/dashboards/cluster-overview.sty @@ -0,0 +1,793 @@ +# Dashboard: Portworx - Cluster overview. +nodes: +- _type: Dashboard + identifier: urn:stackpack:portworx:dashboard:cluster-overview + name: Portworx - Cluster overview + description: Capacity, health and performance of a Portworx cluster (mirrors the Portworx cluster dashboard). + dashboard: + metadata: + project: '{"saveVariables":false}' + spec: + panels: + cfd4ff6e63668cb6ee770: + spec: + plugin: + kind: StatChart + spec: + calculation: last + format: + unit: short + decimalPlaces: 0 + sparkline: true + thresholds: + defaultColor: '#33A02C' + steps: [] + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max(px_cluster_status_nodes_online{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) + alias: Nodes online + display: + name: Nodes online + description: Cluster members online. + 179676c29b1640d96c82f: + spec: + plugin: + kind: StatChart + spec: + calculation: last + format: + unit: short + decimalPlaces: 0 + sparkline: true + thresholds: + defaultColor: '#33A02C' + steps: + - color: '#E31A1C' + value: 1 + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max(px_cluster_status_nodes_storage_down{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) + alias: Storage nodes down + display: + name: Storage nodes down + description: Storage nodes whose storage is down. + 4b6013059829feb59af32: + spec: + plugin: + kind: StatChart + spec: + calculation: last + format: + unit: short + decimalPlaces: 0 + sparkline: true + thresholds: + defaultColor: '#33A02C' + steps: + - color: '#E31A1C' + value: 2 + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max(px_cluster_status_quorum{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) + alias: Quorum + display: + name: Quorum + description: 1 means the cluster has quorum. + 867efc3d44e8b8da439ad: + spec: + plugin: + kind: StatChart + spec: + calculation: last + format: + unit: short + decimalPlaces: 0 + sparkline: true + thresholds: + defaultColor: '#1F78B4' + steps: [] + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: count(max by (volumeid) (px_volume_capacity_bytes{k8s_cluster_name="${cluster}", cluster="${pxcluster}"})) + alias: Volumes + display: + name: Volumes + description: Portworx volumes. + 8f6d99854ec2753b33c83: + spec: + plugin: + kind: StatChart + spec: + calculation: last + format: + unit: percentunit + decimalPlaces: 0 + sparkline: true + thresholds: + defaultColor: '#33A02C' + steps: + - color: '#FF7F00' + value: 0.7 + - color: '#E31A1C' + value: 0.8 + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum(px_cluster_disk_utilized_bytes{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) / sum(px_cluster_disk_total_bytes{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) + alias: Capacity used + display: + name: Capacity used + description: Share of raw pool capacity in use. + 853139287ac16ef969757: + spec: + plugin: + kind: StatChart + spec: + calculation: last + format: + unit: short + decimalPlaces: 0 + sparkline: true + thresholds: + defaultColor: '#E31A1C' + steps: + - color: '#FF7F00' + value: 5 + - color: '#33A02C' + value: 30 + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: min(px_node_status_license_expiry{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) + alias: License days left + display: + name: License days left + description: Days until the license expires. + 3f8686d134bb2781aa284: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: bytes(IEC) + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum(px_cluster_disk_utilized_bytes{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) + alias: Used + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum(px_cluster_disk_total_bytes{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) + alias: Total + display: + name: Storage capacity + description: Used and total raw pool capacity. + 783d832ede5a3ad3b569f: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: bytes(IEC) + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum(max by (volumeid) (px_volume_capacity_bytes{k8s_cluster_name="${cluster}", cluster="${pxcluster}"})) + alias: Provisioned + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum(px_cluster_disk_total_bytes{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) + alias: Raw + display: + name: Provisioned vs raw capacity + description: Sum of volume sizes compared with raw capacity. + 209ae9d9d7791282d9c1b: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max(px_cluster_status_nodes_online{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) + alias: Online + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max(px_cluster_status_nodes_offline{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) + alias: Offline + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max(px_cluster_status_nodes_storage_down{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) + alias: Storage down + display: + name: Nodes + description: Members online/offline and storage down. + b84e2781e06592fc2dc36: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: bytes/sec(IEC) + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum(max by (volumeid) (px_volume_readthroughput{k8s_cluster_name="${cluster}", cluster="${pxcluster}"})) + alias: Read + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum(max by (volumeid) (px_volume_writethroughput{k8s_cluster_name="${cluster}", cluster="${pxcluster}"})) + alias: Write + display: + name: Volume throughput + description: Read and write throughput of all volumes. + 702ef8f72fd6c5c6f9bef: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: ops/sec + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum(max by (volumeid) (rate(px_volume_reads_total{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}[${__rate_interval}]))) + alias: Read + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum(max by (volumeid) (rate(px_volume_writes_total{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}[${__rate_interval}]))) + alias: Write + display: + name: Volume IOPS + description: Read and write operations per second of all volumes. + be2619cedb8712bbcd69d: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: s + decimalPlaces: 4 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (k8s_cluster_name, node) (px_node_stats_read_latency_seconds{k8s_cluster_name="${cluster}", cluster="${pxcluster}", coordinator="1"}) + alias: read ${node} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (k8s_cluster_name, node) (px_node_stats_write_latency_seconds{k8s_cluster_name="${cluster}", cluster="${pxcluster}", coordinator="1"}) + alias: write ${node} + display: + name: I/O latency per node + description: Average latency of the I/O each node coordinates. + 282fec98c095175ae684d: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: percent + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (k8s_cluster_name, node) (px_cluster_cpu_percent{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) + alias: ${node} + display: + name: CPU per node + description: CPU used on each node. + 37235948878747ddea4e3: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: percent + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (k8s_cluster_name, node) (px_cluster_memory_utilized_percent{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) + alias: ${node} + display: + name: Memory per node + description: Memory used on each node. + 457a58e4916f322f2f48d: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (k8s_cluster_name, node) (px_cluster_pendingio{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) + alias: ${node} + display: + name: Pending I/O per node + description: I/O waiting in Portworx on each node. + ad04e6a1ab0069b991b46: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: bytes/sec(IEC) + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (k8s_cluster_name, node) (rate(px_network_io_sent_bytes_total{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}[${__rate_interval}])) + alias: sent ${node} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (k8s_cluster_name, node) (rate(px_network_io_received_bytes_total{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}[${__rate_interval}])) + alias: received ${node} + display: + name: Network traffic per node + description: Bytes Portworx sends between nodes. + 6eab488af3ff3e36b4773: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max(px_cluster_trashcan_volumes{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) + alias: In trashcan + display: + name: Volumes in trashcan + description: Deleted volumes still kept in the trashcan; they keep using pool capacity. + 80653d35c7bb3cd2530f9: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (k8s_cluster_name, node) (px_kvdb_health_state_node_view{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) + alias: ${node} + display: + name: KVDB health per node + description: 1 healthy, 2 unhealthy. + 2403628a8b692423b9403: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: s + decimalPlaces: 4 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: histogram_quantile(0.99, sum by (k8s_cluster_name, node, le) (rate(px_kvdb_watch_latency_seconds_bucket{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}[${__rate_interval}]))) + alias: ${node} + display: + name: KVDB watch latency (p99) + description: 99th percentile latency of KVDB watches per node. + layouts: + - kind: Grid + spec: + items: + - x: 0 + y: 0 + width: 4 + height: 1 + content: + $ref: '#/spec/panels/cfd4ff6e63668cb6ee770' + - x: 4 + y: 0 + width: 4 + height: 1 + content: + $ref: '#/spec/panels/179676c29b1640d96c82f' + - x: 8 + y: 0 + width: 4 + height: 1 + content: + $ref: '#/spec/panels/4b6013059829feb59af32' + - x: 12 + y: 0 + width: 4 + height: 1 + content: + $ref: '#/spec/panels/867efc3d44e8b8da439ad' + - x: 16 + y: 0 + width: 4 + height: 1 + content: + $ref: '#/spec/panels/8f6d99854ec2753b33c83' + - x: 20 + y: 0 + width: 4 + height: 1 + content: + $ref: '#/spec/panels/853139287ac16ef969757' + - x: 0 + y: 1 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/3f8686d134bb2781aa284' + - x: 8 + y: 1 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/783d832ede5a3ad3b569f' + - x: 16 + y: 1 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/209ae9d9d7791282d9c1b' + - x: 0 + y: 3 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/b84e2781e06592fc2dc36' + - x: 8 + y: 3 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/702ef8f72fd6c5c6f9bef' + - x: 16 + y: 3 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/be2619cedb8712bbcd69d' + - x: 0 + y: 5 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/282fec98c095175ae684d' + - x: 8 + y: 5 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/37235948878747ddea4e3' + - x: 16 + y: 5 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/457a58e4916f322f2f48d' + - x: 0 + y: 7 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/ad04e6a1ab0069b991b46' + - x: 8 + y: 7 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/6eab488af3ff3e36b4773' + - x: 16 + y: 7 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/80653d35c7bb3cd2530f9' + - x: 0 + y: 9 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/2403628a8b692423b9403' + variables: + - kind: ListVariable + spec: + plugin: + kind: MetricLabelValues + spec: + labelName: k8s_cluster_name + matchers: + - px_cluster_status_size + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: cluster + display: + name: Cluster + description: Kubernetes cluster + - kind: ListVariable + spec: + plugin: + kind: MetricPromQL + spec: + labelName: cluster + expr: px_cluster_status_size{k8s_cluster_name="${cluster}"} + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: pxcluster + display: + name: Portworx cluster + description: Portworx StorageCluster diff --git a/stackpacks/portworx/settings/dashboards/nodes.sty b/stackpacks/portworx/settings/dashboards/nodes.sty new file mode 100644 index 0000000..42a434f --- /dev/null +++ b/stackpacks/portworx/settings/dashboards/nodes.sty @@ -0,0 +1,508 @@ +# Dashboard: Portworx - Nodes and pools. +nodes: +- _type: Dashboard + identifier: urn:stackpack:portworx:dashboard:nodes + name: Portworx - Nodes and pools + description: Per node and per pool capacity and performance (mirrors the Portworx node and performance dashboards). + dashboard: + metadata: + project: '{"saveVariables":false}' + spec: + panels: + 6043d9c21f5ca511bae12: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: percentunit + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (k8s_cluster_name, node) (px_cluster_disk_utilized_bytes{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) / max by (k8s_cluster_name, node) (px_cluster_disk_total_bytes{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) + alias: ${node} + display: + name: Storage used per node + description: Used share of each node's pool capacity. + aa98989b33c22d6639dff: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: percentunit + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (k8s_cluster_name, node, pool) (px_pool_stats_used_bytes{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) / max by (k8s_cluster_name, node, pool) (px_pool_stats_total_bytes{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) + alias: ${node} ${pool} + display: + name: Pool usage + description: Used share of each storage pool. + d6562c2273ab7e4b039c8: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (k8s_cluster_name, node) (px_node_status_node_status{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) + alias: ${node} + display: + name: Node status + description: 2 is OK; 3 offline, 5 not in quorum, 8 storage down. + f9c717ceefc0b43c5bab0: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: s + decimalPlaces: 4 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (k8s_cluster_name, node) (px_node_stats_read_latency_seconds{k8s_cluster_name="${cluster}", cluster="${pxcluster}", coordinator="1"}) + alias: ${node} + display: + name: Read latency per node + description: Average read latency of coordinated I/O. + 461e2ec56635fad3fc78d: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: s + decimalPlaces: 4 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (k8s_cluster_name, node) (px_node_stats_write_latency_seconds{k8s_cluster_name="${cluster}", cluster="${pxcluster}", coordinator="1"}) + alias: ${node} + display: + name: Write latency per node + description: Average write latency of coordinated I/O. + 0b7952737ea15497b5237: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: ops/sec + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (k8s_cluster_name, node) (px_node_stats_read_iops{k8s_cluster_name="${cluster}", cluster="${pxcluster}", coordinator="1"}) + max by (k8s_cluster_name, node) (px_node_stats_write_iops{k8s_cluster_name="${cluster}", cluster="${pxcluster}", coordinator="1"}) + alias: ${node} + display: + name: IOPS per node + description: Read plus write operations per second of coordinated I/O. + 871ca34adc88e35878abe: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: bytes/sec(IEC) + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (k8s_cluster_name, node) (px_node_stats_readthroughput{k8s_cluster_name="${cluster}", cluster="${pxcluster}", coordinator="1"}) + max by (k8s_cluster_name, node) (px_node_stats_writethroughput{k8s_cluster_name="${cluster}", cluster="${pxcluster}", coordinator="1"}) + alias: ${node} + display: + name: Throughput per node + description: Read plus write throughput of coordinated I/O. + c83bab232f4046e6b1b6f: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: s + decimalPlaces: 4 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (k8s_cluster_name, node, pool) (px_pool_stats_write_latency_seconds{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) + alias: ${node} ${pool} + display: + name: Pool write latency + description: Average write latency of each pool. + 667f919d9d6402caf1d64: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: s + decimalPlaces: 4 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (k8s_cluster_name, node, pool) (px_pool_stats_flush_latency_seconds{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) + alias: ${node} ${pool} + display: + name: Pool flush latency + description: Average flush latency of each pool. + ffe576922a9c9d953e17b: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: s + decimalPlaces: 4 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (k8s_cluster_name, node, disk) (px_disk_stats_read_latency_seconds{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) + alias: read ${node} ${disk} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (k8s_cluster_name, node, disk) (px_disk_stats_write_latency_seconds{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) + alias: write ${node} ${disk} + display: + name: Disk latency + description: Average read and write latency of each disk. + fa0b584419ee80509783d: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (k8s_cluster_name, node) (px_node_stats_volume_replica_count{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) + alias: ${node} + display: + name: Volume replicas per node + description: Volume replicas stored on each node. + 4a7380d5763ca54a4c82e: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: bytes(IEC) + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (k8s_cluster_name, node) (px_proc_stats_res{k8s_cluster_name="${cluster}", cluster="${pxcluster}"}) + alias: ${node} + display: + name: Portworx process memory + description: Resident memory of the Portworx processes per node. + layouts: + - kind: Grid + spec: + items: + - x: 0 + y: 0 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/6043d9c21f5ca511bae12' + - x: 8 + y: 0 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/aa98989b33c22d6639dff' + - x: 16 + y: 0 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/d6562c2273ab7e4b039c8' + - x: 0 + y: 2 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/f9c717ceefc0b43c5bab0' + - x: 8 + y: 2 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/461e2ec56635fad3fc78d' + - x: 16 + y: 2 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/0b7952737ea15497b5237' + - x: 0 + y: 4 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/871ca34adc88e35878abe' + - x: 8 + y: 4 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/c83bab232f4046e6b1b6f' + - x: 16 + y: 4 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/667f919d9d6402caf1d64' + - x: 0 + y: 6 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/ffe576922a9c9d953e17b' + - x: 8 + y: 6 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/fa0b584419ee80509783d' + - x: 16 + y: 6 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/4a7380d5763ca54a4c82e' + variables: + - kind: ListVariable + spec: + plugin: + kind: MetricLabelValues + spec: + labelName: k8s_cluster_name + matchers: + - px_cluster_status_size + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: cluster + display: + name: Cluster + description: Kubernetes cluster + - kind: ListVariable + spec: + plugin: + kind: MetricPromQL + spec: + labelName: cluster + expr: px_cluster_status_size{k8s_cluster_name="${cluster}"} + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: pxcluster + display: + name: Portworx cluster + description: Portworx StorageCluster diff --git a/stackpacks/portworx/settings/dashboards/volumes.sty b/stackpacks/portworx/settings/dashboards/volumes.sty new file mode 100644 index 0000000..85d5b12 --- /dev/null +++ b/stackpacks/portworx/settings/dashboards/volumes.sty @@ -0,0 +1,617 @@ +# Dashboard: Portworx - Volumes. +nodes: +- _type: Dashboard + identifier: urn:stackpack:portworx:dashboard:volumes + name: Portworx - Volumes + description: Usage, replication and performance of Portworx volumes (mirrors the Portworx volume dashboard). + dashboard: + metadata: + project: '{"saveVariables":false}' + spec: + panels: + 1579ee38b12d44ffdb736: + spec: + plugin: + kind: StatChart + spec: + calculation: last + format: + unit: short + decimalPlaces: 0 + sparkline: true + thresholds: + defaultColor: '#1F78B4' + steps: [] + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: count(max by (volumeid) (px_volume_capacity_bytes{k8s_cluster_name="${cluster}", cluster="${pxcluster}", volumename!=""})) + alias: Volumes + display: + name: Volumes + description: Portworx volumes. + aa4fa3221f6c530ba95ca: + spec: + plugin: + kind: StatChart + spec: + calculation: last + format: + unit: short + decimalPlaces: 0 + sparkline: true + thresholds: + defaultColor: '#1F78B4' + steps: [] + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: count(max by (volumeid) (px_volume_attached{k8s_cluster_name="${cluster}", cluster="${pxcluster}", volumename!=""}) > 0) or vector(0) + alias: Attached + display: + name: Attached + description: Attached volumes. + 2c5b8f69b1b20b088c450: + spec: + plugin: + kind: StatChart + spec: + calculation: last + format: + unit: short + decimalPlaces: 0 + sparkline: true + thresholds: + defaultColor: '#33A02C' + steps: + - color: '#FF7F00' + value: 1 + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: count(max by (volumeid) (px_volume_halevel{k8s_cluster_name="${cluster}", cluster="${pxcluster}", volumename!=""}) - min by (volumeid) (px_volume_currhalevel{k8s_cluster_name="${cluster}", cluster="${pxcluster}", volumename!=""}) > 0) or vector(0) + alias: Not fully replicated + display: + name: Not fully replicated + description: Volumes with fewer replicas in sync than configured. + 2d7caf583756eed724f24: + spec: + plugin: + kind: StatChart + spec: + calculation: last + format: + unit: short + decimalPlaces: 0 + sparkline: true + thresholds: + defaultColor: '#33A02C' + steps: + - color: '#FF7F00' + value: 1 + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: count(max by (volumeid) (px_volume_replication_status{k8s_cluster_name="${cluster}", cluster="${pxcluster}", volumename!=""}) >= 1 <= 3) or vector(0) + alias: Degraded or resyncing + display: + name: Degraded or resyncing + description: Volumes with replication status 1 (no quorum), 2 (resync) or 3 (degraded). + 4d99823d2932c0afba1b9: + spec: + plugin: + kind: StatChart + spec: + calculation: last + format: + unit: short + decimalPlaces: 0 + sparkline: true + thresholds: + defaultColor: '#33A02C' + steps: + - color: '#E31A1C' + value: 1 + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: count(max by (volumeid) (px_volume_usage_bytes{k8s_cluster_name="${cluster}", cluster="${pxcluster}", volumename!=""}) / max by (volumeid) (px_volume_capacity_bytes{k8s_cluster_name="${cluster}", cluster="${pxcluster}", volumename!=""}) > 0.8) or vector(0) + alias: Above 80% full + display: + name: Above 80% full + description: Volumes using more than 80% of their size. + bf2b2be2ecb8f63cfffc3: + spec: + plugin: + kind: StatChart + spec: + calculation: last + format: + unit: bytes(IEC) + decimalPlaces: 0 + sparkline: true + thresholds: + defaultColor: '#1F78B4' + steps: [] + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum(max by (volumeid) (px_volume_capacity_bytes{k8s_cluster_name="${cluster}", cluster="${pxcluster}", volumename!=""})) + alias: Provisioned + display: + name: Provisioned + description: Sum of volume sizes. + 50d9229f7cbfe677d0a72: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: percentunit + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: topk(10, max by (namespace, pvc) (px_volume_usage_bytes{k8s_cluster_name="${cluster}", cluster="${pxcluster}", volumename!=""}) / max by (namespace, pvc) (px_volume_capacity_bytes{k8s_cluster_name="${cluster}", cluster="${pxcluster}", volumename!=""})) + alias: ${namespace}/${pvc} + display: + name: Top 10 volumes by usage + description: Used share of the volume size. + 7f5ec0394a796c81402d2: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: s + decimalPlaces: 4 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: topk(10, max by (namespace, pvc) (px_volume_read_latency_seconds{k8s_cluster_name="${cluster}", cluster="${pxcluster}", volumename!=""})) + alias: ${namespace}/${pvc} + display: + name: Top 10 read latency + description: Average read latency per volume. + e7ec15e862c3d6bbab620: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: s + decimalPlaces: 4 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: topk(10, max by (namespace, pvc) (px_volume_write_latency_seconds{k8s_cluster_name="${cluster}", cluster="${pxcluster}", volumename!=""})) + alias: ${namespace}/${pvc} + display: + name: Top 10 write latency + description: Average write latency per volume. + 7f3e96187ee73431a4062: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: ops/sec + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: topk(10, max by (namespace, pvc) (px_volume_read_iops{k8s_cluster_name="${cluster}", cluster="${pxcluster}", volumename!=""}) + max by (namespace, pvc) (px_volume_write_iops{k8s_cluster_name="${cluster}", cluster="${pxcluster}", volumename!=""})) + alias: ${namespace}/${pvc} + display: + name: Top 10 IOPS + description: Read plus write operations per second per volume. + 953d446c51bd8179a4175: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: bytes/sec(IEC) + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: topk(10, max by (namespace, pvc) (px_volume_readthroughput{k8s_cluster_name="${cluster}", cluster="${pxcluster}", volumename!=""}) + max by (namespace, pvc) (px_volume_writethroughput{k8s_cluster_name="${cluster}", cluster="${pxcluster}", volumename!=""})) + alias: ${namespace}/${pvc} + display: + name: Top 10 throughput + description: Read plus write throughput per volume. + 495e074ca7ce0814905fa: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: topk(10, max by (namespace, pvc) (px_volume_depth_io{k8s_cluster_name="${cluster}", cluster="${pxcluster}", volumename!=""})) + alias: ${namespace}/${pvc} + display: + name: Top 10 queue depth + description: I/O operations in progress per volume. + 684ebd1910bbcb63eed09: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (namespace, pvc) (px_volume_replication_status{k8s_cluster_name="${cluster}", cluster="${pxcluster}", volumename!=""}) > 0 + alias: ${namespace}/${pvc} + display: + name: Replication status + description: 'Volumes whose status is not 0 (up): 1 no quorum, 2 resync, 3 degraded, 4 detached, 5 restore.' + 00cf10217a66571779d47: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (namespace, pvc) (px_volume_halevel{k8s_cluster_name="${cluster}", cluster="${pxcluster}", volumename!=""}) - min by (namespace, pvc) (px_volume_currhalevel{k8s_cluster_name="${cluster}", cluster="${pxcluster}", volumename!=""}) > 0 + alias: ${namespace}/${pvc} + display: + name: Missing replicas + description: Configured replicas minus replicas in sync, per volume. + 07371603ede3ec157e9e0: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: topk(10, max by (namespace, pvc) (delta(px_volume_num_long_reads{k8s_cluster_name="${cluster}", cluster="${pxcluster}", volumename!=""}[10m])) + max by (namespace, pvc) (delta(px_volume_num_long_writes{k8s_cluster_name="${cluster}", cluster="${pxcluster}", volumename!=""}[10m])) + max by (namespace, pvc) (delta(px_volume_num_long_flushes{k8s_cluster_name="${cluster}", cluster="${pxcluster}", volumename!=""}[10m]))) + alias: ${namespace}/${pvc} + display: + name: Top 10 long I/Os + description: Reads, writes and flushes counted as long per 10 minutes, per volume. + d7ea877a629f0e59fabbe: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: percentunit + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: topk(10, max by (namespace, pvc) (px_volume_fs_usage_bytes{k8s_cluster_name="${cluster}", cluster="${pxcluster}", volumename!=""}) / max by (namespace, pvc) (px_volume_fs_capacity_bytes{k8s_cluster_name="${cluster}", cluster="${pxcluster}", volumename!=""} > 0)) + alias: ${namespace}/${pvc} + display: + name: Filesystem usage + description: Filesystem used share per volume (only for mounted filesystems). + layouts: + - kind: Grid + spec: + items: + - x: 0 + y: 0 + width: 4 + height: 1 + content: + $ref: '#/spec/panels/1579ee38b12d44ffdb736' + - x: 4 + y: 0 + width: 4 + height: 1 + content: + $ref: '#/spec/panels/aa4fa3221f6c530ba95ca' + - x: 8 + y: 0 + width: 4 + height: 1 + content: + $ref: '#/spec/panels/2c5b8f69b1b20b088c450' + - x: 12 + y: 0 + width: 4 + height: 1 + content: + $ref: '#/spec/panels/2d7caf583756eed724f24' + - x: 16 + y: 0 + width: 4 + height: 1 + content: + $ref: '#/spec/panels/4d99823d2932c0afba1b9' + - x: 20 + y: 0 + width: 4 + height: 1 + content: + $ref: '#/spec/panels/bf2b2be2ecb8f63cfffc3' + - x: 0 + y: 1 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/50d9229f7cbfe677d0a72' + - x: 8 + y: 1 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/7f5ec0394a796c81402d2' + - x: 16 + y: 1 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/e7ec15e862c3d6bbab620' + - x: 0 + y: 3 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/7f3e96187ee73431a4062' + - x: 8 + y: 3 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/953d446c51bd8179a4175' + - x: 16 + y: 3 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/495e074ca7ce0814905fa' + - x: 0 + y: 5 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/684ebd1910bbcb63eed09' + - x: 8 + y: 5 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/00cf10217a66571779d47' + - x: 16 + y: 5 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/07371603ede3ec157e9e0' + - x: 0 + y: 7 + width: 8 + height: 2 + content: + $ref: '#/spec/panels/d7ea877a629f0e59fabbe' + variables: + - kind: ListVariable + spec: + plugin: + kind: MetricLabelValues + spec: + labelName: k8s_cluster_name + matchers: + - px_cluster_status_size + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: cluster + display: + name: Cluster + description: Kubernetes cluster + - kind: ListVariable + spec: + plugin: + kind: MetricPromQL + spec: + labelName: cluster + expr: px_cluster_status_size{k8s_cluster_name="${cluster}"} + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: pxcluster + display: + name: Portworx cluster + description: Portworx StorageCluster diff --git a/stackpacks/portworx/settings/main-menu.sty b/stackpacks/portworx/settings/main-menu.sty new file mode 100644 index 0000000..b1d0c32 --- /dev/null +++ b/stackpacks/portworx/settings/main-menu.sty @@ -0,0 +1,8 @@ +# Main menu group: the left-hand navigation section for the Portworx lists. +nodes: + - _type: MainMenuGroup + name: Portworx + iconbase64: !icon portworx.svg + defaultOpen: true + items: [] + identifier: "urn:stackpack:portworx:main-menu-group:portworx" diff --git a/stackpacks/portworx/settings/monitors/clusters.sty b/stackpacks/portworx/settings/monitors/clusters.sty new file mode 100644 index 0000000..a301880 --- /dev/null +++ b/stackpacks/portworx/settings/monitors/clusters.sty @@ -0,0 +1,128 @@ +# Monitors: Portworx clusters, ported from the Portworx operator's alert rules. +nodes: +- _type: Monitor + name: Portworx - Storage node down + description: One or more storage nodes report their storage down for 5 minutes (Portworx alert PortworxStorageNodeDown). + tags: + - portworx + - storage + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:portworx:shared:monitor:storage-node-down + arguments: + metric: + query: max by (k8s_cluster_name, cluster) (min_over_time(px_cluster_status_nodes_storage_down{k8s_cluster_name!=""}[5m])) + unit: short + aliasTemplate: Storage nodes down + comparator: GT + threshold: 0.0 + failureState: CRITICAL + urnTemplate: urn:portworx:/${k8s_cluster_name}:cluster/${cluster} + titleTemplate: Storage nodes down + intervalSeconds: 60 + remediationHint: !include remediation-hints/storage-node-down.md.hbs + status: ENABLED +- _type: Monitor + name: Portworx - Cluster member down + description: Fewer nodes are online than the cluster size for 5 minutes (Portworx alert PortworxMemberDown). + tags: + - portworx + - storage + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:portworx:shared:monitor:member-down + arguments: + metric: + query: min_over_time((max by (k8s_cluster_name, cluster) (px_cluster_status_size{k8s_cluster_name!=""}) - max by (k8s_cluster_name, cluster) (px_cluster_status_nodes_online{k8s_cluster_name!=""}))[5m:1m]) + unit: short + aliasTemplate: Members down + comparator: GT + threshold: 0.0 + failureState: CRITICAL + urnTemplate: urn:portworx:/${k8s_cluster_name}:cluster/${cluster} + titleTemplate: Cluster members down + intervalSeconds: 60 + remediationHint: !include remediation-hints/member-down.md.hbs + status: ENABLED +- _type: Monitor + name: Portworx - Cluster quorum unhealthy + description: The cluster reports a quorum problem for 5 minutes (Portworx alert PortworxQuorumUnhealthy). + tags: + - portworx + - storage + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:portworx:shared:monitor:quorum-unhealthy + arguments: + metric: + query: max by (k8s_cluster_name, cluster) (min_over_time(px_cluster_status_quorum{k8s_cluster_name!=""}[5m])) + unit: short + aliasTemplate: Quorum state + comparator: GT + threshold: 1.0 + failureState: CRITICAL + urnTemplate: urn:portworx:/${k8s_cluster_name}:cluster/${cluster} + titleTemplate: Cluster quorum + intervalSeconds: 60 + remediationHint: !include remediation-hints/quorum-unhealthy.md.hbs + status: ENABLED +- _type: Monitor + name: Portworx - KVDB unhealthy + description: The Portworx key-value database is unhealthy from the cluster's view (Portworx alert PXKvdbClusterViewUnhealthy). + tags: + - portworx + - storage + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:portworx:shared:monitor:kvdb-cluster-unhealthy + arguments: + metric: + query: max by (k8s_cluster_name, cluster) (px_kvdb_health_state_cluster_view{k8s_cluster_name!=""} == bool 2) + unit: short + aliasTemplate: Unhealthy (1 = yes) + comparator: GT + threshold: 0.0 + failureState: CRITICAL + urnTemplate: urn:portworx:/${k8s_cluster_name}:cluster/${cluster} + titleTemplate: KVDB health (cluster view) + intervalSeconds: 60 + remediationHint: !include remediation-hints/kvdb-cluster-unhealthy.md.hbs + status: ENABLED +- _type: Monitor + name: Portworx - License expires soon + description: The Portworx license expires in less than 5 days (Portworx alert PortworxLicenseExpiry). + tags: + - portworx + - storage + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:portworx:shared:monitor:license-expiry + arguments: + metric: + query: min by (k8s_cluster_name, cluster) (px_node_status_license_expiry{k8s_cluster_name!=""}) + unit: short + aliasTemplate: Days left + comparator: LT + threshold: 5.0 + failureState: DEVIATING + urnTemplate: urn:portworx:/${k8s_cluster_name}:cluster/${cluster} + titleTemplate: License days left + intervalSeconds: 3600 + remediationHint: !include remediation-hints/license-expiry.md.hbs + status: ENABLED +- _type: Monitor + name: Portworx - License invalid + description: Portworx reports an invalid license (Portworx alert PortworxLicenseInvalid). + tags: + - portworx + - storage + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:portworx:shared:monitor:license-invalid + arguments: + metric: + query: max by (k8s_cluster_name, cluster) (px_license_type{k8s_cluster_name!=""} == bool 9) + unit: short + aliasTemplate: Invalid (1 = yes) + comparator: GT + threshold: 0.0 + failureState: CRITICAL + urnTemplate: urn:portworx:/${k8s_cluster_name}:cluster/${cluster} + titleTemplate: License state + intervalSeconds: 300 + remediationHint: !include remediation-hints/license-invalid.md.hbs + status: ENABLED diff --git a/stackpacks/portworx/settings/monitors/storage-nodes.sty b/stackpacks/portworx/settings/monitors/storage-nodes.sty new file mode 100644 index 0000000..6765588 --- /dev/null +++ b/stackpacks/portworx/settings/monitors/storage-nodes.sty @@ -0,0 +1,170 @@ +# Monitors: Portworx storage nodes, ported from the Portworx operator's alert rules. +nodes: +- _type: Monitor + name: Portworx - Storage node almost full + description: More than 80% of the node's Portworx pool capacity is used (Portworx alert PortworxStorageUsageCritical). + tags: + - portworx + - storage + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:portworx:shared:monitor:storage-usage-critical + arguments: + metric: + query: 100 * max by (k8s_cluster_name, node) (min_over_time(px_cluster_disk_utilized_bytes{k8s_cluster_name!=""}[5m])) / max by (k8s_cluster_name, node) (px_cluster_disk_total_bytes{k8s_cluster_name!=""}) + unit: percent + aliasTemplate: Used % + comparator: GT + threshold: 80.0 + failureState: CRITICAL + urnTemplate: urn:portworx:/${k8s_cluster_name}:node/${node} + titleTemplate: Portworx storage used + intervalSeconds: 60 + remediationHint: !include remediation-hints/storage-usage-critical.md.hbs + status: ENABLED +- _type: Monitor + name: Portworx - Storage node filling up + description: More than 70% of the node's Portworx pool capacity is used (Portworx alert PortworxStorageUsage). + tags: + - portworx + - storage + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:portworx:shared:monitor:storage-usage-warning + arguments: + metric: + query: 100 * max by (k8s_cluster_name, node) (min_over_time(px_cluster_disk_utilized_bytes{k8s_cluster_name!=""}[5m])) / max by (k8s_cluster_name, node) (px_cluster_disk_total_bytes{k8s_cluster_name!=""}) + unit: percent + aliasTemplate: Used % + comparator: GT + threshold: 70.0 + failureState: DEVIATING + urnTemplate: urn:portworx:/${k8s_cluster_name}:node/${node} + titleTemplate: Portworx storage used + intervalSeconds: 60 + remediationHint: !include remediation-hints/storage-usage-warning.md.hbs + status: ENABLED +- _type: Monitor + name: Portworx - Storage node will fill within 2 weeks + description: Less than 30% of the node's pool capacity is free and the last hour's trend fills it within 14 days (Portworx alert PortworxStorageWillFill). + tags: + - portworx + - storage + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:portworx:shared:monitor:storage-will-fill + arguments: + metric: + query: max by (k8s_cluster_name, node) (predict_linear(px_cluster_disk_available_bytes{k8s_cluster_name!=""}[1h], 14 * 86400)) and on (k8s_cluster_name, node) (100 * max by (k8s_cluster_name, node) (px_cluster_disk_available_bytes{k8s_cluster_name!=""}) / max by (k8s_cluster_name, node) (px_cluster_disk_total_bytes{k8s_cluster_name!=""}) < 30) + unit: bytes(IEC) + aliasTemplate: Free space predicted in 14 days + comparator: LT + threshold: 0.0 + failureState: DEVIATING + urnTemplate: urn:portworx:/${k8s_cluster_name}:node/${node} + titleTemplate: Predicted free space + intervalSeconds: 300 + remediationHint: !include remediation-hints/storage-will-fill.md.hbs + status: ENABLED +- _type: Monitor + name: Portworx - Storage node not OK + description: Portworx reports this node in a state other than OK for 5 minutes (offline, not in quorum, storage down...). + tags: + - portworx + - storage + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:portworx:shared:monitor:node-status + arguments: + metric: + query: max by (k8s_cluster_name, node) (min_over_time((px_node_status_node_status{k8s_cluster_name!=""} != bool 2)[5m:1m])) + unit: short + aliasTemplate: Not OK (1 = yes) + comparator: GT + threshold: 0.0 + failureState: CRITICAL + urnTemplate: urn:portworx:/${k8s_cluster_name}:node/${node} + titleTemplate: Portworx node status + intervalSeconds: 60 + remediationHint: !include remediation-hints/node-status.md.hbs + status: ENABLED +- _type: Monitor + name: Portworx - Storage pool offline or full + description: A storage pool on this node is offline (0), full (2) or not found (3). Maintenance (4) and background activity such as a rebalance or expansion (6) are ignored. + tags: + - portworx + - storage + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:portworx:shared:monitor:pool-status + arguments: + metric: + query: max by (k8s_cluster_name, node, pool) ((px_pool_stats_status{k8s_cluster_name!=""} == bool 0) + (px_pool_stats_status{k8s_cluster_name!=""} == bool 2) + (px_pool_stats_status{k8s_cluster_name!=""} == bool 3)) + unit: short + aliasTemplate: Offline, full or not found (1 = yes) + comparator: GT + threshold: 0.0 + failureState: CRITICAL + urnTemplate: urn:portworx:/${k8s_cluster_name}:node/${node} + titleTemplate: Pool ${pool} status + intervalSeconds: 60 + remediationHint: !include remediation-hints/pool-status.md.hbs + status: ENABLED +- _type: Monitor + name: Portworx - Storage pool degraded + description: A storage pool on this node is degraded (status 5). + tags: + - portworx + - storage + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:portworx:shared:monitor:pool-degraded + arguments: + metric: + query: max by (k8s_cluster_name, node, pool) (px_pool_stats_status{k8s_cluster_name!=""} == bool 5) + unit: short + aliasTemplate: Degraded (1 = yes) + comparator: GT + threshold: 0.0 + failureState: DEVIATING + urnTemplate: urn:portworx:/${k8s_cluster_name}:node/${node} + titleTemplate: Pool ${pool} degraded + intervalSeconds: 60 + remediationHint: !include remediation-hints/pool-degraded.md.hbs + status: ENABLED +- _type: Monitor + name: Portworx - Storage pool nearly full + description: A single pool is close to the point where Portworx marks it full (free space below 10% of the pool or 100 GiB, whichever is smaller). A full pool puts the node in StorageDown and its volumes become read-only. Fires when free space drops below 1.5 times that limit, even if the node's total capacity looks fine. + tags: + - portworx + - storage + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:portworx:shared:monitor:pool-nearly-full + arguments: + metric: + query: max by (k8s_cluster_name, node, pool) (px_pool_stats_available_bytes{k8s_cluster_name!=""}) - 1.5 * clamp_max(0.1 * max by (k8s_cluster_name, node, pool) (px_pool_stats_total_bytes{k8s_cluster_name!=""}), 107374182400) + unit: bytes(IEC) + aliasTemplate: Free space above 1.5x the full limit + comparator: LT + threshold: 0.0 + failureState: CRITICAL + urnTemplate: urn:portworx:/${k8s_cluster_name}:node/${node} + titleTemplate: Pool ${pool} free space + intervalSeconds: 60 + remediationHint: !include remediation-hints/pool-nearly-full.md.hbs + status: ENABLED +- _type: Monitor + name: Portworx - KVDB unhealthy on node + description: The node sees the Portworx key-value database as unhealthy for 5 minutes (Portworx alert PXKvdbNodeViewUnhealthy). + tags: + - portworx + - storage + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:portworx:shared:monitor:kvdb-node-unhealthy + arguments: + metric: + query: max by (k8s_cluster_name, node) (min_over_time((px_kvdb_health_state_node_view{k8s_cluster_name!=""} == bool 2)[5m:1m])) + unit: short + aliasTemplate: Unhealthy (1 = yes) + comparator: GT + threshold: 0.0 + failureState: CRITICAL + urnTemplate: urn:portworx:/${k8s_cluster_name}:node/${node} + titleTemplate: KVDB health (node view) + intervalSeconds: 60 + remediationHint: !include remediation-hints/kvdb-node-unhealthy.md.hbs + status: ENABLED diff --git a/stackpacks/portworx/settings/monitors/volumes.sty b/stackpacks/portworx/settings/monitors/volumes.sty new file mode 100644 index 0000000..66e3c99 --- /dev/null +++ b/stackpacks/portworx/settings/monitors/volumes.sty @@ -0,0 +1,128 @@ +# Monitors: Portworx volumes, ported from the Portworx operator's alert rules. +nodes: +- _type: Monitor + name: Portworx - Volume almost full + description: More than 80% of the volume is used (Portworx alert PortworxVolumeUsageCritical). + tags: + - portworx + - storage + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:portworx:shared:monitor:volume-usage-critical + arguments: + metric: + query: 100 * max by (k8s_cluster_name, volumename) (min_over_time(px_volume_usage_bytes{k8s_cluster_name!="", volumename!=""}[5m])) / max by (k8s_cluster_name, volumename) (px_volume_capacity_bytes{k8s_cluster_name!="", volumename!=""}) + unit: percent + aliasTemplate: Used % + comparator: GT + threshold: 80.0 + failureState: CRITICAL + urnTemplate: urn:portworx:/${k8s_cluster_name}:volume/${volumename} + titleTemplate: Volume used + intervalSeconds: 60 + remediationHint: !include remediation-hints/volume-usage-critical.md.hbs + status: ENABLED +- _type: Monitor + name: Portworx - Volume filling up + description: More than 70% of the volume is used (Portworx alert PortworxVolumeUsage). + tags: + - portworx + - storage + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:portworx:shared:monitor:volume-usage-warning + arguments: + metric: + query: 100 * max by (k8s_cluster_name, volumename) (min_over_time(px_volume_usage_bytes{k8s_cluster_name!="", volumename!=""}[5m])) / max by (k8s_cluster_name, volumename) (px_volume_capacity_bytes{k8s_cluster_name!="", volumename!=""}) + unit: percent + aliasTemplate: Used % + comparator: GT + threshold: 70.0 + failureState: DEVIATING + urnTemplate: urn:portworx:/${k8s_cluster_name}:volume/${volumename} + titleTemplate: Volume used + intervalSeconds: 60 + remediationHint: !include remediation-hints/volume-usage-warning.md.hbs + status: ENABLED +- _type: Monitor + name: Portworx - Volume replication unhealthy + description: The volume is not in quorum, resyncing or degraded (Portworx alerts PortworxVolumeNotInQuorum, PortworxVolumeInResync and PortworxVolumeDegraded). + tags: + - portworx + - storage + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:portworx:shared:monitor:volume-replication + arguments: + metric: + query: max by (k8s_cluster_name, volumename) ((px_volume_replication_status{k8s_cluster_name!="", volumename!=""} >= bool 1) * (px_volume_replication_status{k8s_cluster_name!="", volumename!=""} <= bool 3)) + unit: short + aliasTemplate: Unhealthy (1 = yes) + comparator: GT + threshold: 0.0 + failureState: DEVIATING + urnTemplate: urn:portworx:/${k8s_cluster_name}:volume/${volumename} + titleTemplate: Volume replication + intervalSeconds: 60 + remediationHint: !include remediation-hints/volume-replication.md.hbs + status: ENABLED +- _type: Monitor + name: Portworx - Volume down + description: For 5 minutes Portworx has reported the volume status as down (3) or its state as error (6). + tags: + - portworx + - storage + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:portworx:shared:monitor:volume-down + arguments: + metric: + query: max by (k8s_cluster_name, volumename) (min_over_time(((max by (k8s_cluster_name, volumename) (px_volume_status{k8s_cluster_name!="", volumename!=""}) == bool 3) + (max by (k8s_cluster_name, volumename) (px_volume_state{k8s_cluster_name!="", volumename!=""}) == bool 6) > bool 0)[5m:1m])) + unit: short + aliasTemplate: Down or error (1 = yes) + comparator: GT + threshold: 0.0 + failureState: CRITICAL + urnTemplate: urn:portworx:/${k8s_cluster_name}:volume/${volumename} + titleTemplate: Volume status + intervalSeconds: 60 + remediationHint: !include remediation-hints/volume-down.md.hbs + status: ENABLED +- _type: Monitor + name: Portworx - Volume filesystem needs inspection + description: The filesystem check found errors that need manual inspection (filesystem health 3). Mounts and resizes can fail. + tags: + - portworx + - storage + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:portworx:shared:monitor:volume-fs-needs-inspection + arguments: + metric: + query: max by (k8s_cluster_name, volumename) (px_volume_fs_health_status{k8s_cluster_name!="", volumename!=""} == bool 3) + unit: short + aliasTemplate: Needs inspection (1 = yes) + comparator: GT + threshold: 0.0 + failureState: CRITICAL + urnTemplate: urn:portworx:/${k8s_cluster_name}:volume/${volumename} + titleTemplate: Filesystem health + intervalSeconds: 60 + remediationHint: !include remediation-hints/volume-fs-needs-inspection.md.hbs + status: ENABLED +- _type: Monitor + name: Portworx - Volume filesystem has fixable errors + description: The filesystem check found errors that Portworx can fix (filesystem health 2). + tags: + - portworx + - storage + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:portworx:shared:monitor:volume-fs-safe-to-fix + arguments: + metric: + query: max by (k8s_cluster_name, volumename) (px_volume_fs_health_status{k8s_cluster_name!="", volumename!=""} == bool 2) + unit: short + aliasTemplate: Fixable errors (1 = yes) + comparator: GT + threshold: 0.0 + failureState: DEVIATING + urnTemplate: urn:portworx:/${k8s_cluster_name}:volume/${volumename} + titleTemplate: Filesystem health + intervalSeconds: 60 + remediationHint: !include remediation-hints/volume-fs-safe-to-fix.md.hbs + status: ENABLED diff --git a/stackpacks/portworx/settings/presentations/cluster.sty b/stackpacks/portworx/settings/presentations/cluster.sty new file mode 100644 index 0000000..a9fb652 --- /dev/null +++ b/stackpacks/portworx/settings/presentations/cluster.sty @@ -0,0 +1,382 @@ +# Presentation: Portworx cluster. +nodes: +- _type: ComponentPresentation + name: Portworx cluster + description: How a Portworx cluster is shown + identifier: urn:stackpack:portworx:component-presentation:cluster + binding: + _type: ComponentPresentationQueryBinding + query: label = "stackpack:portworx" AND label = "source-type:portworx cluster" + rank: + specificity: 210 + presentation: + icon: !icon cluster.svg + filters: + - filterId: px_k8s_cluster + overview: + name: + singular: Portworx cluster + plural: Portworx clusters + title: Clusters + mainMenu: + group: Portworx + order: 30 + columns: + - columnId: health + title: Health + projection: + _type: HealthProjection + value: healthState + - columnId: name + title: Name + projection: + _type: ComponentLinkProjection + name: name + identifier: identifiers[0] + - columnId: k8s_cluster + title: Kubernetes cluster + projection: + _type: TextProjection + value: tags.singleValue('cluster-name') + - columnId: version + title: Version + projection: + _type: TextProjection + value: tags.singleValue('portworx.version') + - columnId: phase + title: Phase + projection: + _type: TextProjection + value: tags.singleValue('portworx.phase') + - columnId: capacity_used + title: Capacity used + projection: + _type: MetricProjection + sparkline: true + decimalPlaces: 0 + unit: percentunit + query: sum by (k8s_cluster_name, cluster) (px_cluster_disk_utilized_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"}) / sum by (k8s_cluster_name, cluster) (px_cluster_disk_total_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"}) + metricId: px-cluster-capacity-usage + - columnId: nodes_online + title: Nodes online + projection: + _type: MetricProjection + sparkline: true + decimalPlaces: 0 + unit: short + query: max by (k8s_cluster_name, cluster) (px_cluster_status_nodes_online{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"}) + metricId: px-cluster-nodes + sort: + - columnId: health + - columnId: name + highlight: + title: Portworx cluster + fields: + - fieldId: k8s_cluster + title: Kubernetes cluster + order: 100.0 + projection: + _type: TextProjection + value: tags.singleValue('cluster-name') + - fieldId: namespace + title: Namespace + order: 90.0 + projection: + _type: TextProjection + value: tags.singleValue('namespace') + - fieldId: version + title: Version + order: 80.0 + projection: + _type: TextProjection + value: tags.singleValue('portworx.version') + - fieldId: phase + title: Phase + order: 70.0 + projection: + _type: TextProjection + value: tags.singleValue('portworx.phase') + - fieldId: uid + title: Cluster UID + order: 60.0 + projection: + _type: TextProjection + value: tags.singleValue('portworx.cluster-uid') + - fieldId: age + title: Age + order: 50.0 + projection: + _type: DurationProjection + startTime: tags.singleValue('k8s.creationTimestamp') + - fieldId: labels + title: Labels + order: 10.0 + projection: + _type: TagProjection + provisioning: + topologySourceSelector: source.mapping.identifier == 'urn:stackpack:portworx:otel-component-mapping:cluster' + showConfiguration: true + showStatus: true + relatedResources: + - resourceId: storage_nodes + title: Storage nodes + order: 1.0 + topologyQuery: (withNeighborsOf(direction = "up", components = (identifier = "${identifiers[0]}"), levels = "1")) and label = "source-type:portworx node" + presentationIdentifier: urn:stackpack:portworx:component-presentation:storage-node + - resourceId: volumes + title: Volumes + order: 2.0 + topologyQuery: (withNeighborsOf(direction = "up", components = (identifier = "${identifiers[0]}"), levels = "1")) and label = "source-type:portworx volume" + presentationIdentifier: urn:stackpack:portworx:component-presentation:volume + events: + showEvents: true + relatedResourcesQuery: identifier = "${identifiers[0]}" + metrics: + - sectionId: capacity + title: Capacity + order: 100 + metrics: + - metricId: px-cluster-capacity-usage + order: 100 + - metricId: px-cluster-capacity + order: 90 + - sectionId: health + title: Health + order: 90 + metrics: + - metricId: px-cluster-nodes + order: 100 + - metricId: px-cluster-quorum + order: 90 + summary: + metrics: + - summaryMetricId: px_cluster_capacity_usage_summary + name: Capacity used + order: 100 + decimalPlaces: 0 + unit: percentunit + query: sum by (k8s_cluster_name, cluster) (px_cluster_disk_utilized_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"}) / sum by (k8s_cluster_name, cluster) (px_cluster_disk_total_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"}) + metricId: px-cluster-capacity-usage + - summaryMetricId: px_cluster_nodes_summary + name: Nodes online + order: 90 + decimalPlaces: 2 + unit: short + query: max by (k8s_cluster_name, cluster) (px_cluster_status_nodes_online{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"}) + metricId: px-cluster-nodes + metricPerspective: + tabs: + - tabId: portworx + title: Portworx + order: 50 + sections: + - sectionId: capacity + title: Capacity + order: 100 + metrics: + - metricId: px-cluster-capacity-usage + - metricId: px-cluster-capacity + - metricId: px-cluster-provisioned + - metricId: px-cluster-volumes + - sectionId: health + title: Health + order: 90 + metrics: + - metricId: px-cluster-nodes + - metricId: px-cluster-quorum + - metricId: px-cluster-kvdb + - metricId: px-cluster-license + - sectionId: performance + title: Performance + order: 80 + metrics: + - metricId: px-cluster-throughput + - metricId: px-cluster-iops + - metricId: px-cluster-latency + - metricId: px-cluster-pending-io + - sectionId: resources + title: Resources + order: 70 + metrics: + - metricId: px-cluster-cpu + - metricId: px-cluster-memory + metrics: + - metricId: px-cluster-capacity-usage + name: Storage capacity used + description: Share of the raw pool capacity of all storage nodes that is in use. + metricQueries: + - expression: sum(px_cluster_disk_utilized_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"}) / sum(px_cluster_disk_total_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"}) + alias: Used + chart: + _type: GaugeChart + unit: percentunit + decimalPlaces: 0 + max: 1 + calculation: last + thresholds: + defaultColor: '#33A02C' + steps: + - color: '#E31A1C' + value: 0.8 + - color: '#FF7F00' + value: 0.7 + mode: absolute + - metricId: px-cluster-capacity + name: Storage capacity + description: Used and total raw pool capacity across all storage nodes. + metricQueries: + - expression: sum(px_cluster_disk_utilized_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"}) + alias: Used + - expression: sum(px_cluster_disk_total_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"}) + alias: Total + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - metricId: px-cluster-provisioned + name: Provisioned volume capacity + description: Sum of the sizes of all Portworx volumes, compared with the raw capacity. Volumes are thin provisioned and each replica uses raw capacity. + metricQueries: + - expression: sum(max by (volumeid) (px_volume_capacity_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"})) + alias: Provisioned + - expression: sum(px_cluster_disk_total_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"}) + alias: Raw capacity + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - metricId: px-cluster-nodes + name: Nodes + description: Cluster members online and offline, and storage nodes whose storage is down. + metricQueries: + - expression: max(px_cluster_status_nodes_online{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"}) + alias: Online + - expression: max(px_cluster_status_nodes_offline{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"}) + alias: Offline + - expression: max(px_cluster_status_nodes_storage_down{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"}) + alias: Storage down + chart: + _type: TimeSeriesChart + unit: short + - metricId: px-cluster-quorum + name: Quorum + description: 'Cluster quorum state as reported by Portworx: 1 means the cluster has quorum.' + metricQueries: + - expression: max(px_cluster_status_quorum{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"}) + alias: Quorum + chart: + _type: StatChart + unit: short + decimals: 0 + calculation: last + sparkline: true + thresholds: + defaultColor: '#33A02C' + steps: + - color: '#E31A1C' + value: 2 + - metricId: px-cluster-volumes + name: Volumes + description: Number of Portworx volumes, how many are attached, and deleted volumes still in the trashcan (they keep using pool capacity). + metricQueries: + - expression: count(max by (volumeid) (px_volume_capacity_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"})) + alias: Volumes + - expression: count(max by (volumeid) (px_volume_attached{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"}) > 0) + alias: Attached + - expression: max(px_cluster_trashcan_volumes{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"}) + alias: In trashcan + chart: + _type: TimeSeriesChart + unit: short + - metricId: px-cluster-throughput + name: Volume throughput + description: Read and write throughput of all Portworx volumes. + metricQueries: + - expression: sum(max by (volumeid) (px_volume_readthroughput{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"})) + alias: Read + - expression: sum(max by (volumeid) (px_volume_writethroughput{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"})) + alias: Write + chart: + _type: TimeSeriesChart + unit: bytes/sec(IEC) + - metricId: px-cluster-iops + name: Volume IOPS + description: Read and write operations per second of all Portworx volumes. + metricQueries: + - expression: sum(max by (volumeid) (rate(px_volume_reads_total{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"}[${__rate_interval}]))) + alias: Read + - expression: sum(max by (volumeid) (rate(px_volume_writes_total{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"}[${__rate_interval}]))) + alias: Write + chart: + _type: TimeSeriesChart + unit: ops/sec + - metricId: px-cluster-latency + name: I/O latency per node + description: Average read and write latency of the I/O each node coordinates. + metricQueries: + - expression: max by (k8s_cluster_name, node) (px_node_stats_read_latency_seconds{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}", coordinator="1"}) + alias: read ${node} + componentIdentifierTemplate: urn:portworx:/${k8s_cluster_name}:node/${node} + - expression: max by (k8s_cluster_name, node) (px_node_stats_write_latency_seconds{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}", coordinator="1"}) + alias: write ${node} + componentIdentifierTemplate: urn:portworx:/${k8s_cluster_name}:node/${node} + chart: + _type: TimeSeriesChart + unit: s + - metricId: px-cluster-cpu + name: Portworx CPU per node + description: CPU used on each node, as reported by Portworx. + metricQueries: + - expression: max by (k8s_cluster_name, node) (px_cluster_cpu_percent{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"}) + alias: ${node} + componentIdentifierTemplate: urn:portworx:/${k8s_cluster_name}:node/${node} + chart: + _type: TimeSeriesChart + unit: percent + - metricId: px-cluster-memory + name: Memory per node + description: Memory used on each node, as reported by Portworx. + metricQueries: + - expression: max by (k8s_cluster_name, node) (px_cluster_memory_utilized_percent{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"}) + alias: ${node} + componentIdentifierTemplate: urn:portworx:/${k8s_cluster_name}:node/${node} + chart: + _type: TimeSeriesChart + unit: percent + - metricId: px-cluster-pending-io + name: Pending I/O per node + description: I/O operations waiting in Portworx on each node. + metricQueries: + - expression: max by (k8s_cluster_name, node) (px_cluster_pendingio{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"}) + alias: ${node} + componentIdentifierTemplate: urn:portworx:/${k8s_cluster_name}:node/${node} + chart: + _type: TimeSeriesChart + unit: short + - metricId: px-cluster-kvdb + name: KVDB health per node + description: 'Each node''s view of the Portworx key-value database: 1 healthy, 2 unhealthy.' + metricQueries: + - expression: max by (k8s_cluster_name, node) (px_kvdb_health_state_node_view{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"}) + alias: ${node} + componentIdentifierTemplate: urn:portworx:/${k8s_cluster_name}:node/${node} + chart: + _type: TimeSeriesChart + unit: short + - metricId: px-cluster-license + name: License days left + description: Days until the Portworx license expires (lowest value reported by any node). + metricQueries: + - expression: min(px_node_status_license_expiry{k8s_cluster_name="${tags.singleValue('cluster-name')}", cluster="${name}"}) + alias: Days + chart: + _type: StatChart + unit: short + decimals: 0 + calculation: last + sparkline: true + thresholds: + defaultColor: '#E31A1C' + steps: + - color: '#FF7F00' + value: 5 + - color: '#33A02C' + value: 30 diff --git a/stackpacks/portworx/settings/presentations/common-filters.sty b/stackpacks/portworx/settings/presentations/common-filters.sty new file mode 100644 index 0000000..8295fd6 --- /dev/null +++ b/stackpacks/portworx/settings/presentations/common-filters.sty @@ -0,0 +1,29 @@ +# Presentation: Portworx common filters. +nodes: +- _type: ComponentPresentation + name: Portworx common filters + description: Filters shared by the Portworx component lists + identifier: urn:stackpack:portworx:component-presentation:common-filters + binding: + _type: ComponentPresentationQueryBinding + query: label = "stackpack:portworx" + rank: + specificity: 1 + presentation: + filters: + - filterId: px_k8s_cluster + displayName: + singular: Cluster + plural: Clusters + filter: + _type: TagFilter + tagKey: cluster-name + menuSection: Portworx + - filterId: px_cluster + displayName: + singular: Portworx cluster + plural: Portworx clusters + filter: + _type: TagFilter + tagKey: portworx.cluster + menuSection: Portworx diff --git a/stackpacks/portworx/settings/presentations/storage-nodes.sty b/stackpacks/portworx/settings/presentations/storage-nodes.sty new file mode 100644 index 0000000..7536bfd --- /dev/null +++ b/stackpacks/portworx/settings/presentations/storage-nodes.sty @@ -0,0 +1,277 @@ +# Presentation: Portworx storage node. +nodes: +- _type: ComponentPresentation + name: Portworx storage node + description: Adds the Portworx list and metrics to Kubernetes nodes that run Portworx + identifier: urn:stackpack:portworx:component-presentation:storage-node + binding: + _type: ComponentPresentationQueryBinding + query: label = "stackpack:portworx" AND label = "source-type:portworx node" + rank: + specificity: 110 + presentation: + icon: !icon storage-node.svg + filters: + - filterId: px_k8s_cluster + - filterId: px_cluster + overview: + name: + singular: Portworx node + plural: Portworx nodes + title: Storage nodes + mainMenu: + group: Portworx + order: 20 + columns: + - columnId: health + title: Health + projection: + _type: HealthProjection + value: healthState + - columnId: name + title: Name + projection: + _type: ComponentLinkProjection + name: name + identifier: identifiers[0] + - columnId: k8s_cluster + title: Kubernetes cluster + projection: + _type: TextProjection + value: tags.singleValue('cluster-name') + - columnId: px_cluster + title: Portworx cluster + projection: + _type: TextProjection + value: tags.singleValue('portworx.cluster') + - columnId: phase + title: Phase + projection: + _type: TextProjection + value: tags.singleValue('portworx.phase') + - columnId: version + title: Version + projection: + _type: TextProjection + value: tags.singleValue('portworx.version') + - columnId: kvdb + title: KVDB member + projection: + _type: TextProjection + value: tags.singleValue('portworx.kvdb-member') + - columnId: capacity_used + title: Capacity used + projection: + _type: MetricProjection + sparkline: true + decimalPlaces: 0 + unit: percentunit + query: max by (k8s_cluster_name, node) (px_cluster_disk_utilized_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}"}) / max by (k8s_cluster_name, node) (px_cluster_disk_total_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}"}) + metricId: px-node-capacity-usage + sort: + - columnId: health + - columnId: name + summary: + metrics: + - summaryMetricId: px_node_capacity_usage_summary + name: Portworx capacity used + order: 100 + decimalPlaces: 0 + unit: percentunit + query: max by (k8s_cluster_name, node) (px_cluster_disk_utilized_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}"}) / max by (k8s_cluster_name, node) (px_cluster_disk_total_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}"}) + metricId: px-node-capacity-usage + - summaryMetricId: px_node_latency_summary + name: Portworx write latency + order: 90 + decimalPlaces: 2 + unit: s + query: max by (k8s_cluster_name, node) (px_node_stats_write_latency_seconds{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}", coordinator="1"}) + metricId: px-node-latency + metricPerspective: + tabs: + - tabId: portworx + title: Portworx + order: 50 + sections: + - sectionId: capacity + title: Capacity + order: 100 + metrics: + - metricId: px-node-capacity-usage + - metricId: px-node-capacity + - metricId: px-node-pool-usage + - metricId: px-node-replicas + - sectionId: health + title: Health + order: 90 + metrics: + - metricId: px-node-status + - metricId: px-node-pool-status + - metricId: px-node-kvdb + - sectionId: performance + title: Performance + order: 80 + metrics: + - metricId: px-node-latency + - metricId: px-node-iops + - metricId: px-node-throughput + - metricId: px-node-disk-latency + - sectionId: resources + title: Resources + order: 70 + metrics: + - metricId: px-node-cpu-memory + - metricId: px-node-network + metrics: + - metricId: px-node-capacity-usage + name: Storage capacity used + description: Share of this node's Portworx pool capacity that is in use. + metricQueries: + - expression: max(px_cluster_disk_utilized_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}"}) / max(px_cluster_disk_total_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}"}) + alias: Used + chart: + _type: GaugeChart + unit: percentunit + decimalPlaces: 0 + max: 1 + calculation: last + thresholds: + defaultColor: '#33A02C' + steps: + - color: '#E31A1C' + value: 0.8 + - color: '#FF7F00' + value: 0.7 + mode: absolute + - metricId: px-node-capacity + name: Storage capacity + description: Used and total Portworx pool capacity on this node. + metricQueries: + - expression: max(px_cluster_disk_utilized_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}"}) + alias: Used + - expression: max(px_cluster_disk_total_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}"}) + alias: Total + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - metricId: px-node-status + name: Portworx node status + description: 'Portworx status code of this node: 2 is OK; 3 offline, 5 not in quorum, 8 storage down.' + metricQueries: + - expression: max(px_node_status_node_status{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}"}) + alias: Status + chart: + _type: StatChart + unit: short + decimals: 0 + calculation: last + sparkline: true + thresholds: + defaultColor: '#E31A1C' + steps: + - color: '#33A02C' + value: 2 + - color: '#E31A1C' + value: 3 + - metricId: px-node-pool-usage + name: Pool usage + description: Used share of each storage pool on this node. + metricQueries: + - expression: max by (pool) (px_pool_stats_used_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}"}) / max by (pool) (px_pool_stats_total_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}"}) + alias: pool ${pool} + chart: + _type: TimeSeriesChart + unit: percentunit + - metricId: px-node-pool-status + name: Pool status + description: 'Status of each storage pool: 1 online, 0 offline, 2 full, 3 not found, 4 maintenance, 5 degraded, 6 background activity.' + metricQueries: + - expression: max by (pool) (px_pool_stats_status{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}"}) + alias: pool ${pool} + chart: + _type: TimeSeriesChart + unit: short + - metricId: px-node-latency + name: I/O latency + description: Average latency of the I/O this node coordinates. + metricQueries: + - expression: max(px_node_stats_read_latency_seconds{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}", coordinator="1"}) + alias: Read + - expression: max(px_node_stats_write_latency_seconds{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}", coordinator="1"}) + alias: Write + chart: + _type: TimeSeriesChart + unit: s + - metricId: px-node-iops + name: IOPS + description: Read and write operations per second this node coordinates. + metricQueries: + - expression: max(px_node_stats_read_iops{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}", coordinator="1"}) + alias: Read + - expression: max(px_node_stats_write_iops{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}", coordinator="1"}) + alias: Write + chart: + _type: TimeSeriesChart + unit: ops/sec + - metricId: px-node-throughput + name: Throughput + description: Read and write throughput this node coordinates. + metricQueries: + - expression: max(px_node_stats_readthroughput{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}", coordinator="1"}) + alias: Read + - expression: max(px_node_stats_writethroughput{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}", coordinator="1"}) + alias: Write + chart: + _type: TimeSeriesChart + unit: bytes/sec(IEC) + - metricId: px-node-disk-latency + name: Disk latency + description: Average read and write latency of each disk in the pools. + metricQueries: + - expression: max by (disk) (px_disk_stats_read_latency_seconds{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}"}) + alias: read ${disk} + - expression: max by (disk) (px_disk_stats_write_latency_seconds{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}"}) + alias: write ${disk} + chart: + _type: TimeSeriesChart + unit: s + - metricId: px-node-network + name: Portworx network traffic + description: Bytes Portworx sends to and receives from other nodes. + metricQueries: + - expression: max(rate(px_network_io_sent_bytes_total{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}"}[${__rate_interval}])) + alias: Sent + - expression: max(rate(px_network_io_received_bytes_total{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}"}[${__rate_interval}])) + alias: Received + chart: + _type: TimeSeriesChart + unit: bytes/sec(IEC) + - metricId: px-node-cpu-memory + name: CPU and memory + description: CPU and memory used on this node, as reported by Portworx. + metricQueries: + - expression: max(px_cluster_cpu_percent{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}"}) + alias: CPU + - expression: max(px_cluster_memory_utilized_percent{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}"}) + alias: Memory + chart: + _type: TimeSeriesChart + unit: percent + - metricId: px-node-replicas + name: Volume replicas + description: Number of volume replicas stored on this node. + metricQueries: + - expression: max(px_node_stats_volume_replica_count{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}"}) + alias: Replicas + chart: + _type: TimeSeriesChart + unit: short + - metricId: px-node-kvdb + name: KVDB health + description: 'This node''s view of the Portworx key-value database: 1 healthy, 2 unhealthy.' + metricQueries: + - expression: max(px_kvdb_health_state_node_view{k8s_cluster_name="${tags.singleValue('cluster-name')}", node="${name}"}) + alias: Node view + chart: + _type: TimeSeriesChart + unit: short diff --git a/stackpacks/portworx/settings/presentations/volumes.sty b/stackpacks/portworx/settings/presentations/volumes.sty new file mode 100644 index 0000000..44f7558 --- /dev/null +++ b/stackpacks/portworx/settings/presentations/volumes.sty @@ -0,0 +1,225 @@ +# Presentation: Portworx volume. +nodes: +- _type: ComponentPresentation + name: Portworx volume + description: Adds the Portworx list and metrics to persistent volumes that Portworx provides + identifier: urn:stackpack:portworx:component-presentation:volume + binding: + _type: ComponentPresentationQueryBinding + query: label = "stackpack:portworx" AND label = "source-type:portworx volume" + rank: + specificity: 110 + presentation: + icon: !icon volume.svg + filters: + - filterId: px_k8s_cluster + - filterId: px_cluster + overview: + name: + singular: Portworx volume + plural: Portworx volumes + title: Volumes + mainMenu: + group: Portworx + order: 10 + columns: + - columnId: health + title: Health + projection: + _type: HealthProjection + value: healthState + - columnId: name + title: Name + projection: + _type: ComponentLinkProjection + name: name + identifier: identifiers[0] + - columnId: pvc + title: Claim + projection: + _type: TextProjection + value: tags.singleValue('persistent-volume-claim') + - columnId: k8s_cluster + title: Kubernetes cluster + projection: + _type: TextProjection + value: tags.singleValue('cluster-name') + - columnId: replicas + title: Replicas + projection: + _type: TextProjection + value: tags.singleValue('portworx.replicas') + - columnId: used + title: Used + projection: + _type: MetricProjection + sparkline: true + decimalPlaces: 0 + unit: percentunit + query: max by (k8s_cluster_name, volumename) (px_volume_usage_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", volumename="${name}"}) / max by (k8s_cluster_name, volumename) (px_volume_capacity_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", volumename="${name}"}) + metricId: px-volume-usage + sort: + - columnId: health + - columnId: name + summary: + metrics: + - summaryMetricId: px_volume_usage_summary + name: Volume used + order: 100 + decimalPlaces: 0 + unit: percentunit + query: max by (k8s_cluster_name, volumename) (px_volume_usage_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", volumename="${name}"}) / max by (k8s_cluster_name, volumename) (px_volume_capacity_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", volumename="${name}"}) + metricId: px-volume-usage + - summaryMetricId: px_volume_latency_summary + name: Write latency + order: 90 + decimalPlaces: 2 + unit: s + query: max by (k8s_cluster_name, volumename) (px_volume_write_latency_seconds{k8s_cluster_name="${tags.singleValue('cluster-name')}", volumename="${name}"}) + metricId: px-volume-latency + metricPerspective: + tabs: + - tabId: portworx + title: Portworx + order: 50 + sections: + - sectionId: capacity + title: Capacity + order: 100 + metrics: + - metricId: px-volume-usage + - metricId: px-volume-capacity + - sectionId: replication + title: Replication + order: 90 + metrics: + - metricId: px-volume-replication + - metricId: px-volume-ha + - sectionId: performance + title: Performance + order: 80 + metrics: + - metricId: px-volume-latency + - metricId: px-volume-iops + - metricId: px-volume-throughput + - metricId: px-volume-depth + - metricId: px-volume-long-io + metrics: + - metricId: px-volume-usage + name: Volume used + description: Share of the volume size that is in use. + metricQueries: + - expression: max(px_volume_usage_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", volumename="${name}"}) / max(px_volume_capacity_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", volumename="${name}"}) + alias: Used + chart: + _type: GaugeChart + unit: percentunit + decimalPlaces: 0 + max: 1 + calculation: last + thresholds: + defaultColor: '#33A02C' + steps: + - color: '#E31A1C' + value: 0.8 + - color: '#FF7F00' + value: 0.7 + mode: absolute + - metricId: px-volume-capacity + name: Volume capacity + description: Used bytes, filesystem used bytes and size of the volume. + metricQueries: + - expression: max(px_volume_usage_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", volumename="${name}"}) + alias: Used + - expression: max(px_volume_fs_usage_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", volumename="${name}"}) + alias: Filesystem used + - expression: max(px_volume_capacity_bytes{k8s_cluster_name="${tags.singleValue('cluster-name')}", volumename="${name}"}) + alias: Size + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - metricId: px-volume-replication + name: Replication status + description: 0 up, 1 not in quorum, 2 resync, 3 degraded, 4 detached, 5 restore. + metricQueries: + - expression: max(px_volume_replication_status{k8s_cluster_name="${tags.singleValue('cluster-name')}", volumename="${name}"}) + alias: Status + chart: + _type: StatChart + unit: short + decimals: 0 + calculation: last + sparkline: true + thresholds: + defaultColor: '#33A02C' + steps: + - color: '#FF7F00' + value: 1 + - color: '#1F78B4' + value: 4 + - metricId: px-volume-ha + name: Replicas + description: Configured replication factor and replicas currently in sync. + metricQueries: + - expression: max(px_volume_halevel{k8s_cluster_name="${tags.singleValue('cluster-name')}", volumename="${name}"}) + alias: Configured + - expression: min(px_volume_currhalevel{k8s_cluster_name="${tags.singleValue('cluster-name')}", volumename="${name}"}) + alias: Current + chart: + _type: TimeSeriesChart + unit: short + - metricId: px-volume-latency + name: Latency + description: Average read and write latency of the volume. + metricQueries: + - expression: max(px_volume_read_latency_seconds{k8s_cluster_name="${tags.singleValue('cluster-name')}", volumename="${name}"}) + alias: Read + - expression: max(px_volume_write_latency_seconds{k8s_cluster_name="${tags.singleValue('cluster-name')}", volumename="${name}"}) + alias: Write + chart: + _type: TimeSeriesChart + unit: s + - metricId: px-volume-iops + name: IOPS + description: Read and write operations per second on the volume. + metricQueries: + - expression: max(px_volume_read_iops{k8s_cluster_name="${tags.singleValue('cluster-name')}", volumename="${name}"}) + alias: Read + - expression: max(px_volume_write_iops{k8s_cluster_name="${tags.singleValue('cluster-name')}", volumename="${name}"}) + alias: Write + chart: + _type: TimeSeriesChart + unit: ops/sec + - metricId: px-volume-throughput + name: Throughput + description: Read and write throughput of the volume. + metricQueries: + - expression: max(px_volume_readthroughput{k8s_cluster_name="${tags.singleValue('cluster-name')}", volumename="${name}"}) + alias: Read + - expression: max(px_volume_writethroughput{k8s_cluster_name="${tags.singleValue('cluster-name')}", volumename="${name}"}) + alias: Write + chart: + _type: TimeSeriesChart + unit: bytes/sec(IEC) + - metricId: px-volume-depth + name: Queue depth + description: I/O operations in progress on the volume. + metricQueries: + - expression: max(px_volume_depth_io{k8s_cluster_name="${tags.singleValue('cluster-name')}", volumename="${name}"}) + alias: In progress + chart: + _type: TimeSeriesChart + unit: short + - metricId: px-volume-long-io + name: Long I/Os + description: Reads, writes and flushes that Portworx counted as long, per 10 minutes. Long I/Os are the most direct sign of stalled storage. + metricQueries: + - expression: max(delta(px_volume_num_long_reads{k8s_cluster_name="${tags.singleValue('cluster-name')}", volumename="${name}"}[10m])) + alias: Reads + - expression: max(delta(px_volume_num_long_writes{k8s_cluster_name="${tags.singleValue('cluster-name')}", volumename="${name}"}[10m])) + alias: Writes + - expression: max(delta(px_volume_num_long_flushes{k8s_cluster_name="${tags.singleValue('cluster-name')}", volumename="${name}"}[10m])) + alias: Flushes + chart: + _type: TimeSeriesChart + unit: short diff --git a/stackpacks/portworx/settings/relation-mappings/node-member-of-cluster.sty b/stackpacks/portworx/settings/relation-mappings/node-member-of-cluster.sty new file mode 100644 index 0000000..045cd9b --- /dev/null +++ b/stackpacks/portworx/settings/relation-mappings/node-member-of-cluster.sty @@ -0,0 +1,31 @@ +# Relation mapping: Portworx storage node -> the Portworx cluster that owns its StorageNode. +nodes: + - _type: "OtelRelationMapping" + name: "Portworx node member of cluster" + description: "Links a Portworx storage node to its Portworx cluster" + identifier: "urn:stackpack:portworx:otel-relation-mapping:node-member-of-cluster" + input: + signal: + - "LOGS" + resource: + scope: + log: + action: "log.body['type'] in ['ADDED', 'MODIFIED'] ? 'CREATE' : log.body['type'] == 'DELETED' ? 'DELETE' : 'CONTINUE'" + condition: >- + log.eventName == 'KubernetesCustomResourceEvent' && + log.attributes['k8s.resource.group'] == 'core.libopenstorage.org' && + log.body['object']['kind'] == 'StorageNode' && + 'ownerReferences' in log.body['object']['metadata'] + vars: + - name: "clusterName" + value: "resource.attributes['k8s.cluster.name']" + - name: "name" + value: "log.attributes['k8s.object.name']" + - name: "pxCluster" + value: "log.body['object']['metadata']['ownerReferences'][0]['name']" + output: + sourceId: "'urn:portworx:/' + vars.clusterName + ':node/' + vars.name" + targetId: "'urn:portworx:/' + vars.clusterName + ':cluster/' + vars.pxCluster" + typeName: "'member of'" + dependencyType: "'ONE_WAY'" + expireAfter: 900000 diff --git a/stackpacks/portworx/settings/relation-mappings/volume-provided-by-cluster.sty b/stackpacks/portworx/settings/relation-mappings/volume-provided-by-cluster.sty new file mode 100644 index 0000000..417fbc1 --- /dev/null +++ b/stackpacks/portworx/settings/relation-mappings/volume-provided-by-cluster.sty @@ -0,0 +1,29 @@ +# Relation mapping: Portworx volume -> the Portworx cluster that provides it. +nodes: + - _type: "OtelRelationMapping" + name: "Portworx volume provided by cluster" + description: "Links a Portworx volume to its Portworx cluster" + identifier: "urn:stackpack:portworx:otel-relation-mapping:volume-provided-by-cluster" + input: + signal: + - "METRICS" + resource: + condition: "'k8s.cluster.name' in resource.attributes" + scope: + metric: + condition: "metric.name == 'px_volume_replication_status'" + datapoint: + action: "'CREATE'" + condition: >- + 'volumename' in datapoint.attributes && + 'cluster' in datapoint.attributes && + datapoint.attributes['volumename'] != '' + vars: + - name: "clusterName" + value: "resource.attributes['k8s.cluster.name']" + output: + sourceId: "'urn:portworx:/' + vars.clusterName + ':volume/' + datapoint.attributes['volumename']" + targetId: "'urn:portworx:/' + vars.clusterName + ':cluster/' + datapoint.attributes['cluster']" + typeName: "'provided by'" + dependencyType: "'ONE_WAY'" + expireAfter: 900000 diff --git a/stackpacks/portworx/stackpack.yaml b/stackpacks/portworx/stackpack.yaml new file mode 100644 index 0000000..5198b79 --- /dev/null +++ b/stackpacks/portworx/stackpack.yaml @@ -0,0 +1,17 @@ +name: "portworx" +version: "0.1.0" +schemaVersion: "2.0" +displayName: "Portworx" +categories: [ "Storage", "Portworx", "Kubernetes" ] +logoUrl: !resource "logo.svg" +overviewUrl: !resource "overview.md" +detailedOverviewUrl: !resource "detailed-overview.md" +configurationUrls: + NOT_INSTALLED: !resource "configuration.md" + PROVISIONING: !resource "provisioning.md" + WAITING_FOR_DATA: !resource "waitingfordata.md" + INSTALLED: !resource "enabled.md" + DEPROVISIONING: !resource "deprovisioning.md" + ERROR: !resource "error.md" +provision: + companionStackPacks: [] From 85140db6ea88d5c180899acc2ea340e8a79c16ef Mon Sep 17 00:00:00 2001 From: Alejandro Bonilla Date: Tue, 10 Nov 2026 07:41:02 -0700 Subject: [PATCH 2/2] fix(portworx): address community contribution review feedback Mark the StackPack experimental/community-maintained, add the Maintainer line and tested SUSE Observability version the new per-pack CI and CONTRIBUTING.md require, and document required permissions and removal steps. Fix the "Volume down" monitor: it used "+" to combine px_volume_status and px_volume_state, which inner-joins the two series and drops a volume that only reports one of them. Use "or" fallbacks so either metric alone still triggers the alert. --- stackpacks/portworx/README.md | 16 +++++++++++++++- .../portworx/resources/detailed-overview.md | 2 ++ stackpacks/portworx/resources/overview.md | 2 +- .../portworx/settings/monitors/volumes.sty | 7 +++++-- 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/stackpacks/portworx/README.md b/stackpacks/portworx/README.md index 637ce27..9f94f9c 100644 --- a/stackpacks/portworx/README.md +++ b/stackpacks/portworx/README.md @@ -1,5 +1,9 @@ # Portworx StackPack +Maintainer: @abonillabeeche + +**Experimental, community-maintained.** This is a third-party StackPack: the maintainer above owns support and updates, not SUSE. Report issues and compatibility problems against this repository. + Monitors Portworx Enterprise 3.x on Kubernetes. User-facing setup is in `resources/configuration.md`. ## Data path @@ -21,8 +25,18 @@ Monitors Portworx Enterprise 3.x on Kubernetes. User-facing setup is in `resourc - Volume metrics are reported by every node that holds a replica, and some of those series carry only `volumeid`, so volume queries use `max by` and `volumename!=""`. - `px_volume_reads_total` / `px_volume_writes_total` carry only `volumeid`. Per-volume IOPS use the `px_volume_read_iops` / `px_volume_write_iops` gauges. - Volume performance metrics only exist while a volume is attached. +- "Volume down" alerts on `px_volume_status == 3` or `px_volume_state == 6`. The query does not assume Portworx reports both metrics for every volume: either one alone is enough to alert, so a volume missing one of the two still has its failure caught by the other. - A volume with no matching Kubernetes PV stays a standalone `portworx volume` component. On the test cluster, one Portworx node kept exporting series for two deleted volumes, so they showed up this way until the series stopped. +## Required permissions + +- Creating the `ServiceMonitor` needs `create`/`get`/`list` on `servicemonitors.monitoring.coreos.com` in the namespace where Portworx runs. +- Adding `core.libopenstorage.org` to the agent's resource collector needs permission to change the `suse-observability-agent` Helm release's values (or the Fleet bundle, Rancher app or Harvester add-on that manages it), plus the agent's existing RBAC already covers reading `StorageCluster`/`StorageNode` once that API group is included. + +## Removing the integration + +Uninstalling the StackPack stops Portworx data from being processed, but it does not remove what you added for it: delete the `so-agent-portworx` ServiceMonitor and drop `core.libopenstorage.org` from the agent's `crDiscovery.apiGroups.include` if you no longer need them. + ## Tested with -Portworx 3.7.1 (operator-installed, internal KVDB) on a 3-node RKE2 / SUSE Virtualization cluster, with agent chart 1.6.12. +Portworx 3.7.1 (operator-installed, internal KVDB) on a 3-node RKE2 / SUSE Virtualization cluster, with agent chart 1.6.12, against SUSE Observability platform version 2.12.0. diff --git a/stackpacks/portworx/resources/detailed-overview.md b/stackpacks/portworx/resources/detailed-overview.md index 25ab1fb..c599852 100644 --- a/stackpacks/portworx/resources/detailed-overview.md +++ b/stackpacks/portworx/resources/detailed-overview.md @@ -1,5 +1,7 @@ ## Portworx StackPack +**Experimental, community-maintained.** This StackPack is contributed and supported by its community maintainer, not by SUSE. See the StackPack's README for maintainer contact, tested versions and removal steps. + The Portworx StackPack shows your Portworx clusters, storage nodes and volumes in SUSE Observability and tracks their health with monitors ported from the Portworx operator's alert rules. ### What you get diff --git a/stackpacks/portworx/resources/overview.md b/stackpacks/portworx/resources/overview.md index 97377a2..d6db9e9 100644 --- a/stackpacks/portworx/resources/overview.md +++ b/stackpacks/portworx/resources/overview.md @@ -1 +1 @@ -Monitor [Portworx Enterprise](https://docs.portworx.com/) storage clusters on Kubernetes: capacity, quorum, storage node and volume health, and I/O performance, with the alerts Portworx ships for Prometheus. +Monitor [Portworx Enterprise](https://docs.portworx.com/) storage clusters on Kubernetes: capacity, quorum, storage node and volume health, and I/O performance, with the alerts Portworx ships for Prometheus. Experimental, community-maintained: see the StackPack README for support and update ownership. diff --git a/stackpacks/portworx/settings/monitors/volumes.sty b/stackpacks/portworx/settings/monitors/volumes.sty index 66e3c99..786d29d 100644 --- a/stackpacks/portworx/settings/monitors/volumes.sty +++ b/stackpacks/portworx/settings/monitors/volumes.sty @@ -65,7 +65,7 @@ nodes: status: ENABLED - _type: Monitor name: Portworx - Volume down - description: For 5 minutes Portworx has reported the volume status as down (3) or its state as error (6). + description: For 5 minutes Portworx has reported the volume status as down (3) or its state as error (6). Either metric alone is enough to alert; they are not required to be reported together. tags: - portworx - storage @@ -73,7 +73,10 @@ nodes: identifier: urn:stackpack:portworx:shared:monitor:volume-down arguments: metric: - query: max by (k8s_cluster_name, volumename) (min_over_time(((max by (k8s_cluster_name, volumename) (px_volume_status{k8s_cluster_name!="", volumename!=""}) == bool 3) + (max by (k8s_cluster_name, volumename) (px_volume_state{k8s_cluster_name!="", volumename!=""}) == bool 6) > bool 0)[5m:1m])) + # "+" alone would inner-join px_volume_status and px_volume_state, so a volume reporting only + # one of the two metrics would drop out of the result. The "or" fallbacks keep that volume's + # single metric instead of silently losing its failure signal. + query: max by (k8s_cluster_name, volumename) (min_over_time(clamp_max(((max by (k8s_cluster_name, volumename) (px_volume_status{k8s_cluster_name!="", volumename!=""}) == bool 3) + (max by (k8s_cluster_name, volumename) (px_volume_state{k8s_cluster_name!="", volumename!=""}) == bool 6)) or (max by (k8s_cluster_name, volumename) (px_volume_status{k8s_cluster_name!="", volumename!=""}) == bool 3) or (max by (k8s_cluster_name, volumename) (px_volume_state{k8s_cluster_name!="", volumename!=""}) == bool 6), 1)[5m:1m])) unit: short aliasTemplate: Down or error (1 = yes) comparator: GT