diff --git a/README.md b/README.md index f8098f9..49c0e03 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ a host][install]. | `config/` | One YAML file per deployed daemon instance, organised by subsystem. | | `src/hispec/` | Installable package: the `HispecDaemon` base class and the `driver/` submodules. | | `systemd/` | Template unit, per-instance env files, operator commands and installer. See [Running the daemons with systemd][systemd-ops]. | +| `grafana/` | Grafana dashboards. | | `docs/` | Sphinx documentation sources. | | `tests/` | Driver unit tests. | | `etc/` | Vendored externals: `camera-interface`, `PIPython`. | diff --git a/docs/operations/systemd.md b/docs/operations/systemd.md index fbfc8a2..fd55a19 100644 --- a/docs/operations/systemd.md +++ b/docs/operations/systemd.md @@ -523,6 +523,11 @@ Then re-run `hispec install`, which also fetches the agent, and restart the daemons. `install` prints the `systemctl enable` lines for the units, and `hispec doctor` checks each piece. Without `otel.env` nothing is exported. +`grafana/hispec-telemetry-dashboard.json` is the Grafana board for it: request +rates, failures and latency per daemon and keyword, traces, daemon logs and host +metrics, from hispec's hosts only. It picks the datasources whose names contain +`HISPEC`. + ## Host setup (admins) Installing and re-running `install.sh` is described in {doc}`../install`. diff --git a/grafana/hispec-telemetry-dashboard.json b/grafana/hispec-telemetry-dashboard.json new file mode 100644 index 0000000..f557f30 --- /dev/null +++ b/grafana/hispec-telemetry-dashboard.json @@ -0,0 +1,1030 @@ +{ + "uid": "hispec-telemetry", + "title": "HISPEC telemetry", + "description": "Requests, traces, logs and host metrics from the HISPEC daemons, through libby's OpenTelemetry export.", + "tags": [ + "hispec", + "libby", + "opentelemetry" + ], + "timezone": "browser", + "refresh": "1m", + "time": { + "from": "now-6h", + "to": "now" + }, + "graphTooltip": 1, + "schemaVersion": 39, + "editable": true, + "templating": { + "list": [ + { + "name": "prometheus", + "label": "Prometheus", + "type": "datasource", + "query": "prometheus", + "regex": "/HISPEC/i", + "hide": 0, + "current": {} + }, + { + "name": "loki", + "label": "Loki", + "type": "datasource", + "query": "loki", + "regex": "/HISPEC/i", + "hide": 0, + "current": {} + }, + { + "name": "tempo", + "label": "Tempo", + "type": "datasource", + "query": "tempo", + "regex": "/HISPEC/i", + "hide": 0, + "current": {} + }, + { + "name": "host", + "label": "Host", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "query": { + "query": "label_values(system_cpu_load_average_1m{job=\"hispec\"}, host_name)", + "refId": "host" + }, + "definition": "label_values(system_cpu_load_average_1m{job=\"hispec\"}, host_name)", + "refresh": 2, + "includeAll": true, + "multi": true, + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1, + "hide": 0 + }, + { + "name": "group", + "label": "Group", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "query": { + "query": "label_values(rpc_server_call_duration_seconds_count{host_name=~\"$host\"}, service_namespace)", + "refId": "group" + }, + "definition": "label_values(rpc_server_call_duration_seconds_count{host_name=~\"$host\"}, service_namespace)", + "refresh": 2, + "includeAll": true, + "multi": true, + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1, + "hide": 0, + "allValue": ".*" + }, + { + "name": "daemon", + "label": "Daemon", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "query": { + "query": "label_values(rpc_server_call_duration_seconds_count{host_name=~\"$host\", service_namespace=~\"$group\"}, job)", + "refId": "daemon" + }, + "definition": "label_values(rpc_server_call_duration_seconds_count{host_name=~\"$host\", service_namespace=~\"$group\"}, job)", + "refresh": 2, + "includeAll": true, + "multi": true, + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1, + "hide": 0, + "allValue": ".*" + }, + { + "name": "search", + "label": "Log search", + "type": "textbox", + "query": "", + "current": { + "text": "", + "value": "" + }, + "hide": 0 + } + ] + }, + "panels": [ + { + "type": "row", + "title": "Overview", + "collapsed": false, + "panels": [], + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 1 + }, + "id": 1 + }, + { + "type": "stat", + "title": "Requests/s", + "description": "Requests the selected daemons handled, keywords and keys.* alike.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "sum(rate(rpc_server_call_duration_seconds_count{host_name=~\"$host\", service_namespace=~\"$group\", job=~\"$daemon\"}[$__rate_interval]))", + "legendFormat": "", + "refId": "A", + "interval": "1m" + } + ], + "gridPos": { + "x": 0, + "y": 1, + "w": 6, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "area" + }, + "id": 2 + }, + { + "type": "stat", + "title": "Failed requests", + "description": "Requests that ended in an error: a refused value, a raised exception, or an error response.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "sum(rate(rpc_server_call_duration_seconds_count{host_name=~\"$host\", service_namespace=~\"$group\", job=~\"$daemon\", error_type!=\"\"}[$__rate_interval])) or vector(0)", + "legendFormat": "", + "refId": "A", + "interval": "1m" + } + ], + "gridPos": { + "x": 6, + "y": 1, + "w": 6, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.001 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "area" + }, + "id": 3 + }, + { + "type": "stat", + "title": "p95 latency", + "description": "95th percentile time a daemon took to handle a request.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(rpc_server_call_duration_seconds_bucket{host_name=~\"$host\", service_namespace=~\"$group\", job=~\"$daemon\"}[$__rate_interval])))", + "legendFormat": "", + "refId": "A", + "interval": "1m" + } + ], + "gridPos": { + "x": 12, + "y": 1, + "w": 6, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "area" + }, + "id": 4 + }, + { + "type": "stat", + "title": "Daemons reporting", + "description": "Daemons that exported request metrics in the last 5 minutes.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "count(count by (job) (rpc_server_call_duration_seconds_count{host_name=~\"$host\", service_namespace=~\"$group\", job=~\"$daemon\"}))", + "legendFormat": "", + "refId": "A", + "interval": "1m" + } + ], + "gridPos": { + "x": 18, + "y": 1, + "w": 6, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "area" + }, + "id": 5 + }, + { + "type": "row", + "title": "Requests by daemon", + "collapsed": false, + "panels": [], + "gridPos": { + "x": 0, + "y": 5, + "w": 24, + "h": 1 + }, + "id": 6 + }, + { + "type": "timeseries", + "title": "Requests/s", + "description": "Every request each daemon handled.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "label_replace(sum by (job) (rate(rpc_server_call_duration_seconds_count{host_name=~\"$host\", service_namespace=~\"$group\", job=~\"$daemon\"}[$__rate_interval])), \"daemon\", \"$1\", \"job\", \".*/(.*)\")", + "legendFormat": "{{daemon}}", + "refId": "A", + "interval": "1m" + } + ], + "gridPos": { + "x": 0, + "y": 6, + "w": 8, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 7 + }, + { + "type": "timeseries", + "title": "Failures/s", + "description": "Failed requests by daemon and error type.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "label_replace(sum by (job, error_type) (rate(rpc_server_call_duration_seconds_count{host_name=~\"$host\", service_namespace=~\"$group\", job=~\"$daemon\", error_type!=\"\"}[$__rate_interval])), \"daemon\", \"$1\", \"job\", \".*/(.*)\")", + "legendFormat": "{{daemon}} {{error_type}}", + "refId": "A", + "interval": "1m" + } + ], + "gridPos": { + "x": 8, + "y": 6, + "w": 8, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 8 + }, + { + "type": "timeseries", + "title": "p95 latency", + "description": "95th percentile handling time per daemon.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "label_replace(histogram_quantile(0.95, sum by (job, le) (rate(rpc_server_call_duration_seconds_bucket{host_name=~\"$host\", service_namespace=~\"$group\", job=~\"$daemon\"}[$__rate_interval]))), \"daemon\", \"$1\", \"job\", \".*/(.*)\")", + "legendFormat": "{{daemon}}", + "refId": "A", + "interval": "1m" + } + ], + "gridPos": { + "x": 16, + "y": 6, + "w": 8, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 9 + }, + { + "type": "row", + "title": "Keywords", + "collapsed": false, + "panels": [], + "gridPos": { + "x": 0, + "y": 14, + "w": 24, + "h": 1 + }, + "id": 10 + }, + { + "type": "table", + "title": "Busiest keywords", + "description": "Keywords shown or modified most often in the selected range; keys.* services left out.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "label_replace(topk(25, round(sum by (job, rpc_method) (increase(rpc_server_call_duration_seconds_count{host_name=~\"$host\", service_namespace=~\"$group\", job=~\"$daemon\", rpc_method!~\"keys\\\\..*\"}[$__range]))) > 0), \"daemon\", \"$1\", \"job\", \".*/(.*)\")", + "legendFormat": "", + "refId": "A", + "interval": "1m", + "instant": true, + "range": false, + "format": "table" + } + ], + "gridPos": { + "x": 0, + "y": 15, + "w": 12, + "h": 9 + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {}, + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "job": true + }, + "renameByName": { + "daemon": "Daemon", + "rpc_method": "Keyword", + "Value": "Requests" + } + } + } + ], + "id": 11 + }, + { + "type": "table", + "title": "Failing keywords", + "description": "Keywords whose requests failed in the selected range, and how.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "label_replace(topk(25, round(sum by (job, rpc_method, error_type) (increase(rpc_server_call_duration_seconds_count{host_name=~\"$host\", service_namespace=~\"$group\", job=~\"$daemon\", error_type!=\"\"}[$__range]))) > 0), \"daemon\", \"$1\", \"job\", \".*/(.*)\")", + "legendFormat": "", + "refId": "A", + "interval": "1m", + "instant": true, + "range": false, + "format": "table" + } + ], + "gridPos": { + "x": 12, + "y": 15, + "w": 12, + "h": 9 + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {}, + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "job": true + }, + "renameByName": { + "daemon": "Daemon", + "rpc_method": "Keyword", + "error_type": "Error", + "Value": "Failures" + } + } + } + ], + "id": 12 + }, + { + "type": "row", + "title": "Traces", + "collapsed": false, + "panels": [], + "gridPos": { + "x": 0, + "y": 24, + "w": 24, + "h": 1 + }, + "id": 13 + }, + { + "type": "nodeGraph", + "title": "Service map", + "description": "Who calls whom, from Tempo's service graph metrics. Needs the Tempo datasource's service graph set to this Prometheus.", + "datasource": { + "type": "tempo", + "uid": "${tempo}" + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${tempo}" + }, + "queryType": "serviceMap", + "refId": "A" + } + ], + "gridPos": { + "x": 0, + "y": 25, + "w": 24, + "h": 12 + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {}, + "id": 14 + }, + { + "type": "table", + "title": "Recent failed requests", + "description": "Traces with a failed span; open one to see where it failed and the log lines written during it.", + "datasource": { + "type": "tempo", + "uid": "${tempo}" + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${tempo}" + }, + "queryType": "traceql", + "query": "{ status = error && resource.host.name =~ \"${host:regex}\" }", + "limit": 20, + "tableType": "traces", + "refId": "A" + } + ], + "gridPos": { + "x": 0, + "y": 37, + "w": 12, + "h": 9 + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {}, + "id": 15 + }, + { + "type": "table", + "title": "Slow requests", + "description": "Requests a daemon took over half a second to handle.", + "datasource": { + "type": "tempo", + "uid": "${tempo}" + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${tempo}" + }, + "queryType": "traceql", + "query": "{ kind = server && duration > 500ms && resource.host.name =~ \"${host:regex}\" }", + "limit": 20, + "tableType": "traces", + "refId": "A" + } + ], + "gridPos": { + "x": 12, + "y": 37, + "w": 12, + "h": 9 + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {}, + "id": 16 + }, + { + "type": "row", + "title": "Logs", + "collapsed": false, + "panels": [], + "gridPos": { + "x": 0, + "y": 46, + "w": 24, + "h": 1 + }, + "id": 17 + }, + { + "type": "timeseries", + "title": "Errors logged", + "description": "ERROR lines per unit, from the journal.", + "datasource": { + "type": "loki", + "uid": "${loki}" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${loki}" + }, + "refId": "A", + "legendFormat": "{{systemd_unit}}", + "expr": "sum by (systemd_unit) (count_over_time({service_name=\"hispec\"} | detected_level=\"error\" [$__interval]))" + } + ], + "gridPos": { + "x": 0, + "y": 47, + "w": 24, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 18 + }, + { + "type": "logs", + "title": "Daemon logs", + "description": "Journal lines from every daemon; a trace_id in a line links to its trace.", + "datasource": { + "type": "loki", + "uid": "${loki}" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${loki}" + }, + "refId": "A", + "expr": "{service_name=\"hispec\"} | systemd_unit=~\"hispec@.*\" |= \"$search\"" + } + ], + "gridPos": { + "x": 0, + "y": 55, + "w": 24, + "h": 12 + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "showTime": true, + "wrapLogMessage": true, + "sortOrder": "Descending", + "enableLogDetails": true + }, + "id": 19 + }, + { + "type": "row", + "title": "Host", + "collapsed": false, + "panels": [], + "gridPos": { + "x": 0, + "y": 67, + "w": 24, + "h": 1 + }, + "id": 20 + }, + { + "type": "timeseries", + "title": "Load average", + "description": "System load, 1 and 15 minute averages.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "system_cpu_load_average_1m{job=\"hispec\", host_name=~\"$host\"}", + "legendFormat": "{{host_name}} 1m", + "refId": "A", + "interval": "1m" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "system_cpu_load_average_15m{job=\"hispec\", host_name=~\"$host\"}", + "legendFormat": "{{host_name}} 15m", + "refId": "B", + "interval": "1m" + } + ], + "gridPos": { + "x": 0, + "y": 68, + "w": 6, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 21 + }, + { + "type": "timeseries", + "title": "Memory used", + "description": "Share of memory in use.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "sum by (host_name) (system_memory_usage_bytes{job=\"hispec\", host_name=~\"$host\", state=\"used\"}) / sum by (host_name) (system_memory_usage_bytes{job=\"hispec\", host_name=~\"$host\"})", + "legendFormat": "{{host_name}}", + "refId": "A", + "interval": "1m" + } + ], + "gridPos": { + "x": 6, + "y": 68, + "w": 6, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 22 + }, + { + "type": "timeseries", + "title": "Disk used", + "description": "Share of each filesystem in use; snap images left out.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "sum by (host_name, mountpoint) (system_filesystem_usage_bytes{job=\"hispec\", host_name=~\"$host\", state=\"used\", type!=\"squashfs\"}) / sum by (host_name, mountpoint) (system_filesystem_usage_bytes{job=\"hispec\", host_name=~\"$host\", type!=\"squashfs\"})", + "legendFormat": "{{host_name}} {{mountpoint}}", + "refId": "A", + "interval": "1m" + } + ], + "gridPos": { + "x": 12, + "y": 68, + "w": 6, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 23 + }, + { + "type": "timeseries", + "title": "Network", + "description": "Bytes in and out, loopback left out.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "sum by (host_name, direction) (rate(system_network_io_bytes_total{job=\"hispec\", host_name=~\"$host\", device!=\"lo\"}[$__rate_interval]))", + "legendFormat": "{{host_name}} {{direction}}", + "refId": "A", + "interval": "1m" + } + ], + "gridPos": { + "x": 18, + "y": 68, + "w": 6, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "Bps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 24 + } + ] +}