diff --git a/config/common/keygrabber.yaml b/config/common/keygrabber.yaml index 485f838..68ac131 100644 --- a/config/common/keygrabber.yaml +++ b/config/common/keygrabber.yaml @@ -1,9 +1,10 @@ -# Keyword archiver: polls the LRIS2 daemons and writes to InfluxDB for Grafana +# Keyword archiver: polls the LRIS2 daemons and writes to InfluxDB and Prometheus for Grafana # # Usage: # daemons/keygrabber -c config/common/keygrabber.yaml # # The InfluxDB token comes from the environment variable named in token_env. +# The prometheus sink exports to OTEL_EXPORTER_OTLP_ENDPOINT from /etc/lris2/otel.env. peer_id: keygrabber group_id: lris2 @@ -15,6 +16,8 @@ sinks: org: caltech bucket: LRIS2 token_env: LRIS2_INFLUX_TOKEN + prometheus: + type: otlp # Reads run concurrently across peers; a single daemon still answers one at a time workers: 4 diff --git a/grafana/lris2-telemetry-dashboard.json b/grafana/lris2-telemetry-dashboard.json new file mode 100644 index 0000000..eabd039 --- /dev/null +++ b/grafana/lris2-telemetry-dashboard.json @@ -0,0 +1,1030 @@ +{ + "uid": "lris2-telemetry", + "title": "LRIS2 telemetry", + "description": "Requests, traces, logs and host metrics from the LRIS2 daemons, through libby's OpenTelemetry export.", + "tags": [ + "lris2", + "libby", + "opentelemetry" + ], + "timezone": "browser", + "refresh": "1m", + "time": { + "from": "now-6h", + "to": "now" + }, + "graphTooltip": 1, + "schemaVersion": 39, + "editable": true, + "templating": { + "list": [ + { + "name": "prometheus", + "label": "Prometheus", + "type": "datasource", + "query": "prometheus", + "regex": "/LRIS2/i", + "hide": 0, + "current": {} + }, + { + "name": "loki", + "label": "Loki", + "type": "datasource", + "query": "loki", + "regex": "/LRIS2/i", + "hide": 0, + "current": {} + }, + { + "name": "tempo", + "label": "Tempo", + "type": "datasource", + "query": "tempo", + "regex": "/LRIS2/i", + "hide": 0, + "current": {} + }, + { + "name": "host", + "label": "Host", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "query": { + "query": "label_values(system_cpu_load_average_1m{job=\"lris2\"}, host_name)", + "refId": "host" + }, + "definition": "label_values(system_cpu_load_average_1m{job=\"lris2\"}, host_name)", + "refresh": 2, + "includeAll": true, + "multi": true, + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1, + "hide": 0 + }, + { + "name": "group", + "label": "Group", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "query": { + "query": "label_values(rpc_server_call_duration_seconds_count{host_name=~\"$host\"}, service_namespace)", + "refId": "group" + }, + "definition": "label_values(rpc_server_call_duration_seconds_count{host_name=~\"$host\"}, service_namespace)", + "refresh": 2, + "includeAll": true, + "multi": true, + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1, + "hide": 0, + "allValue": ".*" + }, + { + "name": "daemon", + "label": "Daemon", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "query": { + "query": "label_values(rpc_server_call_duration_seconds_count{host_name=~\"$host\", service_namespace=~\"$group\"}, job)", + "refId": "daemon" + }, + "definition": "label_values(rpc_server_call_duration_seconds_count{host_name=~\"$host\", service_namespace=~\"$group\"}, job)", + "refresh": 2, + "includeAll": true, + "multi": true, + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1, + "hide": 0, + "allValue": ".*" + }, + { + "name": "search", + "label": "Log search", + "type": "textbox", + "query": "", + "current": { + "text": "", + "value": "" + }, + "hide": 0 + } + ] + }, + "panels": [ + { + "type": "row", + "title": "Overview", + "collapsed": false, + "panels": [], + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 1 + }, + "id": 1 + }, + { + "type": "stat", + "title": "Requests/s", + "description": "Requests the selected daemons handled, keywords and keys.* alike.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "sum(rate(rpc_server_call_duration_seconds_count{host_name=~\"$host\", service_namespace=~\"$group\", job=~\"$daemon\"}[$__rate_interval]))", + "legendFormat": "", + "refId": "A", + "interval": "1m" + } + ], + "gridPos": { + "x": 0, + "y": 1, + "w": 6, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "area" + }, + "id": 2 + }, + { + "type": "stat", + "title": "Failed requests", + "description": "Requests that ended in an error: a refused value, a raised exception, or an error response.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "sum(rate(rpc_server_call_duration_seconds_count{host_name=~\"$host\", service_namespace=~\"$group\", job=~\"$daemon\", error_type!=\"\"}[$__rate_interval])) or vector(0)", + "legendFormat": "", + "refId": "A", + "interval": "1m" + } + ], + "gridPos": { + "x": 6, + "y": 1, + "w": 6, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.001 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "area" + }, + "id": 3 + }, + { + "type": "stat", + "title": "p95 latency", + "description": "95th percentile time a daemon took to handle a request.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(rpc_server_call_duration_seconds_bucket{host_name=~\"$host\", service_namespace=~\"$group\", job=~\"$daemon\"}[$__rate_interval])))", + "legendFormat": "", + "refId": "A", + "interval": "1m" + } + ], + "gridPos": { + "x": 12, + "y": 1, + "w": 6, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "area" + }, + "id": 4 + }, + { + "type": "stat", + "title": "Daemons reporting", + "description": "Daemons that exported request metrics in the last 5 minutes.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "count(count by (job) (rpc_server_call_duration_seconds_count{host_name=~\"$host\", service_namespace=~\"$group\", job=~\"$daemon\"}))", + "legendFormat": "", + "refId": "A", + "interval": "1m" + } + ], + "gridPos": { + "x": 18, + "y": 1, + "w": 6, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "area" + }, + "id": 5 + }, + { + "type": "row", + "title": "Requests by daemon", + "collapsed": false, + "panels": [], + "gridPos": { + "x": 0, + "y": 5, + "w": 24, + "h": 1 + }, + "id": 6 + }, + { + "type": "timeseries", + "title": "Requests/s", + "description": "Every request each daemon handled.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "label_replace(sum by (job) (rate(rpc_server_call_duration_seconds_count{host_name=~\"$host\", service_namespace=~\"$group\", job=~\"$daemon\"}[$__rate_interval])), \"daemon\", \"$1\", \"job\", \".*/(.*)\")", + "legendFormat": "{{daemon}}", + "refId": "A", + "interval": "1m" + } + ], + "gridPos": { + "x": 0, + "y": 6, + "w": 8, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 7 + }, + { + "type": "timeseries", + "title": "Failures/s", + "description": "Failed requests by daemon and error type.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "label_replace(sum by (job, error_type) (rate(rpc_server_call_duration_seconds_count{host_name=~\"$host\", service_namespace=~\"$group\", job=~\"$daemon\", error_type!=\"\"}[$__rate_interval])), \"daemon\", \"$1\", \"job\", \".*/(.*)\")", + "legendFormat": "{{daemon}} {{error_type}}", + "refId": "A", + "interval": "1m" + } + ], + "gridPos": { + "x": 8, + "y": 6, + "w": 8, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 8 + }, + { + "type": "timeseries", + "title": "p95 latency", + "description": "95th percentile handling time per daemon.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "label_replace(histogram_quantile(0.95, sum by (job, le) (rate(rpc_server_call_duration_seconds_bucket{host_name=~\"$host\", service_namespace=~\"$group\", job=~\"$daemon\"}[$__rate_interval]))), \"daemon\", \"$1\", \"job\", \".*/(.*)\")", + "legendFormat": "{{daemon}}", + "refId": "A", + "interval": "1m" + } + ], + "gridPos": { + "x": 16, + "y": 6, + "w": 8, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 9 + }, + { + "type": "row", + "title": "Keywords", + "collapsed": false, + "panels": [], + "gridPos": { + "x": 0, + "y": 14, + "w": 24, + "h": 1 + }, + "id": 10 + }, + { + "type": "table", + "title": "Busiest keywords", + "description": "Keywords shown or modified most often in the selected range; keys.* services left out.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "label_replace(topk(25, round(sum by (job, rpc_method) (increase(rpc_server_call_duration_seconds_count{host_name=~\"$host\", service_namespace=~\"$group\", job=~\"$daemon\", rpc_method!~\"keys\\\\..*\"}[$__range]))) > 0), \"daemon\", \"$1\", \"job\", \".*/(.*)\")", + "legendFormat": "", + "refId": "A", + "interval": "1m", + "instant": true, + "range": false, + "format": "table" + } + ], + "gridPos": { + "x": 0, + "y": 15, + "w": 12, + "h": 9 + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {}, + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "job": true + }, + "renameByName": { + "daemon": "Daemon", + "rpc_method": "Keyword", + "Value": "Requests" + } + } + } + ], + "id": 11 + }, + { + "type": "table", + "title": "Failing keywords", + "description": "Keywords whose requests failed in the selected range, and how.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "label_replace(topk(25, round(sum by (job, rpc_method, error_type) (increase(rpc_server_call_duration_seconds_count{host_name=~\"$host\", service_namespace=~\"$group\", job=~\"$daemon\", error_type!=\"\"}[$__range]))) > 0), \"daemon\", \"$1\", \"job\", \".*/(.*)\")", + "legendFormat": "", + "refId": "A", + "interval": "1m", + "instant": true, + "range": false, + "format": "table" + } + ], + "gridPos": { + "x": 12, + "y": 15, + "w": 12, + "h": 9 + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {}, + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "job": true + }, + "renameByName": { + "daemon": "Daemon", + "rpc_method": "Keyword", + "error_type": "Error", + "Value": "Failures" + } + } + } + ], + "id": 12 + }, + { + "type": "row", + "title": "Traces", + "collapsed": false, + "panels": [], + "gridPos": { + "x": 0, + "y": 24, + "w": 24, + "h": 1 + }, + "id": 13 + }, + { + "type": "nodeGraph", + "title": "Service map", + "description": "Who calls whom, from Tempo's service graph metrics. Needs the Tempo datasource's service graph set to this Prometheus.", + "datasource": { + "type": "tempo", + "uid": "${tempo}" + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${tempo}" + }, + "queryType": "serviceMap", + "refId": "A" + } + ], + "gridPos": { + "x": 0, + "y": 25, + "w": 24, + "h": 12 + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {}, + "id": 14 + }, + { + "type": "table", + "title": "Recent failed requests", + "description": "Traces with a failed span; open one to see where it failed and the log lines written during it.", + "datasource": { + "type": "tempo", + "uid": "${tempo}" + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${tempo}" + }, + "queryType": "traceql", + "query": "{ status = error && resource.host.name =~ \"${host:regex}\" }", + "limit": 20, + "tableType": "traces", + "refId": "A" + } + ], + "gridPos": { + "x": 0, + "y": 37, + "w": 12, + "h": 9 + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {}, + "id": 15 + }, + { + "type": "table", + "title": "Slow requests", + "description": "Requests a daemon took over half a second to handle.", + "datasource": { + "type": "tempo", + "uid": "${tempo}" + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${tempo}" + }, + "queryType": "traceql", + "query": "{ kind = server && duration > 500ms && resource.host.name =~ \"${host:regex}\" }", + "limit": 20, + "tableType": "traces", + "refId": "A" + } + ], + "gridPos": { + "x": 12, + "y": 37, + "w": 12, + "h": 9 + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {}, + "id": 16 + }, + { + "type": "row", + "title": "Logs", + "collapsed": false, + "panels": [], + "gridPos": { + "x": 0, + "y": 46, + "w": 24, + "h": 1 + }, + "id": 17 + }, + { + "type": "timeseries", + "title": "Errors logged", + "description": "ERROR lines per unit, from the journal.", + "datasource": { + "type": "loki", + "uid": "${loki}" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${loki}" + }, + "refId": "A", + "legendFormat": "{{systemd_unit}}", + "expr": "sum by (systemd_unit) (count_over_time({service_name=\"lris2\"} | detected_level=\"error\" [$__interval]))" + } + ], + "gridPos": { + "x": 0, + "y": 47, + "w": 24, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 18 + }, + { + "type": "logs", + "title": "Daemon logs", + "description": "Journal lines from every daemon; a trace_id in a line links to its trace.", + "datasource": { + "type": "loki", + "uid": "${loki}" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${loki}" + }, + "refId": "A", + "expr": "{service_name=\"lris2\"} | systemd_unit=~\"lris2@.*\" |= \"$search\"" + } + ], + "gridPos": { + "x": 0, + "y": 55, + "w": 24, + "h": 12 + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "showTime": true, + "wrapLogMessage": true, + "sortOrder": "Descending", + "enableLogDetails": true + }, + "id": 19 + }, + { + "type": "row", + "title": "Host", + "collapsed": false, + "panels": [], + "gridPos": { + "x": 0, + "y": 67, + "w": 24, + "h": 1 + }, + "id": 20 + }, + { + "type": "timeseries", + "title": "Load average", + "description": "System load, 1 and 15 minute averages.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "system_cpu_load_average_1m{job=\"lris2\", host_name=~\"$host\"}", + "legendFormat": "{{host_name}} 1m", + "refId": "A", + "interval": "1m" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "system_cpu_load_average_15m{job=\"lris2\", host_name=~\"$host\"}", + "legendFormat": "{{host_name}} 15m", + "refId": "B", + "interval": "1m" + } + ], + "gridPos": { + "x": 0, + "y": 68, + "w": 6, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 21 + }, + { + "type": "timeseries", + "title": "Memory used", + "description": "Share of memory in use.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "sum by (host_name) (system_memory_usage_bytes{job=\"lris2\", host_name=~\"$host\", state=\"used\"}) / sum by (host_name) (system_memory_usage_bytes{job=\"lris2\", host_name=~\"$host\"})", + "legendFormat": "{{host_name}}", + "refId": "A", + "interval": "1m" + } + ], + "gridPos": { + "x": 6, + "y": 68, + "w": 6, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 22 + }, + { + "type": "timeseries", + "title": "Disk used", + "description": "Share of each filesystem in use; snap images left out.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "sum by (host_name, mountpoint) (system_filesystem_usage_bytes{job=\"lris2\", host_name=~\"$host\", state=\"used\", type!=\"squashfs\"}) / sum by (host_name, mountpoint) (system_filesystem_usage_bytes{job=\"lris2\", host_name=~\"$host\", type!=\"squashfs\"})", + "legendFormat": "{{host_name}} {{mountpoint}}", + "refId": "A", + "interval": "1m" + } + ], + "gridPos": { + "x": 12, + "y": 68, + "w": 6, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 23 + }, + { + "type": "timeseries", + "title": "Network", + "description": "Bytes in and out, loopback left out.", + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "expr": "sum by (host_name, direction) (rate(system_network_io_bytes_total{job=\"lris2\", host_name=~\"$host\", device!=\"lo\"}[$__rate_interval]))", + "legendFormat": "{{host_name}} {{direction}}", + "refId": "A", + "interval": "1m" + } + ], + "gridPos": { + "x": 18, + "y": 68, + "w": 6, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "Bps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 24 + } + ] +} diff --git a/pyproject.toml b/pyproject.toml index 8fc5c00..42a665e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,8 +15,9 @@ readme = "README.md" dependencies = [ # dashboard extra: lris2-dashboard.service "instrumentctl[dashboard]@git+https://github.com/CaltechOpticalObservatories/instrumentctl.git", - # influxdb extra: needed by daemons/keygrabber - "libby[influxdb]@git+https://github.com/CaltechOpticalObservatories/libby.git", + # influxdb extra: the keygrabber's InfluxDB sink; otel: traces, metrics and + # the keygrabber's Prometheus sink, exported when /etc/lris2/otel.env is set + "libby[influxdb,otel]@git+https://github.com/CaltechOpticalObservatories/libby.git", "pyyaml", ] diff --git a/src/cli/instrument.toml b/src/cli/instrument.toml index 466ca69..87387cc 100644 --- a/src/cli/instrument.toml +++ b/src/cli/instrument.toml @@ -7,3 +7,9 @@ env_prefix = "LRIS2" [help] example_instance = "rpress" example_subsystem = "r" + +# The lris2 host role runs the telemetry backend; each host's /etc/lris2/otel.env points there +[telemetry] +backend_host = "lris2" +# Grafana reads Prometheus, Tempo and Loki from another host +query_bind = "0.0.0.0" diff --git a/systemd/instances/keygrabber.env b/systemd/instances/keygrabber.env index dc737c1..e80e4ff 100644 --- a/systemd/instances/keygrabber.env +++ b/systemd/instances/keygrabber.env @@ -1,4 +1,4 @@ -# Keyword archiver: polls the LRIS2 daemons into InfluxDB for Grafana +# Keyword archiver: polls the LRIS2 daemons into InfluxDB and Prometheus for Grafana # # lris2 deploy keygrabber #