diff --git a/.gitattributes b/.gitattributes index 11a48c1820..a3750227c1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -11,3 +11,6 @@ crates/openshell-core/src/proto/openshell.*.rs linguist-generated # Vendored OCSF schemas fetched from schema.ocsf.io crates/openshell-ocsf-validate/schemas/v*/** linguist-generated + +# Grafana dashboard JSON templates consumed by the monitoring mixin +deploy/openshell-mixin/dashboards/*.json linguist-generated diff --git a/.github/workflows/branch-checks.yml b/.github/workflows/branch-checks.yml index db22a74913..abe5b9c781 100644 --- a/.github/workflows/branch-checks.yml +++ b/.github/workflows/branch-checks.yml @@ -133,6 +133,63 @@ jobs: || github.event.merge_group.base_sha || needs.pr_metadata.outputs.base_sha }} + monitoring-changes: + name: Detect monitoring mixin changes + needs: pr_metadata + if: needs.pr_metadata.outputs.should_run == 'true' + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + changed: ${{ steps.changes.outputs.changed }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Check mixin paths + id: changes + shell: bash + env: + BASE_SHA: ${{ github.event.merge_group.base_sha || needs.pr_metadata.outputs.base_sha }} + EVENT_NAME: ${{ github.event_name }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" = workflow_dispatch ]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + if [ -z "$BASE_SHA" ]; then + echo "::error::Missing base SHA for monitoring mixin change detection." >&2 + exit 1 + fi + changed=$(git diff --name-only "${BASE_SHA}...HEAD" -- deploy/openshell-mixin/) + if [ -n "$changed" ]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + else + echo "changed=false" >> "$GITHUB_OUTPUT" + fi + + monitoring: + name: Monitoring Mixin + needs: [pr_metadata, monitoring-changes] + if: needs.pr_metadata.outputs.should_run == 'true' && needs.monitoring-changes.outputs.changed == 'true' + runs-on: linux-amd64-cpu8 + timeout-minutes: 15 + container: + image: ghcr.io/nvidia/openshell/ci:9cb72baa2e61a1b5f12407e6e82da7fdba0aa722@sha256:67a9a0c32cb99825e6d3e9d9eec45d67149ea1ff7c11a1b1e1b3480d2d3df684 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Install monitoring prerequisites + run: mise install --locked go python uv + + - name: Check monitoring mixin + run: mise run monitoring:check + mise-lockfile: name: mise Lockfile needs: pr_metadata diff --git a/CI.md b/CI.md index a1a1b8bc77..184b5e2021 100644 --- a/CI.md +++ b/CI.md @@ -17,6 +17,14 @@ Manual admission does not change the bot's automatic trust policy for ready PRs. Merge queue validation is a second integration gate for `main`. After a PR has passed the required PR-head statuses, a maintainer adds it to the merge queue. GitHub creates a temporary merge-group branch that combines the latest `main`, the queued PR, and any earlier queued PRs. The same required `OpenShell / ...` status contexts are then published against the merge-group SHA before GitHub merges it. +### Monitoring dashboards + +`Monitoring Mixin` runs in Branch Checks when files under +`deploy/openshell-mixin/` change relative to the PR or merge queue base. It skips +unrelated changes. Manual Branch Checks runs always include this check. +The job runs `mise run monitoring:check` to validate formatting, run the +structural tests, and render the dashboards. Use the same command locally. + ### Protobuf API compatibility `Protobuf Compatibility` runs in Branch Checks and Pre-release through the shared diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3e800debdf..c133e7ad11 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -373,6 +373,8 @@ These are the primary `mise` tasks for day-to-day development: | `mise run ci` | Full local CI checks (lint, compile/type checks, tests) | | `mise run docs` | Validate Fern docs locally | | `mise run helm:docs` | Regenerate the Helm chart README | +| `mise run monitoring:generate` | Render OpenShell Grafana dashboards | +| `mise run monitoring:check` | Check and render the monitoring mixin | | `mise run clean` | Clean build artifacts | ## Project Structure diff --git a/deploy/openshell-mixin/.gitignore b/deploy/openshell-mixin/.gitignore new file mode 100644 index 0000000000..99b4a7c2ed --- /dev/null +++ b/deploy/openshell-mixin/.gitignore @@ -0,0 +1,4 @@ +build/ +.tools/ +__pycache__/ +vendor/ diff --git a/deploy/openshell-mixin/README.md b/deploy/openshell-mixin/README.md new file mode 100644 index 0000000000..eb37b76bb1 --- /dev/null +++ b/deploy/openshell-mixin/README.md @@ -0,0 +1,86 @@ +# OpenShell monitoring mixin + +Reusable Grafana dashboards for OpenShell, packaged as a [monitoring mixin](https://monitoring.mixins.dev/). Render the mixin to dashboard JSON and import it into your Grafana instance. Configure collection and datasources in your own monitoring environment. + +## Dashboards + +| Dashboard | Datasources | Purpose | +| --- | --- | --- | +| OpenShell operational health | Prometheus; optional Tempo | Gateway readiness, RPC/HTTP traffic and latency, supervisor sessions, routing, relay pressure, and mutation locks | +| OpenShell operational logs | Loki | Gateway, supervisor, and workload diagnostics | +| OpenShell OCSF logs | Loki | Structured security and audit events from gateways and supervisors | + +Dashboard UIDs and panel IDs remain stable across renders. Datasource variables are visible and independently selectable. Credentials, tenant headers, datasource provisioning, and Grafana folders belong in the consuming environment. + +## Render and import + +From the repository root, render the dashboards with mise: + +```shell +mise run monitoring:generate +``` + +The tasks declare Go Jsonnet and jsonnetfmt **v0.22.0** as task-specific tools; mise installs them when needed. + +Import the JSON files from `deploy/openshell-mixin/build/dashboards/` through Grafana's dashboard import screen, or use your existing Grafana file provisioning, API, or Kubernetes sidecar workflow. Select the appropriate metrics, logs, and optional traces datasource in each dashboard, then choose the cluster and gateway instance. + +Files under `dashboards/` are source templates containing `__OS_*__` tokens; import the rendered files from `build/dashboards/`. + +The standard mixin exports are `grafanaDashboards`, `prometheusAlerts`, and `prometheusRules`. The two rule exports contain empty groups; no alerts or recording rules are defined. `monitoring:generate` writes their YAML alongside the dashboard directory, but they need not be installed. + +## Customize + +Defaults live in [config.libsonnet](config.libsonnet). Override `_config.openshell` in a consumer Jsonnet file. [examples/custom.jsonnet](examples/custom.jsonnet) demonstrates a different metric selector and cluster label, namespace regex, disabled traces, and datasource name filters: + +```shell +MIXIN=examples/custom.jsonnet OUT=build/custom mise run monitoring:generate +``` + +`MIXIN` and `OUT` are relative to `deploy/openshell-mixin`; `OUT` also accepts an absolute path. + +- `metricsSelector` contains nonempty PromQL label matchers without braces. +- Metric and log label settings let you match your collection pipeline. Label names and TraceQL attribute names must be valid identifiers for their query languages. +- Regex values are escaped before insertion into queries. Datasource regexes match Grafana datasource names, not UIDs. +- Set `tracesEnabled: false` when Tempo is unavailable. This removes the trace panel and its datasource variable. + +The package has no Jsonnet dependencies. After it is published, consumers can pin its subtree with jsonnet-bundler v0.6.0: + +```shell +jb init +jb install github.com/NVIDIA/OpenShell/deploy/openshell-mixin@COMMIT_SHA +``` + +Replace `COMMIT_SHA` with a published immutable revision containing this package. Import `github.com/NVIDIA/OpenShell/deploy/openshell-mixin/mixin.libsonnet` from the consumer's vendor directory and compose its exports into your existing monitoring resources, including kube-prometheus ConfigMaps where applicable. + +## Collection requirements + +| Signal | Required collection contract | +| --- | --- | +| Metrics | OpenShell gateway metrics, with default scrape job `openshell-gateway` and `cluster` / `instance` labels. Cluster discovery uses `openshell_server_readiness_database_healthy`. See the [gateway metrics reference](../../docs/observability/gateway-metrics.mdx) for emitted families. | +| Logs | Loki labels `k8s_cluster_name`, `k8s_namespace_name`, and `service_name`. Preserve `k8s_pod_name`, `log_iostream`, and `log_kind` as queryable metadata. Operational service names are `openshell-gateway`, `supervisor`, and `agent`. | +| OCSF | Preserve the event JSON body and supply `log_kind="ocsf"`, `sandbox_name`, `sandbox_id`, `ocsf_class_uid`, and `ocsf_severity_id` as queryable metadata when available. Collect events from both the gateway and supervisors. | +| Traces | Tempo spans with `resource.service.name="openshell-gateway"` and `resource.k8s.cluster.name`. Cluster values must match the metric cluster labels. Service and cluster attribute settings can be overridden. | + +The default namespace regex is `openshell.*`. Map collection-specific labels, metadata, and service names to this contract or adjust the mixin configuration. Use consistent cluster values across metrics, logs, and traces. + +RPC and HTTP summaries show weighted means, not aggregate p95. Streaming RPC completion durations include session lifetime. Mutation-lock histograms support the displayed fleet p95. Routed retries count as separate attempts; supervisor sessions are not sandbox counts. Missing observations remain absent. Threshold colors provide provisional operational guidance, not SLOs. OCSF class IDs include the emitted value `0`. + +## Validate in your environment + +The local checks require the repository's `uv` Python runner and validate Jsonnet formatting, standard exports, dashboard identity, datasource references, portability, and configuration overrides: + +```shell +mise run monitoring:check +``` + +Branch Checks runs `mise run monitoring:check` when files in `deploy/openshell-mixin/` change. Manual Branch Checks runs always include it. + +Users validate queries and visual behavior against their own collection pipeline and Grafana version: + +1. Confirm cluster and instance selectors populate from the intended deployment. +2. Generate representative gateway and sandbox activity, then check readiness, request rates, latency, and supervisor sessions. +3. Confirm operational logs and gateway/supervisor OCSF events appear with the expected metadata filters. +4. If traces are enabled, open a gateway trace from the trace table and confirm the cluster matches. +5. Investigate query errors and missing data before treating a panel as evidence of health. Exercise failure paths separately; an empty error series does not prove those paths work. + +`monitoring:check` checks formatting, runs the structural tests, and renders the default mixin. Use `monitoring:format` to format Jsonnet files, or run `monitoring:format:check` and `monitoring:test` individually. These tasks do not start a telemetry stack or execute live backend queries. diff --git a/deploy/openshell-mixin/config.libsonnet b/deploy/openshell-mixin/config.libsonnet new file mode 100644 index 0000000000..959566a073 --- /dev/null +++ b/deploy/openshell-mixin/config.libsonnet @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: Apache-2.0 +{ + _config+:: { + openshell+: { + // Nonempty PromQL label matchers, without braces; caller supplies syntax. + metricsSelector: 'job="openshell-gateway"', + metricsClusterLabel: 'cluster', + metricsInstanceLabel: 'instance', + logsClusterLabel: 'k8s_cluster_name', + logsNamespaceLabel: 'k8s_namespace_name', + namespaceRegex: 'openshell.*', + tracesEnabled: true, + tracesClusterAttribute: 'resource.k8s.cluster.name', + tracesService: 'openshell-gateway', + // Match datasource names. Empty means all datasources of that type. + datasourceRegex: { metrics: '', logs: '', traces: '' }, + }, + }, +} diff --git a/deploy/openshell-mixin/dashboards.libsonnet b/deploy/openshell-mixin/dashboards.libsonnet new file mode 100644 index 0000000000..5b5c7474ce --- /dev/null +++ b/deploy/openshell-mixin/dashboards.libsonnet @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: Apache-2.0 +{ + local config = self._config.openshell, + local replacements = { + __OS_METRICS_SELECTOR__: config.metricsSelector, + __OS_METRICS_CLUSTER_LABEL__: config.metricsClusterLabel, + __OS_METRICS_INSTANCE_LABEL__: config.metricsInstanceLabel, + __OS_LOGS_CLUSTER_LABEL__: config.logsClusterLabel, + __OS_LOGS_NAMESPACE_LABEL__: config.logsNamespaceLabel, + __OS_NAMESPACE_REGEX__: std.manifestJsonEx(config.namespaceRegex, ''), + __OS_TRACES_CLUSTER_ATTRIBUTE__: config.tracesClusterAttribute, + __OS_TRACES_SERVICE__: std.manifestJsonEx(config.tracesService, ''), + }, + local render(value) = + if std.isArray(value) then [render(v) for v in value] + else if std.isObject(value) then { [k]: render(value[k]) for k in std.objectFields(value) } + else if std.isString(value) then + std.foldl(function(s, token) std.strReplace(s, token, replacements[token]), std.objectFields(replacements), value) + else value, + local dashboard(raw) = + local d = render(raw); + d { + panels: [p for p in d.panels if config.tracesEnabled || (std.get(std.get(p, 'datasource', {}), 'type', '') != 'tempo' && !(p.type == 'row' && p.title == 'Gateway tracing'))], + templating+: { + list: [ + v + (if v.type == 'datasource' then { regex: config.datasourceRegex[v.name] } else {}) + for v in d.templating.list + if config.tracesEnabled || v.name != 'traces' + ], + }, + }, + grafanaDashboards: { + 'openshell-operational-health.json': dashboard(import 'dashboards/openshell-operational-health.json'), + 'openshell-ocsf-logs.json': dashboard(import 'dashboards/openshell-ocsf-logs.json'), + 'openshell-operational-logs.json': dashboard(import 'dashboards/openshell-operational-logs.json'), + }, +} diff --git a/deploy/openshell-mixin/dashboards/openshell-ocsf-logs.json b/deploy/openshell-mixin/dashboards/openshell-ocsf-logs.json new file mode 100644 index 0000000000..68668f32a8 --- /dev/null +++ b/deploy/openshell-mixin/dashboards/openshell-ocsf-logs.json @@ -0,0 +1,422 @@ +{ + "id": null, + "uid": "openshell-ocsf-logs", + "title": "OpenShell / OCSF logs", + "description": "Structured gateway and supervisor security and audit events from Loki. Select a datasource and cluster, then filter by sandbox, event class, or severity. Expand an event to inspect its complete OCSF JSON and metadata. Missing events do not establish health.", + "tags": [ + "openshell", + "ocsf" + ], + "timezone": "browser", + "schemaVersion": 40, + "version": 1, + "editable": true, + "refresh": "30s", + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "annotations": { + "list": [] + }, + "links": [ + { + "type": "dashboards", + "title": "OpenShell dashboards", + "tags": [ + "openshell" + ], + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "targetBlank": false + } + ], + "templating": { + "list": [ + { + "name": "logs", + "label": "Logs source", + "type": "datasource", + "query": "loki", + "regex": "", + "refresh": 1, + "hide": 0, + "options": [] + }, + { + "name": "cluster", + "label": "Cluster", + "type": "query", + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "query": { + "type": 1, + "label": "__OS_LOGS_CLUSTER_LABEL__", + "stream": "{__OS_LOGS_NAMESPACE_LABEL__=~__OS_NAMESPACE_REGEX__}", + "refId": "clusterVariableQuery" + }, + "definition": "label_values({__OS_LOGS_NAMESPACE_LABEL__=~__OS_NAMESPACE_REGEX__}, __OS_LOGS_CLUSTER_LABEL__)", + "refresh": 1, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [] + }, + { + "name": "sandbox", + "label": "Sandbox name (regex)", + "type": "textbox", + "query": ".*", + "current": { + "text": ".*", + "value": ".*" + }, + "options": [ + { + "text": ".*", + "value": ".*", + "selected": true + } + ] + }, + { + "name": "sandbox_id", + "label": "Sandbox ID (regex)", + "type": "textbox", + "query": ".*", + "current": { + "text": ".*", + "value": ".*" + }, + "options": [ + { + "text": ".*", + "value": ".*", + "selected": true + } + ] + }, + { + "name": "class_uid", + "label": "OCSF class UID (regex)", + "type": "textbox", + "query": ".*", + "current": { + "text": ".*", + "value": ".*" + }, + "options": [ + { + "text": ".*", + "value": ".*", + "selected": true + } + ] + }, + { + "name": "severity", + "label": "OCSF severity ID", + "type": "custom", + "query": "0,1,2,3,4,5,6,99", + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [ + { + "text": "0", + "value": "0", + "selected": false + }, + { + "text": "1", + "value": "1", + "selected": false + }, + { + "text": "2", + "value": "2", + "selected": false + }, + { + "text": "3", + "value": "3", + "selected": false + }, + { + "text": "4", + "value": "4", + "selected": false + }, + { + "text": "5", + "value": "5", + "selected": false + }, + { + "text": "6", + "value": "6", + "selected": false + }, + { + "text": "99", + "value": "99", + "selected": false + } + ] + }, + { + "name": "search", + "label": "Message search (regex)", + "type": "textbox", + "query": "", + "current": { + "text": "", + "value": "" + }, + "options": [ + { + "text": "", + "value": "", + "selected": true + } + ] + } + ] + }, + "panels": [ + { + "id": 1, + "title": "Structured audit events", + "type": "text", + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 3 + }, + "options": { + "mode": "markdown", + "content": "View complete OCSF JSON events; expand a log line for sandbox and Kubernetes metadata. Class IDs are preserved as emitted (including legacy class `0`). Severity IDs: 0 unknown, 1 informational, 2 low, 3 medium, 4 high, 5 critical, 6 fatal, 99 other.\n\nSelect the Loki datasource and cluster to inspect. Regex text filters default to `.*`; do not include backticks. Counts cover the selected time range and filters; absent events do not establish health." + } + }, + { + "id": 2, + "title": "OCSF events in selected range", + "type": "stat", + "gridPos": { + "x": 0, + "y": 3, + "w": 6, + "h": 5 + }, + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "expr": "sum(count_over_time({__OS_LOGS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_LOGS_NAMESPACE_LABEL__=~__OS_NAMESPACE_REGEX__} | log_kind=\"ocsf\" | sandbox_name=~`${sandbox:raw}` | sandbox_id=~`${sandbox_id:raw}` | ocsf_class_uid=~`${class_uid:raw}` | ocsf_severity_id=~\"${severity:regex}\" |~ `${search:raw}` [$__range]))", + "queryType": "instant", + "editorMode": "code", + "legendFormat": "" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short", + "noValue": "No observations" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "none", + "graphMode": "none", + "textMode": "auto", + "orientation": "auto" + } + }, + { + "id": 3, + "title": "Events by severity ID", + "type": "bargauge", + "gridPos": { + "x": 6, + "y": 3, + "w": 18, + "h": 5 + }, + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "expr": "sum by (ocsf_severity_id) (count_over_time({__OS_LOGS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_LOGS_NAMESPACE_LABEL__=~__OS_NAMESPACE_REGEX__} | log_kind=\"ocsf\" | sandbox_name=~`${sandbox:raw}` | sandbox_id=~`${sandbox_id:raw}` | ocsf_class_uid=~`${class_uid:raw}` | ocsf_severity_id=~\"${severity:regex}\" |~ `${search:raw}` [$__range]))", + "queryType": "instant", + "editorMode": "code", + "legendFormat": "Severity {{ocsf_severity_id}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short", + "noValue": "No observations" + }, + "overrides": [] + }, + "options": { + "orientation": "horizontal", + "displayMode": "basic", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + } + }, + { + "id": 4, + "title": "Event rate by OCSF class UID", + "type": "timeseries", + "gridPos": { + "x": 0, + "y": 8, + "w": 24, + "h": 7 + }, + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "expr": "sum by (ocsf_class_uid) (rate({__OS_LOGS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_LOGS_NAMESPACE_LABEL__=~__OS_NAMESPACE_REGEX__} | log_kind=\"ocsf\" | sandbox_name=~`${sandbox:raw}` | sandbox_id=~`${sandbox_id:raw}` | ocsf_class_uid=~`${class_uid:raw}` | ocsf_severity_id=~\"${severity:regex}\" |~ `${search:raw}` [$__auto]))", + "queryType": "range", + "editorMode": "code", + "legendFormat": "Class {{ocsf_class_uid}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "ops", + "noValue": "No observations", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 5, + "title": "OCSF event JSON", + "type": "logs", + "gridPos": { + "x": 0, + "y": 15, + "w": 24, + "h": 15 + }, + "description": "Up to 1,000 matching structured OCSF events, newest first. Full JSON is retained; expand a line to inspect metadata. Sandbox fields describe the audited sandbox, not a scrape target.", + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "expr": "{__OS_LOGS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_LOGS_NAMESPACE_LABEL__=~__OS_NAMESPACE_REGEX__} | log_kind=\"ocsf\" | sandbox_name=~`${sandbox:raw}` | sandbox_id=~`${sandbox_id:raw}` | ocsf_class_uid=~`${class_uid:raw}` | ocsf_severity_id=~\"${severity:regex}\" |~ `${search:raw}`", + "queryType": "range", + "editorMode": "code", + "legendFormat": "", + "maxLines": 1000 + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short", + "noValue": "No observations" + }, + "overrides": [] + }, + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": true, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + } + } + ] +} diff --git a/deploy/openshell-mixin/dashboards/openshell-operational-health.json b/deploy/openshell-mixin/dashboards/openshell-operational-health.json new file mode 100644 index 0000000000..6a0a16df2b --- /dev/null +++ b/deploy/openshell-mixin/dashboards/openshell-operational-health.json @@ -0,0 +1,1629 @@ +{ + "id": null, + "uid": "openshell-operational-health", + "title": "OpenShell / Operational health", + "description": "Gateway readiness, request rates and latency, supervisor sessions, routing, relay capacity, and mutation locks from OpenShell metrics. Select a Prometheus datasource, cluster, and gateway target. Optional gateway trace search uses Tempo and the selected cluster. Missing observations do not establish health; latency panels show means unless labeled p95.", + "tags": [ + "openshell" + ], + "timezone": "browser", + "schemaVersion": 40, + "version": 1, + "editable": true, + "refresh": "30s", + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "annotations": { + "list": [] + }, + "templating": { + "list": [ + { + "name": "metrics", + "label": "Metrics source", + "type": "datasource", + "query": "prometheus", + "refresh": 1, + "options": [], + "hide": 0, + "regex": "" + }, + { + "name": "traces", + "label": "Trace source", + "type": "datasource", + "query": "tempo", + "regex": "", + "refresh": 1, + "hide": 0, + "options": [] + }, + { + "name": "cluster", + "label": "Cluster", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "definition": "label_values(openshell_server_readiness_database_healthy{__OS_METRICS_SELECTOR__}, __OS_METRICS_CLUSTER_LABEL__)", + "query": { + "query": "label_values(openshell_server_readiness_database_healthy{__OS_METRICS_SELECTOR__}, __OS_METRICS_CLUSTER_LABEL__)", + "refId": "clusterVariableQuery" + }, + "refresh": 1, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [] + }, + { + "name": "instance", + "label": "Gateway target", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "definition": "label_values(openshell_server_readiness_database_healthy{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\"}, __OS_METRICS_INSTANCE_LABEL__)", + "query": { + "query": "label_values(openshell_server_readiness_database_healthy{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\"}, __OS_METRICS_INSTANCE_LABEL__)", + "refId": "instanceVariableQuery" + }, + "refresh": 1, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [] + } + ] + }, + "links": [ + { + "type": "dashboards", + "title": "OpenShell dashboards", + "tags": [ + "openshell" + ], + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "targetBlank": false + } + ], + "panels": [ + { + "id": 1, + "type": "text", + "title": "Read this dashboard", + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 5 + }, + "options": { + "mode": "markdown", + "content": "**OpenShell operational health** \nOpenShell-emitted health metrics: database readiness, completed RPC/HTTP throughput and latency, routed attempt outcomes, supervisor sessions, relay pressure, and mutation locks. **No observations is not a healthy zero.** RPC counts represent completed requests; streaming calls contribute when they end. Routed retries count as separate attempts. No collector, backend, scrape-generated availability, or Kubernetes metrics are included. The tracing section searches gateway spans in the selected datasource and cluster; click a trace ID to inspect it. Gateway target filters apply only to metrics." + } + }, + { + "id": 2, + "title": "Mean database probe latency", + "type": "stat", + "gridPos": { + "x": 0, + "y": 5, + "w": 4, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Mean database probe duration across selected gateway replicas, weighted by probe count. No completed probes means no observations.", + "targets": [ + { + "refId": "A", + "expr": "sum(rate(openshell_server_readiness_database_probe_duration_seconds_sum{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"}[$__rate_interval])) / (sum(rate(openshell_server_readiness_database_probe_duration_seconds_count{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"}[$__rate_interval])) > 0)", + "legendFormat": "", + "range": false, + "instant": true, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "No observations", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "null", + "result": { + "text": "No observations", + "color": "gray" + } + } + } + ] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + } + }, + { + "id": 3, + "title": "Database ready", + "type": "stat", + "gridPos": { + "x": 4, + "y": 5, + "w": 4, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Minimum database readiness across selected targets. Does not represent Kubernetes Deployment/StatefulSet readiness.", + "targets": [ + { + "refId": "A", + "expr": "min(openshell_server_readiness_database_healthy{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"})", + "legendFormat": "", + "range": false, + "instant": true, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No observations", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "NOT READY", + "color": "red" + }, + "1": { + "text": "READY", + "color": "green" + } + } + }, + { + "type": "special", + "options": { + "match": "null", + "result": { + "text": "NO DATA", + "color": "gray" + } + } + } + ] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + } + }, + { + "id": 4, + "title": "Supervisor sessions", + "type": "stat", + "gridPos": { + "x": 8, + "y": 5, + "w": 4, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Active supervisor control sessions on selected gateway replicas. This is not a count of sandboxes or users. Zero can be legitimate.", + "targets": [ + { + "refId": "A", + "expr": "sum(openshell_server_supervisor_sessions{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"})", + "legendFormat": "", + "range": false, + "instant": true, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No observations", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + } + }, + { + "id": 5, + "title": "Completed RPCs / sec", + "type": "stat", + "gridPos": { + "x": 12, + "y": 5, + "w": 4, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Completed RPCs per second, including internal and supervisor calls. Streaming calls are counted when they end.", + "targets": [ + { + "refId": "A", + "expr": "sum(rate(openshell_server_grpc_requests_total{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"}[$__rate_interval]))", + "legendFormat": "", + "range": false, + "instant": true, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "noValue": "No observations", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + } + }, + { + "id": 6, + "title": "Service-side failure ratio", + "type": "stat", + "gridPos": { + "x": 16, + "y": 5, + "w": 4, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Fraction of completed RPCs with codes 2,4,8,10,12,13,14,15. Expected caller/authentication errors are excluded. Thresholds are provisional, not SLOs.", + "targets": [ + { + "refId": "A", + "expr": "(sum(rate(openshell_server_grpc_requests_total{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\",code=~\"2|4|8|10|12|13|14|15\"}[$__rate_interval])) or (0 * sum(rate(openshell_server_grpc_requests_total{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"}[$__rate_interval])))) / clamp_min(sum(rate(openshell_server_grpc_requests_total{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"}[$__rate_interval])), 0.000001)", + "legendFormat": "", + "range": false, + "instant": true, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "noValue": "No observations", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.01 + }, + { + "color": "red", + "value": 0.05 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + } + }, + { + "id": 7, + "title": "Relay capacity used", + "type": "stat", + "gridPos": { + "x": 20, + "y": 5, + "w": 4, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Highest pending-relay occupancy across selected replicas. Thresholds of 70% and 90% are provisional.", + "targets": [ + { + "refId": "A", + "expr": "max(openshell_server_relay_pending{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"} / clamp_min(openshell_server_relay_pending_capacity{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"}, 1))", + "legendFormat": "", + "range": false, + "instant": true, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "noValue": "No observations", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + } + }, + { + "id": 8, + "type": "row", + "title": "Traffic, latency and readiness", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 9, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 9, + "title": "Completed RPCs by method", + "type": "timeseries", + "gridPos": { + "x": 0, + "y": 10, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Completed RPCs per second, grouped by method across selected gateway replicas. Streaming calls are counted when they end.", + "targets": [ + { + "refId": "A", + "expr": "sum by (method) (rate(openshell_server_grpc_requests_total{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"}[$__rate_interval]))", + "legendFormat": "{{method}}", + "range": true, + "instant": false, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "noValue": "No observations", + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "axisPlacement": "auto" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + } + } + }, + { + "id": 10, + "title": "Mean completed RPC latency by method", + "type": "timeseries", + "gridPos": { + "x": 12, + "y": 10, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Mean duration of completed RPCs by method, calculated from summed durations and counts across selected replicas. Streaming durations include the full session lifetime. This is a mean, not p95 or p99; no completed calls means no observations.", + "targets": [ + { + "refId": "A", + "expr": "sum by (method) (rate(openshell_server_grpc_request_duration_seconds_sum{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"}[$__rate_interval])) / (sum by (method) (rate(openshell_server_grpc_request_duration_seconds_count{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"}[$__rate_interval])) > 0)", + "legendFormat": "{{method}}", + "range": true, + "instant": false, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "No observations", + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "axisPlacement": "auto" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + } + } + }, + { + "id": 11, + "title": "Non-OK RPC outcomes", + "type": "timeseries", + "gridPos": { + "x": 0, + "y": 17, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Includes caller and auth errors as well as service failures. Codes: 1 cancelled, 2 unknown, 3 invalid argument, 4 deadline, 5 not found, 6 exists, 7 denied, 8 exhausted, 9 precondition, 10 aborted, 11 range, 12 unimplemented, 13 internal, 14 unavailable, 15 data loss, 16 unauthenticated.", + "targets": [ + { + "refId": "A", + "expr": "sum by (method, code) (rate(openshell_server_grpc_requests_total{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\",code!=\"0\"}[$__rate_interval]))", + "legendFormat": "{{method}} \u00b7 code {{code}}", + "range": true, + "instant": false, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "noValue": "No observations", + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "axisPlacement": "auto" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + } + } + }, + { + "id": 12, + "title": "Database readiness and supervisor sessions", + "type": "timeseries", + "gridPos": { + "x": 12, + "y": 17, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Gateway-reported state. Sessions are not sandbox inventory; database readiness is not pod readiness.", + "targets": [ + { + "refId": "A", + "expr": "min(openshell_server_readiness_database_healthy{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"})", + "legendFormat": "Database ready (0/1)", + "range": true, + "instant": false, + "editorMode": "code" + }, + { + "refId": "B", + "expr": "sum(openshell_server_supervisor_sessions{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"})", + "legendFormat": "Supervisor sessions", + "range": true, + "instant": false, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No observations", + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "axisPlacement": "auto" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + } + } + }, + { + "id": 21, + "title": "HTTP traffic, latency and outcomes", + "type": "row", + "collapsed": false, + "panels": [], + "gridPos": { + "x": 0, + "y": 24, + "w": 24, + "h": 1 + } + }, + { + "id": 22, + "title": "HTTP requests by path", + "type": "timeseries", + "gridPos": { + "x": 0, + "y": 25, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "HTTP requests per second, grouped by the gateway's path label. gRPC traffic is shown separately. An unknown path cannot be attributed to a specific endpoint. Missing series appear as No observations.", + "targets": [ + { + "refId": "A", + "expr": "sum by (path) (rate(openshell_server_http_requests_total{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"}[$__rate_interval]))", + "legendFormat": "{{path}}", + "range": true, + "instant": false, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "noValue": "No observations", + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "axisPlacement": "auto" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + } + } + }, + { + "id": 23, + "title": "Mean HTTP latency by path and status", + "type": "timeseries", + "gridPos": { + "x": 12, + "y": 25, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Mean HTTP request duration by path and response status, calculated from summed durations and counts across selected replicas. This is a mean, not p95 or p99; no new duration observations means no mean is available.", + "targets": [ + { + "refId": "A", + "expr": "sum by (path, status) (rate(openshell_server_http_request_duration_seconds_sum{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"}[$__rate_interval])) / (sum by (path, status) (rate(openshell_server_http_request_duration_seconds_count{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"}[$__rate_interval])) > 0)", + "legendFormat": "{{path}} \u00b7 HTTP {{status}}", + "range": true, + "instant": false, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "No observations", + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "axisPlacement": "auto" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + } + } + }, + { + "id": 24, + "title": "HTTP client and server errors", + "type": "timeseries", + "gridPos": { + "x": 0, + "y": 32, + "w": 24, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "4xx caller/authentication responses and 5xx server responses are shown separately. Expected denials, bad requests and not-found responses are not automatically platform availability failures. Redirects are not classified as errors.", + "targets": [ + { + "refId": "A", + "expr": "sum by (path, status) (rate(openshell_server_http_requests_total{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\",status=~\"4..\"}[$__rate_interval]))", + "legendFormat": "Client/auth \u00b7 {{path}} \u00b7 HTTP {{status}}", + "range": true, + "instant": false, + "editorMode": "code" + }, + { + "refId": "B", + "expr": "sum by (path, status) (rate(openshell_server_http_requests_total{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\",status=~\"5..\"}[$__rate_interval]))", + "legendFormat": "Server \u00b7 {{path}} \u00b7 HTTP {{status}}", + "range": true, + "instant": false, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "noValue": "No observations", + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "axisPlacement": "auto" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + } + } + }, + { + "id": 25, + "title": "Local and peer routing attempts", + "type": "row", + "collapsed": false, + "panels": [], + "gridPos": { + "x": 0, + "y": 39, + "w": 24, + "h": 1 + } + }, + { + "id": 26, + "title": "Routed attempts by operation and route", + "type": "timeseries", + "gridPos": { + "x": 0, + "y": 40, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Local relay setup and outbound peer attempts completed or cancelled by each replica. Each retry counts separately: attempts are not unique logical requests or end-to-end sandbox operations.", + "targets": [ + { + "refId": "A", + "expr": "sum by (operation, route, relay_kind) (rate(openshell_server_routed_request_attempts_total{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"}[$__rate_interval]))", + "legendFormat": "{{operation}} \u00b7 {{route}} \u00b7 {{relay_kind}}", + "range": true, + "instant": false, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "noValue": "No observations", + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "axisPlacement": "auto" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + } + } + }, + { + "id": 27, + "title": "Routed attempt outcomes", + "type": "timeseries", + "gridPos": { + "x": 12, + "y": 40, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Local and peer routing attempts grouped by outcome and gRPC status. Cancellations and caller errors are included. Each retry counts separately, so a failed attempt may be followed by a successful retry.", + "targets": [ + { + "refId": "A", + "expr": "sum by (operation, route, outcome, grpc_code) (rate(openshell_server_routed_request_attempts_total{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"}[$__rate_interval]))", + "legendFormat": "{{operation}} \u00b7 {{route}} \u00b7 {{outcome}} \u00b7 {{grpc_code}}", + "range": true, + "instant": false, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "noValue": "No observations", + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "axisPlacement": "auto" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + } + } + }, + { + "id": 13, + "type": "row", + "title": "Relay and mutation pressure", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 47, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 14, + "title": "Pending relay channels and capacity", + "type": "timeseries", + "gridPos": { + "x": 0, + "y": 48, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Channels waiting for supervisor callback; summed across selected replicas.", + "targets": [ + { + "refId": "A", + "expr": "sum(openshell_server_relay_pending{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"})", + "legendFormat": "Pending", + "range": true, + "instant": false, + "editorMode": "code" + }, + { + "refId": "B", + "expr": "sum(openshell_server_relay_pending_capacity{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"})", + "legendFormat": "Capacity", + "range": true, + "instant": false, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No observations", + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "axisPlacement": "auto" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + } + } + }, + { + "id": 15, + "title": "Relay expirations and rejections", + "type": "timeseries", + "gridPos": { + "x": 12, + "y": 48, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Relay expirations and rejections per second across selected gateway replicas. Rejections are grouped by reason. These counts describe relay setup failures, not unique failed sandbox operations.", + "targets": [ + { + "refId": "A", + "expr": "sum(rate(openshell_server_relay_expired_total{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"}[$__rate_interval]))", + "legendFormat": "Expired", + "range": true, + "instant": false, + "editorMode": "code" + }, + { + "refId": "B", + "expr": "sum by (reason) (rate(openshell_server_relay_rejected_total{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"}[$__rate_interval]))", + "legendFormat": "Rejected \u00b7 {{reason}}", + "range": true, + "instant": false, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No observations", + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "axisPlacement": "auto" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + } + } + }, + { + "id": 16, + "title": "Mutation lock wait p95", + "type": "timeseries", + "gridPos": { + "x": 0, + "y": 55, + "w": 8, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "95th percentile of mutation lock wait time, calculated from histogram buckets across selected replicas. No lock observations in the rate window means no observations.", + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le, scope) (rate(openshell_server_mutation_lock_wait_seconds_bucket{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"}[$__rate_interval])))", + "legendFormat": "{{scope}}", + "range": true, + "instant": false, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "No observations", + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "axisPlacement": "auto" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + } + } + }, + { + "id": 17, + "title": "Mutation lock hold p95", + "type": "timeseries", + "gridPos": { + "x": 8, + "y": 55, + "w": 8, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "95th percentile of mutation lock hold time, calculated from histogram buckets across selected replicas. Includes locks released after cancellation.", + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le, scope) (rate(openshell_server_mutation_lock_hold_seconds_bucket{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"}[$__rate_interval])))", + "legendFormat": "{{scope}}", + "range": true, + "instant": false, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "No observations", + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "axisPlacement": "auto" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + } + } + }, + { + "id": 18, + "title": "Mutation failures and deadline-scan skips", + "type": "timeseries", + "gridPos": { + "x": 16, + "y": 55, + "w": 8, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "A skipped scan indicates busy mutation locks; it is not itself a sandbox provisioning timeout.", + "targets": [ + { + "refId": "A", + "expr": "sum by (scope) (rate(openshell_server_mutation_lock_errors_total{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"}[$__rate_interval]))", + "legendFormat": "Lock error \u00b7 {{scope}}", + "range": true, + "instant": false, + "editorMode": "code" + }, + { + "refId": "B", + "expr": "sum by (scope) (rate(openshell_server_mutation_lock_timeouts_total{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"}[$__rate_interval]))", + "legendFormat": "Lock timeout \u00b7 {{scope}}", + "range": true, + "instant": false, + "editorMode": "code" + }, + { + "refId": "C", + "expr": "sum by (scope) (rate(openshell_server_provisioning_deadline_skips_total{__OS_METRICS_SELECTOR__,__OS_METRICS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_METRICS_INSTANCE_LABEL__=~\"${instance:regex}\"}[$__rate_interval]))", + "legendFormat": "Deadline scan skipped \u00b7 {{scope}}", + "range": true, + "instant": false, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No observations", + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "axisPlacement": "auto" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + } + } + }, + { + "id": 19, + "type": "row", + "title": "Gateway tracing", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 62, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 20, + "type": "table", + "title": "OpenShell gateway traces", + "description": "Up to 20 matching gateway traces in the selected datasource, cluster, and dashboard time range. Click a trace ID to inspect its spans in Explore. Gateway target is a metric scrape address and does not filter traces. Missing results do not indicate gateway health.", + "gridPos": { + "x": 0, + "y": 63, + "w": 24, + "h": 10 + }, + "datasource": { + "type": "tempo", + "uid": "${traces}" + }, + "targets": [ + { + "refId": "A", + "queryType": "traceql", + "query": "{resource.service.name = __OS_TRACES_SERVICE__ && __OS_TRACES_CLUSTER_ATTRIBUTE__ =~ \"${cluster:regex}\"}", + "tableType": "traces", + "limit": 20 + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "nested": true + }, + "indexByName": { + "startTime": 0, + "traceService": 1, + "traceName": 2, + "traceDuration": 3, + "traceID": 4 + } + } + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "sortBy": [ + { + "displayName": "Start time", + "desc": true + } + ] + } + } + ], + "fiscalYearStartMonth": 0 +} diff --git a/deploy/openshell-mixin/dashboards/openshell-operational-logs.json b/deploy/openshell-mixin/dashboards/openshell-operational-logs.json new file mode 100644 index 0000000000..9e32b678d5 --- /dev/null +++ b/deploy/openshell-mixin/dashboards/openshell-operational-logs.json @@ -0,0 +1,342 @@ +{ + "id": null, + "uid": "openshell-operational-logs", + "title": "OpenShell / Operational logs", + "description": "Gateway, supervisor, and workload pod logs from Loki. Select a datasource and cluster, then filter by component, pod, stream, or message. Open the OCSF logs dashboard for structured security and audit events.", + "tags": [ + "openshell" + ], + "timezone": "browser", + "schemaVersion": 40, + "version": 1, + "editable": true, + "refresh": "30s", + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "annotations": { + "list": [] + }, + "links": [ + { + "type": "dashboards", + "title": "OpenShell dashboards", + "tags": [ + "openshell" + ], + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "targetBlank": false + } + ], + "templating": { + "list": [ + { + "name": "logs", + "label": "Logs source", + "type": "datasource", + "query": "loki", + "regex": "", + "refresh": 1, + "hide": 0, + "options": [] + }, + { + "name": "cluster", + "label": "Cluster", + "type": "query", + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "query": { + "type": 1, + "label": "__OS_LOGS_CLUSTER_LABEL__", + "stream": "{__OS_LOGS_NAMESPACE_LABEL__=~__OS_NAMESPACE_REGEX__}", + "refId": "clusterVariableQuery" + }, + "definition": "label_values({__OS_LOGS_NAMESPACE_LABEL__=~__OS_NAMESPACE_REGEX__}, __OS_LOGS_CLUSTER_LABEL__)", + "refresh": 1, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [] + }, + { + "name": "component", + "label": "Component", + "type": "custom", + "query": "openshell-gateway,supervisor,agent,gateway", + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [ + { + "text": "openshell-gateway", + "value": "openshell-gateway", + "selected": false + }, + { + "text": "supervisor", + "value": "supervisor", + "selected": false + }, + { + "text": "agent", + "value": "agent", + "selected": false + }, + { + "selected": false, + "text": "gateway", + "value": "gateway" + } + ] + }, + { + "name": "pod", + "label": "Pod name (regex)", + "type": "textbox", + "query": ".*", + "current": { + "text": ".*", + "value": ".*" + }, + "options": [ + { + "text": ".*", + "value": ".*", + "selected": true + } + ] + }, + { + "name": "stream", + "label": "Stream", + "type": "custom", + "query": "stdout,stderr", + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [ + { + "text": "stdout", + "value": "stdout", + "selected": false + }, + { + "text": "stderr", + "value": "stderr", + "selected": false + } + ] + }, + { + "name": "search", + "label": "Message search (regex)", + "type": "textbox", + "query": "", + "current": { + "text": "", + "value": "" + }, + "options": [ + { + "text": "", + "value": "", + "selected": true + } + ] + } + ] + }, + "panels": [ + { + "id": 1, + "title": "Operational diagnostics", + "type": "text", + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 3 + }, + "options": { + "mode": "markdown", + "content": "Components: **openshell-gateway** (gateway), **supervisor** (sandbox supervisor), **agent** (sandbox main-process stdout/stderr). Expand a line for pod/container metadata. OCSF JSON and shorthand console events are excluded; open **OCSF logs** in the OpenShell dashboards menu for audit events.\n\nSelect the Loki datasource and cluster to inspect. Regex text filters default to `.*`; do not include backticks. Separate exec/PTY output may not appear in pod logs. Log volume and stderr are not health or error classifications." + } + }, + { + "id": 2, + "title": "Log lines in selected range", + "type": "stat", + "gridPos": { + "x": 0, + "y": 3, + "w": 6, + "h": 5 + }, + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "expr": "sum(count_over_time({__OS_LOGS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_LOGS_NAMESPACE_LABEL__=~__OS_NAMESPACE_REGEX__,service_name=~\"openshell-gateway|supervisor|agent\"} | service_name=~\"${component:regex}\" | log_kind!=\"ocsf\" !~ ` OCSF(-JSON)? ` | k8s_pod_name=~`${pod:raw}` | log_iostream=~\"${stream:regex}\" |~ `${search:raw}` [$__range]))", + "queryType": "instant", + "editorMode": "code", + "legendFormat": "" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short", + "noValue": "No observations" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "none", + "graphMode": "none", + "textMode": "auto", + "orientation": "auto" + } + }, + { + "id": 3, + "title": "Log rate by component", + "type": "timeseries", + "gridPos": { + "x": 6, + "y": 3, + "w": 18, + "h": 5 + }, + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "expr": "sum by (service_name) (rate({__OS_LOGS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_LOGS_NAMESPACE_LABEL__=~__OS_NAMESPACE_REGEX__,service_name=~\"openshell-gateway|supervisor|agent\"} | service_name=~\"${component:regex}\" | log_kind!=\"ocsf\" !~ ` OCSF(-JSON)? ` | k8s_pod_name=~`${pod:raw}` | log_iostream=~\"${stream:regex}\" |~ `${search:raw}` [$__auto]))", + "queryType": "range", + "editorMode": "code", + "legendFormat": "{{service_name}}" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "ops", + "noValue": "No observations", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 4, + "title": "Operational pod logs", + "type": "logs", + "gridPos": { + "x": 0, + "y": 8, + "w": 24, + "h": 18 + }, + "description": "Up to 1,000 matching operational log lines, newest first. Expand a line for pod, component, and stream metadata. OCSF JSON and shorthand console records are excluded. Separate exec or terminal output may not appear in pod logs.", + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "expr": "{__OS_LOGS_CLUSTER_LABEL__=~\"${cluster:regex}\",__OS_LOGS_NAMESPACE_LABEL__=~__OS_NAMESPACE_REGEX__,service_name=~\"openshell-gateway|supervisor|agent\"} | service_name=~\"${component:regex}\" | log_kind!=\"ocsf\" !~ ` OCSF(-JSON)? ` | k8s_pod_name=~`${pod:raw}` | log_iostream=~\"${stream:regex}\" |~ `${search:raw}`", + "queryType": "range", + "editorMode": "code", + "legendFormat": "", + "maxLines": 1000 + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short", + "noValue": "No observations" + }, + "overrides": [] + }, + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": true, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + } + } + ] +} diff --git a/deploy/openshell-mixin/examples/custom.jsonnet b/deploy/openshell-mixin/examples/custom.jsonnet new file mode 100644 index 0000000000..bab3e53446 --- /dev/null +++ b/deploy/openshell-mixin/examples/custom.jsonnet @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: Apache-2.0 +// Render from the package directory: make MIXIN=examples/custom.jsonnet generate +(import '../mixin.libsonnet') + { + _config+:: { + openshell+: { + metricsSelector: 'job=~"gateway|openshell-gateway"', + metricsClusterLabel: 'kubernetes_cluster', + namespaceRegex: 'openshell.*|team-sandboxes', + tracesEnabled: false, + datasourceRegex+: { metrics: '^production-metrics$', logs: '^production-logs$' }, + }, + }, +} diff --git a/deploy/openshell-mixin/jsonnetfile.json b/deploy/openshell-mixin/jsonnetfile.json new file mode 100644 index 0000000000..746f0752c5 --- /dev/null +++ b/deploy/openshell-mixin/jsonnetfile.json @@ -0,0 +1,5 @@ +{ + "version": 1, + "dependencies": [], + "legacyImports": false +} diff --git a/deploy/openshell-mixin/mixin.libsonnet b/deploy/openshell-mixin/mixin.libsonnet new file mode 100644 index 0000000000..c510f28729 --- /dev/null +++ b/deploy/openshell-mixin/mixin.libsonnet @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: Apache-2.0 +(import 'config.libsonnet') + +(import 'dashboards.libsonnet') + +{ + // Stable extension points; no unreviewed alert thresholds or recording rules. + prometheusAlerts: { groups: [] }, + prometheusRules: { groups: [] }, +} diff --git a/deploy/openshell-mixin/tests/test_mixin.py b/deploy/openshell-mixin/tests/test_mixin.py new file mode 100644 index 0000000000..71403ea575 --- /dev/null +++ b/deploy/openshell-mixin/tests/test_mixin.py @@ -0,0 +1,143 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Exercise the public Jsonnet API, including late-bound consumer overrides.""" + +import json +import os +import subprocess +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def evaluate(expression): + return json.loads( + subprocess.check_output( + [os.environ.get("JSONNET", "jsonnet"), "-e", expression], + cwd=ROOT, + ) + ) + + +def objects(value): + if isinstance(value, dict): + yield value + for child in value.values(): + yield from objects(child) + elif isinstance(value, list): + for child in value: + yield from objects(child) + + +class MixinTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.default = evaluate('import "mixin.libsonnet"') + cls.custom = evaluate('import "examples/custom.jsonnet"') + + def test_standard_exports(self): + self.assertEqual( + set(self.default), + {"grafanaDashboards", "prometheusAlerts", "prometheusRules"}, + ) + self.assertEqual(self.default["prometheusAlerts"], {"groups": []}) + self.assertEqual(self.default["prometheusRules"], {"groups": []}) + self.assertEqual(len(self.default["grafanaDashboards"]), 3) + self.assertEqual(len(self.custom["grafanaDashboards"]), 3) + + def test_portability_and_identity(self): + for result in (self.default, self.custom): + encoded = json.dumps(result) + for forbidden in ( + "__OS_", + "${tenant", + "openshell-tools", + "openshell-density", + "ag34rbum5xgqoe", + "firecracker", + ): + self.assertNotIn(forbidden, encoded) + for name, dashboard in result["grafanaDashboards"].items(): + self.assertEqual(dashboard["uid"], name.removesuffix(".json")) + self.assertIsNone(dashboard["id"]) + variables = {v["name"] for v in dashboard["templating"]["list"]} + for obj in objects(dashboard): + if obj.get("type") == "datasource" and "name" in obj: + self.assertNotIn("current", obj) + self.assertEqual(obj["hide"], 0) + if "datasource" in obj and isinstance(obj["datasource"], dict): + uid = obj["datasource"].get("uid", "") + if uid.startswith("${"): + self.assertIn(uid[2:-1], variables) + + def test_consumer_overrides(self): + default = self.default["grafanaDashboards"]["openshell-operational-health.json"] + custom = self.custom["grafanaDashboards"]["openshell-operational-health.json"] + self.assertTrue( + any( + 'job="openshell-gateway"' in obj.get("expr", "") + for obj in objects(default["panels"]) + ) + ) + queries = [ + obj[k] + for obj in objects(custom) + for k in ("expr", "definition") + if k in obj + ] + self.assertTrue(queries) + for query in queries: + self.assertIn('job=~"gateway|openshell-gateway"', query) + self.assertNotIn('job="openshell-gateway"', query) + self.assertNotIn("cluster=~", query.replace("kubernetes_cluster=~", "")) + self.assertNotIn('"type": "tempo"', json.dumps(custom)) + self.assertNotIn("traces", {v["name"] for v in custom["templating"]["list"]}) + metrics = next( + v for v in custom["templating"]["list"] if v["name"] == "metrics" + ) + self.assertEqual(metrics["regex"], "^production-metrics$") + + def test_log_label_overrides_and_escaping(self): + namespace = 'team"\\namespace.*' + overrides = json.dumps( + { + "logsClusterLabel": "cluster_name", + "logsNamespaceLabel": "namespace", + "namespaceRegex": namespace, + } + ) + result = evaluate( + '(import "mixin.libsonnet") + { _config+:: { openshell+: ' + + overrides + + " } }" + ) + dashboard = result["grafanaDashboards"]["openshell-ocsf-logs.json"] + cluster = next( + v for v in dashboard["templating"]["list"] if v["name"] == "cluster" + ) + self.assertEqual(cluster["query"]["label"], "cluster_name") + self.assertEqual( + cluster["query"]["stream"], + "{namespace=~" + json.dumps(namespace, separators=(",", ":")) + "}", + ) + for obj in objects(dashboard["panels"]): + if "expr" in obj: + self.assertIn('cluster_name=~"${cluster:regex}"', obj["expr"]) + self.assertNotIn("k8s_namespace_name", obj["expr"]) + + def test_shared_panel_references(self): + for dashboard in self.custom["grafanaDashboards"].values(): + panels = [ + p for p in objects(dashboard["panels"]) if "id" in p and "type" in p + ] + ids = [p["id"] for p in panels] + self.assertEqual(len(ids), len(set(ids))) + for obj in objects(dashboard["panels"]): + if "panelId" in obj: + self.assertIn(obj["panelId"], ids) + + +if __name__ == "__main__": + unittest.main() diff --git a/mise.lock b/mise.lock index ec8cccf089..053dbc2087 100644 --- a/mise.lock +++ b/mise.lock @@ -287,6 +287,14 @@ url = "https://dl.google.com/go/go1.26.7.windows-amd64.zip" version = "2.12.2" backend = "go:github.com/golangci/golangci-lint/v2/cmd/golangci-lint" +[[tools."go:github.com/google/go-jsonnet/cmd/jsonnet"]] +version = "0.22.0" +backend = "go:github.com/google/go-jsonnet/cmd/jsonnet" + +[[tools."go:github.com/google/go-jsonnet/cmd/jsonnetfmt"]] +version = "0.22.0" +backend = "go:github.com/google/go-jsonnet/cmd/jsonnetfmt" + [[tools."go:golang.org/x/tools/cmd/goimports"]] version = "0.48.0" backend = "go:golang.org/x/tools/cmd/goimports" diff --git a/tasks/monitoring.toml b/tasks/monitoring.toml new file mode 100644 index 0000000000..c4018e48ae --- /dev/null +++ b/tasks/monitoring.toml @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Portable dashboard generation and structural checks; no telemetry stack. + +["monitoring:generate"] +description = "Render OpenShell Grafana dashboards and empty rule exports" +dir = "deploy/openshell-mixin" +tools = { "go:github.com/google/go-jsonnet/cmd/jsonnet" = "0.22.0" } +run = ''' +set -eu +mixin="${MIXIN:-mixin.libsonnet}" +out="${OUT:-build}" +mkdir -p "$out/dashboards" +jsonnet -J vendor -m "$out/dashboards" -e "(import \"$mixin\").grafanaDashboards" +jsonnet -J vendor -S -e "std.manifestYamlDoc((import \"$mixin\").prometheusAlerts)" > "$out/prometheus_alerts.yaml" +jsonnet -J vendor -S -e "std.manifestYamlDoc((import \"$mixin\").prometheusRules)" > "$out/prometheus_rules.yaml" +''' + +["monitoring:format"] +description = "Format the OpenShell monitoring mixin" +dir = "deploy/openshell-mixin" +tools = { "go:github.com/google/go-jsonnet/cmd/jsonnetfmt" = "0.22.0" } +run = "jsonnetfmt -i *.libsonnet examples/*.jsonnet" + +["monitoring:format:check"] +description = "Check OpenShell monitoring mixin formatting" +dir = "deploy/openshell-mixin" +tools = { "go:github.com/google/go-jsonnet/cmd/jsonnetfmt" = "0.22.0" } +run = "jsonnetfmt --test *.libsonnet examples/*.jsonnet" + +["monitoring:test"] +description = "Check dashboard exports, portability, and configuration overrides" +dir = "deploy/openshell-mixin" +tools = { "go:github.com/google/go-jsonnet/cmd/jsonnet" = "0.22.0" } +run = "uv run --no-project python tests/test_mixin.py" + +["monitoring:check"] +description = "Check and render the OpenShell monitoring mixin" +depends = ["monitoring:format:check", "monitoring:test", "monitoring:generate"]