feat(observability): add reusable OpenShell Grafana monitoring mixin

.gitattributes

@@ -11,3 +11,6 @@ crates/openshell-core/src/proto/openshell.*.rs linguist-generated
 
 # Vendored OCSF schemas fetched from schema.ocsf.io
 crates/openshell-ocsf-validate/schemas/v*/** linguist-generated
+
+# Grafana dashboard JSON templates consumed by the monitoring mixin
+deploy/openshell-mixin/dashboards/*.json linguist-generated

.github/workflows/branch-checks.yml

@@ -133,6 +133,63 @@ jobs:
                 || github.event.merge_group.base_sha
                 || needs.pr_metadata.outputs.base_sha }}
 
+  monitoring-changes:
+    name: Detect monitoring mixin changes
+    needs: pr_metadata
+    if: needs.pr_metadata.outputs.should_run == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    outputs:
+      changed: ${{ steps.changes.outputs.changed }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Check mixin paths
+        id: changes
+        shell: bash
+        env:
+          BASE_SHA: ${{ github.event.merge_group.base_sha || needs.pr_metadata.outputs.base_sha }}
+          EVENT_NAME: ${{ github.event_name }}
+        run: |
+          set -euo pipefail
+          if [ "$EVENT_NAME" = workflow_dispatch ]; then
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          if [ -z "$BASE_SHA" ]; then
+            echo "::error::Missing base SHA for monitoring mixin change detection." >&2
+            exit 1
+          fi
+          changed=$(git diff --name-only "${BASE_SHA}...HEAD" -- deploy/openshell-mixin/)
+          if [ -n "$changed" ]; then
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          fi
+
+  monitoring:
+    name: Monitoring Mixin
+    needs: [pr_metadata, monitoring-changes]
+    if: needs.pr_metadata.outputs.should_run == 'true' && needs.monitoring-changes.outputs.changed == 'true'
+    runs-on: linux-amd64-cpu8
+    timeout-minutes: 15
+    container:
+      image: ghcr.io/nvidia/openshell/ci:9cb72baa2e61a1b5f12407e6e82da7fdba0aa722@sha256:67a9a0c32cb99825e6d3e9d9eec45d67149ea1ff7c11a1b1e1b3480d2d3df684
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Install monitoring prerequisites
+        run: mise install --locked go python uv
+
+      - name: Check monitoring mixin
+        run: mise run monitoring:check
+
   mise-lockfile:
     name: mise Lockfile
     needs: pr_metadata

CI.md

@@ -17,6 +17,14 @@ Manual admission does not change the bot's automatic trust policy for ready PRs.
 
 Merge queue validation is a second integration gate for `main`. After a PR has passed the required PR-head statuses, a maintainer adds it to the merge queue. GitHub creates a temporary merge-group branch that combines the latest `main`, the queued PR, and any earlier queued PRs. The same required `OpenShell / ...` status contexts are then published against the merge-group SHA before GitHub merges it.
 
+### Monitoring dashboards
+
+`Monitoring Mixin` runs in Branch Checks when files under
+`deploy/openshell-mixin/` change relative to the PR or merge queue base. It skips
+unrelated changes. Manual Branch Checks runs always include this check.
+The job runs `mise run monitoring:check` to validate formatting, run the
+structural tests, and render the dashboards. Use the same command locally.
+
 ### Protobuf API compatibility
 
 `Protobuf Compatibility` runs in Branch Checks and Pre-release through the shared

CONTRIBUTING.md

@@ -373,6 +373,8 @@ These are the primary `mise` tasks for day-to-day development:
 | `mise run ci`        | Full local CI checks (lint, compile/type checks, tests) |
 | `mise run docs`      | Validate Fern docs locally                              |
 | `mise run helm:docs` | Regenerate the Helm chart README                        |
+| `mise run monitoring:generate` | Render OpenShell Grafana dashboards           |
+| `mise run monitoring:check` | Check and render the monitoring mixin            |
 | `mise run clean`     | Clean build artifacts                                   |
 
 ## Project Structure

deploy/openshell-mixin/.gitignore

@@ -0,0 +1,4 @@
+build/
+.tools/
+__pycache__/
+vendor/

deploy/openshell-mixin/README.md

@@ -0,0 +1,86 @@
+# OpenShell monitoring mixin
+
+Reusable Grafana dashboards for OpenShell, packaged as a [monitoring mixin](https://monitoring.mixins.dev/). Render the mixin to dashboard JSON and import it into your Grafana instance. Configure collection and datasources in your own monitoring environment.
+
+## Dashboards
+
+| Dashboard | Datasources | Purpose |
+| --- | --- | --- |
+| OpenShell operational health | Prometheus; optional Tempo | Gateway readiness, RPC/HTTP traffic and latency, supervisor sessions, routing, relay pressure, and mutation locks |
+| OpenShell operational logs | Loki | Gateway, supervisor, and workload diagnostics |
+| OpenShell OCSF logs | Loki | Structured security and audit events from gateways and supervisors |
+
+Dashboard UIDs and panel IDs remain stable across renders. Datasource variables are visible and independently selectable. Credentials, tenant headers, datasource provisioning, and Grafana folders belong in the consuming environment.
+
+## Render and import
+
+From the repository root, render the dashboards with mise:
+
+```shell
+mise run monitoring:generate
+```
+
+The tasks declare Go Jsonnet and jsonnetfmt **v0.22.0** as task-specific tools; mise installs them when needed.
+
+Import the JSON files from `deploy/openshell-mixin/build/dashboards/` through Grafana's dashboard import screen, or use your existing Grafana file provisioning, API, or Kubernetes sidecar workflow. Select the appropriate metrics, logs, and optional traces datasource in each dashboard, then choose the cluster and gateway instance.
+
+Files under `dashboards/` are source templates containing `__OS_*__` tokens; import the rendered files from `build/dashboards/`.
+
+The standard mixin exports are `grafanaDashboards`, `prometheusAlerts`, and `prometheusRules`. The two rule exports contain empty groups; no alerts or recording rules are defined. `monitoring:generate` writes their YAML alongside the dashboard directory, but they need not be installed.
+
+## Customize
+
+Defaults live in [config.libsonnet](config.libsonnet). Override `_config.openshell` in a consumer Jsonnet file. [examples/custom.jsonnet](examples/custom.jsonnet) demonstrates a different metric selector and cluster label, namespace regex, disabled traces, and datasource name filters:
+
+```shell
+MIXIN=examples/custom.jsonnet OUT=build/custom mise run monitoring:generate
+```
+
+`MIXIN` and `OUT` are relative to `deploy/openshell-mixin`; `OUT` also accepts an absolute path.
+
+- `metricsSelector` contains nonempty PromQL label matchers without braces.
+- Metric and log label settings let you match your collection pipeline. Label names and TraceQL attribute names must be valid identifiers for their query languages.
+- Regex values are escaped before insertion into queries. Datasource regexes match Grafana datasource names, not UIDs.
+- Set `tracesEnabled: false` when Tempo is unavailable. This removes the trace panel and its datasource variable.
+
+The package has no Jsonnet dependencies. After it is published, consumers can pin its subtree with jsonnet-bundler v0.6.0:
+
+```shell
+jb init
+jb install github.com/NVIDIA/OpenShell/deploy/openshell-mixin@COMMIT_SHA
+```
+
+Replace `COMMIT_SHA` with a published immutable revision containing this package. Import `github.com/NVIDIA/OpenShell/deploy/openshell-mixin/mixin.libsonnet` from the consumer's vendor directory and compose its exports into your existing monitoring resources, including kube-prometheus ConfigMaps where applicable.
+
+## Collection requirements
+
+| Signal | Required collection contract |
+| --- | --- |
+| Metrics | OpenShell gateway metrics, with default scrape job `openshell-gateway` and `cluster` / `instance` labels. Cluster discovery uses `openshell_server_readiness_database_healthy`. See the [gateway metrics reference](../../docs/observability/gateway-metrics.mdx) for emitted families. |
+| Logs | Loki labels `k8s_cluster_name`, `k8s_namespace_name`, and `service_name`. Preserve `k8s_pod_name`, `log_iostream`, and `log_kind` as queryable metadata. Operational service names are `openshell-gateway`, `supervisor`, and `agent`. |
+| OCSF | Preserve the event JSON body and supply `log_kind="ocsf"`, `sandbox_name`, `sandbox_id`, `ocsf_class_uid`, and `ocsf_severity_id` as queryable metadata when available. Collect events from both the gateway and supervisors. |
+| Traces | Tempo spans with `resource.service.name="openshell-gateway"` and `resource.k8s.cluster.name`. Cluster values must match the metric cluster labels. Service and cluster attribute settings can be overridden. |
+
+The default namespace regex is `openshell.*`. Map collection-specific labels, metadata, and service names to this contract or adjust the mixin configuration. Use consistent cluster values across metrics, logs, and traces.
+
+RPC and HTTP summaries show weighted means, not aggregate p95. Streaming RPC completion durations include session lifetime. Mutation-lock histograms support the displayed fleet p95. Routed retries count as separate attempts; supervisor sessions are not sandbox counts. Missing observations remain absent. Threshold colors provide provisional operational guidance, not SLOs. OCSF class IDs include the emitted value `0`.
+
+## Validate in your environment
+
+The local checks require the repository's `uv` Python runner and validate Jsonnet formatting, standard exports, dashboard identity, datasource references, portability, and configuration overrides:
+
+```shell
+mise run monitoring:check
+```
+
+Branch Checks runs `mise run monitoring:check` when files in `deploy/openshell-mixin/` change. Manual Branch Checks runs always include it.
+
+Users validate queries and visual behavior against their own collection pipeline and Grafana version:
+
+1. Confirm cluster and instance selectors populate from the intended deployment.
+2. Generate representative gateway and sandbox activity, then check readiness, request rates, latency, and supervisor sessions.
+3. Confirm operational logs and gateway/supervisor OCSF events appear with the expected metadata filters.
+4. If traces are enabled, open a gateway trace from the trace table and confirm the cluster matches.
+5. Investigate query errors and missing data before treating a panel as evidence of health. Exercise failure paths separately; an empty error series does not prove those paths work.
+
+`monitoring:check` checks formatting, runs the structural tests, and renders the default mixin. Use `monitoring:format` to format Jsonnet files, or run `monitoring:format:check` and `monitoring:test` individually. These tasks do not start a telemetry stack or execute live backend queries.

deploy/openshell-mixin/config.libsonnet

@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: Apache-2.0
+{
+  _config+:: {
+    openshell+: {
+      // Nonempty PromQL label matchers, without braces; caller supplies syntax.
+      metricsSelector: 'job="openshell-gateway"',
+      metricsClusterLabel: 'cluster',
+      metricsInstanceLabel: 'instance',
+      logsClusterLabel: 'k8s_cluster_name',
+      logsNamespaceLabel: 'k8s_namespace_name',
+      namespaceRegex: 'openshell.*',
+      tracesEnabled: true,
+      tracesClusterAttribute: 'resource.k8s.cluster.name',
+      tracesService: 'openshell-gateway',
+      // Match datasource names. Empty means all datasources of that type.
+      datasourceRegex: { metrics: '', logs: '', traces: '' },
+    },
+  },
+}

deploy/openshell-mixin/dashboards.libsonnet

@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: Apache-2.0
+{
+  local config = self._config.openshell,
+  local replacements = {
+    __OS_METRICS_SELECTOR__: config.metricsSelector,
+    __OS_METRICS_CLUSTER_LABEL__: config.metricsClusterLabel,
+    __OS_METRICS_INSTANCE_LABEL__: config.metricsInstanceLabel,
+    __OS_LOGS_CLUSTER_LABEL__: config.logsClusterLabel,
+    __OS_LOGS_NAMESPACE_LABEL__: config.logsNamespaceLabel,
+    __OS_NAMESPACE_REGEX__: std.manifestJsonEx(config.namespaceRegex, ''),
+    __OS_TRACES_CLUSTER_ATTRIBUTE__: config.tracesClusterAttribute,
+    __OS_TRACES_SERVICE__: std.manifestJsonEx(config.tracesService, ''),
+  },
+  local render(value) =
+    if std.isArray(value) then [render(v) for v in value]
+    else if std.isObject(value) then { [k]: render(value[k]) for k in std.objectFields(value) }
+    else if std.isString(value) then
+      std.foldl(function(s, token) std.strReplace(s, token, replacements[token]), std.objectFields(replacements), value)
+    else value,
+  local dashboard(raw) =
+    local d = render(raw);
+    d {
+      panels: [p for p in d.panels if config.tracesEnabled || (std.get(std.get(p, 'datasource', {}), 'type', '') != 'tempo' && !(p.type == 'row' && p.title == 'Gateway tracing'))],
+      templating+: {
+        list: [
+          v + (if v.type == 'datasource' then { regex: config.datasourceRegex[v.name] } else {})
+          for v in d.templating.list
+          if config.tracesEnabled || v.name != 'traces'
+        ],
+      },
+    },
+  grafanaDashboards: {
+    'openshell-operational-health.json': dashboard(import 'dashboards/openshell-operational-health.json'),
+    'openshell-ocsf-logs.json': dashboard(import 'dashboards/openshell-ocsf-logs.json'),
+    'openshell-operational-logs.json': dashboard(import 'dashboards/openshell-operational-logs.json'),
+  },
+}