erain · Nov 10, 2026 · Nov 5, 2026 · Nov 6, 2026
diff --git a/AGENTS.md b/AGENTS.md
@@ -27,6 +27,8 @@ internal/store/        memory + postgres backends (schema.sql and schema.go chan
 internal/substrate     The only place that talks to the Substrate Control API
 internal/workspace     git clone / inline files / skills dir (MCP is spec-only; agents read it via the metadata server)
 internal/{tunnel,guest,metadata}  k8s port-forward, in-sandbox guest client, metadata server
+internal/telemetry     ax-server's OpenTelemetry setup: OTLP traces, metrics, and events (Emit, stdout fallback), Prometheus /metrics (:9090), JSON slog with trace IDs
+internal/axattr        OpenTelemetry attribute names (ax.task.*, and Substrate's ate.* for actor identity)
 pkg/apis/v1alpha1      *.proto (source of truth) -> *.pb.go (generated), types.go, resources.go
 test/e2e               Kind-based; compiled by `go test ./...` but self-skips without AX_E2E=true
 hack/                  E2E provisioning scripts
@@ -40,7 +42,7 @@ skills/                Developer skills (SKILL.md format). .agents/skills, .clau
 export GOTOOLCHAIN=auto          # go.mod wants 1.27.1; older local Go auto-downloads it
 go mod tidy && git diff --exit-code go.mod go.sum
 go build ./cmd/ax ./cmd/ax-server ./cmd/ax-task-runner
-go test ./...                    # ~20 s, 16 packages with tests; Postgres + E2E tiers self-skip
+go test ./...                    # ~20 s, 17 packages with tests; Postgres + E2E tiers self-skip
 go vet ./... && gofmt -l .       # not run by CI; both are clean at HEAD
 ```
 

diff --git a/README.md b/README.md
@@ -141,6 +141,7 @@ Want to see the whole lifecycle end to end? Run [`./demo.sh`](demo.sh). It appli
 | [Sandbox](docs/sandbox.md) | See what the runner does on boot and what your command can rely on: metadata server, guest services, environment. |
 | [Runners](docs/runner.md) | Understand the contract between the control plane and the task container, and build your own runner image to replace the default. |
 | [Networking](docs/networking.md) | Reach a running task through the atenet router from the cluster, your laptop, or a gRPC client. |
+| [Observability](docs/observability.md) | Collect ax-server's logs, metrics, traces, and events, and configure where they go. |
 | [Architecture](DESIGN.md) | Understand how the control plane fits together, plus the [API reference](DESIGN.md#api-reference). |
 | [Development](docs/development.md) | Build, test, and ship changes to AX itself. |
 | [Roadmap](docs/roadmap.md) | See planned milestones across core specs, actor architecture, agentic environments, and governance. |

diff --git a/cmd/ax-server/main.go b/cmd/ax-server/main.go
@@ -31,13 +31,15 @@ import (
 	"github.com/google/ax/internal/server"
 	"github.com/google/ax/internal/store/postgres"
 	"github.com/google/ax/internal/substrate"
+	"github.com/google/ax/internal/telemetry"
 	"github.com/google/ax/pkg/apis/v1alpha1"
 	"github.com/jackc/pgx/v5/pgconn"
 )
 
 func main() {
 	var (
 		listenAddr              string
+		metricsAddr             string
 		storeURL                string
 		substrateEndpoint       string
 		substrateAuthority      string
@@ -51,6 +53,7 @@ func main() {
 	)
 
 	flag.StringVar(&listenAddr, "addr", ":8080", "HTTP listen address")
+	flag.StringVar(&metricsAddr, "metrics-addr", ":9090", "Listen address for Prometheus /metrics; empty disables it")
 	flag.StringVar(&storeURL, "store", "postgres://ax@localhost:5432/ax?sslmode=disable", "PostgreSQL connection URL (env: AX_STORE). libpq variables such as PGPASSWORD fill in what it leaves out")
 	flag.StringVar(&substrateEndpoint, "substrate-endpoint", "api.ate-system.svc.cluster.local:443", "Agent Substrate Control API endpoint")
 	flag.StringVar(&substrateAuthority, "substrate-authority", "api.ate-system.svc", "Authority / TLS ServerName for Substrate endpoint")
@@ -76,11 +79,25 @@ func main() {
 		defaultTaskImage = envImage
 	}
 
-	logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}))
+	logger := slog.New(telemetry.NewLogHandler(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo})))
 	slog.SetDefault(logger)
 
+	tel, err := telemetry.Init(context.Background(), "ax-server")
+	if err != nil {
+		slog.Error("initializing telemetry", "error", err)
+		os.Exit(1)
+	}
+	defer func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		if err := tel.Shutdown(ctx); err != nil {
+			slog.Warn("flushing telemetry", "error", err)
+		}
+	}()
+
 	slog.Info("starting ax-server",
 		"listenAddr", listenAddr,
+		"metricsAddr", metricsAddr,
 		"store", describeStore(storeURL),
 		"substrateEndpoint", substrateEndpoint,
 		"template", defaultTemplate,
@@ -133,12 +150,29 @@ func main() {
 		}
 	}()
 
+	// Metrics get their own listener so the probe and API port stays as it is.
+	var metricsServer *http.Server
+	if metricsAddr != "" {
+		mux := http.NewServeMux()
+		mux.Handle("GET /metrics", tel.MetricsHandler())
+		metricsServer = &http.Server{Addr: metricsAddr, Handler: mux, ReadHeaderTimeout: 10 * time.Second}
+		go func() {
+			if err := metricsServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+				slog.Error("metrics server failed", "error", err)
+				os.Exit(1)
+			}
+		}()
+	}
+
 	<-ctx.Done()
 	slog.Info("shutting down ax-server")
 
 	shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer shutdownCancel()
 	_ = httpServer.Shutdown(shutdownCtx)
+	if metricsServer != nil {
+		_ = metricsServer.Shutdown(shutdownCtx)
+	}
 }
 
 // describeStore summarizes a store connection string for logs. It parses the

diff --git a/deploy/ax-server.yaml b/deploy/ax-server.yaml
@@ -47,6 +47,25 @@ roleRef:
   kind: ClusterRole
   name: ax-server
 ---
+# OpenTelemetry settings for ax-server, read with envFrom. Point ax-server at a
+# collector here; see docs/observability.md. On a cluster that runs Agent
+# Substrate, the endpoint can be the one in ate-system's ate-otel-config.
+# Editing this ConfigMap does not restart ax-server; follow a change with
+# `kubectl -n ax-system rollout restart deployment/ax-server`.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ax-otel-config
+  namespace: ax-system
+data:
+  # OTLP export is off until you set a collector. Metrics stay on /metrics
+  # (port 9090) and events go to stdout. To export, set the endpoint and
+  # remove the two exporter lines; add OTEL_LOGS_EXPORTER: otlp to send
+  # events too.
+  # OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector.observability.svc:4317
+  OTEL_TRACES_EXPORTER: none
+  OTEL_METRICS_EXPORTER: prometheus
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -79,7 +98,26 @@ spec:
             - "--substrate-ca-file=/run/servicedns-ca/trust-bundle.pem"
             - "--template=default-template"
             - "--template-atespace=ax-system"
+          envFrom:
+            - configMapRef:
+                name: ax-otel-config
           env:
+            # The resource attributes come from the downward API. Kubernetes
+            # expands $(VAR) only for a variable listed earlier.
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_UID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.uid
+            - name: OTEL_RESOURCE_ATTRIBUTES
+              value: "k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID)"
             - name: PGPASSWORD
               valueFrom:
                 secretKeyRef:
@@ -92,6 +130,8 @@ spec:
           ports:
             - containerPort: 8080
               name: http
+            - containerPort: 9090
+              name: metrics
           resources:
             requests:
               cpu: 100m
@@ -162,6 +202,9 @@ spec:
     - port: 8080
       targetPort: 8080
       name: http
+    - port: 9090
+      targetPort: 9090
+      name: metrics
   selector:
     app.kubernetes.io/name: ax-server
 
diff --git a/docs/observability.md b/docs/observability.md
@@ -0,0 +1,106 @@
+# Observability
+
+ax-server sets up OpenTelemetry itself and needs no sidecar. It exports
+traces, metrics, and events over OTLP/gRPC to a collector you choose, and
+serves metrics for Prometheus as well.
+
+| Signal | Where it goes | Default in the code | Default in `deploy/ax-server.yaml` |
+|---|---|---|---|
+| Logs | stdout, one JSON object per line | Always on | Always on |
+| Metrics | OTLP push, and Prometheus `/metrics` on port 9090 (`--metrics-addr`) | Both on | `/metrics` only |
+| Traces | OTLP push; 10% of new traces sampled | On | Off |
+| Events | stdout as JSON log lines, or OTLP log records | stdout | stdout |
+
+Health probes and the AX API stay on port 8080. The bundled manifests expose
+port 9090 as `metrics` on the `ax-server` Service, so a Prometheus scrape
+config or a GKE `PodMonitoring` can target it. `--metrics-addr=` turns
+`/metrics` off.
+
+## Connect a collector
+
+The manifests read every `OTEL_*` setting from the `ax-otel-config` ConfigMap
+in `ax-system`. `deploy/ax-server.yaml` ships with OTLP export off, so an
+installation without a collector logs no export errors and loses nothing:
+metrics stay on `/metrics` and events go to stdout. To export, edit the
+ConfigMap and restart ax-server:
+
+```yaml
+data:
+  OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector.observability.svc:4317
+  OTEL_LOGS_EXPORTER: otlp        # also send events; add ,console to keep them on stdout
+  # delete OTEL_TRACES_EXPORTER: none and OTEL_METRICS_EXPORTER: prometheus
+```
+
+```bash
+kubectl -n ax-system rollout restart deployment/ax-server
+```
+
+On a cluster that runs Agent Substrate, the endpoint can be the one in
+`ate-otel-config` in `ate-system`, so ax-server and Substrate telemetry reach
+the same backend. The E2E manifests point at the collector that Substrate's
+kind overlay deploys.
+
+The manifests also set `OTEL_RESOURCE_ATTRIBUTES` from the downward API:
+`k8s.namespace.name`, `k8s.pod.name`, `k8s.pod.uid`, and `service.instance.id`
+set to the pod UID, as Substrate does. This variable is in the Deployment's
+`env`, which overrides `envFrom`, so the same key in `ax-otel-config` has no
+effect. Add other resource attributes to the value in the Deployment.
+
+## Configuration
+
+Telemetry follows the standard
+[OpenTelemetry environment variables](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/).
+As the specification asks, ax-server warns about a value it does not
+recognize and ignores it, which keeps the default in the table below.
+
+| Variable | Effect |
+|---|---|
+| `OTEL_EXPORTER_OTLP_ENDPOINT` | Collector address, for example `http://otel-collector.observability.svc:4317`. The SDK default is `localhost:4317`. An `http://` endpoint is reached without TLS. |
+| `OTEL_TRACES_EXPORTER` | `otlp` (default) or `none`. With `none`, traces are still sampled and propagated, only not sent. |
+| `OTEL_METRICS_EXPORTER` | Controls only the OTLP push: `otlp` (default) turns it on; `prometheus` or `none` turns it off. `/metrics` is controlled by `--metrics-addr`. |
+| `OTEL_LOGS_EXPORTER` | Where events go: `none` (default) writes them to stdout; `otlp` sends them to the collector instead; `otlp,console` does both. |
+| `OTEL_METRIC_EXPORT_INTERVAL` | Milliseconds between metric pushes; the SDK default is 60000. |
+| `OTEL_TRACES_SAMPLER`, `OTEL_TRACES_SAMPLER_ARG` | Replace the default sampler, `parentbased_traceidratio` at `0.1`. With a `parentbased_*` sampler, ax-server always follows the sampling decision of the caller. An unknown sampler, or a ratio sampler whose argument is missing or outside [0, 1], keeps the default. |
+| `OTEL_SERVICE_NAME`, `OTEL_RESOURCE_ATTRIBUTES` | Override the resource. The code defaults are `service.name=ax-server` and `service.instance.id` set to the hostname. |
+| `OTEL_GO_X_CARDINALITY_LIMIT` | Attribute sets each metric instrument keeps per collection; the SDK default is 2000. |
+
+The OTLP metric push runs every interval whether or not anything recorded a
+metric, so with the push on and no reachable collector the SDK logs an export
+error each interval. Traces and events are exported only once ax-server
+records them.
+
+## Logs
+
+ax-server writes structured JSON logs to stdout. A line logged while a trace
+is active carries `trace_id`, `span_id`, and `trace_flags`, the fields the
+OpenTelemetry specification defines for trace context in non-OTLP logs, so a
+log backend can link it to the trace. ax-server's log lines are not sent over
+OTLP; collect them from stdout as you would for any pod.
+
+## Events
+
+Structured events, such as task state changes, use the OTLP logs signal:
+each is a log record with an event name. With `OTEL_LOGS_EXPORTER` unset or
+`none`, an event is written to stdout as a JSON log line instead, with the
+event name in `event`, so it is never dropped because OTLP is off. This
+follows Agent Substrate, whose actor events default to stdout the same way.
+
+OTLP event records are batched, so a slow collector never delays the task
+operation that emits them. Records still queued when ax-server exits
+ungracefully are lost, unless `console` also wrote them to stdout.
+
+## Metrics
+
+Each instrument keeps at most 2000 attribute sets per collection (the SDK
+default, which `OTEL_GO_X_CARDINALITY_LIMIT` overrides). Further sets are
+folded into one series labelled `otel_metric_overflow="true"`, so an
+unbounded label cannot exhaust memory. `/metrics` also serves the Go runtime
+and process metrics (`go_*`, `process_*`); they are not pushed over OTLP.
+
+## Attribute names
+
+Spans, events, and log attributes use the names in
+[`internal/axattr`](../internal/axattr/axattr.go). Attributes that identify an
+actor reuse Agent Substrate's names, `ate.atespace` and `ate.actor.uid`, so
+ax-server and Substrate records join on them. Identity attributes such as
+`ax.task.id`, `ate.atespace`, and `ate.actor.uid` never appear on metrics.
diff --git a/go.mod b/go.mod
@@ -6,21 +6,46 @@ require (
 	github.com/agent-substrate/env v0.0.11-0.20260912052224-4468a200b170
 	github.com/cenkalti/backoff/v5 v5.0.3
 	github.com/jackc/pgx/v5 v5.11.0
+	github.com/prometheus/client_golang v1.24.1
 	go.ate.dev/substrate v1.0.0
+	go.opentelemetry.io/otel v1.45.0
+	go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.21.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.45.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0
+	go.opentelemetry.io/otel/exporters/prometheus v0.65.0
+	go.opentelemetry.io/otel/log v0.21.0
+	go.opentelemetry.io/otel/metric v1.45.0
+	go.opentelemetry.io/otel/sdk v1.45.0
+	go.opentelemetry.io/otel/sdk/log v0.21.0
+	go.opentelemetry.io/otel/sdk/metric v1.45.0
+	go.opentelemetry.io/otel/trace v1.45.0
+	go.opentelemetry.io/proto/otlp v1.11.0
 	google.golang.org/grpc v1.83.2
 	google.golang.org/protobuf v1.36.12
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/go-logr/logr v1.4.4 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
-	github.com/kr/pretty v0.3.1 // indirect
-	github.com/rogpeppe/go-internal v1.14.1 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.70.1 // indirect
+	github.com/prometheus/otlptranslator v1.0.0 // indirect
+	github.com/prometheus/procfs v0.21.1 // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect
 	golang.org/x/net v0.60.0 // indirect
 	golang.org/x/sync v0.23.0 // indirect
 	golang.org/x/sys v0.48.0 // indirect
 	golang.org/x/text v0.42.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect
 )