diff --git a/.changeset/classify-clickhouse-proxy-failures.md b/.changeset/classify-clickhouse-proxy-failures.md new file mode 100644 index 000000000..0bb5e8e84 --- /dev/null +++ b/.changeset/classify-clickhouse-proxy-failures.md @@ -0,0 +1,9 @@ +--- +'@hyperdx/api': patch +--- + +feat: Classify failed ClickHouse proxy requests on the server span and the +`clickhouse_proxy.query` SLI (`error_category`), including errors ClickHouse +writes after a 200 response. The span also records the upstream status, the +ClickHouse error code and type, and the body of upstream errors that carry no +ClickHouse code. diff --git a/agent_docs/observability.md b/agent_docs/observability.md index 1de5356b4..b8950ac54 100644 --- a/agent_docs/observability.md +++ b/agent_docs/observability.md @@ -1,8 +1,8 @@ # Observability & Instrumentation -ClickStack is an observability product, so our own code should be a shining example -of well-instrumented software. When you add or change a feature, instrument it -as part of the change - not as an afterthought. +ClickStack is an observability product, so our own code should be a shining +example of well-instrumented software. When you add or change a feature, +instrument it as part of the change - not as an afterthought. This guide covers the conventions and the shared helpers. The principles are aligned with the @@ -11,9 +11,9 @@ aligned with the ## TL;DR (the non-negotiables) 1. **Every team-scoped operation carries team + user context.** Use - `setBusinessContext(...)` so `hyperdx.team.id` / `user.id` end up on - the trace. Auth middleware already does this for HTTP requests; background - jobs and other entry points must do it themselves. + `setBusinessContext(...)` so `hyperdx.team.id` / `user.id` end up on the + trace. Auth middleware already does this for HTTP requests; background jobs + and other entry points must do it themselves. 2. **If something is worth a log, put it on the span first.** The active span is the wide event for the current unit of work — attach the fact there as an attribute. If it also marks a countable event worth aggregating (an error, a @@ -30,14 +30,15 @@ aligned with the already in flight, and put point-in-time / gauge-style values (sizes, counts, depths, current state) there rather than in a gauge. This is an internal engineering preference, not a rule against metrics — counters and histograms - remain first-class, feed alerts and SLOs, and many ClickStack deployments depend - heavily on them. See [Wide events over gauges](#wide-events-over-gauges). + remain first-class, feed alerts and SLOs, and many ClickStack deployments + depend heavily on them. See + [Wide events over gauges](#wide-events-over-gauges). 5. **Use the shared helpers** in [`packages/api/src/utils/instrumentation.ts`](../packages/api/src/utils/instrumentation.ts). Don't hand-roll tracer/meter lifecycle. -6. **A new surface that queries ClickHouse declares its query attribution.** - One provider or one `withQueryAttribution` scope at the entry point, not a - change at each query. See [Query attribution](#query-attribution). +6. **A new surface that queries ClickHouse declares its query attribution.** One + provider or one `withQueryAttribution` scope at the entry point, not a change + at each query. See [Query attribution](#query-attribution). ## The shared helper library @@ -116,7 +117,7 @@ Standard attribute keys: | -------------------------- | ---------------------------------------------------------- | | `hyperdx.team.id` | Owning team (multi-tenancy boundary). Set this everywhere. | | `user.id` | Acting user (OTel `user.*` semconv). | -| `user.email` | Acting user's email (OTel `user.*` semconv). | +| `user.email` | Acting user's email (OTel `user.*` semconv). | | `hyperdx..` | Domain IDs, e.g. `hyperdx.alert.id`, `hyperdx.source.id`. | | `feature_flag.` | Evaluated feature/config flag state. | @@ -124,8 +125,8 @@ Standard attribute keys: > **Scope:** this is ClickStack's _internal_ instrumentation philosophy for the > code in this repo — not a recommendation that users pick wide events over -> metrics. Metrics are a first-class ClickStack signal that many operators rely on; -> nothing here discourages them. What follows is simply how _we_ prefer to +> metrics. Metrics are a first-class ClickStack signal that many operators rely +> on; nothing here discourages them. What follows is simply how _we_ prefer to > instrument our own services. We instrument in the spirit of @@ -214,10 +215,10 @@ To make a piece of functionality SLO-able, wrap it with `withOperationMetrics` a streaming proxy). Both emit a standard pair of SLI signals tagged with a stable, low-cardinality `operation` name and `outcome` (`success` | `error`): -| Metric | Type | Use as | -| ------------------------------- | --------- | --------------- | +| Metric | Type | Use as | +| ------------------------------- | --------- | ---------------- | | `hyperdx.operation.requests` | counter | availability SLI | -| `hyperdx.operation.duration_ms` | histogram | latency SLI | +| `hyperdx.operation.duration_ms` | histogram | latency SLI | ```ts import { withOperationMetrics } from '@/utils/instrumentation'; @@ -230,8 +231,8 @@ const chartConfig = await withOperationMetrics( ``` Because every operation reports through the same two metrics, an SLO is just a -filter on `operation` — availability = `outcome:success` / total, latency = -the duration histogram. Keep `operation` a constant (e.g. `ai.assistant`, +filter on `operation` — availability = `outcome:success` / total, latency = the +duration histogram. Keep `operation` a constant (e.g. `ai.assistant`, `clickhouse_proxy.query`); never interpolate IDs or user input into it. Reach for this on functionality with real failure modes worth a target (external dependencies, query proxies, AI calls) — not on thin CRUD that the HTTP @@ -239,18 +240,18 @@ auto-instrumentation and error middleware already cover. ## Query attribution -Every ClickHouse query ClickStack runs is tagged with what asked for it, so a row -in the customer's `system.query_log` points back to a dashboard tile, a saved -search, an alert, or an MCP tool. This is the one piece of our instrumentation -that lands in *their* data rather than ours, so it stays small, carries no -identity, and must never be able to fail a query. +Every ClickHouse query ClickStack runs is tagged with what asked for it, so a +row in the customer's `system.query_log` points back to a dashboard tile, a +saved search, an alert, or an MCP tool. This is the one piece of our +instrumentation that lands in _their_ data rather than ours, so it stays small, +carries no identity, and must never be able to fail a query. Two channels carry it, set by `BaseClickhouseClient.query`: -| Where | Shape | What it answers | -| --- | --- | --- | +| Where | Shape | What it answers | +| ------------- | ----------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------- | | `log_comment` | JSON payload, e.g. `{"v":1,"surface":"dashboard","dashboard":"…","tile":"…"}` | After the fact: `GROUP BY JSONExtractString(log_comment, 'tile')` over `system.query_log` | -| `query_id` | `hdx--` | Right now: reading `system.processes` while a query is still running | +| `query_id` | `hdx--` | Right now: reading `system.processes` while a query is still running | The vocabulary lives in [`packages/common-utils/src/clickhouse/attribution.ts`](../packages/common-utils/src/clickhouse/attribution.ts). @@ -304,14 +305,13 @@ calls `/api/v1/prometheus/query{,_range}`, and `/labels` and `buildLogComment(attribution)` in the `x-hyperdx-query-attribution` header (`QUERY_ATTRIBUTION_HEADER`), and the router parses it with `parseLogComment` into a `withQueryAttribution` scope layered over the auth middleware's `api` -scope. The browser's surface and ids win; `trace` and `label` stay the -server's. Queries the route runs through a ClickHouse client (label lookups -before 26.9, and the `prometheusQuery()`/`prometheusQueryRange()` fallback) -pick the scope up as usual. Requests proxied to ClickHouse's Prometheus HTTP -API (queries on 26.6+, label lookups on 26.9+) are tagged -by `clickhousePrometheusRequest`: `log_comment` as a URL setting, `query_id` as -the `X-ClickHouse-Query-Id` header, because the handler rejects `query_id` as an -unknown setting in the URL. +scope. The browser's surface and ids win; `trace` and `label` stay the server's. +Queries the route runs through a ClickHouse client (label lookups before 26.9, +and the `prometheusQuery()`/`prometheusQueryRange()` fallback) pick the scope up +as usual. Requests proxied to ClickHouse's Prometheus HTTP API (queries on +26.6+, label lookups on 26.9+) are tagged by `clickhousePrometheusRequest`: +`log_comment` as a URL setting, `query_id` as the `X-ClickHouse-Query-Id` +header, because the handler rejects `query_id` as an unknown setting in the URL. ### Rules @@ -356,7 +356,7 @@ unknown setting in the URL. - SLO operation metrics (`withOperationMetrics`) on an external dependency: `packages/api/src/routers/api/ai.ts` - SLO operation metrics (`recordOperationOutcome`) on a streaming proxy: - `packages/api/src/routers/api/clickhouseProxy.ts` + `packages/api/src/utils/proxyOutcome.ts` - End-to-end + sub-operation SLO metrics on a background job (alert evaluation and its data fetch): `packages/api/src/tasks/checkAlerts/index.ts` (`alerts.evaluate`, `alerts.query`) diff --git a/packages/api/src/routers/api/__tests__/clickhouseProxyOutcome.int.test.ts b/packages/api/src/routers/api/__tests__/clickhouseProxyOutcome.int.test.ts new file mode 100644 index 000000000..2e27e8ce9 --- /dev/null +++ b/packages/api/src/routers/api/__tests__/clickhouseProxyOutcome.int.test.ts @@ -0,0 +1,251 @@ +import { INVALID_SPAN_CONTEXT, trace } from '@opentelemetry/api'; +import http from 'http'; + +import { getLoggedInAgent, getServer } from '@/fixtures'; +import Connection from '@/models/connection'; +import * as instrumentation from '@/utils/instrumentation'; + +jest.unmock('http-proxy-middleware'); + +const TIMEOUT_TRAILER = + 'Code: 159. DB::Exception: Timeout exceeded: elapsed 15.0 seconds, maximum: 15. (TIMEOUT_EXCEEDED) (version 25.3.1.1)\n'; + +// Each scenario is what ClickHouse (or the endpoint in front of it) does for +// the query id in the request. +const scenarios: Record< + string, + (req: http.IncomingMessage, res: http.ServerResponse) => void +> = { + ok: (_req, res) => { + res.writeHead(200); + res.end('{"a":1}\n'); + }, + 'mid-stream': (_req, res) => { + res.writeHead(200); + res.write('{"a":1}\n'); + res.end(TIMEOUT_TRAILER); + }, + tagged: (_req, res) => { + const tag = 'abcdefghijklmnop'; + res.writeHead(200, { 'X-ClickHouse-Exception-Tag': tag }); + res.write('{"a":1}\n'); + res.end( + `\r\n__exception__\r\n${tag}\r\n${TIMEOUT_TRAILER}${Buffer.byteLength(TIMEOUT_TRAILER)} ${tag}\r\n__exception__\r\n`, + ); + }, + overloaded: (_req, res) => { + res.writeHead(500, { 'X-ClickHouse-Exception-Code': '202' }); + res.end( + 'Code: 202. DB::Exception: Too many simultaneous queries. (TOO_MANY_SIMULTANEOUS_QUERIES)\n', + ); + }, + unavailable: (_req, res) => { + res.writeHead(503); + res.end(`Service is starting up${' '.repeat(10_000)}