diff --git a/.changeset/source-vars-2.md b/.changeset/source-vars-2.md new file mode 100644 index 000000000..32fd79c38 --- /dev/null +++ b/.changeset/source-vars-2.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/api': minor +--- + +feat: support data source and connection variables in the external API and MCP diff --git a/packages/api/openapi.json b/packages/api/openapi.json index 65ea9dff2..cfa7b0b8c 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -1883,7 +1883,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -1980,7 +1980,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2057,7 +2057,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2144,7 +2144,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2227,7 +2227,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2288,7 +2288,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2343,7 +2343,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2437,7 +2437,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2495,7 +2495,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2559,7 +2559,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2599,7 +2599,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to mine patterns from.", + "description": "ID of the data source to mine patterns from, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2664,7 +2664,7 @@ }, "connectionId": { "type": "string", - "description": "ID of the ClickHouse connection to execute the query against.", + "description": "ID of the ClickHouse connection to execute the query against, or\n`$connectionVariable` of a CONNECTION filter on the dashboard.\n", "example": "65f5e4a3b9e77c001a567890" }, "sqlTemplate": { @@ -2675,7 +2675,7 @@ }, "sourceId": { "type": "string", - "description": "Optional ID of the data source associated with this Raw SQL chart. Used for applying dashboard filters.", + "description": "Optional ID of the data source associated with this Raw SQL chart, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. Used for applying dashboard filters.", "example": "65f5e4a3b9e77c001a567890" }, "numberFormat": { @@ -4031,6 +4031,12 @@ }, { "$ref": "#/components/schemas/AdhocFilterInput" + }, + { + "$ref": "#/components/schemas/DataSourceFilterInput" + }, + { + "$ref": "#/components/schemas/ConnectionFilterInput" } ], "discriminator": { @@ -4040,7 +4046,179 @@ "STATIC_LIST": "#/components/schemas/StaticListFilterInput", "PROMETHEUS_LABEL": "#/components/schemas/PrometheusLabelFilterInput", "SQL_QUERY": "#/components/schemas/SqlQueryFilterInput", - "ADHOC": "#/components/schemas/AdhocFilterInput" + "ADHOC": "#/components/schemas/AdhocFilterInput", + "DATA_SOURCE": "#/components/schemas/DataSourceFilterInput", + "CONNECTION": "#/components/schemas/ConnectionFilterInput" + } + } + }, + "DataSourceFilterInput": { + "type": "object", + "description": "A variable whose single, required value is a source. Tiles reference it\nas `$variableName` in their sourceId, and other filters in sourceId or\nappliesToSourceIds, so viewers can switch the source those read. Tiles\nthat use it do not load until a source is selected, and cannot have\nalerts.\n", + "required": [ + "type", + "name", + "allowedSourceKinds" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "DATA_SOURCE" + ], + "description": "Filter type discriminator. Must be \"DATA_SOURCE\".", + "example": "DATA_SOURCE" + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name of the variable's dropdown", + "example": "Logs source" + }, + "variableName": { + "type": "string", + "maxLength": 64, + "pattern": "^[a-zA-Z][a-zA-Z0-9_]*$", + "description": "Token tiles and filters reference the selected source by, as `$variableName`.", + "example": "logs" + }, + "isGlobalRequirement": { + "type": "boolean", + "default": false, + "description": "Blocks every tile on the dashboard until a value is selected. False\n(the default) blocks only the tiles that reference the variable.\n", + "example": true + }, + "minSelections": { + "type": "integer", + "enum": [ + 1 + ], + "default": 1, + "description": "Must be 1 if provided. The variable always requires a selection.", + "example": 1 + }, + "maxSelections": { + "type": "integer", + "enum": [ + 1 + ], + "default": 1, + "description": "Must be 1 if provided. The variable is always single-select.", + "example": 1 + }, + "isBroadcastEnabled": { + "type": "boolean", + "enum": [ + false + ], + "default": false, + "description": "Must be false or omitted.", + "example": false + }, + "isVariableEnabled": { + "type": "boolean", + "enum": [ + true + ], + "default": true, + "description": "Must be true if provided, and is true when omitted.", + "example": true + }, + "allowedSourceKinds": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "log", + "trace", + "session", + "metric", + "promql" + ] + }, + "description": "Kinds of sources offered. A field can reference the variable if it\naccepts at least one listed kind. A tile whose selected source is a\nkind it cannot use shows an error instead of running.\n", + "example": [ + "log" + ] + }, + "connectionId": { + "type": "string", + "description": "Optional connection the offered sources are limited to: a\nconnection ID, or `$name` of a CONNECTION filter on the dashboard.\n", + "example": "$conn" + } + } + }, + "ConnectionFilterInput": { + "type": "object", + "description": "A variable whose single, required value is a connection. Raw SQL tiles\nreference it as `$variableName` in their connectionId, and data source\nvariables in theirs. Tiles that use it do not load until a connection is\nselected, and cannot have alerts.\n", + "required": [ + "type", + "name" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "CONNECTION" + ], + "description": "Filter type discriminator. Must be \"CONNECTION\".", + "example": "CONNECTION" + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name of the variable's dropdown", + "example": "Connection" + }, + "variableName": { + "type": "string", + "maxLength": 64, + "pattern": "^[a-zA-Z][a-zA-Z0-9_]*$", + "description": "Token raw SQL tiles reference the selected connection by, as `$variableName`.", + "example": "conn" + }, + "isGlobalRequirement": { + "type": "boolean", + "default": false, + "description": "Blocks every tile on the dashboard until a value is selected. False\n(the default) blocks only the tiles that reference the variable.\n", + "example": true + }, + "minSelections": { + "type": "integer", + "enum": [ + 1 + ], + "default": 1, + "description": "Must be 1 if provided. The variable always requires a selection.", + "example": 1 + }, + "maxSelections": { + "type": "integer", + "enum": [ + 1 + ], + "default": 1, + "description": "Must be 1 if provided. The variable is always single-select.", + "example": 1 + }, + "isBroadcastEnabled": { + "type": "boolean", + "enum": [ + false + ], + "default": false, + "description": "Must be false or omitted.", + "example": false + }, + "isVariableEnabled": { + "type": "boolean", + "enum": [ + true + ], + "default": true, + "description": "Must be true if provided, and is true when omitted.", + "example": true } } }, @@ -4076,7 +4254,7 @@ }, "sourceId": { "type": "string", - "description": "Source ID this dashboard filter key applies to", + "description": "Source ID this dashboard filter key applies to, or `$name` of a DATA_SOURCE filter on the dashboard", "example": "65f5e4a3b9e77c001a111111" }, "sourceMetricType": { @@ -4469,7 +4647,7 @@ "items": { "type": "string" }, - "description": "IDs of the sources whose keys and values the filter suggests, all of sourceType.", + "description": "IDs of the sources whose keys and values the filter suggests, all of sourceType, or `$name` of data source variables.", "example": [ "65f5e4a3b9e77c001a111111" ] @@ -4493,7 +4671,7 @@ "items": { "type": "string" }, - "description": "IDs of the sources whose tiles the conditions apply to, all of\nsourceType. Omit or send an empty array to apply to the tiles of\nthe filter's sources.\n", + "description": "IDs of the sources whose tiles the conditions apply to, all of\nsourceType, or `$name` of data source variables. Omit or send an\nempty array to apply to the tiles of the filter's sources.\n", "example": [ "65f5e4a3b9e77c001a111111" ] diff --git a/packages/api/src/mcp/__tests__/dashboards/patchDashboard.int.test.ts b/packages/api/src/mcp/__tests__/dashboards/patchDashboard.int.test.ts index eef1d0838..1285ec729 100644 --- a/packages/api/src/mcp/__tests__/dashboards/patchDashboard.int.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/patchDashboard.int.test.ts @@ -1400,4 +1400,86 @@ describe('MCP Dashboard Tools - clickstack_patch_dashboard', () => { expect(tile.config.showOperandSeries).toBe(false); }); }); + + describe('reference variables', () => { + const createDashboard = () => + new Dashboard({ + name: 'Variable dashboard', + tiles: [ + { + id: 'tile', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + name: 'Tile', + source: ctx.traceSource._id.toString(), + displayType: 'line', + select: [{ aggFn: 'count', valueExpression: '' }], + where: '', + }, + }, + ], + filters: [ + { + id: 'ds-filter', + type: 'DATA_SOURCE', + name: 'Traces source', + variableName: 'traces', + allowedSourceKinds: [SourceKind.Trace], + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }, + ], + team: ctx.team._id, + }).save(); + + const patchSource = (dashboardId: string, sourceId: string) => + callTool(ctx.client!, 'clickstack_patch_dashboard', { + dashboardId, + tileId: 'tile', + tile: { + name: 'Tile', + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + }); + + it('patches a tile to read from a declared data source variable', async () => { + const dashboard = await createDashboard(); + + const result = await patchSource(dashboard._id.toString(), '$traces'); + + expect(result.isError).toBeFalsy(); + const stored = await Dashboard.findById(dashboard._id).lean(); + expect(stored?.tiles[0].config).toMatchObject({ source: '$traces' }); + }); + + it('rejects a patch referencing an undeclared variable', async () => { + const dashboard = await createDashboard(); + + const result = await patchSource(dashboard._id.toString(), '$nope'); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain( + 'does not name a data source variable', + ); + }); + + it('accepts a patch that keeps a reference the dashboard no longer satisfies', async () => { + const dashboard = await createDashboard(); + await patchSource(dashboard._id.toString(), '$traces'); + await Dashboard.updateOne({ _id: dashboard._id }, { filters: [] }); + + const result = await patchSource(dashboard._id.toString(), '$traces'); + + expect(result.isError).toBeFalsy(); + }); + }); }); diff --git a/packages/api/src/mcp/__tests__/dashboards/queryTile.int.test.ts b/packages/api/src/mcp/__tests__/dashboards/queryTile.int.test.ts index 48de1b8f6..5042fc502 100644 --- a/packages/api/src/mcp/__tests__/dashboards/queryTile.int.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/queryTile.int.test.ts @@ -11,6 +11,7 @@ import { seedExponentialHistogramMetric, } from '@/fixtures'; import { callTool, getFirstText } from '@/mcp/__tests__/mcpTestUtils'; +import Dashboard from '@/models/dashboard'; import { Source } from '@/models/source'; import { setupDashboardTests } from './setup'; @@ -1373,4 +1374,85 @@ describe('MCP Dashboard Tools - clickstack_query_tile', () => { expect(rows.map(r => r.ServiceName)).toEqual(['svc-e', 'svc-d', 'svc-c']); }); }); + + describe('reference variables', () => { + const createVariableDashboard = (savedFilterValues: unknown[] = []) => + new Dashboard({ + name: 'Variable dashboard', + tiles: [ + { + id: 'variable-tile', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + name: 'Count', + source: '$traces', + displayType: 'number', + select: [{ aggFn: 'count', valueExpression: '' }], + where: '', + }, + }, + ], + filters: [ + { + id: 'ds-filter', + type: 'DATA_SOURCE', + name: 'Traces source', + variableName: 'traces', + allowedSourceKinds: [SourceKind.Trace], + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }, + ], + savedFilterValues, + team: ctx.team._id, + }).save(); + + it("runs a tile on its data source variable's saved selection", async () => { + const dashboard = await createVariableDashboard([ + { + type: 'variable', + name: 'traces', + values: [ctx.traceSource._id.toString()], + }, + ]); + + const result = await callTool(ctx.client!, 'clickstack_query_tile', { + dashboardId: dashboard._id.toString(), + tileId: 'variable-tile', + }); + + expect(result.isError).toBeFalsy(); + }); + + it('runs a tile on a selection passed in variableValues', async () => { + const dashboard = await createVariableDashboard(); + + const result = await callTool(ctx.client!, 'clickstack_query_tile', { + dashboardId: dashboard._id.toString(), + tileId: 'variable-tile', + variableValues: [ + { name: 'traces', values: [ctx.traceSource._id.toString()] }, + ], + }); + + expect(result.isError).toBeFalsy(); + }); + + it('reports a data source variable with no selection', async () => { + const dashboard = await createVariableDashboard(); + + const result = await callTool(ctx.client!, 'clickstack_query_tile', { + dashboardId: dashboard._id.toString(), + tileId: 'variable-tile', + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('$traces variable'); + }); + }); }); diff --git a/packages/api/src/mcp/__tests__/dashboards/queryTiles.int.test.ts b/packages/api/src/mcp/__tests__/dashboards/queryTiles.int.test.ts index afaeff978..6b154c40e 100644 --- a/packages/api/src/mcp/__tests__/dashboards/queryTiles.int.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/queryTiles.int.test.ts @@ -594,4 +594,59 @@ describe('MCP Dashboard Tools - clickstack_query_tiles', () => { expect(text).toContain('prod, staging'); }); }); + + it('reports a tile whose data source variable has no selection without failing the batch', async () => { + const sourceId = ctx.traceSource._id.toString(); + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'Batch reference variable test', + tiles: [ + { + name: 'Fixed', + config: { + displayType: 'number', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + { + name: 'Variable', + config: { + displayType: 'number', + sourceId: '$traces', + select: [{ aggFn: 'count' }], + }, + }, + ], + filters: [ + { + type: 'DATA_SOURCE', + name: 'Traces source', + variableName: 'traces', + allowedSourceKinds: [SourceKind.Trace], + }, + ], + }, + ); + expect(createResult.isError).toBeFalsy(); + const dashboard = JSON.parse(getFirstText(createResult)); + + const result = await callTool(ctx.client!, 'clickstack_query_tiles', { + dashboardId: dashboard.id, + ...wideRange(), + }); + + expect(result.isError).toBeFalsy(); + const { tiles } = JSON.parse(getFirstText(result)); + expect(tiles).toEqual([ + expect.objectContaining({ name: 'Fixed', status: 'ok' }), + expect.objectContaining({ + name: 'Variable', + status: 'error', + error: expect.stringContaining('$traces variable'), + }), + ]); + }); }); diff --git a/packages/api/src/mcp/__tests__/dashboards/queryTilesHandler.test.ts b/packages/api/src/mcp/__tests__/dashboards/queryTilesHandler.test.ts index 3b270d960..597ef883c 100644 --- a/packages/api/src/mcp/__tests__/dashboards/queryTilesHandler.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/queryTilesHandler.test.ts @@ -17,6 +17,15 @@ jest.mock('@/mcp/tools/query/helpers', () => { }; }); +jest.mock('@/mcp/tools/dashboards/variables', () => { + const actual = jest.requireActual('@/mcp/tools/dashboards/variables'); + return { + ...actual, + getReferenceTargets: () => + Promise.resolve({ sources: [], connectionIds: new Set() }), + }; +}); + jest.mock('@/models/dashboard', () => ({ __esModule: true, default: { findOne: (...args: unknown[]) => mockFindOne(...args) }, diff --git a/packages/api/src/mcp/__tests__/dashboards/saveDashboard.int.test.ts b/packages/api/src/mcp/__tests__/dashboards/saveDashboard.int.test.ts index 9624eca73..721e25064 100644 --- a/packages/api/src/mcp/__tests__/dashboards/saveDashboard.int.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/saveDashboard.int.test.ts @@ -4132,4 +4132,107 @@ describe('MCP Dashboard Tools - clickstack_save_dashboard', () => { ]); }); }); + + describe('reference variables', () => { + const filters = [ + { + type: 'CONNECTION', + name: 'Connection', + variableName: 'conn', + isGlobalRequirement: true, + }, + { + type: 'DATA_SOURCE', + name: 'Traces source', + variableName: 'traces', + allowedSourceKinds: [SourceKind.Trace], + connectionId: '$conn', + }, + ]; + + it('saves and returns variables and tiles that reference them', async () => { + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Variable dashboard', + tiles: [ + { + name: 'Builder', + config: { + displayType: 'line', + sourceId: '$traces', + select: [{ aggFn: 'count' }], + }, + }, + { + name: 'Raw SQL', + config: { + configType: 'sql', + displayType: 'table', + connectionId: '$conn', + sqlTemplate: 'SELECT 1', + }, + }, + ], + filters, + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.filters).toEqual([ + expect.objectContaining(filters[0]), + expect.objectContaining(filters[1]), + ]); + expect( + output.tiles.map((t: ExternalDashboardTileWithId) => t.config), + ).toEqual([ + expect.objectContaining({ sourceId: '$traces' }), + expect.objectContaining({ connectionId: '$conn' }), + ]); + }); + + it('rejects a tile referencing an undeclared variable', async () => { + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Variable dashboard', + tiles: [ + { + name: 'Builder', + config: { + displayType: 'line', + sourceId: '$traces', + select: [{ aggFn: 'count' }], + }, + }, + ], + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain( + 'does not name a data source variable', + ); + }); + + it('checks an update without filters against the stored variables', async () => { + const builderTile = (sourceId: string) => ({ + name: 'Builder', + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'count' }], + }, + }); + const created = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Variable dashboard', + tiles: [builderTile(ctx.traceSource._id.toString())], + filters, + }); + expect(created.isError).toBeFalsy(); + + const updated = await callTool(ctx.client!, 'clickstack_save_dashboard', { + id: JSON.parse(getFirstText(created)).id, + name: 'Variable dashboard', + tiles: [builderTile('$traces')], + }); + + expect(updated.isError).toBeFalsy(); + }); + }); }); diff --git a/packages/api/src/mcp/__tests__/dashboards/validation.test.ts b/packages/api/src/mcp/__tests__/dashboards/validation.test.ts index bd6ffe6a6..0e9034c09 100644 --- a/packages/api/src/mcp/__tests__/dashboards/validation.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/validation.test.ts @@ -11,6 +11,7 @@ import { getRawSqlMissingSourceError, getRawSqlTileMacroWarnings, getRawSqlTilesMissingRequiredSource, + getSavedFilterValueWarnings, getTileVariableWarnings, } from '@/mcp/tools/dashboards/validation'; import type { @@ -982,3 +983,31 @@ describe('getFilterVariableWarnings', () => { }); }); }); + +describe('getSavedFilterValueWarnings', () => { + const filters = [ + { type: 'DATA_SOURCE' as const, name: 'Traces', variableName: 'traces' }, + { type: 'CONNECTION' as const, name: 'Cluster', variableName: 'conn' }, + ]; + + it('accepts saved values for data source and connection variables', () => { + expect( + getSavedFilterValueWarnings( + [ + { type: 'variable', name: 'traces', values: ['s1'] }, + { type: 'variable', name: 'conn', values: ['c1'] }, + ], + filters, + ), + ).toEqual([]); + }); + + it('warns about a saved value no filter declares', () => { + expect( + getSavedFilterValueWarnings( + [{ type: 'variable', name: 'nope', values: ['x'] }], + filters, + ), + ).toHaveLength(1); + }); +}); diff --git a/packages/api/src/mcp/__tests__/dashboards/variables.test.ts b/packages/api/src/mcp/__tests__/dashboards/variables.test.ts index 6d3bcc42b..9d5e0e45a 100644 --- a/packages/api/src/mcp/__tests__/dashboards/variables.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/variables.test.ts @@ -1,10 +1,16 @@ +import { SourceKind } from '@hyperdx/common-utils/dist/types'; import mongoose from 'mongoose'; import { + type ReferenceTargets, resolveDashboardVariables, + resolveTileVariableReferences, withResolvedFilterVariableNames, } from '@/mcp/tools/dashboards/variables'; -import type { ExternalDashboardFilterWithId } from '@/utils/zod'; +import type { + ExternalDashboardFilterWithId, + ExternalDashboardTileWithId, +} from '@/utils/zod'; const sourceId = new mongoose.Types.ObjectId().toString(); @@ -322,3 +328,282 @@ describe('withResolvedFilterVariableNames', () => { ]); }); }); + +describe('resolveTileVariableReferences', () => { + const objectId = () => new mongoose.Types.ObjectId().toString(); + const connectionId = objectId(); + const otherConnectionId = objectId(); + const traceSourceId = objectId(); + const logSourceId = objectId(); + const metricSourceId = objectId(); + const promqlSourceId = objectId(); + const otherConnectionLogSourceId = objectId(); + const targets: ReferenceTargets = { + sources: [ + { id: traceSourceId, kind: SourceKind.Trace, connection: connectionId }, + { id: logSourceId, kind: SourceKind.Log, connection: connectionId }, + { id: metricSourceId, kind: SourceKind.Metric, connection: connectionId }, + { id: promqlSourceId, kind: SourceKind.Promql, connection: connectionId }, + { + id: otherConnectionLogSourceId, + kind: SourceKind.Log, + connection: otherConnectionId, + }, + ], + connectionIds: new Set([connectionId, otherConnectionId]), + }; + + const referenceVariableFields = { + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + } as const; + const connectionVariable: ExternalDashboardFilterWithId = { + ...referenceVariableFields, + id: objectId(), + type: 'CONNECTION', + name: 'Connection', + variableName: 'conn', + }; + const tracesVariable = ( + connection?: string, + ): ExternalDashboardFilterWithId => ({ + ...referenceVariableFields, + id: objectId(), + type: 'DATA_SOURCE', + name: 'Traces source', + variableName: 'traces', + allowedSourceKinds: [SourceKind.Trace], + connectionId: connection, + }); + + const sourceVariable = ( + allowedSourceKinds: SourceKind[], + ): ExternalDashboardFilterWithId => ({ + ...referenceVariableFields, + id: objectId(), + type: 'DATA_SOURCE', + name: 'Source', + variableName: 'src', + allowedSourceKinds, + }); + + const builderTile: ExternalDashboardTileWithId = { + id: 'builder', + name: 'Builder', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + displayType: 'number', + sourceId: '$traces', + select: [{ aggFn: 'count', where: '' }], + }, + }; + const rawSqlTile: ExternalDashboardTileWithId = { + id: 'raw-sql', + name: 'Raw SQL', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + configType: 'sql', + displayType: 'table', + connectionId: '$conn', + sqlTemplate: 'SELECT 1', + }, + }; + + const promqlTile: ExternalDashboardTileWithId = { + id: 'promql', + name: 'PromQL', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + configType: 'promql', + displayType: 'line', + sourceId: '$src', + expressions: [{ expression: 'up' }], + }, + } as ExternalDashboardTileWithId; + const searchTile: ExternalDashboardTileWithId = { + id: 'search', + name: 'Search', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + displayType: 'search', + sourceId: '$src', + select: '', + where: '', + whereLanguage: 'sql', + }, + }; + const rawSqlSourceTile: ExternalDashboardTileWithId = { + ...rawSqlTile, + id: 'raw-sql-source', + config: { + configType: 'sql', + displayType: 'table', + connectionId: '$conn', + sourceId: '$src', + sqlTemplate: 'SELECT 1', + }, + }; + + const resolve = ( + tile: ExternalDashboardTileWithId, + filters: ExternalDashboardFilterWithId[], + selections: Record, + ) => + resolveTileVariableReferences( + tile, + filters, + Object.entries(selections).map(([name, values]) => ({ name, values })), + [], + targets, + ); + + it("resolves a connection reference to the variable's selection", () => { + const result = resolve(rawSqlTile, [connectionVariable], { + conn: [connectionId], + }); + + expect(result).toEqual({ + tile: { + ...rawSqlTile, + config: { ...rawSqlTile.config, connectionId }, + }, + }); + }); + + it('resolves a source on the connection its connection variable selects', () => { + const result = resolve( + builderTile, + [connectionVariable, tracesVariable('$conn')], + { conn: [connectionId], traces: [traceSourceId] }, + ); + + expect(result).toEqual({ + tile: { + ...builderTile, + config: { ...builderTile.config, sourceId: traceSourceId }, + }, + }); + }); + + it('resolves a source of a kind the tile accepts', () => { + const result = resolve( + promqlTile, + [sourceVariable([SourceKind.Log, SourceKind.Promql])], + { src: [promqlSourceId] }, + ); + + expect(result).toEqual({ + tile: { + ...promqlTile, + config: { ...promqlTile.config, sourceId: promqlSourceId }, + }, + }); + }); + + it.each<{ + case: string; + tile: ExternalDashboardTileWithId; + selection: string; + kinds: SourceKind[]; + }>([ + { + case: 'a log source on a PromQL tile', + tile: promqlTile, + selection: logSourceId, + kinds: [SourceKind.Log, SourceKind.Promql], + }, + { + case: 'a metric source on a search tile', + tile: searchTile, + selection: metricSourceId, + kinds: [SourceKind.Log, SourceKind.Metric], + }, + ])('rejects $case', ({ tile, selection, kinds }) => { + expect( + resolve(tile, [sourceVariable(kinds)], { src: [selection] }), + ).toEqual({ + error: expect.stringContaining('this tile only accepts'), + }); + }); + + it('resolves a raw SQL source on the connection the query runs on', () => { + const result = resolve( + rawSqlSourceTile, + [connectionVariable, sourceVariable([SourceKind.Log])], + { conn: [connectionId], src: [logSourceId] }, + ); + + expect(result).toEqual({ + tile: { + ...rawSqlSourceTile, + config: { + ...rawSqlSourceTile.config, + connectionId, + sourceId: logSourceId, + }, + }, + }); + }); + + it('rejects a raw SQL source on another connection than the query', () => { + const result = resolve( + rawSqlSourceTile, + [connectionVariable, sourceVariable([SourceKind.Log])], + { conn: [connectionId], src: [otherConnectionLogSourceId] }, + ); + + expect(result).toEqual({ + error: expect.stringContaining( + `the source is on connection "${otherConnectionId}"`, + ), + }); + }); + + it.each<{ + case: string; + tile: ExternalDashboardTileWithId; + filters: ExternalDashboardFilterWithId[]; + selections: Record; + variable: string; + }>([ + { + case: 'a source selection that is not a source', + tile: builderTile, + filters: [tracesVariable()], + selections: { traces: [connectionId] }, + variable: '$traces', + }, + { + case: 'a source variable whose connection variable has no selection', + tile: builderTile, + filters: [connectionVariable, tracesVariable('$conn')], + selections: { traces: [traceSourceId] }, + variable: '$conn', + }, + { + case: 'a connection selection that is not a connection', + tile: rawSqlTile, + filters: [connectionVariable], + selections: { conn: [traceSourceId] }, + variable: '$conn', + }, + ])('rejects $case', ({ tile, filters, selections, variable }) => { + expect(resolve(tile, filters, selections)).toEqual({ + error: expect.stringContaining(`${variable} variable`), + }); + }); +}); diff --git a/packages/api/src/mcp/prompts/dashboards/content.ts b/packages/api/src/mcp/prompts/dashboards/content.ts index bbfb7167d..4db680859 100644 --- a/packages/api/src/mcp/prompts/dashboards/content.ts +++ b/packages/api/src/mcp/prompts/dashboards/content.ts @@ -137,6 +137,8 @@ Apply these before calling clickstack_save_dashboard. Each rule is enforced by t 9h. USE AN ADHOC FILTER TO LET THE USER BUILD FREE-FORM CONDITIONS. filters: [{ type: "ADHOC", name, sourceType: "sql", sources: [""], variableName? }] adds an input where the user picks any column or map key, an operator, and a value; clicking a value in a tile adds a condition too. Every condition is ANDed onto the tiles of its sources (or of appliesToSourceIds) with no per-tile wiring. It is never a variable: its variableName only names the entry its conditions are saved under. Seed conditions with savedFilterValues: [{ type: "adhoc", name: "", conditions: [{ key: "ServiceName", operator: "IN", value: "api" }] }]; IN conditions on the same key are ORed. Declare at most one ad hoc filter per source, since a click updates every ad hoc filter on the clicked tile's source. +9i. TO LET VIEWERS SWITCH THE SOURCE OR CONNECTION, USE A DATA_SOURCE OR CONNECTION FILTER. filters: [{ type: "DATA_SOURCE", name: "Logs source", variableName: "logs", allowedSourceKinds: ["log"] }] adds a required source picker; tiles then set sourceId: "$logs" (filters may use it in sourceId or appliesToSourceIds). A CONNECTION filter does the same for raw SQL tiles' connectionId, and a DATA_SOURCE filter's optional connectionId limits its sources to one connection. Keep allowedSourceKinds to kinds charts treat alike (log + trace, or one kind). Tiles that use either cannot have alerts, and clickstack_query_tile needs a selection for the variable (variableValues or the dashboard's saved value). + 10. UPDATE IS REPLACE, NOT MERGE. clickstack_save_dashboard with an id overwrites tiles, containers, and filters in their entirety. Call clickstack_get_dashboard first when you only want to add or rename one entry; do not send a partial set or you will silently drop everything you omitted. 11. GROUP RELATED TILES INTO CONTAINERS. REQUIRED at five or more tiles, no exceptions. An ungrouped wall of nine or ten tiles is a readability failure even when each tile is correct in isolation. Containers are the right way to introduce structure; markdown tiles for section labels are not. diff --git a/packages/api/src/mcp/tools/dashboards/patchDashboard.ts b/packages/api/src/mcp/tools/dashboards/patchDashboard.ts index 36d5f43a2..c19c9d871 100644 --- a/packages/api/src/mcp/tools/dashboards/patchDashboard.ts +++ b/packages/api/src/mcp/tools/dashboards/patchDashboard.ts @@ -201,6 +201,7 @@ export function registerPatchDashboard({ teamId, tiles: [mergedTile], existingTiles: existingDashboard.tiles ?? [], + existingFilters: existingDashboard.filters, containers: existingDashboard.containers ?? [], }); if (validationError) { diff --git a/packages/api/src/mcp/tools/dashboards/queryTile.ts b/packages/api/src/mcp/tools/dashboards/queryTile.ts index b6cd385b5..1e610b0cd 100644 --- a/packages/api/src/mcp/tools/dashboards/queryTile.ts +++ b/packages/api/src/mcp/tools/dashboards/queryTile.ts @@ -13,7 +13,12 @@ import { getRawSqlTileMacroWarnings, getTileVariableWarnings, } from './validation'; -import { mcpVariableValuesParam, resolveDashboardVariables } from './variables'; +import { + getReferenceTargets, + mcpVariableValuesParam, + resolveDashboardVariables, + resolveTileVariableReferences, +} from './variables'; export function registerQueryTile({ context, @@ -79,8 +84,8 @@ export function registerQueryTile({ const externalDashboard = convertToExternalDashboard(dashboard, { hidePromql: !isPromqlEnabled(context), }); - const tile = externalDashboard.tiles.find(t => t.id === tileId); - if (!tile) { + const storedTile = externalDashboard.tiles.find(t => t.id === tileId); + if (!storedTile) { return mcpUserError( `Tile not found: ${tileId}. Available tile IDs: ${externalDashboard.tiles.map(t => t.id).join(', ')}`, ); @@ -94,6 +99,18 @@ export function registerQueryTile({ return mcpUserError(resolvedVariables.error); } + const resolvedTile = resolveTileVariableReferences( + storedTile, + externalDashboard.filters, + variableValues, + externalDashboard.savedFilterValues, + await getReferenceTargets(teamId.toString()), + ); + if ('error' in resolvedTile) { + return mcpUserError(resolvedTile.error); + } + const { tile } = resolvedTile; + const result = await runConfigTile( teamId.toString(), tile, diff --git a/packages/api/src/mcp/tools/dashboards/queryTiles.ts b/packages/api/src/mcp/tools/dashboards/queryTiles.ts index 44af35c95..a1716461f 100644 --- a/packages/api/src/mcp/tools/dashboards/queryTiles.ts +++ b/packages/api/src/mcp/tools/dashboards/queryTiles.ts @@ -19,7 +19,12 @@ import { getRawSqlTileMacroWarnings, getTileVariableWarnings, } from './validation'; -import { mcpVariableValuesParam, resolveDashboardVariables } from './variables'; +import { + getReferenceTargets, + mcpVariableValuesParam, + resolveDashboardVariables, + resolveTileVariableReferences, +} from './variables'; /** * How many tiles to query against ClickHouse at once. Kept lowish so a batch @@ -230,6 +235,8 @@ export function registerQueryTiles({ .slice(MAX_TILES_PER_CALL) .map(t => t.id); + const referenceTargets = await getReferenceTargets(teamId.toString()); + // Fix one shared deadline for the whole batch that every tile races. const deadlineAt = Date.now() + BATCH_DEADLINE_MS; @@ -251,6 +258,17 @@ export function registerQueryTiles({ return { ...base, status: 'skipped' }; } + const resolvedTile = resolveTileVariableReferences( + tile, + externalDashboard.filters, + variableValues, + externalDashboard.savedFilterValues, + referenceTargets, + ); + if ('error' in resolvedTile) { + return { ...base, status: 'error', error: resolvedTile.error }; + } + // Run the query under the shared deadline. It's passed as a thunk // so withDeadline can skip issuing it entirely once the budget is // spent (see withDeadline). withDeadline hands the thunk an @@ -263,10 +281,13 @@ export function registerQueryTiles({ try { const result = await withDeadline( signal => - runConfigTile(teamId.toString(), tile, startDate, endDate, { - abortSignal: signal, - variables, - }), + runConfigTile( + teamId.toString(), + resolvedTile.tile, + startDate, + endDate, + { abortSignal: signal, variables }, + ), deadlineAt, ); diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index 090d00522..11acfd67b 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -26,6 +26,7 @@ import { PromqlQueryTypeSchema, PromqlReducerSchema, SearchConditionTrimmedLanguageSchema, + SourceKind, SQLIntervalSchema, SqlQueryDashboardFilterSchema, StaticListDashboardFilterSchema, @@ -41,10 +42,18 @@ import { externalAdhocSavedFilterValueSchema, externalSqlSavedFilterValueSchema, externalVariableSavedFilterValueSchema, + objectIdOrVariableReferenceSchema, objectIdSchema, tagsSchema, } from '@/utils/zod'; +const SOURCE_VARIABLE_REFERENCE_NOTE = + 'May instead be `$name`, a reference to a DATA_SOURCE filter (variable) on this ' + + 'dashboard, so viewers can switch the source; the variable must offer at least ' + + 'one kind this field accepts.'; + +const SOURCE_ID_DESCRIPTION = `Source ID - call clickstack_list_sources. ${SOURCE_VARIABLE_REFERENCE_NOTE}`; + /** * Metric type values exposed on dashboard tile select items. Restricted to * the kinds the query renderer can translate today; summary is intentionally @@ -651,7 +660,7 @@ const mcpTimeChartDisplayFields = { }; const mcpRowListSourceFields = { - sourceId: z.string().describe('Source ID – call clickstack_list_sources'), + sourceId: z.string().describe(SOURCE_ID_DESCRIPTION), where: z .string() .optional() @@ -739,7 +748,7 @@ export const mcpLineTileSchema = mcpTileLayoutSchema.extend({ .object({ ...rejectedTileWhereFields, displayType: z.literal('line').describe('Line chart over time'), - sourceId: z.string().describe('Source ID – call clickstack_list_sources'), + sourceId: z.string().describe(SOURCE_ID_DESCRIPTION), select: z .array(mcpTileSelectItemSchema) .min(1) @@ -776,7 +785,7 @@ export const mcpLineTileSchema = mcpTileLayoutSchema.extend({ }); const mcpStackedTimeChartConfigFields = { - sourceId: z.string().describe('Source ID – call clickstack_list_sources'), + sourceId: z.string().describe(SOURCE_ID_DESCRIPTION), select: z .array(mcpTileSelectItemSchema) .min(1) @@ -828,7 +837,7 @@ const mcpTableTileSchema = mcpTileLayoutSchema.extend({ .object({ ...rejectedTileWhereFields, displayType: z.literal('table').describe('Tabular aggregated data'), - sourceId: z.string().describe('Source ID – call clickstack_list_sources'), + sourceId: z.string().describe(SOURCE_ID_DESCRIPTION), select: z .array(mcpTileSelectItemSchema) .min(1) @@ -884,7 +893,7 @@ export const mcpNumberTileSchema = mcpTileLayoutSchema.extend({ displayType: z .literal('number') .describe('Single aggregate scalar value'), - sourceId: z.string().describe('Source ID – call clickstack_list_sources'), + sourceId: z.string().describe(SOURCE_ID_DESCRIPTION), select: z .array(mcpTileSelectItemSchema) .min(1) @@ -925,7 +934,7 @@ const mcpPieTileSchema = mcpTileLayoutSchema.extend({ .object({ ...rejectedTileWhereFields, displayType: z.literal('pie').describe('Pie chart'), - sourceId: z.string().describe('Source ID – call clickstack_list_sources'), + sourceId: z.string().describe(SOURCE_ID_DESCRIPTION), select: z .array(mcpTileSelectItemSchema) .length(1) @@ -973,7 +982,7 @@ const mcpCategoricalBarTileSchema = mcpTileLayoutSchema.extend({ displayType: z .literal('bar') .describe('Bar chart — one bar per group value (not a time series)'), - sourceId: z.string().describe('Source ID – call clickstack_list_sources'), + sourceId: z.string().describe(SOURCE_ID_DESCRIPTION), select: z .array(mcpTileSelectItemSchema) .length(1) @@ -1061,7 +1070,8 @@ const mcpHeatmapTileSchema = mcpTileLayoutSchema.extend({ .string() .describe( 'Source ID. Must be a Trace source today; use clickstack_list_sources and ' + - 'pick one whose kind is "trace".', + 'pick one whose kind is "trace". ' + + SOURCE_VARIABLE_REFERENCE_NOTE, ), select: z .array(mcpHeatmapSelectItemSchema) @@ -1106,7 +1116,8 @@ const mcpHeatmapSeriesTileSchema = mcpTileLayoutSchema.extend({ sourceId: z .string() .describe( - 'Source ID of a Trace, Log, or Metric source - call clickstack_list_sources', + 'Source ID of a Trace, Log, or Metric source - call clickstack_list_sources. ' + + SOURCE_VARIABLE_REFERENCE_NOTE, ), select: z .array(mcpTileSelectItemSchema) @@ -1203,7 +1214,9 @@ export const mcpSqlTileSchema = mcpTileLayoutSchema.extend({ connectionId: z .string() .describe( - 'Connection ID (not sourceId) – call clickstack_list_sources to find available connections', + 'Connection ID (not sourceId) – call clickstack_list_sources to find available connections. ' + + 'May instead be `$name`, a reference to a CONNECTION filter (variable) on this ' + + 'dashboard, so viewers can switch the connection the query runs on.', ), sourceId: z .string() @@ -1213,7 +1226,8 @@ export const mcpSqlTileSchema = mcpTileLayoutSchema.extend({ 'ALWAYS set this for raw SQL tiles UNLESS the query reads from multiple tables ' + '(e.g. JOINs or sub-queries spanning several sources), in which case omit it. ' + 'sourceId is REQUIRED by two macros: $__filters and $__sourceTable. ' + - 'The sourceId must belong to the same connection as connectionId.', + 'The sourceId must belong to the same connection as connectionId. ' + + SOURCE_VARIABLE_REFERENCE_NOTE, ), sqlTemplate: z.string().describe(` Raw ClickHouse SQL query. SQL guidelines: @@ -1341,7 +1355,8 @@ const mcpPromqlTileSchema = mcpTileLayoutSchema.extend({ sourceId: z .string() .describe( - 'ID of a source whose kind is "promql" – call clickstack_list_sources', + 'ID of a source whose kind is "promql" – call clickstack_list_sources. ' + + SOURCE_VARIABLE_REFERENCE_NOTE, ), expressions: z .array(mcpPromqlExpressionSchema) @@ -1670,9 +1685,10 @@ const mcpQueryExpressionFilterSchema = z "not declared in any of this dashboard's filters is silently dropped at click time. " + 'Declare an expression here for every column you plan to drive via row-click.', ), - sourceId: objectIdSchema.describe( + sourceId: objectIdOrVariableReferenceSchema.describe( 'Source the filter values are pulled from (for the dropdown). ' + - 'Get IDs from clickstack_list_sources.', + 'Get IDs from clickstack_list_sources. ' + + SOURCE_VARIABLE_REFERENCE_NOTE, ), sourceMetricType: z .nativeEnum(MetricsDataType) @@ -1701,7 +1717,7 @@ const mcpQueryExpressionFilterSchema = z 'Query language of `where`. Set it explicitly.', ), appliesToSourceIds: z - .array(objectIdSchema) + .array(objectIdOrVariableReferenceSchema) .optional() .describe( 'Source IDs (from clickstack_list_sources) that this filter is applied to. ' + @@ -1805,8 +1821,9 @@ const mcpPromqlLabelFilterSchema = z .describe( "Filter type discriminator for a filter whose dropdown lists a Prometheus label's values.", ), - sourceId: objectIdSchema.describe( - 'ID of a source whose kind is "promql". Get IDs from clickstack_list_sources.', + sourceId: objectIdOrVariableReferenceSchema.describe( + 'ID of a source whose kind is "promql". Get IDs from clickstack_list_sources. ' + + SOURCE_VARIABLE_REFERENCE_NOTE, ), label: PromqlLabelDashboardFilterSchema.shape.label.describe( 'Label whose values fill the dropdown, e.g. "job". ' + @@ -1846,11 +1863,11 @@ const mcpAdhocFilterSchema = z '"sql": keys are the columns and map keys of Log, Trace, or Session sources.', ), sources: z - .array(objectIdSchema) + .array(objectIdOrVariableReferenceSchema) .min(1) .describe( 'IDs of the Log, Trace, or Session sources (from clickstack_list_sources) ' + - 'whose keys and values the filter suggests.', + 'whose keys and values the filter suggests, or `$name` of DATA_SOURCE filters.', ), where: z .string() @@ -1863,7 +1880,7 @@ const mcpAdhocFilterSchema = z 'Query language of `where`.', ), appliesToSourceIds: z - .array(objectIdSchema) + .array(objectIdOrVariableReferenceSchema) .optional() .describe( 'IDs of the sources whose tiles the conditions apply to, of the same kind ' + @@ -1885,31 +1902,91 @@ const mcpPromqlAdhocFilterSchema = mcpAdhocFilterSchema.extend({ 'as label matchers to every selector in PromQL tiles.', ), sources: z - .array(objectIdSchema) + .array(objectIdOrVariableReferenceSchema) .min(1) .describe( 'IDs of the sources (from clickstack_list_sources) whose keys and values ' + - 'the filter suggests: Log, Trace, or Session sources for sourceType ' + - '"sql", promql sources for "promql".', + 'the filter suggests, or `$name` of DATA_SOURCE filters: Log, Trace, or ' + + 'Session sources for sourceType "sql", promql sources for "promql".', ), where: mcpAdhocFilterSchema.shape.where.describe( `${mcpAdhocFilterSchema.shape.where.description} sourceType "sql" only; rejected when sourceType is "promql".`, ), }); +// Reference variables are always single-select and required, so they take +// none of the selection-limit fields. +const mcpReferenceVariableBaseShape = { + id: mcpDashboardFilterBaseShape.id, + name: mcpDashboardFilterBaseShape.name, + variableName: mcpDashboardFilterBaseShape.variableName, + // Always required, so the base description's minSelections condition doesn't apply + isGlobalRequirement: mcpDashboardFilterBaseShape.isGlobalRequirement.describe( + 'Block EVERY tile on the dashboard until a value is selected, not just ' + + 'the ones that reference this variable.', + ), +}; + +const mcpDataSourceFilterSchema = z + .object({ + ...mcpReferenceVariableBaseShape, + type: z + .literal(DashboardFilterType.enum.DATA_SOURCE) + .describe( + 'Filter type discriminator for a variable whose value is a source.', + ), + allowedSourceKinds: z + .array(z.nativeEnum(SourceKind)) + .min(1) + .describe( + 'Kinds of sources offered in the dropdown. A tile or filter can reference the ' + + 'variable if it accepts at least one listed kind. A tile whose selected source ' + + 'is a kind it cannot use (e.g. a log source on a PromQL tile) errors instead of ' + + 'running, so prefer kinds that charts treat alike (log + trace, or a single kind).', + ), + connectionId: objectIdOrVariableReferenceSchema + .optional() + .describe( + 'Optional connection the offered sources are limited to: a connection ID, or ' + + '`$name` of a CONNECTION filter on this dashboard.', + ), + }) + .describe( + 'A variable whose single, required value is a source. Tiles and other filters ' + + 'reference it as `$variableName` in sourceId (or appliesToSourceIds) instead of a ' + + 'source ID; tiles stay blocked until a source is selected.', + ); + +const mcpConnectionFilterSchema = z + .object({ + ...mcpReferenceVariableBaseShape, + type: z + .literal(DashboardFilterType.enum.CONNECTION) + .describe( + 'Filter type discriminator for a variable whose value is a connection.', + ), + }) + .describe( + 'A variable whose single, required value is a connection. Raw SQL tiles reference ' + + 'it as `$variableName` in connectionId, and DATA_SOURCE filters in connectionId.', + ); + const mcpDashboardFilterSchema = z .discriminatedUnion('type', [ mcpQueryExpressionFilterSchema, mcpStaticListFilterSchema, mcpSqlQueryFilterSchema, mcpAdhocFilterSchema, + mcpDataSourceFilterSchema, + mcpConnectionFilterSchema, ]) .describe( 'A dashboard-level filter the user can adjust in the dashboard filter bar. ' + 'QUERY_EXPRESSION queries its dropdown values from a source column; ' + 'STATIC_LIST declares them inline; SQL_QUERY reads them from a SQL query. ' + 'STATIC_LIST and SQL_QUERY are always variable-only. ADHOC holds free-form ' + - 'conditions and always broadcasts.', + 'conditions and always broadcasts. DATA_SOURCE and CONNECTION select the ' + + 'source or connection that tiles referencing them as `$variableName` use.', ); const mcpPromqlDashboardFilterSchema = z @@ -1919,6 +1996,8 @@ const mcpPromqlDashboardFilterSchema = z mcpSqlQueryFilterSchema, mcpPromqlLabelFilterSchema, mcpPromqlAdhocFilterSchema, + mcpDataSourceFilterSchema, + mcpConnectionFilterSchema, ]) .describe( 'A dashboard-level filter the user can adjust in the dashboard filter bar. ' + @@ -1926,7 +2005,8 @@ const mcpPromqlDashboardFilterSchema = z 'STATIC_LIST declares them inline; SQL_QUERY reads them from a SQL query; ' + 'PROMETHEUS_LABEL lists the values of a label on a promql source. Those three ' + 'are always variable-only. ADHOC holds free-form conditions on SQL or promql ' + - 'sources and always broadcasts.', + 'sources and always broadcasts. DATA_SOURCE and CONNECTION select the source ' + + 'or connection that tiles referencing them as `$variableName` use.', ); export type McpDashboardFilter = z.infer; @@ -2011,7 +2091,16 @@ const mcpFiltersParam = z ']\n' + 'with savedFilterValues: [{ "type": "adhoc", "name": "conditions", "conditions": ' + '[{ "key": "ServiceName", "operator": "IN", "value": "api" }, ' + - '{ "key": "ServiceName", "operator": "IN", "value": "web" }] }].', + '{ "key": "ServiceName", "operator": "IN", "value": "web" }] }].\n\n' + + 'Example (data source variable: viewers pick which logs source every tile reads):\n' + + '[\n' + + ' { "type": "DATA_SOURCE", "name": "Logs source", "variableName": "logs",\n' + + ' "allowedSourceKinds": ["log"] }\n' + + ']\n' + + 'Tiles then set "sourceId": "$logs". A CONNECTION filter works the same way for ' + + 'raw SQL tiles ("connectionId": "$conn"); a DATA_SOURCE filter may set ' + + '"connectionId" to limit its sources to one connection. Alerts are not supported ' + + 'on tiles that use either.', ); const PROMQL_FILTERS_EXAMPLE = diff --git a/packages/api/src/mcp/tools/dashboards/validation.ts b/packages/api/src/mcp/tools/dashboards/validation.ts index d678b3772..d323c86ac 100644 --- a/packages/api/src/mcp/tools/dashboards/validation.ts +++ b/packages/api/src/mcp/tools/dashboards/validation.ts @@ -6,6 +6,8 @@ import { getFilterVariableName, isAdhocConditionSupported, isFilterVariableEnabled, + isReferenceVariableFilterType, + pairWithVariableNames, validateSqlQueryFilterTemplate, } from '@hyperdx/common-utils/dist/filters'; import { @@ -264,7 +266,8 @@ export function getFilterVariableWarnings( * `name` matches no filter of its kind, or an ad hoc condition its filter's * source type can't express, is silently ignored when the dashboard loads, so * it looks like a saved default but never applies. Warn so the agent can fix - * it. + * it. Data source and connection variables count as variable-enabled filters, + * though they are not text variables. */ export function getSavedFilterValueWarnings( savedFilterValues: DashboardFilterValue[] | undefined, @@ -273,11 +276,16 @@ export function getSavedFilterValueWarnings( | undefined, ): string[] { if (!savedFilterValues?.length) return []; - const declaredNames = new Set( - getDashboardVariableDeclarations(filters).map( + const declaredNames = new Set([ + ...getDashboardVariableDeclarations(filters).map( declaration => declaration.name, ), - ); + ...pairWithVariableNames( + (filters ?? []).filter(filter => + isReferenceVariableFilterType(filter.type), + ), + ).map(({ name }) => name), + ]); const adhocSourceTypeByName = new Map( (filters ?? []).flatMap(filter => filter.type === 'ADHOC' && filter.sourceType diff --git a/packages/api/src/mcp/tools/dashboards/variables.ts b/packages/api/src/mcp/tools/dashboards/variables.ts index f1b98a611..7f5e91cb9 100644 --- a/packages/api/src/mcp/tools/dashboards/variables.ts +++ b/packages/api/src/mcp/tools/dashboards/variables.ts @@ -2,18 +2,48 @@ import { getDashboardVariableDeclarations, getDashboardVariableFilters, getFilterVariableName, + isDataSourceFilter, isFilterVariableEnabled, isStaticListFilter, } from '@hyperdx/common-utils/dist/filters'; -import type { ChartVariable } from '@hyperdx/common-utils/dist/types'; +import { getSavedConfigAllowedSourceKinds } from '@hyperdx/common-utils/dist/guards'; +import { + getReferenceVariableFilters, + getStringField, + isSourceOfferedByVariable, + parseVariableReference, +} from '@hyperdx/common-utils/dist/referenceVariables'; +import type { + ChartVariable, + SavedChartConfig, + SourceKind, +} from '@hyperdx/common-utils/dist/types'; import { isFilterSingleSelect } from '@hyperdx/common-utils/dist/variables'; import { z } from 'zod'; +import { getConnectionsByTeam } from '@/controllers/connection'; +import { getSources } from '@/controllers/sources'; +import { + convertToInternalTileConfig, + type ExternalDashboard, + isConfigTile, + isRawSqlExternalTileConfig, +} from '@/routers/external-api/v2/utils/dashboards'; import type { ExternalDashboardFilter, ExternalDashboardFilterWithId, + ExternalDashboardTileWithId, } from '@/utils/zod'; +type ExternalFilters = + | (ExternalDashboardFilter | ExternalDashboardFilterWithId)[] + | undefined; + +/** Names of the data source and connection variables a dashboard declares. */ +function getReferenceVariableNames(filters: ExternalFilters): Set { + return new Set(getReferenceVariableFilters(filters).map(({ name }) => name)); +} + /** Optional per-variable selection for a tile run. */ export const mcpVariableValuesParam = z .array( @@ -28,7 +58,9 @@ export const mcpVariableValuesParam = z .array(z.string()) .describe( 'Values to treat as selected. An empty array means nothing is selected for this variable. ' + - "For a STATIC_LIST filter's variable, every value must be one of the filter's declared options.", + "For a STATIC_LIST filter's variable, every value must be one of the filter's declared options. " + + 'For a DATA_SOURCE or CONNECTION variable, pass one source or connection ID; it ' + + "overrides the dashboard's saved selection.", ), }), ) @@ -65,9 +97,11 @@ export function resolveDashboardVariables( if (!variableValues?.length) return { variables }; const byName = new Map(variables.map(variable => [variable.name, variable])); + // Resolved separately, by resolveTileVariableReferences + const referenceVariableNames = getReferenceVariableNames(filters); const unknownVariableValues = variableValues .map(({ name }) => name) - .filter(name => !byName.has(name)); + .filter(name => !byName.has(name) && !referenceVariableNames.has(name)); if (unknownVariableValues.length > 0) { const available = variables.length > 0 @@ -121,12 +155,197 @@ export function resolveDashboardVariables( // Override the default empty selection with the given variableValues for (const { name, values } of variableValues) { - byName.get(name)!.values = values; + const variable = byName.get(name); + if (variable) variable.values = values; } return { variables }; } +/** The team's sources and connections that a reference variable may select. */ +export type ReferenceTargets = { + sources: { + id: string; + kind: SourceKind; + connection: string; + disabled?: boolean; + }[]; + connectionIds: ReadonlySet; +}; + +export async function getReferenceTargets( + teamId: string, +): Promise { + const [sources, connections] = await Promise.all([ + getSources(teamId), + getConnectionsByTeam(teamId), + ]); + return { + sources: sources.map(source => ({ + id: source._id.toString(), + kind: source.kind, + connection: source.connection.toString(), + disabled: source.disabled, + })), + connectionIds: new Set(connections.map(c => c._id.toString())), + }; +} + +/** + * Replace a tile's `$name` sourceId / connectionId with the source or + * connection its data source or connection variable selects: the caller's + * variableValues first, then the dashboard's saved selection. A tile with no + * selection cannot run, as on the dashboard, so that is an error naming the + * variable. So is a selection the dashboard's picker would not offer, such as + * a source of another kind or on another connection. + */ +export function resolveTileVariableReferences< + T extends ExternalDashboardTileWithId, +>( + tile: T, + filters: ExternalFilters, + variableValues: McpVariableValues | undefined, + savedFilterValues: ExternalDashboard['savedFilterValues'] | undefined, + targets: ReferenceTargets, +): { tile: T } | { error: string } { + if (!('config' in tile) || tile.config == null) return { tile }; + const selections = new Map(); + for (const value of savedFilterValues ?? []) { + if (value.type === 'variable') selections.set(value.name, value.values); + } + // resolveDashboardVariables has already rejected unknown names + for (const { name, values } of variableValues ?? []) { + selections.set(name, values); + } + const sourceVariables = new Map( + getReferenceVariableFilters(filters).flatMap(({ filter, name }) => + isDataSourceFilter(filter) ? [[name, filter] as const] : [], + ), + ); + + const isSourceOffered = (name: string, id: string) => { + const variable = sourceVariables.get(name); + const source = targets.sources.find(source => source.id === id); + if (variable == null || source == null) return false; + // An unselected connection variable stays a reference, which matches no source + const connectionName = parseVariableReference(variable.connectionId); + const connection = + connectionName == null + ? variable.connectionId + : (selections.get(connectionName)?.[0] ?? variable.connectionId); + return isSourceOfferedByVariable(source, { + allowedSourceKinds: variable.allowedSourceKinds, + connection, + }); + }; + + const describeOffer = (name: string) => { + const variable = sourceVariables.get(name); + if (variable == null) return 'no sources'; + const connectionName = parseVariableReference(variable.connectionId); + const connection = + connectionName != null + ? ` on the connection the $${connectionName} variable selects` + : variable.connectionId + ? ` on connection ${variable.connectionId}` + : ''; + return `enabled ${variable.allowedSourceKinds.join(', ')} sources${connection}`; + }; + + let config: object = tile.config; + for (const field of ['sourceId', 'connectionId'] as const) { + const name = parseVariableReference(getStringField(config, field)); + if (name == null) continue; + const subject = `Tile "${tile.id}" reads its ${field} from the $${name} variable`; + const values = selections.get(name) ?? []; + if (values.length !== 1) { + return { + error: + values.length === 0 + ? `${subject}, which has no selection. Pass one ID for it in variableValues, or save a selection on the dashboard.` + : `${subject}, which accepts exactly one value.`, + }; + } + const [id] = values; + const isOffered = + field === 'connectionId' + ? targets.connectionIds.has(id) + : isSourceOffered(name, id); + if (!isOffered) { + const offer = + field === 'connectionId' + ? 'connections on this team' + : describeOffer(name); + return { + error: `${subject}, which selects "${id}" but only offers ${offer}.`, + }; + } + config = { ...config, [field]: id }; + } + + const resolved = { ...tile, config }; + const tileError = getResolvedSourceError(resolved, tile.config, targets); + return tileError != null ? { error: tileError } : { tile: resolved }; +} + +/** + * Checks a resolved source the variable offers but the tile can't use, which + * the dashboard shows as an error in place of the tile: a kind the tile's + * chart type doesn't accept, or, for raw SQL, a source on another connection + * than the query runs on. + */ +function getResolvedSourceError( + tile: ExternalDashboardTileWithId, + storedConfig: object, + targets: ReferenceTargets, +): string | null { + if (!isConfigTile(tile)) return null; + const sourceId = getStringField(tile.config, 'sourceId'); + const source = targets.sources.find(s => s.id === sourceId); + if (source == null) return null; + const sourceName = parseVariableReference( + getStringField(storedConfig, 'sourceId'), + ); + const connectionName = parseVariableReference( + getStringField(storedConfig, 'connectionId'), + ); + + if (sourceName != null) { + let config: SavedChartConfig; + try { + config = convertToInternalTileConfig(tile).config; + } catch { + // A shape the converter refuses fails when the tile runs + return null; + } + const allowedKinds = getSavedConfigAllowedSourceKinds(config); + if (!allowedKinds.includes(source.kind)) { + return ( + `Tile "${tile.id}" reads its sourceId from the $${sourceName} variable, which ` + + `selects "${source.id}", a ${source.kind} source, but this tile only accepts ` + + `${allowedKinds.join(', ')} sources.` + ); + } + } + + if ( + (sourceName != null || connectionName != null) && + isRawSqlExternalTileConfig(tile.config) && + source.connection !== tile.config.connectionId + ) { + const from = [sourceName, connectionName] + .filter((name): name is string => name != null) + .map(name => `$${name}`) + .join(' and '); + return ( + `Tile "${tile.id}" resolves source "${source.id}" and connection ` + + `"${tile.config.connectionId}" from ${from}, but the source is on connection ` + + `"${source.connection}". Select a source on the connection the query runs on.` + ); + } + return null; +} + /** Options of each static filter, keyed by the variable name it answers to. */ function buildStaticOptionsByName( filters: ExternalDashboardFilter[] | undefined, diff --git a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts index 92a72d976..38a9e736c 100644 --- a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts +++ b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts @@ -8511,6 +8511,381 @@ describe('External API v2 Dashboards - new format', () => { }); }); + describe('reference variables', () => { + const connectionFilter = { + type: 'CONNECTION' as const, + name: 'Connection', + variableName: 'conn', + }; + const dataSourceFilter = { + type: 'DATA_SOURCE' as const, + name: 'Traces source', + variableName: 'traces', + allowedSourceKinds: [SourceKind.Trace], + connectionId: '$conn', + }; + const rawSqlTile = { + name: 'Raw SQL', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + configType: 'sql', + displayType: 'table', + connectionId: '$conn', + sqlTemplate: 'SELECT 1', + }, + }; + + it('round-trips the variables, their values and the tiles that use them', async () => { + const globalConnectionFilter = { + ...connectionFilter, + isGlobalRequirement: true, + }; + const savedFilterValues = [ + { type: 'variable', name: 'conn', values: [connection._id.toString()] }, + { + type: 'variable', + name: 'traces', + values: [traceSource._id.toString()], + }, + ]; + const created = await authRequest('post', BASE_URL) + .send({ + name: 'Variable Dashboard', + tiles: [makeExternalTile({ sourceId: '$traces' }), rawSqlTile], + tags: [], + filters: [globalConnectionFilter, dataSourceFilter], + savedFilterValues, + }) + .expect(200); + + const fetched = await authRequest( + 'get', + `${BASE_URL}/${created.body.data.id}`, + ).expect(200); + expect(fetched.body.data.filters).toEqual([ + expect.objectContaining(globalConnectionFilter), + expect.objectContaining(dataSourceFilter), + ]); + expect(fetched.body.data.tiles.map(t => t.config)).toEqual([ + expect.objectContaining({ sourceId: '$traces' }), + expect.objectContaining({ connectionId: '$conn' }), + ]); + expect(fetched.body.data.savedFilterValues).toEqual(savedFilterValues); + + const echoed = await authRequest( + 'put', + `${BASE_URL}/${created.body.data.id}`, + ) + .send(fetched.body.data) + .expect(200); + expect(echoed.body.data.filters).toEqual(fetched.body.data.filters); + expect(echoed.body.data.tiles.map(t => t.config)).toEqual( + fetched.body.data.tiles.map(t => t.config), + ); + }); + + it('rejects a tile referencing an undeclared variable', async () => { + const res = await authRequest('post', BASE_URL) + .send({ + name: 'Variable Dashboard', + tiles: [makeExternalTile({ sourceId: '$traces' })], + tags: [], + }) + .expect(400); + expect(res.body.message).toMatch(/does not name a data source variable/); + }); + + it('rejects a connection reference to an undeclared connection variable', async () => { + const res = await authRequest('post', BASE_URL) + .send({ + name: 'Variable Dashboard', + tiles: [rawSqlTile], + tags: [], + filters: [{ ...dataSourceFilter, connectionId: undefined }], + }) + .expect(400); + expect(res.body.message).toMatch(/does not name a connection variable/); + }); + + it('accepts a PromQL tile and PROMETHEUS_LABEL filter on a PromQL data source variable', async () => { + await authRequest('post', BASE_URL) + .send({ + name: 'Variable Dashboard', + tiles: [ + { + name: 'PromQL', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + configType: 'promql', + displayType: 'line', + sourceId: '$prom', + expressions: [{ expression: 'up' }], + }, + }, + ], + tags: [], + filters: [ + { + type: 'DATA_SOURCE', + name: 'PromQL source', + variableName: 'prom', + allowedSourceKinds: [SourceKind.Promql], + }, + { + type: 'PROMETHEUS_LABEL', + name: 'Job', + sourceId: '$prom', + label: 'job', + }, + ], + }) + .expect(200); + }); + + it('accepts an ADHOC filter whose sources reference a data source variable', async () => { + const adhocFilter = { + type: 'ADHOC', + name: 'Conditions', + sourceType: 'sql', + sources: ['$traces'], + appliesToSourceIds: ['$traces'], + }; + await authRequest('post', BASE_URL) + .send({ + name: 'Variable Dashboard', + tiles: [], + tags: [], + filters: [omit(dataSourceFilter, 'connectionId'), adhocFilter], + }) + .expect(200); + + const res = await authRequest('post', BASE_URL) + .send({ + name: 'Variable Dashboard', + tiles: [], + tags: [], + filters: [adhocFilter], + }) + .expect(400); + expect(res.body.message).toMatch(/does not name a data source variable/); + }); + + it('accepts a raw SQL tile with a fixed source on a connection variable', async () => { + await authRequest('post', BASE_URL) + .send({ + name: 'Variable Dashboard', + tiles: [ + { + ...rawSqlTile, + config: { + ...rawSqlTile.config, + sourceId: traceSource._id.toString(), + }, + }, + ], + tags: [], + filters: [connectionFilter], + }) + .expect(200); + }); + + it("rejects a raw SQL tile whose data source variable is limited to another connection than the tile's", async () => { + const otherConnection = await Connection.create({ + team: team._id, + name: 'Other Connection', + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + + const res = await authRequest('post', BASE_URL) + .send({ + name: 'Variable Dashboard', + tiles: [ + { + ...rawSqlTile, + config: { + ...rawSqlTile.config, + connectionId: connection._id.toString(), + sourceId: '$traces', + }, + }, + ], + tags: [], + filters: [ + { + ...dataSourceFilter, + connectionId: otherConnection._id.toString(), + }, + ], + }) + .expect(400); + expect(res.body.message).toBe( + 'The following source IDs do not match the specified connections: $traces', + ); + }); + + it('checks a PUT without filters against the stored variables', async () => { + const dashboard = await createTestDashboard({ + filters: [ + { id: new ObjectId().toString(), ...connectionFilter }, + { + id: new ObjectId().toString(), + ...omit(dataSourceFilter, 'connectionId'), + connection: '$conn', + }, + ], + }); + + await authRequest('put', `${BASE_URL}/${dashboard._id}`) + .send({ + name: 'Variable Dashboard', + tiles: [makeExternalTile({ sourceId: '$traces' })], + tags: [], + }) + .expect(200); + }); + + // Only references an update adds are checked, so editing a variable + // doesn't fail on the tiles that already use it. + it('accepts a PUT whose filter change leaves stored references unsatisfied', async () => { + const variable = { ...dataSourceFilter, connectionId: undefined }; + const created = await authRequest('post', BASE_URL) + .send({ + name: 'Variable Dashboard', + tiles: [makeExternalTile({ sourceId: '$traces' })], + tags: [], + filters: [variable], + }) + .expect(200); + const fetched = await authRequest( + 'get', + `${BASE_URL}/${created.body.data.id}`, + ).expect(200); + const url = `${BASE_URL}/${created.body.data.id}`; + const [storedVariable] = fetched.body.data.filters; + + await authRequest('put', url) + .send({ + ...fetched.body.data, + filters: [ + { ...storedVariable, allowedSourceKinds: [SourceKind.Promql] }, + ], + }) + .expect(200); + await authRequest('put', url) + .send({ ...fetched.body.data, filters: [] }) + .expect(200); + + const res = await authRequest('put', url) + .send({ + ...fetched.body.data, + filters: [], + tiles: [ + ...fetched.body.data.tiles, + makeExternalTile({ sourceId: '$nope' }), + ], + }) + .expect(400); + expect(res.body.message).toMatch(/does not name a data source variable/); + }); + + it("accepts a PUT that moves a raw SQL tile's data source variable to another connection", async () => { + const otherConnection = await Connection.create({ + team: team._id, + name: 'Other Connection', + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + const created = await authRequest('post', BASE_URL) + .send({ + name: 'Variable Dashboard', + tiles: [ + { + ...rawSqlTile, + config: { + ...rawSqlTile.config, + connectionId: connection._id.toString(), + sourceId: '$traces', + }, + }, + ], + tags: [], + filters: [ + { ...dataSourceFilter, connectionId: connection._id.toString() }, + ], + }) + .expect(200); + const fetched = await authRequest( + 'get', + `${BASE_URL}/${created.body.data.id}`, + ).expect(200); + const [storedVariable] = fetched.body.data.filters; + + await authRequest('put', `${BASE_URL}/${created.body.data.id}`) + .send({ + ...fetched.body.data, + filters: [ + { + ...storedVariable, + connectionId: otherConnection._id.toString(), + }, + ], + }) + .expect(200); + }); + + it('deletes the alert of a tile that moves to a data source variable', async () => { + const tile = makeExternalTile({ sourceId: traceSource._id.toString() }); + const created = await authRequest('post', BASE_URL) + .send({ + name: 'Variable Dashboard', + tiles: [tile], + tags: [], + filters: [connectionFilter, dataSourceFilter], + }) + .expect(200); + const { id: dashboardId, tiles } = created.body.data; + const webhook = await Webhook.create({ + name: 'Test Webhook', + service: WebhookService.Slack, + url: 'https://hooks.slack.com/test', + team: team._id, + }); + const alert = await Alert.create({ + team: team._id, + dashboard: dashboardId, + tileId: tiles[0].id, + source: AlertSource.TILE, + threshold: 100, + interval: '1h', + thresholdType: AlertThresholdType.ABOVE, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + }); + + await authRequest('put', `${BASE_URL}/${dashboardId}`) + .send({ + ...created.body.data, + tiles: [ + { + ...tiles[0], + config: { ...tiles[0].config, sourceId: '$traces' }, + }, + ], + }) + .expect(200); + + expect(await Alert.findById(alert._id)).toBeNull(); + }); + }); + describe('ad hoc filters', () => { const adhocFilter = () => ({ id: new ObjectId().toString(), diff --git a/packages/api/src/routers/external-api/v2/dashboards.ts b/packages/api/src/routers/external-api/v2/dashboards.ts index 0b6bacea0..cae8943ea 100644 --- a/packages/api/src/routers/external-api/v2/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/dashboards.ts @@ -705,7 +705,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "line" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -792,7 +792,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "stacked_bar" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -862,7 +862,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "stacked_line" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -938,7 +938,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "table" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -1012,7 +1012,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "number" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -1071,7 +1071,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "pie" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -1124,7 +1124,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "bar" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -1217,7 +1217,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "distribution" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -1265,7 +1265,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "series" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -1314,7 +1314,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "search" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: string @@ -1345,7 +1345,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "event_patterns" * sourceId: * type: string - * description: ID of the data source to mine patterns from. + * description: ID of the data source to mine patterns from, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: string @@ -1395,7 +1395,9 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "sql" * connectionId: * type: string - * description: ID of the ClickHouse connection to execute the query against. + * description: | + * ID of the ClickHouse connection to execute the query against, or + * `$connectionVariable` of a CONNECTION filter on the dashboard. * example: "65f5e4a3b9e77c001a567890" * sqlTemplate: * type: string @@ -1404,7 +1406,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "SELECT count() FROM otel_logs WHERE timestamp > now() - INTERVAL 1 HOUR" * sourceId: * type: string - * description: Optional ID of the data source associated with this Raw SQL chart. Used for applying dashboard filters. + * description: Optional ID of the data source associated with this Raw SQL chart, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. Used for applying dashboard filters. * example: "65f5e4a3b9e77c001a567890" * numberFormat: * $ref: '#/components/schemas/NumberFormat' @@ -2420,6 +2422,8 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * - $ref: '#/components/schemas/PrometheusLabelFilterInput' * - $ref: '#/components/schemas/SqlQueryFilterInput' * - $ref: '#/components/schemas/AdhocFilterInput' + * - $ref: '#/components/schemas/DataSourceFilterInput' + * - $ref: '#/components/schemas/ConnectionFilterInput' * discriminator: * propertyName: type * mapping: @@ -2428,6 +2432,145 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * PROMETHEUS_LABEL: '#/components/schemas/PrometheusLabelFilterInput' * SQL_QUERY: '#/components/schemas/SqlQueryFilterInput' * ADHOC: '#/components/schemas/AdhocFilterInput' + * DATA_SOURCE: '#/components/schemas/DataSourceFilterInput' + * CONNECTION: '#/components/schemas/ConnectionFilterInput' + * + * DataSourceFilterInput: + * type: object + * description: | + * A variable whose single, required value is a source. Tiles reference it + * as `$variableName` in their sourceId, and other filters in sourceId or + * appliesToSourceIds, so viewers can switch the source those read. Tiles + * that use it do not load until a source is selected, and cannot have + * alerts. + * required: + * - type + * - name + * - allowedSourceKinds + * properties: + * type: + * type: string + * enum: [DATA_SOURCE] + * description: Filter type discriminator. Must be "DATA_SOURCE". + * example: "DATA_SOURCE" + * name: + * type: string + * minLength: 1 + * description: Display name of the variable's dropdown + * example: "Logs source" + * variableName: + * type: string + * maxLength: 64 + * pattern: '^[a-zA-Z][a-zA-Z0-9_]*$' + * description: Token tiles and filters reference the selected source by, as `$variableName`. + * example: "logs" + * isGlobalRequirement: + * type: boolean + * default: false + * description: | + * Blocks every tile on the dashboard until a value is selected. False + * (the default) blocks only the tiles that reference the variable. + * example: true + * minSelections: + * type: integer + * enum: [1] + * default: 1 + * description: Must be 1 if provided. The variable always requires a selection. + * example: 1 + * maxSelections: + * type: integer + * enum: [1] + * default: 1 + * description: Must be 1 if provided. The variable is always single-select. + * example: 1 + * isBroadcastEnabled: + * type: boolean + * enum: [false] + * default: false + * description: Must be false or omitted. + * example: false + * isVariableEnabled: + * type: boolean + * enum: [true] + * default: true + * description: Must be true if provided, and is true when omitted. + * example: true + * allowedSourceKinds: + * type: array + * minItems: 1 + * items: + * type: string + * enum: [log, trace, session, metric, promql] + * description: | + * Kinds of sources offered. A field can reference the variable if it + * accepts at least one listed kind. A tile whose selected source is a + * kind it cannot use shows an error instead of running. + * example: ["log"] + * connectionId: + * type: string + * description: | + * Optional connection the offered sources are limited to: a + * connection ID, or `$name` of a CONNECTION filter on the dashboard. + * example: "$conn" + * + * ConnectionFilterInput: + * type: object + * description: | + * A variable whose single, required value is a connection. Raw SQL tiles + * reference it as `$variableName` in their connectionId, and data source + * variables in theirs. Tiles that use it do not load until a connection is + * selected, and cannot have alerts. + * required: + * - type + * - name + * properties: + * type: + * type: string + * enum: [CONNECTION] + * description: Filter type discriminator. Must be "CONNECTION". + * example: "CONNECTION" + * name: + * type: string + * minLength: 1 + * description: Display name of the variable's dropdown + * example: "Connection" + * variableName: + * type: string + * maxLength: 64 + * pattern: '^[a-zA-Z][a-zA-Z0-9_]*$' + * description: Token raw SQL tiles reference the selected connection by, as `$variableName`. + * example: "conn" + * isGlobalRequirement: + * type: boolean + * default: false + * description: | + * Blocks every tile on the dashboard until a value is selected. False + * (the default) blocks only the tiles that reference the variable. + * example: true + * minSelections: + * type: integer + * enum: [1] + * default: 1 + * description: Must be 1 if provided. The variable always requires a selection. + * example: 1 + * maxSelections: + * type: integer + * enum: [1] + * default: 1 + * description: Must be 1 if provided. The variable is always single-select. + * example: 1 + * isBroadcastEnabled: + * type: boolean + * enum: [false] + * default: false + * description: Must be false or omitted. + * example: false + * isVariableEnabled: + * type: boolean + * enum: [true] + * default: true + * description: Must be true if provided, and is true when omitted. + * example: true * * QueryExpressionFilterInput: * type: object @@ -2458,7 +2601,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "environment" * sourceId: * type: string - * description: Source ID this dashboard filter key applies to + * description: Source ID this dashboard filter key applies to, or `$name` of a DATA_SOURCE filter on the dashboard * example: "65f5e4a3b9e77c001a111111" * sourceMetricType: * type: string @@ -2854,7 +2997,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * minItems: 1 * items: * type: string - * description: IDs of the sources whose keys and values the filter suggests, all of sourceType. + * description: IDs of the sources whose keys and values the filter suggests, all of sourceType, or `$name` of data source variables. * example: ["65f5e4a3b9e77c001a111111"] * where: * type: string @@ -2874,8 +3017,8 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * type: string * description: | * IDs of the sources whose tiles the conditions apply to, all of - * sourceType. Omit or send an empty array to apply to the tiles of - * the filter's sources. + * sourceType, or `$name` of data source variables. Omit or send an + * empty array to apply to the tiles of the filter's sources. * example: ["65f5e4a3b9e77c001a111111"] * isBroadcastEnabled: * type: boolean diff --git a/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts b/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts index 9ab380298..200f6f26b 100644 --- a/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts +++ b/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts @@ -8,6 +8,7 @@ import { MetricsDataType, PromqlReducer, PromqlSavedChartConfig, + SourceKind, } from '@hyperdx/common-utils/dist/types'; import mongoose from 'mongoose'; import { z } from 'zod'; @@ -539,6 +540,30 @@ describe('convertToExternalDashboard orphan-ref heal', () => { expect(hidden.savedFilterValues).toEqual([adhocValue]); }); + it('keeps data source variables that offer promql sources with hidePromql', () => { + const doc = makeDoc({ + filters: [ + { + id: 'mixed', + type: 'DATA_SOURCE', + name: 'mixed', + variableName: 'mixed', + allowedSourceKinds: [SourceKind.Metric, SourceKind.Promql], + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }, + ], + }); + + expect( + convertToExternalDashboard(doc, { hidePromql: true }).filters?.map( + f => f.id, + ), + ).toEqual(['mixed']); + }); + // seriesLimit is three-state (matching the internal schema): omitted = // default cap, 0 = unlimited, positive N = top-N. All three must survive a // GET->PUT round-trip: 0 and N pass through, null maps to absent (the diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index e48e86cff..fbde606d5 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -7,6 +7,7 @@ import { } from '@hyperdx/common-utils/dist/core/utils'; import { collectRefinementIssues, + getVariableReferenceError, validateDashboardContainersStructure, validateDashboardFilterFieldGating, validateDashboardFilterModes, @@ -25,6 +26,10 @@ import { isPromqlSavedChartConfig, isRawSqlSavedChartConfig, } from '@hyperdx/common-utils/dist/guards'; +import { + getSourceVariableDeclarations, + parseVariableReference, +} from '@hyperdx/common-utils/dist/referenceVariables'; import { AggregateFunctionSchema, BuilderSavedChartConfig, @@ -58,7 +63,6 @@ import { getConnectionsByTeam } from '@/controllers/connection'; import { getSources } from '@/controllers/sources'; import Dashboard, { DashboardDocument } from '@/models/dashboard'; import { - getExternalCompatibleFilters, translateExternalChartToTileConfig, translateExternalFilterToFilter, translateFilterToExternalFilter, @@ -813,7 +817,7 @@ export function convertToExternalDashboard( convertTileToExternalChart(tile, containerById, dashboardId), ), tags: dashboard.tags || [], - filters: getExternalCompatibleFilters(dashboard.filters) + filters: (dashboard.filters ?? []) .filter(filter => !(hidePromql && isPromqlFilter(filter))) .map(translateFilterToExternalFilter), savedQuery: dashboard.savedQuery ?? null, @@ -1329,7 +1333,12 @@ function getMissingSources( const existingSourceIds = new Set( sources.map(source => source._id.toString()), ); - return [...sourceIds].filter(sourceId => !existingSourceIds.has(sourceId)); + // Variable references are checked by validateDashboardVariableReferences + return [...sourceIds].filter( + sourceId => + parseVariableReference(sourceId) == null && + !existingSourceIds.has(sourceId), + ); } /** @@ -1587,16 +1596,18 @@ async function getMissingConnections( filters: (ExternalDashboardFilter | ExternalDashboardFilterWithId)[] = [], ): Promise { const connectionIds = new Set(); + // Variable references are checked by validateDashboardVariableReferences + const addConnectionId = (id: string | undefined) => { + if (id && parseVariableReference(id) == null) connectionIds.add(id); + }; for (const tile of tiles) { if (isConfigTile(tile) && isRawSqlExternalTileConfig(tile.config)) { - connectionIds.add(tile.config.connectionId); + addConnectionId(tile.config.connectionId); } } for (const filter of filters) { - if ('connectionId' in filter) { - connectionIds.add(filter.connectionId); - } + if ('connectionId' in filter) addConnectionId(filter.connectionId); } if (connectionIds.size === 0) return []; @@ -1699,8 +1710,26 @@ export function convertExternalFiltersToInternal( function getSourceConnectionMismatches( sources: SourceForValidation[], tiles: ExternalDashboardTileWithId[], + filters: DashboardFilter[], + existingTiles: DashboardDocument['tiles'] = [], ): string[] { + const existingConfigById = new Map( + existingTiles.map(tile => [tile.id, tile.config]), + ); const sourceById = new Map(sources.map(s => [s._id.toString(), s])); + const variableConnectionByName = new Map( + getSourceVariableDeclarations(filters).map(({ name, connection }) => [ + name, + connection, + ]), + ); + // A source variable's sources are on its declared connection, if any + const getSourceConnection = (sourceId: string) => { + const name = parseVariableReference(sourceId); + return name == null + ? sourceById.get(sourceId)?.connection.toString() + : variableConnectionByName.get(name); + }; const mismatched: string[] = []; for (const tile of tiles) { @@ -1709,9 +1738,30 @@ function getSourceConnectionMismatches( isRawSqlExternalTileConfig(tile.config) && tile.config.sourceId ) { - const source = sourceById.get(tile.config.sourceId); - if (source && source.connection.toString() !== tile.config.connectionId) { - mismatched.push(tile.config.sourceId); + const { connectionId, sourceId } = tile.config; + // Like other variable references, one the update leaves as it was + // isn't checked again; the dashboard shows the mismatch instead + const existingConfig = tile.id + ? existingConfigById.get(tile.id) + : undefined; + if ( + parseVariableReference(sourceId) != null && + existingConfig != null && + Reflect.get(existingConfig, 'source') === sourceId && + Reflect.get(existingConfig, 'connection') === connectionId + ) { + continue; + } + const sourceConnection = getSourceConnection(sourceId); + // A connection reference is only known once selected, so compare two + // IDs, or two references, but not one of each + if ( + sourceConnection != null && + (parseVariableReference(sourceConnection) == null) === + (parseVariableReference(connectionId) == null) && + sourceConnection !== connectionId + ) { + mismatched.push(sourceId); } } } @@ -1726,9 +1776,16 @@ export type TileValidationContext = { tiles: ExternalDashboardTileWithId[]; /** Filters to check for missing source IDs (create/full-update paths). */ filters?: (ExternalDashboardFilter | ExternalDashboardFilterWithId)[]; - /** Existing internal tiles for scoping heatmap change detection (update paths). */ + /** + * Existing internal tiles for scoping heatmap change detection and the + * variable reference check (update paths). + */ existingTiles?: DashboardDocument['tiles']; - /** Existing filters, whose source references aren't re-checked (update paths). */ + /** + * Existing internal filters (update paths). Their source references aren't + * re-checked, and tile variable references are checked against them when + * the request carries no `filters`. + */ existingFilters?: DashboardFilter[]; /** Container set to validate tile containerId/tabId refs against. */ containers: DashboardContainer[]; @@ -1755,6 +1812,26 @@ export async function validateDashboardTiles( return tileRefIssues.join('; '); } + // Stored IDs are kept so that the references an update leaves as they were + // can be told apart from the ones it adds, which are the only ones checked + const referenceFilters = filters + ? convertExternalFiltersToInternal( + filters, + new Set(existingFilters?.map(filter => filter.id)), + ) + : (existingFilters ?? []); + const variableReferenceError = getVariableReferenceError( + convertExternalTilesToInternal( + tiles, + new Set(existingTiles?.map(tile => tile.id)), + ), + referenceFilters, + existingTiles && { tiles: existingTiles, filters: existingFilters }, + ); + if (variableReferenceError) { + return variableReferenceError; + } + // Fetch sources/connections/onClick targets in parallel. const [ sources, @@ -1779,6 +1856,8 @@ export async function validateDashboardTiles( const sourceConnectionMismatches = getSourceConnectionMismatches( sources, tiles, + referenceFilters, + existingTiles, ); if (sourceConnectionMismatches.length > 0) { return `The following source IDs do not match the specified connections: ${sourceConnectionMismatches.join(', ')}`; @@ -1818,10 +1897,10 @@ export async function validateDashboardTiles( // PromQL tiles query through their source's connection and TimeSeries // table, which only a PromQL source carries. const nonPromqlTileSourceIds = getInvalidSourceIds( - getChangedPromqlTileSourceIds(tiles, existingTiles).map(id => ({ - id, - kinds: [SourceKind.Promql], - })), + getChangedPromqlTileSourceIds(tiles, existingTiles) + // A reference's kinds are checked by validateDashboardVariableReferences + .filter(id => parseVariableReference(id) == null) + .map(id => ({ id, kinds: [SourceKind.Promql] })), sources, ); if (nonPromqlTileSourceIds.length > 0) { @@ -1838,7 +1917,7 @@ export async function validateDashboardTiles( translateExternalFilterToFilter({ id: '', ...filter }), ), existingFilters, - ), + ).filter(ref => parseVariableReference(ref.id) == null), sources, ); if (filterSourceError != null) return filterSourceError; diff --git a/packages/api/src/utils/externalApi.ts b/packages/api/src/utils/externalApi.ts index 91f627748..164e59d7f 100644 --- a/packages/api/src/utils/externalApi.ts +++ b/packages/api/src/utils/externalApi.ts @@ -3,7 +3,6 @@ import { isFilterBroadcastEnabled, isFilterRequired, isFilterVariableEnabled, - isReferenceVariableFilter, } from '@hyperdx/common-utils/dist/filters'; import { AlertErrorType, @@ -239,24 +238,8 @@ export function translateExternalChartToTileConfig( }; } -/** Filter types the external API and MCP don't expose yet. */ -export type ExternalCompatibleDashboardFilter = Exclude< - DashboardFilter, - { type: 'DATA_SOURCE' | 'CONNECTION' } ->; - -/** The filters the external API and MCP may read. */ -export function getExternalCompatibleFilters( - filters: DashboardFilter[] | undefined, -): ExternalCompatibleDashboardFilter[] { - return (filters ?? []).filter( - (filter): filter is ExternalCompatibleDashboardFilter => - !isReferenceVariableFilter(filter), - ); -} - export function translateFilterToExternalFilter( - filter: ExternalCompatibleDashboardFilter, + filter: DashboardFilter, ): ExternalDashboardFilterWithId { // Omitted from responses unless the filter is required, so a GET response // can be PUT back verbatim. @@ -286,6 +269,15 @@ export function translateFilterToExternalFilter( connectionId: filter.connection.toString(), }; + case 'DATA_SOURCE': + return { + ...omit(filter, 'connection'), + ...(filter.connection ? { connectionId: filter.connection } : {}), + }; + + case 'CONNECTION': + return filter; + case 'QUERY_EXPRESSION': { // Ignore variableName and appliesToSourceIds if the filter is not in a mode that uses them const ignoredKeys = [ @@ -327,6 +319,15 @@ export function translateExternalFilterToFilter( case 'QUERY_EXPRESSION': return { ...omit(filter, 'sourceId'), source: filter.sourceId }; + case 'DATA_SOURCE': + return { + ...omit(filter, 'connectionId'), + ...(filter.connectionId ? { connection: filter.connectionId } : {}), + }; + + case 'CONNECTION': + return filter; + default: filter satisfies never; return filter; diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index 05cd5c23d..09bd5ab19 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -1,4 +1,5 @@ import { validateChartConfigFormulas } from '@hyperdx/common-utils/dist/dashboardValidation'; +import { parseVariableReference } from '@hyperdx/common-utils/dist/referenceVariables'; import { addDuplicateTileIdIssues, AdhocDashboardFilterSchema, @@ -12,8 +13,10 @@ import { AlertThresholdType, BackgroundChartSchema, ChartPaletteTokenSchema, + ConnectionDashboardFilterSchema, DASHBOARD_CONTAINER_ID_MAX, DASHBOARD_MAX_TILES, + DataSourceDashboardFilterSchema, HeatmapScaleTypeSchema, LineInterpolationSchema, MAX_LEGEND_TEMPLATE_LENGTH, @@ -54,6 +57,17 @@ export const objectIdSchema = z.string().refine(val => { return Types.ObjectId.isValid(val); }, 'Invalid ObjectId'); +/** + * A source or connection field: an ObjectId, or a `$name` reference to the + * dashboard's data source or connection variable of that name. + */ +export const objectIdOrVariableReferenceSchema = z + .string() + .refine( + val => Types.ObjectId.isValid(val) || parseVariableReference(val) != null, + 'Invalid ObjectId or $variable reference', + ); + /** * A repeatable query param. Express's parser yields a bare string for a single * occurrence and an array for several, and `?p[]=x` lands on the same key as @@ -175,7 +189,7 @@ export { tagsSchema }; const externalQueryExpressionFilterShape = QueryExpressionDashboardFilterSchema.omit({ source: true }).extend({ - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, }); // A static filter's mode flags each accept exactly one value, so callers may @@ -188,7 +202,7 @@ const externalStaticListFilterShape = StaticListDashboardFilterSchema.extend({ const externalPromqlLabelFilterShape = PromqlLabelDashboardFilterSchema.omit({ source: true, }).extend({ - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, isBroadcastEnabled: z.literal(false).default(false), isVariableEnabled: z.literal(true).default(true), }); @@ -204,12 +218,32 @@ const externalSqlQueryFilterShape = SqlQueryDashboardFilterSchema.omit({ // Field names match the stored filter, so it needs no translation. const externalAdhocFilterShape = AdhocDashboardFilterSchema.extend({ - sources: z.array(objectIdSchema).min(1), - appliesToSourceIds: z.array(objectIdSchema).optional(), + sources: z.array(objectIdOrVariableReferenceSchema).min(1), + appliesToSourceIds: z.array(objectIdOrVariableReferenceSchema).optional(), isBroadcastEnabled: z.literal(true).default(true), isVariableEnabled: z.literal(true).default(true), }); +// Reference variables are always single-select and required, so these +// fields, like the mode flags, accept one value each and may be omitted. +const referenceVariableFixedFields = { + minSelections: z.literal(1).default(1), + maxSelections: z.literal(1).default(1), + isBroadcastEnabled: z.literal(false).default(false), + isVariableEnabled: z.literal(true).default(true), +}; + +const externalDataSourceFilterShape = DataSourceDashboardFilterSchema.omit({ + connection: true, +}).extend({ + connectionId: objectIdOrVariableReferenceSchema.optional(), + ...referenceVariableFixedFields, +}); + +const externalConnectionFilterShape = ConnectionDashboardFilterSchema.extend( + referenceVariableFixedFields, +); + export const externalDashboardFilterSchemaWithId = z.discriminatedUnion( 'type', [ @@ -218,6 +252,8 @@ export const externalDashboardFilterSchemaWithId = z.discriminatedUnion( externalPromqlLabelFilterShape.strict(), externalSqlQueryFilterShape.strict(), externalAdhocFilterShape.strict(), + externalDataSourceFilterShape.strict(), + externalConnectionFilterShape.strict(), ], ); @@ -236,6 +272,8 @@ export const externalDashboardFilterSchema = z.discriminatedUnion('type', [ externalPromqlLabelFilterShape.omit({ id: true }).strict(), externalSqlQueryFilterShape.omit({ id: true }).strict(), externalAdhocFilterShape.omit({ id: true }).strict(), + externalDataSourceFilterShape.omit({ id: true }).strict(), + externalConnectionFilterShape.omit({ id: true }).strict(), ]); export type ExternalDashboardFilter = z.infer< @@ -356,9 +394,9 @@ export type ExternalDashboardSelectItem = z.infer< const externalDashboardRawSqlChartConfigBaseSchema = z.object({ configType: z.literal('sql'), - connectionId: objectIdSchema, + connectionId: objectIdOrVariableReferenceSchema, sqlTemplate: z.string().max(100000), - sourceId: objectIdSchema.optional(), + sourceId: objectIdOrVariableReferenceSchema.optional(), numberFormat: NumberFormatSchema.optional(), }); @@ -381,7 +419,7 @@ const externalFormulasSchema = z.array(MetricFormulaSchema).max(10).optional(); const externalShowOperandSeriesSchema = z.boolean().optional(); const externalDashboardTimeChartConfigSchema = z.object({ - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, select: z.array(externalDashboardSelectItemSchema).min(1).max(20), groupBy: z.string().max(10000).optional(), asRatio: z.boolean().optional(), @@ -452,7 +490,7 @@ const externalDashboardStackedLineRawSqlChartConfigSchema = const externalDashboardTableChartConfigSchema = z.object({ displayType: z.literal('table'), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, select: z.array(externalDashboardSelectItemSchema).min(1).max(20), groupBy: z.string().max(10000).optional(), having: z.string().max(10000).optional(), @@ -492,7 +530,7 @@ const externalDashboardCategoricalBarRawSqlChartConfigSchema = const externalDashboardNumberChartConfigSchema = z.object({ displayType: z.literal('number'), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, // Exactly one select item unless `formulas` is set: a formula number tile // displays the (single) formula value and its select entries are the // formula's operands (e.g. A / (A + B)). The one-item rule for @@ -532,7 +570,7 @@ const externalDashboardNumberChartConfigSchema = z.object({ const externalDashboardPieChartConfigSchema = z.object({ displayType: z.literal('pie'), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, select: z.array(externalDashboardSelectItemSchema).length(1), groupBy: z.string().max(10000).optional(), orderBy: z.string().max(10000).optional(), @@ -543,7 +581,7 @@ const externalDashboardPieChartConfigSchema = z.object({ const externalDashboardCategoricalBarChartConfigSchema = z.object({ displayType: z.literal('bar'), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, select: z.array(externalDashboardSelectItemSchema).length(1), groupBy: z.string().max(10000).optional(), orderBy: z.string().max(10000).optional(), @@ -580,7 +618,7 @@ export type ExternalDashboardHeatmapSelectItem = z.infer< const externalDashboardHeatmapChartConfigSchema = z.object({ displayType: z.literal('heatmap'), heatmapMode: z.literal('distribution').optional(), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, select: z.array(externalDashboardHeatmapSelectItemSchema).length(1), where: z.string().max(10000).optional().default(''), // `whereLanguageSchema` (an alias for `SearchConditionLanguageSchema`) @@ -594,7 +632,7 @@ const externalDashboardHeatmapChartConfigSchema = z.object({ const externalDashboardHeatmapSeriesChartConfigSchema = z.object({ displayType: z.literal('heatmap'), heatmapMode: z.literal('series'), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, select: z.array(externalDashboardSelectItemSchema).length(1), groupBy: z.string().max(10000).optional(), numberFormat: NumberFormatSchema.optional(), @@ -602,7 +640,7 @@ const externalDashboardHeatmapSeriesChartConfigSchema = z.object({ const externalDashboardSearchChartConfigSchema = z.object({ displayType: z.literal('search'), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, select: z.string().max(10000), where: z.string().max(10000).optional().default(''), whereLanguage: whereLanguageSchema, @@ -626,7 +664,7 @@ const externalDashboardMarkdownChartConfigSchema = z.object({ const externalDashboardEventPatternsChartConfigSchema = z.object({ displayType: z.literal('event_patterns'), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, select: z.string().max(10000).optional().default(''), where: z.string().max(10000).optional().default(''), whereLanguage: whereLanguageSchema, @@ -720,7 +758,7 @@ const externalPromqlLegendTemplateSchema = z const externalPromqlChartConfigBaseSchema = z.object({ configType: z.literal('promql'), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, // Sets the Prometheus step and `$__interval` of range queries. granularity: z.union([SQLIntervalSchema, z.literal('auto')]).optional(), numberFormat: NumberFormatSchema.optional(), diff --git a/packages/app/src/components/ChartEditor/utils.ts b/packages/app/src/components/ChartEditor/utils.ts index 5aa209879..ba2239df5 100644 --- a/packages/app/src/components/ChartEditor/utils.ts +++ b/packages/app/src/components/ChartEditor/utils.ts @@ -13,10 +13,13 @@ import { validateRawSqlForAlert, } from '@hyperdx/common-utils/dist/core/utils'; import { - getHeatmapSourceKinds, + getAllowedSourceKinds, isBuilderSavedChartConfig, + isPromqlDisplayType, isPromqlSavedChartConfig, isRawSqlSavedChartConfig, + isStringSelectDisplayType, + toBuilderHeatmapMode, } from '@hyperdx/common-utils/dist/guards'; import { MACRO_SUGGESTIONS } from '@hyperdx/common-utils/dist/macros'; import { QUERY_PARAMS_BY_DISPLAY_TYPE } from '@hyperdx/common-utils/dist/rawSqlParams'; @@ -26,12 +29,10 @@ import { ChartVariable, DisplayType, getSampleWeightExpression, - HeatmapMode, isLogSource, isMetricSource, isRangeThresholdType, isTraceSource, - NON_PROMQL_SOURCE_KINDS, PromqlChartConfig, PromqlHeatmapMode, PromqlSavedChartConfig, @@ -178,68 +179,6 @@ export const isRawSqlDisplayType = ( displayType === DisplayType.Bar || displayType === DisplayType.Number; -/** - * Display types that store `select` as a plain string (column expression) - * rather than a structured `DerivedColumn[]` series array. These types - * don't use the builder series editor and skip series-level validation. - */ -export const isStringSelectDisplayType = ( - displayType: DisplayType | undefined, -): displayType is DisplayType.Search | DisplayType.EventPatterns => - displayType === DisplayType.Search || - displayType === DisplayType.EventPatterns; - -export const isPromqlDisplayType = ( - displayType: DisplayType | undefined, -): displayType is - | DisplayType.Table - | DisplayType.Line - | DisplayType.StackedBar - | DisplayType.StackedLine - | DisplayType.Pie - | DisplayType.Bar - | DisplayType.Number - | DisplayType.Heatmap => - displayType === DisplayType.Table || - displayType === DisplayType.Line || - displayType === DisplayType.StackedBar || - displayType === DisplayType.StackedLine || - displayType === DisplayType.Pie || - displayType === DisplayType.Bar || - displayType === DisplayType.Number || - displayType === DisplayType.Heatmap; - -/** - * Search and event patterns list raw rows out of the source's `from` table. A - * metric source keeps its rows in `metricTables` and leaves `from.tableName` - * empty, so there is nothing for them to read. - */ -const ROW_LISTING_SOURCE_KINDS = NON_PROMQL_SOURCE_KINDS.filter( - kind => kind !== SourceKind.Metric, -); - -/** Source kinds the Data Source picker offers, given the editor mode and the display type. */ -export function getAllowedSourceKinds({ - configType, - displayType, - heatmapMode, -}: { - configType: ChartEditorFormState['configType']; - displayType: DisplayType | undefined; - heatmapMode?: PromqlHeatmapMode; -}): SourceKind[] { - if (configType === 'promql' && isPromqlDisplayType(displayType)) { - return [SourceKind.Promql]; - } - if (displayType === DisplayType.Heatmap) { - return [...getHeatmapSourceKinds(toBuilderHeatmapMode(heatmapMode))]; - } - if (isStringSelectDisplayType(displayType)) { - return ROW_LISTING_SOURCE_KINDS; - } - return [...NON_PROMQL_SOURCE_KINDS]; -} - const isCustomOrderByDisplayType = ( displayType: DisplayType | undefined, ): displayType is DisplayType.Table | DisplayType.Bar | DisplayType.Pie => @@ -252,6 +191,15 @@ const isCustomOrderByDisplayType = ( // ChartEditorControls and the server-side surfaces cannot drift. export { isFormulaDisplayType, isFormulaSourceKind }; +// Shared with MCP tile runs, which reject a variable's source selection the +// tile can't use, as the dashboard does. +export { + getAllowedSourceKinds, + isPromqlDisplayType, + isStringSelectDisplayType, + toBuilderHeatmapMode, +}; + export function convertFormStateToSavedChartConfig( form: ChartEditorFormState, source: TSource | undefined, @@ -569,13 +517,6 @@ function formBuilderHeatmap( : { mode: toBuilderHeatmapMode(form.heatmap.mode) }; } -/** A heatmap mode as a builder heatmap's, which has no histogram mode. */ -export function toBuilderHeatmapMode( - mode: PromqlHeatmapMode | undefined, -): HeatmapMode | undefined { - return mode === 'histogram' ? undefined : mode; -} - const DEFAULT_PROMQL_PLACEHOLDER = "rate(http_requests_total{service='api'}[5m])"; diff --git a/packages/common-utils/src/__tests__/guards.test.ts b/packages/common-utils/src/__tests__/guards.test.ts index e7789a0c9..26c9cfb30 100644 --- a/packages/common-utils/src/__tests__/guards.test.ts +++ b/packages/common-utils/src/__tests__/guards.test.ts @@ -1,5 +1,6 @@ import { displayTypeRequiresSource, + getSavedConfigAllowedSourceKinds, isBuilderSavedChartConfig, isRawSqlSavedChartConfig, } from '@/guards'; @@ -9,6 +10,8 @@ import { DisplayType, isChartPaletteToken, resolveChartPaletteToken, + SavedChartConfig, + SourceKind, walkRawDashboardTileColors, } from '@/types'; @@ -367,3 +370,46 @@ describe('walkRawDashboardTileColors', () => { expect(walkRawDashboardTileColors(input, () => 'x')).toBe(input); }); }); + +describe('getSavedConfigAllowedSourceKinds', () => { + it.each<{ + case: string; + config: Partial; + kinds: SourceKind[]; + }>([ + { + case: 'a PromQL chart', + config: { configType: 'promql', displayType: DisplayType.Line }, + kinds: [SourceKind.Promql], + }, + { + case: 'a builder search', + config: { displayType: DisplayType.Search }, + kinds: [SourceKind.Log, SourceKind.Trace, SourceKind.Session], + }, + { + case: 'a builder distribution heatmap', + config: { displayType: DisplayType.Heatmap }, + kinds: [SourceKind.Trace], + }, + { + case: 'a builder series heatmap', + config: { displayType: DisplayType.Heatmap, heatmap: { mode: 'series' } }, + kinds: [SourceKind.Trace, SourceKind.Log, SourceKind.Metric], + }, + { + case: 'a raw SQL table', + config: { configType: 'sql', displayType: DisplayType.Table }, + kinds: [ + SourceKind.Log, + SourceKind.Trace, + SourceKind.Session, + SourceKind.Metric, + ], + }, + ])('lists the kinds for $case', ({ config, kinds }) => { + expect( + getSavedConfigAllowedSourceKinds(config as SavedChartConfig).sort(), + ).toEqual([...kinds].sort()); + }); +}); diff --git a/packages/common-utils/src/guards.ts b/packages/common-utils/src/guards.ts index 2b6d71657..fcc009fe3 100644 --- a/packages/common-utils/src/guards.ts +++ b/packages/common-utils/src/guards.ts @@ -5,7 +5,9 @@ import { ChartConfigWithOptDateRange, DisplayType, HeatmapMode, + NON_PROMQL_SOURCE_KINDS, PromqlChartConfig, + PromqlHeatmapMode, PromqlSavedChartConfig, RawSqlChartConfig, RawSqlSavedChartConfig, @@ -94,3 +96,89 @@ export function displayTypeRequiresSource( ): boolean { return displayType !== DisplayType.Markdown; } + +/** + * Display types that store `select` as a plain string (column expression) + * rather than a structured `DerivedColumn[]` series array. These types + * don't use the builder series editor and skip series-level validation. + */ +export const isStringSelectDisplayType = ( + displayType: DisplayType | undefined, +): displayType is DisplayType.Search | DisplayType.EventPatterns => + displayType === DisplayType.Search || + displayType === DisplayType.EventPatterns; + +export const isPromqlDisplayType = ( + displayType: DisplayType | undefined, +): displayType is + | DisplayType.Table + | DisplayType.Line + | DisplayType.StackedBar + | DisplayType.StackedLine + | DisplayType.Pie + | DisplayType.Bar + | DisplayType.Number + | DisplayType.Heatmap => + displayType === DisplayType.Table || + displayType === DisplayType.Line || + displayType === DisplayType.StackedBar || + displayType === DisplayType.StackedLine || + displayType === DisplayType.Pie || + displayType === DisplayType.Bar || + displayType === DisplayType.Number || + displayType === DisplayType.Heatmap; + +/** A heatmap mode as a builder heatmap's, which has no histogram mode. */ +export function toBuilderHeatmapMode( + mode: PromqlHeatmapMode | undefined, +): HeatmapMode | undefined { + return mode === 'histogram' ? undefined : mode; +} + +/** + * Search and event patterns list raw rows out of the source's `from` table. A + * metric source keeps its rows in `metricTables` and leaves `from.tableName` + * empty, so there is nothing for them to read. + */ +const ROW_LISTING_SOURCE_KINDS = NON_PROMQL_SOURCE_KINDS.filter( + kind => kind !== SourceKind.Metric, +); + +/** Source kinds a chart can use, given its config type and display type. */ +export function getAllowedSourceKinds({ + configType, + displayType, + heatmapMode, +}: { + configType: 'builder' | 'sql' | 'promql' | undefined; + displayType: DisplayType | undefined; + heatmapMode?: PromqlHeatmapMode; +}): SourceKind[] { + if (configType === 'promql' && isPromqlDisplayType(displayType)) { + return [SourceKind.Promql]; + } + if (displayType === DisplayType.Heatmap) { + return [...getHeatmapSourceKinds(toBuilderHeatmapMode(heatmapMode))]; + } + if (isStringSelectDisplayType(displayType)) { + return ROW_LISTING_SOURCE_KINDS; + } + return [...NON_PROMQL_SOURCE_KINDS]; +} + +/** `getAllowedSourceKinds` for a saved chart config. */ +export function getSavedConfigAllowedSourceKinds( + config: SavedChartConfig, +): SourceKind[] { + if (isRawSqlSavedChartConfig(config)) { + return getAllowedSourceKinds({ + configType: 'sql', + displayType: config.displayType, + }); + } + return getAllowedSourceKinds({ + configType: isPromqlSavedChartConfig(config) ? 'promql' : 'builder', + displayType: config.displayType, + heatmapMode: config.heatmap?.mode, + }); +}