From bb94ed5d59336db9af87015629e64e4dd144c69a Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Mon, 16 Nov 2026 11:09:05 -0500 Subject: [PATCH 01/18] feat: add data source and connection variable types to dashboards --- .changeset/source-vars-1.md | 7 + packages/api/src/controllers/alerts.ts | 6 + .../api/__tests__/dashboard.int.test.ts | 124 ++++ packages/api/src/routers/api/dashboards.ts | 29 +- .../external-api/v2/utils/dashboards.ts | 20 +- .../__tests__/provisionDashboards.int.test.ts | 35 + .../src/tasks/provisionDashboards/index.ts | 11 +- packages/api/src/utils/externalApi.ts | 19 +- .../app/src/components/ChartEditor/utils.ts | 7 +- .../DashboardFiltersList.tsx | 34 +- .../__tests__/DashboardFiltersModal.test.tsx | 1 + .../__tests__/filterFormState.test.ts | 5 +- .../src/__tests__/referenceVariables.test.ts | 702 ++++++++++++++++++ packages/common-utils/src/core/utils.ts | 70 +- .../common-utils/src/dashboardFilterValues.ts | 16 + .../common-utils/src/dashboardValidation.ts | 146 +++- packages/common-utils/src/filters.ts | 55 +- .../common-utils/src/referenceVariables.ts | 360 +++++++++ packages/common-utils/src/types.ts | 67 +- 19 files changed, 1637 insertions(+), 77 deletions(-) create mode 100644 .changeset/source-vars-1.md create mode 100644 packages/common-utils/src/__tests__/referenceVariables.test.ts create mode 100644 packages/common-utils/src/referenceVariables.ts diff --git a/.changeset/source-vars-1.md b/.changeset/source-vars-1.md new file mode 100644 index 000000000..eeacef4d1 --- /dev/null +++ b/.changeset/source-vars-1.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/common-utils': minor +'@hyperdx/api': minor +'@hyperdx/app': patch +--- + +feat: add data source and connection variable types to dashboards diff --git a/packages/api/src/controllers/alerts.ts b/packages/api/src/controllers/alerts.ts index f1c69d0ee..f49cf85a6 100644 --- a/packages/api/src/controllers/alerts.ts +++ b/packages/api/src/controllers/alerts.ts @@ -9,6 +9,7 @@ import { isPromqlSavedChartConfig, isRawSqlSavedChartConfig, } from '@hyperdx/common-utils/dist/guards'; +import { configUsesReferenceVariable } from '@hyperdx/common-utils/dist/referenceVariables'; import { isRangeThresholdType } from '@hyperdx/common-utils/dist/types'; import { groupBy } from 'lodash'; import { type FilterQuery, Types } from 'mongoose'; @@ -97,6 +98,11 @@ export const validateAlertInput = async ( // The alert checker cannot evaluate these tiles, so the alert would never run. if (tile.config != null && !savedChartConfigSupportsAlerts(tile.config)) { + if (configUsesReferenceVariable(tile.config)) { + throw new Api400Error( + 'Alerts are not supported on tiles that use a data source or connection variable', + ); + } if (isPromqlSavedChartConfig(tile.config)) { throw new Api400Error('Alerts are not supported on PromQL tiles'); } diff --git a/packages/api/src/routers/api/__tests__/dashboard.int.test.ts b/packages/api/src/routers/api/__tests__/dashboard.int.test.ts index 405e260fd..de6f975a8 100644 --- a/packages/api/src/routers/api/__tests__/dashboard.int.test.ts +++ b/packages/api/src/routers/api/__tests__/dashboard.int.test.ts @@ -1519,6 +1519,130 @@ describe('dashboard router', () => { ); }); + describe('data source variables', () => { + const dataSourceFilter = { + id: new Types.ObjectId().toString(), + type: 'DATA_SOURCE' as const, + name: 'Logs source', + variableName: 'logs', + allowedSourceKinds: [SourceKind.Log], + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }; + + it('persists a tile and a filter that reference the variable', async () => { + const tile = makeTile({ sourceId: '$logs' }); + const queriedFilter = { + id: new Types.ObjectId().toString(), + type: 'QUERY_EXPRESSION' as const, + name: 'Service', + expression: 'ServiceName', + source: '$logs', + }; + + const created = await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + tiles: [tile], + filters: [dataSourceFilter, queriedFilter], + }) + .expect(200); + + expect(created.body.tiles[0].config.source).toBe('$logs'); + expect(created.body.filters).toEqual([dataSourceFilter, queriedFilter]); + }); + + it('rejects a reference to an unknown variable', async () => { + const res = await agent + .post('/dashboards') + .send({ ...MOCK_DASHBOARD, tiles: [makeTile({ sourceId: '$nope' })] }) + .expect(400); + + expect(JSON.stringify(res.body)).toMatch( + /does not name a data source variable/, + ); + }); + + it('rejects a variable offering kinds the reference cannot use', async () => { + const res = await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + tiles: [makeTile({ sourceId: '$logs' })], + filters: [ + { + ...dataSourceFilter, + allowedSourceKinds: [SourceKind.Log, SourceKind.Promql], + }, + ], + }) + .expect(400); + + expect(JSON.stringify(res.body)).toMatch(/promql/); + }); + + it('validates a PATCH of tiles against the stored filters', async () => { + const created = await agent + .post('/dashboards') + .send({ ...MOCK_DASHBOARD, filters: [dataSourceFilter] }) + .expect(200); + + await agent + .patch(`/dashboards/${created.body.id}`) + .send({ tiles: [makeTile({ sourceId: '$logs' })] }) + .expect(200); + await agent + .patch(`/dashboards/${created.body.id}`) + .send({ filters: [] }) + .expect(400); + }); + + it('drops an alert on a tile that uses the variable', async () => { + await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + tiles: [ + makeTile({ + sourceId: '$logs', + alert: makeMockAlert(webhook._id.toString()), + }), + ], + filters: [dataSourceFilter], + }) + .expect(200); + + expect(await Alert.countDocuments({ team: team._id })).toBe(0); + }); + + it('rejects a tile alert on a tile that uses the variable', async () => { + const created = await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + tiles: [makeTile({ sourceId: '$logs' })], + filters: [dataSourceFilter], + }) + .expect(200); + + const res = await agent + .post('/alerts') + .send( + makeAlertInput({ + dashboardId: created.body.id, + tileId: created.body.tiles[0].id, + webhookId: webhook._id.toString(), + }), + ) + .expect(400); + + expect(res.body.message).toMatch(/data source or connection variable/); + }); + }); + describe('promql-label filters', () => { const createPromqlSource = () => Source.create({ diff --git a/packages/api/src/routers/api/dashboards.ts b/packages/api/src/routers/api/dashboards.ts index e88f39e32..35c2781de 100644 --- a/packages/api/src/routers/api/dashboards.ts +++ b/packages/api/src/routers/api/dashboards.ts @@ -1,9 +1,11 @@ import { + getVariableReferenceError, validateDashboardFilterModes, validateDashboardFilterOptionUniqueness, validateDashboardFilterVariableNames, } from '@hyperdx/common-utils/dist/dashboardValidation'; import { isSqlQueryFilter } from '@hyperdx/common-utils/dist/filters'; +import { parseVariableReference } from '@hyperdx/common-utils/dist/referenceVariables'; import { DashboardFilter, DashboardSchema, @@ -77,7 +79,10 @@ async function validateFilterSources( filters: DashboardFilter[] = [], existingFilters: DashboardFilter[] = [], ): Promise { - const refs = getChangedFilterSourceRefs(filters, existingFilters); + const refs = getChangedFilterSourceRefs(filters, existingFilters).filter( + // Variable references are checked by validateDashboardVariableReferences + ref => parseVariableReference(ref.id) == null, + ); if (refs.length === 0) return null; return getFilterSourceRefError(refs, await getSources(teamId.toString())); } @@ -181,10 +186,9 @@ router.post( // dashboard to the provisioner. const dashboard = _.omit(req.body, 'provisioned'); - const filterError = await validateFilterReferences( - teamId, - req.body.filters, - ); + const filterError = + getVariableReferenceError(req.body.tiles, req.body.filters) ?? + (await validateFilterReferences(teamId, req.body.filters)); if (filterError != null) { return res.status(400).json({ message: filterError }); } @@ -229,11 +233,16 @@ router.patch( // null still signals field removal downstream. const updates = DashboardWithoutIdSchema.partial().parse(req.body); - const filterError = await validateFilterReferences( - teamId, - req.body.filters, - dashboard.filters, - ); + const filterError = + getVariableReferenceError( + req.body.tiles ?? dashboard.tiles, + req.body.filters ?? dashboard.filters, + ) ?? + (await validateFilterReferences( + teamId, + req.body.filters, + dashboard.filters, + )); if (filterError != null) { return res.status(400).json({ message: filterError }); } diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index 0590b30fc..067980de9 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -6,6 +6,7 @@ import { TIME_SERIES_DISPLAY_TYPE_BY_NAME, } from '@hyperdx/common-utils/dist/core/utils'; import { + collectRefinementIssues, validateDashboardContainersStructure, validateDashboardFilterFieldGating, validateDashboardFilterModes, @@ -59,6 +60,7 @@ import Dashboard, { DashboardDocument } from '@/models/dashboard'; import { translateExternalChartToTileConfig, translateExternalFilterToFilter, + getExternalCompatibleFilters, translateFilterToExternalFilter, } from '@/utils/externalApi'; import { @@ -811,7 +813,7 @@ export function convertToExternalDashboard( convertTileToExternalChart(tile, containerById, dashboardId), ), tags: dashboard.tags || [], - filters: (dashboard.filters ?? []) + filters: getExternalCompatibleFilters(dashboard.filters) .filter(filter => !(hidePromql && isPromqlFilter(filter))) .map(translateFilterToExternalFilter), savedQuery: dashboard.savedQuery ?? null, @@ -1951,15 +1953,13 @@ export function collectTileContainerRefIssues( containers: DashboardContainer[], tiles: ExternalDashboardTileWithId[], ): string[] { - const schema = z.object({}).superRefine((_, ctx) => { - const containerById = new Map( - containers.map(c => [c.id, c]), - ); - validateDashboardTileContainerRefs(containerById, tiles, ctx); - }); - const result = schema.safeParse({}); - if (result.success) return []; - return result.error.issues.map(issue => { + const containerById = new Map( + containers.map(c => [c.id, c]), + ); + const issues = collectRefinementIssues(ctx => + validateDashboardTileContainerRefs(containerById, tiles, ctx), + ); + return issues.map(issue => { const path = issue.path.length > 0 ? `${issue.path.join('.')}: ` : ''; return `${path}${issue.message}`; }); diff --git a/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts b/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts index 3e1759481..c91e79b5b 100644 --- a/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts +++ b/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts @@ -143,6 +143,41 @@ describe('provisionDashboards', () => { // rejects the legacy enum and `readDashboardFiles` skips the file // outright, so a provisioned dashboard authored against #2265 // wouldn't ship at all. + it('parses a tile that references a declared data source variable', () => { + fs.writeFileSync( + path.join(tmpDir, 'source-variable.json'), + JSON.stringify({ + name: 'Source Variable', + tiles: [makeTile({ sourceId: '$logs' })], + filters: [ + { + id: 'ds', + type: 'DATA_SOURCE', + name: 'Logs source', + variableName: 'logs', + allowedSourceKinds: ['log'], + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }, + ], + }), + ); + expect(readDashboardFiles(tmpDir)).toHaveLength(1); + }); + + it('skips a tile that references an undeclared data source variable', () => { + fs.writeFileSync( + path.join(tmpDir, 'unknown-variable.json'), + JSON.stringify({ + name: 'Unknown Variable', + tiles: [makeTile({ sourceId: '$logs' })], + }), + ); + expect(readDashboardFiles(tmpDir)).toEqual([]); + }); + it('migrates legacy chart-N tile colors before schema validation', () => { const tile = makeTile(); (tile.config as any).color = 'chart-1'; diff --git a/packages/api/src/tasks/provisionDashboards/index.ts b/packages/api/src/tasks/provisionDashboards/index.ts index cf10427c7..4199d9618 100644 --- a/packages/api/src/tasks/provisionDashboards/index.ts +++ b/packages/api/src/tasks/provisionDashboards/index.ts @@ -1,4 +1,7 @@ -import { validateDashboardFilterOptionUniqueness } from '@hyperdx/common-utils/dist/dashboardValidation'; +import { + validateDashboardFilterOptionUniqueness, + validateDashboardVariableReferences, +} from '@hyperdx/common-utils/dist/dashboardValidation'; import { DashboardWithoutId, DashboardWithoutIdSchema, @@ -30,8 +33,10 @@ function migrateLegacyDashboardTileColorsRaw(raw: unknown): unknown { } const provisionedDashboardSchema = DashboardWithoutIdSchema.superRefine( - (data, ctx) => - validateDashboardFilterOptionUniqueness(data.filters ?? [], ctx), + (data, ctx) => { + validateDashboardFilterOptionUniqueness(data.filters ?? [], ctx); + validateDashboardVariableReferences(data.tiles, data.filters ?? [], ctx); + }, ); export function readDashboardFiles(dir: string): DashboardWithoutId[] { diff --git a/packages/api/src/utils/externalApi.ts b/packages/api/src/utils/externalApi.ts index a626c5e49..91f627748 100644 --- a/packages/api/src/utils/externalApi.ts +++ b/packages/api/src/utils/externalApi.ts @@ -3,6 +3,7 @@ import { isFilterBroadcastEnabled, isFilterRequired, isFilterVariableEnabled, + isReferenceVariableFilter, } from '@hyperdx/common-utils/dist/filters'; import { AlertErrorType, @@ -238,8 +239,24 @@ export function translateExternalChartToTileConfig( }; } +/** Filter types the external API and MCP don't expose yet. */ +export type ExternalCompatibleDashboardFilter = Exclude< + DashboardFilter, + { type: 'DATA_SOURCE' | 'CONNECTION' } +>; + +/** The filters the external API and MCP may read. */ +export function getExternalCompatibleFilters( + filters: DashboardFilter[] | undefined, +): ExternalCompatibleDashboardFilter[] { + return (filters ?? []).filter( + (filter): filter is ExternalCompatibleDashboardFilter => + !isReferenceVariableFilter(filter), + ); +} + export function translateFilterToExternalFilter( - filter: DashboardFilter, + filter: ExternalCompatibleDashboardFilter, ): ExternalDashboardFilterWithId { // Omitted from responses unless the filter is required, so a GET response // can be PUT back verbatim. diff --git a/packages/app/src/components/ChartEditor/utils.ts b/packages/app/src/components/ChartEditor/utils.ts index 5607dcdaa..5aa209879 100644 --- a/packages/app/src/components/ChartEditor/utils.ts +++ b/packages/app/src/components/ChartEditor/utils.ts @@ -31,6 +31,7 @@ import { isMetricSource, isRangeThresholdType, isTraceSource, + NON_PROMQL_SOURCE_KINDS, PromqlChartConfig, PromqlHeatmapMode, PromqlSavedChartConfig, @@ -208,10 +209,6 @@ export const isPromqlDisplayType = ( displayType === DisplayType.Number || displayType === DisplayType.Heatmap; -const NON_PROMQL_SOURCE_KINDS = Object.values(SourceKind).filter( - kind => kind !== SourceKind.Promql, -); - /** * Search and event patterns list raw rows out of the source's `from` table. A * metric source keeps its rows in `metricTables` and leaves `from.tableName` @@ -240,7 +237,7 @@ export function getAllowedSourceKinds({ if (isStringSelectDisplayType(displayType)) { return ROW_LISTING_SOURCE_KINDS; } - return NON_PROMQL_SOURCE_KINDS; + return [...NON_PROMQL_SOURCE_KINDS]; } const isCustomOrderByDisplayType = ( diff --git a/packages/app/src/components/DashboardFiltersModal/DashboardFiltersList.tsx b/packages/app/src/components/DashboardFiltersModal/DashboardFiltersList.tsx index 2f7cd9ece..6828e205e 100644 --- a/packages/app/src/components/DashboardFiltersModal/DashboardFiltersList.tsx +++ b/packages/app/src/components/DashboardFiltersModal/DashboardFiltersList.tsx @@ -5,6 +5,7 @@ import { isFilterReferenceableVariable, isFilterRequired, } from '@hyperdx/common-utils/dist/filters'; +import { getObjectName } from '@hyperdx/common-utils/dist/referenceVariables'; import { DashboardFilter } from '@hyperdx/common-utils/dist/types'; import { Button, @@ -17,16 +18,20 @@ import { import { IconAsterisk, IconBuildingBroadcastTower, + IconDatabase, IconFilter, IconLabel, IconList, IconPencil, IconRefresh, IconSearch, + IconServer, IconSql, IconTrash, } from '@tabler/icons-react'; +import { SOURCE_KIND_LABELS } from '@/components/Sources/SourceForm/constants'; +import { useConnections } from '@/connection'; import { useSources } from '@/source'; import { MODAL_SIZE } from './constants'; @@ -54,7 +59,7 @@ function getSourceNames( sources?: { id: string; name: string }[], ) { return ids - .map(id => sources?.find(s => s.id === id)?.name) + .map(id => getObjectName(id, sources)) .filter((name): name is string => !!name) .join(', '); } @@ -63,19 +68,20 @@ function getSourceNames( function getValuesAttribute( filter: DashboardFilter, sources?: { id: string; name: string }[], + connections?: { id: string; name: string }[], ): DashboardFilterAttribute { switch (filter.type) { case 'QUERY_EXPRESSION': return { icon: , tooltip: 'Source the dropdown values are queried from', - label: sources?.find(s => s.id === filter.source)?.name ?? '', + label: getObjectName(filter.source, sources) ?? '', }; case 'PROMETHEUS_LABEL': return { icon: , tooltip: 'PromQL source and label the dropdown values come from', - label: [sources?.find(s => s.id === filter.source)?.name, filter.label] + label: [getObjectName(filter.source, sources), filter.label] .filter(Boolean) .join(' · '), }; @@ -98,6 +104,25 @@ function getValuesAttribute( tooltip: 'The dropdown values come from a SQL query', label: 'SQL query', }; + case 'DATA_SOURCE': + return { + icon: , + tooltip: + 'Kinds of sources offered, and the connection they are limited to', + label: [ + filter.allowedSourceKinds + .map(kind => SOURCE_KIND_LABELS[kind]) + .join(', '), + filter.connection && getObjectName(filter.connection, connections), + ] + .filter(Boolean) + .join(' · '), + }; + case 'CONNECTION': + return { + icon: , + label: 'Connections', + }; default: filter satisfies never; // exhaustive check return { icon: , label: '' }; @@ -129,6 +154,7 @@ export const DashboardFiltersList = ({ onAddNew, }: DashboardFiltersListProps) => { const { data: sources } = useSources(); + const { data: connections } = useConnections(); return ( {filters.map(filter => { - const attributes = [getValuesAttribute(filter, sources)]; + const attributes = [getValuesAttribute(filter, sources, connections)]; const broadcastTarget = getBroadcastTargetDisplay(filter, sources); if (!hideAppliesTo && broadcastTarget != null) { diff --git a/packages/app/src/components/DashboardFiltersModal/__tests__/DashboardFiltersModal.test.tsx b/packages/app/src/components/DashboardFiltersModal/__tests__/DashboardFiltersModal.test.tsx index 91d55296f..04b1d99c6 100644 --- a/packages/app/src/components/DashboardFiltersModal/__tests__/DashboardFiltersModal.test.tsx +++ b/packages/app/src/components/DashboardFiltersModal/__tests__/DashboardFiltersModal.test.tsx @@ -12,6 +12,7 @@ import { FilterFormControl } from '@/components/DashboardFiltersModal/filterForm jest.mock('@/useConfirm', () => ({ useConfirm: () => jest.fn() })); jest.mock('@/source', () => ({ useSources: () => ({ data: [] }) })); +jest.mock('@/connection', () => ({ useConnections: () => ({ data: [] }) })); // Mantine's Combobox calls scrollIntoView when its dropdown opens; jsdom lacks it. window.HTMLElement.prototype.scrollIntoView = jest.fn(); diff --git a/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts b/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts index 63e0497c8..27a9bcb66 100644 --- a/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts +++ b/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts @@ -327,7 +327,10 @@ describe('toSavedFilter', () => { /** The minimum a form needs to save, per filter type the form supports. */ const byType: Record< - Exclude, + Exclude< + DashboardFilter['type'], + 'ADHOC' | 'SQL_QUERY' | 'DATA_SOURCE' | 'CONNECTION' + >, Partial > = { QUERY_EXPRESSION: { diff --git a/packages/common-utils/src/__tests__/referenceVariables.test.ts b/packages/common-utils/src/__tests__/referenceVariables.test.ts new file mode 100644 index 000000000..278dca4a9 --- /dev/null +++ b/packages/common-utils/src/__tests__/referenceVariables.test.ts @@ -0,0 +1,702 @@ +import { + convertToDashboardTemplate, + savedChartConfigSupportsAlerts, +} from '@/core/utils'; +import { getBlockingRequiredFilterNames } from '@/dashboardFilterValues'; +import { + collectRefinementIssues, + getVariableReferenceError, + validateDashboardVariableReferences, +} from '@/dashboardValidation'; +import { + getDashboardVariableDeclarations, + isReferenceVariableFilterType, + validateVariableName, +} from '@/filters'; +import { + configUsesReferenceVariable, + getConfigReferenceVariables, + getConnectionVariableNames, + getObjectName, + getSourceVariableDeclarations, + isSourceOfferedByVariable, + isSourceVariableReferenceAccepted, + parseVariableReference, + resolveConfigReferences, + resolveFilterReferences, + resolveVariableReference, + restoreConfigReferences, +} from '@/referenceVariables'; +import { + AdhocDashboardFilter, + ConnectionDashboardFilter, + DashboardFilter, + DashboardFilterSchema, + DataSourceDashboardFilter, + DisplayType, + QueryExpressionDashboardFilter, + RawSqlSavedChartConfig, + SavedChartConfig, + SourceKind, +} from '@/types'; + +const dataSourceFilter = ( + overrides: Partial = {}, +): DataSourceDashboardFilter => ({ + id: 'ds', + type: 'DATA_SOURCE', + name: 'Logs source', + variableName: 'logs', + allowedSourceKinds: [SourceKind.Log], + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + ...overrides, +}); + +const queriedFilter = ( + overrides: Partial = {}, +): QueryExpressionDashboardFilter => ({ + id: 'q', + type: 'QUERY_EXPRESSION', + name: 'Service', + expression: 'ServiceName', + source: '$logs', + ...overrides, +}); + +const connectionFilter = ( + overrides: Partial = {}, +): ConnectionDashboardFilter => ({ + id: 'conn-filter', + type: 'CONNECTION', + name: 'Connection', + variableName: 'conn', + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + ...overrides, +}); + +const rawSqlConfig = ( + overrides: Partial = {}, +): RawSqlSavedChartConfig => ({ + configType: 'sql', + displayType: DisplayType.Table, + sqlTemplate: 'SELECT 1', + connection: 'conn-1', + ...overrides, +}); + +const lineConfig = (source: string): SavedChartConfig => ({ + source, + displayType: DisplayType.Line, + select: [{ aggFn: 'count', valueExpression: '' }], + where: '', + whereLanguage: 'lucene', +}); + +describe('DataSourceDashboardFilterSchema', () => { + it('accepts a data source filter', () => { + expect(DashboardFilterSchema.safeParse(dataSourceFilter()).success).toBe( + true, + ); + }); + + it('requires at least one allowed kind', () => { + expect( + DashboardFilterSchema.safeParse( + dataSourceFilter({ allowedSourceKinds: [] }), + ).success, + ).toBe(false); + }); + + it('is always single-select and required', () => { + expect( + DashboardFilterSchema.safeParse({ + ...dataSourceFilter(), + maxSelections: undefined, + }).success, + ).toBe(false); + expect( + DashboardFilterSchema.safeParse({ + ...dataSourceFilter(), + minSelections: 0, + }).success, + ).toBe(false); + }); + + it('rejects unknown keys', () => { + expect( + DashboardFilterSchema.safeParse({ + ...dataSourceFilter(), + expression: 'ServiceName', + }).success, + ).toBe(false); + }); + + it('accepts a global requirement on data source and connection variables', () => { + expect( + DashboardFilterSchema.safeParse( + dataSourceFilter({ isGlobalRequirement: true }), + ).success, + ).toBe(true); + expect( + DashboardFilterSchema.safeParse( + connectionFilter({ isGlobalRequirement: true }), + ).success, + ).toBe(true); + }); +}); + +describe('parseVariableReference', () => { + it.each([ + ['$logs', 'logs'], + ['$Logs_2', 'Logs_2'], + ['65f1c0ffee0000000000abcd', undefined], + ['$', undefined], + ['$1abc', undefined], + ['x$logs', undefined], + ['$logs ', undefined], + [undefined, undefined], + ])('parses %p as %p', (source, expected) => { + expect(parseVariableReference(source)).toBe(expected); + }); +}); + +describe('resolveVariableReference', () => { + it('passes source IDs through and resolves references', () => { + expect(resolveVariableReference('abc', new Map([['logs', 'x']]))).toBe( + 'abc', + ); + expect(resolveVariableReference('$logs', new Map([['logs', 'x']]))).toBe( + 'x', + ); + expect(resolveVariableReference('$logs', new Map())).toBeUndefined(); + expect(resolveVariableReference(undefined, new Map())).toBeUndefined(); + }); +}); + +describe('resolveConfigReferences', () => { + const values = new Map([ + ['logs', 'abc'], + ['conn', 'conn-1'], + ]); + + it('replaces a source reference with the selected source', () => { + expect(resolveConfigReferences(lineConfig('$logs'), values)).toEqual( + lineConfig('abc'), + ); + }); + + it('replaces a connection reference with the selected connection', () => { + expect( + resolveConfigReferences(rawSqlConfig({ connection: '$conn' }), values), + ).toEqual(rawSqlConfig({ connection: 'conn-1' })); + }); + + it('leaves unselected references and IDs as written', () => { + expect(resolveConfigReferences(lineConfig('$logs'), new Map())).toEqual( + lineConfig('$logs'), + ); + expect(resolveConfigReferences(lineConfig('abc'), values)).toEqual( + lineConfig('abc'), + ); + }); +}); + +describe('getConfigReferenceVariables', () => { + it('names the variables a config source and connection reference', () => { + expect( + getConfigReferenceVariables( + rawSqlConfig({ connection: '$conn', source: '$logs' }), + ), + ).toEqual(['logs', 'conn']); + expect(getConfigReferenceVariables(lineConfig('abc'))).toEqual([]); + }); +}); + +describe('isSourceVariableReferenceAccepted', () => { + const declarations = [ + { name: 'logs', allowedSourceKinds: [SourceKind.Log] }, + { name: 'any', allowedSourceKinds: [SourceKind.Log, SourceKind.Promql] }, + ]; + + it('accepts a variable whose every kind the field accepts', () => { + expect( + isSourceVariableReferenceAccepted('$logs', declarations, [ + SourceKind.Log, + SourceKind.Trace, + ]), + ).toBe(true); + }); + + it('rejects a variable offering a kind the field does not accept', () => { + expect( + isSourceVariableReferenceAccepted('$any', declarations, [SourceKind.Log]), + ).toBe(false); + }); + + it('rejects an unknown variable and a source ID', () => { + expect( + isSourceVariableReferenceAccepted('$nope', declarations, [ + SourceKind.Log, + ]), + ).toBe(false); + expect( + isSourceVariableReferenceAccepted('abc', declarations, [SourceKind.Log]), + ).toBe(false); + }); +}); + +describe('resolveFilterReferences', () => { + it('resolves source and appliesToSourceIds', () => { + const { filter, unresolvedVariables } = resolveFilterReferences( + queriedFilter({ appliesToSourceIds: ['$logs', 'other'] }), + new Map([['logs', 'abc']]), + ); + expect(filter.source).toBe('abc'); + expect(filter.appliesToSourceIds).toEqual(['abc', 'other']); + expect(unresolvedVariables).toEqual([]); + }); + + it('leaves unselected references as written so they match no tile', () => { + const adhoc: AdhocDashboardFilter = { + id: 'a', + type: 'ADHOC', + name: 'Conditions', + sourceType: 'sql', + sources: ['$logs'], + isVariableEnabled: true, + }; + const { filter, unresolvedVariables } = resolveFilterReferences( + adhoc, + new Map(), + ); + expect(filter.sources).toEqual(['$logs']); + expect(unresolvedVariables).toEqual(['logs']); + }); +}); + +describe('dashboard variable declarations', () => { + it('keeps the first of two variables sharing a name', () => { + expect( + getSourceVariableDeclarations([ + dataSourceFilter(), + dataSourceFilter({ + id: 'ds2', + allowedSourceKinds: [SourceKind.Trace], + }), + ]), + ).toEqual([ + { + name: 'logs', + allowedSourceKinds: [SourceKind.Log], + connection: undefined, + }, + ]); + }); + + it('keeps reference variables out of text variables', () => { + const filters: DashboardFilter[] = [dataSourceFilter(), connectionFilter()]; + expect(getDashboardVariableDeclarations(filters)).toEqual([]); + expect(getSourceVariableDeclarations(filters)).toEqual([ + { name: 'logs', allowedSourceKinds: [SourceKind.Log] }, + ]); + expect(getConnectionVariableNames(filters)).toEqual(['conn']); + }); + + it('shares the variable name namespace with other filters', () => { + expect( + validateVariableName({ + value: 'logs', + otherFilters: [dataSourceFilter()], + }), + ).toMatch(/used by another filter/); + }); +}); + +describe('alerts', () => { + it('are not supported on tiles that use a connection variable', () => { + expect( + savedChartConfigSupportsAlerts( + rawSqlConfig({ displayType: DisplayType.Line, connection: '$conn' }), + ), + ).toBe(false); + }); + + it('are not supported on tiles that use a data source variable', () => { + expect(configUsesReferenceVariable(lineConfig('$logs'))).toBe(true); + expect(savedChartConfigSupportsAlerts(lineConfig('$logs'))).toBe(false); + expect(savedChartConfigSupportsAlerts(lineConfig('abc'))).toBe(true); + }); +}); + +describe('getBlockingRequiredFilterNames (connections)', () => { + it('blocks a tile until its connection variable has a selection', () => { + expect( + getBlockingRequiredFilterNames({ + config: rawSqlConfig({ connection: '$conn' }), + sourceId: undefined, + unsatisfiedRequiredFilters: [connectionFilter()], + referencedVariables: [], + referenceVariableNames: ['conn'], + }), + ).toEqual(['Connection']); + }); +}); + +describe('getBlockingRequiredFilterNames', () => { + it('blocks a tile until its data source variable has a selection', () => { + const unsatisfied = [dataSourceFilter()]; + expect( + getBlockingRequiredFilterNames({ + config: lineConfig('$logs'), + sourceId: undefined, + unsatisfiedRequiredFilters: unsatisfied, + referencedVariables: [], + referenceVariableNames: ['logs'], + }), + ).toEqual(['Logs source']); + expect( + getBlockingRequiredFilterNames({ + config: lineConfig('abc'), + sourceId: 'abc', + unsatisfiedRequiredFilters: unsatisfied, + referencedVariables: [], + }), + ).toEqual([]); + }); + + it('blocks every tile when the data source variable is a global requirement', () => { + expect( + getBlockingRequiredFilterNames({ + config: lineConfig('abc'), + sourceId: 'abc', + unsatisfiedRequiredFilters: [ + dataSourceFilter({ isGlobalRequirement: true }), + ], + referencedVariables: [], + }), + ).toEqual(['Logs source']); + }); +}); + +describe('validateDashboardVariableReferences', () => { + const collectIssues = ( + tiles: { config: SavedChartConfig }[], + filters: DashboardFilter[], + ) => + collectRefinementIssues(ctx => + validateDashboardVariableReferences(tiles, filters, ctx), + ); + + it('accepts references to declared variables of compatible kinds', () => { + expect( + collectIssues( + [{ config: lineConfig('$logs') }], + [dataSourceFilter(), queriedFilter()], + ), + ).toEqual([]); + }); + + it('accepts connection references to a declared connection variable', () => { + expect( + collectIssues( + [{ config: rawSqlConfig({ connection: '$conn' }) }], + [connectionFilter(), dataSourceFilter({ connection: '$conn' })], + ), + ).toEqual([]); + }); + + it('rejects a connection reference that names no connection variable', () => { + const issues = collectIssues( + [{ config: rawSqlConfig({ connection: '$logs' }) }], + [dataSourceFilter({ connection: '$nope' })], + ); + expect(issues.map(issue => issue.path)).toEqual([ + ['tiles', 0, 'config', 'connection'], + ['filters', 0, 'connection'], + ]); + }); + + it('reports each issue as path: message', () => { + expect( + getVariableReferenceError( + [{ config: rawSqlConfig({ connection: '$logs' }) }], + [dataSourceFilter({ connection: '$nope' })], + ), + ).toBe( + 'tiles.0.config.connection: "$logs" does not name a connection variable on this dashboard; ' + + 'filters.0.connection: "$nope" does not name a connection variable on this dashboard', + ); + expect( + getVariableReferenceError( + [{ config: lineConfig('$logs') }], + [dataSourceFilter()], + ), + ).toBeNull(); + }); + + it('rejects a source reference to a connection variable', () => { + expect( + collectIssues([{ config: lineConfig('$conn') }], [connectionFilter()]), + ).toHaveLength(1); + }); + + it('rejects a reference to an unknown variable', () => { + const issues = collectIssues([{ config: lineConfig('$nope') }], []); + expect(issues).toHaveLength(1); + expect(issues[0].path).toEqual(['tiles', 0, 'config', 'source']); + }); + + it.each<[string, { config: SavedChartConfig }[], DashboardFilter[]]>([ + [ + 'a PromQL tile', + [ + { + config: { + configType: 'promql', + source: '$logs', + displayType: DisplayType.Line, + promqlExpression: [{ expression: 'up' }], + }, + }, + ], + [], + ], + [ + 'a PromQL label filter', + [], + [ + { + id: 'p', + type: 'PROMETHEUS_LABEL', + name: 'Job', + source: '$logs', + label: 'job', + isBroadcastEnabled: false, + isVariableEnabled: true, + }, + ], + ], + [ + 'an ad hoc filter of another source type', + [], + [ + { + id: 'a', + type: 'ADHOC', + name: 'Conditions', + sourceType: 'promql', + sources: ['$logs'], + isVariableEnabled: true, + }, + ], + ], + [ + "a queried filter's applies-to list", + [], + [ + queriedFilter({ + source: 'abc', + appliesToSourceIds: ['$logs'], + }), + ], + ], + ])( + 'rejects a logs-or-PromQL variable used by %s', + (_label, tiles, filters) => { + const issues = collectIssues(tiles, [ + dataSourceFilter({ + allowedSourceKinds: [SourceKind.Log, SourceKind.Promql], + }), + ...filters, + ]); + expect(issues.length).toBeGreaterThan(0); + }, + ); + + it('rejects a variable offering kinds the field does not accept', () => { + const issues = collectIssues( + [], + [ + dataSourceFilter({ + allowedSourceKinds: [SourceKind.Log, SourceKind.Promql], + }), + queriedFilter(), + ], + ); + expect(issues).toHaveLength(1); + expect(issues[0].path).toEqual(['filters', 1, 'source']); + expect(issues[0].message).toMatch(/promql/); + }); +}); + +describe('convertToDashboardTemplate (connections)', () => { + it('keeps connection references and names a data source connection', () => { + const template = convertToDashboardTemplate( + { + id: 'd', + name: 'Dashboard', + tags: [], + tiles: [ + { + id: 't', + x: 0, + y: 0, + w: 6, + h: 4, + config: rawSqlConfig({ connection: '$conn' }), + }, + ], + filters: [ + connectionFilter(), + dataSourceFilter({ connection: 'conn-1' }), + ], + savedFilterValues: [{ type: 'variable', name: 'conn', values: ['c'] }], + }, + [], + [{ id: 'conn-1', name: 'Primary', host: '', username: '' }], + ); + expect(template.tiles[0].config).toMatchObject({ connection: '$conn' }); + expect(template.filters?.[1]).toMatchObject({ connection: 'Primary' }); + expect(template.savedFilterValues).toBeUndefined(); + }); +}); + +describe('convertToDashboardTemplate', () => { + it('keeps references and drops the saved data source selection', () => { + const template = convertToDashboardTemplate( + { + id: 'd', + name: 'Dashboard', + tags: [], + tiles: [ + { id: 't', x: 0, y: 0, w: 6, h: 4, config: lineConfig('$logs') }, + ], + filters: [dataSourceFilter(), queriedFilter()], + savedFilterValues: [ + { type: 'variable', name: 'logs', values: ['abc'] }, + { type: 'variable', name: 'env', values: ['prod'] }, + ], + }, + [], + ); + expect(template.tiles[0].config).toMatchObject({ source: '$logs' }); + expect(template.filters?.[1]).toMatchObject({ source: '$logs' }); + expect(template.savedFilterValues).toEqual([ + { type: 'variable', name: 'env', values: ['prod'] }, + ]); + }); +}); + +describe('resolveFilterReferences (connections)', () => { + it("resolves a data source variable's connection reference", () => { + expect( + resolveFilterReferences( + dataSourceFilter({ connection: '$conn' }), + new Map([['conn', 'conn-1']]), + ), + ).toEqual({ + filter: dataSourceFilter({ connection: 'conn-1' }), + unresolvedVariables: [], + }); + expect( + resolveFilterReferences( + dataSourceFilter({ connection: '$conn' }), + new Map(), + ).unresolvedVariables, + ).toEqual(['conn']); + }); +}); + +describe('restoreConfigReferences', () => { + it('puts stored source and connection references back', () => { + const stored = rawSqlConfig({ connection: '$conn', source: '$logs' }); + const resolved = resolveConfigReferences( + stored, + new Map([ + ['conn', 'conn-1'], + ['logs', 'abc'], + ]), + ); + expect( + restoreConfigReferences( + { ...resolved, displayType: DisplayType.Number }, + stored, + ), + ).toEqual({ ...stored, displayType: DisplayType.Number }); + }); + + it('keeps fields the stored config holds as IDs', () => { + expect( + restoreConfigReferences( + rawSqlConfig({ connection: 'conn-2' }), + rawSqlConfig({ connection: 'conn-1' }), + ), + ).toEqual(rawSqlConfig({ connection: 'conn-2' })); + }); +}); + +describe('isSourceOfferedByVariable', () => { + const source = { kind: SourceKind.Log, connection: 'conn-1' }; + + it('offers an enabled source of an allowed kind on the variable connection', () => { + expect( + isSourceOfferedByVariable(source, { + allowedSourceKinds: [SourceKind.Log], + connection: 'conn-1', + }), + ).toBe(true); + expect( + isSourceOfferedByVariable(source, { + allowedSourceKinds: [SourceKind.Log], + }), + ).toBe(true); + }); + + it.each([ + ['another kind', source, { allowedSourceKinds: [SourceKind.Trace] }], + [ + 'another connection', + source, + { allowedSourceKinds: [SourceKind.Log], connection: 'conn-2' }, + ], + [ + 'an unresolved connection reference', + source, + { allowedSourceKinds: [SourceKind.Log], connection: '$conn' }, + ], + [ + 'a disabled source', + { ...source, disabled: true }, + { allowedSourceKinds: [SourceKind.Log] }, + ], + ])('does not offer %s', (_label, candidate, variable) => { + expect(isSourceOfferedByVariable(candidate, variable)).toBe(false); + }); +}); + +describe('getObjectName', () => { + const objects = [{ id: 'abc', name: 'Logs' }]; + + it('names an object, keeps a reference, and is undefined for a missing ID', () => { + expect(getObjectName('abc', objects)).toBe('Logs'); + expect(getObjectName('$logs', objects)).toBe('$logs'); + expect(getObjectName('nope', objects)).toBeUndefined(); + }); +}); + +describe('isReferenceVariableFilterType', () => { + it.each([ + ['DATA_SOURCE', true], + ['CONNECTION', true], + ['STATIC_LIST', false], + [undefined, false], + ] as const)('%p is %p', (type, expected) => { + expect(isReferenceVariableFilterType(type)).toBe(expected); + }); +}); diff --git a/packages/common-utils/src/core/utils.ts b/packages/common-utils/src/core/utils.ts index b3e4546e3..c0b6ea41d 100644 --- a/packages/common-utils/src/core/utils.ts +++ b/packages/common-utils/src/core/utils.ts @@ -8,6 +8,8 @@ export { default as objectHash } from 'object-hash'; import { isAdhocFilter, + isConnectionFilter, + isDataSourceFilter, isQueryExpressionFilter, isSqlQueryFilter, isStaticListFilter, @@ -27,6 +29,11 @@ import { replaceMacros, } from '@/macros'; import { QUERY_PARAMS, RawSqlQueryParam } from '@/rawSqlParams'; +import { + configUsesReferenceVariable, + getObjectName, + getReferenceVariableFilters, +} from '@/referenceVariables'; import { BuilderChartConfig, BuilderChartConfigWithDateRange, @@ -915,51 +922,44 @@ export function convertToDashboardTemplate( }; }; + // A variable reference is portable as written; '' marks a missing object + const getSourceName = (id: string) => getObjectName(id, sources) ?? ''; + const getConnectionName = (id: string) => + getObjectName(id, connections) ?? ''; + const convertToTileTemplate = ( input: Dashboard['tiles'][0], - sources: TSource[], - connections: Connection[], ): TileTemplate => { const tile = TileTemplateSchema.strip().parse(structuredClone(input)); // Extract name from source/connection or default to '' if not found const tileConfig = tile.config; if (isBuilderSavedChartConfig(tileConfig)) { - tileConfig.source = ( - sources.find(source => source.id === tileConfig.source) ?? { name: '' } - ).name; + tileConfig.source = getSourceName(tileConfig.source); } else if (isRawSqlSavedChartConfig(tileConfig)) { - tileConfig.connection = ( - connections.find(conn => conn.id === tileConfig.connection) ?? { - name: '', - } - ).name; + tileConfig.connection = getConnectionName(tileConfig.connection); if (tileConfig.source) { - tileConfig.source = - sources.find(source => source.id === tileConfig.source)?.name ?? ''; + tileConfig.source = getSourceName(tileConfig.source); } } convertOnClickTargetIdToName(tileConfig); return tile; }; - const convertToFilterTemplate = ( - input: DashboardFilter, - sources: TSource[], - connections: Connection[], - ): DashboardFilter => { + const convertToFilterTemplate = (input: DashboardFilter): DashboardFilter => { const filter = DashboardFilterSchema.parse(structuredClone(input)); - // A static filter references nothing in the workspace - if (isStaticListFilter(filter)) return filter; - + // These reference nothing in the workspace + if (isStaticListFilter(filter) || isConnectionFilter(filter)) return filter; if (isSqlQueryFilter(filter)) { - filter.connection = - connections.find(conn => conn.id === filter.connection)?.name ?? ''; + filter.connection = getConnectionName(filter.connection); + return filter; + } + if (isDataSourceFilter(filter)) { + if (filter.connection) { + filter.connection = getConnectionName(filter.connection) || undefined; + } return filter; } - - const getSourceName = (id: string) => - sources.find(source => source.id === id)?.name ?? ''; if (isAdhocFilter(filter)) { filter.sources = filter.sources.map(getSourceName); @@ -977,15 +977,13 @@ export function convertToDashboardTemplate( }; for (const tile of input.tiles) { - output.tiles.push(convertToTileTemplate(tile, sources, connections)); + output.tiles.push(convertToTileTemplate(tile)); } if (input.filters) { output.filters = []; for (const filter of input.filters ?? []) { - output.filters.push( - convertToFilterTemplate(filter, sources, connections), - ); + output.filters.push(convertToFilterTemplate(filter)); } } @@ -998,8 +996,16 @@ export function convertToDashboardTemplate( output.savedQueryLanguage = input.savedQueryLanguage ?? null; } - if (input.savedFilterValues && input.savedFilterValues.length > 0) { - output.savedFilterValues = structuredClone(input.savedFilterValues); + // A reference variable's saved value is an ID from this workspace + const referenceVariableNames = new Set( + getReferenceVariableFilters(input.filters).map(({ name }) => name), + ); + const savedFilterValues = (input.savedFilterValues ?? []).filter( + value => + !(value.type === 'variable' && referenceVariableNames.has(value.name)), + ); + if (savedFilterValues.length > 0) { + output.savedFilterValues = structuredClone(savedFilterValues); } return output; @@ -1738,6 +1744,8 @@ export function displayTypeSupportsPromQLAlerts( export function savedChartConfigSupportsAlerts( config: SavedChartConfig, ): boolean { + // An alert must query one fixed source, not the viewer's current selection. + if (configUsesReferenceVariable(config)) return false; if (isPromqlSavedChartConfig(config)) { return displayTypeSupportsPromQLAlerts(config.displayType); } diff --git a/packages/common-utils/src/dashboardFilterValues.ts b/packages/common-utils/src/dashboardFilterValues.ts index 2ea039b02..a0e7922d1 100644 --- a/packages/common-utils/src/dashboardFilterValues.ts +++ b/packages/common-utils/src/dashboardFilterValues.ts @@ -14,6 +14,7 @@ import { } from '@/filters'; import { isPromqlSavedChartConfig, isRawSqlSavedChartConfig } from '@/guards'; import { hasMacro, isMissingFiltersMacro } from '@/macros'; +import { getReferenceVariableFilters } from '@/referenceVariables'; import { AdhocDashboardFilter, AdhocFilterValue, @@ -269,6 +270,8 @@ export function getAdhocFilterTargetsForTile( * 2. The tile references the filter's variable. * 3. The filter applies via broadcast to the tile's source, and the tile * applies broadcast conditions (`consumesBroadcastFilters`). + * 4. The filter is a data source or connection variable the tile's stored + * config references. */ export function getBlockingRequiredFilters( unsatisfiedRequiredFilters: DashboardFilter[], @@ -277,8 +280,16 @@ export function getBlockingRequiredFilters( referencedVariableNames?: readonly string[]; /** Whether the tile's query applies broadcast filter conditions. */ consumesBroadcastFilters: boolean; + /** The reference variables the tile's stored config references. */ + referenceVariableNames?: readonly string[]; }, ): DashboardFilter[] { + const referenceVariableNames = new Set(tile.referenceVariableNames ?? []); + const referenceVariableFilterIds = new Set( + getReferenceVariableFilters(unsatisfiedRequiredFilters) + .filter(({ name }) => referenceVariableNames.has(name)) + .map(({ filter }) => filter.id), + ); const referencedNamesSet = new Set(tile.referencedVariableNames ?? []); const referencedFilterIds = new Set( getDashboardVariableFilters(unsatisfiedRequiredFilters) @@ -290,6 +301,7 @@ export function getBlockingRequiredFilters( filter => isFilterGlobalRequirement(filter) || referencedFilterIds.has(filter.id) || + referenceVariableFilterIds.has(filter.id) || (tile.consumesBroadcastFilters && doesFilterApplyToSource(filter, tile.sourceId)), ); @@ -301,15 +313,19 @@ export function getBlockingRequiredFilterNames({ sourceId, unsatisfiedRequiredFilters, referencedVariables, + referenceVariableNames, }: { config: SavedChartConfig | ChartConfigWithOptDateRange; sourceId: string | undefined; unsatisfiedRequiredFilters: DashboardFilter[] | undefined; /** The tile's variables, already narrowed to the ones it references. */ referencedVariables: readonly { name: string }[] | undefined; + /** The reference variables the tile's stored config references. */ + referenceVariableNames?: readonly string[]; }): string[] { return getBlockingRequiredFilters(unsatisfiedRequiredFilters ?? [], { sourceId, + referenceVariableNames, referencedVariableNames: referencedVariables?.map( variable => variable.name, ), diff --git a/packages/common-utils/src/dashboardValidation.ts b/packages/common-utils/src/dashboardValidation.ts index 57faefa10..8dabf9be9 100644 --- a/packages/common-utils/src/dashboardValidation.ts +++ b/packages/common-utils/src/dashboardValidation.ts @@ -7,7 +7,22 @@ import { isFilterBroadcastEnabled, isFilterVariableEnabled, } from './filters'; -import { DashboardContainerSchema, DisplayType } from './types'; +import { + getConfigReferenceFields, + getConnectionVariableNames, + getFilterReferenceFields, + getReferenceFieldValues, + getSourceVariableFilters, + getStringField, + getUnacceptedSourceKinds, + parseVariableReference, + ReferenceField, +} from './referenceVariables'; +import { + DashboardContainerSchema, + DashboardFilter, + DisplayType, +} from './types'; // Inputs shared by the internal `DashboardSchema` refinement and the // external API body schema: the validation only depends on the @@ -395,3 +410,132 @@ export function validateChartConfigFormulas( }); } } + +/** The issues a refinement raises, run on its own outside a schema. */ +export function collectRefinementIssues( + refine: (ctx: z.RefinementCtx) => void, +): z.ZodIssue[] { + const result = z + .object({}) + .superRefine((_value, ctx) => refine(ctx)) + .safeParse({}); + return result.success ? [] : result.error.issues; +} + +/** + * Every reference variable reference must name a variable of the right type + * on the dashboard: `$name` in a source field names a data source variable + * offering only kinds the field accepts; `$name` in a connection field names a + * connection variable. The fields are those of `getConfigReferenceFields` and + * `getFilterReferenceFields`. + * + * Issues raised: + * - An unknown variable, or one of the wrong type (path `[i].config.` or `[i].[j]`). + * - A data source variable offering kinds the field does not accept (same paths). + */ +export function validateDashboardVariableReferences( + tiles: { config: object }[], + filters: DashboardFilter[], + ctx: z.RefinementCtx, + paths?: { + tilesPath?: (string | number)[]; + filtersPath?: (string | number)[]; + }, +): void { + const tilesPath = paths?.tilesPath ?? ['tiles']; + const filtersPath = paths?.filtersPath ?? ['filters']; + const kindsByName = new Map( + getSourceVariableFilters(filters).map(({ filter, name }) => [ + name, + filter.allowedSourceKinds, + ]), + ); + const connectionVariableNames = new Set(getConnectionVariableNames(filters)); + + const check = ( + value: string, + field: ReferenceField, + path: (string | number)[], + ) => { + const name = parseVariableReference(value); + if (name == null) return; + if (field.target === 'connection') { + if (connectionVariableNames.has(name)) return; + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `"${value}" does not name a connection variable on this dashboard`, + path, + }); + return; + } + const kinds = kindsByName.get(name); + if (kinds == null) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `"${value}" does not name a data source variable on this dashboard`, + path, + }); + return; + } + const unaccepted = getUnacceptedSourceKinds( + { allowedSourceKinds: kinds }, + field.acceptedKinds, + ); + if (unaccepted.length > 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Data source variable "${value}" offers ${unaccepted.join(', ')} sources, which cannot be used here`, + path, + }); + } + }; + + const checkFields = ( + object: object, + fields: ReferenceField[], + path: (string | number)[], + ) => { + for (const field of fields) { + const value = getStringField(object, field.key); + if (value != null) { + check(value, field, [...path, field.key]); + } else { + getReferenceFieldValues(object, field.key).forEach((entry, i) => + check(entry, field, [...path, field.key, i]), + ); + } + } + }; + + tiles.forEach((tile, tileIdx) => + checkFields(tile.config, getConfigReferenceFields(tile.config), [ + ...tilesPath, + tileIdx, + 'config', + ]), + ); + filters.forEach((filter, filterIdx) => + checkFields(filter, getFilterReferenceFields(filter), [ + ...filtersPath, + filterIdx, + ]), + ); +} + +/** + * `validateDashboardVariableReferences` outside a schema, for handlers that + * check a merged update against the stored remainder. Returns the issues as + * `path: message` joined by `; `, or null. + */ +export function getVariableReferenceError( + tiles: { config: object }[] = [], + filters: DashboardFilter[] = [], +): string | null { + const issues = collectRefinementIssues(ctx => + validateDashboardVariableReferences(tiles, filters, ctx), + ); + if (issues.length === 0) return null; + return issues + .map(issue => `${issue.path.join('.')}: ${issue.message}`) + .join('; '); +} diff --git a/packages/common-utils/src/filters.ts b/packages/common-utils/src/filters.ts index ed946aede..71c7513eb 100644 --- a/packages/common-utils/src/filters.ts +++ b/packages/common-utils/src/filters.ts @@ -998,6 +998,37 @@ export function isSqlQueryFilter( return filter.type === 'SQL_QUERY'; } +/** Type guard for DATA_SOURCE type filters. */ +export function isDataSourceFilter( + filter: T, +): filter is Extract { + return filter.type === 'DATA_SOURCE'; +} + +/** Type guard for CONNECTION type filters. */ +export function isConnectionFilter( + filter: T, +): filter is Extract { + return filter.type === 'CONNECTION'; +} + +/** + * Whether a filter type is a reference variable: a data source or connection + * variable, whose value is an ID that tiles and filters use in place of one. + */ +export function isReferenceVariableFilterType( + type: DashboardFilterKind | undefined, +): type is 'DATA_SOURCE' | 'CONNECTION' { + return type === 'DATA_SOURCE' || type === 'CONNECTION'; +} + +/** Type guard for reference variable filters; see `isReferenceVariableFilterType`. */ +export function isReferenceVariableFilter< + T extends { type: DashboardFilterKind }, +>(filter: T): filter is Extract { + return isReferenceVariableFilterType(filter.type); +} + /** * The sources an ADHOC filter broadcasts its conditions to: its * `appliesToSourceIds`, or its own `sources` when that is unset or empty. @@ -1317,17 +1348,33 @@ export type FilterForVariableDeclaration = { /** * The variable-enabled filters a dashboard declares, paired with the name each - * one answers to, in filter order. + * one answers to, in filter order. Data source variables are left out: their + * value selects a tile's source or connection and is never substituted into + * query text. */ export function getDashboardVariableFilters< T extends FilterForVariableDeclaration, >(filters: T[] | undefined): { filter: T; name: string }[] { + return pairWithVariableNames( + (filters ?? []).filter( + filter => + isFilterReferenceableVariable(filter) && + !isReferenceVariableFilterType(filter.type), + ), + ); +} + +/** + * Each filter paired with the name it answers to, skipping filters with no + * usable name. + */ +export function pairWithVariableNames< + T extends { name: string; variableName?: string }, +>(filters: T[]): { filter: T; name: string }[] { const results: { filter: T; name: string }[] = []; const takenNames = new Set(); - for (const filter of filters ?? []) { - if (!isFilterReferenceableVariable(filter)) continue; - + for (const filter of filters) { // There shouldn't be any duplicate names, but if there are then the first one wins. const name = getFilterVariableName(filter); if (!name || takenNames.has(name)) continue; diff --git a/packages/common-utils/src/referenceVariables.ts b/packages/common-utils/src/referenceVariables.ts new file mode 100644 index 000000000..e3f28e28a --- /dev/null +++ b/packages/common-utils/src/referenceVariables.ts @@ -0,0 +1,360 @@ +import { castArray, difference, isString, uniq } from 'lodash'; + +import { + getAdhocFilterSourceKinds, + isConnectionFilter, + isDataSourceFilter, + isReferenceVariableFilter, + pairWithVariableNames, + QUERY_EXPRESSION_FILTER_SOURCE_KINDS, +} from '@/filters'; +import { + ConnectionDashboardFilter, + DASHBOARD_VARIABLE_NAME_PATTERN_ANCHORED, + DashboardFilter, + DataSourceDashboardFilter, + NON_PROMQL_SOURCE_KINDS, + SourceKind, +} from '@/types'; + +/** + * The variable name a field references, if it holds a `$name` reference + * rather than an ID. The whole field is the token. + */ +export function parseVariableReference( + value: string | undefined | null, +): string | undefined { + if (!value?.startsWith('$')) return undefined; + const name = value.slice(1); + return DASHBOARD_VARIABLE_NAME_PATTERN_ANCHORED.test(name) ? name : undefined; +} + +export function toVariableReference(name: string): string { + return `$${name}`; +} + +/** A field's value when it is a string; undefined when absent or not a string. */ +export function getStringField( + object: object, + field: string, +): string | undefined { + const value: unknown = Reflect.get(object, field); + return typeof value === 'string' ? value : undefined; +} + +/** + * An object's name, or the reference itself for a variable reference. + * Undefined when no object has the ID. + */ +export function getObjectName( + id: string, + objects: readonly { id: string; name: string }[] | undefined, +): string | undefined { + if (parseVariableReference(id) != null) return id; + return objects?.find(object => object.id === id)?.name; +} + +/** Selected ID per reference variable name; undefined when unselected. */ +export type ReferenceVariableValues = ReadonlyMap; + +/** A data source variable's identity, as offered by source pickers. */ +export type SourceVariableDeclaration = { + name: string; + allowedSourceKinds: SourceKind[]; + /** A connection ID or connection variable reference the sources are limited to. */ + connection?: string; +}; + +/** + * The kinds a data source variable can resolve to that a field accepting + * `acceptedKinds` cannot use. + */ +export function getUnacceptedSourceKinds( + variable: Pick, + acceptedKinds: readonly SourceKind[], +): SourceKind[] { + return difference(variable.allowedSourceKinds, acceptedKinds); +} + +/** + * Whether a field accepting `acceptedKinds` can use a data source variable, + * whatever source it selects. + */ +export function isSourceVariableAccepted( + variable: Pick, + acceptedKinds: readonly SourceKind[], +): boolean { + return getUnacceptedSourceKinds(variable, acceptedKinds).length === 0; +} + +/** `isSourceVariableAccepted` for the variable that `source` references. */ +export function isSourceVariableReferenceAccepted( + source: string | undefined, + declarations: SourceVariableDeclaration[], + acceptedKinds: readonly SourceKind[], +): boolean { + const name = parseVariableReference(source); + const variable = declarations.find(v => v.name === name); + return variable != null && isSourceVariableAccepted(variable, acceptedKinds); +} + +/** + * Whether a data source variable can select `source`. `connection` is the + * variable's connection already resolved; a still unresolved reference + * matches no source. + */ +export function isSourceOfferedByVariable( + source: { kind: SourceKind; connection: string; disabled?: boolean }, + variable: Pick< + SourceVariableDeclaration, + 'allowedSourceKinds' | 'connection' + >, +): boolean { + return ( + !source.disabled && + variable.allowedSourceKinds.includes(source.kind) && + (!variable.connection || source.connection === variable.connection) + ); +} + +/** The data source variables a dashboard declares, in filter order. */ +export function getSourceVariableFilters( + filters: DashboardFilter[] | undefined, +): { filter: DataSourceDashboardFilter; name: string }[] { + return pairWithVariableNames((filters ?? []).filter(isDataSourceFilter)); +} + +/** The connection variables a dashboard declares, in filter order. */ +export function getConnectionVariableFilters( + filters: DashboardFilter[] | undefined, +): { filter: ConnectionDashboardFilter; name: string }[] { + return pairWithVariableNames((filters ?? []).filter(isConnectionFilter)); +} + +/** + * The data source and connection variables a dashboard declares, in filter + * order. Structural so the external API's filter shape fits too. + */ +export function getReferenceVariableFilters< + T extends { + type: DashboardFilter['type']; + name: string; + variableName?: string; + }, +>( + filters: T[] | undefined, +): { + filter: Extract; + name: string; +}[] { + return pairWithVariableNames( + (filters ?? []).filter(isReferenceVariableFilter), + ); +} + +export function getSourceVariableDeclarations( + filters: DashboardFilter[] | undefined, +): SourceVariableDeclaration[] { + return getSourceVariableFilters(filters).map(({ filter, name }) => ({ + name, + allowedSourceKinds: filter.allowedSourceKinds, + connection: filter.connection, + })); +} + +export function getConnectionVariableNames( + filters: DashboardFilter[] | undefined, +): string[] { + return getConnectionVariableFilters(filters).map(({ name }) => name); +} + +/** + * Resolve a field: an ID passes through, a reference resolves to the + * variable's selection (undefined while unselected). + */ +export function resolveVariableReference( + value: string | undefined, + values: ReferenceVariableValues | undefined, +): string | undefined { + const name = parseVariableReference(value); + if (name == null) return value; + return values?.get(name); +} + +/** + * A field that can hold a reference variable reference: what it may reference, + * and, for a source field, the source kinds it accepts. + */ +export type ReferenceField = + | { key: K; target: 'source'; acceptedKinds: readonly SourceKind[] } + | { key: K; target: 'connection' }; + +/** The chart config fields reference resolution reads, whatever the config type. */ +type ReferenceHoldingConfig = { + source?: string; + connection?: string; + configType?: string; +}; + +/** The fields of a chart config that can hold a reference. */ +export function getConfigReferenceFields( + config: ReferenceHoldingConfig, +): ReferenceField<'source' | 'connection'>[] { + const isPromql = config.configType === 'promql'; + return [ + { + key: 'source', + target: 'source', + acceptedKinds: isPromql ? [SourceKind.Promql] : NON_PROMQL_SOURCE_KINDS, + }, + { key: 'connection', target: 'connection' }, + ]; +} + +/** The fields of a filter that can hold a reference. */ +export function getFilterReferenceFields( + filter: DashboardFilter, +): ReferenceField< + 'source' | 'sources' | 'appliesToSourceIds' | 'connection' +>[] { + switch (filter.type) { + case 'QUERY_EXPRESSION': + return (['source', 'appliesToSourceIds'] as const).map(key => ({ + key, + target: 'source', + acceptedKinds: QUERY_EXPRESSION_FILTER_SOURCE_KINDS, + })); + case 'PROMETHEUS_LABEL': + return [ + { key: 'source', target: 'source', acceptedKinds: [SourceKind.Promql] }, + ]; + case 'ADHOC': + return (['sources', 'appliesToSourceIds'] as const).map(key => ({ + key, + target: 'source', + acceptedKinds: getAdhocFilterSourceKinds(filter.sourceType), + })); + case 'DATA_SOURCE': + return [{ key: 'connection', target: 'connection' }]; + case 'STATIC_LIST': + case 'CONNECTION': + return []; + default: + filter satisfies never; + return []; + } +} + +/** A field's references and IDs: one for a string field, each entry of a list. */ +export function getReferenceFieldValues(object: object, key: string): string[] { + return castArray(Reflect.get(object, key)).filter(isString); +} + +/** Each field's value with `resolve` applied, keeping strings and lists as such. */ +function mapReferenceFields( + object: T, + keys: readonly string[], + resolve: (value: string) => string, +): T { + let mapped = object; + for (const key of keys) { + const value: unknown = Reflect.get(object, key); + if (typeof value === 'string') { + const next = resolve(value); + if (next !== value) mapped = { ...mapped, [key]: next }; + } else if (Array.isArray(value)) { + const next = value.map(v => (typeof v === 'string' ? resolve(v) : v)); + if (next.some((v, i) => v !== value.at(i))) { + mapped = { ...mapped, [key]: next }; + } + } + } + return mapped; +} + +/** The variables named by references in the given fields, deduplicated. */ +function getReferencedVariables( + object: object, + keys: readonly string[], +): string[] { + return uniq( + keys + .flatMap(key => getReferenceFieldValues(object, key)) + .map(parseVariableReference) + .filter((name): name is string => name != null), + ); +} + +/** The reference variables a chart config's `source` and `connection` name. */ +export function getConfigReferenceVariables( + config: ReferenceHoldingConfig, +): string[] { + return getReferencedVariables( + config, + getConfigReferenceFields(config).map(({ key }) => key), + ); +} + +/** Whether a chart config's source or connection is a variable reference. */ +export function configUsesReferenceVariable( + config: ReferenceHoldingConfig, +): boolean { + return getConfigReferenceVariables(config).length > 0; +} + +/** + * Replace the references in a config's `source` and `connection` with the + * selected IDs. An unselected reference is left as written. Returns `config` + * itself when nothing changes. + */ +export function resolveConfigReferences( + config: T, + values: ReferenceVariableValues | undefined, +): T { + return mapReferenceFields( + config, + getConfigReferenceFields(config).map(({ key }) => key), + value => resolveVariableReference(value, values) || value, + ); +} + +/** + * The inverse of `resolveConfigReferences`: put the references `stored` holds + * back on a config derived from its resolved form. + */ +export function restoreConfigReferences( + config: T, + stored: ReferenceHoldingConfig, +): T { + let restored = config; + for (const { key } of getConfigReferenceFields(stored)) { + const value = getStringField(stored, key); + if (parseVariableReference(value) != null && key in config) { + restored = { ...restored, [key]: value }; + } + } + return restored; +} + +/** + * Replace the references in a filter's source and connection fields with the + * selected IDs. An unselected reference is left as written, so it matches no + * tile's source rather than widening a list to every tile, and is reported in + * `unresolvedVariables`. + */ +export function resolveFilterReferences( + filter: T, + values: ReferenceVariableValues | undefined, +): { filter: T; unresolvedVariables: string[] } { + const keys = getFilterReferenceFields(filter).map(({ key }) => key); + return { + filter: mapReferenceFields( + filter, + keys, + value => resolveVariableReference(value, values) || value, + ), + unresolvedVariables: getReferencedVariables(filter, keys).filter( + name => !values?.get(name), + ), + }; +} diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 67a347e59..c733b1bc7 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -2309,6 +2309,19 @@ export const DashboardContainerSchema = z.object({ export type DashboardContainer = z.infer; +export enum SourceKind { + Log = 'log', + Trace = 'trace', + Session = 'session', + Metric = 'metric', + Promql = 'promql', +} + +/** The source kinds builder and raw SQL charts can query. */ +export const NON_PROMQL_SOURCE_KINDS: readonly SourceKind[] = Object.values( + SourceKind, +).filter(kind => kind !== SourceKind.Promql); + /** Type of dashboard filter, determining how its dropdown values are populated. */ export const DashboardFilterType = z.enum([ 'QUERY_EXPRESSION', @@ -2316,6 +2329,8 @@ export const DashboardFilterType = z.enum([ 'PROMETHEUS_LABEL', 'ADHOC', 'SQL_QUERY', + 'DATA_SOURCE', + 'CONNECTION', ]); /** Allowed variable names for dashboard filters. Alphanumeric + underscore, must start with a letter. */ @@ -2474,12 +2489,51 @@ export const AdhocDashboardFilterSchema = dashboardFilterBaseSchema }) .strict(); +/** + * A variable whose single selected value is a source ID. Tiles and other + * filters reference it as `$variableName` in place of a source ID. Always + * single-select and required, since a tile cannot query without a source. + */ +export const DataSourceDashboardFilterSchema = dashboardFilterBaseSchema + .extend({ + type: z.literal(DashboardFilterType.enum.DATA_SOURCE), + /** The kinds of sources offered as values. */ + allowedSourceKinds: z.array(z.nativeEnum(SourceKind)).min(1), + /** + * Limits the offered sources to one connection: a connection ID, or a + * `$name` reference to a connection variable. + */ + connection: z.string().min(1).optional(), + minSelections: z.literal(1), + maxSelections: z.literal(1), + isBroadcastEnabled: z.literal(false), + isVariableEnabled: z.literal(true), + }) + .strict(); + +/** + * A variable whose single selected value is a connection ID. Raw SQL tiles + * and data source variables reference it as `$variableName` in place of a + * connection ID. + */ +export const ConnectionDashboardFilterSchema = dashboardFilterBaseSchema + .extend({ + type: z.literal(DashboardFilterType.enum.CONNECTION), + minSelections: z.literal(1), + maxSelections: z.literal(1), + isBroadcastEnabled: z.literal(false), + isVariableEnabled: z.literal(true), + }) + .strict(); + export const DashboardFilterSchema = z.discriminatedUnion('type', [ QueryExpressionDashboardFilterSchema, StaticListDashboardFilterSchema, PromqlLabelDashboardFilterSchema, AdhocDashboardFilterSchema, SqlQueryDashboardFilterSchema, + DataSourceDashboardFilterSchema, + ConnectionDashboardFilterSchema, ]); export type QueryExpressionDashboardFilter = z.infer< @@ -2495,6 +2549,12 @@ export type AdhocDashboardFilter = z.infer; export type SqlQueryDashboardFilter = z.infer< typeof SqlQueryDashboardFilterSchema >; +export type DataSourceDashboardFilter = z.infer< + typeof DataSourceDashboardFilterSchema +>; +export type ConnectionDashboardFilter = z.infer< + typeof ConnectionDashboardFilterSchema +>; export type DashboardFilter = z.infer; export enum PresetDashboard { @@ -2650,13 +2710,6 @@ export type Team = z.infer; // -------------------------- // TABLE SOURCES // -------------------------- -export enum SourceKind { - Log = 'log', - Trace = 'trace', - Session = 'session', - Metric = 'metric', - Promql = 'promql', -} // -------------------------- // TABLE SOURCE FORM VALIDATION From 078c4bee1ce5e10d117abcc12f12f09ad5850b3d Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Mon, 16 Nov 2026 14:17:50 -0500 Subject: [PATCH 02/18] fix: check only the variable references a dashboard update adds, and accept variables with some usable kinds --- .../api/__tests__/dashboard.int.test.ts | 66 ++++++- packages/api/src/routers/api/dashboards.ts | 3 + .../src/__tests__/referenceVariables.test.ts | 109 +++++++++-- .../common-utils/src/dashboardValidation.ts | 180 ++++++++++++------ .../common-utils/src/referenceVariables.ts | 21 +- 5 files changed, 279 insertions(+), 100 deletions(-) diff --git a/packages/api/src/routers/api/__tests__/dashboard.int.test.ts b/packages/api/src/routers/api/__tests__/dashboard.int.test.ts index de6f975a8..4a710a555 100644 --- a/packages/api/src/routers/api/__tests__/dashboard.int.test.ts +++ b/packages/api/src/routers/api/__tests__/dashboard.int.test.ts @@ -1566,8 +1566,23 @@ describe('dashboard router', () => { ); }); - it('rejects a variable offering kinds the reference cannot use', async () => { + it('rejects a variable offering no kind the reference can use', async () => { const res = await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + tiles: [makeTile({ sourceId: '$logs' })], + filters: [ + { ...dataSourceFilter, allowedSourceKinds: [SourceKind.Promql] }, + ], + }) + .expect(400); + + expect(res.body.message).toMatch(/offers no kind of source/); + }); + + it('accepts a variable offering some kinds the reference cannot use', async () => { + await agent .post('/dashboards') .send({ ...MOCK_DASHBOARD, @@ -1579,9 +1594,7 @@ describe('dashboard router', () => { }, ], }) - .expect(400); - - expect(JSON.stringify(res.body)).toMatch(/promql/); + .expect(200); }); it('validates a PATCH of tiles against the stored filters', async () => { @@ -1594,10 +1607,51 @@ describe('dashboard router', () => { .patch(`/dashboards/${created.body.id}`) .send({ tiles: [makeTile({ sourceId: '$logs' })] }) .expect(200); - await agent + const res = await agent .patch(`/dashboards/${created.body.id}`) - .send({ filters: [] }) + .send({ + tiles: [ + makeTile({ sourceId: '$logs' }), + makeTile({ sourceId: '$nope' }), + ], + }) .expect(400); + expect(res.body.message).toMatch(/does not name a data source variable/); + }); + + // Only references an update adds are checked, so editing a variable + // doesn't fail on the tiles and filters that already use it. + it('saves a filter edit that leaves stored references unsatisfied', async () => { + const adhocFilter = { + id: new Types.ObjectId().toString(), + type: 'ADHOC' as const, + name: 'Conditions', + sourceType: 'sql' as const, + sources: ['$logs'], + isVariableEnabled: true, + }; + const created = await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + tiles: [makeTile({ sourceId: '$logs' })], + filters: [dataSourceFilter, adhocFilter], + }) + .expect(200); + + await agent + .patch(`/dashboards/${created.body.id}`) + .send({ + filters: [ + { ...dataSourceFilter, allowedSourceKinds: [SourceKind.Promql] }, + adhocFilter, + ], + }) + .expect(200); + await agent + .patch(`/dashboards/${created.body.id}`) + .send({ filters: [adhocFilter] }) + .expect(200); }); it('drops an alert on a tile that uses the variable', async () => { diff --git a/packages/api/src/routers/api/dashboards.ts b/packages/api/src/routers/api/dashboards.ts index 35c2781de..e17520007 100644 --- a/packages/api/src/routers/api/dashboards.ts +++ b/packages/api/src/routers/api/dashboards.ts @@ -234,9 +234,12 @@ router.patch( const updates = DashboardWithoutIdSchema.partial().parse(req.body); const filterError = + // Only references this update adds are checked, so a filter edit + // can't fail on tiles and filters it didn't touch getVariableReferenceError( req.body.tiles ?? dashboard.tiles, req.body.filters ?? dashboard.filters, + dashboard, ) ?? (await validateFilterReferences( teamId, diff --git a/packages/common-utils/src/__tests__/referenceVariables.test.ts b/packages/common-utils/src/__tests__/referenceVariables.test.ts index 278dca4a9..0fd4cfb94 100644 --- a/packages/common-utils/src/__tests__/referenceVariables.test.ts +++ b/packages/common-utils/src/__tests__/referenceVariables.test.ts @@ -222,6 +222,7 @@ describe('isSourceVariableReferenceAccepted', () => { const declarations = [ { name: 'logs', allowedSourceKinds: [SourceKind.Log] }, { name: 'any', allowedSourceKinds: [SourceKind.Log, SourceKind.Promql] }, + { name: 'prom', allowedSourceKinds: [SourceKind.Promql] }, ]; it('accepts a variable whose every kind the field accepts', () => { @@ -233,9 +234,17 @@ describe('isSourceVariableReferenceAccepted', () => { ).toBe(true); }); - it('rejects a variable offering a kind the field does not accept', () => { + it('accepts a variable offering some kinds the field does not accept', () => { expect( isSourceVariableReferenceAccepted('$any', declarations, [SourceKind.Log]), + ).toBe(true); + }); + + it('rejects a variable offering no kind the field accepts', () => { + expect( + isSourceVariableReferenceAccepted('$prom', declarations, [ + SourceKind.Log, + ]), ).toBe(false); }); @@ -452,9 +461,12 @@ describe('validateDashboardVariableReferences', () => { expect(issues[0].path).toEqual(['tiles', 0, 'config', 'source']); }); - it.each<[string, { config: SavedChartConfig }[], DashboardFilter[]]>([ + it.each< + [string, SourceKind[], { config: SavedChartConfig }[], DashboardFilter[]] + >([ [ 'a PromQL tile', + [SourceKind.Log, SourceKind.Trace], [ { config: { @@ -469,6 +481,7 @@ describe('validateDashboardVariableReferences', () => { ], [ 'a PromQL label filter', + [SourceKind.Log, SourceKind.Trace], [], [ { @@ -484,6 +497,7 @@ describe('validateDashboardVariableReferences', () => { ], [ 'an ad hoc filter of another source type', + [SourceKind.Log, SourceKind.Trace], [], [ { @@ -498,6 +512,7 @@ describe('validateDashboardVariableReferences', () => { ], [ "a queried filter's applies-to list", + [SourceKind.Promql], [], [ queriedFilter({ @@ -507,31 +522,87 @@ describe('validateDashboardVariableReferences', () => { ], ], ])( - 'rejects a logs-or-PromQL variable used by %s', - (_label, tiles, filters) => { + 'rejects a variable offering no kind usable by %s', + (_label, allowedSourceKinds, tiles, filters) => { const issues = collectIssues(tiles, [ - dataSourceFilter({ - allowedSourceKinds: [SourceKind.Log, SourceKind.Promql], - }), + dataSourceFilter({ allowedSourceKinds }), ...filters, ]); expect(issues.length).toBeGreaterThan(0); }, ); - it('rejects a variable offering kinds the field does not accept', () => { - const issues = collectIssues( - [], - [ - dataSourceFilter({ - allowedSourceKinds: [SourceKind.Log, SourceKind.Promql], + it('accepts a variable offering some kinds the field does not accept', () => { + expect( + collectIssues( + [], + [ + dataSourceFilter({ + allowedSourceKinds: [SourceKind.Log, SourceKind.Promql], + }), + queriedFilter(), + ], + ), + ).toEqual([]); + }); + + describe('with the stored dashboard', () => { + const storedTiles = [{ id: 't1', config: lineConfig('$logs') }]; + const storedFilters = [dataSourceFilter(), queriedFilter()]; + const promqlVariable = dataSourceFilter({ + allowedSourceKinds: [SourceKind.Promql], + }); + + it('skips references the update leaves as they were', () => { + expect( + getVariableReferenceError( + storedTiles, + [promqlVariable, queriedFilter()], + { + tiles: storedTiles, + filters: storedFilters, + }, + ), + ).toBeNull(); + expect( + getVariableReferenceError(storedTiles, [queriedFilter()], { + tiles: storedTiles, + filters: storedFilters, }), - queriedFilter(), - ], - ); - expect(issues).toHaveLength(1); - expect(issues[0].path).toEqual(['filters', 1, 'source']); - expect(issues[0].message).toMatch(/promql/); + ).toBeNull(); + }); + + it('checks references the update adds', () => { + const issues = collectRefinementIssues(ctx => + validateDashboardVariableReferences( + [...storedTiles, { id: 't2', config: lineConfig('$logs') }], + [promqlVariable, queriedFilter()], + ctx, + { existing: { tiles: storedTiles, filters: storedFilters } }, + ), + ); + expect(issues.map(issue => issue.path)).toEqual([ + ['tiles', 1, 'config', 'source'], + ]); + }); + + it('checks a reference again when its field accepts other kinds', () => { + const promqlTile = { + id: 't1', + config: { + configType: 'promql', + source: '$logs', + displayType: DisplayType.Line, + promqlExpression: [{ expression: 'up' }], + } satisfies SavedChartConfig, + }; + expect( + getVariableReferenceError([promqlTile], storedFilters, { + tiles: storedTiles, + filters: storedFilters, + }), + ).toMatch(/tiles\.0\.config\.source/); + }); }); }); diff --git a/packages/common-utils/src/dashboardValidation.ts b/packages/common-utils/src/dashboardValidation.ts index 8dabf9be9..20a4ac309 100644 --- a/packages/common-utils/src/dashboardValidation.ts +++ b/packages/common-utils/src/dashboardValidation.ts @@ -14,7 +14,7 @@ import { getReferenceFieldValues, getSourceVariableFilters, getStringField, - getUnacceptedSourceKinds, + isSourceVariableAccepted, parseVariableReference, ReferenceField, } from './referenceVariables'; @@ -422,28 +422,99 @@ export function collectRefinementIssues( return result.success ? [] : result.error.issues; } +type ReferenceTile = { id?: string; config: object }; + +type ReferenceOccurrence = { + value: string; + field: ReferenceField; + path: (string | number)[]; + /** Same for the same reference in the same field of the same tile or filter. */ + key: string; +}; + +function getReferenceOccurrences( + tiles: ReferenceTile[], + filters: DashboardFilter[], + tilesPath: (string | number)[], + filtersPath: (string | number)[], +): ReferenceOccurrence[] { + const occurrences: ReferenceOccurrence[] = []; + const add = ( + owner: string, + object: object, + fields: ReferenceField[], + path: (string | number)[], + ) => { + for (const field of fields) { + // A field that now accepts other kinds counts as a new reference + const fieldKey = + field.target === 'source' + ? `${field.key}:${field.acceptedKinds.join(',')}` + : field.key; + const value = getStringField(object, field.key); + const entries = + value != null + ? [{ value, path: [...path, field.key] }] + : getReferenceFieldValues(object, field.key).map((entry, i) => ({ + value: entry, + path: [...path, field.key, i], + })); + for (const entry of entries) { + if (parseVariableReference(entry.value) == null) continue; + occurrences.push({ + ...entry, + field, + key: JSON.stringify([owner, fieldKey, entry.value]), + }); + } + } + }; + + tiles.forEach((tile, tileIdx) => + add( + `tile:${tile.id ?? tileIdx}`, + tile.config, + getConfigReferenceFields(tile.config), + [...tilesPath, tileIdx, 'config'], + ), + ); + filters.forEach((filter, filterIdx) => + add(`filter:${filter.id}`, filter, getFilterReferenceFields(filter), [ + ...filtersPath, + filterIdx, + ]), + ); + return occurrences; +} + /** * Every reference variable reference must name a variable of the right type * on the dashboard: `$name` in a source field names a data source variable - * offering only kinds the field accepts; `$name` in a connection field names a - * connection variable. The fields are those of `getConfigReferenceFields` and - * `getFilterReferenceFields`. + * offering at least one kind the field accepts; `$name` in a connection field + * names a connection variable. The fields are those of + * `getConfigReferenceFields` and `getFilterReferenceFields`. + * + * With `existing`, the stored dashboard an update applies to, only references + * the update adds are checked. A filter edit can then leave stored tiles and + * filters with references it no longer satisfies, which the dashboard shows + * as errors when it's viewed. * * Issues raised: * - An unknown variable, or one of the wrong type (path `[i].config.` or `[i].[j]`). - * - A data source variable offering kinds the field does not accept (same paths). + * - A data source variable offering no kind the field accepts (same paths). */ export function validateDashboardVariableReferences( - tiles: { config: object }[], + tiles: ReferenceTile[], filters: DashboardFilter[], ctx: z.RefinementCtx, - paths?: { + options?: { tilesPath?: (string | number)[]; filtersPath?: (string | number)[]; + existing?: { tiles: ReferenceTile[]; filters: DashboardFilter[] }; }, ): void { - const tilesPath = paths?.tilesPath ?? ['tiles']; - const filtersPath = paths?.filtersPath ?? ['filters']; + const tilesPath = options?.tilesPath ?? ['tiles']; + const filtersPath = options?.filtersPath ?? ['filters']; const kindsByName = new Map( getSourceVariableFilters(filters).map(({ filter, name }) => [ name, @@ -451,22 +522,35 @@ export function validateDashboardVariableReferences( ]), ); const connectionVariableNames = new Set(getConnectionVariableNames(filters)); + const existingKeys = new Set( + options?.existing + ? getReferenceOccurrences( + options.existing.tiles, + options.existing.filters, + tilesPath, + filtersPath, + ).map(({ key }) => key) + : [], + ); - const check = ( - value: string, - field: ReferenceField, - path: (string | number)[], - ) => { + const occurrences = getReferenceOccurrences( + tiles, + filters, + tilesPath, + filtersPath, + ); + for (const { value, field, path, key } of occurrences) { + if (existingKeys.has(key)) continue; const name = parseVariableReference(value); - if (name == null) return; + if (name == null) continue; if (field.target === 'connection') { - if (connectionVariableNames.has(name)) return; + if (connectionVariableNames.has(name)) continue; ctx.addIssue({ code: z.ZodIssueCode.custom, message: `"${value}" does not name a connection variable on this dashboard`, path, }); - return; + continue; } const kinds = kindsByName.get(name); if (kinds == null) { @@ -475,64 +559,40 @@ export function validateDashboardVariableReferences( message: `"${value}" does not name a data source variable on this dashboard`, path, }); - return; + continue; } - const unaccepted = getUnacceptedSourceKinds( - { allowedSourceKinds: kinds }, - field.acceptedKinds, - ); - if (unaccepted.length > 0) { + if ( + !isSourceVariableAccepted( + { allowedSourceKinds: kinds }, + field.acceptedKinds, + ) + ) { ctx.addIssue({ code: z.ZodIssueCode.custom, - message: `Data source variable "${value}" offers ${unaccepted.join(', ')} sources, which cannot be used here`, + message: `Data source variable "${value}" offers no kind of source that can be used here`, path, }); } - }; - - const checkFields = ( - object: object, - fields: ReferenceField[], - path: (string | number)[], - ) => { - for (const field of fields) { - const value = getStringField(object, field.key); - if (value != null) { - check(value, field, [...path, field.key]); - } else { - getReferenceFieldValues(object, field.key).forEach((entry, i) => - check(entry, field, [...path, field.key, i]), - ); - } - } - }; - - tiles.forEach((tile, tileIdx) => - checkFields(tile.config, getConfigReferenceFields(tile.config), [ - ...tilesPath, - tileIdx, - 'config', - ]), - ); - filters.forEach((filter, filterIdx) => - checkFields(filter, getFilterReferenceFields(filter), [ - ...filtersPath, - filterIdx, - ]), - ); + } } /** * `validateDashboardVariableReferences` outside a schema, for handlers that - * check a merged update against the stored remainder. Returns the issues as - * `path: message` joined by `; `, or null. + * check a merged update against the stored dashboard, passed as `existing`. + * Returns the issues as `path: message` joined by `; `, or null. */ export function getVariableReferenceError( - tiles: { config: object }[] = [], + tiles: ReferenceTile[] = [], filters: DashboardFilter[] = [], + existing?: { tiles?: ReferenceTile[]; filters?: DashboardFilter[] }, ): string | null { const issues = collectRefinementIssues(ctx => - validateDashboardVariableReferences(tiles, filters, ctx), + validateDashboardVariableReferences(tiles, filters, ctx, { + existing: existing && { + tiles: existing.tiles ?? [], + filters: existing.filters ?? [], + }, + }), ); if (issues.length === 0) return null; return issues diff --git a/packages/common-utils/src/referenceVariables.ts b/packages/common-utils/src/referenceVariables.ts index e3f28e28a..622442a15 100644 --- a/packages/common-utils/src/referenceVariables.ts +++ b/packages/common-utils/src/referenceVariables.ts @@ -1,4 +1,4 @@ -import { castArray, difference, isString, uniq } from 'lodash'; +import { castArray, isString, uniq } from 'lodash'; import { getAdhocFilterSourceKinds, @@ -66,25 +66,15 @@ export type SourceVariableDeclaration = { }; /** - * The kinds a data source variable can resolve to that a field accepting - * `acceptedKinds` cannot use. - */ -export function getUnacceptedSourceKinds( - variable: Pick, - acceptedKinds: readonly SourceKind[], -): SourceKind[] { - return difference(variable.allowedSourceKinds, acceptedKinds); -} - -/** - * Whether a field accepting `acceptedKinds` can use a data source variable, - * whatever source it selects. + * Whether a field accepting `acceptedKinds` can use a data source variable: at + * least one kind it can resolve to fits. A selection of another kind is + * handled where the reference is resolved. */ export function isSourceVariableAccepted( variable: Pick, acceptedKinds: readonly SourceKind[], ): boolean { - return getUnacceptedSourceKinds(variable, acceptedKinds).length === 0; + return variable.allowedSourceKinds.some(kind => acceptedKinds.includes(kind)); } /** `isSourceVariableAccepted` for the variable that `source` references. */ @@ -237,6 +227,7 @@ export function getFilterReferenceFields( case 'DATA_SOURCE': return [{ key: 'connection', target: 'connection' }]; case 'STATIC_LIST': + case 'SQL_QUERY': case 'CONNECTION': return []; default: From d638aa19f89701efa8d17eabeb1487e4600629bc Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Tue, 17 Nov 2026 14:21:21 -0500 Subject: [PATCH 03/18] test: add isBroadcastEnabled to ad hoc filter fixtures --- packages/api/src/routers/api/__tests__/dashboard.int.test.ts | 1 + packages/api/src/routers/external-api/v2/utils/dashboards.ts | 2 +- packages/common-utils/src/__tests__/referenceVariables.test.ts | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/api/src/routers/api/__tests__/dashboard.int.test.ts b/packages/api/src/routers/api/__tests__/dashboard.int.test.ts index 4a710a555..e1b5e4cdb 100644 --- a/packages/api/src/routers/api/__tests__/dashboard.int.test.ts +++ b/packages/api/src/routers/api/__tests__/dashboard.int.test.ts @@ -1628,6 +1628,7 @@ describe('dashboard router', () => { name: 'Conditions', sourceType: 'sql' as const, sources: ['$logs'], + isBroadcastEnabled: true, isVariableEnabled: true, }; const created = await agent diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index 067980de9..e48e86cff 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -58,9 +58,9 @@ import { getConnectionsByTeam } from '@/controllers/connection'; import { getSources } from '@/controllers/sources'; import Dashboard, { DashboardDocument } from '@/models/dashboard'; import { + getExternalCompatibleFilters, translateExternalChartToTileConfig, translateExternalFilterToFilter, - getExternalCompatibleFilters, translateFilterToExternalFilter, } from '@/utils/externalApi'; import { diff --git a/packages/common-utils/src/__tests__/referenceVariables.test.ts b/packages/common-utils/src/__tests__/referenceVariables.test.ts index 0fd4cfb94..3b5384c4f 100644 --- a/packages/common-utils/src/__tests__/referenceVariables.test.ts +++ b/packages/common-utils/src/__tests__/referenceVariables.test.ts @@ -278,6 +278,7 @@ describe('resolveFilterReferences', () => { name: 'Conditions', sourceType: 'sql', sources: ['$logs'], + isBroadcastEnabled: true, isVariableEnabled: true, }; const { filter, unresolvedVariables } = resolveFilterReferences( @@ -506,6 +507,7 @@ describe('validateDashboardVariableReferences', () => { name: 'Conditions', sourceType: 'promql', sources: ['$logs'], + isBroadcastEnabled: true, isVariableEnabled: true, }, ], From d569edc3b21e093224fc1d4ed2a788ffd423eea2 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Mon, 16 Nov 2026 12:17:34 -0500 Subject: [PATCH 04/18] feat: support data source and connection variables in the external API and MCP --- .changeset/source-vars-2.md | 5 + packages/api/openapi.json | 140 +++++++-- .../dashboards/patchDashboard.int.test.ts | 72 +++++ .../dashboards/queryTile.int.test.ts | 82 +++++ .../dashboards/queryTiles.int.test.ts | 55 ++++ .../dashboards/queryTilesHandler.test.ts | 9 + .../dashboards/saveDashboard.int.test.ts | 103 +++++++ .../__tests__/dashboards/validation.test.ts | 29 ++ .../__tests__/dashboards/variables.test.ts | 148 ++++++++- .../api/src/mcp/prompts/dashboards/content.ts | 2 + .../mcp/tools/dashboards/patchDashboard.ts | 1 + .../api/src/mcp/tools/dashboards/queryTile.ts | 23 +- .../src/mcp/tools/dashboards/queryTiles.ts | 31 +- .../api/src/mcp/tools/dashboards/schemas.ts | 140 +++++++-- .../src/mcp/tools/dashboards/validation.ts | 16 +- .../api/src/mcp/tools/dashboards/variables.ts | 159 +++++++++- .../__tests__/dashboards.int.test.ts | 285 ++++++++++++++++++ .../src/routers/external-api/v2/dashboards.ts | 128 ++++++-- .../external-api/v2/utils/dashboards.ts | 81 ++++- packages/api/src/utils/externalApi.ts | 37 +-- packages/api/src/utils/zod.ts | 70 ++++- 21 files changed, 1489 insertions(+), 127 deletions(-) create mode 100644 .changeset/source-vars-2.md diff --git a/.changeset/source-vars-2.md b/.changeset/source-vars-2.md new file mode 100644 index 000000000..32fd79c38 --- /dev/null +++ b/.changeset/source-vars-2.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/api': minor +--- + +feat: support data source and connection variables in the external API and MCP diff --git a/packages/api/openapi.json b/packages/api/openapi.json index 65ea9dff2..9f1e13056 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -1883,7 +1883,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -1980,7 +1980,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2057,7 +2057,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2144,7 +2144,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2227,7 +2227,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2288,7 +2288,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2343,7 +2343,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2437,7 +2437,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2495,7 +2495,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2559,7 +2559,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to query.", + "description": "ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2599,7 +2599,7 @@ }, "sourceId": { "type": "string", - "description": "ID of the data source to mine patterns from.", + "description": "ID of the data source to mine patterns from, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard.", "example": "65f5e4a3b9e77c001a111111" }, "select": { @@ -2664,7 +2664,7 @@ }, "connectionId": { "type": "string", - "description": "ID of the ClickHouse connection to execute the query against.", + "description": "ID of the ClickHouse connection to execute the query against, or\n`$connectionVariable` of a CONNECTION filter on the dashboard.\n", "example": "65f5e4a3b9e77c001a567890" }, "sqlTemplate": { @@ -2675,7 +2675,7 @@ }, "sourceId": { "type": "string", - "description": "Optional ID of the data source associated with this Raw SQL chart. Used for applying dashboard filters.", + "description": "Optional ID of the data source associated with this Raw SQL chart, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. Used for applying dashboard filters.", "example": "65f5e4a3b9e77c001a567890" }, "numberFormat": { @@ -4031,6 +4031,12 @@ }, { "$ref": "#/components/schemas/AdhocFilterInput" + }, + { + "$ref": "#/components/schemas/DataSourceFilterInput" + }, + { + "$ref": "#/components/schemas/ConnectionFilterInput" } ], "discriminator": { @@ -4040,7 +4046,107 @@ "STATIC_LIST": "#/components/schemas/StaticListFilterInput", "PROMETHEUS_LABEL": "#/components/schemas/PrometheusLabelFilterInput", "SQL_QUERY": "#/components/schemas/SqlQueryFilterInput", - "ADHOC": "#/components/schemas/AdhocFilterInput" + "ADHOC": "#/components/schemas/AdhocFilterInput", + "DATA_SOURCE": "#/components/schemas/DataSourceFilterInput", + "CONNECTION": "#/components/schemas/ConnectionFilterInput" + } + } + }, + "DataSourceFilterInput": { + "type": "object", + "description": "A variable whose single, required value is a source. Tiles reference it\nas `$variableName` in their sourceId, and other filters in sourceId or\nappliesToSourceIds, so viewers can switch the source those read. Tiles\nthat use it do not load until a source is selected, and cannot have\nalerts.\n", + "required": [ + "type", + "name", + "allowedSourceKinds" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "DATA_SOURCE" + ], + "description": "Filter type discriminator. Must be \"DATA_SOURCE\".", + "example": "DATA_SOURCE" + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name of the variable's dropdown", + "example": "Logs source" + }, + "variableName": { + "type": "string", + "maxLength": 64, + "pattern": "^[a-zA-Z][a-zA-Z0-9_]*$", + "description": "Token tiles and filters reference the selected source by, as `$variableName`.", + "example": "logs" + }, + "isGlobalRequirement": { + "type": "boolean", + "default": false, + "description": "Blocks every tile on the dashboard until a value is selected. False\n(the default) blocks only the tiles that reference the variable.\n", + "example": true + }, + "allowedSourceKinds": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "log", + "trace", + "session", + "metric", + "promql" + ] + }, + "description": "Kinds of sources offered. A field can only reference the variable\nif it accepts every listed kind.\n", + "example": [ + "log" + ] + }, + "connectionId": { + "type": "string", + "description": "Optional connection the offered sources are limited to: a\nconnection ID, or `$name` of a CONNECTION filter on the dashboard.\n", + "example": "$conn" + } + } + }, + "ConnectionFilterInput": { + "type": "object", + "description": "A variable whose single, required value is a connection. Raw SQL tiles\nreference it as `$variableName` in their connectionId, and data source\nvariables in theirs. Tiles that use it do not load until a connection is\nselected, and cannot have alerts.\n", + "required": [ + "type", + "name" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "CONNECTION" + ], + "description": "Filter type discriminator. Must be \"CONNECTION\".", + "example": "CONNECTION" + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name of the variable's dropdown", + "example": "Connection" + }, + "variableName": { + "type": "string", + "maxLength": 64, + "pattern": "^[a-zA-Z][a-zA-Z0-9_]*$", + "description": "Token raw SQL tiles reference the selected connection by, as `$variableName`.", + "example": "conn" + }, + "isGlobalRequirement": { + "type": "boolean", + "default": false, + "description": "Blocks every tile on the dashboard until a value is selected. False\n(the default) blocks only the tiles that reference the variable.\n", + "example": true } } }, @@ -4076,7 +4182,7 @@ }, "sourceId": { "type": "string", - "description": "Source ID this dashboard filter key applies to", + "description": "Source ID this dashboard filter key applies to, or `$name` of a DATA_SOURCE filter on the dashboard", "example": "65f5e4a3b9e77c001a111111" }, "sourceMetricType": { @@ -4469,7 +4575,7 @@ "items": { "type": "string" }, - "description": "IDs of the sources whose keys and values the filter suggests, all of sourceType.", + "description": "IDs of the sources whose keys and values the filter suggests, all of sourceType, or `$name` of data source variables.", "example": [ "65f5e4a3b9e77c001a111111" ] @@ -4493,7 +4599,7 @@ "items": { "type": "string" }, - "description": "IDs of the sources whose tiles the conditions apply to, all of\nsourceType. Omit or send an empty array to apply to the tiles of\nthe filter's sources.\n", + "description": "IDs of the sources whose tiles the conditions apply to, all of\nsourceType, or `$name` of data source variables. Omit or send an\nempty array to apply to the tiles of the filter's sources.\n", "example": [ "65f5e4a3b9e77c001a111111" ] diff --git a/packages/api/src/mcp/__tests__/dashboards/patchDashboard.int.test.ts b/packages/api/src/mcp/__tests__/dashboards/patchDashboard.int.test.ts index eef1d0838..21e4e6b12 100644 --- a/packages/api/src/mcp/__tests__/dashboards/patchDashboard.int.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/patchDashboard.int.test.ts @@ -1400,4 +1400,76 @@ describe('MCP Dashboard Tools - clickstack_patch_dashboard', () => { expect(tile.config.showOperandSeries).toBe(false); }); }); + + describe('reference variables', () => { + const createDashboard = () => + new Dashboard({ + name: 'Variable dashboard', + tiles: [ + { + id: 'tile', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + name: 'Tile', + source: ctx.traceSource._id.toString(), + displayType: 'line', + select: [{ aggFn: 'count', valueExpression: '' }], + where: '', + }, + }, + ], + filters: [ + { + id: 'ds-filter', + type: 'DATA_SOURCE', + name: 'Traces source', + variableName: 'traces', + allowedSourceKinds: [SourceKind.Trace], + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }, + ], + team: ctx.team._id, + }).save(); + + const patchSource = (dashboardId: string, sourceId: string) => + callTool(ctx.client!, 'clickstack_patch_dashboard', { + dashboardId, + tileId: 'tile', + tile: { + name: 'Tile', + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + }); + + it('patches a tile to read from a declared data source variable', async () => { + const dashboard = await createDashboard(); + + const result = await patchSource(dashboard._id.toString(), '$traces'); + + expect(result.isError).toBeFalsy(); + const stored = await Dashboard.findById(dashboard._id).lean(); + expect(stored?.tiles[0].config).toMatchObject({ source: '$traces' }); + }); + + it('rejects a patch referencing an undeclared variable', async () => { + const dashboard = await createDashboard(); + + const result = await patchSource(dashboard._id.toString(), '$nope'); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain( + 'does not name a data source variable', + ); + }); + }); }); diff --git a/packages/api/src/mcp/__tests__/dashboards/queryTile.int.test.ts b/packages/api/src/mcp/__tests__/dashboards/queryTile.int.test.ts index 48de1b8f6..5042fc502 100644 --- a/packages/api/src/mcp/__tests__/dashboards/queryTile.int.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/queryTile.int.test.ts @@ -11,6 +11,7 @@ import { seedExponentialHistogramMetric, } from '@/fixtures'; import { callTool, getFirstText } from '@/mcp/__tests__/mcpTestUtils'; +import Dashboard from '@/models/dashboard'; import { Source } from '@/models/source'; import { setupDashboardTests } from './setup'; @@ -1373,4 +1374,85 @@ describe('MCP Dashboard Tools - clickstack_query_tile', () => { expect(rows.map(r => r.ServiceName)).toEqual(['svc-e', 'svc-d', 'svc-c']); }); }); + + describe('reference variables', () => { + const createVariableDashboard = (savedFilterValues: unknown[] = []) => + new Dashboard({ + name: 'Variable dashboard', + tiles: [ + { + id: 'variable-tile', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + name: 'Count', + source: '$traces', + displayType: 'number', + select: [{ aggFn: 'count', valueExpression: '' }], + where: '', + }, + }, + ], + filters: [ + { + id: 'ds-filter', + type: 'DATA_SOURCE', + name: 'Traces source', + variableName: 'traces', + allowedSourceKinds: [SourceKind.Trace], + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }, + ], + savedFilterValues, + team: ctx.team._id, + }).save(); + + it("runs a tile on its data source variable's saved selection", async () => { + const dashboard = await createVariableDashboard([ + { + type: 'variable', + name: 'traces', + values: [ctx.traceSource._id.toString()], + }, + ]); + + const result = await callTool(ctx.client!, 'clickstack_query_tile', { + dashboardId: dashboard._id.toString(), + tileId: 'variable-tile', + }); + + expect(result.isError).toBeFalsy(); + }); + + it('runs a tile on a selection passed in variableValues', async () => { + const dashboard = await createVariableDashboard(); + + const result = await callTool(ctx.client!, 'clickstack_query_tile', { + dashboardId: dashboard._id.toString(), + tileId: 'variable-tile', + variableValues: [ + { name: 'traces', values: [ctx.traceSource._id.toString()] }, + ], + }); + + expect(result.isError).toBeFalsy(); + }); + + it('reports a data source variable with no selection', async () => { + const dashboard = await createVariableDashboard(); + + const result = await callTool(ctx.client!, 'clickstack_query_tile', { + dashboardId: dashboard._id.toString(), + tileId: 'variable-tile', + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('$traces variable'); + }); + }); }); diff --git a/packages/api/src/mcp/__tests__/dashboards/queryTiles.int.test.ts b/packages/api/src/mcp/__tests__/dashboards/queryTiles.int.test.ts index afaeff978..6b154c40e 100644 --- a/packages/api/src/mcp/__tests__/dashboards/queryTiles.int.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/queryTiles.int.test.ts @@ -594,4 +594,59 @@ describe('MCP Dashboard Tools - clickstack_query_tiles', () => { expect(text).toContain('prod, staging'); }); }); + + it('reports a tile whose data source variable has no selection without failing the batch', async () => { + const sourceId = ctx.traceSource._id.toString(); + const createResult = await callTool( + ctx.client!, + 'clickstack_save_dashboard', + { + name: 'Batch reference variable test', + tiles: [ + { + name: 'Fixed', + config: { + displayType: 'number', + sourceId, + select: [{ aggFn: 'count' }], + }, + }, + { + name: 'Variable', + config: { + displayType: 'number', + sourceId: '$traces', + select: [{ aggFn: 'count' }], + }, + }, + ], + filters: [ + { + type: 'DATA_SOURCE', + name: 'Traces source', + variableName: 'traces', + allowedSourceKinds: [SourceKind.Trace], + }, + ], + }, + ); + expect(createResult.isError).toBeFalsy(); + const dashboard = JSON.parse(getFirstText(createResult)); + + const result = await callTool(ctx.client!, 'clickstack_query_tiles', { + dashboardId: dashboard.id, + ...wideRange(), + }); + + expect(result.isError).toBeFalsy(); + const { tiles } = JSON.parse(getFirstText(result)); + expect(tiles).toEqual([ + expect.objectContaining({ name: 'Fixed', status: 'ok' }), + expect.objectContaining({ + name: 'Variable', + status: 'error', + error: expect.stringContaining('$traces variable'), + }), + ]); + }); }); diff --git a/packages/api/src/mcp/__tests__/dashboards/queryTilesHandler.test.ts b/packages/api/src/mcp/__tests__/dashboards/queryTilesHandler.test.ts index 3b270d960..597ef883c 100644 --- a/packages/api/src/mcp/__tests__/dashboards/queryTilesHandler.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/queryTilesHandler.test.ts @@ -17,6 +17,15 @@ jest.mock('@/mcp/tools/query/helpers', () => { }; }); +jest.mock('@/mcp/tools/dashboards/variables', () => { + const actual = jest.requireActual('@/mcp/tools/dashboards/variables'); + return { + ...actual, + getReferenceTargets: () => + Promise.resolve({ sources: [], connectionIds: new Set() }), + }; +}); + jest.mock('@/models/dashboard', () => ({ __esModule: true, default: { findOne: (...args: unknown[]) => mockFindOne(...args) }, diff --git a/packages/api/src/mcp/__tests__/dashboards/saveDashboard.int.test.ts b/packages/api/src/mcp/__tests__/dashboards/saveDashboard.int.test.ts index 9624eca73..721e25064 100644 --- a/packages/api/src/mcp/__tests__/dashboards/saveDashboard.int.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/saveDashboard.int.test.ts @@ -4132,4 +4132,107 @@ describe('MCP Dashboard Tools - clickstack_save_dashboard', () => { ]); }); }); + + describe('reference variables', () => { + const filters = [ + { + type: 'CONNECTION', + name: 'Connection', + variableName: 'conn', + isGlobalRequirement: true, + }, + { + type: 'DATA_SOURCE', + name: 'Traces source', + variableName: 'traces', + allowedSourceKinds: [SourceKind.Trace], + connectionId: '$conn', + }, + ]; + + it('saves and returns variables and tiles that reference them', async () => { + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Variable dashboard', + tiles: [ + { + name: 'Builder', + config: { + displayType: 'line', + sourceId: '$traces', + select: [{ aggFn: 'count' }], + }, + }, + { + name: 'Raw SQL', + config: { + configType: 'sql', + displayType: 'table', + connectionId: '$conn', + sqlTemplate: 'SELECT 1', + }, + }, + ], + filters, + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.filters).toEqual([ + expect.objectContaining(filters[0]), + expect.objectContaining(filters[1]), + ]); + expect( + output.tiles.map((t: ExternalDashboardTileWithId) => t.config), + ).toEqual([ + expect.objectContaining({ sourceId: '$traces' }), + expect.objectContaining({ connectionId: '$conn' }), + ]); + }); + + it('rejects a tile referencing an undeclared variable', async () => { + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Variable dashboard', + tiles: [ + { + name: 'Builder', + config: { + displayType: 'line', + sourceId: '$traces', + select: [{ aggFn: 'count' }], + }, + }, + ], + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain( + 'does not name a data source variable', + ); + }); + + it('checks an update without filters against the stored variables', async () => { + const builderTile = (sourceId: string) => ({ + name: 'Builder', + config: { + displayType: 'line', + sourceId, + select: [{ aggFn: 'count' }], + }, + }); + const created = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Variable dashboard', + tiles: [builderTile(ctx.traceSource._id.toString())], + filters, + }); + expect(created.isError).toBeFalsy(); + + const updated = await callTool(ctx.client!, 'clickstack_save_dashboard', { + id: JSON.parse(getFirstText(created)).id, + name: 'Variable dashboard', + tiles: [builderTile('$traces')], + }); + + expect(updated.isError).toBeFalsy(); + }); + }); }); diff --git a/packages/api/src/mcp/__tests__/dashboards/validation.test.ts b/packages/api/src/mcp/__tests__/dashboards/validation.test.ts index bd6ffe6a6..0e9034c09 100644 --- a/packages/api/src/mcp/__tests__/dashboards/validation.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/validation.test.ts @@ -11,6 +11,7 @@ import { getRawSqlMissingSourceError, getRawSqlTileMacroWarnings, getRawSqlTilesMissingRequiredSource, + getSavedFilterValueWarnings, getTileVariableWarnings, } from '@/mcp/tools/dashboards/validation'; import type { @@ -982,3 +983,31 @@ describe('getFilterVariableWarnings', () => { }); }); }); + +describe('getSavedFilterValueWarnings', () => { + const filters = [ + { type: 'DATA_SOURCE' as const, name: 'Traces', variableName: 'traces' }, + { type: 'CONNECTION' as const, name: 'Cluster', variableName: 'conn' }, + ]; + + it('accepts saved values for data source and connection variables', () => { + expect( + getSavedFilterValueWarnings( + [ + { type: 'variable', name: 'traces', values: ['s1'] }, + { type: 'variable', name: 'conn', values: ['c1'] }, + ], + filters, + ), + ).toEqual([]); + }); + + it('warns about a saved value no filter declares', () => { + expect( + getSavedFilterValueWarnings( + [{ type: 'variable', name: 'nope', values: ['x'] }], + filters, + ), + ).toHaveLength(1); + }); +}); diff --git a/packages/api/src/mcp/__tests__/dashboards/variables.test.ts b/packages/api/src/mcp/__tests__/dashboards/variables.test.ts index 6d3bcc42b..45270e65e 100644 --- a/packages/api/src/mcp/__tests__/dashboards/variables.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/variables.test.ts @@ -1,10 +1,16 @@ +import { SourceKind } from '@hyperdx/common-utils/dist/types'; import mongoose from 'mongoose'; import { + type ReferenceTargets, resolveDashboardVariables, + resolveTileVariableReferences, withResolvedFilterVariableNames, } from '@/mcp/tools/dashboards/variables'; -import type { ExternalDashboardFilterWithId } from '@/utils/zod'; +import type { + ExternalDashboardFilterWithId, + ExternalDashboardTileWithId, +} from '@/utils/zod'; const sourceId = new mongoose.Types.ObjectId().toString(); @@ -322,3 +328,143 @@ describe('withResolvedFilterVariableNames', () => { ]); }); }); + +describe('resolveTileVariableReferences', () => { + const objectId = () => new mongoose.Types.ObjectId().toString(); + const connectionId = objectId(); + const traceSourceId = objectId(); + const targets: ReferenceTargets = { + sources: [ + { id: traceSourceId, kind: SourceKind.Trace, connection: connectionId }, + ], + connectionIds: new Set([connectionId]), + }; + + const referenceVariableFields = { + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + } as const; + const connectionVariable: ExternalDashboardFilterWithId = { + ...referenceVariableFields, + id: objectId(), + type: 'CONNECTION', + name: 'Connection', + variableName: 'conn', + }; + const tracesVariable = ( + connection?: string, + ): ExternalDashboardFilterWithId => ({ + ...referenceVariableFields, + id: objectId(), + type: 'DATA_SOURCE', + name: 'Traces source', + variableName: 'traces', + allowedSourceKinds: [SourceKind.Trace], + connectionId: connection, + }); + + const builderTile: ExternalDashboardTileWithId = { + id: 'builder', + name: 'Builder', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + displayType: 'number', + sourceId: '$traces', + select: [{ aggFn: 'count', where: '' }], + }, + }; + const rawSqlTile: ExternalDashboardTileWithId = { + id: 'raw-sql', + name: 'Raw SQL', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + configType: 'sql', + displayType: 'table', + connectionId: '$conn', + sqlTemplate: 'SELECT 1', + }, + }; + + const resolve = ( + tile: ExternalDashboardTileWithId, + filters: ExternalDashboardFilterWithId[], + selections: Record, + ) => + resolveTileVariableReferences( + tile, + filters, + Object.entries(selections).map(([name, values]) => ({ name, values })), + [], + targets, + ); + + it("resolves a connection reference to the variable's selection", () => { + const result = resolve(rawSqlTile, [connectionVariable], { + conn: [connectionId], + }); + + expect(result).toEqual({ + tile: { + ...rawSqlTile, + config: { ...rawSqlTile.config, connectionId }, + }, + }); + }); + + it('resolves a source on the connection its connection variable selects', () => { + const result = resolve( + builderTile, + [connectionVariable, tracesVariable('$conn')], + { conn: [connectionId], traces: [traceSourceId] }, + ); + + expect(result).toEqual({ + tile: { + ...builderTile, + config: { ...builderTile.config, sourceId: traceSourceId }, + }, + }); + }); + + it.each<{ + case: string; + tile: ExternalDashboardTileWithId; + filters: ExternalDashboardFilterWithId[]; + selections: Record; + variable: string; + }>([ + { + case: 'a source selection that is not a source', + tile: builderTile, + filters: [tracesVariable()], + selections: { traces: [connectionId] }, + variable: '$traces', + }, + { + case: 'a source variable whose connection variable has no selection', + tile: builderTile, + filters: [connectionVariable, tracesVariable('$conn')], + selections: { traces: [traceSourceId] }, + variable: '$conn', + }, + { + case: 'a connection selection that is not a connection', + tile: rawSqlTile, + filters: [connectionVariable], + selections: { conn: [traceSourceId] }, + variable: '$conn', + }, + ])('rejects $case', ({ tile, filters, selections, variable }) => { + expect(resolve(tile, filters, selections)).toEqual({ + error: expect.stringContaining(`${variable} variable`), + }); + }); +}); diff --git a/packages/api/src/mcp/prompts/dashboards/content.ts b/packages/api/src/mcp/prompts/dashboards/content.ts index bbfb7167d..4db680859 100644 --- a/packages/api/src/mcp/prompts/dashboards/content.ts +++ b/packages/api/src/mcp/prompts/dashboards/content.ts @@ -137,6 +137,8 @@ Apply these before calling clickstack_save_dashboard. Each rule is enforced by t 9h. USE AN ADHOC FILTER TO LET THE USER BUILD FREE-FORM CONDITIONS. filters: [{ type: "ADHOC", name, sourceType: "sql", sources: [""], variableName? }] adds an input where the user picks any column or map key, an operator, and a value; clicking a value in a tile adds a condition too. Every condition is ANDed onto the tiles of its sources (or of appliesToSourceIds) with no per-tile wiring. It is never a variable: its variableName only names the entry its conditions are saved under. Seed conditions with savedFilterValues: [{ type: "adhoc", name: "", conditions: [{ key: "ServiceName", operator: "IN", value: "api" }] }]; IN conditions on the same key are ORed. Declare at most one ad hoc filter per source, since a click updates every ad hoc filter on the clicked tile's source. +9i. TO LET VIEWERS SWITCH THE SOURCE OR CONNECTION, USE A DATA_SOURCE OR CONNECTION FILTER. filters: [{ type: "DATA_SOURCE", name: "Logs source", variableName: "logs", allowedSourceKinds: ["log"] }] adds a required source picker; tiles then set sourceId: "$logs" (filters may use it in sourceId or appliesToSourceIds). A CONNECTION filter does the same for raw SQL tiles' connectionId, and a DATA_SOURCE filter's optional connectionId limits its sources to one connection. Keep allowedSourceKinds to kinds charts treat alike (log + trace, or one kind). Tiles that use either cannot have alerts, and clickstack_query_tile needs a selection for the variable (variableValues or the dashboard's saved value). + 10. UPDATE IS REPLACE, NOT MERGE. clickstack_save_dashboard with an id overwrites tiles, containers, and filters in their entirety. Call clickstack_get_dashboard first when you only want to add or rename one entry; do not send a partial set or you will silently drop everything you omitted. 11. GROUP RELATED TILES INTO CONTAINERS. REQUIRED at five or more tiles, no exceptions. An ungrouped wall of nine or ten tiles is a readability failure even when each tile is correct in isolation. Containers are the right way to introduce structure; markdown tiles for section labels are not. diff --git a/packages/api/src/mcp/tools/dashboards/patchDashboard.ts b/packages/api/src/mcp/tools/dashboards/patchDashboard.ts index 36d5f43a2..c19c9d871 100644 --- a/packages/api/src/mcp/tools/dashboards/patchDashboard.ts +++ b/packages/api/src/mcp/tools/dashboards/patchDashboard.ts @@ -201,6 +201,7 @@ export function registerPatchDashboard({ teamId, tiles: [mergedTile], existingTiles: existingDashboard.tiles ?? [], + existingFilters: existingDashboard.filters, containers: existingDashboard.containers ?? [], }); if (validationError) { diff --git a/packages/api/src/mcp/tools/dashboards/queryTile.ts b/packages/api/src/mcp/tools/dashboards/queryTile.ts index b6cd385b5..1e610b0cd 100644 --- a/packages/api/src/mcp/tools/dashboards/queryTile.ts +++ b/packages/api/src/mcp/tools/dashboards/queryTile.ts @@ -13,7 +13,12 @@ import { getRawSqlTileMacroWarnings, getTileVariableWarnings, } from './validation'; -import { mcpVariableValuesParam, resolveDashboardVariables } from './variables'; +import { + getReferenceTargets, + mcpVariableValuesParam, + resolveDashboardVariables, + resolveTileVariableReferences, +} from './variables'; export function registerQueryTile({ context, @@ -79,8 +84,8 @@ export function registerQueryTile({ const externalDashboard = convertToExternalDashboard(dashboard, { hidePromql: !isPromqlEnabled(context), }); - const tile = externalDashboard.tiles.find(t => t.id === tileId); - if (!tile) { + const storedTile = externalDashboard.tiles.find(t => t.id === tileId); + if (!storedTile) { return mcpUserError( `Tile not found: ${tileId}. Available tile IDs: ${externalDashboard.tiles.map(t => t.id).join(', ')}`, ); @@ -94,6 +99,18 @@ export function registerQueryTile({ return mcpUserError(resolvedVariables.error); } + const resolvedTile = resolveTileVariableReferences( + storedTile, + externalDashboard.filters, + variableValues, + externalDashboard.savedFilterValues, + await getReferenceTargets(teamId.toString()), + ); + if ('error' in resolvedTile) { + return mcpUserError(resolvedTile.error); + } + const { tile } = resolvedTile; + const result = await runConfigTile( teamId.toString(), tile, diff --git a/packages/api/src/mcp/tools/dashboards/queryTiles.ts b/packages/api/src/mcp/tools/dashboards/queryTiles.ts index 44af35c95..a1716461f 100644 --- a/packages/api/src/mcp/tools/dashboards/queryTiles.ts +++ b/packages/api/src/mcp/tools/dashboards/queryTiles.ts @@ -19,7 +19,12 @@ import { getRawSqlTileMacroWarnings, getTileVariableWarnings, } from './validation'; -import { mcpVariableValuesParam, resolveDashboardVariables } from './variables'; +import { + getReferenceTargets, + mcpVariableValuesParam, + resolveDashboardVariables, + resolveTileVariableReferences, +} from './variables'; /** * How many tiles to query against ClickHouse at once. Kept lowish so a batch @@ -230,6 +235,8 @@ export function registerQueryTiles({ .slice(MAX_TILES_PER_CALL) .map(t => t.id); + const referenceTargets = await getReferenceTargets(teamId.toString()); + // Fix one shared deadline for the whole batch that every tile races. const deadlineAt = Date.now() + BATCH_DEADLINE_MS; @@ -251,6 +258,17 @@ export function registerQueryTiles({ return { ...base, status: 'skipped' }; } + const resolvedTile = resolveTileVariableReferences( + tile, + externalDashboard.filters, + variableValues, + externalDashboard.savedFilterValues, + referenceTargets, + ); + if ('error' in resolvedTile) { + return { ...base, status: 'error', error: resolvedTile.error }; + } + // Run the query under the shared deadline. It's passed as a thunk // so withDeadline can skip issuing it entirely once the budget is // spent (see withDeadline). withDeadline hands the thunk an @@ -263,10 +281,13 @@ export function registerQueryTiles({ try { const result = await withDeadline( signal => - runConfigTile(teamId.toString(), tile, startDate, endDate, { - abortSignal: signal, - variables, - }), + runConfigTile( + teamId.toString(), + resolvedTile.tile, + startDate, + endDate, + { abortSignal: signal, variables }, + ), deadlineAt, ); diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index 090d00522..29e504a37 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -26,6 +26,7 @@ import { PromqlQueryTypeSchema, PromqlReducerSchema, SearchConditionTrimmedLanguageSchema, + SourceKind, SQLIntervalSchema, SqlQueryDashboardFilterSchema, StaticListDashboardFilterSchema, @@ -41,10 +42,18 @@ import { externalAdhocSavedFilterValueSchema, externalSqlSavedFilterValueSchema, externalVariableSavedFilterValueSchema, + objectIdOrVariableReferenceSchema, objectIdSchema, tagsSchema, } from '@/utils/zod'; +const SOURCE_VARIABLE_REFERENCE_NOTE = + 'May instead be `$name`, a reference to a DATA_SOURCE filter (variable) on this ' + + 'dashboard, so viewers can switch the source; the variable must only offer kinds ' + + 'this field accepts.'; + +const SOURCE_ID_DESCRIPTION = `Source ID - call clickstack_list_sources. ${SOURCE_VARIABLE_REFERENCE_NOTE}`; + /** * Metric type values exposed on dashboard tile select items. Restricted to * the kinds the query renderer can translate today; summary is intentionally @@ -651,7 +660,7 @@ const mcpTimeChartDisplayFields = { }; const mcpRowListSourceFields = { - sourceId: z.string().describe('Source ID – call clickstack_list_sources'), + sourceId: z.string().describe(SOURCE_ID_DESCRIPTION), where: z .string() .optional() @@ -739,7 +748,7 @@ export const mcpLineTileSchema = mcpTileLayoutSchema.extend({ .object({ ...rejectedTileWhereFields, displayType: z.literal('line').describe('Line chart over time'), - sourceId: z.string().describe('Source ID – call clickstack_list_sources'), + sourceId: z.string().describe(SOURCE_ID_DESCRIPTION), select: z .array(mcpTileSelectItemSchema) .min(1) @@ -776,7 +785,7 @@ export const mcpLineTileSchema = mcpTileLayoutSchema.extend({ }); const mcpStackedTimeChartConfigFields = { - sourceId: z.string().describe('Source ID – call clickstack_list_sources'), + sourceId: z.string().describe(SOURCE_ID_DESCRIPTION), select: z .array(mcpTileSelectItemSchema) .min(1) @@ -828,7 +837,7 @@ const mcpTableTileSchema = mcpTileLayoutSchema.extend({ .object({ ...rejectedTileWhereFields, displayType: z.literal('table').describe('Tabular aggregated data'), - sourceId: z.string().describe('Source ID – call clickstack_list_sources'), + sourceId: z.string().describe(SOURCE_ID_DESCRIPTION), select: z .array(mcpTileSelectItemSchema) .min(1) @@ -884,7 +893,7 @@ export const mcpNumberTileSchema = mcpTileLayoutSchema.extend({ displayType: z .literal('number') .describe('Single aggregate scalar value'), - sourceId: z.string().describe('Source ID – call clickstack_list_sources'), + sourceId: z.string().describe(SOURCE_ID_DESCRIPTION), select: z .array(mcpTileSelectItemSchema) .min(1) @@ -925,7 +934,7 @@ const mcpPieTileSchema = mcpTileLayoutSchema.extend({ .object({ ...rejectedTileWhereFields, displayType: z.literal('pie').describe('Pie chart'), - sourceId: z.string().describe('Source ID – call clickstack_list_sources'), + sourceId: z.string().describe(SOURCE_ID_DESCRIPTION), select: z .array(mcpTileSelectItemSchema) .length(1) @@ -973,7 +982,7 @@ const mcpCategoricalBarTileSchema = mcpTileLayoutSchema.extend({ displayType: z .literal('bar') .describe('Bar chart — one bar per group value (not a time series)'), - sourceId: z.string().describe('Source ID – call clickstack_list_sources'), + sourceId: z.string().describe(SOURCE_ID_DESCRIPTION), select: z .array(mcpTileSelectItemSchema) .length(1) @@ -1061,7 +1070,8 @@ const mcpHeatmapTileSchema = mcpTileLayoutSchema.extend({ .string() .describe( 'Source ID. Must be a Trace source today; use clickstack_list_sources and ' + - 'pick one whose kind is "trace".', + 'pick one whose kind is "trace". ' + + SOURCE_VARIABLE_REFERENCE_NOTE, ), select: z .array(mcpHeatmapSelectItemSchema) @@ -1106,7 +1116,8 @@ const mcpHeatmapSeriesTileSchema = mcpTileLayoutSchema.extend({ sourceId: z .string() .describe( - 'Source ID of a Trace, Log, or Metric source - call clickstack_list_sources', + 'Source ID of a Trace, Log, or Metric source - call clickstack_list_sources. ' + + SOURCE_VARIABLE_REFERENCE_NOTE, ), select: z .array(mcpTileSelectItemSchema) @@ -1203,7 +1214,9 @@ export const mcpSqlTileSchema = mcpTileLayoutSchema.extend({ connectionId: z .string() .describe( - 'Connection ID (not sourceId) – call clickstack_list_sources to find available connections', + 'Connection ID (not sourceId) – call clickstack_list_sources to find available connections. ' + + 'May instead be `$name`, a reference to a CONNECTION filter (variable) on this ' + + 'dashboard, so viewers can switch the connection the query runs on.', ), sourceId: z .string() @@ -1213,7 +1226,8 @@ export const mcpSqlTileSchema = mcpTileLayoutSchema.extend({ 'ALWAYS set this for raw SQL tiles UNLESS the query reads from multiple tables ' + '(e.g. JOINs or sub-queries spanning several sources), in which case omit it. ' + 'sourceId is REQUIRED by two macros: $__filters and $__sourceTable. ' + - 'The sourceId must belong to the same connection as connectionId.', + 'The sourceId must belong to the same connection as connectionId. ' + + SOURCE_VARIABLE_REFERENCE_NOTE, ), sqlTemplate: z.string().describe(` Raw ClickHouse SQL query. SQL guidelines: @@ -1341,7 +1355,8 @@ const mcpPromqlTileSchema = mcpTileLayoutSchema.extend({ sourceId: z .string() .describe( - 'ID of a source whose kind is "promql" – call clickstack_list_sources', + 'ID of a source whose kind is "promql" – call clickstack_list_sources. ' + + SOURCE_VARIABLE_REFERENCE_NOTE, ), expressions: z .array(mcpPromqlExpressionSchema) @@ -1670,9 +1685,10 @@ const mcpQueryExpressionFilterSchema = z "not declared in any of this dashboard's filters is silently dropped at click time. " + 'Declare an expression here for every column you plan to drive via row-click.', ), - sourceId: objectIdSchema.describe( + sourceId: objectIdOrVariableReferenceSchema.describe( 'Source the filter values are pulled from (for the dropdown). ' + - 'Get IDs from clickstack_list_sources.', + 'Get IDs from clickstack_list_sources. ' + + SOURCE_VARIABLE_REFERENCE_NOTE, ), sourceMetricType: z .nativeEnum(MetricsDataType) @@ -1701,7 +1717,7 @@ const mcpQueryExpressionFilterSchema = z 'Query language of `where`. Set it explicitly.', ), appliesToSourceIds: z - .array(objectIdSchema) + .array(objectIdOrVariableReferenceSchema) .optional() .describe( 'Source IDs (from clickstack_list_sources) that this filter is applied to. ' + @@ -1805,8 +1821,9 @@ const mcpPromqlLabelFilterSchema = z .describe( "Filter type discriminator for a filter whose dropdown lists a Prometheus label's values.", ), - sourceId: objectIdSchema.describe( - 'ID of a source whose kind is "promql". Get IDs from clickstack_list_sources.', + sourceId: objectIdOrVariableReferenceSchema.describe( + 'ID of a source whose kind is "promql". Get IDs from clickstack_list_sources. ' + + SOURCE_VARIABLE_REFERENCE_NOTE, ), label: PromqlLabelDashboardFilterSchema.shape.label.describe( 'Label whose values fill the dropdown, e.g. "job". ' + @@ -1846,11 +1863,11 @@ const mcpAdhocFilterSchema = z '"sql": keys are the columns and map keys of Log, Trace, or Session sources.', ), sources: z - .array(objectIdSchema) + .array(objectIdOrVariableReferenceSchema) .min(1) .describe( 'IDs of the Log, Trace, or Session sources (from clickstack_list_sources) ' + - 'whose keys and values the filter suggests.', + 'whose keys and values the filter suggests, or `$name` of DATA_SOURCE filters.', ), where: z .string() @@ -1863,7 +1880,7 @@ const mcpAdhocFilterSchema = z 'Query language of `where`.', ), appliesToSourceIds: z - .array(objectIdSchema) + .array(objectIdOrVariableReferenceSchema) .optional() .describe( 'IDs of the sources whose tiles the conditions apply to, of the same kind ' + @@ -1885,31 +1902,90 @@ const mcpPromqlAdhocFilterSchema = mcpAdhocFilterSchema.extend({ 'as label matchers to every selector in PromQL tiles.', ), sources: z - .array(objectIdSchema) + .array(objectIdOrVariableReferenceSchema) .min(1) .describe( 'IDs of the sources (from clickstack_list_sources) whose keys and values ' + - 'the filter suggests: Log, Trace, or Session sources for sourceType ' + - '"sql", promql sources for "promql".', + 'the filter suggests, or `$name` of DATA_SOURCE filters: Log, Trace, or ' + + 'Session sources for sourceType "sql", promql sources for "promql".', ), where: mcpAdhocFilterSchema.shape.where.describe( `${mcpAdhocFilterSchema.shape.where.description} sourceType "sql" only; rejected when sourceType is "promql".`, ), }); +// Reference variables are always single-select and required, so they take +// none of the selection-limit fields. +const mcpReferenceVariableBaseShape = { + id: mcpDashboardFilterBaseShape.id, + name: mcpDashboardFilterBaseShape.name, + variableName: mcpDashboardFilterBaseShape.variableName, + // Always required, so the base description's minSelections condition doesn't apply + isGlobalRequirement: mcpDashboardFilterBaseShape.isGlobalRequirement.describe( + 'Block EVERY tile on the dashboard until a value is selected, not just ' + + 'the ones that reference this variable.', + ), +}; + +const mcpDataSourceFilterSchema = z + .object({ + ...mcpReferenceVariableBaseShape, + type: z + .literal(DashboardFilterType.enum.DATA_SOURCE) + .describe( + 'Filter type discriminator for a variable whose value is a source.', + ), + allowedSourceKinds: z + .array(z.nativeEnum(SourceKind)) + .min(1) + .describe( + 'Kinds of sources offered in the dropdown. A tile can only use the variable ' + + 'if it accepts every listed kind, so keep kinds that charts treat alike ' + + '(log + trace, or a single kind).', + ), + connectionId: objectIdOrVariableReferenceSchema + .optional() + .describe( + 'Optional connection the offered sources are limited to: a connection ID, or ' + + '`$name` of a CONNECTION filter on this dashboard.', + ), + }) + .describe( + 'A variable whose single, required value is a source. Tiles and other filters ' + + 'reference it as `$variableName` in sourceId (or appliesToSourceIds) instead of a ' + + 'source ID; tiles stay blocked until a source is selected.', + ); + +const mcpConnectionFilterSchema = z + .object({ + ...mcpReferenceVariableBaseShape, + type: z + .literal(DashboardFilterType.enum.CONNECTION) + .describe( + 'Filter type discriminator for a variable whose value is a connection.', + ), + }) + .describe( + 'A variable whose single, required value is a connection. Raw SQL tiles reference ' + + 'it as `$variableName` in connectionId, and DATA_SOURCE filters in connectionId.', + ); + const mcpDashboardFilterSchema = z .discriminatedUnion('type', [ mcpQueryExpressionFilterSchema, mcpStaticListFilterSchema, mcpSqlQueryFilterSchema, mcpAdhocFilterSchema, + mcpDataSourceFilterSchema, + mcpConnectionFilterSchema, ]) .describe( 'A dashboard-level filter the user can adjust in the dashboard filter bar. ' + 'QUERY_EXPRESSION queries its dropdown values from a source column; ' + 'STATIC_LIST declares them inline; SQL_QUERY reads them from a SQL query. ' + 'STATIC_LIST and SQL_QUERY are always variable-only. ADHOC holds free-form ' + - 'conditions and always broadcasts.', + 'conditions and always broadcasts. DATA_SOURCE and CONNECTION select the ' + + 'source or connection that tiles referencing them as `$variableName` use.', ); const mcpPromqlDashboardFilterSchema = z @@ -1919,6 +1995,8 @@ const mcpPromqlDashboardFilterSchema = z mcpSqlQueryFilterSchema, mcpPromqlLabelFilterSchema, mcpPromqlAdhocFilterSchema, + mcpDataSourceFilterSchema, + mcpConnectionFilterSchema, ]) .describe( 'A dashboard-level filter the user can adjust in the dashboard filter bar. ' + @@ -1926,7 +2004,8 @@ const mcpPromqlDashboardFilterSchema = z 'STATIC_LIST declares them inline; SQL_QUERY reads them from a SQL query; ' + 'PROMETHEUS_LABEL lists the values of a label on a promql source. Those three ' + 'are always variable-only. ADHOC holds free-form conditions on SQL or promql ' + - 'sources and always broadcasts.', + 'sources and always broadcasts. DATA_SOURCE and CONNECTION select the source ' + + 'or connection that tiles referencing them as `$variableName` use.', ); export type McpDashboardFilter = z.infer; @@ -2011,7 +2090,16 @@ const mcpFiltersParam = z ']\n' + 'with savedFilterValues: [{ "type": "adhoc", "name": "conditions", "conditions": ' + '[{ "key": "ServiceName", "operator": "IN", "value": "api" }, ' + - '{ "key": "ServiceName", "operator": "IN", "value": "web" }] }].', + '{ "key": "ServiceName", "operator": "IN", "value": "web" }] }].\n\n' + + 'Example (data source variable: viewers pick which logs source every tile reads):\n' + + '[\n' + + ' { "type": "DATA_SOURCE", "name": "Logs source", "variableName": "logs",\n' + + ' "allowedSourceKinds": ["log"] }\n' + + ']\n' + + 'Tiles then set "sourceId": "$logs". A CONNECTION filter works the same way for ' + + 'raw SQL tiles ("connectionId": "$conn"); a DATA_SOURCE filter may set ' + + '"connectionId" to limit its sources to one connection. Alerts are not supported ' + + 'on tiles that use either.', ); const PROMQL_FILTERS_EXAMPLE = diff --git a/packages/api/src/mcp/tools/dashboards/validation.ts b/packages/api/src/mcp/tools/dashboards/validation.ts index d678b3772..63b47c5fd 100644 --- a/packages/api/src/mcp/tools/dashboards/validation.ts +++ b/packages/api/src/mcp/tools/dashboards/validation.ts @@ -7,6 +7,8 @@ import { isAdhocConditionSupported, isFilterVariableEnabled, validateSqlQueryFilterTemplate, + isReferenceVariableFilterType, + pairWithVariableNames, } from '@hyperdx/common-utils/dist/filters'; import { isBuilderSavedChartConfig, @@ -264,7 +266,8 @@ export function getFilterVariableWarnings( * `name` matches no filter of its kind, or an ad hoc condition its filter's * source type can't express, is silently ignored when the dashboard loads, so * it looks like a saved default but never applies. Warn so the agent can fix - * it. + * it. Data source and connection variables count as variable-enabled filters, + * though they are not text variables. */ export function getSavedFilterValueWarnings( savedFilterValues: DashboardFilterValue[] | undefined, @@ -273,11 +276,16 @@ export function getSavedFilterValueWarnings( | undefined, ): string[] { if (!savedFilterValues?.length) return []; - const declaredNames = new Set( - getDashboardVariableDeclarations(filters).map( + const declaredNames = new Set([ + ...getDashboardVariableDeclarations(filters).map( declaration => declaration.name, ), - ); + ...pairWithVariableNames( + (filters ?? []).filter(filter => + isReferenceVariableFilterType(filter.type), + ), + ).map(({ name }) => name), + ]); const adhocSourceTypeByName = new Map( (filters ?? []).flatMap(filter => filter.type === 'ADHOC' && filter.sourceType diff --git a/packages/api/src/mcp/tools/dashboards/variables.ts b/packages/api/src/mcp/tools/dashboards/variables.ts index f1b98a611..d0c7a7548 100644 --- a/packages/api/src/mcp/tools/dashboards/variables.ts +++ b/packages/api/src/mcp/tools/dashboards/variables.ts @@ -2,18 +2,41 @@ import { getDashboardVariableDeclarations, getDashboardVariableFilters, getFilterVariableName, + isDataSourceFilter, isFilterVariableEnabled, isStaticListFilter, } from '@hyperdx/common-utils/dist/filters'; -import type { ChartVariable } from '@hyperdx/common-utils/dist/types'; +import { + getReferenceVariableFilters, + getStringField, + isSourceOfferedByVariable, + parseVariableReference, +} from '@hyperdx/common-utils/dist/referenceVariables'; +import type { + ChartVariable, + SourceKind, +} from '@hyperdx/common-utils/dist/types'; import { isFilterSingleSelect } from '@hyperdx/common-utils/dist/variables'; import { z } from 'zod'; +import { getConnectionsByTeam } from '@/controllers/connection'; +import { getSources } from '@/controllers/sources'; +import type { ExternalDashboard } from '@/routers/external-api/v2/utils/dashboards'; import type { ExternalDashboardFilter, ExternalDashboardFilterWithId, + ExternalDashboardTileWithId, } from '@/utils/zod'; +type ExternalFilters = + | (ExternalDashboardFilter | ExternalDashboardFilterWithId)[] + | undefined; + +/** Names of the data source and connection variables a dashboard declares. */ +function getReferenceVariableNames(filters: ExternalFilters): Set { + return new Set(getReferenceVariableFilters(filters).map(({ name }) => name)); +} + /** Optional per-variable selection for a tile run. */ export const mcpVariableValuesParam = z .array( @@ -28,7 +51,9 @@ export const mcpVariableValuesParam = z .array(z.string()) .describe( 'Values to treat as selected. An empty array means nothing is selected for this variable. ' + - "For a STATIC_LIST filter's variable, every value must be one of the filter's declared options.", + "For a STATIC_LIST filter's variable, every value must be one of the filter's declared options. " + + 'For a DATA_SOURCE or CONNECTION variable, pass one source or connection ID; it ' + + "overrides the dashboard's saved selection.", ), }), ) @@ -65,9 +90,11 @@ export function resolveDashboardVariables( if (!variableValues?.length) return { variables }; const byName = new Map(variables.map(variable => [variable.name, variable])); + // Resolved separately, by resolveTileVariableReferences + const referenceVariableNames = getReferenceVariableNames(filters); const unknownVariableValues = variableValues .map(({ name }) => name) - .filter(name => !byName.has(name)); + .filter(name => !byName.has(name) && !referenceVariableNames.has(name)); if (unknownVariableValues.length > 0) { const available = variables.length > 0 @@ -121,12 +148,136 @@ export function resolveDashboardVariables( // Override the default empty selection with the given variableValues for (const { name, values } of variableValues) { - byName.get(name)!.values = values; + const variable = byName.get(name); + if (variable) variable.values = values; } return { variables }; } +/** The team's sources and connections that a reference variable may select. */ +export type ReferenceTargets = { + sources: { + id: string; + kind: SourceKind; + connection: string; + disabled?: boolean; + }[]; + connectionIds: ReadonlySet; +}; + +export async function getReferenceTargets( + teamId: string, +): Promise { + const [sources, connections] = await Promise.all([ + getSources(teamId), + getConnectionsByTeam(teamId), + ]); + return { + sources: sources.map(source => ({ + id: source._id.toString(), + kind: source.kind, + connection: source.connection.toString(), + disabled: source.disabled, + })), + connectionIds: new Set(connections.map(c => c._id.toString())), + }; +} + +/** + * Replace a tile's `$name` sourceId / connectionId with the source or + * connection its data source or connection variable selects: the caller's + * variableValues first, then the dashboard's saved selection. A tile with no + * selection cannot run, as on the dashboard, so that is an error naming the + * variable. So is a selection the dashboard's picker would not offer, such as + * a source of another kind or on another connection. + */ +export function resolveTileVariableReferences< + T extends ExternalDashboardTileWithId, +>( + tile: T, + filters: ExternalFilters, + variableValues: McpVariableValues | undefined, + savedFilterValues: ExternalDashboard['savedFilterValues'] | undefined, + targets: ReferenceTargets, +): { tile: T } | { error: string } { + if (!('config' in tile) || tile.config == null) return { tile }; + const selections = new Map(); + for (const value of savedFilterValues ?? []) { + if (value.type === 'variable') selections.set(value.name, value.values); + } + // resolveDashboardVariables has already rejected unknown names + for (const { name, values } of variableValues ?? []) { + selections.set(name, values); + } + const sourceVariables = new Map( + getReferenceVariableFilters(filters).flatMap(({ filter, name }) => + isDataSourceFilter(filter) ? [[name, filter] as const] : [], + ), + ); + + const isSourceOffered = (name: string, id: string) => { + const variable = sourceVariables.get(name); + const source = targets.sources.find(source => source.id === id); + if (variable == null || source == null) return false; + // An unselected connection variable stays a reference, which matches no source + const connectionName = parseVariableReference(variable.connectionId); + const connection = + connectionName == null + ? variable.connectionId + : (selections.get(connectionName)?.[0] ?? variable.connectionId); + return isSourceOfferedByVariable(source, { + allowedSourceKinds: variable.allowedSourceKinds, + connection, + }); + }; + + const describeOffer = (name: string) => { + const variable = sourceVariables.get(name); + if (variable == null) return 'no sources'; + const connectionName = parseVariableReference(variable.connectionId); + const connection = + connectionName != null + ? ` on the connection the $${connectionName} variable selects` + : variable.connectionId + ? ` on connection ${variable.connectionId}` + : ''; + return `enabled ${variable.allowedSourceKinds.join(', ')} sources${connection}`; + }; + + let config: object = tile.config; + for (const field of ['sourceId', 'connectionId'] as const) { + const name = parseVariableReference(getStringField(config, field)); + if (name == null) continue; + const subject = `Tile "${tile.id}" reads its ${field} from the $${name} variable`; + const values = selections.get(name) ?? []; + if (values.length !== 1) { + return { + error: + values.length === 0 + ? `${subject}, which has no selection. Pass one ID for it in variableValues, or save a selection on the dashboard.` + : `${subject}, which accepts exactly one value.`, + }; + } + const [id] = values; + const isOffered = + field === 'connectionId' + ? targets.connectionIds.has(id) + : isSourceOffered(name, id); + if (!isOffered) { + const offer = + field === 'connectionId' + ? 'connections on this team' + : describeOffer(name); + return { + error: `${subject}, which selects "${id}" but only offers ${offer}.`, + }; + } + config = { ...config, [field]: id }; + } + return { tile: { ...tile, config } }; +} + /** Options of each static filter, keyed by the variable name it answers to. */ function buildStaticOptionsByName( filters: ExternalDashboardFilter[] | undefined, diff --git a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts index 92a72d976..57d2e35be 100644 --- a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts +++ b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts @@ -8511,6 +8511,291 @@ describe('External API v2 Dashboards - new format', () => { }); }); + describe('reference variables', () => { + const connectionFilter = { + type: 'CONNECTION' as const, + name: 'Connection', + variableName: 'conn', + }; + const dataSourceFilter = { + type: 'DATA_SOURCE' as const, + name: 'Traces source', + variableName: 'traces', + allowedSourceKinds: [SourceKind.Trace], + connectionId: '$conn', + }; + const rawSqlTile = { + name: 'Raw SQL', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + configType: 'sql', + displayType: 'table', + connectionId: '$conn', + sqlTemplate: 'SELECT 1', + }, + }; + + it('round-trips the variables, their values and the tiles that use them', async () => { + const globalConnectionFilter = { + ...connectionFilter, + isGlobalRequirement: true, + }; + const savedFilterValues = [ + { type: 'variable', name: 'conn', values: [connection._id.toString()] }, + { + type: 'variable', + name: 'traces', + values: [traceSource._id.toString()], + }, + ]; + const created = await authRequest('post', BASE_URL) + .send({ + name: 'Variable Dashboard', + tiles: [makeExternalTile({ sourceId: '$traces' }), rawSqlTile], + tags: [], + filters: [globalConnectionFilter, dataSourceFilter], + savedFilterValues, + }) + .expect(200); + + const fetched = await authRequest( + 'get', + `${BASE_URL}/${created.body.data.id}`, + ).expect(200); + expect(fetched.body.data.filters).toEqual([ + expect.objectContaining(globalConnectionFilter), + expect.objectContaining(dataSourceFilter), + ]); + expect(fetched.body.data.tiles.map(t => t.config)).toEqual([ + expect.objectContaining({ sourceId: '$traces' }), + expect.objectContaining({ connectionId: '$conn' }), + ]); + expect(fetched.body.data.savedFilterValues).toEqual(savedFilterValues); + + const echoed = await authRequest( + 'put', + `${BASE_URL}/${created.body.data.id}`, + ) + .send(fetched.body.data) + .expect(200); + expect(echoed.body.data.filters).toEqual(fetched.body.data.filters); + expect(echoed.body.data.tiles.map(t => t.config)).toEqual( + fetched.body.data.tiles.map(t => t.config), + ); + }); + + it('rejects a tile referencing an undeclared variable', async () => { + const res = await authRequest('post', BASE_URL) + .send({ + name: 'Variable Dashboard', + tiles: [makeExternalTile({ sourceId: '$traces' })], + tags: [], + }) + .expect(400); + expect(res.body.message).toMatch(/does not name a data source variable/); + }); + + it('rejects a connection reference to an undeclared connection variable', async () => { + const res = await authRequest('post', BASE_URL) + .send({ + name: 'Variable Dashboard', + tiles: [rawSqlTile], + tags: [], + filters: [{ ...dataSourceFilter, connectionId: undefined }], + }) + .expect(400); + expect(res.body.message).toMatch(/does not name a connection variable/); + }); + + it('accepts a PromQL tile and PROMETHEUS_LABEL filter on a PromQL data source variable', async () => { + await authRequest('post', BASE_URL) + .send({ + name: 'Variable Dashboard', + tiles: [ + { + name: 'PromQL', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + configType: 'promql', + displayType: 'line', + sourceId: '$prom', + expressions: [{ expression: 'up' }], + }, + }, + ], + tags: [], + filters: [ + { + type: 'DATA_SOURCE', + name: 'PromQL source', + variableName: 'prom', + allowedSourceKinds: [SourceKind.Promql], + }, + { + type: 'PROMETHEUS_LABEL', + name: 'Job', + sourceId: '$prom', + label: 'job', + }, + ], + }) + .expect(200); + }); + + it('accepts an ADHOC filter whose sources reference a data source variable', async () => { + const adhocFilter = { + type: 'ADHOC', + name: 'Conditions', + sourceType: 'sql', + sources: ['$traces'], + appliesToSourceIds: ['$traces'], + }; + await authRequest('post', BASE_URL) + .send({ + name: 'Variable Dashboard', + tiles: [], + tags: [], + filters: [omit(dataSourceFilter, 'connectionId'), adhocFilter], + }) + .expect(200); + + const res = await authRequest('post', BASE_URL) + .send({ + name: 'Variable Dashboard', + tiles: [], + tags: [], + filters: [adhocFilter], + }) + .expect(400); + expect(res.body.message).toMatch(/does not name a data source variable/); + }); + + it('accepts a raw SQL tile with a fixed source on a connection variable', async () => { + await authRequest('post', BASE_URL) + .send({ + name: 'Variable Dashboard', + tiles: [ + { + ...rawSqlTile, + config: { + ...rawSqlTile.config, + sourceId: traceSource._id.toString(), + }, + }, + ], + tags: [], + filters: [connectionFilter], + }) + .expect(200); + }); + + it("rejects a raw SQL tile whose data source variable is limited to another connection than the tile's", async () => { + const otherConnection = await Connection.create({ + team: team._id, + name: 'Other Connection', + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + + const res = await authRequest('post', BASE_URL) + .send({ + name: 'Variable Dashboard', + tiles: [ + { + ...rawSqlTile, + config: { + ...rawSqlTile.config, + connectionId: connection._id.toString(), + sourceId: '$traces', + }, + }, + ], + tags: [], + filters: [ + { + ...dataSourceFilter, + connectionId: otherConnection._id.toString(), + }, + ], + }) + .expect(400); + expect(res.body.message).toBe( + 'The following source IDs do not match the specified connections: $traces', + ); + }); + + it('checks a PUT without filters against the stored variables', async () => { + const dashboard = await createTestDashboard({ + filters: [ + { id: new ObjectId().toString(), ...connectionFilter }, + { + id: new ObjectId().toString(), + ...omit(dataSourceFilter, 'connectionId'), + connection: '$conn', + }, + ], + }); + + await authRequest('put', `${BASE_URL}/${dashboard._id}`) + .send({ + name: 'Variable Dashboard', + tiles: [makeExternalTile({ sourceId: '$traces' })], + tags: [], + }) + .expect(200); + }); + + it('deletes the alert of a tile that moves to a data source variable', async () => { + const tile = makeExternalTile({ sourceId: traceSource._id.toString() }); + const created = await authRequest('post', BASE_URL) + .send({ + name: 'Variable Dashboard', + tiles: [tile], + tags: [], + filters: [connectionFilter, dataSourceFilter], + }) + .expect(200); + const { id: dashboardId, tiles } = created.body.data; + const webhook = await Webhook.create({ + name: 'Test Webhook', + service: WebhookService.Slack, + url: 'https://hooks.slack.com/test', + team: team._id, + }); + const alert = await Alert.create({ + team: team._id, + dashboard: dashboardId, + tileId: tiles[0].id, + source: AlertSource.TILE, + threshold: 100, + interval: '1h', + thresholdType: AlertThresholdType.ABOVE, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + }); + + await authRequest('put', `${BASE_URL}/${dashboardId}`) + .send({ + ...created.body.data, + tiles: [ + { + ...tiles[0], + config: { ...tiles[0].config, sourceId: '$traces' }, + }, + ], + }) + .expect(200); + + expect(await Alert.findById(alert._id)).toBeNull(); + }); + }); + describe('ad hoc filters', () => { const adhocFilter = () => ({ id: new ObjectId().toString(), diff --git a/packages/api/src/routers/external-api/v2/dashboards.ts b/packages/api/src/routers/external-api/v2/dashboards.ts index 0b6bacea0..e37a2889f 100644 --- a/packages/api/src/routers/external-api/v2/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/dashboards.ts @@ -705,7 +705,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "line" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -792,7 +792,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "stacked_bar" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -862,7 +862,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "stacked_line" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -938,7 +938,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "table" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -1012,7 +1012,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "number" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -1071,7 +1071,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "pie" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -1124,7 +1124,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "bar" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -1217,7 +1217,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "distribution" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -1265,7 +1265,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "series" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: array @@ -1314,7 +1314,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "search" * sourceId: * type: string - * description: ID of the data source to query. + * description: ID of the data source to query, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: string @@ -1345,7 +1345,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "event_patterns" * sourceId: * type: string - * description: ID of the data source to mine patterns from. + * description: ID of the data source to mine patterns from, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. * example: "65f5e4a3b9e77c001a111111" * select: * type: string @@ -1395,7 +1395,9 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "sql" * connectionId: * type: string - * description: ID of the ClickHouse connection to execute the query against. + * description: | + * ID of the ClickHouse connection to execute the query against, or + * `$connectionVariable` of a CONNECTION filter on the dashboard. * example: "65f5e4a3b9e77c001a567890" * sqlTemplate: * type: string @@ -1404,7 +1406,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "SELECT count() FROM otel_logs WHERE timestamp > now() - INTERVAL 1 HOUR" * sourceId: * type: string - * description: Optional ID of the data source associated with this Raw SQL chart. Used for applying dashboard filters. + * description: Optional ID of the data source associated with this Raw SQL chart, or `$sourceVariable` of a DATA_SOURCE filter on the dashboard. Used for applying dashboard filters. * example: "65f5e4a3b9e77c001a567890" * numberFormat: * $ref: '#/components/schemas/NumberFormat' @@ -2420,6 +2422,8 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * - $ref: '#/components/schemas/PrometheusLabelFilterInput' * - $ref: '#/components/schemas/SqlQueryFilterInput' * - $ref: '#/components/schemas/AdhocFilterInput' + * - $ref: '#/components/schemas/DataSourceFilterInput' + * - $ref: '#/components/schemas/ConnectionFilterInput' * discriminator: * propertyName: type * mapping: @@ -2428,6 +2432,96 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * PROMETHEUS_LABEL: '#/components/schemas/PrometheusLabelFilterInput' * SQL_QUERY: '#/components/schemas/SqlQueryFilterInput' * ADHOC: '#/components/schemas/AdhocFilterInput' + * DATA_SOURCE: '#/components/schemas/DataSourceFilterInput' + * CONNECTION: '#/components/schemas/ConnectionFilterInput' + * + * DataSourceFilterInput: + * type: object + * description: | + * A variable whose single, required value is a source. Tiles reference it + * as `$variableName` in their sourceId, and other filters in sourceId or + * appliesToSourceIds, so viewers can switch the source those read. Tiles + * that use it do not load until a source is selected, and cannot have + * alerts. + * required: + * - type + * - name + * - allowedSourceKinds + * properties: + * type: + * type: string + * enum: [DATA_SOURCE] + * description: Filter type discriminator. Must be "DATA_SOURCE". + * example: "DATA_SOURCE" + * name: + * type: string + * minLength: 1 + * description: Display name of the variable's dropdown + * example: "Logs source" + * variableName: + * type: string + * maxLength: 64 + * pattern: '^[a-zA-Z][a-zA-Z0-9_]*$' + * description: Token tiles and filters reference the selected source by, as `$variableName`. + * example: "logs" + * isGlobalRequirement: + * type: boolean + * default: false + * description: | + * Blocks every tile on the dashboard until a value is selected. False + * (the default) blocks only the tiles that reference the variable. + * example: true + * allowedSourceKinds: + * type: array + * minItems: 1 + * items: + * type: string + * enum: [log, trace, session, metric, promql] + * description: | + * Kinds of sources offered. A field can only reference the variable + * if it accepts every listed kind. + * example: ["log"] + * connectionId: + * type: string + * description: | + * Optional connection the offered sources are limited to: a + * connection ID, or `$name` of a CONNECTION filter on the dashboard. + * example: "$conn" + * + * ConnectionFilterInput: + * type: object + * description: | + * A variable whose single, required value is a connection. Raw SQL tiles + * reference it as `$variableName` in their connectionId, and data source + * variables in theirs. Tiles that use it do not load until a connection is + * selected, and cannot have alerts. + * required: + * - type + * - name + * properties: + * type: + * type: string + * enum: [CONNECTION] + * description: Filter type discriminator. Must be "CONNECTION". + * example: "CONNECTION" + * name: + * type: string + * minLength: 1 + * description: Display name of the variable's dropdown + * example: "Connection" + * variableName: + * type: string + * maxLength: 64 + * pattern: '^[a-zA-Z][a-zA-Z0-9_]*$' + * description: Token raw SQL tiles reference the selected connection by, as `$variableName`. + * example: "conn" + * isGlobalRequirement: + * type: boolean + * default: false + * description: | + * Blocks every tile on the dashboard until a value is selected. False + * (the default) blocks only the tiles that reference the variable. + * example: true * * QueryExpressionFilterInput: * type: object @@ -2458,7 +2552,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "environment" * sourceId: * type: string - * description: Source ID this dashboard filter key applies to + * description: Source ID this dashboard filter key applies to, or `$name` of a DATA_SOURCE filter on the dashboard * example: "65f5e4a3b9e77c001a111111" * sourceMetricType: * type: string @@ -2854,7 +2948,7 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * minItems: 1 * items: * type: string - * description: IDs of the sources whose keys and values the filter suggests, all of sourceType. + * description: IDs of the sources whose keys and values the filter suggests, all of sourceType, or `$name` of data source variables. * example: ["65f5e4a3b9e77c001a111111"] * where: * type: string @@ -2874,8 +2968,8 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * type: string * description: | * IDs of the sources whose tiles the conditions apply to, all of - * sourceType. Omit or send an empty array to apply to the tiles of - * the filter's sources. + * sourceType, or `$name` of data source variables. Omit or send an + * empty array to apply to the tiles of the filter's sources. * example: ["65f5e4a3b9e77c001a111111"] * isBroadcastEnabled: * type: boolean diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index e48e86cff..1b379c6e0 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -7,6 +7,7 @@ import { } from '@hyperdx/common-utils/dist/core/utils'; import { collectRefinementIssues, + getVariableReferenceError, validateDashboardContainersStructure, validateDashboardFilterFieldGating, validateDashboardFilterModes, @@ -25,6 +26,10 @@ import { isPromqlSavedChartConfig, isRawSqlSavedChartConfig, } from '@hyperdx/common-utils/dist/guards'; +import { + getSourceVariableDeclarations, + parseVariableReference, +} from '@hyperdx/common-utils/dist/referenceVariables'; import { AggregateFunctionSchema, BuilderSavedChartConfig, @@ -58,7 +63,6 @@ import { getConnectionsByTeam } from '@/controllers/connection'; import { getSources } from '@/controllers/sources'; import Dashboard, { DashboardDocument } from '@/models/dashboard'; import { - getExternalCompatibleFilters, translateExternalChartToTileConfig, translateExternalFilterToFilter, translateFilterToExternalFilter, @@ -813,7 +817,7 @@ export function convertToExternalDashboard( convertTileToExternalChart(tile, containerById, dashboardId), ), tags: dashboard.tags || [], - filters: getExternalCompatibleFilters(dashboard.filters) + filters: (dashboard.filters ?? []) .filter(filter => !(hidePromql && isPromqlFilter(filter))) .map(translateFilterToExternalFilter), savedQuery: dashboard.savedQuery ?? null, @@ -1329,7 +1333,12 @@ function getMissingSources( const existingSourceIds = new Set( sources.map(source => source._id.toString()), ); - return [...sourceIds].filter(sourceId => !existingSourceIds.has(sourceId)); + // Variable references are checked by validateDashboardVariableReferences + return [...sourceIds].filter( + sourceId => + parseVariableReference(sourceId) == null && + !existingSourceIds.has(sourceId), + ); } /** @@ -1587,16 +1596,18 @@ async function getMissingConnections( filters: (ExternalDashboardFilter | ExternalDashboardFilterWithId)[] = [], ): Promise { const connectionIds = new Set(); + // Variable references are checked by validateDashboardVariableReferences + const addConnectionId = (id: string | undefined) => { + if (id && parseVariableReference(id) == null) connectionIds.add(id); + }; for (const tile of tiles) { if (isConfigTile(tile) && isRawSqlExternalTileConfig(tile.config)) { - connectionIds.add(tile.config.connectionId); + addConnectionId(tile.config.connectionId); } } for (const filter of filters) { - if ('connectionId' in filter) { - connectionIds.add(filter.connectionId); - } + if ('connectionId' in filter) addConnectionId(filter.connectionId); } if (connectionIds.size === 0) return []; @@ -1699,8 +1710,22 @@ export function convertExternalFiltersToInternal( function getSourceConnectionMismatches( sources: SourceForValidation[], tiles: ExternalDashboardTileWithId[], + filters: DashboardFilter[], ): string[] { const sourceById = new Map(sources.map(s => [s._id.toString(), s])); + const variableConnectionByName = new Map( + getSourceVariableDeclarations(filters).map(({ name, connection }) => [ + name, + connection, + ]), + ); + // A source variable's sources are on its declared connection, if any + const getSourceConnection = (sourceId: string) => { + const name = parseVariableReference(sourceId); + return name == null + ? sourceById.get(sourceId)?.connection.toString() + : variableConnectionByName.get(name); + }; const mismatched: string[] = []; for (const tile of tiles) { @@ -1709,9 +1734,17 @@ function getSourceConnectionMismatches( isRawSqlExternalTileConfig(tile.config) && tile.config.sourceId ) { - const source = sourceById.get(tile.config.sourceId); - if (source && source.connection.toString() !== tile.config.connectionId) { - mismatched.push(tile.config.sourceId); + const { connectionId, sourceId } = tile.config; + const sourceConnection = getSourceConnection(sourceId); + // A connection reference is only known once selected, so compare two + // IDs, or two references, but not one of each + if ( + sourceConnection != null && + (parseVariableReference(sourceConnection) == null) === + (parseVariableReference(connectionId) == null) && + sourceConnection !== connectionId + ) { + mismatched.push(sourceId); } } } @@ -1728,7 +1761,11 @@ export type TileValidationContext = { filters?: (ExternalDashboardFilter | ExternalDashboardFilterWithId)[]; /** Existing internal tiles for scoping heatmap change detection (update paths). */ existingTiles?: DashboardDocument['tiles']; - /** Existing filters, whose source references aren't re-checked (update paths). */ + /** + * Existing internal filters (update paths). Their source references aren't + * re-checked, and tile variable references are checked against them when + * the request carries no `filters`. + */ existingFilters?: DashboardFilter[]; /** Container set to validate tile containerId/tabId refs against. */ containers: DashboardContainer[]; @@ -1755,6 +1792,17 @@ export async function validateDashboardTiles( return tileRefIssues.join('; '); } + const referenceFilters = filters + ? convertExternalFiltersToInternal(filters) + : (existingFilters ?? []); + const variableReferenceError = getVariableReferenceError( + convertExternalTilesToInternal(tiles), + referenceFilters, + ); + if (variableReferenceError) { + return variableReferenceError; + } + // Fetch sources/connections/onClick targets in parallel. const [ sources, @@ -1779,6 +1827,7 @@ export async function validateDashboardTiles( const sourceConnectionMismatches = getSourceConnectionMismatches( sources, tiles, + referenceFilters, ); if (sourceConnectionMismatches.length > 0) { return `The following source IDs do not match the specified connections: ${sourceConnectionMismatches.join(', ')}`; @@ -1818,10 +1867,10 @@ export async function validateDashboardTiles( // PromQL tiles query through their source's connection and TimeSeries // table, which only a PromQL source carries. const nonPromqlTileSourceIds = getInvalidSourceIds( - getChangedPromqlTileSourceIds(tiles, existingTiles).map(id => ({ - id, - kinds: [SourceKind.Promql], - })), + getChangedPromqlTileSourceIds(tiles, existingTiles) + // A reference's kinds are checked by validateDashboardVariableReferences + .filter(id => parseVariableReference(id) == null) + .map(id => ({ id, kinds: [SourceKind.Promql] })), sources, ); if (nonPromqlTileSourceIds.length > 0) { @@ -1838,7 +1887,7 @@ export async function validateDashboardTiles( translateExternalFilterToFilter({ id: '', ...filter }), ), existingFilters, - ), + ).filter(ref => parseVariableReference(ref.id) == null), sources, ); if (filterSourceError != null) return filterSourceError; diff --git a/packages/api/src/utils/externalApi.ts b/packages/api/src/utils/externalApi.ts index 91f627748..164e59d7f 100644 --- a/packages/api/src/utils/externalApi.ts +++ b/packages/api/src/utils/externalApi.ts @@ -3,7 +3,6 @@ import { isFilterBroadcastEnabled, isFilterRequired, isFilterVariableEnabled, - isReferenceVariableFilter, } from '@hyperdx/common-utils/dist/filters'; import { AlertErrorType, @@ -239,24 +238,8 @@ export function translateExternalChartToTileConfig( }; } -/** Filter types the external API and MCP don't expose yet. */ -export type ExternalCompatibleDashboardFilter = Exclude< - DashboardFilter, - { type: 'DATA_SOURCE' | 'CONNECTION' } ->; - -/** The filters the external API and MCP may read. */ -export function getExternalCompatibleFilters( - filters: DashboardFilter[] | undefined, -): ExternalCompatibleDashboardFilter[] { - return (filters ?? []).filter( - (filter): filter is ExternalCompatibleDashboardFilter => - !isReferenceVariableFilter(filter), - ); -} - export function translateFilterToExternalFilter( - filter: ExternalCompatibleDashboardFilter, + filter: DashboardFilter, ): ExternalDashboardFilterWithId { // Omitted from responses unless the filter is required, so a GET response // can be PUT back verbatim. @@ -286,6 +269,15 @@ export function translateFilterToExternalFilter( connectionId: filter.connection.toString(), }; + case 'DATA_SOURCE': + return { + ...omit(filter, 'connection'), + ...(filter.connection ? { connectionId: filter.connection } : {}), + }; + + case 'CONNECTION': + return filter; + case 'QUERY_EXPRESSION': { // Ignore variableName and appliesToSourceIds if the filter is not in a mode that uses them const ignoredKeys = [ @@ -327,6 +319,15 @@ export function translateExternalFilterToFilter( case 'QUERY_EXPRESSION': return { ...omit(filter, 'sourceId'), source: filter.sourceId }; + case 'DATA_SOURCE': + return { + ...omit(filter, 'connectionId'), + ...(filter.connectionId ? { connection: filter.connectionId } : {}), + }; + + case 'CONNECTION': + return filter; + default: filter satisfies never; return filter; diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index 05cd5c23d..09bd5ab19 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -1,4 +1,5 @@ import { validateChartConfigFormulas } from '@hyperdx/common-utils/dist/dashboardValidation'; +import { parseVariableReference } from '@hyperdx/common-utils/dist/referenceVariables'; import { addDuplicateTileIdIssues, AdhocDashboardFilterSchema, @@ -12,8 +13,10 @@ import { AlertThresholdType, BackgroundChartSchema, ChartPaletteTokenSchema, + ConnectionDashboardFilterSchema, DASHBOARD_CONTAINER_ID_MAX, DASHBOARD_MAX_TILES, + DataSourceDashboardFilterSchema, HeatmapScaleTypeSchema, LineInterpolationSchema, MAX_LEGEND_TEMPLATE_LENGTH, @@ -54,6 +57,17 @@ export const objectIdSchema = z.string().refine(val => { return Types.ObjectId.isValid(val); }, 'Invalid ObjectId'); +/** + * A source or connection field: an ObjectId, or a `$name` reference to the + * dashboard's data source or connection variable of that name. + */ +export const objectIdOrVariableReferenceSchema = z + .string() + .refine( + val => Types.ObjectId.isValid(val) || parseVariableReference(val) != null, + 'Invalid ObjectId or $variable reference', + ); + /** * A repeatable query param. Express's parser yields a bare string for a single * occurrence and an array for several, and `?p[]=x` lands on the same key as @@ -175,7 +189,7 @@ export { tagsSchema }; const externalQueryExpressionFilterShape = QueryExpressionDashboardFilterSchema.omit({ source: true }).extend({ - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, }); // A static filter's mode flags each accept exactly one value, so callers may @@ -188,7 +202,7 @@ const externalStaticListFilterShape = StaticListDashboardFilterSchema.extend({ const externalPromqlLabelFilterShape = PromqlLabelDashboardFilterSchema.omit({ source: true, }).extend({ - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, isBroadcastEnabled: z.literal(false).default(false), isVariableEnabled: z.literal(true).default(true), }); @@ -204,12 +218,32 @@ const externalSqlQueryFilterShape = SqlQueryDashboardFilterSchema.omit({ // Field names match the stored filter, so it needs no translation. const externalAdhocFilterShape = AdhocDashboardFilterSchema.extend({ - sources: z.array(objectIdSchema).min(1), - appliesToSourceIds: z.array(objectIdSchema).optional(), + sources: z.array(objectIdOrVariableReferenceSchema).min(1), + appliesToSourceIds: z.array(objectIdOrVariableReferenceSchema).optional(), isBroadcastEnabled: z.literal(true).default(true), isVariableEnabled: z.literal(true).default(true), }); +// Reference variables are always single-select and required, so these +// fields, like the mode flags, accept one value each and may be omitted. +const referenceVariableFixedFields = { + minSelections: z.literal(1).default(1), + maxSelections: z.literal(1).default(1), + isBroadcastEnabled: z.literal(false).default(false), + isVariableEnabled: z.literal(true).default(true), +}; + +const externalDataSourceFilterShape = DataSourceDashboardFilterSchema.omit({ + connection: true, +}).extend({ + connectionId: objectIdOrVariableReferenceSchema.optional(), + ...referenceVariableFixedFields, +}); + +const externalConnectionFilterShape = ConnectionDashboardFilterSchema.extend( + referenceVariableFixedFields, +); + export const externalDashboardFilterSchemaWithId = z.discriminatedUnion( 'type', [ @@ -218,6 +252,8 @@ export const externalDashboardFilterSchemaWithId = z.discriminatedUnion( externalPromqlLabelFilterShape.strict(), externalSqlQueryFilterShape.strict(), externalAdhocFilterShape.strict(), + externalDataSourceFilterShape.strict(), + externalConnectionFilterShape.strict(), ], ); @@ -236,6 +272,8 @@ export const externalDashboardFilterSchema = z.discriminatedUnion('type', [ externalPromqlLabelFilterShape.omit({ id: true }).strict(), externalSqlQueryFilterShape.omit({ id: true }).strict(), externalAdhocFilterShape.omit({ id: true }).strict(), + externalDataSourceFilterShape.omit({ id: true }).strict(), + externalConnectionFilterShape.omit({ id: true }).strict(), ]); export type ExternalDashboardFilter = z.infer< @@ -356,9 +394,9 @@ export type ExternalDashboardSelectItem = z.infer< const externalDashboardRawSqlChartConfigBaseSchema = z.object({ configType: z.literal('sql'), - connectionId: objectIdSchema, + connectionId: objectIdOrVariableReferenceSchema, sqlTemplate: z.string().max(100000), - sourceId: objectIdSchema.optional(), + sourceId: objectIdOrVariableReferenceSchema.optional(), numberFormat: NumberFormatSchema.optional(), }); @@ -381,7 +419,7 @@ const externalFormulasSchema = z.array(MetricFormulaSchema).max(10).optional(); const externalShowOperandSeriesSchema = z.boolean().optional(); const externalDashboardTimeChartConfigSchema = z.object({ - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, select: z.array(externalDashboardSelectItemSchema).min(1).max(20), groupBy: z.string().max(10000).optional(), asRatio: z.boolean().optional(), @@ -452,7 +490,7 @@ const externalDashboardStackedLineRawSqlChartConfigSchema = const externalDashboardTableChartConfigSchema = z.object({ displayType: z.literal('table'), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, select: z.array(externalDashboardSelectItemSchema).min(1).max(20), groupBy: z.string().max(10000).optional(), having: z.string().max(10000).optional(), @@ -492,7 +530,7 @@ const externalDashboardCategoricalBarRawSqlChartConfigSchema = const externalDashboardNumberChartConfigSchema = z.object({ displayType: z.literal('number'), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, // Exactly one select item unless `formulas` is set: a formula number tile // displays the (single) formula value and its select entries are the // formula's operands (e.g. A / (A + B)). The one-item rule for @@ -532,7 +570,7 @@ const externalDashboardNumberChartConfigSchema = z.object({ const externalDashboardPieChartConfigSchema = z.object({ displayType: z.literal('pie'), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, select: z.array(externalDashboardSelectItemSchema).length(1), groupBy: z.string().max(10000).optional(), orderBy: z.string().max(10000).optional(), @@ -543,7 +581,7 @@ const externalDashboardPieChartConfigSchema = z.object({ const externalDashboardCategoricalBarChartConfigSchema = z.object({ displayType: z.literal('bar'), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, select: z.array(externalDashboardSelectItemSchema).length(1), groupBy: z.string().max(10000).optional(), orderBy: z.string().max(10000).optional(), @@ -580,7 +618,7 @@ export type ExternalDashboardHeatmapSelectItem = z.infer< const externalDashboardHeatmapChartConfigSchema = z.object({ displayType: z.literal('heatmap'), heatmapMode: z.literal('distribution').optional(), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, select: z.array(externalDashboardHeatmapSelectItemSchema).length(1), where: z.string().max(10000).optional().default(''), // `whereLanguageSchema` (an alias for `SearchConditionLanguageSchema`) @@ -594,7 +632,7 @@ const externalDashboardHeatmapChartConfigSchema = z.object({ const externalDashboardHeatmapSeriesChartConfigSchema = z.object({ displayType: z.literal('heatmap'), heatmapMode: z.literal('series'), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, select: z.array(externalDashboardSelectItemSchema).length(1), groupBy: z.string().max(10000).optional(), numberFormat: NumberFormatSchema.optional(), @@ -602,7 +640,7 @@ const externalDashboardHeatmapSeriesChartConfigSchema = z.object({ const externalDashboardSearchChartConfigSchema = z.object({ displayType: z.literal('search'), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, select: z.string().max(10000), where: z.string().max(10000).optional().default(''), whereLanguage: whereLanguageSchema, @@ -626,7 +664,7 @@ const externalDashboardMarkdownChartConfigSchema = z.object({ const externalDashboardEventPatternsChartConfigSchema = z.object({ displayType: z.literal('event_patterns'), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, select: z.string().max(10000).optional().default(''), where: z.string().max(10000).optional().default(''), whereLanguage: whereLanguageSchema, @@ -720,7 +758,7 @@ const externalPromqlLegendTemplateSchema = z const externalPromqlChartConfigBaseSchema = z.object({ configType: z.literal('promql'), - sourceId: objectIdSchema, + sourceId: objectIdOrVariableReferenceSchema, // Sets the Prometheus step and `$__interval` of range queries. granularity: z.union([SQLIntervalSchema, z.literal('auto')]).optional(), numberFormat: NumberFormatSchema.optional(), From 6d96f92e0d0acc8c86c5b4c7dbd6ce50379bb3a4 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Mon, 16 Nov 2026 14:24:51 -0500 Subject: [PATCH 05/18] fix: check only the variable references an external API or MCP dashboard update adds --- .../dashboards/patchDashboard.int.test.ts | 10 +++ .../api/src/mcp/tools/dashboards/schemas.ts | 4 +- .../__tests__/dashboards.int.test.ts | 90 +++++++++++++++++++ .../external-api/v2/utils/dashboards.ts | 36 +++++++- 4 files changed, 135 insertions(+), 5 deletions(-) diff --git a/packages/api/src/mcp/__tests__/dashboards/patchDashboard.int.test.ts b/packages/api/src/mcp/__tests__/dashboards/patchDashboard.int.test.ts index 21e4e6b12..1285ec729 100644 --- a/packages/api/src/mcp/__tests__/dashboards/patchDashboard.int.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/patchDashboard.int.test.ts @@ -1471,5 +1471,15 @@ describe('MCP Dashboard Tools - clickstack_patch_dashboard', () => { 'does not name a data source variable', ); }); + + it('accepts a patch that keeps a reference the dashboard no longer satisfies', async () => { + const dashboard = await createDashboard(); + await patchSource(dashboard._id.toString(), '$traces'); + await Dashboard.updateOne({ _id: dashboard._id }, { filters: [] }); + + const result = await patchSource(dashboard._id.toString(), '$traces'); + + expect(result.isError).toBeFalsy(); + }); }); }); diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index 29e504a37..5bc7dda1b 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -49,8 +49,8 @@ import { const SOURCE_VARIABLE_REFERENCE_NOTE = 'May instead be `$name`, a reference to a DATA_SOURCE filter (variable) on this ' + - 'dashboard, so viewers can switch the source; the variable must only offer kinds ' + - 'this field accepts.'; + 'dashboard, so viewers can switch the source; the variable must offer at least ' + + 'one kind this field accepts.'; const SOURCE_ID_DESCRIPTION = `Source ID - call clickstack_list_sources. ${SOURCE_VARIABLE_REFERENCE_NOTE}`; diff --git a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts index 57d2e35be..38a9e736c 100644 --- a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts +++ b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts @@ -8752,6 +8752,96 @@ describe('External API v2 Dashboards - new format', () => { .expect(200); }); + // Only references an update adds are checked, so editing a variable + // doesn't fail on the tiles that already use it. + it('accepts a PUT whose filter change leaves stored references unsatisfied', async () => { + const variable = { ...dataSourceFilter, connectionId: undefined }; + const created = await authRequest('post', BASE_URL) + .send({ + name: 'Variable Dashboard', + tiles: [makeExternalTile({ sourceId: '$traces' })], + tags: [], + filters: [variable], + }) + .expect(200); + const fetched = await authRequest( + 'get', + `${BASE_URL}/${created.body.data.id}`, + ).expect(200); + const url = `${BASE_URL}/${created.body.data.id}`; + const [storedVariable] = fetched.body.data.filters; + + await authRequest('put', url) + .send({ + ...fetched.body.data, + filters: [ + { ...storedVariable, allowedSourceKinds: [SourceKind.Promql] }, + ], + }) + .expect(200); + await authRequest('put', url) + .send({ ...fetched.body.data, filters: [] }) + .expect(200); + + const res = await authRequest('put', url) + .send({ + ...fetched.body.data, + filters: [], + tiles: [ + ...fetched.body.data.tiles, + makeExternalTile({ sourceId: '$nope' }), + ], + }) + .expect(400); + expect(res.body.message).toMatch(/does not name a data source variable/); + }); + + it("accepts a PUT that moves a raw SQL tile's data source variable to another connection", async () => { + const otherConnection = await Connection.create({ + team: team._id, + name: 'Other Connection', + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + const created = await authRequest('post', BASE_URL) + .send({ + name: 'Variable Dashboard', + tiles: [ + { + ...rawSqlTile, + config: { + ...rawSqlTile.config, + connectionId: connection._id.toString(), + sourceId: '$traces', + }, + }, + ], + tags: [], + filters: [ + { ...dataSourceFilter, connectionId: connection._id.toString() }, + ], + }) + .expect(200); + const fetched = await authRequest( + 'get', + `${BASE_URL}/${created.body.data.id}`, + ).expect(200); + const [storedVariable] = fetched.body.data.filters; + + await authRequest('put', `${BASE_URL}/${created.body.data.id}`) + .send({ + ...fetched.body.data, + filters: [ + { + ...storedVariable, + connectionId: otherConnection._id.toString(), + }, + ], + }) + .expect(200); + }); + it('deletes the alert of a tile that moves to a data source variable', async () => { const tile = makeExternalTile({ sourceId: traceSource._id.toString() }); const created = await authRequest('post', BASE_URL) diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index 1b379c6e0..fbde606d5 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -1711,7 +1711,11 @@ function getSourceConnectionMismatches( sources: SourceForValidation[], tiles: ExternalDashboardTileWithId[], filters: DashboardFilter[], + existingTiles: DashboardDocument['tiles'] = [], ): string[] { + const existingConfigById = new Map( + existingTiles.map(tile => [tile.id, tile.config]), + ); const sourceById = new Map(sources.map(s => [s._id.toString(), s])); const variableConnectionByName = new Map( getSourceVariableDeclarations(filters).map(({ name, connection }) => [ @@ -1735,6 +1739,19 @@ function getSourceConnectionMismatches( tile.config.sourceId ) { const { connectionId, sourceId } = tile.config; + // Like other variable references, one the update leaves as it was + // isn't checked again; the dashboard shows the mismatch instead + const existingConfig = tile.id + ? existingConfigById.get(tile.id) + : undefined; + if ( + parseVariableReference(sourceId) != null && + existingConfig != null && + Reflect.get(existingConfig, 'source') === sourceId && + Reflect.get(existingConfig, 'connection') === connectionId + ) { + continue; + } const sourceConnection = getSourceConnection(sourceId); // A connection reference is only known once selected, so compare two // IDs, or two references, but not one of each @@ -1759,7 +1776,10 @@ export type TileValidationContext = { tiles: ExternalDashboardTileWithId[]; /** Filters to check for missing source IDs (create/full-update paths). */ filters?: (ExternalDashboardFilter | ExternalDashboardFilterWithId)[]; - /** Existing internal tiles for scoping heatmap change detection (update paths). */ + /** + * Existing internal tiles for scoping heatmap change detection and the + * variable reference check (update paths). + */ existingTiles?: DashboardDocument['tiles']; /** * Existing internal filters (update paths). Their source references aren't @@ -1792,12 +1812,21 @@ export async function validateDashboardTiles( return tileRefIssues.join('; '); } + // Stored IDs are kept so that the references an update leaves as they were + // can be told apart from the ones it adds, which are the only ones checked const referenceFilters = filters - ? convertExternalFiltersToInternal(filters) + ? convertExternalFiltersToInternal( + filters, + new Set(existingFilters?.map(filter => filter.id)), + ) : (existingFilters ?? []); const variableReferenceError = getVariableReferenceError( - convertExternalTilesToInternal(tiles), + convertExternalTilesToInternal( + tiles, + new Set(existingTiles?.map(tile => tile.id)), + ), referenceFilters, + existingTiles && { tiles: existingTiles, filters: existingFilters }, ); if (variableReferenceError) { return variableReferenceError; @@ -1828,6 +1857,7 @@ export async function validateDashboardTiles( sources, tiles, referenceFilters, + existingTiles, ); if (sourceConnectionMismatches.length > 0) { return `The following source IDs do not match the specified connections: ${sourceConnectionMismatches.join(', ')}`; From e0ba99b835a53a5e30748275a5fd0878b8ac6407 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Tue, 17 Nov 2026 10:34:59 -0500 Subject: [PATCH 06/18] fix: Check source var kinds against tile --- packages/api/openapi.json | 2 +- .../__tests__/dashboards/variables.test.ts | 141 +++++++++++++++++- .../api/src/mcp/tools/dashboards/schemas.ts | 7 +- .../src/mcp/tools/dashboards/validation.ts | 2 +- .../api/src/mcp/tools/dashboards/variables.ts | 72 ++++++++- .../src/routers/external-api/v2/dashboards.ts | 5 +- .../app/src/components/ChartEditor/utils.ts | 85 ++--------- .../common-utils/src/__tests__/guards.test.ts | 46 ++++++ packages/common-utils/src/guards.ts | 88 +++++++++++ 9 files changed, 366 insertions(+), 82 deletions(-) diff --git a/packages/api/openapi.json b/packages/api/openapi.json index 9f1e13056..95979cd65 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -4101,7 +4101,7 @@ "promql" ] }, - "description": "Kinds of sources offered. A field can only reference the variable\nif it accepts every listed kind.\n", + "description": "Kinds of sources offered. A field can reference the variable if it\naccepts at least one listed kind. A tile whose selected source is a\nkind it cannot use shows an error instead of running.\n", "example": [ "log" ] diff --git a/packages/api/src/mcp/__tests__/dashboards/variables.test.ts b/packages/api/src/mcp/__tests__/dashboards/variables.test.ts index 45270e65e..9d5e0e45a 100644 --- a/packages/api/src/mcp/__tests__/dashboards/variables.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/variables.test.ts @@ -332,12 +332,25 @@ describe('withResolvedFilterVariableNames', () => { describe('resolveTileVariableReferences', () => { const objectId = () => new mongoose.Types.ObjectId().toString(); const connectionId = objectId(); + const otherConnectionId = objectId(); const traceSourceId = objectId(); + const logSourceId = objectId(); + const metricSourceId = objectId(); + const promqlSourceId = objectId(); + const otherConnectionLogSourceId = objectId(); const targets: ReferenceTargets = { sources: [ { id: traceSourceId, kind: SourceKind.Trace, connection: connectionId }, + { id: logSourceId, kind: SourceKind.Log, connection: connectionId }, + { id: metricSourceId, kind: SourceKind.Metric, connection: connectionId }, + { id: promqlSourceId, kind: SourceKind.Promql, connection: connectionId }, + { + id: otherConnectionLogSourceId, + kind: SourceKind.Log, + connection: otherConnectionId, + }, ], - connectionIds: new Set([connectionId]), + connectionIds: new Set([connectionId, otherConnectionId]), }; const referenceVariableFields = { @@ -365,6 +378,17 @@ describe('resolveTileVariableReferences', () => { connectionId: connection, }); + const sourceVariable = ( + allowedSourceKinds: SourceKind[], + ): ExternalDashboardFilterWithId => ({ + ...referenceVariableFields, + id: objectId(), + type: 'DATA_SOURCE', + name: 'Source', + variableName: 'src', + allowedSourceKinds, + }); + const builderTile: ExternalDashboardTileWithId = { id: 'builder', name: 'Builder', @@ -393,6 +417,47 @@ describe('resolveTileVariableReferences', () => { }, }; + const promqlTile: ExternalDashboardTileWithId = { + id: 'promql', + name: 'PromQL', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + configType: 'promql', + displayType: 'line', + sourceId: '$src', + expressions: [{ expression: 'up' }], + }, + } as ExternalDashboardTileWithId; + const searchTile: ExternalDashboardTileWithId = { + id: 'search', + name: 'Search', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + displayType: 'search', + sourceId: '$src', + select: '', + where: '', + whereLanguage: 'sql', + }, + }; + const rawSqlSourceTile: ExternalDashboardTileWithId = { + ...rawSqlTile, + id: 'raw-sql-source', + config: { + configType: 'sql', + displayType: 'table', + connectionId: '$conn', + sourceId: '$src', + sqlTemplate: 'SELECT 1', + }, + }; + const resolve = ( tile: ExternalDashboardTileWithId, filters: ExternalDashboardFilterWithId[], @@ -434,6 +499,80 @@ describe('resolveTileVariableReferences', () => { }); }); + it('resolves a source of a kind the tile accepts', () => { + const result = resolve( + promqlTile, + [sourceVariable([SourceKind.Log, SourceKind.Promql])], + { src: [promqlSourceId] }, + ); + + expect(result).toEqual({ + tile: { + ...promqlTile, + config: { ...promqlTile.config, sourceId: promqlSourceId }, + }, + }); + }); + + it.each<{ + case: string; + tile: ExternalDashboardTileWithId; + selection: string; + kinds: SourceKind[]; + }>([ + { + case: 'a log source on a PromQL tile', + tile: promqlTile, + selection: logSourceId, + kinds: [SourceKind.Log, SourceKind.Promql], + }, + { + case: 'a metric source on a search tile', + tile: searchTile, + selection: metricSourceId, + kinds: [SourceKind.Log, SourceKind.Metric], + }, + ])('rejects $case', ({ tile, selection, kinds }) => { + expect( + resolve(tile, [sourceVariable(kinds)], { src: [selection] }), + ).toEqual({ + error: expect.stringContaining('this tile only accepts'), + }); + }); + + it('resolves a raw SQL source on the connection the query runs on', () => { + const result = resolve( + rawSqlSourceTile, + [connectionVariable, sourceVariable([SourceKind.Log])], + { conn: [connectionId], src: [logSourceId] }, + ); + + expect(result).toEqual({ + tile: { + ...rawSqlSourceTile, + config: { + ...rawSqlSourceTile.config, + connectionId, + sourceId: logSourceId, + }, + }, + }); + }); + + it('rejects a raw SQL source on another connection than the query', () => { + const result = resolve( + rawSqlSourceTile, + [connectionVariable, sourceVariable([SourceKind.Log])], + { conn: [connectionId], src: [otherConnectionLogSourceId] }, + ); + + expect(result).toEqual({ + error: expect.stringContaining( + `the source is on connection "${otherConnectionId}"`, + ), + }); + }); + it.each<{ case: string; tile: ExternalDashboardTileWithId; diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index 5bc7dda1b..11acfd67b 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -1939,9 +1939,10 @@ const mcpDataSourceFilterSchema = z .array(z.nativeEnum(SourceKind)) .min(1) .describe( - 'Kinds of sources offered in the dropdown. A tile can only use the variable ' + - 'if it accepts every listed kind, so keep kinds that charts treat alike ' + - '(log + trace, or a single kind).', + 'Kinds of sources offered in the dropdown. A tile or filter can reference the ' + + 'variable if it accepts at least one listed kind. A tile whose selected source ' + + 'is a kind it cannot use (e.g. a log source on a PromQL tile) errors instead of ' + + 'running, so prefer kinds that charts treat alike (log + trace, or a single kind).', ), connectionId: objectIdOrVariableReferenceSchema .optional() diff --git a/packages/api/src/mcp/tools/dashboards/validation.ts b/packages/api/src/mcp/tools/dashboards/validation.ts index 63b47c5fd..d323c86ac 100644 --- a/packages/api/src/mcp/tools/dashboards/validation.ts +++ b/packages/api/src/mcp/tools/dashboards/validation.ts @@ -6,9 +6,9 @@ import { getFilterVariableName, isAdhocConditionSupported, isFilterVariableEnabled, - validateSqlQueryFilterTemplate, isReferenceVariableFilterType, pairWithVariableNames, + validateSqlQueryFilterTemplate, } from '@hyperdx/common-utils/dist/filters'; import { isBuilderSavedChartConfig, diff --git a/packages/api/src/mcp/tools/dashboards/variables.ts b/packages/api/src/mcp/tools/dashboards/variables.ts index d0c7a7548..7f5e91cb9 100644 --- a/packages/api/src/mcp/tools/dashboards/variables.ts +++ b/packages/api/src/mcp/tools/dashboards/variables.ts @@ -6,6 +6,7 @@ import { isFilterVariableEnabled, isStaticListFilter, } from '@hyperdx/common-utils/dist/filters'; +import { getSavedConfigAllowedSourceKinds } from '@hyperdx/common-utils/dist/guards'; import { getReferenceVariableFilters, getStringField, @@ -14,6 +15,7 @@ import { } from '@hyperdx/common-utils/dist/referenceVariables'; import type { ChartVariable, + SavedChartConfig, SourceKind, } from '@hyperdx/common-utils/dist/types'; import { isFilterSingleSelect } from '@hyperdx/common-utils/dist/variables'; @@ -21,7 +23,12 @@ import { z } from 'zod'; import { getConnectionsByTeam } from '@/controllers/connection'; import { getSources } from '@/controllers/sources'; -import type { ExternalDashboard } from '@/routers/external-api/v2/utils/dashboards'; +import { + convertToInternalTileConfig, + type ExternalDashboard, + isConfigTile, + isRawSqlExternalTileConfig, +} from '@/routers/external-api/v2/utils/dashboards'; import type { ExternalDashboardFilter, ExternalDashboardFilterWithId, @@ -275,7 +282,68 @@ export function resolveTileVariableReferences< } config = { ...config, [field]: id }; } - return { tile: { ...tile, config } }; + + const resolved = { ...tile, config }; + const tileError = getResolvedSourceError(resolved, tile.config, targets); + return tileError != null ? { error: tileError } : { tile: resolved }; +} + +/** + * Checks a resolved source the variable offers but the tile can't use, which + * the dashboard shows as an error in place of the tile: a kind the tile's + * chart type doesn't accept, or, for raw SQL, a source on another connection + * than the query runs on. + */ +function getResolvedSourceError( + tile: ExternalDashboardTileWithId, + storedConfig: object, + targets: ReferenceTargets, +): string | null { + if (!isConfigTile(tile)) return null; + const sourceId = getStringField(tile.config, 'sourceId'); + const source = targets.sources.find(s => s.id === sourceId); + if (source == null) return null; + const sourceName = parseVariableReference( + getStringField(storedConfig, 'sourceId'), + ); + const connectionName = parseVariableReference( + getStringField(storedConfig, 'connectionId'), + ); + + if (sourceName != null) { + let config: SavedChartConfig; + try { + config = convertToInternalTileConfig(tile).config; + } catch { + // A shape the converter refuses fails when the tile runs + return null; + } + const allowedKinds = getSavedConfigAllowedSourceKinds(config); + if (!allowedKinds.includes(source.kind)) { + return ( + `Tile "${tile.id}" reads its sourceId from the $${sourceName} variable, which ` + + `selects "${source.id}", a ${source.kind} source, but this tile only accepts ` + + `${allowedKinds.join(', ')} sources.` + ); + } + } + + if ( + (sourceName != null || connectionName != null) && + isRawSqlExternalTileConfig(tile.config) && + source.connection !== tile.config.connectionId + ) { + const from = [sourceName, connectionName] + .filter((name): name is string => name != null) + .map(name => `$${name}`) + .join(' and '); + return ( + `Tile "${tile.id}" resolves source "${source.id}" and connection ` + + `"${tile.config.connectionId}" from ${from}, but the source is on connection ` + + `"${source.connection}". Select a source on the connection the query runs on.` + ); + } + return null; } /** Options of each static filter, keyed by the variable name it answers to. */ diff --git a/packages/api/src/routers/external-api/v2/dashboards.ts b/packages/api/src/routers/external-api/v2/dashboards.ts index e37a2889f..c1e8ec0b2 100644 --- a/packages/api/src/routers/external-api/v2/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/dashboards.ts @@ -2478,8 +2478,9 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * type: string * enum: [log, trace, session, metric, promql] * description: | - * Kinds of sources offered. A field can only reference the variable - * if it accepts every listed kind. + * Kinds of sources offered. A field can reference the variable if it + * accepts at least one listed kind. A tile whose selected source is a + * kind it cannot use shows an error instead of running. * example: ["log"] * connectionId: * type: string diff --git a/packages/app/src/components/ChartEditor/utils.ts b/packages/app/src/components/ChartEditor/utils.ts index 5aa209879..ba2239df5 100644 --- a/packages/app/src/components/ChartEditor/utils.ts +++ b/packages/app/src/components/ChartEditor/utils.ts @@ -13,10 +13,13 @@ import { validateRawSqlForAlert, } from '@hyperdx/common-utils/dist/core/utils'; import { - getHeatmapSourceKinds, + getAllowedSourceKinds, isBuilderSavedChartConfig, + isPromqlDisplayType, isPromqlSavedChartConfig, isRawSqlSavedChartConfig, + isStringSelectDisplayType, + toBuilderHeatmapMode, } from '@hyperdx/common-utils/dist/guards'; import { MACRO_SUGGESTIONS } from '@hyperdx/common-utils/dist/macros'; import { QUERY_PARAMS_BY_DISPLAY_TYPE } from '@hyperdx/common-utils/dist/rawSqlParams'; @@ -26,12 +29,10 @@ import { ChartVariable, DisplayType, getSampleWeightExpression, - HeatmapMode, isLogSource, isMetricSource, isRangeThresholdType, isTraceSource, - NON_PROMQL_SOURCE_KINDS, PromqlChartConfig, PromqlHeatmapMode, PromqlSavedChartConfig, @@ -178,68 +179,6 @@ export const isRawSqlDisplayType = ( displayType === DisplayType.Bar || displayType === DisplayType.Number; -/** - * Display types that store `select` as a plain string (column expression) - * rather than a structured `DerivedColumn[]` series array. These types - * don't use the builder series editor and skip series-level validation. - */ -export const isStringSelectDisplayType = ( - displayType: DisplayType | undefined, -): displayType is DisplayType.Search | DisplayType.EventPatterns => - displayType === DisplayType.Search || - displayType === DisplayType.EventPatterns; - -export const isPromqlDisplayType = ( - displayType: DisplayType | undefined, -): displayType is - | DisplayType.Table - | DisplayType.Line - | DisplayType.StackedBar - | DisplayType.StackedLine - | DisplayType.Pie - | DisplayType.Bar - | DisplayType.Number - | DisplayType.Heatmap => - displayType === DisplayType.Table || - displayType === DisplayType.Line || - displayType === DisplayType.StackedBar || - displayType === DisplayType.StackedLine || - displayType === DisplayType.Pie || - displayType === DisplayType.Bar || - displayType === DisplayType.Number || - displayType === DisplayType.Heatmap; - -/** - * Search and event patterns list raw rows out of the source's `from` table. A - * metric source keeps its rows in `metricTables` and leaves `from.tableName` - * empty, so there is nothing for them to read. - */ -const ROW_LISTING_SOURCE_KINDS = NON_PROMQL_SOURCE_KINDS.filter( - kind => kind !== SourceKind.Metric, -); - -/** Source kinds the Data Source picker offers, given the editor mode and the display type. */ -export function getAllowedSourceKinds({ - configType, - displayType, - heatmapMode, -}: { - configType: ChartEditorFormState['configType']; - displayType: DisplayType | undefined; - heatmapMode?: PromqlHeatmapMode; -}): SourceKind[] { - if (configType === 'promql' && isPromqlDisplayType(displayType)) { - return [SourceKind.Promql]; - } - if (displayType === DisplayType.Heatmap) { - return [...getHeatmapSourceKinds(toBuilderHeatmapMode(heatmapMode))]; - } - if (isStringSelectDisplayType(displayType)) { - return ROW_LISTING_SOURCE_KINDS; - } - return [...NON_PROMQL_SOURCE_KINDS]; -} - const isCustomOrderByDisplayType = ( displayType: DisplayType | undefined, ): displayType is DisplayType.Table | DisplayType.Bar | DisplayType.Pie => @@ -252,6 +191,15 @@ const isCustomOrderByDisplayType = ( // ChartEditorControls and the server-side surfaces cannot drift. export { isFormulaDisplayType, isFormulaSourceKind }; +// Shared with MCP tile runs, which reject a variable's source selection the +// tile can't use, as the dashboard does. +export { + getAllowedSourceKinds, + isPromqlDisplayType, + isStringSelectDisplayType, + toBuilderHeatmapMode, +}; + export function convertFormStateToSavedChartConfig( form: ChartEditorFormState, source: TSource | undefined, @@ -569,13 +517,6 @@ function formBuilderHeatmap( : { mode: toBuilderHeatmapMode(form.heatmap.mode) }; } -/** A heatmap mode as a builder heatmap's, which has no histogram mode. */ -export function toBuilderHeatmapMode( - mode: PromqlHeatmapMode | undefined, -): HeatmapMode | undefined { - return mode === 'histogram' ? undefined : mode; -} - const DEFAULT_PROMQL_PLACEHOLDER = "rate(http_requests_total{service='api'}[5m])"; diff --git a/packages/common-utils/src/__tests__/guards.test.ts b/packages/common-utils/src/__tests__/guards.test.ts index e7789a0c9..26c9cfb30 100644 --- a/packages/common-utils/src/__tests__/guards.test.ts +++ b/packages/common-utils/src/__tests__/guards.test.ts @@ -1,5 +1,6 @@ import { displayTypeRequiresSource, + getSavedConfigAllowedSourceKinds, isBuilderSavedChartConfig, isRawSqlSavedChartConfig, } from '@/guards'; @@ -9,6 +10,8 @@ import { DisplayType, isChartPaletteToken, resolveChartPaletteToken, + SavedChartConfig, + SourceKind, walkRawDashboardTileColors, } from '@/types'; @@ -367,3 +370,46 @@ describe('walkRawDashboardTileColors', () => { expect(walkRawDashboardTileColors(input, () => 'x')).toBe(input); }); }); + +describe('getSavedConfigAllowedSourceKinds', () => { + it.each<{ + case: string; + config: Partial; + kinds: SourceKind[]; + }>([ + { + case: 'a PromQL chart', + config: { configType: 'promql', displayType: DisplayType.Line }, + kinds: [SourceKind.Promql], + }, + { + case: 'a builder search', + config: { displayType: DisplayType.Search }, + kinds: [SourceKind.Log, SourceKind.Trace, SourceKind.Session], + }, + { + case: 'a builder distribution heatmap', + config: { displayType: DisplayType.Heatmap }, + kinds: [SourceKind.Trace], + }, + { + case: 'a builder series heatmap', + config: { displayType: DisplayType.Heatmap, heatmap: { mode: 'series' } }, + kinds: [SourceKind.Trace, SourceKind.Log, SourceKind.Metric], + }, + { + case: 'a raw SQL table', + config: { configType: 'sql', displayType: DisplayType.Table }, + kinds: [ + SourceKind.Log, + SourceKind.Trace, + SourceKind.Session, + SourceKind.Metric, + ], + }, + ])('lists the kinds for $case', ({ config, kinds }) => { + expect( + getSavedConfigAllowedSourceKinds(config as SavedChartConfig).sort(), + ).toEqual([...kinds].sort()); + }); +}); diff --git a/packages/common-utils/src/guards.ts b/packages/common-utils/src/guards.ts index 2b6d71657..fcc009fe3 100644 --- a/packages/common-utils/src/guards.ts +++ b/packages/common-utils/src/guards.ts @@ -5,7 +5,9 @@ import { ChartConfigWithOptDateRange, DisplayType, HeatmapMode, + NON_PROMQL_SOURCE_KINDS, PromqlChartConfig, + PromqlHeatmapMode, PromqlSavedChartConfig, RawSqlChartConfig, RawSqlSavedChartConfig, @@ -94,3 +96,89 @@ export function displayTypeRequiresSource( ): boolean { return displayType !== DisplayType.Markdown; } + +/** + * Display types that store `select` as a plain string (column expression) + * rather than a structured `DerivedColumn[]` series array. These types + * don't use the builder series editor and skip series-level validation. + */ +export const isStringSelectDisplayType = ( + displayType: DisplayType | undefined, +): displayType is DisplayType.Search | DisplayType.EventPatterns => + displayType === DisplayType.Search || + displayType === DisplayType.EventPatterns; + +export const isPromqlDisplayType = ( + displayType: DisplayType | undefined, +): displayType is + | DisplayType.Table + | DisplayType.Line + | DisplayType.StackedBar + | DisplayType.StackedLine + | DisplayType.Pie + | DisplayType.Bar + | DisplayType.Number + | DisplayType.Heatmap => + displayType === DisplayType.Table || + displayType === DisplayType.Line || + displayType === DisplayType.StackedBar || + displayType === DisplayType.StackedLine || + displayType === DisplayType.Pie || + displayType === DisplayType.Bar || + displayType === DisplayType.Number || + displayType === DisplayType.Heatmap; + +/** A heatmap mode as a builder heatmap's, which has no histogram mode. */ +export function toBuilderHeatmapMode( + mode: PromqlHeatmapMode | undefined, +): HeatmapMode | undefined { + return mode === 'histogram' ? undefined : mode; +} + +/** + * Search and event patterns list raw rows out of the source's `from` table. A + * metric source keeps its rows in `metricTables` and leaves `from.tableName` + * empty, so there is nothing for them to read. + */ +const ROW_LISTING_SOURCE_KINDS = NON_PROMQL_SOURCE_KINDS.filter( + kind => kind !== SourceKind.Metric, +); + +/** Source kinds a chart can use, given its config type and display type. */ +export function getAllowedSourceKinds({ + configType, + displayType, + heatmapMode, +}: { + configType: 'builder' | 'sql' | 'promql' | undefined; + displayType: DisplayType | undefined; + heatmapMode?: PromqlHeatmapMode; +}): SourceKind[] { + if (configType === 'promql' && isPromqlDisplayType(displayType)) { + return [SourceKind.Promql]; + } + if (displayType === DisplayType.Heatmap) { + return [...getHeatmapSourceKinds(toBuilderHeatmapMode(heatmapMode))]; + } + if (isStringSelectDisplayType(displayType)) { + return ROW_LISTING_SOURCE_KINDS; + } + return [...NON_PROMQL_SOURCE_KINDS]; +} + +/** `getAllowedSourceKinds` for a saved chart config. */ +export function getSavedConfigAllowedSourceKinds( + config: SavedChartConfig, +): SourceKind[] { + if (isRawSqlSavedChartConfig(config)) { + return getAllowedSourceKinds({ + configType: 'sql', + displayType: config.displayType, + }); + } + return getAllowedSourceKinds({ + configType: isPromqlSavedChartConfig(config) ? 'promql' : 'builder', + displayType: config.displayType, + heatmapMode: config.heatmap?.mode, + }); +} From d1bfdde2d5a9479b4201ce79fe9bed80870cb204 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Tue, 17 Nov 2026 15:34:11 -0500 Subject: [PATCH 07/18] review: Address review findings --- packages/api/openapi.json | 72 +++++++++++++++++++ .../dashboards/promqlTiles.int.test.ts | 59 +++++++++++++++ .../api/src/mcp/tools/dashboards/schemas.ts | 21 +++++- .../src/routers/external-api/v2/dashboards.ts | 48 +++++++++++++ .../v2/utils/__tests__/dashboards.test.ts | 31 ++++++++ .../external-api/v2/utils/dashboards.ts | 5 +- 6 files changed, 233 insertions(+), 3 deletions(-) diff --git a/packages/api/openapi.json b/packages/api/openapi.json index 95979cd65..cfa7b0b8c 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -4088,6 +4088,42 @@ "description": "Blocks every tile on the dashboard until a value is selected. False\n(the default) blocks only the tiles that reference the variable.\n", "example": true }, + "minSelections": { + "type": "integer", + "enum": [ + 1 + ], + "default": 1, + "description": "Must be 1 if provided. The variable always requires a selection.", + "example": 1 + }, + "maxSelections": { + "type": "integer", + "enum": [ + 1 + ], + "default": 1, + "description": "Must be 1 if provided. The variable is always single-select.", + "example": 1 + }, + "isBroadcastEnabled": { + "type": "boolean", + "enum": [ + false + ], + "default": false, + "description": "Must be false or omitted.", + "example": false + }, + "isVariableEnabled": { + "type": "boolean", + "enum": [ + true + ], + "default": true, + "description": "Must be true if provided, and is true when omitted.", + "example": true + }, "allowedSourceKinds": { "type": "array", "minItems": 1, @@ -4147,6 +4183,42 @@ "default": false, "description": "Blocks every tile on the dashboard until a value is selected. False\n(the default) blocks only the tiles that reference the variable.\n", "example": true + }, + "minSelections": { + "type": "integer", + "enum": [ + 1 + ], + "default": 1, + "description": "Must be 1 if provided. The variable always requires a selection.", + "example": 1 + }, + "maxSelections": { + "type": "integer", + "enum": [ + 1 + ], + "default": 1, + "description": "Must be 1 if provided. The variable is always single-select.", + "example": 1 + }, + "isBroadcastEnabled": { + "type": "boolean", + "enum": [ + false + ], + "default": false, + "description": "Must be false or omitted.", + "example": false + }, + "isVariableEnabled": { + "type": "boolean", + "enum": [ + true + ], + "default": true, + "description": "Must be true if provided, and is true when omitted.", + "example": true } } }, diff --git a/packages/api/src/mcp/__tests__/dashboards/promqlTiles.int.test.ts b/packages/api/src/mcp/__tests__/dashboards/promqlTiles.int.test.ts index 61250c8c0..bf5372873 100644 --- a/packages/api/src/mcp/__tests__/dashboards/promqlTiles.int.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/promqlTiles.int.test.ts @@ -562,6 +562,65 @@ describe('MCP Dashboard Tools - PromQL tiles and label filters', () => { expect(await Dashboard.countDocuments({ team: ctx.team._id })).toBe(0); }); + it('rejects a DATA_SOURCE filter that offers promql sources', async () => { + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'PromQL off', + tiles: [], + filters: [ + { + type: 'DATA_SOURCE', + name: 'Metrics', + variableName: 'metrics', + allowedSourceKinds: ['promql'], + }, + ], + }); + + expect(result.isError).toBe(true); + expect(await Dashboard.countDocuments({ team: ctx.team._id })).toBe(0); + }); + + it('hides and keeps DATA_SOURCE filters that offer promql sources', async () => { + const dataSource = (name: string, allowedSourceKinds: string[]) => ({ + id: name, + type: 'DATA_SOURCE' as const, + name, + variableName: name, + allowedSourceKinds, + minSelections: 1 as const, + maxSelections: 1 as const, + isBroadcastEnabled: false as const, + isVariableEnabled: true as const, + }); + const dashboard = await new Dashboard({ + name: 'Source variables', + team: ctx.team._id, + tiles: [], + filters: [ + dataSource('logs', ['log']), + dataSource('metrics', ['promql']), + ], + }).save(); + + const get = await callTool(ctx.client!, 'clickstack_get_dashboard', { + id: dashboard._id.toString(), + }); + expect(get.isError).toBeFalsy(); + expect(parseOutput(get).filters.map((f: { id: string }) => f.id)).toEqual( + ['logs'], + ); + + const save = await callTool(ctx.client!, 'clickstack_save_dashboard', { + id: dashboard._id.toString(), + name: 'Source variables', + tiles: [], + filters: [{ ...dataSource('logs', ['log']), name: 'Renamed' }], + }); + expect(save.isError).toBeFalsy(); + const stored = await Dashboard.findById(dashboard._id).lean(); + expect(stored?.filters?.map(f => f.name)).toEqual(['Renamed', 'metrics']); + }); + it('rejects a PromQL tile', async () => { const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { name: 'PromQL off', diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index 11acfd67b..589066ce4 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -1927,7 +1927,7 @@ const mcpReferenceVariableBaseShape = { ), }; -const mcpDataSourceFilterSchema = z +const mcpPromqlDataSourceFilterSchema = z .object({ ...mcpReferenceVariableBaseShape, type: z @@ -1957,6 +1957,23 @@ const mcpDataSourceFilterSchema = z 'source ID; tiles stay blocked until a source is selected.', ); +const mcpDataSourceFilterSchema = mcpPromqlDataSourceFilterSchema.extend({ + allowedSourceKinds: z + .array( + z.enum([ + SourceKind.Log, + SourceKind.Trace, + SourceKind.Session, + SourceKind.Metric, + ]), + ) + .min(1) + .describe( + mcpPromqlDataSourceFilterSchema.shape.allowedSourceKinds.description ?? + '', + ), +}); + const mcpConnectionFilterSchema = z .object({ ...mcpReferenceVariableBaseShape, @@ -1996,7 +2013,7 @@ const mcpPromqlDashboardFilterSchema = z mcpSqlQueryFilterSchema, mcpPromqlLabelFilterSchema, mcpPromqlAdhocFilterSchema, - mcpDataSourceFilterSchema, + mcpPromqlDataSourceFilterSchema, mcpConnectionFilterSchema, ]) .describe( diff --git a/packages/api/src/routers/external-api/v2/dashboards.ts b/packages/api/src/routers/external-api/v2/dashboards.ts index c1e8ec0b2..cae8943ea 100644 --- a/packages/api/src/routers/external-api/v2/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/dashboards.ts @@ -2471,6 +2471,30 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * Blocks every tile on the dashboard until a value is selected. False * (the default) blocks only the tiles that reference the variable. * example: true + * minSelections: + * type: integer + * enum: [1] + * default: 1 + * description: Must be 1 if provided. The variable always requires a selection. + * example: 1 + * maxSelections: + * type: integer + * enum: [1] + * default: 1 + * description: Must be 1 if provided. The variable is always single-select. + * example: 1 + * isBroadcastEnabled: + * type: boolean + * enum: [false] + * default: false + * description: Must be false or omitted. + * example: false + * isVariableEnabled: + * type: boolean + * enum: [true] + * default: true + * description: Must be true if provided, and is true when omitted. + * example: true * allowedSourceKinds: * type: array * minItems: 1 @@ -2523,6 +2547,30 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * Blocks every tile on the dashboard until a value is selected. False * (the default) blocks only the tiles that reference the variable. * example: true + * minSelections: + * type: integer + * enum: [1] + * default: 1 + * description: Must be 1 if provided. The variable always requires a selection. + * example: 1 + * maxSelections: + * type: integer + * enum: [1] + * default: 1 + * description: Must be 1 if provided. The variable is always single-select. + * example: 1 + * isBroadcastEnabled: + * type: boolean + * enum: [false] + * default: false + * description: Must be false or omitted. + * example: false + * isVariableEnabled: + * type: boolean + * enum: [true] + * default: true + * description: Must be true if provided, and is true when omitted. + * example: true * * QueryExpressionFilterInput: * type: object diff --git a/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts b/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts index 9ab380298..51e1c8b3e 100644 --- a/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts +++ b/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts @@ -8,6 +8,7 @@ import { MetricsDataType, PromqlReducer, PromqlSavedChartConfig, + SourceKind, } from '@hyperdx/common-utils/dist/types'; import mongoose from 'mongoose'; import { z } from 'zod'; @@ -539,6 +540,36 @@ describe('convertToExternalDashboard orphan-ref heal', () => { expect(hidden.savedFilterValues).toEqual([adhocValue]); }); + it('hides data source variables that offer promql sources with hidePromql', () => { + const dataSource = (id: string, allowedSourceKinds: SourceKind[]) => ({ + id, + type: 'DATA_SOURCE' as const, + name: id, + variableName: id, + allowedSourceKinds, + minSelections: 1 as const, + maxSelections: 1 as const, + isBroadcastEnabled: false as const, + isVariableEnabled: true as const, + }); + const doc = makeDoc({ + filters: [ + dataSource('logs', [SourceKind.Log]), + dataSource('mixed', [SourceKind.Metric, SourceKind.Promql]), + ], + }); + + expect(convertToExternalDashboard(doc).filters?.map(f => f.id)).toEqual([ + 'logs', + 'mixed', + ]); + expect( + convertToExternalDashboard(doc, { hidePromql: true }).filters?.map( + f => f.id, + ), + ).toEqual(['logs']); + }); + // seriesLimit is three-state (matching the internal schema): omitted = // default cap, 0 = unlimited, positive N = top-N. All three must survive a // GET->PUT round-trip: 0 and N pass through, null maps to absent (the diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index fbde606d5..50498a8e1 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -17,6 +17,7 @@ import { } from '@hyperdx/common-utils/dist/dashboardValidation'; import { isAdhocFilter, + isDataSourceFilter, isPrometheusLabelFilter, isQueryExpressionFilter, } from '@hyperdx/common-utils/dist/filters'; @@ -784,7 +785,9 @@ function convertTileToExternalChart( export function isPromqlFilter(filter: DashboardFilter): boolean { return ( isPrometheusLabelFilter(filter) || - (isAdhocFilter(filter) && filter.sourceType === 'promql') + (isAdhocFilter(filter) && filter.sourceType === 'promql') || + (isDataSourceFilter(filter) && + filter.allowedSourceKinds.includes(SourceKind.Promql)) ); } From ef79c9163f5fc62f2b9696ca78b2b1f8075728bb Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Tue, 17 Nov 2026 15:49:07 -0500 Subject: [PATCH 08/18] fix: show data source variables that offer promql sources while PromQL is disabled --- .../dashboards/promqlTiles.int.test.ts | 59 ------------------- .../api/src/mcp/tools/dashboards/schemas.ts | 21 +------ .../v2/utils/__tests__/dashboards.test.ts | 32 ++++------ .../external-api/v2/utils/dashboards.ts | 5 +- 4 files changed, 16 insertions(+), 101 deletions(-) diff --git a/packages/api/src/mcp/__tests__/dashboards/promqlTiles.int.test.ts b/packages/api/src/mcp/__tests__/dashboards/promqlTiles.int.test.ts index bf5372873..61250c8c0 100644 --- a/packages/api/src/mcp/__tests__/dashboards/promqlTiles.int.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/promqlTiles.int.test.ts @@ -562,65 +562,6 @@ describe('MCP Dashboard Tools - PromQL tiles and label filters', () => { expect(await Dashboard.countDocuments({ team: ctx.team._id })).toBe(0); }); - it('rejects a DATA_SOURCE filter that offers promql sources', async () => { - const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { - name: 'PromQL off', - tiles: [], - filters: [ - { - type: 'DATA_SOURCE', - name: 'Metrics', - variableName: 'metrics', - allowedSourceKinds: ['promql'], - }, - ], - }); - - expect(result.isError).toBe(true); - expect(await Dashboard.countDocuments({ team: ctx.team._id })).toBe(0); - }); - - it('hides and keeps DATA_SOURCE filters that offer promql sources', async () => { - const dataSource = (name: string, allowedSourceKinds: string[]) => ({ - id: name, - type: 'DATA_SOURCE' as const, - name, - variableName: name, - allowedSourceKinds, - minSelections: 1 as const, - maxSelections: 1 as const, - isBroadcastEnabled: false as const, - isVariableEnabled: true as const, - }); - const dashboard = await new Dashboard({ - name: 'Source variables', - team: ctx.team._id, - tiles: [], - filters: [ - dataSource('logs', ['log']), - dataSource('metrics', ['promql']), - ], - }).save(); - - const get = await callTool(ctx.client!, 'clickstack_get_dashboard', { - id: dashboard._id.toString(), - }); - expect(get.isError).toBeFalsy(); - expect(parseOutput(get).filters.map((f: { id: string }) => f.id)).toEqual( - ['logs'], - ); - - const save = await callTool(ctx.client!, 'clickstack_save_dashboard', { - id: dashboard._id.toString(), - name: 'Source variables', - tiles: [], - filters: [{ ...dataSource('logs', ['log']), name: 'Renamed' }], - }); - expect(save.isError).toBeFalsy(); - const stored = await Dashboard.findById(dashboard._id).lean(); - expect(stored?.filters?.map(f => f.name)).toEqual(['Renamed', 'metrics']); - }); - it('rejects a PromQL tile', async () => { const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { name: 'PromQL off', diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index 589066ce4..11acfd67b 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -1927,7 +1927,7 @@ const mcpReferenceVariableBaseShape = { ), }; -const mcpPromqlDataSourceFilterSchema = z +const mcpDataSourceFilterSchema = z .object({ ...mcpReferenceVariableBaseShape, type: z @@ -1957,23 +1957,6 @@ const mcpPromqlDataSourceFilterSchema = z 'source ID; tiles stay blocked until a source is selected.', ); -const mcpDataSourceFilterSchema = mcpPromqlDataSourceFilterSchema.extend({ - allowedSourceKinds: z - .array( - z.enum([ - SourceKind.Log, - SourceKind.Trace, - SourceKind.Session, - SourceKind.Metric, - ]), - ) - .min(1) - .describe( - mcpPromqlDataSourceFilterSchema.shape.allowedSourceKinds.description ?? - '', - ), -}); - const mcpConnectionFilterSchema = z .object({ ...mcpReferenceVariableBaseShape, @@ -2013,7 +1996,7 @@ const mcpPromqlDashboardFilterSchema = z mcpSqlQueryFilterSchema, mcpPromqlLabelFilterSchema, mcpPromqlAdhocFilterSchema, - mcpPromqlDataSourceFilterSchema, + mcpDataSourceFilterSchema, mcpConnectionFilterSchema, ]) .describe( diff --git a/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts b/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts index 51e1c8b3e..200f6f26b 100644 --- a/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts +++ b/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts @@ -540,34 +540,28 @@ describe('convertToExternalDashboard orphan-ref heal', () => { expect(hidden.savedFilterValues).toEqual([adhocValue]); }); - it('hides data source variables that offer promql sources with hidePromql', () => { - const dataSource = (id: string, allowedSourceKinds: SourceKind[]) => ({ - id, - type: 'DATA_SOURCE' as const, - name: id, - variableName: id, - allowedSourceKinds, - minSelections: 1 as const, - maxSelections: 1 as const, - isBroadcastEnabled: false as const, - isVariableEnabled: true as const, - }); + it('keeps data source variables that offer promql sources with hidePromql', () => { const doc = makeDoc({ filters: [ - dataSource('logs', [SourceKind.Log]), - dataSource('mixed', [SourceKind.Metric, SourceKind.Promql]), + { + id: 'mixed', + type: 'DATA_SOURCE', + name: 'mixed', + variableName: 'mixed', + allowedSourceKinds: [SourceKind.Metric, SourceKind.Promql], + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }, ], }); - expect(convertToExternalDashboard(doc).filters?.map(f => f.id)).toEqual([ - 'logs', - 'mixed', - ]); expect( convertToExternalDashboard(doc, { hidePromql: true }).filters?.map( f => f.id, ), - ).toEqual(['logs']); + ).toEqual(['mixed']); }); // seriesLimit is three-state (matching the internal schema): omitted = diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index 50498a8e1..fbde606d5 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -17,7 +17,6 @@ import { } from '@hyperdx/common-utils/dist/dashboardValidation'; import { isAdhocFilter, - isDataSourceFilter, isPrometheusLabelFilter, isQueryExpressionFilter, } from '@hyperdx/common-utils/dist/filters'; @@ -785,9 +784,7 @@ function convertTileToExternalChart( export function isPromqlFilter(filter: DashboardFilter): boolean { return ( isPrometheusLabelFilter(filter) || - (isAdhocFilter(filter) && filter.sourceType === 'promql') || - (isDataSourceFilter(filter) && - filter.allowedSourceKinds.includes(SourceKind.Promql)) + (isAdhocFilter(filter) && filter.sourceType === 'promql') ); } From f408b8282de7e37117c0eddf421bdc733e8cad48 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Mon, 16 Nov 2026 13:02:30 -0500 Subject: [PATCH 09/18] feat: add data source variables to dashboard filters and tiles --- .changeset/source-vars-3.md | 5 + packages/app/src/DBDashboardPage.tsx | 200 +++++++++++++----- packages/app/src/DashboardFilters.tsx | 60 +++++- .../AdhocDashboardFilter.tsx | 15 +- .../__tests__/AdhocDashboardFilter.test.tsx | 19 ++ .../DashboardFilterEditForm.tsx | 52 +++-- .../DataSourceFilterEditForm.tsx | 66 ++++++ .../__tests__/filterFormState.test.ts | 105 ++++++++- .../DashboardFiltersModal/filterFormState.ts | 32 ++- .../ReferenceVariableDashboardFilter.tsx | 159 ++++++++++++++ .../app/src/components/SelectControlled.tsx | 32 +-- .../ReferenceVariableDashboardFilter.test.tsx | 108 ++++++++++ .../app/src/components/sourceSelectUtils.tsx | 12 ++ .../__tests__/useDashboardFilters.test.tsx | 180 ++++++++++++++++ .../usePresetDashboardFilters.test.tsx | 3 + .../app/src/hooks/useDashboardFilters.tsx | 104 ++++++++- .../dashboard-required-filters.spec.ts | 2 +- .../dashboard-source-variables.spec.ts | 109 ++++++++++ .../tests/e2e/page-objects/DashboardPage.ts | 42 +++- 19 files changed, 1207 insertions(+), 98 deletions(-) create mode 100644 .changeset/source-vars-3.md create mode 100644 packages/app/src/components/DashboardFiltersModal/DataSourceFilterEditForm.tsx create mode 100644 packages/app/src/components/ReferenceVariableDashboardFilter.tsx create mode 100644 packages/app/src/components/__tests__/ReferenceVariableDashboardFilter.test.tsx create mode 100644 packages/app/tests/e2e/features/dashboard-source-variables.spec.ts diff --git a/.changeset/source-vars-3.md b/.changeset/source-vars-3.md new file mode 100644 index 000000000..743757dbd --- /dev/null +++ b/.changeset/source-vars-3.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/app': minor +--- + +feat: add data source variables to dashboard filters and tiles diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx index f5000d1a7..35c39c806 100644 --- a/packages/app/src/DBDashboardPage.tsx +++ b/packages/app/src/DBDashboardPage.tsx @@ -28,11 +28,9 @@ import { } from '@hyperdx/common-utils/dist/core/metadata'; import { convertToDashboardTemplate, - displayTypeSupportsBuilderAlerts, - displayTypeSupportsPromQLAlerts, - displayTypeSupportsRawSqlAlerts, Granularity, isTimeSeriesDisplayType, + savedChartConfigSupportsAlerts, } from '@hyperdx/common-utils/dist/core/utils'; import { getAdhocFilterTargetsForTile, @@ -56,6 +54,11 @@ import { isImportableDashboard, } from '@hyperdx/common-utils/dist/iac'; import { isMissingFiltersMacro } from '@hyperdx/common-utils/dist/macros'; +import { + getConfigReferenceVariables, + resolveConfigReferences, + restoreConfigReferences, +} from '@hyperdx/common-utils/dist/referenceVariables'; import { AlertState, BroadcastFilter, @@ -455,6 +458,7 @@ const Tile = ({ filters, variables, unsatisfiedRequiredFilters, + referenceVariableNames, showAlertAnnotations, showReleaseAnnotations, isLive, @@ -489,6 +493,12 @@ const Tile = ({ variables?: ChartVariable[]; /** The dashboard's required filters that have nothing selected. */ unsatisfiedRequiredFilters?: DashboardFilter[]; + /** + * The data source and connection variables the tile's stored config + * references. `chart` carries their selections, or the references while + * unselected. + */ + referenceVariableNames?: string[]; // When true, draw alert firing/recovery annotations on this tile's chart. showAlertAnnotations?: boolean; // When true, draw release markers on this tile's chart. @@ -604,8 +614,33 @@ const Tile = ({ id: chart.config.source, }); + const usesReferenceVariable = !!referenceVariableNames?.length; + const isSourceMissing = !!chart.config.source && isSourceFetched && source == null; + // A data source or connection variable can select a source on another + // connection than the one a raw SQL tile queries. + const isSourceOnOtherConnection = + usesReferenceVariable && + source != null && + isRawSqlSavedChartConfig(chart.config) && + source.connection !== chart.config.connection; + // A variable's kinds can change after tiles use it, and saves don't check a + // source's kind against the tile's display type. + const isSourceKindUnsupported = + usesReferenceVariable && + source != null && + !getAllowedSourceKinds({ + configType: isPromqlSavedChartConfig(chart.config) + ? 'promql' + : isRawSqlSavedChartConfig(chart.config) + ? 'sql' + : 'builder', + displayType: chart.config.displayType, + heatmapMode: isRawSqlSavedChartConfig(chart.config) + ? undefined + : chart.config.heatmap?.mode, + }).includes(source.kind); const isSourceUnset = !!chart.config && isBuilderSavedChartConfig(chart.config) && @@ -655,9 +690,15 @@ const Tile = ({ sourceId: chart.config.source, unsatisfiedRequiredFilters, referencedVariables: tileVariables, + referenceVariableNames, }), ), - [unsatisfiedRequiredFilters, chart.config, tileVariables], + [ + unsatisfiedRequiredFilters, + chart.config, + tileVariables, + referenceVariableNames, + ], ); const missingRequiredFilterNames = useMemo( () => JSON.parse(serializedMissingRequiredFilterNames), @@ -929,16 +970,13 @@ const Tile = ({ return url && addTimeZone(url); }, [dateRange, queriedConfig, source, addTimeZone]); + // An alert must query one fixed source and connection + const supportsAlerts = + !usesReferenceVariable && savedChartConfigSupportsAlerts(chart.config); + const hoverToolbar = useMemo(() => { if (readOnly) return null; - const isRawSql = isRawSqlSavedChartConfig(chart.config); - const isPromQL = isPromqlSavedChartConfig(chart.config); - const displayTypeSupportsAlerts = isRawSql - ? displayTypeSupportsRawSqlAlerts(chart.config.displayType) - : isPromQL - ? displayTypeSupportsPromQLAlerts(chart.config.displayType) - : displayTypeSupportsBuilderAlerts(chart.config.displayType); const canMoveToGroup = onMoveToGroup && moveTargets && moveTargets.length > 0; return ( @@ -968,7 +1006,7 @@ const Tile = ({ )} - {displayTypeSupportsAlerts && + {supportsAlerts && (alert ? ( // Existing alert: bell with a colored status dot indicator. { if (readOnly) return null; - const isRawSql = isRawSqlSavedChartConfig(chart.config); - const isPromQL = isPromqlSavedChartConfig(chart.config); - const showAlerts = isRawSql - ? displayTypeSupportsRawSqlAlerts(chart.config.displayType) - : isPromQL - ? displayTypeSupportsPromQLAlerts(chart.config.displayType) - : displayTypeSupportsBuilderAlerts(chart.config.displayType); const canMoveToGroup = onMoveToGroup && moveTargets && moveTargets.length > 0; return ( @@ -1113,7 +1144,7 @@ const Tile = ({ Replay search )} - {showAlerts && ( + {supportsAlerts && ( <> - {`Missing required filters: ${missingRequiredFilterNames.join( + {`Missing required filters or variables: ${missingRequiredFilterNames.join( ', ', )}. Select a value for each to load this tile.`} @@ -1269,6 +1300,20 @@ const Tile = ({ The data source for this tile is not set. Edit the tile to select a data source. + ) : isSourceOnOtherConnection ? ( + + The selected data source is on a different connection than this + tile's query. Select a source on the same connection. + + ) : isSourceKindUnsupported ? ( + + The selected data source can't be used with this chart type. + Select a different source. + ) : ( <> {effectiveQueriedConfig && @@ -1501,6 +1546,8 @@ const Tile = ({ filterWarning, isSourceMissing, isSourceUnset, + isSourceOnOtherConnection, + isSourceKindUnsupported, isBlockedByRequiredFilters, missingRequiredFilterNames, hasBeenVisible, @@ -1900,28 +1947,6 @@ function DBDashboardPage({ const { data: allDashboards } = useDashboards(); const [highlightedTileId] = useQueryState('highlightedTileId'); - const tableConnections = useMemo(() => { - if (!dashboard) return []; - const tc: TableConnection[] = []; - - for (const { config } of dashboard.tiles) { - if (!isBuilderSavedChartConfig(config)) continue; - const source = sources?.find(v => v.id === config.source); - if (!source || source.disabled) continue; - // TODO: will need to update this when we allow for multiple metrics per chart - const firstSelect = config.select[0]; - const metricType = - typeof firstSelect !== 'string' ? firstSelect?.metricType : undefined; - const tableName = getMetricTableName(source, metricType); - if (!tableName) continue; - tc.push({ - ...tcFromSource(source), - tableName, - }); - } - - return tc; - }, [dashboard, sources]); const [granularity, setGranularity] = useQueryState( 'granularity', @@ -1966,8 +1991,60 @@ function DBDashboardPage({ ignoredVariableNames, getFilterQueriesForSource, variables, + referenceVariableValues, + resolvedFilters, + unresolvedReferenceVariablesByFilterId, unsatisfiedRequiredFilters, - } = useDashboardFilters(filters); + } = useDashboardFilters(filters, { sources }); + // Tiles as they query, by ID: source and connection variable references + // replaced by the selections. Edits and saves read `dashboard.tiles`, so the + // stored references survive. + const resolvedTiles = useMemo( + () => + new Map( + (dashboard?.tiles ?? []).map(tile => { + const config = resolveConfigReferences( + tile.config, + referenceVariableValues, + ); + return [tile.id, config === tile.config ? tile : { ...tile, config }]; + }), + ), + [dashboard?.tiles, referenceVariableValues], + ); + // Kept apart from `resolvedTiles` so a tile's list only changes with its config + const referenceVariableNamesByTileId = useMemo( + () => + new Map( + (dashboard?.tiles ?? []).map(tile => [ + tile.id, + getConfigReferenceVariables(tile.config), + ]), + ), + [dashboard?.tiles], + ); + + const tableConnections = useMemo(() => { + const tc: TableConnection[] = []; + + for (const { config } of resolvedTiles.values()) { + if (!isBuilderSavedChartConfig(config)) continue; + const source = sources?.find(v => v.id === config.source); + if (!source || source.disabled) continue; + // TODO: will need to update this when we allow for multiple metrics per chart + const firstSelect = config.select[0]; + const metricType = + typeof firstSelect !== 'string' ? firstSelect?.metricType : undefined; + const tableName = getMetricTableName(source, metricType); + if (!tableName) continue; + tc.push({ + ...tcFromSource(source), + tableName, + }); + } + + return tc; + }, [resolvedTiles, sources]); const dashboardReady = !!dashboard?.id && @@ -2406,10 +2483,10 @@ function DBDashboardPage({ // The filter bar is hidden in kiosk mode, so a click would add conditions // the viewer can't see or remove. if (isKioskMode) return handlers; - for (const tile of dashboard?.tiles ?? []) { + for (const tile of resolvedTiles.values()) { const sourceId = 'source' in tile.config ? tile.config.source : undefined; const targetIds = getAdhocFilterTargetsForTile( - filters, + resolvedFilters, tile.config, sourceId, ).map(filter => filter.id); @@ -2420,19 +2497,23 @@ function DBDashboardPage({ } } return handlers; - }, [isKioskMode, dashboard?.tiles, filters, applyAdhocAction]); + }, [isKioskMode, resolvedTiles, resolvedFilters, applyAdhocAction]); const renderTileComponent = useCallback( (chart: Tile) => { + const resolvedChart = resolvedTiles.get(chart.id) ?? chart; // Resolve the tile's source ID so per-source-scoped filters can be // narrowed to only the tiles they target. Builder and RawSQL configs // both carry a `source` field; markdown / other configs don't. const tileSourceId = - 'source' in chart.config ? chart.config.source : undefined; + 'source' in resolvedChart.config + ? resolvedChart.config.source + : undefined; return ( setEditedTile(chart)} readOnly={isKioskMode} @@ -2459,7 +2540,14 @@ function DBDashboardPage({ c => c.id === chart.id, ); if (chartIndex === -1) return; - draft.tiles[chartIndex] = newChart; + // Keep the source as stored, not as resolved for display + draft.tiles[chartIndex] = { + ...newChart, + config: restoreConfigReferences( + newChart.config, + chart.config, + ), + }; }), ); } @@ -2532,6 +2620,8 @@ function DBDashboardPage({ ); }, [ + resolvedTiles, + referenceVariableNamesByTileId, dashboard, searchedTimeRange, isRefreshEnabled, @@ -3324,7 +3414,7 @@ function DBDashboardPage({ )} {!isKioskMode && ( )} {/* Selection indicator */} diff --git a/packages/app/src/DashboardFilters.tsx b/packages/app/src/DashboardFilters.tsx index eb7846246..f9350b4d2 100644 --- a/packages/app/src/DashboardFilters.tsx +++ b/packages/app/src/DashboardFilters.tsx @@ -5,12 +5,14 @@ import { getFilterVariableName, getPendingFilterValuesVariables, isAdhocFilter, + isDataSourceFilter, isFilterGlobalRequirement, isFilterReferenceableVariable, isFilterRequired, isQueryExpressionFilter, isStaticListFilter, } from '@hyperdx/common-utils/dist/filters'; +import { ReferenceVariableValues } from '@hyperdx/common-utils/dist/referenceVariables'; import { AdhocFilterCondition, ChartVariable, @@ -28,6 +30,7 @@ import { getPendingVariablesTooltip, } from './components/DashboardFilterLabel'; import { FilterLinkToggle } from './components/FilterLinkToggle'; +import { DataSourceDashboardFilter } from './components/ReferenceVariableDashboardFilter'; import { VirtualMultiSelect } from './components/VirtualMultiSelect/VirtualMultiSelect'; import { useDashboardFilterValues } from './hooks/useDashboardFilterValues'; @@ -173,6 +176,13 @@ interface DashboardFilterProps { variables?: ChartVariable[]; /** Required filters with nothing selected, which can block ad hoc filters. */ unsatisfiedRequiredFilters?: DashboardFilter[]; + /** The selected ID of each data source and connection variable, by name. */ + referenceVariableValues?: ReferenceVariableValues; + /** + * The unselected reference variables each filter's fields reference, by + * filter ID. `filters` are otherwise expected with their references resolved. + */ + unresolvedReferenceVariablesByFilterId?: ReadonlyMap; } const DashboardFilters = ({ @@ -184,6 +194,8 @@ const DashboardFilters = ({ onSetAdhocConditions, variables, unsatisfiedRequiredFilters, + referenceVariableValues, + unresolvedReferenceVariablesByFilterId, }: DashboardFilterProps) => { // "Link" mode (opt-in, off by default): each dropdown's values are narrowed by // the others' selections. Off by default because contingent value lookups @@ -191,13 +203,22 @@ const DashboardFilters = ({ // on, all of a source's facets are computed in a single groupUniqArrayIf scan. const [linked, setLinked] = useState(false); + // A filter whose data source variable has no selection has nothing to query yet + const queryableFilters = useMemo( + () => + filters.filter( + filter => !unresolvedReferenceVariablesByFilterId?.has(filter.id), + ), + [filters, unresolvedReferenceVariablesByFilterId], + ); + const { data: filterValuesById, erroredFilterIds, filterErrorMessages, isFetching, } = useDashboardFilterValues({ - filters, + filters: queryableFilters, dateRange, variables, // Only narrow by sibling selections when linked. @@ -214,6 +235,25 @@ const DashboardFilters = ({ return ( {filters.map(filter => { + const unresolvedVariables = + unresolvedReferenceVariablesByFilterId?.get(filter.id) ?? []; + if (isDataSourceFilter(filter)) { + return ( + + onSetFilterValue(filter.id, value ? [value] : []) + } + effect={getFilterEffect(filter)} + requiredTooltip={getRequiredFilterTooltip(filter)} + /> + ); + } if (isAdhocFilter(filter)) { return ( ); } @@ -238,14 +279,17 @@ const DashboardFilters = ({ // Fall back to the hook-level fetching state only until this filter's // query has produced an entry; once it has (even with empty values), // honor its own loading flag. - const isLoadingValues = queriedFilterValues - ? queriedFilterValues.isLoading - : isFetching; + const isLoadingValues = + unresolvedVariables.length === 0 && + (queriedFilterValues ? queriedFilterValues.isLoading : isFetching); // Only a queried dropdown has a values query that can await a - // variable's selection. - const pendingVariables = isQueryExpressionFilter(filter) - ? getPendingFilterValuesVariables(filter, variables) - : undefined; + // variable's selection, besides any data source variable it reads from. + const pendingVariables = [ + ...unresolvedVariables, + ...(isQueryExpressionFilter(filter) + ? getPendingFilterValuesVariables(filter, variables) + : []), + ]; return ( 0; + const allPendingVariables = [...unresolvedVariables, ...pendingVariables]; const toggleAdding = () => { if (isDisabled) return; @@ -136,9 +145,9 @@ export function AdhocDashboardFilter({ testId={`dashboard-filter-blocked-${filter.name}`} /> )} - {!isDisabled && pendingVariables.length > 0 && ( + {!disabledReason && allPendingVariables.length > 0 && ( )} diff --git a/packages/app/src/components/AdhocDashboardFilter/__tests__/AdhocDashboardFilter.test.tsx b/packages/app/src/components/AdhocDashboardFilter/__tests__/AdhocDashboardFilter.test.tsx index 3f767a0e3..a870b9b8f 100644 --- a/packages/app/src/components/AdhocDashboardFilter/__tests__/AdhocDashboardFilter.test.tsx +++ b/packages/app/src/components/AdhocDashboardFilter/__tests__/AdhocDashboardFilter.test.tsx @@ -151,6 +151,25 @@ describe('AdhocDashboardFilter', () => { ); }); + it('waits for an unselected data source variable its sources reference', async () => { + const onChange = renderFilter([existing], filter, { + unresolvedVariables: ['ds'], + }); + + expect( + screen.getByTestId('dashboard-filter-pending-variable-Conditions'), + ).toBeInTheDocument(); + expect(addButton()).toHaveAttribute('aria-disabled', 'true'); + await userEvent.click(addButton()); + await userEvent.click(screen.getByTestId('adhoc-condition-pill')); + expect(screen.queryByTestId(`${editorId}-key`)).not.toBeInTheDocument(); + + await userEvent.click( + screen.getByRole('button', { name: 'Remove filter' }), + ); + expect(onChange).toHaveBeenCalledWith([]); + }); + it('edits a condition from its pill', async () => { const onChange = renderFilter([ existing, diff --git a/packages/app/src/components/DashboardFiltersModal/DashboardFilterEditForm.tsx b/packages/app/src/components/DashboardFiltersModal/DashboardFilterEditForm.tsx index f1a384dd6..cd08b76a2 100644 --- a/packages/app/src/components/DashboardFiltersModal/DashboardFilterEditForm.tsx +++ b/packages/app/src/components/DashboardFiltersModal/DashboardFilterEditForm.tsx @@ -3,6 +3,7 @@ import { useForm, useWatch } from 'react-hook-form'; import { deriveVariableName, getFilterVariableName, + isReferenceVariableFilterType, } from '@hyperdx/common-utils/dist/filters'; import { ChartVariable, @@ -33,6 +34,7 @@ import { useZIndex } from '@/zIndex'; import { AdhocFilterEditForm } from './AdhocFilterEditForm'; import { MODAL_SIZE } from './constants'; import { CustomInputWrapper } from './CustomInputWrapper'; +import { DataSourceFilterEditForm } from './DataSourceFilterEditForm'; import { FilterFormValues, toFormValues, @@ -64,6 +66,7 @@ const FILTER_TYPE_OPTIONS = [ }, ] : []), + { value: DashboardFilterType.enum.DATA_SOURCE, label: 'Data source' }, ]; interface DashboardFilterEditFormProps { @@ -184,9 +187,30 @@ export const DashboardFilterEditForm = ({ return variables?.filter(variable => variable.name !== ownName); }, [variables, filter]); + // Ad hoc conditions and reference variables have fixed selection rules. + // Reference variables are always required, so only the global choice is left. + const isReferenceVariable = isReferenceVariableFilterType(formFilterType); + const showSelectionOptions = + showRequiredFilterOptions && + formFilterType !== 'ADHOC' && + !isReferenceVariable; + const showGlobalRequirementOption = + showRequiredFilterOptions && (isReferenceVariable || !!isRequired); + const isNew = !filter; const showTypeInput = !!showVariableOptions; + const globalRequirementCheckbox = ( + + ); + return ( - {formFilterType === 'STATIC_LIST' ? ( + {formFilterType === 'DATA_SOURCE' ? ( + + ) : formFilterType === 'STATIC_LIST' ? ( - {showRequiredFilterOptions && formFilterType !== 'ADHOC' && ( + {showSelectionOptions && ( - {!!isRequired && ( - - - + {showGlobalRequirementOption && ( + {globalRequirementCheckbox} )} )} + {isReferenceVariable && showGlobalRequirementOption && ( + + + {globalRequirementCheckbox} + + )} {formState.errors.root && ( IS_PROMQL_ENABLED || kind !== SourceKind.Promql) + .map(kind => ({ value: kind, label: SOURCE_KIND_LABELS[kind] })); + +interface DataSourceFilterEditFormProps { + control: FilterFormControl; + /** Filters other than the one being edited, used to keep variable names unique. */ + otherFilters: DashboardFilter[]; +} + +/** Form for editing a variable whose value is a source. */ +export const DataSourceFilterEditForm = ({ + control, + otherFilters, +}: DataSourceFilterEditFormProps) => { + const { errors } = useFormState({ control }); + + return ( + <> + + + (value?.length ?? 0) > 0 || 'Select at least one source kind', + }} + render={({ field: { onChange, value } }) => ( + ( + + {SOURCE_KIND_ICONS[option.value]} + {option.label} + + )} + /> + )} + /> + + + + + ); +}; diff --git a/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts b/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts index 27a9bcb66..172959493 100644 --- a/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts +++ b/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts @@ -1,4 +1,4 @@ -import { DashboardFilter } from '@hyperdx/common-utils/dist/types'; +import { DashboardFilter, SourceKind } from '@hyperdx/common-utils/dist/types'; import { FilterFormValues, @@ -318,6 +318,74 @@ describe('toSavedFilter', () => { ).toThrow(); }); + it('saves a data source filter as single-select and required', () => { + const saved = toSavedFilter( + formValues({ + type: 'DATA_SOURCE', + name: 'Logs source', + allowedSourceKinds: [SourceKind.Log], + isRequired: false, + isSingleSelect: false, + expression: 'ignored', + source: 'ignored', + }), + ); + + expect(saved).toEqual({ + id: expect.any(String), + type: 'DATA_SOURCE', + name: 'Logs source', + variableName: 'Logs_source', + allowedSourceKinds: [SourceKind.Log], + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }); + expect(toFormValues(saved).allowedSourceKinds).toEqual([SourceKind.Log]); + }); + + it('saves a data source filter limited to a connection variable', () => { + expect( + toSavedFilter( + formValues({ + type: 'DATA_SOURCE', + name: 'Logs source', + allowedSourceKinds: [SourceKind.Log], + connection: '$conn', + }), + ), + ).toMatchObject({ connection: '$conn' }); + }); + + it('saves a connection filter as single-select and required', () => { + const saved = toSavedFilter( + formValues({ + type: 'CONNECTION', + name: 'Cluster', + isSingleSelect: false, + source: 'ignored', + }), + ); + + expect(saved).toEqual({ + id: expect.any(String), + type: 'CONNECTION', + name: 'Cluster', + variableName: 'Cluster', + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }); + }); + + it('rejects a data source filter without source kinds', () => { + expect(() => + toSavedFilter(formValues({ type: 'DATA_SOURCE', name: 'Source' })), + ).toThrow(); + }); + it('rejects values the schema cannot accept', () => { expect(() => toSavedFilter(formValues({ type: 'STATIC_LIST', options: [] })), @@ -325,7 +393,10 @@ describe('toSavedFilter', () => { }); }); -/** The minimum a form needs to save, per filter type the form supports. */ +/** + * The minimum a form needs to save, per filter type with configurable + * selection limits. + */ const byType: Record< Exclude< DashboardFilter['type'], @@ -435,6 +506,36 @@ describe('isGlobalRequirement', () => { ).not.toHaveProperty('isGlobalRequirement'); }, ); + + // Reference variables are always required, so the form has no "Required" + // checkbox for them to gate the requirement's scope. + it('round-trips a dashboard-wide data source requirement without isRequired', () => { + const saved = toSavedFilter( + formValues({ + type: 'DATA_SOURCE', + name: 'Source', + allowedSourceKinds: [SourceKind.Log], + isRequired: false, + isGlobalRequirement: true, + }), + ); + + expect(saved).toMatchObject({ isGlobalRequirement: true }); + expect(toFormValues(saved).isGlobalRequirement).toBe(true); + }); + + it('emits no key for a data source filter that blocks only its own tiles', () => { + expect( + toSavedFilter( + formValues({ + type: 'DATA_SOURCE', + name: 'Source', + allowedSourceKinds: [SourceKind.Log], + isGlobalRequirement: false, + }), + ), + ).not.toHaveProperty('isGlobalRequirement'); + }); }); describe('maxSelections', () => { diff --git a/packages/app/src/components/DashboardFiltersModal/filterFormState.ts b/packages/app/src/components/DashboardFiltersModal/filterFormState.ts index 7b681fe2f..caa45a695 100644 --- a/packages/app/src/components/DashboardFiltersModal/filterFormState.ts +++ b/packages/app/src/components/DashboardFiltersModal/filterFormState.ts @@ -5,11 +5,13 @@ import { isFilterGlobalRequirement, isFilterRequired, isFilterVariableEnabled, + isReferenceVariableFilterType, } from '@hyperdx/common-utils/dist/filters'; import { AdhocDashboardFilter, DashboardFilter, DashboardFilterSchema, + DataSourceDashboardFilter, PromqlLabelDashboardFilter, QueryExpressionDashboardFilter, StaticListDashboardFilter, @@ -53,7 +55,8 @@ export type FilterFormValues = { | 'isGlobalRequirement' | 'maxSelections' > & - Pick; + Pick & + Pick; export type FilterFormControl = Control; @@ -65,6 +68,7 @@ export const toFormValues = ( const staticList = filter?.type === 'STATIC_LIST' ? filter : undefined; const promqlLabel = filter?.type === 'PROMETHEUS_LABEL' ? filter : undefined; const adhoc = filter?.type === 'ADHOC' ? filter : undefined; + const dataSource = filter?.type === 'DATA_SOURCE' ? filter : undefined; return { id: filter?.id ?? crypto.randomUUID(), @@ -100,6 +104,10 @@ export const toFormValues = ( // ADHOC fields sourceType: adhoc?.sourceType ?? 'sql', sources: adhoc?.sources ?? [], + + // DATA_SOURCE fields + allowedSourceKinds: dataSource?.allowedSourceKinds ?? [], + connection: dataSource?.connection, }; }; @@ -140,6 +148,28 @@ export const toSavedFilter = (values: FilterFormValues): DashboardFilter => { }); } + // Reference variables are always single-select and required, so only the + // global requirement checkbox is consulted + if (isReferenceVariableFilterType(values.type)) { + return DashboardFilterSchema.parse({ + id: values.id, + type: values.type, + name: values.name, + ...(values.type === 'DATA_SOURCE' + ? { + allowedSourceKinds: values.allowedSourceKinds, + connection: values.connection || undefined, + } + : {}), + minSelections: 1, + maxSelections: 1, + ...(isGlobalRequirement ? { isGlobalRequirement: true } : {}), + isBroadcastEnabled: false, + isVariableEnabled: true, + variableName: getFilterVariableName(values), + }); + } + const appliesTo = values.appliesToSourceIds?.filter(id => !!id?.length); const trimmedWhere = values.where?.trim() ?? ''; const whereFields = { diff --git a/packages/app/src/components/ReferenceVariableDashboardFilter.tsx b/packages/app/src/components/ReferenceVariableDashboardFilter.tsx new file mode 100644 index 000000000..79f2a6782 --- /dev/null +++ b/packages/app/src/components/ReferenceVariableDashboardFilter.tsx @@ -0,0 +1,159 @@ +import { useCallback, useMemo } from 'react'; +import { DataSourceDashboardFilter as DataSourceFilter } from '@hyperdx/common-utils/dist/types'; +import { + Center, + CheckIcon, + ComboboxItem, + Group, + Select, + Stack, + Text, +} from '@mantine/core'; +import { IconStack } from '@tabler/icons-react'; + +import { isValueInSelectData } from '@/components/SelectControlled'; +import { + getSourceOptionIcon, + sourceSelectFilter, + SourceSelectGroup, + useFilteredSortedSourceItems, + useSourceKindMap, +} from '@/components/sourceSelectUtils'; +import { useSources } from '@/source'; + +import { + DashboardFilterLabel, + FilterCaution, + FilterEffect, + getPendingVariablesTooltip, +} from './DashboardFilterLabel'; + +import sourceSelectStyles from '@styles/SourceSelectControlled.module.scss'; + +type ReferenceVariableFilterProps = { + value: string | undefined; + onChange: (id: string | undefined) => void; + effect: FilterEffect; + requiredTooltip: string; +}; + +/** + * The filter bar control for a reference variable: a single picker over the + * IDs the variable can select. + */ +function ReferenceVariableSelect({ + name, + items, + value, + onChange, + effect, + requiredTooltip, + pendingVariables, + placeholder, + renderIcon, +}: ReferenceVariableFilterProps & { + name: string; + items: ComboboxItem[] | SourceSelectGroup[]; + pendingVariables?: string[]; + placeholder: string; + renderIcon: (value: string | null) => React.ReactNode; +}) { + // A selection that is no longer offered (deleted, disabled, or no longer + // allowed by the variable) shows as empty, so it reads as unselected. + const isOffered = useMemo( + () => isValueInSelectData(items, value), + [items, value], + ); + const selected = value && isOffered ? value : null; + + const renderOption = useCallback( + ({ option, checked }: { option: ComboboxItem; checked?: boolean }) => ( + + {checked ? ( +
+ +
+ ) : ( + renderIcon(option.value) + )} + {option.label} +
+ ), + [renderIcon], + ); + + return ( + + + {selected == null && ( + + )} + {!!pendingVariables?.length && ( + + )} + + ); }, @@ -1701,3 +1702,110 @@ describe('DBEditTimeChartForm - Raw SQL data source variables', () => { expect(screen.queryByTestId('preview-blocked')).not.toBeInTheDocument(); }); }); + +describe('DBEditTimeChartForm - Alerts on data source variables', () => { + const metricsVariable: DataSourceDashboardFilter = { + id: 'metrics', + type: 'DATA_SOURCE', + name: 'metrics', + variableName: 'metrics', + allowedSourceKinds: [SourceKind.Metric], + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }; + + const series = { + aggFn: 'avg' as const, + aggCondition: '', + aggConditionLanguage: 'lucene' as const, + valueExpression: 'Value', + metricType: MetricsDataType.Gauge, + metricName: 'test.metric.gauge', + }; + + const completeAlert = { + ...DEFAULT_TILE_ALERT, + channels: [{ type: 'webhook' as const, webhookId: 'webhook-1' }], + }; + + const renderWithAlert = ( + alert: NonNullable, + onSave: jest.Mock, + ) => + renderWithMantine( + + + + + , + ); + + const removalNotice = /This tile's alert will be removed when you save\./; + + beforeEach(() => { + jest.clearAllMocks(); + mockUseSource.mockImplementation(({ id }: { id?: string }) => ({ + data: + id === 'metric-source' + ? { + id: 'metric-source', + kind: SourceKind.Metric, + name: 'Test Metric Source', + from: { databaseName: 'default', tableName: '' }, + connection: 'default', + timestampValueExpression: 'Timestamp', + metricTables: { gauge: 'metrics.gauge' }, + } + : undefined, + })); + }); + + it('keeps the alert when the source goes to a variable and back', async () => { + const onSave = jest.fn(); + renderWithAlert(completeAlert, onSave); + const sourceSelector = screen.getByTestId('source-selector'); + + await userEvent.selectOptions(sourceSelector, '$metrics'); + expect(screen.getByText(removalNotice)).toBeInTheDocument(); + expect(screen.queryByTestId('alert-button')).not.toBeInTheDocument(); + + await userEvent.selectOptions(sourceSelector, 'metric-source'); + expect(screen.queryByText(removalNotice)).not.toBeInTheDocument(); + await userEvent.click(screen.getByTestId('chart-save-button')); + + await waitFor(() => expect(onSave).toHaveBeenCalledTimes(1)); + expect(onSave.mock.calls[0][0].alert).toMatchObject(completeAlert); + }); + + it('saves without the alert while the source is a variable, even an invalid one', async () => { + const onSave = jest.fn(); + // No webhook picked, so the alert alone would block the save + renderWithAlert(DEFAULT_TILE_ALERT, onSave); + + await userEvent.selectOptions( + screen.getByTestId('source-selector'), + '$metrics', + ); + await userEvent.click(screen.getByTestId('chart-save-button')); + + await waitFor(() => expect(onSave).toHaveBeenCalledTimes(1)); + const saved = onSave.mock.calls[0][0]; + expect(saved.source).toBe('$metrics'); + expect(saved.alert).toBeUndefined(); + }); +}); From f2404905cfcf3519e65310c03929577aeb15153f Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Fri, 13 Nov 2026 14:38:45 -0500 Subject: [PATCH 17/18] feat: add connection variables for raw SQL tiles and connection-scoped data source variables --- .changeset/source-vars-5.md | 5 + packages/app/src/DBDashboardImportPage.tsx | 140 ++++++++++-------- packages/app/src/DashboardFilters.tsx | 34 +++-- .../__tests__/DBDashboardImportPage.test.tsx | 64 ++++++++ .../ChartEditor/RawSqlChartEditor.tsx | 27 +++- .../app/src/components/ConnectionSelect.tsx | 50 ++++--- .../DashboardFilterEditForm.tsx | 9 +- .../DataSourceFilterEditForm.tsx | 15 ++ .../DataSourceFilterEditForm.test.tsx | 52 ++++++- .../ReferenceVariableDashboardFilter.tsx | 28 +++- packages/app/src/components/SourceSelect.tsx | 7 + .../ReferenceVariableDashboardFilter.test.tsx | 43 +++++- .../__tests__/SourceSelect.test.tsx | 30 ++++ .../app/src/components/sourceSelectUtils.tsx | 46 ++++-- packages/app/src/hooks/useConnectionItems.ts | 17 +++ .../e2e/components/ChartEditorComponent.ts | 13 ++ .../dashboard-source-variables.spec.ts | 75 +++++++++- .../tests/e2e/page-objects/DashboardPage.ts | 29 +++- 18 files changed, 569 insertions(+), 115 deletions(-) create mode 100644 .changeset/source-vars-5.md create mode 100644 packages/app/src/hooks/useConnectionItems.ts diff --git a/.changeset/source-vars-5.md b/.changeset/source-vars-5.md new file mode 100644 index 000000000..0704548e1 --- /dev/null +++ b/.changeset/source-vars-5.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/app': minor +--- + +feat: add connection variables for raw SQL tiles and connection-scoped data source variables diff --git a/packages/app/src/DBDashboardImportPage.tsx b/packages/app/src/DBDashboardImportPage.tsx index 9f7eda309..749908f98 100644 --- a/packages/app/src/DBDashboardImportPage.tsx +++ b/packages/app/src/DBDashboardImportPage.tsx @@ -24,6 +24,7 @@ import { type DashboardFilterQueryIssue, getAdhocFilterSourceKinds, isAdhocFilter, + isDataSourceFilter, isPrometheusLabelFilter, isQueryExpressionFilter, isSqlQueryFilter, @@ -80,6 +81,7 @@ import { import SelectControlled from './components/SelectControlled'; import { SourceMultiSelectControlled } from './components/SourceMultiSelect'; +import { useConnectionItems } from './hooks/useConnectionItems'; import api from './api'; import { usePageTitle } from './brand'; import { useConnections } from './connection'; @@ -448,8 +450,8 @@ function resolveAppliesToSources( } /** - * A data source variable reference (`$name`) is portable as written, so - * import neither maps it nor asks the user to. + * A data source or connection variable reference (`$name`) is portable as + * written, so import neither maps it nor asks the user to. */ const isVariableRef = (name: string | undefined) => parseVariableReference(name) != null; @@ -467,6 +469,18 @@ const isSourceBackedTemplateFilter = ( (isQueryExpressionFilter(filter) || isPrometheusLabelFilter(filter)) && !isVariableRef(filter.source); +/** + * Returns the filter if it names a connection import must map: a SQL query + * filter, or a data source variable limited to a connection. + */ +const connectionBackedTemplateFilter = (filter: DashboardFilter | undefined) => + filter && + (isSqlQueryFilter(filter) || isDataSourceFilter(filter)) && + filter.connection && + !isVariableRef(filter.connection) + ? filter + : undefined; + /** Returns the filter if it names a source, or `undefined` if it doesn't. */ const sourceBackedTemplateFilter = (filter: DashboardFilter | undefined) => filter && isSourceBackedTemplateFilter(filter) ? filter : undefined; @@ -529,7 +543,7 @@ export function buildMappingFormSchema( connectionMappings: z.array(MappingValueSchema), /** A list of filter source mappings, ordered by input filter index */ filterSourceMappings: z.array(MappingValueSchema).optional(), - /** A list of filter connection mappings, ordered by input filter index. Only applicable for SQL query filters */ + /** Connection mappings of SQL query filters and data source variables, ordered by input filter index */ filterConnectionMappings: z.array(MappingValueSchema).optional(), /** * Per-filter applies-to source mappings. Each entry is an array of mapped @@ -572,7 +586,11 @@ export function buildMappingFormSchema( } // A raw SQL tile always gets a connection row, and its connection is // not optional — leaving it unpicked used to throw on `connection!.id`. - if (isRawSqlSavedChartConfig(config) && !data.connectionMappings[idx]) { + if ( + isRawSqlSavedChartConfig(config) && + !isVariableRef(config.connection) && + !data.connectionMappings[idx] + ) { ctx.addIssue({ code: z.ZodIssueCode.custom, path: ['connectionMappings', idx], @@ -588,6 +606,16 @@ export function buildMappingFormSchema( }); input.filters?.forEach((filter, idx) => { + if ( + connectionBackedTemplateFilter(filter) && + !data.filterConnectionMappings?.[idx] + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['filterConnectionMappings', idx], + message: 'Select a connection for this filter', + }); + } if (isAdhocFilter(filter)) { const kinds = getAdhocFilterSourceKinds(filter.sourceType); const kindMessage = @@ -623,17 +651,6 @@ export function buildMappingFormSchema( return; } - if (isSqlQueryFilter(filter)) { - if (!data.filterConnectionMappings?.[idx]) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ['filterConnectionMappings', idx], - message: 'Select a connection for this filter', - }); - } - return; - } - if (!isSourceBackedTemplateFilter(filter)) return; const mappedSourceId = data.filterSourceMappings?.[idx]; if (!mappedSourceId) { @@ -670,6 +687,7 @@ export function Mapping({ input }: { input: DashboardTemplate }) { const router = useRouter(); const { data: sources } = useSources(); const { data: connections } = useConnections(); + const connectionItems = useConnectionItems(); const { data: dashboards } = useDashboards(); const { data: existingTags } = api.useTags(); const [dashboardId] = useQueryState('dashboardId', parseAsString); @@ -722,17 +740,19 @@ export function Mapping({ input }: { input: DashboardTemplate }) { const findConnectionIdByName = (name: string) => connections.find(conn => conn.name.toLowerCase() === name.toLowerCase()) - ?.id || ''; + ?.id ?? ''; const connectionMappings = input.tiles.map(tile => { const config = tile.config as SavedChartConfig; - if (!isRawSqlSavedChartConfig(config)) return ''; + if (!isRawSqlSavedChartConfig(config) || isVariableRef(config.connection)) + return ''; return findConnectionIdByName(config.connection); }); - const filterConnectionMappings = input.filters?.map(filter => - isSqlQueryFilter(filter) ? findConnectionIdByName(filter.connection) : '', - ); + const filterConnectionMappings = input.filters?.map(filter => { + const connection = connectionBackedTemplateFilter(filter)?.connection; + return connection ? findConnectionIdByName(connection) : ''; + }); const filterSourceMappings = input.filters?.map(filter => { const sourceName = sourceBackedTemplateFilter(filter)?.source; @@ -791,8 +811,8 @@ export function Mapping({ input }: { input: DashboardTemplate }) { setValue('tileSourceMappings', tileSourceMappings); setValue('connectionMappings', connectionMappings); - setValue('filterSourceMappings', filterSourceMappings); setValue('filterConnectionMappings', filterConnectionMappings); + setValue('filterSourceMappings', filterSourceMappings); multiMappingSlotsRef.current.clear(); filterAppliesToSourceMappings?.forEach((mapping, idx) => { setValue(`filterAppliesToSourceMappings.${idx}`, mapping); @@ -983,8 +1003,8 @@ export function Mapping({ input }: { input: DashboardTemplate }) { sources, ]); - // Propagate connection mapping changes to other RawSQL tiles and SQL query - // filters with the same input connection + // Propagate connection mapping changes to other RawSQL tiles and filters + // with the same input connection useEffect(() => { if (isUpdatingRef.current) return; if (!input.tiles) return; @@ -1017,9 +1037,11 @@ export function Mapping({ input }: { input: DashboardTemplate }) { ); if (filterIdx !== -1) { prevFilterConnectionMappingsRef.current = filterConnectionMappings; - const filter = input.filters?.[filterIdx]; - if (filter && isSqlQueryFilter(filter)) { - inputConnectionName = filter.connection; + const connection = connectionBackedTemplateFilter( + input.filters?.[filterIdx], + )?.connection; + if (connection) { + inputConnectionName = connection; connectionId = filterConnectionMappings[filterIdx] ?? ''; } } @@ -1044,8 +1066,8 @@ export function Mapping({ input }: { input: DashboardTemplate }) { ?.map((filter, index) => ({ filter, index })) .filter( ({ filter }) => - isSqlQueryFilter(filter) && - filter.connection === inputConnectionName, + connectionBackedTemplateFilter(filter)?.connection === + inputConnectionName, ) .map(({ index }) => `filterConnectionMappings.${index}` as const) ?? []; @@ -1159,12 +1181,14 @@ export function Mapping({ input }: { input: DashboardTemplate }) { }; if (isRawSqlSavedChartConfig(tile.config)) { - const connection = findConnection(data.connectionMappings[idx]); + const connection = isVariableRef(tile.config.connection) + ? tile.config.connection + : findConnection(data.connectionMappings[idx])!.id; return { ...tile, config: applyOnClick({ ...tile.config, - connection: connection!.id, + connection, ...(source ? { source: source.id } : {}), }), }; @@ -1206,11 +1230,13 @@ export function Mapping({ input }: { input: DashboardTemplate }) { appliesToSourceIds, }; } - if (isSqlQueryFilter(filter)) { - const connection = findConnection( - data.filterConnectionMappings?.[idx], - ); - return { ...filter, connection: connection!.id }; + if (connectionBackedTemplateFilter(filter)) { + return { + ...filter, + // The form schema requires this mapping + connection: findConnection(data.filterConnectionMappings?.[idx])! + .id, + }; } const source = findSource(data.filterSourceMappings?.[idx]); return { @@ -1322,7 +1348,7 @@ export function Mapping({ input }: { input: DashboardTemplate }) { )} {/** Mapping for the tile's connection, if it's a raw sql tile */} - {isRawSql && ( + {isRawSql && !isVariableRef(config.connection) && ( {tile.config.name} Data connection @@ -1331,10 +1357,7 @@ export function Mapping({ input }: { input: DashboardTemplate }) { ({ - value: conn.id, - label: conn.name, - }))} + data={connectionItems} placeholder="Select a connection" /> @@ -1394,6 +1417,25 @@ export function Mapping({ input }: { input: DashboardTemplate }) { : undefined; return ( + {connectionBackedTemplateFilter(filter) && ( + + {filter.name} (Filter) + Data connection + + {connectionBackedTemplateFilter(filter)?.connection} + + + + + + + + )} {isSourceBackedTemplateFilter(filter) && ( {filter.name} (Filter) @@ -1418,24 +1460,6 @@ export function Mapping({ input }: { input: DashboardTemplate }) { )} - {isSqlQueryFilter(filter) && ( - - {filter.name} (Filter) - Data connection - {filter.connection} - - ({ - value: conn.id, - label: conn.name, - }))} - placeholder="Select a connection" - /> - - - )} {isAdhocFilter(filter) && !!adhocSourceNames?.length && ( {filter.name} (Filter) diff --git a/packages/app/src/DashboardFilters.tsx b/packages/app/src/DashboardFilters.tsx index 3e155c733..ff17338cf 100644 --- a/packages/app/src/DashboardFilters.tsx +++ b/packages/app/src/DashboardFilters.tsx @@ -10,6 +10,7 @@ import { isFilterReferenceableVariable, isFilterRequired, isQueryExpressionFilter, + isReferenceVariableFilter, isStaticListFilter, } from '@hyperdx/common-utils/dist/filters'; import { ReferenceVariableValues } from '@hyperdx/common-utils/dist/referenceVariables'; @@ -31,7 +32,10 @@ import { getPendingVariablesTooltip, } from './components/DashboardFilterLabel'; import { FilterLinkToggle } from './components/FilterLinkToggle'; -import { DataSourceDashboardFilter } from './components/ReferenceVariableDashboardFilter'; +import { + ConnectionDashboardFilter, + DataSourceDashboardFilter, +} from './components/ReferenceVariableDashboardFilter'; import { VirtualMultiSelect } from './components/VirtualMultiSelect/VirtualMultiSelect'; import { useDashboardFilterValues } from './hooks/useDashboardFilterValues'; @@ -248,20 +252,28 @@ const DashboardFilters = ({ const unselectedVariables = unresolvedVariables.filter( name => !incompatibleVariables.includes(name), ); - if (isDataSourceFilter(filter)) { - return ( + if (isReferenceVariableFilter(filter)) { + const referenceProps = { + value: referenceVariableValues?.get( + getFilterVariableName(filter) ?? '', + ), + onChange: (value: string | undefined) => + onSetFilterValue(filter.id, value ? [value] : []), + effect: getFilterEffect(filter), + requiredTooltip: getRequiredFilterTooltip(filter), + }; + return isDataSourceFilter(filter) ? ( - onSetFilterValue(filter.id, value ? [value] : []) - } - effect={getFilterEffect(filter)} - requiredTooltip={getRequiredFilterTooltip(filter)} + {...referenceProps} + /> + ) : ( + ); } diff --git a/packages/app/src/__tests__/DBDashboardImportPage.test.tsx b/packages/app/src/__tests__/DBDashboardImportPage.test.tsx index 1083ba8ca..81502709f 100644 --- a/packages/app/src/__tests__/DBDashboardImportPage.test.tsx +++ b/packages/app/src/__tests__/DBDashboardImportPage.test.tsx @@ -811,3 +811,67 @@ describe('Dashboard import - data source variables', () => { ]); }); }); + +describe('Dashboard import - connection variables', () => { + const connectionVariableTemplate = DashboardTemplateSchema.parse({ + version: '0.1.0', + name: 'Connection Variables', + tiles: [ + { + id: 'tile-rawsql', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + name: 'tile-rawsql', + displayType: 'table', + configType: 'sql', + connection: '$conn', + sqlTemplate: 'SELECT 1', + }, + }, + ], + filters: [ + { + id: 'conn', + type: 'CONNECTION', + name: 'Cluster', + variableName: 'conn', + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }, + { + id: 'ds', + type: 'DATA_SOURCE', + name: 'Logs source', + variableName: 'logs', + allowedSourceKinds: [SourceKind.Log], + connection: 'Default', + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }, + ], + }); + + it('keeps connection references and maps a data source variable connection', async () => { + renderWithMantine(); + + const finish = await screen.findByRole('button', { + name: /finish import/i, + }); + expect(screen.queryByText('$conn')).not.toBeInTheDocument(); + await act(async () => { + fireEvent.click(finish); + }); + + await waitFor(() => expect(mockMutateAsync).toHaveBeenCalledTimes(1)); + const payload = mockMutateAsync.mock.calls[0][0]; + expect(payload.tiles[0].config.connection).toBe('$conn'); + expect(payload.filters[1].connection).toBe('conn-default'); + }); +}); diff --git a/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx b/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx index bfc60685a..895c1685b 100644 --- a/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx +++ b/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx @@ -154,10 +154,11 @@ export default function RawSqlChartEditor({ const { data: sources } = useSources(); const displayType = useWatch({ control, name: 'displayType' }); - const connection = useWatch({ control, name: 'connection' }); + // Source and connection variable references autocomplete and expand macros + // as the source and connection they currently select. + const storedConnection = useWatch({ control, name: 'connection' }); + const connection = useResolvedReference(storedConnection); const storedSource = useWatch({ control, name: 'source' }); - // A data source variable reference expands macros as the source it - // currently selects. const source = useResolvedReference(storedSource); const sqlTemplate = useWatch({ control, name: 'sqlTemplate' }); const chartName = useWatch({ control, name: 'name' }); @@ -217,12 +218,25 @@ export default function RawSqlChartEditor({ prevConnection, sources, }); - if (update?.field === 'connection') { + // A connection variable stays as picked; the tile shows a mismatch + // instead of having its reference replaced + if ( + update?.field === 'connection' && + parseVariableReference(storedConnection) == null + ) { setValue('connection', update.value); } else if (update?.field === 'source') { setValue('source', update.value); } - }, [connection, prevConnection, prevSource, setValue, storedSource, sources]); + }, [ + connection, + storedConnection, + prevConnection, + prevSource, + setValue, + storedSource, + sources, + ]); const placeholderSQl = SQL_PLACEHOLDERS[displayType ?? DisplayType.Table]; @@ -284,6 +298,8 @@ export default function RawSqlChartEditor({ control={control} name="connection" size="xs" + allowConnectionVariables + data-testid="connection-selector" /> @@ -305,6 +321,7 @@ export default function RawSqlChartEditor({ allowedSourceKinds={allowedSourceKinds} allowSourceVariables connectionId={connection} + variableConnection={storedConnection} size="xs" clearable placeholder="None" diff --git a/packages/app/src/components/ConnectionSelect.tsx b/packages/app/src/components/ConnectionSelect.tsx index 2422377cf..eccaacc5b 100644 --- a/packages/app/src/components/ConnectionSelect.tsx +++ b/packages/app/src/components/ConnectionSelect.tsx @@ -1,35 +1,51 @@ -import { useMemo } from 'react'; -import { UseControllerProps } from 'react-hook-form'; +import { UseControllerProps, useWatch } from 'react-hook-form'; +import { parseVariableReference } from '@hyperdx/common-utils/dist/referenceVariables'; +import { SelectProps } from '@mantine/core'; import { IconServer } from '@tabler/icons-react'; import SelectControlled from '@/components/SelectControlled'; -import { useConnections } from '@/connection'; +import { + useWithConnectionVariableItems, + VARIABLE_ICON, +} from '@/components/sourceSelectUtils'; +import { useConnectionItems } from '@/hooks/useConnectionItems'; export function ConnectionSelectControlled({ size, + allowConnectionVariables = false, + placeholder = 'Connection', + clearable = false, ...props -}: { size?: string } & UseControllerProps) { - const { data } = useConnections(); - - const values = useMemo( - () => - data?.map(d => ({ - value: d.id, - label: d.name, - })), - [data], +}: { + size?: string; + /** Whether to offer the dashboard's connection variables. */ + allowConnectionVariables?: boolean; +} & Pick & + UseControllerProps) { + const selected = useWatch({ control: props.control, name: props.name }); + const connectionItems = useConnectionItems(); + const items = useWithConnectionVariableItems( + connectionItems, + allowConnectionVariables, ); return ( } + placeholder={placeholder} + leftSection={ + parseVariableReference(selected) != null ? ( + VARIABLE_ICON + ) : ( + + ) + } maxDropdownHeight={280} size={size} /> diff --git a/packages/app/src/components/DashboardFiltersModal/DashboardFilterEditForm.tsx b/packages/app/src/components/DashboardFiltersModal/DashboardFilterEditForm.tsx index cd08b76a2..cb8c5d784 100644 --- a/packages/app/src/components/DashboardFiltersModal/DashboardFilterEditForm.tsx +++ b/packages/app/src/components/DashboardFiltersModal/DashboardFilterEditForm.tsx @@ -43,6 +43,7 @@ import { import { PromqlLabelFilterEditForm } from './PromqlLabelFilterEditForm'; import { QueryExpressionFilterEditForm } from './QueryExpressionFilterEditForm'; import { StaticListFilterEditForm } from './StaticListFilterEditForm'; +import { VariableNameInput } from './VariableNameInput'; const FILTER_TYPE_OPTIONS = [ { @@ -67,6 +68,7 @@ const FILTER_TYPE_OPTIONS = [ ] : []), { value: DashboardFilterType.enum.DATA_SOURCE, label: 'Data source' }, + { value: DashboardFilterType.enum.CONNECTION, label: 'Connection' }, ]; interface DashboardFilterEditFormProps { @@ -260,7 +262,12 @@ export const DashboardFilterEditForm = ({ message="Failed to load filter" showErrorMessage > - {formFilterType === 'DATA_SOURCE' ? ( + {formFilterType === 'CONNECTION' ? ( + + ) : formFilterType === 'DATA_SOURCE' ? ( + + + + {hasMixedKinds && ( ({ +jest.mock('@/connection', () => ({ + useConnections: () => ({ data: [{ id: 'conn-1', name: 'Primary' }] }), +})); + +function Harness({ + allowedSourceKinds, + connection, + onSave, +}: { + allowedSourceKinds: SourceKind[]; + connection?: string; + onSave?: (values: FilterFormValues) => void; +}) { + const { control, getValues } = useForm({ defaultValues: { ...toFormValues(), type: 'DATA_SOURCE', + name: 'Logs source', allowedSourceKinds, + connection, }, }); - return ; + return ( + <> + + + + ); } const warning = () => screen.queryByTestId('filter-mixed-source-kinds-warning'); @@ -35,4 +58,27 @@ describe('DataSourceFilterEditForm', () => { ); expect(warning()).not.toBeInTheDocument(); }); + + it('saves no connection once the connection is cleared', async () => { + const onSave = jest.fn(); + renderWithMantine( + , + ); + + const input = screen.getByTestId('filter-data-source-connection-input'); + expect(input).toHaveValue('Primary'); + const clearButton = input.parentElement?.querySelector('button'); + if (!clearButton) throw new Error('Expected a clear button'); + await userEvent.click(clearButton); + expect(input).toHaveValue(''); + await userEvent.click(screen.getByRole('button', { name: 'Save' })); + + expect(toSavedFilter(onSave.mock.calls[0][0])).toMatchObject({ + connection: undefined, + }); + }); }); diff --git a/packages/app/src/components/ReferenceVariableDashboardFilter.tsx b/packages/app/src/components/ReferenceVariableDashboardFilter.tsx index 27b494358..0f163c5f1 100644 --- a/packages/app/src/components/ReferenceVariableDashboardFilter.tsx +++ b/packages/app/src/components/ReferenceVariableDashboardFilter.tsx @@ -1,6 +1,7 @@ import { useCallback, useMemo } from 'react'; import { isSourceOfferedByVariable } from '@hyperdx/common-utils/dist/referenceVariables'; import { + ConnectionDashboardFilter as ConnectionFilter, DataSourceDashboardFilter as DataSourceFilter, TSource, } from '@hyperdx/common-utils/dist/types'; @@ -13,7 +14,7 @@ import { Stack, Text, } from '@mantine/core'; -import { IconStack } from '@tabler/icons-react'; +import { IconServer, IconStack } from '@tabler/icons-react'; import { isValueInSelectData } from '@/components/SelectControlled'; import { @@ -23,6 +24,7 @@ import { useFilteredSortedSourceItems, useSourceKindMap, } from '@/components/sourceSelectUtils'; +import { useConnectionItems } from '@/hooks/useConnectionItems'; import { useSources } from '@/source'; import { @@ -42,8 +44,8 @@ type ReferenceVariableFilterProps = { }; /** - * The filter bar control for a reference variable: a single picker over the - * IDs the variable can select. + * The filter bar control shared by data source and connection variables: a + * single picker over the IDs the variable can select. */ function ReferenceVariableSelect({ name, @@ -164,3 +166,23 @@ export const DataSourceDashboardFilter = ({ /> ); }; + +const renderConnectionIcon = () => ; + +/** A connection variable's picker. */ +export const ConnectionDashboardFilter = ({ + filter, + ...props +}: ReferenceVariableFilterProps & { filter: ConnectionFilter }) => { + const items = useConnectionItems({ sorted: true }); + + return ( + + ); +}; diff --git a/packages/app/src/components/SourceSelect.tsx b/packages/app/src/components/SourceSelect.tsx index 2dd72a808..845b7e1cf 100644 --- a/packages/app/src/components/SourceSelect.tsx +++ b/packages/app/src/components/SourceSelect.tsx @@ -163,6 +163,7 @@ function SourceSelectControlledComponent({ connectionId, isSourceAllowed, allowSourceVariables = false, + variableConnection, comboboxProps, ...props }: { @@ -178,6 +179,11 @@ function SourceSelectControlledComponent({ isSourceAllowed?: (source: TSource) => boolean; /** Whether to offer the dashboard's data source variables. */ allowSourceVariables?: boolean; + /** + * The connection data source variables must be limited to, as stored (it + * may be a connection variable reference). Defaults to `connectionId`. + */ + variableConnection?: string; } & UseControllerProps & SelectProps) { const { data } = useSources(); @@ -221,6 +227,7 @@ function SourceSelectControlledComponent({ const items = useWithSourceVariableItems(sourceItems, { enabled: allowSourceVariables, allowedSourceKinds, + connection: variableConnection ?? connectionId, }); const hasSelection = !!selectedSourceId; diff --git a/packages/app/src/components/__tests__/ReferenceVariableDashboardFilter.test.tsx b/packages/app/src/components/__tests__/ReferenceVariableDashboardFilter.test.tsx index 10ab34464..b09fe5a5b 100644 --- a/packages/app/src/components/__tests__/ReferenceVariableDashboardFilter.test.tsx +++ b/packages/app/src/components/__tests__/ReferenceVariableDashboardFilter.test.tsx @@ -5,11 +5,22 @@ import { import { screen } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; -import { DataSourceDashboardFilter } from '@/components/ReferenceVariableDashboardFilter'; +import { + ConnectionDashboardFilter, + DataSourceDashboardFilter, +} from '@/components/ReferenceVariableDashboardFilter'; jest.mock('@/source', () => ({ useSources: () => ({ data: mockSources }), })); +jest.mock('@/connection', () => ({ + useConnections: () => ({ + data: [ + { id: 'conn-b', name: 'Secondary' }, + { id: 'conn-a', name: 'Primary' }, + ], + }), +})); // Mantine's Combobox calls scrollIntoView when its dropdown opens; jsdom lacks it. window.HTMLElement.prototype.scrollIntoView = jest.fn(); @@ -106,3 +117,33 @@ describe('DataSourceDashboardFilter', () => { ).toBeInTheDocument(); }); }); + +describe('ConnectionDashboardFilter', () => { + it('offers the connections and reports the picked one', async () => { + const onChange = jest.fn(); + renderWithMantine( + , + ); + + await userEvent.click( + screen.getByTestId('dashboard-filter-select-Cluster'), + ); + await userEvent.click(await screen.findByText('Secondary')); + expect(onChange).toHaveBeenCalledWith('conn-b'); + }); +}); diff --git a/packages/app/src/components/__tests__/SourceSelect.test.tsx b/packages/app/src/components/__tests__/SourceSelect.test.tsx index b4dbdbbf0..90c17febc 100644 --- a/packages/app/src/components/__tests__/SourceSelect.test.tsx +++ b/packages/app/src/components/__tests__/SourceSelect.test.tsx @@ -333,6 +333,36 @@ describe('SourceSelectControlled (data source variables)', () => { expect(screen.getByText('Sources')).toBeInTheDocument(); }); + it('offers only variables unlimited or limited to its connection', async () => { + asMock(useSources).mockReturnValue({ data: SOURCES }); + function ConnectionHarness() { + const { control } = useForm(); + return ( + + + + ); + } + renderWithMantine(); + await userEvent.click(screen.getByPlaceholderText('Data Source')); + + expect(await screen.findByText('$any')).toBeInTheDocument(); + expect(screen.getByText('$mine')).toBeInTheDocument(); + expect(screen.queryByText('$other')).not.toBeInTheDocument(); + }); + it('offers no variables unless the picker opts in', async () => { asMock(useSources).mockReturnValue({ data: SOURCES }); renderWithMantine(); diff --git a/packages/app/src/components/sourceSelectUtils.tsx b/packages/app/src/components/sourceSelectUtils.tsx index 4a06f779b..04c555c48 100644 --- a/packages/app/src/components/sourceSelectUtils.tsx +++ b/packages/app/src/components/sourceSelectUtils.tsx @@ -1,5 +1,6 @@ import { createContext, use, useMemo } from 'react'; import { + getConnectionVariableNames, getSourceVariableDeclarations, isSourceVariableAccepted, parseVariableReference, @@ -30,7 +31,7 @@ export const SOURCE_KIND_ICONS: Record = { [SourceKind.Promql]: , }; -const VARIABLE_ICON = ; +export const VARIABLE_ICON = ; export function getSourceOptionIcon( value: string | undefined, @@ -42,16 +43,17 @@ export function getSourceOptionIcon( type ReferenceVariablesContextValue = { sourceVariables: SourceVariableDeclaration[]; + connectionVariables: string[]; values: ReferenceVariableValues; }; const ReferenceVariablesContext = createContext( - { sourceVariables: [], values: new Map() }, + { sourceVariables: [], connectionVariables: [], values: new Map() }, ); /** - * Offers the data source variables `filters` declare to the pickers below it - * that opt in, and their selections to `useResolvedReference`. + * Offers the data source and connection variables `filters` declare to the + * pickers below it that opt in, and their selections to `useResolvedReference`. */ export function ReferenceVariablesProvider({ filters, @@ -65,6 +67,7 @@ export function ReferenceVariablesProvider({ const value = useMemo( () => ({ sourceVariables: getSourceVariableDeclarations(filters), + connectionVariables: getConnectionVariableNames(filters), values, }), [filters, values], @@ -77,8 +80,8 @@ export function ReferenceVariablesProvider({ } /** - * The ID a source field currently queries: an ID as is, or the selection of - * the variable it references. + * The ID a source or connection field currently queries: an ID as is, or the + * selection of the variable it references. */ export function useResolvedReference( value: string | undefined, @@ -103,17 +106,20 @@ const toVariableItem = (name: string): ComboboxItem => { /** * A source picker's items, preceded by a "Variables" group of the data source * variables it can offer. A variable is offered if at least one kind it can - * resolve to is allowed; a selection of another kind shows as an error where - * the reference is used. + * resolve to is allowed, and, for a picker limited to `connection`, only if it + * is unlimited or limited to that same connection (ID or variable reference). + * A selection of another kind shows as an error where the reference is used. */ export function useWithSourceVariableItems( sourceItems: ComboboxItem[] | SourceSelectGroup[], { enabled, allowedSourceKinds, + connection, }: { enabled: boolean; allowedSourceKinds?: SourceKind[]; + connection?: string; }, ): (ComboboxItem | SourceSelectGroup)[] { const { sourceVariables } = use(ReferenceVariablesContext); @@ -122,8 +128,11 @@ export function useWithSourceVariableItems( const items = sourceVariables .filter( variable => - !allowedSourceKinds || - isSourceVariableAccepted(variable, allowedSourceKinds), + (!allowedSourceKinds || + isSourceVariableAccepted(variable, allowedSourceKinds)) && + (!connection || + !variable.connection || + variable.connection === connection), ) .map(variable => toVariableItem(variable.name)); if (items.length === 0) return sourceItems; @@ -138,7 +147,22 @@ export function useWithSourceVariableItems( ? [{ group: 'Sources', items: ungrouped }] : sourceItems), ]; - }, [enabled, sourceVariables, allowedSourceKinds, sourceItems]); + }, [enabled, sourceVariables, allowedSourceKinds, connection, sourceItems]); +} + +/** A connection picker's items, preceded by a "Variables" group of connection variables. */ +export function useWithConnectionVariableItems( + connectionItems: ComboboxItem[], + enabled: boolean, +): (ComboboxItem | SourceSelectGroup)[] { + const { connectionVariables } = use(ReferenceVariablesContext); + return useMemo(() => { + if (!enabled || connectionVariables.length === 0) return connectionItems; + return [ + { group: 'Variables', items: connectionVariables.map(toVariableItem) }, + { group: 'Connections', items: connectionItems }, + ]; + }, [enabled, connectionVariables, connectionItems]); } /** Header for sources that have no section assigned. Internal to this module. */ diff --git a/packages/app/src/hooks/useConnectionItems.ts b/packages/app/src/hooks/useConnectionItems.ts new file mode 100644 index 000000000..811bb0b16 --- /dev/null +++ b/packages/app/src/hooks/useConnectionItems.ts @@ -0,0 +1,17 @@ +import { useMemo } from 'react'; + +import { useConnections } from '@/connection'; + +/** The team's connections as select options, optionally sorted by name. */ +export function useConnectionItems({ sorted = false } = {}) { + const { data } = useConnections(); + return useMemo(() => { + const items = (data ?? []).map(connection => ({ + value: connection.id, + label: connection.name, + })); + return sorted + ? items.sort((a, b) => a.label.localeCompare(b.label)) + : items; + }, [data, sorted]); +} diff --git a/packages/app/tests/e2e/components/ChartEditorComponent.ts b/packages/app/tests/e2e/components/ChartEditorComponent.ts index fb24092b7..25e68e9bd 100644 --- a/packages/app/tests/e2e/components/ChartEditorComponent.ts +++ b/packages/app/tests/e2e/components/ChartEditorComponent.ts @@ -446,6 +446,19 @@ export class ChartEditorComponent { } } + /** The raw SQL editor's connection picker input. */ + get connection(): Locator { + return this.editorForm().getByTestId('connection-selector'); + } + + /** Pick the raw SQL editor's connection, by option label. */ + async selectConnection(connectionName: string) { + await this.connection.click(); + await this.page + .getByRole('option', { name: connectionName, exact: true }) + .click(); + } + /** Open the Data Source dropdown. */ async openSourcePicker() { await this.sourceSelector.click(); diff --git a/packages/app/tests/e2e/features/dashboard-source-variables.spec.ts b/packages/app/tests/e2e/features/dashboard-source-variables.spec.ts index 8c5d73610..89c61fc55 100644 --- a/packages/app/tests/e2e/features/dashboard-source-variables.spec.ts +++ b/packages/app/tests/e2e/features/dashboard-source-variables.spec.ts @@ -1,12 +1,16 @@ /** - * A `DATA_SOURCE` dashboard filter is a single-select variable whose value is - * a source. Tiles and other filters reference it as `$name` in place of a - * source, block until it has a selection, and cannot carry alerts. + * `DATA_SOURCE` and `CONNECTION` dashboard filters are single-select variables + * whose value is a source or a connection. Tiles and other filters reference + * them as `$name` in place of an ID, block until they have a selection, and + * cannot carry alerts. */ +import { DisplayType } from '@hyperdx/common-utils/dist/types'; + import { DashboardPage } from '../page-objects/DashboardPage'; import { SERVICES } from '../seed-clickhouse'; import { expect, test } from '../utils/base-test'; import { + DEFAULT_CONNECTION_NAME, DEFAULT_LOGS_SOURCE_NAME, DEFAULT_METRICS_SOURCE_NAME, DEFAULT_TRACES_SOURCE_NAME, @@ -187,6 +191,71 @@ test.describe( }); }); + test('drives a raw SQL tile from connection and source variables', async () => { + const editor = dashboardPage.chartEditor; + + await test.step('Create a connection variable and a source variable limited to it', async () => { + await dashboardPage.createNewDashboard(); + await dashboardPage.openEditFiltersModal(); + await dashboardPage.addConnectionFilterToDashboard( + 'Cluster', + 'cluster', + ); + await dashboardPage.addDataSourceFilterToDashboard( + 'Logs source', + ['Logs'], + 'logs', + { connectionLabel: '$cluster' }, + ); + await expect( + dashboardPage.getFilterItemByName('Logs source'), + ).toContainText('$cluster'); + await dashboardPage.closeFiltersModal(); + await expect( + dashboardPage.getFilterPendingVariableWarning('Logs source'), + ).toBeVisible(); + }); + + await test.step('A raw SQL tile on the variables cannot have an alert', async () => { + await dashboardPage.addTile(); + await expect(editor.nameInput).toBeVisible(); + await editor.waitForDataToLoad(); + await editor.setChartType(DisplayType.Number); + await editor.setChartName('Variable count'); + await editor.switchToSqlMode(); + await expect(editor.alertButton).toBeVisible(); + await editor.selectConnection('$cluster'); + await editor.selectSource('$logs'); + await expect(editor.alertButton).toHaveCount(0); + await editor.typeSqlQuery('SELECT count() FROM $__sourceTable'); + await editor.save(); + await expect( + dashboardPage.getTileMissingRequiredFilters(0), + ).toContainText('Cluster'); + }); + + await test.step('Selecting a connection, then a source on it, loads the tile', async () => { + await dashboardPage.selectReferenceFilterValue( + 'Cluster', + DEFAULT_CONNECTION_NAME, + ); + await dashboardPage.selectReferenceFilterValue( + 'Logs source', + DEFAULT_LOGS_SOURCE_NAME, + ); + await expect( + dashboardPage.getTileMissingRequiredFilters(0), + ).toHaveCount(0); + await expect(dashboardPage.getNumberTileValue(0)).toBeVisible(); + }); + + await test.step('The editor keeps both references', async () => { + await dashboardPage.editTile(0); + await expect(editor.connection).toHaveValue('$cluster'); + await expect(editor.source).toHaveValue('$logs'); + }); + }); + // A filter edit can leave other filters with a variable they can't always // use. It still saves, and the filter shows an error while it can't. test("saves a kind other filters can't use, and flags them while it's selected", async ({ diff --git a/packages/app/tests/e2e/page-objects/DashboardPage.ts b/packages/app/tests/e2e/page-objects/DashboardPage.ts index 86130cde3..71b1b843a 100644 --- a/packages/app/tests/e2e/page-objects/DashboardPage.ts +++ b/packages/app/tests/e2e/page-objects/DashboardPage.ts @@ -1404,7 +1404,8 @@ export class DashboardPage { | 'Static values' | 'PromQL label values' | 'Ad hoc keys and values' - | 'Data source', + | 'Data source' + | 'Connection', ) { await this.getFilterTypePicker().click(); await this.getFilterOption(label).click(); @@ -1493,7 +1494,9 @@ export class DashboardPage { name: string, kindLabels: string[], variableName: string, - options?: Pick, + options?: Pick & { + connectionLabel?: string; + }, ) { await this.addFiltersButton.click(); await this.selectFilterType('Data source'); @@ -1505,6 +1508,15 @@ export class DashboardPage { await kindsInput.click(); await this.getFilterOption(label).click(); } + if (options?.connectionLabel !== undefined) { + // Close the kinds dropdown, which covers this input; Escape would close + // the modal instead + await this.getFilterNameInput().click(); + await this.page + .getByTestId('filter-data-source-connection-input') + .click(); + await this.getFilterOption(options.connectionLabel).click(); + } await this.variableNameInput.fill(variableName); if (options?.globalRequirement !== undefined) { await this.globalRequirementCheckbox.setChecked( @@ -1518,6 +1530,19 @@ export class DashboardPage { }); } + /** Add a connection variable and wait for it in the filters list. */ + async addConnectionFilterToDashboard(name: string, variableName: string) { + await this.addFiltersButton.click(); + await this.selectFilterType('Connection'); + await this.getFilterNameInput().fill(name); + await this.variableNameInput.fill(variableName); + await this.page.getByTestId('save-filter-button').click(); + await this.getFilterItemByName(name).waitFor({ + state: 'visible', + timeout: 10000, + }); + } + /** Pick a value in a data source or connection variable's filter bar select. */ async selectReferenceFilterValue(filterName: string, optionLabel: string) { await this.getFilterSelectByName(filterName).click(); From 0a3f111c83ecdacb2919562ace59c98f9c5b2a67 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Tue, 17 Nov 2026 08:21:19 -0500 Subject: [PATCH 18/18] feat: let SQL query filters run on a connection variable --- .changeset/source-vars-5.md | 5 ++- packages/api/openapi.json | 4 +- .../dashboards/saveDashboard.int.test.ts | 20 +++++++++ .../api/src/mcp/prompts/dashboards/content.ts | 2 +- .../api/src/mcp/tools/dashboards/schemas.ts | 8 ++-- .../api/__tests__/dashboard.int.test.ts | 36 ++++++++++++++++ packages/api/src/routers/api/dashboards.ts | 8 +++- .../__tests__/dashboards.int.test.ts | 29 +++++++++++++ .../src/routers/external-api/v2/dashboards.ts | 8 ++-- packages/api/src/utils/zod.ts | 2 +- .../src/__tests__/referenceVariables.test.ts | 41 +++++++++++++++++++ .../common-utils/src/referenceVariables.ts | 2 +- 12 files changed, 151 insertions(+), 14 deletions(-) diff --git a/.changeset/source-vars-5.md b/.changeset/source-vars-5.md index 0704548e1..49079c548 100644 --- a/.changeset/source-vars-5.md +++ b/.changeset/source-vars-5.md @@ -1,5 +1,8 @@ --- '@hyperdx/app': minor +'@hyperdx/api': minor +'@hyperdx/common-utils': minor --- -feat: add connection variables for raw SQL tiles and connection-scoped data source variables +feat: add connection variables for raw SQL tiles, SQL query filters, and +connection-scoped data source variables diff --git a/packages/api/openapi.json b/packages/api/openapi.json index cfa7b0b8c..d3d0e8dc9 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -4151,7 +4151,7 @@ }, "ConnectionFilterInput": { "type": "object", - "description": "A variable whose single, required value is a connection. Raw SQL tiles\nreference it as `$variableName` in their connectionId, and data source\nvariables in theirs. Tiles that use it do not load until a connection is\nselected, and cannot have alerts.\n", + "description": "A variable whose single, required value is a connection. Raw SQL tiles\nreference it as `$variableName` in their connectionId, and data source\nvariables and SQL query filters in theirs. Tiles that use it do not load\nuntil a connection is selected, and cannot have alerts.\n", "required": [ "type", "name" @@ -4548,7 +4548,7 @@ }, "connectionId": { "type": "string", - "description": "ID of the ClickHouse connection the query runs on.", + "description": "ClickHouse connection the query runs on: a connection ID, or\n`$name` of a CONNECTION filter on the dashboard.\n", "example": "65f5e4a3b9e77c001a567890" }, "sqlTemplate": { diff --git a/packages/api/src/mcp/__tests__/dashboards/saveDashboard.int.test.ts b/packages/api/src/mcp/__tests__/dashboards/saveDashboard.int.test.ts index 721e25064..b750454e5 100644 --- a/packages/api/src/mcp/__tests__/dashboards/saveDashboard.int.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/saveDashboard.int.test.ts @@ -4189,6 +4189,26 @@ describe('MCP Dashboard Tools - clickstack_save_dashboard', () => { ]); }); + it('saves a SQL query filter that runs on a connection variable', async () => { + const sqlFilter = { + type: 'SQL_QUERY', + name: 'Environment', + variableName: 'env', + connectionId: '$conn', + sqlTemplate: 'SELECT DISTINCT env FROM envs', + }; + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Variable dashboard', + tiles: [], + filters: [filters[0], sqlFilter], + }); + + expect(result.isError).toBeFalsy(); + expect(JSON.parse(getFirstText(result)).filters[1]).toMatchObject( + sqlFilter, + ); + }); + it('rejects a tile referencing an undeclared variable', async () => { const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { name: 'Variable dashboard', diff --git a/packages/api/src/mcp/prompts/dashboards/content.ts b/packages/api/src/mcp/prompts/dashboards/content.ts index 4db680859..125b3c16e 100644 --- a/packages/api/src/mcp/prompts/dashboards/content.ts +++ b/packages/api/src/mcp/prompts/dashboards/content.ts @@ -137,7 +137,7 @@ Apply these before calling clickstack_save_dashboard. Each rule is enforced by t 9h. USE AN ADHOC FILTER TO LET THE USER BUILD FREE-FORM CONDITIONS. filters: [{ type: "ADHOC", name, sourceType: "sql", sources: [""], variableName? }] adds an input where the user picks any column or map key, an operator, and a value; clicking a value in a tile adds a condition too. Every condition is ANDed onto the tiles of its sources (or of appliesToSourceIds) with no per-tile wiring. It is never a variable: its variableName only names the entry its conditions are saved under. Seed conditions with savedFilterValues: [{ type: "adhoc", name: "", conditions: [{ key: "ServiceName", operator: "IN", value: "api" }] }]; IN conditions on the same key are ORed. Declare at most one ad hoc filter per source, since a click updates every ad hoc filter on the clicked tile's source. -9i. TO LET VIEWERS SWITCH THE SOURCE OR CONNECTION, USE A DATA_SOURCE OR CONNECTION FILTER. filters: [{ type: "DATA_SOURCE", name: "Logs source", variableName: "logs", allowedSourceKinds: ["log"] }] adds a required source picker; tiles then set sourceId: "$logs" (filters may use it in sourceId or appliesToSourceIds). A CONNECTION filter does the same for raw SQL tiles' connectionId, and a DATA_SOURCE filter's optional connectionId limits its sources to one connection. Keep allowedSourceKinds to kinds charts treat alike (log + trace, or one kind). Tiles that use either cannot have alerts, and clickstack_query_tile needs a selection for the variable (variableValues or the dashboard's saved value). +9i. TO LET VIEWERS SWITCH THE SOURCE OR CONNECTION, USE A DATA_SOURCE OR CONNECTION FILTER. filters: [{ type: "DATA_SOURCE", name: "Logs source", variableName: "logs", allowedSourceKinds: ["log"] }] adds a required source picker; tiles then set sourceId: "$logs" (filters may use it in sourceId or appliesToSourceIds). A CONNECTION filter does the same for raw SQL tiles' and SQL_QUERY filters' connectionId, and a DATA_SOURCE filter's optional connectionId limits its sources to one connection. Keep allowedSourceKinds to kinds charts treat alike (log + trace, or one kind). Tiles that use either cannot have alerts, and clickstack_query_tile needs a selection for the variable (variableValues or the dashboard's saved value). 10. UPDATE IS REPLACE, NOT MERGE. clickstack_save_dashboard with an id overwrites tiles, containers, and filters in their entirety. Call clickstack_get_dashboard first when you only want to add or rename one entry; do not send a partial set or you will silently drop everything you omitted. diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index 11acfd67b..0440330f1 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -1794,8 +1794,9 @@ const mcpSqlQueryFilterSchema = z .describe( 'Filter type discriminator for a filter whose dropdown values come from a ClickHouse SQL query.', ), - connectionId: objectIdSchema.describe( - 'ClickHouse connection the query runs on. Get IDs from clickstack_list_sources.', + connectionId: objectIdOrVariableReferenceSchema.describe( + 'ClickHouse connection the query runs on: a connection ID from ' + + 'clickstack_list_sources, or `$name` of a CONNECTION filter on this dashboard.', ), sqlTemplate: SqlQueryDashboardFilterSchema.shape.sqlTemplate.describe( 'ClickHouse SELECT whose FIRST column becomes the dropdown values (cast to ' + @@ -1968,7 +1969,8 @@ const mcpConnectionFilterSchema = z }) .describe( 'A variable whose single, required value is a connection. Raw SQL tiles reference ' + - 'it as `$variableName` in connectionId, and DATA_SOURCE filters in connectionId.', + 'it as `$variableName` in connectionId, and DATA_SOURCE and SQL_QUERY filters in ' + + 'connectionId.', ); const mcpDashboardFilterSchema = z diff --git a/packages/api/src/routers/api/__tests__/dashboard.int.test.ts b/packages/api/src/routers/api/__tests__/dashboard.int.test.ts index e1b5e4cdb..d6065229c 100644 --- a/packages/api/src/routers/api/__tests__/dashboard.int.test.ts +++ b/packages/api/src/routers/api/__tests__/dashboard.int.test.ts @@ -2145,6 +2145,42 @@ describe('dashboard router', () => { expect(updated.body.filters).toEqual([moved]); }); + + const connectionFilter = { + id: new Types.ObjectId().toString(), + type: 'CONNECTION' as const, + name: 'Connection', + variableName: 'conn', + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }; + + it('accepts a connection variable as the connection', async () => { + const filter = makeSqlFilter({ connection: '$conn' }); + + const created = await agent + .post('/dashboards') + .send({ ...MOCK_DASHBOARD, filters: [connectionFilter, filter] }) + .expect(200); + + expect(created.body.filters).toEqual([connectionFilter, filter]); + }); + + it('rejects a reference that names no connection variable', async () => { + const response = await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + filters: [makeSqlFilter({ connection: '$conn' })], + }) + .expect(400); + + expect(response.body.message).toMatch( + /does not name a connection variable/, + ); + }); }); describe('ad hoc filters', () => { diff --git a/packages/api/src/routers/api/dashboards.ts b/packages/api/src/routers/api/dashboards.ts index e17520007..ac54d3e74 100644 --- a/packages/api/src/routers/api/dashboards.ts +++ b/packages/api/src/routers/api/dashboards.ts @@ -90,7 +90,8 @@ async function validateFilterSources( /** * Rejects SQL_QUERY filters whose connection doesn't belong to the team. As in * `validateFilterSources`, a connection the stored filter with the same ID - * already used is skipped. Returns an error message, or null. + * already used is skipped, and so is a variable reference. Returns an error + * message, or null. */ async function validateFilterConnections( teamId: ObjectId, @@ -106,7 +107,10 @@ async function validateFilterConnections( filters .filter(isSqlQueryFilter) .filter( - filter => existingConnectionById.get(filter.id) !== filter.connection, + filter => + // Variable references are checked by validateDashboardVariableReferences + parseVariableReference(filter.connection) == null && + existingConnectionById.get(filter.id) !== filter.connection, ) .map(filter => filter.connection), ); diff --git a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts index 38a9e736c..88faec017 100644 --- a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts +++ b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts @@ -6329,6 +6329,35 @@ describe('External API v2 Dashboards - new format', () => { expect(response.body.message).toContain(connectionId); }); + it('accepts a connection variable as the connection', async () => { + const response = await sendFilters([ + { + id: new ObjectId().toString(), + type: 'CONNECTION', + name: 'Connection', + variableName: 'conn', + }, + sqlFilterInput({ connectionId: '$conn' }), + ]); + + expect(response.status).toBe(200); + expect(response.body.data.filters[1]).toMatchObject({ + type: 'SQL_QUERY', + connectionId: '$conn', + }); + }); + + it('rejects a reference that names no connection variable', async () => { + const response = await sendFilters([ + sqlFilterInput({ connectionId: '$conn' }), + ]); + + expect(response.status).toBe(400); + expect(response.body.message).toMatch( + /does not name a connection variable/, + ); + }); + it.each([ ['broadcast enabled', { isBroadcastEnabled: true }], ['not variable-enabled', { isVariableEnabled: false }], diff --git a/packages/api/src/routers/external-api/v2/dashboards.ts b/packages/api/src/routers/external-api/v2/dashboards.ts index cae8943ea..511775a03 100644 --- a/packages/api/src/routers/external-api/v2/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/dashboards.ts @@ -2518,8 +2518,8 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * description: | * A variable whose single, required value is a connection. Raw SQL tiles * reference it as `$variableName` in their connectionId, and data source - * variables in theirs. Tiles that use it do not load until a connection is - * selected, and cannot have alerts. + * variables and SQL query filters in theirs. Tiles that use it do not load + * until a connection is selected, and cannot have alerts. * required: * - type * - name @@ -2905,7 +2905,9 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "Environment" * connectionId: * type: string - * description: ID of the ClickHouse connection the query runs on. + * description: | + * ClickHouse connection the query runs on: a connection ID, or + * `$name` of a CONNECTION filter on the dashboard. * example: "65f5e4a3b9e77c001a567890" * sqlTemplate: * type: string diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index 09bd5ab19..87df22e50 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -211,7 +211,7 @@ const externalPromqlLabelFilterShape = PromqlLabelDashboardFilterSchema.omit({ const externalSqlQueryFilterShape = SqlQueryDashboardFilterSchema.omit({ connection: true, }).extend({ - connectionId: objectIdSchema, + connectionId: objectIdOrVariableReferenceSchema, isBroadcastEnabled: z.literal(false).default(false), isVariableEnabled: z.literal(true).default(true), }); diff --git a/packages/common-utils/src/__tests__/referenceVariables.test.ts b/packages/common-utils/src/__tests__/referenceVariables.test.ts index f1559f9b2..f41a77b4a 100644 --- a/packages/common-utils/src/__tests__/referenceVariables.test.ts +++ b/packages/common-utils/src/__tests__/referenceVariables.test.ts @@ -38,6 +38,7 @@ import { RawSqlSavedChartConfig, SavedChartConfig, SourceKind, + SqlQueryDashboardFilter, } from '@/types'; const dataSourceFilter = ( @@ -80,6 +81,20 @@ const connectionFilter = ( ...overrides, }); +const sqlQueryFilter = ( + overrides: Partial = {}, +): SqlQueryDashboardFilter => ({ + id: 'sql', + type: 'SQL_QUERY', + name: 'Environment', + variableName: 'env', + connection: '$conn', + sqlTemplate: 'SELECT DISTINCT env FROM envs', + isBroadcastEnabled: false, + isVariableEnabled: true, + ...overrides, +}); + const rawSqlConfig = ( overrides: Partial = {}, ): RawSqlSavedChartConfig => ({ @@ -458,6 +473,17 @@ describe('validateDashboardVariableReferences', () => { ]); }); + it("checks a SQL query filter's connection reference", () => { + expect(collectIssues([], [connectionFilter(), sqlQueryFilter()])).toEqual( + [], + ); + expect( + collectIssues([], [sqlQueryFilter({ connection: '$nope' })]).map( + issue => issue.path, + ), + ).toEqual([['filters', 0, 'connection']]); + }); + it('reports each issue as path: message', () => { expect( getVariableReferenceError( @@ -654,6 +680,7 @@ describe('convertToDashboardTemplate (connections)', () => { filters: [ connectionFilter(), dataSourceFilter({ connection: 'conn-1' }), + sqlQueryFilter(), ], savedFilterValues: [{ type: 'variable', name: 'conn', values: ['c'] }], }, @@ -662,6 +689,7 @@ describe('convertToDashboardTemplate (connections)', () => { ); expect(template.tiles[0].config).toMatchObject({ connection: '$conn' }); expect(template.filters?.[1]).toMatchObject({ connection: 'Primary' }); + expect(template.filters?.[2]).toMatchObject({ connection: '$conn' }); expect(template.savedFilterValues).toBeUndefined(); }); }); @@ -693,6 +721,19 @@ describe('convertToDashboardTemplate', () => { }); describe('resolveFilterReferences (connections)', () => { + it("resolves a SQL query filter's connection reference", () => { + expect( + resolveFilterReferences(sqlQueryFilter(), new Map([['conn', 'conn-1']])), + ).toEqual({ + filter: sqlQueryFilter({ connection: 'conn-1' }), + unresolvedVariables: [], + incompatibleVariables: [], + }); + expect( + resolveFilterReferences(sqlQueryFilter(), new Map()).unresolvedVariables, + ).toEqual(['conn']); + }); + it("resolves a data source variable's connection reference", () => { expect( resolveFilterReferences( diff --git a/packages/common-utils/src/referenceVariables.ts b/packages/common-utils/src/referenceVariables.ts index 4ce72c87d..c3009d445 100644 --- a/packages/common-utils/src/referenceVariables.ts +++ b/packages/common-utils/src/referenceVariables.ts @@ -225,9 +225,9 @@ export function getFilterReferenceFields( acceptedKinds: getAdhocFilterSourceKinds(filter.sourceType), })); case 'DATA_SOURCE': + case 'SQL_QUERY': return [{ key: 'connection', target: 'connection' }]; case 'STATIC_LIST': - case 'SQL_QUERY': case 'CONNECTION': return []; default: