From f2404905cfcf3519e65310c03929577aeb15153f Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Fri, 13 Nov 2026 14:38:45 -0500 Subject: [PATCH 1/2] feat: add connection variables for raw SQL tiles and connection-scoped data source variables --- .changeset/source-vars-5.md | 5 + packages/app/src/DBDashboardImportPage.tsx | 140 ++++++++++-------- packages/app/src/DashboardFilters.tsx | 34 +++-- .../__tests__/DBDashboardImportPage.test.tsx | 64 ++++++++ .../ChartEditor/RawSqlChartEditor.tsx | 27 +++- .../app/src/components/ConnectionSelect.tsx | 50 ++++--- .../DashboardFilterEditForm.tsx | 9 +- .../DataSourceFilterEditForm.tsx | 15 ++ .../DataSourceFilterEditForm.test.tsx | 52 ++++++- .../ReferenceVariableDashboardFilter.tsx | 28 +++- packages/app/src/components/SourceSelect.tsx | 7 + .../ReferenceVariableDashboardFilter.test.tsx | 43 +++++- .../__tests__/SourceSelect.test.tsx | 30 ++++ .../app/src/components/sourceSelectUtils.tsx | 46 ++++-- packages/app/src/hooks/useConnectionItems.ts | 17 +++ .../e2e/components/ChartEditorComponent.ts | 13 ++ .../dashboard-source-variables.spec.ts | 75 +++++++++- .../tests/e2e/page-objects/DashboardPage.ts | 29 +++- 18 files changed, 569 insertions(+), 115 deletions(-) create mode 100644 .changeset/source-vars-5.md create mode 100644 packages/app/src/hooks/useConnectionItems.ts diff --git a/.changeset/source-vars-5.md b/.changeset/source-vars-5.md new file mode 100644 index 000000000..0704548e1 --- /dev/null +++ b/.changeset/source-vars-5.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/app': minor +--- + +feat: add connection variables for raw SQL tiles and connection-scoped data source variables diff --git a/packages/app/src/DBDashboardImportPage.tsx b/packages/app/src/DBDashboardImportPage.tsx index 9f7eda309..749908f98 100644 --- a/packages/app/src/DBDashboardImportPage.tsx +++ b/packages/app/src/DBDashboardImportPage.tsx @@ -24,6 +24,7 @@ import { type DashboardFilterQueryIssue, getAdhocFilterSourceKinds, isAdhocFilter, + isDataSourceFilter, isPrometheusLabelFilter, isQueryExpressionFilter, isSqlQueryFilter, @@ -80,6 +81,7 @@ import { import SelectControlled from './components/SelectControlled'; import { SourceMultiSelectControlled } from './components/SourceMultiSelect'; +import { useConnectionItems } from './hooks/useConnectionItems'; import api from './api'; import { usePageTitle } from './brand'; import { useConnections } from './connection'; @@ -448,8 +450,8 @@ function resolveAppliesToSources( } /** - * A data source variable reference (`$name`) is portable as written, so - * import neither maps it nor asks the user to. + * A data source or connection variable reference (`$name`) is portable as + * written, so import neither maps it nor asks the user to. */ const isVariableRef = (name: string | undefined) => parseVariableReference(name) != null; @@ -467,6 +469,18 @@ const isSourceBackedTemplateFilter = ( (isQueryExpressionFilter(filter) || isPrometheusLabelFilter(filter)) && !isVariableRef(filter.source); +/** + * Returns the filter if it names a connection import must map: a SQL query + * filter, or a data source variable limited to a connection. + */ +const connectionBackedTemplateFilter = (filter: DashboardFilter | undefined) => + filter && + (isSqlQueryFilter(filter) || isDataSourceFilter(filter)) && + filter.connection && + !isVariableRef(filter.connection) + ? filter + : undefined; + /** Returns the filter if it names a source, or `undefined` if it doesn't. */ const sourceBackedTemplateFilter = (filter: DashboardFilter | undefined) => filter && isSourceBackedTemplateFilter(filter) ? filter : undefined; @@ -529,7 +543,7 @@ export function buildMappingFormSchema( connectionMappings: z.array(MappingValueSchema), /** A list of filter source mappings, ordered by input filter index */ filterSourceMappings: z.array(MappingValueSchema).optional(), - /** A list of filter connection mappings, ordered by input filter index. Only applicable for SQL query filters */ + /** Connection mappings of SQL query filters and data source variables, ordered by input filter index */ filterConnectionMappings: z.array(MappingValueSchema).optional(), /** * Per-filter applies-to source mappings. Each entry is an array of mapped @@ -572,7 +586,11 @@ export function buildMappingFormSchema( } // A raw SQL tile always gets a connection row, and its connection is // not optional — leaving it unpicked used to throw on `connection!.id`. - if (isRawSqlSavedChartConfig(config) && !data.connectionMappings[idx]) { + if ( + isRawSqlSavedChartConfig(config) && + !isVariableRef(config.connection) && + !data.connectionMappings[idx] + ) { ctx.addIssue({ code: z.ZodIssueCode.custom, path: ['connectionMappings', idx], @@ -588,6 +606,16 @@ export function buildMappingFormSchema( }); input.filters?.forEach((filter, idx) => { + if ( + connectionBackedTemplateFilter(filter) && + !data.filterConnectionMappings?.[idx] + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['filterConnectionMappings', idx], + message: 'Select a connection for this filter', + }); + } if (isAdhocFilter(filter)) { const kinds = getAdhocFilterSourceKinds(filter.sourceType); const kindMessage = @@ -623,17 +651,6 @@ export function buildMappingFormSchema( return; } - if (isSqlQueryFilter(filter)) { - if (!data.filterConnectionMappings?.[idx]) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ['filterConnectionMappings', idx], - message: 'Select a connection for this filter', - }); - } - return; - } - if (!isSourceBackedTemplateFilter(filter)) return; const mappedSourceId = data.filterSourceMappings?.[idx]; if (!mappedSourceId) { @@ -670,6 +687,7 @@ export function Mapping({ input }: { input: DashboardTemplate }) { const router = useRouter(); const { data: sources } = useSources(); const { data: connections } = useConnections(); + const connectionItems = useConnectionItems(); const { data: dashboards } = useDashboards(); const { data: existingTags } = api.useTags(); const [dashboardId] = useQueryState('dashboardId', parseAsString); @@ -722,17 +740,19 @@ export function Mapping({ input }: { input: DashboardTemplate }) { const findConnectionIdByName = (name: string) => connections.find(conn => conn.name.toLowerCase() === name.toLowerCase()) - ?.id || ''; + ?.id ?? ''; const connectionMappings = input.tiles.map(tile => { const config = tile.config as SavedChartConfig; - if (!isRawSqlSavedChartConfig(config)) return ''; + if (!isRawSqlSavedChartConfig(config) || isVariableRef(config.connection)) + return ''; return findConnectionIdByName(config.connection); }); - const filterConnectionMappings = input.filters?.map(filter => - isSqlQueryFilter(filter) ? findConnectionIdByName(filter.connection) : '', - ); + const filterConnectionMappings = input.filters?.map(filter => { + const connection = connectionBackedTemplateFilter(filter)?.connection; + return connection ? findConnectionIdByName(connection) : ''; + }); const filterSourceMappings = input.filters?.map(filter => { const sourceName = sourceBackedTemplateFilter(filter)?.source; @@ -791,8 +811,8 @@ export function Mapping({ input }: { input: DashboardTemplate }) { setValue('tileSourceMappings', tileSourceMappings); setValue('connectionMappings', connectionMappings); - setValue('filterSourceMappings', filterSourceMappings); setValue('filterConnectionMappings', filterConnectionMappings); + setValue('filterSourceMappings', filterSourceMappings); multiMappingSlotsRef.current.clear(); filterAppliesToSourceMappings?.forEach((mapping, idx) => { setValue(`filterAppliesToSourceMappings.${idx}`, mapping); @@ -983,8 +1003,8 @@ export function Mapping({ input }: { input: DashboardTemplate }) { sources, ]); - // Propagate connection mapping changes to other RawSQL tiles and SQL query - // filters with the same input connection + // Propagate connection mapping changes to other RawSQL tiles and filters + // with the same input connection useEffect(() => { if (isUpdatingRef.current) return; if (!input.tiles) return; @@ -1017,9 +1037,11 @@ export function Mapping({ input }: { input: DashboardTemplate }) { ); if (filterIdx !== -1) { prevFilterConnectionMappingsRef.current = filterConnectionMappings; - const filter = input.filters?.[filterIdx]; - if (filter && isSqlQueryFilter(filter)) { - inputConnectionName = filter.connection; + const connection = connectionBackedTemplateFilter( + input.filters?.[filterIdx], + )?.connection; + if (connection) { + inputConnectionName = connection; connectionId = filterConnectionMappings[filterIdx] ?? ''; } } @@ -1044,8 +1066,8 @@ export function Mapping({ input }: { input: DashboardTemplate }) { ?.map((filter, index) => ({ filter, index })) .filter( ({ filter }) => - isSqlQueryFilter(filter) && - filter.connection === inputConnectionName, + connectionBackedTemplateFilter(filter)?.connection === + inputConnectionName, ) .map(({ index }) => `filterConnectionMappings.${index}` as const) ?? []; @@ -1159,12 +1181,14 @@ export function Mapping({ input }: { input: DashboardTemplate }) { }; if (isRawSqlSavedChartConfig(tile.config)) { - const connection = findConnection(data.connectionMappings[idx]); + const connection = isVariableRef(tile.config.connection) + ? tile.config.connection + : findConnection(data.connectionMappings[idx])!.id; return { ...tile, config: applyOnClick({ ...tile.config, - connection: connection!.id, + connection, ...(source ? { source: source.id } : {}), }), }; @@ -1206,11 +1230,13 @@ export function Mapping({ input }: { input: DashboardTemplate }) { appliesToSourceIds, }; } - if (isSqlQueryFilter(filter)) { - const connection = findConnection( - data.filterConnectionMappings?.[idx], - ); - return { ...filter, connection: connection!.id }; + if (connectionBackedTemplateFilter(filter)) { + return { + ...filter, + // The form schema requires this mapping + connection: findConnection(data.filterConnectionMappings?.[idx])! + .id, + }; } const source = findSource(data.filterSourceMappings?.[idx]); return { @@ -1322,7 +1348,7 @@ export function Mapping({ input }: { input: DashboardTemplate }) { )} {/** Mapping for the tile's connection, if it's a raw sql tile */} - {isRawSql && ( + {isRawSql && !isVariableRef(config.connection) && ( {tile.config.name} Data connection @@ -1331,10 +1357,7 @@ export function Mapping({ input }: { input: DashboardTemplate }) { ({ - value: conn.id, - label: conn.name, - }))} + data={connectionItems} placeholder="Select a connection" /> @@ -1394,6 +1417,25 @@ export function Mapping({ input }: { input: DashboardTemplate }) { : undefined; return ( + {connectionBackedTemplateFilter(filter) && ( + + {filter.name} (Filter) + Data connection + + {connectionBackedTemplateFilter(filter)?.connection} + + + + + + + + )} {isSourceBackedTemplateFilter(filter) && ( {filter.name} (Filter) @@ -1418,24 +1460,6 @@ export function Mapping({ input }: { input: DashboardTemplate }) { )} - {isSqlQueryFilter(filter) && ( - - {filter.name} (Filter) - Data connection - {filter.connection} - - ({ - value: conn.id, - label: conn.name, - }))} - placeholder="Select a connection" - /> - - - )} {isAdhocFilter(filter) && !!adhocSourceNames?.length && ( {filter.name} (Filter) diff --git a/packages/app/src/DashboardFilters.tsx b/packages/app/src/DashboardFilters.tsx index 3e155c733..ff17338cf 100644 --- a/packages/app/src/DashboardFilters.tsx +++ b/packages/app/src/DashboardFilters.tsx @@ -10,6 +10,7 @@ import { isFilterReferenceableVariable, isFilterRequired, isQueryExpressionFilter, + isReferenceVariableFilter, isStaticListFilter, } from '@hyperdx/common-utils/dist/filters'; import { ReferenceVariableValues } from '@hyperdx/common-utils/dist/referenceVariables'; @@ -31,7 +32,10 @@ import { getPendingVariablesTooltip, } from './components/DashboardFilterLabel'; import { FilterLinkToggle } from './components/FilterLinkToggle'; -import { DataSourceDashboardFilter } from './components/ReferenceVariableDashboardFilter'; +import { + ConnectionDashboardFilter, + DataSourceDashboardFilter, +} from './components/ReferenceVariableDashboardFilter'; import { VirtualMultiSelect } from './components/VirtualMultiSelect/VirtualMultiSelect'; import { useDashboardFilterValues } from './hooks/useDashboardFilterValues'; @@ -248,20 +252,28 @@ const DashboardFilters = ({ const unselectedVariables = unresolvedVariables.filter( name => !incompatibleVariables.includes(name), ); - if (isDataSourceFilter(filter)) { - return ( + if (isReferenceVariableFilter(filter)) { + const referenceProps = { + value: referenceVariableValues?.get( + getFilterVariableName(filter) ?? '', + ), + onChange: (value: string | undefined) => + onSetFilterValue(filter.id, value ? [value] : []), + effect: getFilterEffect(filter), + requiredTooltip: getRequiredFilterTooltip(filter), + }; + return isDataSourceFilter(filter) ? ( - onSetFilterValue(filter.id, value ? [value] : []) - } - effect={getFilterEffect(filter)} - requiredTooltip={getRequiredFilterTooltip(filter)} + {...referenceProps} + /> + ) : ( + ); } diff --git a/packages/app/src/__tests__/DBDashboardImportPage.test.tsx b/packages/app/src/__tests__/DBDashboardImportPage.test.tsx index 1083ba8ca..81502709f 100644 --- a/packages/app/src/__tests__/DBDashboardImportPage.test.tsx +++ b/packages/app/src/__tests__/DBDashboardImportPage.test.tsx @@ -811,3 +811,67 @@ describe('Dashboard import - data source variables', () => { ]); }); }); + +describe('Dashboard import - connection variables', () => { + const connectionVariableTemplate = DashboardTemplateSchema.parse({ + version: '0.1.0', + name: 'Connection Variables', + tiles: [ + { + id: 'tile-rawsql', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + name: 'tile-rawsql', + displayType: 'table', + configType: 'sql', + connection: '$conn', + sqlTemplate: 'SELECT 1', + }, + }, + ], + filters: [ + { + id: 'conn', + type: 'CONNECTION', + name: 'Cluster', + variableName: 'conn', + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }, + { + id: 'ds', + type: 'DATA_SOURCE', + name: 'Logs source', + variableName: 'logs', + allowedSourceKinds: [SourceKind.Log], + connection: 'Default', + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }, + ], + }); + + it('keeps connection references and maps a data source variable connection', async () => { + renderWithMantine(); + + const finish = await screen.findByRole('button', { + name: /finish import/i, + }); + expect(screen.queryByText('$conn')).not.toBeInTheDocument(); + await act(async () => { + fireEvent.click(finish); + }); + + await waitFor(() => expect(mockMutateAsync).toHaveBeenCalledTimes(1)); + const payload = mockMutateAsync.mock.calls[0][0]; + expect(payload.tiles[0].config.connection).toBe('$conn'); + expect(payload.filters[1].connection).toBe('conn-default'); + }); +}); diff --git a/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx b/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx index bfc60685a..895c1685b 100644 --- a/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx +++ b/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx @@ -154,10 +154,11 @@ export default function RawSqlChartEditor({ const { data: sources } = useSources(); const displayType = useWatch({ control, name: 'displayType' }); - const connection = useWatch({ control, name: 'connection' }); + // Source and connection variable references autocomplete and expand macros + // as the source and connection they currently select. + const storedConnection = useWatch({ control, name: 'connection' }); + const connection = useResolvedReference(storedConnection); const storedSource = useWatch({ control, name: 'source' }); - // A data source variable reference expands macros as the source it - // currently selects. const source = useResolvedReference(storedSource); const sqlTemplate = useWatch({ control, name: 'sqlTemplate' }); const chartName = useWatch({ control, name: 'name' }); @@ -217,12 +218,25 @@ export default function RawSqlChartEditor({ prevConnection, sources, }); - if (update?.field === 'connection') { + // A connection variable stays as picked; the tile shows a mismatch + // instead of having its reference replaced + if ( + update?.field === 'connection' && + parseVariableReference(storedConnection) == null + ) { setValue('connection', update.value); } else if (update?.field === 'source') { setValue('source', update.value); } - }, [connection, prevConnection, prevSource, setValue, storedSource, sources]); + }, [ + connection, + storedConnection, + prevConnection, + prevSource, + setValue, + storedSource, + sources, + ]); const placeholderSQl = SQL_PLACEHOLDERS[displayType ?? DisplayType.Table]; @@ -284,6 +298,8 @@ export default function RawSqlChartEditor({ control={control} name="connection" size="xs" + allowConnectionVariables + data-testid="connection-selector" /> @@ -305,6 +321,7 @@ export default function RawSqlChartEditor({ allowedSourceKinds={allowedSourceKinds} allowSourceVariables connectionId={connection} + variableConnection={storedConnection} size="xs" clearable placeholder="None" diff --git a/packages/app/src/components/ConnectionSelect.tsx b/packages/app/src/components/ConnectionSelect.tsx index 2422377cf..eccaacc5b 100644 --- a/packages/app/src/components/ConnectionSelect.tsx +++ b/packages/app/src/components/ConnectionSelect.tsx @@ -1,35 +1,51 @@ -import { useMemo } from 'react'; -import { UseControllerProps } from 'react-hook-form'; +import { UseControllerProps, useWatch } from 'react-hook-form'; +import { parseVariableReference } from '@hyperdx/common-utils/dist/referenceVariables'; +import { SelectProps } from '@mantine/core'; import { IconServer } from '@tabler/icons-react'; import SelectControlled from '@/components/SelectControlled'; -import { useConnections } from '@/connection'; +import { + useWithConnectionVariableItems, + VARIABLE_ICON, +} from '@/components/sourceSelectUtils'; +import { useConnectionItems } from '@/hooks/useConnectionItems'; export function ConnectionSelectControlled({ size, + allowConnectionVariables = false, + placeholder = 'Connection', + clearable = false, ...props -}: { size?: string } & UseControllerProps) { - const { data } = useConnections(); - - const values = useMemo( - () => - data?.map(d => ({ - value: d.id, - label: d.name, - })), - [data], +}: { + size?: string; + /** Whether to offer the dashboard's connection variables. */ + allowConnectionVariables?: boolean; +} & Pick & + UseControllerProps) { + const selected = useWatch({ control: props.control, name: props.name }); + const connectionItems = useConnectionItems(); + const items = useWithConnectionVariableItems( + connectionItems, + allowConnectionVariables, ); return ( } + placeholder={placeholder} + leftSection={ + parseVariableReference(selected) != null ? ( + VARIABLE_ICON + ) : ( + + ) + } maxDropdownHeight={280} size={size} /> diff --git a/packages/app/src/components/DashboardFiltersModal/DashboardFilterEditForm.tsx b/packages/app/src/components/DashboardFiltersModal/DashboardFilterEditForm.tsx index cd08b76a2..cb8c5d784 100644 --- a/packages/app/src/components/DashboardFiltersModal/DashboardFilterEditForm.tsx +++ b/packages/app/src/components/DashboardFiltersModal/DashboardFilterEditForm.tsx @@ -43,6 +43,7 @@ import { import { PromqlLabelFilterEditForm } from './PromqlLabelFilterEditForm'; import { QueryExpressionFilterEditForm } from './QueryExpressionFilterEditForm'; import { StaticListFilterEditForm } from './StaticListFilterEditForm'; +import { VariableNameInput } from './VariableNameInput'; const FILTER_TYPE_OPTIONS = [ { @@ -67,6 +68,7 @@ const FILTER_TYPE_OPTIONS = [ ] : []), { value: DashboardFilterType.enum.DATA_SOURCE, label: 'Data source' }, + { value: DashboardFilterType.enum.CONNECTION, label: 'Connection' }, ]; interface DashboardFilterEditFormProps { @@ -260,7 +262,12 @@ export const DashboardFilterEditForm = ({ message="Failed to load filter" showErrorMessage > - {formFilterType === 'DATA_SOURCE' ? ( + {formFilterType === 'CONNECTION' ? ( + + ) : formFilterType === 'DATA_SOURCE' ? ( + + + + {hasMixedKinds && ( ({ +jest.mock('@/connection', () => ({ + useConnections: () => ({ data: [{ id: 'conn-1', name: 'Primary' }] }), +})); + +function Harness({ + allowedSourceKinds, + connection, + onSave, +}: { + allowedSourceKinds: SourceKind[]; + connection?: string; + onSave?: (values: FilterFormValues) => void; +}) { + const { control, getValues } = useForm({ defaultValues: { ...toFormValues(), type: 'DATA_SOURCE', + name: 'Logs source', allowedSourceKinds, + connection, }, }); - return ; + return ( + <> + + + + ); } const warning = () => screen.queryByTestId('filter-mixed-source-kinds-warning'); @@ -35,4 +58,27 @@ describe('DataSourceFilterEditForm', () => { ); expect(warning()).not.toBeInTheDocument(); }); + + it('saves no connection once the connection is cleared', async () => { + const onSave = jest.fn(); + renderWithMantine( + , + ); + + const input = screen.getByTestId('filter-data-source-connection-input'); + expect(input).toHaveValue('Primary'); + const clearButton = input.parentElement?.querySelector('button'); + if (!clearButton) throw new Error('Expected a clear button'); + await userEvent.click(clearButton); + expect(input).toHaveValue(''); + await userEvent.click(screen.getByRole('button', { name: 'Save' })); + + expect(toSavedFilter(onSave.mock.calls[0][0])).toMatchObject({ + connection: undefined, + }); + }); }); diff --git a/packages/app/src/components/ReferenceVariableDashboardFilter.tsx b/packages/app/src/components/ReferenceVariableDashboardFilter.tsx index 27b494358..0f163c5f1 100644 --- a/packages/app/src/components/ReferenceVariableDashboardFilter.tsx +++ b/packages/app/src/components/ReferenceVariableDashboardFilter.tsx @@ -1,6 +1,7 @@ import { useCallback, useMemo } from 'react'; import { isSourceOfferedByVariable } from '@hyperdx/common-utils/dist/referenceVariables'; import { + ConnectionDashboardFilter as ConnectionFilter, DataSourceDashboardFilter as DataSourceFilter, TSource, } from '@hyperdx/common-utils/dist/types'; @@ -13,7 +14,7 @@ import { Stack, Text, } from '@mantine/core'; -import { IconStack } from '@tabler/icons-react'; +import { IconServer, IconStack } from '@tabler/icons-react'; import { isValueInSelectData } from '@/components/SelectControlled'; import { @@ -23,6 +24,7 @@ import { useFilteredSortedSourceItems, useSourceKindMap, } from '@/components/sourceSelectUtils'; +import { useConnectionItems } from '@/hooks/useConnectionItems'; import { useSources } from '@/source'; import { @@ -42,8 +44,8 @@ type ReferenceVariableFilterProps = { }; /** - * The filter bar control for a reference variable: a single picker over the - * IDs the variable can select. + * The filter bar control shared by data source and connection variables: a + * single picker over the IDs the variable can select. */ function ReferenceVariableSelect({ name, @@ -164,3 +166,23 @@ export const DataSourceDashboardFilter = ({ /> ); }; + +const renderConnectionIcon = () => ; + +/** A connection variable's picker. */ +export const ConnectionDashboardFilter = ({ + filter, + ...props +}: ReferenceVariableFilterProps & { filter: ConnectionFilter }) => { + const items = useConnectionItems({ sorted: true }); + + return ( + + ); +}; diff --git a/packages/app/src/components/SourceSelect.tsx b/packages/app/src/components/SourceSelect.tsx index 2dd72a808..845b7e1cf 100644 --- a/packages/app/src/components/SourceSelect.tsx +++ b/packages/app/src/components/SourceSelect.tsx @@ -163,6 +163,7 @@ function SourceSelectControlledComponent({ connectionId, isSourceAllowed, allowSourceVariables = false, + variableConnection, comboboxProps, ...props }: { @@ -178,6 +179,11 @@ function SourceSelectControlledComponent({ isSourceAllowed?: (source: TSource) => boolean; /** Whether to offer the dashboard's data source variables. */ allowSourceVariables?: boolean; + /** + * The connection data source variables must be limited to, as stored (it + * may be a connection variable reference). Defaults to `connectionId`. + */ + variableConnection?: string; } & UseControllerProps & SelectProps) { const { data } = useSources(); @@ -221,6 +227,7 @@ function SourceSelectControlledComponent({ const items = useWithSourceVariableItems(sourceItems, { enabled: allowSourceVariables, allowedSourceKinds, + connection: variableConnection ?? connectionId, }); const hasSelection = !!selectedSourceId; diff --git a/packages/app/src/components/__tests__/ReferenceVariableDashboardFilter.test.tsx b/packages/app/src/components/__tests__/ReferenceVariableDashboardFilter.test.tsx index 10ab34464..b09fe5a5b 100644 --- a/packages/app/src/components/__tests__/ReferenceVariableDashboardFilter.test.tsx +++ b/packages/app/src/components/__tests__/ReferenceVariableDashboardFilter.test.tsx @@ -5,11 +5,22 @@ import { import { screen } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; -import { DataSourceDashboardFilter } from '@/components/ReferenceVariableDashboardFilter'; +import { + ConnectionDashboardFilter, + DataSourceDashboardFilter, +} from '@/components/ReferenceVariableDashboardFilter'; jest.mock('@/source', () => ({ useSources: () => ({ data: mockSources }), })); +jest.mock('@/connection', () => ({ + useConnections: () => ({ + data: [ + { id: 'conn-b', name: 'Secondary' }, + { id: 'conn-a', name: 'Primary' }, + ], + }), +})); // Mantine's Combobox calls scrollIntoView when its dropdown opens; jsdom lacks it. window.HTMLElement.prototype.scrollIntoView = jest.fn(); @@ -106,3 +117,33 @@ describe('DataSourceDashboardFilter', () => { ).toBeInTheDocument(); }); }); + +describe('ConnectionDashboardFilter', () => { + it('offers the connections and reports the picked one', async () => { + const onChange = jest.fn(); + renderWithMantine( + , + ); + + await userEvent.click( + screen.getByTestId('dashboard-filter-select-Cluster'), + ); + await userEvent.click(await screen.findByText('Secondary')); + expect(onChange).toHaveBeenCalledWith('conn-b'); + }); +}); diff --git a/packages/app/src/components/__tests__/SourceSelect.test.tsx b/packages/app/src/components/__tests__/SourceSelect.test.tsx index b4dbdbbf0..90c17febc 100644 --- a/packages/app/src/components/__tests__/SourceSelect.test.tsx +++ b/packages/app/src/components/__tests__/SourceSelect.test.tsx @@ -333,6 +333,36 @@ describe('SourceSelectControlled (data source variables)', () => { expect(screen.getByText('Sources')).toBeInTheDocument(); }); + it('offers only variables unlimited or limited to its connection', async () => { + asMock(useSources).mockReturnValue({ data: SOURCES }); + function ConnectionHarness() { + const { control } = useForm(); + return ( + + + + ); + } + renderWithMantine(); + await userEvent.click(screen.getByPlaceholderText('Data Source')); + + expect(await screen.findByText('$any')).toBeInTheDocument(); + expect(screen.getByText('$mine')).toBeInTheDocument(); + expect(screen.queryByText('$other')).not.toBeInTheDocument(); + }); + it('offers no variables unless the picker opts in', async () => { asMock(useSources).mockReturnValue({ data: SOURCES }); renderWithMantine(); diff --git a/packages/app/src/components/sourceSelectUtils.tsx b/packages/app/src/components/sourceSelectUtils.tsx index 4a06f779b..04c555c48 100644 --- a/packages/app/src/components/sourceSelectUtils.tsx +++ b/packages/app/src/components/sourceSelectUtils.tsx @@ -1,5 +1,6 @@ import { createContext, use, useMemo } from 'react'; import { + getConnectionVariableNames, getSourceVariableDeclarations, isSourceVariableAccepted, parseVariableReference, @@ -30,7 +31,7 @@ export const SOURCE_KIND_ICONS: Record = { [SourceKind.Promql]: , }; -const VARIABLE_ICON = ; +export const VARIABLE_ICON = ; export function getSourceOptionIcon( value: string | undefined, @@ -42,16 +43,17 @@ export function getSourceOptionIcon( type ReferenceVariablesContextValue = { sourceVariables: SourceVariableDeclaration[]; + connectionVariables: string[]; values: ReferenceVariableValues; }; const ReferenceVariablesContext = createContext( - { sourceVariables: [], values: new Map() }, + { sourceVariables: [], connectionVariables: [], values: new Map() }, ); /** - * Offers the data source variables `filters` declare to the pickers below it - * that opt in, and their selections to `useResolvedReference`. + * Offers the data source and connection variables `filters` declare to the + * pickers below it that opt in, and their selections to `useResolvedReference`. */ export function ReferenceVariablesProvider({ filters, @@ -65,6 +67,7 @@ export function ReferenceVariablesProvider({ const value = useMemo( () => ({ sourceVariables: getSourceVariableDeclarations(filters), + connectionVariables: getConnectionVariableNames(filters), values, }), [filters, values], @@ -77,8 +80,8 @@ export function ReferenceVariablesProvider({ } /** - * The ID a source field currently queries: an ID as is, or the selection of - * the variable it references. + * The ID a source or connection field currently queries: an ID as is, or the + * selection of the variable it references. */ export function useResolvedReference( value: string | undefined, @@ -103,17 +106,20 @@ const toVariableItem = (name: string): ComboboxItem => { /** * A source picker's items, preceded by a "Variables" group of the data source * variables it can offer. A variable is offered if at least one kind it can - * resolve to is allowed; a selection of another kind shows as an error where - * the reference is used. + * resolve to is allowed, and, for a picker limited to `connection`, only if it + * is unlimited or limited to that same connection (ID or variable reference). + * A selection of another kind shows as an error where the reference is used. */ export function useWithSourceVariableItems( sourceItems: ComboboxItem[] | SourceSelectGroup[], { enabled, allowedSourceKinds, + connection, }: { enabled: boolean; allowedSourceKinds?: SourceKind[]; + connection?: string; }, ): (ComboboxItem | SourceSelectGroup)[] { const { sourceVariables } = use(ReferenceVariablesContext); @@ -122,8 +128,11 @@ export function useWithSourceVariableItems( const items = sourceVariables .filter( variable => - !allowedSourceKinds || - isSourceVariableAccepted(variable, allowedSourceKinds), + (!allowedSourceKinds || + isSourceVariableAccepted(variable, allowedSourceKinds)) && + (!connection || + !variable.connection || + variable.connection === connection), ) .map(variable => toVariableItem(variable.name)); if (items.length === 0) return sourceItems; @@ -138,7 +147,22 @@ export function useWithSourceVariableItems( ? [{ group: 'Sources', items: ungrouped }] : sourceItems), ]; - }, [enabled, sourceVariables, allowedSourceKinds, sourceItems]); + }, [enabled, sourceVariables, allowedSourceKinds, connection, sourceItems]); +} + +/** A connection picker's items, preceded by a "Variables" group of connection variables. */ +export function useWithConnectionVariableItems( + connectionItems: ComboboxItem[], + enabled: boolean, +): (ComboboxItem | SourceSelectGroup)[] { + const { connectionVariables } = use(ReferenceVariablesContext); + return useMemo(() => { + if (!enabled || connectionVariables.length === 0) return connectionItems; + return [ + { group: 'Variables', items: connectionVariables.map(toVariableItem) }, + { group: 'Connections', items: connectionItems }, + ]; + }, [enabled, connectionVariables, connectionItems]); } /** Header for sources that have no section assigned. Internal to this module. */ diff --git a/packages/app/src/hooks/useConnectionItems.ts b/packages/app/src/hooks/useConnectionItems.ts new file mode 100644 index 000000000..811bb0b16 --- /dev/null +++ b/packages/app/src/hooks/useConnectionItems.ts @@ -0,0 +1,17 @@ +import { useMemo } from 'react'; + +import { useConnections } from '@/connection'; + +/** The team's connections as select options, optionally sorted by name. */ +export function useConnectionItems({ sorted = false } = {}) { + const { data } = useConnections(); + return useMemo(() => { + const items = (data ?? []).map(connection => ({ + value: connection.id, + label: connection.name, + })); + return sorted + ? items.sort((a, b) => a.label.localeCompare(b.label)) + : items; + }, [data, sorted]); +} diff --git a/packages/app/tests/e2e/components/ChartEditorComponent.ts b/packages/app/tests/e2e/components/ChartEditorComponent.ts index fb24092b7..25e68e9bd 100644 --- a/packages/app/tests/e2e/components/ChartEditorComponent.ts +++ b/packages/app/tests/e2e/components/ChartEditorComponent.ts @@ -446,6 +446,19 @@ export class ChartEditorComponent { } } + /** The raw SQL editor's connection picker input. */ + get connection(): Locator { + return this.editorForm().getByTestId('connection-selector'); + } + + /** Pick the raw SQL editor's connection, by option label. */ + async selectConnection(connectionName: string) { + await this.connection.click(); + await this.page + .getByRole('option', { name: connectionName, exact: true }) + .click(); + } + /** Open the Data Source dropdown. */ async openSourcePicker() { await this.sourceSelector.click(); diff --git a/packages/app/tests/e2e/features/dashboard-source-variables.spec.ts b/packages/app/tests/e2e/features/dashboard-source-variables.spec.ts index 8c5d73610..89c61fc55 100644 --- a/packages/app/tests/e2e/features/dashboard-source-variables.spec.ts +++ b/packages/app/tests/e2e/features/dashboard-source-variables.spec.ts @@ -1,12 +1,16 @@ /** - * A `DATA_SOURCE` dashboard filter is a single-select variable whose value is - * a source. Tiles and other filters reference it as `$name` in place of a - * source, block until it has a selection, and cannot carry alerts. + * `DATA_SOURCE` and `CONNECTION` dashboard filters are single-select variables + * whose value is a source or a connection. Tiles and other filters reference + * them as `$name` in place of an ID, block until they have a selection, and + * cannot carry alerts. */ +import { DisplayType } from '@hyperdx/common-utils/dist/types'; + import { DashboardPage } from '../page-objects/DashboardPage'; import { SERVICES } from '../seed-clickhouse'; import { expect, test } from '../utils/base-test'; import { + DEFAULT_CONNECTION_NAME, DEFAULT_LOGS_SOURCE_NAME, DEFAULT_METRICS_SOURCE_NAME, DEFAULT_TRACES_SOURCE_NAME, @@ -187,6 +191,71 @@ test.describe( }); }); + test('drives a raw SQL tile from connection and source variables', async () => { + const editor = dashboardPage.chartEditor; + + await test.step('Create a connection variable and a source variable limited to it', async () => { + await dashboardPage.createNewDashboard(); + await dashboardPage.openEditFiltersModal(); + await dashboardPage.addConnectionFilterToDashboard( + 'Cluster', + 'cluster', + ); + await dashboardPage.addDataSourceFilterToDashboard( + 'Logs source', + ['Logs'], + 'logs', + { connectionLabel: '$cluster' }, + ); + await expect( + dashboardPage.getFilterItemByName('Logs source'), + ).toContainText('$cluster'); + await dashboardPage.closeFiltersModal(); + await expect( + dashboardPage.getFilterPendingVariableWarning('Logs source'), + ).toBeVisible(); + }); + + await test.step('A raw SQL tile on the variables cannot have an alert', async () => { + await dashboardPage.addTile(); + await expect(editor.nameInput).toBeVisible(); + await editor.waitForDataToLoad(); + await editor.setChartType(DisplayType.Number); + await editor.setChartName('Variable count'); + await editor.switchToSqlMode(); + await expect(editor.alertButton).toBeVisible(); + await editor.selectConnection('$cluster'); + await editor.selectSource('$logs'); + await expect(editor.alertButton).toHaveCount(0); + await editor.typeSqlQuery('SELECT count() FROM $__sourceTable'); + await editor.save(); + await expect( + dashboardPage.getTileMissingRequiredFilters(0), + ).toContainText('Cluster'); + }); + + await test.step('Selecting a connection, then a source on it, loads the tile', async () => { + await dashboardPage.selectReferenceFilterValue( + 'Cluster', + DEFAULT_CONNECTION_NAME, + ); + await dashboardPage.selectReferenceFilterValue( + 'Logs source', + DEFAULT_LOGS_SOURCE_NAME, + ); + await expect( + dashboardPage.getTileMissingRequiredFilters(0), + ).toHaveCount(0); + await expect(dashboardPage.getNumberTileValue(0)).toBeVisible(); + }); + + await test.step('The editor keeps both references', async () => { + await dashboardPage.editTile(0); + await expect(editor.connection).toHaveValue('$cluster'); + await expect(editor.source).toHaveValue('$logs'); + }); + }); + // A filter edit can leave other filters with a variable they can't always // use. It still saves, and the filter shows an error while it can't. test("saves a kind other filters can't use, and flags them while it's selected", async ({ diff --git a/packages/app/tests/e2e/page-objects/DashboardPage.ts b/packages/app/tests/e2e/page-objects/DashboardPage.ts index 86130cde3..71b1b843a 100644 --- a/packages/app/tests/e2e/page-objects/DashboardPage.ts +++ b/packages/app/tests/e2e/page-objects/DashboardPage.ts @@ -1404,7 +1404,8 @@ export class DashboardPage { | 'Static values' | 'PromQL label values' | 'Ad hoc keys and values' - | 'Data source', + | 'Data source' + | 'Connection', ) { await this.getFilterTypePicker().click(); await this.getFilterOption(label).click(); @@ -1493,7 +1494,9 @@ export class DashboardPage { name: string, kindLabels: string[], variableName: string, - options?: Pick, + options?: Pick & { + connectionLabel?: string; + }, ) { await this.addFiltersButton.click(); await this.selectFilterType('Data source'); @@ -1505,6 +1508,15 @@ export class DashboardPage { await kindsInput.click(); await this.getFilterOption(label).click(); } + if (options?.connectionLabel !== undefined) { + // Close the kinds dropdown, which covers this input; Escape would close + // the modal instead + await this.getFilterNameInput().click(); + await this.page + .getByTestId('filter-data-source-connection-input') + .click(); + await this.getFilterOption(options.connectionLabel).click(); + } await this.variableNameInput.fill(variableName); if (options?.globalRequirement !== undefined) { await this.globalRequirementCheckbox.setChecked( @@ -1518,6 +1530,19 @@ export class DashboardPage { }); } + /** Add a connection variable and wait for it in the filters list. */ + async addConnectionFilterToDashboard(name: string, variableName: string) { + await this.addFiltersButton.click(); + await this.selectFilterType('Connection'); + await this.getFilterNameInput().fill(name); + await this.variableNameInput.fill(variableName); + await this.page.getByTestId('save-filter-button').click(); + await this.getFilterItemByName(name).waitFor({ + state: 'visible', + timeout: 10000, + }); + } + /** Pick a value in a data source or connection variable's filter bar select. */ async selectReferenceFilterValue(filterName: string, optionLabel: string) { await this.getFilterSelectByName(filterName).click(); From 0a3f111c83ecdacb2919562ace59c98f9c5b2a67 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Tue, 17 Nov 2026 08:21:19 -0500 Subject: [PATCH 2/2] feat: let SQL query filters run on a connection variable --- .changeset/source-vars-5.md | 5 ++- packages/api/openapi.json | 4 +- .../dashboards/saveDashboard.int.test.ts | 20 +++++++++ .../api/src/mcp/prompts/dashboards/content.ts | 2 +- .../api/src/mcp/tools/dashboards/schemas.ts | 8 ++-- .../api/__tests__/dashboard.int.test.ts | 36 ++++++++++++++++ packages/api/src/routers/api/dashboards.ts | 8 +++- .../__tests__/dashboards.int.test.ts | 29 +++++++++++++ .../src/routers/external-api/v2/dashboards.ts | 8 ++-- packages/api/src/utils/zod.ts | 2 +- .../src/__tests__/referenceVariables.test.ts | 41 +++++++++++++++++++ .../common-utils/src/referenceVariables.ts | 2 +- 12 files changed, 151 insertions(+), 14 deletions(-) diff --git a/.changeset/source-vars-5.md b/.changeset/source-vars-5.md index 0704548e1..49079c548 100644 --- a/.changeset/source-vars-5.md +++ b/.changeset/source-vars-5.md @@ -1,5 +1,8 @@ --- '@hyperdx/app': minor +'@hyperdx/api': minor +'@hyperdx/common-utils': minor --- -feat: add connection variables for raw SQL tiles and connection-scoped data source variables +feat: add connection variables for raw SQL tiles, SQL query filters, and +connection-scoped data source variables diff --git a/packages/api/openapi.json b/packages/api/openapi.json index cfa7b0b8c..d3d0e8dc9 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -4151,7 +4151,7 @@ }, "ConnectionFilterInput": { "type": "object", - "description": "A variable whose single, required value is a connection. Raw SQL tiles\nreference it as `$variableName` in their connectionId, and data source\nvariables in theirs. Tiles that use it do not load until a connection is\nselected, and cannot have alerts.\n", + "description": "A variable whose single, required value is a connection. Raw SQL tiles\nreference it as `$variableName` in their connectionId, and data source\nvariables and SQL query filters in theirs. Tiles that use it do not load\nuntil a connection is selected, and cannot have alerts.\n", "required": [ "type", "name" @@ -4548,7 +4548,7 @@ }, "connectionId": { "type": "string", - "description": "ID of the ClickHouse connection the query runs on.", + "description": "ClickHouse connection the query runs on: a connection ID, or\n`$name` of a CONNECTION filter on the dashboard.\n", "example": "65f5e4a3b9e77c001a567890" }, "sqlTemplate": { diff --git a/packages/api/src/mcp/__tests__/dashboards/saveDashboard.int.test.ts b/packages/api/src/mcp/__tests__/dashboards/saveDashboard.int.test.ts index 721e25064..b750454e5 100644 --- a/packages/api/src/mcp/__tests__/dashboards/saveDashboard.int.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/saveDashboard.int.test.ts @@ -4189,6 +4189,26 @@ describe('MCP Dashboard Tools - clickstack_save_dashboard', () => { ]); }); + it('saves a SQL query filter that runs on a connection variable', async () => { + const sqlFilter = { + type: 'SQL_QUERY', + name: 'Environment', + variableName: 'env', + connectionId: '$conn', + sqlTemplate: 'SELECT DISTINCT env FROM envs', + }; + const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Variable dashboard', + tiles: [], + filters: [filters[0], sqlFilter], + }); + + expect(result.isError).toBeFalsy(); + expect(JSON.parse(getFirstText(result)).filters[1]).toMatchObject( + sqlFilter, + ); + }); + it('rejects a tile referencing an undeclared variable', async () => { const result = await callTool(ctx.client!, 'clickstack_save_dashboard', { name: 'Variable dashboard', diff --git a/packages/api/src/mcp/prompts/dashboards/content.ts b/packages/api/src/mcp/prompts/dashboards/content.ts index 4db680859..125b3c16e 100644 --- a/packages/api/src/mcp/prompts/dashboards/content.ts +++ b/packages/api/src/mcp/prompts/dashboards/content.ts @@ -137,7 +137,7 @@ Apply these before calling clickstack_save_dashboard. Each rule is enforced by t 9h. USE AN ADHOC FILTER TO LET THE USER BUILD FREE-FORM CONDITIONS. filters: [{ type: "ADHOC", name, sourceType: "sql", sources: [""], variableName? }] adds an input where the user picks any column or map key, an operator, and a value; clicking a value in a tile adds a condition too. Every condition is ANDed onto the tiles of its sources (or of appliesToSourceIds) with no per-tile wiring. It is never a variable: its variableName only names the entry its conditions are saved under. Seed conditions with savedFilterValues: [{ type: "adhoc", name: "", conditions: [{ key: "ServiceName", operator: "IN", value: "api" }] }]; IN conditions on the same key are ORed. Declare at most one ad hoc filter per source, since a click updates every ad hoc filter on the clicked tile's source. -9i. TO LET VIEWERS SWITCH THE SOURCE OR CONNECTION, USE A DATA_SOURCE OR CONNECTION FILTER. filters: [{ type: "DATA_SOURCE", name: "Logs source", variableName: "logs", allowedSourceKinds: ["log"] }] adds a required source picker; tiles then set sourceId: "$logs" (filters may use it in sourceId or appliesToSourceIds). A CONNECTION filter does the same for raw SQL tiles' connectionId, and a DATA_SOURCE filter's optional connectionId limits its sources to one connection. Keep allowedSourceKinds to kinds charts treat alike (log + trace, or one kind). Tiles that use either cannot have alerts, and clickstack_query_tile needs a selection for the variable (variableValues or the dashboard's saved value). +9i. TO LET VIEWERS SWITCH THE SOURCE OR CONNECTION, USE A DATA_SOURCE OR CONNECTION FILTER. filters: [{ type: "DATA_SOURCE", name: "Logs source", variableName: "logs", allowedSourceKinds: ["log"] }] adds a required source picker; tiles then set sourceId: "$logs" (filters may use it in sourceId or appliesToSourceIds). A CONNECTION filter does the same for raw SQL tiles' and SQL_QUERY filters' connectionId, and a DATA_SOURCE filter's optional connectionId limits its sources to one connection. Keep allowedSourceKinds to kinds charts treat alike (log + trace, or one kind). Tiles that use either cannot have alerts, and clickstack_query_tile needs a selection for the variable (variableValues or the dashboard's saved value). 10. UPDATE IS REPLACE, NOT MERGE. clickstack_save_dashboard with an id overwrites tiles, containers, and filters in their entirety. Call clickstack_get_dashboard first when you only want to add or rename one entry; do not send a partial set or you will silently drop everything you omitted. diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index 11acfd67b..0440330f1 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -1794,8 +1794,9 @@ const mcpSqlQueryFilterSchema = z .describe( 'Filter type discriminator for a filter whose dropdown values come from a ClickHouse SQL query.', ), - connectionId: objectIdSchema.describe( - 'ClickHouse connection the query runs on. Get IDs from clickstack_list_sources.', + connectionId: objectIdOrVariableReferenceSchema.describe( + 'ClickHouse connection the query runs on: a connection ID from ' + + 'clickstack_list_sources, or `$name` of a CONNECTION filter on this dashboard.', ), sqlTemplate: SqlQueryDashboardFilterSchema.shape.sqlTemplate.describe( 'ClickHouse SELECT whose FIRST column becomes the dropdown values (cast to ' + @@ -1968,7 +1969,8 @@ const mcpConnectionFilterSchema = z }) .describe( 'A variable whose single, required value is a connection. Raw SQL tiles reference ' + - 'it as `$variableName` in connectionId, and DATA_SOURCE filters in connectionId.', + 'it as `$variableName` in connectionId, and DATA_SOURCE and SQL_QUERY filters in ' + + 'connectionId.', ); const mcpDashboardFilterSchema = z diff --git a/packages/api/src/routers/api/__tests__/dashboard.int.test.ts b/packages/api/src/routers/api/__tests__/dashboard.int.test.ts index e1b5e4cdb..d6065229c 100644 --- a/packages/api/src/routers/api/__tests__/dashboard.int.test.ts +++ b/packages/api/src/routers/api/__tests__/dashboard.int.test.ts @@ -2145,6 +2145,42 @@ describe('dashboard router', () => { expect(updated.body.filters).toEqual([moved]); }); + + const connectionFilter = { + id: new Types.ObjectId().toString(), + type: 'CONNECTION' as const, + name: 'Connection', + variableName: 'conn', + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }; + + it('accepts a connection variable as the connection', async () => { + const filter = makeSqlFilter({ connection: '$conn' }); + + const created = await agent + .post('/dashboards') + .send({ ...MOCK_DASHBOARD, filters: [connectionFilter, filter] }) + .expect(200); + + expect(created.body.filters).toEqual([connectionFilter, filter]); + }); + + it('rejects a reference that names no connection variable', async () => { + const response = await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + filters: [makeSqlFilter({ connection: '$conn' })], + }) + .expect(400); + + expect(response.body.message).toMatch( + /does not name a connection variable/, + ); + }); }); describe('ad hoc filters', () => { diff --git a/packages/api/src/routers/api/dashboards.ts b/packages/api/src/routers/api/dashboards.ts index e17520007..ac54d3e74 100644 --- a/packages/api/src/routers/api/dashboards.ts +++ b/packages/api/src/routers/api/dashboards.ts @@ -90,7 +90,8 @@ async function validateFilterSources( /** * Rejects SQL_QUERY filters whose connection doesn't belong to the team. As in * `validateFilterSources`, a connection the stored filter with the same ID - * already used is skipped. Returns an error message, or null. + * already used is skipped, and so is a variable reference. Returns an error + * message, or null. */ async function validateFilterConnections( teamId: ObjectId, @@ -106,7 +107,10 @@ async function validateFilterConnections( filters .filter(isSqlQueryFilter) .filter( - filter => existingConnectionById.get(filter.id) !== filter.connection, + filter => + // Variable references are checked by validateDashboardVariableReferences + parseVariableReference(filter.connection) == null && + existingConnectionById.get(filter.id) !== filter.connection, ) .map(filter => filter.connection), ); diff --git a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts index 38a9e736c..88faec017 100644 --- a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts +++ b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts @@ -6329,6 +6329,35 @@ describe('External API v2 Dashboards - new format', () => { expect(response.body.message).toContain(connectionId); }); + it('accepts a connection variable as the connection', async () => { + const response = await sendFilters([ + { + id: new ObjectId().toString(), + type: 'CONNECTION', + name: 'Connection', + variableName: 'conn', + }, + sqlFilterInput({ connectionId: '$conn' }), + ]); + + expect(response.status).toBe(200); + expect(response.body.data.filters[1]).toMatchObject({ + type: 'SQL_QUERY', + connectionId: '$conn', + }); + }); + + it('rejects a reference that names no connection variable', async () => { + const response = await sendFilters([ + sqlFilterInput({ connectionId: '$conn' }), + ]); + + expect(response.status).toBe(400); + expect(response.body.message).toMatch( + /does not name a connection variable/, + ); + }); + it.each([ ['broadcast enabled', { isBroadcastEnabled: true }], ['not variable-enabled', { isVariableEnabled: false }], diff --git a/packages/api/src/routers/external-api/v2/dashboards.ts b/packages/api/src/routers/external-api/v2/dashboards.ts index cae8943ea..511775a03 100644 --- a/packages/api/src/routers/external-api/v2/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/dashboards.ts @@ -2518,8 +2518,8 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * description: | * A variable whose single, required value is a connection. Raw SQL tiles * reference it as `$variableName` in their connectionId, and data source - * variables in theirs. Tiles that use it do not load until a connection is - * selected, and cannot have alerts. + * variables and SQL query filters in theirs. Tiles that use it do not load + * until a connection is selected, and cannot have alerts. * required: * - type * - name @@ -2905,7 +2905,9 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * example: "Environment" * connectionId: * type: string - * description: ID of the ClickHouse connection the query runs on. + * description: | + * ClickHouse connection the query runs on: a connection ID, or + * `$name` of a CONNECTION filter on the dashboard. * example: "65f5e4a3b9e77c001a567890" * sqlTemplate: * type: string diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index 09bd5ab19..87df22e50 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -211,7 +211,7 @@ const externalPromqlLabelFilterShape = PromqlLabelDashboardFilterSchema.omit({ const externalSqlQueryFilterShape = SqlQueryDashboardFilterSchema.omit({ connection: true, }).extend({ - connectionId: objectIdSchema, + connectionId: objectIdOrVariableReferenceSchema, isBroadcastEnabled: z.literal(false).default(false), isVariableEnabled: z.literal(true).default(true), }); diff --git a/packages/common-utils/src/__tests__/referenceVariables.test.ts b/packages/common-utils/src/__tests__/referenceVariables.test.ts index f1559f9b2..f41a77b4a 100644 --- a/packages/common-utils/src/__tests__/referenceVariables.test.ts +++ b/packages/common-utils/src/__tests__/referenceVariables.test.ts @@ -38,6 +38,7 @@ import { RawSqlSavedChartConfig, SavedChartConfig, SourceKind, + SqlQueryDashboardFilter, } from '@/types'; const dataSourceFilter = ( @@ -80,6 +81,20 @@ const connectionFilter = ( ...overrides, }); +const sqlQueryFilter = ( + overrides: Partial = {}, +): SqlQueryDashboardFilter => ({ + id: 'sql', + type: 'SQL_QUERY', + name: 'Environment', + variableName: 'env', + connection: '$conn', + sqlTemplate: 'SELECT DISTINCT env FROM envs', + isBroadcastEnabled: false, + isVariableEnabled: true, + ...overrides, +}); + const rawSqlConfig = ( overrides: Partial = {}, ): RawSqlSavedChartConfig => ({ @@ -458,6 +473,17 @@ describe('validateDashboardVariableReferences', () => { ]); }); + it("checks a SQL query filter's connection reference", () => { + expect(collectIssues([], [connectionFilter(), sqlQueryFilter()])).toEqual( + [], + ); + expect( + collectIssues([], [sqlQueryFilter({ connection: '$nope' })]).map( + issue => issue.path, + ), + ).toEqual([['filters', 0, 'connection']]); + }); + it('reports each issue as path: message', () => { expect( getVariableReferenceError( @@ -654,6 +680,7 @@ describe('convertToDashboardTemplate (connections)', () => { filters: [ connectionFilter(), dataSourceFilter({ connection: 'conn-1' }), + sqlQueryFilter(), ], savedFilterValues: [{ type: 'variable', name: 'conn', values: ['c'] }], }, @@ -662,6 +689,7 @@ describe('convertToDashboardTemplate (connections)', () => { ); expect(template.tiles[0].config).toMatchObject({ connection: '$conn' }); expect(template.filters?.[1]).toMatchObject({ connection: 'Primary' }); + expect(template.filters?.[2]).toMatchObject({ connection: '$conn' }); expect(template.savedFilterValues).toBeUndefined(); }); }); @@ -693,6 +721,19 @@ describe('convertToDashboardTemplate', () => { }); describe('resolveFilterReferences (connections)', () => { + it("resolves a SQL query filter's connection reference", () => { + expect( + resolveFilterReferences(sqlQueryFilter(), new Map([['conn', 'conn-1']])), + ).toEqual({ + filter: sqlQueryFilter({ connection: 'conn-1' }), + unresolvedVariables: [], + incompatibleVariables: [], + }); + expect( + resolveFilterReferences(sqlQueryFilter(), new Map()).unresolvedVariables, + ).toEqual(['conn']); + }); + it("resolves a data source variable's connection reference", () => { expect( resolveFilterReferences( diff --git a/packages/common-utils/src/referenceVariables.ts b/packages/common-utils/src/referenceVariables.ts index 4ce72c87d..c3009d445 100644 --- a/packages/common-utils/src/referenceVariables.ts +++ b/packages/common-utils/src/referenceVariables.ts @@ -225,9 +225,9 @@ export function getFilterReferenceFields( acceptedKinds: getAdhocFilterSourceKinds(filter.sourceType), })); case 'DATA_SOURCE': + case 'SQL_QUERY': return [{ key: 'connection', target: 'connection' }]; case 'STATIC_LIST': - case 'SQL_QUERY': case 'CONNECTION': return []; default: