Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/source-vars-4.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@hyperdx/app': minor
---

feat: let tiles and filters use a data source variable as their source
108 changes: 94 additions & 14 deletions packages/app/src/DBDashboardImportPage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@ import { Controller, useForm, useWatch } from 'react-hook-form';
import { z } from 'zod';
import { zodResolver } from '@hookform/resolvers/zod';
import { convertToDashboardDocument } from '@hyperdx/common-utils/dist/core/utils';
import {
collectRefinementIssues,
validateDashboardVariableReferences,
} from '@hyperdx/common-utils/dist/dashboardValidation';
import {
type DashboardFilterQueryIssue,
getAdhocFilterSourceKinds,
Expand All @@ -34,6 +38,7 @@ import {
displayTypeRequiresSource,
isRawSqlSavedChartConfig,
} from '@hyperdx/common-utils/dist/guards';
import { parseVariableReference } from '@hyperdx/common-utils/dist/referenceVariables';
import {
type DashboardFilter,
type DashboardTemplate,
Expand Down Expand Up @@ -130,6 +135,32 @@ function CollapsibleMessage({
);
}

/**
* A `$name` that names no variable of the right type in the template. The API
* rejects these too, but only once the mappings are filled in, and local mode
* doesn't check at all.
*/
export function getTemplateVariableReferenceIssues(
template: DashboardTemplate,
): string[] {
return collectRefinementIssues(ctx =>
validateDashboardVariableReferences(
template.tiles,
template.filters ?? [],
ctx,
),
).map(({ path, message }) => {
const [collection, index] = path;
const owner =
collection === 'tiles' && typeof index === 'number'
? `Tile "${template.tiles[index]?.config.name ?? index}"`
: collection === 'filters' && typeof index === 'number'
? `Filter "${template.filters?.[index]?.name ?? index}"`
: path.join('.');
return `${owner}: ${message}`;
});
}

function FileSelection({
onComplete,
}: {
Expand Down Expand Up @@ -203,6 +234,24 @@ function FileSelection({
return;
}

const referenceIssues = getTemplateVariableReferenceIssues(result.data);
if (referenceIssues.length > 0) {
onComplete(null);
setError({
message: 'Failed to Import Dashboard',
details: (
<Stack gap={0}>
{referenceIssues.map(issue => (
<Text key={issue} c="red">

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔵 minor — New import error list uses raw c="red", Title Case copy, and keys that can collide

The conventions say new status text should use <Text variant="danger">, not c="red", and UI copy should be sentence case ("Failed to import dashboard"). Also, key={issue} collides when two tiles or filters have the same name and the same bad reference, because getTemplateVariableReferenceIssues builds identical strings for them. That causes React duplicate-key warnings. This block also repeats the schema-error block just above it. Fix: move both into one helper that renders string[] with variant="danger" and index-based keys.

Advisory. Fix if it is a small defect in code this PR changes; otherwise reply in the thread. Do not widen the PR or touch files it did not already change. How to respond

{issue}
</Text>
))}
</Stack>
),
});
return;
}

// The schema only guarantees saved filter values are well-shaped, not that
// their condition strings parse. Surface a non-blocking warning so users
// who hand-edited or generated the file know a value won't actually filter,
Expand Down Expand Up @@ -398,11 +447,25 @@ function resolveAppliesToSources(
return { ids: slots.filter(Boolean), slots };
}

/**
* A data source variable reference (`$name`) is portable as written, so
* import neither maps it nor asks the user to.
*/
const isVariableRef = (name: string | undefined) =>

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What happens on import if a tile references $name and the template declares no data source variable with that name?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Before it would have failed when saving due to API validations, now I have added front-end validations on the import page 👍

parseVariableReference(name) != null;

const sourceVariableRefs = (names: string[] | undefined) =>
names?.filter(isVariableRef) ?? [];

const mappableSourceNames = (names: string[] | undefined) =>
names?.filter(name => !isVariableRef(name));

/** Whether the filter names a single source that import must map. */
const isSourceBackedTemplateFilter = (
filter: DashboardFilter,
): filter is QueryExpressionDashboardFilter | PromqlLabelDashboardFilter =>
isQueryExpressionFilter(filter) || isPrometheusLabelFilter(filter);
(isQueryExpressionFilter(filter) || isPrometheusLabelFilter(filter)) &&
!isVariableRef(filter.source);

/** Returns the filter if it names a source, or `undefined` if it doesn't. */
const sourceBackedTemplateFilter = (filter: DashboardFilter | undefined) =>
Expand Down Expand Up @@ -497,6 +560,7 @@ export function buildMappingFormSchema(
// meaningful source even when the template names one.
if (
config.source &&
!isVariableRef(config.source) &&
displayTypeRequiresSource(config.displayType) &&
!data.tileSourceMappings[idx]
) {
Expand Down Expand Up @@ -533,7 +597,10 @@ export function buildMappingFormSchema(
const mappedSourceIds = data.filterAdhocSourceMappings?.[idx]?.filter(
id => !!id,
);
if (!mappedSourceIds?.length) {
if (
!mappedSourceIds?.length &&
sourceVariableRefs(filter.sources).length === 0
) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
path: ['filterAdhocSourceMappings', idx],
Expand Down Expand Up @@ -646,7 +713,7 @@ export function Mapping({ input }: { input: DashboardTemplate }) {

const tileSourceMappings = input.tiles.map(tile => {
const config = tile.config as SavedChartConfig;
if (!config.source) return '';
if (!config.source || isVariableRef(config.source)) return '';
const match = sources.find(
source => source.name.toLowerCase() === config.source!.toLowerCase(),
);
Expand Down Expand Up @@ -1118,15 +1185,24 @@ export function Mapping({ input }: { input: DashboardTemplate }) {

// Zip the source mappings with the input filters
const zippedFilters = input.filters?.map((filter, idx) => {
const appliesTo = data.filterAppliesToSourceMappings?.[idx]?.filter(
id => !!id?.length,
);
const appliesToSourceIds = appliesTo?.length ? appliesTo : undefined;
// Data source variable references carry over unmapped
const appliesTo = [
...sourceVariableRefs(
appliesToTemplateFilter(filter)?.appliesToSourceIds,
),
...(data.filterAppliesToSourceMappings?.[idx]?.filter(
id => !!id?.length,
) ?? []),
];
const appliesToSourceIds = appliesTo.length ? appliesTo : undefined;
if (isAdhocFilter(filter)) {
return {
...filter,
sources:
data.filterAdhocSourceMappings?.[idx]?.filter(id => !!id) ?? [],
sources: [
...sourceVariableRefs(filter.sources),
...(data.filterAdhocSourceMappings?.[idx]?.filter(id => !!id) ??
[]),
],
appliesToSourceIds,
};
}
Expand Down Expand Up @@ -1227,7 +1303,7 @@ export function Mapping({ input }: { input: DashboardTemplate }) {
return (
<Fragment key={tile.id}>
{/** Mapping for the tile's source, if one exists (they're optional for raw sql tiles) */}
{tile.config.source && (
{tile.config.source && !isVariableRef(tile.config.source) && (
<Table.Tr>
<Table.Td>{tile.config.name}</Table.Td>
<Table.Td>Data source</Table.Td>
Expand Down Expand Up @@ -1310,8 +1386,12 @@ export function Mapping({ input }: { input: DashboardTemplate }) {

{/** Map filter sources */}
{input.filters?.map((filter, i) => {
const appliesToSourceIds =
appliesToTemplateFilter(filter)?.appliesToSourceIds;
const appliesToSourceIds = mappableSourceNames(
appliesToTemplateFilter(filter)?.appliesToSourceIds,
);
const adhocSourceNames = isAdhocFilter(filter)
? mappableSourceNames(filter.sources)
: undefined;
return (
<Fragment key={filter.id}>
{isSourceBackedTemplateFilter(filter) && (
Expand Down Expand Up @@ -1356,11 +1436,11 @@ export function Mapping({ input }: { input: DashboardTemplate }) {
</Table.Td>
</Table.Tr>
)}
{isAdhocFilter(filter) && (
{isAdhocFilter(filter) && !!adhocSourceNames?.length && (
<Table.Tr>
<Table.Td>{filter.name} (Filter)</Table.Td>
<Table.Td>Data sources</Table.Td>
<Table.Td>{filter.sources.join(', ')}</Table.Td>
<Table.Td>{adhocSourceNames.join(', ')}</Table.Td>
<Table.Td>
<SourceMultiSelectControlled
control={control}
Expand Down
8 changes: 6 additions & 2 deletions packages/app/src/DBDashboardPage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ import ResourceTerraformPopover from '@/components/Iac/ResourceTerraformPopover'
import { InlineNameInput } from '@/components/InlineNameInput/InlineNameInput';
import { PageHeader } from '@/components/PageHeader';
import { PageLayout } from '@/components/PageLayout';
import { ReferenceVariablesProvider } from '@/components/sourceSelectUtils';
import { SqlVariablesProvider } from '@/components/SQLEditor/variableCompletions';
import { TimePicker } from '@/components/TimePicker';
import {
Expand Down Expand Up @@ -3306,7 +3307,10 @@ function DBDashboardPage({
// `<PageLayout>` return shallow and prevents the tile-grid tree below
// from being wrapped in an extra indentation level.
const dashboardBody = (
<>
<ReferenceVariablesProvider
filters={filters}
values={referenceVariableValues}
>
<Head>
<title>{title}</title>
</Head>
Expand Down Expand Up @@ -3601,7 +3605,7 @@ function DBDashboardPage({
variables={variables}
/>
)}
</>
</ReferenceVariablesProvider>
);

return (
Expand Down
89 changes: 88 additions & 1 deletion packages/app/src/__tests__/DBDashboardImportPage.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,11 @@ import {
} from '@hyperdx/common-utils/dist/types';
import { act, fireEvent, screen, waitFor } from '@testing-library/react';

import { buildMappingFormSchema, Mapping } from '@/DBDashboardImportPage';
import {
buildMappingFormSchema,
getTemplateVariableReferenceIssues,
Mapping,
} from '@/DBDashboardImportPage';

// Sources/connections the import maps the template's source *names* onto. The
// auto-mapping effect in `Mapping` matches by name (case-insensitive), so a
Expand Down Expand Up @@ -724,3 +728,86 @@ describe('Dashboard import - ad hoc filter sources', () => {
expect(issuesFor(['src-logs'])).toBeUndefined();
});
});

describe('Dashboard import - data source variables', () => {
const sourceVariableTemplate = DashboardTemplateSchema.parse({
version: '0.1.0',
name: 'Source Variables',
tiles: [builderTile('tile-line', 'line', '$logs', 0)],
filters: [
{
id: 'ds',
type: 'DATA_SOURCE',
name: 'Logs source',
variableName: 'logs',
allowedSourceKinds: [SourceKind.Log],
minSelections: 1,
maxSelections: 1,
isBroadcastEnabled: false,
isVariableEnabled: true,
},
{
id: 'svc',
type: 'QUERY_EXPRESSION',
name: 'Service',
expression: 'ServiceName',
source: '$logs',
},
{
id: 'adhoc',
type: 'ADHOC',
name: 'Conditions',
sourceType: 'sql',
sources: ['$logs', 'Traces'],
isBroadcastEnabled: true,
isVariableEnabled: true,
},
],
Comment thread
pulpdrew marked this conversation as resolved.
Resolved
});

it('keeps references as written without asking for a mapping', async () => {
renderWithMantine(<Mapping input={sourceVariableTemplate} />);

const finish = await screen.findByRole('button', {
name: /finish import/i,
});
expect(screen.queryByText('$logs')).not.toBeInTheDocument();
await act(async () => {
fireEvent.click(finish);
});

await waitFor(() => expect(mockMutateAsync).toHaveBeenCalledTimes(1));
const payload = mockMutateAsync.mock.calls[0][0];
expect(payload.tiles[0].config.source).toBe('$logs');
expect(payload.filters[1].source).toBe('$logs');
expect(payload.filters[2].sources).toEqual(['$logs', 'src-traces']);
});

it('accepts references to the variables the template declares', () => {
expect(getTemplateVariableReferenceIssues(sourceVariableTemplate)).toEqual(
[],
);
});

it('reports references to variables the template does not declare', () => {
const template = DashboardTemplateSchema.parse({
version: '0.1.0',
name: 'Undeclared',
tiles: [builderTile('tile-line', 'line', '$traces', 0)],
filters: [
{
id: 'svc',
type: 'QUERY_EXPRESSION',
name: 'Service',
expression: 'ServiceName',
source: '$traces',
},
],
});

expect(getTemplateVariableReferenceIssues(template)).toEqual([
'Tile "tile-line": "$traces" does not name a data source variable on this dashboard',
'Filter "Service": "$traces" does not name a data source variable on this dashboard',
]);
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import { IconCirclePlus } from '@tabler/icons-react';

import { HeatmapModeControl } from '@/components/DBEditTimeChartForm/HeatmapModeControl';
import { SourceSelectControlled } from '@/components/SourceSelect';
import { useResolvedReference } from '@/components/sourceSelectUtils';
import { usePromqlMetricNames } from '@/hooks/usePromqlMetadata';
import { useSource } from '@/source';

Expand Down Expand Up @@ -80,7 +81,7 @@ export default function PromqlChartEditor({
);
}, []);

const sourceId = useWatch({ control, name: 'source' });
const sourceId = useResolvedReference(useWatch({ control, name: 'source' }));
const displayType = useWatch({ control, name: 'displayType' });
const { data: source } = useSource({ id: sourceId });
// The form can still hold a non-PromQL source right after switching a tile
Expand All @@ -107,6 +108,7 @@ export default function PromqlChartEditor({
name="source"
data-testid="source-selector"
allowedSourceKinds={allowedSourceKinds}
allowSourceVariables
/>
</Group>
{displayType === DisplayType.Heatmap && (
Expand Down
Loading
Loading