Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .changeset/faster-exact-filter-values.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
---
'@hyperdx/common-utils': patch
'@hyperdx/app': patch
---

perf: load search filter values faster in exact mode by reading several keys on
one Map column with a single `groupUniqArrayMap`, and by querying plain columns
and each Map or JSON column separately so slow attribute maps don't hold up the
rest
77 changes: 76 additions & 1 deletion packages/app/src/hooks/__tests__/useMetadata.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -600,7 +600,8 @@ describe('useGetKeyValues', () => {
const allKeys = [
"LogAttributes['a']",
"LogAttributes['b']",
"LogAttributes['c']",
// Quoting doesn't make it a different column
"`LogAttributes`['c']",
"`Resource-Attributes`['x']",
'ServiceName',
'SeverityText',
Expand All @@ -612,6 +613,7 @@ describe('useGetKeyValues', () => {
"`Resource-Attributes`['x']",
'ServiceName',
'SeverityText',
'SpanName',
];

const { result } = renderHook(
Expand Down Expand Up @@ -647,6 +649,79 @@ describe('useGetKeyValues', () => {
expect(getKeyValuesWithMVs.mock.calls[0][0].keyConditions).toBeUndefined();
});

describe("'exact' mode with plain columns and nested keys", () => {
const mixedKeys = [
"LogAttributes['a']",
'ServiceName',
"ResourceAttributes['b']",
'`__hdx_materialized_k8s.pod.name`',
"`LogAttributes`['c']",
'Body.`user`',
];
const renderMixed = () =>
renderValues({
mode: 'exact',
chartConfigs: [logsConfig],
keys: mixedKeys,
});

it('shows plain columns before the nested keys finish', async () => {
let releaseNested: () => void = () => {};
const nested = new Promise<void>(resolve => {
releaseNested = resolve;
});
const getKeyValuesWithMVs = jest
.spyOn(mockMetadata, 'getKeyValuesWithMVs')
.mockImplementation(async ({ keys }) => {
if (keys.includes('ServiceName'))
return [{ key: 'ServiceName', value: ['api'] }];
if (keys.includes("ResourceAttributes['b']"))
return [{ key: "ResourceAttributes['b']", value: ['y'] }];
await nested;
return [{ key: keys[0], value: ['x'] }];
});

const { result } = renderMixed();

await waitFor(() =>
expect(result.current.data).toEqual(
expect.arrayContaining([
{ key: 'ServiceName', value: ['api'] },
{ key: "ResourceAttributes['b']", value: ['y'] },
]),
),
);
expect(result.current.data).toHaveLength(2);
expect(result.current.isFetching).toBe(true);
expect(getKeyValuesWithMVs.mock.calls.map(([args]) => args.keys)).toEqual(
[
['ServiceName', '`__hdx_materialized_k8s.pod.name`', 'Body.`user`'],
["LogAttributes['a']", "`LogAttributes`['c']"],
["ResourceAttributes['b']"],
],
);

releaseNested();

await waitForSettled(result);
expect(result.current.data).toHaveLength(3);
});

it('errors when either group fails', async () => {
jest
.spyOn(mockMetadata, 'getKeyValuesWithMVs')
.mockImplementation(async ({ keys }) => {
if (keys.includes('ServiceName')) return [];
throw new Error('nested failed');
});

const { result } = renderMixed();

await waitFor(() => expect(result.current.isError).toBe(true));
expect(result.current.error?.message).toBe('nested failed');
});
});

it("retries a failed 'exact' mode lookup per the client's defaults", async () => {
queryClient.setDefaultOptions({ queries: { retry: 1, retryDelay: 0 } });
mockLookups.exact(new Error('transient'), [
Expand Down
120 changes: 76 additions & 44 deletions packages/app/src/hooks/useMetadata.tsx
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { useEffect, useMemo, useRef, useState } from 'react';
import { groupBy } from 'lodash';
import objectHash from 'object-hash';
import {
ColumnMeta,
Expand All @@ -15,6 +16,7 @@ import {
TableConnection,
TableMetadata,
tcFromSource,
unquoteIdentifier,
} from '@hyperdx/common-utils/dist/core/metadata';
import {
FilterState,
Expand Down Expand Up @@ -458,6 +460,46 @@ export function useFilterKeysFetchLimit() {
return me?.team?.filterKeysFetchLimit ?? DEFAULT_FILTER_KEYS_FETCH_LIMIT;
}

/** `LogAttributes['k']` -> `LogAttributes`; `ServiceName` -> null */
function getMapColumn(key: string): string | null {
const path = parseKeyPath(key);
return path.length > 1 ? unquoteIdentifier(path[0]) : null;
}

/** Indexes of the keys left after keeping the first `max` keys of each Map. */
function indexesWithinMapKeyLimit(keys: string[], max: number): number[] {
const countByMap = new Map<string, number>();
return keys.flatMap((key, i) => {
const mapColumn = getMapColumn(key);
if (mapColumn === null) return [i];
const count = (countByMap.get(mapColumn) ?? 0) + 1;
countByMap.set(mapColumn, count);
return count <= max ? [i] : [];
});
}

/** One group for all native columns, plus one group per Map column. */
function groupKeysByMapColumn(keys: string[]): string[][] {
const nativeColumnKeys = keys.filter(key => getMapColumn(key) === null);
const mapKeys = keys.filter(key => getMapColumn(key) !== null);
return [
nativeColumnKeys,
...Object.values(groupBy(mapKeys, getMapColumn)),
].filter(group => group.length > 0);
}

/** Yields each promise's value as it resolves; throws on the first rejection. */
async function* inCompletionOrder<T>(promises: Promise<T>[]) {
const pending = new Map(
promises.map((promise, i) => [i, promise.then(value => ({ i, value }))]),
);
while (pending.size > 0) {
const { i, value } = await Promise.race(pending.values());
pending.delete(i);
yield value;
}
}

/**
* Throw the first rejection when every lookup failed, otherwise return what
* succeeded, so one failing source doesn't empty a multi-source result.
Expand All @@ -477,7 +519,9 @@ async function settleAllOrThrow<T>(promises: Promise<T>[]): Promise<T[]> {
*
* - `'exact'` reads metadata MVs, else the raw table, sampling the first
* `max_rows_to_read` rows unless `disableRowLimit` is set. Supports
* `keyConditions`.
* `keyConditions`. Without them, native columns share one query and each Map
* column gets its own; when no values are showing yet, each query's values
* land in `data` as it finishes.
* - `'all'` tries text indexes, then metadata MVs, then the raw table, so it is
* cheaper for typeahead. It skips JSON-column keys, and its lookups return no
* values instead of failing, so it rarely errors. When no values are showing
Expand Down Expand Up @@ -525,9 +569,9 @@ export function useMultipleGetKeyValues(
const { data: sources, isLoading: isLoadingSources } = useSources();
const maxKeys = useFilterKeysFetchLimit();

// 'exact' lookups don't stream, so they keep the client's defaults rather
// than the streaming hook's. An unset default stays `undefined` here, which
// React Query reads as its own default. `refetchOnWindowFocus` is React
// 'exact' lookups yield one chunk per query, so they keep the client's
// defaults rather than the streaming hook's. An unset default stays `undefined` here,
// which React Query reads as its own default. `refetchOnWindowFocus` is React
// Query's default outright: the client's can be a function typed against
// `Query<unknown>`, which doesn't fit this query's data.
const { retry, structuralSharing } =
Expand Down Expand Up @@ -567,30 +611,10 @@ export function useMultipleGetKeyValues(
if (keys.length === 0) return;
const usableMVsOverride =
chartConfigsArr.length === 1 ? metadataMVsOverride : undefined;
// The limit applies per Map column, so one wide Map (e.g. LogAttributes)
// can't crowd out the others. Top-level columns share a single limit.
const { limitedKeys, limitedKeyConditions } = keys.reduce<{
countByColumn: Map<string, number>;
limitedKeys: string[];
limitedKeyConditions?: (FilterState | undefined)[];
}>(
(acc, key, i) => {
const path = parseKeyPath(key);
const column = path.length > 1 ? path[0] : '';
const count = acc.countByColumn.get(column) ?? 0;
acc.countByColumn.set(column, count + 1);
if (count < maxKeys) {
acc.limitedKeys.push(key);
acc.limitedKeyConditions?.push(keyConditions?.[i]);
}
return acc;
},
{
countByColumn: new Map(),
limitedKeys: [],
limitedKeyConditions: keyConditions ? [] : undefined,
},
);
const keptIndexes = indexesWithinMapKeyLimit(keys, maxKeys);
const limitedKeys = keptIndexes.map(i => keys[i]);
const limitedKeyConditions =
keyConditions && keptIndexes.map(i => keyConditions[i]);

const lookups = chartConfigsArr.map((chartConfig, i) => {
const source = chartConfig.source
Expand All @@ -605,22 +629,30 @@ export function useMultipleGetKeyValues(
});

if (mode === 'exact') {
const results = await settleAllOrThrow(
lookups.map(({ chartConfig, source }) =>
metadata.getKeyValuesWithMVs({
chartConfig,
keys: limitedKeys,
keyConditions: limitedKeyConditions,
limit,
disableRowLimit,
source,
signal,
}),
),
);
// Numeric values have always passed through unconverted here; `useQuery`
// used to hide the mismatch with `Facet`. HDX-1548.
yield results.flat() as Facet[];
// A key's condition can reference any sibling key, so faceted lookups
// stay in one query.
const keyGroups = limitedKeyConditions
? [limitedKeys]
: groupKeysByMapColumn(limitedKeys);
const fetchGroup = async (groupKeys: string[]) => {
const results = await settleAllOrThrow(
lookups.map(({ chartConfig, source }) =>
metadata.getKeyValuesWithMVs({
chartConfig,
keys: groupKeys,
keyConditions: limitedKeyConditions,
limit,
disableRowLimit,
source,
signal,
}),
),
);
// Numeric values have always passed through unconverted here; `useQuery`
// used to hide the mismatch with `Facet`. HDX-1548.
return results.flat() as Facet[];
};
yield* inCompletionOrder(keyGroups.map(fetchGroup));
return;
}

Expand Down
32 changes: 32 additions & 0 deletions packages/common-utils/src/__tests__/metadata.int.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,38 @@ describe('Metadata Integration Tests', () => {
);
});

it('should handle several keys on one Map column', async () => {
const result = await metadata.getKeyValues({
chartConfig,
keys: [
"ResourceAttributes['k8s.pod.name']",
'SeverityText',
"`ResourceAttributes`['env']",
"ResourceAttributes['missing']",
],
disableRowLimit,
source,
});

expect(result).toEqual([
{
key: "ResourceAttributes['k8s.pod.name']",
value: expect.arrayContaining(['pod1', 'pod2']),
},
{
key: 'SeverityText',
value: expect.arrayContaining(['info', 'error', 'warning']),
},
{
key: "`ResourceAttributes`['env']",
value: expect.arrayContaining(['prod', 'staging']),
},
{ key: "ResourceAttributes['missing']", value: [] },
]);
expect(result[0].value).toHaveLength(2);
expect(result[2].value).toHaveLength(2);
});

it('should return an empty list when no keys are provided', async () => {
const resultEmpty = await metadata.getKeyValues({
chartConfig,
Expand Down
Loading
Loading