From eef2b23b74e8231612bbb1731d1e0641f54c6be3 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Tue, 17 Nov 2026 10:21:33 -0500 Subject: [PATCH 1/3] feat: speed up exact filters --- .changeset/faster-exact-filter-values.md | 9 ++ .../src/hooks/__tests__/useMetadata.test.tsx | 74 ++++++++++++ packages/app/src/hooks/useMetadata.tsx | 67 ++++++++--- .../src/__tests__/metadata.int.test.ts | 32 +++++ .../src/__tests__/metadata.test.ts | 112 ++++++++++++++++++ packages/common-utils/src/core/metadata.ts | 100 ++++++++++++++-- 6 files changed, 366 insertions(+), 28 deletions(-) create mode 100644 .changeset/faster-exact-filter-values.md diff --git a/.changeset/faster-exact-filter-values.md b/.changeset/faster-exact-filter-values.md new file mode 100644 index 000000000..017dae532 --- /dev/null +++ b/.changeset/faster-exact-filter-values.md @@ -0,0 +1,9 @@ +--- +'@hyperdx/common-utils': patch +'@hyperdx/app': patch +--- + +perf: load search filter values faster in exact mode by reading several keys on +one Map column with a single `groupUniqArrayMap`, and by querying plain columns +and each Map or JSON column separately so slow attribute maps don't hold up the +rest diff --git a/packages/app/src/hooks/__tests__/useMetadata.test.tsx b/packages/app/src/hooks/__tests__/useMetadata.test.tsx index 46c92c95e..52e187974 100644 --- a/packages/app/src/hooks/__tests__/useMetadata.test.tsx +++ b/packages/app/src/hooks/__tests__/useMetadata.test.tsx @@ -647,6 +647,80 @@ describe('useGetKeyValues', () => { expect(getKeyValuesWithMVs.mock.calls[0][0].keyConditions).toBeUndefined(); }); + describe("'exact' mode with plain columns and nested keys", () => { + const mixedKeys = [ + "LogAttributes['a']", + 'ServiceName', + "ResourceAttributes['b']", + '`__hdx_materialized_k8s.pod.name`', + "`LogAttributes`['c']", + 'Body.`user`', + ]; + const renderMixed = () => + renderValues({ + mode: 'exact', + chartConfigs: [logsConfig], + keys: mixedKeys, + }); + + it('shows plain columns before the nested keys finish', async () => { + let releaseNested: () => void = () => {}; + const nested = new Promise(resolve => { + releaseNested = resolve; + }); + const getKeyValuesWithMVs = jest + .spyOn(mockMetadata, 'getKeyValuesWithMVs') + .mockImplementation(async ({ keys }) => { + if (keys.includes('ServiceName')) + return [{ key: 'ServiceName', value: ['api'] }]; + if (keys.includes("ResourceAttributes['b']")) + return [{ key: "ResourceAttributes['b']", value: ['y'] }]; + await nested; + return [{ key: keys[0], value: ['x'] }]; + }); + + const { result } = renderMixed(); + + await waitFor(() => + expect(result.current.data).toEqual( + expect.arrayContaining([ + { key: 'ServiceName', value: ['api'] }, + { key: "ResourceAttributes['b']", value: ['y'] }, + ]), + ), + ); + expect(result.current.data).toHaveLength(2); + expect(result.current.isFetching).toBe(true); + expect(getKeyValuesWithMVs.mock.calls.map(([args]) => args.keys)).toEqual( + [ + ["LogAttributes['a']", "`LogAttributes`['c']"], + ['ServiceName', '`__hdx_materialized_k8s.pod.name`'], + ["ResourceAttributes['b']"], + ['Body.`user`'], + ], + ); + + releaseNested(); + + await waitForSettled(result); + expect(result.current.data).toHaveLength(4); + }); + + it('errors when either group fails', async () => { + jest + .spyOn(mockMetadata, 'getKeyValuesWithMVs') + .mockImplementation(async ({ keys }) => { + if (keys.includes('ServiceName')) return []; + throw new Error('nested failed'); + }); + + const { result } = renderMixed(); + + await waitFor(() => expect(result.current.isError).toBe(true)); + expect(result.current.error?.message).toBe('nested failed'); + }); + }); + it("retries a failed 'exact' mode lookup per the client's defaults", async () => { queryClient.setDefaultOptions({ queries: { retry: 1, retryDelay: 0 } }); mockLookups.exact(new Error('transient'), [ diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx index fb965ed26..4838a7882 100644 --- a/packages/app/src/hooks/useMetadata.tsx +++ b/packages/app/src/hooks/useMetadata.tsx @@ -1,4 +1,5 @@ import { useEffect, useMemo, useRef, useState } from 'react'; +import { groupBy } from 'lodash'; import objectHash from 'object-hash'; import { ColumnMeta, @@ -15,6 +16,7 @@ import { TableConnection, TableMetadata, tcFromSource, + unquoteIdentifier, } from '@hyperdx/common-utils/dist/core/metadata'; import { FilterState, @@ -458,6 +460,12 @@ export function useFilterKeysFetchLimit() { return me?.team?.filterKeysFetchLimit ?? DEFAULT_FILTER_KEYS_FETCH_LIMIT; } +/** '' for a plain column, else the column a nested key reads from. */ +const keyQueryGroup = (key: string) => { + const column = /^(`[^`]+`|[A-Za-z_][A-Za-z0-9_]*)/.exec(key)?.[0] ?? key; + return column === key ? '' : unquoteIdentifier(column); +}; + /** * Throw the first rejection when every lookup failed, otherwise return what * succeeded, so one failing source doesn't empty a multi-source result. @@ -477,7 +485,9 @@ async function settleAllOrThrow(promises: Promise[]): Promise { * * - `'exact'` reads metadata MVs, else the raw table, sampling the first * `max_rows_to_read` rows unless `disableRowLimit` is set. Supports - * `keyConditions`. + * `keyConditions`. Without them, plain columns share one query and each Map / + * JSON column gets its own; when no values are showing yet, each query's + * values land in `data` as it finishes. * - `'all'` tries text indexes, then metadata MVs, then the raw table, so it is * cheaper for typeahead. It skips JSON-column keys, and its lookups return no * values instead of failing, so it rarely errors. When no values are showing @@ -525,9 +535,9 @@ export function useMultipleGetKeyValues( const { data: sources, isLoading: isLoadingSources } = useSources(); const maxKeys = useFilterKeysFetchLimit(); - // 'exact' lookups don't stream, so they keep the client's defaults rather - // than the streaming hook's. An unset default stays `undefined` here, which - // React Query reads as its own default. `refetchOnWindowFocus` is React + // 'exact' lookups yield a few large chunks, so they keep the client's defaults + // rather than the streaming hook's. An unset default stays `undefined` here, + // which React Query reads as its own default. `refetchOnWindowFocus` is React // Query's default outright: the client's can be a function typed against // `Query`, which doesn't fit this query's data. const { retry, structuralSharing } = @@ -605,22 +615,43 @@ export function useMultipleGetKeyValues( }); if (mode === 'exact') { - const results = await settleAllOrThrow( - lookups.map(({ chartConfig, source }) => - metadata.getKeyValuesWithMVs({ - chartConfig, - keys: limitedKeys, - keyConditions: limitedKeyConditions, - limit, - disableRowLimit, - source, - signal, - }), + // Map / JSON keys are far slower to read than plain columns, so plain + // columns share one query and each Map / JSON column gets its own: one + // wide map can't hold up the rest. Faceted lookups stay in one query: a + // key's condition can reference any sibling key. + const keyGroups = limitedKeyConditions + ? [limitedKeys] + : Object.values(groupBy(limitedKeys, keyQueryGroup)); + const groupResults = keyGroups.map(keys => + settleAllOrThrow( + lookups.map(({ chartConfig, source }) => + metadata.getKeyValuesWithMVs({ + chartConfig, + keys, + keyConditions: limitedKeyConditions, + limit, + disableRowLimit, + source, + signal, + }), + ), ), ); - // Numeric values have always passed through unconverted here; `useQuery` - // used to hide the mismatch with `Facet`. HDX-1548. - yield results.flat() as Facet[]; + // Unlike `mergeStreams`, a failed group fails the lookup rather than + // silently leaving its keys empty. + const pending = new Map( + groupResults.map((results, i) => [ + i, + results.then(r => [i, r] as const), + ]), + ); + while (pending.size > 0) { + const [i, results] = await Promise.race(pending.values()); + pending.delete(i); + // Numeric values have always passed through unconverted here; `useQuery` + // used to hide the mismatch with `Facet`. HDX-1548. + yield results.flat() as Facet[]; + } return; } diff --git a/packages/common-utils/src/__tests__/metadata.int.test.ts b/packages/common-utils/src/__tests__/metadata.int.test.ts index 5d782bc87..328c04191 100644 --- a/packages/common-utils/src/__tests__/metadata.int.test.ts +++ b/packages/common-utils/src/__tests__/metadata.int.test.ts @@ -190,6 +190,38 @@ describe('Metadata Integration Tests', () => { ); }); + it('should handle several keys on one Map column', async () => { + const result = await metadata.getKeyValues({ + chartConfig, + keys: [ + "ResourceAttributes['k8s.pod.name']", + 'SeverityText', + "`ResourceAttributes`['env']", + "ResourceAttributes['missing']", + ], + disableRowLimit, + source, + }); + + expect(result).toEqual([ + { + key: "ResourceAttributes['k8s.pod.name']", + value: expect.arrayContaining(['pod1', 'pod2']), + }, + { + key: 'SeverityText', + value: expect.arrayContaining(['info', 'error', 'warning']), + }, + { + key: "`ResourceAttributes`['env']", + value: expect.arrayContaining(['prod', 'staging']), + }, + { key: "ResourceAttributes['missing']", value: [] }, + ]); + expect(result[0].value).toHaveLength(2); + expect(result[2].value).toHaveLength(2); + }); + it('should return an empty list when no keys are provided', async () => { const resultEmpty = await metadata.getKeyValues({ chartConfig, diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index e12561b5e..f25357738 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1133,6 +1133,118 @@ describe('Metadata', () => { }, }); }); + + describe('with several keys on one Map column', () => { + beforeEach(() => { + jest.spyOn(metadata, 'getColumn').mockImplementation(({ column }) => + Promise.resolve( + column === 'LogAttributes' + ? ({ + name: 'LogAttributes', + type: 'Map(LowCardinality(String), String)', + } as any) + : undefined, + ), + ); + }); + + it('reads them with one groupUniqArrayMap', async () => { + const renderChartConfigSpy = jest.spyOn( + renderChartConfigModule, + 'renderChartConfig', + ); + + await metadata.getKeyValues({ + chartConfig: mockChartConfig, + keys: [ + "LogAttributes['a']", + 'SeverityText', + "`LogAttributes`['b\\'c']", + "LogAttributes['d']", + ], + limit: 10, + source, + }); + + const actualConfig = renderChartConfigSpy.mock.calls[0][0]; + if (!isBuilderChartConfig(actualConfig)) + throw new Error('Expected builder config'); + expect(actualConfig.with?.[0]).toMatchObject({ + chartConfig: { + select: + 'SeverityText as param1, ' + + "mapFilter((k, v) -> k IN ('a', 'b\\'c', 'd'), LogAttributes) as mapParam0", + }, + }); + expect(actualConfig.select).toBe( + [ + "groupUniqArrayMap(10)(mapParam0)['a'] AS param0", + 'groupUniqArray(10)(param1) AS param1', + "groupUniqArrayMap(10)(mapParam0)['b\\'c'] AS param2", + "groupUniqArrayMap(10)(mapParam0)['d'] AS param3", + ].join(', '), + ); + }); + + it('keeps faceted keys on their own predicate', async () => { + const renderChartConfigSpy = jest.spyOn( + renderChartConfigModule, + 'renderChartConfig', + ); + + await metadata.getKeyValues({ + chartConfig: mockChartConfig, + keys: ["LogAttributes['a']", "LogAttributes['b']", 'SeverityText'], + keyConditions: [ + { + SeverityText: { + included: new Set(['info']), + excluded: new Set(), + }, + }, + ], + limit: 10, + disableRowLimit: true, + source, + }); + + const actualConfig = renderChartConfigSpy.mock.calls[0][0]; + if (!isBuilderChartConfig(actualConfig)) + throw new Error('Expected builder config'); + // Only one unconstrained key is left on the Map, so nothing to share. + expect(actualConfig.select).toBe( + [ + "groupUniqArrayIf(10)(LogAttributes['a'], (SeverityText IN ('info'))) AS param0", + "groupUniqArray(10)(LogAttributes['b']) AS param1", + 'groupUniqArray(10)(SeverityText) AS param2', + ].join(', '), + ); + }); + + it('shares the Map read when the row limit is disabled', async () => { + const renderChartConfigSpy = jest.spyOn( + renderChartConfigModule, + 'renderChartConfig', + ); + + await metadata.getKeyValues({ + chartConfig: mockChartConfig, + keys: ["LogAttributes['a']", "LogAttributes['b']"], + limit: 10, + disableRowLimit: true, + source, + }); + + const actualConfig = renderChartConfigSpy.mock.calls[0][0]; + if (!isBuilderChartConfig(actualConfig)) + throw new Error('Expected builder config'); + const filter = "mapFilter((k, v) -> k IN ('a', 'b'), LogAttributes)"; + expect(actualConfig.select).toBe( + `groupUniqArrayMap(10)(${filter})['a'] AS param0, ` + + `groupUniqArrayMap(10)(${filter})['b'] AS param1`, + ); + }); + }); }); // Each of the four fetch strategies emits a distinct SQL shape (map-text- diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 7dcc2e4c7..6bfd41ab4 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -184,6 +184,10 @@ const inlineNonNegativeInt = (value: number, label: string): string => { return String(value); }; +// `Col['key']` or `` `Col`['key'] `` with a well-formed single-quoted key. +const MAP_SUBSCRIPT_REGEX = + /^(`[^`]+`|[A-Za-z_][A-Za-z0-9_]*)\['((?:[^'\\]|\\.)*)'\]$/; + /** Strip one level of `…` / "…" identifier quoting, if present. */ export const unquoteIdentifier = (identifier: string): string => { if ( @@ -584,6 +588,60 @@ export class Metadata { return rendered; } + /** + * Groups `Map['key']` expressions by their Map column, for columns with two + * or more keys. Each group's `filter` keeps just those keys, so one + * `groupUniqArrayMap` reads them all: a separate `Map['key']` lookup per key + * scans the whole map every time, which made wide attribute maps dominate + * value queries. + */ + private async groupMapSubscripts({ + databaseName, + tableName, + connectionId, + keys, + }: { + databaseName: string; + tableName: string; + connectionId: string; + keys: string[]; + }): Promise<{ + filters: string[]; + byKey: Map; + }> { + const literalsByColumn = new Map>(); + for (const key of keys) { + const match = MAP_SUBSCRIPT_REGEX.exec(key); + if (!match) continue; + const column = unquoteIdentifier(match[1]); + const literals = literalsByColumn.get(column) ?? new Map(); + literals.set(key, `'${match[2]}'`); + literalsByColumn.set(column, literals); + } + + const filters: string[] = []; + const byKey = new Map(); + for (const [column, literals] of literalsByColumn) { + const uniqueLiterals = [...new Set(literals.values())]; + if (uniqueLiterals.length < 2) continue; + const columnMeta = await this.getColumn({ + databaseName, + tableName, + column, + connectionId, + }); + if (convertCHDataTypeToJSType(columnMeta?.type ?? '') !== JSDataType.Map) + continue; + filters.push( + `mapFilter((k, v) -> k IN (${uniqueLiterals.join(', ')}), ${quoteIdentifierIfNeeded(column)})`, + ); + for (const [key, literal] of literals) { + byKey.set(key, { filter: filters.length - 1, literal }); + } + } + return { filters, byKey }; + } + private async queryTableMetadata({ database, table, @@ -3308,6 +3366,16 @@ export class Metadata { }); const renderKey = (key: string) => renderedByKey.get(key) ?? key; const renderedKeys = keys.map(renderKey); + // Faceted keys keep their own predicate, so only the rest can share a + // Map read. + const mapSubscripts = await this.groupMapSubscripts({ + databaseName: chartConfig.from.databaseName, + tableName: chartConfig.from.tableName, + connectionId: chartConfig.connection, + keys: renderedKeys.filter( + (_, i) => !(disableRowLimit && keyConditions?.at(i)), + ), + }); // When disableRowLimit is true, query directly without CTE // Otherwise, use CTE with row limits for sampling @@ -3323,12 +3391,16 @@ export class Metadata { // physical column than the aggregate wrapping it. const condition = state && filterStateToPredicate(state, renderKey); + const mapSubscript = mapSubscripts.byKey.get(k); // `groupUniqArrayIf` lets each key be constrained by its own // predicate, so multiple faceted value lists are computed in a // single scan instead of one query per key. - return condition - ? `groupUniqArrayIf(${limit})(${k}, ${condition}) AS param${i}` - : `groupUniqArray(${limit})(${k}) AS param${i}`; + if (condition) + return `groupUniqArrayIf(${limit})(${k}, ${condition}) AS param${i}`; + // ClickHouse computes the repeated aggregate once. + if (mapSubscript) + return `groupUniqArrayMap(${limit})(${mapSubscripts.filters[mapSubscript.filter]})[${mapSubscript.literal}] AS param${i}`; + return `groupUniqArray(${limit})(${k}) AS param${i}`; }) .join(', '), } @@ -3338,8 +3410,14 @@ export class Metadata { // than selecting just the JSON paths corresponding to the given keys. // paramN aliases are used to avoid issues with special characters or complex expressions in keys. const selectExpr = - renderedKeys.map((k, i) => `${k} as param${i}`).join(', ') || - '*'; + [ + ...renderedKeys.flatMap((k, i) => + mapSubscripts.byKey.has(k) ? [] : [`${k} as param${i}`], + ), + ...mapSubscripts.filters.map( + (filter, i) => `${filter} as mapParam${i}`, + ), + ].join(', ') || '*'; return { with: [ @@ -3359,11 +3437,13 @@ export class Metadata { isSubquery: true, }, ], - select: keys - .map( - (_, i) => - `groupUniqArray(${limit})(param${i}) AS param${i}`, - ) + select: renderedKeys + .map((k, i) => { + const mapSubscript = mapSubscripts.byKey.get(k); + return mapSubscript + ? `groupUniqArrayMap(${limit})(mapParam${mapSubscript.filter})[${mapSubscript.literal}] AS param${i}` + : `groupUniqArray(${limit})(param${i}) AS param${i}`; + }) .join(', '), connection: chartConfig.connection, from: { databaseName: '', tableName: 'sampledData' }, From 4af4d1857133651a550d51e7855f9e9d433c38e7 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Tue, 17 Nov 2026 10:43:45 -0500 Subject: [PATCH 2/3] fix lint --- packages/common-utils/src/__tests__/metadata.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index f25357738..a9c674727 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1142,7 +1142,7 @@ describe('Metadata', () => { ? ({ name: 'LogAttributes', type: 'Map(LowCardinality(String), String)', - } as any) + } as ColumnMeta) : undefined, ), ); From 6fa248550045318402f817d3f656d7e6654640e2 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Tue, 17 Nov 2026 10:57:05 -0500 Subject: [PATCH 3/3] revamp --- .../src/hooks/__tests__/useMetadata.test.tsx | 9 +- packages/app/src/hooks/useMetadata.tsx | 115 +++++++++--------- packages/common-utils/src/core/metadata.ts | 12 +- 3 files changed, 67 insertions(+), 69 deletions(-) diff --git a/packages/app/src/hooks/__tests__/useMetadata.test.tsx b/packages/app/src/hooks/__tests__/useMetadata.test.tsx index 52e187974..cefcb44b5 100644 --- a/packages/app/src/hooks/__tests__/useMetadata.test.tsx +++ b/packages/app/src/hooks/__tests__/useMetadata.test.tsx @@ -600,7 +600,8 @@ describe('useGetKeyValues', () => { const allKeys = [ "LogAttributes['a']", "LogAttributes['b']", - "LogAttributes['c']", + // Quoting doesn't make it a different column + "`LogAttributes`['c']", "`Resource-Attributes`['x']", 'ServiceName', 'SeverityText', @@ -612,6 +613,7 @@ describe('useGetKeyValues', () => { "`Resource-Attributes`['x']", 'ServiceName', 'SeverityText', + 'SpanName', ]; const { result } = renderHook( @@ -693,17 +695,16 @@ describe('useGetKeyValues', () => { expect(result.current.isFetching).toBe(true); expect(getKeyValuesWithMVs.mock.calls.map(([args]) => args.keys)).toEqual( [ + ['ServiceName', '`__hdx_materialized_k8s.pod.name`', 'Body.`user`'], ["LogAttributes['a']", "`LogAttributes`['c']"], - ['ServiceName', '`__hdx_materialized_k8s.pod.name`'], ["ResourceAttributes['b']"], - ['Body.`user`'], ], ); releaseNested(); await waitForSettled(result); - expect(result.current.data).toHaveLength(4); + expect(result.current.data).toHaveLength(3); }); it('errors when either group fails', async () => { diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx index 4838a7882..1655d8b97 100644 --- a/packages/app/src/hooks/useMetadata.tsx +++ b/packages/app/src/hooks/useMetadata.tsx @@ -460,11 +460,45 @@ export function useFilterKeysFetchLimit() { return me?.team?.filterKeysFetchLimit ?? DEFAULT_FILTER_KEYS_FETCH_LIMIT; } -/** '' for a plain column, else the column a nested key reads from. */ -const keyQueryGroup = (key: string) => { - const column = /^(`[^`]+`|[A-Za-z_][A-Za-z0-9_]*)/.exec(key)?.[0] ?? key; - return column === key ? '' : unquoteIdentifier(column); -}; +/** `LogAttributes['k']` -> `LogAttributes`; `ServiceName` -> null */ +function getMapColumn(key: string): string | null { + const path = parseKeyPath(key); + return path.length > 1 ? unquoteIdentifier(path[0]) : null; +} + +/** Indexes of the keys left after keeping the first `max` keys of each Map. */ +function indexesWithinMapKeyLimit(keys: string[], max: number): number[] { + const countByMap = new Map(); + return keys.flatMap((key, i) => { + const mapColumn = getMapColumn(key); + if (mapColumn === null) return [i]; + const count = (countByMap.get(mapColumn) ?? 0) + 1; + countByMap.set(mapColumn, count); + return count <= max ? [i] : []; + }); +} + +/** One group for all native columns, plus one group per Map column. */ +function groupKeysByMapColumn(keys: string[]): string[][] { + const nativeColumnKeys = keys.filter(key => getMapColumn(key) === null); + const mapKeys = keys.filter(key => getMapColumn(key) !== null); + return [ + nativeColumnKeys, + ...Object.values(groupBy(mapKeys, getMapColumn)), + ].filter(group => group.length > 0); +} + +/** Yields each promise's value as it resolves; throws on the first rejection. */ +async function* inCompletionOrder(promises: Promise[]) { + const pending = new Map( + promises.map((promise, i) => [i, promise.then(value => ({ i, value }))]), + ); + while (pending.size > 0) { + const { i, value } = await Promise.race(pending.values()); + pending.delete(i); + yield value; + } +} /** * Throw the first rejection when every lookup failed, otherwise return what @@ -485,9 +519,9 @@ async function settleAllOrThrow(promises: Promise[]): Promise { * * - `'exact'` reads metadata MVs, else the raw table, sampling the first * `max_rows_to_read` rows unless `disableRowLimit` is set. Supports - * `keyConditions`. Without them, plain columns share one query and each Map / - * JSON column gets its own; when no values are showing yet, each query's - * values land in `data` as it finishes. + * `keyConditions`. Without them, native columns share one query and each Map + * column gets its own; when no values are showing yet, each query's values + * land in `data` as it finishes. * - `'all'` tries text indexes, then metadata MVs, then the raw table, so it is * cheaper for typeahead. It skips JSON-column keys, and its lookups return no * values instead of failing, so it rarely errors. When no values are showing @@ -535,8 +569,8 @@ export function useMultipleGetKeyValues( const { data: sources, isLoading: isLoadingSources } = useSources(); const maxKeys = useFilterKeysFetchLimit(); - // 'exact' lookups yield a few large chunks, so they keep the client's defaults - // rather than the streaming hook's. An unset default stays `undefined` here, + // 'exact' lookups yield one chunk per query, so they keep the client's + // defaults rather than the streaming hook's. An unset default stays `undefined` here, // which React Query reads as its own default. `refetchOnWindowFocus` is React // Query's default outright: the client's can be a function typed against // `Query`, which doesn't fit this query's data. @@ -577,30 +611,10 @@ export function useMultipleGetKeyValues( if (keys.length === 0) return; const usableMVsOverride = chartConfigsArr.length === 1 ? metadataMVsOverride : undefined; - // The limit applies per Map column, so one wide Map (e.g. LogAttributes) - // can't crowd out the others. Top-level columns share a single limit. - const { limitedKeys, limitedKeyConditions } = keys.reduce<{ - countByColumn: Map; - limitedKeys: string[]; - limitedKeyConditions?: (FilterState | undefined)[]; - }>( - (acc, key, i) => { - const path = parseKeyPath(key); - const column = path.length > 1 ? path[0] : ''; - const count = acc.countByColumn.get(column) ?? 0; - acc.countByColumn.set(column, count + 1); - if (count < maxKeys) { - acc.limitedKeys.push(key); - acc.limitedKeyConditions?.push(keyConditions?.[i]); - } - return acc; - }, - { - countByColumn: new Map(), - limitedKeys: [], - limitedKeyConditions: keyConditions ? [] : undefined, - }, - ); + const keptIndexes = indexesWithinMapKeyLimit(keys, maxKeys); + const limitedKeys = keptIndexes.map(i => keys[i]); + const limitedKeyConditions = + keyConditions && keptIndexes.map(i => keyConditions[i]); const lookups = chartConfigsArr.map((chartConfig, i) => { const source = chartConfig.source @@ -615,19 +629,17 @@ export function useMultipleGetKeyValues( }); if (mode === 'exact') { - // Map / JSON keys are far slower to read than plain columns, so plain - // columns share one query and each Map / JSON column gets its own: one - // wide map can't hold up the rest. Faceted lookups stay in one query: a - // key's condition can reference any sibling key. + // A key's condition can reference any sibling key, so faceted lookups + // stay in one query. const keyGroups = limitedKeyConditions ? [limitedKeys] - : Object.values(groupBy(limitedKeys, keyQueryGroup)); - const groupResults = keyGroups.map(keys => - settleAllOrThrow( + : groupKeysByMapColumn(limitedKeys); + const fetchGroup = async (groupKeys: string[]) => { + const results = await settleAllOrThrow( lookups.map(({ chartConfig, source }) => metadata.getKeyValuesWithMVs({ chartConfig, - keys, + keys: groupKeys, keyConditions: limitedKeyConditions, limit, disableRowLimit, @@ -635,23 +647,12 @@ export function useMultipleGetKeyValues( signal, }), ), - ), - ); - // Unlike `mergeStreams`, a failed group fails the lookup rather than - // silently leaving its keys empty. - const pending = new Map( - groupResults.map((results, i) => [ - i, - results.then(r => [i, r] as const), - ]), - ); - while (pending.size > 0) { - const [i, results] = await Promise.race(pending.values()); - pending.delete(i); + ); // Numeric values have always passed through unconverted here; `useQuery` // used to hide the mismatch with `Facet`. HDX-1548. - yield results.flat() as Facet[]; - } + return results.flat() as Facet[]; + }; + yield* inCompletionOrder(keyGroups.map(fetchGroup)); return; } diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 6bfd41ab4..739ad6c77 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -184,10 +184,6 @@ const inlineNonNegativeInt = (value: number, label: string): string => { return String(value); }; -// `Col['key']` or `` `Col`['key'] `` with a well-formed single-quoted key. -const MAP_SUBSCRIPT_REGEX = - /^(`[^`]+`|[A-Za-z_][A-Za-z0-9_]*)\['((?:[^'\\]|\\.)*)'\]$/; - /** Strip one level of `…` / "…" identifier quoting, if present. */ export const unquoteIdentifier = (identifier: string): string => { if ( @@ -611,11 +607,11 @@ export class Metadata { }> { const literalsByColumn = new Map>(); for (const key of keys) { - const match = MAP_SUBSCRIPT_REGEX.exec(key); - if (!match) continue; - const column = unquoteIdentifier(match[1]); + const path = parseKeyPath(key); + if (path.length < 2) continue; + const column = unquoteIdentifier(path[0]); const literals = literalsByColumn.get(column) ?? new Map(); - literals.set(key, `'${match[2]}'`); + literals.set(key, `'${path[1]}'`); literalsByColumn.set(column, literals); }