diff --git a/.changeset/numeric-string-comparisons.md b/.changeset/numeric-string-comparisons.md new file mode 100644 index 000000000..f03e8d1ee --- /dev/null +++ b/.changeset/numeric-string-comparisons.md @@ -0,0 +1,13 @@ +--- +'@hyperdx/common-utils': patch +'@hyperdx/api': patch +'@hyperdx/app': patch +--- + +fix: support numeric range searches on String attributes + +Range searches with numeric bounds on a String-typed field, such as +`LogAttributes.duration_ms:[10 TO 20]`, failed with "no supertype for types +String, UInt8". They now compare with `toFloat64OrNull`; values that aren't +numbers don't match. `>`, `<`, `>=` and `<=` still compare String fields as +text; use a range such as `:[1000 TO *]` for a numeric comparison. diff --git a/packages/common-utils/src/__tests__/numericStringComparisons.int.test.ts b/packages/common-utils/src/__tests__/numericStringComparisons.int.test.ts new file mode 100644 index 000000000..0fddc273b --- /dev/null +++ b/packages/common-utils/src/__tests__/numericStringComparisons.int.test.ts @@ -0,0 +1,141 @@ +import { ClickHouseClient, createClient } from '@clickhouse/client'; + +import { parameterizedQueryToSql } from '@/clickhouse'; +import { ClickhouseClient as HdxClickhouseClient } from '@/clickhouse/node'; +import { Metadata, MetadataCache } from '@/core/metadata'; +import { renderChartConfig } from '@/core/renderChartConfig'; +import { ChartConfigWithOptDateRange } from '@/types'; + +// Unit tests only compare SQL text, which is how ranges with numeric bounds on +// String attributes shipped while ClickHouse rejected them. These run each +// search against ClickHouse and check which rows come back. +describe('numeric range bounds on String attributes', () => { + let client: ClickHouseClient; + let hdxClient: HdxClickhouseClient; + let metadata: Metadata; + + const DATABASE = 'default'; + const TABLE_NAME = 'numeric_string_comparisons_int_test'; + + // Code mirrors gc_pause_ms in a FixedString, padded with zero bytes. + const ROWS: Array<{ + Id: number; + Severity: string; + LogAttributes: Record; + Code: string; + }> = [ + { + Id: 1, + Severity: 'debug', + LogAttributes: { gc_pause_ms: '5' }, + Code: '5', + }, + { + Id: 2, + Severity: 'info', + LogAttributes: { gc_pause_ms: '10' }, + Code: '10', + }, + { + Id: 3, + Severity: 'error', + LogAttributes: { gc_pause_ms: '17.25' }, + Code: '17.25', + }, + { + Id: 4, + Severity: 'info', + LogAttributes: { gc_pause_ms: '100' }, + Code: '100', + }, + { + Id: 5, + Severity: 'info', + LogAttributes: { gc_pause_ms: '150' }, + Code: '150', + }, + { + Id: 6, + Severity: 'info', + LogAttributes: { gc_pause_ms: 'abc' }, + Code: 'abc', + }, + { Id: 7, Severity: 'info', LogAttributes: { other: 'x' }, Code: '' }, + ]; + + const queryIds = async (where: string) => { + const config: ChartConfigWithOptDateRange = { + connection: 'test-connection', + from: { databaseName: DATABASE, tableName: TABLE_NAME }, + select: 'Id', + where, + whereLanguage: 'lucene', + timestampValueExpression: 'Timestamp', + dateRange: [new Date('2025-01-01'), new Date('2025-01-02')], + orderBy: 'Id', + }; + const sql = parameterizedQueryToSql( + await renderChartConfig(config, metadata, undefined), + ); + const result = await client.query({ query: sql, format: 'JSONEachRow' }); + const rows = await result.json<{ Id: number }>(); + return rows.map(r => Number(r.Id)); + }; + + beforeAll(async () => { + const host = process.env.CLICKHOUSE_HOST || 'http://localhost:8123'; + const username = process.env.CLICKHOUSE_USER || 'default'; + const password = process.env.CLICKHOUSE_PASSWORD || ''; + + client = createClient({ url: host, username, password }); + hdxClient = new HdxClickhouseClient({ host, username, password }); + + await client.command({ + query: `CREATE OR REPLACE TABLE ${DATABASE}.${TABLE_NAME} ( + Timestamp DateTime64(9), + Id UInt32, + Severity Enum8('debug' = 1, 'info' = 5, 'error' = 9), + LogAttributes Map(LowCardinality(String), String), + GcPause String MATERIALIZED LogAttributes['gc_pause_ms'], + Code FixedString(8) + ) + ENGINE = MergeTree + ORDER BY (Timestamp)`, + }); + + await client.insert({ + table: `${DATABASE}.${TABLE_NAME}`, + values: ROWS.map(r => ({ ...r, Timestamp: '2025-01-01 12:00:00' })), + format: 'JSONEachRow', + }); + }); + + beforeEach(() => { + metadata = new Metadata(hdxClient, new MetadataCache()); + }); + + afterAll(async () => { + await client.command({ + query: `DROP TABLE IF EXISTS ${DATABASE}.${TABLE_NAME}`, + }); + await hdxClient.close(); + await client.close(); + }); + + it.each([ + ['LogAttributes.gc_pause_ms:[10 TO 100]', [2, 3, 4]], + ['LogAttributes.gc_pause_ms:{10 TO 100}', [3]], + ['LogAttributes.gc_pause_ms:[100 TO *]', [4, 5]], + // Missing (7) and non-numeric (6) values match a negated range. + ['-LogAttributes.gc_pause_ms:[10 TO 100]', [1, 5, 6, 7]], + // Comparisons stay lexical: '5', '17.25' and 'abc' sort after '100'. + ['LogAttributes.gc_pause_ms:>100', [1, 3, 5, 6]], + // A materialized String column and a FixedString compare the same way. + ['GcPause:[10 TO 100]', [2, 3, 4]], + ['Code:[10 TO 100]', [2, 3, 4]], + // Enum columns keep comparing by their numeric value. + ['Severity:[1 TO 5]', [1, 2, 4, 5, 6, 7]], + ])('%s returns the matching rows', async (where, ids) => { + expect(await queryIds(where)).toEqual(ids); + }); +}); diff --git a/packages/common-utils/src/__tests__/queryParser.test.ts b/packages/common-utils/src/__tests__/queryParser.test.ts index 85b7b5097..df3d48219 100644 --- a/packages/common-utils/src/__tests__/queryParser.test.ts +++ b/packages/common-utils/src/__tests__/queryParser.test.ts @@ -91,6 +91,7 @@ describe('CustomSchemaSQLSerializerV2 - json', () => { }, found: true, propertyType: 'json', + columnType: 'JSON', isArray: false, }); const field2 = 'ResourceAttributesJSON.test.nest'; @@ -104,6 +105,7 @@ describe('CustomSchemaSQLSerializerV2 - json', () => { }, found: true, propertyType: 'json', + columnType: 'JSON', isArray: false, }); }); @@ -362,7 +364,7 @@ describe('CustomSchemaSQLSerializerV2 - json', () => { }, { lucene: 'foo:[1 TO 5]', - sql: '((foo BETWEEN 1 AND 5))', + sql: '((ifNull(toFloat64OrNull(foo) BETWEEN 1 AND 5, 0)))', english: 'foo is between 1 and 5', }, { @@ -766,6 +768,96 @@ describe('CustomSchemaSQLSerializerV2 - json', () => { }); }); +describe('CustomSchemaSQLSerializerV2 - numeric range bounds on String values', () => { + const metadata = getMetadata( + new ClickhouseClient({ host: 'http://localhost:8123' }), + ); + // The OTel schema's attribute maps hold every value as a String. + metadata.getColumn = jest.fn().mockImplementation(async ({ column }) => { + if (column === 'LogAttributes') { + return { + name: 'LogAttributes', + type: 'Map(LowCardinality(String), String)', + }; + } else if (column === 'Duration') { + return { name: 'Duration', type: 'UInt64' }; + } else if (column === 'ServiceName') { + return { name: 'ServiceName', type: 'LowCardinality(String)' }; + } else if (column === 'Severity') { + return { name: 'Severity', type: "Enum8('debug' = 1, 'info' = 5)" }; + } + return undefined; + }); + metadata.getMaterializedColumnsLookupTable = jest + .fn() + .mockImplementation(async () => new Map()); + + const serializer = new CustomSchemaSQLSerializerV2({ + metadata, + databaseName: 'testName', + tableName: 'testTable', + connectionId: 'testId', + implicitColumnExpression: 'Body', + }); + + const attr = "toFloat64OrNull(`LogAttributes`['gc_pause_ms'])"; + const hint = "indexHint(mapContains(`LogAttributes`, 'gc_pause_ms'))"; + + it.each([ + { + lucene: 'LogAttributes.gc_pause_ms:[10 TO 20]', + sql: `((ifNull(${attr} BETWEEN 10 AND 20, 0) AND ${hint}))`, + }, + { + lucene: 'LogAttributes.gc_pause_ms:[-2.5 TO 1e3]', + sql: `((ifNull(${attr} BETWEEN -2.5 AND 1000, 0) AND ${hint}))`, + }, + { + lucene: 'LogAttributes.gc_pause_ms:{10 TO 20}', + sql: `((ifNull(${attr} > 10, 0) AND ifNull(${attr} < 20, 0) AND ${hint}))`, + }, + { + lucene: 'LogAttributes.gc_pause_ms:[10 TO *]', + sql: `((ifNull(${attr} >= 10, 0) AND ${hint}))`, + }, + { + lucene: '-LogAttributes.gc_pause_ms:[10 TO 20]', + sql: `((NOT ifNull(${attr} BETWEEN 10 AND 20, 0)))`, + }, + { + lucene: 'ServiceName:[1 TO 5]', + sql: '((ifNull(toFloat64OrNull(ServiceName) BETWEEN 1 AND 5, 0)))', + }, + ])('compares "$lucene" as numbers', async ({ lucene, sql }) => { + const builder = new SearchQueryBuilder(lucene, serializer); + expect(await builder.build()).toBe(sql); + }); + + it.each([ + { + // Comparisons stay lexical: fixed-width values such as dates rely on it. + lucene: 'LogAttributes.gc_pause_ms:>100', + sql: `((\`LogAttributes\`['gc_pause_ms'] > '100' AND ${hint}))`, + }, + { + lucene: 'LogAttributes.version:[a TO m]', + sql: "((`LogAttributes`['version'] BETWEEN 'a' AND 'm' AND indexHint(mapContains(`LogAttributes`, 'version'))))", + }, + { + lucene: 'Duration:[10 TO 20]', + sql: '((Duration BETWEEN 10 AND 20))', + }, + { + // Enum compares to numbers already; toFloat64OrNull would reject it. + lucene: 'Severity:[1 TO 5]', + sql: '((Severity BETWEEN 1 AND 5))', + }, + ])('leaves "$lucene" unchanged', async ({ lucene, sql }) => { + const builder = new SearchQueryBuilder(lucene, serializer); + expect(await builder.build()).toBe(sql); + }); +}); + describe('CustomSchemaSQLSerializerV2 - range bounds', () => { const metadata = getMetadata( new ClickhouseClient({ host: 'http://localhost:8123' }), @@ -2433,6 +2525,7 @@ describe('CustomSchemaSQLSerializerV2 - Array and Nested Fields', () => { column: 'Events.Name', found: true, propertyType: JSDataType.String, + columnType: 'Array(String)', isArray: true, }); @@ -2442,6 +2535,7 @@ describe('CustomSchemaSQLSerializerV2 - Array and Nested Fields', () => { column: 'Events.Count', found: true, propertyType: JSDataType.Number, + columnType: 'Array(UInt64)', isArray: true, }); @@ -2451,6 +2545,7 @@ describe('CustomSchemaSQLSerializerV2 - Array and Nested Fields', () => { column: 'Events.IsAvailable', found: true, propertyType: JSDataType.Bool, + columnType: 'Array(Bool)', isArray: true, }); }); diff --git a/packages/common-utils/src/queryParser.ts b/packages/common-utils/src/queryParser.ts index e6a34a8e9..eff632811 100644 --- a/packages/common-utils/src/queryParser.ts +++ b/packages/common-utils/src/queryParser.ts @@ -15,6 +15,7 @@ import { isClickHouseVersionAtLeast, supportsDirectReadMap, } from '@/core/clickhouseVersion'; +import { stripTypeWrappers } from '@/core/eventDeltas'; import { Metadata, parseKeyPath, @@ -24,6 +25,7 @@ import { } from '@/core/metadata'; import { parseTokenizerFromTextIndex, + parseToNumber, splitAndTrimWithBracket, } from '@/core/utils'; import { UseTextIndex } from '@/types'; @@ -130,6 +132,46 @@ function normalizeChExpression(expr: string): string { export const IMPLICIT_FIELD = ''; const RANGE_UNBOUNDED = '*'; +// Decimal numbers only; parseToNumber also accepts hex such as 0x10. +const isNumericLiteral = (term: string) => + /^[-+.\deE]+$/.test(term) && parseToNumber(term) !== undefined; + +/** + * Map attributes (and other String columns) hold numbers as text, and a range + * with numeric bounds on them fails with "no supertype for types String, + * UInt8". Enum, UUID and IP columns also map to JSDataType.String but + * toFloat64OrNull rejects them, so check the type. + */ +function isStringComparedToNumbers( + columnType: string | undefined, + terms: string[], +) { + if (!columnType) { + return false; + } + return ( + (stripTypeWrappers(columnType) === 'String' || + isCHFixedStringType(columnType)) && + terms.length > 0 && + terms.every(isNumericLiteral) + ); +} + +/** + * Values that aren't numbers convert to NULL. ifNull makes them a non-match + * rather than NULL, so a NOT around the range still includes them. + */ +function numericComparison( + column: string | undefined, + comparison: string, + values: number[], +) { + return SqlString.format( + `ifNull(toFloat64OrNull(${column}) ${comparison}, 0)`, + values, + ); +} + // Type guards for lucene AST types function isNodeTerm(node: lucene.Node | lucene.AST): node is lucene.NodeTerm { return 'term' in node && node.term != null; @@ -485,6 +527,8 @@ export abstract class SQLSerializer implements Serializer { column?: string; columnJSON?: { string: string; number: string }; propertyType?: JSDataType; + /** The ClickHouse type of the resolved column or map value. */ + columnType?: string; /** * The resolved column is FixedString. Token functions (hasToken, * hasAllTokens) reject that haystack and need CAST(... AS String). @@ -798,7 +842,7 @@ export abstract class SQLSerializer implements Serializer { context: SerializerContext, inclusive: lucene.NodeRangedTerm['inclusive'] = 'both', ) { - const { column, found, mapKeyIndexExpression, isArray } = + const { column, found, mapKeyIndexExpression, isArray, columnType } = await this.getColumnForField(field, context); if (!found) { return this.NOT_FOUND_QUERY; @@ -818,31 +862,36 @@ export abstract class SQLSerializer implements Serializer { if (isStartUnbounded && isEndUnbounded) { return this.isNotNull(field, isNegatedField, context); } + const compareAsNumbers = isStringComparedToNumbers( + columnType, + [start, end].filter(bound => bound !== RANGE_UNBOUNDED), + ); if (!isStartUnbounded && !isEndUnbounded && inclusive === 'both') { + if (compareAsNumbers) { + return `(${isNegatedField ? 'NOT ' : ''}${numericComparison(column, 'BETWEEN ? AND ?', [Number(start), Number(end)])}${expressionPostfix})`; + } return SqlString.format( `(${column} ${isNegatedField ? 'NOT ' : ''}BETWEEN ? AND ?${expressionPostfix})`, [this.attemptToParseNumber(start), this.attemptToParseNumber(end)], ); } + const compareBound = (operator: string, bound: string) => + compareAsNumbers + ? numericComparison(column, `${operator} ?`, [Number(bound)]) + : SqlString.format(`${column} ${operator} ?`, [ + this.attemptToParseNumber(bound), + ]); const bounds: string[] = []; if (!isStartUnbounded) { const operator = inclusive === 'both' || inclusive === 'left' ? '>=' : '>'; - bounds.push( - SqlString.format(`${column} ${operator} ?`, [ - this.attemptToParseNumber(start), - ]), - ); + bounds.push(compareBound(operator, start)); } if (!isEndUnbounded) { const operator = inclusive === 'both' || inclusive === 'right' ? '<=' : '<'; - bounds.push( - SqlString.format(`${column} ${operator} ?`, [ - this.attemptToParseNumber(end), - ]), - ); + bounds.push(compareBound(operator, end)); } return isNegatedField ? `(NOT (${bounds.join(' AND ')})${expressionPostfix})` @@ -2031,6 +2080,7 @@ export class CustomSchemaSQLSerializerV2 extends SQLSerializer { column: expression.columnExpression, columnJSON: expression?.columnExpressionJSON, propertyType: type ?? undefined, + columnType: expression.columnType, isArray, found: expression.found, mapKey: expression.mapKey,