From 0e2e65fd1afcff29b8a90b4713f554ef4113da7d Mon Sep 17 00:00:00 2001 From: aryainguz Date: Tue, 17 Nov 2026 18:15:19 +0530 Subject: [PATCH 1/6] fix(common-utils): compare numeric search bounds as numbers on String attributes Attribute maps store every value as a String, so `field:>100` compared text ('17.25' > '100' is true) and ranges with numeric bounds failed with "no supertype for types String, UInt8". When every bound is a numeric literal and the field is String-typed, compare toFloat64OrNull(field) instead. ifNull keeps non-numeric and missing values a non-match, so a surrounding NOT still includes them. --- .changeset/numeric-string-comparisons.md | 13 ++ .../src/__tests__/queryParser.test.ts | 115 +++++++++++++++++- packages/common-utils/src/queryParser.ts | 75 ++++++++++-- 3 files changed, 191 insertions(+), 12 deletions(-) create mode 100644 .changeset/numeric-string-comparisons.md diff --git a/.changeset/numeric-string-comparisons.md b/.changeset/numeric-string-comparisons.md new file mode 100644 index 000000000..2f69f86bb --- /dev/null +++ b/.changeset/numeric-string-comparisons.md @@ -0,0 +1,13 @@ +--- +'@hyperdx/common-utils': patch +'@hyperdx/api': patch +'@hyperdx/app': patch +--- + +fix: compare numeric search bounds as numbers on String attributes + +Searches like `LogAttributes.duration_ms:>1000` compared the attribute's text +value to `'1000'` as strings, so `'17.25' > '100'` matched and real matches +were missed, and ranges such as `:[10 TO 20]` failed with "no supertype for +types String, UInt8". A numeric bound on a String-typed field now compares +with `toFloat64OrNull`; values that aren't numbers don't match. diff --git a/packages/common-utils/src/__tests__/queryParser.test.ts b/packages/common-utils/src/__tests__/queryParser.test.ts index 85b7b5097..00657122e 100644 --- a/packages/common-utils/src/__tests__/queryParser.test.ts +++ b/packages/common-utils/src/__tests__/queryParser.test.ts @@ -362,7 +362,7 @@ describe('CustomSchemaSQLSerializerV2 - json', () => { }, { lucene: 'foo:[1 TO 5]', - sql: '((foo BETWEEN 1 AND 5))', + sql: '((ifNull(toFloat64OrNull(foo) BETWEEN 1 AND 5, 0)))', english: 'foo is between 1 and 5', }, { @@ -766,6 +766,119 @@ describe('CustomSchemaSQLSerializerV2 - json', () => { }); }); +describe('CustomSchemaSQLSerializerV2 - numeric bounds on String values', () => { + const metadata = getMetadata( + new ClickhouseClient({ host: 'http://localhost:8123' }), + ); + // The OTel schema's attribute maps hold every value as a String. + metadata.getColumn = jest.fn().mockImplementation(async ({ column }) => { + if (column === 'LogAttributes') { + return { + name: 'LogAttributes', + type: 'Map(LowCardinality(String), String)', + }; + } else if (column === 'Duration') { + return { name: 'Duration', type: 'UInt64' }; + } else if (column === 'ServiceName') { + return { name: 'ServiceName', type: 'LowCardinality(String)' }; + } + return undefined; + }); + metadata.getMaterializedColumnsLookupTable = jest + .fn() + .mockImplementation(async () => new Map()); + + const serializer = new CustomSchemaSQLSerializerV2({ + metadata, + databaseName: 'testName', + tableName: 'testTable', + connectionId: 'testId', + implicitColumnExpression: 'Body', + }); + + const attr = "toFloat64OrNull(`LogAttributes`['gc_pause_ms'])"; + const hint = "indexHint(mapContains(`LogAttributes`, 'gc_pause_ms'))"; + + it.each([ + { + lucene: 'LogAttributes.gc_pause_ms:>100', + sql: `((ifNull(${attr} > 100, 0) AND ${hint}))`, + }, + { + lucene: 'LogAttributes.gc_pause_ms:>=100', + sql: `((ifNull(${attr} >= 100, 0) AND ${hint}))`, + }, + { + lucene: 'LogAttributes.gc_pause_ms:<100', + sql: `((ifNull(${attr} < 100, 0) AND ${hint}))`, + }, + { + lucene: 'LogAttributes.gc_pause_ms:<=100', + sql: `((ifNull(${attr} <= 100, 0) AND ${hint}))`, + }, + { + lucene: 'LogAttributes.gc_pause_ms:>-2.5', + sql: `((ifNull(${attr} > -2.5, 0) AND ${hint}))`, + }, + { + lucene: 'LogAttributes.gc_pause_ms:>1e3', + sql: `((ifNull(${attr} > 1000, 0) AND ${hint}))`, + }, + { + lucene: '-LogAttributes.gc_pause_ms:>100', + sql: `((ifNull(${attr} <= 100, 0) AND ${hint}))`, + }, + { + lucene: 'LogAttributes.gc_pause_ms:[10 TO 20]', + sql: `((ifNull(${attr} BETWEEN 10 AND 20, 0) AND ${hint}))`, + }, + { + lucene: 'LogAttributes.gc_pause_ms:{10 TO 20}', + sql: `((ifNull(${attr} > 10, 0) AND ifNull(${attr} < 20, 0) AND ${hint}))`, + }, + { + lucene: 'LogAttributes.gc_pause_ms:[10 TO *]', + sql: `((ifNull(${attr} >= 10, 0) AND ${hint}))`, + }, + { + lucene: '-LogAttributes.gc_pause_ms:[10 TO 20]', + sql: `((NOT ifNull(${attr} BETWEEN 10 AND 20, 0)))`, + }, + { + lucene: 'ServiceName:[1 TO 5]', + sql: '((ifNull(toFloat64OrNull(ServiceName) BETWEEN 1 AND 5, 0)))', + }, + ])('compares "$lucene" as numbers', async ({ lucene, sql }) => { + const builder = new SearchQueryBuilder(lucene, serializer); + expect(await builder.build()).toBe(sql); + }); + + it.each([ + { + // A non-numeric bound keeps the string comparison. + lucene: 'LogAttributes.version:>abc', + sql: "((`LogAttributes`['version'] > 'abc' AND indexHint(mapContains(`LogAttributes`, 'version'))))", + }, + { + // Hex is not a numeric literal here. + lucene: 'LogAttributes.version:>0x10', + sql: "((`LogAttributes`['version'] > '0x10' AND indexHint(mapContains(`LogAttributes`, 'version'))))", + }, + { + // Numeric columns are compared by ClickHouse already. + lucene: 'Duration:>100', + sql: "((Duration > '100'))", + }, + { + lucene: 'Duration:[10 TO 20]', + sql: '((Duration BETWEEN 10 AND 20))', + }, + ])('leaves "$lucene" unchanged', async ({ lucene, sql }) => { + const builder = new SearchQueryBuilder(lucene, serializer); + expect(await builder.build()).toBe(sql); + }); +}); + describe('CustomSchemaSQLSerializerV2 - range bounds', () => { const metadata = getMetadata( new ClickhouseClient({ host: 'http://localhost:8123' }), diff --git a/packages/common-utils/src/queryParser.ts b/packages/common-utils/src/queryParser.ts index e6a34a8e9..2ba64cdb4 100644 --- a/packages/common-utils/src/queryParser.ts +++ b/packages/common-utils/src/queryParser.ts @@ -130,6 +130,42 @@ function normalizeChExpression(expr: string): string { export const IMPLICIT_FIELD = ''; const RANGE_UNBOUNDED = '*'; +// Decimal and exponent notation only, so hex such as 0x10 stays a string. +const isNumericLiteral = (term: string) => + /^[-+.\deE]+$/.test(term) && Number.isFinite(Number(term)); + +/** + * Map attributes (and other String columns) hold numbers as text. Compared as + * strings, `'17.25' > '100'` is true, and BETWEEN with numeric bounds fails + * with "no supertype for types String, UInt8", so numeric bounds on a String + * field need a numeric comparison. + */ +function isStringComparedToNumbers( + propertyType: JSDataType | undefined, + terms: string[], +) { + return ( + propertyType === JSDataType.String && + terms.length > 0 && + terms.every(isNumericLiteral) + ); +} + +/** + * Values that aren't numbers convert to NULL. ifNull makes them a non-match + * rather than NULL, so a NOT around the search still matches them. + */ +function numericComparison( + column: string | undefined, + comparison: string, + values: number[], +) { + return SqlString.format( + `ifNull(toFloat64OrNull(${column}) ${comparison}, 0)`, + values, + ); +} + // Type guards for lucene AST types function isNodeTerm(node: lucene.Node | lucene.AST): node is lucene.NodeTerm { return 'term' in node && node.term != null; @@ -678,6 +714,9 @@ export abstract class SQLSerializer implements Serializer { [term], ); } + if (isStringComparedToNumbers(propertyType, [term])) { + return `(${numericComparison(column, '>= ?', [Number(term)])}${expressionPostfix})`; + } return SqlString.format(`(${column} >= ?${expressionPostfix})`, [term]); } @@ -705,6 +744,9 @@ export abstract class SQLSerializer implements Serializer { [term], ); } + if (isStringComparedToNumbers(propertyType, [term])) { + return `(${numericComparison(column, '<= ?', [Number(term)])}${expressionPostfix})`; + } return SqlString.format(`(${column} <= ?${expressionPostfix})`, [term]); } @@ -732,6 +774,9 @@ export abstract class SQLSerializer implements Serializer { [term], ); } + if (isStringComparedToNumbers(propertyType, [term])) { + return `(${numericComparison(column, '< ?', [Number(term)])}${expressionPostfix})`; + } return SqlString.format(`(${column} < ?${expressionPostfix})`, [term]); } @@ -759,6 +804,9 @@ export abstract class SQLSerializer implements Serializer { [term], ); } + if (isStringComparedToNumbers(propertyType, [term])) { + return `(${numericComparison(column, '> ?', [Number(term)])}${expressionPostfix})`; + } return SqlString.format(`(${column} > ?${expressionPostfix})`, [term]); } @@ -798,7 +846,7 @@ export abstract class SQLSerializer implements Serializer { context: SerializerContext, inclusive: lucene.NodeRangedTerm['inclusive'] = 'both', ) { - const { column, found, mapKeyIndexExpression, isArray } = + const { column, found, mapKeyIndexExpression, isArray, propertyType } = await this.getColumnForField(field, context); if (!found) { return this.NOT_FOUND_QUERY; @@ -818,31 +866,36 @@ export abstract class SQLSerializer implements Serializer { if (isStartUnbounded && isEndUnbounded) { return this.isNotNull(field, isNegatedField, context); } + const compareAsNumbers = isStringComparedToNumbers( + propertyType, + [start, end].filter(bound => bound !== RANGE_UNBOUNDED), + ); if (!isStartUnbounded && !isEndUnbounded && inclusive === 'both') { + if (compareAsNumbers) { + return `(${isNegatedField ? 'NOT ' : ''}${numericComparison(column, 'BETWEEN ? AND ?', [Number(start), Number(end)])}${expressionPostfix})`; + } return SqlString.format( `(${column} ${isNegatedField ? 'NOT ' : ''}BETWEEN ? AND ?${expressionPostfix})`, [this.attemptToParseNumber(start), this.attemptToParseNumber(end)], ); } + const compareBound = (operator: string, bound: string) => + compareAsNumbers + ? numericComparison(column, `${operator} ?`, [Number(bound)]) + : SqlString.format(`${column} ${operator} ?`, [ + this.attemptToParseNumber(bound), + ]); const bounds: string[] = []; if (!isStartUnbounded) { const operator = inclusive === 'both' || inclusive === 'left' ? '>=' : '>'; - bounds.push( - SqlString.format(`${column} ${operator} ?`, [ - this.attemptToParseNumber(start), - ]), - ); + bounds.push(compareBound(operator, start)); } if (!isEndUnbounded) { const operator = inclusive === 'both' || inclusive === 'right' ? '<=' : '<'; - bounds.push( - SqlString.format(`${column} ${operator} ?`, [ - this.attemptToParseNumber(end), - ]), - ); + bounds.push(compareBound(operator, end)); } return isNegatedField ? `(NOT (${bounds.join(' AND ')})${expressionPostfix})` From b94f4b97a3f0fc61f0d8c73a5bc0dbe0e671224b Mon Sep 17 00:00:00 2001 From: aryainguz Date: Tue, 17 Nov 2026 18:48:31 +0530 Subject: [PATCH 2/6] refactor(common-utils): reuse parseToNumber for numeric search bounds --- packages/common-utils/src/queryParser.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/common-utils/src/queryParser.ts b/packages/common-utils/src/queryParser.ts index 2ba64cdb4..dc5d2d533 100644 --- a/packages/common-utils/src/queryParser.ts +++ b/packages/common-utils/src/queryParser.ts @@ -24,6 +24,7 @@ import { } from '@/core/metadata'; import { parseTokenizerFromTextIndex, + parseToNumber, splitAndTrimWithBracket, } from '@/core/utils'; import { UseTextIndex } from '@/types'; @@ -130,9 +131,9 @@ function normalizeChExpression(expr: string): string { export const IMPLICIT_FIELD = ''; const RANGE_UNBOUNDED = '*'; -// Decimal and exponent notation only, so hex such as 0x10 stays a string. +// parseToNumber also accepts hex such as 0x10; keep those as strings. const isNumericLiteral = (term: string) => - /^[-+.\deE]+$/.test(term) && Number.isFinite(Number(term)); + /^[-+.\deE]+$/.test(term) && parseToNumber(term) !== undefined; /** * Map attributes (and other String columns) hold numbers as text. Compared as From 71a2b78bde5d1013c07b2337b54a3c450486dd3c Mon Sep 17 00:00:00 2001 From: aryainguz Date: Tue, 17 Nov 2026 19:05:22 +0530 Subject: [PATCH 3/6] fix(common-utils): keep negated comparisons and non-String types consistent - `-field:>x` is rewritten to the opposite operator, so missing and non-numeric values now match it, as for `-field:[a TO b]` and NOT. - Only String and FixedString columns get toFloat64OrNull. Enum, UUID and IP types also map to JSDataType.String but ClickHouse rejects toFloat64OrNull for them, and Enum already compares to numbers. - Integration test runs each search against ClickHouse and checks the returned rows. --- .../numericStringComparisons.int.test.ts | 109 ++++++++++++++++++ .../src/__tests__/queryParser.test.ts | 25 +++- packages/common-utils/src/queryParser.ts | 77 +++++++++---- 3 files changed, 190 insertions(+), 21 deletions(-) create mode 100644 packages/common-utils/src/__tests__/numericStringComparisons.int.test.ts diff --git a/packages/common-utils/src/__tests__/numericStringComparisons.int.test.ts b/packages/common-utils/src/__tests__/numericStringComparisons.int.test.ts new file mode 100644 index 000000000..81f3747da --- /dev/null +++ b/packages/common-utils/src/__tests__/numericStringComparisons.int.test.ts @@ -0,0 +1,109 @@ +import { ClickHouseClient, createClient } from '@clickhouse/client'; + +import { parameterizedQueryToSql } from '@/clickhouse'; +import { ClickhouseClient as HdxClickhouseClient } from '@/clickhouse/node'; +import { Metadata, MetadataCache } from '@/core/metadata'; +import { renderChartConfig } from '@/core/renderChartConfig'; +import { ChartConfigWithOptDateRange } from '@/types'; + +// Unit tests only compare SQL text, which is how string comparisons of +// numeric attributes shipped. These run each search against ClickHouse and +// check which rows come back. +describe('numeric search bounds on String attributes', () => { + let client: ClickHouseClient; + let hdxClient: HdxClickhouseClient; + let metadata: Metadata; + + const DATABASE = 'default'; + const TABLE_NAME = 'numeric_string_comparisons_int_test'; + + const ROWS: Array<{ + Id: number; + Severity: string; + LogAttributes: Record; + }> = [ + { Id: 1, Severity: 'debug', LogAttributes: { gc_pause_ms: '5' } }, + { Id: 2, Severity: 'info', LogAttributes: { gc_pause_ms: '10' } }, + { Id: 3, Severity: 'error', LogAttributes: { gc_pause_ms: '17.25' } }, + { Id: 4, Severity: 'info', LogAttributes: { gc_pause_ms: '100' } }, + { Id: 5, Severity: 'info', LogAttributes: { gc_pause_ms: '150' } }, + { Id: 6, Severity: 'info', LogAttributes: { gc_pause_ms: 'abc' } }, + { Id: 7, Severity: 'info', LogAttributes: { other: 'x' } }, + ]; + + const queryIds = async (where: string) => { + const config: ChartConfigWithOptDateRange = { + connection: 'test-connection', + from: { databaseName: DATABASE, tableName: TABLE_NAME }, + select: 'Id', + where, + whereLanguage: 'lucene', + timestampValueExpression: 'Timestamp', + dateRange: [new Date('2025-01-01'), new Date('2025-01-02')], + orderBy: 'Id', + }; + const sql = parameterizedQueryToSql( + await renderChartConfig(config, metadata, undefined), + ); + const result = await client.query({ query: sql, format: 'JSONEachRow' }); + const rows = await result.json<{ Id: number }>(); + return rows.map(r => Number(r.Id)); + }; + + beforeAll(async () => { + const host = process.env.CLICKHOUSE_HOST || 'http://localhost:8123'; + const username = process.env.CLICKHOUSE_USER || 'default'; + const password = process.env.CLICKHOUSE_PASSWORD || ''; + + client = createClient({ url: host, username, password }); + hdxClient = new HdxClickhouseClient({ host, username, password }); + + await client.command({ + query: `CREATE OR REPLACE TABLE ${DATABASE}.${TABLE_NAME} ( + Timestamp DateTime64(9), + Id UInt32, + Severity Enum8('debug' = 1, 'info' = 5, 'error' = 9), + LogAttributes Map(LowCardinality(String), String) + ) + ENGINE = MergeTree + ORDER BY (Timestamp)`, + }); + + await client.insert({ + table: `${DATABASE}.${TABLE_NAME}`, + values: ROWS.map(r => ({ ...r, Timestamp: '2025-01-01 12:00:00' })), + format: 'JSONEachRow', + }); + }); + + beforeEach(() => { + metadata = new Metadata(hdxClient, new MetadataCache()); + }); + + afterAll(async () => { + await client.command({ + query: `DROP TABLE IF EXISTS ${DATABASE}.${TABLE_NAME}`, + }); + await hdxClient.close(); + await client.close(); + }); + + it.each([ + ['LogAttributes.gc_pause_ms:>100', [5]], + ['LogAttributes.gc_pause_ms:>=100', [4, 5]], + ['LogAttributes.gc_pause_ms:<10', [1]], + ['LogAttributes.gc_pause_ms:<=10', [1, 2]], + ['LogAttributes.gc_pause_ms:[10 TO 100]', [2, 3, 4]], + ['LogAttributes.gc_pause_ms:{10 TO 100}', [3]], + ['LogAttributes.gc_pause_ms:[100 TO *]', [4, 5]], + // Missing (7) and non-numeric (6) values match every negated form. + ['-LogAttributes.gc_pause_ms:>100', [1, 2, 3, 4, 6, 7]], + ['NOT LogAttributes.gc_pause_ms:>100', [1, 2, 3, 4, 6, 7]], + ['-LogAttributes.gc_pause_ms:<10', [2, 3, 4, 5, 6, 7]], + ['-LogAttributes.gc_pause_ms:[10 TO 100]', [1, 5, 6, 7]], + // Enum columns keep comparing by their numeric value. + ['Severity:[1 TO 5]', [1, 2, 4, 5, 6, 7]], + ])('%s returns the matching rows', async (where, ids) => { + expect(await queryIds(where)).toEqual(ids); + }); +}); diff --git a/packages/common-utils/src/__tests__/queryParser.test.ts b/packages/common-utils/src/__tests__/queryParser.test.ts index 00657122e..3a66234e7 100644 --- a/packages/common-utils/src/__tests__/queryParser.test.ts +++ b/packages/common-utils/src/__tests__/queryParser.test.ts @@ -91,6 +91,7 @@ describe('CustomSchemaSQLSerializerV2 - json', () => { }, found: true, propertyType: 'json', + columnType: 'JSON', isArray: false, }); const field2 = 'ResourceAttributesJSON.test.nest'; @@ -104,6 +105,7 @@ describe('CustomSchemaSQLSerializerV2 - json', () => { }, found: true, propertyType: 'json', + columnType: 'JSON', isArray: false, }); }); @@ -781,6 +783,10 @@ describe('CustomSchemaSQLSerializerV2 - numeric bounds on String values', () => return { name: 'Duration', type: 'UInt64' }; } else if (column === 'ServiceName') { return { name: 'ServiceName', type: 'LowCardinality(String)' }; + } else if (column === 'Severity') { + return { name: 'Severity', type: "Enum8('debug' = 1, 'info' = 5)" }; + } else if (column === 'SrcIp') { + return { name: 'SrcIp', type: 'IPv4' }; } return undefined; }); @@ -825,8 +831,13 @@ describe('CustomSchemaSQLSerializerV2 - numeric bounds on String values', () => sql: `((ifNull(${attr} > 1000, 0) AND ${hint}))`, }, { + // Missing and non-numeric values match, as for NOT and -field:[a TO b]. lucene: '-LogAttributes.gc_pause_ms:>100', - sql: `((ifNull(${attr} <= 100, 0) AND ${hint}))`, + sql: `((ifNull(${attr} <= 100, 1)))`, + }, + { + lucene: '-LogAttributes.gc_pause_ms:<10', + sql: `((ifNull(${attr} >= 10, 1)))`, }, { lucene: 'LogAttributes.gc_pause_ms:[10 TO 20]', @@ -873,6 +884,15 @@ describe('CustomSchemaSQLSerializerV2 - numeric bounds on String values', () => lucene: 'Duration:[10 TO 20]', sql: '((Duration BETWEEN 10 AND 20))', }, + { + // Enum compares to numbers already; toFloat64OrNull would reject it. + lucene: 'Severity:[1 TO 5]', + sql: '((Severity BETWEEN 1 AND 5))', + }, + { + lucene: 'SrcIp:>100', + sql: "((SrcIp > '100'))", + }, ])('leaves "$lucene" unchanged', async ({ lucene, sql }) => { const builder = new SearchQueryBuilder(lucene, serializer); expect(await builder.build()).toBe(sql); @@ -2546,6 +2566,7 @@ describe('CustomSchemaSQLSerializerV2 - Array and Nested Fields', () => { column: 'Events.Name', found: true, propertyType: JSDataType.String, + columnType: 'Array(String)', isArray: true, }); @@ -2555,6 +2576,7 @@ describe('CustomSchemaSQLSerializerV2 - Array and Nested Fields', () => { column: 'Events.Count', found: true, propertyType: JSDataType.Number, + columnType: 'Array(UInt64)', isArray: true, }); @@ -2564,6 +2586,7 @@ describe('CustomSchemaSQLSerializerV2 - Array and Nested Fields', () => { column: 'Events.IsAvailable', found: true, propertyType: JSDataType.Bool, + columnType: 'Array(Bool)', isArray: true, }); }); diff --git a/packages/common-utils/src/queryParser.ts b/packages/common-utils/src/queryParser.ts index dc5d2d533..99b22f646 100644 --- a/packages/common-utils/src/queryParser.ts +++ b/packages/common-utils/src/queryParser.ts @@ -15,6 +15,7 @@ import { isClickHouseVersionAtLeast, supportsDirectReadMap, } from '@/core/clickhouseVersion'; +import { stripTypeWrappers } from '@/core/eventDeltas'; import { Metadata, parseKeyPath, @@ -139,34 +140,60 @@ const isNumericLiteral = (term: string) => * Map attributes (and other String columns) hold numbers as text. Compared as * strings, `'17.25' > '100'` is true, and BETWEEN with numeric bounds fails * with "no supertype for types String, UInt8", so numeric bounds on a String - * field need a numeric comparison. + * field need a numeric comparison. Enum, UUID and IP columns also map to + * JSDataType.String but toFloat64OrNull rejects them, so check the type. */ function isStringComparedToNumbers( - propertyType: JSDataType | undefined, + columnType: string | undefined, terms: string[], ) { + const baseType = columnType ? stripTypeWrappers(columnType) : ''; return ( - propertyType === JSDataType.String && + (baseType === 'String' || baseType.startsWith('FixedString(')) && terms.length > 0 && terms.every(isNumericLiteral) ); } /** - * Values that aren't numbers convert to NULL. ifNull makes them a non-match - * rather than NULL, so a NOT around the search still matches them. + * Values that aren't numbers convert to NULL. ifNull makes them a plain match + * or non-match rather than NULL, so a NOT around the search still works. */ function numericComparison( column: string | undefined, comparison: string, values: number[], + matchNonNumeric = false, ) { return SqlString.format( - `ifNull(toFloat64OrNull(${column}) ${comparison}, 0)`, + `ifNull(toFloat64OrNull(${column}) ${comparison}, ${matchNonNumeric ? 1 : 0})`, values, ); } +/** + * `-field:>x` reaches gt/gte/lt/lte as the opposite operator (`field:<=x`), + * so missing and non-numeric values must match it, as they do for + * `-field:[a TO b]`. The map-key index hint would skip rows without the key, + * so negated comparisons leave it off, as range() does. + */ +function numericBound( + column: string | undefined, + operator: string, + term: string, + context: SerializerContext, + expressionPostfix: string, +) { + const negated = context.isNegatedComparison === true; + const comparison = numericComparison( + column, + `${operator} ?`, + [Number(term)], + negated, + ); + return `(${comparison}${negated ? '' : expressionPostfix})`; +} + // Type guards for lucene AST types function isNodeTerm(node: lucene.Node | lucene.AST): node is lucene.NodeTerm { return 'term' in node && node.term != null; @@ -262,6 +289,8 @@ interface SerializerContext { */ bodyExpression?: string; isNegatedAndParenthesized?: boolean; + /** Set when the visitor turned `-field:>x` into the opposite operator. */ + isNegatedComparison?: boolean; } interface Serializer { @@ -522,6 +551,8 @@ export abstract class SQLSerializer implements Serializer { column?: string; columnJSON?: { string: string; number: string }; propertyType?: JSDataType; + /** The ClickHouse type of the resolved column or map value. */ + columnType?: string; /** * The resolved column is FixedString. Token functions (hasToken, * hasAllTokens) reject that haystack and need CAST(... AS String). @@ -697,6 +728,7 @@ export abstract class SQLSerializer implements Serializer { columnJSON, found, propertyType, + columnType, isArray, mapKeyIndexExpression, } = await this.getColumnForField(field, context); @@ -715,8 +747,8 @@ export abstract class SQLSerializer implements Serializer { [term], ); } - if (isStringComparedToNumbers(propertyType, [term])) { - return `(${numericComparison(column, '>= ?', [Number(term)])}${expressionPostfix})`; + if (isStringComparedToNumbers(columnType, [term])) { + return numericBound(column, '>=', term, context, expressionPostfix); } return SqlString.format(`(${column} >= ?${expressionPostfix})`, [term]); } @@ -727,6 +759,7 @@ export abstract class SQLSerializer implements Serializer { columnJSON, found, propertyType, + columnType, isArray, mapKeyIndexExpression, } = await this.getColumnForField(field, context); @@ -745,8 +778,8 @@ export abstract class SQLSerializer implements Serializer { [term], ); } - if (isStringComparedToNumbers(propertyType, [term])) { - return `(${numericComparison(column, '<= ?', [Number(term)])}${expressionPostfix})`; + if (isStringComparedToNumbers(columnType, [term])) { + return numericBound(column, '<=', term, context, expressionPostfix); } return SqlString.format(`(${column} <= ?${expressionPostfix})`, [term]); } @@ -757,6 +790,7 @@ export abstract class SQLSerializer implements Serializer { columnJSON, found, propertyType, + columnType, isArray, mapKeyIndexExpression, } = await this.getColumnForField(field, context); @@ -775,8 +809,8 @@ export abstract class SQLSerializer implements Serializer { [term], ); } - if (isStringComparedToNumbers(propertyType, [term])) { - return `(${numericComparison(column, '< ?', [Number(term)])}${expressionPostfix})`; + if (isStringComparedToNumbers(columnType, [term])) { + return numericBound(column, '<', term, context, expressionPostfix); } return SqlString.format(`(${column} < ?${expressionPostfix})`, [term]); } @@ -787,6 +821,7 @@ export abstract class SQLSerializer implements Serializer { columnJSON, found, propertyType, + columnType, isArray, mapKeyIndexExpression, } = await this.getColumnForField(field, context); @@ -805,8 +840,8 @@ export abstract class SQLSerializer implements Serializer { [term], ); } - if (isStringComparedToNumbers(propertyType, [term])) { - return `(${numericComparison(column, '> ?', [Number(term)])}${expressionPostfix})`; + if (isStringComparedToNumbers(columnType, [term])) { + return numericBound(column, '>', term, context, expressionPostfix); } return SqlString.format(`(${column} > ?${expressionPostfix})`, [term]); } @@ -847,7 +882,7 @@ export abstract class SQLSerializer implements Serializer { context: SerializerContext, inclusive: lucene.NodeRangedTerm['inclusive'] = 'both', ) { - const { column, found, mapKeyIndexExpression, isArray, propertyType } = + const { column, found, mapKeyIndexExpression, isArray, columnType } = await this.getColumnForField(field, context); if (!found) { return this.NOT_FOUND_QUERY; @@ -868,7 +903,7 @@ export abstract class SQLSerializer implements Serializer { return this.isNotNull(field, isNegatedField, context); } const compareAsNumbers = isStringComparedToNumbers( - propertyType, + columnType, [start, end].filter(bound => bound !== RANGE_UNBOUNDED), ); if (!isStartUnbounded && !isEndUnbounded && inclusive === 'both') { @@ -2085,6 +2120,7 @@ export class CustomSchemaSQLSerializerV2 extends SQLSerializer { column: expression.columnExpression, columnJSON: expression?.columnExpressionJSON, propertyType: type ?? undefined, + columnType: expression.columnType, isArray, found: expression.found, mapKey: expression.mapKey, @@ -2135,27 +2171,28 @@ async function nodeTerm( return serializer.isNotNull(field, isNegatedField, context); } + const negatedContext = { ...context, isNegatedComparison: true }; if (!nodeTerm.quoted && term.substring(0, 2) === '>=') { if (isNegatedField) { - return serializer.lt(field, term.slice(2), context); + return serializer.lt(field, term.slice(2), negatedContext); } return serializer.gte(field, term.slice(2), context); } if (!nodeTerm.quoted && term.substring(0, 2) === '<=') { if (isNegatedField) { - return serializer.gt(field, term.slice(2), context); + return serializer.gt(field, term.slice(2), negatedContext); } return serializer.lte(field, term.slice(2), context); } if (!nodeTerm.quoted && term[0] === '>') { if (isNegatedField) { - return serializer.lte(field, term.slice(1), context); + return serializer.lte(field, term.slice(1), negatedContext); } return serializer.gt(field, term.slice(1), context); } if (!nodeTerm.quoted && term[0] === '<') { if (isNegatedField) { - return serializer.gte(field, term.slice(1), context); + return serializer.gte(field, term.slice(1), negatedContext); } return serializer.lt(field, term.slice(1), context); } From 8f730dedd1ac4aca739deb2446de639652dc45e3 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Tue, 17 Nov 2026 19:11:09 +0530 Subject: [PATCH 4/6] test(common-utils): cover materialized String and FixedString numeric comparisons --- .../numericStringComparisons.int.test.ts | 57 ++++++++++++++++--- 1 file changed, 49 insertions(+), 8 deletions(-) diff --git a/packages/common-utils/src/__tests__/numericStringComparisons.int.test.ts b/packages/common-utils/src/__tests__/numericStringComparisons.int.test.ts index 81f3747da..1a6478fe7 100644 --- a/packages/common-utils/src/__tests__/numericStringComparisons.int.test.ts +++ b/packages/common-utils/src/__tests__/numericStringComparisons.int.test.ts @@ -17,18 +17,50 @@ describe('numeric search bounds on String attributes', () => { const DATABASE = 'default'; const TABLE_NAME = 'numeric_string_comparisons_int_test'; + // Code mirrors gc_pause_ms in a FixedString, padded with zero bytes. const ROWS: Array<{ Id: number; Severity: string; LogAttributes: Record; + Code: string; }> = [ - { Id: 1, Severity: 'debug', LogAttributes: { gc_pause_ms: '5' } }, - { Id: 2, Severity: 'info', LogAttributes: { gc_pause_ms: '10' } }, - { Id: 3, Severity: 'error', LogAttributes: { gc_pause_ms: '17.25' } }, - { Id: 4, Severity: 'info', LogAttributes: { gc_pause_ms: '100' } }, - { Id: 5, Severity: 'info', LogAttributes: { gc_pause_ms: '150' } }, - { Id: 6, Severity: 'info', LogAttributes: { gc_pause_ms: 'abc' } }, - { Id: 7, Severity: 'info', LogAttributes: { other: 'x' } }, + { + Id: 1, + Severity: 'debug', + LogAttributes: { gc_pause_ms: '5' }, + Code: '5', + }, + { + Id: 2, + Severity: 'info', + LogAttributes: { gc_pause_ms: '10' }, + Code: '10', + }, + { + Id: 3, + Severity: 'error', + LogAttributes: { gc_pause_ms: '17.25' }, + Code: '17.25', + }, + { + Id: 4, + Severity: 'info', + LogAttributes: { gc_pause_ms: '100' }, + Code: '100', + }, + { + Id: 5, + Severity: 'info', + LogAttributes: { gc_pause_ms: '150' }, + Code: '150', + }, + { + Id: 6, + Severity: 'info', + LogAttributes: { gc_pause_ms: 'abc' }, + Code: 'abc', + }, + { Id: 7, Severity: 'info', LogAttributes: { other: 'x' }, Code: '' }, ]; const queryIds = async (where: string) => { @@ -63,7 +95,9 @@ describe('numeric search bounds on String attributes', () => { Timestamp DateTime64(9), Id UInt32, Severity Enum8('debug' = 1, 'info' = 5, 'error' = 9), - LogAttributes Map(LowCardinality(String), String) + LogAttributes Map(LowCardinality(String), String), + GcPause String MATERIALIZED LogAttributes['gc_pause_ms'], + Code FixedString(8) ) ENGINE = MergeTree ORDER BY (Timestamp)`, @@ -101,6 +135,13 @@ describe('numeric search bounds on String attributes', () => { ['NOT LogAttributes.gc_pause_ms:>100', [1, 2, 3, 4, 6, 7]], ['-LogAttributes.gc_pause_ms:<10', [2, 3, 4, 5, 6, 7]], ['-LogAttributes.gc_pause_ms:[10 TO 100]', [1, 5, 6, 7]], + // A materialized String column and a FixedString compare the same way. + ['GcPause:>100', [5]], + ['GcPause:[10 TO 100]', [2, 3, 4]], + ['-GcPause:>100', [1, 2, 3, 4, 6, 7]], + ['Code:>100', [5]], + ['Code:[10 TO 100]', [2, 3, 4]], + ['-Code:>100', [1, 2, 3, 4, 6, 7]], // Enum columns keep comparing by their numeric value. ['Severity:[1 TO 5]', [1, 2, 4, 5, 6, 7]], ])('%s returns the matching rows', async (where, ids) => { From 07cfe5840f1bffc30677973906998f04a8ecbd66 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Wed, 18 Nov 2026 03:38:17 +0530 Subject: [PATCH 5/6] fix(common-utils): limit numeric String comparisons to range searches `>`, `<`, `>=` and `<=` on String fields compare as text again. Fixed-width values such as dates and versions rely on that, and switching to a numeric comparison would make those searches match nothing without an error. Ranges with numeric bounds still compare with toFloat64OrNull, since they failed with "no supertype" before. --- .changeset/numeric-string-comparisons.md | 12 ++-- .../numericStringComparisons.int.test.ts | 23 ++----- .../src/__tests__/queryParser.test.ts | 61 +++-------------- packages/common-utils/src/queryParser.ts | 66 ++++--------------- 4 files changed, 34 insertions(+), 128 deletions(-) diff --git a/.changeset/numeric-string-comparisons.md b/.changeset/numeric-string-comparisons.md index 2f69f86bb..f03e8d1ee 100644 --- a/.changeset/numeric-string-comparisons.md +++ b/.changeset/numeric-string-comparisons.md @@ -4,10 +4,10 @@ '@hyperdx/app': patch --- -fix: compare numeric search bounds as numbers on String attributes +fix: support numeric range searches on String attributes -Searches like `LogAttributes.duration_ms:>1000` compared the attribute's text -value to `'1000'` as strings, so `'17.25' > '100'` matched and real matches -were missed, and ranges such as `:[10 TO 20]` failed with "no supertype for -types String, UInt8". A numeric bound on a String-typed field now compares -with `toFloat64OrNull`; values that aren't numbers don't match. +Range searches with numeric bounds on a String-typed field, such as +`LogAttributes.duration_ms:[10 TO 20]`, failed with "no supertype for types +String, UInt8". They now compare with `toFloat64OrNull`; values that aren't +numbers don't match. `>`, `<`, `>=` and `<=` still compare String fields as +text; use a range such as `:[1000 TO *]` for a numeric comparison. diff --git a/packages/common-utils/src/__tests__/numericStringComparisons.int.test.ts b/packages/common-utils/src/__tests__/numericStringComparisons.int.test.ts index 1a6478fe7..0fddc273b 100644 --- a/packages/common-utils/src/__tests__/numericStringComparisons.int.test.ts +++ b/packages/common-utils/src/__tests__/numericStringComparisons.int.test.ts @@ -6,10 +6,10 @@ import { Metadata, MetadataCache } from '@/core/metadata'; import { renderChartConfig } from '@/core/renderChartConfig'; import { ChartConfigWithOptDateRange } from '@/types'; -// Unit tests only compare SQL text, which is how string comparisons of -// numeric attributes shipped. These run each search against ClickHouse and -// check which rows come back. -describe('numeric search bounds on String attributes', () => { +// Unit tests only compare SQL text, which is how ranges with numeric bounds on +// String attributes shipped while ClickHouse rejected them. These run each +// search against ClickHouse and check which rows come back. +describe('numeric range bounds on String attributes', () => { let client: ClickHouseClient; let hdxClient: HdxClickhouseClient; let metadata: Metadata; @@ -123,25 +123,16 @@ describe('numeric search bounds on String attributes', () => { }); it.each([ - ['LogAttributes.gc_pause_ms:>100', [5]], - ['LogAttributes.gc_pause_ms:>=100', [4, 5]], - ['LogAttributes.gc_pause_ms:<10', [1]], - ['LogAttributes.gc_pause_ms:<=10', [1, 2]], ['LogAttributes.gc_pause_ms:[10 TO 100]', [2, 3, 4]], ['LogAttributes.gc_pause_ms:{10 TO 100}', [3]], ['LogAttributes.gc_pause_ms:[100 TO *]', [4, 5]], - // Missing (7) and non-numeric (6) values match every negated form. - ['-LogAttributes.gc_pause_ms:>100', [1, 2, 3, 4, 6, 7]], - ['NOT LogAttributes.gc_pause_ms:>100', [1, 2, 3, 4, 6, 7]], - ['-LogAttributes.gc_pause_ms:<10', [2, 3, 4, 5, 6, 7]], + // Missing (7) and non-numeric (6) values match a negated range. ['-LogAttributes.gc_pause_ms:[10 TO 100]', [1, 5, 6, 7]], + // Comparisons stay lexical: '5', '17.25' and 'abc' sort after '100'. + ['LogAttributes.gc_pause_ms:>100', [1, 3, 5, 6]], // A materialized String column and a FixedString compare the same way. - ['GcPause:>100', [5]], ['GcPause:[10 TO 100]', [2, 3, 4]], - ['-GcPause:>100', [1, 2, 3, 4, 6, 7]], - ['Code:>100', [5]], ['Code:[10 TO 100]', [2, 3, 4]], - ['-Code:>100', [1, 2, 3, 4, 6, 7]], // Enum columns keep comparing by their numeric value. ['Severity:[1 TO 5]', [1, 2, 4, 5, 6, 7]], ])('%s returns the matching rows', async (where, ids) => { diff --git a/packages/common-utils/src/__tests__/queryParser.test.ts b/packages/common-utils/src/__tests__/queryParser.test.ts index 3a66234e7..df3d48219 100644 --- a/packages/common-utils/src/__tests__/queryParser.test.ts +++ b/packages/common-utils/src/__tests__/queryParser.test.ts @@ -768,7 +768,7 @@ describe('CustomSchemaSQLSerializerV2 - json', () => { }); }); -describe('CustomSchemaSQLSerializerV2 - numeric bounds on String values', () => { +describe('CustomSchemaSQLSerializerV2 - numeric range bounds on String values', () => { const metadata = getMetadata( new ClickhouseClient({ host: 'http://localhost:8123' }), ); @@ -785,8 +785,6 @@ describe('CustomSchemaSQLSerializerV2 - numeric bounds on String values', () => return { name: 'ServiceName', type: 'LowCardinality(String)' }; } else if (column === 'Severity') { return { name: 'Severity', type: "Enum8('debug' = 1, 'info' = 5)" }; - } else if (column === 'SrcIp') { - return { name: 'SrcIp', type: 'IPv4' }; } return undefined; }); @@ -806,43 +804,14 @@ describe('CustomSchemaSQLSerializerV2 - numeric bounds on String values', () => const hint = "indexHint(mapContains(`LogAttributes`, 'gc_pause_ms'))"; it.each([ - { - lucene: 'LogAttributes.gc_pause_ms:>100', - sql: `((ifNull(${attr} > 100, 0) AND ${hint}))`, - }, - { - lucene: 'LogAttributes.gc_pause_ms:>=100', - sql: `((ifNull(${attr} >= 100, 0) AND ${hint}))`, - }, - { - lucene: 'LogAttributes.gc_pause_ms:<100', - sql: `((ifNull(${attr} < 100, 0) AND ${hint}))`, - }, - { - lucene: 'LogAttributes.gc_pause_ms:<=100', - sql: `((ifNull(${attr} <= 100, 0) AND ${hint}))`, - }, - { - lucene: 'LogAttributes.gc_pause_ms:>-2.5', - sql: `((ifNull(${attr} > -2.5, 0) AND ${hint}))`, - }, - { - lucene: 'LogAttributes.gc_pause_ms:>1e3', - sql: `((ifNull(${attr} > 1000, 0) AND ${hint}))`, - }, - { - // Missing and non-numeric values match, as for NOT and -field:[a TO b]. - lucene: '-LogAttributes.gc_pause_ms:>100', - sql: `((ifNull(${attr} <= 100, 1)))`, - }, - { - lucene: '-LogAttributes.gc_pause_ms:<10', - sql: `((ifNull(${attr} >= 10, 1)))`, - }, { lucene: 'LogAttributes.gc_pause_ms:[10 TO 20]', sql: `((ifNull(${attr} BETWEEN 10 AND 20, 0) AND ${hint}))`, }, + { + lucene: 'LogAttributes.gc_pause_ms:[-2.5 TO 1e3]', + sql: `((ifNull(${attr} BETWEEN -2.5 AND 1000, 0) AND ${hint}))`, + }, { lucene: 'LogAttributes.gc_pause_ms:{10 TO 20}', sql: `((ifNull(${attr} > 10, 0) AND ifNull(${attr} < 20, 0) AND ${hint}))`, @@ -866,19 +835,13 @@ describe('CustomSchemaSQLSerializerV2 - numeric bounds on String values', () => it.each([ { - // A non-numeric bound keeps the string comparison. - lucene: 'LogAttributes.version:>abc', - sql: "((`LogAttributes`['version'] > 'abc' AND indexHint(mapContains(`LogAttributes`, 'version'))))", - }, - { - // Hex is not a numeric literal here. - lucene: 'LogAttributes.version:>0x10', - sql: "((`LogAttributes`['version'] > '0x10' AND indexHint(mapContains(`LogAttributes`, 'version'))))", + // Comparisons stay lexical: fixed-width values such as dates rely on it. + lucene: 'LogAttributes.gc_pause_ms:>100', + sql: `((\`LogAttributes\`['gc_pause_ms'] > '100' AND ${hint}))`, }, { - // Numeric columns are compared by ClickHouse already. - lucene: 'Duration:>100', - sql: "((Duration > '100'))", + lucene: 'LogAttributes.version:[a TO m]', + sql: "((`LogAttributes`['version'] BETWEEN 'a' AND 'm' AND indexHint(mapContains(`LogAttributes`, 'version'))))", }, { lucene: 'Duration:[10 TO 20]', @@ -889,10 +852,6 @@ describe('CustomSchemaSQLSerializerV2 - numeric bounds on String values', () => lucene: 'Severity:[1 TO 5]', sql: '((Severity BETWEEN 1 AND 5))', }, - { - lucene: 'SrcIp:>100', - sql: "((SrcIp > '100'))", - }, ])('leaves "$lucene" unchanged', async ({ lucene, sql }) => { const builder = new SearchQueryBuilder(lucene, serializer); expect(await builder.build()).toBe(sql); diff --git a/packages/common-utils/src/queryParser.ts b/packages/common-utils/src/queryParser.ts index 99b22f646..44b1a89d1 100644 --- a/packages/common-utils/src/queryParser.ts +++ b/packages/common-utils/src/queryParser.ts @@ -137,11 +137,10 @@ const isNumericLiteral = (term: string) => /^[-+.\deE]+$/.test(term) && parseToNumber(term) !== undefined; /** - * Map attributes (and other String columns) hold numbers as text. Compared as - * strings, `'17.25' > '100'` is true, and BETWEEN with numeric bounds fails - * with "no supertype for types String, UInt8", so numeric bounds on a String - * field need a numeric comparison. Enum, UUID and IP columns also map to - * JSDataType.String but toFloat64OrNull rejects them, so check the type. + * Map attributes (and other String columns) hold numbers as text, and a range + * with numeric bounds on them fails with "no supertype for types String, + * UInt8". Enum, UUID and IP columns also map to JSDataType.String but + * toFloat64OrNull rejects them, so check the type. */ function isStringComparedToNumbers( columnType: string | undefined, @@ -156,44 +155,20 @@ function isStringComparedToNumbers( } /** - * Values that aren't numbers convert to NULL. ifNull makes them a plain match - * or non-match rather than NULL, so a NOT around the search still works. + * Values that aren't numbers convert to NULL. ifNull makes them a non-match + * rather than NULL, so a NOT around the range still includes them. */ function numericComparison( column: string | undefined, comparison: string, values: number[], - matchNonNumeric = false, ) { return SqlString.format( - `ifNull(toFloat64OrNull(${column}) ${comparison}, ${matchNonNumeric ? 1 : 0})`, + `ifNull(toFloat64OrNull(${column}) ${comparison}, 0)`, values, ); } -/** - * `-field:>x` reaches gt/gte/lt/lte as the opposite operator (`field:<=x`), - * so missing and non-numeric values must match it, as they do for - * `-field:[a TO b]`. The map-key index hint would skip rows without the key, - * so negated comparisons leave it off, as range() does. - */ -function numericBound( - column: string | undefined, - operator: string, - term: string, - context: SerializerContext, - expressionPostfix: string, -) { - const negated = context.isNegatedComparison === true; - const comparison = numericComparison( - column, - `${operator} ?`, - [Number(term)], - negated, - ); - return `(${comparison}${negated ? '' : expressionPostfix})`; -} - // Type guards for lucene AST types function isNodeTerm(node: lucene.Node | lucene.AST): node is lucene.NodeTerm { return 'term' in node && node.term != null; @@ -289,8 +264,6 @@ interface SerializerContext { */ bodyExpression?: string; isNegatedAndParenthesized?: boolean; - /** Set when the visitor turned `-field:>x` into the opposite operator. */ - isNegatedComparison?: boolean; } interface Serializer { @@ -728,7 +701,6 @@ export abstract class SQLSerializer implements Serializer { columnJSON, found, propertyType, - columnType, isArray, mapKeyIndexExpression, } = await this.getColumnForField(field, context); @@ -747,9 +719,6 @@ export abstract class SQLSerializer implements Serializer { [term], ); } - if (isStringComparedToNumbers(columnType, [term])) { - return numericBound(column, '>=', term, context, expressionPostfix); - } return SqlString.format(`(${column} >= ?${expressionPostfix})`, [term]); } @@ -759,7 +728,6 @@ export abstract class SQLSerializer implements Serializer { columnJSON, found, propertyType, - columnType, isArray, mapKeyIndexExpression, } = await this.getColumnForField(field, context); @@ -778,9 +746,6 @@ export abstract class SQLSerializer implements Serializer { [term], ); } - if (isStringComparedToNumbers(columnType, [term])) { - return numericBound(column, '<=', term, context, expressionPostfix); - } return SqlString.format(`(${column} <= ?${expressionPostfix})`, [term]); } @@ -790,7 +755,6 @@ export abstract class SQLSerializer implements Serializer { columnJSON, found, propertyType, - columnType, isArray, mapKeyIndexExpression, } = await this.getColumnForField(field, context); @@ -809,9 +773,6 @@ export abstract class SQLSerializer implements Serializer { [term], ); } - if (isStringComparedToNumbers(columnType, [term])) { - return numericBound(column, '<', term, context, expressionPostfix); - } return SqlString.format(`(${column} < ?${expressionPostfix})`, [term]); } @@ -821,7 +782,6 @@ export abstract class SQLSerializer implements Serializer { columnJSON, found, propertyType, - columnType, isArray, mapKeyIndexExpression, } = await this.getColumnForField(field, context); @@ -840,9 +800,6 @@ export abstract class SQLSerializer implements Serializer { [term], ); } - if (isStringComparedToNumbers(columnType, [term])) { - return numericBound(column, '>', term, context, expressionPostfix); - } return SqlString.format(`(${column} > ?${expressionPostfix})`, [term]); } @@ -2171,28 +2128,27 @@ async function nodeTerm( return serializer.isNotNull(field, isNegatedField, context); } - const negatedContext = { ...context, isNegatedComparison: true }; if (!nodeTerm.quoted && term.substring(0, 2) === '>=') { if (isNegatedField) { - return serializer.lt(field, term.slice(2), negatedContext); + return serializer.lt(field, term.slice(2), context); } return serializer.gte(field, term.slice(2), context); } if (!nodeTerm.quoted && term.substring(0, 2) === '<=') { if (isNegatedField) { - return serializer.gt(field, term.slice(2), negatedContext); + return serializer.gt(field, term.slice(2), context); } return serializer.lte(field, term.slice(2), context); } if (!nodeTerm.quoted && term[0] === '>') { if (isNegatedField) { - return serializer.lte(field, term.slice(1), negatedContext); + return serializer.lte(field, term.slice(1), context); } return serializer.gt(field, term.slice(1), context); } if (!nodeTerm.quoted && term[0] === '<') { if (isNegatedField) { - return serializer.gte(field, term.slice(1), negatedContext); + return serializer.gte(field, term.slice(1), context); } return serializer.lt(field, term.slice(1), context); } From f0cdc45addda5dae0a21a582d2b2924986ac236d Mon Sep 17 00:00:00 2001 From: aryainguz Date: Wed, 18 Nov 2026 03:44:04 +0530 Subject: [PATCH 6/6] refactor(common-utils): reuse isCHFixedStringType for numeric range check --- packages/common-utils/src/queryParser.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/common-utils/src/queryParser.ts b/packages/common-utils/src/queryParser.ts index 44b1a89d1..eff632811 100644 --- a/packages/common-utils/src/queryParser.ts +++ b/packages/common-utils/src/queryParser.ts @@ -132,7 +132,7 @@ function normalizeChExpression(expr: string): string { export const IMPLICIT_FIELD = ''; const RANGE_UNBOUNDED = '*'; -// parseToNumber also accepts hex such as 0x10; keep those as strings. +// Decimal numbers only; parseToNumber also accepts hex such as 0x10. const isNumericLiteral = (term: string) => /^[-+.\deE]+$/.test(term) && parseToNumber(term) !== undefined; @@ -146,9 +146,12 @@ function isStringComparedToNumbers( columnType: string | undefined, terms: string[], ) { - const baseType = columnType ? stripTypeWrappers(columnType) : ''; + if (!columnType) { + return false; + } return ( - (baseType === 'String' || baseType.startsWith('FixedString(')) && + (stripTypeWrappers(columnType) === 'String' || + isCHFixedStringType(columnType)) && terms.length > 0 && terms.every(isNumericLiteral) );