Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions .changeset/numeric-string-comparisons.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
---
'@hyperdx/common-utils': patch
'@hyperdx/api': patch
'@hyperdx/app': patch
---

fix: support numeric range searches on String attributes

Range searches with numeric bounds on a String-typed field, such as
`LogAttributes.duration_ms:[10 TO 20]`, failed with "no supertype for types
String, UInt8". They now compare with `toFloat64OrNull`; values that aren't
numbers don't match. `>`, `<`, `>=` and `<=` still compare String fields as
text; use a range such as `:[1000 TO *]` for a numeric comparison.
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
import { ClickHouseClient, createClient } from '@clickhouse/client';

import { parameterizedQueryToSql } from '@/clickhouse';
import { ClickhouseClient as HdxClickhouseClient } from '@/clickhouse/node';
import { Metadata, MetadataCache } from '@/core/metadata';
import { renderChartConfig } from '@/core/renderChartConfig';
import { ChartConfigWithOptDateRange } from '@/types';

// Unit tests only compare SQL text, which is how ranges with numeric bounds on
// String attributes shipped while ClickHouse rejected them. These run each
// search against ClickHouse and check which rows come back.
describe('numeric range bounds on String attributes', () => {
let client: ClickHouseClient;
let hdxClient: HdxClickhouseClient;
let metadata: Metadata;

const DATABASE = 'default';
const TABLE_NAME = 'numeric_string_comparisons_int_test';

// Code mirrors gc_pause_ms in a FixedString, padded with zero bytes.
const ROWS: Array<{
Id: number;
Severity: string;
LogAttributes: Record<string, string>;
Code: string;
}> = [
{
Id: 1,
Severity: 'debug',
LogAttributes: { gc_pause_ms: '5' },
Code: '5',
},
{
Id: 2,
Severity: 'info',
LogAttributes: { gc_pause_ms: '10' },
Code: '10',
},
{
Id: 3,
Severity: 'error',
LogAttributes: { gc_pause_ms: '17.25' },
Code: '17.25',
},
{
Id: 4,
Severity: 'info',
LogAttributes: { gc_pause_ms: '100' },
Code: '100',
},
{
Id: 5,
Severity: 'info',
LogAttributes: { gc_pause_ms: '150' },
Code: '150',
},
{
Id: 6,
Severity: 'info',
LogAttributes: { gc_pause_ms: 'abc' },
Code: 'abc',
},
{ Id: 7, Severity: 'info', LogAttributes: { other: 'x' }, Code: '' },
];

const queryIds = async (where: string) => {
const config: ChartConfigWithOptDateRange = {
connection: 'test-connection',
from: { databaseName: DATABASE, tableName: TABLE_NAME },
select: 'Id',
where,
whereLanguage: 'lucene',
timestampValueExpression: 'Timestamp',
dateRange: [new Date('2025-01-01'), new Date('2025-01-02')],
orderBy: 'Id',
};
const sql = parameterizedQueryToSql(
await renderChartConfig(config, metadata, undefined),
);
const result = await client.query({ query: sql, format: 'JSONEachRow' });
const rows = await result.json<{ Id: number }>();
return rows.map(r => Number(r.Id));
};

beforeAll(async () => {
const host = process.env.CLICKHOUSE_HOST || 'http://localhost:8123';
const username = process.env.CLICKHOUSE_USER || 'default';
const password = process.env.CLICKHOUSE_PASSWORD || '';

client = createClient({ url: host, username, password });
hdxClient = new HdxClickhouseClient({ host, username, password });

await client.command({
query: `CREATE OR REPLACE TABLE ${DATABASE}.${TABLE_NAME} (
Timestamp DateTime64(9),
Id UInt32,
Severity Enum8('debug' = 1, 'info' = 5, 'error' = 9),
LogAttributes Map(LowCardinality(String), String),
GcPause String MATERIALIZED LogAttributes['gc_pause_ms'],
Code FixedString(8)
)
ENGINE = MergeTree
ORDER BY (Timestamp)`,
});

await client.insert({
table: `${DATABASE}.${TABLE_NAME}`,
values: ROWS.map(r => ({ ...r, Timestamp: '2025-01-01 12:00:00' })),
format: 'JSONEachRow',
});
});

beforeEach(() => {
metadata = new Metadata(hdxClient, new MetadataCache());
});

afterAll(async () => {
await client.command({
query: `DROP TABLE IF EXISTS ${DATABASE}.${TABLE_NAME}`,
});
await hdxClient.close();
await client.close();
});

it.each([
Comment thread
Aryainguz marked this conversation as resolved.
Resolved
['LogAttributes.gc_pause_ms:[10 TO 100]', [2, 3, 4]],
['LogAttributes.gc_pause_ms:{10 TO 100}', [3]],
['LogAttributes.gc_pause_ms:[100 TO *]', [4, 5]],
// Missing (7) and non-numeric (6) values match a negated range.
['-LogAttributes.gc_pause_ms:[10 TO 100]', [1, 5, 6, 7]],
// Comparisons stay lexical: '5', '17.25' and 'abc' sort after '100'.
['LogAttributes.gc_pause_ms:>100', [1, 3, 5, 6]],
// A materialized String column and a FixedString compare the same way.
['GcPause:[10 TO 100]', [2, 3, 4]],
['Code:[10 TO 100]', [2, 3, 4]],
// Enum columns keep comparing by their numeric value.
['Severity:[1 TO 5]', [1, 2, 4, 5, 6, 7]],
])('%s returns the matching rows', async (where, ids) => {
expect(await queryIds(where)).toEqual(ids);
});
});
97 changes: 96 additions & 1 deletion packages/common-utils/src/__tests__/queryParser.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ describe('CustomSchemaSQLSerializerV2 - json', () => {
},
found: true,
propertyType: 'json',
columnType: 'JSON',
isArray: false,
});
const field2 = 'ResourceAttributesJSON.test.nest';
Expand All @@ -104,6 +105,7 @@ describe('CustomSchemaSQLSerializerV2 - json', () => {
},
found: true,
propertyType: 'json',
columnType: 'JSON',
isArray: false,
});
});
Expand Down Expand Up @@ -362,7 +364,7 @@ describe('CustomSchemaSQLSerializerV2 - json', () => {
},
{
lucene: 'foo:[1 TO 5]',
sql: '((foo BETWEEN 1 AND 5))',
sql: '((ifNull(toFloat64OrNull(foo) BETWEEN 1 AND 5, 0)))',
english: 'foo is between 1 and 5',
},
{
Expand Down Expand Up @@ -766,6 +768,96 @@ describe('CustomSchemaSQLSerializerV2 - json', () => {
});
});

describe('CustomSchemaSQLSerializerV2 - numeric range bounds on String values', () => {
const metadata = getMetadata(
new ClickhouseClient({ host: 'http://localhost:8123' }),
);
// The OTel schema's attribute maps hold every value as a String.
metadata.getColumn = jest.fn().mockImplementation(async ({ column }) => {
if (column === 'LogAttributes') {
return {
name: 'LogAttributes',
type: 'Map(LowCardinality(String), String)',
};
} else if (column === 'Duration') {
return { name: 'Duration', type: 'UInt64' };
} else if (column === 'ServiceName') {
return { name: 'ServiceName', type: 'LowCardinality(String)' };
} else if (column === 'Severity') {
return { name: 'Severity', type: "Enum8('debug' = 1, 'info' = 5)" };
}
return undefined;
});
metadata.getMaterializedColumnsLookupTable = jest
.fn()
.mockImplementation(async () => new Map());

const serializer = new CustomSchemaSQLSerializerV2({
metadata,
databaseName: 'testName',
tableName: 'testTable',
connectionId: 'testId',
implicitColumnExpression: 'Body',
});

const attr = "toFloat64OrNull(`LogAttributes`['gc_pause_ms'])";
const hint = "indexHint(mapContains(`LogAttributes`, 'gc_pause_ms'))";

it.each([
{
lucene: 'LogAttributes.gc_pause_ms:[10 TO 20]',
sql: `((ifNull(${attr} BETWEEN 10 AND 20, 0) AND ${hint}))`,
},
{
lucene: 'LogAttributes.gc_pause_ms:[-2.5 TO 1e3]',
sql: `((ifNull(${attr} BETWEEN -2.5 AND 1000, 0) AND ${hint}))`,
},
{
lucene: 'LogAttributes.gc_pause_ms:{10 TO 20}',
sql: `((ifNull(${attr} > 10, 0) AND ifNull(${attr} < 20, 0) AND ${hint}))`,
},
{
lucene: 'LogAttributes.gc_pause_ms:[10 TO *]',
sql: `((ifNull(${attr} >= 10, 0) AND ${hint}))`,
},
{
lucene: '-LogAttributes.gc_pause_ms:[10 TO 20]',
sql: `((NOT ifNull(${attr} BETWEEN 10 AND 20, 0)))`,
},
{
lucene: 'ServiceName:[1 TO 5]',
sql: '((ifNull(toFloat64OrNull(ServiceName) BETWEEN 1 AND 5, 0)))',
},
])('compares "$lucene" as numbers', async ({ lucene, sql }) => {
const builder = new SearchQueryBuilder(lucene, serializer);
expect(await builder.build()).toBe(sql);
});

it.each([
{
// Comparisons stay lexical: fixed-width values such as dates rely on it.
lucene: 'LogAttributes.gc_pause_ms:>100',
sql: `((\`LogAttributes\`['gc_pause_ms'] > '100' AND ${hint}))`,
},
{
lucene: 'LogAttributes.version:[a TO m]',
sql: "((`LogAttributes`['version'] BETWEEN 'a' AND 'm' AND indexHint(mapContains(`LogAttributes`, 'version'))))",
},
{
lucene: 'Duration:[10 TO 20]',
sql: '((Duration BETWEEN 10 AND 20))',
},
{
// Enum compares to numbers already; toFloat64OrNull would reject it.
lucene: 'Severity:[1 TO 5]',
sql: '((Severity BETWEEN 1 AND 5))',
},
])('leaves "$lucene" unchanged', async ({ lucene, sql }) => {
const builder = new SearchQueryBuilder(lucene, serializer);
expect(await builder.build()).toBe(sql);
});
});

describe('CustomSchemaSQLSerializerV2 - range bounds', () => {
const metadata = getMetadata(
new ClickhouseClient({ host: 'http://localhost:8123' }),
Expand Down Expand Up @@ -2433,6 +2525,7 @@ describe('CustomSchemaSQLSerializerV2 - Array and Nested Fields', () => {
column: 'Events.Name',
found: true,
propertyType: JSDataType.String,
columnType: 'Array(String)',
isArray: true,
});

Expand All @@ -2442,6 +2535,7 @@ describe('CustomSchemaSQLSerializerV2 - Array and Nested Fields', () => {
column: 'Events.Count',
found: true,
propertyType: JSDataType.Number,
columnType: 'Array(UInt64)',
isArray: true,
});

Expand All @@ -2451,6 +2545,7 @@ describe('CustomSchemaSQLSerializerV2 - Array and Nested Fields', () => {
column: 'Events.IsAvailable',
found: true,
propertyType: JSDataType.Bool,
columnType: 'Array(Bool)',
isArray: true,
});
});
Expand Down
Loading
Loading