From ee7d3658b21711bd992a5b9a95df8a7749188789 Mon Sep 17 00:00:00 2001 From: Alex Fedotyev <61838744+alex-fedotyev@users.noreply.github.com> Date: Mon, 16 Nov 2026 18:26:41 +0000 Subject: [PATCH] feat(common-utils): add a compact form of the pattern predicate buildPatternPredicate repeats the separator class between every token. With all ASCII punctuation as delimiters that is about 30 characters per token. The app keeps pattern filters in the URL, and with this form two exclusions of long patterns already pass Node's default 16 KB header limit. buildCompactPatternPredicate builds the same match in a different shape: replaceRegexpAll() turns every separator run in the value into one space, and the regex matches tokens joined by single spaces, so the separator class appears once. On 8 templates mined from the demo logs, 8 exclusions take 11K characters in the URL instead of 104K, and each template matches the same number of events in ClickHouse with either form. The cost is speed: those 8 exclusions over 15 minutes of logs took 0.95 s instead of 0.45 s. It returns null for a template whose token holds one of the extra delimiters, where the two forms would disagree. parsePatternPredicate reads both forms. Part of FR-385. Co-Authored-By: Claude Opus 5.5 --- .changeset/compact-pattern-predicate.md | 9 + .../src/__tests__/pattern-predicate.test.ts | 233 +++++++++++++++--- packages/common-utils/src/drain/index.ts | 1 + .../src/drain/pattern-predicate.ts | 101 +++++++- 4 files changed, 301 insertions(+), 43 deletions(-) create mode 100644 .changeset/compact-pattern-predicate.md diff --git a/.changeset/compact-pattern-predicate.md b/.changeset/compact-pattern-predicate.md new file mode 100644 index 000000000..f5ded73f6 --- /dev/null +++ b/.changeset/compact-pattern-predicate.md @@ -0,0 +1,9 @@ +--- +'@hyperdx/common-utils': minor +--- + +feat: add `buildCompactPatternPredicate` + +It builds the same match as `buildPatternPredicate` in much shorter SQL, for +pattern filters that are kept in a URL. `parsePatternPredicate` reads both +forms. diff --git a/packages/common-utils/src/__tests__/pattern-predicate.test.ts b/packages/common-utils/src/__tests__/pattern-predicate.test.ts index 5a0f0eeb9..8791957ab 100644 --- a/packages/common-utils/src/__tests__/pattern-predicate.test.ts +++ b/packages/common-utils/src/__tests__/pattern-predicate.test.ts @@ -1,5 +1,6 @@ import { TemplateMinerConfig } from '@/drain/config'; import { + buildCompactPatternPredicate, buildPatternPredicate, MAX_PREDICATE_TEMPLATE_LENGTH, parsePatternPredicate, @@ -15,15 +16,57 @@ const withPunctuation = { extraDelimiters: PUNCTUATION }; // RE2 spells code points \x{HHHH} and categories \pZ; JS (with the u flag) // spells them \u{HHHH} and \p{Z}, and has no POSIX classes. RE2's \s is // [\t\n\f\r ]; JS's also covers Unicode spaces and BOM. +function toJsSource(re2: string): string { + return re2 + .replace(/(? + literal.replace(/''/g, "'").replace(/\\\\/g, '\\'); + +// Evaluates a compact predicate the way ClickHouse would: replaceRegexpAll, +// then match. +function matchesCompact(predicate: string, value: string): boolean { + const call = + /^match\(replaceRegexpAll\(.+, '((?:[^'\\]|\\\\|'')*)', ' '\), '((?:[^'\\]|\\\\|'')*)'\)$/s.exec( + predicate, + ); + if (call == null) throw new Error(`not a compact predicate: ${predicate}`); + const separatorRun = RegExp(toJsSource(unescapeSql(call[1])), 'gu'); + return toJsRegex(unescapeSql(call[2])).test(value.replace(separatorRun, ' ')); +} + +const MESSAGES = [ + 'Connection to db-1:5432 timed out after 30s (retry 1/3)', + 'Connection to db-2:5432 timed out after 30s (retry 2/3)', + 'user=alice action=login status=ok', + 'user=bob action=login status=ok', + '[2026-04-17T08:35:57.725Z] "GET /api/products/2ZYF HTTP/1.1" 200 -', + '[2026-04-17T08:36:02.101Z] "GET /api/products/L9EC HTTP/1.1" 200 -', + 'path=C:\\temp\\a.txt mode=[r^w] note={a|b} cost=$5 ~ok', + 'path=C:\\temp\\b.txt mode=[r^w] note={a|b} cost=$7 ~ok', +]; + +function mineClusters(): { template: string; members: string[] }[] { + const config = new TemplateMinerConfig(); + config.drainExtraDelimiters = PUNCTUATION; + const miner = new TemplateMiner(config); + const members = new Map(); + for (const message of MESSAGES) { + const { clusterId } = miner.addLogMessage(message); + members.set(clusterId, [...(members.get(clusterId) ?? []), message]); + } + return [...members].map(([clusterId, clusterMembers]) => ({ + template: miner.drain.getCluster(clusterId)!.getTemplate(), + members: clusterMembers, + })); } describe('templateToRegex', () => { @@ -95,31 +138,125 @@ describe('templateToRegex', () => { }); it('matches every member of every cluster mined with the same delimiters', () => { - const config = new TemplateMinerConfig(); - config.drainExtraDelimiters = PUNCTUATION; - const miner = new TemplateMiner(config); - const messages = [ - 'Connection to db-1:5432 timed out after 30s (retry 1/3)', - 'Connection to db-2:5432 timed out after 30s (retry 2/3)', - 'user=alice action=login status=ok', - 'user=bob action=login status=ok', - '[2026-04-17T08:35:57.725Z] "GET /api/products/2ZYF HTTP/1.1" 200 -', - '[2026-04-17T08:36:02.101Z] "GET /api/products/L9EC HTTP/1.1" 200 -', - 'path=C:\\temp\\a.txt mode=[r^w] note={a|b} cost=$5 ~ok', - 'path=C:\\temp\\b.txt mode=[r^w] note={a|b} cost=$7 ~ok', + for (const { template, members } of mineClusters()) { + const regex = toJsRegex(templateToRegex(template, withPunctuation)); + for (const message of MESSAGES) { + expect(regex.test(message)).toBe(members.includes(message)); + } + } + }); +}); + +const PARTIAL: PatternRegexOptions = { + extraDelimiters: ['/', ':', '-', ']', '\\', '^'], +}; + +// The option sets the equivalence and round-trip tests run under. +const OPTION_SETS: (PatternRegexOptions | undefined)[] = [ + withPunctuation, + undefined, + PARTIAL, + { extraDelimiters: PUNCTUATION, whitespace: 'python' }, +]; + +describe('buildCompactPatternPredicate', () => { + it('matches exactly what the full predicate matches', () => { + const values = [ + ...MESSAGES, + ' GET /api/v1/users/9 404 3ms.', + '', + ' \t ', + 'GET api v1 users 9 404', + // Separators outside ASCII, and ones only one whitespace set has. + 'GET\u00a0api\u2003v1\x0busers\u3000 9 404 3ms', + 'GET api\ufeffv1 users 9 404 3ms', + 'GET api v1 users 9\x1e404\x853ms', ]; - const members = new Map(); - for (const message of messages) { - const { clusterId } = miner.addLogMessage(message); - members.set(clusterId, [...(members.get(clusterId) ?? []), message]); + const templates = [ + ...mineClusters().map(c => c.template), + 'GET api v1 users <*> <*> <*>', + '<*>', + '', + ]; + for (const options of OPTION_SETS) { + for (const template of templates) { + const full = toJsRegex(templateToRegex(template, options)); + const compact = buildCompactPatternPredicate( + 'Body', + template, + options, + )!; + for (const value of values) { + expect([template, value, matchesCompact(compact, value)]).toEqual([ + template, + value, + full.test(value), + ]); + } + } } - for (const [clusterId, clusterMembers] of members) { - const template = miner.drain.getCluster(clusterId)!.getTemplate(); - const regex = toJsRegex(templateToRegex(template, withPunctuation)); - for (const message of messages) { - expect(regex.test(message)).toBe(clusterMembers.includes(message)); + }); + + it('is much shorter than the full predicate for long templates', () => { + const template = Array.from({ length: 40 }, (_, i) => + i % 3 === 0 ? '<*>' : `word${i}`, + ).join(' '); + const full = buildPatternPredicate('Body', template, withPunctuation)!; + const compact = buildCompactPatternPredicate( + 'Body', + template, + withPunctuation, + )!; + expect(compact.length * 4).toBeLessThan(full.length); + }); + + it('escapes regex and SQL syntax in tokens', () => { + // Mined without punctuation delimiters, tokens keep their punctuation. + const cases: [PatternRegexOptions | undefined, string][] = [ + [undefined, 'GET /api/v1.2 (ok) $5 a|b'], + [undefined, "user=<*> it's C:\\tmp 3ms."], + [PARTIAL, 'GET api v1.2 (ok) $5 a|b'], + ]; + const values = [ + 'GET /api/v1.2 (ok) $5 a|b', + 'GET /api/v1x2 (ok) $5 a|b', + 'GET api v1.2 (ok) $5 a|b', + 'GET/api/v1.2 (ok) $5 a|b', + "user=<*> it's C:\\tmp 3ms.", + "user=<*> it's C:\\tmp 3msX", + ]; + for (const [options, template] of cases) { + const full = toJsRegex(templateToRegex(template, options)); + const compact = buildCompactPatternPredicate('Body', template, options)!; + for (const value of values) { + expect([template, value, matchesCompact(compact, value)]).toEqual([ + template, + value, + full.test(value), + ]); } + expect(parsePatternPredicate(compact, options)?.template).toBe(template); } + // `.` is a literal, so only the template's own text matches. + const compact = buildCompactPatternPredicate( + 'Body', + 'GET /api/v1.2 (ok) $5 a|b', + )!; + expect(matchesCompact(compact, 'GET /api/v1.2 (ok) $5 a|b')).toBe(true); + expect(matchesCompact(compact, 'GET /api/v1x2 (ok) $5 a|b')).toBe(false); + }); + + it('builds no predicate for a token that holds a delimiter', () => { + // Mined without delimiters, then built with them: normalizing would split + // `db-1:5432`, so the compact form can't match what the full one does. + const template = 'Connection to db-1:5432 timed out'; + expect( + buildCompactPatternPredicate('Body', template, withPunctuation), + ).toBeNull(); + expect(buildPatternPredicate('Body', template, withPunctuation)).not.toBe( + null, + ); + expect(buildCompactPatternPredicate('Body', template)).not.toBeNull(); }); }); @@ -136,12 +273,19 @@ describe('parsePatternPredicate', () => { ['Body', 'GET api v1 users <*> <*> <*>'], ["JSONExtractString(Body, 'message')", 'user <*> action login status ok'], ['Body', '<*>'], - ])('reads back what buildPatternPredicate built (%s)', (expr, template) => { - const predicate = buildPatternPredicate(expr, template, withPunctuation)!; - expect(parsePatternPredicate(predicate, withPunctuation)).toEqual({ - valueExpression: expr, - template, - }); + ])('reads back what either builder built (%s)', (expr, template) => { + for (const options of OPTION_SETS) { + for (const build of [ + buildPatternPredicate, + buildCompactPatternPredicate, + ]) { + const predicate = build(expr, template, options)!; + expect(parsePatternPredicate(predicate, options)).toEqual({ + valueExpression: expr, + template, + }); + } + } }); it('reads back what buildPatternPredicate built without options', () => { @@ -167,6 +311,25 @@ describe('parsePatternPredicate', () => { expect(parsePatternPredicate("ServiceName IN ('api')")).toBeNull(); expect(parsePatternPredicate("match(Body, 'timed out')")).toBeNull(); // A predicate only reads back with the options it was built with. + expect( + parsePatternPredicate( + "match(replaceRegexpAll(Body, '[ ]+', ' '), '^ ?timed out ?$')", + withPunctuation, + ), + ).toBeNull(); + // The real separator class with a hand-written regex: only the round trip + // through compactRegex rejects it. + const compact = buildCompactPatternPredicate( + 'Body', + 'timed out', + withPunctuation, + )!; + const handWritten = compact.replace( + "'^ ?timed out ?$'", + "'^ ?timed.* out ?$'", + ); + expect(handWritten).not.toBe(compact); + expect(parsePatternPredicate(handWritten, withPunctuation)).toBeNull(); expect( parsePatternPredicate( buildPatternPredicate('Body', 'Sending <*> rows')!, diff --git a/packages/common-utils/src/drain/index.ts b/packages/common-utils/src/drain/index.ts index 65c433ea1..b0b64d73c 100644 --- a/packages/common-utils/src/drain/index.ts +++ b/packages/common-utils/src/drain/index.ts @@ -21,6 +21,7 @@ export { buildNoisyBodyMatcher, selectNoisyPatterns } from './noisy-matcher'; export type { PatternRegexOptions } from './pattern-predicate'; export { ASCII_PUNCTUATION, + buildCompactPatternPredicate, buildPatternPredicate, MAX_PREDICATE_TEMPLATE_LENGTH, parsePatternPredicate, diff --git a/packages/common-utils/src/drain/pattern-predicate.ts b/packages/common-utils/src/drain/pattern-predicate.ts index d7de608bf..8b5ed5623 100644 --- a/packages/common-utils/src/drain/pattern-predicate.ts +++ b/packages/common-utils/src/drain/pattern-predicate.ts @@ -70,6 +70,13 @@ function separatorClass(options?: PatternRegexOptions): string { ); } +// A template is its tokens joined by spaces. Drain3 keeps a BOM inside a +// token, where JS `\s` would split. +function tokensOf(template: string, options?: PatternRegexOptions): string[] { + const split = options?.whitespace === 'python' ? /[^\S\u{FEFF}]+/u : /\s+/; + return template.split(split).filter(t => t.length > 0); +} + /** Anchored RE2 regex that a value matches when it fits the Drain template. */ export function templateToRegex( template: string, @@ -77,10 +84,7 @@ export function templateToRegex( ): string { const separators = separatorClass(options); const separator = `[${separators}]`; - // Drain3 keeps a BOM inside a token, where JS `\s` would split. - const split = options?.whitespace === 'python' ? /[^\S\u{FEFF}]+/u : /\s+/; - const tokens = template.split(split).filter(t => t.length > 0); - const body = tokens + const body = tokensOf(template, options) .map(t => t === PATTERN_WILDCARD ? `[^${separators}]+` : escapeRe2Literal(t), ) @@ -101,18 +105,99 @@ export function buildPatternPredicate( return `match(${valueExpression}, '${escapeSqlString(templateToRegex(template, options))}')`; } +// The compact regex: tokens joined by single spaces, at most one space at +// each end, and one run of non-spaces per wildcard. +const COMPACT_START = '^ ?'; +const COMPACT_END = ' ?$'; +const COMPACT_WILDCARD = '[^ ]+'; + +// Regex over a value whose separator runs were replaced by single spaces. +function compactRegex(template: string, options?: PatternRegexOptions): string { + const body = tokensOf(template, options) + .map(t => (t === PATTERN_WILDCARD ? COMPACT_WILDCARD : escapeRe2Literal(t))) + .join(' '); + return `${COMPACT_START}${body}${COMPACT_END}`; +} + +/** + * The same match as buildPatternPredicate in much shorter SQL: separator runs + * in the value become single spaces first, so the separator class appears + * once instead of between every token. For conditions that are stored or + * shared, such as search filters kept in the URL. + * + * Null when a template token holds one of the extra delimiters: normalizing + * would split it, so the two forms would disagree. Templates mined with the + * same options never have one. As in buildPatternPredicate, `valueExpression` + * goes into the SQL as is: pass a column expression, never user input. + */ +export function buildCompactPatternPredicate( + valueExpression: string, + template: string, + options?: PatternRegexOptions, +): string | null { + if (template.length > MAX_PREDICATE_TEMPLATE_LENGTH) return null; + const delimiters = options?.extraDelimiters ?? []; + const splitsToken = tokensOf(template, options).some( + t => t !== PATTERN_WILDCARD && [...t].some(c => delimiters.includes(c)), + ); + if (splitsToken) return null; + const separatorRun = `[${separatorClass(options)}]+`; + const normalized = `replaceRegexpAll(${valueExpression}, '${escapeSqlString(separatorRun)}', ' ')`; + return `match(${normalized}, '${escapeSqlString(compactRegex(template, options))}')`; +} + +const unescapeSql = (literal: string) => + literal.replace(/''/g, "'").replace(/\\\\/g, '\\'); +const unescapeRe2Literal = (token: string) => token.replace(/\\(.)/g, '$1'); + +function parseCompactPredicate( + condition: string, + options?: PatternRegexOptions, +): { valueExpression: string; template: string } | null { + const call = + /^match\(replaceRegexpAll\((.+), '((?:[^'\\]|\\\\|'')*)', ' '\), '((?:[^'\\]|\\\\|'')*)'\)$/s.exec( + condition, + ); + if (call == null) return null; + const [, valueExpression, separatorLiteral, regexLiteral] = call; + if (unescapeSql(separatorLiteral) !== `[${separatorClass(options)}]+`) { + return null; + } + const regex = unescapeSql(regexLiteral); + if (!regex.startsWith(COMPACT_START) || !regex.endsWith(COMPACT_END)) { + return null; + } + const inner = regex.slice( + COMPACT_START.length, + regex.length - COMPACT_END.length, + ); + // COMPACT_WILDCARD itself contains a space, so match tokens rather than + // splitting on spaces. + const template = (inner.match(/\[\^ \]\+|[^ ]+/g) ?? []) + .map(part => + part === COMPACT_WILDCARD ? PATTERN_WILDCARD : unescapeRe2Literal(part), + ) + .join(' '); + return compactRegex(template, options) === regex + ? { valueExpression, template } + : null; +} + /** - * Reverse of buildPatternPredicate: the value expression and template of a - * predicate it built with the same options, or null for any other condition. + * Reverse of buildPatternPredicate and buildCompactPatternPredicate: the + * value expression and template of a predicate either built with the same + * options, or null for any other condition. */ export function parsePatternPredicate( condition: string, options?: PatternRegexOptions, ): { valueExpression: string; template: string } | null { + const compact = parseCompactPredicate(condition, options); + if (compact != null) return compact; const call = /^match\((.+), '((?:[^'\\]|\\\\|'')*)'\)$/s.exec(condition); if (call == null) return null; const [, valueExpression, literal] = call; - const regex = literal.replace(/''/g, "'").replace(/\\\\/g, '\\'); + const regex = unescapeSql(literal); const separators = separatorClass(options); const separator = `[${separators}]`; @@ -125,7 +210,7 @@ export function parsePatternPredicate( .map(part => part === `[^${separators}]+` ? PATTERN_WILDCARD - : part.replace(/\\(.)/g, '$1'), + : unescapeRe2Literal(part), ) .join(' '); // Only what this module builds counts, so a hand-written match() isn't