diff --git a/.changeset/escape-sequence-remnant.md b/.changeset/escape-sequence-remnant.md new file mode 100644 index 0000000..e7210ba --- /dev/null +++ b/.changeset/escape-sequence-remnant.md @@ -0,0 +1,14 @@ +--- +'@moderation-api/unicode-spoofing': patch +--- + +Strip whole ANSI control sequences (CSI) from `normalized`, not just the ESC. + +`ok \x1b[31mred\x1b[0m done` normalized to `ok [31mred[0m done`: only ESC is an +illegal code point, so the rest of each colour code was left in the text. A +well-formed ECMA-48 control sequence (ESC `[`, parameter bytes, intermediate +bytes, final byte) is now one `illegal` finding, whose `word` is the whole +sequence, and is removed whole, giving `ok red done`; its bytes no longer +tokenize as part of the neighbouring word. An ESC not followed by a complete +control sequence — including other escape types such as OSC strings — is still +removed on its own, as before. diff --git a/src/analyze.ts b/src/analyze.ts index cccee8e..8961f04 100644 --- a/src/analyze.ts +++ b/src/analyze.ts @@ -215,6 +215,24 @@ function isIllegalCodePoint(cp: number): boolean { return false; } +/* eslint-disable no-control-regex -- ESC is what these patterns match */ +/** + * An ECMA-48 control sequence (CSI): ESC "[", parameter bytes 0x30–0x3F, + * intermediate bytes 0x20–0x2F, one final byte 0x40–0x7E — e.g. the colour + * code "\x1b[31m". Only ESC is an illegal code point, but removing it alone + * would leave "[31m" in the text, so a well-formed sequence goes whole. + */ +const CONTROL_SEQUENCE_RE = /\x1b\[[\x30-\x3f]*[\x20-\x2f]*[\x40-\x7e]/g; +const CONTROL_SEQUENCE_AT_RE = new RegExp(CONTROL_SEQUENCE_RE.source, 'y'); +/* eslint-enable no-control-regex */ + +/** `text` with control sequences blanked to spaces of the same length, so + * their bytes ("31m") do not tokenize as words while offsets stay valid. */ +function blankControlSequences(text: string): string { + if (!text.includes('\x1b')) return text; + return text.replace(CONTROL_SEQUENCE_RE, (seq) => ' '.repeat(seq.length)); +} + /** * U+FFFD REPLACEMENT CHARACTER: a decoder emitted this because bytes it was * handed were not valid in the encoding it assumed — "José" arriving as @@ -446,11 +464,12 @@ function analyzeToken( export function analyze(text: string, options: AnalyzeOptions = {}): AnalysisResult { const expectedScripts = new Set(options.expectedScripts ?? []); - const tokens = [...text.matchAll(TOKEN_RE)]; + const wordText = blankControlSequences(text); + const tokens = [...wordText.matchAll(TOKEN_RE)]; // Dominant script: most frequent primary script across all letters. const scriptCounts = new Map(); - for (const ch of text) { + for (const ch of wordText) { if (!LETTER_RE.test(ch) || isInvisibleChar(ch)) continue; const s = primaryScript(ch); if (s === 'Common' || s === 'Inherited' || s === 'Unknown') continue; @@ -564,17 +583,21 @@ export function analyze(text: string, options: AnalyzeOptions = {}): AnalysisRes // Illegal code points live *between* tokens (the token scanner drops them), // so they are found in a separate pass over the raw text. Each becomes its - // own finding and is stripped from the normalized output. + // own finding and is stripped from the normalized output, together with the + // rest of the control sequence an ESC introduces. for (let i = 0; i < text.length;) { const cp = text.codePointAt(i)!; - const width = cp > 0xffff ? 2 : 1; + let end = i + (cp > 0xffff ? 2 : 1); if (isIllegalCodePoint(cp)) { - const char = String.fromCodePoint(cp); + if (cp === 0x1b) { + CONTROL_SEQUENCE_AT_RE.lastIndex = i; + if (CONTROL_SEQUENCE_AT_RE.test(text)) end = CONTROL_SEQUENCE_AT_RE.lastIndex; + } signals.illegal = true; - words.push({ word: char, index: i, signals: ['illegal'], scripts: [] }); - replacements.push({ start: i, end: i + width, value: '' }); + words.push({ word: text.slice(i, end), index: i, signals: ['illegal'], scripts: [] }); + replacements.push({ start: i, end, value: '' }); } - i += width; + i = end; } // Decode damage. A run reports as ONE finding: a single destroyed character diff --git a/test/analyze.test.ts b/test/analyze.test.ts index 4ad838f..25f5462 100644 --- a/test/analyze.test.ts +++ b/test/analyze.test.ts @@ -87,6 +87,26 @@ describe('analyze — spoofed content', () => { } }); + it('strips a whole terminal escape sequence, not just the ESC', () => { + const r = analyze('ok \x1b[31mred\x1b[0m done'); + expect(r.signals.illegal).toBe(true); + expect(r.normalized).toBe('ok red done'); + expect(r.words.map((w) => w.word)).toEqual(['\x1b[31m', '\x1b[0m']); + }); + + it('keeps sequence bytes out of the word next to them', () => { + // "1m" is part of the sequence, not of the word "НОТ" that follows it. + const r = analyze('\x1b[1mНОТ deal \x1b[38;5;196mnow\x1b[?25l'); + expect(r.normalized).toBe('HOT deal now'); + expect(r.words.map((w) => w.word)).toContain('НОТ'); + }); + + it('removes only the ESC of an unterminated escape sequence', () => { + // No final byte: the bytes after ESC are not provably part of a sequence. + expect(analyze('a\x1b[31').normalized).toBe('a[31'); + expect(analyze('a\x1b[3é').normalized).toBe('a[3é'); + }); + it('reports codepoint-accurate word offsets', () => { const r = analyze(CUSTOMER_SAMPLE); for (const w of r.words) { diff --git a/test/cases.data.ts b/test/cases.data.ts index 7e1b50e..5e075f7 100644 --- a/test/cases.data.ts +++ b/test/cases.data.ts @@ -83,6 +83,20 @@ export const CASES: SpoofCase[] = [ status: 'supported', expect: { spoofed: false }, }, + { + category: 'illegal', + name: 'strips a whole ANSI colour sequence, not just the ESC', + input: 'ok \x1b[31mred\x1b[0m done', + status: 'supported', + expect: { spoofed: true, signals: { illegal: true }, normalized: 'ok red done' }, + }, + { + category: 'illegal (must NOT flag)', + name: 'leaves bracketed text that looks like a sequence body alone ([31m)', + input: 'see note [31m] and [0m', + status: 'supported', + expect: { spoofed: false, normalized: 'see note [31m] and [0m' }, + }, { category: 'digit systems', name: 'flags fullwidth digits masquerading as ASCII (１２３)',