diff --git a/.changeset/variation-selector-supplement.md b/.changeset/variation-selector-supplement.md new file mode 100644 index 0000000..b71f34f --- /dev/null +++ b/.changeset/variation-selector-supplement.md @@ -0,0 +1,25 @@ +--- +'@moderation-api/unicode-spoofing': minor +--- + +Recognise the variation selector supplement (VS17–VS256, U+E0100–U+E01EF). + +Only VS1–VS16 and the Mongolian selectors were treated as variation +selectors, so a supplement selector after a Latin letter passed unreported — +one hidden byte per letter (`h\u{E0151}e\u{E0152}l\u{E0153}lo`). And because +these selectors are combining marks, a valid ideographic variation sequence +followed by more selectors (`辻\u{E0100}\u{E0101}\u{E0102}`) was reported as +`zalgo` and normalized with the valid selector stripped too. + +A supplement selector is now legitimate only directly after a Han or kana +letter (an ideographic variation sequence) and is reported as `invisible` +anywhere else: after another letter, a combining mark, a digit, an apostrophe, +or at the start of a word. A base takes one selector, so further selectors on +it are reported and removed while the first one is kept. VS1–VS16 directly +after a combining mark are reported too; at the start of a word they are left +alone, since there they end the emoji or symbol before the word (`❤️you`). A valid +selector no longer counts toward a `zalgo` stack and is never stripped as a +zalgo mark; stray selectors still count. + +This also removes a quadratic path: each selector used to walk back over the +whole run of selectors before it to find its base. diff --git a/src/analyze.ts b/src/analyze.ts index cccee8e..22562d0 100644 --- a/src/analyze.ts +++ b/src/analyze.ts @@ -76,9 +76,14 @@ const INVISIBLE_MARKS = new Set([ 0x110b1, // KAITHI VOWEL SIGN I ]); -/** Variation selectors, incl. the Mongolian free variation selectors. */ +/** Variation selectors (VS1–VS16, the VS17–VS256 supplement), incl. the + * Mongolian free variation selectors. */ function isVariationSelector(cp: number): boolean { - return (cp >= 0xfe00 && cp <= 0xfe0f) || (cp >= 0x180b && cp <= 0x180d); + return ( + (cp >= 0xfe00 && cp <= 0xfe0f) || + (cp >= 0xe0100 && cp <= 0xe01ef) || + (cp >= 0x180b && cp <= 0x180d) + ); } /** @@ -96,6 +101,14 @@ const VARIATION_BASE_SCRIPTS = new Set([ 'Mongolian', ]); +/** Scripts with ideographic variation sequences: the only bases a selector + * from the supplement (VS17–VS256) is registered on. */ +const IDEOGRAPHIC_BASE_SCRIPTS = new Set([ + 'Han', + 'Hiragana', + 'Katakana', +]); + /** A handful of characters are letters AND emoji — U+2139 "ℹ" is the common * one — so they take the emoji presentation selector legitimately. */ const PICTOGRAPHIC_RE = /\p{Extended_Pictographic}/u; @@ -302,25 +315,45 @@ function analyzeToken( for (let i = 0; i < chars.length; i += 1) { const cp = chars[i]!.codePointAt(0)!; if (!isVariationSelector(cp)) continue; - // The base is the last character that is not itself a selector. - let base: string | undefined; - for (let j = i - 1; j >= 0; j -= 1) { - if (!isVariationSelector(chars[j]!.codePointAt(0)!)) { - base = chars[j]; - break; - } + const base = chars[i - 1]; + const supplement = cp >= 0xe0100; + // At the word start the base sits outside the word: VS1–VS16 there end an + // emoji or symbol sequence ("❤️you", "≨︀"), which is not this pass's to + // judge; a supplement selector has no such base. + if (base === undefined) { + if (supplement) strayVariation.add(i); + continue; + } + // A base takes one selector; one after another has no sequence to join. + if (isVariationSelector(base.codePointAt(0)!)) { + strayVariation.add(i); + continue; } - if (base === undefined || !LETTER_RE.test(base) || PICTOGRAPHIC_RE.test(base)) continue; - if (!VARIATION_BASE_SCRIPTS.has(primaryScript(base))) strayVariation.add(i); + // A supplement selector is registered only directly on an ideograph or + // kana letter. VS1–VS16 also form sequences with digits and symbols + // (keycaps, slashed zero), but never with a combining mark. + if (!LETTER_RE.test(base)) { + if (supplement || MARK_RE.test(base)) strayVariation.add(i); + continue; + } + const baseScript = primaryScript(base); + const stray = supplement + ? !IDEOGRAPHIC_BASE_SCRIPTS.has(baseScript) + : !PICTOGRAPHIC_RE.test(base) && !VARIATION_BASE_SCRIPTS.has(baseScript); + if (stray) strayVariation.add(i); } const hasInvisible = chars.some((ch, i) => isInvisibleChar(ch) && !inertAt[i]) || strayVariation.size > 0; if (hasInvisible && !scriptExempt) signals.push('invisible'); - // zalgo: combining marks stacked beyond orthographic depth. + // zalgo: combining marks stacked beyond orthographic depth. Variation + // selectors are category Mn too; one that forms a valid sequence neither + // counts nor breaks a stack, a stray one counts like any other mark. let markRun = 0; let maxMarkRun = 0; - for (const ch of chars) { + for (let i = 0; i < chars.length; i += 1) { + const ch = chars[i]!; + if (isVariationSelector(ch.codePointAt(0)!) && !strayVariation.has(i)) continue; markRun = MARK_RE.test(ch) ? markRun + 1 : 0; if (markRun > maxMarkRun) maxMarkRun = markRun; } @@ -423,7 +456,8 @@ function analyzeToken( // would edit the message without cause. if (dropInvisible && ((isInvisibleChar(ch) && !inertAt[i]) || strayVariation.has(i))) return ''; - if (zalgo && MARK_RE.test(ch)) return ''; + const validSelector = isVariationSelector(ch.codePointAt(0)!) && !strayVariation.has(i); + if (zalgo && MARK_RE.test(ch) && !validSelector) return ''; if (nfkcFold) return ch; return foldLookalikes ? foldChar(ch) : ch; }) diff --git a/test/cases.data.ts b/test/cases.data.ts index 7e1b50e..e741922 100644 --- a/test/cases.data.ts +++ b/test/cases.data.ts @@ -91,4 +91,18 @@ export const CASES: SpoofCase[] = [ expect: { spoofed: true }, ref: 'https://github.com/moderation-api/unicode-spoofing/issues/1', }, + { + category: 'invisible', + name: 'flags a byte hidden after every letter in supplement variation selectors', + input: 'h\u{E0151}e\u{E0152}l\u{E0153}lo', + status: 'supported', + expect: { spoofed: true, signals: { invisible: true }, normalized: 'hello' }, + }, + { + category: 'multilingual (must NOT flag)', + name: 'leaves a Japanese name with an ideographic variation sequence alone', + input: '辻\u{E0100}さん', + status: 'supported', + expect: { spoofed: false, normalized: '辻\u{E0100}さん' }, + }, ]; diff --git a/test/invisible.test.ts b/test/invisible.test.ts index d93f5be..c147703 100644 --- a/test/invisible.test.ts +++ b/test/invisible.test.ts @@ -204,11 +204,80 @@ describe('variation selectors', () => { expect(r.normalized).toBe('hithere'); }); + it('flags a supplement selector (VS17-VS256) on a Latin letter', () => { + const r = analyze('x\u{E0110}'); + expect(r.signals.invisible).toBe(true); + expect(r.signals.zalgo).toBe(false); + expect(r.normalized).toBe('x'); + }); + + it('flags a byte hidden after every letter in supplement selectors', () => { + const r = analyze('h\u{E0151}e\u{E0152}l\u{E0153}lo'); + expect(r.signals.invisible).toBe(true); + expect(r.normalized).toBe('hello'); + expect(wellFormed(r.normalized)).toBe(true); + }); + + it('keeps the first selector on an ideograph and strips the rest, without zalgo', () => { + const r = analyze('\u8FBB\u{E0100}\u{E0101}\u{E0102}'); + expect(r.signals.invisible).toBe(true); + expect(r.signals.zalgo).toBe(false); + expect(r.normalized).toBe('\u8FBB\u{E0100}'); + }); + + it('flags a supplement selector anywhere but directly on an ideograph', () => { + // Carriers other than letters: a decomposed accent, a digit, an + // apostrophe, the word start. + for (const text of [ + 'cafe\u0301\u{E0141}', + 'h3\u{E0151}llo', + "don'\u{E0151}t", + '\u{E0101}hello', + ]) { + expect(analyze(text).signals.invisible, JSON.stringify(text)).toBe(true); + } + }); + + it('leaves a symbol selector alone when a letter follows it directly', () => { + // The word starts at the selector; its base is the emoji or symbol before. + for (const text of [ + 'I \u2764\uFE0Fyou', + '\u2600\uFE0Fsunny', + '\u00A9\uFE0FAcme', + '#\uFE0F\u20E3word', + '\u263A\uFE0Eok', + 'x\u2268\uFE00y', + ]) { + const r = analyze(text); + expect(r.spoofed, JSON.stringify(text)).toBe(false); + expect(r.changed, JSON.stringify(text)).toBe(false); + } + }); + + it('still counts stray selectors toward a zalgo stack', () => { + for (const text of ['Vie\u0302\u0323\u{E0101}t Nam', 'a\u0301\u0302\uFE00']) { + const r = analyze(text); + expect(r.signals.zalgo, JSON.stringify(text)).toBe(true); + expect(r.signals.invisible, JSON.stringify(text)).toBe(true); + } + }); + + it('keeps an ideographic variation sequence when the word is stripped of zalgo', () => { + const r = analyze('\u8FBB\u{E0100}\u0301\u0302\u0303'); + expect(r.signals.zalgo).toBe(true); + expect(r.normalized).toBe('\u8FBB\u{E0100}'); + }); + it('leaves registered variation sequences alone', () => { const LEGITIMATE: [string, string][] = [ ['emoji presentation', '\u2764\uFE0F'], ['keycap', '#\uFE0F\u20E3'], ['ideographic variant', '\u8FBB\uFE00'], + ['ideographic variation sequence (VS17)', '\u8FBB\u{E0100}'], + [ + 'ideographic variation sequences in running text', + '\u845B\u{E0101}\u57CE\u3055\u{E0100}\u3093', + ], ['Mongolian free variation selector', '\u1820\u180B'], ]; for (const [name, str] of LEGITIMATE) { diff --git a/test/performance.test.ts b/test/performance.test.ts index 43fea96..7ba2992 100644 --- a/test/performance.test.ts +++ b/test/performance.test.ts @@ -79,4 +79,13 @@ describe('analyze performance', () => { const elapsed = medianMs(() => analyze(zalgo), 3); expect(elapsed).toBeLessThan(250); }); + + it('handles a long run of variation selectors quickly', () => { + // Each selector is judged against the character right before it; looking + // further back for a base would make a run of n selectors cost n². + const run = 'x' + '\uFE01'.repeat(40_000); + + const elapsed = medianMs(() => analyze(run), 3); + expect(elapsed).toBeLessThan(250); + }); });