fix: recognise the variation selector supplement (VS17–VS256)

.changeset/variation-selector-supplement.md

@@ -0,0 +1,25 @@
+---
+'@moderation-api/unicode-spoofing': minor
+---
+
+Recognise the variation selector supplement (VS17–VS256, U+E0100–U+E01EF).
+
+Only VS1–VS16 and the Mongolian selectors were treated as variation
+selectors, so a supplement selector after a Latin letter passed unreported —
+one hidden byte per letter (`h\u{E0151}e\u{E0152}l\u{E0153}lo`). And because
+these selectors are combining marks, a valid ideographic variation sequence
+followed by more selectors (`辻\u{E0100}\u{E0101}\u{E0102}`) was reported as
+`zalgo` and normalized with the valid selector stripped too.
+
+A supplement selector is now legitimate only directly after a Han or kana
+letter (an ideographic variation sequence) and is reported as `invisible`
+anywhere else: after another letter, a combining mark, a digit, an apostrophe,
+or at the start of a word. A base takes one selector, so further selectors on
+it are reported and removed while the first one is kept. VS1–VS16 directly
+after a combining mark are reported too; at the start of a word they are left
+alone, since there they end the emoji or symbol before the word (`❤️you`). A valid
+selector no longer counts toward a `zalgo` stack and is never stripped as a
+zalgo mark; stray selectors still count.
+
+This also removes a quadratic path: each selector used to walk back over the
+whole run of selectors before it to find its base.

src/analyze.ts

@@ -76,9 +76,14 @@ const INVISIBLE_MARKS = new Set([
   0x110b1, // KAITHI VOWEL SIGN I
 ]);
 
-/** Variation selectors, incl. the Mongolian free variation selectors. */
+/** Variation selectors (VS1–VS16, the VS17–VS256 supplement), incl. the
+ * Mongolian free variation selectors. */
 function isVariationSelector(cp: number): boolean {
-  return (cp >= 0xfe00 && cp <= 0xfe0f) || (cp >= 0x180b && cp <= 0x180d);
+  return (
+    (cp >= 0xfe00 && cp <= 0xfe0f) ||
+    (cp >= 0xe0100 && cp <= 0xe01ef) ||
+    (cp >= 0x180b && cp <= 0x180d)
+  );
 }
 
 /**
@@ -96,6 +101,14 @@ const VARIATION_BASE_SCRIPTS = new Set<ScriptName | PseudoScript>([
   'Mongolian',
 ]);
 
+/** Scripts with ideographic variation sequences: the only bases a selector
+ * from the supplement (VS17–VS256) is registered on. */
+const IDEOGRAPHIC_BASE_SCRIPTS = new Set<ScriptName | PseudoScript>([
+  'Han',
+  'Hiragana',
+  'Katakana',
+]);
+
 /** A handful of characters are letters AND emoji — U+2139 "ℹ" is the common
  * one — so they take the emoji presentation selector legitimately. */
 const PICTOGRAPHIC_RE = /\p{Extended_Pictographic}/u;
@@ -302,25 +315,45 @@ function analyzeToken(
   for (let i = 0; i < chars.length; i += 1) {
     const cp = chars[i]!.codePointAt(0)!;
     if (!isVariationSelector(cp)) continue;
-    // The base is the last character that is not itself a selector.
-    let base: string | undefined;
-    for (let j = i - 1; j >= 0; j -= 1) {
-      if (!isVariationSelector(chars[j]!.codePointAt(0)!)) {
-        base = chars[j];
-        break;
-      }
+    const base = chars[i - 1];
+    const supplement = cp >= 0xe0100;
+    // At the word start the base sits outside the word: VS1–VS16 there end an
+    // emoji or symbol sequence ("❤️you", "≨︀"), which is not this pass's to
+    // judge; a supplement selector has no such base.
+    if (base === undefined) {
+      if (supplement) strayVariation.add(i);
+      continue;
+    }
+    // A base takes one selector; one after another has no sequence to join.
+    if (isVariationSelector(base.codePointAt(0)!)) {
+      strayVariation.add(i);
+      continue;
     }
-    if (base === undefined || !LETTER_RE.test(base) || PICTOGRAPHIC_RE.test(base)) continue;
-    if (!VARIATION_BASE_SCRIPTS.has(primaryScript(base))) strayVariation.add(i);
+    // A supplement selector is registered only directly on an ideograph or
+    // kana letter. VS1–VS16 also form sequences with digits and symbols
+    // (keycaps, slashed zero), but never with a combining mark.
+    if (!LETTER_RE.test(base)) {
+      if (supplement || MARK_RE.test(base)) strayVariation.add(i);
+      continue;
+    }
+    const baseScript = primaryScript(base);
+    const stray = supplement
+      ? !IDEOGRAPHIC_BASE_SCRIPTS.has(baseScript)
+      : !PICTOGRAPHIC_RE.test(base) && !VARIATION_BASE_SCRIPTS.has(baseScript);
+    if (stray) strayVariation.add(i);
   }
   const hasInvisible =
     chars.some((ch, i) => isInvisibleChar(ch) && !inertAt[i]) || strayVariation.size > 0;
   if (hasInvisible && !scriptExempt) signals.push('invisible');
 
-  // zalgo: combining marks stacked beyond orthographic depth.
+  // zalgo: combining marks stacked beyond orthographic depth. Variation
+  // selectors are category Mn too; one that forms a valid sequence neither
+  // counts nor breaks a stack, a stray one counts like any other mark.
   let markRun = 0;
   let maxMarkRun = 0;
-  for (const ch of chars) {
+  for (let i = 0; i < chars.length; i += 1) {
+    const ch = chars[i]!;
+    if (isVariationSelector(ch.codePointAt(0)!) && !strayVariation.has(i)) continue;
     markRun = MARK_RE.test(ch) ? markRun + 1 : 0;
     if (markRun > maxMarkRun) maxMarkRun = markRun;
   }
@@ -423,7 +456,8 @@ function analyzeToken(
         // would edit the message without cause.
         if (dropInvisible && ((isInvisibleChar(ch) && !inertAt[i]) || strayVariation.has(i)))
           return '';
-        if (zalgo && MARK_RE.test(ch)) return '';
+        const validSelector = isVariationSelector(ch.codePointAt(0)!) && !strayVariation.has(i);
+        if (zalgo && MARK_RE.test(ch) && !validSelector) return '';
         if (nfkcFold) return ch;
         return foldLookalikes ? foldChar(ch) : ch;
       })

test/cases.data.ts

@@ -91,4 +91,18 @@ export const CASES: SpoofCase[] = [
     expect: { spoofed: true },
     ref: 'https://github.com/moderation-api/unicode-spoofing/issues/1',
   },
+  {
+    category: 'invisible',
+    name: 'flags a byte hidden after every letter in supplement variation selectors',
+    input: 'h\u{E0151}e\u{E0152}l\u{E0153}lo',
+    status: 'supported',
+    expect: { spoofed: true, signals: { invisible: true }, normalized: 'hello' },
+  },
+  {
+    category: 'multilingual (must NOT flag)',
+    name: 'leaves a Japanese name with an ideographic variation sequence alone',
+    input: '辻\u{E0100}さん',
+    status: 'supported',
+    expect: { spoofed: false, normalized: '辻\u{E0100}さん' },
+  },
 ];

test/invisible.test.ts

@@ -204,11 +204,80 @@ describe('variation selectors', () => {
     expect(r.normalized).toBe('hithere');
   });
 
+  it('flags a supplement selector (VS17-VS256) on a Latin letter', () => {
+    const r = analyze('x\u{E0110}');
+    expect(r.signals.invisible).toBe(true);
+    expect(r.signals.zalgo).toBe(false);
+    expect(r.normalized).toBe('x');
+  });
+
+  it('flags a byte hidden after every letter in supplement selectors', () => {
+    const r = analyze('h\u{E0151}e\u{E0152}l\u{E0153}lo');
+    expect(r.signals.invisible).toBe(true);
+    expect(r.normalized).toBe('hello');
+    expect(wellFormed(r.normalized)).toBe(true);
+  });
+
+  it('keeps the first selector on an ideograph and strips the rest, without zalgo', () => {
+    const r = analyze('\u8FBB\u{E0100}\u{E0101}\u{E0102}');
+    expect(r.signals.invisible).toBe(true);
+    expect(r.signals.zalgo).toBe(false);
+    expect(r.normalized).toBe('\u8FBB\u{E0100}');
+  });
+
+  it('flags a supplement selector anywhere but directly on an ideograph', () => {
+    // Carriers other than letters: a decomposed accent, a digit, an
+    // apostrophe, the word start.
+    for (const text of [
+      'cafe\u0301\u{E0141}',
+      'h3\u{E0151}llo',
+      "don'\u{E0151}t",
+      '\u{E0101}hello',
+    ]) {
+      expect(analyze(text).signals.invisible, JSON.stringify(text)).toBe(true);
+    }
+  });
+
+  it('leaves a symbol selector alone when a letter follows it directly', () => {
+    // The word starts at the selector; its base is the emoji or symbol before.
+    for (const text of [
+      'I \u2764\uFE0Fyou',
+      '\u2600\uFE0Fsunny',
+      '\u00A9\uFE0FAcme',
+      '#\uFE0F\u20E3word',
+      '\u263A\uFE0Eok',
+      'x\u2268\uFE00y',
+    ]) {
+      const r = analyze(text);
+      expect(r.spoofed, JSON.stringify(text)).toBe(false);
+      expect(r.changed, JSON.stringify(text)).toBe(false);
+    }
+  });
+
+  it('still counts stray selectors toward a zalgo stack', () => {
+    for (const text of ['Vie\u0302\u0323\u{E0101}t Nam', 'a\u0301\u0302\uFE00']) {
+      const r = analyze(text);
+      expect(r.signals.zalgo, JSON.stringify(text)).toBe(true);
+      expect(r.signals.invisible, JSON.stringify(text)).toBe(true);
+    }
+  });
+
+  it('keeps an ideographic variation sequence when the word is stripped of zalgo', () => {
+    const r = analyze('\u8FBB\u{E0100}\u0301\u0302\u0303');
+    expect(r.signals.zalgo).toBe(true);
+    expect(r.normalized).toBe('\u8FBB\u{E0100}');
+  });
+
   it('leaves registered variation sequences alone', () => {
     const LEGITIMATE: [string, string][] = [
       ['emoji presentation', '\u2764\uFE0F'],
       ['keycap', '#\uFE0F\u20E3'],
       ['ideographic variant', '\u8FBB\uFE00'],
+      ['ideographic variation sequence (VS17)', '\u8FBB\u{E0100}'],
+      [
+        'ideographic variation sequences in running text',
+        '\u845B\u{E0101}\u57CE\u3055\u{E0100}\u3093',
+      ],
       ['Mongolian free variation selector', '\u1820\u180B'],
     ];
     for (const [name, str] of LEGITIMATE) {

test/performance.test.ts

@@ -79,4 +79,13 @@ describe('analyze performance', () => {
     const elapsed = medianMs(() => analyze(zalgo), 3);
     expect(elapsed).toBeLessThan(250);
   });
+
+  it('handles a long run of variation selectors quickly', () => {
+    // Each selector is judged against the character right before it; looking
+    // further back for a base would make a run of n selectors cost n².
+    const run = 'x' + '\uFE01'.repeat(40_000);
+
+    const elapsed = medianMs(() => analyze(run), 3);
+    expect(elapsed).toBeLessThan(250);
+  });
 });