From f23a97991b285e4a8eda8541422ab2b86dd0590c Mon Sep 17 00:00:00 2001 From: cycanic <79452786+cycanic@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:50:25 +0400 Subject: [PATCH] fix: judge the format-character script exemption per letter One letter of a FORMAT_CHAR_SCRIPTS script exempted its whole token from `invisible` and `zalgo`. A bidi override inside a Hebrew word, tag characters inside an Arabic or Devanagari word, or a run of zero-width spaces inside a Persian word therefore passed and stayed in `normalized`. Judge each hidden character and mark by the nearest letters around it (script-neutral letters such as the tatweel are skipped). Kept: a single ZWNJ/ZWJ/ZWSP (or the ZWJ ZWNJ and ZWJ ZWNJ ZWJ idioms) in a word of these scripts with no other-script letter on either side, so joiners next to numerals and Thai/Khmer/Myanmar word breaks stay clean; a single implicit bidi mark or isolate next to an RTL letter or next to a number in an RTL word; CGJ between marks of such a letter; a format character of a neighbouring letter's script (never a blank glyph); and mark stacks on these letters. Everything else is judged as in a Latin word, and only the offending characters are stripped. Myanmar joins VARIATION_BASE_SCRIPTS to prevent a regression: its words were exempt as a whole, so its standardized dotted forms (letter + VS1, StandardizedVariants.txt) were never judged; per-letter judging would report them. Among FORMAT_CHAR_SCRIPTS only Mongolian (already listed) and Myanmar have standardized variants. Co-Authored-By: Claude Opus 5.5 --- .changeset/script-exemption-per-letter.md | 39 +++++ README.md | 9 +- src/analyze.ts | 173 ++++++++++++++++++++-- test/cases.data.ts | 21 +++ test/invisible.test.ts | 91 ++++++++++++ 5 files changed, 315 insertions(+), 18 deletions(-) create mode 100644 .changeset/script-exemption-per-letter.md diff --git a/.changeset/script-exemption-per-letter.md b/.changeset/script-exemption-per-letter.md new file mode 100644 index 0000000..6d5e4c1 --- /dev/null +++ b/.changeset/script-exemption-per-letter.md @@ -0,0 +1,39 @@ +--- +'@moderation-api/unicode-spoofing': minor +--- + +Judge the `FORMAT_CHAR_SCRIPTS` exemption per letter instead of per word. + +A single Hebrew, Arabic, Persian or Indic letter exempted its whole word from +`invisible` and `zalgo`, so a bidi override (`שלום`), tag characters +(`ميخ`, `कष`) or a run of zero-width spaces (`می<5×ZWSP>خواهم`) +inside such a word passed undetected and stayed in `normalized`. + +Hidden characters and marks are now judged by the letters next to them. Still +exempt: + +- a single ZWNJ, ZWJ or ZWSP in a word of these scripts with no letter of + another script on either side (Persian ZWNJ, also next to numerals; Indic + conjuncts; Malayalam chillu; Thai, Khmer and Myanmar word breaks), and the + joiner sequences ZWJ ZWNJ (final form) and ZWJ ZWNJ ZWJ (ligature break); +- a single implicit bidi mark (LRM, RLM, ALM) or isolate next to a + right-to-left letter, or next to a number in a right-to-left word; +- a combining grapheme joiner between the marks of such a letter; +- a format character of a neighbouring letter's script (Mongolian vowel + separator, Syriac abbreviation mark, invisible Khmer vowels); +- mark stacks on these letters. + +Script-neutral letters such as the Arabic tatweel are read as part of the +letters around them. + +Everything else inside such a word is judged as in a Latin word, and only the +offending characters are removed. Newly reported there: bidi overrides and +embeddings (an `RLE…PDF` wrapper is reported where an `RLI…PDI` isolate is +not), runs of bidi marks or isolates, tag characters, runs of zero-width +characters, Hangul fillers, soft hyphens, word joiners, BOMs, a CGJ not +following a mark, variation selectors on letters with no registered sequence +(`שלום`), and zalgo stacks on a non-letter base (`ש1` + 3 marks). + +Myanmar joins the scripts whose letters take a variation selector, so its +standardized dotted forms (`က`) stay unreported now that its words are no +longer exempt as a whole. diff --git a/README.md b/README.md index 7ccff53..1203429 100644 --- a/README.md +++ b/README.md @@ -231,7 +231,7 @@ against a name instead of a hardcoded string. | `SCRIPT_NAMES` | Every script the classifier can name — the possible values of `dominantScript` and `words[].scripts` | | `SUPPORTED_SCRIPTS` | The subset of `SCRIPT_NAMES` this runtime's Unicode tables support (all of them, on a current engine) | | `PSEUDO_SCRIPTS` | `Common`, `Inherited`, `Unknown` — classifications that are not scripts and never appear in a finding | -| `FORMAT_CHAR_SCRIPTS` | Scripts exempt from `invisible` and `zalgo`, because joiners and stacked marks are their orthography | +| `FORMAT_CHAR_SCRIPTS` | Scripts whose letters may carry joiners and stacked marks without firing `invisible` or `zalgo` | | `LEGITIMATE_SCRIPT_COMBINATIONS` | Script mixes that never fire `mixed_script` (Japanese, Korean, Bopomofo-annotated Chinese) | | `ZALGO_MARK_RUN` | Combining marks per base at which `zalgo` fires | | `UNICODE_VERSION`, `DATA_DATE` | Which UTS #39 confusables table is compiled in | @@ -305,8 +305,11 @@ analyze(text, { expectedScripts: ['Cyrillic'] }); // real Russian traffic - Japanese (Han + kana), Korean (Han + Hangul), and Bopomofo-annotated Chinese are legitimate single-word script mixes and are exempt. - Scripts whose orthography uses joiners/zero-width characters (Arabic, Indic - scripts, Persian ZWNJ, …) are exempt from `invisible`; scripts with stacked - marks (Hebrew points, Arabic tashkeel, …) are exempt from `zalgo`. + scripts, Persian ZWNJ, …) may carry a single ZWNJ/ZWJ/ZWSP between their + letters, and right-to-left text its bidi marks, without firing `invisible`; + stacked marks on their letters (Hebrew points, Arabic tashkeel, …) never fire + `zalgo`. The exemption is judged per letter: a bidi override, tag characters + or a zero-width run inside such a word are still reported. - Invisible characters that build a sequence are left alone: emoji ZWJ sequences, keycaps, the tag characters in `🏴󠁧󠁢󠁳󠁣󠁴󠁿`, emoji presentation selectors (`❤️`, `ℹ️`), ideographic variants (`辻︀`) and Mongolian FVS. diff --git a/src/analyze.ts b/src/analyze.ts index cccee8e..b0dfc5c 100644 --- a/src/analyze.ts +++ b/src/analyze.ts @@ -66,7 +66,7 @@ const BLANK_CHARS = new Set([ /** * Combining marks that render as nothing. They are category Mn rather than Cf, * so the format-character rule misses them. Each is legitimate inside its own - * script (Khmer, Kaithi) — which the `scriptExempt` check below preserves — and + * script (Khmer, Kaithi) — which the per-letter check (`isOrthographicInvisible`) preserves — and * pure obfuscation inside a Latin word. */ const INVISIBLE_MARKS = new Set([ @@ -94,6 +94,7 @@ const VARIATION_BASE_SCRIPTS = new Set([ 'Hiragana', 'Katakana', 'Mongolian', + 'Myanmar', // dotted letter forms (StandardizedVariants.txt) ]); /** A handful of characters are letters AND emoji — U+2139 "ℹ" is the common @@ -109,8 +110,10 @@ function isInvisibleChar(ch: string): boolean { /** * Scripts in which format characters (ZWJ/ZWNJ/ZWSP…) or stacked combining - * marks are part of normal orthography. Tokens whose letters touch these - * scripts are exempt from the invisible and zalgo signals. + * marks are part of normal orthography. Exemptions are judged per letter: a + * mark stack on a letter of these scripts is never zalgo, and a single + * joiner or zero-width space between their letters is not invisible — see + * `isOrthographicInvisible`. The rest of the word is judged as usual. */ export const FORMAT_CHAR_SCRIPTS: readonly ScriptName[] = [ 'Arabic', @@ -139,6 +142,128 @@ export const FORMAT_CHAR_SCRIPTS: readonly ScriptName[] = [ const FORMAT_CHAR_SCRIPT_SET = new Set(FORMAT_CHAR_SCRIPTS); +/** Scripts written right to left, whose text needs the implicit bidi marks. */ +const RTL_SCRIPT_SET = new Set(['Arabic', 'Hebrew', 'Nko', 'Syriac', 'Thaana']); + +/** Script of a neighbouring letter; `undefined` when there is none. */ +type NeighbourScript = ScriptName | PseudoScript | undefined; + +const isFormatCharScript = (s: NeighbourScript) => + s !== undefined && FORMAT_CHAR_SCRIPT_SET.has(s as ScriptName); +const isRtlScript = (s: NeighbourScript) => s !== undefined && RTL_SCRIPT_SET.has(s as ScriptName); + +/** + * Joiner sequences with a defined meaning: ZWJ ZWNJ ZWJ breaks a ligature + * while keeping cursive joining (The Unicode Standard, §23.2), and ZWJ ZWNJ + * forces a final form (Kurdish and older Persian heh). + */ +const JOINER_IDIOMS = new Set(['\u200D\u200C\u200D', '\u200D\u200C']); + +/** + * The run of invisible characters around `i`, or `undefined` when it is longer + * than any legitimate run (three). Bounded so a long run costs no more than a + * short one. + */ +function shortInvisibleRun(chars: readonly string[], i: number): string | undefined { + let start = i; + let end = i + 1; + while (start > 0 && isInvisibleChar(chars[start - 1]!)) { + start -= 1; + if (end - start > 3) return undefined; + } + while (end < chars.length && isInvisibleChar(chars[end]!)) { + end += 1; + if (end - start > 3) return undefined; + } + return chars.slice(start, end).join(''); +} + +/** + * Whether the invisible character at `i` is part of the orthography of the + * letters around it, given the scripts of the nearest letter before and after + * it in the word (marks, invisibles and script-neutral letters such as the + * tatweel skipped; `undefined` when a digit, punctuation or the word edge + * comes first) and whether the word has letters of a FORMAT_CHAR_SCRIPTS or + * right-to-left script at all: + * - a single ZWSP/ZWNJ/ZWJ (or a joiner idiom) in a word of such a script, + * with no letter of another script on either side (Persian ZWNJ, Indic + * conjunct control, Malayalam chillu, Thai/Khmer/Myanmar word breaks, + * also next to numerals); + * - a single implicit bidi mark or isolate next to a right-to-left letter, + * or in a right-to-left word with no letter on either side (next to a + * number); + * - a combining grapheme joiner between marks of such a letter (it orders + * vowel points, e.g. in Biblical Hebrew); + * - a format character or invisible mark of the script of a neighbouring + * letter (Mongolian vowel separator, Syriac abbreviation mark, the + * invisible Khmer vowels). Blank glyphs never: they occupy space. + * Bidi overrides and embeddings, tag characters, runs of zero-width + * characters or bidi marks, and anything next to letters of other scripts + * stay invisible. + */ +function isOrthographicInvisible( + chars: readonly string[], + i: number, + before: NeighbourScript, + after: NeighbourScript, + word: { formatChar: boolean; rtl: boolean }, +): boolean { + const ch = chars[i]!; + const cp = ch.codePointAt(0)!; + if (cp === 0x200b || cp === 0x200c || cp === 0x200d) { + const run = shortInvisibleRun(chars, i); + return ( + run !== undefined && + (run.length === 1 || JOINER_IDIOMS.has(run)) && + word.formatChar && + (before === undefined || isFormatCharScript(before)) && + (after === undefined || isFormatCharScript(after)) + ); + } + if (cp === 0x200e || cp === 0x200f || cp === 0x061c || (cp >= 0x2066 && cp <= 0x2069)) { + return ( + shortInvisibleRun(chars, i)?.length === 1 && + (isRtlScript(before) || + isRtlScript(after) || + (word.rtl && before === undefined && after === undefined)) + ); + } + if (cp === 0x034f) { + return isFormatCharScript(before) && i > 0 && MARK_RE.test(chars[i - 1]!); + } + if (BLANK_CHARS.has(cp)) return false; + const own = primaryScript(ch); + return ( + own !== 'Common' && + own !== 'Inherited' && + own !== 'Unknown' && + (own === before || own === after) + ); +} + +/** + * For each position, the script of the nearest letter in direction `step`, + * skipping marks, invisibles and script-neutral letters (the Arabic tatweel + * is Common); `undefined` when a non-letter or the word edge comes first. + */ +function nearestLetterScripts(chars: readonly string[], step: 1 | -1): NeighbourScript[] { + const out = new Array(chars.length); + let script: NeighbourScript; + for (let k = 0; k < chars.length; k += 1) { + const i = step === 1 ? chars.length - 1 - k : k; + out[i] = script; + const ch = chars[i]!; + if (MARK_RE.test(ch) || isInvisibleChar(ch)) continue; + if (!LETTER_RE.test(ch)) { + script = undefined; + continue; + } + const s = primaryScript(ch); + if (s !== 'Common' && s !== 'Inherited') script = s; + } + return out; +} + /** * Zero-width AND non-reordering: these render as nothing and move nothing. * Deliberately excludes the other invisibles, which stay reportable wherever @@ -293,7 +418,15 @@ function analyzeToken( } const { scripts, mixed } = analyzeWordScripts(letters); - const scriptExempt = scripts.some((s) => FORMAT_CHAR_SCRIPT_SET.has(s)); + // Hidden characters and marks are judged by the letters next to them; most + // words have neither, so the neighbour scripts are only resolved on demand. + const judged = chars.some((ch) => MARK_RE.test(ch) || isInvisibleChar(ch)); + const letterBefore = judged ? nearestLetterScripts(chars, -1) : []; + const letterAfter = judged ? nearestLetterScripts(chars, 1) : []; + const wordScripts = { + formatChar: scripts.some(isFormatCharScript), + rtl: scripts.some(isRtlScript), + }; // invisible: characters that render as nothing inside a word of a script that // does not use them — format characters, blank glyphs, invisible marks, and @@ -313,18 +446,29 @@ function analyzeToken( if (base === undefined || !LETTER_RE.test(base) || PICTOGRAPHIC_RE.test(base)) continue; if (!VARIATION_BASE_SCRIPTS.has(primaryScript(base))) strayVariation.add(i); } - const hasInvisible = - chars.some((ch, i) => isInvisibleChar(ch) && !inertAt[i]) || strayVariation.size > 0; - if (hasInvisible && !scriptExempt) signals.push('invisible'); - - // zalgo: combining marks stacked beyond orthographic depth. + const strayInvisible = judged + ? chars.map( + (ch, i) => + isInvisibleChar(ch) && + !inertAt[i] && + !isOrthographicInvisible(chars, i, letterBefore[i], letterAfter[i], wordScripts), + ) + : []; + const hasInvisible = strayInvisible.includes(true) || strayVariation.size > 0; + if (hasInvisible) signals.push('invisible'); + + // zalgo: combining marks stacked beyond orthographic depth. Marks on a + // letter of a script that stacks them are orthography at any depth. + const orthographicMark = judged + ? chars.map((ch, i) => MARK_RE.test(ch) && isFormatCharScript(letterBefore[i])) + : []; let markRun = 0; let maxMarkRun = 0; - for (const ch of chars) { - markRun = MARK_RE.test(ch) ? markRun + 1 : 0; + for (let i = 0; i < chars.length; i += 1) { + markRun = MARK_RE.test(chars[i]!) && !orthographicMark[i] ? markRun + 1 : 0; if (markRun > maxMarkRun) maxMarkRun = markRun; } - const zalgo = maxMarkRun >= ZALGO_MARK_RUN && !scriptExempt; + const zalgo = maxMarkRun >= ZALGO_MARK_RUN; if (zalgo) signals.push('zalgo'); if (mixed) signals.push('mixed_script'); @@ -421,9 +565,8 @@ function analyzeToken( .map((ch, i) => { // Inert characters survive: they were not evidence, so removing them // would edit the message without cause. - if (dropInvisible && ((isInvisibleChar(ch) && !inertAt[i]) || strayVariation.has(i))) - return ''; - if (zalgo && MARK_RE.test(ch)) return ''; + if (dropInvisible && (strayInvisible[i] || strayVariation.has(i))) return ''; + if (zalgo && MARK_RE.test(ch) && !orthographicMark[i]) return ''; if (nfkcFold) return ch; return foldLookalikes ? foldChar(ch) : ch; }) diff --git a/test/cases.data.ts b/test/cases.data.ts index 7e1b50e..e3db55a 100644 --- a/test/cases.data.ts +++ b/test/cases.data.ts @@ -91,4 +91,25 @@ export const CASES: SpoofCase[] = [ expect: { spoofed: true }, ref: 'https://github.com/moderation-api/unicode-spoofing/issues/1', }, + { + category: 'invisible', + name: 'flags a bidi override inside a Hebrew word', + input: 'של\u202Eום', + status: 'supported', + expect: { spoofed: true, signals: { invisible: true }, normalized: 'שלום' }, + }, + { + category: 'invisible', + name: 'flags a zero-width run inside a Persian word', + input: 'می\u200B\u200B\u200B\u200B\u200Bخواهم', + status: 'supported', + expect: { spoofed: true, signals: { invisible: true }, normalized: 'میخواهم' }, + }, + { + category: 'multilingual (must NOT flag)', + name: 'leaves Persian and Hindi joiners and Thai word breaks alone', + input: 'می\u200Cخواهم ۲۰\u200Cساله، क्\u200Cष ราคา\u200B100\u200Bบาท', + status: 'supported', + expect: { spoofed: false }, + }, ]; diff --git a/test/invisible.test.ts b/test/invisible.test.ts index d93f5be..40d987f 100644 --- a/test/invisible.test.ts +++ b/test/invisible.test.ts @@ -367,3 +367,94 @@ describe('sequence exemption is not a loophole', () => { expect(analyze('a = ​= b').spoofed).toBe(true); }); }); + +describe('script exemptions are judged per letter', () => { + // Joiners and stacked marks are orthography in the FORMAT_CHAR_SCRIPTS, so + // they are exempt next to those letters — but one such letter must not + // exempt everything else hidden in its word. + it.each([ + [ + 'a bidi override inside a Hebrew word', + '\u05E9\u05DC\u202E\u05D5\u05DD', + '\u05E9\u05DC\u05D5\u05DD', + ], + [ + 'tag characters inside an Arabic word', + '\u0645\u064A\u{E0041}\u{E0042}\u062E', + '\u0645\u064A\u062E', + ], + [ + 'a zero-width run inside a Persian word', + '\u0645\u06CC\u200B\u200B\u200B\u200B\u200B\u062E\u0648\u0627\u0647\u0645', + '\u0645\u06CC\u062E\u0648\u0627\u0647\u0645', + ], + ['a tag character inside a Devanagari word', '\u0915\u{E0041}\u0937', '\u0915\u0937'], + ['a joiner between Hebrew and Latin letters', '\u05E9\u200Dx', '\u05E9x'], + ['a Hangul filler inside a Hangul word', '\uD55C\u3164\uAE00', '\uD55C\uAE00'], + [ + 'a run of bidi marks inside a Hebrew word', + '\u05E9\u05DC\u200F\u200E\u200E\u200F\u200F\u200E\u05D5\u05DD', + '\u05E9\u05DC\u05D5\u05DD', + ], + [ + 'a run of isolates inside an Arabic word', + '\u0645\u0631\u2066\u2067\u2068\u2069\u062D\u0628\u0627', + '\u0645\u0631\u062D\u0628\u0627', + ], + ])('flags %s', (_name, input, normalized) => { + const r = analyze(input); + expect(r.signals.invisible).toBe(true); + expect(r.normalized).toBe(normalized); + }); + + it('flags zalgo on a Latin letter next to a Hebrew one, keeping the Hebrew points', () => { + const r = analyze('\u05E9\u05B8a\u0301\u0302\u0303\u0304'); + expect(r.signals.zalgo).toBe(true); + expect(r.normalized).toBe('\u05E9\u05B8a'); + }); + + it.each([ + ['Malayalam chillu (ZWJ at word end)', '\u0D1E\u0D3E\u0D28\u0D4D\u200D'], + ['Thai word break (ZWSP)', '\u0E20\u0E32\u0E29\u0E32\u200B\u0E44\u0E17\u0E22'], + ['Hebrew with a right-to-left mark', '\u05E9\u05DC\u05D5\u05DD\u200F!'], + ['Arabic wrapped in isolates', '\u2067\u0645\u0631\u062D\u0628\u0627\u2069'], + ['Mongolian vowel separator', '\u1820\u180E\u1821'], + ['Syriac abbreviation mark (it opens the abbreviation)', '\u070F\u072B\u0720\u0721'], + [ + 'Biblical Hebrew point order (CGJ)', + '\u05D9\u05B0\u05E8\u05D5\u05BC\u05E9\u05B8\u034F\u05B7\u05DC\u05B4\u05DD', + ], + ['Myanmar dotted form (VS1)', '\u1000\uFE00'], + ['deep Tibetan stack', '\u0F40\u0F71\u0F72\u0F7E\u0F83'], + [ + 'Persian ZWNJ after a numeral', + '\u06F3\u200C\u0628\u0639\u062F\u06CC \u06F2\u06F0\u200C\u0633\u0627\u0644\u0647', + ], + ['Thai ZWSP around a number', '\u0E23\u0E32\u0E04\u0E32\u200B100\u200B\u0E1A\u0E32\u0E17'], + [ + 'Khmer ZWSP around a number', + '\u1786\u17D2\u1793\u17B6\u17C6\u200B\u17E2\u17E0\u17E2\u17E4\u200B\u1793\u17C1\u17C7', + ], + [ + 'Khmer ZWSP after punctuation and at a word start', + '\u1781\u17D2\u1798\u17C2\u179A\u17D4\u200B\u1797\u17B6\u179F\u17B6 \u200B\u1781\u17D2\u1798\u17C2\u179A', + ], + [ + 'Myanmar ZWSP before a number', + '\u1019\u103C\u1014\u103A\u200B\u1042\u1040\u1042\u1044\u200B\u1001\u102F', + ], + ['letter form shown with a leading ZWJ', '\u200D\u0647 \u200D\u1820'], + ['Khmer ordinal before a word break', '\u1791\u17B8\u17E4\u200B'], + ['Persian ZWNJ after a numeral at the word end', '\u0633\u0627\u0644\u06F2\u06F0\u200C'], + ['Thai ZWSP between numbers', '\u0E1B\u0E35\u200B25\u200B67'], + ['right-to-left mark next to a number', '\u200F16\u05D8\u05DF \u05D12024\u200F'], + ['marks on a tatweel', '\u0628\u0640\u064E\u064F\u0651\u0628'], + ['joiners after a tatweel', '\u0647\u0640\u200D \u0628\u0640\u200C\u0628'], + ['Kurdish final heh (ZWJ ZWNJ)', '\u0626\u0647\u200D\u200C\u0645'], + ['ligature break keeping cursive joining (ZWJ ZWNJ ZWJ)', '\u0644\u200D\u200C\u200D\u0627'], + ])('leaves %s alone', (_name, str) => { + const r = analyze(str); + expect(r.spoofed).toBe(false); + expect(r.changed).toBe(false); + }); +});