diff --git a/.changeset/keep-curly-apostrophe.md b/.changeset/keep-curly-apostrophe.md new file mode 100644 index 0000000..5b97fd5 --- /dev/null +++ b/.changeset/keep-curly-apostrophe.md @@ -0,0 +1,11 @@ +--- +'@moderation-api/unicode-spoofing': patch +--- + +Keep the typographic apostrophe in `normalized` when a lookalike word is folded. + +`Ivа’s` (with a Cyrillic `а`) normalized to `Iva's`: the fold ran every +character of the word through the confusables map, including U+2019, whose +prototype is the ASCII `'`. Only the letters are lookalikes, so the apostrophe +is now kept as typed and the word normalizes to `Iva’s`. Detection is +unchanged. diff --git a/src/analyze.ts b/src/analyze.ts index cccee8e..3b2a83a 100644 --- a/src/analyze.ts +++ b/src/analyze.ts @@ -425,6 +425,9 @@ function analyzeToken( return ''; if (zalgo && MARK_RE.test(ch)) return ''; if (nfkcFold) return ch; + // The apostrophe is punctuation the token admits, not a lookalike: + // folding it would turn "Ivа’s" into "Iva's" rather than "Iva’s". + if (ch === '’') return ch; return foldLookalikes ? foldChar(ch) : ch; }) .join(''); diff --git a/test/analyze.test.ts b/test/analyze.test.ts index 4ad838f..62b8fb2 100644 --- a/test/analyze.test.ts +++ b/test/analyze.test.ts @@ -133,6 +133,14 @@ describe('analyze — legitimate content', () => { expect(r.signals.mixed_script).toBe(true); }); + it('keeps the curly apostrophe when it folds a lookalike word', () => { + // Only the letters are lookalikes; the apostrophe is the author's own + // punctuation and survives normalization as typed. + expect(analyze('Ivа’s').normalized).toBe('Iva’s'); + expect(analyze("Ivа's").normalized).toBe("Iva's"); + expect(analyze('the boss’ НОТ’ deal').normalized).toBe('the boss’ HOT’ deal'); + }); + // A word already written in Latin cannot be impersonating Latin, so the only // thing the skeleton test measures there is whether the fold happens to reach // ASCII — which it does for ordinary European letters, and arbitrarily: "æ" diff --git a/test/cases.data.ts b/test/cases.data.ts index 7e1b50e..cfd1473 100644 --- a/test/cases.data.ts +++ b/test/cases.data.ts @@ -64,6 +64,13 @@ export interface SpoofCase { } export const CASES: SpoofCase[] = [ + { + category: 'confusable', + name: 'keeps the curly apostrophe of a folded word (Ivа’s → Iva’s)', + input: 'Ivа’s', + status: 'supported', + expect: { spoofed: true, signals: { mixed_script: true }, normalized: 'Iva’s' }, + }, { category: 'confusable', name: 'flags a Cyrillic whole-word lookalike and normalizes it (НОТ → HOT)',