From 618199701e61e9915ef80142f981056d14b9cc51 Mon Sep 17 00:00:00 2001 From: cycanic <79452786+cycanic@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:50:24 +0400 Subject: [PATCH] fix: keep the curly apostrophe when folding a lookalike word MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fold ran every character of an affected token through the confusables map, including U+2019, whose prototype is ASCII "'". So "Ivа’s" (Cyrillic а) normalized to "Iva's" instead of "Iva’s". The apostrophe is punctuation the token admits, not a lookalike; leave it as typed. Detection is unchanged. Co-Authored-By: Claude Opus 5.5 --- .changeset/keep-curly-apostrophe.md | 11 +++++++++++ src/analyze.ts | 3 +++ test/analyze.test.ts | 8 ++++++++ test/cases.data.ts | 7 +++++++ 4 files changed, 29 insertions(+) create mode 100644 .changeset/keep-curly-apostrophe.md diff --git a/.changeset/keep-curly-apostrophe.md b/.changeset/keep-curly-apostrophe.md new file mode 100644 index 0000000..5b97fd5 --- /dev/null +++ b/.changeset/keep-curly-apostrophe.md @@ -0,0 +1,11 @@ +--- +'@moderation-api/unicode-spoofing': patch +--- + +Keep the typographic apostrophe in `normalized` when a lookalike word is folded. + +`Ivа’s` (with a Cyrillic `а`) normalized to `Iva's`: the fold ran every +character of the word through the confusables map, including U+2019, whose +prototype is the ASCII `'`. Only the letters are lookalikes, so the apostrophe +is now kept as typed and the word normalizes to `Iva’s`. Detection is +unchanged. diff --git a/src/analyze.ts b/src/analyze.ts index cccee8e..3b2a83a 100644 --- a/src/analyze.ts +++ b/src/analyze.ts @@ -425,6 +425,9 @@ function analyzeToken( return ''; if (zalgo && MARK_RE.test(ch)) return ''; if (nfkcFold) return ch; + // The apostrophe is punctuation the token admits, not a lookalike: + // folding it would turn "Ivа’s" into "Iva's" rather than "Iva’s". + if (ch === '’') return ch; return foldLookalikes ? foldChar(ch) : ch; }) .join(''); diff --git a/test/analyze.test.ts b/test/analyze.test.ts index 4ad838f..62b8fb2 100644 --- a/test/analyze.test.ts +++ b/test/analyze.test.ts @@ -133,6 +133,14 @@ describe('analyze — legitimate content', () => { expect(r.signals.mixed_script).toBe(true); }); + it('keeps the curly apostrophe when it folds a lookalike word', () => { + // Only the letters are lookalikes; the apostrophe is the author's own + // punctuation and survives normalization as typed. + expect(analyze('Ivа’s').normalized).toBe('Iva’s'); + expect(analyze("Ivа's").normalized).toBe("Iva's"); + expect(analyze('the boss’ НОТ’ deal').normalized).toBe('the boss’ HOT’ deal'); + }); + // A word already written in Latin cannot be impersonating Latin, so the only // thing the skeleton test measures there is whether the fold happens to reach // ASCII — which it does for ordinary European letters, and arbitrarily: "æ" diff --git a/test/cases.data.ts b/test/cases.data.ts index 7e1b50e..cfd1473 100644 --- a/test/cases.data.ts +++ b/test/cases.data.ts @@ -64,6 +64,13 @@ export interface SpoofCase { } export const CASES: SpoofCase[] = [ + { + category: 'confusable', + name: 'keeps the curly apostrophe of a folded word (Ivа’s → Iva’s)', + input: 'Ivа’s', + status: 'supported', + expect: { spoofed: true, signals: { mixed_script: true }, normalized: 'Iva’s' }, + }, { category: 'confusable', name: 'flags a Cyrillic whole-word lookalike and normalizes it (НОТ → HOT)',