From 983162a296abde5e5e57de448e076a316e225bc1 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 16 Sep 2026 20:23:05 +0000 Subject: [PATCH] fix: count the prefix penalty in codepoints, not UTF-16 code units [patch] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CalculateScoreCore passed the raw UTF-16 offset strIdx to PenalizeNonPatternCharacters, but that formula multiplies its argument by unmatchedPrefixLetterPenalty and so means it as a count of skipped characters. Since the codepoint-aware refactor the loop advances strIdx by 1 or 2 code units per iteration, so a supplementary-plane prefix was penalized twice over: "😁😁😁y" matching "y" scored -8 where the character-count-equivalent "abcy" scored -6. Track a codepoint counter alongside strIdx and pass that instead, and rename the parameter to precedingCodepointCount so the contract states which unit it wants. Fixes #77 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QGbFXFrsMi1r2B64mxE2mp --- FuzzySearch.Test/FuzzyTests.cs | 31 +++++++++++++++++++++++++++++++ FuzzySearch/Fuzzy.cs | 16 ++++++++++++---- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/FuzzySearch.Test/FuzzyTests.cs b/FuzzySearch.Test/FuzzyTests.cs index ac772fe..178a0ae 100644 --- a/FuzzySearch.Test/FuzzyTests.cs +++ b/FuzzySearch.Test/FuzzyTests.cs @@ -606,6 +606,37 @@ public void Contains_SurrogatePairPattern_DoesNotMatchADifferentSurrogatePair() Assert.IsFalse(result, "Two supplementary-plane characters sharing a high surrogate are still different characters."); } + [TestMethod] + public void Contains_WithScore_SupplementaryPlanePrefix_PenalizedPerCharacterNotPerCodeUnit() + { + // Arrange: both subjects have exactly three unmatched characters before the match, but the emoji prefix + // occupies six UTF-16 code units rather than three. + string emojiSubject = $"{Emoji}{Emoji}{Emoji}y"; + string asciiSubject = "abcy"; + + // Act + Fuzzy.Contains(emojiSubject, "y", out int emojiScore); + Fuzzy.Contains(asciiSubject, "y", out int asciiScore); + + // Assert + Assert.AreEqual(asciiScore, emojiScore, "The prefix penalty should count skipped characters, not their UTF-16 width."); + } + + [TestMethod] + public void Contains_WithScore_SupplementaryPlanePrefix_DoesNotExceedTheUncappedPenalty() + { + // Arrange: two skipped codepoints occupy four UTF-16 code units, which is still under the penalty cap, + // so a code-unit count would show through as a larger penalty rather than being hidden by the cap. + string emojiSubject = $"{Emoji}{Emoji}y"; + + // Act + Fuzzy.Contains(emojiSubject, "y", out int emojiScore); + Fuzzy.Contains("aby", "y", out int asciiScore); + + // Assert + Assert.AreEqual(asciiScore, emojiScore, "Two skipped supplementary-plane characters should cost the same as two skipped ASCII characters."); + } + #endregion #region Integration Tests diff --git a/FuzzySearch/Fuzzy.cs b/FuzzySearch/Fuzzy.cs index e9d3b22..95f13b4 100644 --- a/FuzzySearch/Fuzzy.cs +++ b/FuzzySearch/Fuzzy.cs @@ -171,6 +171,10 @@ internal static int CalculateScoreCore(ReadOnlySpan subject, ReadOnlySpan< int patternLength = pattern.Length; int strIdx = 0; int strLength = subject.Length; + + // Counts codepoints rather than code units, so the prefix penalty reflects skipped characters and not the + // UTF-16 width of those characters. + int strCodepointIdx = 0; bool prevMatched = false; bool prevLower = false; bool prevSeparator = true; // true if first letter match gets separator bonus @@ -210,7 +214,7 @@ internal static int CalculateScoreCore(ReadOnlySpan subject, ReadOnlySpan< { int newScore = 0; - score = PenalizeNonPatternCharacters(score, patternIdx, strIdx); + score = PenalizeNonPatternCharacters(score, patternIdx, strCodepointIdx); newScore = ApplyBonuses(prevMatched, prevLower, prevSeparator, strChar, strLower, strUpper, newScore); @@ -249,6 +253,7 @@ internal static int CalculateScoreCore(ReadOnlySpan subject, ReadOnlySpan< prevSeparator = strChar is '_' or ' '; strIdx += strCharLength; + strCodepointIdx++; } // Apply score for last match @@ -390,15 +395,18 @@ internal static int ApplyBonuses(bool prevMatched, bool prevLower, bool prevSepa /// /// The current score to apply penalties to. /// The current index in the pattern. - /// The current index in the subject span. + /// + /// The number of subject codepoints preceding the match. This is a count of characters, not of UTF-16 code units, + /// so a supplementary-plane prefix is penalized the same as a basic-plane prefix of the same length. + /// /// The updated score after applying any applicable penalties. - internal static int PenalizeNonPatternCharacters(int score, int patternIdx, int strIdx) + internal static int PenalizeNonPatternCharacters(int score, int patternIdx, int precedingCodepointCount) { // Apply penalty for each letter before the first pattern match // Note: Math.Max because penalties are negative values. So max is smallest penalty. if (patternIdx == 0) { - int penalty = Math.Max(strIdx * unmatchedPrefixLetterPenalty, maxPrefixPenalty); + int penalty = Math.Max(precedingCodepointCount * unmatchedPrefixLetterPenalty, maxPrefixPenalty); score += penalty; }