From c768cb8005b5f853f780f5b43d71bb580339fe7e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Sep 2026 13:34:38 +0000 Subject: [PATCH 1/2] fix: match canonically equivalent NFC and NFD text [patch] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fuzzy.Contains and CalculateScore compared subject and pattern per UTF-16 char with no Unicode normalization, so precomposed text (e.g. "café" with U+00E9) never matched its canonically equivalent decomposed form ("cafe" + U+0301). NFD is the normal form for filenames on HFS+/APFS, so consumers matching against such paths silently missed matches for visibly identical text. Both inputs are now normalized to FormC before matching. The common case costs nothing: ASCII-only text is already in FormC, so it is detected by an allocation-free scan and passed through unchanged. Only text containing a non-ASCII character pays for the conversion. Text that is not well-formed Unicode (a lone surrogate) cannot be normalized, so it is compared as given rather than throwing out of a method that previously never threw. Normalization needs the runtime's globalization data, which invariant mode does not provide, so the test project opts out of invariant globalization and the limitation is documented on the type. Fixes #70 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01W78KwkEpp9G2ZNXfm6VVqL --- FuzzySearch.Test/FuzzySearch.Test.csproj | 2 + FuzzySearch.Test/FuzzyTests.cs | 64 ++++++++++++++++ FuzzySearch/Fuzzy.cs | 95 +++++++++++++++++++++++- 3 files changed, 159 insertions(+), 2 deletions(-) diff --git a/FuzzySearch.Test/FuzzySearch.Test.csproj b/FuzzySearch.Test/FuzzySearch.Test.csproj index 753be6e..15bac1b 100644 --- a/FuzzySearch.Test/FuzzySearch.Test.csproj +++ b/FuzzySearch.Test/FuzzySearch.Test.csproj @@ -6,6 +6,8 @@ true net10.0 + + false diff --git a/FuzzySearch.Test/FuzzyTests.cs b/FuzzySearch.Test/FuzzyTests.cs index 97f6511..be20ff2 100644 --- a/FuzzySearch.Test/FuzzyTests.cs +++ b/FuzzySearch.Test/FuzzyTests.cs @@ -451,6 +451,70 @@ public void CalculateScore_EmptyPattern_ZeroScoreAndPatternPresent() #endregion + #region Unicode Normalization Tests + + // "café" precomposed (NFC): the accented letter is a single code point U+00E9. + private const string PrecomposedCafe = "café"; + + // "café" decomposed (NFD): a plain 'e' followed by U+0301 COMBINING ACUTE ACCENT. + private const string DecomposedCafe = "café"; + + [TestMethod] + public void Contains_DecomposedSubject_MatchesPrecomposedPattern() + { + // Act + bool result = Fuzzy.Contains(DecomposedCafe, PrecomposedCafe); + + // Assert + Assert.IsTrue(result, "A decomposed (NFD) subject should match its canonically equivalent precomposed (NFC) pattern."); + } + + [TestMethod] + public void Contains_PrecomposedSubject_MatchesDecomposedPattern() + { + // Act + bool result = Fuzzy.Contains(PrecomposedCafe, DecomposedCafe); + + // Assert + Assert.IsTrue(result, "A precomposed (NFC) subject should match its canonically equivalent decomposed (NFD) pattern."); + } + + [TestMethod] + public void Contains_WithScore_CanonicallyEquivalentForms_ScoreIdentically() + { + // Act + Fuzzy.Contains(PrecomposedCafe, PrecomposedCafe, out int precomposedScore); + Fuzzy.Contains(DecomposedCafe, PrecomposedCafe, out int decomposedScore); + + // Assert + Assert.AreEqual(precomposedScore, decomposedScore, "Canonically equivalent text should produce the same score."); + } + + [TestMethod] + public void Contains_DecomposedSubject_StillRejectsNonMatchingPattern() + { + // Act + bool result = Fuzzy.Contains(DecomposedCafe, "zzz"); + + // Assert + Assert.IsFalse(result, "Normalization should not turn a non-match into a match."); + } + + [TestMethod] + public void Contains_MalformedUnicode_ComparesAsGivenWithoutThrowing() + { + // Arrange: a lone high surrogate is not well-formed Unicode and cannot be normalized. + string subject = "caf\uD83D"; + + // Act + bool result = Fuzzy.Contains(subject, "caf"); + + // Assert + Assert.IsTrue(result, "Text that cannot be normalized should still be matched as it was given."); + } + + #endregion + #region Integration Tests [TestMethod] diff --git a/FuzzySearch/Fuzzy.cs b/FuzzySearch/Fuzzy.cs index 6cfd9ea..79ff288 100644 --- a/FuzzySearch/Fuzzy.cs +++ b/FuzzySearch/Fuzzy.cs @@ -1,6 +1,7 @@ // Copyright (c) 2023-2026 ktsu-dev contributors using System.Runtime.CompilerServices; +using System.Text; [assembly: InternalsVisibleTo("ktsu.FuzzySearch.Test")] @@ -12,8 +13,16 @@ namespace ktsu.FuzzySearch; /// Provides fuzzy string matching capabilities, allowing for approximate string matching and scoring. /// /// +/// /// This implementation uses a scoring system that rewards consecutive matches, matches after separator characters, /// and matches across camelCase boundaries, while penalizing unmatched characters. +/// +/// +/// Input is normalized to before matching, so canonically equivalent text +/// matches whether it is precomposed (NFC) or decomposed (NFD). Normalization relies on the runtime's globalization +/// data: in an application running with invariant globalization enabled, +/// is a no-op and the two forms of the same text will not match. +/// /// public static class Fuzzy { @@ -43,7 +52,23 @@ public static class Fuzzy /// /// true if the subject contains all characters from the pattern in sequence, or the pattern is empty and the subject is not; otherwise, false. /// - public static bool Contains(ReadOnlySpan subject, ReadOnlySpan pattern) + /// + /// Subject and pattern are normalized to before comparison, so canonically + /// equivalent text matches regardless of whether it is precomposed (NFC) or decomposed (NFD). + /// + public static bool Contains(ReadOnlySpan subject, ReadOnlySpan pattern) => + ContainsCore(NormalizeForComparison(subject), NormalizeForComparison(pattern)); + + /// + /// Determines whether the specified subject contains all characters from the pattern in sequence, assuming both + /// spans are already normalized to a common form. + /// + /// The span of characters to search within. + /// The sequence of characters to search for. + /// + /// true if the subject contains all characters from the pattern in sequence, or the pattern is empty and the subject is not; otherwise, false. + /// + internal static bool ContainsCore(ReadOnlySpan subject, ReadOnlySpan pattern) { if (pattern.IsEmpty) { @@ -96,7 +121,25 @@ public static bool Contains(ReadOnlySpan subject, ReadOnlySpan patte /// or the pattern is empty and the subject is not; otherwise, false. /// /// A score representing the quality of the match. Higher scores indicate better matches. - internal static int CalculateScore(ReadOnlySpan subject, ReadOnlySpan pattern, out bool wholePatternIsPresent) + /// + /// Subject and pattern are normalized to before comparison, so canonically + /// equivalent text matches regardless of whether it is precomposed (NFC) or decomposed (NFD). + /// + internal static int CalculateScore(ReadOnlySpan subject, ReadOnlySpan pattern, out bool wholePatternIsPresent) => + CalculateScoreCore(NormalizeForComparison(subject), NormalizeForComparison(pattern), out wholePatternIsPresent); + + /// + /// Calculates a fuzzy match score between the subject span and pattern span, assuming both spans are already + /// normalized to a common form. + /// + /// The span of characters to search within. + /// The sequence of characters to search for. + /// + /// When this method returns, contains true if the entire pattern was found in the subject, + /// or the pattern is empty and the subject is not; otherwise, false. + /// + /// A score representing the quality of the match. Higher scores indicate better matches. + internal static int CalculateScoreCore(ReadOnlySpan subject, ReadOnlySpan pattern, out bool wholePatternIsPresent) { if (pattern.IsEmpty) { @@ -199,6 +242,54 @@ internal static int CalculateScore(ReadOnlySpan subject, ReadOnlySpan + /// Normalizes a span to so that canonically equivalent text compares equal. + /// + /// The span to normalize. + /// + /// The normalized text, or itself when it is already in that form or cannot be normalized. + /// + /// + /// ASCII-only text is always already in , so the common case is served by a + /// scan that allocates nothing. Only text containing a non-ASCII character pays for the conversion. + /// + internal static ReadOnlySpan NormalizeForComparison(ReadOnlySpan value) + { + if (IsAscii(value)) + { + return value; + } + + try + { + return value.ToString().Normalize(NormalizationForm.FormC).AsSpan(); + } + catch (ArgumentException) + { + // Text that is not well-formed Unicode (for example a lone surrogate) cannot be normalized. + // Compare it as it was given rather than failing the match outright. + return value; + } + } + + /// + /// Determines whether every character in the span is an ASCII character. + /// + /// The span to inspect. + /// true if the span contains only ASCII characters; otherwise, false. + internal static bool IsAscii(ReadOnlySpan value) + { + foreach (char c in value) + { + if (c > 0x7F) + { + return false; + } + } + + return true; + } + /// /// Applies bonus scores for various match characteristics. /// From cc4ae2cd941231ffd1c9d3199ea39fba2eca14b4 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Sep 2026 13:47:21 +0000 Subject: [PATCH 2/2] fix: keep the Contains overloads adjacent [patch] Introducing ContainsCore between the two public Contains overloads split them apart, which SonarQube flags as S4136. Move ContainsCore below both. No behaviour change. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01W78KwkEpp9G2ZNXfm6VVqL --- FuzzySearch/Fuzzy.cs | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/FuzzySearch/Fuzzy.cs b/FuzzySearch/Fuzzy.cs index 79ff288..2e1edfa 100644 --- a/FuzzySearch/Fuzzy.cs +++ b/FuzzySearch/Fuzzy.cs @@ -59,6 +59,24 @@ public static class Fuzzy public static bool Contains(ReadOnlySpan subject, ReadOnlySpan pattern) => ContainsCore(NormalizeForComparison(subject), NormalizeForComparison(pattern)); + /// + /// Determines whether the specified subject contains all characters from the pattern in sequence, and calculates a match score. + /// + /// The span of characters to search within. + /// The sequence of characters to search for. + /// + /// When this method returns, contains the calculated match score if the pattern is found; otherwise, + /// the score reflects how close the match was. + /// + /// + /// true if the subject contains all characters from the pattern in sequence, or the pattern is empty and the subject is not; otherwise, false. + /// + public static bool Contains(ReadOnlySpan subject, ReadOnlySpan pattern, out int outScore) + { + outScore = CalculateScore(subject, pattern, out bool wholePatternPresent); + return wholePatternPresent; + } + /// /// Determines whether the specified subject contains all characters from the pattern in sequence, assuming both /// spans are already normalized to a common form. @@ -93,24 +111,6 @@ internal static bool ContainsCore(ReadOnlySpan subject, ReadOnlySpan return patternIdx == patternLength; } - /// - /// Determines whether the specified subject contains all characters from the pattern in sequence, and calculates a match score. - /// - /// The span of characters to search within. - /// The sequence of characters to search for. - /// - /// When this method returns, contains the calculated match score if the pattern is found; otherwise, - /// the score reflects how close the match was. - /// - /// - /// true if the subject contains all characters from the pattern in sequence, or the pattern is empty and the subject is not; otherwise, false. - /// - public static bool Contains(ReadOnlySpan subject, ReadOnlySpan pattern, out int outScore) - { - outScore = CalculateScore(subject, pattern, out bool wholePatternPresent); - return wholePatternPresent; - } - /// /// Calculates a fuzzy match score between the subject span and pattern span. ///