diff --git a/FuzzySearch.Test/FuzzySearch.Test.csproj b/FuzzySearch.Test/FuzzySearch.Test.csproj index 753be6e..15bac1b 100644 --- a/FuzzySearch.Test/FuzzySearch.Test.csproj +++ b/FuzzySearch.Test/FuzzySearch.Test.csproj @@ -6,6 +6,8 @@ true net10.0 + + false diff --git a/FuzzySearch.Test/FuzzyTests.cs b/FuzzySearch.Test/FuzzyTests.cs index 97f6511..be20ff2 100644 --- a/FuzzySearch.Test/FuzzyTests.cs +++ b/FuzzySearch.Test/FuzzyTests.cs @@ -451,6 +451,70 @@ public void CalculateScore_EmptyPattern_ZeroScoreAndPatternPresent() #endregion + #region Unicode Normalization Tests + + // "café" precomposed (NFC): the accented letter is a single code point U+00E9. + private const string PrecomposedCafe = "café"; + + // "café" decomposed (NFD): a plain 'e' followed by U+0301 COMBINING ACUTE ACCENT. + private const string DecomposedCafe = "café"; + + [TestMethod] + public void Contains_DecomposedSubject_MatchesPrecomposedPattern() + { + // Act + bool result = Fuzzy.Contains(DecomposedCafe, PrecomposedCafe); + + // Assert + Assert.IsTrue(result, "A decomposed (NFD) subject should match its canonically equivalent precomposed (NFC) pattern."); + } + + [TestMethod] + public void Contains_PrecomposedSubject_MatchesDecomposedPattern() + { + // Act + bool result = Fuzzy.Contains(PrecomposedCafe, DecomposedCafe); + + // Assert + Assert.IsTrue(result, "A precomposed (NFC) subject should match its canonically equivalent decomposed (NFD) pattern."); + } + + [TestMethod] + public void Contains_WithScore_CanonicallyEquivalentForms_ScoreIdentically() + { + // Act + Fuzzy.Contains(PrecomposedCafe, PrecomposedCafe, out int precomposedScore); + Fuzzy.Contains(DecomposedCafe, PrecomposedCafe, out int decomposedScore); + + // Assert + Assert.AreEqual(precomposedScore, decomposedScore, "Canonically equivalent text should produce the same score."); + } + + [TestMethod] + public void Contains_DecomposedSubject_StillRejectsNonMatchingPattern() + { + // Act + bool result = Fuzzy.Contains(DecomposedCafe, "zzz"); + + // Assert + Assert.IsFalse(result, "Normalization should not turn a non-match into a match."); + } + + [TestMethod] + public void Contains_MalformedUnicode_ComparesAsGivenWithoutThrowing() + { + // Arrange: a lone high surrogate is not well-formed Unicode and cannot be normalized. + string subject = "caf\uD83D"; + + // Act + bool result = Fuzzy.Contains(subject, "caf"); + + // Assert + Assert.IsTrue(result, "Text that cannot be normalized should still be matched as it was given."); + } + + #endregion + #region Integration Tests [TestMethod] diff --git a/FuzzySearch/Fuzzy.cs b/FuzzySearch/Fuzzy.cs index 6cfd9ea..2e1edfa 100644 --- a/FuzzySearch/Fuzzy.cs +++ b/FuzzySearch/Fuzzy.cs @@ -1,6 +1,7 @@ // Copyright (c) 2023-2026 ktsu-dev contributors using System.Runtime.CompilerServices; +using System.Text; [assembly: InternalsVisibleTo("ktsu.FuzzySearch.Test")] @@ -12,8 +13,16 @@ namespace ktsu.FuzzySearch; /// Provides fuzzy string matching capabilities, allowing for approximate string matching and scoring. /// /// +/// /// This implementation uses a scoring system that rewards consecutive matches, matches after separator characters, /// and matches across camelCase boundaries, while penalizing unmatched characters. +/// +/// +/// Input is normalized to before matching, so canonically equivalent text +/// matches whether it is precomposed (NFC) or decomposed (NFD). Normalization relies on the runtime's globalization +/// data: in an application running with invariant globalization enabled, +/// is a no-op and the two forms of the same text will not match. +/// /// public static class Fuzzy { @@ -43,7 +52,41 @@ public static class Fuzzy /// /// true if the subject contains all characters from the pattern in sequence, or the pattern is empty and the subject is not; otherwise, false. /// - public static bool Contains(ReadOnlySpan subject, ReadOnlySpan pattern) + /// + /// Subject and pattern are normalized to before comparison, so canonically + /// equivalent text matches regardless of whether it is precomposed (NFC) or decomposed (NFD). + /// + public static bool Contains(ReadOnlySpan subject, ReadOnlySpan pattern) => + ContainsCore(NormalizeForComparison(subject), NormalizeForComparison(pattern)); + + /// + /// Determines whether the specified subject contains all characters from the pattern in sequence, and calculates a match score. + /// + /// The span of characters to search within. + /// The sequence of characters to search for. + /// + /// When this method returns, contains the calculated match score if the pattern is found; otherwise, + /// the score reflects how close the match was. + /// + /// + /// true if the subject contains all characters from the pattern in sequence, or the pattern is empty and the subject is not; otherwise, false. + /// + public static bool Contains(ReadOnlySpan subject, ReadOnlySpan pattern, out int outScore) + { + outScore = CalculateScore(subject, pattern, out bool wholePatternPresent); + return wholePatternPresent; + } + + /// + /// Determines whether the specified subject contains all characters from the pattern in sequence, assuming both + /// spans are already normalized to a common form. + /// + /// The span of characters to search within. + /// The sequence of characters to search for. + /// + /// true if the subject contains all characters from the pattern in sequence, or the pattern is empty and the subject is not; otherwise, false. + /// + internal static bool ContainsCore(ReadOnlySpan subject, ReadOnlySpan pattern) { if (pattern.IsEmpty) { @@ -69,25 +112,25 @@ public static bool Contains(ReadOnlySpan subject, ReadOnlySpan patte } /// - /// Determines whether the specified subject contains all characters from the pattern in sequence, and calculates a match score. + /// Calculates a fuzzy match score between the subject span and pattern span. /// /// The span of characters to search within. /// The sequence of characters to search for. - /// - /// When this method returns, contains the calculated match score if the pattern is found; otherwise, - /// the score reflects how close the match was. + /// + /// When this method returns, contains true if the entire pattern was found in the subject, + /// or the pattern is empty and the subject is not; otherwise, false. /// - /// - /// true if the subject contains all characters from the pattern in sequence, or the pattern is empty and the subject is not; otherwise, false. - /// - public static bool Contains(ReadOnlySpan subject, ReadOnlySpan pattern, out int outScore) - { - outScore = CalculateScore(subject, pattern, out bool wholePatternPresent); - return wholePatternPresent; - } + /// A score representing the quality of the match. Higher scores indicate better matches. + /// + /// Subject and pattern are normalized to before comparison, so canonically + /// equivalent text matches regardless of whether it is precomposed (NFC) or decomposed (NFD). + /// + internal static int CalculateScore(ReadOnlySpan subject, ReadOnlySpan pattern, out bool wholePatternIsPresent) => + CalculateScoreCore(NormalizeForComparison(subject), NormalizeForComparison(pattern), out wholePatternIsPresent); /// - /// Calculates a fuzzy match score between the subject span and pattern span. + /// Calculates a fuzzy match score between the subject span and pattern span, assuming both spans are already + /// normalized to a common form. /// /// The span of characters to search within. /// The sequence of characters to search for. @@ -96,7 +139,7 @@ public static bool Contains(ReadOnlySpan subject, ReadOnlySpan patte /// or the pattern is empty and the subject is not; otherwise, false. /// /// A score representing the quality of the match. Higher scores indicate better matches. - internal static int CalculateScore(ReadOnlySpan subject, ReadOnlySpan pattern, out bool wholePatternIsPresent) + internal static int CalculateScoreCore(ReadOnlySpan subject, ReadOnlySpan pattern, out bool wholePatternIsPresent) { if (pattern.IsEmpty) { @@ -199,6 +242,54 @@ internal static int CalculateScore(ReadOnlySpan subject, ReadOnlySpan + /// Normalizes a span to so that canonically equivalent text compares equal. + /// + /// The span to normalize. + /// + /// The normalized text, or itself when it is already in that form or cannot be normalized. + /// + /// + /// ASCII-only text is always already in , so the common case is served by a + /// scan that allocates nothing. Only text containing a non-ASCII character pays for the conversion. + /// + internal static ReadOnlySpan NormalizeForComparison(ReadOnlySpan value) + { + if (IsAscii(value)) + { + return value; + } + + try + { + return value.ToString().Normalize(NormalizationForm.FormC).AsSpan(); + } + catch (ArgumentException) + { + // Text that is not well-formed Unicode (for example a lone surrogate) cannot be normalized. + // Compare it as it was given rather than failing the match outright. + return value; + } + } + + /// + /// Determines whether every character in the span is an ASCII character. + /// + /// The span to inspect. + /// true if the span contains only ASCII characters; otherwise, false. + internal static bool IsAscii(ReadOnlySpan value) + { + foreach (char c in value) + { + if (c > 0x7F) + { + return false; + } + } + + return true; + } + /// /// Applies bonus scores for various match characteristics. ///