diff --git a/FuzzySearch.Test/FuzzyTests.cs b/FuzzySearch.Test/FuzzyTests.cs index e31c606..f880d21 100644 --- a/FuzzySearch.Test/FuzzyTests.cs +++ b/FuzzySearch.Test/FuzzyTests.cs @@ -583,6 +583,46 @@ public void Contains_MalformedUnicode_ComparesAsGivenWithoutThrowing() Assert.IsTrue(result, "Text that cannot be normalized should still be matched as it was given."); } + [TestMethod] + public void Contains_DecomposedSubjectWithLoneSurrogate_StillMatchesPrecomposedPattern() + { + // Act + bool result = Fuzzy.Contains(DecomposedCafe + " \uD800", PrecomposedCafe); + + // Assert + Assert.IsTrue(result, "An unrelated lone surrogate must not switch off normalization for the rest of the subject."); + } + + [TestMethod] + public void Contains_WithScore_DecomposedSubjectWithLoneSurrogate_StillMatchesPrecomposedPattern() + { + // Act + bool result = Fuzzy.Contains(DecomposedCafe + " \uD800", PrecomposedCafe, out _); + + // Assert + Assert.IsTrue(result, "The scoring overload must agree that the normalized text is present."); + } + + [TestMethod] + public void Contains_PatternWithLoneSurrogate_IsStillNormalized() + { + // Act + bool result = Fuzzy.Contains(PrecomposedCafe + " \uDC00", DecomposedCafe + " \uDC00"); + + // Assert + Assert.IsTrue(result, "A lone surrogate in the pattern must not switch off normalization for the rest of it."); + } + + [TestMethod] + public void NormalizeForComparison_LoneSurrogate_KeepsItInPlaceAndNormalizesAroundIt() + { + // Act + string result = Fuzzy.NormalizeForComparison(DecomposedCafe + "\uD800" + DecomposedCafe + "\uD83D\uDE01").ToString(); + + // Assert + Assert.AreEqual(PrecomposedCafe + "\uD800" + PrecomposedCafe + "\uD83D\uDE01", result); + } + #endregion #region Surrogate Pair Tests diff --git a/FuzzySearch/Fuzzy.cs b/FuzzySearch/Fuzzy.cs index b62af59..bfef38d 100644 --- a/FuzzySearch/Fuzzy.cs +++ b/FuzzySearch/Fuzzy.cs @@ -359,9 +359,56 @@ internal static ReadOnlySpan NormalizeForComparison(ReadOnlySpan val } catch (ArgumentException) { - // Text that is not well-formed Unicode (for example a lone surrogate) cannot be normalized. - // Compare it as it was given rather than failing the match outright. - return value; + // Text that is not well-formed Unicode (for example a lone surrogate) cannot be normalized as a + // whole. Normalize the well-formed text around it instead, so one stray surrogate does not + // switch off canonical equivalence for the rest of the string. + return NormalizeWellFormedRuns(value); + } + } + + /// + /// Normalizes each well-formed run of to , leaving + /// every unpaired surrogate where it was. + /// + /// Text containing at least one unpaired surrogate. + /// The text with its well-formed runs normalized. + /// + /// An unpaired surrogate has no canonical composition with its neighbours, so treating it as a run boundary gives + /// the same result as normalizing the valid text on either side of it. + /// + private static ReadOnlySpan NormalizeWellFormedRuns(ReadOnlySpan value) + { + StringBuilder builder = new(value.Length); + int runStart = 0; + + for (int i = 0; i < value.Length; i++) + { + char c = value[i]; + if (!char.IsSurrogate(c)) + { + continue; + } + + if (char.IsHighSurrogate(c) && i + 1 < value.Length && char.IsLowSurrogate(value[i + 1])) + { + i++; + continue; + } + + AppendNormalized(builder, value[runStart..i]); + builder.Append(c); + runStart = i + 1; + } + + AppendNormalized(builder, value[runStart..]); + return builder.ToString().AsSpan(); + + static void AppendNormalized(StringBuilder builder, ReadOnlySpan run) + { + if (!run.IsEmpty) + { + builder.Append(run.ToString().Normalize(NormalizationForm.FormC)); + } } }