diff --git a/Sorting.Test/NaturalStringComparerTests.cs b/Sorting.Test/NaturalStringComparerTests.cs index da415ed..a89de38 100644 --- a/Sorting.Test/NaturalStringComparerTests.cs +++ b/Sorting.Test/NaturalStringComparerTests.cs @@ -229,4 +229,72 @@ public void Compare_WithMixedNumbersAndText_SortsCorrectly() CollectionAssert.AreEqual(sorted, unsorted); } + + [TestMethod] + public void Compare_NonAsciiDigitAgainstText_OrdersLikeTheAsciiDigit() + { + // A digit chunk against a text chunk used to compare code points, so the Arabic-Indic five + // sorted after letters while the ASCII five, which it equals, sorted before them. + Assert.IsLessThan(0, _comparer.Compare("\u0665", "10")); + Assert.IsLessThan(0, _comparer.Compare("10", "z")); + Assert.IsLessThan(0, _comparer.Compare("\u0665", "z")); + Assert.IsGreaterThan(0, _comparer.Compare("z", "\u0665")); + Assert.IsGreaterThan(0, _comparer.Compare("\u0665", "-")); + Assert.IsGreaterThan(0, _comparer.Compare("5", "-")); + Assert.IsLessThan(0, _comparer.Compare("\u0665", "~")); + Assert.IsLessThan(0, _comparer.Compare("5", "~")); + } + + [TestMethod] + public void Compare_SortOfMixedScriptDigitsAndText_DoesNotDependOnInputOrder() + { + string[] expected = ["\u0665", "10", "z"]; + + string[] forward = ["\u0665", "10", "z"]; + string[] backward = ["z", "10", "\u0665"]; + Array.Sort(forward, _comparer); + Array.Sort(backward, _comparer); + + CollectionAssert.AreEqual(expected, forward); + CollectionAssert.AreEqual(expected, backward); + } + + [TestMethod] + public void Compare_IsTransitiveOverMixedScriptDigitsAndText() + { + string[] values = + [ + "", "0", "5", "05", "10", "\u0665", "\u0661\u0660", "\U0001D7D7", "\U0001D7CF\U0001D7CE", + "\u0969", "a", "z", "Z", " ", "-", "_", "~", "\u00e9", "\U0001F600", + "a5", "a\u0665", "a10", "a-", "a z", "5a", "\u0665a", "10a", "-5", "~5", + ]; + + foreach (string a in values) + { + foreach (string b in values) + { + Assert.AreEqual(Math.Sign(_comparer.Compare(a, b)), -Math.Sign(_comparer.Compare(b, a)), $"antisymmetry: '{a}' vs '{b}'"); + + foreach (string c in values) + { + int ab = Math.Sign(_comparer.Compare(a, b)); + int bc = Math.Sign(_comparer.Compare(b, c)); + if (ab <= 0 && bc <= 0) + { + Assert.IsLessThanOrEqualTo(0, _comparer.Compare(a, c), $"transitivity: '{a}' <= '{b}' <= '{c}'"); + } + } + } + } + } + + [TestMethod] + public void Compare_DigitsOutsideTheBasicMultilingualPlane_ComparedByNumericValue() + { + // Mathematical bold digits nine, and one followed by zero, are surrogate pairs in UTF-16. + Assert.IsLessThan(0, _comparer.Compare("file\U0001D7D7", "file\U0001D7CF\U0001D7CE")); + Assert.IsLessThan(0, _comparer.Compare("file\U0001D7D7", "file10")); + Assert.IsGreaterThan(0, _comparer.Compare("file\U0001D7D7", "file8")); + Assert.AreEqual(0, _comparer.Compare("file\U0001D7D7", "file9")); + } } diff --git a/Sorting/NaturalStringComparer.cs b/Sorting/NaturalStringComparer.cs index b884e81..108c6f6 100644 --- a/Sorting/NaturalStringComparer.cs +++ b/Sorting/NaturalStringComparer.cs @@ -3,23 +3,19 @@ namespace ktsu.Sorting; using System.Globalization; -using System.Text.RegularExpressions; +using System.Text; /// /// Comparer that performs a natural comparison between strings, correctly comparing embedded numbers. /// /// /// A run of Unicode decimal digits (category Nd) is a number, whatever script it is written -/// in, and is compared by the value it spells rather than by its code points. +/// in and whether or not it lies outside the Basic Multilingual Plane, and is compared by the value +/// it spells rather than by its code points. A number compared with text orders exactly as the +/// equivalent ASCII digits would, so every digit script sorts in the same place relative to text. /// public partial class NaturalStringComparer : IComparer { - /// - /// Regular expression to match alphanumeric chunks in a string. - /// - [System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "SYSLIB1045:Convert to 'GeneratedRegexAttribute'.", Justification = "")] - private static Regex CreateNaturalChunkRegex() => new(@"(\d+)|(\D+)"); - /// /// Compares two strings using natural sorting, where embedded numbers are compared as numeric values. /// @@ -52,20 +48,19 @@ public int Compare(string? x, string? y) return 0; } - Regex regex = CreateNaturalChunkRegex(); - Match[] xMatches = [.. regex.Matches(x).Cast()]; - Match[] yMatches = [.. regex.Matches(y).Cast()]; + List xChunks = SplitIntoChunks(x); + List yChunks = SplitIntoChunks(y); int i = 0, j = 0; - while (i < xMatches.Length && j < yMatches.Length) + while (i < xChunks.Count && j < yChunks.Count) { - Match xMatch = xMatches[i++]; - Match yMatch = yMatches[j++]; + Chunk xChunk = xChunks[i++]; + Chunk yChunk = yChunks[j++]; // If both chunks are numeric, compare them as numbers - if (char.IsDigit(xMatch.Value[0]) && char.IsDigit(yMatch.Value[0])) + if (xChunk.IsNumeric && yChunk.IsNumeric) { - int numComparison = CompareNumericChunks(xMatch.Value, yMatch.Value); + int numComparison = CompareNumericChunks(xChunk.Value, yChunk.Value); if (numComparison != 0) { return numComparison; @@ -73,7 +68,11 @@ public int Compare(string? x, string? y) } else // Otherwise, compare them as strings { - int stringComparison = string.Compare(xMatch.Value, yMatch.Value, StringComparison.Ordinal); + // A numeric chunk holds ASCII digits and a text chunk never starts with a digit, so a + // number against text is decided by the first character, the same way for every + // digit script. Comparing a non-ASCII digit's own code point here would sort it after + // letters while its ASCII equal sorts before them, which makes the order intransitive. + int stringComparison = string.Compare(xChunk.Value, yChunk.Value, StringComparison.Ordinal); if (stringComparison != 0) { return stringComparison; @@ -82,7 +81,54 @@ public int Compare(string? x, string? y) } // If we've exhausted one sequence but not the other, the shorter one comes first - return xMatches.Length.CompareTo(yMatches.Length); + return xChunks.Count.CompareTo(yChunks.Count); + } + + /// + /// Splits a string into alternating runs of decimal digits and of other text, walking it by code + /// point so that a digit encoded as a surrogate pair still counts as a digit. + /// + /// + /// A numeric chunk's value is rewritten in ASCII digits, so that numbers from every script compare + /// alike both with each other and with text. A text chunk keeps its original characters. + /// + /// The string to split. + /// The chunks of , in order. + private static List SplitIntoChunks(string value) + { + List chunks = []; + StringBuilder current = new(); + bool currentIsNumeric = false; + int index = 0; + while (index < value.Length) + { + bool isDigit = CharUnicodeInfo.GetUnicodeCategory(value, index) == UnicodeCategory.DecimalDigitNumber; + if (current.Length > 0 && isDigit != currentIsNumeric) + { + chunks.Add(new Chunk(currentIsNumeric, current.ToString())); + current.Clear(); + } + + currentIsNumeric = isDigit; + int width = char.IsSurrogatePair(value, index) ? 2 : 1; + if (isDigit) + { + current.Append((char)('0' + CharUnicodeInfo.GetDecimalDigitValue(value, index))); + } + else + { + current.Append(value, index, width); + } + + index += width; + } + + if (current.Length > 0) + { + chunks.Add(new Chunk(currentIsNumeric, current.ToString())); + } + + return chunks; } /// @@ -90,11 +136,10 @@ public int Compare(string? x, string? y) /// code points. /// /// - /// Both chunks come from the \d+ alternative of the chunk regex, so every character is a - /// Unicode decimal digit (category Nd) and has a decimal value of 0-9. Comparing those - /// values, rather than the raw UTF-16 code points, is what keeps non-ASCII digit scripts - /// ordering by magnitude: '٥' (Arabic-Indic five) is numerically less than '9', - /// even though its code point is far greater. + /// Both chunks come from , which has already rewritten every Unicode + /// decimal digit (category Nd) as the ASCII digit of the same value. That is what keeps + /// non-ASCII digit scripts ordering by magnitude: '٥' (Arabic-Indic five) is numerically + /// less than '9', even though its code point is far greater. /// /// First digit chunk to compare. /// Second digit chunk to compare. @@ -116,7 +161,7 @@ private static int CompareNumericChunks(string xChunk, string yChunk) // Same digit count, so the first differing digit decides for (int offset = 0; offset < xDigits; offset++) { - int digitComparison = DigitValue(xChunk[xStart + offset]).CompareTo(DigitValue(yChunk[yStart + offset])); + int digitComparison = xChunk[xStart + offset].CompareTo(yChunk[yStart + offset]); if (digitComparison != 0) { return digitComparison; @@ -136,7 +181,7 @@ private static int CompareNumericChunks(string xChunk, string yChunk) private static int SkipLeadingZeros(string chunk) { int index = 0; - while (index < chunk.Length - 1 && DigitValue(chunk[index]) == 0) + while (index < chunk.Length - 1 && chunk[index] == '0') { index++; } @@ -145,10 +190,9 @@ private static int SkipLeadingZeros(string chunk) } /// - /// Returns the decimal value of a Unicode decimal digit, so that digits from any script compare - /// by magnitude. + /// A run of decimal digits, held as ASCII digits, or a run of other text. /// - /// The digit character, which the chunk regex guarantees is category Nd. - /// The digit's value of 0-9. - private static int DigitValue(char digit) => CharUnicodeInfo.GetDecimalDigitValue(digit); + /// Whether the chunk is a run of decimal digits. + /// The chunk's ASCII digits when numeric, otherwise its original text. + private readonly record struct Chunk(bool IsNumeric, string Value); }