From 04664849b483401e5a28c0b99b3b75fe8e160308 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 21 Sep 2026 18:27:29 +0000 Subject: [PATCH] fix: compare non-ASCII digits by numeric value in NaturalStringComparer [patch] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Chunk detection used char.IsDigit and \d+, which match any Unicode Nd digit, but CompareNumericChunks then normalized with TrimStart('0') and fell back to an ordinal compare — both of which assume ASCII 0x30-0x39. A chunk containing a non-ASCII digit silently degraded to a raw UTF-16 code-point comparison, so Compare("٥", "9") ranked five above nine, contradicting the type's own documented numeric ordering. Compare digit chunks by the value each digit spells, via CharUnicodeInfo.GetDecimalDigitValue, and skip leading zeros by value rather than by the ASCII '0' character. ASCII ordering is unchanged: for equal-length ASCII chunks, comparing digit values gives the same result as the ordinal compare it replaces. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01FyZyutu7Xna2FUQK6o8KAC --- Sorting.Test/NaturalStringComparerTests.cs | 42 +++++++++++++ Sorting/NaturalStringComparer.cs | 68 ++++++++++++++++++---- 2 files changed, 100 insertions(+), 10 deletions(-) diff --git a/Sorting.Test/NaturalStringComparerTests.cs b/Sorting.Test/NaturalStringComparerTests.cs index 9ff08fa..da415ed 100644 --- a/Sorting.Test/NaturalStringComparerTests.cs +++ b/Sorting.Test/NaturalStringComparerTests.cs @@ -101,6 +101,48 @@ public void Compare_StringsWithLeadingZeros_HandledCorrectly() Assert.IsLessThan(0, _comparer.Compare("file005", "file06")); // 5 < 6 numerically } + [TestMethod] + public void Compare_NonAsciiDigits_ComparedByNumericValue() + { + // Arabic-Indic five is numerically less than ASCII nine, despite the far greater code point + Assert.IsLessThan(0, _comparer.Compare("٥", "9")); + Assert.IsGreaterThan(0, _comparer.Compare("9", "٥")); + + // The same holds within a single non-ASCII script + Assert.IsLessThan(0, _comparer.Compare("٥", "٩")); // Arabic-Indic 5 < 9 + + // Devanagari digits order by value too + Assert.IsLessThan(0, _comparer.Compare("५", "३०")); // 5 < 30 + } + + [TestMethod] + public void Compare_NonAsciiDigits_EqualValuesAreEqual() + { + // Arabic-Indic five and ASCII five spell the same number + Assert.AreEqual(0, _comparer.Compare("٥", "5")); + Assert.AreEqual(0, _comparer.Compare("file٥", "file5")); + } + + [TestMethod] + public void Compare_NonAsciiLeadingZeros_NormalizedLikeAsciiZeros() + { + // An Arabic-Indic zero is a leading zero, so both chunks reduce to the single digit 0 + Assert.AreEqual(0, _comparer.Compare("٠0", "0")); + Assert.AreEqual(0, _comparer.Compare("٠٥", "5")); + + // An all-zeros chunk compares as zero, whatever the script + Assert.AreEqual(0, _comparer.Compare("٠٠", "0")); + Assert.IsLessThan(0, _comparer.Compare("٠٠", "1")); + } + + [TestMethod] + public void Compare_MixedScriptDigits_ComparedByNumericValue() + { + // A single chunk may mix scripts; it still spells one number + Assert.IsLessThan(0, _comparer.Compare("file1٥", "file20")); // 15 < 20 + Assert.AreEqual(0, _comparer.Compare("file1٥", "file15")); + } + [TestMethod] public void Compare_DifferentLengthStrings_ShorterComesFirst() { diff --git a/Sorting/NaturalStringComparer.cs b/Sorting/NaturalStringComparer.cs index 361844d..b884e81 100644 --- a/Sorting/NaturalStringComparer.cs +++ b/Sorting/NaturalStringComparer.cs @@ -2,11 +2,16 @@ namespace ktsu.Sorting; +using System.Globalization; using System.Text.RegularExpressions; /// /// Comparer that performs a natural comparison between strings, correctly comparing embedded numbers. /// +/// +/// A run of Unicode decimal digits (category Nd) is a number, whatever script it is written +/// in, and is compared by the value it spells rather than by its code points. +/// public partial class NaturalStringComparer : IComparer { /// @@ -80,27 +85,70 @@ public int Compare(string? x, string? y) return xMatches.Length.CompareTo(yMatches.Length); } + /// + /// Compares two chunks of decimal digits by the numeric value they spell, rather than by their + /// code points. + /// + /// + /// Both chunks come from the \d+ alternative of the chunk regex, so every character is a + /// Unicode decimal digit (category Nd) and has a decimal value of 0-9. Comparing those + /// values, rather than the raw UTF-16 code points, is what keeps non-ASCII digit scripts + /// ordering by magnitude: '٥' (Arabic-Indic five) is numerically less than '9', + /// even though its code point is far greater. + /// + /// First digit chunk to compare. + /// Second digit chunk to compare. + /// A negative number, zero, or a positive number, as for . private static int CompareNumericChunks(string xChunk, string yChunk) { - string xTrimmed = xChunk.TrimStart('0'); - string yTrimmed = yChunk.TrimStart('0'); + int xStart = SkipLeadingZeros(xChunk); + int yStart = SkipLeadingZeros(yChunk); - if (xTrimmed.Length == 0) + // With leading zeros gone, the chunk spelling more digits is the larger number + int xDigits = xChunk.Length - xStart; + int yDigits = yChunk.Length - yStart; + int lengthComparison = xDigits.CompareTo(yDigits); + if (lengthComparison != 0) { - xTrimmed = "0"; + return lengthComparison; } - if (yTrimmed.Length == 0) + // Same digit count, so the first differing digit decides + for (int offset = 0; offset < xDigits; offset++) { - yTrimmed = "0"; + int digitComparison = DigitValue(xChunk[xStart + offset]).CompareTo(DigitValue(yChunk[yStart + offset])); + if (digitComparison != 0) + { + return digitComparison; + } } - int lengthComparison = xTrimmed.Length.CompareTo(yTrimmed.Length); - if (lengthComparison != 0) + return 0; + } + + /// + /// Returns the index of the first digit in that is not a zero, or + /// chunk.Length - 1 when the chunk is all zeros, so a chunk of zeros compares as a + /// single zero digit. + /// + /// The digit chunk to scan. + /// The index at which the chunk's significant digits begin. + private static int SkipLeadingZeros(string chunk) + { + int index = 0; + while (index < chunk.Length - 1 && DigitValue(chunk[index]) == 0) { - return lengthComparison; + index++; } - return string.Compare(xTrimmed, yTrimmed, StringComparison.Ordinal); + return index; } + + /// + /// Returns the decimal value of a Unicode decimal digit, so that digits from any script compare + /// by magnitude. + /// + /// The digit character, which the chunk regex guarantees is category Nd. + /// The digit's value of 0-9. + private static int DigitValue(char digit) => CharUnicodeInfo.GetDecimalDigitValue(digit); }