From 05d879d38742bf6cf680b2ea96dbcabb2b2f8757 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Thu, 2 Jul 2026 13:50:55 -0700 Subject: [PATCH 1/8] Add ToUpperOrdinal and ToLowerOrdinal casing APIs Implement ordinal (simple, one-to-one) casing on Char, Rune, String, and MemoryExtensions, matching the mappings used by OrdinalIgnoreCase comparisons. String.ToUpperOrdinal/ToLowerOrdinal return the same instance when an all-ASCII string is already in the requested case. --- .../vm/wasm/browser/callhelpers-pinvoke.cpp | 2 + .../vm/wasm/wasi/callhelpers-pinvoke.cpp | 2 + .../Common/src/Interop/Interop.Casing.cs | 3 + .../System.Memory/ref/System.Memory.cs | 2 + .../System.Private.CoreLib/src/System/Char.cs | 15 + .../src/System/Globalization/Ordinal.cs | 25 ++ .../System/Globalization/OrdinalCasing.Icu.cs | 121 ++++++ .../src/System/Globalization/TextInfo.cs | 18 + .../System/MemoryExtensions.Globalization.cs | 22 + .../src/System/String.Manipulation.cs | 112 +++++ .../src/System/Text/Rune.cs | 40 ++ .../System.Runtime/ref/System.Runtime.cs | 6 + .../System.Globalization.Tests.csproj | 1 + .../Globalization/OrdinalCasingTests.cs | 398 ++++++++++++++++++ .../System.Globalization.Native.def | 3 +- .../System.Globalization.Native/entrypoints.c | 1 + .../System.Globalization.Native/pal_casing.h | 2 + .../System.Globalization.Native/pal_common.c | 34 ++ 18 files changed, 806 insertions(+), 1 deletion(-) create mode 100644 src/libraries/System.Runtime/tests/System.Globalization.Tests/System/Globalization/OrdinalCasingTests.cs diff --git a/src/coreclr/vm/wasm/browser/callhelpers-pinvoke.cpp b/src/coreclr/vm/wasm/browser/callhelpers-pinvoke.cpp index 1c820b6d7b18ca..f98cd238f18612 100644 --- a/src/coreclr/vm/wasm/browser/callhelpers-pinvoke.cpp +++ b/src/coreclr/vm/wasm/browser/callhelpers-pinvoke.cpp @@ -45,6 +45,7 @@ extern "C" { int32_t GlobalizationNative_IndexOf (void *, void *, int32_t, void *, int32_t, int32_t, void *); void GlobalizationNative_InitICUFunctions (void *, void *, void *, void *); void GlobalizationNative_InitOrdinalCasingPage (int32_t, void *); + void GlobalizationNative_InitOrdinalLowerCasingPage (int32_t, void *); int32_t GlobalizationNative_IsNormalized (int32_t, void *, int32_t); int32_t GlobalizationNative_IsPredefinedLocale (void *); int32_t GlobalizationNative_LastIndexOf (void *, void *, int32_t, void *, int32_t, int32_t, void *); @@ -182,6 +183,7 @@ static const Entry s_libSystem_Globalization_Native [] = { DllImportEntry(GlobalizationNative_IndexOf) // System.Private.CoreLib DllImportEntry(GlobalizationNative_InitICUFunctions) // System.Private.CoreLib DllImportEntry(GlobalizationNative_InitOrdinalCasingPage) // System.Private.CoreLib + DllImportEntry(GlobalizationNative_InitOrdinalLowerCasingPage) // System.Private.CoreLib DllImportEntry(GlobalizationNative_IsNormalized) // System.Private.CoreLib DllImportEntry(GlobalizationNative_IsPredefinedLocale) // System.Private.CoreLib DllImportEntry(GlobalizationNative_LastIndexOf) // System.Private.CoreLib diff --git a/src/coreclr/vm/wasm/wasi/callhelpers-pinvoke.cpp b/src/coreclr/vm/wasm/wasi/callhelpers-pinvoke.cpp index 8e63502831632b..a11e4f28d1911c 100644 --- a/src/coreclr/vm/wasm/wasi/callhelpers-pinvoke.cpp +++ b/src/coreclr/vm/wasm/wasi/callhelpers-pinvoke.cpp @@ -44,6 +44,7 @@ extern "C" { int32_t GlobalizationNative_IndexOf (void *, void *, int32_t, void *, int32_t, int32_t, void *); void GlobalizationNative_InitICUFunctions (void *, void *, void *, void *); void GlobalizationNative_InitOrdinalCasingPage (int32_t, void *); + void GlobalizationNative_InitOrdinalLowerCasingPage (int32_t, void *); int32_t GlobalizationNative_IsNormalized (int32_t, void *, int32_t); int32_t GlobalizationNative_IsPredefinedLocale (void *); int32_t GlobalizationNative_LastIndexOf (void *, void *, int32_t, void *, int32_t, int32_t, void *); @@ -232,6 +233,7 @@ static const Entry s_libSystem_Globalization_Native [] = { DllImportEntry(GlobalizationNative_IndexOf) // System.Private.CoreLib DllImportEntry(GlobalizationNative_InitICUFunctions) // System.Private.CoreLib DllImportEntry(GlobalizationNative_InitOrdinalCasingPage) // System.Private.CoreLib + DllImportEntry(GlobalizationNative_InitOrdinalLowerCasingPage) // System.Private.CoreLib DllImportEntry(GlobalizationNative_IsNormalized) // System.Private.CoreLib DllImportEntry(GlobalizationNative_IsPredefinedLocale) // System.Private.CoreLib DllImportEntry(GlobalizationNative_LastIndexOf) // System.Private.CoreLib diff --git a/src/libraries/Common/src/Interop/Interop.Casing.cs b/src/libraries/Common/src/Interop/Interop.Casing.cs index 62743b2c6abd47..c479545ce1949c 100644 --- a/src/libraries/Common/src/Interop/Interop.Casing.cs +++ b/src/libraries/Common/src/Interop/Interop.Casing.cs @@ -21,5 +21,8 @@ internal static partial class Globalization [LibraryImport(Libraries.GlobalizationNative, EntryPoint = "GlobalizationNative_InitOrdinalCasingPage", StringMarshalling = StringMarshalling.Utf16)] internal static unsafe partial void InitOrdinalCasingPage(int pageNumber, char* pTarget); + + [LibraryImport(Libraries.GlobalizationNative, EntryPoint = "GlobalizationNative_InitOrdinalLowerCasingPage", StringMarshalling = StringMarshalling.Utf16)] + internal static unsafe partial void InitOrdinalLowerCasingPage(int pageNumber, char* pTarget); } } diff --git a/src/libraries/System.Memory/ref/System.Memory.cs b/src/libraries/System.Memory/ref/System.Memory.cs index f373f41d19195f..87e41a13e50f48 100644 --- a/src/libraries/System.Memory/ref/System.Memory.cs +++ b/src/libraries/System.Memory/ref/System.Memory.cs @@ -499,8 +499,10 @@ public static void Sort(this System.Span keys, Sy public static bool StartsWith(this System.ReadOnlySpan span, T value, System.Collections.Generic.IEqualityComparer? comparer = null) { throw null; } public static int ToLower(this System.ReadOnlySpan source, System.Span destination, System.Globalization.CultureInfo? culture) { throw null; } public static int ToLowerInvariant(this System.ReadOnlySpan source, System.Span destination) { throw null; } + public static int ToLowerOrdinal(this System.ReadOnlySpan source, System.Span destination) { throw null; } public static int ToUpper(this System.ReadOnlySpan source, System.Span destination, System.Globalization.CultureInfo? culture) { throw null; } public static int ToUpperInvariant(this System.ReadOnlySpan source, System.Span destination) { throw null; } + public static int ToUpperOrdinal(this System.ReadOnlySpan source, System.Span destination) { throw null; } public static System.Memory Trim(this System.Memory memory) { throw null; } public static System.ReadOnlyMemory Trim(this System.ReadOnlyMemory memory) { throw null; } public static System.ReadOnlySpan Trim(this System.ReadOnlySpan span) { throw null; } diff --git a/src/libraries/System.Private.CoreLib/src/System/Char.cs b/src/libraries/System.Private.CoreLib/src/System/Char.cs index e0018e74006088..7ef4b0a4588fd3 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Char.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Char.cs @@ -529,6 +529,14 @@ public static char ToUpper(char c) // Converts a character to upper-case for invariant culture. public static char ToUpperInvariant(char c) => TextInfo.ToUpperInvariant(c); + /// + /// Converts a character to uppercase using the casing rules used by + /// comparisons. + /// + /// The character to convert. + /// The uppercase equivalent of . + public static char ToUpperOrdinal(char c) => TextInfo.ToUpperOrdinal(c); + /*===================================ToLower==================================== ** ==============================================================================*/ @@ -558,6 +566,13 @@ public static char ToLower(char c) // Converts a character to lower-case for invariant culture. public static char ToLowerInvariant(char c) => TextInfo.ToLowerInvariant(c); + /// + /// Converts a character to lowercase using ordinal (simple, one-to-one) casing rules. + /// + /// The character to convert. + /// The lowercase equivalent of . + public static char ToLowerOrdinal(char c) => TextInfo.ToLowerOrdinal(c); + // // IConvertible implementation // diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs index b9ed56a9fb06a4..4a2c2ec64fd875 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs @@ -707,5 +707,30 @@ internal static int ToUpperOrdinal(ReadOnlySpan source, Span destina OrdinalCasing.ToUpperOrdinal(source, destination); return source.Length; } + + internal static int ToLowerOrdinal(ReadOnlySpan source, Span destination) + { + if (source.Overlaps(destination)) + ThrowHelper.ThrowInvalidOperationException(ExceptionResource.InvalidOperation_SpanOverlappedOperation); + + // Assuming that changing case does not affect length + if (destination.Length < source.Length) + return -1; + + if (GlobalizationMode.Invariant) + { + InvariantModeCasing.ToLower(source, destination); + return source.Length; + } + + if (GlobalizationMode.UseNls) + { + TextInfo.Invariant.ChangeCaseToLower(source, destination); // this is the best so far for NLS. + return source.Length; + } + + OrdinalCasing.ToLowerOrdinal(source, destination); + return source.Length; + } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/OrdinalCasing.Icu.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/OrdinalCasing.Icu.cs index cff1e976199b0f..57c6cb6375a8ae 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/OrdinalCasing.Icu.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/OrdinalCasing.Icu.cs @@ -43,6 +43,35 @@ internal static partial class OrdinalCasing // Every cell is array of 512 character for uppercasing mapping. private static readonly ushort[]?[] s_casingTable = InitCasingTable(); + // s_lowerBasicLatin is covering the lower casing for the Basic Latin & C0 Controls range. + // we are not lazy initializing this range because it is the most common used range and we'll cache it anyway very early. + private static readonly ushort[] s_lowerBasicLatin = + [ + // Lower Casing + + /* 0000-000f */ 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + /* 0010-001f */ 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + /* 0020-002f */ 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + /* 0030-003f */ 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + /* 0040-004f */ 0x0040, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + /* 0050-005f */ 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + /* 0060-006f */ 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + /* 0070-007f */ 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + /* 0080-008f */ 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + /* 0090-009f */ 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + /* 00a0-00af */ 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + /* 00b0-00bf */ 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf, + /* 00c0-00cf */ 0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + /* 00d0-00df */ 0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00d7, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00df, + /* 00e0-00ef */ 0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + /* 00f0-00ff */ 0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff, + ]; + + // s_lowerCasingTable mirrors s_casingTable but holds the simple lower casing mapping for the BMP. + // It is pre-seeded with the shared NoCasingPage for the permanently uncased pages (see InitLowerCasingTable); + // remaining pages are initialized lazily through ICU and identity pages collapse to the shared NoCasingPage. + private static readonly ushort[]?[] s_lowerCasingTable = InitLowerCasingTable(); + /* The table is initialized to: { @@ -172,6 +201,57 @@ internal static void ToUpperOrdinal(ReadOnlySpan source, Span destin } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static char ToLower(char c) + { + int pageNumber = ((int)c) >> 8; + if (pageNumber == 0) // optimize for ASCII range + { + return (char)s_lowerBasicLatin[(int)c]; + } + + ushort[]? casingTable = s_lowerCasingTable[pageNumber]; + + if (casingTable == NoCasingPage) + { + return c; + } + + casingTable ??= InitOrdinalLowerCasingPage(pageNumber); + + return (char)casingTable[((int)c) & 0xFF]; + } + + internal static void ToLowerOrdinal(ReadOnlySpan source, Span destination) + { + Debug.Assert(!GlobalizationMode.Invariant); + Debug.Assert(!GlobalizationMode.UseNls); + + for (int i = 0; i < source.Length; i++) + { + char c = source[i]; + if (c <= '\u00FF') // optimize ASCII/Latin + { + destination[i] = (char)s_lowerBasicLatin[c]; + continue; + } + + if (char.IsHighSurrogate(c) && i < source.Length - 1) + { + char cl = source[i + 1]; + if (char.IsLowSurrogate(cl)) + { + // well formed surrogates + SurrogateCasing.ToLower(c, cl, out destination[i], out destination[i + 1]); + i++; // skip the low surrogate + continue; + } + } + + destination[i] = ToLower(c); + } + } + internal static int CompareStringIgnoreCase(ref char strA, int lengthA, ref char strB, int lengthB) { Debug.Assert(!GlobalizationMode.Invariant); @@ -428,5 +508,46 @@ private static unsafe ushort[] InitOrdinalCasingPage(int pageNumber) Volatile.Write(ref s_casingTable[pageNumber], casingTable); return casingTable; } + + private static ushort[]?[] InitLowerCasingTable() + { + // Reuse the upper-casing NoCasing bitmap to pre-seed the lower-casing table. Every page that bitmap + // marks (the permanently uncased CJK/Hangul/PUA blocks) is also identity under lower casing, so seeding + // these pages with the shared NoCasingPage avoids first-use native calls without risking a wrong mapping. + ushort[]?[] table = new ushort[]?[s_casingTableInit.Length * 8]; + for (int i = 0; i < s_casingTableInit.Length * 8; ++i) + { + // The bits are in reverse order + byte val = (byte)(s_casingTableInit[i / 8] >> (7 - (i % 8))); + if ((val & 1) == 1) + table[i] = NoCasingPage; + } + table[0] = s_lowerBasicLatin; + return table; + } + + private static unsafe ushort[] InitOrdinalLowerCasingPage(int pageNumber) + { + Debug.Assert(pageNumber > 0 && pageNumber < 256); + + ushort[] casingTable = new ushort[256]; + fixed (ushort* table = casingTable) + { + char* pTable = (char*)table; + Interop.Globalization.InitOrdinalLowerCasingPage(pageNumber, pTable); + } + + // If the page doesn't change any character, collapse it to the shared NoCasingPage to avoid + // retaining the 512-byte buffer. We still return the fully populated table for the current lookup. + int pageBase = pageNumber << 8; + int i = 0; + while (i < 256 && casingTable[i] == (ushort)(pageBase + i)) + { + i++; + } + + Volatile.Write(ref s_lowerCasingTable[pageNumber], i == 256 ? NoCasingPage : casingTable); + return casingTable; + } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/TextInfo.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/TextInfo.cs index 9423ad9523800c..8d4ebdbc8cd4f7 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/TextInfo.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/TextInfo.cs @@ -204,6 +204,24 @@ internal static char ToUpperOrdinal(char c) return OrdinalCasing.ToUpper(c); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static char ToLowerOrdinal(char c) + { + if (GlobalizationMode.Invariant) + { + return InvariantModeCasing.ToLower(c); + } + + if (GlobalizationMode.UseNls) + { + return char.IsAscii(c) + ? ToLowerAsciiInvariant(c) + : Invariant.ChangeCase(c, toUpper: false); + } + + return OrdinalCasing.ToLower(c); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] internal void ChangeCaseToLower(ReadOnlySpan source, Span destination) { diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.Globalization.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.Globalization.cs index c2569a96c32bc4..15db9c4bbe27d7 100644 --- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.Globalization.cs +++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.Globalization.cs @@ -330,6 +330,28 @@ public static int ToUpperInvariant(this ReadOnlySpan source, Span de return source.Length; } + /// + /// Copies the characters from the source span into the destination, converting each character to uppercase + /// using the casing rules used by comparisons. + /// + /// The source span. + /// The destination span which contains the transformed characters. + /// The number of characters written into the destination span. If the destination is too small, returns -1. + /// The source and destination buffers overlap. + public static int ToUpperOrdinal(this ReadOnlySpan source, Span destination) => + Ordinal.ToUpperOrdinal(source, destination); + + /// + /// Copies the characters from the source span into the destination, converting each character to lowercase + /// using ordinal (simple, one-to-one) casing rules. + /// + /// The source span. + /// The destination span which contains the transformed characters. + /// The number of characters written into the destination span. If the destination is too small, returns -1. + /// The source and destination buffers overlap. + public static int ToLowerOrdinal(this ReadOnlySpan source, Span destination) => + Ordinal.ToLowerOrdinal(source, destination); + /// /// Determines whether the end of the matches the specified when compared using the specified option. /// diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs index 7b68b4aa46f105..7874b2ac4b3c45 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs @@ -11,6 +11,7 @@ using System.Runtime.InteropServices; using System.Runtime.Intrinsics; using System.Text; +using System.Text.Unicode; namespace System { @@ -2385,6 +2386,117 @@ public string ToUpperInvariant() return TextInfo.ToUpperInvariant(this); } + /// + /// Returns a copy of this string converted to uppercase using the casing rules used by + /// comparisons. + /// + /// The uppercase equivalent of the current string. + /// + /// The conversion uses a simple, one-to-one mapping that preserves the length of the string. + /// Two strings are equal under if and only if + /// their results are ordinally equal. + /// + public string ToUpperOrdinal() + { + if (Length == 0) + { + return Empty; + } + + return ChangeCaseOrdinal(toUpper: true); + } + + /// + /// Returns a copy of this string converted to lowercase using ordinal (simple, one-to-one) casing rules. + /// + /// The lowercase equivalent of the current string. + /// + /// The conversion uses a simple, one-to-one mapping that preserves the length of the string. + /// + public string ToLowerOrdinal() + { + if (Length == 0) + { + return Empty; + } + + return ChangeCaseOrdinal(toUpper: false); + } + + private unsafe string ChangeCaseOrdinal(bool toUpper) + { + Debug.Assert(Length > 0); + + int consumed; + + // Fast path: scan the leading run of ASCII characters that is already in the + // requested case. If the entire string is ASCII and needs no change, return the + // same instance to avoid an allocation. This mirrors the behavior of + // TextInfo.ChangeCaseCommon used by ToUpper(Invariant)/ToLower. + fixed (char* pSource = this) + { + nuint currIdx = 0; // in chars + + if (Length >= 2) + { + nuint lastIndexWhereCanReadTwoChars = (uint)Length - 2; + do + { + // Read 2 chars (one 32-bit integer) at a time. + uint tempValue = Unsafe.ReadUnaligned(pSource + currIdx); + if (!Utf16Utility.AllCharsInUInt32AreAscii(tempValue) || + (toUpper + ? Utf16Utility.UInt32ContainsAnyLowercaseAsciiChar(tempValue) + : Utf16Utility.UInt32ContainsAnyUppercaseAsciiChar(tempValue))) + { + goto MustChange; + } + + currIdx += 2; + } while (currIdx <= lastIndexWhereCanReadTwoChars); + } + + // If there's a single character left, check it now. + if ((Length & 1) != 0) + { + uint tempValue = pSource[currIdx]; + if (tempValue > 0x7Fu || + (toUpper + ? ((tempValue - 'a') <= (uint)('z' - 'a')) + : ((tempValue - 'A') <= (uint)('Z' - 'A')))) + { + goto MustChange; + } + } + + // The whole string is ASCII and already in the requested case. + return this; + + MustChange: + consumed = (int)currIdx; + } + + // The leading [0, consumed) chars are ASCII already in the requested case, so copy + // them verbatim and run the ordinal casing over the remainder. + string result = FastAllocateString(Length); + Span resultSpan = new Span(ref result.GetRawStringData(), Length); + if (consumed > 0) + { + this.AsSpan(0, consumed).CopyTo(resultSpan); + } + + if (toUpper) + { + Globalization.Ordinal.ToUpperOrdinal(this.AsSpan(consumed), resultSpan.Slice(consumed)); + } + else + { + Globalization.Ordinal.ToLowerOrdinal(this.AsSpan(consumed), resultSpan.Slice(consumed)); + } + + return result; + } + // Trims the whitespace from both ends of the string. Whitespace is defined by // char.IsWhiteSpace. // diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Rune.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Rune.cs index 632535b3bab0c2..5af3c1f8740862 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Rune.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Rune.cs @@ -1591,6 +1591,46 @@ public static Rune ToUpperInvariant(Rune value) #endif } +#if SYSTEM_PRIVATE_CORELIB + /// + /// Returns a copy of converted to uppercase using the casing rules used by + /// comparisons. + /// + public static Rune ToUpperOrdinal(Rune value) + { + if (value.IsAscii) + { + return UnsafeCreate(Utf16Utility.ConvertAllAsciiCharsInUInt32ToUppercase(value._value)); + } + + if (value.IsBmp) + { + return UnsafeCreate(TextInfo.ToUpperOrdinal((char)value._value)); + } + + // Supplementary characters use the same simple scalar mapping as OrdinalIgnoreCase comparisons. + return UnsafeCreate(CharUnicodeInfo.ToUpper(value._value)); + } + + /// + /// Returns a copy of converted to lowercase using ordinal (simple, one-to-one) casing rules. + /// + public static Rune ToLowerOrdinal(Rune value) + { + if (value.IsAscii) + { + return UnsafeCreate(Utf16Utility.ConvertAllAsciiCharsInUInt32ToLowercase(value._value)); + } + + if (value.IsBmp) + { + return UnsafeCreate(TextInfo.ToLowerOrdinal((char)value._value)); + } + + return UnsafeCreate(CharUnicodeInfo.ToLower(value._value)); + } +#endif + /// int IComparable.CompareTo(object? obj) { diff --git a/src/libraries/System.Runtime/ref/System.Runtime.cs b/src/libraries/System.Runtime/ref/System.Runtime.cs index b4e1e0dbe0bb05..1cf4a0c6fe3f58 100644 --- a/src/libraries/System.Runtime/ref/System.Runtime.cs +++ b/src/libraries/System.Runtime/ref/System.Runtime.cs @@ -1253,12 +1253,14 @@ public CannotUnloadAppDomainException(string? message, System.Exception? innerEx public static char ToLower(char c) { throw null; } public static char ToLower(char c, System.Globalization.CultureInfo culture) { throw null; } public static char ToLowerInvariant(char c) { throw null; } + public static char ToLowerOrdinal(char c) { throw null; } public override string ToString() { throw null; } public static string ToString(char c) { throw null; } public string ToString(System.IFormatProvider? provider) { throw null; } public static char ToUpper(char c) { throw null; } public static char ToUpper(char c, System.Globalization.CultureInfo culture) { throw null; } public static char ToUpperInvariant(char c) { throw null; } + public static char ToUpperOrdinal(char c) { throw null; } public static bool TryParse([System.Diagnostics.CodeAnalysis.NotNullWhenAttribute(true)] string? s, out char result) { throw null; } } public sealed partial class CharEnumerator : System.Collections.Generic.IEnumerator, System.Collections.IEnumerator, System.ICloneable, System.IDisposable @@ -5852,11 +5854,13 @@ public void CopyTo(System.Span destination) { } public string ToLower() { throw null; } public string ToLower(System.Globalization.CultureInfo? culture) { throw null; } public string ToLowerInvariant() { throw null; } + public string ToLowerOrdinal() { throw null; } public override string ToString() { throw null; } public string ToString(System.IFormatProvider? provider) { throw null; } public string ToUpper() { throw null; } public string ToUpper(System.Globalization.CultureInfo? culture) { throw null; } public string ToUpperInvariant() { throw null; } + public string ToUpperOrdinal() { throw null; } public string Trim() { throw null; } public string Trim(char trimChar) { throw null; } public string Trim(System.Text.Rune trimRune) { throw null; } @@ -16174,9 +16178,11 @@ public enum NormalizationForm static bool System.IUtf8SpanParsable.TryParse(System.ReadOnlySpan utf8Text, System.IFormatProvider? provider, out System.Text.Rune result) { throw null; } public static System.Text.Rune ToLower(System.Text.Rune value, System.Globalization.CultureInfo culture) { throw null; } public static System.Text.Rune ToLowerInvariant(System.Text.Rune value) { throw null; } + public static System.Text.Rune ToLowerOrdinal(System.Text.Rune value) { throw null; } public override string ToString() { throw null; } public static System.Text.Rune ToUpper(System.Text.Rune value, System.Globalization.CultureInfo culture) { throw null; } public static System.Text.Rune ToUpperInvariant(System.Text.Rune value) { throw null; } + public static System.Text.Rune ToUpperOrdinal(System.Text.Rune value) { throw null; } public static bool TryCreate(char highSurrogate, char lowSurrogate, out System.Text.Rune result) { throw null; } public static bool TryCreate(char ch, out System.Text.Rune result) { throw null; } public static bool TryCreate(int value, out System.Text.Rune result) { throw null; } diff --git a/src/libraries/System.Runtime/tests/System.Globalization.Tests/System.Globalization.Tests.csproj b/src/libraries/System.Runtime/tests/System.Globalization.Tests/System.Globalization.Tests.csproj index 7c0d235a7f2a60..501cd8c54d8a8f 100644 --- a/src/libraries/System.Runtime/tests/System.Globalization.Tests/System.Globalization.Tests.csproj +++ b/src/libraries/System.Runtime/tests/System.Globalization.Tests/System.Globalization.Tests.csproj @@ -119,6 +119,7 @@ + diff --git a/src/libraries/System.Runtime/tests/System.Globalization.Tests/System/Globalization/OrdinalCasingTests.cs b/src/libraries/System.Runtime/tests/System.Globalization.Tests/System/Globalization/OrdinalCasingTests.cs new file mode 100644 index 00000000000000..730767b7e39560 --- /dev/null +++ b/src/libraries/System.Runtime/tests/System.Globalization.Tests/System/Globalization/OrdinalCasingTests.cs @@ -0,0 +1,398 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Text; +using Microsoft.DotNet.RemoteExecutor; +using Xunit; + +namespace System.Globalization.Tests +{ + public class OrdinalCasingTests + { + private const int FirstSurrogate = 0xD800; + private const int LastSurrogate = 0xDFFF; + + // Pairs that are equal under StringComparison.OrdinalIgnoreCase including the well known + // characters that fold to the same uppercase letter even though they are not the ASCII forms. + public static IEnumerable OrdinalIgnoreCaseEqualPairs() + { + yield return new object[] { "", "" }; + yield return new object[] { "abc", "ABC" }; + yield return new object[] { "Hello", "hELLO" }; + yield return new object[] { "\u00B5", "\u039C" }; // MICRO SIGN folds to GREEK CAPITAL MU + yield return new object[] { "\u017F", "\u017F" }; // LATIN SMALL LETTER LONG S stays itself (no OIC fold to 'S') + yield return new object[] { "\u0131", "\u0131" }; // LATIN SMALL LETTER DOTLESS I stays itself (no OIC fold to 'I') + yield return new object[] { "\u03C3", "\u03A3" }; // GREEK SMALL SIGMA vs GREEK CAPITAL SIGMA + yield return new object[] { "\u03C2", "\u03A3" }; // GREEK SMALL FINAL SIGMA vs GREEK CAPITAL SIGMA + yield return new object[] { "\u0561\u0562", "\u0531\u0532" }; // Armenian small vs capital + } + + public static IEnumerable OrdinalIgnoreCaseNotEqualPairs() + { + yield return new object[] { "abc", "abd" }; + yield return new object[] { "abc", "ab" }; + yield return new object[] { "\u017F", "S" }; // LONG S is not OIC equal to 'S' + yield return new object[] { "\u017F", "s" }; + yield return new object[] { "\u0131", "I" }; // DOTLESS I is not OIC equal to 'I' + yield return new object[] { "\u0131", "i" }; + yield return new object[] { "\u212A", "K" }; // KELVIN SIGN is not OIC equal to 'K' + yield return new object[] { "\u212A", "k" }; + yield return new object[] { "K", "L" }; + } + + [Theory] + [MemberData(nameof(OrdinalIgnoreCaseEqualPairs))] + public void ToUpperOrdinal_MatchesOrdinalIgnoreCase_ForEqualStrings(string left, string right) + { + bool oic = string.Equals(left, right, StringComparison.OrdinalIgnoreCase); + Assert.True(oic); + Assert.Equal(left.ToUpperOrdinal(), right.ToUpperOrdinal()); + } + + [Theory] + [MemberData(nameof(OrdinalIgnoreCaseNotEqualPairs))] + public void ToUpperOrdinal_MatchesOrdinalIgnoreCase_ForNotEqualStrings(string left, string right) + { + bool oic = string.Equals(left, right, StringComparison.OrdinalIgnoreCase); + Assert.False(oic); + bool foldEqual = string.Equals(left.ToUpperOrdinal(), right.ToUpperOrdinal(), StringComparison.Ordinal); + Assert.False(foldEqual); + } + + [Fact] + public void ToUpperOrdinal_EveryBmpChar_IsOrdinalIgnoreCaseEqualToItsFold() + { + for (int c = 0; c <= 0xFFFF; c++) + { + if (c >= FirstSurrogate && c <= LastSurrogate) + { + continue; + } + + string original = ((char)c).ToString(); + string upper = original.ToUpperOrdinal(); + + Assert.Equal(1, upper.Length); + Assert.True(string.Equals(original, upper, StringComparison.OrdinalIgnoreCase), + $"U+{c:X4} is not OrdinalIgnoreCase-equal to its ToUpperOrdinal fold U+{(int)upper[0]:X4}"); + + // The String, Char and span overloads must agree. + Assert.Equal(upper[0], char.ToUpperOrdinal((char)c)); + + Span destination = stackalloc char[1]; + Assert.Equal(1, original.AsSpan().ToUpperOrdinal(destination)); + Assert.Equal(upper[0], destination[0]); + } + } + + [Fact] + public void ToLowerOrdinal_EveryBmpChar_AgreesAcrossOverloads() + { + for (int c = 0; c <= 0xFFFF; c++) + { + if (c >= FirstSurrogate && c <= LastSurrogate) + { + continue; + } + + string original = ((char)c).ToString(); + string lower = original.ToLowerOrdinal(); + + Assert.Equal(1, lower.Length); + Assert.Equal(lower[0], char.ToLowerOrdinal((char)c)); + + Span destination = stackalloc char[1]; + Assert.Equal(1, original.AsSpan().ToLowerOrdinal(destination)); + Assert.Equal(lower[0], destination[0]); + } + } + + // ToLowerOrdinal must stay consistent with OrdinalIgnoreCase: a character is always + // OrdinalIgnoreCase-equal to its ordinal lower-cased form. This guards the special handling that keeps + // characters whose simple lower mapping would leave their class unchanged (for example the Kelvin, Ohm + // and Angstrom signs) instead of folding them into a character of a different class. + [Fact] + public void ToLowerOrdinal_EveryBmpChar_IsOrdinalIgnoreCaseEqualToItself() + { + for (int c = 0; c <= 0xFFFF; c++) + { + if (c >= FirstSurrogate && c <= LastSurrogate) + { + continue; + } + + string original = ((char)c).ToString(); + string lower = original.ToLowerOrdinal(); + bool equal = string.Equals(original, lower, StringComparison.OrdinalIgnoreCase); + Assert.True(equal); + } + } + + // Validates the ordinal lower casing value for the whole BMP, including the pages that are pre-seeded as + // no-casing, against char.ToLowerInvariant. Invariant lowering matches the simple lower mapping everywhere, + // so it is used as an independent oracle: where it would move a character out of its ordinal upper-casing + // class the ordinal value keeps the original character, otherwise the two must agree. A page wrongly marked + // as no-casing would return the original character while the oracle returns the lowercased form. + [Fact] + public void ToLowerOrdinal_EveryBmpChar_MatchesInvariantLoweringWithinClass() + { + for (int c = 0; c <= 0xFFFF; c++) + { + if (c >= FirstSurrogate && c <= LastSurrogate) + { + continue; + } + + char ordinalLower = char.ToLowerOrdinal((char)c); + char invariantLower = char.ToLowerInvariant((char)c); + + if (char.ToUpperOrdinal(invariantLower) != char.ToUpperOrdinal((char)c)) + { + Assert.Equal((char)c, ordinalLower); + } + else + { + Assert.Equal(invariantLower, ordinalLower); + } + } + } + + [Fact] + public void Casing_Ascii_IsCorrect() + { + for (int c = 0; c < 0x80; c++) + { + char expectedUpper = (c >= 'a' && c <= 'z') ? (char)(c - 0x20) : (char)c; + char expectedLower = (c >= 'A' && c <= 'Z') ? (char)(c + 0x20) : (char)c; + + Assert.Equal(expectedUpper, char.ToUpperOrdinal((char)c)); + Assert.Equal(expectedLower, char.ToLowerOrdinal((char)c)); + } + } + + public static IEnumerable CharUpperData() + { + yield return new object[] { 'a', 'A' }; + yield return new object[] { 'z', 'Z' }; + yield return new object[] { 'A', 'A' }; + yield return new object[] { '5', '5' }; + yield return new object[] { '\u00E0', '\u00C0' }; // a-grave -> A-grave + yield return new object[] { '\u03B1', '\u0391' }; // greek small alpha -> capital alpha + yield return new object[] { '\u0450', '\u0400' }; // cyrillic small ie -> capital + yield return new object[] { '\u00B5', '\u039C' }; // MICRO SIGN -> GREEK CAPITAL MU + yield return new object[] { '\u212A', '\u212A' }; // KELVIN SIGN is not folded by ordinal upper + yield return new object[] { '\u0131', '\u0131' }; // DOTLESS I unchanged + yield return new object[] { '\u017F', '\u017F' }; // LONG S unchanged + } + + public static IEnumerable CharLowerData() + { + yield return new object[] { 'A', 'a' }; + yield return new object[] { 'Z', 'z' }; + yield return new object[] { 'a', 'a' }; + yield return new object[] { '5', '5' }; + yield return new object[] { '\u00C0', '\u00E0' }; // A-grave -> a-grave + yield return new object[] { '\u0391', '\u03B1' }; // greek capital alpha -> small alpha + yield return new object[] { '\u0400', '\u0450' }; // cyrillic capital -> small ie + yield return new object[] { '\u212A', '\u212A' }; // KELVIN SIGN stays itself (not OIC equal to 'k') + yield return new object[] { '\u212B', '\u212B' }; // ANGSTROM SIGN stays itself (not OIC equal to small a-ring) + yield return new object[] { '\u2126', '\u2126' }; // OHM SIGN stays itself (not OIC equal to small omega) + yield return new object[] { '\u1E9E', '\u1E9E' }; // CAPITAL SHARP S stays itself (not OIC equal to small sharp s) + yield return new object[] { '\u03F4', '\u03F4' }; // GREEK CAPITAL THETA SYMBOL stays itself (not OIC equal to small theta) + yield return new object[] { '\u0130', '\u0130' }; // CAPITAL I WITH DOT ABOVE stays itself (simple 'i' would leave its class) + yield return new object[] { 'I', 'i' }; // ordinal lowering ignores Turkish rules + } + + [Theory] + [MemberData(nameof(CharUpperData))] + public void Char_ToUpperOrdinal(char input, char expected) + { + Assert.Equal(expected, char.ToUpperOrdinal(input)); + Assert.Equal(expected.ToString(), input.ToString().ToUpperOrdinal()); + } + + [Theory] + [MemberData(nameof(CharLowerData))] + public void Char_ToLowerOrdinal(char input, char expected) + { + Assert.Equal(expected, char.ToLowerOrdinal(input)); + Assert.Equal(expected.ToString(), input.ToString().ToLowerOrdinal()); + } + + [Fact] + public void String_Empty_ReturnsEmpty() + { + Assert.Same(string.Empty, "".ToUpperOrdinal()); + Assert.Same(string.Empty, "".ToLowerOrdinal()); + } + + [Fact] + public void String_PreservesLength() + { + string value = "Stra\u00DFe \u212A \u0131\u017F Test 123"; + Assert.Equal(value.Length, value.ToUpperOrdinal().Length); + Assert.Equal(value.Length, value.ToLowerOrdinal().Length); + } + + [Theory] + [InlineData("A")] + [InlineData("AB")] + [InlineData("ABC")] + [InlineData("HELLO WORLD")] + [InlineData("HELLO WORLD 123!")] + [InlineData("123 !@#$ ")] + public void ToUpperOrdinal_AlreadyUpperAscii_ReturnsSameInstance(string value) + { + Assert.Same(value, value.ToUpperOrdinal()); + } + + [Theory] + [InlineData("a")] + [InlineData("ab")] + [InlineData("abc")] + [InlineData("hello world")] + [InlineData("hello world 123!")] + [InlineData("123 !@#$ ")] + public void ToLowerOrdinal_AlreadyLowerAscii_ReturnsSameInstance(string value) + { + Assert.Same(value, value.ToLowerOrdinal()); + } + + [Theory] + [InlineData("abc", "ABC")] // change at the start + [InlineData("aBcD", "ABCD")] // interleaved + [InlineData("ABCd", "ABCD")] // change at the end (odd length) + [InlineData("aBCDEFGH", "ABCDEFGH")] + [InlineData("ABCDEFGa", "ABCDEFGA")] + [InlineData("ABC\u00E9", "ABC\u00C9")] // ascii prefix already upper, non-ascii tail changes + [InlineData("aBc\u00E9", "ABC\u00C9")] // change begins in the ascii prefix + public void ToUpperOrdinal_NeedsChange_ProducesCorrectNewInstance(string value, string expected) + { + string result = value.ToUpperOrdinal(); + Assert.Equal(expected, result); + Assert.NotSame(value, result); + } + + [Theory] + [InlineData("ABC", "abc")] + [InlineData("aBcD", "abcd")] + [InlineData("abcD", "abcd")] + [InlineData("Abcdefgh", "abcdefgh")] + [InlineData("abcdefgH", "abcdefgh")] + [InlineData("abc\u00C9", "abc\u00E9")] + [InlineData("aBc\u00C9", "abc\u00E9")] + public void ToLowerOrdinal_NeedsChange_ProducesCorrectNewInstance(string value, string expected) + { + string result = value.ToLowerOrdinal(); + Assert.Equal(expected, result); + Assert.NotSame(value, result); + } + + [Fact] + public void Span_DestinationTooSmall_ReturnsNegativeOne() + { + ReadOnlySpan source = "abc"; + Span destination = stackalloc char[2]; + Assert.Equal(-1, source.ToUpperOrdinal(destination)); + Assert.Equal(-1, source.ToLowerOrdinal(destination)); + } + + [Fact] + public void Span_ExactSize_WritesAllCharacters() + { + ReadOnlySpan source = "aBcD"; + Span upper = stackalloc char[4]; + Span lower = stackalloc char[4]; + + Assert.Equal(4, source.ToUpperOrdinal(upper)); + Assert.Equal(4, source.ToLowerOrdinal(lower)); + Assert.Equal("ABCD", upper.ToString()); + Assert.Equal("abcd", lower.ToString()); + } + + [Fact] + public void Span_OverlappingBuffers_Throws() + { + char[] buffer = { 'a', 'b', 'c', 'd' }; + Assert.Throws(() => ((ReadOnlySpan)buffer).ToUpperOrdinal(buffer)); + Assert.Throws(() => ((ReadOnlySpan)buffer).ToLowerOrdinal(buffer)); + } + + public static IEnumerable RuneUpperData() + { + yield return new object[] { new Rune('a'), new Rune('A') }; + yield return new object[] { new Rune('5'), new Rune('5') }; + yield return new object[] { new Rune('\u03B1'), new Rune('\u0391') }; // greek + yield return new object[] { new Rune('\u212A'), new Rune('\u212A') }; // KELVIN SIGN is not folded by ordinal upper + yield return new object[] { new Rune(0x10428), new Rune(0x10400) }; // DESERET small ee -> capital long i + } + + public static IEnumerable RuneLowerData() + { + yield return new object[] { new Rune('A'), new Rune('a') }; + yield return new object[] { new Rune('5'), new Rune('5') }; + yield return new object[] { new Rune('\u0391'), new Rune('\u03B1') }; // greek + yield return new object[] { new Rune('\u212A'), new Rune('\u212A') }; // KELVIN SIGN stays itself (not OIC equal to 'k') + yield return new object[] { new Rune(0x10400), new Rune(0x10428) }; // DESERET capital -> small + } + + [Theory] + [MemberData(nameof(RuneUpperData))] + public void Rune_ToUpperOrdinal(Rune input, Rune expected) + { + Assert.Equal(expected, Rune.ToUpperOrdinal(input)); + + if (input.IsBmp) + { + Assert.Equal((char)expected.Value, char.ToUpperOrdinal((char)input.Value)); + } + } + + [Theory] + [MemberData(nameof(RuneLowerData))] + public void Rune_ToLowerOrdinal(Rune input, Rune expected) + { + Assert.Equal(expected, Rune.ToLowerOrdinal(input)); + + if (input.IsBmp) + { + Assert.Equal((char)expected.Value, char.ToLowerOrdinal((char)input.Value)); + } + } + + [Fact] + public void OrdinalIgnoreCase_Equivalence_HoldsInInvariantMode() + { + var options = new RemoteInvokeOptions(); + options.RuntimeConfigurationOptions.Add("System.Globalization.Invariant", true); + + RemoteExecutor.Invoke(static () => + { + foreach (object[] pair in OrdinalIgnoreCaseEqualPairs()) + { + string left = (string)pair[0]; + string right = (string)pair[1]; + + bool oic = string.Equals(left, right, StringComparison.OrdinalIgnoreCase); + bool upperEqual = string.Equals(left.ToUpperOrdinal(), right.ToUpperOrdinal(), StringComparison.Ordinal); + Assert.Equal(oic, upperEqual); + } + + // Verify the ToUpperOrdinal-equals-OrdinalIgnoreCase contract over the whole BMP in invariant mode. + for (int c = 0; c <= 0xFFFF; c++) + { + if (c >= FirstSurrogate && c <= LastSurrogate) + { + continue; + } + + string original = ((char)c).ToString(); + string upper = original.ToUpperOrdinal(); + bool selfOic = string.Equals(original, upper, StringComparison.OrdinalIgnoreCase); + Assert.True(selfOic); + } + }, options).Dispose(); + } + } +} diff --git a/src/native/libs/System.Globalization.Native/System.Globalization.Native.def b/src/native/libs/System.Globalization.Native/System.Globalization.Native.def index 3338d4c36994a2..162e0a0c2dc032 100644 --- a/src/native/libs/System.Globalization.Native/System.Globalization.Native.def +++ b/src/native/libs/System.Globalization.Native/System.Globalization.Native.def @@ -36,4 +36,5 @@ EXPORTS GlobalizationNative_WindowsIdToIanaId GlobalizationNative_ToAscii GlobalizationNative_ToUnicode - GlobalizationNative_InitOrdinalCasingPage \ No newline at end of file + GlobalizationNative_InitOrdinalCasingPage + GlobalizationNative_InitOrdinalLowerCasingPage \ No newline at end of file diff --git a/src/native/libs/System.Globalization.Native/entrypoints.c b/src/native/libs/System.Globalization.Native/entrypoints.c index b687ea837daac6..ffe0347e312786 100644 --- a/src/native/libs/System.Globalization.Native/entrypoints.c +++ b/src/native/libs/System.Globalization.Native/entrypoints.c @@ -84,6 +84,7 @@ static const Entry s_globalizationNative[] = DllImportEntry(GlobalizationNative_ToAscii) DllImportEntry(GlobalizationNative_ToUnicode) DllImportEntry(GlobalizationNative_InitOrdinalCasingPage) + DllImportEntry(GlobalizationNative_InitOrdinalLowerCasingPage) }; EXTERN_C const void* GlobalizationResolveDllImport(const char* name); diff --git a/src/native/libs/System.Globalization.Native/pal_casing.h b/src/native/libs/System.Globalization.Native/pal_casing.h index 5371ba3bb08868..58f1b702ce9fdd 100644 --- a/src/native/libs/System.Globalization.Native/pal_casing.h +++ b/src/native/libs/System.Globalization.Native/pal_casing.h @@ -6,6 +6,8 @@ PALEXPORT void GlobalizationNative_InitOrdinalCasingPage(int32_t pageNumber, UChar* pTarget); +PALEXPORT void GlobalizationNative_InitOrdinalLowerCasingPage(int32_t pageNumber, UChar* pTarget); + PALEXPORT void GlobalizationNative_ChangeCase(const UChar* lpSrc, int32_t cwSrcLength, UChar* lpDst, diff --git a/src/native/libs/System.Globalization.Native/pal_common.c b/src/native/libs/System.Globalization.Native/pal_common.c index 5904a1c3bdbe25..6ad57d2e588870 100644 --- a/src/native/libs/System.Globalization.Native/pal_common.c +++ b/src/native/libs/System.Globalization.Native/pal_common.c @@ -19,3 +19,37 @@ void GlobalizationNative_InitOrdinalCasingPage(int32_t pageNumber, UChar* pTarge pTarget[0x7F] = (UChar)0x017F; // // 017F;LATIN SMALL LETTER LONG S } } + +void GlobalizationNative_InitOrdinalLowerCasingPage(int32_t pageNumber, UChar* pTarget) +{ + pageNumber <<= 8; + for (int i = 0; i < 256; i++) + { + // Unfortunately, to ensure one-to-one simple mapping we have to call u_tolower on every character. + // Using string casing ICU APIs cannot give such results even when using NULL locale to force root behavior. + pTarget[i] = (UChar) u_tolower((UChar32)(pageNumber + i)); + } + + // Keep a few characters unchanged because their simple lower casing would move them out of their + // ordinal (simple) upper-casing equivalence class. Ordinal lower casing must stay consistent with + // OrdinalIgnoreCase comparisons, i.e. a character has to remain OrdinalIgnoreCase-equal to its + // lower-cased form, exactly like the upper-casing page keeps Turkish 'i' and long 's' unchanged. + // The affected characters are handled per originating page, mirroring the upper-casing page above. + switch (pageNumber) + { + case 0x0100: + pTarget[0x30] = (UChar)0x0130; // LATIN CAPITAL LETTER I WITH DOT ABOVE + break; + case 0x0300: + pTarget[0xF4] = (UChar)0x03F4; // GREEK CAPITAL THETA SYMBOL + break; + case 0x1E00: + pTarget[0x9E] = (UChar)0x1E9E; // LATIN CAPITAL LETTER SHARP S + break; + case 0x2100: + pTarget[0x26] = (UChar)0x2126; // OHM SIGN + pTarget[0x2A] = (UChar)0x212A; // KELVIN SIGN + pTarget[0x2B] = (UChar)0x212B; // ANGSTROM SIGN + break; + } +} From e8415d65c9ab3f15024edfdd0b354ff6bf9fd7e6 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed <10833894+tarekgh@users.noreply.github.com> Date: Thu, 2 Jul 2026 14:23:56 -0700 Subject: [PATCH 2/8] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../src/System/Globalization/OrdinalCasing.Icu.cs | 4 ++-- .../System/Globalization/OrdinalCasingTests.cs | 6 +++--- src/native/libs/System.Globalization.Native/entrypoints.c | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/OrdinalCasing.Icu.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/OrdinalCasing.Icu.cs index 57c6cb6375a8ae..dbdab5da339b73 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/OrdinalCasing.Icu.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/OrdinalCasing.Icu.cs @@ -43,8 +43,8 @@ internal static partial class OrdinalCasing // Every cell is array of 512 character for uppercasing mapping. private static readonly ushort[]?[] s_casingTable = InitCasingTable(); - // s_lowerBasicLatin is covering the lower casing for the Basic Latin & C0 Controls range. - // we are not lazy initializing this range because it is the most common used range and we'll cache it anyway very early. + // s_lowerBasicLatin covers the lower casing for the Basic Latin & C0 Controls range. + // We don't lazily initialize this range because it is the most commonly used range and we'll cache it anyway very early. private static readonly ushort[] s_lowerBasicLatin = [ // Lower Casing diff --git a/src/libraries/System.Runtime/tests/System.Globalization.Tests/System/Globalization/OrdinalCasingTests.cs b/src/libraries/System.Runtime/tests/System.Globalization.Tests/System/Globalization/OrdinalCasingTests.cs index 730767b7e39560..89a73c68f0e618 100644 --- a/src/libraries/System.Runtime/tests/System.Globalization.Tests/System/Globalization/OrdinalCasingTests.cs +++ b/src/libraries/System.Runtime/tests/System.Globalization.Tests/System/Globalization/OrdinalCasingTests.cs @@ -110,10 +110,10 @@ public void ToLowerOrdinal_EveryBmpChar_AgreesAcrossOverloads() // ToLowerOrdinal must stay consistent with OrdinalIgnoreCase: a character is always // OrdinalIgnoreCase-equal to its ordinal lower-cased form. This guards the special handling that keeps - // characters whose simple lower mapping would leave their class unchanged (for example the Kelvin, Ohm - // and Angstrom signs) instead of folding them into a character of a different class. + // characters unchanged when their simple lower mapping would move them out of their ordinal upper-casing + // equivalence class (for example the Kelvin, Ohm, and Angstrom signs) instead of folding them into a different class. [Fact] - public void ToLowerOrdinal_EveryBmpChar_IsOrdinalIgnoreCaseEqualToItself() + public void ToLowerOrdinal_EveryBmpChar_IsOrdinalIgnoreCaseEqualToItsFold() { for (int c = 0; c <= 0xFFFF; c++) { diff --git a/src/native/libs/System.Globalization.Native/entrypoints.c b/src/native/libs/System.Globalization.Native/entrypoints.c index ffe0347e312786..14a64d3f2b988a 100644 --- a/src/native/libs/System.Globalization.Native/entrypoints.c +++ b/src/native/libs/System.Globalization.Native/entrypoints.c @@ -81,10 +81,10 @@ static const Entry s_globalizationNative[] = DllImportEntry(GlobalizationNative_NormalizeStringNative) DllImportEntry(GlobalizationNative_StartsWithNative) #endif - DllImportEntry(GlobalizationNative_ToAscii) - DllImportEntry(GlobalizationNative_ToUnicode) - DllImportEntry(GlobalizationNative_InitOrdinalCasingPage) - DllImportEntry(GlobalizationNative_InitOrdinalLowerCasingPage) + DllImportEntry(GlobalizationNative_ToAscii) + DllImportEntry(GlobalizationNative_ToUnicode) + DllImportEntry(GlobalizationNative_InitOrdinalCasingPage) + DllImportEntry(GlobalizationNative_InitOrdinalLowerCasingPage) }; EXTERN_C const void* GlobalizationResolveDllImport(const char* name); From 1e4789acbd0f9412394bf68a284a7b0a7ff8441e Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Thu, 2 Jul 2026 14:40:52 -0700 Subject: [PATCH 3/8] Add param and returns XML docs to Rune.ToUpperOrdinal/ToLowerOrdinal --- src/libraries/System.Private.CoreLib/src/System/Text/Rune.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Rune.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Rune.cs index 5af3c1f8740862..61673fc1da8d51 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Rune.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Rune.cs @@ -1596,6 +1596,8 @@ public static Rune ToUpperInvariant(Rune value) /// Returns a copy of converted to uppercase using the casing rules used by /// comparisons. /// + /// The character to convert. + /// The uppercase equivalent of . public static Rune ToUpperOrdinal(Rune value) { if (value.IsAscii) @@ -1615,6 +1617,8 @@ public static Rune ToUpperOrdinal(Rune value) /// /// Returns a copy of converted to lowercase using ordinal (simple, one-to-one) casing rules. /// + /// The character to convert. + /// The lowercase equivalent of . public static Rune ToLowerOrdinal(Rune value) { if (value.IsAscii) From 16dad80eb33fd3bc37928cb8f92117b6ff9be0d6 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Thu, 2 Jul 2026 15:51:32 -0700 Subject: [PATCH 4/8] Keep ToLowerOrdinal OrdinalIgnoreCase-consistent in invariant and NLS modes Invariant and NLS simple lowering could move characters such as the Kelvin, Ohm and Angstrom signs out of their ordinal upper-casing class, so ToLowerOrdinal stopped being consistent with OrdinalIgnoreCase in those modes. Keep the original character when its simple lower mapping would change its ordinal upper-casing form, and extend the invariant-mode test to cover ToLowerOrdinal across the BMP. --- .../src/System/Globalization/Ordinal.cs | 27 +++++++++++++++++++ .../src/System/Globalization/TextInfo.cs | 13 +++++++-- .../Globalization/OrdinalCasingTests.cs | 9 +++++++ 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs index 4a2c2ec64fd875..9aa65e1dc66a9c 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs @@ -720,17 +720,44 @@ internal static int ToLowerOrdinal(ReadOnlySpan source, Span destina if (GlobalizationMode.Invariant) { InvariantModeCasing.ToLower(source, destination); + PreserveOrdinalLowerCasingClass(source, destination); return source.Length; } if (GlobalizationMode.UseNls) { TextInfo.Invariant.ChangeCaseToLower(source, destination); // this is the best so far for NLS. + PreserveOrdinalLowerCasingClass(source, destination); return source.Length; } OrdinalCasing.ToLowerOrdinal(source, destination); return source.Length; } + + // Ordinal lower casing must never move a character out of its ordinal upper-casing class, otherwise it + // would stop being consistent with OrdinalIgnoreCase (for example the Kelvin, Ohm and Angstrom signs). The + // ICU ordinal table encodes this directly, but invariant and NLS simple lowering do not, so restore any + // BMP character whose simple lower mapping changed its ordinal upper-casing form. + private static void PreserveOrdinalLowerCasingClass(ReadOnlySpan source, Span destination) + { + for (int i = 0; i < source.Length; i++) + { + char c = source[i]; + + // The special characters are all single BMP scalars, so surrogate pairs never need adjustment. + if (char.IsHighSurrogate(c) && i + 1 < source.Length && char.IsLowSurrogate(source[i + 1])) + { + i++; + continue; + } + + char lower = destination[i]; + if (lower != c && !char.IsAscii(c) && TextInfo.ToUpperOrdinal(lower) != TextInfo.ToUpperOrdinal(c)) + { + destination[i] = c; + } + } + } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/TextInfo.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/TextInfo.cs index 8d4ebdbc8cd4f7..c3196897f3bcef 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/TextInfo.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/TextInfo.cs @@ -209,19 +209,28 @@ internal static char ToLowerOrdinal(char c) { if (GlobalizationMode.Invariant) { - return InvariantModeCasing.ToLower(c); + return char.IsAscii(c) + ? ToLowerAsciiInvariant(c) + : PreserveOrdinalLowerCasingClass(c, InvariantModeCasing.ToLower(c)); } if (GlobalizationMode.UseNls) { return char.IsAscii(c) ? ToLowerAsciiInvariant(c) - : Invariant.ChangeCase(c, toUpper: false); + : PreserveOrdinalLowerCasingClass(c, Invariant.ChangeCase(c, toUpper: false)); } return OrdinalCasing.ToLower(c); } + // Ordinal lower casing must never move a character out of its ordinal upper-casing class, otherwise it + // would stop being consistent with OrdinalIgnoreCase (for example the Kelvin, Ohm and Angstrom signs). The + // ICU ordinal table encodes this directly, but invariant and NLS simple lowering do not, so keep the original + // character whenever its simple lower mapping would change its ordinal upper-casing form. + private static char PreserveOrdinalLowerCasingClass(char c, char lower) => + lower == c || ToUpperOrdinal(lower) == ToUpperOrdinal(c) ? lower : c; + [MethodImpl(MethodImplOptions.AggressiveInlining)] internal void ChangeCaseToLower(ReadOnlySpan source, Span destination) { diff --git a/src/libraries/System.Runtime/tests/System.Globalization.Tests/System/Globalization/OrdinalCasingTests.cs b/src/libraries/System.Runtime/tests/System.Globalization.Tests/System/Globalization/OrdinalCasingTests.cs index 89a73c68f0e618..b8f5ae7f7c8a21 100644 --- a/src/libraries/System.Runtime/tests/System.Globalization.Tests/System/Globalization/OrdinalCasingTests.cs +++ b/src/libraries/System.Runtime/tests/System.Globalization.Tests/System/Globalization/OrdinalCasingTests.cs @@ -391,6 +391,15 @@ public void OrdinalIgnoreCase_Equivalence_HoldsInInvariantMode() string upper = original.ToUpperOrdinal(); bool selfOic = string.Equals(original, upper, StringComparison.OrdinalIgnoreCase); Assert.True(selfOic); + + // ToLowerOrdinal has its own invariant-mode path (InvariantModeCasing.ToLower) that must keep + // the same OrdinalIgnoreCase consistency, including the characters (Kelvin, Ohm, Angstrom, ...) + // whose simple lower mapping would otherwise move them out of their ordinal upper-casing class. + string lower = original.ToLowerOrdinal(); + Assert.Equal(1, lower.Length); + Assert.True(string.Equals(original, lower, StringComparison.OrdinalIgnoreCase), + $"U+{c:X4} is not OrdinalIgnoreCase-equal to its ToLowerOrdinal fold U+{(int)lower[0]:X4} in invariant mode"); + Assert.Equal(lower[0], char.ToLowerOrdinal((char)c)); } }, options).Dispose(); } From 53931e6d9f0a611c1e2da251cdf084035dc1b489 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Mon, 6 Jul 2026 12:40:49 -0700 Subject: [PATCH 5/8] Fix OrdinalCasingTests failures on NLS and platforms without RemoteExecutor Guard the MICRO SIGN and GREEK SMALL FINAL SIGMA fold cases so they only run under ICU and invariant casing, since NLS does not treat them as OrdinalIgnoreCase-equal. Gate OrdinalIgnoreCase_Equivalence_HoldsInInvariantMode on RemoteExecutor.IsSupported so it is skipped on wasm and android instead of throwing PlatformNotSupportedException. --- .../Globalization/OrdinalCasingTests.cs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/libraries/System.Runtime/tests/System.Globalization.Tests/System/Globalization/OrdinalCasingTests.cs b/src/libraries/System.Runtime/tests/System.Globalization.Tests/System/Globalization/OrdinalCasingTests.cs index b8f5ae7f7c8a21..025962e89fab31 100644 --- a/src/libraries/System.Runtime/tests/System.Globalization.Tests/System/Globalization/OrdinalCasingTests.cs +++ b/src/libraries/System.Runtime/tests/System.Globalization.Tests/System/Globalization/OrdinalCasingTests.cs @@ -20,12 +20,18 @@ public static IEnumerable OrdinalIgnoreCaseEqualPairs() yield return new object[] { "", "" }; yield return new object[] { "abc", "ABC" }; yield return new object[] { "Hello", "hELLO" }; - yield return new object[] { "\u00B5", "\u039C" }; // MICRO SIGN folds to GREEK CAPITAL MU yield return new object[] { "\u017F", "\u017F" }; // LATIN SMALL LETTER LONG S stays itself (no OIC fold to 'S') yield return new object[] { "\u0131", "\u0131" }; // LATIN SMALL LETTER DOTLESS I stays itself (no OIC fold to 'I') yield return new object[] { "\u03C3", "\u03A3" }; // GREEK SMALL SIGMA vs GREEK CAPITAL SIGMA - yield return new object[] { "\u03C2", "\u03A3" }; // GREEK SMALL FINAL SIGMA vs GREEK CAPITAL SIGMA yield return new object[] { "\u0561\u0562", "\u0531\u0532" }; // Armenian small vs capital + + // MICRO SIGN and GREEK SMALL FINAL SIGMA fold to their Greek capitals only under ICU and invariant + // casing. NLS does not treat these as OrdinalIgnoreCase-equal, so only include them when not on NLS. + if (!PlatformDetection.IsNlsGlobalization) + { + yield return new object[] { "\u00B5", "\u039C" }; // MICRO SIGN folds to GREEK CAPITAL MU + yield return new object[] { "\u03C2", "\u03A3" }; // GREEK SMALL FINAL SIGMA vs GREEK CAPITAL SIGMA + } } public static IEnumerable OrdinalIgnoreCaseNotEqualPairs() @@ -180,10 +186,15 @@ public static IEnumerable CharUpperData() yield return new object[] { '\u00E0', '\u00C0' }; // a-grave -> A-grave yield return new object[] { '\u03B1', '\u0391' }; // greek small alpha -> capital alpha yield return new object[] { '\u0450', '\u0400' }; // cyrillic small ie -> capital - yield return new object[] { '\u00B5', '\u039C' }; // MICRO SIGN -> GREEK CAPITAL MU yield return new object[] { '\u212A', '\u212A' }; // KELVIN SIGN is not folded by ordinal upper yield return new object[] { '\u0131', '\u0131' }; // DOTLESS I unchanged yield return new object[] { '\u017F', '\u017F' }; // LONG S unchanged + + // MICRO SIGN folds to GREEK CAPITAL MU only under ICU and invariant casing. NLS leaves it unchanged. + if (!PlatformDetection.IsNlsGlobalization) + { + yield return new object[] { '\u00B5', '\u039C' }; // MICRO SIGN -> GREEK CAPITAL MU + } } public static IEnumerable CharLowerData() @@ -361,7 +372,7 @@ public void Rune_ToLowerOrdinal(Rune input, Rune expected) } } - [Fact] + [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] public void OrdinalIgnoreCase_Equivalence_HoldsInInvariantMode() { var options = new RemoteInvokeOptions(); From 691e2f128d0cb39d384b876ceb85aa3217246c1c Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Mon, 6 Jul 2026 13:46:20 -0700 Subject: [PATCH 6/8] Avoid per-character ToUpperOrdinal in PreserveOrdinalLowerCasingClass Upper-case the source and lowered spans once each instead of calling TextInfo.ToUpperOrdinal per non-ASCII character, which is expensive under NLS. --- .../src/System/Globalization/Ordinal.cs | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs index 9aa65e1dc66a9c..338b64e704bcfe 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Buffers; using System.Diagnostics; using System.Numerics; using System.Runtime.CompilerServices; @@ -741,6 +742,27 @@ internal static int ToLowerOrdinal(ReadOnlySpan source, Span destina // BMP character whose simple lower mapping changed its ordinal upper-casing form. private static void PreserveOrdinalLowerCasingClass(ReadOnlySpan source, Span destination) { + // Upper-case the original text and the lowered result once over the whole span instead of calling + // TextInfo.ToUpperOrdinal per character. Per-character upper-casing is expensive under NLS, where each + // non-ASCII character would otherwise route through a separate invariant ChangeCase call. + const int StackAllocThreshold = 256; + + char[]? sourceUpperRented = null; + char[]? loweredUpperRented = null; + + Span sourceUpper = source.Length <= StackAllocThreshold + ? stackalloc char[StackAllocThreshold] + : (sourceUpperRented = ArrayPool.Shared.Rent(source.Length)); + sourceUpper = sourceUpper.Slice(0, source.Length); + + Span loweredUpper = source.Length <= StackAllocThreshold + ? stackalloc char[StackAllocThreshold] + : (loweredUpperRented = ArrayPool.Shared.Rent(source.Length)); + loweredUpper = loweredUpper.Slice(0, source.Length); + + ToUpperOrdinal(source, sourceUpper); + ToUpperOrdinal(destination.Slice(0, source.Length), loweredUpper); + for (int i = 0; i < source.Length; i++) { char c = source[i]; @@ -752,12 +774,21 @@ private static void PreserveOrdinalLowerCasingClass(ReadOnlySpan source, S continue; } - char lower = destination[i]; - if (lower != c && !char.IsAscii(c) && TextInfo.ToUpperOrdinal(lower) != TextInfo.ToUpperOrdinal(c)) + if (destination[i] != c && !char.IsAscii(c) && sourceUpper[i] != loweredUpper[i]) { destination[i] = c; } } + + if (sourceUpperRented is not null) + { + ArrayPool.Shared.Return(sourceUpperRented); + } + + if (loweredUpperRented is not null) + { + ArrayPool.Shared.Return(loweredUpperRented); + } } } } From 09b10046b5b7cc7c9ff06d4b54e3b70de07836f0 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Mon, 6 Jul 2026 15:19:46 -0700 Subject: [PATCH 7/8] Fast-path PreserveOrdinalLowerCasingClass to the affected characters Only a fixed set of five BMP scalars (Greek capital theta symbol, Latin capital sharp S, Ohm, Kelvin and Angstrom signs) fall out of their ordinal upper-casing class under simple invariant or NLS lowering. Scan the source once with SearchValues and skip the pass entirely when none are present, instead of upper-casing the whole span on every lowering. --- .../src/System/Globalization/Ordinal.cs | 66 ++++++------------- 1 file changed, 21 insertions(+), 45 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs index 338b64e704bcfe..459e771fc3d6af 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs @@ -736,58 +736,34 @@ internal static int ToLowerOrdinal(ReadOnlySpan source, Span destina return source.Length; } + // The only BMP scalars whose simple invariant/NLS lower mapping moves them out of their ordinal + // upper-casing class. Each one is its own ordinal upper-casing (ToUpperOrdinal(c) == c) yet lowers to a + // different letter, so simple lowering would break OrdinalIgnoreCase consistency for them. The ICU ordinal + // table already keeps them unchanged; invariant and NLS lowering do not, so they must be restored. + // U+03F4 GREEK CAPITAL THETA SYMBOL, U+1E9E LATIN CAPITAL LETTER SHARP S, U+2126 OHM SIGN, + // U+212A KELVIN SIGN, U+212B ANGSTROM SIGN. The ToLowerOrdinal_EveryBmpChar tests would fail if this set + // ever became incomplete. + private static readonly SearchValues s_ordinalLowerCasingClassRestoreChars = + SearchValues.Create("\u03F4\u1E9E\u2126\u212A\u212B"); + // Ordinal lower casing must never move a character out of its ordinal upper-casing class, otherwise it - // would stop being consistent with OrdinalIgnoreCase (for example the Kelvin, Ohm and Angstrom signs). The - // ICU ordinal table encodes this directly, but invariant and NLS simple lowering do not, so restore any - // BMP character whose simple lower mapping changed its ordinal upper-casing form. + // would stop being consistent with OrdinalIgnoreCase (for example the Kelvin, Ohm and Angstrom signs). + // Only the fixed set above is ever affected, so skip the pass entirely unless one of them is present. private static void PreserveOrdinalLowerCasingClass(ReadOnlySpan source, Span destination) { - // Upper-case the original text and the lowered result once over the whole span instead of calling - // TextInfo.ToUpperOrdinal per character. Per-character upper-casing is expensive under NLS, where each - // non-ASCII character would otherwise route through a separate invariant ChangeCase call. - const int StackAllocThreshold = 256; - - char[]? sourceUpperRented = null; - char[]? loweredUpperRented = null; - - Span sourceUpper = source.Length <= StackAllocThreshold - ? stackalloc char[StackAllocThreshold] - : (sourceUpperRented = ArrayPool.Shared.Rent(source.Length)); - sourceUpper = sourceUpper.Slice(0, source.Length); - - Span loweredUpper = source.Length <= StackAllocThreshold - ? stackalloc char[StackAllocThreshold] - : (loweredUpperRented = ArrayPool.Shared.Rent(source.Length)); - loweredUpper = loweredUpper.Slice(0, source.Length); - - ToUpperOrdinal(source, sourceUpper); - ToUpperOrdinal(destination.Slice(0, source.Length), loweredUpper); - - for (int i = 0; i < source.Length; i++) - { - char c = source[i]; - - // The special characters are all single BMP scalars, so surrogate pairs never need adjustment. - if (char.IsHighSurrogate(c) && i + 1 < source.Length && char.IsLowSurrogate(source[i + 1])) - { - i++; - continue; - } - - if (destination[i] != c && !char.IsAscii(c) && sourceUpper[i] != loweredUpper[i]) - { - destination[i] = c; - } - } - - if (sourceUpperRented is not null) + int i = source.IndexOfAny(s_ordinalLowerCasingClassRestoreChars); + if (i < 0) { - ArrayPool.Shared.Return(sourceUpperRented); + return; } - if (loweredUpperRented is not null) + // Restore each affected character to its original form; its ordinal lower casing is itself. + for (; i < source.Length; i++) { - ArrayPool.Shared.Return(loweredUpperRented); + if (s_ordinalLowerCasingClassRestoreChars.Contains(source[i])) + { + destination[i] = source[i]; + } } } } From 514da087985f6505a350023bf2eb258f5e64910b Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Tue, 7 Jul 2026 13:00:22 -0700 Subject: [PATCH 8/8] Use Ascii casing and SearchValues in ordinal casing fast paths Fast-path the leading ASCII run of the ICU ToUpperOrdinal/ToLowerOrdinal span helpers with Ascii.ToUpper/ToLower, and replace the hand-rolled unsafe scan in the string ChangeCaseOrdinal fast path with a vectorized IndexOfAnyExcept over a SearchValues of the ASCII characters already in the requested case. --- .../System/Globalization/OrdinalCasing.Icu.cs | 10 ++- .../src/System/String.Manipulation.cs | 72 ++++++++----------- 2 files changed, 38 insertions(+), 44 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/OrdinalCasing.Icu.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/OrdinalCasing.Icu.cs index dbdab5da339b73..d4ce25256b1169 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/OrdinalCasing.Icu.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/OrdinalCasing.Icu.cs @@ -176,7 +176,10 @@ internal static void ToUpperOrdinal(ReadOnlySpan source, Span destin Debug.Assert(!GlobalizationMode.Invariant); Debug.Assert(!GlobalizationMode.UseNls); - for (int i = 0; i < source.Length; i++) + // Quickly upper-case the leading run of ASCII characters, then handle the remainder. + Ascii.ToUpper(source, destination, out int start); + + for (int i = start; i < source.Length; i++) { char c = source[i]; if (c <= '\u00FF') // optimize ASCII/Latin @@ -227,7 +230,10 @@ internal static void ToLowerOrdinal(ReadOnlySpan source, Span destin Debug.Assert(!GlobalizationMode.Invariant); Debug.Assert(!GlobalizationMode.UseNls); - for (int i = 0; i < source.Length; i++) + // Quickly lower-case the leading run of ASCII characters, then handle the remainder. + Ascii.ToLower(source, destination, out int start); + + for (int i = start; i < source.Length; i++) { char c = source[i]; if (c <= '\u00FF') // optimize ASCII/Latin diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs index ca14ce8ce850c6..53b13b146f9ad2 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs @@ -2614,57 +2614,45 @@ public string ToLowerOrdinal() return ChangeCaseOrdinal(toUpper: false); } - private unsafe string ChangeCaseOrdinal(bool toUpper) - { - Debug.Assert(Length > 0); + // All ASCII characters except the lowercase letters 'a'-'z'. The first character outside this set is + // the first one ToUpperOrdinal must change (a lowercase ASCII letter or a non-ASCII character). + private static readonly SearchValues s_asciiExceptLowercase = CreateAsciiExcept('a', 'z'); - int consumed; + // All ASCII characters except the uppercase letters 'A'-'Z'. The first character outside this set is + // the first one ToLowerOrdinal must change (an uppercase ASCII letter or a non-ASCII character). + private static readonly SearchValues s_asciiExceptUppercase = CreateAsciiExcept('A', 'Z'); - // Fast path: scan the leading run of ASCII characters that is already in the - // requested case. If the entire string is ASCII and needs no change, return the - // same instance to avoid an allocation. This mirrors the behavior of - // TextInfo.ChangeCaseCommon used by ToUpper(Invariant)/ToLower. - fixed (char* pSource = this) + private static SearchValues CreateAsciiExcept(char firstLetter, char lastLetter) + { + int letterCount = lastLetter - firstLetter + 1; + Span asciiChars = stackalloc char[128 - letterCount]; + int count = 0; + for (int c = 0; c < 128; c++) { - nuint currIdx = 0; // in chars - - if (Length >= 2) + if ((uint)(c - firstLetter) >= (uint)letterCount) { - nuint lastIndexWhereCanReadTwoChars = (uint)Length - 2; - do - { - // Read 2 chars (one 32-bit integer) at a time. - uint tempValue = Unsafe.ReadUnaligned(pSource + currIdx); - if (!Utf16Utility.AllCharsInUInt32AreAscii(tempValue) || - (toUpper - ? Utf16Utility.UInt32ContainsAnyLowercaseAsciiChar(tempValue) - : Utf16Utility.UInt32ContainsAnyUppercaseAsciiChar(tempValue))) - { - goto MustChange; - } - - currIdx += 2; - } while (currIdx <= lastIndexWhereCanReadTwoChars); + asciiChars[count++] = (char)c; } + } - // If there's a single character left, check it now. - if ((Length & 1) != 0) - { - uint tempValue = pSource[currIdx]; - if (tempValue > 0x7Fu || - (toUpper - ? ((tempValue - 'a') <= (uint)('z' - 'a')) - : ((tempValue - 'A') <= (uint)('Z' - 'A')))) - { - goto MustChange; - } - } + return SearchValues.Create(asciiChars); + } + + private string ChangeCaseOrdinal(bool toUpper) + { + Debug.Assert(Length > 0); + + // Fast path: scan the leading run of ASCII characters that is already in the requested case. + // If the entire string is ASCII and needs no change, return the same instance to avoid an + // allocation. This mirrors the behavior of TextInfo.ChangeCaseCommon used by ToUpper(Invariant)/ToLower. + int consumed = toUpper + ? this.AsSpan().IndexOfAnyExcept(s_asciiExceptLowercase) + : this.AsSpan().IndexOfAnyExcept(s_asciiExceptUppercase); + if (consumed < 0) + { // The whole string is ASCII and already in the requested case. return this; - - MustChange: - consumed = (int)currIdx; } // The leading [0, consumed) chars are ASCII already in the requested case, so copy