From 7bc0abfa4663c10fb533149c498a2bc6a2acc2a7 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Mon, 17 Aug 2026 00:32:53 +0000 Subject: [PATCH 1/2] [JSC] Yarr: regenerate the non-unicode /i canonicalization table with the Unicode 16 and 17 case pairs YarrCanonicalizeUCS2.cpp is a committed file produced by running YarrCanonicalizeUCS2.js in a JS shell, so it carries the case mappings of whatever ICU that shell had. It was last regenerated before the Unicode 16 data existed, while the /iu table (YarrCanonicalizeUnicode.cpp) is generated at build time from ucd/CaseFolding.txt, which is at 17.0.0. As a result the eight BMP simple case pairs added in Unicode 16 and 17 (U+019B/U+A7DC, U+0264/U+A7CB, U+A7CC/U+A7CD, U+A7CE/U+A7CF, U+A7D2/U+A7D3, U+A7D4/U+A7D5, U+A7DA/U+A7DB, U+1C89/U+1C8A) are CanonicalizeUnique under /i but fold under /iu and in String.prototype.toUpperCase. This is the unmodified output of YarrCanonicalizeUCS2.js run under ICU 78.3 (Unicode 17). The only entries that change are the ones for those sixteen code units; the character sets, the range count and the Latin-1 table are unchanged. --- ...xp-ignore-case-unicode-16-17-case-pairs.js | 95 +++++++++++++++++++ .../yarr/YarrCanonicalizeUCS2.cpp | 16 ++-- 2 files changed, 103 insertions(+), 8 deletions(-) create mode 100644 JSTests/stress/regexp-ignore-case-unicode-16-17-case-pairs.js diff --git a/JSTests/stress/regexp-ignore-case-unicode-16-17-case-pairs.js b/JSTests/stress/regexp-ignore-case-unicode-16-17-case-pairs.js new file mode 100644 index 0000000000000..d0bde1d94cca3 --- /dev/null +++ b/JSTests/stress/regexp-ignore-case-unicode-16-17-case-pairs.js @@ -0,0 +1,95 @@ +//@ runDefault("--useRegExpJIT=true") +//@ runNoJIT("--useRegExpJIT=false") + +// Non-unicode /i matching uses the committed table in YarrCanonicalizeUCS2.cpp. +// That table has to contain the BMP simple case pairs added in Unicode 16 and 17; +// /iu (generated from ucd/CaseFolding.txt at build time) already had them. + +function shouldBe(actual, expected, message) { + if (actual !== expected) + throw new Error(message + ": expected " + expected + " but got " + actual); +} + +function escapeForClass(ch) { + return "\\u" + ch.charCodeAt(0).toString(16).padStart(4, "0"); +} + +function checkPair(lower, upper, flags) { + const lo = String.fromCharCode(lower); + const up = String.fromCharCode(upper); + const name = "U+" + lower.toString(16) + "/U+" + upper.toString(16) + " /" + flags; + + for (const [pattern, input] of [[lo, up], [up, lo]]) { + const escaped = escapeForClass(pattern); + shouldBe(new RegExp(escaped, flags).test(input), true, name + " atom"); + shouldBe(new RegExp("^" + escaped + "$", flags).test(input), true, name + " anchored atom"); + shouldBe(new RegExp("[" + escaped + "]", flags).test(input), true, name + " class"); + shouldBe(new RegExp("[^" + escaped + "]", flags).test(input), false, name + " negated class"); + shouldBe(new RegExp("x" + escaped + "y", flags).test("x" + input + "y"), true, name + " inside a longer atom"); + shouldBe(new RegExp("(" + escaped + ")\\1", flags).test(pattern + input), true, name + " backreference"); + shouldBe(("a" + input + "b").replace(new RegExp(escaped, flags), "-"), "a-b", name + " replace"); + } +} + +function checkUnrelated(a, b, flags) { + const name = "U+" + a.toString(16) + " vs U+" + b.toString(16) + " /" + flags; + shouldBe(new RegExp(escapeForClass(String.fromCharCode(a)), flags).test(String.fromCharCode(b)), false, name + " atom"); + shouldBe(new RegExp("[" + escapeForClass(String.fromCharCode(a)) + "]", flags).test(String.fromCharCode(b)), false, name + " class"); +} + +// [lower, upper] +const newPairs = [ + // Unicode 16 + [0x019b, 0xa7dc], // LATIN LETTER LAMBDA WITH STROKE + [0x0264, 0xa7cb], // LATIN LETTER RAMS HORN + [0xa7cd, 0xa7cc], // LATIN LETTER S WITH DIAGONAL STROKE + [0xa7db, 0xa7da], // LATIN LETTER LAMBDA + [0x1c8a, 0x1c89], // CYRILLIC LETTER TJE + // Unicode 17 + [0xa7cf, 0xa7ce], // LATIN LETTER PHARYNGEAL VOICED FRICATIVE + [0xa7d3, 0xa7d2], // LATIN LETTER DOUBLE THORN (capital is new, the small letter is from Unicode 14) + [0xa7d5, 0xa7d4], // LATIN LETTER DOUBLE WYNN (same) +]; + +// Pairs next to the new entries whose table ranges were merged or split by the update. +const neighbouringPairs = [ + [0x019a, 0x023d], // LATIN LETTER L WITH BAR + [0x0263, 0x0194], // LATIN LETTER GAMMA + [0x0265, 0xa78d], // LATIN LETTER TURNED H + [0xa7d1, 0xa7d0], // LATIN LETTER CLOSED INSULAR G + [0xa7d7, 0xa7d6], // LATIN LETTER MIDDLE SCOTS S + [0xa7d9, 0xa7d8], // LATIN LETTER SIGMOID S + [0xa7f6, 0xa7f5], // LATIN LETTER REVERSED HALF H + [0xa64b, 0xa64a], // CYRILLIC LETTER MONOGRAPH UK (also equivalent to U+1C88 below) +]; + +for (let i = 0; i < 50; ++i) { + for (const [lower, upper] of newPairs) { + checkPair(lower, upper, "i"); + checkPair(lower, upper, "iu"); + } + for (const [lower, upper] of neighbouringPairs) { + checkPair(lower, upper, "i"); + checkPair(lower, upper, "iu"); + } + + // U+1C88 CYRILLIC SMALL LETTER UNBLENDED UK is in the same set as U+A64A/U+A64B. + checkPair(0x1c88, 0xa64a, "i"); + checkPair(0x1c88, 0xa64b, "i"); + + // The new pairs must not leak into the code units around them. + checkUnrelated(0x1c8a, 0x1c8b, "i"); // U+1C8B is unassigned + checkUnrelated(0x1c89, 0x1c88, "i"); + checkUnrelated(0xa7cb, 0xa7ca, "i"); // U+A7CA pairs with U+A7C9 + checkUnrelated(0xa7dc, 0xa7dd, "i"); // U+A7DD is unassigned + checkUnrelated(0xa7dc, 0xa7db, "i"); + checkUnrelated(0x019b, 0x019a, "i"); + checkUnrelated(0x0264, 0x0263, "i"); +} + +// Case-insensitive ranges in classes cover the new partners too. +shouldBe(/[\ua7cc-\ua7db]/i.test("\ua7cd"), true, "range containing both halves"); +shouldBe(/[\u0190-\u01a0]/i.test("\ua7dc"), true, "U+A7DC falls into a range holding U+019B"); +shouldBe(/[\ua7c0-\ua7ff]/i.test("\u019b"), true, "U+019B falls into a range holding U+A7DC"); +shouldBe(/[\ua7c0-\ua7ff]/i.test("\u0264"), true, "U+0264 falls into a range holding U+A7CB"); +shouldBe(/[\u1c80-\u1c8f]/i.test("\u1c8a"), true, "range holding both TJE letters"); diff --git a/Source/JavaScriptCore/yarr/YarrCanonicalizeUCS2.cpp b/Source/JavaScriptCore/yarr/YarrCanonicalizeUCS2.cpp index 010a55370de5d..6cc72dba52444 100644 --- a/Source/JavaScriptCore/yarr/YarrCanonicalizeUCS2.cpp +++ b/Source/JavaScriptCore/yarr/YarrCanonicalizeUCS2.cpp @@ -128,7 +128,7 @@ constinit const CanonicalizationRange ucs2RangeInfo[ucs2CanonicalizationRanges] { 0x0197, 0x0197, 0x00d1, CanonicalizeRangeLo }, { 0x0198, 0x0199, 0x0000, CanonicalizeAlternatingAligned }, { 0x019a, 0x019a, 0x00a3, CanonicalizeRangeLo }, - { 0x019b, 0x019b, 0x0000, CanonicalizeUnique }, + { 0x019b, 0x019b, 0xa641, CanonicalizeRangeLo }, { 0x019c, 0x019c, 0x00d3, CanonicalizeRangeLo }, { 0x019d, 0x019d, 0x00d5, CanonicalizeRangeLo }, { 0x019e, 0x019e, 0x0082, CanonicalizeRangeLo }, @@ -193,7 +193,7 @@ constinit const CanonicalizationRange ucs2RangeInfo[ucs2CanonicalizationRanges] { 0x0261, 0x0261, 0xa54b, CanonicalizeRangeLo }, { 0x0262, 0x0262, 0x0000, CanonicalizeUnique }, { 0x0263, 0x0263, 0x00cf, CanonicalizeRangeHi }, - { 0x0264, 0x0264, 0x0000, CanonicalizeUnique }, + { 0x0264, 0x0264, 0xa567, CanonicalizeRangeLo }, { 0x0265, 0x0265, 0xa528, CanonicalizeRangeLo }, { 0x0266, 0x0266, 0xa544, CanonicalizeRangeLo }, { 0x0267, 0x0267, 0x0000, CanonicalizeUnique }, @@ -369,7 +369,8 @@ constinit const CanonicalizationRange ucs2RangeInfo[ucs2CanonicalizationRanges] { 0x1c86, 0x1c86, 0x0013, CanonicalizeSet }, { 0x1c87, 0x1c87, 0x0014, CanonicalizeSet }, { 0x1c88, 0x1c88, 0x0016, CanonicalizeSet }, - { 0x1c89, 0x1c8f, 0x0000, CanonicalizeUnique }, + { 0x1c89, 0x1c8a, 0x0000, CanonicalizeAlternatingUnaligned }, + { 0x1c8b, 0x1c8f, 0x0000, CanonicalizeUnique }, { 0x1c90, 0x1cba, 0x0bc0, CanonicalizeRangeHi }, { 0x1cbb, 0x1cbc, 0x0000, CanonicalizeUnique }, { 0x1cbd, 0x1cbf, 0x0bc0, CanonicalizeRangeHi }, @@ -529,11 +530,10 @@ constinit const CanonicalizationRange ucs2RangeInfo[ucs2CanonicalizationRanges] { 0xa7c5, 0xa7c5, 0xa543, CanonicalizeRangeHi }, { 0xa7c6, 0xa7c6, 0x8a38, CanonicalizeRangeHi }, { 0xa7c7, 0xa7ca, 0x0000, CanonicalizeAlternatingUnaligned }, - { 0xa7cb, 0xa7cf, 0x0000, CanonicalizeUnique }, - { 0xa7d0, 0xa7d1, 0x0000, CanonicalizeAlternatingAligned }, - { 0xa7d2, 0xa7d5, 0x0000, CanonicalizeUnique }, - { 0xa7d6, 0xa7d9, 0x0000, CanonicalizeAlternatingAligned }, - { 0xa7da, 0xa7f4, 0x0000, CanonicalizeUnique }, + { 0xa7cb, 0xa7cb, 0xa567, CanonicalizeRangeHi }, + { 0xa7cc, 0xa7db, 0x0000, CanonicalizeAlternatingAligned }, + { 0xa7dc, 0xa7dc, 0xa641, CanonicalizeRangeHi }, + { 0xa7dd, 0xa7f4, 0x0000, CanonicalizeUnique }, { 0xa7f5, 0xa7f6, 0x0000, CanonicalizeAlternatingUnaligned }, { 0xa7f7, 0xab52, 0x0000, CanonicalizeUnique }, { 0xab53, 0xab53, 0x03a0, CanonicalizeRangeHi }, From f867433cea3a463badab0e5858af49b788468605 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Mon, 17 Aug 2026 01:49:50 +0000 Subject: [PATCH 2/2] Test: make the class range assertions depend on the folding The tested character now lies outside each range, so the assertions only pass when the range's partner is canonicalized into the class. --- .../regexp-ignore-case-unicode-16-17-case-pairs.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/JSTests/stress/regexp-ignore-case-unicode-16-17-case-pairs.js b/JSTests/stress/regexp-ignore-case-unicode-16-17-case-pairs.js index d0bde1d94cca3..9ae738da2ab9d 100644 --- a/JSTests/stress/regexp-ignore-case-unicode-16-17-case-pairs.js +++ b/JSTests/stress/regexp-ignore-case-unicode-16-17-case-pairs.js @@ -87,9 +87,10 @@ for (let i = 0; i < 50; ++i) { checkUnrelated(0x0264, 0x0263, "i"); } -// Case-insensitive ranges in classes cover the new partners too. -shouldBe(/[\ua7cc-\ua7db]/i.test("\ua7cd"), true, "range containing both halves"); -shouldBe(/[\u0190-\u01a0]/i.test("\ua7dc"), true, "U+A7DC falls into a range holding U+019B"); -shouldBe(/[\ua7c0-\ua7ff]/i.test("\u019b"), true, "U+019B falls into a range holding U+A7DC"); -shouldBe(/[\ua7c0-\ua7ff]/i.test("\u0264"), true, "U+0264 falls into a range holding U+A7CB"); -shouldBe(/[\u1c80-\u1c8f]/i.test("\u1c8a"), true, "range holding both TJE letters"); +// Case-insensitive ranges in classes cover the new partners too. Each tested +// character lies outside the range and can only match through its partner. +shouldBe(/[\ua7cc-\ua7da]/i.test("\ua7db"), true, "U+A7DB matches through U+A7DA at the end of the range"); +shouldBe(/[\u0190-\u01a0]/i.test("\ua7dc"), true, "U+A7DC matches through U+019B inside the range"); +shouldBe(/[\ua7c0-\ua7ff]/i.test("\u019b"), true, "U+019B matches through U+A7DC inside the range"); +shouldBe(/[\ua7c0-\ua7ff]/i.test("\u0264"), true, "U+0264 matches through U+A7CB inside the range"); +shouldBe(/[\u1c80-\u1c89]/i.test("\u1c8a"), true, "U+1C8A matches through U+1C89 at the end of the range");