url: accept Unicode 16.0 IDNA-mapped hostnames and reject invalid Punycode labels

scripts/regenerate-uts46-override.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# Regenerates src/jsc/bindings/icu_uts46_override.nrm.
+#
+# The bundled ICU prebuilts (ICU 75.1 on Linux/musl, ICU 73.2 on Windows) ship
+# uts46.nrm data derived from Unicode 15.x's IdnaMappingTable.txt, which marks
+# several late-casefolded capitals as "disallowed" (U+04C0, U+10A0..10C5,
+# U+2132, U+2183, ...). Unicode 16.0's IdnaMappingTable.txt changed these to
+# "mapped". This script produces a uts46.nrm carrying the Unicode 16.0 mappings
+# in Nrm2 format version 4 (the format ICU 73/75's Normalizer2 reads), which
+# bun_icu_decompress.cpp swaps in at runtime via the udata hook.
+#
+# Delete the override once the oven-sh/WebKit prebuilt bundles ICU 76 or later.
+set -euo pipefail
+
+ICU_TOOLCHAIN_TAG=release-75-1   # whose gennorm2 to use (emits Nrm2 format v4)
+ICU_TOOLCHAIN_SRC=icu4c-75_1-src.tgz
+ICU_DATA_TAG=release-76-1        # whose norm2/uts46.txt to compile
+
+cd "$(dirname "$0")/.."
+OUT="src/jsc/bindings/icu_uts46_override.nrm"
+WORK="$(mktemp -d)"
+trap 'rm -rf "$WORK"' EXIT
+
+curl -fL "https://github.com/unicode-org/icu/releases/download/${ICU_TOOLCHAIN_TAG}/${ICU_TOOLCHAIN_SRC}" \
+  | tar -xz -C "$WORK"
+pushd "$WORK/icu/source" >/dev/null
+./configure --enable-static --disable-shared --with-data-packaging=archive \
+  --disable-samples --disable-tests --disable-extras --disable-icuio >/dev/null
+make -j"$(nproc)" >/dev/null
+popd >/dev/null
+
+curl -fL "https://raw.githubusercontent.com/unicode-org/icu/${ICU_DATA_TAG}/icu4c/source/data/unidata/norm2/uts46.txt" \
+  -o "$WORK/icu/source/data/unidata/norm2/uts46.txt"
+
+LD_LIBRARY_PATH="$WORK/icu/source/lib:$WORK/icu/source/stubdata" \
+  "$WORK/icu/source/bin/gennorm2" -o "$OUT" \
+  -s "$WORK/icu/source/data/unidata/norm2" nfc.txt uts46.txt
+
+# Format-version sanity check: byte 16 must be 4.
+fmt=$(od -An -t u1 -j 16 -N 1 "$OUT" | tr -d ' ')
+if [ "$fmt" != "4" ]; then
+  echo "error: generated $OUT has Nrm2 format version $fmt, expected 4" >&2
+  exit 1
+fi
+
+echo "wrote $OUT ($(wc -c <"$OUT") bytes)"

src/jsc/bindings/bun_icu_decompress.cpp

@@ -46,6 +46,42 @@ extern "C" __attribute__((weak)) const unsigned int bun_icu_zstd_dict_size;
 
 namespace Bun {
 
+// Replacement uts46.nrm carrying the Unicode 16.0 IdnaMappingTable (UTS #46
+// rev. 33) in Nrm2 format version 4, readable by the ICU 73/75 the prebuilts
+// bundle. Regenerate via scripts/regenerate-uts46-override.sh.
+alignas(16) static constexpr uint8_t s_uts46Override[] = {
+#embed "icu_uts46_override.nrm"
+};
+
+// The bundled prebuilts' uts46.nrm predates Unicode 16.0, which reclassified
+// U+04C0, U+10A0..10C5, U+2132, U+2183 et al. from "disallowed" to "mapped".
+// Match by 48-byte prefix (DataHeader + first four Nrm2 indexes, unique per *.nrm).
+static const void* maybeOverrideUTS46(const void* p, int32_t* length)
+{
+    // clang-format off
+    static constexpr uint8_t kUTS46Prefix75[48] = {
+        0x20, 0x00, 0xda, 0x27, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x4e, 0x72, 0x6d, 0x32,
+        0x04, 0x00, 0x00, 0x00, 0x0f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x50, 0x00, 0x00, 0x00, 0xc0, 0x93, 0x00, 0x00, 0x8c, 0xe8, 0x00, 0x00, 0x8c, 0xe9, 0x00, 0x00,
+    };
+    static constexpr uint8_t kUTS46Prefix73[48] = {
+        0x20, 0x00, 0xda, 0x27, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x4e, 0x72, 0x6d, 0x32,
+        0x04, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x50, 0x00, 0x00, 0x00, 0x84, 0x93, 0x00, 0x00, 0x4c, 0xe8, 0x00, 0x00, 0x4c, 0xe9, 0x00, 0x00,
+    };
+    // clang-format on
+    static_assert(s_uts46Override[12] == 'N' && s_uts46Override[13] == 'r' && s_uts46Override[16] == 4,
+        "icu_uts46_override.nrm must be Nrm2 format version 4");
+
+    if (*length >= static_cast<int32_t>(sizeof(kUTS46Prefix75))
+        && (std::memcmp(p, kUTS46Prefix75, sizeof(kUTS46Prefix75)) == 0
+            || std::memcmp(p, kUTS46Prefix73, sizeof(kUTS46Prefix73)) == 0)) {
+        *length = static_cast<int32_t>(sizeof(s_uts46Override));
+        return s_uts46Override;
+    }
+    return p;
+}
+
 class ICUDecompressor {
 public:
     static ICUDecompressor& get()
@@ -119,7 +155,7 @@ extern "C" const void* bun_icu_maybe_decompress(const void* p, int32_t* length)
     uint32_t magic;
     std::memcpy(&magic, p, sizeof(magic));
     if (magic != ZSTD_MAGICNUMBER) [[likely]]
-        return p;
+        return Bun::maybeOverrideUTS46(p, length);
     return Bun::ICUDecompressor::get().decompress(p, length);
 }
 

test/js/web/url/url.test.ts

@@ -1,6 +1,7 @@
 import { describe, expect, it, test } from "bun:test";
-import { bunEnv, bunExe } from "harness";
+import { bunEnv, bunExe, isLinux, isWindows } from "harness";
 import { resolveObjectURL } from "node:buffer";
+import { domainToASCII } from "node:url";
 
 describe("url", () => {
   it("URL throws", () => {
@@ -332,3 +333,42 @@ describe("object URL prefix check", () => {
     });
   }, 60_000);
 });
+
+// Unicode 16.0 UTS #46 reclassified several codepoints from "disallowed" to
+// "mapped"/"ignored"; the bundled ICU predates that. The uts46.nrm override in
+// bun_icu_decompress.cpp only applies on Linux/Windows (udata hook platforms).
+describe.skipIf(!(isLinux || isWindows))("IDNA UTS #46 Unicode 16.0 mappings", () => {
+  it("maps late-casefolded capitals instead of rejecting them", () => {
+    expect(
+      [
+        "https://\u04C0/", // CYRILLIC LETTER PALOCHKA
+        "https://\u10AC/", // GEORGIAN CAPITAL LETTER NAR
+        "https://a\u10B5/", // a + GEORGIAN CAPITAL LETTER KHAR
+        "https://\u2132/", // TURNED CAPITAL F
+        "https://\u2183/", // ROMAN NUMERAL REVERSED ONE HUNDRED
+        "https://\u1874\u10A0/", // MONGOLIAN LETTER ... + GEORGIAN CAPITAL AN
+        "https://\uA846\u3002\u2183\u0FB5\uB1AE-/", // WPT IdnaTestV2 "V3 (ignored)" case
+      ].map(href => new URL(href).hostname),
+    ).toEqual(["xn--s5a", "xn--3kj", "xn--a-hws", "xn--73g", "xn--r5g", "xn--h9e436h", "xn--fc9a.xn----qmg097k469k"]);
+  });
+
+  it("ignores format controls reclassified in Unicode 16.0", () => {
+    expect(
+      [
+        "https://a\u180Eb/", // MONGOLIAN VOWEL SEPARATOR
+        "https://a\u2061b/", // FUNCTION APPLICATION
+        "https://a\u3164b/", // HANGUL FILLER
+      ].map(href => new URL(href).hostname),
+    ).toEqual(["ab", "ab", "ab"]);
+  });
+
+  it("node:url domainToASCII applies the same mapping", () => {
+    expect(domainToASCII("\u10AC")).toBe("xn--3kj");
+    expect(domainToASCII("\u2183")).toBe("xn--r5g");
+  });
+
+  it("still maps pre-existing UTS #46 cases", () => {
+    expect(new URL("https://M\u00FCnchen.de/").hostname).toBe("xn--mnchen-3ya.de");
+    expect(new URL("https://\u0391/").hostname).toBe("xn--mxa");
+  });
+});