url: accept Unicode 16.0 IDNA-mapped hostnames and reject invalid Punycode labels

scripts/regenerate-uts46-override.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Regenerates src/jsc/bindings/icu_uts46_override.nrm: a Unicode 16.0 UTS #46
+# uts46.nrm in Nrm2 format v4 (readable by the bundled ICU 73/75), swapped in at
+# runtime by bun_icu_decompress.cpp. Drop once the WebKit prebuilt ships ICU 76+.
+set -euo pipefail
+
+ICU_TOOLCHAIN_TAG=release-75-1   # whose gennorm2 to use (emits Nrm2 format v4)
+ICU_TOOLCHAIN_SRC=icu4c-75_1-src.tgz
+ICU_TOOLCHAIN_SHA256=cb968df3e4d2e87e8b11c49a5d01c787bd13b9545280fc6642f826527618caef
+ICU_DATA_TAG=release-76-1        # whose norm2/uts46.txt to compile
+ICU_DATA_SHA256=fda2c1c636d71db2cc685ca7671aff8efa27e83d84ee322cc3ce1375e53300d8
+
+REPO_ROOT="$(cd -- "$(dirname -- "$0")/.." && pwd)"
+OUT="$REPO_ROOT/src/jsc/bindings/icu_uts46_override.nrm"
+WORK="$(mktemp -d)"
+trap 'rm -rf "$WORK"' EXIT
+
+fetch() {
+  curl -fL --proto '=https' --tlsv1.2 "$1" -o "$2"
+  echo "$3  $2" | sha256sum -c -
+}
+
+fetch "https://github.com/unicode-org/icu/releases/download/${ICU_TOOLCHAIN_TAG}/${ICU_TOOLCHAIN_SRC}" \
+  "$WORK/icu.tgz" "$ICU_TOOLCHAIN_SHA256"
+tar -xzf "$WORK/icu.tgz" -C "$WORK"
+pushd "$WORK/icu/source" >/dev/null
+./configure --enable-static --disable-shared --with-data-packaging=archive \
+  --disable-samples --disable-tests --disable-extras --disable-icuio >/dev/null
+make -j"$(nproc)" >/dev/null
+popd >/dev/null
+
+fetch "https://raw.githubusercontent.com/unicode-org/icu/${ICU_DATA_TAG}/icu4c/source/data/unidata/norm2/uts46.txt" \
+  "$WORK/icu/source/data/unidata/norm2/uts46.txt" "$ICU_DATA_SHA256"
+
+LD_LIBRARY_PATH="$WORK/icu/source/lib:$WORK/icu/source/stubdata" \
+  "$WORK/icu/source/bin/gennorm2" -o "$OUT" \
+  -s "$WORK/icu/source/data/unidata/norm2" nfc.txt uts46.txt
+
+fmt=$(od -An -t u1 -j 16 -N 1 "$OUT" | tr -d ' ')
+[ "$fmt" = "4" ] || { echo "error: $OUT has Nrm2 format version $fmt, expected 4" >&2; exit 1; }
+
+echo "wrote $OUT ($(wc -c <"$OUT") bytes, md5 $(md5sum "$OUT" | cut -d' ' -f1))"

src/jsc/bindings/BunIDNA.cpp

@@ -0,0 +1,61 @@
+#include "root.h"
+#include "BunIDNA.h"
+
+#include <unicode/uidna.h>
+#include <wtf/URL.h>
+#include <wtf/URLParser.h>
+#include <wtf/Vector.h>
+#include <wtf/text/StringView.h>
+#include <wtf/text/WTFString.h>
+
+namespace Bun {
+
+bool domainHasACELabel(WTF::StringView domain)
+{
+    unsigned labelStart = 0;
+    while (true) {
+        if (domain.substring(labelStart).startsWithIgnoringASCIICase("xn--"_s))
+            return true;
+        size_t dot = domain.find('.', labelStart);
+        if (dot == WTF::notFound)
+            return false;
+        labelStart = static_cast<unsigned>(dot) + 1;
+    }
+}
+
+WTF::String domainToASCII(WTF::StringView domain)
+{
+    std::array<char16_t, WTF::URLParser::hostnameBufferLength> stackBuffer;
+    WTF::Vector<char16_t> heapBuffer;
+    std::span<char16_t> buffer { stackBuffer };
+    while (true) {
+        UErrorCode error = U_ZERO_ERROR;
+        UIDNAInfo processingDetails = UIDNA_INFO_INITIALIZER;
+        int32_t length = uidna_nameToASCII(&WTF::URLParser::internationalDomainNameTranscoder(), domain.upconvertedCharacters(), domain.length(), buffer.data(), static_cast<int32_t>(buffer.size()), &processingDetails, &error);
+        if (U_SUCCESS(error) && !(processingDetails.errors & ~WTF::URLParser::allowedNameToASCIIErrors) && length > 0)
+            return WTF::String { buffer.first(static_cast<size_t>(length)) };
+        // ICU's preflight convention: on overflow, `length` is the required
+        // size. Retry once so a domain longer than the stack buffer (a host
+        // the spec places no length limit on) is not treated as invalid.
+        if (error != U_BUFFER_OVERFLOW_ERROR || length <= 0 || !heapBuffer.isEmpty())
+            return {};
+        heapBuffer.grow(static_cast<size_t>(length));
+        buffer = heapBuffer.mutableSpan();
+    }
+}
+
+bool urlHostIsValidIDNA(const WTF::URL& url)
+{
+    // Only special-scheme URLs have a domain host; the host of every other
+    // scheme is opaque and the URL Standard does not apply IDNA to it.
+    if (!url.hasSpecialScheme())
+        return true;
+    // A parsed special host is already ASCII; UTS-46 "domain to ASCII" can
+    // only still fail on it when a label uses the Punycode "xn--" prefix.
+    auto host = url.host();
+    if (!domainHasACELabel(host))
+        return true;
+    return !domainToASCII(host).isNull();
+}
+
+} // namespace Bun

src/jsc/bindings/BunIDNA.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include "root.h"
+
+#include <wtf/Forward.h>
+
+namespace Bun {
+
+// Whether any dot-separated label of `domain` starts with the Punycode ACE
+// prefix "xn--" (ASCII case-insensitive). Those are the only all-ASCII
+// labels that UTS-46 "domain to ASCII" can reject.
+bool domainHasACELabel(WTF::StringView domain);
+
+// UTS-46 "domain to ASCII" with the URL Standard's options (beStrict = false):
+// https://url.spec.whatwg.org/#concept-domain-to-ascii
+// Returns a null String when `domain` is not a valid IDNA domain.
+WTF::String domainToASCII(WTF::StringView domain);
+
+// WTF::URLParser never runs UTS-46 on an all-ASCII host, so an "xn--" label
+// that does not decode to a valid IDNA label parses successfully. The URL
+// Standard's host parser requires that to fail; re-check such hosts here.
+bool urlHostIsValidIDNA(const WTF::URL&);
+
+} // namespace Bun

src/jsc/bindings/DOMURL.cpp

@@ -29,6 +29,7 @@
 #include "ActiveDOMObject.h"
 // #include "Blob.h"
 // #include "BlobURL.h"
+#include "BunIDNA.h"
 // #include "MemoryCache.h"
 // #include "PublicURLManager.h"
 // #include "ResourceRequest.h"
@@ -55,6 +56,14 @@ static inline String redact(const String& input)
     return makeString('"', input, '"');
 }
 
+// WTF::URLParser skips UTS-46 validation for all-ASCII hosts, so "xn--"
+// (Punycode) labels that are not valid IDNA parse successfully; the URL
+// Standard's host parser requires them to fail. Re-check them here.
+static bool isValidCompleteURL(const URL& url)
+{
+    return url.isValid() && Bun::urlHostIsValidIDNA(url);
+}
+
 inline DOMURL::DOMURL(URL&& completeURL)
     : m_url(WTF::move(completeURL))
     , m_initialURLCostForGC(static_cast<uint16_t>(std::min<size_t>(m_url.string().impl()->costDuringGC(), std::numeric_limits<uint16_t>::max())))
@@ -65,24 +74,25 @@ inline DOMURL::DOMURL(URL&& completeURL)
 ExceptionOr<Ref<DOMURL>> DOMURL::create(const String& url)
 {
     URL completeURL { url };
-    if (!completeURL.isValid())
+    if (!isValidCompleteURL(completeURL))
         return Exception { InvalidURLError, makeString(redact(url), " cannot be parsed as a URL."_s) };
     return adoptRef(*new DOMURL(WTF::move(completeURL)));
 }
 
 ExceptionOr<Ref<DOMURL>> DOMURL::create(const String& url, const URL& base)
 {
-    ASSERT(base.isValid() || base.isNull());
+    // Private overload: create(url, String base), the only caller, already rejected an invalid base.
+    ASSERT(base.isNull() || isValidCompleteURL(base));
     URL completeURL { base, url };
-    if (!completeURL.isValid())
+    if (!isValidCompleteURL(completeURL))
         return Exception { InvalidURLError, makeString(redact(url), " cannot be parsed as a URL."_s) };
     return adoptRef(*new DOMURL(WTF::move(completeURL)));
 }
 
 ExceptionOr<Ref<DOMURL>> DOMURL::create(const String& url, const String& base)
 {
     URL baseURL { base };
-    if (!base.isNull() && !baseURL.isValid())
+    if (!base.isNull() && !isValidCompleteURL(baseURL))
         return Exception { InvalidURLError, makeString(redact(url), " cannot be parsed as a URL against "_s, redact(base)) };
     return create(url, baseURL);
 }
@@ -92,9 +102,12 @@ DOMURL::~DOMURL() = default;
 static URL parseInternal(const String& url, const String& base)
 {
     URL baseURL { base };
-    if (!base.isNull() && !baseURL.isValid())
+    if (!base.isNull() && !isValidCompleteURL(baseURL))
+        return {};
+    URL completeURL { baseURL, url };
+    if (!isValidCompleteURL(completeURL))
         return {};
-    return { baseURL, url };
+    return completeURL;
 }
 
 RefPtr<DOMURL> DOMURL::parse(const String& url, const String& base)
@@ -113,10 +126,8 @@ bool DOMURL::canParse(const String& url, const String& base)
 ExceptionOr<void> DOMURL::setHref(const String& url)
 {
     URL completeURL { URL {}, url };
-    if (!completeURL.isValid()) {
-
+    if (!isValidCompleteURL(completeURL))
         return Exception { InvalidURLError, makeString(redact(url), " cannot be parsed as a URL."_s) };
-    }
     m_url = WTF::move(completeURL);
     if (m_searchParams)
         m_searchParams->updateFromAssociatedURL();

src/jsc/bindings/NodeURL.cpp

@@ -1,4 +1,5 @@
 #include "NodeURL.h"
+#include "BunIDNA.h"
 #include "wtf/URLParser.h"
 #include <unicode/uidna.h>
 
@@ -50,25 +51,15 @@ JSC_DEFINE_HOST_FUNCTION(jsDomainToASCII, (JSC::JSGlobalObject * globalObject, J
     )
         return JSC::JSValue::encode(jsEmptyString(vm));
 
-    if (domain.containsOnlyASCII())
+    // An "xn--" (ACE) label must decode to a valid IDNA label, so it cannot
+    // take the ASCII fast path: https://url.spec.whatwg.org/#concept-domain-to-ascii
+    if (domain.containsOnlyASCII() && !domainHasACELabel(domain))
         return JSC::JSValue::encode(arg0);
-    if (domain.is8Bit())
-        domain.convertTo16Bit();
 
-    constexpr static int allowedNameToASCIIErrors = UIDNA_ERROR_EMPTY_LABEL | UIDNA_ERROR_LABEL_TOO_LONG | UIDNA_ERROR_DOMAIN_NAME_TOO_LONG | UIDNA_ERROR_LEADING_HYPHEN | UIDNA_ERROR_TRAILING_HYPHEN | UIDNA_ERROR_HYPHEN_3_4;
-    constexpr static size_t hostnameBufferLength = 2048;
-
-    auto encoder = &WTF::URLParser::internationalDomainNameTranscoder();
-    char16_t hostnameBuffer[hostnameBufferLength];
-    UErrorCode error = U_ZERO_ERROR;
-    UIDNAInfo processingDetails = UIDNA_INFO_INITIALIZER;
-    const auto span = domain.span16();
-    int32_t numCharactersConverted = uidna_nameToASCII(encoder, span.data(), span.size(), hostnameBuffer, hostnameBufferLength, &processingDetails, &error);
-
-    if (U_SUCCESS(error) && !(processingDetails.errors & ~allowedNameToASCIIErrors) && numCharactersConverted) {
-        return JSC::JSValue::encode(JSC::jsString(vm, WTF::String(std::span { hostnameBuffer, static_cast<unsigned int>(numCharactersConverted) })));
-    }
-    return JSC::JSValue::encode(jsEmptyString(vm));
+    auto ascii = domainToASCII(domain);
+    if (ascii.isNull())
+        return JSC::JSValue::encode(jsEmptyString(vm));
+    return JSC::JSValue::encode(JSC::jsString(vm, ascii));
 }
 
 JSC_DEFINE_HOST_FUNCTION(jsDomainToUnicode, (JSC::JSGlobalObject * globalObject, JSC::CallFrame* callFrame))

src/jsc/bindings/URLDecomposition.cpp

@@ -25,6 +25,7 @@
 
 #include "URLDecomposition.h"
 
+#include "BunIDNA.h"
 #include <wtf/text/StringToIntegerConversion.h>
 
 namespace WebCore {
@@ -38,7 +39,9 @@ String URLDecomposition::origin() const
     if (fullURL.protocolIsBlob()) {
         const String& path = fullURL.path().toString();
         const URL subUrl { URL {}, path };
-        if (subUrl.isValid()) {
+        // An inner host with an invalid Punycode label makes this parse fail
+        // per the spec, which yields an opaque ("null") origin.
+        if (subUrl.isValid() && Bun::urlHostIsValidIDNA(subUrl)) {
             if (subUrl.protocolIsInHTTPFamily() or subUrl.protocolIsInFTPFamily() or subUrl.protocolIs("ws"_s) or subUrl.protocolIs("wss"_s) or subUrl.protocolIsFile())
                 return subUrl.protocolHostAndPort();
         }

src/jsc/bindings/bun_icu_decompress.cpp

@@ -46,6 +46,42 @@ extern "C" __attribute__((weak)) const unsigned int bun_icu_zstd_dict_size;
 
 namespace Bun {
 
+// Replacement uts46.nrm carrying the Unicode 16.0 IdnaMappingTable (UTS #46
+// rev. 33) in Nrm2 format version 4, readable by the ICU 73/75 the prebuilts
+// bundle. Regenerate via scripts/regenerate-uts46-override.sh.
+alignas(16) static constexpr uint8_t s_uts46Override[] = {
+#embed "icu_uts46_override.nrm"
+};
+
+// The bundled prebuilts' uts46.nrm predates Unicode 16.0, which reclassified
+// U+04C0, U+10A0..10C5, U+2132, U+2183 et al. from "disallowed" to "mapped".
+// Match by 48-byte prefix (DataHeader + first four Nrm2 indexes, unique per *.nrm).
+static const void* maybeOverrideUTS46(const void* p, int32_t* length)
+{
+    // clang-format off
+    static constexpr uint8_t kUTS46Prefix75[48] = {
+        0x20, 0x00, 0xda, 0x27, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x4e, 0x72, 0x6d, 0x32,
+        0x04, 0x00, 0x00, 0x00, 0x0f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x50, 0x00, 0x00, 0x00, 0xc0, 0x93, 0x00, 0x00, 0x8c, 0xe8, 0x00, 0x00, 0x8c, 0xe9, 0x00, 0x00,
+    };
+    static constexpr uint8_t kUTS46Prefix73[48] = {
+        0x20, 0x00, 0xda, 0x27, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x4e, 0x72, 0x6d, 0x32,
+        0x04, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x50, 0x00, 0x00, 0x00, 0x84, 0x93, 0x00, 0x00, 0x4c, 0xe8, 0x00, 0x00, 0x4c, 0xe9, 0x00, 0x00,
+    };
+    // clang-format on
+    static_assert(s_uts46Override[12] == 'N' && s_uts46Override[13] == 'r' && s_uts46Override[16] == 4,
+        "icu_uts46_override.nrm must be Nrm2 format version 4");
+
+    if (*length >= static_cast<int32_t>(sizeof(kUTS46Prefix75))
+        && (std::memcmp(p, kUTS46Prefix75, sizeof(kUTS46Prefix75)) == 0
+            || std::memcmp(p, kUTS46Prefix73, sizeof(kUTS46Prefix73)) == 0)) {
+        *length = static_cast<int32_t>(sizeof(s_uts46Override));
+        return s_uts46Override;
+    }
+    return p;
+}
+
 class ICUDecompressor {
 public:
     static ICUDecompressor& get()
@@ -119,7 +155,7 @@ extern "C" const void* bun_icu_maybe_decompress(const void* p, int32_t* length)
     uint32_t magic;
     std::memcpy(&magic, p, sizeof(magic));
     if (magic != ZSTD_MAGICNUMBER) [[likely]]
-        return p;
+        return Bun::maybeOverrideUTS46(p, length);
     return Bun::ICUDecompressor::get().decompress(p, length);
 }
 

test/js/node/url/url-domain-ascii-unicode.test.js

@@ -99,3 +99,27 @@ describe("url.domainToUnicode", () => {
     });
   }
 });
+
+// https://url.spec.whatwg.org/#concept-domain-to-ascii
+// An "xn--" (ACE) label must decode to a valid IDNA label; "xn--a-ecp"
+// decodes to U+0061 U+2488 (DIGIT ONE FULL STOP), which is disallowed.
+describe("url.domainToASCII and invalid Punycode (xn--) labels", () => {
+  const invalidPunycode = ["xn--a-ecp.example", "sub.xn--a-ecp.example", "XN--A-ECP.example", "xn--pokxncvks", "xn--"];
+  for (const domain of invalidPunycode) {
+    test(`-> '${domain}' is ''`, () => {
+      expect(url.domainToASCII(domain)).toBe("");
+    });
+  }
+  test("a valid ACE label is preserved and lowercased", () => {
+    expect(url.domainToASCII("XN--BCHER-KVA.DE")).toBe("xn--bcher-kva.de");
+    expect(url.domainToASCII("xn--fiqs8s")).toBe("xn--fiqs8s");
+  });
+  test("a label that merely contains 'xn--' is not an ACE label", () => {
+    expect(url.domainToASCII("axn--a-ecp.example")).toBe("axn--a-ecp.example");
+  });
+  test("a domain with an ACE label longer than ICU's stack buffer still round-trips", () => {
+    // 3017 code units forces the U_BUFFER_OVERFLOW_ERROR retry in Bun::domainToASCII.
+    const domain = "xn--bcher-kva." + Buffer.alloc(3000, "a").toString() + ".de";
+    expect(url.domainToASCII(domain)).toBe(domain);
+  });
+});