From 816f8930e74cc710858648911f88286c4964f689 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Tue, 18 Aug 2026 00:19:39 +0000 Subject: [PATCH] test: add forceUTF16 harness helper; the slice-off-a-wide-char idiom yields 8-bit strings Several tests forced 16-bit string storage with (s + "\u0100").slice(0, -1) or the same with an emoji. JSC serves that slice straight from the rope's first fiber, which is the original 8-bit string, so these "16-bit path" tests were running the 8-bit path a second time. forceUTF16 builds the string through a utf16le round trip and asserts it really is 16-bit; the five call sites now use it. --- test/harness.ts | 21 +++++++++++++++++++++ test/js/bun/util/sliceAnsi-fuzz.test.ts | 9 ++++----- test/js/bun/util/sliceAnsi.test.ts | 8 +++----- test/js/node/buffer.test.js | 10 ++++------ test/js/web/fetch/headers.test.ts | 8 +++----- 5 files changed, 35 insertions(+), 21 deletions(-) diff --git a/test/harness.ts b/test/harness.ts index b3167919ec84..fd2db2b7496a 100644 --- a/test/harness.ts +++ b/test/harness.ts @@ -1606,6 +1606,27 @@ String.prototype.isUTF16 = function () { return require("bun:internal-for-testing").jscInternals.isUTF16String(this); }; +/** + * Returns a copy of `s` stored as a 16-bit (UTF-16) JSC string, so a test can + * run the 16-bit code path of an API on content that would otherwise be stored + * as Latin-1. Throws if the result is not actually 16-bit. + * + * `(s + "\u0100").slice(0, -1)` does not do this: slicing a rope whose first + * fiber covers the whole range hands back that fiber, i.e. the original 8-bit + * string. + * + * Strings shorter than 2 code units cannot be forced (JSC interns the empty + * string and single Latin-1 characters as 8-bit) and are returned as-is. + */ +export function forceUTF16(s: string): string { + if (s.length < 2) return s; + const out = Buffer.from(s, "utf16le").toString("utf16le"); + if (!out.isUTF16()) { + throw new Error(`forceUTF16: ${JSON.stringify(s)} was not stored as a 16-bit string`); + } + return out; +} + interface BunHarnessTestMatchers { toBeLatin1String(): void; toBeUTF16String(): void; diff --git a/test/js/bun/util/sliceAnsi-fuzz.test.ts b/test/js/bun/util/sliceAnsi-fuzz.test.ts index 1bd831ab612b..f62e84acce29 100644 --- a/test/js/bun/util/sliceAnsi-fuzz.test.ts +++ b/test/js/bun/util/sliceAnsi-fuzz.test.ts @@ -2,7 +2,7 @@ // These complement sliceAnsi.test.ts with property-based and adversarial cases. import { describe, expect, test } from "bun:test"; -import { isASAN, isDebug } from "harness"; +import { forceUTF16, isASAN, isDebug } from "harness"; // Seeded PRNG for reproducibility. Change seed to explore different cases. function makeRng(seed: number) { @@ -572,9 +572,8 @@ describe("sliceAnsi encoding equivalence", () => { for (let i = 0; i < 50; i++) { // Build a string that COULD be Latin-1 (all < 0x100). const latin1 = randomAnsiAscii(rng, 10, 50); - // Force to UTF-16 by concatenating then removing a high char. - const utf16 = (latin1 + "\u{1F600}").slice(0, -2); - // Now latin1 is probably Latin-1, utf16 is definitely UTF-16. Same content. + // Same content in 16-bit storage. + const utf16 = forceUTF16(latin1); for (const a of [0, 2, 5]) { for (const b of [10, 20, 100]) { expect(Bun.sliceAnsi(utf16, a, b)).toBe(Bun.sliceAnsi(latin1, a, b)); @@ -586,7 +585,7 @@ describe("sliceAnsi encoding equivalence", () => { test("Latin-1-range non-ASCII in both encodings", () => { // Chars 0x80-0xFF exist in both encodings. 0xA9 (©), 0xE9 (é), etc. const s8 = "\u00A9\u00E9\u00DF\u00F1"; // ©éßñ — likely Latin-1 internally - const s16 = (s8 + "\u{1F600}").slice(0, -2); // force UTF-16 + const s16 = forceUTF16(s8); for (let a = 0; a <= 4; a++) { for (let b = a; b <= 4; b++) { expect(Bun.sliceAnsi(s16, a, b)).toBe(Bun.sliceAnsi(s8, a, b)); diff --git a/test/js/bun/util/sliceAnsi.test.ts b/test/js/bun/util/sliceAnsi.test.ts index 28451848551d..6b5c36c7df92 100644 --- a/test/js/bun/util/sliceAnsi.test.ts +++ b/test/js/bun/util/sliceAnsi.test.ts @@ -1,4 +1,5 @@ import { describe, expect, test } from "bun:test"; +import { forceUTF16 } from "harness"; // Constants matching the upstream slice-ansi test suite const ESCAPE = "\u001B"; @@ -1589,11 +1590,8 @@ describe("Bun.sliceAnsi", () => { }); test("UTF-16 ASCII fast path (string forced to 16-bit)", () => { - // Force a string into UTF-16 representation by including then removing a wide char. - // JSC doesn't re-compact to Latin-1, so this exercises the uint16_t SIMD lane path. - const wide = "hello world" + "\u00ff".slice(0, 0); // stays Latin-1 actually - // Better: concat with a surrogate, then slice it off — result stays UTF-16 - const utf16 = ("hello world" + "\u{1F600}").slice(0, 11); + // ASCII content in 16-bit storage exercises the uint16_t SIMD lane path. + const utf16 = forceUTF16("hello world"); expect(Bun.sliceAnsi(utf16, 0, 5)).toBe("hello"); expect(Bun.sliceAnsi(utf16, 6, 11)).toBe("world"); expect(Bun.sliceAnsi(utf16)).toBe(utf16); diff --git a/test/js/node/buffer.test.js b/test/js/node/buffer.test.js index 4ceb2edb6d92..f542c873d313 100644 --- a/test/js/node/buffer.test.js +++ b/test/js/node/buffer.test.js @@ -1,6 +1,6 @@ import { Buffer, SlowBuffer, isAscii, isUtf8, kMaxLength } from "buffer"; import { afterEach, beforeEach, describe, expect, it } from "bun:test"; -import { bunEnv, bunExe, gc, isASAN, isDebug, nodeExe, withoutAggressiveGC } from "harness"; +import { bunEnv, bunExe, forceUTF16, gc, isASAN, isDebug, nodeExe, withoutAggressiveGC } from "harness"; import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; import os from "node:os"; @@ -1124,8 +1124,6 @@ for (let withOverridenBufferWrite of [false, true]) { } return s; }; - // keeps only ASCII hex characters but forces two-byte string storage - const toUTF16 = s => (s + "\u0100").slice(0, -1); it("decodes valid input at every length around the vector widths", () => { for (const pairs of [15, 16, 17, 31, 32, 33, 48, 63, 64, 65, 127, 128, 129, 255, 256, 1024]) { @@ -1138,7 +1136,7 @@ for (let withOverridenBufferWrite of [false, true]) { const fromLatin1 = Buffer.from(hex, "hex"); expect(fromLatin1).toEqual(expected); - const fromUTF16 = Buffer.from(toUTF16(hex), "hex"); + const fromUTF16 = Buffer.from(forceUTF16(hex), "hex"); expect(fromUTF16).toEqual(expected); } } @@ -1154,7 +1152,7 @@ for (let withOverridenBufferWrite of [false, true]) { const expected = referenceHexDecode(hex); expect(expected.length).toBe(Math.floor(pos / 2)); expect(Buffer.from(hex, "hex")).toEqual(expected); - expect(Buffer.from(toUTF16(hex), "hex")).toEqual(expected); + expect(Buffer.from(forceUTF16(hex), "hex")).toEqual(expected); } } }); @@ -1234,7 +1232,7 @@ for (let withOverridenBufferWrite of [false, true]) { // 16-bit string path const utf16Target = Buffer.alloc(pairs); - expect(utf16Target.write(toUTF16(hex), "hex")).toBe(pairs); + expect(utf16Target.write(forceUTF16(hex), "hex")).toBe(pairs); expect(utf16Target).toEqual(expected); }); }); diff --git a/test/js/web/fetch/headers.test.ts b/test/js/web/fetch/headers.test.ts index b514a241674c..7edbdd1f27d5 100644 --- a/test/js/web/fetch/headers.test.ts +++ b/test/js/web/fetch/headers.test.ts @@ -2,6 +2,7 @@ import { beforeAll, describe, expect, test } from "bun:test"; // Namespace import so a missing binding fails only the kernel tests below // (accessing an absent export is `undefined`), not the whole file. import * as internalForTesting from "bun:internal-for-testing"; +import { forceUTF16 } from "harness"; beforeAll(() => { // expect(Headers).toBeDefined(); @@ -715,16 +716,13 @@ describe("Headers", () => { }); // An all-ASCII WTF string can still be stored as 16-bit, which takes a - // separate kernel. Force 16-bit storage by appending a code unit > 0xFF and - // slicing it back off, then run the same checks on the 16-bit path. - const to16 = (s: string) => (s + "\u0100").slice(0, -1); - + // separate kernel; run the same checks on the 16-bit path. test("16-bit: matches a scalar reference across lengths and alignments", () => { const alphabet = "AZaz09@[]^_`{|}~-.Mm"; for (let len = 0; len <= 160; len++) { let s = ""; for (let i = 0; i < len; i++) s += alphabet[(i * 7 + len) % alphabet.length]; - expect(lowercaseHeaderNameSIMD(to16(s))).toBe(scalarLower(s)); + expect(lowercaseHeaderNameSIMD(forceUTF16(s))).toBe(scalarLower(s)); } });