Skip to content

Decode module sources with the // @bun pragma as UTF-8 #37162

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
robobun wants to merge 5 commits into
base: main
Choose a base branch
from
+79 −2
Open

Decode module sources with the // @bun pragma as UTF-8 #37162

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
f2c971a
Decode module sources with the // @bun pragma as UTF-8
robobun Aug 7, 2026
8246d41
Set has_loaded on transpiler cache hits for the entry point
robobun Aug 7, 2026
1cf6eba
Assert no new cache entry is written on the cache-hit run
robobun Aug 7, 2026
89b7b73
Set has_loaded when the entry point carries the // @bun pragma
robobun Aug 7, 2026
2f3024c
ci: retrigger
robobun Aug 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/jsc/RuntimeTranspilerStore.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -994,7 +994,7 @@ impl TranspilerJob {
_ => (ptr::null_mut(), 0),
};
self.resolved_source = OwnedResolvedSource::from(ResolvedSource {
source_code: String::clone_latin1(&parse_result.source.contents),
source_code: String::clone_utf8(&parse_result.source.contents),
already_bundled: true,
bytecode_cache,
bytecode_cache_size,
Expand Down
10 changes: 9 additions & 1 deletion src/runtime/jsc_hooks.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2788,7 +2788,7 @@ fn transpile_source_code_inner(
_ => (core::ptr::null_mut(), 0),
};
return Ok(OwnedResolvedSource::from(ResolvedSource {
source_code: bun_core::String::clone_latin1(&source.contents),
source_code: bun_core::String::clone_utf8(&source.contents),
Comment thread
robobun marked this conversation as resolved.
specifier: input_specifier.dupe_ref(),
source_url: create_if_different(input_specifier, path.text),
already_bundled: true,
Expand Down Expand Up @@ -2931,6 +2931,14 @@ fn transpile_source_code_inner(
};
let (bytecode_cache, bytecode_cache_size) =
node_compile_cache_blob.unwrap_or((core::ptr::null_mut(), 0));
if is_main {
// Without this, a cache hit for the entry point leaves
// `has_loaded` false and later imports with unknown
// extensions fall back to `Loader::Tsx` instead of
// `Loader::File`.
// SAFETY: per fn contract — `jsc_vm` is the live per-thread VM.
unsafe { (*jsc_vm).has_loaded = true };
}
return Ok(OwnedResolvedSource::from(ResolvedSource {
source_code,
specifier: input_specifier.dupe_ref(),
Expand Down
34 changes: 34 additions & 0 deletions test/bundler/transpiler/runtime-transpiler.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,40 @@ describe("// @bun", () => {
expect(stdout.toString()).toBe("Hello world!\n");
expect(exitCode).toBe(0);
});

// https://github.com/oven-sh/bun/issues/37161
const nonAscii = "\u2014\u00b7\u2713 K\u00e4ufer";
test("raw utf-8 decodes as utf-8, not latin-1 (import)", async () => {
using dir = tempDir("bun-pragma-utf8-import", {
"pragma.js": `// @bun\nexport const text = "${nonAscii}";\n`,
});
const { text } = await import(`${dir}/pragma.js`);
expect(text).toBe(nonAscii);
});

test("raw utf-8 decodes as utf-8, not latin-1 (require)", async () => {
using dir = tempDir("bun-pragma-utf8-require", {
"pragma.js": `// @bun\nexport const text = "${nonAscii}";\n`,
});
const { text } = require(`${dir}/pragma.js`);
expect(text).toBe(nonAscii);
});

test("raw utf-8 decodes as utf-8, not latin-1 (entry point)", async () => {
using dir = tempDir("bun-pragma-utf8-entry", {
"pragma.js": `// @bun\nconst DASH = "\u2014";\nconsole.log([...DASH].map(c => c.codePointAt(0).toString(16)).join(" "));\n`,
});
await using proc = Bun.spawn({
cmd: [bunExe(), "pragma.js"],
env: bunEnv,
cwd: String(dir),
stderr: "pipe",
});
const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]);
expect(stderr).toBe("");
expect(stdout).toBe("2014\n");
expect(exitCode).toBe(0);
});
});

describe("json imports", () => {
Expand Down
11 changes: 11 additions & 0 deletions test/cli/run/transpiler-cache.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,17 @@ describe("transpiler cache", () => {
expect(await bunRun(join(temp_dir, "a.js"), env)).toSpawn("a");
expect(!existsSync(cache_dir)).toBeTrue();
});
test("cache hit on the entry point keeps unknown-extension imports on the file loader", async () => {
writeFileSync(join(temp_dir, "asset.hbs"), "<!DOCTYPE html>\n<html></html>\n");
const padding = "// " + Buffer.alloc(8 * 1024, "x").toString() + "\n";
writeFileSync(join(temp_dir, "a.js"), `import asset from "./asset.hbs";\n${padding}console.log(typeof asset);\n`);
expect(await bunRun(join(temp_dir, "a.js"), env)).toSpawn("string");
expect(newCacheCount()).toBe(1);
// The second run loads the entry point from the transpiler cache; the
// unknown extension must still get the file loader, not the tsx fallback.
expect(await bunRun(join(temp_dir, "a.js"), env)).toSpawn("string");
expect(newCacheCount()).toBe(0);
});
Comment thread
coderabbitai[bot] marked this conversation as resolved.
test("it is indeed content addressable", async () => {
writeFileSync(join(temp_dir, "a.js"), dummyFile(50 * 1024, "1", "b"));
expect(await bunRun(join(temp_dir, "a.js"), env)).toSpawn("b");
Expand Down