ci: add linux aarch64 ASAN build and test lanes to PR pipelines

.buildkite/ci.mjs

@@ -140,6 +140,9 @@ const buildPlatforms = [
   { os: "linux", arch: "aarch64", distro: "amazonlinux", release: "2023", features: ["docker"] },
   { os: "linux", arch: "x64", distro: "amazonlinux", release: "2023", features: ["docker"] },
   { os: "linux", arch: "x64", baseline: true, distro: "amazonlinux", release: "2023", features: ["docker"] },
+  // ASAN lanes are PR-only (filtered out on main — see getPipelineOptions and
+  // includeASAN in getPipeline) and never ship in releases.
+  { os: "linux", arch: "aarch64", profile: "asan", distro: "amazonlinux", release: "2023", features: ["docker"] },
   { os: "linux", arch: "x64", profile: "asan", distro: "amazonlinux", release: "2023", features: ["docker"] },
   { os: "linux", arch: "aarch64", abi: "musl", distro: "alpine", release: "3.23" },
   { os: "linux", arch: "x64", abi: "musl", distro: "alpine", release: "3.23" },
@@ -194,6 +197,7 @@ const testPlatforms = [
   { os: "linux", arch: "aarch64", distro: "debian", release: "13", tier: "latest" },
   { os: "linux", arch: "x64", distro: "debian", release: "13", tier: "latest" },
   { os: "linux", arch: "x64", baseline: true, distro: "debian", release: "13", tier: "latest" },
+  { os: "linux", arch: "aarch64", profile: "asan", distro: "debian", release: "13", tier: "latest" },
   { os: "linux", arch: "x64", profile: "asan", distro: "debian", release: "13", tier: "latest" },
   { os: "linux", arch: "aarch64", distro: "ubuntu", release: "25.04", tier: "latest" },
   { os: "linux", arch: "x64", distro: "ubuntu", release: "25.04", tier: "latest" },
@@ -688,7 +692,12 @@ function getTargetTriplet(platform) {
  * @returns {boolean}
  */
 function needsBaselineVerification(platform) {
-  const { os, arch, baseline } = platform;
+  const { os, arch, baseline, profile } = platform;
+  // ASAN builds are PR-only test artifacts that never ship, and they don't
+  // produce the ${triplet}-profile.zip this step downloads (the asan link
+  // uploads ${triplet}-asan.zip instead — see scripts/build/ci.ts). The
+  // release lanes already cover instruction-policy verification.
+  if (profile === "asan") return false;
   if (os === "linux") return (arch === "x64" && baseline) || arch === "aarch64";
   if (os === "windows") return arch === "x64" && baseline;
   return false;

scripts/build/deps/webkit.ts

@@ -7,7 +7,7 @@
 // -lto variants built with ThinLTO (per-module summaries for cross-language
 // importing), and the Windows ICU data table filtered + per-item zstd
 // compressed (lazily decompressed via bun_icu_decompress.cpp).
-export const WEBKIT_VERSION = "963f8758c29e965471c191668d5776a1a1b014b6";
+export const WEBKIT_VERSION = "f18cf9c267c72ff3d1ca31eec864ebcbeefd29b5";
 
 /**
  * WebKit (JavaScriptCore) — the JS engine.

src/jsc/ResolvedSource.rs

@@ -45,6 +45,13 @@ pub struct ResolvedSource {
     // -- Bytecode cache fields --
     pub bytecode_cache: *mut u8,
     pub bytecode_cache_size: usize,
+    /// Whether `bytecode_cache` is a heap allocation (`heap::into_raw`'d
+    /// `Box<[u8]>`) that the C++ `JSC::CachedBytecode` destructor must free.
+    /// `false` when the bytes point into the standalone executable's embedded
+    /// section (`bun build --compile`). Deliberately separate from
+    /// `source_code_needs_deref`, which C++ mutates while consuming the
+    /// source string.
+    pub bytecode_cache_needs_deref: bool,
     pub module_info: *mut c_void,
     /// The file path used as the source origin for bytecode cache validation.
     /// JSC validates bytecode by checking if the origin URL matches exactly what
@@ -68,6 +75,7 @@ impl Default for ResolvedSource {
             already_bundled: false,
             bytecode_cache: core::ptr::null_mut(),
             bytecode_cache_size: 0,
+            bytecode_cache_needs_deref: false,
             module_info: core::ptr::null_mut(),
             bytecode_origin_path: BunString::empty(),
         }
@@ -139,6 +147,19 @@ impl Drop for OwnedResolvedSource {
         if self.0.source_code_needs_deref {
             self.0.source_code.deref();
         }
+        if self.0.bytecode_cache_needs_deref && !self.0.bytecode_cache.is_null() {
+            // SAFETY: `bytecode_cache_needs_deref` marks the pointer as the
+            // `heap::into_raw`'d `Box<[u8]>` built on the runtime
+            // `// @bun @bytecode` path. `Drop` only runs when `into_ffi()` was
+            // never called, so Rust is still the sole owner of the buffer
+            // (the C++ `CachedBytecode` destructor frees it otherwise).
+            unsafe {
+                drop(bun_core::heap::take(core::ptr::slice_from_raw_parts_mut(
+                    self.0.bytecode_cache,
+                    self.0.bytecode_cache_size,
+                )));
+            }
+        }
         self.0.specifier.deref();
         self.0.source_url.deref();
         self.0.bytecode_origin_path.deref();

src/jsc/RuntimeTranspilerStore.rs

@@ -1032,17 +1032,33 @@ impl TranspilerJob {
         }
 
         if !matches!(parse_result.already_bundled, AlreadyBundled::None) {
-            let bytecode_slice = parse_result.already_bundled.bytecode_slice();
+            // Move the bytecode out of `parse_result` and transfer ownership
+            // to C++ (matches `transpile_source_code_inner` in jsc_hooks.rs and
+            // the Zig default_allocator semantics): `parse_result` is dropped
+            // when this function returns, so a borrowed `bytecode_slice()`
+            // pointer would dangle by the time the JS thread builds the
+            // `JSC::CachedBytecode` over it. `bytecode_cache_needs_deref`
+            // tells the C++ CachedBytecode destructor to free the buffer.
+            let already_bundled = core::mem::take(&mut parse_result.already_bundled);
+            let is_commonjs_module = already_bundled.is_common_js();
+            let (bytecode_cache, bytecode_cache_size) = match already_bundled {
+                AlreadyBundled::Bytecode(bytes) | AlreadyBundled::BytecodeCjs(bytes) => {
+                    let len = bytes.len();
+                    if len == 0 {
+                        (ptr::null_mut(), 0)
+                    } else {
+                        (bun_core::heap::into_raw(bytes).cast::<u8>(), len)
+                    }
+                }
+                _ => (ptr::null_mut(), 0),
+            };
             self.resolved_source = OwnedResolvedSource::from(ResolvedSource {
                 source_code: String::clone_latin1(&parse_result.source.contents),
                 already_bundled: true,
-                bytecode_cache: if !bytecode_slice.is_empty() {
-                    bytecode_slice.as_ptr().cast_mut()
-                } else {
-                    ptr::null_mut()
-                },
-                bytecode_cache_size: bytecode_slice.len(),
-                is_commonjs_module: parse_result.already_bundled.is_common_js(),
+                bytecode_cache,
+                bytecode_cache_size,
+                bytecode_cache_needs_deref: !bytecode_cache.is_null(),
+                is_commonjs_module,
                 tag: this_tag,
                 ..Default::default()
             });

src/jsc/VirtualMachine.rs

@@ -4495,10 +4495,19 @@ impl VirtualMachine {
         // each stored map and `deinit()`s the sibling `saved_source_map_table`.
         drop(core::mem::take(&mut self.source_mappings));
 
-        if let Some(rare) = self.rare_data.take() {
+        if self.rare_data.is_some() {
+            // Must run BEFORE `rare_data.take()`: `CronJob::clear_all_for_vm`
+            // reads `vm.rare_data` to drain the job list and early-returns on
+            // `None`. Calling it after the take() made it a no-op, leaking
+            // every still-registered CronJob (and tripping the
+            // `cron_jobs.is_empty()` assert in `RareData::drop` when a job
+            // survived finalization, e.g. `process.exit()` from inside a cron
+            // callback).
             if let Some(hooks) = runtime_hooks() {
                 (hooks.cron_clear_all_teardown)(self);
             }
+        }
+        if let Some(rare) = self.rare_data.take() {
             // Paired with `rare_data()`'s register_root_region. Without this,
             // every terminated Worker leaves a stale LSAN root entry pointing
             // into a freed arena.

src/jsc/bindings/ZigSourceProvider.cpp

@@ -124,11 +124,20 @@ Ref<SourceProvider> SourceProvider::create(
             const auto destructorNoOp = [](const void* ptr) {
                 // no-op, for bun build --compile.
             };
-            const auto destructor = resolvedSource.needsDeref ? destructorPtr : destructorNoOp;
+            // NOTE: do not gate this on `needsDeref` — that flag tracks the
+            // `source_code` string +1 and is cleared above (and mutated by
+            // `JSCommonJSModule::evaluate`'s wrapper-override path) before we
+            // get here, which both leaked runtime `// @bun @bytecode` buffers
+            // and could free embedded standalone-binary bytecode.
+            const auto destructor = resolvedSource.bytecode_cache_needs_deref ? destructorPtr : destructorNoOp;
 
             auto origin = getSourceOrigin();
 
             Ref<JSC::CachedBytecode> bytecode = JSC::CachedBytecode::create(std::span<uint8_t>(resolvedSource.bytecode_cache, resolvedSource.bytecode_cache_size), destructor, {});
+            // The CachedBytecode now owns the buffer; clear the ownership
+            // marker (mirroring the `needsDeref` clearing above) so the copy
+            // stored in m_resolvedSource cannot be misread as still owning it.
+            resolvedSource.bytecode_cache_needs_deref = false;
             auto provider = adoptRef(*new SourceProvider(
                 globalObject->bunVM(),
                 resolvedSource,

src/jsc/bindings/headers-handwritten.h

@@ -130,6 +130,10 @@ typedef struct ResolvedSource {
     // -- Bytecode cache fields --
     uint8_t* bytecode_cache;
     size_t bytecode_cache_size;
+    // Whether `bytecode_cache` is heap-owned (Rust `Box<[u8]>` via
+    // `heap::into_raw`) and must be freed by the CachedBytecode destructor.
+    // False for `bun build --compile`, where it points into the executable.
+    bool bytecode_cache_needs_deref;
     void* module_info;
     // File path used as source origin for bytecode cache validation.
     // Converted to file:// URL. If empty, origin is derived from source_url.

src/jsc/bindings/node/JSNodeHTTPServerSocket.cpp

@@ -247,7 +247,10 @@ void JSNodeHTTPServerSocket::onData(const char* data, int length, bool last)
 
     WebCore::ScriptExecutionContext* scriptExecutionContext = globalObject->scriptExecutionContext();
 
-    if (scriptExecutionContext) {
+    // Mirror onClose: never allocate JS cells or post tasks while the VM is
+    // shutting down (this can be reached from uws close callbacks during the
+    // final GC, when allocation is forbidden and posted tasks never run).
+    if (scriptExecutionContext && !globalObject->isShuttingDown()) {
         auto scope = DECLARE_TOP_EXCEPTION_SCOPE(globalObject->vm());
         JSC::JSUint8Array* buffer = WebCore::createBuffer(globalObject, std::span<const uint8_t>(reinterpret_cast<const uint8_t*>(data), length));
         auto chunk = JSC::JSValue(buffer);

src/runtime/bake/DevServer.rs

@@ -1208,6 +1208,12 @@ impl Drop for DevServer {
                     // SAFETY: stored ref from `init_from_any_blob`; no live borrow.
                     unsafe { StaticRoute::deref_(cached.as_ptr()) };
                 }
+                // Zig `RouteBundle.deinit`: `html.html_bundle.deref()` — release
+                // the intrusive ref taken by `get_or_put_route_bundle` when the
+                // bundle was stored. `html_bundle` is a raw `*mut`, so dropping
+                // the Vec would otherwise leak the route (and its HTMLBundle).
+                // SAFETY: the slot holds a counted ref taken at store time.
+                unsafe { bun_ptr::RefCount::<HTMLBundleRoute>::deref(html.html_bundle) };
             }
         }
 

src/runtime/jsc_hooks.rs

@@ -2732,6 +2732,9 @@ fn transpile_source_code_inner(
                         already_bundled: true,
                         bytecode_cache,
                         bytecode_cache_size,
+                        // C++ owns the buffer (heap::into_raw above); the
+                        // CachedBytecode destructor frees it.
+                        bytecode_cache_needs_deref: !bytecode_cache.is_null(),
                         is_commonjs_module,
                         ..Default::default()
                     }));

src/runtime/server/mod.rs

@@ -1715,6 +1715,38 @@
         // for the server's lifetime); single-threaded JS context, no aliasing `&mut`.
         let vm = unsafe { &mut *self.vm_mut() };
 
+        if vm.is_shutting_down() {
+            // The VM is shutting down (`is_shutting_down` is set in `on_exit`,
+            // before `Zig__GlobalObject__destructOnExit`'s final GC /
+            // `lastChanceToFinalize()` — typically we get here from a server
+            // wrapper finalizer during that last collection): the event loop
+            // will never tick again, so the App.close + deinit task pair below would
+            // be freed *unrun* by `EventLoop::deinit()` and the entire server
+            // graph (config, user_routes, HTMLBundle routes, …) would leak.
+            // Run both steps synchronously instead, preserving the
+            // close-before-destroy ordering (`~TemplatedApp` requires `close()`
+            // to have run — destroying without closing leaks the listen-socket
+            // polls). The re-entrancy hazard the task split guards against
+            // (App.close firing handlers mid-GC) is moot here: this branch is
+            // only reachable with no listener, no pending requests, and no
+            // active websockets, and `Listener` finalizers already
+            // `closeAll()` synchronously on this same path (their closed
+            // sockets are drained right after `destructOnExit`).
+            if !self.flags.contains(ServerFlags::TERMINATED) {
+                self.flags.insert(ServerFlags::TERMINATED);
+                if let Some(app) = self.app {
+                    // S012: `NewApp<SSL>` is a ZST opaque — safe `*mut → &mut` deref.
+                    bun_opaque::opaque_deref_mut(app).close();
+                }
+            }
+            // SAFETY: `self` is the unique heap server pointer (its JS wrapper
+            // has been finalized — `schedule_deinit` is only reached with
+            // `js_value == Finalized`). `deinit` frees it; `self` must not be
+            // touched after this call, so return immediately.
+            Self::deinit(std::ptr::from_mut::<Self>(self));
+            return;
+        }
+
         if !self.flags.contains(ServerFlags::TERMINATED) {
             // App.close can cause finalizers to run.
             // scheduleDeinit can be called inside a finalizer.

src/watcher/Watcher.rs

@@ -267,7 +267,12 @@ impl Watcher {
                     let _ = bun_sys::close(fd);
                 }
             }
-            // watchlist freed by Drop on Box
+            // `MultiArrayList`'s `Drop` is slab-only and never runs column
+            // destructors; drop the rows first or every `Cow::Owned`
+            // `file_path` leaks (matches Zig, where the watchlist allocator
+            // owned the dupes).
+            me.watchlist.drop_elements();
+            // watchlist slab freed by Drop on Box
             // SAFETY: this was heap-allocated by caller of init()
             drop(unsafe { bun_core::heap::take(this) });
         }
@@ -318,7 +323,11 @@ impl Watcher {
                     let _ = bun_sys::close(fd);
                 }
             }
-            // watchlist freed by Drop below
+            // `MultiArrayList`'s `Drop` is slab-only and never runs column
+            // destructors; drop the rows first or every `Cow::Owned`
+            // `file_path` leaks.
+            me.watchlist.drop_elements();
+            // watchlist slab freed by Drop below
         }
 
         // Close trace file if open
@@ -397,6 +406,12 @@ impl Watcher {
             if item == last_item || self.watchlist.len() <= item as usize {
                 continue;
             }
+            // `swap_remove` is a bitwise row overwrite that never runs column
+            // destructors — take the `Cow::Owned` file_path out first or it
+            // leaks (matches Zig, where the watchlist allocator owned the dupe).
+            drop(core::mem::take(
+                &mut self.watchlist.items_file_path_mut()[item as usize],
+            ));
             self.watchlist.swap_remove(item as usize);
 
             // swapRemove put a different entry at `item`, but its kqueue registration still
@@ -1093,6 +1108,7 @@ pub enum WatchItemKind {
 /// the unsafe generic `Slice::items::<F>(field)`.
 pub trait WatchItemColumns {
     fn items_file_path(&self) -> &[Cow<'static, [u8]>];
+    fn items_file_path_mut(&mut self) -> &mut [Cow<'static, [u8]>];
     fn items_hash(&self) -> &[u32];
     fn items_fd(&self) -> &[Fd];
     fn items_fd_mut(&mut self) -> &mut [Fd];
@@ -1106,6 +1122,9 @@ impl WatchItemColumns for WatchList {
     fn items_file_path(&self) -> &[Cow<'static, [u8]>] {
         self.items::<"file_path", Cow<'static, [u8]>>()
     }
+    fn items_file_path_mut(&mut self) -> &mut [Cow<'static, [u8]>] {
+        self.items_mut::<"file_path", Cow<'static, [u8]>>()
+    }
     fn items_hash(&self) -> &[u32] {
         self.items::<"hash", u32>()
     }
@@ -1131,6 +1150,9 @@ impl WatchItemColumns for bun_collections::multi_array_list::Slice<WatchItem> {
     fn items_file_path(&self) -> &[Cow<'static, [u8]>] {
         self.items::<"file_path", Cow<'static, [u8]>>()
     }
+    fn items_file_path_mut(&mut self) -> &mut [Cow<'static, [u8]>] {
+        self.items_mut::<"file_path", Cow<'static, [u8]>>()
+    }
     fn items_hash(&self) -> &[u32] {
         self.items::<"hash", u32>()
     }