test: make the ported Node suite run leak-clean under the ASAN runner, and fix the teardown UAFs and leaks it surfaces

.buildkite/ci.mjs

@@ -306,7 +306,17 @@
     manual: {
       permit_on_passed: true,
     },
-    automatic: false,
+    // Self-heal infra deaths once instead of leaving the build failed until a
+    // human notices and clicks retry:
+    //   -1  = agent lost / process killed (box died, agent restarted)
+    //   255 = step timeout kill (timeout_in_minutes SIGTERM cascade)
+    // User-canceled jobs are state=canceled, which never triggers automatic
+    // retry, so this cannot resurrect deliberately canceled builds. limit: 1
+    // caps the cost when a suite genuinely crashes with these statuses.
+    automatic: [
+      { exit_status: -1, limit: 1 },
+      { exit_status: 255, limit: 1 },
+    ],
   };
 }
 
@@ -824,7 +834,7 @@
     retry: getRetry(),
     cancel_on_build_failing: isMergeQueue(),
     parallelism: os === "darwin" ? 2 : os === "windows" ? 8 : 20,
-    timeout_in_minutes: profile === "asan" || os === "windows" ? 45 : os === "darwin" ? 40 : 30,
+    timeout_in_minutes: profile === "asan" || os === "windows" || os === "darwin" ? 45 : 30,
     env: {
       ASAN_OPTIONS: "allow_user_segv_handler=1:disable_coredump=0:detect_leaks=0",
       // Platform smoke check: runner.node.mjs asserts the agent matches what
@@ -837,7 +847,8 @@
       EXPECTED_PLATFORM_ARCH: platform.arch,
       ...(platform.abi ? { EXPECTED_PLATFORM_ABI: platform.abi } : {}),
       ...(platform.os === "linux" && platform.distro ? { EXPECTED_PLATFORM_DISTRO: platform.distro } : {}),
-      ...(platform.os === "linux" || (platform.os === "darwin" && platform.arch === "aarch64" && platform.tier === "latest")
+      ...(platform.os === "linux" ||
+      (platform.os === "darwin" && platform.arch === "aarch64" && platform.tier === "latest")
         ? { EXPECTED_PLATFORM_RELEASE: platform.release }
         : {}),
     },
@@ -849,6 +860,31 @@
 }
 
 /**
+ * CI image lifecycle
+ * ------------------
+ * Build/test agents boot from pre-baked cloud images (AWS AMIs for Linux,
+ * Azure Shared Image Gallery for Windows). The image a job requests is
+ * `${getImageKey(platform)}-v${N}`, where N is the `# Version:` comment at the
+ * top of scripts/bootstrap.sh (Linux) or scripts/bootstrap.ps1 (Windows).
+ *
+ * To change what's installed on a CI machine:
+ *
+ *   1. Edit bootstrap.sh / bootstrap.ps1 and bump its `# Version:` line.
+ *   2. Open a PR whose **commit subject** contains `[build images]` (or
+ *      `[build linux images]` / `[build windows images]` to scope it). This
+ *      bakes throwaway `…-build-<buildNumber>` images and runs the full
+ *      build+test pipeline against them so you can verify the change.
+ *   3. Once green, amend/force-push the subject to `[publish images]` (or the
+ *      scoped variant). This bakes the real `…-vN` images that normal CI will
+ *      pick up. Publishing replaces the live tag in place — for Windows it
+ *      deletes the existing gallery version before the new one finishes — so
+ *      don't cancel a publish run mid-bake.
+ *   4. Merge the PR **after** the publish run is green. By then the `…-vN`
+ *      images already exist, so the post-merge `main` build runs immediately
+ *      instead of everyone waiting 2-3 h on a bake.
+ *
+ * These tags are ignored on `main` — image bakes happen on the PR only.
+ *
  * @param {Platform} platform
  * @param {PipelineOptions} options
  * @returns {Step}
@@ -976,27 +1012,29 @@
 const BINARY_SIZE_THRESHOLD_MB = 0.5;
 
 /**
- * @param {Platform[]} buildPlatforms
+ * @param {Platform[]} releasePlatforms
  * @param {PipelineOptions} options
  * @param {{ signed: boolean }} [extra]
  * @returns {Step}
  */
-function getReleaseStep(buildPlatforms, options, { signed = false } = {}) {
+function getReleaseStep(releasePlatforms, options, { signed = false } = {}) {
   const { canary } = options;
   const revision = typeof canary === "number" ? canary : 1;
 
   // When signing ran, depend on windows-sign instead of the raw Windows builds
   // so we wait for signed artifacts before releasing.
   const depends_on = signed
-    ? [...buildPlatforms.filter(p => p.os !== "windows").map(p => `${getTargetKey(p)}-build-bun`), "windows-sign"]
-    : buildPlatforms.map(platform => `${getTargetKey(platform)}-build-bun`);
+    ? [...releasePlatforms.filter(p => p.os !== "windows").map(p => `${getTargetKey(p)}-build-bun`), "windows-sign"]
+    : releasePlatforms.map(platform => `${getTargetKey(platform)}-build-bun`);
 
   return {
     key: "release",
     label: getBuildkiteEmoji("rocket"),
-    agents: {
-      queue: "test-darwin",
-    },
+    agents: getEc2Agent(
+      buildPlatforms.find(p => p.os === "linux" && p.arch === "aarch64" && p.distro === "amazonlinux"),
+      options,
+      { instanceType: "c8g.large" },
+    ),
     depends_on,
     env: {
       CANARY: revision,
@@ -1350,6 +1388,8 @@
     };
   }
 
+  // BUILDKITE_MESSAGE is the commit subject line only — option tags like
+  // [publish images] must appear in the subject, not the commit body.
   const commitMessage = getCommitMessage();
 
   /**
@@ -1368,6 +1408,23 @@
   const isCanary =
     !parseBoolean(getEnv("RELEASE", false) || "false") &&
     !/\[(release|build release|release build)\]/i.test(commitMessage);
+
+  let buildImages = parseOption(/\[(build (?:(?:windows|linux) )?images?)\]/i);
+  let publishImages = parseOption(/\[(publish (?:(?:windows|linux) )?images?)\]/i);
+  let imageFilter = (commitMessage.match(/\[(?:build|publish) (windows|linux) images?\]/i) || [])[1]?.toLowerCase();
+
+  // Image bake/publish is meant to happen on the PR; the squash-merge commit
+  // subject often still carries the [publish images] tag, which would re-run
+  // the multi-hour bake on main and (because publish replaces the live image
+  // tag) briefly delete the images CI runs on. Ignore the tag on main and run
+  // a normal build instead.
+  if (isMainBranch() && (buildImages || publishImages)) {
+    console.log(`Ignoring [${publishImages || buildImages}] on main branch — images are built and published from PRs.`);
+    buildImages = false;
+    publishImages = false;
+    imageFilter = undefined;
+  }
+
   return {
     canary: isCanary ? canary : 0,
     skipEverything: parseOption(/\[(skip ci|no ci)\]/i),
@@ -1376,10 +1433,10 @@
     skipTests: parseOption(/\[(skip tests?|no tests?|only builds?)\]/i),
     skipSizeCheck: parseOption(/\[(skip size( check)?|allow size)\]/i),
     signWindows: parseOption(/\[(sign windows)\]/i),
-    buildImages: parseOption(/\[(build (?:(?:windows|linux) )?images?)\]/i),
+    buildImages,
     dryRun: parseOption(/\[(dry run)\]/i),
-    publishImages: parseOption(/\[(publish (?:(?:windows|linux) )?images?)\]/i),
-    imageFilter: (commitMessage.match(/\[(?:build|publish) (windows|linux) images?\]/i) || [])[1]?.toLowerCase(),
+    publishImages,
+    imageFilter,
     buildPlatforms: Array.from(buildPlatformsMap.values()),
     testPlatforms: Array.from(testPlatformsMap.values()),
   };

.buildkite/scripts/upload-release.sh

@@ -61,22 +61,104 @@ function run_command() {
   { set +x; } 2>/dev/null
 }
 
+function maybe_sudo() {
+  if [ "$(id -u)" -eq 0 ]; then
+    run_command "$@"
+  elif command -v sudo &> /dev/null; then
+    run_command sudo "$@"
+  else
+    run_command "$@"
+  fi
+}
+
+function package_manager_install() {
+  if command -v dnf &> /dev/null; then
+    maybe_sudo dnf install -y "$@"
+  elif command -v yum &> /dev/null; then
+    maybe_sudo yum install -y "$@"
+  elif command -v apt-get &> /dev/null; then
+    export DEBIAN_FRONTEND=noninteractive
+    maybe_sudo apt-get install -y "$@"
+  elif command -v apk &> /dev/null; then
+    maybe_sudo apk add "$@"
+  else
+    echo "error: No supported package manager found to install: $*"
+    exit 1
+  fi
+}
+
+function install_gh_linux() {
+  local arch
+  case "$(uname -m)" in
+    x86_64 | amd64) arch="amd64" ;;
+    aarch64 | arm64) arch="arm64" ;;
+    *) echo "error: Unsupported architecture: $(uname -m)"; exit 1 ;;
+  esac
+  # Resolve the version from the releases/latest redirect, not the REST API: the API is rate
+  # limited to 60 req/hour per IP (GITHUB_TOKEN is not exported yet), and piping curl into a
+  # short-circuiting reader such as `grep -m1` makes curl exit 23 (EPIPE) under pipefail.
+  local url version
+  url="$(curl -fsSLI -o /dev/null -w '%{url_effective}' "https://github.com/cli/cli/releases/latest")"
+  version="${url##*/tag/v}"
+  if [ -z "$version" ] || [ "$version" == "$url" ]; then
+    echo "error: Cannot determine latest gh release version from: $url"
+    exit 1
+  fi
+  local dir
+  dir="$(mktemp -d)"
+  run_command curl -fsSL "https://github.com/cli/cli/releases/download/v${version}/gh_${version}_linux_${arch}.tar.gz" -o "$dir/gh.tar.gz"
+  run_command tar -xzf "$dir/gh.tar.gz" -C "$dir" --strip-components=1
+  maybe_sudo install -m 0755 "$dir/bin/gh" /usr/local/bin/gh
+  rm -rf "$dir"
+}
+
+function install_aws_linux() {
+  command -v unzip &> /dev/null || package_manager_install unzip
+  local dir
+  dir="$(mktemp -d)"
+  run_command curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "$dir/awscliv2.zip"
+  run_command unzip -q "$dir/awscliv2.zip" -d "$dir"
+  maybe_sudo "$dir/aws/install" --update
+  rm -rf "$dir"
+}
+
+function install_sentry_cli_linux() {
+  # The installer drops a single static binary into INSTALL_DIR.
+  maybe_sudo bash -c "curl -fsSL https://sentry.io/get-cli/ | INSTALL_DIR=/usr/local/bin sh"
+}
+
 function assert_command() {
   local command="$1"
   local package="$2"
   local help_url="$3"
+  if command -v "$command" &> /dev/null; then
+    return
+  fi
+  echo "warning: $command is not installed, installing..."
+  if command -v brew &> /dev/null; then
+    HOMEBREW_NO_AUTO_UPDATE=1 run_command brew install "$package"
+  elif [ "$(uname -s)" == "Linux" ]; then
+    case "$command" in
+      gh) install_gh_linux ;;
+      aws) install_aws_linux ;;
+      sentry-cli) install_sentry_cli_linux ;;
+      *) echo "error: Don't know how to install $command on Linux"; exit 1 ;;
+    esac
+  else
+    echo "error: Cannot install $command, please install it"
+    if [ -n "$help_url" ]; then
+      echo ""
+      echo "hint: See $help_url for help"
+    fi
+    exit 1
+  fi
   if ! command -v "$command" &> /dev/null; then
-    echo "warning: $command is not installed, installing..."
-    if command -v brew &> /dev/null; then
-      HOMEBREW_NO_AUTO_UPDATE=1 run_command brew install "$package"
-    else
-      echo "error: Cannot install $command, please install it"
-      if [ -n "$help_url" ]; then
-        echo ""
-        echo "hint: See $help_url for help"
-      fi
-      exit 1
+    echo "error: Failed to install $command"
+    if [ -n "$help_url" ]; then
+      echo ""
+      echo "hint: See $help_url for help"
     fi
+    exit 1
   fi
 }
 

.buildkite/update-test-durations.yml

@@ -0,0 +1,18 @@
+# Regenerates test/expected-durations.json from recent Buildkite runs and
+# uploads it as a build artifact. Attach a weekly schedule to this pipeline in
+# the Buildkite UI (it is not wired into ci.mjs so it never runs on PRs).
+#
+# The runner uses the checked-in copy for sharding; refresh that copy by
+# downloading this artifact and committing it when the shard balance drifts.
+steps:
+  - label: ":stopwatch: update-test-durations"
+    if: build.source == "schedule" || build.source == "ui"
+    agents:
+      queue: build-linux
+    command: |
+      node scripts/update-test-durations.mjs --builds 5
+      buildkite-agent artifact upload test/expected-durations.json
+    env:
+      # The script reads BUILDKITE_API_TOKEN; the agent environment hook
+      # already exports a read-scoped token under this name.
+      BUILDKITE_API_TOKEN: "$BUILDKITE_API_TOKEN"

.claude/commands/upgrade-boringssl.md

@@ -0,0 +1,100 @@
+---
+description: Upgrade Bun's BoringSSL fork (oven-sh/boringssl) to the latest upstream google/boringssl
+---
+
+Bun pins BoringSSL by **commit SHA** in `scripts/build/deps/boringssl.ts` (`BORINGSSL_COMMIT`). The build downloads a tarball from `oven-sh/boringssl` at that SHA — there is no submodule and `vendor/boringssl/` is git-ignored.
+
+The fork carries a small patch set on top of upstream (see "Preserved patches" below). Upgrading means: merge `google/boringssl` into `oven-sh/boringssl@master`, push, then bump the SHA + regenerate source lists in Bun.
+
+## Steps
+
+### 1. Clone the fork and merge upstream
+
+```sh
+git clone https://github.com/oven-sh/boringssl.git /tmp/boringssl
+cd /tmp/boringssl
+git remote add upstream https://github.com/google/boringssl.git
+git fetch upstream
+git log --oneline $(git merge-base HEAD upstream/main)..HEAD   # our patches
+git merge upstream/main
+```
+
+Resolve conflicts **preserving the fork's additions**. Most conflicts are upstream's periodic `|...|` → `` `...` `` doc-comment restyle landing adjacent to a line we added — keep our line + upstream's comment style. For `include/openssl/nid.h`, keep upstream's new NIDs **and** ours (our NID numbers are from OpenSSL's range and don't collide with BoringSSL's sequential allocation).
+
+### 2. Verify the merged tree builds
+
+```sh
+cmake -B build -GNinja -DCMAKE_BUILD_TYPE=Release
+ninja -C build crypto ssl decrepit
+```
+
+This catches mis-resolved conflicts before they reach Bun's CI.
+
+### 3. Push to the fork
+
+The default branch is **`master`** (not `main`).
+
+```sh
+git push origin HEAD:master
+NEW_SHA=$(git rev-parse HEAD)
+```
+
+### 4. Bump Bun
+
+In the bun repo:
+
+- `scripts/build/deps/boringssl.ts` — set `BORINGSSL_COMMIT` to `$NEW_SHA`.
+- `test/js/node/process/process.test.js` — update the `boringssl:` entry in `expectedVersions` to `$NEW_SHA`.
+- Regenerate the source lists (the file's header comment has the exact one-liner). Only `gen/sources.json` is authoritative — diff old vs new and apply the delta:
+
+  ```sh
+  rm -rf vendor/boringssl   # force re-fetch on next build
+  bun bd --target=clone-boringssl
+  bun -e 'const j=require("./vendor/boringssl/gen/sources.json");
+          const f=l=>l.map(JSON.stringify).join(", ");
+          for(const k of ["bcm","crypto","ssl","decrepit"]) console.log(k,"\n",f(j[k].srcs));
+          console.log("asm\n",f([...j.bcm.asm,...j.crypto.asm]));
+          console.log("nasm\n",f([...j.bcm.nasm,...j.crypto.nasm]))'
+  ```
+
+### 5. Build and test locally
+
+```sh
+rm -rf vendor/boringssl
+bun bd -p 'require("crypto").createHash("sha3-256").update("hi").digest("hex")'
+bun bd test test/js/node/crypto/ test/js/bun/crypto/
+bun bd test test/js/node/tls/ test/js/web/fetch/fetch.tls.test.ts
+```
+
+### 6. Open the Bun PR
+
+```sh
+git checkout -b claude/boringssl-<upstream-short-sha>
+git commit -am "deps: upgrade BoringSSL to <upstream-short-sha>"
+git push -u origin HEAD
+gh pr create
+```
+
+Then `bun run ci:watch` and fix anything that turns up.
+
+## Preserved patches (what conflicts to expect)
+
+`git diff $(git merge-base HEAD upstream/main)..HEAD --stat` — currently ~35 files, ~550 insertions:
+
+- **SHA-512/224** — `crypto/fipsmodule/sha/sha512.cc.inc`, `crypto/sha/sha512.cc`, `include/openssl/{sha2,nid,digest}.h`
+- **SHA3-224/256/384/512 as `EVP_MD`** — `crypto/digest/digest_extra.cc`, `crypto/fipsmodule/{digest/digests.cc.inc,keccak/*}`, `include/openssl/{digest,nid}.h`
+- **HMAC-SHA3** — `crypto/hmac/hmac_test*.{cc,txt}`
+- **BLAKE2b-512** — `crypto/blake2/blake2.cc`, `include/openssl/blake2.h`
+- **RIPEMD160 in `crypto/` (not `decrepit/`) + `EVP_ripemd160` lookup** — `crypto/ripemd/ripemd.cc` (moved), `crypto/digest/digest_extra.cc`, `include/openssl/digest.h`, `gen/sources.*`, `build.json`
+- **`EVP_PBE_validate_scrypt_params`** — `crypto/evp/scrypt.cc`, `include/openssl/evp.h`
+- **Electron `SSL_want` / `EVP_CIPHER_do_all_sorted`** — `ssl/ssl_lib.cc` (return `rwstate` directly), `ssl/ssl_test.cc` (drops the corresponding test block), `decrepit/evp/evp_do_all.cc`, `crypto/cipher/get_cipher.cc`, `include/openssl/cipher.h`
+- **MLDSA stack-frame pragma** — `crypto/fipsmodule/mldsa/mldsa.cc.inc`
+
+If upstream upstreams any of these (check `git grep` on `upstream/main` before re-applying), drop the fork's copy.
+
+## Things that have broken before
+
+- **`SSL_CTX` / `SSL_ECH_KEYS` / `SSL_CREDENTIAL` made opaque** — Bun's Rust FFI (`src/boringssl_sys/boringssl.rs`) treats them as opaque already, so this is fine, but check `packages/bun-usockets/src/crypto/openssl.c` for any direct field access.
+- **`BIO_read`/`BIO_write` error-value narrowing** — can change `SSL_read` error paths over memory BIOs (`SSLWrapper` for TLS-over-duplex). If `node-tls-connect.test.ts` crashes in `flush_pending_events`, see `src/runtime/socket/UpgradedDuplex.rs::teardown` and `WindowsNamedPipe.rs`'s `WRAPPER_BUSY` for the re-entrant-drop guard.
+- **Per-handshake allocation churn (PQ key shares)** grows under ASAN quarantine; RSS-delta tests like `tls-connect-socket-churn.test.ts` may need their `isASAN` bound raised. The `sslCtxLiveCount` check is the real regression guard there — if that passes and LSAN is clean, raise the RSS bound.
+- **`asn1_string_st` / `GENERAL_NAME_st` layout** — Bun mirrors these in `src/boringssl_sys/boringssl.rs`; diff `include/openssl/{asn1,x509v3}.h` for field changes.

.claude/commands/upgrade-webkit.md

@@ -36,6 +36,6 @@ To do that:
 
 Things to check for a successful upgrade:
 
-- Did Source/JavaScriptCore/runtime/JSType.h change? The enum values must align with Bun's mirror in src/jsc/JSType.rs (src/jsc/JSType.zig is a non-compiled porting reference, not the live code).
+- Did Source/JavaScriptCore/runtime/JSType.h change? The enum values must align with Bun's mirror in src/jsc/JSType.rs.
 - Were there any changes to the WebCore code generator? If there are C++ compilation errors, check for differences in the generated reference code in vendor/WebKit/Source/WebCore/bindings/scripts/test/JS/
 - If the merge touched the fork's .github/workflows, the release tarball names must still match prebuiltSuffix() in scripts/build/deps/webkit.ts