ci(verify-baseline): parse //@ JSC flags, skip v128 wasm fixtures on x64, re-enable --jit-stress on WebKit changes

.buildkite/ci.mjs

@@ -746,7 +746,8 @@ function getEmulatorBinary(platform) {
  */
 function hasWebKitChanges(options) {
   const { changedFiles = [] } = options;
-  return changedFiles.some(file => file.includes("SetupWebKit.cmake"));
+  // vendor/WebKit is gitignored; WebKit version bumps land here.
+  return changedFiles.some(file => file === "scripts/build/deps/webkit.ts");
 }
 
 /**
@@ -759,10 +760,15 @@ function getVerifyBaselineStep(platform, options) {
   const targetKey = getTargetKey(platform);
   const triplet = getTargetTriplet(platform);
   const emulator = getEmulatorBinary(platform);
-  const jitStressFlag = hasWebKitChanges(options) ? " --jit-stress" : "";
   // Android binaries need /system/bin/linker64 + a bionic sysroot, neither of which exist on the
   // build host, so qemu-user cannot load them; only the static instruction scan is meaningful.
-  const skipEmulationFlag = abi === "android" ? " --skip-emulation" : "";
+  const skipEmulation = abi === "android";
+  const skipEmulationFlag = skipEmulation ? " --skip-emulation" : "";
+  // Windows SDE (Pin) pays ~45s startup per fixture; 80+ serial fixtures would
+  // take ~1h. qemu-Nehalem on Linux already verifies the same x64-no-AVX JIT
+  // output at ~1s/fixture, so Windows keeps only the static scan + SIMD test.
+  const wantJitStress = hasWebKitChanges(options) && os !== "windows" && !skipEmulation;
+  const jitStressFlag = wantJitStress ? " --jit-stress" : "";
 
   // Scan bun-profile, not bun. The stripped binary has no .symtab (ELF) and
   // no companion .pdb (PE) — the static scanner would emit <no-symbol@addr>
@@ -814,7 +820,7 @@ function getVerifyBaselineStep(platform, options) {
     agents,
     retry: getRetry(),
     cancel_on_build_failing: isMergeQueue(),
-    timeout_in_minutes: hasWebKitChanges(options) ? 30 : 10,
+    timeout_in_minutes: wantJitStress ? 30 : 10,
     command: [
       ...setupCommands,
       `cargo build --release --manifest-path scripts/verify-baseline-static/Cargo.toml${os === "windows" ? " || exit /b 1" : ""}`,

scripts/verify-baseline.ts

@@ -11,6 +11,7 @@
 
 import { readdirSync } from "node:fs";
 import { basename, dirname, join, resolve } from "node:path";
+import { parseJSCFlags, wasmSIMDFixtures } from "../test/js/bun/jsc-stress/jsc-flags";
 // @ts-ignore — utils.mjs has JSDoc types but no .d.ts
 import { markBuildkiteStepReported } from "./utils.mjs";
 
@@ -92,12 +93,27 @@ console.log();
 let instructionFailures = 0;
 let otherFailures = 0;
 let passed = 0;
+let skipped = 0;
 const failedTests: string[] = [];
 
+// Mirrors the relevant bits of test/harness.ts's bunEnv so fixtures behave the
+// same here as under jsc-stress.test.ts.
+const fixtureBaseEnv: Record<string, string | undefined> = {
+  ...process.env,
+  BUN_DEBUG_QUIET_LOGS: "1",
+  NO_COLOR: "1",
+  BUN_GARBAGE_COLLECTOR_LEVEL: process.env.BUN_GARBAGE_COLLECTOR_LEVEL || "0",
+  BUN_FEATURE_FLAG_INTERNAL_FOR_TESTING: "1",
+};
+// harness.ts strips this ad-hoc agent override so it can't leak into fixtures.
+delete fixtureBaseEnv.JSC_useJIT;
+
 interface RunTestOptions {
   cwd?: string;
   /** Tee output live to the console while still capturing it for analysis */
   live?: boolean;
+  /** Extra environment (e.g. parsed `//@` JSC flags) for the spawned binary */
+  env?: Record<string, string>;
 }
 
 /** Read a stream, write each chunk to a writable, and return the full text. */
@@ -118,6 +134,7 @@ async function runTest(label: string, binaryArgs: string[], options?: RunTestOpt
   const proc = Bun.spawn([...config.runnerCmd, binary, ...binaryArgs], {
     // config.cwd takes priority — SDE on Windows must run from its own directory for Pin DLL resolution
     cwd: config.cwd ?? options?.cwd,
+    env: { ...fixtureBaseEnv, ...(options?.env ?? {}) },
     stdout: "pipe",
     stderr: "pipe",
   });
@@ -241,7 +258,10 @@ if (values["skip-emulation"]) {
   console.log(`--- JS fixtures (DFG/FTL) — ${jsFixtures.length} tests`);
   for (let i = 0; i < jsFixtures.length; i++) {
     const fixture = jsFixtures[i];
-    await runTest(`[${i + 1}/${jsFixtures.length}] ${fixture}`, ["--preload", preloadPath, join(fixturesDir, fixture)]);
+    const fixturePath = join(fixturesDir, fixture);
+    await runTest(`[${i + 1}/${jsFixtures.length}] ${fixture}`, ["--preload", preloadPath, fixturePath], {
+      env: parseJSCFlags(fixturePath),
+    });
   }
 
   const wasmFixtures = readdirSync(wasmFixturesDir)
@@ -251,11 +271,20 @@ if (values["skip-emulation"]) {
   console.log(`--- Wasm fixtures (BBQ/OMG) — ${wasmFixtures.length} tests`);
   for (let i = 0; i < wasmFixtures.length; i++) {
     const fixture = wasmFixtures[i];
-    await runTest(
-      `[${i + 1}/${wasmFixtures.length}] ${fixture}`,
-      ["--preload", preloadPath, join(wasmFixturesDir, fixture)],
-      { cwd: wasmFixturesDir },
-    );
+    const fixturePath = join(wasmFixturesDir, fixture);
+    // Nehalem has no AVX; JSC's recomputeDependentOptions() force-disables
+    // useWasmSIMD there, so v128-typed modules fail to parse. A real baseline
+    // CPU never runs the wasm-SIMD JIT path, so there is nothing to verify.
+    if (!isAarch64 && wasmSIMDFixtures.has(fixture)) {
+      console.log(`--- [${i + 1}/${wasmFixtures.length}] ${fixture}`);
+      console.log("    SKIP (uses wasm v128; JSC disables wasm SIMD on x64 without AVX)");
+      skipped++;
+      continue;
+    }
+    await runTest(`[${i + 1}/${wasmFixtures.length}] ${fixture}`, ["--preload", preloadPath, fixturePath], {
+      cwd: wasmFixturesDir,
+      env: parseJSCFlags(fixturePath),
+    });
   }
 } else {
   console.log();
@@ -266,6 +295,7 @@ if (values["skip-emulation"]) {
 console.log();
 console.log("--- Summary");
 console.log(`    Passed: ${passed}`);
+if (skipped) console.log(`    Skipped: ${skipped}`);
 console.log(`    Instruction failures: ${instructionFailures}`);
 console.log(`    Other failures: ${otherFailures} (not CPU instruction issues)`);
 console.log();

test/js/bun/jsc-stress/jsc-flags.test.ts

@@ -0,0 +1,89 @@
+// Covers the helper shared by jsc-stress.test.ts and scripts/verify-baseline.ts
+// and keeps verify-baseline's Nehalem skip list in sync with the wasm fixtures.
+
+import { describe, expect, test } from "bun:test";
+import { readdirSync } from "fs";
+import { bunEnv, bunExe, isDebug } from "harness";
+import path from "path";
+import { parseJSCFlags, wasmSIMDFixtures } from "./jsc-flags";
+
+const fixturesDir = path.join(import.meta.dir, "fixtures");
+const wasmFixturesDir = path.join(fixturesDir, "wasm");
+const preloadPath = path.join(import.meta.dir, "preload.js");
+// Same headroom jsc-stress.test.ts gives debug builds for JIT tier-up loops.
+const fixtureTimeout = isDebug ? 180_000 : undefined;
+
+describe("parseJSCFlags", () => {
+  test("runDefaultWasm", () => {
+    expect(parseJSCFlags(path.join(wasmFixturesDir, "bbq-osr-with-exceptions.js"))).toEqual({
+      BUN_JSC_useDollarVM: "1",
+      BUN_JSC_jitPolicyScale: "0.1",
+    });
+  });
+
+  test("runDefault", () => {
+    expect(parseJSCFlags(path.join(wasmFixturesDir, "omg-tail-call-clobber-scratch-register.js"))).toEqual({
+      BUN_JSC_jitPolicyScale: "0",
+    });
+  });
+
+  test("runFTLNoCJIT implies useFTLJIT / !useConcurrentJIT", () => {
+    expect(parseJSCFlags(path.join(fixturesDir, "licm-no-pre-header.js"))).toEqual({
+      BUN_JSC_useFTLJIT: "true",
+      BUN_JSC_useConcurrentJIT: "false",
+      BUN_JSC_createPreHeaders: "false",
+    });
+  });
+
+  test("no directive", () => {
+    expect(parseJSCFlags(path.join(wasmFixturesDir, "ipint-bbq-osr-with-try2.js"))).toEqual({});
+  });
+});
+
+// Nehalem has no AVX so JSC disables useWasmSIMD there, making v128 an invalid
+// wasm type. Assert wasmSIMDFixtures lists exactly the fixtures that fail to
+// parse without SIMD so verify-baseline's skip list stays correct.
+describe.concurrent("wasmSIMDFixtures matches fixtures that require wasm SIMD", () => {
+  const allWasmFixtures = readdirSync(wasmFixturesDir)
+    .filter(f => f.endsWith(".js"))
+    .sort();
+
+  test("every listed fixture exists on disk", () => {
+    const onDisk = new Set(allWasmFixtures);
+    expect([...wasmSIMDFixtures].filter(f => !onDisk.has(f))).toEqual([]);
+  });
+
+  for (const fixture of allWasmFixtures) {
+    test(
+      fixture,
+      async () => {
+        const fixturePath = path.join(wasmFixturesDir, fixture);
+        await using proc = Bun.spawn({
+          cmd: [bunExe(), "--preload", preloadPath, fixturePath],
+          // Simulate the Nehalem path: no AVX => JSC disables wasm SIMD.
+          env: { ...bunEnv, ...parseJSCFlags(fixturePath), BUN_JSC_useWasmSIMD: "false" },
+          cwd: wasmFixturesDir,
+          stdout: "pipe",
+          stderr: "pipe",
+        });
+        const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]);
+
+        if (wasmSIMDFixtures.has(fixture)) {
+          // Must fail to parse with the characteristic error; if it passes, it
+          // no longer needs SIMD and should be removed from wasmSIMDFixtures.
+          expect(stderr).toContain("WebAssembly.Module doesn't parse");
+          expect(exitCode).not.toBe(0);
+        } else {
+          // Must pass without SIMD; if it fails with a parse error, add it to
+          // wasmSIMDFixtures so verify-baseline skips it under Nehalem.
+          if (exitCode !== 0) {
+            console.log("stdout:", stdout);
+            console.log("stderr:", stderr);
+          }
+          expect(exitCode).toBe(0);
+        }
+      },
+      fixtureTimeout,
+    );
+  }
+});

test/js/bun/jsc-stress/jsc-flags.ts

@@ -0,0 +1,48 @@
+// Shared between jsc-stress.test.ts and scripts/verify-baseline.ts so both
+// spawn fixtures with the same JSC options.
+
+import fs from "fs";
+
+/**
+ * Parse JSC option flags from //@ directives at the top of a test file.
+ * Converts --flag=value to BUN_JSC_flag=value environment variables.
+ *
+ * Supported directives:
+ *   //@ runDefault("--flag=value", ...)
+ *   //@ runFTLNoCJIT("--flag=value", ...)
+ *   //@ runDefaultWasm("--flag=value", ...)
+ */
+export function parseJSCFlags(filePath: string): Record<string, string> {
+  const content = fs.readFileSync(filePath, "utf-8");
+  const env: Record<string, string> = {};
+
+  for (const line of content.split("\n")) {
+    if (line === "// @bun" || line.trim() === "") continue;
+    if (!line.startsWith("//@")) break;
+
+    const match = line.match(/^\/\/@ (runDefault|runFTLNoCJIT|runDefaultWasm)\((.*)\)/);
+    if (!match) continue;
+
+    const [, mode, argsStr] = match;
+
+    // runFTLNoCJIT implies these flags (from WebKit's run-jsc-stress-tests)
+    if (mode === "runFTLNoCJIT") {
+      env["BUN_JSC_useFTLJIT"] = "true";
+      env["BUN_JSC_useConcurrentJIT"] = "false";
+    }
+
+    // Parse explicit flags: "--key=value"
+    const flagPattern = /"--(\w+)=([^"]+)"/g;
+    let flagMatch;
+    while ((flagMatch = flagPattern.exec(argsStr)) !== null) {
+      env[`BUN_JSC_${flagMatch[1]}`] = flagMatch[2];
+    }
+  }
+
+  return env;
+}
+
+// Wasm fixtures whose modules declare v128 (0x7B). JSC's Options.cpp disables
+// useWasmSIMD on x86_64 without AVX, so these fail to parse under Nehalem
+// emulation; verify-baseline.ts skips them on x64.
+export const wasmSIMDFixtures = new Set(["bbq-osr-with-exceptions.js", "omg-tail-call-clobber-scratch-register.js"]);

test/js/bun/jsc-stress/jsc-stress.test.ts

@@ -1,50 +1,11 @@
 import { describe, expect, test } from "bun:test";
-import fs from "fs";
 import { bunEnv, bunExe, isDebug } from "harness";
 import path from "path";
+import { parseJSCFlags } from "./jsc-flags";
 
 const fixturesDir = path.join(import.meta.dir, "fixtures");
 const wasmFixturesDir = path.join(fixturesDir, "wasm");
 
-/**
- * Parse JSC option flags from //@ directives at the top of a test file.
- * Converts --flag=value to BUN_JSC_flag=value environment variables.
- *
- * Supported directives:
- *   //@ runDefault("--flag=value", ...)
- *   //@ runFTLNoCJIT("--flag=value", ...)
- *   //@ runDefaultWasm("--flag=value", ...)
- */
-function parseJSCFlags(filePath: string): Record<string, string> {
-  const content = fs.readFileSync(filePath, "utf-8");
-  const env: Record<string, string> = {};
-
-  for (const line of content.split("\n")) {
-    if (line === "// @bun" || line.trim() === "") continue;
-    if (!line.startsWith("//@")) break;
-
-    const match = line.match(/^\/\/@ (runDefault|runFTLNoCJIT|runDefaultWasm)\((.*)\)/);
-    if (!match) continue;
-
-    const [, mode, argsStr] = match;
-
-    // runFTLNoCJIT implies these flags (from WebKit's run-jsc-stress-tests)
-    if (mode === "runFTLNoCJIT") {
-      env["BUN_JSC_useFTLJIT"] = "true";
-      env["BUN_JSC_useConcurrentJIT"] = "false";
-    }
-
-    // Parse explicit flags: "--key=value"
-    const flagPattern = /"--(\w+)=([^"]+)"/g;
-    let flagMatch;
-    while ((flagMatch = flagPattern.exec(argsStr)) !== null) {
-      env[`BUN_JSC_${flagMatch[1]}`] = flagMatch[2];
-    }
-  }
-
-  return env;
-}
-
 const jsFixtures = [
   // FTL - Math intrinsics
   "ftl-arithsin.js",