diff --git a/Cargo.lock b/Cargo.lock index 3ae3d5fb2386..32632519ce15 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1672,6 +1672,7 @@ dependencies = [ "bun_parsers", "bun_patch_jsc", "bun_paths", + "bun_perf", "bun_picohttp", "bun_ptr", "bun_resolve_builtins", diff --git a/src/js/internal-for-testing.ts b/src/js/internal-for-testing.ts index 10064f1f8b30..110a93a5e813 100644 --- a/src/js/internal-for-testing.ts +++ b/src/js/internal-for-testing.ts @@ -229,6 +229,27 @@ export const timerInternals = { timerClockMs: $newZigFunction("runtime/timer/Timer.zig", "internal_bindings.timerClockMs", 0), }; +export const hwTimerInternals = { + /** + * Run the x64 TSC-frequency decision (`bun_perf::hw_timer`) on caller-supplied CPUID values: + * (hypervisor, hvMaxLeaf, hvTscKhz, leaf15Eax, leaf15Ebx, leaf15Ecx) -> Hz (0 = OS-clock fallback). + */ + resolveTscFrequency: $newZigFunction("hw_timer.zig", "resolveTscFrequency", 6) as ( + hypervisor: boolean, + hvMaxLeaf: number, + hvTscKhz: number, + leaf15Eax: number, + leaf15Ebx: number, + leaf15Ecx: number, + ) => number, + /** The frequency hw_timer would calibrate with on this machine, plus a counter/OS-clock sample pair. */ + calibrationState: $newZigFunction("hw_timer.zig", "calibrationState", 0) as () => { + frequencyHz: number; + counter: number; + osNs: number; + }, +}; + export const decodeURIComponentSIMD = $newCppFunction( "decodeURIComponentSIMD.cpp", "jsFunctionDecodeURIComponentSIMD", diff --git a/src/perf/hw_timer.rs b/src/perf/hw_timer.rs index 3bec3eae94b7..705ad086446d 100644 --- a/src/perf/hw_timer.rs +++ b/src/perf/hw_timer.rs @@ -7,17 +7,27 @@ //! `mach_absolute_time`, with ~24 ns resolution instead of ~12 µs. On Windows //! it replaces `GetTickCount64`'s ~15.6 ms granularity. //! -//! `now_ns()` is calibrated once against the OS monotonic clock so its values -//! share an epoch with `bun.getRoughTickCount()`. For pure A→B deltas where the -//! epoch doesn't matter, `read_counter()` is the cheapest possible read. +//! The calibrated clock is anchored once against the OS monotonic clock so its +//! values share an epoch with `bun.getRoughTickCount()`. For pure A→B deltas +//! where the epoch doesn't matter, `read_counter()` is the cheapest possible +//! read. //! -//! On x64 Linux/Windows where the TSC frequency isn't exposed by CPUID 0x15, -//! `now_ns()` reads the OS high-res clock per call (vDSO/QPC, ~20 ns) instead — +//! On x64 Linux/Windows the TSC frequency comes from CPUID: leaf 0x15 on bare +//! metal, or the hypervisor timing leaf 0x4000_0010 inside a guest (leaf 0x15 +//! describes the host part's crystal there, not the rate the guest's `rdtsc` +//! actually ticks at). When neither source is trustworthy, the calibrated +//! clock reads the OS high-res clock per call (vDSO/QPC, ~20 ns) instead — //! still sub-µs resolution. //! //! See WebKit r312153 (UnbarrieredMonotonicTime) for the original design and //! drift/monotonicity measurements on Darwin/arm64. +#[cfg(all( + target_arch = "x86_64", + any(target_os = "macos", target_os = "freebsd") +))] +use core::ffi::{c_char, c_int, c_void}; + /// Raw counter read. No barriers. /// - aarch64: `CNTVCT_EL0` (fixed-frequency virtual counter) /// - x86_64: `rdtsc` @@ -54,3 +64,276 @@ pub fn read_counter() -> u64 { #[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] compile_error!("hw_timer::read_counter: unsupported architecture"); } + +/// Counter frequency in Hz, or 0 if it can't be learned without spinning. +/// All paths that return non-zero already imply invariant/constant-rate TSC. +fn read_frequency() -> u64 { + #[cfg(target_arch = "aarch64")] + { + // Architectural register; always populated. + let ret: u64; + // SAFETY: reading CNTFRQ_EL0 is side-effect-free and always valid at EL0. + unsafe { + core::arch::asm!( + "mrs {ret}, CNTFRQ_EL0", + ret = out(reg) ret, + options(nomem, nostack, preserves_flags), + ); + } + return ret; + } + + #[cfg(target_arch = "x86_64")] + { + #[cfg(any(target_os = "macos", target_os = "freebsd"))] + { + // Kernel's own boot-time TSC calibration. Only present (and only + // meaningful) when the kernel has decided TSC is usable. + const NAME: &core::ffi::CStr = if cfg!(target_os = "macos") { + c"machdep.tsc.frequency" + } else { + c"machdep.tsc_freq" + }; + let mut hz: u64 = 0; + let mut hz_len: usize = core::mem::size_of::(); + // SAFETY: NAME is NUL-terminated; oldp/oldlenp point to valid stack locals. + unsafe { + let _ = sysctlbyname( + NAME.as_ptr(), + core::ptr::from_mut::(&mut hz).cast::(), + &raw mut hz_len, + core::ptr::null(), + 0, + ); + } + return hz; + } + + #[cfg(not(any(target_os = "macos", target_os = "freebsd")))] + { + // Linux/Windows: require invariant TSC (CPUID 0x8000_0007 EDX[8]) so + // rdtsc is monotonic across cores and P/C-states, whichever source + // the frequency comes from below. + if cpuid(0x8000_0000, 0).eax < 0x8000_0007 || cpuid(0x8000_0007, 0).edx & (1 << 8) == 0 + { + return 0; + } + + let hypervisor = cpuid(1, 0).ecx & (1 << 31) != 0; + let (hv_max_leaf, hv_tsc_khz) = if hypervisor { + let max_leaf = cpuid(0x4000_0000, 0).eax; + let khz = if max_leaf >= HYPERVISOR_TIMING_LEAF { + cpuid(HYPERVISOR_TIMING_LEAF, 0).eax + } else { + 0 + }; + (max_leaf, khz) + } else { + (0, 0) + }; + + let leaf_15 = if cpuid(0, 0).eax >= 0x15 { + cpuid(0x15, 0) + } else { + CpuidResult { + eax: 0, + ebx: 0, + ecx: 0, + edx: 0, + } + }; + + return resolve_x64_tsc_frequency(X64TscCpuidInfo { + hypervisor, + hv_max_leaf, + hv_tsc_khz, + leaf_15_eax: leaf_15.eax, + leaf_15_ebx: leaf_15.ebx, + leaf_15_ecx: leaf_15.ecx, + }); + } + } + + #[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] + compile_error!("hw_timer::read_frequency: unsupported target"); +} + +/// Hypervisor "timing information" leaf (VMware interface, also implemented by +/// KVM when it wants guests to know the TSC rate): EAX is the guest's TSC +/// frequency in kHz, already accounting for TSC scaling. +const HYPERVISOR_TIMING_LEAF: u32 = 0x4000_0010; + +/// Reject hypervisor-advertised TSC rates outside [100 MHz, 10 GHz]; real +/// parts sit well inside this range, so anything else is a broken leaf. +const MIN_PLAUSIBLE_TSC_HZ: u64 = 100_000_000; +const MAX_PLAUSIBLE_TSC_HZ: u64 = 10_000_000_000; + +/// Raw CPUID values that decide the x64 TSC frequency. Gathered from the live +/// CPU by `read_frequency()`; built with synthetic values by the +/// `bun:internal-for-testing` binding so the decision logic can be exercised +/// off the exact hardware that exhibits a mis-calibration. +#[derive(Clone, Copy)] +pub struct X64TscCpuidInfo { + /// CPUID.1:ECX[31] — running under a hypervisor. + pub hypervisor: bool, + /// CPUID.0x4000_0000:EAX — highest hypervisor leaf (0 when not read). + pub hv_max_leaf: u32, + /// CPUID.0x4000_0010:EAX — guest TSC frequency in kHz (0 when not read). + pub hv_tsc_khz: u32, + /// CPUID.0x15:EAX — denominator of the TSC/crystal ratio (0 when absent). + pub leaf_15_eax: u32, + /// CPUID.0x15:EBX — numerator of the TSC/crystal ratio (0 when absent). + pub leaf_15_ebx: u32, + /// CPUID.0x15:ECX — crystal clock frequency in Hz (0 when absent). + pub leaf_15_ecx: u32, +} + +/// Decide the TSC frequency (Hz) from raw CPUID values, or 0 when no +/// trustworthy source exists and the calibrated clock must stay on the OS +/// clock. +/// +/// Unlike the Zig reference, leaf 0x15 is never trusted under a hypervisor. +/// Inside a guest the TSC the OS hands out may be scaled/emulated, so leaf +/// 0x15 (host crystal info leaked through the VMM) need not match the rate +/// `rdtsc` actually ticks at — on some GCP/KVM hosts it is off by ~2.8×, which +/// skews every deadline derived from a clock calibrated with it. The only +/// CPUID source trusted under a hypervisor is the timing leaf the hypervisor +/// itself publishes; otherwise we fall back to the OS clock, which the guest +/// kernel already calibrates correctly. +pub fn resolve_x64_tsc_frequency(info: X64TscCpuidInfo) -> u64 { + if info.hypervisor { + if info.hv_max_leaf >= HYPERVISOR_TIMING_LEAF { + let hz = u64::from(info.hv_tsc_khz) * 1000; + if (MIN_PLAUSIBLE_TSC_HZ..=MAX_PLAUSIBLE_TSC_HZ).contains(&hz) { + return hz; + } + } + return 0; + } + // Bare metal: CPUID 0x15 is the architectural crystal-clock ratio (exact on + // Intel Skylake+ when fully populated). AMD and older Intel leave the + // fields zero — fall back to vDSO/QPC per call. + if info.leaf_15_eax != 0 && info.leaf_15_ebx != 0 && info.leaf_15_ecx != 0 { + return u64::from(info.leaf_15_ecx) * u64::from(info.leaf_15_ebx) + / u64::from(info.leaf_15_eax); + } + 0 +} + +/// Point-in-time view of the values the TSC calibration works from, for +/// `bun:internal-for-testing`. +pub struct CalibrationSnapshot { + /// What `read_frequency()` reports on this machine (0 ⇒ OS-clock fallback). + pub frequency_hz: u64, + /// `read_counter()` sampled immediately before `os_ns`. + pub counter: u64, + /// OS monotonic clock in nanoseconds. + pub os_ns: u64, +} + +/// Snapshot the counter against the OS monotonic clock plus the frequency the +/// HW path would calibrate with, so tests can verify the two agree. +pub fn calibration_snapshot() -> CalibrationSnapshot { + let frequency_hz = read_frequency(); + let counter = read_counter(); + let os_ns = os_monotonic_ns(); + CalibrationSnapshot { + frequency_hz, + counter, + os_ns, + } +} + +#[cfg(all( + target_arch = "x86_64", + not(any(target_os = "macos", target_os = "freebsd")) +))] +struct CpuidResult { + eax: u32, + ebx: u32, + ecx: u32, + edx: u32, +} + +#[cfg(all( + target_arch = "x86_64", + not(any(target_os = "macos", target_os = "freebsd")) +))] +#[inline] +fn cpuid(leaf: u32, subleaf: u32) -> CpuidResult { + // Rust inline asm reserves `rbx` (LLVM PIC base), so use the std intrinsic, + // which handles the xchg dance internally instead of raw asm. + // (`__cpuid_count` is a safe fn on x86_64 — cpuid is baseline.) + let r = core::arch::x86_64::__cpuid_count(leaf, subleaf); + CpuidResult { + eax: r.eax, + ebx: r.ebx, + ecx: r.ecx, + edx: r.edx, + } +} + +/// OS high-res monotonic clock. Used as the anchor the counter is measured +/// against in `calibration_snapshot()`. +fn os_monotonic_ns() -> u64 { + #[cfg(windows)] + { + // QPF is a constant read from KUSER_SHARED_DATA; no need to cache. + let mut counter: i64 = 0; + let mut freq: i64 = 0; + // QPC/QPF are declared `safe` in bun_sys (the out-param is a valid + // `&mut` and they never fail on XP+), so no `unsafe` block is needed. + bun_sys::windows::QueryPerformanceCounter(&mut counter); + bun_sys::windows::QueryPerformanceFrequency(&mut freq); + return u64::try_from((counter as u128) * (NS_PER_S as u128) / (freq as u128)).unwrap(); + } + #[cfg(not(windows))] + { + let mut spec = libc::timespec { + tv_sec: 0, + tv_nsec: 0, + }; + #[cfg(any(target_os = "linux", target_os = "android"))] + { + // CLOCK_MONOTONIC, not _RAW: guaranteed vDSO (no syscall). _RAW only + // joined the vDSO in 5.3. + // SAFETY: spec is a valid out-pointer. + unsafe { + let _ = libc::clock_gettime(libc::CLOCK_MONOTONIC, &raw mut spec); + } + } + #[cfg(target_os = "macos")] + { + // SAFETY: spec is a valid out-pointer. + unsafe { + let _ = libc::clock_gettime(libc::CLOCK_MONOTONIC_RAW, &raw mut spec); + } + } + #[cfg(not(any(target_os = "linux", target_os = "android", target_os = "macos")))] + { + // SAFETY: spec is a valid out-pointer. + unsafe { + let _ = libc::clock_gettime(libc::CLOCK_MONOTONIC, &raw mut spec); + } + } + (spec.tv_sec as u64) + .wrapping_mul(NS_PER_S) + .wrapping_add(spec.tv_nsec as u64) + } +} + +use bun_core::time::NS_PER_S; + +#[cfg(all( + target_arch = "x86_64", + any(target_os = "macos", target_os = "freebsd") +))] +unsafe extern "C" { + fn sysctlbyname( + name: *const c_char, + oldp: *mut c_void, + oldlenp: *mut usize, + newp: *const c_void, + newlen: usize, + ) -> c_int; +} diff --git a/src/runtime/Cargo.toml b/src/runtime/Cargo.toml index 74f6d8bb7f1a..445d571f1c3e 100644 --- a/src/runtime/Cargo.toml +++ b/src/runtime/Cargo.toml @@ -71,6 +71,7 @@ bun_output.workspace = true bun_spawn.workspace = true bun_sql_jsc.workspace = true bun_paths.workspace = true +bun_perf.workspace = true bun_picohttp.workspace = true bun_ptr.workspace = true bun_resolve_builtins.workspace = true diff --git a/src/runtime/dispatch_js2native.rs b/src/runtime/dispatch_js2native.rs index 4af6e63fec56..97baab8f07ed 100644 --- a/src/runtime/dispatch_js2native.rs +++ b/src/runtime/dispatch_js2native.rs @@ -76,6 +76,66 @@ pub(crate) fn bun_get_use_system_ca( Ok(JSValue::js_boolean(v)) } +// Test-only bindings for `bun_perf::hw_timer`. The bodies live here (like +// `bun_get_use_system_ca` above) because `bun_perf` is a low-tier crate with +// no `*_jsc` sibling and must not depend on `bun_jsc`. + +/// `hwTimerInternals.resolveTscFrequency(hypervisor, hvMaxLeaf, hvTscKhz, +/// leaf15Eax, leaf15Ebx, leaf15Ecx)` — run the x64 TSC-frequency decision on +/// caller-supplied CPUID values so tests can cover hypervisor/bare-metal +/// combinations this machine doesn't exhibit. +pub(crate) fn perf_hw_timer_resolve_tsc_frequency( + global: &JSGlobalObject, + frame: &CallFrame, +) -> JsResult { + let got = frame.arguments_count() as usize; + if got < 6 { + return Err(global.throw_not_enough_arguments("resolveTscFrequency", 6, got)); + } + let uint = |i: usize| -> JsResult { + let v = frame.argument(i).coerce_to_int64(global)?; + Ok(v.clamp(0, i64::from(u32::MAX)) as u32) + }; + let info = bun_perf::hw_timer::X64TscCpuidInfo { + hypervisor: frame.argument(0).to_boolean(), + hv_max_leaf: uint(1)?, + hv_tsc_khz: uint(2)?, + leaf_15_eax: uint(3)?, + leaf_15_ebx: uint(4)?, + leaf_15_ecx: uint(5)?, + }; + Ok(JSValue::js_number_from_uint64( + bun_perf::hw_timer::resolve_x64_tsc_frequency(info), + )) +} + +/// `hwTimerInternals.calibrationState()` — the frequency `bun_perf::hw_timer` +/// would calibrate with on this machine plus a counter/OS-clock sample pair, +/// so tests can measure the real counter rate and compare. +pub(crate) fn perf_hw_timer_calibration_state( + global: &JSGlobalObject, + _frame: &CallFrame, +) -> JsResult { + let snapshot = bun_perf::hw_timer::calibration_snapshot(); + let result = JSValue::create_empty_object(global, 3); + result.put( + global, + b"frequencyHz", + JSValue::js_number_from_uint64(snapshot.frequency_hz), + ); + result.put( + global, + b"counter", + JSValue::js_number_from_uint64(snapshot.counter), + ); + result.put( + global, + b"osNs", + JSValue::js_number_from_uint64(snapshot.os_ns), + ); + Ok(result) +} + mod css { pub use bun_css_jsc::css_internals::{ _test, attr_test, minify_error_test_with_options, minify_test, minify_test_with_options, diff --git a/test/js/bun/perf/hw-timer-tsc-frequency.test.ts b/test/js/bun/perf/hw-timer-tsc-frequency.test.ts new file mode 100644 index 000000000000..beb5b2de7cfd --- /dev/null +++ b/test/js/bun/perf/hw-timer-tsc-frequency.test.ts @@ -0,0 +1,77 @@ +import { expect, test } from "bun:test"; + +// Looked up at call time (instead of a named ESM import) so that running this +// file against a build without the binding reports each test as failed with a +// clear error instead of aborting the whole file at module load. +const { hwTimerInternals } = require("bun:internal-for-testing"); + +// CPUID 0x15 values the way a Skylake-class part reports them on bare metal: +// TSC/crystal ratio = 188/2 with a 24 MHz crystal -> 2.256 GHz. +const leaf15 = { eax: 2, ebx: 188, ecx: 24_000_000 }; +const leaf15Hz = (leaf15.ecx * leaf15.ebx) / leaf15.eax; + +test("hw_timer exposes the TSC frequency decision to bun:internal-for-testing", () => { + expect(hwTimerInternals).toBeDefined(); + expect(typeof hwTimerInternals.resolveTscFrequency).toBe("function"); + expect(typeof hwTimerInternals.calibrationState).toBe("function"); +}); + +test("CPUID leaf 0x15 is not trusted inside a hypervisor guest", () => { + // The GCP/KVM mis-calibration shape: the guest sees the hypervisor bit plus a + // populated leaf 0x15 describing the *host* crystal, which does not match the + // rate the guest's (scaled) TSC actually ticks at. With no hypervisor timing + // leaf available there is no trustworthy CPUID source, so the decision must + // be 0 (OS-clock fallback) instead of the leaf-0x15 value. + expect(hwTimerInternals.resolveTscFrequency(true, 0x4000_0001, 0, leaf15.eax, leaf15.ebx, leaf15.ecx)).toBe(0); +}); + +test("hypervisor timing leaf wins over leaf 0x15 in a guest", () => { + // KVM/VMware advertise the guest TSC rate (in kHz) via leaf 0x4000_0010; that + // value accounts for TSC scaling, so it is the only CPUID source trusted + // under a hypervisor — even when leaf 0x15 is also populated. + expect(hwTimerInternals.resolveTscFrequency(true, 0x4000_0010, 2_899_987, leaf15.eax, leaf15.ebx, leaf15.ecx)).toBe( + 2_899_987_000, + ); +}); + +test("implausible hypervisor timing-leaf values fall back to the OS clock", () => { + // Timing leaf advertised but empty. + expect(hwTimerInternals.resolveTscFrequency(true, 0x4000_0010, 0, leaf15.eax, leaf15.ebx, leaf15.ecx)).toBe(0); + // 1 kHz and ~4.3 THz are not real TSC rates. + expect(hwTimerInternals.resolveTscFrequency(true, 0x4000_0010, 1, leaf15.eax, leaf15.ebx, leaf15.ecx)).toBe(0); + expect(hwTimerInternals.resolveTscFrequency(true, 0xffff_ffff, 0xffff_ffff, leaf15.eax, leaf15.ebx, leaf15.ecx)).toBe( + 0, + ); +}); + +test("bare metal still uses CPUID leaf 0x15", () => { + expect(hwTimerInternals.resolveTscFrequency(false, 0, 0, leaf15.eax, leaf15.ebx, leaf15.ecx)).toBe(leaf15Hz); + // Partially populated leaf 0x15 (AMD, pre-Skylake Intel) stays on the OS clock. + expect(hwTimerInternals.resolveTscFrequency(false, 0, 0, leaf15.eax, leaf15.ebx, 0)).toBe(0); + expect(hwTimerInternals.resolveTscFrequency(false, 0, 0, 0, 0, 0)).toBe(0); +}); + +test("calibration frequency matches the OS monotonic clock on this machine", async () => { + // End-to-end guard for the original report (setTimeout firing at ~2.8x the + // requested delay on some KVM guests): whatever frequency hw_timer decides to + // calibrate with must describe the rate the hardware counter actually ticks + // at, as measured against the OS monotonic clock. + const a = hwTimerInternals.calibrationState(); + expect(a.frequencyHz).toBeGreaterThanOrEqual(0); + if (a.frequencyHz === 0) { + // No trustworthy CPUID frequency on this machine; hw_timer reads the OS + // clock per call, which cannot mis-calibrate. + return; + } + + await Bun.sleep(150); + + const b = hwTimerInternals.calibrationState(); + expect(b.osNs).toBeGreaterThan(a.osNs); + const measuredHz = ((b.counter - a.counter) / (b.osNs - a.osNs)) * 1e9; + expect(measuredHz).toBeGreaterThan(0); + // Scheduling noise between the counter read and the clock read inside one + // sample is far below a millisecond over a 150 ms window (<1%), while the + // failure mode being guarded against is a 2.8x mismatch. + expect(Math.abs(measuredHz - a.frequencyHz) / measuredHz).toBeLessThan(0.2); +});