From 50b829ff31c169e155f55301bd6799191d4fddf3 Mon Sep 17 00:00:00 2001 From: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com> Date: Wed, 15 Jul 2026 16:37:56 -0700 Subject: [PATCH] Prevent MSR state leaking across restore KVM denies guest MSR access by default. SandboxConfiguration::allow_msrs permits selected MSRs. MSHV and WHP have no per-MSR filter. Hyperlight captures exposed retained MSR state at VM creation and resets it on restore. Captured MSR state persists in OCI snapshots. KVM denials report the MSR index. Unsupported accesses on MSHV and WHP raise a guest general protection fault. Both failures poison the sandbox. Signed-off-by: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com> --- CHANGELOG.md | 4 + Justfile | 4 + docs/msr.md | 304 ++++ docs/snapshot-versioning.md | 12 + src/hyperlight_host/src/error.rs | 14 + .../src/hypervisor/hyperlight_vm/mod.rs | 27 + .../src/hypervisor/hyperlight_vm/x86_64.rs | 151 +- .../src/hypervisor/regs/x86_64/mod.rs | 2 + .../src/hypervisor/regs/x86_64/msrs.rs | 483 +++++++ .../hypervisor/regs/x86_64/special_regs.rs | 10 + .../hypervisor/virtual_machine/kvm/x86_64.rs | 336 ++++- .../src/hypervisor/virtual_machine/mod.rs | 118 ++ .../hypervisor/virtual_machine/mshv/x86_64.rs | 140 +- .../src/hypervisor/virtual_machine/whp.rs | 203 ++- src/hyperlight_host/src/mem/mgr.rs | 8 + src/hyperlight_host/src/sandbox/config.rs | 128 +- .../src/sandbox/initialized_multi_use.rs | 1254 ++++++++++++++++- .../src/sandbox/snapshot/file/config.rs | 85 ++ .../src/sandbox/snapshot/file/mod.rs | 17 + .../src/sandbox/snapshot/file_tests.rs | 194 +++ .../src/sandbox/snapshot/mod.rs | 56 + src/tests/rust_guests/simpleguest/src/main.rs | 81 ++ 22 files changed, 3605 insertions(+), 26 deletions(-) create mode 100644 docs/msr.md create mode 100644 src/hyperlight_host/src/hypervisor/regs/x86_64/msrs.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index a7c7c6ae7..fd6e60137 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added ### Changed +* **Breaking:** Retained guest MSR state is restored on snapshot restore. KVM + denies guest MSR reads and writes by default. MSHV and WHP reset all exposed + retained MSR state. `SandboxConfiguration::allow_msrs` permits specific MSRs + by @ludfjig in https://github.com/hyperlight-dev/hyperlight/pull/991 ### Removed diff --git a/Justfile b/Justfile index 1eb4dd3a0..d12fc7793 100644 --- a/Justfile +++ b/Justfile @@ -243,6 +243,10 @@ test-isolated target=default-target features="" : {{ cargo-cmd }} test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} {{ target-triple-flag }} -p hyperlight-host --test integration_test -- log_message --exact --ignored @# CPU vendor check, gated to known CI runner hardware {{ cargo-cmd }} test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} {{ target-triple-flag }} -p hyperlight-host --lib -- sandbox::snapshot::file::config::tests::cpu_vendor_current_is_recognized --exact --ignored + @# Slow host-dependent MSR audit. Run once per x86_64 CI profile. + {{ if features == "" { if hyperlight-target-arch == "x86_64" { cargo-cmd + " test --profile=" + (if target == "debug" { "dev" } else { target }) + " " + target-triple-flag + " -p hyperlight-host --lib -- sandbox::initialized_multi_use::tests::msr_tests::test_no_msr_leaks_across_restore_full_window_sweep --exact --ignored --nocapture" } else { "" } } else { "" } }} + @# LAPIC-enabled MSHV can expose additional MSRs. Audit it once in debug. + {{ if features == "mshv3,hw-interrupts" { if target == "debug" { if hyperlight-target-arch == "x86_64" { cargo-cmd + " test --no-default-features -F mshv3,hw-interrupts --profile=dev " + target-triple-flag + " -p hyperlight-host --lib -- sandbox::initialized_multi_use::tests::msr_tests::test_no_msr_leaks_across_restore_full_window_sweep --exact --ignored --nocapture" } else { "" } } else { "" } } else { "" } }} @# metrics tests {{ cargo-cmd }} test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F function_call_metrics," + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} {{ target-triple-flag }} -p hyperlight-host --lib -- metrics::tests::test_metrics_are_emitted --exact diff --git a/docs/msr.md b/docs/msr.md new file mode 100644 index 000000000..316f097c5 --- /dev/null +++ b/docs/msr.md @@ -0,0 +1,304 @@ +# MSR state across restore + +## Requirement + +A snapshot restore must remove all model-specific register (MSR) state written +after the snapshot. The guest must observe the MSR values saved with the +restored state. + +## Reset set + +The reset set contains every MSR whose guest-written value can persist. Each +running snapshot stores values for this set. A snapshot created from a guest +binary has no saved MSR values, so restore uses the baseline captured when the +VM was created. + +`MSR_TABLE` lists the MSRs that hold retained state restore must write. A +write-only command MSR holds no state, so it is absent from the table. + +The resolved reset set contains the backend core set, required MTRRs, and the +validated allow list. Hyperlight sorts and deduplicates the indices before +capturing the initialization baseline. + +The required invariant is: + +```text +guest-writable retained state => host-readable and host-writable state +``` + +Host-readable state need not be guest-writable. Extra reset entries are safe. +`EFER`, `APIC_BASE`, `FS_BASE`, and `GS_BASE` belong to the special-register +state. + +The two halves are established differently. Resolution checks the +host-readable half at run time: a candidate index enters the set only when the +host read succeeds, so an unreadable MSR is dropped. Nothing checks the +host-writable half at run time. It holds by construction. Every reset MSR is +stored in VP register state that the Hyper-V host interface both reads and +writes, except `KERNEL_GS_BASE`, a real register the host reads and writes +directly. Round-trip tests plant a guest value, restore, and assert that it does +not survive. Every future entry must remain host-readable and host-writable. + +## Reset set justification + +Each entry is grounded in how the Hyper-V hypervisor handles a guest access to +that register, confirmed against the Hyper-V source. + +A register is reset when the guest can write it and Hyper-V keeps the written +value. Hyper-V keeps it in one of two ways: it stores the value in the VP's +saved register state, or it lets the guest write the real register directly. +Only `FS_BASE`, `GS_BASE`, and `KERNEL_GS_BASE` are written directly. Every +other register below is intercepted and stored. Hyper-V stores the value on both +Intel and AMD hosts. + +Interception alone is not the test. Hyper-V also intercepts registers the guest +can only read, or that return a host-derived value. Those keep no +guest-controlled state and are not reset. Each row below names the guest state +that persists. + +| MSR (index) | Retained guest state | +| --- | --- | +| SYSENTER CS, ESP, EIP (`0x174`-`0x176`) | Guest write retained. | +| STAR, LSTAR, CSTAR, SFMASK (`0xC000_0081`-`0xC000_0084`) | Guest write retained (syscall targets). | +| KERNEL_GS_BASE (`0xC000_0102`) | Guest write retained. Written to the real register, and reachable through `SWAPGS` without a `WRMSR`. | +| PAT (`0x277`) | Guest write retained. | +| DEBUGCTL (`0x1D9`) | Guest write retained. | +| SPEC_CTRL (`0x48`) | Guest write retained. | +| CET U_CET, S_CET, PL0-3_SSP, INTERRUPT_SSP_TABLE_ADDR (`0x6A0`, `0x6A2`, `0x6A4`-`0x6A8`) | Guest write retained. | +| XSS (`0xDA0`) | Guest write retained. | +| TSC (`0x10`) | Guest write retained. Hyper-V forbids intercepting its implemented TSC, so restore rewrites the captured value. | +| TSC_ADJUST (`0x3B`) | Guest write retained, independent of TSC. | +| TSC_AUX (`0xC000_0103`) | Guest write retained. | +| MTRRs (`0x2FF`, `0x200`-`0x21F`, `0x250`, `0x258`-`0x259`, `0x268`-`0x26F`) | Guest write retained (memory-type state). | +| TSX_CTRL (`0x122`) | Guest write retained. | +| XFD, XFD_ERR (`0x1C4`, `0x1C5`) | Guest write retained. | +| UMWAIT_CONTROL (`0xE1`) | Guest write retained. Intel only. | +| TSC_DEADLINE (`0x6E0`) | Guest write retained. | +| BNDCFGS (`0xD90`) | Guest write retained when the host supports MPX. A guest access faults otherwise. | +| MPERF, APERF (`0xE7`, `0xE8`) | Guest write retained in per-VP counters. | + +Write-only command MSRs hold no state. A guest write performs an action and +leaves nothing to restore, so they are absent from the reset table and cannot +be allowed. + +| MSR (index) | Behavior | +| --- | --- | +| PRED_CMD (`0x49`) | Guest write issues a prediction barrier. | +| FLUSH_CMD (`0x10B`) | Guest write flushes caches. | + +Some registers are deliberately absent because guest access cannot leave state +outside another reset mechanism. + +| MSR (index) | Why excluded | +| --- | --- | +| MISC_ENABLE (`0x1A0`) | Intercepted, but Hyper-V discards a guest write and returns a fixed value. No retained state. On AMD the access faults. See below. | +| FRED (`0x1CC`-`0x1D4`) | Retained only when the host exposes FRED, which Hyperlight does not. A guest access faults otherwise. | +| PASID (`0xD93`) | MSHV exposes ENQCMD on capable Intel hosts. PASID is a supervisor XSAVE component, so XSAVE reset clears it. WHP does not expose ENQCMD. | +| PMU: PMC0, PERFEVTSEL0, FIXED_CTR_CTRL, PERF_GLOBAL_CTRL (`0xC1`, `0x186`, `0x38D`, `0x38F`) | Heads of the performance-monitoring class. Hyper-V leaves these unimplemented for the guest and installs guest-accessible descriptors, sized to the CPU counter count, only when perfmon is enabled. Hyperlight enables no perfmon, so a guest access faults and retains nothing. | +| LBR: LBR_SELECT, LASTBRANCH_TOS, LBR_CTL, LBR_DEPTH (`0x1C8`, `0x1C9`, `0x14CE`, `0x14CF`) | Last-branch registers, gated with perfmon. A guest access faults and retains nothing while perfmon stays off. | + +Hyper-V virtualizes `BNDCFGS`, `FRED`, and `PASID` only when the matching CPU +feature is exposed. `BNDCFGS` is reset because MPX is exposed by default on +capable hosts. `FRED` stays excluded because Hyperlight does not expose it. +MSHV can expose ENQCMD, but its XSAVE state mask then includes PASID. Hyperlight +clears PASID during XSAVE reset before restoring MSRs. The performance-monitoring +and last-branch registers remain inaccessible while perfmon is off. + +## Snapshot validation + +Snapshot MSR entries are untrusted. A snapshot records the reset values and the +capturing sandbox's allow list. `validate_snapshot` enforces two rules against +the destination VM's reset set: + +* The snapshot's allow list must be a subset of the destination's. A + destination that allows at least as much accepts the snapshot. +* Every supplied index must belong to the destination reset set. + +Indices the destination resets but the snapshot omits take the destination's +creation-time baseline. A rejected restore poisons the sandbox before the guest +can run. Equivalent allow lists produce the same sorted reset set, regardless of +insertion order. + +## Restore across allow lists + +A restore or `from_snapshot` succeeds when the destination allow list is a +superset of the snapshot's, on every backend. The snapshot's allowed MSRs keep +their captured values. An MSR the destination allows but the snapshot did not +resets to the destination baseline. A non-superset allow list is rejected +uniformly. + +The rule is backend independent even though each backend sizes its reset set +differently. KVM derives its reset set from the allow list. MSHV and WHP reset +the full host table. The allow-list subset check gates the restore before either +reset set is applied, so a flow that succeeds on one backend succeeds on all. + +The superset check is the common rule across backends. MSHV and WHP accept any +allow list on their own. The shared check gives every backend KVM's constraint. + +## Allow list + +`SandboxConfiguration::allow_msrs` adds indices to the requested allow list. It +enforces capacity only. VM creation verifies that each index is resettable and +supported by the selected backend. + +KVM requires the index in `KVM_GET_MSR_INDEX_LIST` and a successful host read +and write. MSHV and WHP require a named-register mapping and a successful host +read. + +At most 16 distinct MSRs may be requested. KVM also limits the resulting +contiguous filter groups to 16. + +## KVM + +KVM installs a deny filter over the full MSR space. Allowed indices form the +only guest `RDMSR` and `WRMSR` paths through that filter. A denied access exits +to Hyperlight, injects `#GP`, and poisons the sandbox. The denied write stores +no state. + +The KVM reset set contains the allow list plus `KERNEL_GS_BASE` and `TSC`. +`KERNEL_GS_BASE` is required because `WRGSBASE` followed by `SWAPGS` changes it +without `WRMSR`. `TSC` gives restore the same clock semantics on every backend. + +KVM does not filter x2APIC indices `0x800..=0x8FF`. Hyperlight keeps the APIC in +xAPIC mode, where MSR access to that range raises `#GP`. `APIC_BASE` is not an +allowable MSR, so a guest cannot enable x2APIC. Snapshots created by Hyperlight +therefore retain `APIC_BASE.EXTD = 0`. File snapshots serialize `APIC_BASE` +without semantic validation, so the caller must trust the snapshot source as +required by the snapshot format. + +## MTRRs + +MSHV and WHP read `IA32_MTRRCAP` when the VM is created. The required set +contains `MTRR_DEF_TYPE`, each variable pair reported by `VCNT`, and all fixed +MTRRs. + +Hyper-V accepts fixed-MTRR writes even when `MTRRCAP.FIX` is clear. All fixed +MTRRs are therefore required. Hyper-V supports at most 16 variable pairs. VM +creation fails when the count is larger or a required MTRR cannot be read. + +## MSHV + +MSHV has no per-MSR filter. Hyper-V permits an MSR intercept only for an +unimplemented index, which already faults for the guest, and cannot intercept +the implemented MSRs that hold retained state. Isolation therefore comes from +reset, not a deny filter. + +The MSHV reset set contains every table entry that has a Hyper-V +register mapping and can be read, plus the allow list. + +`msr_to_hv_reg_name` determines which indices the get and set path can reach. +The enumerated host index list does not identify retained state, so it does not +define the reset set. + +MSHV maps `IA32_XSS` through `MSR_IA32_REGISTER_U_XSS`. It maps `IA32_MPERF` +and `IA32_APERF` to the per-VP `MCount` and `ACount` registers. TSX control, +XFD, MPX (`BNDCFGS`), WAITPKG (`UMWAIT_CONTROL`), and the TSC deadline timer +enter the reset set when their host-register probes succeed. + +MSHV enables every host-supported processor feature unless the caller supplies +an explicit disabled-feature mask. Hyperlight supplies no mask. On capable +Intel hosts this can expose ENQCMD and its PASID MSR. MSHV reports PASID in the +partition XSAVE state mask, and Hyperlight's XSAVE reset clears it. + +## WHP + +WHP has no per-MSR filter. Its reset set contains every table entry +that has a WHP register name and can be read, plus the allow list. + +WHP uses Germanium compatibility. Speculation control is off in its default +feature banks, and perfmon (the PMU and architectural LBR) is a separate +property WHP leaves off. Experimental `DEBUGCTL` bits stay disabled. The WHP +API defines no FRED feature and its supported feature mask omits ENQCMD, so WHP +cannot expose FRED or PASID. + +Each guest MSR write is either captured for restore or unsupported by the +partition. Unsupported writes store no state. + +## TSC + +MSHV and WHP expose `TSC` as a host-writable register. Hyper-V stores `TSC` and +`TSC_ADJUST` independently, so restoring `TSC_ADJUST` cannot undo a guest +`WRMSR(TSC)`. + +While time is running, Hyper-V preserves `TSC - TSC_ADJUST`: writing `TSC` +adds the same delta to `TSC_ADJUST`, and writing `TSC_ADJUST` adds its delta to +the internal TSC offset. Restoring `TSC` followed by `TSC_ADJUST` therefore +cancels any guest-controlled delta. Freezing partition time is not required for +isolation. + +Hyper-V does not permit an intercept for its implemented `TSC` MSR. Restore +must therefore write the captured `TSC` value. KVM also restores `TSC` so all +backends rewind guest time with the rest of the snapshot state. + +## Feature exposure + +On MSHV and WHP a guest reaches an MSR only when the hypervisor exposes that +CPU feature to the partition. This gives three cases: + +* Not exposed. Features the partition does not enable, such as the + performance-monitoring unit, last-branch records, and FRED. Hyper-V may still + model the register, but a guest access faults and stores no state until the + feature is exposed. +* Exposed by default. Features the host CPU supports, such as TSC deadline, + UMWAIT, TSX control, CET, `MPERF`/`APERF`, XFD, AMX, and MPX. Their MSRs + must be in the reset set. +* Reset through another state class. MSHV can expose ENQCMD and PASID on + capable Intel hosts. PASID is cleared by XSAVE reset, so it is not duplicated + in the MSR reset set. + +MSHV and WHP enable partition features differently. MSHV creates the partition +without an explicit feature mask, so it enables every processor feature the host +supports. WHP starts from the host-supported set with speculation control off. +MSHV exposes the broader surface and determines which registers the reset set +must cover. + +Perfmon is not part of either default. The performance-monitoring unit and the +last-branch registers are a separate opt-in partition property, off by default +on both backends. Hyperlight never enables it, so those registers stay +unreachable regardless of the enable-everything processor-feature default. + +Only reachable, retained MSRs need coverage, and retained state is always held +in a host-readable and writable register. The mapped registers therefore bound +the reset set: coverage is complete when every mapped register is in the reset +table and reset. + +## Host-addressable but not guest-writable + +A host register mapping does not imply the guest can write the MSR. +`IA32_MISC_ENABLE` (`0x1A0`) is the notable case. Hyper-V emulates it, discards +a guest write, and returns a fixed value to the guest regardless of what was +written. A guest cannot change it to any value, so it retains no guest state +and needs no reset. On AMD the guest access faults. + +## Failed access reporting + +A KVM-denied or Hyper-V-unsupported MSR access does not persist and poisons the +sandbox. The error type and its detail differ by backend. + +* KVM traps the access at the deny filter. Hyperlight reports + `MsrReadViolation` or `MsrWriteViolation`, naming the MSR index and, for a + write, the value. The report is host-verified. +* MSHV and WHP have no host MSR trap. An unsupported access faults inside the + guest as a general protection fault from Hyper-V, so Hyperlight reports + `GuestAborted`. The message records the fault and the faulting instruction but + does not identify the MSR. An exposed MSR can succeed even when absent from + the allow list. Its retained state must be in the reset table. + +Future work: the guest exception handler could decode a faulting `RDMSR` or +`WRMSR` and report the index, promoting the abort to a typed MSR violation on +MSHV and WHP. That index would be guest-reported and therefore advisory. It is +not implemented. + +## Limitations + +KVM's security boundary is structural because its deny filter bounds guest +writes. MSHV and WHP depend on the reset table and exposed processor +features. + +The filterless backend tests run on one CPU model per runner. Model-specific +state absent on that CPU is not exercised. A backend that exposes a new +retained MSR feature needs a matching table entry before Hyperlight can use it +safely. + +The ignored full-window audit probes fixed index ranges with a small set of +values. It cannot prove that every vendor MSR or accepted value is covered. diff --git a/docs/snapshot-versioning.md b/docs/snapshot-versioning.md index ec3dca233..0d465f651 100644 --- a/docs/snapshot-versioning.md +++ b/docs/snapshot-versioning.md @@ -44,6 +44,18 @@ The config blob also records `hyperlight_version`, the `CARGO_PKG_VERSION` of the host crate at write time. This is informational only. The loader records it for diagnostics and does not gate loading on it. +## Compatibility cleanup + +Record compatibility paths here when a future hard snapshot break can remove +them. + +### Optional MSR fields + +The persisted `msrs` and `allowed_msrs` fields are optional so snapshots made +before MSR capture remain loadable. At the next hard break, make them required +vectors and remove `serde(default)` and the missing-field fallback. Keep the +in-memory fields optional while `Snapshot` represents pre-init state. + ## Enforcement The format is large and easy to change by accident. Two mechanisms diff --git a/src/hyperlight_host/src/error.rs b/src/hyperlight_host/src/error.rs index c6738374d..0b982eed6 100644 --- a/src/hyperlight_host/src/error.rs +++ b/src/hyperlight_host/src/error.rs @@ -154,6 +154,16 @@ pub enum HyperlightError { #[error("Memory Access Violation at address {0:#x} of type {1}, but memory is marked as {2}")] MemoryAccessViolation(u64, MemoryRegionFlags, MemoryRegionFlags), + /// A denied guest MSR read. + #[cfg(all(target_arch = "x86_64", kvm))] + #[error("Guest read from denied MSR {0:#x}")] + MsrReadViolation(u32), + + /// A denied guest MSR write. + #[cfg(all(target_arch = "x86_64", kvm))] + #[error("Guest write of {1:#x} to denied MSR {0:#x}")] + MsrWriteViolation(u32, u64), + /// Memory Allocation Failed. #[error("Memory Allocation Failed with OS Error {0:?}.")] MemoryAllocationFailed(Option), @@ -352,6 +362,10 @@ impl HyperlightError { // as poisoning here too for defense in depth. | HyperlightError::HyperlightVmError(HyperlightVmError::Restore(_)) => true, + #[cfg(all(target_arch = "x86_64", kvm))] + HyperlightError::MsrReadViolation(_) + | HyperlightError::MsrWriteViolation(_, _) => true, + // These errors poison the sandbox because they can leave // it in an inconsistent state due to snapshot restore // failing partway through diff --git a/src/hyperlight_host/src/hypervisor/hyperlight_vm/mod.rs b/src/hyperlight_host/src/hypervisor/hyperlight_vm/mod.rs index 56f7d635d..2d9ad6c3e 100644 --- a/src/hyperlight_host/src/hypervisor/hyperlight_vm/mod.rs +++ b/src/hyperlight_host/src/hypervisor/hyperlight_vm/mod.rs @@ -163,6 +163,16 @@ impl DispatchGuestCallError { region_flags, }) => HyperlightError::MemoryAccessViolation(addr, access_type, region_flags), + #[cfg(all(target_arch = "x86_64", kvm))] + DispatchGuestCallError::Run(RunVmError::MsrReadViolation(msr_index)) => { + HyperlightError::MsrReadViolation(msr_index) + } + + #[cfg(all(target_arch = "x86_64", kvm))] + DispatchGuestCallError::Run(RunVmError::MsrWriteViolation { msr_index, value }) => { + HyperlightError::MsrWriteViolation(msr_index, value) + } + // Leave others as is other => HyperlightVmError::DispatchGuestCall(other).into(), }; @@ -213,6 +223,12 @@ pub enum RunVmError { MmioReadUnmapped(u64), #[error("MMIO WRITE access to unmapped address {0:#x}")] MmioWriteUnmapped(u64), + #[cfg(all(target_arch = "x86_64", kvm))] + #[error("Guest read from denied MSR {0:#x}")] + MsrReadViolation(u32), + #[cfg(all(target_arch = "x86_64", kvm))] + #[error("Guest write of {value:#x} to denied MSR {msr_index:#x}")] + MsrWriteViolation { msr_index: u32, value: u64 }, #[error("vCPU run failed: {0}")] RunVcpu(#[from] RunVcpuError), #[error("Unexpected VM exit: {0}")] @@ -409,6 +425,9 @@ pub(crate) struct HyperlightVm { pub(super) trace_info: MemTraceInfo, #[cfg(crashdump)] pub(super) rt_cfg: SandboxRuntimeConfig, + /// MSRs restored on snapshot restore. + #[cfg(target_arch = "x86_64")] + pub(super) msr_reset: crate::hypervisor::regs::MsrResetState, } impl HyperlightVm { @@ -732,6 +751,14 @@ impl HyperlightVm { } } } + #[cfg(all(target_arch = "x86_64", kvm))] + Ok(VmExit::MsrRead(msr_index)) => { + break Err(RunVmError::MsrReadViolation(msr_index)); + } + #[cfg(all(target_arch = "x86_64", kvm))] + Ok(VmExit::MsrWrite { msr_index, value }) => { + break Err(RunVmError::MsrWriteViolation { msr_index, value }); + } Ok(VmExit::Cancelled()) => { // If cancellation was not requested for this specific guest function call, // the vcpu was interrupted by a stale cancellation. This can occur when: diff --git a/src/hyperlight_host/src/hypervisor/hyperlight_vm/x86_64.rs b/src/hyperlight_host/src/hypervisor/hyperlight_vm/x86_64.rs index 82206f787..408c7812c 100644 --- a/src/hyperlight_host/src/hypervisor/hyperlight_vm/x86_64.rs +++ b/src/hyperlight_host/src/hypervisor/hyperlight_vm/x86_64.rs @@ -41,9 +41,12 @@ use crate::hypervisor::gdb::{ #[cfg(gdb)] use crate::hypervisor::gdb::{DebugError, DebugMemoryAccessError}; use crate::hypervisor::regs::{ - CommonDebugRegs, CommonFpu, CommonRegisters, CommonSpecialRegisters, + CommonDebugRegs, CommonFpu, CommonRegisters, CommonSpecialRegisters, MsrEntry, MsrResetState, + core_reset_indices, is_mtrr_reset_index, }; -#[cfg(not(gdb))] +#[cfg(kvm)] +use crate::hypervisor::regs::{MSR_KERNEL_GS_BASE, MSR_TSC}; +#[cfg(any(not(gdb), mshv3, target_os = "windows"))] use crate::hypervisor::virtual_machine::VirtualMachine; #[cfg(kvm)] use crate::hypervisor::virtual_machine::kvm::KvmVm; @@ -69,6 +72,26 @@ use crate::sandbox::trace::MemTraceInfo; #[cfg(crashdump)] use crate::sandbox::uninitialized::SandboxRuntimeConfig; +#[cfg(gdb)] +type BoxedVm = Box; +#[cfg(not(gdb))] +type BoxedVm = Box; + +/// Determines the MTRR reset indices for an MSHV or WHP backend and validates +/// its allow list. Both hosts lack an MSR filter, so the allow list adds reset +/// state. +#[cfg(any(mshv3, target_os = "windows"))] +fn determine_reset_msrs( + vm: &dyn VirtualMachine, + allowed: &[u32], +) -> std::result::Result, VmError> { + use crate::hypervisor::virtual_machine::{mtrr_reset_indices, validate_allowed_msrs}; + + let required_mtrrs = mtrr_reset_indices(vm).map_err(VmError::CreateVm)?; + validate_allowed_msrs(vm, allowed).map_err(VmError::CreateVm)?; + Ok(required_mtrrs) +} + impl HyperlightVm { /// Create a new HyperlightVm instance (will not run vm until calling `initialise`) #[instrument(err(Debug), skip_all, parent = Span::current(), level = "Trace")] @@ -85,18 +108,27 @@ impl HyperlightVm { #[cfg(crashdump)] rt_cfg: SandboxRuntimeConfig, #[cfg(feature = "mem_profile")] trace_info: MemTraceInfo, ) -> std::result::Result { - #[cfg(gdb)] - type VmType = Box; - #[cfg(not(gdb))] - type VmType = Box; - - let vm: VmType = match get_available_hypervisor() { + let (vm, required_mtrrs): (BoxedVm, Vec) = match get_available_hypervisor() { #[cfg(kvm)] - Some(HypervisorType::Kvm) => Box::new(KvmVm::new().map_err(VmError::CreateVm)?), + Some(HypervisorType::Kvm) => { + let kvm_vm = KvmVm::new().map_err(VmError::CreateVm)?; + kvm_vm + .configure_msr_access(config.get_allowed_msrs()) + .map_err(VmError::CreateVm)?; + (Box::new(kvm_vm), Vec::new()) + } #[cfg(mshv3)] - Some(HypervisorType::Mshv) => Box::new(MshvVm::new().map_err(VmError::CreateVm)?), + Some(HypervisorType::Mshv) => { + let vm: BoxedVm = Box::new(MshvVm::new().map_err(VmError::CreateVm)?); + let required_mtrrs = determine_reset_msrs(vm.as_ref(), config.get_allowed_msrs())?; + (vm, required_mtrrs) + } #[cfg(target_os = "windows")] - Some(HypervisorType::Whp) => Box::new(WhpVm::new().map_err(VmError::CreateVm)?), + Some(HypervisorType::Whp) => { + let vm: BoxedVm = Box::new(WhpVm::new().map_err(VmError::CreateVm)?); + let required_mtrrs = determine_reset_msrs(vm.as_ref(), config.get_allowed_msrs())?; + (vm, required_mtrrs) + } None => return Err(CreateHyperlightVmError::NoHypervisorFound), }; @@ -136,6 +168,10 @@ impl HyperlightVm { }), }); + // The MSRs reset on restore and captured in snapshots. + let msr_reset = + Self::capture_msr_reset_state(&vm, required_mtrrs, config.get_allowed_msrs())?; + let snapshot_slot = 0u32; let scratch_slot = 1u32; #[cfg_attr(not(gdb), allow(unused_mut))] @@ -168,6 +204,7 @@ impl HyperlightVm { trace_info, #[cfg(crashdump)] rt_cfg, + msr_reset, }; ret.update_snapshot_mapping(snapshot_mem)?; @@ -199,6 +236,50 @@ impl HyperlightVm { Ok(ret) } + /// Determines this VM's MSR reset set: the MSRs reset on restore and + /// captured in snapshots. `required_mtrrs` is the VCNT-driven MTRR set + /// gathered at creation. `allowed` is the validated allow list. + fn capture_msr_reset_state( + vm: &BoxedVm, + required_mtrrs: Vec, + allowed: &[u32], + ) -> std::result::Result { + let core: Vec = match get_available_hypervisor() { + #[cfg(kvm)] + Some(HypervisorType::Kvm) => vec![MSR_KERNEL_GS_BASE, MSR_TSC], // GS_BASE is needed for correctness. TSC is to match mshv/whp behavior + #[cfg(mshv3)] + Some(HypervisorType::Mshv) => Self::probe_core_reset_indices(vm), + #[cfg(target_os = "windows")] + Some(HypervisorType::Whp) => Self::probe_core_reset_indices(vm), + // The VM already exists, so a hypervisor was found. The `None` + // arm in `new` returned before this point. + None => unreachable!(), + }; + let mut indices: Vec = core + .into_iter() + .chain(required_mtrrs) + .chain(allowed.iter().copied()) + .collect(); + indices.sort_unstable(); + indices.dedup(); + let baseline = vm.msrs(&indices).map_err(VmError::Register)?; + let mut allowed = allowed.to_vec(); + allowed.sort_unstable(); + allowed.dedup(); + Ok(MsrResetState::new(baseline, allowed)) + } + + /// Core stateful MSRs a filterless host can read. + /// + /// MTRRs come from the VCNT-driven required set, so they are excluded. + #[cfg(any(mshv3, target_os = "windows"))] + fn probe_core_reset_indices(vm: &BoxedVm) -> Vec { + core_reset_indices() + .filter(|i| !is_mtrr_reset_index(*i)) + .filter(|i| vm.msrs(&[*i]).is_ok()) + .collect() + } + /// Initialise the internally stored vCPU with the given PEB address and /// random number seed, then run it until a HLT instruction. #[instrument(err(Debug), skip_all, parent = Span::current(), level = "Trace")] @@ -270,6 +351,54 @@ impl HyperlightVm { Ok(self.vm.sregs()?) } + /// Returns the current values of the MSR reset set. + pub(crate) fn get_msr_reset_state(&self) -> Result, AccessPageTableError> { + Ok(self.vm.msrs(&self.msr_reset.indices())?) + } + + /// Returns this VM's requested allow list, sorted and deduplicated. + pub(crate) fn get_msr_allow_list(&self) -> Vec { + self.msr_reset.allowed().to_vec() + } + + /// Restores snapshot MSRs or the initialization baseline. + pub(crate) fn restore_msrs( + &mut self, + snap_msrs: Option<&Vec>, + snap_allowed: Option<&[u32]>, + ) -> std::result::Result<(), ResetVcpuError> { + match snap_msrs { + // No captured MSRs. Use this VM's baseline. + None => self.vm.set_msrs(self.msr_reset.baseline())?, + // Reset the MSRs to the snapshot's captured values. The snapshot's + // allow list must be a subset of this VM's, and every captured + // index must be in this reset set, so validation rejects a + // mismatch first. + Some(msrs) => { + let entries = self + .msr_reset + .validate_snapshot(msrs, snap_allowed.unwrap_or(&[]))?; + self.vm.set_msrs(&entries)?; + } + } + Ok(()) + } + + /// Reads arbitrary backend MSRs for tests. + #[cfg(all(test, mshv3, target_arch = "x86_64"))] + pub(crate) fn capture_msrs_for_test( + &self, + indices: &[u32], + ) -> std::result::Result, RegisterError> { + self.vm.msrs(indices) + } + + /// Attempts one backend MSR write for tests. + #[cfg(all(test, mshv3, target_arch = "x86_64"))] + pub(crate) fn try_set_msr_for_test(&self, index: u32, value: u64) -> bool { + self.vm.set_msrs(&[MsrEntry { index, value }]).is_ok() + } + /// Dispatch a call from the host to the guest using the given pointer /// to the dispatch function _in the guest's address space_. /// diff --git a/src/hyperlight_host/src/hypervisor/regs/x86_64/mod.rs b/src/hyperlight_host/src/hypervisor/regs/x86_64/mod.rs index 88724d95a..12d65f056 100644 --- a/src/hyperlight_host/src/hypervisor/regs/x86_64/mod.rs +++ b/src/hyperlight_host/src/hypervisor/regs/x86_64/mod.rs @@ -16,11 +16,13 @@ limitations under the License. mod debug_regs; mod fpu; +mod msrs; mod special_regs; mod standard_regs; pub(crate) use debug_regs::*; pub(crate) use fpu::*; +pub(crate) use msrs::*; pub(crate) use special_regs::*; pub(crate) use standard_regs::*; diff --git a/src/hyperlight_host/src/hypervisor/regs/x86_64/msrs.rs b/src/hyperlight_host/src/hypervisor/regs/x86_64/msrs.rs new file mode 100644 index 000000000..86a688182 --- /dev/null +++ b/src/hyperlight_host/src/hypervisor/regs/x86_64/msrs.rs @@ -0,0 +1,483 @@ +/* +Copyright 2025 The Hyperlight Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +//! Model-specific register (MSR) state restored with a snapshot. +//! +//! The reset set contains every MSR whose guest-written state can persist. +//! Allowing guest access to an MSR is a separate concern. Allowed MSRs still +//! reset. + +use serde::{Deserialize, Serialize}; + +/// A single MSR captured for reset. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct MsrEntry { + /// The index passed to `RDMSR` and `WRMSR`. + pub index: u32, + /// The captured value. + pub value: u64, +} + +/// The MSR reset set and initialization baseline for one VM. +#[derive(Debug, Clone)] +pub(crate) struct MsrResetState { + /// Creation-time value for every MSR this VM resets on restore and + /// captures in a snapshot: core indices, required MTRRs, and the allow + /// list. Sorted by index and deduplicated. + baseline: Vec, + /// The user-supplied allow list (`SandboxConfiguration::allow_msrs`), + /// sorted and deduplicated. A subset of the `baseline` indices, kept as a + /// separate copy so restore can compare it, as a superset, against a + /// snapshot's allow list. + allowed: Vec, +} + +impl MsrResetState { + /// Builds the reset state from the creation-time baseline entries and the + /// requested allow list. + pub fn new(baseline: Vec, allowed: Vec) -> Self { + Self { baseline, allowed } + } + + /// The creation-time baseline entries. + pub fn baseline(&self) -> &[MsrEntry] { + &self.baseline + } + + /// This VM's requested allow list. + pub fn allowed(&self) -> &[u32] { + &self.allowed + } + + /// The MSR indices this VM resets, for host reads. + pub fn indices(&self) -> Vec { + self.baseline.iter().map(|entry| entry.index).collect() + } + + /// Resolves an untrusted snapshot's MSRs against this VM's reset set. + /// + /// The snapshot's allow list must be a subset of this VM's, so a + /// destination that allows at least as much accepts the snapshot on every + /// backend. Each supplied index must belong to this reset set. Returns the + /// entries to write: the snapshot's value for each reset index, or the + /// baseline where the snapshot omits it. + pub fn validate_snapshot( + &self, + snapshot: &[MsrEntry], + snap_allowed: &[u32], + ) -> Result, crate::hypervisor::virtual_machine::RegisterError> { + use crate::hypervisor::virtual_machine::RegisterError; + + let missing: Vec = snap_allowed + .iter() + .copied() + .filter(|index| !self.allowed.contains(index)) + .collect(); + if !missing.is_empty() { + return Err(RegisterError::SnapshotMsrNotAllowed { missing }); + } + + for entry in snapshot { + if !self.baseline.iter().any(|base| base.index == entry.index) { + return Err(RegisterError::InvalidSnapshotMsrIndex { index: entry.index }); + } + } + + Ok(self + .baseline + .iter() + .map(|base| MsrEntry { + index: base.index, + value: snapshot + .iter() + .find(|entry| entry.index == base.index) + .map_or(base.value, |entry| entry.value), + }) + .collect()) + } +} + +pub(crate) const MSR_TSC: u32 = 0x10; +pub(crate) const MSR_TSC_ADJUST: u32 = 0x3B; +pub(crate) const MSR_SPEC_CTRL: u32 = 0x48; +pub(crate) const MSR_UMWAIT_CONTROL: u32 = 0xE1; +pub(crate) const MSR_MPERF: u32 = 0xE7; +pub(crate) const MSR_APERF: u32 = 0xE8; +pub(crate) const MSR_MTRR_CAP: u32 = 0xFE; +pub(crate) const MSR_TSX_CTRL: u32 = 0x122; +pub(crate) const MSR_SYSENTER_CS: u32 = 0x174; +pub(crate) const MSR_SYSENTER_ESP: u32 = 0x175; +pub(crate) const MSR_SYSENTER_EIP: u32 = 0x176; +pub(crate) const MSR_XFD: u32 = 0x1C4; +pub(crate) const MSR_XFD_ERR: u32 = 0x1C5; +pub(crate) const MSR_DEBUGCTL: u32 = 0x1D9; +pub(crate) const MSR_MTRR_FIX64K_00000: u32 = 0x250; +pub(crate) const MSR_PAT: u32 = 0x277; +pub(crate) const MSR_MTRR_DEF_TYPE: u32 = 0x2FF; +pub(crate) const MSR_U_CET: u32 = 0x6A0; +pub(crate) const MSR_S_CET: u32 = 0x6A2; +pub(crate) const MSR_PL0_SSP: u32 = 0x6A4; +pub(crate) const MSR_PL1_SSP: u32 = 0x6A5; +pub(crate) const MSR_PL2_SSP: u32 = 0x6A6; +pub(crate) const MSR_PL3_SSP: u32 = 0x6A7; +pub(crate) const MSR_INTERRUPT_SSP_TABLE_ADDR: u32 = 0x6A8; +pub(crate) const MSR_TSC_DEADLINE: u32 = 0x6E0; +pub(crate) const MSR_BNDCFGS: u32 = 0xD90; +pub(crate) const MSR_XSS: u32 = 0xDA0; +pub(crate) const MSR_STAR: u32 = 0xC000_0081; +pub(crate) const MSR_LSTAR: u32 = 0xC000_0082; +pub(crate) const MSR_CSTAR: u32 = 0xC000_0083; +pub(crate) const MSR_SFMASK: u32 = 0xC000_0084; +pub(crate) const MSR_KERNEL_GS_BASE: u32 = 0xC000_0102; +pub(crate) const MSR_TSC_AUX: u32 = 0xC000_0103; +pub(crate) const MSR_VIRT_SPEC_CTRL: u32 = 0xC001_011F; + +const HYPERV_VARIABLE_MTRR_COUNT: u8 = 16; + +// Every guest-writable retained value must be host-readable and host-writable. +// EFER, APIC_BASE, FS_BASE, and GS_BASE are part of the special-register state. +// PRED_CMD (0x49) and FLUSH_CMD (0x10B) are intentionally absent: they are +// write-only commands with no retained state, so they cannot be reset and +// therefore cannot be allowed. +const MSR_TABLE: &[u32] = &[ + // Guest and host access use the matching SYSENTER state. + MSR_SYSENTER_CS, + MSR_SYSENTER_ESP, + MSR_SYSENTER_EIP, + // WHP exposes no writable DEBUGCTL bits under Germanium compatibility. + MSR_DEBUGCTL, + // Guest and host access use the same PAT state. + MSR_PAT, + // Guest and host access use the matching syscall state. + MSR_STAR, + MSR_LSTAR, + MSR_CSTAR, + MSR_SFMASK, + // SWAPGS and host access use the same KERNEL_GS_BASE state. + MSR_KERNEL_GS_BASE, + // Guest and host access use the same SPEC_CTRL state. + MSR_SPEC_CTRL, + // AMD virtualized SSBD control. Guest-writable only where the host exposes + // the legacy VIRT_SPEC_CTRL SSBD mechanism. Host probing omits it otherwise. + MSR_VIRT_SPEC_CTRL, + // Host probing omits CET state when CET is unavailable. + MSR_U_CET, + MSR_S_CET, + // Host probing omits shadow-stack state when it is unavailable. + MSR_PL0_SSP, + MSR_PL1_SSP, + MSR_PL2_SSP, + MSR_PL3_SSP, + MSR_INTERRUPT_SSP_TABLE_ADDR, + // MSHV maps XSS through its U_XSS alias. + MSR_XSS, + // Guest and host access use the same virtual counter. + MSR_TSC, + // Hyper-V stores TSC_ADJUST independently from TSC. + MSR_TSC_ADJUST, + // Host probing omits TSC_AUX when RDTSCP is unavailable. + MSR_TSC_AUX, + // Hyper-V exposes MPERF and APERF as per-VP counters. + MSR_MPERF, + MSR_APERF, + // Host probing omits TSX_CTRL when TSX control is unavailable. + MSR_TSX_CTRL, + // Host probing omits XFD state when XFD is unavailable. + MSR_XFD, + MSR_XFD_ERR, + // Feature MSRs enabled by default on capable hosts. Host probing omits + // each when its feature is unavailable. + MSR_UMWAIT_CONTROL, + MSR_TSC_DEADLINE, + MSR_BNDCFGS, + // Hyper-V accepts fixed-MTRR writes even when MTRRCAP.FIX is clear. + MSR_MTRR_DEF_TYPE, + 0x200, // PHYSBASE0 + 0x201, // PHYSMASK0 + 0x202, // PHYSBASE1 + 0x203, // PHYSMASK1 + 0x204, // PHYSBASE2 + 0x205, // PHYSMASK2 + 0x206, // PHYSBASE3 + 0x207, // PHYSMASK3 + 0x208, // PHYSBASE4 + 0x209, // PHYSMASK4 + 0x20A, // PHYSBASE5 + 0x20B, // PHYSMASK5 + 0x20C, // PHYSBASE6 + 0x20D, // PHYSMASK6 + 0x20E, // PHYSBASE7 + 0x20F, // PHYSMASK7 + 0x210, // PHYSBASE8 + 0x211, // PHYSMASK8 + 0x212, // PHYSBASE9 + 0x213, // PHYSMASK9 + 0x214, // PHYSBASEA + 0x215, // PHYSMASKA + 0x216, // PHYSBASEB + 0x217, // PHYSMASKB + 0x218, // PHYSBASEC + 0x219, // PHYSMASKC + 0x21A, // PHYSBASED + 0x21B, // PHYSMASKD + 0x21C, // PHYSBASEE + 0x21D, // PHYSMASKE + 0x21E, // PHYSBASEF + 0x21F, // PHYSMASKF + MSR_MTRR_FIX64K_00000, // FIX64K_00000 + 0x258, // FIX16K_80000 + 0x259, // FIX16K_A0000 + 0x268, // FIX4K_C0000 + 0x269, // FIX4K_C8000 + 0x26A, // FIX4K_D0000 + 0x26B, // FIX4K_D8000 + 0x26C, // FIX4K_E0000 + 0x26D, // FIX4K_E8000 + 0x26E, // FIX4K_F0000 + 0x26F, // FIX4K_F8000 +]; + +/// Whether an MSR carries retained state eligible for the reset set. +pub(crate) fn is_resettable_msr(index: u32) -> bool { + MSR_TABLE.contains(&index) +} + +/// Returns core stateful indices for host filtering. +pub(crate) fn core_reset_indices() -> impl Iterator { + MSR_TABLE.iter().copied() +} + +pub(crate) fn hyperv_mtrr_reset_indices( + mtrr_cap: u64, +) -> Result, crate::hypervisor::virtual_machine::CreateVmError> { + use crate::hypervisor::virtual_machine::CreateVmError; + + let advertised = (mtrr_cap & 0xff) as u8; + if advertised > HYPERV_VARIABLE_MTRR_COUNT { + return Err(CreateVmError::UnexpectedVariableMtrrCount { + advertised, + maximum: HYPERV_VARIABLE_MTRR_COUNT, + }); + } + + let mut indices = Vec::with_capacity(1 + usize::from(advertised) * 2 + 11); + indices.push(MSR_MTRR_DEF_TYPE); + indices.extend((0..u32::from(advertised) * 2).map(|offset| 0x200 + offset)); + indices.extend([ + MSR_MTRR_FIX64K_00000, + 0x258, + 0x259, + 0x268, + 0x269, + 0x26A, + 0x26B, + 0x26C, + 0x26D, + 0x26E, + 0x26F, + ]); + Ok(indices) +} + +pub(crate) fn is_mtrr_reset_index(index: u32) -> bool { + index == MSR_MTRR_DEF_TYPE + || (0x200..=0x21F).contains(&index) + || matches!( + index, + MSR_MTRR_FIX64K_00000 + | 0x258 + | 0x259 + | 0x268 + | 0x269 + | 0x26A + | 0x26B + | 0x26C + | 0x26D + | 0x26E + | 0x26F + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::hypervisor::virtual_machine::{CreateVmError, RegisterError}; + + fn state() -> MsrResetState { + MsrResetState { + baseline: vec![ + MsrEntry { + index: MSR_SYSENTER_CS, + value: 0x1C, + }, + MsrEntry { + index: MSR_SYSENTER_ESP, + value: 0x2C, + }, + MsrEntry { + index: MSR_KERNEL_GS_BASE, + value: 0x3C, + }, + ], + allowed: vec![MSR_SYSENTER_CS, MSR_SYSENTER_ESP], + } + } + + #[test] + fn hyperv_mtrr_indices_follow_guest_capability() { + assert_eq!( + hyperv_mtrr_reset_indices(2).unwrap(), + [ + MSR_MTRR_DEF_TYPE, + 0x200, + 0x201, + 0x202, + 0x203, + MSR_MTRR_FIX64K_00000, + 0x258, + 0x259, + 0x268, + 0x269, + 0x26A, + 0x26B, + 0x26C, + 0x26D, + 0x26E, + 0x26F, + ] + ); + let fixed_bit_does_not_change_reset_set = hyperv_mtrr_reset_indices(2 | (1 << 8)).unwrap(); + assert_eq!( + fixed_bit_does_not_change_reset_set, + hyperv_mtrr_reset_indices(2).unwrap() + ); + } + + #[test] + fn hyperv_mtrr_count_rejects_more_than_sixteen_pairs() { + let indices = hyperv_mtrr_reset_indices(16).unwrap(); + assert!(indices.contains(&0x21F)); + assert_eq!(indices.last(), Some(&0x26F)); + assert!(indices.iter().all(|&index| is_resettable_msr(index))); + assert!(matches!( + hyperv_mtrr_reset_indices(17), + Err(CreateVmError::UnexpectedVariableMtrrCount { + advertised: 17, + maximum: 16 + }) + )); + } + + #[test] + fn snapshot_msr_validation_accepts_exact_canonical_set() { + let supplied = vec![ + MsrEntry { + index: MSR_SYSENTER_CS, + value: 1, + }, + MsrEntry { + index: MSR_SYSENTER_ESP, + value: 2, + }, + MsrEntry { + index: MSR_KERNEL_GS_BASE, + value: 3, + }, + ]; + + assert_eq!( + state() + .validate_snapshot(&supplied, &[MSR_SYSENTER_CS]) + .unwrap(), + supplied + ); + } + + #[test] + fn snapshot_msr_validation_baselines_omitted_indices() { + // A snapshot covering a subset of this VM's reset set is accepted. + // The omitted index takes the creation-time baseline value. + let supplied = vec![ + MsrEntry { + index: MSR_SYSENTER_CS, + value: 1, + }, + MsrEntry { + index: MSR_KERNEL_GS_BASE, + value: 3, + }, + ]; + + assert_eq!( + state().validate_snapshot(&supplied, &[]).unwrap(), + vec![ + MsrEntry { + index: MSR_SYSENTER_CS, + value: 1 + }, + MsrEntry { + index: MSR_SYSENTER_ESP, + value: 0x2C + }, + MsrEntry { + index: MSR_KERNEL_GS_BASE, + value: 3 + }, + ] + ); + } + + #[test] + fn snapshot_msr_validation_rejects_non_superset_allow_list() { + // The snapshot allows an MSR the destination does not. + assert!(matches!( + state().validate_snapshot(&[], &[MSR_KERNEL_GS_BASE]), + Err(RegisterError::SnapshotMsrNotAllowed { missing }) if missing == vec![MSR_KERNEL_GS_BASE] + )); + } + + #[test] + fn snapshot_msr_validation_rejects_index_outside_reset_set() { + let supplied = vec![ + MsrEntry { + index: MSR_SYSENTER_CS, + value: 1, + }, + MsrEntry { + index: MSR_SYSENTER_ESP, + value: 2, + }, + MsrEntry { + index: MSR_PAT, + value: 3, + }, + ]; + + assert!(matches!( + state().validate_snapshot(&supplied, &[]), + Err(RegisterError::InvalidSnapshotMsrIndex { index }) if index == MSR_PAT + )); + } + + #[test] + fn snapshot_msr_validation_accepts_empty_canonical_set() { + let state = MsrResetState { + baseline: Vec::new(), + allowed: Vec::new(), + }; + assert!(state.validate_snapshot(&[], &[]).unwrap().is_empty()); + } +} diff --git a/src/hyperlight_host/src/hypervisor/regs/x86_64/special_regs.rs b/src/hyperlight_host/src/hypervisor/regs/x86_64/special_regs.rs index e67ab3b49..adecfb24a 100644 --- a/src/hyperlight_host/src/hypervisor/regs/x86_64/special_regs.rs +++ b/src/hyperlight_host/src/hypervisor/regs/x86_64/special_regs.rs @@ -677,6 +677,16 @@ mod tests { assert_eq!(original, roundtrip); } + /// The guest boots in xAPIC mode. `APIC_BASE` bit 10 (`EXTD`) enables + /// x2APIC, which would make the `0x800`-`0x8FF` MSR range legal. Keeping it + /// clear is why a guest write to an x2APIC MSR faults. + #[test] + fn standard_defaults_leave_x2apic_disabled() { + const X2APIC_ENABLE: u64 = 1 << 10; + let sregs = CommonSpecialRegisters::standard_64bit_defaults(0x1000); + assert_eq!(sregs.apic_base & X2APIC_ENABLE, 0); + } + #[cfg(mshv3)] #[test] fn round_trip_mshv_sregs() { diff --git a/src/hyperlight_host/src/hypervisor/virtual_machine/kvm/x86_64.rs b/src/hyperlight_host/src/hypervisor/virtual_machine/kvm/x86_64.rs index 3dc8ec87a..87f9f111e 100644 --- a/src/hyperlight_host/src/hypervisor/virtual_machine/kvm/x86_64.rs +++ b/src/hyperlight_host/src/hypervisor/virtual_machine/kvm/x86_64.rs @@ -14,16 +14,21 @@ See the License for the specific language governing permissions and limitations under the License. */ +use std::collections::HashSet; use std::sync::LazyLock; use hyperlight_common::outb::VmAction; #[cfg(gdb)] use kvm_bindings::kvm_guest_debug; use kvm_bindings::{ - kvm_debugregs, kvm_fpu, kvm_regs, kvm_sregs, kvm_userspace_memory_region, kvm_xsave, + Msrs, kvm_debugregs, kvm_enable_cap, kvm_fpu, kvm_msr_entry, kvm_regs, kvm_sregs, + kvm_userspace_memory_region, kvm_xsave, }; use kvm_ioctls::Cap::UserMemory; -use kvm_ioctls::{Kvm, VcpuExit, VcpuFd, VmFd}; +use kvm_ioctls::{ + Cap, Kvm, MsrExitReason, MsrFilterDefaultAction, MsrFilterRange, MsrFilterRangeFlags, VcpuExit, + VcpuFd, VmFd, +}; use tracing::{Span, instrument}; #[cfg(feature = "trace_guest")] use tracing_opentelemetry::OpenTelemetrySpanExt; @@ -34,7 +39,7 @@ use vmm_sys_util::eventfd::EventFd; use crate::hypervisor::gdb::{DebugError, DebuggableVm}; use crate::hypervisor::regs::{ CommonDebugRegs, CommonFpu, CommonRegisters, CommonSpecialRegisters, FP_CONTROL_WORD_DEFAULT, - MXCSR_DEFAULT, + MXCSR_DEFAULT, MsrEntry, is_resettable_msr, }; #[cfg(test)] use crate::hypervisor::virtual_machine::XSAVE_BUFFER_SIZE; @@ -110,6 +115,42 @@ pub(crate) struct KvmVm { static KVM: LazyLock> = LazyLock::new(|| Kvm::new().map_err(|e| CreateVmError::HypervisorNotAvailable(e.into()))); +/// Cached host indices reported by `KVM_GET_MSR_INDEX_LIST`. +/// An empty set makes support checks fail closed. +static HOST_MSR_INDICES: LazyLock> = LazyLock::new(|| match KVM.as_ref() { + Ok(kvm) => match kvm.get_msr_index_list() { + Ok(list) => list.as_slice().iter().copied().collect(), + Err(e) => { + tracing::warn!("KVM_GET_MSR_INDEX_LIST failed: {e}"); + HashSet::new() + } + }, + Err(_) => HashSet::new(), +}); + +/// Returns the set of MSR indices the host KVM supports for get/set. +pub(crate) fn host_msr_indices() -> &'static HashSet { + &HOST_MSR_INDICES +} + +/// KVM allows at most this many MSR filter ranges. +const KVM_MSR_FILTER_MAX_RANGES: usize = 16; + +/// Returns the smallest contiguous ranges covering the supplied indices. +fn coalesce_msr_ranges(indices: &[u32]) -> Vec<(u32, usize)> { + let mut sorted: Vec = indices.to_vec(); + sorted.sort_unstable(); + sorted.dedup(); + let mut groups: Vec<(u32, usize)> = Vec::new(); + for idx in sorted { + match groups.last_mut() { + Some((base, count)) if *base + *count as u32 == idx => *count += 1, + _ => groups.push((idx, 1)), + } + } + groups +} + #[cfg(feature = "hw-interrupts")] impl KvmVm { /// Create the in-kernel IRQ chip and register an irqfd for GSI 0. @@ -228,6 +269,20 @@ impl KvmVm { } Ok(VcpuExit::MmioRead(addr, _)) => return Ok(VmExit::MmioRead(addr)), Ok(VcpuExit::MmioWrite(addr, _)) => return Ok(VmExit::MmioWrite(addr)), + // Complete filtered access through the default run path. + Ok(VcpuExit::X86Rdmsr(msr_exit)) => { + let msr_index = msr_exit.index; + *msr_exit.error = 1; + self.complete_filtered_msr_exit()?; + return Ok(VmExit::MsrRead(msr_index)); + } + Ok(VcpuExit::X86Wrmsr(msr_exit)) => { + let msr_index = msr_exit.index; + let value = msr_exit.data; + *msr_exit.error = 1; + self.complete_filtered_msr_exit()?; + return Ok(VmExit::MsrWrite { msr_index, value }); + } #[cfg(gdb)] Ok(VcpuExit::Debug(debug_exit)) => { return Ok(VmExit::Debug { @@ -250,6 +305,23 @@ impl KvmVm { } } + /// Finish a filtered MSR access after its `error` flag was set. + /// Reentering `KVM_RUN` injects `#GP` for the pending access. + /// immediate_exit halts the guest right after, so it never runs past + /// the fault. That reentry returns `EINTR`, which is expected here. + fn complete_filtered_msr_exit(&mut self) -> std::result::Result<(), RunVcpuError> { + self.vcpu_fd.set_kvm_immediate_exit(1); + // `.err()` drops the `Ok(VcpuExit)`, which borrows `vcpu_fd`, keeping + // only the owned error so the borrow ends before the next call. + let err = self.vcpu_fd.run().err(); + self.vcpu_fd.set_kvm_immediate_exit(0); + match err { + None => Ok(()), + Some(e) if e.errno() == libc::EINTR => Ok(()), + Some(e) => Err(RunVcpuError::Unknown(e.into())), + } + } + #[cfg(feature = "hw-interrupts")] fn handle_pv_timer_config(&mut self, data: &[u8]) { use super::super::x86_64::hw_interrupts::handle_pv_timer_config; @@ -275,6 +347,21 @@ impl KvmVm { Ok(VcpuExit::IoOut(port, data)) => Ok(VmExit::IoOut(port, data.to_vec())), Ok(VcpuExit::MmioRead(addr, _)) => Ok(VmExit::MmioRead(addr)), Ok(VcpuExit::MmioWrite(addr, _)) => Ok(VmExit::MmioWrite(addr)), + // Reentering KVM_RUN completes the failed MSR exit and injects #GP. + // immediate_exit prevents further guest execution. + Ok(VcpuExit::X86Rdmsr(msr_exit)) => { + let msr_index = msr_exit.index; + *msr_exit.error = 1; + self.complete_filtered_msr_exit()?; + Ok(VmExit::MsrRead(msr_index)) + } + Ok(VcpuExit::X86Wrmsr(msr_exit)) => { + let msr_index = msr_exit.index; + let value = msr_exit.data; + *msr_exit.error = 1; + self.complete_filtered_msr_exit()?; + Ok(VmExit::MsrWrite { msr_index, value }) + } #[cfg(gdb)] Ok(VcpuExit::Debug(debug_exit)) => Ok(VmExit::Debug { dr6: debug_exit.dr6, @@ -292,6 +379,150 @@ impl KvmVm { ))), } } + + /// Installs a deny filter containing the validated allow list. + /// Requires `KVM_CAP_X86_USER_SPACE_MSR` and `KVM_CAP_X86_MSR_FILTER`. + pub(crate) fn configure_msr_access( + &self, + allowed: &[u32], + ) -> std::result::Result<(), CreateVmError> { + let hv = KVM.as_ref().map_err(|e| e.clone())?; + if !hv.check_extension(Cap::X86UserSpaceMsr) || !hv.check_extension(Cap::X86MsrFilter) { + tracing::error!( + "KVM does not support KVM_CAP_X86_USER_SPACE_MSR or KVM_CAP_X86_MSR_FILTER." + ); + return Err(CreateVmError::MsrFilterNotSupported); + } + + // Every permitted guest write must have restorable host state. + for &msr in allowed { + self.validate_allowed_msr(msr)?; + } + + // Tell KVM to exit to userspace on filtered MSR access. + let cap = kvm_enable_cap { + cap: Cap::X86UserSpaceMsr as u32, + args: [MsrExitReason::Filter.bits() as u64, 0, 0, 0], + ..Default::default() + }; + self.vm_fd + .enable_cap(&cap) + .map_err(|e| CreateVmError::InitializeVm(e.into()))?; + + // Each contiguous group consumes one KVM filter range. + let groups = coalesce_msr_ranges(allowed); + if groups.len() > KVM_MSR_FILTER_MAX_RANGES { + return Err(CreateVmError::TooManyMsrRanges(groups.len())); + } + + // The bitmaps must live through set_msr_filter. + let bitmaps: Vec> = groups + .iter() + .map(|(_, count)| { + let mut bytes = vec![0u8; count.div_ceil(8)]; + for bit in 0..*count { + bytes[bit / 8] |= 1 << (bit % 8); + } + bytes + }) + .collect(); + + // Default deny requires at least one range. + static DENY_BITMAP: [u8; 1] = [0u8]; + let ranges: Vec = if groups.is_empty() { + vec![MsrFilterRange { + flags: MsrFilterRangeFlags::READ | MsrFilterRangeFlags::WRITE, + base: 0, + msr_count: 1, + bitmap: &DENY_BITMAP, + }] + } else { + groups + .iter() + .zip(bitmaps.iter()) + .map(|((base, count), bitmap)| MsrFilterRange { + flags: MsrFilterRangeFlags::READ | MsrFilterRangeFlags::WRITE, + base: *base, + msr_count: *count as u32, + bitmap: bitmap.as_slice(), + }) + .collect() + }; + + self.vm_fd + .set_msr_filter(MsrFilterDefaultAction::DENY, &ranges) + .map_err(|e| CreateVmError::InitializeVm(e.into()))?; + Ok(()) + } + + /// Validates that an allowed MSR has restorable host state. + fn validate_allowed_msr(&self, msr: u32) -> std::result::Result<(), CreateVmError> { + if !is_resettable_msr(msr) { + return Err(CreateVmError::MsrNotAllowable { + msr, + reason: "MSR is not a resettable MSR".to_string(), + }); + } + if !host_msr_indices().contains(&msr) { + return Err(CreateVmError::MsrNotAllowable { + msr, + reason: "MSR is not supported by the host".to_string(), + }); + } + let value = self + .read_msr(msr) + .map_err(|e| CreateVmError::MsrNotAllowable { + msr, + reason: format!("MSR is not readable: {e}"), + })?; + self.write_msr(msr, value) + .map_err(|e| CreateVmError::MsrNotAllowable { + msr, + reason: format!("MSR is not resettable: {e}"), + })?; + Ok(()) + } + + /// Reads one vCPU MSR without the guest filter. + fn read_msr(&self, index: u32) -> std::result::Result { + let mut msrs = Msrs::from_entries(&[kvm_msr_entry { + index, + ..Default::default() + }]) + .map_err(|e| RegisterError::MsrBuild(format!("{e:?}")))?; + let n = self + .vcpu_fd + .get_msrs(&mut msrs) + .map_err(|e| RegisterError::GetMsrs(e.into()))?; + if n != 1 { + return Err(RegisterError::MsrShortCount { + expected: 1, + actual: n, + }); + } + Ok(msrs.as_slice()[0].data) + } + + /// Writes one vCPU MSR without the guest filter. + fn write_msr(&self, index: u32, data: u64) -> std::result::Result<(), RegisterError> { + let msrs = Msrs::from_entries(&[kvm_msr_entry { + index, + data, + ..Default::default() + }]) + .map_err(|e| RegisterError::MsrBuild(format!("{e:?}")))?; + let n = self + .vcpu_fd + .set_msrs(&msrs) + .map_err(|e| RegisterError::SetMsrs(e.into()))?; + if n != 1 { + return Err(RegisterError::MsrShortCount { + expected: 1, + actual: n, + }); + } + Ok(()) + } } impl VirtualMachine for KvmVm { @@ -403,6 +634,66 @@ impl VirtualMachine for KvmVm { Ok(()) } + fn msrs(&self, indices: &[u32]) -> std::result::Result, RegisterError> { + if indices.is_empty() { + return Ok(Vec::new()); + } + let entries: Vec = indices + .iter() + .map(|&index| kvm_msr_entry { + index, + ..Default::default() + }) + .collect(); + let mut msrs = + Msrs::from_entries(&entries).map_err(|e| RegisterError::MsrBuild(format!("{e:?}")))?; + let n = self + .vcpu_fd + .get_msrs(&mut msrs) + .map_err(|e| RegisterError::GetMsrs(e.into()))?; + if n != indices.len() { + return Err(RegisterError::MsrShortCount { + expected: indices.len(), + actual: n, + }); + } + Ok(msrs + .as_slice() + .iter() + .map(|e| MsrEntry { + index: e.index, + value: e.data, + }) + .collect()) + } + + fn set_msrs(&self, msrs: &[MsrEntry]) -> std::result::Result<(), RegisterError> { + let entries: Vec = msrs + .iter() + .map(|e| kvm_msr_entry { + index: e.index, + data: e.value, + ..Default::default() + }) + .collect(); + if entries.is_empty() { + return Ok(()); + } + let kvm_msrs = + Msrs::from_entries(&entries).map_err(|e| RegisterError::MsrBuild(format!("{e:?}")))?; + let n = self + .vcpu_fd + .set_msrs(&kvm_msrs) + .map_err(|e| RegisterError::SetMsrs(e.into()))?; + if n != entries.len() { + return Err(RegisterError::MsrShortCount { + expected: entries.len(), + actual: n, + }); + } + Ok(()) + } + #[allow(dead_code)] fn xsave(&self) -> std::result::Result, RegisterError> { let xsave = self @@ -570,10 +861,45 @@ impl DebuggableVm for KvmVm { } #[cfg(test)] -#[cfg(feature = "hw-interrupts")] -mod hw_interrupt_tests { +mod tests { use super::*; + #[test] + fn coalesces_unsorted_contiguous_indices() { + assert_eq!( + coalesce_msr_ranges(&[0x176, 0x174, 0x175]), + vec![(0x174, 3)] + ); + } + + #[test] + fn deduplicates_indices() { + assert_eq!( + coalesce_msr_ranges(&[0x174, 0x174, 0x176]), + vec![(0x174, 1), (0x176, 1)] + ); + } + + #[test] + fn preserves_sixteen_range_boundary() { + let indices: Vec = (0..KVM_MSR_FILTER_MAX_RANGES as u32) + .map(|index| index * 2) + .collect(); + let ranges = coalesce_msr_ranges(&indices); + assert_eq!(ranges.len(), KVM_MSR_FILTER_MAX_RANGES); + assert!(ranges.iter().all(|(_, count)| *count == 1)); + } + + #[test] + fn scattered_indices_exceed_sixteen_range_limit() { + let indices: Vec = (0..=KVM_MSR_FILTER_MAX_RANGES as u32) + .map(|index| index * 2) + .collect(); + let ranges = coalesce_msr_ranges(&indices); + assert!(ranges.len() > KVM_MSR_FILTER_MAX_RANGES); + } + + #[cfg(feature = "hw-interrupts")] #[test] fn halt_port_is_not_standard_device() { // VmAction::Halt port must not overlap in-kernel PIC/PIT/speaker ports diff --git a/src/hyperlight_host/src/hypervisor/virtual_machine/mod.rs b/src/hyperlight_host/src/hypervisor/virtual_machine/mod.rs index dac344711..4cd8fca02 100644 --- a/src/hyperlight_host/src/hypervisor/virtual_machine/mod.rs +++ b/src/hyperlight_host/src/hypervisor/virtual_machine/mod.rs @@ -21,9 +21,13 @@ use tracing::{Span, instrument}; #[cfg(gdb)] use crate::hypervisor::gdb::DebugError; +#[cfg(target_arch = "x86_64")] +use crate::hypervisor::regs::MsrEntry; use crate::hypervisor::regs::{ CommonDebugRegs, CommonFpu, CommonRegisters, CommonSpecialRegisters, }; +#[cfg(all(target_arch = "x86_64", any(mshv3, target_os = "windows")))] +use crate::hypervisor::regs::{MSR_MTRR_CAP, hyperv_mtrr_reset_indices, is_resettable_msr}; use crate::mem::memory_region::MemoryRegion; #[cfg(feature = "trace_guest")] use crate::sandbox::trace::TraceContext as SandboxTraceContext; @@ -139,6 +143,12 @@ pub(crate) enum VmExit { MmioRead(u64), /// The vCPU tried to write to the given (unmapped) addr MmioWrite(u64), + /// The vCPU tried to read from the given MSR + #[cfg(all(target_arch = "x86_64", kvm))] + MsrRead(u32), + /// The vCPU tried to write to the given MSR with the given value + #[cfg(all(target_arch = "x86_64", kvm))] + MsrWrite { msr_index: u32, value: u64 }, /// The vCPU execution has been cancelled Cancelled(), /// The vCPU has exited for a reason that is not handled by Hyperlight @@ -183,6 +193,30 @@ pub enum CreateVmError { HypervisorNotAvailable(HypervisorError), #[error("Initialize VM failed: {0}")] InitializeVm(HypervisorError), + #[cfg(all(kvm, target_arch = "x86_64"))] + #[error("KVM MSR filtering requires KVM_CAP_X86_USER_SPACE_MSR and KVM_CAP_X86_MSR_FILTER")] + MsrFilterNotSupported, + #[cfg(target_arch = "x86_64")] + #[error("MSR {msr:#x} cannot be allowed: {reason}")] + MsrNotAllowable { msr: u32, reason: String }, + #[cfg(target_arch = "x86_64")] + #[error("Failed to read IA32_MTRRCAP: {0}")] + GetMtrrCap(RegisterError), + #[cfg(target_arch = "x86_64")] + #[error("Guest-visible MTRRs cannot be reset: {0}")] + RequiredMtrrsNotResettable(RegisterError), + #[cfg(target_arch = "x86_64")] + #[error("Guest exposes {advertised} variable MTRR pairs, expected at most {maximum}")] + UnexpectedVariableMtrrCount { advertised: u8, maximum: u8 }, + #[cfg(all(kvm, target_arch = "x86_64"))] + #[error("Too many allowed MSR filter ranges: {0}. Maximum is 16")] + TooManyMsrRanges(usize), + #[cfg(target_os = "windows")] + #[error("Get Partition Property failed: {0}")] + GetPartitionProperty(HypervisorError), + #[cfg(target_os = "windows")] + #[error("WHP exposes {advertised} processor feature banks, expected {expected}")] + UnexpectedProcessorFeatureBankCount { advertised: u32, expected: u32 }, #[error("Set Partition Property failed: {0}")] SetPartitionProperty(HypervisorError), #[cfg(target_os = "windows")] @@ -241,6 +275,38 @@ pub enum RegisterError { }, #[error("Invalid xsave alignment")] InvalidXsaveAlignment, + #[cfg(target_arch = "x86_64")] + #[error("MSR operation not supported on this hypervisor")] + MsrsUnsupported, + #[cfg(target_arch = "x86_64")] + #[error("Failed to build MSR list: {0}")] + MsrBuild(String), + #[cfg(target_arch = "x86_64")] + #[error("Failed to get MSRs: {0}")] + GetMsrs(HypervisorError), + #[cfg(target_arch = "x86_64")] + #[error("Failed to set MSRs: {0}")] + SetMsrs(HypervisorError), + #[cfg(target_arch = "x86_64")] + #[error("Snapshot allows MSRs the destination does not: {missing:x?}")] + SnapshotMsrNotAllowed { + /// Allowed MSR indices present in the snapshot but not the destination. + missing: Vec, + }, + #[cfg(target_arch = "x86_64")] + #[error("Snapshot MSR index {index:#x} is not in this VM's reset set")] + InvalidSnapshotMsrIndex { + /// Architectural MSR index supplied by the snapshot. + index: u32, + }, + #[cfg(all(kvm, target_arch = "x86_64"))] + #[error("MSR batch short count: expected {expected}, applied {actual}")] + MsrShortCount { + /// Number of MSRs requested + expected: usize, + /// Number of MSRs actually applied before KVM stopped + actual: usize, + }, #[cfg(target_os = "windows")] #[error("Failed to get xsave size: {0}")] GetXsaveSize(#[from] HypervisorError), @@ -359,6 +425,13 @@ pub(crate) trait VirtualMachine: Debug + Send { #[allow(dead_code)] fn set_debug_regs(&self, drs: &CommonDebugRegs) -> std::result::Result<(), RegisterError>; + /// Reads the requested MSRs. + #[cfg(target_arch = "x86_64")] + fn msrs(&self, indices: &[u32]) -> std::result::Result, RegisterError>; + /// Writes the supplied MSRs. + #[cfg(target_arch = "x86_64")] + fn set_msrs(&self, msrs: &[MsrEntry]) -> std::result::Result<(), RegisterError>; + /// Get xsave #[allow(dead_code)] #[cfg(not(target_arch = "aarch64"))] @@ -385,6 +458,51 @@ pub(crate) trait VirtualMachine: Debug + Send { fn partition_handle(&self) -> windows::Win32::System::Hypervisor::WHV_PARTITION_HANDLE; } +/// Validates that each allowed MSR is restorable on a filterless (MSHV/WHP) +/// host: reset replays a captured value, so the host must read and write it. +/// Rejects e.g. a variable MTRR above the host VCNT. +#[cfg(all(target_arch = "x86_64", any(mshv3, target_os = "windows")))] +pub(crate) fn validate_allowed_msrs( + vm: &dyn VirtualMachine, + allowed: &[u32], +) -> std::result::Result<(), CreateVmError> { + for &msr in allowed { + if !is_resettable_msr(msr) { + return Err(CreateVmError::MsrNotAllowable { + msr, + reason: "MSR is not a resettable MSR".to_string(), + }); + } + let captured = vm + .msrs(&[msr]) + .map_err(|_| CreateVmError::MsrNotAllowable { + msr, + reason: "MSR cannot be read on this host".to_string(), + })?; + vm.set_msrs(&captured) + .map_err(|_| CreateVmError::MsrNotAllowable { + msr, + reason: "MSR cannot be written on this host".to_string(), + })?; + } + Ok(()) +} + +/// Returns every guest-visible MTRR a filterless (MSHV/WHP) host must reset. +#[cfg(all(target_arch = "x86_64", any(mshv3, target_os = "windows")))] +pub(crate) fn mtrr_reset_indices( + vm: &dyn VirtualMachine, +) -> std::result::Result, CreateVmError> { + let mtrr_cap = vm + .msrs(&[MSR_MTRR_CAP]) + .map_err(CreateVmError::GetMtrrCap)?[0] + .value; + let indices = hyperv_mtrr_reset_indices(mtrr_cap)?; + vm.msrs(&indices) + .map_err(CreateVmError::RequiredMtrrsNotResettable)?; + Ok(indices) +} + #[cfg(test)] mod tests { diff --git a/src/hyperlight_host/src/hypervisor/virtual_machine/mshv/x86_64.rs b/src/hyperlight_host/src/hypervisor/virtual_machine/mshv/x86_64.rs index 1fd50d29a..88f2ae1f3 100644 --- a/src/hyperlight_host/src/hypervisor/virtual_machine/mshv/x86_64.rs +++ b/src/hyperlight_host/src/hypervisor/virtual_machine/mshv/x86_64.rs @@ -31,8 +31,10 @@ use mshv_bindings::{ hv_message_type_HVMSG_X64_HALT, hv_message_type_HVMSG_X64_IO_PORT_INTERCEPT, hv_partition_property_code_HV_PARTITION_PROPERTY_SYNTHETIC_PROC_FEATURES, hv_partition_synthetic_processor_features, hv_register_assoc, - hv_register_name_HV_X64_REGISTER_RIP, hv_register_value, mshv_create_partition_v2, - mshv_user_mem_region, + hv_register_name_HV_X64_REGISTER_MSR_MTRR_PHYS_BASE0, + hv_register_name_HV_X64_REGISTER_MSR_MTRR_PHYS_MASK0, hv_register_name_HV_X64_REGISTER_RIP, + hv_register_name_HV_X64_REGISTER_U_XSS, hv_register_value, mshv_create_partition_v2, + mshv_user_mem_region, msr_to_hv_reg_name as mshv_msr_to_hv_reg_name, }; #[cfg(feature = "hw-interrupts")] use mshv_bindings::{ @@ -50,7 +52,8 @@ use tracing_opentelemetry::OpenTelemetrySpanExt; use crate::hypervisor::gdb::{DebugError, DebuggableVm}; use crate::hypervisor::regs::{ CommonDebugRegs, CommonFpu, CommonRegisters, CommonSpecialRegisters, FP_CONTROL_WORD_DEFAULT, - MXCSR_DEFAULT, + MSR_APERF, MSR_BNDCFGS, MSR_MPERF, MSR_TSC_DEADLINE, MSR_TSX_CTRL, MSR_UMWAIT_CONTROL, + MSR_VIRT_SPEC_CTRL, MSR_XFD, MSR_XFD_ERR, MSR_XSS, MXCSR_DEFAULT, MsrEntry, }; #[cfg(test)] use crate::hypervisor::virtual_machine::XSAVE_BUFFER_SIZE; @@ -77,6 +80,84 @@ pub(crate) fn is_hypervisor_present() -> bool { } } +/// Maps an MSR index to the Hyper-V register used for host reset. +fn msr_to_hv_register_name(index: u32) -> Result { + const HV_X64_REGISTER_BNDCFGS: u32 = 0x0008_007C; + const HV_X64_REGISTER_MCOUNT: u32 = 0x0008_007E; + const HV_X64_REGISTER_A_COUNT: u32 = 0x0008_007F; + const HV_X64_REGISTER_TSX_CTRL: u32 = 0x0008_0088; + const HV_X64_REGISTER_TSC_DEADLINE: u32 = 0x0008_0095; + const HV_X64_REGISTER_UMWAIT_CONTROL: u32 = 0x0008_0098; + const HV_X64_REGISTER_XFD: u32 = 0x0008_0099; + const HV_X64_REGISTER_XFD_ERR: u32 = 0x0008_009A; + const HV_X64_REGISTER_VIRT_SPEC_CTRL: u32 = 0x0008_0086; + + if (0x200..=0x21F).contains(&index) { + let pair = (index - 0x200) / 2; + return Ok(if index & 1 == 0 { + hv_register_name_HV_X64_REGISTER_MSR_MTRR_PHYS_BASE0 + pair + } else { + hv_register_name_HV_X64_REGISTER_MSR_MTRR_PHYS_MASK0 + pair + }); + } + if index == MSR_XSS { + return Ok(hv_register_name_HV_X64_REGISTER_U_XSS); + } + if index == MSR_MPERF { + return Ok(HV_X64_REGISTER_MCOUNT); + } + if index == MSR_APERF { + return Ok(HV_X64_REGISTER_A_COUNT); + } + if index == MSR_TSX_CTRL { + return Ok(HV_X64_REGISTER_TSX_CTRL); + } + if index == MSR_TSC_DEADLINE { + return Ok(HV_X64_REGISTER_TSC_DEADLINE); + } + if index == MSR_UMWAIT_CONTROL { + return Ok(HV_X64_REGISTER_UMWAIT_CONTROL); + } + if index == MSR_BNDCFGS { + return Ok(HV_X64_REGISTER_BNDCFGS); + } + if index == MSR_XFD { + return Ok(HV_X64_REGISTER_XFD); + } + if index == MSR_XFD_ERR { + return Ok(HV_X64_REGISTER_XFD_ERR); + } + if index == MSR_VIRT_SPEC_CTRL { + return Ok(HV_X64_REGISTER_VIRT_SPEC_CTRL); + } + mshv_msr_to_hv_reg_name(index) +} + +#[cfg(test)] +mod msr_mapping_tests { + use super::*; + use crate::hypervisor::regs::core_reset_indices; + + #[test] + fn maps_all_stateful_msrs() { + for index in core_reset_indices() { + assert!( + msr_to_hv_register_name(index).is_ok(), + "missing MSR mapping for {index:#x}" + ); + } + assert_eq!(msr_to_hv_register_name(MSR_MPERF), Ok(0x0008_007E)); + assert_eq!(msr_to_hv_register_name(MSR_APERF), Ok(0x0008_007F)); + assert_eq!(msr_to_hv_register_name(MSR_TSX_CTRL), Ok(0x0008_0088)); + assert_eq!(msr_to_hv_register_name(MSR_TSC_DEADLINE), Ok(0x0008_0095)); + assert_eq!(msr_to_hv_register_name(MSR_UMWAIT_CONTROL), Ok(0x0008_0098)); + assert_eq!(msr_to_hv_register_name(MSR_BNDCFGS), Ok(0x0008_007C)); + assert_eq!(msr_to_hv_register_name(MSR_XFD), Ok(0x0008_0099)); + assert_eq!(msr_to_hv_register_name(MSR_XFD_ERR), Ok(0x0008_009A)); + assert_eq!(msr_to_hv_register_name(MSR_VIRT_SPEC_CTRL), Ok(0x0008_0086)); + } +} + /// A MSHV implementation of a single-vcpu VM #[derive(Debug)] pub(crate) struct MshvVm { @@ -102,13 +183,13 @@ impl MshvVm { let mshv = MSHV.as_ref().map_err(|e| e.clone())?; #[allow(unused_mut)] - let mut pr: mshv_create_partition_v2 = Default::default(); + let mut pr = mshv_create_partition_v2::default(); // Enable LAPIC for hw-interrupts — required for interrupt delivery // via request_virtual_interrupt. #[cfg(feature = "hw-interrupts")] { use mshv_bindings::MSHV_PT_BIT_LAPIC; - pr.pt_flags = 1u64 << MSHV_PT_BIT_LAPIC; + pr.pt_flags |= 1u64 << MSHV_PT_BIT_LAPIC; } // It's important to use create_vm_with_args() (not create_vm()), // because create_vm() sets up a SynIC partition by default. @@ -432,6 +513,55 @@ impl VirtualMachine for MshvVm { Ok(()) } + fn msrs(&self, indices: &[u32]) -> std::result::Result, RegisterError> { + if indices.is_empty() { + return Ok(Vec::new()); + } + let mut registers: Vec = indices + .iter() + .map(|&index| { + Ok(hv_register_assoc { + name: msr_to_hv_register_name(index) + .map_err(|_| RegisterError::MsrsUnsupported)?, + ..Default::default() + }) + }) + .collect::>()?; + self.vcpu_fd + .get_reg(&mut registers) + .map_err(|e| RegisterError::GetMsrs(e.into()))?; + Ok(registers + .iter() + .zip(indices) + .map(|(register, &index)| MsrEntry { + index, + // SAFETY: get_reg initialized each association as a 64-bit register. + value: unsafe { register.value.reg64 }, + }) + .collect()) + } + + fn set_msrs(&self, msrs: &[MsrEntry]) -> std::result::Result<(), RegisterError> { + let registers: Vec = msrs + .iter() + .map(|entry| { + Ok(hv_register_assoc { + name: msr_to_hv_register_name(entry.index) + .map_err(|_| RegisterError::MsrsUnsupported)?, + value: hv_register_value { reg64: entry.value }, + ..Default::default() + }) + }) + .collect::>()?; + if registers.is_empty() { + return Ok(()); + } + self.vcpu_fd + .set_reg(®isters) + .map_err(|e| RegisterError::SetMsrs(e.into()))?; + Ok(()) + } + #[allow(dead_code)] fn xsave(&self) -> std::result::Result, RegisterError> { let xsave = self diff --git a/src/hyperlight_host/src/hypervisor/virtual_machine/whp.rs b/src/hyperlight_host/src/hypervisor/virtual_machine/whp.rs index 6de2b29f1..7f3ab0656 100644 --- a/src/hyperlight_host/src/hypervisor/virtual_machine/whp.rs +++ b/src/hyperlight_host/src/hypervisor/virtual_machine/whp.rs @@ -33,9 +33,13 @@ use windows_result::HRESULT; use crate::hypervisor::gdb::{DebugError, DebuggableVm}; use crate::hypervisor::regs::{ Align16, CommonDebugRegs, CommonFpu, CommonRegisters, CommonSpecialRegisters, - FP_CONTROL_WORD_DEFAULT, MXCSR_DEFAULT, WHP_DEBUG_REGS_NAMES, WHP_DEBUG_REGS_NAMES_LEN, - WHP_FPU_NAMES, WHP_FPU_NAMES_LEN, WHP_REGS_NAMES, WHP_REGS_NAMES_LEN, WHP_SREGS_NAMES, - WHP_SREGS_NAMES_LEN, + FP_CONTROL_WORD_DEFAULT, MSR_APERF, MSR_BNDCFGS, MSR_CSTAR, MSR_INTERRUPT_SSP_TABLE_ADDR, + MSR_KERNEL_GS_BASE, MSR_LSTAR, MSR_MPERF, MSR_MTRR_CAP, MSR_PAT, MSR_PL0_SSP, MSR_PL1_SSP, + MSR_PL2_SSP, MSR_PL3_SSP, MSR_S_CET, MSR_SFMASK, MSR_SPEC_CTRL, MSR_STAR, MSR_SYSENTER_CS, + MSR_SYSENTER_EIP, MSR_SYSENTER_ESP, MSR_TSC_DEADLINE, MSR_TSX_CTRL, MSR_U_CET, + MSR_UMWAIT_CONTROL, MSR_XFD, MSR_XFD_ERR, MXCSR_DEFAULT, MsrEntry, WHP_DEBUG_REGS_NAMES, + WHP_DEBUG_REGS_NAMES_LEN, WHP_FPU_NAMES, WHP_FPU_NAMES_LEN, WHP_REGS_NAMES, WHP_REGS_NAMES_LEN, + WHP_SREGS_NAMES, WHP_SREGS_NAMES_LEN, }; use crate::hypervisor::surrogate_process::SurrogateProcess; use crate::hypervisor::surrogate_process_manager::{ @@ -73,6 +77,143 @@ pub(crate) fn is_hypervisor_present() -> bool { } } +/// Maps an MSR index to the WHP register used for host reset. +fn msr_to_whv_register_name(index: u32) -> Option { + Some(match index { + MSR_MTRR_CAP => WHvX64RegisterMsrMtrrCap, + MSR_SYSENTER_CS => WHvX64RegisterSysenterCs, + MSR_SYSENTER_ESP => WHvX64RegisterSysenterEsp, + MSR_SYSENTER_EIP => WHvX64RegisterSysenterEip, + MSR_PAT => WHvX64RegisterPat, + MSR_STAR => WHvX64RegisterStar, + MSR_LSTAR => WHvX64RegisterLstar, + MSR_CSTAR => WHvX64RegisterCstar, + MSR_SFMASK => WHvX64RegisterSfmask, + MSR_KERNEL_GS_BASE => WHvX64RegisterKernelGsBase, + MSR_SPEC_CTRL => WHvX64RegisterSpecCtrl, + MSR_U_CET => WHvX64RegisterUCet, + MSR_S_CET => WHvX64RegisterSCet, + MSR_PL0_SSP => WHvX64RegisterPl0Ssp, + MSR_PL1_SSP => WHvX64RegisterPl1Ssp, + MSR_PL2_SSP => WHvX64RegisterPl2Ssp, + MSR_PL3_SSP => WHvX64RegisterPl3Ssp, + MSR_INTERRUPT_SSP_TABLE_ADDR => WHvX64RegisterInterruptSspTableAddr, + // TSC and TSC offset/aux. + 0x10 => WHvX64RegisterTsc, + 0x3B => WHvX64RegisterTscAdjust, + 0xC000_0103 => WHvX64RegisterTscAux, + MSR_MPERF => WHvX64RegisterMCount, + MSR_APERF => WHvX64RegisterACount, + MSR_TSX_CTRL => WHvX64RegisterTsxCtrl, + MSR_XFD => WHvX64RegisterXfd, + MSR_XFD_ERR => WHvX64RegisterXfdErr, + MSR_UMWAIT_CONTROL => WHvX64RegisterUmwaitControl, + MSR_TSC_DEADLINE => WHvX64RegisterTscDeadline, + MSR_BNDCFGS => WHvX64RegisterBndcfgs, + // XSAVE supervisor state mask (IA32_XSS). + 0xDA0 => WHvX64RegisterXss, + // MTRRs: def type, variable base/mask pairs 0..=9, fixed ranges. + 0x2FF => WHvX64RegisterMsrMtrrDefType, + 0x200 => WHvX64RegisterMsrMtrrPhysBase0, + 0x201 => WHvX64RegisterMsrMtrrPhysMask0, + 0x202 => WHvX64RegisterMsrMtrrPhysBase1, + 0x203 => WHvX64RegisterMsrMtrrPhysMask1, + 0x204 => WHvX64RegisterMsrMtrrPhysBase2, + 0x205 => WHvX64RegisterMsrMtrrPhysMask2, + 0x206 => WHvX64RegisterMsrMtrrPhysBase3, + 0x207 => WHvX64RegisterMsrMtrrPhysMask3, + 0x208 => WHvX64RegisterMsrMtrrPhysBase4, + 0x209 => WHvX64RegisterMsrMtrrPhysMask4, + 0x20A => WHvX64RegisterMsrMtrrPhysBase5, + 0x20B => WHvX64RegisterMsrMtrrPhysMask5, + 0x20C => WHvX64RegisterMsrMtrrPhysBase6, + 0x20D => WHvX64RegisterMsrMtrrPhysMask6, + 0x20E => WHvX64RegisterMsrMtrrPhysBase7, + 0x20F => WHvX64RegisterMsrMtrrPhysMask7, + 0x210 => WHvX64RegisterMsrMtrrPhysBase8, + 0x211 => WHvX64RegisterMsrMtrrPhysMask8, + 0x212 => WHvX64RegisterMsrMtrrPhysBase9, + 0x213 => WHvX64RegisterMsrMtrrPhysMask9, + 0x214 => WHvX64RegisterMsrMtrrPhysBaseA, + 0x215 => WHvX64RegisterMsrMtrrPhysMaskA, + 0x216 => WHvX64RegisterMsrMtrrPhysBaseB, + 0x217 => WHvX64RegisterMsrMtrrPhysMaskB, + 0x218 => WHvX64RegisterMsrMtrrPhysBaseC, + 0x219 => WHvX64RegisterMsrMtrrPhysMaskC, + 0x21A => WHvX64RegisterMsrMtrrPhysBaseD, + 0x21B => WHvX64RegisterMsrMtrrPhysMaskD, + 0x21C => WHvX64RegisterMsrMtrrPhysBaseE, + 0x21D => WHvX64RegisterMsrMtrrPhysMaskE, + 0x21E => WHvX64RegisterMsrMtrrPhysBaseF, + 0x21F => WHvX64RegisterMsrMtrrPhysMaskF, + 0x250 => WHvX64RegisterMsrMtrrFix64k00000, + 0x258 => WHvX64RegisterMsrMtrrFix16k80000, + 0x259 => WHvX64RegisterMsrMtrrFix16kA0000, + 0x268 => WHvX64RegisterMsrMtrrFix4kC0000, + 0x269 => WHvX64RegisterMsrMtrrFix4kC8000, + 0x26A => WHvX64RegisterMsrMtrrFix4kD0000, + 0x26B => WHvX64RegisterMsrMtrrFix4kD8000, + 0x26C => WHvX64RegisterMsrMtrrFix4kE0000, + 0x26D => WHvX64RegisterMsrMtrrFix4kE8000, + 0x26E => WHvX64RegisterMsrMtrrFix4kF0000, + 0x26F => WHvX64RegisterMsrMtrrFix4kF8000, + _ => return None, + }) +} + +#[cfg(test)] +mod msr_mapping_tests { + use super::*; + use crate::hypervisor::regs::{MSR_DEBUGCTL, MSR_VIRT_SPEC_CTRL, core_reset_indices}; + + #[test] + fn maps_all_stateful_msrs_except_debugctl_and_virt_spec_ctrl() { + for index in core_reset_indices() { + // WHP models neither DEBUGCTL nor VIRT_SPEC_CTRL as a named register. + // Both are safe: WHP does not expose their guest-writable features, so + // a guest cannot leave retained state in them. + if index != MSR_DEBUGCTL && index != MSR_VIRT_SPEC_CTRL { + assert!( + msr_to_whv_register_name(index).is_some(), + "missing MSR mapping for {index:#x}" + ); + } + } + + assert!(msr_to_whv_register_name(MSR_DEBUGCTL).is_none()); + assert!(msr_to_whv_register_name(MSR_VIRT_SPEC_CTRL).is_none()); + assert_eq!( + msr_to_whv_register_name(MSR_MPERF), + Some(WHvX64RegisterMCount) + ); + assert_eq!( + msr_to_whv_register_name(MSR_APERF), + Some(WHvX64RegisterACount) + ); + assert_eq!( + msr_to_whv_register_name(MSR_TSX_CTRL), + Some(WHvX64RegisterTsxCtrl) + ); + assert_eq!(msr_to_whv_register_name(MSR_XFD), Some(WHvX64RegisterXfd)); + assert_eq!( + msr_to_whv_register_name(MSR_XFD_ERR), + Some(WHvX64RegisterXfdErr) + ); + assert_eq!( + msr_to_whv_register_name(MSR_UMWAIT_CONTROL), + Some(WHvX64RegisterUmwaitControl) + ); + assert_eq!( + msr_to_whv_register_name(MSR_TSC_DEADLINE), + Some(WHvX64RegisterTscDeadline) + ); + assert_eq!( + msr_to_whv_register_name(MSR_BNDCFGS), + Some(WHvX64RegisterBndcfgs) + ); + } +} + /// Helper: release a host-side file mapping view and its handle. /// Called from both `unmap_memory` and `WhpVm::drop`. fn release_file_mapping(view_base: *mut c_void, mapping_handle: HandleWrapper) { @@ -177,6 +318,9 @@ impl WhpVm { #[cfg(feature = "hw-interrupts")] Self::enable_lapic_emulation(p)?; + // Hyper-V permits MSR intercepts only for unimplemented indices. + // Implemented MSR isolation therefore depends on reset. + WHvSetupPartition(p).map_err(|e| CreateVmError::InitializeVm(e.into()))?; WHvCreateVirtualProcessor(p, 0, 0) .map_err(|e| CreateVmError::CreateVcpuFd(e.into()))?; @@ -667,6 +811,59 @@ impl VirtualMachine for WhpVm { } } + fn msrs(&self, indices: &[u32]) -> std::result::Result, RegisterError> { + if indices.is_empty() { + return Ok(Vec::new()); + } + // Callers validate every index before reaching this mapping. + let names: Vec = indices + .iter() + .map(|&i| msr_to_whv_register_name(i).ok_or(RegisterError::MsrsUnsupported)) + .collect::>()?; + // SAFETY: WHV_REGISTER_VALUE is a union of plain-old-data fields, so an + // all-zero value is a valid initial state. + let mut values: Vec> = + vec![unsafe { std::mem::zeroed() }; names.len()]; + // SAFETY: names and values have equal length. The call fills each value + // slot from the partition's vp 0 for the given register names. + unsafe { + WHvGetVirtualProcessorRegisters( + self.partition, + 0, + names.as_ptr(), + names.len() as u32, + values.as_mut_ptr() as *mut WHV_REGISTER_VALUE, + ) + .map_err(|e| RegisterError::GetMsrs(e.into()))?; + } + Ok(indices + .iter() + .zip(values) + .map(|(&index, v)| MsrEntry { + index, + // SAFETY: each register was read as a 64-bit MSR value. + value: unsafe { v.0.Reg64 }, + }) + .collect()) + } + + fn set_msrs(&self, msrs: &[MsrEntry]) -> std::result::Result<(), RegisterError> { + let regs: Vec<(WHV_REGISTER_NAME, Align16)> = msrs + .iter() + .map(|e| { + msr_to_whv_register_name(e.index) + .map(|name| (name, Align16(WHV_REGISTER_VALUE { Reg64: e.value }))) + .ok_or(RegisterError::MsrsUnsupported) + }) + .collect::>()?; + if regs.is_empty() { + return Ok(()); + } + self.set_registers(®s) + .map_err(|e| RegisterError::SetMsrs(e.into()))?; + Ok(()) + } + fn debug_regs(&self) -> std::result::Result { let mut whp_debug_regs_values: [Align16; WHP_DEBUG_REGS_NAMES_LEN] = Default::default(); diff --git a/src/hyperlight_host/src/mem/mgr.rs b/src/hyperlight_host/src/mem/mgr.rs index 1804d1c9f..57851f215 100644 --- a/src/hyperlight_host/src/mem/mgr.rs +++ b/src/hyperlight_host/src/mem/mgr.rs @@ -31,6 +31,8 @@ use super::shared_mem::{ ExclusiveSharedMemory, GuestSharedMemory, HostSharedMemory, ReadonlySharedMemory, SharedMemory, }; use crate::hypervisor::regs::CommonSpecialRegisters; +#[cfg(target_arch = "x86_64")] +use crate::hypervisor::regs::MsrEntry; use crate::mem::memory_region::MemoryRegion; #[cfg(crashdump)] use crate::mem::memory_region::{CrashDumpRegion, MemoryRegionFlags, MemoryRegionType}; @@ -300,6 +302,8 @@ where root_pt_gpas: &[u64], rsp_gva: u64, sregs: CommonSpecialRegisters, + #[cfg(target_arch = "x86_64")] msrs: Option>, + #[cfg(target_arch = "x86_64")] allowed_msrs: Option>, entrypoint: NextAction, host_functions: HostFunctionDetails, ) -> Result { @@ -313,6 +317,10 @@ where root_pt_gpas, rsp_gva, sregs, + #[cfg(target_arch = "x86_64")] + msrs, + #[cfg(target_arch = "x86_64")] + allowed_msrs, entrypoint, self.snapshot_count, host_functions, diff --git a/src/hyperlight_host/src/sandbox/config.rs b/src/hyperlight_host/src/sandbox/config.rs index f12387a0b..ba5fffdb2 100644 --- a/src/hyperlight_host/src/sandbox/config.rs +++ b/src/hyperlight_host/src/sandbox/config.rs @@ -29,6 +29,18 @@ pub struct DebugInfo { pub port: u16, } +/// Errors returned when configuring guest MSR access. +#[cfg(target_arch = "x86_64")] +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +pub enum AllowMsrError { + /// The requested allow list exceeds its fixed capacity. + #[error("MSR allow list exceeds its maximum of {maximum} distinct entries")] + CapacityExceeded { + /// Maximum number of distinct allowed MSRs. + maximum: usize, + }, +} + /// The complete set of configuration needed to create a Sandbox #[derive(Copy, Clone, Debug, Eq, PartialEq)] #[repr(C)] @@ -74,6 +86,12 @@ pub struct SandboxConfiguration { interrupt_vcpu_sigrtmin_offset: u8, /// How much writable memory to offer the guest scratch_size: usize, + /// Requested allow list stored inline to keep this type `Copy`. + #[cfg(target_arch = "x86_64")] + allowed_msrs: [u32; Self::MAX_ALLOWED_MSRS], + /// Number of valid entries in `allowed_msrs`. + #[cfg(target_arch = "x86_64")] + allowed_msrs_count: usize, } impl SandboxConfiguration { @@ -93,6 +111,11 @@ impl SandboxConfiguration { pub const DEFAULT_HEAP_SIZE: u64 = 131072; /// The default size of the scratch region pub const DEFAULT_SCRATCH_SIZE: usize = 0x48000; + /// Maximum number of distinct requested MSRs. + /// KVM supports at most 16 MSR filter ranges. Each index may require its + /// own range, so 16 is the portable limit across backends. + #[cfg(target_arch = "x86_64")] + pub const MAX_ALLOWED_MSRS: usize = 16; #[allow(clippy::too_many_arguments)] /// Create a new configuration for a sandbox with the given sizes. @@ -118,6 +141,10 @@ impl SandboxConfiguration { guest_debug_info, #[cfg(crashdump)] guest_core_dump, + #[cfg(target_arch = "x86_64")] + allowed_msrs: [0; Self::MAX_ALLOWED_MSRS], + #[cfg(target_arch = "x86_64")] + allowed_msrs_count: 0, } } @@ -159,6 +186,49 @@ impl SandboxConfiguration { self.interrupt_vcpu_sigrtmin_offset } + /// Adds MSRs to the sandbox allow list. + /// VM creation verifies that the backend can restore them. + /// + /// Duplicate indices, within the slice or against the existing list, are + /// ignored and do not count toward capacity. + /// + /// # Errors + /// + /// Returns [`AllowMsrError::CapacityExceeded`] if the distinct entries + /// would exceed [`Self::MAX_ALLOWED_MSRS`]. The allow list is unchanged on + /// error. + #[cfg(target_arch = "x86_64")] + #[instrument(skip_all, parent = Span::current(), level= "Trace")] + pub fn allow_msrs(&mut self, indices: &[u32]) -> Result<&mut Self, AllowMsrError> { + let additional = indices + .iter() + .enumerate() + .filter(|(position, index)| { + !self.allowed_msrs[..self.allowed_msrs_count].contains(index) + && !indices[..*position].contains(index) + }) + .count(); + if additional > Self::MAX_ALLOWED_MSRS - self.allowed_msrs_count { + return Err(AllowMsrError::CapacityExceeded { + maximum: Self::MAX_ALLOWED_MSRS, + }); + } + for &index in indices { + if !self.allowed_msrs[..self.allowed_msrs_count].contains(&index) { + self.allowed_msrs[self.allowed_msrs_count] = index; + self.allowed_msrs_count += 1; + } + } + Ok(self) + } + + /// Returns the requested allow list. + #[cfg(target_arch = "x86_64")] + #[instrument(skip_all, parent = Span::current(), level= "Trace")] + pub(crate) fn get_allowed_msrs(&self) -> &[u32] { + &self.allowed_msrs[..self.allowed_msrs_count] + } + /// Sets the offset from `SIGRTMIN` to determine the real-time signal used for /// interrupting the VCPU thread. /// @@ -261,7 +331,63 @@ impl Default for SandboxConfiguration { #[cfg(test)] mod tests { - use super::SandboxConfiguration; + use super::{AllowMsrError, SandboxConfiguration}; + + #[test] + #[cfg(target_arch = "x86_64")] + fn msr_allow_list_reports_overflow() { + let mut cfg = SandboxConfiguration::default(); + for index in 0..SandboxConfiguration::MAX_ALLOWED_MSRS as u32 { + cfg.allow_msrs(&[index]).unwrap(); + } + + cfg.allow_msrs(&[0]).unwrap(); + assert_eq!( + cfg.allow_msrs(&[SandboxConfiguration::MAX_ALLOWED_MSRS as u32]), + Err(AllowMsrError::CapacityExceeded { + maximum: SandboxConfiguration::MAX_ALLOWED_MSRS, + }) + ); + } + + #[test] + #[cfg(target_arch = "x86_64")] + fn bulk_msr_allow_list_overflow_is_atomic() { + let mut cfg = SandboxConfiguration::default(); + cfg.allow_msrs(&[1, 2]).unwrap(); + let oversized: Vec = (3..=SandboxConfiguration::MAX_ALLOWED_MSRS as u32 + 1).collect(); + + assert!(matches!( + cfg.allow_msrs(&oversized), + Err(AllowMsrError::CapacityExceeded { .. }) + )); + assert_eq!(cfg.get_allowed_msrs(), &[1, 2]); + } + + #[test] + #[cfg(target_arch = "x86_64")] + fn allow_msrs_dedups_and_preserves_order() { + let mut cfg = SandboxConfiguration::default(); + cfg.allow_msrs(&[0x10]).unwrap(); + cfg.allow_msrs(&[0x20, 0x20, 0x10, 0x30, 0x20]).unwrap(); + // 0x10 already present, 0x20 and 0x30 added once each in first-seen order. + assert_eq!(cfg.get_allowed_msrs(), &[0x10, 0x20, 0x30]); + } + + #[test] + #[cfg(target_arch = "x86_64")] + fn allow_msrs_duplicates_do_not_count_toward_capacity() { + let mut cfg = SandboxConfiguration::default(); + let fill: Vec = (0..SandboxConfiguration::MAX_ALLOWED_MSRS as u32 - 1).collect(); + cfg.allow_msrs(&fill).unwrap(); + // One slot remains. Three copies of one new index count as a single + // distinct entry and fit. + cfg.allow_msrs(&[u32::MAX, u32::MAX, u32::MAX]).unwrap(); + assert_eq!( + cfg.get_allowed_msrs().len(), + SandboxConfiguration::MAX_ALLOWED_MSRS + ); + } #[test] fn overrides() { diff --git a/src/hyperlight_host/src/sandbox/initialized_multi_use.rs b/src/hyperlight_host/src/sandbox/initialized_multi_use.rs index 6f9e87e17..e4dfb155d 100644 --- a/src/hyperlight_host/src/sandbox/initialized_multi_use.rs +++ b/src/hyperlight_host/src/sandbox/initialized_multi_use.rs @@ -159,7 +159,9 @@ impl MultiUseSandbox { /// interrupt behavior. Memory layout fields /// (`input_data_size`, `output_data_size`, `heap_size`, `scratch_size`) /// are always taken from the snapshot. Any values supplied in - /// `config` for those fields are ignored. + /// `config` for those fields are ignored. On x86_64 the `config` MSR + /// allow list must be a superset of the one the snapshot was taken with, + /// or the load fails with an MSR mismatch. /// /// # Examples /// @@ -304,6 +306,15 @@ impl MultiUseSandbox { crate::hypervisor::hyperlight_vm::HyperlightVmError::Restore(e.into()), ) })?; + + // Restore captured MSR state. + #[cfg(target_arch = "x86_64")] + vm.restore_msrs(snapshot.msrs(), snapshot.allowed_msrs()) + .map_err(|e| { + crate::HyperlightError::HyperlightVmError( + crate::hypervisor::hyperlight_vm::HyperlightVmError::Restore(e), + ) + })?; } #[cfg(gdb)] @@ -385,6 +396,13 @@ impl MultiUseSandbox { .vm .get_snapshot_sregs() .map_err(|e| HyperlightError::HyperlightVmError(e.into()))?; + #[cfg(target_arch = "x86_64")] + let msrs = self + .vm + .get_msr_reset_state() + .map_err(|e| HyperlightError::HyperlightVmError(e.into()))?; + #[cfg(target_arch = "x86_64")] + let allowed_msrs = self.vm.get_msr_allow_list(); let entrypoint = self.vm.get_entrypoint(); let host_functions = (&*self.host_funcs.try_lock().map_err(|e| { crate::new_error!("Error locking host_funcs at {}:{}: {}", file!(), line!(), e) @@ -396,6 +414,10 @@ impl MultiUseSandbox { &root_pt_gpas, stack_top_gpa, sregs, + #[cfg(target_arch = "x86_64")] + Some(msrs), + #[cfg(target_arch = "x86_64")] + Some(allowed_msrs), entrypoint, host_functions, )?; @@ -417,6 +439,10 @@ impl MultiUseSandbox { /// [`SnapshotHostFunctionMismatch`](crate::HyperlightError::SnapshotHostFunctionMismatch) /// carrying the missing names and signature differences. /// + /// On x86_64 this sandbox's MSR allow list must be a superset of the one + /// the snapshot was taken with, or the restore poisons with an MSR + /// mismatch. + /// /// ## Poison State Recovery /// /// This method automatically clears any poison state when successful. This is safe because: @@ -543,6 +569,15 @@ impl MultiUseSandbox { HyperlightVmError::Restore(e) })?; + // Restore captured MSR state. + #[cfg(target_arch = "x86_64")] + self.vm + .restore_msrs(snapshot.msrs(), snapshot.allowed_msrs()) + .map_err(|e| { + self.poisoned = true; + HyperlightVmError::Restore(e) + })?; + self.vm.set_stack_top(snapshot.stack_top_gva()); self.vm.set_entrypoint(snapshot.entrypoint()); @@ -2720,6 +2755,1223 @@ mod tests { ); } + #[cfg(target_arch = "x86_64")] + mod msr_tests { + use super::*; + use crate::HostFunctions; + use crate::hypervisor::hyperlight_vm::{CreateHyperlightVmError, HyperlightVmError}; + use crate::hypervisor::virtual_machine::{ + CreateVmError, RegisterError, ResetVcpuError, VmError, + }; + use crate::sandbox::snapshot::Snapshot; + + const KERNEL_GS_BASE: u32 = 0xC000_0102; + const SYSENTER_CS: u32 = 0x174; + + fn assert_msr_not_allowable(error: &HyperlightError, expected: u32) { + assert!( + matches!( + error, + HyperlightError::HyperlightVmError(HyperlightVmError::Create( + CreateHyperlightVmError::Vm(VmError::CreateVm( + CreateVmError::MsrNotAllowable { msr, .. } + )) + )) if *msr == expected + ), + "expected MsrNotAllowable for {expected:#x}, got: {error:?}" + ); + } + + fn assert_invalid_snapshot_msr(error: &HyperlightError) { + assert!( + matches!( + error, + HyperlightError::HyperlightVmError(HyperlightVmError::Restore( + ResetVcpuError::Register(RegisterError::InvalidSnapshotMsrIndex { .. }) + )) + ), + "expected InvalidSnapshotMsrIndex, got: {error:?}" + ); + } + + fn assert_msr_not_allowed(error: &HyperlightError) { + assert!( + matches!( + error, + HyperlightError::HyperlightVmError(HyperlightVmError::Restore( + ResetVcpuError::Register(RegisterError::SnapshotMsrNotAllowed { .. }) + )) + ), + "expected SnapshotMsrNotAllowed, got: {error:?}" + ); + } + + #[test] + fn kernel_gs_base_does_not_leak_through_swapgs() { + let mut sandbox = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + None, + ) + .unwrap() + .evolve() + .unwrap(); + + let original: u64 = sandbox.call("ReadKernelGsBaseViaSwapgs", ()).unwrap(); + let sentinel = if original == 0x0000_7AAA_5555_AAAA { + 0x0000_6BBB_4444_BBBB + } else { + 0x0000_7AAA_5555_AAAA + }; + let snapshot = sandbox.snapshot().unwrap(); + + sandbox + .call::<()>("WriteKernelGsBaseViaSwapgs", sentinel) + .unwrap(); + assert_eq!( + sandbox + .call::("ReadKernelGsBaseViaSwapgs", ()) + .unwrap(), + sentinel + ); + + sandbox.restore(snapshot).unwrap(); + assert_eq!( + sandbox + .call::("ReadKernelGsBaseViaSwapgs", ()) + .unwrap(), + original, + "KERNEL_GS_BASE leaked across restore" + ); + } + + #[test] + fn snapshot_msr_values_survive_full_in_memory_lifecycle() { + let mut config = SandboxConfiguration::default(); + config.allow_msrs(&[KERNEL_GS_BASE]).unwrap(); + let mut source = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(config), + ) + .unwrap() + .evolve() + .unwrap(); + let first = 0x1111; + let second = 0x2222; + let third = 0x3333; + + source + .call::<()>("WriteMSR", (KERNEL_GS_BASE, first)) + .unwrap(); + assert_eq!( + source.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + first + ); + let first_snapshot = source.snapshot().unwrap(); + + source + .call::<()>("WriteMSR", (KERNEL_GS_BASE, second)) + .unwrap(); + assert_eq!( + source.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + second + ); + source.restore(first_snapshot.clone()).unwrap(); + assert_eq!( + source.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + first + ); + + let mut clone = MultiUseSandbox::from_snapshot( + first_snapshot.clone(), + HostFunctions::default(), + Some(config), + ) + .unwrap(); + assert_eq!(clone.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), first); + + clone + .call::<()>("WriteMSR", (KERNEL_GS_BASE, third)) + .unwrap(); + assert_eq!(clone.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), third); + let third_snapshot = clone.snapshot().unwrap(); + source.restore(third_snapshot.clone()).unwrap(); + assert_eq!( + source.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + third + ); + + let mut second_clone = MultiUseSandbox::from_snapshot( + third_snapshot, + HostFunctions::default(), + Some(config), + ) + .unwrap(); + assert_eq!( + second_clone.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + third + ); + second_clone.restore(first_snapshot).unwrap(); + assert_eq!( + second_clone.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + first + ); + } + + #[test] + fn equivalent_msr_configs_are_order_independent_across_sandboxes() { + let source_order = [KERNEL_GS_BASE, SYSENTER_CS]; + let target_order = [SYSENTER_CS, KERNEL_GS_BASE]; + let mut source_config = SandboxConfiguration::default(); + source_config.allow_msrs(&source_order).unwrap(); + let mut source = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(source_config), + ) + .unwrap() + .evolve() + .unwrap(); + source + .call::<()>("WriteMSR", (KERNEL_GS_BASE, 0x4444u64)) + .unwrap(); + assert_eq!( + source.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + 0x4444 + ); + source + .call::<()>("WriteMSR", (SYSENTER_CS, 0x5555u64)) + .unwrap(); + assert_eq!(source.call::("ReadMSR", SYSENTER_CS).unwrap(), 0x5555); + let snapshot = source.snapshot().unwrap(); + + let mut target_config = SandboxConfiguration::default(); + target_config.allow_msrs(&target_order).unwrap(); + let mut target = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(target_config), + ) + .unwrap() + .evolve() + .unwrap(); + target + .call::<()>("WriteMSR", (KERNEL_GS_BASE, 0xAAAAu64)) + .unwrap(); + assert_eq!( + target.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + 0xAAAA + ); + target + .call::<()>("WriteMSR", (SYSENTER_CS, 0xBBBBu64)) + .unwrap(); + assert_eq!(target.call::("ReadMSR", SYSENTER_CS).unwrap(), 0xBBBB); + target.restore(snapshot.clone()).unwrap(); + assert_eq!( + target.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + 0x4444 + ); + assert_eq!(target.call::("ReadMSR", SYSENTER_CS).unwrap(), 0x5555); + + let mut clone = MultiUseSandbox::from_snapshot( + snapshot, + HostFunctions::default(), + Some(target_config), + ) + .unwrap(); + assert_eq!( + clone.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + 0x4444 + ); + assert_eq!(clone.call::("ReadMSR", SYSENTER_CS).unwrap(), 0x5555); + } + + /// A restore succeeds when the destination allow list is a superset + /// of the snapshot's. The snapshot's allowed MSR keeps its captured + /// value. An MSR the destination adds resets to the baseline. + #[test] + fn snapshot_restores_into_superset_allow_list() { + const SYSENTER_ESP: u32 = 0x175; + let sentinel: u64 = 0x1234; + let mut source_config = SandboxConfiguration::default(); + source_config.allow_msrs(&[SYSENTER_CS]).unwrap(); + let mut source = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(source_config), + ) + .unwrap() + .evolve() + .unwrap(); + source + .call::<()>("WriteMSR", (SYSENTER_CS, sentinel)) + .unwrap(); + let snapshot = source.snapshot().unwrap(); + + let mut dest_config = SandboxConfiguration::default(); + dest_config + .allow_msrs(&[SYSENTER_CS, SYSENTER_ESP]) + .unwrap(); + + let mut clone = MultiUseSandbox::from_snapshot( + snapshot.clone(), + HostFunctions::default(), + Some(dest_config), + ) + .unwrap(); + assert_eq!(clone.call::("ReadMSR", SYSENTER_CS).unwrap(), sentinel); + let baseline: u64 = clone.call("ReadMSR", SYSENTER_ESP).unwrap(); + + let mut target = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(dest_config), + ) + .unwrap() + .evolve() + .unwrap(); + target + .call::<()>("WriteMSR", (SYSENTER_ESP, baseline ^ 0x55)) + .unwrap(); + target.restore(snapshot).unwrap(); + assert_eq!( + target.call::("ReadMSR", SYSENTER_CS).unwrap(), + sentinel + ); + // An MSR the destination adds resets to its baseline. + assert_eq!( + target.call::("ReadMSR", SYSENTER_ESP).unwrap(), + baseline + ); + } + + /// A restore is rejected when the snapshot allows an MSR the + /// destination does not. Both restore paths fail and poison the + /// sandbox. + #[test] + fn snapshot_rejects_non_superset_allow_list() { + const SYSENTER_ESP: u32 = 0x175; + let mut source_config = SandboxConfiguration::default(); + source_config.allow_msrs(&[SYSENTER_CS]).unwrap(); + let mut source = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(source_config), + ) + .unwrap() + .evolve() + .unwrap(); + source + .call::<()>("WriteMSR", (SYSENTER_CS, 0x1234u64)) + .unwrap(); + let snapshot = source.snapshot().unwrap(); + + // A destination that allows nothing, and one that allows a + // disjoint MSR, both fail the superset check. + for dest in [&[][..], &[SYSENTER_ESP][..]] { + let mut config = SandboxConfiguration::default(); + config.allow_msrs(dest).unwrap(); + + let err = MultiUseSandbox::from_snapshot( + snapshot.clone(), + HostFunctions::default(), + Some(config), + ) + .expect_err("from_snapshot must reject a non-superset allow list"); + assert_msr_not_allowed(&err); + + let mut target = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(config), + ) + .unwrap() + .evolve() + .unwrap(); + let err = target + .restore(snapshot.clone()) + .expect_err("restore must reject a non-superset allow list"); + assert_msr_not_allowed(&err); + assert!(target.poisoned()); + assert!(matches!( + target.call::("Echo", "hi".to_string()), + Err(HyperlightError::PoisonedSandbox) + )); + } + } + + #[test] + fn from_pre_init_snapshot_uses_local_msr_reset_set() { + let mut config = SandboxConfiguration::default(); + config.allow_msrs(&[KERNEL_GS_BASE]).unwrap(); + let snapshot = Arc::new( + Snapshot::from_env( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + config, + ) + .unwrap(), + ); + assert!(snapshot.msrs().is_none()); + + let mut sandbox = MultiUseSandbox::from_snapshot( + snapshot.clone(), + HostFunctions::default(), + Some(config), + ) + .unwrap(); + let baseline: u64 = sandbox.call("ReadMSR", KERNEL_GS_BASE).unwrap(); + sandbox + .call::<()>("WriteMSR", (KERNEL_GS_BASE, baseline ^ 0x55)) + .unwrap(); + assert_eq!( + sandbox.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + baseline ^ 0x55 + ); + } + + #[test] + #[cfg(target_arch = "x86_64")] + fn snapshot_without_msrs_uses_destination_reset_set() { + let mut source = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + None, + ) + .unwrap() + .evolve() + .unwrap(); + let snapshot = source.snapshot().unwrap(); + source.snapshot = None; + let Ok(mut snap) = Arc::try_unwrap(snapshot) else { + panic!("snapshot should be uniquely owned"); + }; + // A snapshot without MSRs uses the destination baseline. + snap.set_msrs(None); + snap.set_allowed_msrs(None); + let snapshot = Arc::new(snap); + + let mut config = SandboxConfiguration::default(); + config.allow_msrs(&[KERNEL_GS_BASE]).unwrap(); + let mut target = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(config), + ) + .unwrap() + .evolve() + .unwrap(); + let baseline: u64 = target.call("ReadMSR", KERNEL_GS_BASE).unwrap(); + target + .call::<()>("WriteMSR", (KERNEL_GS_BASE, baseline ^ 0x55)) + .unwrap(); + assert_eq!( + target.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + baseline ^ 0x55 + ); + target.restore(snapshot.clone()).unwrap(); + assert_eq!( + target.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + baseline + ); + target + .call::<()>("WriteMSR", (KERNEL_GS_BASE, baseline ^ 0xAA)) + .unwrap(); + assert_eq!( + target.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + baseline ^ 0xAA + ); + + let mut clone = + MultiUseSandbox::from_snapshot(snapshot, HostFunctions::default(), Some(config)) + .unwrap(); + let clone_baseline: u64 = clone.call("ReadMSR", KERNEL_GS_BASE).unwrap(); + clone + .call::<()>("WriteMSR", (KERNEL_GS_BASE, clone_baseline ^ 0xCC)) + .unwrap(); + assert_eq!( + clone.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + clone_baseline ^ 0xCC + ); + } + + #[test] + fn malformed_snapshot_msrs_poison_and_trusted_restore_recovers() { + let indices = [SYSENTER_CS, KERNEL_GS_BASE]; + let mut config = SandboxConfiguration::default(); + config.allow_msrs(&indices).unwrap(); + let mut source = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(config), + ) + .unwrap() + .evolve() + .unwrap(); + let snapshot = source.snapshot().unwrap(); + source.snapshot = None; + let Ok(mut snapshot) = Arc::try_unwrap(snapshot) else { + panic!("snapshot should be uniquely owned"); + }; + let mut msrs = snapshot.msrs().unwrap().clone(); + msrs[0].index = 0xDEAD; + snapshot.set_msrs(Some(msrs)); + let snapshot = Arc::new(snapshot); + + let error = MultiUseSandbox::from_snapshot( + snapshot.clone(), + HostFunctions::default(), + Some(config), + ) + .expect_err("from_snapshot must reject malformed snapshot MSRs"); + assert_invalid_snapshot_msr(&error); + + let mut target = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(config), + ) + .unwrap() + .evolve() + .unwrap(); + let trusted_value: u64 = target.call("ReadMSR", KERNEL_GS_BASE).unwrap(); + let recovery_snapshot = target.snapshot().unwrap(); + target + .call::<()>("WriteMSR", (KERNEL_GS_BASE, trusted_value ^ 0x55)) + .unwrap(); + let error = target + .restore(snapshot) + .expect_err("restore must reject malformed snapshot MSRs"); + assert_invalid_snapshot_msr(&error); + assert!(target.poisoned()); + assert!(matches!( + target.call::("Echo", "hi".to_string()), + Err(HyperlightError::PoisonedSandbox) + )); + + target.restore(recovery_snapshot).unwrap(); + assert!(!target.poisoned()); + assert_eq!( + target.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + trusted_value + ); + } + + #[test] + #[cfg(kvm)] + fn denied_msr_access_poisons_sandbox() { + use crate::hypervisor::virtual_machine::{HypervisorType, get_available_hypervisor}; + + match get_available_hypervisor() { + Some(HypervisorType::Kvm) => {} + _ => { + return; + } + } + + let mut sbox = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + None, + ) + .unwrap() + .evolve() + .unwrap(); + + let snapshot = sbox.snapshot().unwrap(); + let msr_index: u32 = 0xC000_0102; // IA32_KERNEL_GS_BASE + + let result = sbox.call::("ReadMSR", msr_index); + assert!( + matches!( + &result, + Err(HyperlightError::MsrReadViolation(idx)) if *idx == msr_index + ), + "RDMSR 0x{:X}: expected MsrReadViolation, got: {:?}", + msr_index, + result + ); + assert!(sbox.poisoned()); + + sbox.restore(snapshot.clone()).unwrap(); + + let result = sbox.call::<()>("WriteMSR", (msr_index, 0x5u64)); + assert!( + matches!( + &result, + Err(HyperlightError::MsrWriteViolation(idx, _)) if *idx == msr_index + ), + "WRMSR 0x{:X}: expected MsrWriteViolation, got: {:?}", + msr_index, + result + ); + assert!(sbox.poisoned()); + } + + /// A write-only command cannot enter the reset set. + #[test] + #[cfg(target_arch = "x86_64")] + fn test_allow_non_resettable_msr_fails_creation() { + let mut cfg = SandboxConfiguration::default(); + cfg.allow_msrs(&[0x49]).unwrap(); // IA32_PRED_CMD, a write-only command MSR + + let err = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(cfg), + ) + .unwrap() + .evolve() + .unwrap_err(); + + assert_msr_not_allowable(&err, 0x49); + } + + /// Host support cannot authorize an unclassified MSR. + #[test] + #[cfg(kvm)] + fn unclassified_allowed_msr_rejected_at_creation() { + use crate::hypervisor::virtual_machine::{HypervisorType, get_available_hypervisor}; + + if !matches!(get_available_hypervisor(), Some(HypervisorType::Kvm)) { + return; + } + + let mut cfg = SandboxConfiguration::default(); + cfg.allow_msrs(&[0x1A0]).unwrap(); // IA32_MISC_ENABLE: host-probeable, not in MSR_TABLE + + let err = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(cfg), + ) + .unwrap() + .evolve() + .expect_err("an unclassified allowed MSR must be rejected at creation"); + + assert_msr_not_allowable(&err, 0x1A0); + } + + #[test] + #[cfg(target_arch = "x86_64")] + fn test_multiple_allowed_msrs_reset_across_restore() { + // Resettable MSRs the guest may write once allowed. + let msrs: [u32; 4] = [0x174, 0x175, 0x176, 0xC000_0102]; + let mut cfg = SandboxConfiguration::default(); + cfg.allow_msrs(&msrs).unwrap(); + + let mut sbox = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(cfg), + ) + .unwrap() + .evolve() + .unwrap(); + + let baseline_snapshot = sbox.snapshot().unwrap(); + + let value: u64 = 0x1000; + for &msr in &msrs { + sbox.call::<()>("WriteMSR", (msr, value)).unwrap(); + let read_value: u64 = sbox.call("ReadMSR", msr).unwrap(); + assert_eq!(read_value, value, "MSR 0x{msr:X} should be writable"); + } + + sbox.restore(baseline_snapshot).unwrap(); + for &msr in &msrs { + let read_value: u64 = sbox.call("ReadMSR", msr).unwrap(); + assert_ne!( + read_value, value, + "MSR 0x{msr:X} should be reset to baseline across restore" + ); + } + } + + /// An allowed guest write must not survive restore. + #[test] + #[cfg(target_arch = "x86_64")] + fn test_allowed_msr_does_not_leak_across_restore() { + let msr_index: u32 = 0xC000_0102; // IA32_KERNEL_GS_BASE + let sentinel: u64 = 0xCAFE_F00D; + + let mut cfg = SandboxConfiguration::default(); + cfg.allow_msrs(&[msr_index]).unwrap(); + let mut sbox = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(cfg), + ) + .unwrap() + .evolve() + .unwrap(); + + let baseline = sbox.snapshot().unwrap(); + let original: u64 = sbox.call("ReadMSR", msr_index).unwrap(); + assert_ne!( + original, sentinel, + "test sentinel must differ from the baseline value" + ); + + sbox.call::<()>("WriteMSR", (msr_index, sentinel)).unwrap(); + assert_eq!( + sbox.call::("ReadMSR", msr_index).unwrap(), + sentinel, + "sentinel should be observable before restore" + ); + sbox.restore(baseline).unwrap(); + + let after: u64 = sbox.call("ReadMSR", msr_index).unwrap(); + assert_ne!(after, sentinel, "sentinel leaked across restore"); + assert_eq!(after, original, "MSR not reset to its baseline value"); + } + + /// KVM denies DEBUGCTL through its filter and x2APIC through xAPIC mode. + #[test] + #[cfg(all(kvm, target_arch = "x86_64"))] + fn test_debugctl_and_x2apic_msr_denied_by_default() { + use crate::hypervisor::virtual_machine::{HypervisorType, get_available_hypervisor}; + + if !matches!(get_available_hypervisor(), Some(HypervisorType::Kvm)) { + return; + } + + let cases: [(u32, bool); 2] = [(0x1D9, true), (0x800, false)]; + for (msr_index, expect_filter_violation) in cases { + let mut sbox = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + None, + ) + .unwrap() + .evolve() + .unwrap(); + + let result = sbox.call::<()>("WriteMSR", (msr_index, 0x1u64)); + if expect_filter_violation { + assert!( + matches!( + &result, + Err(HyperlightError::MsrWriteViolation(idx, _)) if *idx == msr_index + ), + "WRMSR 0x{msr_index:X}: expected MsrWriteViolation, got: {result:?}" + ); + } else { + assert!( + matches!(&result, Err(HyperlightError::GuestAborted(_, _))), + "WRMSR 0x{msr_index:X}: expected direct #GP, got: {result:?}" + ); + } + assert!( + sbox.poisoned(), + "sandbox should be poisoned after a denied WRMSR to 0x{msr_index:X}" + ); + } + } + + /// A rejected MSR restore poisons the sandbox on every backend. + /// + /// The host register set rejects the noncanonical KERNEL_GS_BASE value + /// on KVM, MSHV, and WHP alike, leaving the sandbox poisoned. + #[test] + #[cfg(target_arch = "x86_64")] + fn rejected_msr_restore_poisons_sandbox() { + let mut sbox = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + None, + ) + .unwrap() + .evolve() + .unwrap(); + + let baseline = sbox.snapshot().unwrap(); + sbox.snapshot = None; + let Ok(mut snap) = Arc::try_unwrap(baseline) else { + panic!("snapshot should be uniquely owned after clearing the cache"); + }; + // A noncanonical KERNEL_GS_BASE value the host set rejects. + let mut msrs = snap.msrs().unwrap().clone(); + let kernel_gs_base = msrs + .iter_mut() + .find(|entry| entry.index == 0xC000_0102) + .expect("KERNEL_GS_BASE should be in the reset set"); + kernel_gs_base.value = 0xDEAD_0000_0000_0000; + snap.set_msrs(Some(msrs)); + let snap = Arc::new(snap); + + let err = sbox + .restore(snap) + .expect_err("restore should fail on a rejected MSR set"); + assert!( + format!("{err:?}").to_lowercase().contains("msr") + || format!("{err:?}").to_lowercase().contains("restore"), + "expected an MSR restore error, got: {err:?}" + ); + assert!( + sbox.poisoned(), + "sandbox should be poisoned after failed restore" + ); + + let call = sbox.call::("Echo", "hi".to_string()); + assert!( + matches!(call, Err(HyperlightError::PoisonedSandbox)), + "poisoned sandbox should reject guest calls, got: {call:?}" + ); + } + + /// Unresettable feature-class MSRs must not retain guest writes. PMU, + /// LBR, and FRED are perfmon or feature gated. The AMD virtualization + /// MSRs are gated on nested-virt capability the sandbox never requests. + #[test] + #[cfg(target_arch = "x86_64")] + fn unresettable_msr_classes_do_not_leak() { + let cases: &[(u32, &str)] = &[ + (0xC1, "PMU IA32_PMC0"), + (0x186, "PMU IA32_PERFEVTSEL0"), + (0x38F, "PMU IA32_PERF_GLOBAL_CTRL"), + (0x1C8, "LBR_SELECT"), + (0x14CE, "arch-LBR IA32_LBR_CTL"), + (0x1D4, "FRED IA32_FRED_CONFIG"), + (0xC001_0114, "AMD VM_CR"), + (0xC001_0117, "AMD VM_HSAVE_PA"), + ]; + + let mut sbox = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + None, + ) + .unwrap() + .evolve() + .unwrap(); + + for &(msr, _name) in cases { + assert_msr_write_does_not_survive_restore(&mut sbox, msr, 0x1); + } + } + + /// A guest write to IA32_MISC_ENABLE leaves no retained state. Hyper-V + /// drops the write on Intel and faults it on AMD. + #[test] + #[cfg(target_arch = "x86_64")] + fn misc_enable_guest_write_does_not_survive_restore() { + let mut sbox = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + None, + ) + .unwrap() + .evolve() + .unwrap(); + assert_msr_write_does_not_survive_restore(&mut sbox, 0x1A0, 1u64 << 40); + } + + /// Every stateful table entry needs runtime reset coverage. + #[test] + #[cfg(target_arch = "x86_64")] + fn runtime_msr_table_entries_are_justified() { + use crate::hypervisor::regs::core_reset_indices; + + #[cfg(kvm)] + let kernel_gs_uses_instruction_side_effect = matches!( + crate::hypervisor::virtual_machine::get_available_hypervisor(), + Some(crate::hypervisor::virtual_machine::HypervisorType::Kvm) + ); + #[cfg(not(kvm))] + let kernel_gs_uses_instruction_side_effect = false; + + let mut sbox = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + None, + ) + .unwrap() + .evolve() + .unwrap(); + + let reset_indices: Vec = sbox + .snapshot() + .unwrap() + .msrs() + .expect("filterless backend should have an MSR reset set") + .iter() + .map(|entry| entry.index) + .collect(); + + for index in core_reset_indices() { + if !reset_indices.contains(&index) { + assert_omitted_msr_does_not_retain(&mut sbox, index); + } else if kernel_gs_uses_instruction_side_effect && index == KERNEL_GS_BASE { + // Direct WRMSR is denied. The dedicated SWAPGS test proves + // the instruction-side mutation is restored. + } else if (index == 0x10 && !kernel_gs_uses_instruction_side_effect) + || matches!(index, 0xE7 | 0xE8) + { + assert_guest_counter_is_writable_and_restored(&mut sbox, index); + } else if let Some(sentinel) = positive_write_sentinel(index) { + assert_guest_msr_is_writable_and_restored(&mut sbox, index, sentinel); + } else { + assert!( + reset_exception_reason(index).is_some(), + "MSR 0x{index:X} is in the reset set without positive guest-write coverage or an explicit reason" + ); + } + } + } + + fn assert_omitted_msr_does_not_retain(sbox: &mut MultiUseSandbox, index: u32) { + let baseline = sbox.snapshot().unwrap(); + let original: u64 = match sbox.call("ReadMSR", index) { + Ok(value) => value, + Err(_) => { + assert!(sbox.poisoned(), "0x{index:X}: fault did not poison sandbox"); + sbox.restore(baseline).unwrap(); + return; + } + }; + let preferred = positive_write_sentinel(index).unwrap_or(original ^ 1); + let candidates = [preferred, original ^ 1, original ^ 2, 0, 1, 0x1000]; + + for candidate in candidates { + if candidate == original { + continue; + } + if sbox.call::<()>("WriteMSR", (index, candidate)).is_err() { + assert!(sbox.poisoned(), "0x{index:X}: fault did not poison sandbox"); + sbox.restore(baseline.clone()).unwrap(); + continue; + } + let written: u64 = sbox.call("ReadMSR", index).unwrap_or_else(|error| { + panic!("0x{index:X}: read after successful write failed: {error:?}") + }); + if written != original { + sbox.restore(baseline).unwrap(); + let after: u64 = sbox.call("ReadMSR", index).unwrap(); + assert_eq!( + after, original, + "0x{index:X}: guest retained a write but the MSR is absent from the reset set" + ); + return; + } + sbox.restore(baseline.clone()).unwrap(); + } + } + + fn positive_write_sentinel(index: u32) -> Option { + match index { + 0x174 => Some(0x10), // SYSENTER_CS + 0x175 | 0x176 => Some(0x1000), // SYSENTER_ESP/EIP + 0x277 => Some(0x0007_0406_0007_0406), // PAT + 0xC000_0081 => Some(0x001B_0008_0000_0000), // STAR + 0xC000_0082 | 0xC000_0083 => Some(0x1000), // LSTAR/CSTAR + 0xC000_0084 => Some(0x200), // SFMASK + 0xC000_0102 => Some(0x1000), // KERNEL_GS_BASE + 0x3B => Some(0x1000), // TSC_ADJUST + 0xC000_0103 => Some(0x5), // TSC_AUX + 0x2FF => Some(0xC00), // MTRR_DEF_TYPE + 0x200..=0x21F if index & 1 == 0 => Some(0x6), // MTRR_PHYSBASEn + 0x200..=0x21F => Some(0x800), // MTRR_PHYSMASKn + 0x250 | 0x258 | 0x259 | 0x268..=0x26F => Some(0x0606_0606_0606_0606), + _ => None, + } + } + + fn reset_exception_reason(index: u32) -> Option<&'static str> { + match index { + 0x10 => Some("KVM denies direct guest TSC MSR access"), + 0x1D9 => Some("DEBUGCTL support depends on exposed debug features"), + 0x48 => Some("SPEC_CTRL writable bits depend on mitigation features"), + 0x6A0 | 0x6A2 | 0x6A4..=0x6A8 => { + Some("CET writable state depends on exposed CET features") + } + 0x122 => Some("TSX_CTRL writable bits depend on exposed TSX features"), + 0x1C4 | 0x1C5 => Some("XFD writable bits depend on exposed XSAVE features"), + 0xE1 => Some("UMWAIT_CONTROL writable bits depend on exposed WAITPKG features"), + 0x6E0 => Some("TSC_DEADLINE writable bits depend on exposed APIC-timer features"), + 0xD90 => Some("BNDCFGS writable bits depend on exposed MPX features"), + 0xDA0 => Some("XSS writable bits depend on exposed XSAVE features"), + 0xC001_011F => { + Some("VIRT_SPEC_CTRL writable bits depend on exposed AMD SSBD virtualization") + } + _ => None, + } + } + + fn assert_guest_msr_is_writable_and_restored( + sbox: &mut MultiUseSandbox, + index: u32, + sentinel: u64, + ) { + let baseline = sbox.snapshot().unwrap(); + let original: u64 = sbox + .call("ReadMSR", index) + .unwrap_or_else(|error| panic!("0x{index:X}: guest RDMSR failed: {error:?}")); + let value = if original == sentinel { 0 } else { sentinel }; + + sbox.call::<()>("WriteMSR", (index, value)) + .unwrap_or_else(|error| panic!("0x{index:X}: guest WRMSR failed: {error:?}")); + let written: u64 = sbox + .call("ReadMSR", index) + .unwrap_or_else(|error| panic!("0x{index:X}: guest read-back failed: {error:?}")); + assert_eq!(written, value, "0x{index:X}: guest write did not stick"); + + sbox.restore(baseline).unwrap(); + let restored: u64 = sbox.call("ReadMSR", index).unwrap(); + assert_eq!( + restored, original, + "0x{index:X}: restore did not recover the baseline" + ); + } + + fn assert_guest_counter_is_writable_and_restored(sbox: &mut MultiUseSandbox, index: u32) { + let baseline = sbox.snapshot().unwrap(); + let original: u64 = sbox.call("ReadMSR", index).unwrap(); + let jump = original.wrapping_add(1 << 60); + + sbox.call::<()>("WriteMSR", (index, jump)).unwrap(); + let written: u64 = sbox.call("ReadMSR", index).unwrap(); + assert!( + written >= jump / 2, + "0x{index:X}: guest write did not stick" + ); + + sbox.restore(baseline).unwrap(); + let restored: u64 = sbox.call("ReadMSR", index).unwrap(); + assert!( + restored < jump / 2, + "0x{index:X}: restore did not pull the counter below the guest-written jump" + ); + } + + /// Verifies that a guest MSR write faults or resets to its baseline. + #[cfg(target_arch = "x86_64")] + fn assert_msr_write_does_not_survive_restore( + sbox: &mut MultiUseSandbox, + msr: u32, + sentinel: u64, + ) { + let baseline = sbox.snapshot().unwrap(); + let original: u64 = match sbox.call("ReadMSR", msr) { + Ok(v) => v, + Err(_) => { + assert!( + sbox.poisoned(), + "0x{msr:X}: a faulting RDMSR should poison the sandbox" + ); + sbox.restore(baseline).unwrap(); + return; + } + }; + assert_ne!( + original, sentinel, + "0x{msr:X}: sentinel must differ from baseline" + ); + + if sbox.call::<()>("WriteMSR", (msr, sentinel)).is_err() { + assert!( + sbox.poisoned(), + "0x{msr:X}: a faulting WRMSR should poison the sandbox" + ); + sbox.restore(baseline).unwrap(); + return; + } + + sbox.restore(baseline).unwrap(); + let after: u64 = sbox.call("ReadMSR", msr).unwrap(); + assert_eq!( + after, original, + "0x{msr:X}: MSR leaked across restore (expected 0x{original:X}, got 0x{after:X})" + ); + } + + /// Audits Hyper-V MSR bitmap ranges for guest state retained by restore. + #[test] + #[ignore = "slow host-dependent hardware MSR audit"] + #[cfg(target_arch = "x86_64")] + fn test_no_msr_leaks_across_restore_full_window_sweep() { + // Free-running counters use a magnitude check after restore. + const FREE_RUNNING: &[u32] = &[ + 0x10, // IA32_TIME_STAMP_COUNTER + 0xE7, // IA32_MPERF + 0xE8, // IA32_APERF + ]; + + #[cfg(kvm)] + if matches!( + crate::hypervisor::virtual_machine::get_available_hypervisor(), + Some(crate::hypervisor::virtual_machine::HypervisorType::Kvm) + ) { + return; + } + + let mut sbox = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + None, + ) + .unwrap() + .evolve() + .unwrap(); + + let baseline = sbox.snapshot().unwrap(); + + // At least one retained write must exercise restore. + let mut readable = 0usize; + let mut exercised: Vec = Vec::new(); + let mut read_only: Vec = Vec::new(); + let mut masked_only: Vec = Vec::new(); + // Collect all free-running leaks for one diagnostic. + let mut free_running_leaked: Vec = Vec::new(); + + // Architectural and low model-specific indices. + let low = 0x0000_0000u32..=0x0000_1FFF; + // Hyper-V synthetic indices. + let hyperv_synthetic = 0x4000_0000u32..=0x4000_1FFF; + // Extended and AMD model-specific indices. + let extended = 0xC000_0000u32..=0xC001_FFFF; + let windows = low.chain(hyperv_synthetic).chain(extended); + for msr in windows { + let original: u64 = match sbox.call("ReadMSR", msr) { + Ok(v) => v, + Err(_) => { + sbox.restore(baseline.clone()).unwrap(); + continue; + } + }; + readable += 1; + + // A large jump distinguishes reset from normal counter progress. + if FREE_RUNNING.contains(&msr) { + let jump = original.wrapping_add(1 << 60); + if sbox.call::<()>("WriteMSR", (msr, jump)).is_err() { + sbox.restore(baseline.clone()).unwrap(); + read_only.push(msr); + continue; + } + let planted = match sbox.call::("ReadMSR", msr) { + Ok(v) => v, + Err(_) => { + sbox.restore(baseline.clone()).unwrap(); + masked_only.push(msr); + continue; + } + }; + if planted < jump / 2 { + sbox.restore(baseline.clone()).unwrap(); + masked_only.push(msr); + continue; + } + sbox.restore(baseline.clone()).unwrap(); + let after: u64 = sbox.call("ReadMSR", msr).unwrap(); + if after < jump / 2 { + exercised.push(msr); + } else { + free_running_leaked.push(msr); + } + continue; + } + + // Multiple candidates cover MSRs with restricted writable bits. + let candidates = [ + original ^ 0x55, + original ^ 0x1, + original ^ (1 << 12), + original ^ (1 << 20), + original ^ (1 << 32), + original.wrapping_add(1), + 0, + ]; + let mut planted = false; + let mut saw_write = false; + for cand in candidates { + if cand == original { + continue; + } + if sbox.call::<()>("WriteMSR", (msr, cand)).is_err() { + sbox.restore(baseline.clone()).unwrap(); + continue; + } + saw_write = true; + match sbox.call::("ReadMSR", msr) { + Ok(v) if v != original => { + planted = true; + break; + } + _ => { + sbox.restore(baseline.clone()).unwrap(); + } + } + } + + if planted { + sbox.restore(baseline.clone()).unwrap(); + match sbox.call::("ReadMSR", msr) { + Ok(after) => assert_eq!( + after, original, + "0x{msr:X}: a guest MSR write leaked across restore \ + (expected 0x{original:X}, got 0x{after:X})" + ), + Err(e) => panic!("0x{msr:X}: read-back after restore failed: {e:?}"), + } + exercised.push(msr); + } else if saw_write { + masked_only.push(msr); + } else { + read_only.push(msr); + } + } + + let fmt = |v: &[u32]| { + v.iter() + .map(|m| format!("0x{m:X}")) + .collect::>() + .join(", ") + }; + eprintln!( + "full-window MSR sweep: readable={readable} exercised={} masked_only={} read_only={}", + exercised.len(), + masked_only.len(), + read_only.len() + ); + eprintln!(" exercised: [{}]", fmt(&exercised)); + eprintln!(" masked_only: [{}]", fmt(&masked_only)); + eprintln!(" read_only: [{}]", fmt(&read_only)); + eprintln!(" free_running_leaked: [{}]", fmt(&free_running_leaked)); + assert!( + free_running_leaked.is_empty(), + "free-running MSRs not reset across restore on this backend: [{}]", + fmt(&free_running_leaked) + ); + assert!( + !exercised.is_empty(), + "sweep was vacuous: no guest MSR write ever retained a value that restore \ + then rolled back, so the rollback path was never exercised" + ); + } + + /// A host TSC write must reset the guest-visible MSHV counter. + #[test] + #[cfg(all(mshv3, target_arch = "x86_64"))] + fn mshv_host_tsc_writeback_resets_guest_tsc() { + use crate::hypervisor::virtual_machine::{HypervisorType, get_available_hypervisor}; + + if !matches!(get_available_hypervisor(), Some(HypervisorType::Mshv)) { + return; + } + + let mut sbox = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + None, + ) + .unwrap() + .evolve() + .unwrap(); + + let base = sbox.vm.capture_msrs_for_test(&[0x10]).unwrap()[0].value; + + let jump = base.wrapping_add(1 << 60); + sbox.call::<()>("WriteMSR", (0x10u32, jump)).unwrap(); + let planted: u64 = sbox.call("ReadMSR", 0x10u32).unwrap(); + assert!( + planted >= jump, + "guest TSC write did not take (planted=0x{planted:X} jump=0x{jump:X})" + ); + + assert!( + sbox.vm.try_set_msr_for_test(0x10, base), + "host set of HV_X64_REGISTER_TSC failed" + ); + + let after: u64 = sbox.call("ReadMSR", 0x10u32).unwrap(); + eprintln!( + "mshv TSC writeback probe: base=0x{base:X} jump=0x{jump:X} planted=0x{planted:X} after=0x{after:X}" + ); + assert!( + after < jump / 2, + "host TSC write-back did NOT reset the guest TSC (after=0x{after:X} still near \ + jump=0x{jump:X}); the reset approach is not viable on this host" + ); + } + } + /// Tests for [`MultiUseSandbox::from_snapshot`] in-memory. mod from_snapshot { use std::sync::Arc; diff --git a/src/hyperlight_host/src/sandbox/snapshot/file/config.rs b/src/hyperlight_host/src/sandbox/snapshot/file/config.rs index 9b8d0792c..982489948 100644 --- a/src/hyperlight_host/src/sandbox/snapshot/file/config.rs +++ b/src/hyperlight_host/src/sandbox/snapshot/file/config.rs @@ -21,6 +21,8 @@ use serde::{Deserialize, Serialize}; use super::media_types::SNAPSHOT_ABI_VERSION; use crate::hypervisor::regs::CommonSpecialRegisters; +#[cfg(target_arch = "x86_64")] +use crate::hypervisor::regs::MsrEntry; use crate::mem::layout::SandboxMemoryLayout; // --- Arch and hypervisor identifiers -------------------------------- @@ -180,6 +182,17 @@ pub(super) struct OciSnapshotConfig { /// Special registers captured from the paused vCPU, restored /// verbatim when resuming the call. pub(super) sregs: CommonSpecialRegisters, + /// MSR reset state captured from the paused vCPU. `None` uses the + /// destination sandbox's reset set. + #[cfg(target_arch = "x86_64")] + #[serde(default)] + pub(super) msrs: Option>, + /// Allow list of the sandbox that captured this snapshot. Present + /// exactly when `msrs` is, and checked against the destination allow + /// list on load. + #[cfg(target_arch = "x86_64")] + #[serde(default)] + pub(super) allowed_msrs: Option>, pub(super) layout: MemoryLayout, /// Total size of the memory blob in bytes (including the guest /// page-table tail, if any). Equal to `self.memory.mem_size()`. @@ -592,6 +605,49 @@ mod tests { assert_eq!(restored, expected); } + /// Captured MSRs survive the serde round-trip through the config, + /// including index and value for every entry. + #[cfg(target_arch = "x86_64")] + #[test] + fn msrs_round_trip_preserves_every_entry() { + let original = gating_config_with_msrs(Some(vec![ + MsrEntry { + index: 0xC000_0102, + value: 0xDEAD_BEEF, + }, + MsrEntry { + index: 0x10, + value: 0x1234_5678_9ABC_DEF0, + }, + ])); + let json = serde_json::to_vec(&original).unwrap(); + let restored: OciSnapshotConfig = serde_json::from_slice(&json).unwrap(); + assert_eq!(restored.msrs, original.msrs); + } + + /// A config JSON with no MSR keys deserializes both fields to `None`. + #[cfg(target_arch = "x86_64")] + #[test] + fn config_without_msr_keys_deserializes_to_none() { + let with = gating_config_with_msrs(Some(vec![MsrEntry { + index: 0x10, + value: 1, + }])); + let mut json: serde_json::Value = + serde_json::from_slice(&serde_json::to_vec(&with).unwrap()).unwrap(); + assert!(json.as_object_mut().unwrap().remove("msrs").is_some()); + assert!( + json.as_object_mut() + .unwrap() + .remove("allowed_msrs") + .is_some() + ); + + let restored: OciSnapshotConfig = serde_json::from_value(json).unwrap(); + assert_eq!(restored.msrs, None); + assert_eq!(restored.allowed_msrs, None); + } + /// Every `ParameterType` survives the round-trip through its serde /// mirror, guarding against a transposed variant in either match. #[test] @@ -676,6 +732,10 @@ mod tests { stack_top_gva: 0x2000, entrypoint_addr: SandboxMemoryLayout::BASE_ADDRESS as u64, sregs: distinct_sregs(), + #[cfg(target_arch = "x86_64")] + msrs: None, + #[cfg(target_arch = "x86_64")] + allowed_msrs: None, layout: MemoryLayout { input_data_size: 0, output_data_size: 0, @@ -693,6 +753,17 @@ mod tests { } } + /// `gating_config` with a chosen MSR set, for serde tests. + #[cfg(target_arch = "x86_64")] + fn gating_config_with_msrs(msrs: Option>) -> OciSnapshotConfig { + let allowed_msrs = msrs.as_ref().map(|_| Vec::new()); + OciSnapshotConfig { + msrs, + allowed_msrs, + ..gating_config() + } + } + /// A snapshot built for a different architecture is rejected. #[test] fn validate_for_load_rejects_arch_mismatch() { @@ -888,6 +959,20 @@ mod schema_pin { 11 ] }, + "msrs": [ + { + "index": 16, + "value": 42 + }, + { + "index": 3221225474, + "value": 3735928559 + } + ], + "allowed_msrs": [ + 16, + 3221225474 + ], "layout": { "input_data_size": 1, "output_data_size": 2, diff --git a/src/hyperlight_host/src/sandbox/snapshot/file/mod.rs b/src/hyperlight_host/src/sandbox/snapshot/file/mod.rs index 59fcd6372..c97aadc2b 100644 --- a/src/hyperlight_host/src/sandbox/snapshot/file/mod.rs +++ b/src/hyperlight_host/src/sandbox/snapshot/file/mod.rs @@ -585,6 +585,10 @@ impl Snapshot { stack_top_gva: self.stack_top_gva, entrypoint_addr, sregs: *sregs, + #[cfg(target_arch = "x86_64")] + msrs: self.msrs.clone(), + #[cfg(target_arch = "x86_64")] + allowed_msrs: self.allowed_msrs.clone(), layout: MemoryLayout { input_data_size: l.input_data_size(), output_data_size: l.output_data_size(), @@ -850,6 +854,15 @@ impl Snapshot { // 8. Build entrypoint + sregs back from the config. let entrypoint = NextAction::Call(cfg.entrypoint_addr); + // `msrs` and `allowed_msrs` travel together. A config with one but + // not the other cannot enforce the allow-list check on restore. + #[cfg(target_arch = "x86_64")] + if cfg.msrs.is_some() != cfg.allowed_msrs.is_some() { + return Err(crate::new_error!( + "snapshot config inconsistent: msrs and allowed_msrs must both be present or both absent" + )); + } + // 9. Reconstitute host_functions metadata. let snapshot_generation = cfg.snapshot_generation; let host_funcs_vec: Vec< @@ -871,6 +884,10 @@ impl Snapshot { load_info: crate::mem::exe::LoadInfo::dummy(), stack_top_gva: cfg.stack_top_gva, sregs: Some(cfg.sregs), + #[cfg(target_arch = "x86_64")] + msrs: cfg.msrs, + #[cfg(target_arch = "x86_64")] + allowed_msrs: cfg.allowed_msrs, entrypoint, snapshot_generation, host_functions, diff --git a/src/hyperlight_host/src/sandbox/snapshot/file_tests.rs b/src/hyperlight_host/src/sandbox/snapshot/file_tests.rs index 691f6f920..bcce218f0 100644 --- a/src/hyperlight_host/src/sandbox/snapshot/file_tests.rs +++ b/src/hyperlight_host/src/sandbox/snapshot/file_tests.rs @@ -207,6 +207,200 @@ fn snapshot_generation_round_trip() { assert_eq!(loaded3.snapshot_generation(), gen3); } +/// A disk round-trip preserves captured MSR reset state. +#[cfg(target_arch = "x86_64")] +#[test] +fn msrs_round_trip_via_disk() { + let snap = create_snapshot(); + let original = snap.msrs().cloned(); + assert!( + original.as_ref().is_some_and(|m| !m.is_empty()), + "a running snapshot should capture a non-empty MSR reset set" + ); + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("snap"); + snap.save(&path, &OciTag::new("latest").unwrap()).unwrap(); + + let loaded = Snapshot::checked_load(&path, OciTag::new("latest").unwrap()).unwrap(); + assert_eq!(loaded.msrs(), original.as_ref()); + // The allow list travels with the MSRs and is Some exactly when they are. + assert_eq!(loaded.allowed_msrs(), Some(&[][..])); +} + +/// A config with no `msrs` key loads and uses the destination reset set. +#[cfg(target_arch = "x86_64")] +#[test] +fn snapshot_without_msrs_key_loads_and_runs() { + let (_dir, path) = save_for_mutation(); + rewrite_config(&path, |cfg| { + let obj = cfg.as_object_mut().unwrap(); + assert!( + obj.remove("msrs").is_some(), + "a running x86_64 snapshot config should carry msrs to remove" + ); + assert!( + obj.remove("allowed_msrs").is_some(), + "a running x86_64 snapshot config should carry allowed_msrs to remove" + ); + }); + + let loaded = Snapshot::checked_load(&path, OciTag::new("latest").unwrap()).unwrap(); + assert_eq!(loaded.msrs(), None); + + let mut sbox = + MultiUseSandbox::from_snapshot(Arc::new(loaded), HostFunctions::default(), None).unwrap(); + assert_eq!(sbox.call::("GetStatic", ()).unwrap(), 0); +} + +/// A snapshot whose reset set includes an allowed MSR carries that +/// MSR's guest-written value across a disk round-trip. Loading with a +/// matching allow config restores the value, and restore-in-place +/// returns it after an overwrite. +#[cfg(target_arch = "x86_64")] +#[test] +fn disk_snapshot_restores_allowed_msr_value() { + const SYSENTER_CS: u32 = 0x174; + let sentinel: u64 = 0xDEAD_BEEF; + + let mut cfg = crate::sandbox::SandboxConfiguration::default(); + cfg.allow_msrs(&[SYSENTER_CS]).unwrap(); + let mut source = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().unwrap()), + Some(cfg), + ) + .unwrap() + .evolve() + .unwrap(); + source + .call::<()>("WriteMSR", (SYSENTER_CS, sentinel)) + .unwrap(); + let snap = source.snapshot().unwrap(); + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("snap"); + snap.save(&path, &OciTag::new("latest").unwrap()).unwrap(); + let loaded = Arc::new(Snapshot::checked_load(&path, OciTag::new("latest").unwrap()).unwrap()); + + let mut cfg = crate::sandbox::SandboxConfiguration::default(); + cfg.allow_msrs(&[SYSENTER_CS]).unwrap(); + let mut sbox = + MultiUseSandbox::from_snapshot(loaded.clone(), HostFunctions::default(), Some(cfg)) + .unwrap(); + assert_eq!(sbox.call::("ReadMSR", SYSENTER_CS).unwrap(), sentinel); + + sbox.call::<()>("WriteMSR", (SYSENTER_CS, sentinel ^ 0x55)) + .unwrap(); + sbox.restore(loaded).unwrap(); + assert_eq!(sbox.call::("ReadMSR", SYSENTER_CS).unwrap(), sentinel); +} + +/// A disk snapshot restores into a destination with a superset allow list. +#[cfg(target_arch = "x86_64")] +#[test] +fn disk_snapshot_restores_into_superset_allow_list() { + const SYSENTER_CS: u32 = 0x174; + const SYSENTER_ESP: u32 = 0x175; + let sentinel: u64 = 0xDEAD_BEEF; + + let mut cfg = crate::sandbox::SandboxConfiguration::default(); + cfg.allow_msrs(&[SYSENTER_CS]).unwrap(); + let mut source = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().unwrap()), + Some(cfg), + ) + .unwrap() + .evolve() + .unwrap(); + source + .call::<()>("WriteMSR", (SYSENTER_CS, sentinel)) + .unwrap(); + let snap = source.snapshot().unwrap(); + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("snap"); + snap.save(&path, &OciTag::new("latest").unwrap()).unwrap(); + let loaded = Arc::new(Snapshot::checked_load(&path, OciTag::new("latest").unwrap()).unwrap()); + + let mut dest_cfg = crate::sandbox::SandboxConfiguration::default(); + dest_cfg.allow_msrs(&[SYSENTER_CS, SYSENTER_ESP]).unwrap(); + let mut sbox = + MultiUseSandbox::from_snapshot(loaded, HostFunctions::default(), Some(dest_cfg)).unwrap(); + assert_eq!(sbox.call::("ReadMSR", SYSENTER_CS).unwrap(), sentinel); +} + +/// A disk snapshot is rejected when it allows an MSR the destination does not. +#[cfg(target_arch = "x86_64")] +#[test] +fn disk_snapshot_non_superset_allow_list_rejected() { + const SYSENTER_CS: u32 = 0x174; + let sentinel: u64 = 0x1234; + + let mut cfg = crate::sandbox::SandboxConfiguration::default(); + cfg.allow_msrs(&[SYSENTER_CS]).unwrap(); + let mut source = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().unwrap()), + Some(cfg), + ) + .unwrap() + .evolve() + .unwrap(); + source + .call::<()>("WriteMSR", (SYSENTER_CS, sentinel)) + .unwrap(); + let snap = source.snapshot().unwrap(); + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("snap"); + snap.save(&path, &OciTag::new("latest").unwrap()).unwrap(); + let loaded = Arc::new(Snapshot::checked_load(&path, OciTag::new("latest").unwrap()).unwrap()); + + // The destination allows nothing, so the snapshot's allowed MSR is not + // a subset of the destination allow list. + let err = MultiUseSandbox::from_snapshot(loaded.clone(), HostFunctions::default(), None) + .expect_err("loading into a non-superset allow list must fail"); + assert!( + format!("{err:?}").contains("SnapshotMsrNotAllowed"), + "expected an MSR allow-list mismatch, got: {err:?}" + ); + + let mut target = create_test_sandbox(); + let err = target + .restore(loaded) + .expect_err("restore into a non-superset allow list must fail"); + assert!( + format!("{err:?}").contains("SnapshotMsrNotAllowed"), + "expected an MSR allow-list mismatch, got: {err:?}" + ); + assert!(target.poisoned()); +} + +/// A config with `msrs` but no `allowed_msrs` cannot enforce the +/// allow-list check, so the load rejects it. +#[cfg(target_arch = "x86_64")] +#[test] +fn disk_snapshot_msrs_without_allow_list_rejected() { + let (_dir, path) = save_for_mutation(); + rewrite_config(&path, |cfg| { + assert!( + cfg.as_object_mut() + .unwrap() + .remove("allowed_msrs") + .is_some(), + "a running x86_64 snapshot config should carry allowed_msrs to remove" + ); + }); + + let err = unwrap_err_snapshot(Snapshot::checked_load( + &path, + OciTag::new("latest").unwrap(), + )); + assert!( + format!("{err:?}").contains("allowed_msrs"), + "expected an msrs/allowed_msrs consistency error, got: {err:?}" + ); +} + #[test] fn pre_init_snapshot_cannot_be_persisted() { let snap = Snapshot::from_env( diff --git a/src/hyperlight_host/src/sandbox/snapshot/mod.rs b/src/hyperlight_host/src/sandbox/snapshot/mod.rs index 9a638a98e..ac940cc59 100644 --- a/src/hyperlight_host/src/sandbox/snapshot/mod.rs +++ b/src/hyperlight_host/src/sandbox/snapshot/mod.rs @@ -32,6 +32,8 @@ use tracing::{Span, instrument}; use crate::Result; use crate::hypervisor::regs::CommonSpecialRegisters; +#[cfg(target_arch = "x86_64")] +use crate::hypervisor::regs::MsrEntry; use crate::mem::exe::{ExeInfo, LoadInfo}; use crate::mem::layout::SandboxMemoryLayout; use crate::mem::memory_region::{GuestMemoryRegion, MemoryRegion, MemoryRegionFlags}; @@ -92,6 +94,18 @@ pub struct Snapshot { /// tables are relocated during snapshot. sregs: Option, + /// MSR reset state captured from a running vCPU. + /// None for a pre-init snapshot or an old on-disk snapshot + /// predating MSR capture. + #[cfg(target_arch = "x86_64")] + msrs: Option>, + + /// The allow list of the sandbox that captured this snapshot, + /// sorted and deduplicated. Present exactly when `msrs` is present. + /// A restore requires the destination allow list to be a superset. + #[cfg(target_arch = "x86_64")] + allowed_msrs: Option>, + /// The next action that should be performed on this snapshot entrypoint: NextAction, @@ -381,6 +395,10 @@ impl Snapshot { load_info, stack_top_gva: exn_stack_top_gva, sregs: None, + #[cfg(target_arch = "x86_64")] + msrs: None, + #[cfg(target_arch = "x86_64")] + allowed_msrs: None, entrypoint: NextAction::Initialise(load_addr + entrypoint_va - base_va), snapshot_generation: 0, host_functions: HostFunctionDetails { @@ -407,6 +425,8 @@ impl Snapshot { root_pt_gpas: &[u64], stack_top_gva: u64, sregs: CommonSpecialRegisters, + #[cfg(target_arch = "x86_64")] msrs: Option>, + #[cfg(target_arch = "x86_64")] allowed_msrs: Option>, entrypoint: NextAction, snapshot_generation: u64, host_functions: HostFunctionDetails, @@ -560,6 +580,10 @@ impl Snapshot { load_info, stack_top_gva, sregs: Some(sregs), + #[cfg(target_arch = "x86_64")] + msrs, + #[cfg(target_arch = "x86_64")] + allowed_msrs, entrypoint, snapshot_generation, host_functions, @@ -603,6 +627,30 @@ impl Snapshot { self.sregs.as_ref() } + /// Returns the captured MSR reset state. + #[cfg(target_arch = "x86_64")] + pub(crate) fn msrs(&self) -> Option<&Vec> { + self.msrs.as_ref() + } + + /// Returns the capturing sandbox's allow list. + #[cfg(target_arch = "x86_64")] + pub(crate) fn allowed_msrs(&self) -> Option<&[u32]> { + self.allowed_msrs.as_deref() + } + + /// Stores captured MSR reset state. + #[cfg(all(test, target_arch = "x86_64"))] + pub(crate) fn set_msrs(&mut self, msrs: Option>) { + self.msrs = msrs; + } + + /// Stores the capturing sandbox's allow list. + #[cfg(all(test, target_arch = "x86_64"))] + pub(crate) fn set_allowed_msrs(&mut self, allowed: Option>) { + self.allowed_msrs = allowed; + } + pub(crate) fn entrypoint(&self) -> NextAction { self.entrypoint } @@ -761,6 +809,10 @@ mod tests { &[pt_base], 0, default_sregs(), + #[cfg(target_arch = "x86_64")] + None, + #[cfg(target_arch = "x86_64")] + None, super::NextAction::None, 1, HostFunctionDetails::default(), @@ -778,6 +830,10 @@ mod tests { &[pt_base], 0, default_sregs(), + #[cfg(target_arch = "x86_64")] + None, + #[cfg(target_arch = "x86_64")] + None, super::NextAction::None, 2, HostFunctionDetails::default(), diff --git a/src/tests/rust_guests/simpleguest/src/main.rs b/src/tests/rust_guests/simpleguest/src/main.rs index 2da2eb7ca..efe6badab 100644 --- a/src/tests/rust_guests/simpleguest/src/main.rs +++ b/src/tests/rust_guests/simpleguest/src/main.rs @@ -1095,6 +1095,87 @@ fn call_host_expect_error(hostfuncname: String) -> Result<()> { Ok(()) } +#[guest_function("ReadMSR")] +#[cfg(target_arch = "x86_64")] +fn read_msr(msr: u32) -> u64 { + let (read_eax, read_edx): (u32, u32); + // SAFETY: The test guest runs at CPL0, and the operands declare every register used. + unsafe { + core::arch::asm!( + "rdmsr", + in("ecx") msr, + out("eax") read_eax, + out("edx") read_edx, + options(nostack, nomem) + ); + } + ((read_edx as u64) << 32) | (read_eax as u64) +} + +#[guest_function("WriteKernelGsBaseViaSwapgs")] +#[cfg(target_arch = "x86_64")] +fn write_kernel_gs_base_via_swapgs(value: u64) { + // SAFETY: The test guest runs at CPL0, and CR4 is restored before returning. + unsafe { + core::arch::asm!( + "mov {original_cr4}, cr4", + "mov {enabled_cr4}, {original_cr4}", + "or {enabled_cr4}, {fsgsbase}", + "mov cr4, {enabled_cr4}", + "wrgsbase {value}", + "swapgs", + "mov cr4, {original_cr4}", + original_cr4 = out(reg) _, + enabled_cr4 = out(reg) _, + fsgsbase = const 1 << 16, + value = in(reg) value, + options(nostack) + ); + } +} + +#[guest_function("ReadKernelGsBaseViaSwapgs")] +#[cfg(target_arch = "x86_64")] +fn read_kernel_gs_base_via_swapgs() -> u64 { + let value: u64; + // SAFETY: The test guest runs at CPL0, and SWAPGS and CR4 are restored before returning. + unsafe { + core::arch::asm!( + "mov {original_cr4}, cr4", + "mov {enabled_cr4}, {original_cr4}", + "or {enabled_cr4}, {fsgsbase}", + "mov cr4, {enabled_cr4}", + "swapgs", + "rdgsbase {value}", + "swapgs", + "mov cr4, {original_cr4}", + original_cr4 = out(reg) _, + enabled_cr4 = out(reg) _, + value = lateout(reg) value, + fsgsbase = const 1 << 16, + options(nostack) + ); + } + value +} + +#[guest_function("WriteMSR")] +#[cfg(target_arch = "x86_64")] +fn write_msr(msr: u32, value: u64) { + let eax = (value & 0xFFFFFFFF) as u32; + let edx = ((value >> 32) & 0xFFFFFFFF) as u32; + // SAFETY: The test guest runs at CPL0, and the operands declare every register used. + unsafe { + core::arch::asm!( + "wrmsr", + in("ecx") msr, + in("eax") eax, + in("edx") edx, + options(nostack, nomem) + ); + } +} + #[hyperlight_guest_bin::main] #[instrument(skip_all, parent = Span::current(), level= "Trace")] fn main() {