From f2fa50bedb93d1c49bc57e9e421a26ed11c4f054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= Date: Thu, 8 May 2025 15:16:04 +0300 Subject: [PATCH 1/3] gdb: add support for windows debugging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Doru Blânzeanu --- docs/how-to-debug-a-hyperlight-guest.md | 8 +- src/hyperlight_host/Cargo.toml | 4 +- src/hyperlight_host/build.rs | 2 +- .../src/hypervisor/gdb/event_loop.rs | 16 +- .../src/hypervisor/gdb/hyperv_debug.rs | 255 ++++++++++++++ src/hyperlight_host/src/hypervisor/gdb/mod.rs | 6 +- .../src/hypervisor/gdb/x86_64_target.rs | 2 + .../src/hypervisor/hyperv_windows.rs | 322 +++++++++++++++++- .../src/hypervisor/hypervisor_handler.rs | 10 +- .../hypervisor/windows_hypervisor_platform.rs | 160 ++++++++- .../src/hypervisor/wrappers.rs | 12 + 11 files changed, 777 insertions(+), 20 deletions(-) create mode 100644 src/hyperlight_host/src/hypervisor/gdb/hyperv_debug.rs diff --git a/docs/how-to-debug-a-hyperlight-guest.md b/docs/how-to-debug-a-hyperlight-guest.md index abf571062..5f7db51bd 100644 --- a/docs/how-to-debug-a-hyperlight-guest.md +++ b/docs/how-to-debug-a-hyperlight-guest.md @@ -1,12 +1,12 @@ -# How to debug a Hyperlight guest using gdb on Linux +# How to debug a Hyperlight guest using gdb or lldb -Hyperlight supports gdb debugging of a **KVM** or **MSHV** guest running inside a Hyperlight sandbox on Linux. +Hyperlight supports gdb debugging of a guest running inside a Hyperlight sandbox on Linux or Windows. When Hyperlight is compiled with the `gdb` feature enabled, a Hyperlight sandbox can be configured to start listening for a gdb connection. ## Supported features -The Hyperlight `gdb` feature enables **KVM** and **MSHV** guest debugging to: +The Hyperlight `gdb` feature enables guest debugging to: - stop at an entry point breakpoint which is automatically set by Hyperlight - add and remove HW breakpoints (maximum 4 set breakpoints at a time) - add and remove SW breakpoints @@ -18,7 +18,7 @@ The Hyperlight `gdb` feature enables **KVM** and **MSHV** guest debugging to: ## Expected behavior Below is a list describing some cases of expected behavior from a gdb debug -session of a guest binary running inside a Hyperlight sandbox on Linux. +session of a guest binary running inside a Hyperlight sandbox. - when the `gdb` feature is enabled and a SandboxConfiguration is provided a debug port, the created sandbox will wait for a gdb client to connect on the diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index f5a4de2ee..bac68f599 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -21,6 +21,8 @@ bench = false # see https://bheisler.github.io/criterion.rs/book/faq.html#cargo- workspace = true [dependencies] +gdbstub = { version = "0.7.5", optional = true } +gdbstub_arch = { version = "0.3.1", optional = true } goblin = { version = "0.9" } rand = { version = "0.9" } cfg-if = { version = "1.0.0" } @@ -65,8 +67,6 @@ windows-version = "0.1" lazy_static = "1.4.0" [target.'cfg(unix)'.dependencies] -gdbstub = { version = "0.7.5", optional = true } -gdbstub_arch = { version = "0.3.1", optional = true } seccompiler = { version = "0.5.0", optional = true } kvm-bindings = { version = "0.11", features = ["fam-wrappers"], optional = true } kvm-ioctls = { version = "0.21", optional = true } diff --git a/src/hyperlight_host/build.rs b/src/hyperlight_host/build.rs index e82f75d9a..82253c6be 100644 --- a/src/hyperlight_host/build.rs +++ b/src/hyperlight_host/build.rs @@ -89,7 +89,7 @@ fn main() -> Result<()> { // Essentially the kvm and mshv features are ignored on windows as long as you use #[cfg(kvm)] and not #[cfg(feature = "kvm")]. // You should never use #[cfg(feature = "kvm")] or #[cfg(feature = "mshv")] in the codebase. cfg_aliases::cfg_aliases! { - gdb: { all(feature = "gdb", debug_assertions, any(feature = "kvm", feature = "mshv2", feature = "mshv3"), target_os = "linux") }, + gdb: { all(feature = "gdb", debug_assertions, any(feature = "kvm", feature = "mshv2", feature = "mshv3")) }, kvm: { all(feature = "kvm", target_os = "linux") }, mshv: { all(any(feature = "mshv2", feature = "mshv3"), target_os = "linux") }, // inprocess feature is aliased with debug_assertions to make it only available in debug-builds. diff --git a/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs b/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs index c21c66042..1956a2197 100644 --- a/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs +++ b/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs @@ -19,6 +19,7 @@ use gdbstub::conn::ConnectionExt; use gdbstub::stub::{ run_blocking, BaseStopReason, DisconnectReason, GdbStub, SingleThreadStopReason, }; +#[cfg(target_os = "linux")] use libc::{pthread_kill, SIGRTMIN}; use super::x86_64_target::HyperlightSandboxTarget; @@ -49,14 +50,18 @@ impl run_blocking::BlockingEventLoop for GdbBlockingEventLoop { // Resume execution if unknown reason for stop let stop_response = match stop_reason { VcpuStopReason::DoneStep => BaseStopReason::DoneStep, - VcpuStopReason::EntryPointBp => BaseStopReason::HwBreak(()), VcpuStopReason::SwBp => BaseStopReason::SwBreak(()), VcpuStopReason::HwBp => BaseStopReason::HwBreak(()), + VcpuStopReason::EntryPointBp => BaseStopReason::HwBreak(()), // This is a consequence of the GDB client sending an interrupt signal // to the target thread VcpuStopReason::Interrupt => BaseStopReason::SignalWithThread { tid: (), + #[cfg(target_os = "linux")] signal: Signal(SIGRTMIN() as u8), + #[cfg(target_os = "windows")] + // TODO: Handle the signal properly + signal: Signal(53u8), }, VcpuStopReason::Unknown => { log::warn!("Unknown stop reason received"); @@ -96,13 +101,20 @@ impl run_blocking::BlockingEventLoop for GdbBlockingEventLoop { /// This function is called when the GDB client sends an interrupt signal. /// Passing `None` defers sending a stop reason to later (e.g. when the target stops). fn on_interrupt( - target: &mut Self::Target, + #[allow(unused_variables)] target: &mut Self::Target, ) -> Result, ::Error> { log::info!("Received interrupt from GDB client - sending signal to target thread"); // Send a signal to the target thread to interrupt it + #[cfg(target_os = "linux")] let ret = unsafe { pthread_kill(target.get_thread_id(), SIGRTMIN()) }; + #[cfg(target_os = "windows")] + let ret = { + // TODO: Implement Windows signal sending + libc::ESRCH + }; + log::info!("pthread_kill returned {}", ret); if ret < 0 && ret != libc::ESRCH { diff --git a/src/hyperlight_host/src/hypervisor/gdb/hyperv_debug.rs b/src/hyperlight_host/src/hypervisor/gdb/hyperv_debug.rs new file mode 100644 index 000000000..0cd6f7067 --- /dev/null +++ b/src/hyperlight_host/src/hypervisor/gdb/hyperv_debug.rs @@ -0,0 +1,255 @@ +/* +Copyright 2024 The Hyperlight Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +use std::collections::HashMap; + +use windows::Win32::System::Hypervisor::WHV_VP_EXCEPTION_CONTEXT; + +use super::arch::{vcpu_stop_reason, MAX_NO_OF_HW_BP}; +use super::{GuestDebug, VcpuStopReason, X86_64Regs, SW_BP_SIZE}; +use crate::hypervisor::windows_hypervisor_platform::VMProcessor; +use crate::hypervisor::wrappers::{WHvDebugRegisters, WHvGeneralRegisters}; +use crate::{new_error, HyperlightError, Result}; + +/// KVM Debug struct +/// This struct is used to abstract the internal details of the kvm +/// guest debugging settings +#[derive(Default)] +pub(crate) struct HypervDebug { + /// vCPU stepping state + single_step: bool, + + /// Array of addresses for HW breakpoints + hw_breakpoints: Vec, + /// Saves the bytes modified to enable SW breakpoints + sw_breakpoints: HashMap, + + /// Debug registers + dbg_cfg: WHvDebugRegisters, +} + +impl HypervDebug { + pub(crate) fn new() -> Self { + Self { + single_step: false, + hw_breakpoints: vec![], + sw_breakpoints: HashMap::new(), + dbg_cfg: WHvDebugRegisters::default(), + } + } + + /// Returns the instruction pointer from the stopped vCPU + fn get_instruction_pointer(&self, vcpu_fd: &VMProcessor) -> Result { + let regs = vcpu_fd + .get_regs() + .map_err(|e| new_error!("Could not retrieve registers from vCPU: {:?}", e))?; + + Ok(regs.rip) + } + + /// This method sets the kvm debugreg fields to enable breakpoints at + /// specific addresses + /// + /// The first 4 debug registers are used to set the addresses + /// The 4th and 5th debug registers are obsolete and not used + /// The 7th debug register is used to enable the breakpoints + /// For more information see: DEBUG REGISTERS chapter in the architecture + /// manual + fn set_debug_config(&mut self, vcpu_fd: &VMProcessor, step: bool) -> Result<()> { + let addrs = &self.hw_breakpoints; + + let mut dbg_cfg = WHvDebugRegisters::default(); + + for (k, addr) in addrs.iter().enumerate() { + match k { + 0 => { + dbg_cfg.dr0 = *addr; + } + 1 => { + dbg_cfg.dr1 = *addr; + } + 2 => { + dbg_cfg.dr2 = *addr; + } + 3 => { + dbg_cfg.dr3 = *addr; + } + _ => { + Err(new_error!("Tried to set more than 4 HW breakpoints"))?; + } + } + dbg_cfg.dr7 |= 1 << (k * 2); + } + + self.dbg_cfg = dbg_cfg; + + vcpu_fd + .set_debug_regs(&self.dbg_cfg) + .map_err(|e| new_error!("Could not set guest debug: {:?}", e))?; + + self.single_step = step; + + let mut regs = vcpu_fd + .get_regs() + .map_err(|e| new_error!("Could not get registers: {:?}", e))?; + + // Set TF Flag to enable Traps + if self.single_step { + regs.rflags |= 1 << 8; // Set the TF flag + } else { + regs.rflags &= !(1 << 8); // Clear the TF flag + } + + vcpu_fd + .set_general_purpose_registers(®s) + .map_err(|e| new_error!("Could not set guest registers: {:?}", e))?; + + Ok(()) + } + + /// Get the reason the vCPU has stopped + pub(crate) fn get_stop_reason( + &mut self, + vcpu_fd: &VMProcessor, + exception: WHV_VP_EXCEPTION_CONTEXT, + entrypoint: u64, + ) -> Result { + let rip = self.get_instruction_pointer(vcpu_fd)?; + let rip = self.translate_gva(vcpu_fd, rip)?; + + let debug_regs = vcpu_fd + .get_debug_regs() + .map_err(|e| new_error!("Could not retrieve registers from vCPU: {:?}", e))?; + + // Check if the vCPU stopped because of a hardware breakpoint + let reason = vcpu_stop_reason( + self.single_step, + rip, + debug_regs.dr6, + entrypoint, + exception.ExceptionType as u32, + &self.hw_breakpoints, + &self.sw_breakpoints, + ); + + if let VcpuStopReason::EntryPointBp = reason { + // In case the hw breakpoint is the entry point, remove it to + // avoid hanging here as gdb does not remove breakpoints it + // has not set. + // Gdb expects the target to be stopped when connected. + self.remove_hw_breakpoint(vcpu_fd, entrypoint)?; + } + + Ok(reason) + } +} + +impl GuestDebug for HypervDebug { + type Vcpu = VMProcessor; + + fn is_hw_breakpoint(&self, addr: &u64) -> bool { + self.hw_breakpoints.contains(addr) + } + fn is_sw_breakpoint(&self, addr: &u64) -> bool { + self.sw_breakpoints.contains_key(addr) + } + fn save_hw_breakpoint(&mut self, addr: &u64) -> bool { + if self.hw_breakpoints.len() >= MAX_NO_OF_HW_BP { + false + } else { + self.hw_breakpoints.push(*addr); + + true + } + } + fn save_sw_breakpoint_data(&mut self, addr: u64, data: [u8; 1]) { + _ = self.sw_breakpoints.insert(addr, data); + } + fn delete_hw_breakpoint(&mut self, addr: &u64) { + self.hw_breakpoints.retain(|&a| a != *addr); + } + fn delete_sw_breakpoint_data(&mut self, addr: &u64) -> Option<[u8; 1]> { + self.sw_breakpoints.remove(addr) + } + + fn read_regs(&self, vcpu_fd: &Self::Vcpu, regs: &mut X86_64Regs) -> Result<()> { + log::debug!("Read registers"); + let vcpu_regs = vcpu_fd + .get_regs() + .map_err(|e| new_error!("Could not read guest registers: {:?}", e))?; + + regs.rax = vcpu_regs.rax; + regs.rbx = vcpu_regs.rbx; + regs.rcx = vcpu_regs.rcx; + regs.rdx = vcpu_regs.rdx; + regs.rsi = vcpu_regs.rsi; + regs.rdi = vcpu_regs.rdi; + regs.rbp = vcpu_regs.rbp; + regs.rsp = vcpu_regs.rsp; + regs.r8 = vcpu_regs.r8; + regs.r9 = vcpu_regs.r9; + regs.r10 = vcpu_regs.r10; + regs.r11 = vcpu_regs.r11; + regs.r12 = vcpu_regs.r12; + regs.r13 = vcpu_regs.r13; + regs.r14 = vcpu_regs.r14; + regs.r15 = vcpu_regs.r15; + + regs.rip = vcpu_regs.rip; + regs.rflags = vcpu_regs.rflags; + + Ok(()) + } + + fn set_single_step(&mut self, vcpu_fd: &Self::Vcpu, enable: bool) -> Result<()> { + self.set_debug_config(vcpu_fd, enable) + } + + fn translate_gva(&self, vcpu_fd: &Self::Vcpu, gva: u64) -> Result { + vcpu_fd + .translate_gva(gva) + .map_err(|_| HyperlightError::TranslateGuestAddress(gva)) + } + + fn write_regs(&self, vcpu_fd: &Self::Vcpu, regs: &X86_64Regs) -> Result<()> { + log::debug!("Write registers"); + let regs = WHvGeneralRegisters { + rax: regs.rax, + rbx: regs.rbx, + rcx: regs.rcx, + rdx: regs.rdx, + rsi: regs.rsi, + rdi: regs.rdi, + rbp: regs.rbp, + rsp: regs.rsp, + r8: regs.r8, + r9: regs.r9, + r10: regs.r10, + r11: regs.r11, + r12: regs.r12, + r13: regs.r13, + r14: regs.r14, + r15: regs.r15, + + rip: regs.rip, + rflags: regs.rflags, + }; + + vcpu_fd + .set_general_purpose_registers(®s) + .map_err(|e| new_error!("Could not write guest registers: {:?}", e)) + } +} diff --git a/src/hyperlight_host/src/hypervisor/gdb/mod.rs b/src/hyperlight_host/src/hypervisor/gdb/mod.rs index 46c26d1dc..dc9fde634 100644 --- a/src/hyperlight_host/src/hypervisor/gdb/mod.rs +++ b/src/hyperlight_host/src/hypervisor/gdb/mod.rs @@ -16,6 +16,8 @@ limitations under the License. mod arch; mod event_loop; +#[cfg(target_os = "windows")] +mod hyperv_debug; #[cfg(kvm)] mod kvm_debug; #[cfg(mshv)] @@ -34,6 +36,8 @@ use gdbstub::conn::ConnectionExt; use gdbstub::stub::GdbStub; use gdbstub::target::TargetError; use hyperlight_common::mem::PAGE_SIZE; +#[cfg(target_os = "windows")] +pub(crate) use hyperv_debug::HypervDebug; #[cfg(kvm)] pub(crate) use kvm_debug::KvmDebug; #[cfg(mshv)] @@ -156,7 +160,7 @@ pub(crate) enum DebugResponse { } /// This trait is used to define common debugging functionality for Hypervisors -pub(crate) trait GuestDebug { +pub(super) trait GuestDebug { /// Type that wraps the vCPU functionality type Vcpu; diff --git a/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs b/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs index a25098bf4..944667e4a 100644 --- a/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs +++ b/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs @@ -36,6 +36,7 @@ pub(crate) struct HyperlightSandboxTarget { /// Hypervisor communication channels hyp_conn: DebugCommChannel, /// Thread ID + #[allow(dead_code)] thread_id: u64, } @@ -61,6 +62,7 @@ impl HyperlightSandboxTarget { } /// Returns the thread ID + #[allow(dead_code)] pub(crate) fn get_thread_id(&self) -> u64 { self.thread_id } diff --git a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs index 44256d1fe..b1e7d2a90 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs @@ -27,10 +27,16 @@ use windows::Win32::System::Hypervisor::{ WHV_MEMORY_ACCESS_TYPE, WHV_PARTITION_HANDLE, WHV_REGISTER_VALUE, WHV_RUN_VP_EXIT_CONTEXT, WHV_RUN_VP_EXIT_REASON, WHV_X64_SEGMENT_REGISTER, WHV_X64_SEGMENT_REGISTER_0, }; +#[cfg(gdb)] +use { + super::gdb::{DebugCommChannel, DebugMsg, DebugResponse, GuestDebug, HypervDebug}, + super::handlers::DbgMemAccessHandlerWrapper, + crate::hypervisor::handlers::DbgMemAccessHandlerCaller, + crate::{log_then_return, HyperlightError}, + std::sync::{Arc, Mutex}, +}; use super::fpu::{FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT}; -#[cfg(gdb)] -use super::handlers::DbgMemAccessHandlerWrapper; use super::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper}; use super::surrogate_process::SurrogateProcess; use super::surrogate_process_manager::*; @@ -47,6 +53,211 @@ use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags}; use crate::mem::ptr::{GuestPtr, RawPtr}; use crate::{debug, new_error, Result}; +#[cfg(gdb)] +mod debug { + use std::sync::{Arc, Mutex}; + + use windows::Win32::System::Hypervisor::WHV_VP_EXCEPTION_CONTEXT; + + use super::{HypervWindowsDriver, *}; + use crate::hypervisor::gdb::{DebugMsg, DebugResponse, VcpuStopReason, X86_64Regs}; + use crate::hypervisor::handlers::DbgMemAccessHandlerCaller; + use crate::{new_error, Result}; + + impl HypervWindowsDriver { + /// Resets the debug information to disable debugging + fn disable_debug(&mut self) -> Result<()> { + let mut debug = HypervDebug::default(); + + debug.set_single_step(&self.processor, false)?; + + self.debug = Some(debug); + + Ok(()) + } + + /// Get the reason the vCPU has stopped + pub(crate) fn get_stop_reason( + &mut self, + exception: WHV_VP_EXCEPTION_CONTEXT, + ) -> Result { + let debug = self + .debug + .as_mut() + .ok_or_else(|| new_error!("Debug is not enabled"))?; + + debug.get_stop_reason(&self.processor, exception, self.entrypoint) + } + + pub(crate) fn process_dbg_request( + &mut self, + req: DebugMsg, + dbg_mem_access_fn: Arc>, + ) -> Result { + if let Some(debug) = self.debug.as_mut() { + match req { + DebugMsg::AddHwBreakpoint(addr) => Ok(DebugResponse::AddHwBreakpoint( + debug + .add_hw_breakpoint(&self.processor, addr) + .map_err(|e| { + log::error!("Failed to add hw breakpoint: {:?}", e); + + e + }) + .is_ok(), + )), + DebugMsg::AddSwBreakpoint(addr) => Ok(DebugResponse::AddSwBreakpoint( + debug + .add_sw_breakpoint(&self.processor, addr, dbg_mem_access_fn) + .map_err(|e| { + log::error!("Failed to add sw breakpoint: {:?}", e); + + e + }) + .is_ok(), + )), + DebugMsg::Continue => { + debug.set_single_step(&self.processor, false).map_err(|e| { + log::error!("Failed to continue execution: {:?}", e); + + e + })?; + + Ok(DebugResponse::Continue) + } + DebugMsg::DisableDebug => { + self.disable_debug().map_err(|e| { + log::error!("Failed to disable debugging: {:?}", e); + + e + })?; + + Ok(DebugResponse::DisableDebug) + } + DebugMsg::GetCodeSectionOffset => { + let offset = dbg_mem_access_fn + .try_lock() + .map_err(|e| { + new_error!("Error locking at {}:{}: {}", file!(), line!(), e) + })? + .get_code_offset() + .map_err(|e| { + log::error!("Failed to get code offset: {:?}", e); + + e + })?; + + Ok(DebugResponse::GetCodeSectionOffset(offset as u64)) + } + DebugMsg::ReadAddr(addr, len) => { + let mut data = vec![0u8; len]; + + debug + .read_addrs(&self.processor, addr, &mut data, dbg_mem_access_fn) + .map_err(|e| { + log::error!("Failed to read from address: {:?}", e); + + e + })?; + + Ok(DebugResponse::ReadAddr(data)) + } + DebugMsg::ReadRegisters => { + let mut regs = X86_64Regs::default(); + + debug + .read_regs(&self.processor, &mut regs) + .map_err(|e| { + log::error!("Failed to read registers: {:?}", e); + + e + }) + .map(|_| DebugResponse::ReadRegisters(regs)) + } + DebugMsg::RemoveHwBreakpoint(addr) => Ok(DebugResponse::RemoveHwBreakpoint( + debug + .remove_hw_breakpoint(&self.processor, addr) + .map_err(|e| { + log::error!("Failed to remove hw breakpoint: {:?}", e); + + e + }) + .is_ok(), + )), + DebugMsg::RemoveSwBreakpoint(addr) => Ok(DebugResponse::RemoveSwBreakpoint( + debug + .remove_sw_breakpoint(&self.processor, addr, dbg_mem_access_fn) + .map_err(|e| { + log::error!("Failed to remove sw breakpoint: {:?}", e); + + e + }) + .is_ok(), + )), + DebugMsg::Step => { + debug.set_single_step(&self.processor, true).map_err(|e| { + log::error!("Failed to enable step instruction: {:?}", e); + + e + })?; + + Ok(DebugResponse::Step) + } + DebugMsg::WriteAddr(addr, data) => { + debug + .write_addrs(&self.processor, addr, &data, dbg_mem_access_fn) + .map_err(|e| { + log::error!("Failed to write to address: {:?}", e); + + e + })?; + + Ok(DebugResponse::WriteAddr) + } + DebugMsg::WriteRegisters(regs) => debug + .write_regs(&self.processor, ®s) + .map_err(|e| { + log::error!("Failed to write registers: {:?}", e); + + e + }) + .map(|_| DebugResponse::WriteRegisters), + } + } else { + Err(new_error!("Debugging is not enabled")) + } + } + + pub(crate) fn recv_dbg_msg(&mut self) -> Result { + let gdb_conn = self + .gdb_conn + .as_mut() + .ok_or_else(|| new_error!("Debug is not enabled"))?; + + gdb_conn.recv().map_err(|e| { + new_error!( + "Got an error while waiting to receive a + message: {:?}", + e + ) + }) + } + + pub(crate) fn send_dbg_msg(&mut self, cmd: DebugResponse) -> Result<()> { + log::debug!("Sending {:?}", cmd); + + let gdb_conn = self + .gdb_conn + .as_mut() + .ok_or_else(|| new_error!("Debug is not enabled"))?; + + gdb_conn + .send(cmd) + .map_err(|e| new_error!("Got an error while sending a response message {:?}", e)) + } + } +} + /// A Hypervisor driver for HyperV-on-Windows. pub(crate) struct HypervWindowsDriver { size: usize, // this is the size of the memory region, excluding the 2 surrounding guard pages @@ -56,6 +267,10 @@ pub(crate) struct HypervWindowsDriver { entrypoint: u64, orig_rsp: GuestPtr, mem_regions: Vec, + #[cfg(gdb)] + debug: Option, + #[cfg(gdb)] + gdb_conn: Option>, } /* This does not automatically impl Send/Sync because the host * address of the shared memory region is a raw pointer, which are @@ -66,6 +281,7 @@ unsafe impl Send for HypervWindowsDriver {} unsafe impl Sync for HypervWindowsDriver {} impl HypervWindowsDriver { + #[allow(clippy::too_many_arguments)] #[instrument(err(Debug), skip_all, parent = Span::current(), level = "Trace")] pub(crate) fn new( mem_regions: Vec, @@ -75,6 +291,8 @@ impl HypervWindowsDriver { entrypoint: u64, rsp: u64, mmap_file_handle: HandleWrapper, + + #[cfg(gdb)] gdb_conn: Option>, ) -> Result { // create and setup hypervisor partition let mut partition = VMPartition::new(1)?; @@ -91,6 +309,16 @@ impl HypervWindowsDriver { let mut proc = VMProcessor::new(partition)?; Self::setup_initial_sregs(&mut proc, pml4_address)?; + #[cfg(gdb)] + let (debug, gdb_conn) = if let Some(gdb_conn) = gdb_conn { + let mut debug = HypervDebug::new(); + debug.add_hw_breakpoint(&proc, entrypoint)?; + + (Some(debug), Some(gdb_conn)) + } else { + (None, None) + }; + // subtract 2 pages for the guard pages, since when we copy memory to and from surrogate process, // we don't want to copy the guard pages themselves (that would cause access violation) let mem_size = raw_size - 2 * PAGE_SIZE_USIZE; @@ -102,6 +330,10 @@ impl HypervWindowsDriver { entrypoint, orig_rsp: GuestPtr::try_from(RawPtr::from(rsp))?, mem_regions, + #[cfg(gdb)] + debug, + #[cfg(gdb)] + gdb_conn, }) } @@ -461,6 +693,18 @@ impl Hypervisor for HypervWindowsDriver { debug!("HyperV Cancelled Details :\n {:#?}", &self); HyperlightExit::Cancelled() } + #[cfg(gdb)] + WHV_RUN_VP_EXIT_REASON(4098i32) => { + // Get information about the exception that triggered the exit + let exception = unsafe { exit_context.Anonymous.VpException }; + + match self.get_stop_reason(exception) { + Ok(reason) => HyperlightExit::Debug(reason), + Err(e) => { + log_then_return!("Error getting stop reason: {}", e); + } + } + } WHV_RUN_VP_EXIT_REASON(_) => { debug!( "HyperV Unexpected Exit Details :#nReason {:#?}\n {:#?}", @@ -489,6 +733,49 @@ impl Hypervisor for HypervWindowsDriver { fn get_memory_regions(&self) -> &[MemoryRegion] { &self.mem_regions } + + #[cfg(gdb)] + fn handle_debug( + &mut self, + dbg_mem_access_fn: Arc>, + stop_reason: super::gdb::VcpuStopReason, + ) -> Result<()> { + self.send_dbg_msg(DebugResponse::VcpuStopped(stop_reason)) + .map_err(|e| new_error!("Couldn't signal vCPU stopped event to GDB thread: {:?}", e))?; + + loop { + log::debug!("Debug wait for event to resume vCPU"); + + // Wait for a message from gdb + let req = self.recv_dbg_msg()?; + + let result = self.process_dbg_request(req, dbg_mem_access_fn.clone()); + + let response = match result { + Ok(response) => response, + // Treat non fatal errors separately so the guest doesn't fail + Err(HyperlightError::TranslateGuestAddress(_)) => DebugResponse::ErrorOccurred, + Err(e) => { + return Err(e); + } + }; + + // If the command was either step or continue, we need to run the vcpu + let cont = matches!( + response, + DebugResponse::Step | DebugResponse::Continue | DebugResponse::DisableDebug + ); + + self.send_dbg_msg(response) + .map_err(|e| new_error!("Couldn't send response to gdb: {:?}", e))?; + + if cont { + break; + } + } + + Ok(()) + } } #[cfg(test)] @@ -497,10 +784,30 @@ pub mod tests { use serial_test::serial; + #[cfg(gdb)] + use crate::hypervisor::handlers::DbgMemAccessHandlerCaller; use crate::hypervisor::handlers::{MemAccessHandler, OutBHandler}; use crate::hypervisor::tests::test_initialise; use crate::Result; + #[cfg(gdb)] + struct DbgMemAccessHandler {} + + #[cfg(gdb)] + impl DbgMemAccessHandlerCaller for DbgMemAccessHandler { + fn read(&mut self, _offset: usize, _data: &mut [u8]) -> Result<()> { + Ok(()) + } + + fn write(&mut self, _offset: usize, _data: &[u8]) -> Result<()> { + Ok(()) + } + + fn get_code_offset(&mut self) -> Result { + Ok(0) + } + } + #[test] #[serial] fn test_init() { @@ -513,6 +820,15 @@ pub mod tests { let func: Box Result<()> + Send> = Box::new(|| -> Result<()> { Ok(()) }); Arc::new(Mutex::new(MemAccessHandler::from(func))) }; - test_initialise(outb_handler, mem_access_handler).unwrap(); + #[cfg(gdb)] + let dbg_mem_access_handler = Arc::new(Mutex::new(DbgMemAccessHandler {})); + + test_initialise( + outb_handler, + mem_access_handler, + #[cfg(gdb)] + dbg_mem_access_handler, + ) + .unwrap(); } } diff --git a/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs b/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs index 591f43a07..9fbc8a768 100644 --- a/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs +++ b/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs @@ -898,7 +898,13 @@ fn set_up_hypervisor_partition( // This is only done when the hypervisor is not in-process #[cfg(gdb)] let gdb_conn = if let Some(DebugInfo { port }) = debug_info { - let gdb_conn = create_gdb_thread(*port, unsafe { pthread_self() }); + #[cfg(target_os = "linux")] + let thread_id = unsafe { pthread_self() }; + #[cfg(target_os = "windows")] + // TODO: Implement Windows thread id retrieval + let thread_id = 0; + + let gdb_conn = create_gdb_thread(*port, thread_id); // in case the gdb thread creation fails, we still want to continue // without gdb @@ -954,6 +960,8 @@ fn set_up_hypervisor_partition( entrypoint_ptr.absolute()?, rsp_ptr.absolute()?, HandleWrapper::from(mmap_file_handle), + #[cfg(gdb)] + gdb_conn, )?; Ok(Box::new(hv)) } diff --git a/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs b/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs index 47e200d38..62974a80a 100644 --- a/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs +++ b/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs @@ -24,6 +24,8 @@ use windows::Win32::System::LibraryLoader::*; use windows_result::HRESULT; use super::wrappers::HandleWrapper; +#[cfg(gdb)] +use crate::hypervisor::wrappers::WHvDebugRegisters; use crate::hypervisor::wrappers::{WHvFPURegisters, WHvGeneralRegisters, WHvSpecialRegisters}; use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags}; use crate::{new_error, Result}; @@ -64,10 +66,16 @@ pub(crate) fn is_hypervisor_present() -> bool { pub(super) struct VMPartition(WHV_PARTITION_HANDLE); impl VMPartition { + /// This is the position of the extended vm exit in partition property + #[cfg(gdb)] + const EXTENDED_VM_EXIT_POS: u32 = 2; + #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")] pub(super) fn new(proc_count: u32) -> Result { let hdl = unsafe { WHvCreatePartition() }?; Self::set_processor_count(&hdl, proc_count)?; + #[cfg(gdb)] + Self::set_extended_vm_exits(&hdl)?; unsafe { WHvSetupPartition(hdl) }?; Ok(Self(hdl)) } @@ -89,6 +97,56 @@ impl VMPartition { Ok(()) } + /// Sets up the debugging exception interception for the partition + /// This is needed for a HyperV partition to be able to intercept debug traps and breakpoints + /// Steps: + /// - set the extended VM exits property to enable extended VM exits + /// - set the exception exit bitmap to include debug trap and breakpoint trap + #[cfg(gdb)] + #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")] + pub fn set_extended_vm_exits(partition_handle: &WHV_PARTITION_HANDLE) -> Result<()> { + let mut property: WHV_PARTITION_PROPERTY = Default::default(); + + // Set the extended VM exits property + property.ExtendedVmExits.AsUINT64 = 1 << Self::EXTENDED_VM_EXIT_POS; + Self::set_property( + partition_handle, + WHvPartitionPropertyCodeExtendedVmExits, + &property, + )?; + + // Set the exception exit bitmap to include debug trap and breakpoint trap + property = Default::default(); + property.ExceptionExitBitmap = (1 << WHvX64ExceptionTypeDebugTrapOrFault.0) + | (1 << WHvX64ExceptionTypeBreakpointTrap.0); + Self::set_property( + partition_handle, + WHvPartitionPropertyCodeExceptionExitBitmap, + &property, + )?; + + Ok(()) + } + + /// Helper function to set partition properties + #[cfg(gdb)] + fn set_property( + partition_handle: &WHV_PARTITION_HANDLE, + property_code: WHV_PARTITION_PROPERTY_CODE, + property: &WHV_PARTITION_PROPERTY, + ) -> Result<()> { + unsafe { + WHvSetPartitionProperty( + *partition_handle, + property_code, + property as *const _ as *const c_void, + std::mem::size_of::() as u32, + )?; + } + + Ok(()) + } + #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")] pub(super) fn map_gpa_range( &mut self, @@ -205,6 +263,31 @@ impl VMProcessor { Ok(Self(part)) } + /// This function is used to translate a guest virtual address to a guest physical address + #[cfg(gdb)] + pub(super) fn translate_gva(&self, gva: u64) -> Result { + let partition_handle = self.get_partition_hdl(); + let mut gpa = 0; + let mut result = WHV_TRANSLATE_GVA_RESULT::default(); + + // Only validate read access because the write access is handled through the + // host memory mapping + let translateflags = WHvTranslateGvaFlagValidateRead; + + unsafe { + WHvTranslateGva( + partition_handle, + 0, + gva, + translateflags, + &mut result, + &mut gpa, + )?; + } + + Ok(gpa) + } + #[instrument(skip_all, parent = Span::current(), level= "Trace")] pub(super) fn get_partition_hdl(&self) -> WHV_PARTITION_HANDLE { let part = &self.0; @@ -213,7 +296,7 @@ impl VMProcessor { #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")] pub(super) fn set_registers( - &mut self, + &self, registers: &[(WHV_REGISTER_NAME, WHV_REGISTER_VALUE)], ) -> Result<()> { let partition_handle = self.get_partition_hdl(); @@ -300,10 +383,7 @@ impl VMProcessor { // Sets the registers for the VMProcessor to the given general purpose registers. // If you want to set other registers, use `set_registers` instead. - pub(super) fn set_general_purpose_registers( - &mut self, - regs: &WHvGeneralRegisters, - ) -> Result<()> { + pub(super) fn set_general_purpose_registers(&self, regs: &WHvGeneralRegisters) -> Result<()> { const LEN: usize = 18; let names: [WHV_REGISTER_NAME; LEN] = [ @@ -416,7 +496,75 @@ impl VMProcessor { } } - pub(super) fn set_fpu(&mut self, regs: &WHvFPURegisters) -> Result<()> { + #[cfg(gdb)] + pub(super) fn set_debug_regs(&self, regs: &WHvDebugRegisters) -> Result<()> { + const LEN: usize = 6; + + let names: [WHV_REGISTER_NAME; LEN] = [ + WHvX64RegisterDr0, + WHvX64RegisterDr1, + WHvX64RegisterDr2, + WHvX64RegisterDr3, + WHvX64RegisterDr6, + WHvX64RegisterDr7, + ]; + + let values: [WHV_REGISTER_VALUE; LEN] = [ + WHV_REGISTER_VALUE { Reg64: regs.dr0 }, + WHV_REGISTER_VALUE { Reg64: regs.dr1 }, + WHV_REGISTER_VALUE { Reg64: regs.dr2 }, + WHV_REGISTER_VALUE { Reg64: regs.dr3 }, + WHV_REGISTER_VALUE { Reg64: regs.dr6 }, + WHV_REGISTER_VALUE { Reg64: regs.dr7 }, + ]; + + unsafe { + WHvSetVirtualProcessorRegisters( + self.get_partition_hdl(), + 0, + names.as_ptr(), + LEN as u32, + values.as_ptr(), + )? + } + + Ok(()) + } + + #[cfg(gdb)] + pub(super) fn get_debug_regs(&self) -> Result { + const LEN: usize = 6; + + let names: [WHV_REGISTER_NAME; LEN] = [ + WHvX64RegisterDr0, + WHvX64RegisterDr1, + WHvX64RegisterDr2, + WHvX64RegisterDr3, + WHvX64RegisterDr6, + WHvX64RegisterDr7, + ]; + + let mut out: [WHV_REGISTER_VALUE; LEN] = unsafe { std::mem::zeroed() }; + unsafe { + WHvGetVirtualProcessorRegisters( + self.get_partition_hdl(), + 0, + names.as_ptr(), + LEN as u32, + out.as_mut_ptr(), + )?; + Ok(WHvDebugRegisters { + dr0: out[0].Reg64, + dr1: out[1].Reg64, + dr2: out[2].Reg64, + dr3: out[3].Reg64, + dr6: out[4].Reg64, + dr7: out[5].Reg64, + }) + } + } + + pub(super) fn set_fpu(&self, regs: &WHvFPURegisters) -> Result<()> { const LEN: usize = 26; let names: [WHV_REGISTER_NAME; LEN] = [ diff --git a/src/hyperlight_host/src/hypervisor/wrappers.rs b/src/hyperlight_host/src/hypervisor/wrappers.rs index aa9943d6a..4b26694e6 100644 --- a/src/hyperlight_host/src/hypervisor/wrappers.rs +++ b/src/hyperlight_host/src/hypervisor/wrappers.rs @@ -80,6 +80,18 @@ pub(super) struct WHvGeneralRegisters { pub rflags: u64, } +/// only used on widos for handling debug registers with the VMProcessor +#[cfg(gdb)] +#[derive(Debug, Default, Copy, Clone, PartialEq)] +pub(super) struct WHvDebugRegisters { + pub dr0: u64, + pub dr1: u64, + pub dr2: u64, + pub dr3: u64, + pub dr6: u64, + pub dr7: u64, +} + #[derive(Debug, Default, Copy, Clone, PartialEq)] pub(super) struct WHvFPURegisters { pub xmm0: u128, From 5dc23137c7f49c9b1dfb478f2aede26f0d00965e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= Date: Thu, 8 May 2025 17:21:48 +0300 Subject: [PATCH 2/3] gdb: weird thing with windows crashing from hv call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Doru Blânzeanu --- .../src/hypervisor/windows_hypervisor_platform.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs b/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs index 62974a80a..a2bfd4dd4 100644 --- a/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs +++ b/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs @@ -518,14 +518,19 @@ impl VMProcessor { WHV_REGISTER_VALUE { Reg64: regs.dr7 }, ]; - unsafe { + let ret_val = unsafe { WHvSetVirtualProcessorRegisters( self.get_partition_hdl(), 0, names.as_ptr(), LEN as u32, values.as_ptr(), - )? + ) + }; + + if ret_val.is_err() { + println!("Failed to set debug registers: {:?}", ret_val); + return Err(new_error!("Call to WHvSetVirtualProcessorRegisters failed")); } Ok(()) From 480fa717388605751e22a1099bc2ec81c2987e11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= Date: Fri, 9 May 2025 14:55:19 +0300 Subject: [PATCH 3/3] gdb: update comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Doru Blânzeanu --- .../src/hypervisor/gdb/event_loop.rs | 32 ++++++++++---- .../src/hypervisor/hypervisor_handler.rs | 4 +- .../hypervisor/windows_hypervisor_platform.rs | 42 ++++--------------- 3 files changed, 35 insertions(+), 43 deletions(-) diff --git a/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs b/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs index 1956a2197..708f16a00 100644 --- a/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs +++ b/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs @@ -55,14 +55,23 @@ impl run_blocking::BlockingEventLoop for GdbBlockingEventLoop { VcpuStopReason::EntryPointBp => BaseStopReason::HwBreak(()), // This is a consequence of the GDB client sending an interrupt signal // to the target thread - VcpuStopReason::Interrupt => BaseStopReason::SignalWithThread { - tid: (), + VcpuStopReason::Interrupt => { #[cfg(target_os = "linux")] - signal: Signal(SIGRTMIN() as u8), + let rsp = BaseStopReason::SignalWithThread { + tid: (), + signal: Signal(SIGRTMIN() as u8), + }; + + // For Windows we don't send a signal, this should not be happen #[cfg(target_os = "windows")] - // TODO: Handle the signal properly - signal: Signal(53u8), - }, + let rsp = BaseStopReason::SignalWithThread { + tid: (), + // This is a placeholder signal, we don't have a real signal + signal: Signal(53u8), + }; + + rsp + } VcpuStopReason::Unknown => { log::warn!("Unknown stop reason received"); @@ -111,8 +120,15 @@ impl run_blocking::BlockingEventLoop for GdbBlockingEventLoop { #[cfg(target_os = "windows")] let ret = { - // TODO: Implement Windows signal sending - libc::ESRCH + // pthread_kill is not implemented on Windows + // We need to use a different method to send a signal to the target thread + // For now, we just log a warning and return an error + // NOTE: The way we make a vCPU stop on windows is to make a windows API call + // to suspend the thread, but for that we need a handle to the thread which we don't have + log::warn!("Windows signal sending not implemented"); + + // For now, we just return an error + -1 }; log::info!("pthread_kill returned {}", ret); diff --git a/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs b/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs index 9fbc8a768..2f1ffd0d4 100644 --- a/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs +++ b/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs @@ -901,7 +901,9 @@ fn set_up_hypervisor_partition( #[cfg(target_os = "linux")] let thread_id = unsafe { pthread_self() }; #[cfg(target_os = "windows")] - // TODO: Implement Windows thread id retrieval + // On Windows, we need to get the thread id from the thread handle because + // we cannot send a signal to the thread without a handle + // This is a placeholder, as we don't have the actual thread handle here let thread_id = 0; let gdb_conn = create_gdb_thread(*port, thread_id); diff --git a/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs b/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs index a2bfd4dd4..3dbe36047 100644 --- a/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs +++ b/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs @@ -498,42 +498,16 @@ impl VMProcessor { #[cfg(gdb)] pub(super) fn set_debug_regs(&self, regs: &WHvDebugRegisters) -> Result<()> { - const LEN: usize = 6; - - let names: [WHV_REGISTER_NAME; LEN] = [ - WHvX64RegisterDr0, - WHvX64RegisterDr1, - WHvX64RegisterDr2, - WHvX64RegisterDr3, - WHvX64RegisterDr6, - WHvX64RegisterDr7, - ]; - - let values: [WHV_REGISTER_VALUE; LEN] = [ - WHV_REGISTER_VALUE { Reg64: regs.dr0 }, - WHV_REGISTER_VALUE { Reg64: regs.dr1 }, - WHV_REGISTER_VALUE { Reg64: regs.dr2 }, - WHV_REGISTER_VALUE { Reg64: regs.dr3 }, - WHV_REGISTER_VALUE { Reg64: regs.dr6 }, - WHV_REGISTER_VALUE { Reg64: regs.dr7 }, + let registers = vec![ + (WHvX64RegisterDr0, WHV_REGISTER_VALUE { Reg64: regs.dr0 }), + (WHvX64RegisterDr1, WHV_REGISTER_VALUE { Reg64: regs.dr1 }), + (WHvX64RegisterDr2, WHV_REGISTER_VALUE { Reg64: regs.dr2 }), + (WHvX64RegisterDr3, WHV_REGISTER_VALUE { Reg64: regs.dr3 }), + (WHvX64RegisterDr6, WHV_REGISTER_VALUE { Reg64: regs.dr6 }), + (WHvX64RegisterDr7, WHV_REGISTER_VALUE { Reg64: regs.dr7 }), ]; - let ret_val = unsafe { - WHvSetVirtualProcessorRegisters( - self.get_partition_hdl(), - 0, - names.as_ptr(), - LEN as u32, - values.as_ptr(), - ) - }; - - if ret_val.is_err() { - println!("Failed to set debug registers: {:?}", ret_val); - return Err(new_error!("Call to WHvSetVirtualProcessorRegisters failed")); - } - - Ok(()) + self.set_registers(®isters) } #[cfg(gdb)]