Enable debugging on windows

docs/how-to-debug-a-hyperlight-guest.md

@@ -1,12 +1,12 @@
-# How to debug a Hyperlight guest using gdb on Linux
+# How to debug a Hyperlight guest using gdb or lldb
 
-Hyperlight supports gdb debugging of a **KVM** or **MSHV** guest running inside a Hyperlight sandbox on Linux.
+Hyperlight supports gdb debugging of a guest running inside a Hyperlight sandbox on Linux or Windows.
 When Hyperlight is compiled with the `gdb` feature enabled, a Hyperlight sandbox can be configured
 to start listening for a gdb connection.
 
 ## Supported features
 
-The Hyperlight `gdb` feature enables **KVM** and **MSHV** guest debugging to:
+The Hyperlight `gdb` feature enables guest debugging to:
    - stop at an entry point breakpoint which is automatically set by Hyperlight
    - add and remove HW breakpoints (maximum 4 set breakpoints at a time)
    - add and remove SW breakpoints
@@ -18,7 +18,7 @@ The Hyperlight `gdb` feature enables **KVM** and **MSHV** guest debugging to:
 ## Expected behavior
 
 Below is a list describing some cases of expected behavior from a gdb debug 
-session of a guest binary running inside a Hyperlight sandbox on Linux.
+session of a guest binary running inside a Hyperlight sandbox.
 
 - when the `gdb` feature is enabled and a SandboxConfiguration is provided a
   debug port, the created sandbox will wait for a gdb client to connect on the

src/hyperlight_host/Cargo.toml

@@ -21,6 +21,8 @@ bench = false # see https://bheisler.github.io/criterion.rs/book/faq.html#cargo-
 workspace = true
 
 [dependencies]
+gdbstub = { version = "0.7.5", optional = true }
+gdbstub_arch = { version = "0.3.1", optional = true }
 goblin = { version = "0.9" }
 rand = { version = "0.9" }
 cfg-if = { version = "1.0.0" }
@@ -65,8 +67,6 @@ windows-version = "0.1"
 lazy_static = "1.4.0"
 
 [target.'cfg(unix)'.dependencies]
-gdbstub = { version = "0.7.5", optional = true }
-gdbstub_arch = { version = "0.3.1", optional = true }
 seccompiler = { version = "0.5.0", optional = true }
 kvm-bindings = { version = "0.11", features = ["fam-wrappers"], optional = true }
 kvm-ioctls = { version = "0.21", optional = true }

src/hyperlight_host/build.rs

@@ -89,7 +89,7 @@ fn main() -> Result<()> {
     // Essentially the kvm and mshv features are ignored on windows as long as you use #[cfg(kvm)] and not #[cfg(feature = "kvm")].
     // You should never use #[cfg(feature = "kvm")] or #[cfg(feature = "mshv")] in the codebase.
     cfg_aliases::cfg_aliases! {
-        gdb: { all(feature = "gdb", debug_assertions, any(feature = "kvm", feature = "mshv2", feature = "mshv3"), target_os = "linux") },
+        gdb: { all(feature = "gdb", debug_assertions, any(feature = "kvm", feature = "mshv2", feature = "mshv3")) },
         kvm: { all(feature = "kvm", target_os = "linux") },
         mshv: { all(any(feature = "mshv2", feature = "mshv3"), target_os = "linux") },
         // inprocess feature is aliased with debug_assertions to make it only available in debug-builds.

src/hyperlight_host/src/hypervisor/gdb/event_loop.rs

@@ -19,6 +19,7 @@ use gdbstub::conn::ConnectionExt;
 use gdbstub::stub::{
     run_blocking, BaseStopReason, DisconnectReason, GdbStub, SingleThreadStopReason,
 };
+#[cfg(target_os = "linux")]
 use libc::{pthread_kill, SIGRTMIN};
 
 use super::x86_64_target::HyperlightSandboxTarget;
@@ -49,15 +50,28 @@ impl run_blocking::BlockingEventLoop for GdbBlockingEventLoop {
                     // Resume execution if unknown reason for stop
                     let stop_response = match stop_reason {
                         VcpuStopReason::DoneStep => BaseStopReason::DoneStep,
-                        VcpuStopReason::EntryPointBp => BaseStopReason::HwBreak(()),
                         VcpuStopReason::SwBp => BaseStopReason::SwBreak(()),
                         VcpuStopReason::HwBp => BaseStopReason::HwBreak(()),
+                        VcpuStopReason::EntryPointBp => BaseStopReason::HwBreak(()),
                         // This is a consequence of the GDB client sending an interrupt signal
                         // to the target thread
-                        VcpuStopReason::Interrupt => BaseStopReason::SignalWithThread {
-                            tid: (),
-                            signal: Signal(SIGRTMIN() as u8),
-                        },
+                        VcpuStopReason::Interrupt => {
+                            #[cfg(target_os = "linux")]
+                            let rsp = BaseStopReason::SignalWithThread {
+                                tid: (),
+                                signal: Signal(SIGRTMIN() as u8),
+                            };
+
+                            // For Windows we don't send a signal, this should not be happen
+                            #[cfg(target_os = "windows")]
+                            let rsp = BaseStopReason::SignalWithThread {
+                                tid: (),
+                                // This is a placeholder signal, we don't have a real signal
+                                signal: Signal(53u8),
+                            };
+
+                            rsp
+                        }
                         VcpuStopReason::Unknown => {
                             log::warn!("Unknown stop reason received");
 
@@ -96,13 +110,27 @@ impl run_blocking::BlockingEventLoop for GdbBlockingEventLoop {
     /// This function is called when the GDB client sends an interrupt signal.
     /// Passing `None` defers sending a stop reason to later (e.g. when the target stops).
     fn on_interrupt(
-        target: &mut Self::Target,
+        #[allow(unused_variables)] target: &mut Self::Target,
     ) -> Result<Option<Self::StopReason>, <Self::Target as gdbstub::target::Target>::Error> {
         log::info!("Received interrupt from GDB client - sending signal to target thread");
 
         // Send a signal to the target thread to interrupt it
+        #[cfg(target_os = "linux")]
         let ret = unsafe { pthread_kill(target.get_thread_id(), SIGRTMIN()) };
 
+        #[cfg(target_os = "windows")]
+        let ret = {
+            // pthread_kill is not implemented on Windows
+            // We need to use a different method to send a signal to the target thread
+            // For now, we just log a warning and return an error
+            // NOTE: The way we make a vCPU stop on windows is to make a windows API call
+            // to suspend the thread, but for that we need a handle to the thread which we don't have
+            log::warn!("Windows signal sending not implemented");
+
+            // For now, we just return an error
+            -1
+        };
+
         log::info!("pthread_kill returned {}", ret);
 
         if ret < 0 && ret != libc::ESRCH {

src/hyperlight_host/src/hypervisor/gdb/hyperv_debug.rs

@@ -0,0 +1,255 @@
+/*
+Copyright 2024 The Hyperlight Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+use std::collections::HashMap;
+
+use windows::Win32::System::Hypervisor::WHV_VP_EXCEPTION_CONTEXT;
+
+use super::arch::{vcpu_stop_reason, MAX_NO_OF_HW_BP};
+use super::{GuestDebug, VcpuStopReason, X86_64Regs, SW_BP_SIZE};
+use crate::hypervisor::windows_hypervisor_platform::VMProcessor;
+use crate::hypervisor::wrappers::{WHvDebugRegisters, WHvGeneralRegisters};
+use crate::{new_error, HyperlightError, Result};
+
+/// KVM Debug struct
+/// This struct is used to abstract the internal details of the kvm
+/// guest debugging settings
+#[derive(Default)]
+pub(crate) struct HypervDebug {
+    /// vCPU stepping state
+    single_step: bool,
+
+    /// Array of addresses for HW breakpoints
+    hw_breakpoints: Vec<u64>,
+    /// Saves the bytes modified to enable SW breakpoints
+    sw_breakpoints: HashMap<u64, [u8; SW_BP_SIZE]>,
+
+    /// Debug registers
+    dbg_cfg: WHvDebugRegisters,
+}
+
+impl HypervDebug {
+    pub(crate) fn new() -> Self {
+        Self {
+            single_step: false,
+            hw_breakpoints: vec![],
+            sw_breakpoints: HashMap::new(),
+            dbg_cfg: WHvDebugRegisters::default(),
+        }
+    }
+
+    /// Returns the instruction pointer from the stopped vCPU
+    fn get_instruction_pointer(&self, vcpu_fd: &VMProcessor) -> Result<u64> {
+        let regs = vcpu_fd
+            .get_regs()
+            .map_err(|e| new_error!("Could not retrieve registers from vCPU: {:?}", e))?;
+
+        Ok(regs.rip)
+    }
+
+    /// This method sets the kvm debugreg fields to enable breakpoints at
+    /// specific addresses
+    ///
+    /// The first 4 debug registers are used to set the addresses
+    /// The 4th and 5th debug registers are obsolete and not used
+    /// The 7th debug register is used to enable the breakpoints
+    /// For more information see: DEBUG REGISTERS chapter in the architecture
+    /// manual
+    fn set_debug_config(&mut self, vcpu_fd: &VMProcessor, step: bool) -> Result<()> {
+        let addrs = &self.hw_breakpoints;
+
+        let mut dbg_cfg = WHvDebugRegisters::default();
+
+        for (k, addr) in addrs.iter().enumerate() {
+            match k {
+                0 => {
+                    dbg_cfg.dr0 = *addr;
+                }
+                1 => {
+                    dbg_cfg.dr1 = *addr;
+                }
+                2 => {
+                    dbg_cfg.dr2 = *addr;
+                }
+                3 => {
+                    dbg_cfg.dr3 = *addr;
+                }
+                _ => {
+                    Err(new_error!("Tried to set more than 4 HW breakpoints"))?;
+                }
+            }
+            dbg_cfg.dr7 |= 1 << (k * 2);
+        }
+
+        self.dbg_cfg = dbg_cfg;
+
+        vcpu_fd
+            .set_debug_regs(&self.dbg_cfg)
+            .map_err(|e| new_error!("Could not set guest debug: {:?}", e))?;
+
+        self.single_step = step;
+
+        let mut regs = vcpu_fd
+            .get_regs()
+            .map_err(|e| new_error!("Could not get registers: {:?}", e))?;
+
+        // Set TF Flag to enable Traps
+        if self.single_step {
+            regs.rflags |= 1 << 8; // Set the TF flag
+        } else {
+            regs.rflags &= !(1 << 8); // Clear the TF flag
+        }
+
+        vcpu_fd
+            .set_general_purpose_registers(&regs)
+            .map_err(|e| new_error!("Could not set guest registers: {:?}", e))?;
+
+        Ok(())
+    }
+
+    /// Get the reason the vCPU has stopped
+    pub(crate) fn get_stop_reason(
+        &mut self,
+        vcpu_fd: &VMProcessor,
+        exception: WHV_VP_EXCEPTION_CONTEXT,
+        entrypoint: u64,
+    ) -> Result<VcpuStopReason> {
+        let rip = self.get_instruction_pointer(vcpu_fd)?;
+        let rip = self.translate_gva(vcpu_fd, rip)?;
+
+        let debug_regs = vcpu_fd
+            .get_debug_regs()
+            .map_err(|e| new_error!("Could not retrieve registers from vCPU: {:?}", e))?;
+
+        // Check if the vCPU stopped because of a hardware breakpoint
+        let reason = vcpu_stop_reason(
+            self.single_step,
+            rip,
+            debug_regs.dr6,
+            entrypoint,
+            exception.ExceptionType as u32,
+            &self.hw_breakpoints,
+            &self.sw_breakpoints,
+        );
+
+        if let VcpuStopReason::EntryPointBp = reason {
+            // In case the hw breakpoint is the entry point, remove it to
+            // avoid hanging here as gdb does not remove breakpoints it
+            // has not set.
+            // Gdb expects the target to be stopped when connected.
+            self.remove_hw_breakpoint(vcpu_fd, entrypoint)?;
+        }
+
+        Ok(reason)
+    }
+}
+
+impl GuestDebug for HypervDebug {
+    type Vcpu = VMProcessor;
+
+    fn is_hw_breakpoint(&self, addr: &u64) -> bool {
+        self.hw_breakpoints.contains(addr)
+    }
+    fn is_sw_breakpoint(&self, addr: &u64) -> bool {
+        self.sw_breakpoints.contains_key(addr)
+    }
+    fn save_hw_breakpoint(&mut self, addr: &u64) -> bool {
+        if self.hw_breakpoints.len() >= MAX_NO_OF_HW_BP {
+            false
+        } else {
+            self.hw_breakpoints.push(*addr);
+
+            true
+        }
+    }
+    fn save_sw_breakpoint_data(&mut self, addr: u64, data: [u8; 1]) {
+        _ = self.sw_breakpoints.insert(addr, data);
+    }
+    fn delete_hw_breakpoint(&mut self, addr: &u64) {
+        self.hw_breakpoints.retain(|&a| a != *addr);
+    }
+    fn delete_sw_breakpoint_data(&mut self, addr: &u64) -> Option<[u8; 1]> {
+        self.sw_breakpoints.remove(addr)
+    }
+
+    fn read_regs(&self, vcpu_fd: &Self::Vcpu, regs: &mut X86_64Regs) -> Result<()> {
+        log::debug!("Read registers");
+        let vcpu_regs = vcpu_fd
+            .get_regs()
+            .map_err(|e| new_error!("Could not read guest registers: {:?}", e))?;
+
+        regs.rax = vcpu_regs.rax;
+        regs.rbx = vcpu_regs.rbx;
+        regs.rcx = vcpu_regs.rcx;
+        regs.rdx = vcpu_regs.rdx;
+        regs.rsi = vcpu_regs.rsi;
+        regs.rdi = vcpu_regs.rdi;
+        regs.rbp = vcpu_regs.rbp;
+        regs.rsp = vcpu_regs.rsp;
+        regs.r8 = vcpu_regs.r8;
+        regs.r9 = vcpu_regs.r9;
+        regs.r10 = vcpu_regs.r10;
+        regs.r11 = vcpu_regs.r11;
+        regs.r12 = vcpu_regs.r12;
+        regs.r13 = vcpu_regs.r13;
+        regs.r14 = vcpu_regs.r14;
+        regs.r15 = vcpu_regs.r15;
+
+        regs.rip = vcpu_regs.rip;
+        regs.rflags = vcpu_regs.rflags;
+
+        Ok(())
+    }
+
+    fn set_single_step(&mut self, vcpu_fd: &Self::Vcpu, enable: bool) -> Result<()> {
+        self.set_debug_config(vcpu_fd, enable)
+    }
+
+    fn translate_gva(&self, vcpu_fd: &Self::Vcpu, gva: u64) -> Result<u64> {
+        vcpu_fd
+            .translate_gva(gva)
+            .map_err(|_| HyperlightError::TranslateGuestAddress(gva))
+    }
+
+    fn write_regs(&self, vcpu_fd: &Self::Vcpu, regs: &X86_64Regs) -> Result<()> {
+        log::debug!("Write registers");
+        let regs = WHvGeneralRegisters {
+            rax: regs.rax,
+            rbx: regs.rbx,
+            rcx: regs.rcx,
+            rdx: regs.rdx,
+            rsi: regs.rsi,
+            rdi: regs.rdi,
+            rbp: regs.rbp,
+            rsp: regs.rsp,
+            r8: regs.r8,
+            r9: regs.r9,
+            r10: regs.r10,
+            r11: regs.r11,
+            r12: regs.r12,
+            r13: regs.r13,
+            r14: regs.r14,
+            r15: regs.r15,
+
+            rip: regs.rip,
+            rflags: regs.rflags,
+        };
+
+        vcpu_fd
+            .set_general_purpose_registers(&regs)
+            .map_err(|e| new_error!("Could not write guest registers: {:?}", e))
+    }
+}