Merge #231

231: Migrate to naked functions r=stlankes a=mkroening

Fixes #222.

Co-authored-by: Martin Kröning <[email protected]>

Author: bors[bot]

src/arch/mod.rs

@@ -62,21 +62,12 @@ pub use crate::arch::x86_64::kernel::processor;
 #[cfg(target_arch = "x86_64")]
 pub use crate::arch::x86_64::kernel::scheduler;
 #[cfg(target_arch = "x86_64")]
+pub use crate::arch::x86_64::kernel::switch;
+#[cfg(target_arch = "x86_64")]
 pub use crate::arch::x86_64::kernel::systemtime::get_boot_time;
 #[cfg(all(target_arch = "x86_64", target_os = "hermit"))]
 pub use crate::arch::x86_64::kernel::{boot_application_processors, boot_processor_init};
 #[cfg(target_arch = "x86_64")]
 pub use crate::arch::x86_64::kernel::{
 	get_processor_count, message_output_init, output_message_buf, output_message_byte,
 };
-
-#[cfg(test)]
-pub unsafe fn switch_to_task(_old_stack: *mut usize, _new_stack: usize) {}
-#[cfg(test)]
-pub unsafe fn switch_to_fpu_owner(_old_stack: *mut usize, _new_stack: usize) {}
-
-#[cfg(not(test))]
-extern "C" {
-	pub fn switch_to_task(old_stack: *mut usize, new_stack: usize);
-	pub fn switch_to_fpu_owner(old_stack: *mut usize, new_stack: usize);
-}

src/arch/x86_64/kernel/gdt.rs

@@ -135,8 +135,6 @@ pub fn add_current_core() {
 	}
 }
 
-#[inline(never)]
-#[no_mangle]
-pub fn set_current_kernel_stack() {
+pub extern "C" fn set_current_kernel_stack() {
 	core_scheduler().set_current_kernel_stack();
 }

src/arch/x86_64/kernel/mod.rs

@@ -42,17 +42,13 @@ pub mod scheduler;
 pub mod serial;
 #[cfg(feature = "smp")]
 mod smp_boot_code;
+#[cfg(not(test))]
+mod start;
+pub mod switch;
 pub mod systemtime;
 #[cfg(feature = "vga")]
 mod vga;
 
-#[cfg(not(test))]
-global_asm!(include_str!("start.s"));
-#[cfg(all(not(test), not(feature = "fsgsbase")))]
-global_asm!(include_str!("switch.s"));
-#[cfg(all(not(test), feature = "fsgsbase"))]
-global_asm!(include_str!("switch_fsgsbase.s"));
-
 const SERIAL_PORT_BAUDRATE: u32 = 115_200;
 
 /// Map between Core ID and per-core scheduler
@@ -89,6 +85,44 @@ pub struct BootInfo {
 	hcmask: [u8; 4],
 }
 
+impl BootInfo {
+	const LAYOUT: Self = Self {
+		magic_number: 0,
+		version: 0,
+		base: 0,
+		limit: 0,
+		image_size: 0,
+		tls_start: 0,
+		tls_filesz: 0,
+		tls_memsz: 0,
+		current_stack_address: 0,
+		current_percore_address: 0,
+		host_logical_addr: 0,
+		boot_gtod: 0,
+		mb_info: 0,
+		cmdline: 0,
+		cmdsize: 0,
+		cpu_freq: 0,
+		boot_processor: 0,
+		cpu_online: 0,
+		possible_cpus: 0,
+		current_boot_id: 0,
+		uartport: 0,
+		single_kernel: 0,
+		uhyve: 0,
+		hcip: [0; 4],
+		hcgateway: [0; 4],
+		hcmask: [0; 4],
+	};
+
+	pub const fn current_stack_address_offset() -> isize {
+		let layout = Self::LAYOUT;
+		let start = ptr::addr_of!(layout);
+		let stack = ptr::addr_of!(layout.current_stack_address);
+		unsafe { stack.cast::<u8>().offset_from(start.cast()) }
+	}
+}
+
 /// Kernel header to announce machine features
 #[cfg(not(target_os = "hermit"))]
 static mut BOOT_INFO: *mut BootInfo = ptr::null_mut();

src/arch/x86_64/kernel/scheduler.rs

@@ -84,6 +84,11 @@ pub enum TaskStacks {
 }
 
 impl TaskStacks {
+	/// Size of the debug marker at the very top of each stack.
+	///
+	/// We have a marker at the very top of the stack for debugging (`0xdeadbeef`), which should not be overridden.
+	pub const MARKER_SIZE: usize = 0x10;
+
 	pub fn new(size: usize) -> TaskStacks {
 		let user_stack_size = if size < KERNEL_STACK_SIZE {
 			KERNEL_STACK_SIZE
@@ -148,9 +153,11 @@ impl TaskStacks {
 
 	pub fn from_boot_stacks() -> TaskStacks {
 		let tss = unsafe { &(*PERCORE.tss.get()) };
-		let stack = VirtAddr::from_usize(tss.rsp[0] as usize + 0x10 - KERNEL_STACK_SIZE);
+		let stack =
+			VirtAddr::from_usize(tss.rsp[0] as usize + Self::MARKER_SIZE - KERNEL_STACK_SIZE);
 		debug!("Using boot stack {:#X}", stack);
-		let ist0 = VirtAddr::from_usize(tss.ist[0] as usize + 0x10 - KERNEL_STACK_SIZE);
+		let ist0 =
+			VirtAddr::from_usize(tss.ist[0] as usize + Self::MARKER_SIZE - KERNEL_STACK_SIZE);
 		debug!("IST0 is located at {:#X}", ist0);
 
 		TaskStacks::Boot(BootStack { stack, ist0 })
@@ -327,15 +334,28 @@ impl Clone for TaskTLS {
 }
 
 #[cfg(not(target_os = "hermit"))]
-extern "C" fn task_start(func: extern "C" fn(usize), arg: usize, user_stack: u64) {}
+extern "C" fn task_start(_f: extern "C" fn(usize), _arg: usize, _user_stack: u64) -> ! {
+	unimplemented!()
+}
 
 #[cfg(target_os = "hermit")]
-extern "C" {
-	fn task_start(func: extern "C" fn(usize), arg: usize, user_stack: u64);
+#[naked]
+extern "C" fn task_start(_f: extern "C" fn(usize), _arg: usize, _user_stack: u64) -> ! {
+	// `f` is in the `rdi` register
+	// `arg` is in the `rsi` register
+	// `user_stack` is in the `rdx` register
+
+	unsafe {
+		asm!(
+			"mov rsp, rdx",
+			"sti",
+			"jmp {task_entry}",
+			task_entry = sym task_entry,
+			options(noreturn)
+		)
+	}
 }
 
-#[inline(never)]
-#[no_mangle]
 extern "C" fn task_entry(func: extern "C" fn(usize), arg: usize) -> ! {
 	// Call the actual entry point of the task.
 	func(arg);
@@ -356,8 +376,8 @@ impl TaskFrame for Task {
 
 		unsafe {
 			// Set a marker for debugging at the very top.
-			let mut stack =
-				self.stacks.get_kernel_stack() + self.stacks.get_kernel_stack_size() - 0x10u64;
+			let mut stack = self.stacks.get_kernel_stack() + self.stacks.get_kernel_stack_size()
+				- TaskStacks::MARKER_SIZE;
 			*stack.as_mut_ptr::<u64>() = 0xDEAD_BEEFu64;
 
 			// Put the State structure expected by the ASM switch() function on the stack.
@@ -378,8 +398,9 @@ impl TaskFrame for Task {
 
 			// Set the task's stack pointer entry to the stack we have just crafted.
 			self.last_stack_pointer = stack;
-			self.user_stack_pointer =
-				self.stacks.get_user_stack() + self.stacks.get_user_stack_size() - 0x10u64;
+			self.user_stack_pointer = self.stacks.get_user_stack()
+				+ self.stacks.get_user_stack_size()
+				- TaskStacks::MARKER_SIZE;
 
 			// rdx is required to intialize the stack
 			(*state).rdx = self.user_stack_pointer.as_u64() - mem::size_of::<u64>() as u64;

src/arch/x86_64/kernel/start.rs

@@ -0,0 +1,27 @@
+use crate::{
+	kernel::{pre_init, scheduler::TaskStacks, BootInfo},
+	KERNEL_STACK_SIZE,
+};
+
+#[no_mangle]
+#[naked]
+pub extern "C" fn _start(_boot_info: &'static mut BootInfo) -> ! {
+	// boot_info is in the `rdi` register
+
+	// validate signature
+	const _F: unsafe fn(&'static mut BootInfo) -> ! = pre_init;
+
+	unsafe {
+		asm!(
+			// initialize stack pointer
+			"mov rsp, [rdi + {current_stack_address_offset}]",
+			"add rsp, {stack_top_offset}",
+			"mov rbp, rsp",
+			"call {pre_init}",
+			current_stack_address_offset = const BootInfo::current_stack_address_offset(),
+			stack_top_offset = const KERNEL_STACK_SIZE - TaskStacks::MARKER_SIZE,
+			pre_init = sym pre_init,
+			options(noreturn)
+		)
+	}
+}

src/arch/x86_64/kernel/start.s

@@ -0,0 +1,145 @@
+use crate::set_current_kernel_stack;
+
+#[cfg(feature = "fsgsbase")]
+macro_rules! push_fs {
+	() => {
+		r#"
+		rdfsbase rax
+		push rax
+		"#
+	};
+}
+
+#[cfg(feature = "fsgsbase")]
+macro_rules! pop_fs {
+	() => {
+		r#"
+		pop rax
+		wrfsbase rax
+		"#
+	};
+}
+
+#[cfg(not(feature = "fsgsbase"))]
+macro_rules! push_fs {
+	() => {
+		r#"
+		mov ecx, 0xc0000100 // FS.Base Model Specific Register
+		rdmsr
+		sub rsp, 8
+		mov [rsp+4], edx
+		mov [rsp], eax
+		"#
+	};
+}
+
+#[cfg(not(feature = "fsgsbase"))]
+macro_rules! pop_fs {
+	() => {
+		r#"
+		mov ecx, 0xc0000100 // FS.Base Model Specific Register
+		mov edx, [rsp+4]
+		mov eax, [rsp]
+		add rsp, 8
+		wrmsr
+		"#
+	};
+}
+
+macro_rules! save_context {
+	() => {
+		concat!(
+			r#"
+			pushfq
+			push rax
+			push rcx
+			push rdx
+			push rbx
+			push rbp
+			push rsi
+			push rdi
+			push r8
+			push r9
+			push r10
+			push r11
+			push r12
+			push r13
+			push r14
+			push r15
+			"#,
+			push_fs!()
+		)
+	};
+}
+
+macro_rules! restore_context {
+	() => {
+		concat!(
+			pop_fs!(),
+			r#"
+			pop r15
+			pop r14
+			pop r13
+			pop r12
+			pop r11
+			pop r10
+			pop r9
+			pop r8
+			pop rdi
+			pop rsi
+			pop rbp
+			pop rbx
+			pop rdx
+			pop rcx
+			pop rax
+			popfq
+			ret
+			"#
+		)
+	};
+}
+
+#[naked]
+pub unsafe extern "C" fn switch_to_task(_old_stack: *mut usize, _new_stack: usize) {
+	// `old_stack` is in `rdi` register
+	// `new_stack` is in `rsi` register
+
+	asm!(
+		save_context!(),
+		// Store the old `rsp` behind `old_stack`
+		"mov [rdi], rsp",
+		// Set `rsp` to `new_stack`
+		"mov rsp, rsi",
+		// Set task switched flag
+		"mov rax, cr0",
+		"or rax, 8",
+		"mov cr0, rax",
+		// Set stack pointer in TSS
+		"call {set_current_kernel_stack}",
+		restore_context!(),
+		set_current_kernel_stack = sym set_current_kernel_stack,
+		options(noreturn)
+	);
+}
+
+/// Performa a context switch to an idle task or a task, which alread is owner
+/// of the FPU.
+#[naked]
+pub unsafe extern "C" fn switch_to_fpu_owner(_old_stack: *mut usize, _new_stack: usize) {
+	// `old_stack` is in `rdi` register
+	// `new_stack` is in `rsi` register
+
+	asm!(
+		save_context!(),
+		// Store the old `rsp` behind `old_stack`
+		"mov [rdi], rsp",
+		// Set `rsp` to `new_stack`
+		"mov rsp, rsi",
+		// Don't set task switched flag, as we switch to fpu owner.
+		// Set stack pointer in TSS
+		"call {set_current_kernel_stack}",
+		restore_context!(),
+		set_current_kernel_stack = sym set_current_kernel_stack,
+		options(noreturn),
+	);
+}