From 2a21898ad22d802aa8cf713f54c9d04572e3a2cb Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Mon, 31 Aug 2020 01:06:03 +0200 Subject: [PATCH 1/4] remove dependencies to rust feature "naked functions" - naked functions does currently not work (rust-lang/rust#75922) - we replaced naked functions by assembly code - using global_asm to include the assembly code --- src/arch/mod.rs | 7 +- src/arch/x86_64/kernel/apic.rs | 9 +- src/arch/x86_64/kernel/gdt.rs | 1 + src/arch/x86_64/kernel/mod.rs | 26 +++++- src/arch/x86_64/kernel/scheduler.rs | 14 ++- src/arch/x86_64/kernel/start.rs | 48 ---------- src/arch/x86_64/kernel/start.s | 32 +++++++ src/arch/x86_64/kernel/switch.rs | 132 ---------------------------- src/arch/x86_64/kernel/switch.s | 119 +++++++++++++++++++++++++ src/lib.rs | 2 +- src/scheduler/mod.rs | 8 +- 11 files changed, 202 insertions(+), 196 deletions(-) delete mode 100755 src/arch/x86_64/kernel/start.rs create mode 100755 src/arch/x86_64/kernel/start.s delete mode 100755 src/arch/x86_64/kernel/switch.rs create mode 100755 src/arch/x86_64/kernel/switch.s diff --git a/src/arch/mod.rs b/src/arch/mod.rs index d3eddce023..626f32e70e 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -60,8 +60,6 @@ pub use crate::arch::x86_64::kernel::processor; #[cfg(target_arch = "x86_64")] pub use crate::arch::x86_64::kernel::scheduler; #[cfg(target_arch = "x86_64")] -pub use crate::arch::x86_64::kernel::switch::{switch_to_fpu_owner, switch_to_task}; -#[cfg(target_arch = "x86_64")] pub use crate::arch::x86_64::kernel::systemtime::get_boot_time; #[cfg(not(test))] #[cfg(target_arch = "x86_64")] @@ -72,3 +70,8 @@ pub use crate::arch::x86_64::kernel::{ pub use crate::arch::x86_64::kernel::{ get_processor_count, message_output_init, output_message_buf, output_message_byte, }; + +extern "C" { + pub fn switch_to_task(old_stack: *mut usize, new_stack: usize); + pub fn switch_to_fpu_owner(old_stack: *mut usize, new_stack: usize); +} diff --git a/src/arch/x86_64/kernel/apic.rs b/src/arch/x86_64/kernel/apic.rs index b25127e1f5..5ddab5f040 100644 --- a/src/arch/x86_64/kernel/apic.rs +++ b/src/arch/x86_64/kernel/apic.rs @@ -518,6 +518,10 @@ pub fn init_next_processor_variables(core_id: CoreId) { } } +extern "C" { + fn _start(); +} + /// Boot all Application Processors /// This algorithm is derived from Intel MultiProcessor Specification 1.4, B.4, but testing has shown /// that a second STARTUP IPI and setting the BIOS Reset Vector are no longer necessary. @@ -559,10 +563,9 @@ pub fn boot_application_processors() { // Set entry point debug!( "Set entry point for application processor to 0x{:x}", - arch::x86_64::kernel::start::_start as u64 + _start as u64 ); - *((SMP_BOOT_CODE_ADDRESS + SMP_BOOT_CODE_OFFSET_ENTRY).as_mut_ptr()) = - arch::x86_64::kernel::start::_start as u64; + *((SMP_BOOT_CODE_ADDRESS + SMP_BOOT_CODE_OFFSET_ENTRY).as_mut_ptr()) = _start as u64; *((SMP_BOOT_CODE_ADDRESS + SMP_BOOT_CODE_OFFSET_BOOTINFO).as_mut_ptr()) = BOOT_INFO as u64; } diff --git a/src/arch/x86_64/kernel/gdt.rs b/src/arch/x86_64/kernel/gdt.rs index 09e6f971e4..e67ca98bdd 100644 --- a/src/arch/x86_64/kernel/gdt.rs +++ b/src/arch/x86_64/kernel/gdt.rs @@ -131,6 +131,7 @@ pub fn add_current_core() { } #[inline(never)] +#[no_mangle] pub fn set_current_kernel_stack() { core_scheduler().set_current_kernel_stack(); } diff --git a/src/arch/x86_64/kernel/mod.rs b/src/arch/x86_64/kernel/mod.rs index ac79d69b7f..84edb127db 100644 --- a/src/arch/x86_64/kernel/mod.rs +++ b/src/arch/x86_64/kernel/mod.rs @@ -25,9 +25,6 @@ pub mod scheduler; pub mod serial; #[cfg(not(test))] mod smp_boot_code; -#[cfg(not(test))] -mod start; -pub mod switch; pub mod systemtime; #[cfg(feature = "vga")] mod vga; @@ -49,7 +46,11 @@ use core::convert::TryInto; #[cfg(feature = "newlib")] use core::slice; use core::{intrinsics, ptr}; -use x86::controlregs::{cr0, cr4}; +use x86::controlregs::{cr0, cr0_write, cr4, Cr0}; + +#[cfg(not(test))] +global_asm!(include_str!("start.s")); +global_asm!(include_str!("switch.s")); const SERIAL_PORT_BAUDRATE: u32 = 115_200; @@ -403,3 +404,20 @@ pub fn print_statistics() { } } } + +#[inline(never)] +#[no_mangle] +unsafe fn pre_init(boot_info: &'static mut BootInfo) -> ! { + // Enable caching + let mut cr0 = cr0(); + cr0.remove(Cr0::CR0_CACHE_DISABLE | Cr0::CR0_NOT_WRITE_THROUGH); + cr0_write(cr0); + + BOOT_INFO = boot_info as *mut BootInfo; + + if boot_info.cpu_online == 0 { + crate::boot_processor_main() + } else { + crate::application_processor_main() + } +} diff --git a/src/arch/x86_64/kernel/scheduler.rs b/src/arch/x86_64/kernel/scheduler.rs index cee505473e..9c8c594f60 100644 --- a/src/arch/x86_64/kernel/scheduler.rs +++ b/src/arch/x86_64/kernel/scheduler.rs @@ -327,14 +327,17 @@ impl Clone for TaskTLS { } #[cfg(test)] -extern "C" fn task_entry(func: extern "C" fn(usize), arg: usize) {} +extern "C" fn task_start(func: extern "C" fn(usize), arg: usize, user_stack: u64) {} #[cfg(not(test))] +extern "C" { + fn task_start(func: extern "C" fn(usize), arg: usize, user_stack: u64); +} + #[inline(never)] -#[naked] +#[no_mangle] extern "C" fn task_entry(func: extern "C" fn(usize), arg: usize) -> ! { // Call the actual entry point of the task. - switch_to_user!(); func(arg); switch_to_kernel!(); @@ -366,7 +369,7 @@ impl TaskFrame for Task { if let Some(tls) = &self.tls { (*state).fs = tls.get_fs().as_u64(); } - (*state).rip = task_entry as u64; + (*state).rip = task_start as u64; (*state).rdi = func as u64; (*state).rsi = arg as u64; @@ -377,6 +380,9 @@ impl TaskFrame for Task { self.last_stack_pointer = stack; self.user_stack_pointer = self.stacks.get_user_stack() + self.stacks.get_user_stack_size() - 0x10u64; + + // rdx is required to intialize the stack + (*state).rdx = self.user_stack_pointer.as_u64() - mem::size_of::() as u64; } } } diff --git a/src/arch/x86_64/kernel/start.rs b/src/arch/x86_64/kernel/start.rs deleted file mode 100755 index bf9613a5f4..0000000000 --- a/src/arch/x86_64/kernel/start.rs +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) 2019 Stefan Lankes, RWTH Aachen University -// -// Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be -// copied, modified, or distributed except according to those terms. - -use crate::application_processor_main; -use crate::arch::x86_64::kernel::{BootInfo, BOOT_INFO}; -use crate::boot_processor_main; -use crate::config::KERNEL_STACK_SIZE; -use crate::x86::controlregs::*; - -unsafe fn cr0_enable_caching() { - let mut cr0 = cr0(); - - // Enable caching. - cr0.remove(Cr0::CR0_CACHE_DISABLE | Cr0::CR0_NOT_WRITE_THROUGH); - - cr0_write(cr0); -} - -#[inline(never)] -#[no_mangle] -#[naked] -pub unsafe extern "C" fn _start(boot_info: &'static mut BootInfo) -> ! { - // initialize stack pointer - llvm_asm!("mov $0, %rsp; mov %rsp, %rbp" - :: "r"(boot_info.current_stack_address + KERNEL_STACK_SIZE as u64 - 0x10) - :: "volatile"); - - pre_init(boot_info); -} - -unsafe fn pre_init(boot_info: &'static mut BootInfo) -> ! { - // - // CR0 CONFIGURATION - // - cr0_enable_caching(); - - BOOT_INFO = boot_info as *mut BootInfo; - - if boot_info.cpu_online == 0 { - boot_processor_main() - } else { - application_processor_main() - } -} diff --git a/src/arch/x86_64/kernel/start.s b/src/arch/x86_64/kernel/start.s new file mode 100755 index 0000000000..7898988c49 --- /dev/null +++ b/src/arch/x86_64/kernel/start.s @@ -0,0 +1,32 @@ +// Copyright (c) 2020 Stefan Lankes, RWTH Aachen University +// +// Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be +// copied, modified, or distributed except according to those terms. + +.section .text +.global _start +.global task_start +.extern pre_init +.extern task_entry + +.align 16 +_start: + // initialize stack pointer + mov $0x7ff0,%rax + add 0x38(%rdi),%rax + mov %rax, %rsp + mov %rsp, %rbp + + call pre_init + +l1: + jmp l1 + +.align 16 +task_start: + mov %rdx, %rsp + sti + jmp task_entry + jmp l1 diff --git a/src/arch/x86_64/kernel/switch.rs b/src/arch/x86_64/kernel/switch.rs deleted file mode 100755 index f187267bb6..0000000000 --- a/src/arch/x86_64/kernel/switch.rs +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright (c) 2017-2018 Stefan Lankes, RWTH Aachen University -// -// Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be -// copied, modified, or distributed except according to those terms. - -use crate::arch::x86_64::kernel::gdt::set_current_kernel_stack; - -/// The function triggers a context switch to a new task. -#[inline(never)] -#[naked] -pub fn switch_to_task(_old_stack: *mut usize, _new_stack: usize) { - // rdi = old_stack => the address to store the old rsp - // rsi = new_stack => stack pointer of the new task - - unsafe { - // store context - llvm_asm!( - "pushfq\n\t\ - push %rax\n\t\ - push %rcx\n\t\ - push %rdx\n\t\ - push %rbx\n\t\ - push %rbp\n\t\ - push %rsi\n\t\ - push %rdi\n\t\ - push %r8\n\t\ - push %r9\n\t\ - push %r10\n\t\ - push %r11\n\t\ - push %r12\n\t\ - push %r13\n\t\ - push %r14\n\t\ - push %r15\n\t\ - rdfsbaseq %rax\n\t\ - push %rax\n\t\ - // store the old stack pointer in the dereferenced first parameter\n\t\ - // and load the new stack pointer in the second parameter.\n\t\ - mov %rsp, (%rdi)\n\t\ - mov %rsi, %rsp\n\t\ - // Set task switched flag \n\t\ - mov %cr0, %rax\n\t\ - or $$8, %rax\n\t\ - mov %rax, %cr0" :::: "volatile" - ); - // set stack pointer in TSS \n\t\ - set_current_kernel_stack(); - // restore context - llvm_asm!( - "pop %rax\n\t\ - wrfsbaseq %rax\n\t\ - pop %r15\n\t\ - pop %r14\n\t\ - pop %r13\n\t\ - pop %r12\n\t\ - pop %r11\n\t\ - pop %r10\n\t\ - pop %r9\n\t\ - pop %r8\n\t\ - pop %rdi\n\t\ - pop %rsi\n\t\ - pop %rbp\n\t\ - pop %rbx\n\t\ - pop %rdx\n\t\ - pop %rcx\n\t\ - pop %rax\n\t\ - popfq" :::: "volatile" - ); - } -} - -/// The function triggers a context switch to an idle task or -/// a task, which is alread owner of the FPU. -/// Consequently the kernel don't set the task switched flag. -#[inline(never)] -#[naked] -pub fn switch_to_fpu_owner(_old_stack: *mut usize, _new_stack: usize) { - // rdi = old_stack => the address to store the old rsp - // rsi = new_stack => stack pointer of the new task - - unsafe { - // store context - llvm_asm!( - "pushfq\n\t\ - push %rax\n\t\ - push %rcx\n\t\ - push %rdx\n\t\ - push %rbx\n\t\ - push %rbp\n\t\ - push %rsi\n\t\ - push %rdi\n\t\ - push %r8\n\t\ - push %r9\n\t\ - push %r10\n\t\ - push %r11\n\t\ - push %r12\n\t\ - push %r13\n\t\ - push %r14\n\t\ - push %r15\n\t\ - rdfsbaseq %rax\n\t\ - push %rax\n\t\ - // store the old stack pointer in the dereferenced first parameter\n\t\ - // and load the new stack pointer in the second parameter.\n\t\ - mov %rsp, (%rdi)\n\t\ - mov %rsi, %rsp" :::: "volatile" - ); - // set stack pointer in TSS \n\t\ - set_current_kernel_stack(); - // restore context - llvm_asm!( - "pop %rax\n\t\ - wrfsbaseq %rax\n\t\ - pop %r15\n\t\ - pop %r14\n\t\ - pop %r13\n\t\ - pop %r12\n\t\ - pop %r11\n\t\ - pop %r10\n\t\ - pop %r9\n\t\ - pop %r8\n\t\ - pop %rdi\n\t\ - pop %rsi\n\t\ - pop %rbp\n\t\ - pop %rbx\n\t\ - pop %rdx\n\t\ - pop %rcx\n\t\ - pop %rax\n\t\ - popfq" :::: "volatile" - ); - } -} diff --git a/src/arch/x86_64/kernel/switch.s b/src/arch/x86_64/kernel/switch.s new file mode 100755 index 0000000000..c69be30bc9 --- /dev/null +++ b/src/arch/x86_64/kernel/switch.s @@ -0,0 +1,119 @@ +// Copyright (c) 2020 Stefan Lankes, RWTH Aachen University +// +// Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be +// copied, modified, or distributed except according to those terms. + +.section .text +.global switch_to_task +.global switch_to_fpu_owner +.extern set_current_kernel_stack + +.align 16 +switch_to_task: + // rdi = old_stack => the address to store the old rsp + // rsi = new_stack => stack pointer of the new task + + pushfq + push %rax + push %rcx + push %rdx + push %rbx + push %rbp + push %rsi + push %rdi + push %r8 + push %r9 + push %r10 + push %r11 + push %r12 + push %r13 + push %r14 + push %r15 + rdfsbaseq %rax + push %rax + // store the old stack pointer in the dereferenced first parameter\n\t\ + // and load the new stack pointer in the second parameter.\n\t\ + mov %rsp, (%rdi) + mov %rsi, %rsp + // Set task switched flag + mov %cr0, %rax + or $8, %rax + mov %rax, %cr0 + // set stack pointer in TSS + call set_current_kernel_stack + // restore context + pop %rax + wrfsbaseq %rax + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %r11 + pop %r10 + pop %r9 + pop %r8 + pop %rdi + pop %rsi + pop %rbp + pop %rbx + pop %rdx + pop %rcx + pop %rax + popfq + ret + +/// The function triggers a context switch to an idle task or +/// a task, which is alread owner of the FPU. +/// Consequently the kernel don't set the task switched flag. +.align 16 +switch_to_fpu_owner: + // rdi = old_stack => the address to store the old rsp + // rsi = new_stack => stack pointer of the new task + + // store context + pushfq + push %rax + push %rcx + push %rdx + push %rbx + push %rbp + push %rsi + push %rdi + push %r8 + push %r9 + push %r10 + push %r11 + push %r12 + push %r13 + push %r14 + push %r15 + rdfsbaseq %rax + push %rax + // store the old stack pointer in the dereferenced first parameter\n\t\ + // and load the new stack pointer in the second parameter.\n\t\ + mov %rsp, (%rdi) + mov %rsi, %rsp + // set stack pointer in TSS + call set_current_kernel_stack + // restore context + pop %rax + wrfsbaseq %rax + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %r11 + pop %r10 + pop %r9 + pop %r8 + pop %rdi + pop %rsi + pop %rbp + pop %rbx + pop %rdx + pop %rcx + pop %rax + popfq + ret diff --git a/src/lib.rs b/src/lib.rs index 00feea11b8..2b7e6a567e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,12 +28,12 @@ #![feature(allocator_api)] #![feature(const_btree_new)] #![feature(const_fn)] +#![feature(global_asm)] #![feature(lang_items)] #![feature(linkage)] #![feature(llvm_asm)] #![feature(panic_info_message)] #![feature(specialization)] -#![feature(naked_functions)] #![feature(nonnull_slice_from_raw_parts)] #![feature(core_intrinsics)] #![feature(alloc_error_handler)] diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index d5ca992e7e..32208dc75d 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -477,9 +477,13 @@ impl PerCoreScheduler { // Finally save our current context and restore the context of the new task. if is_idle || Rc::ptr_eq(&self.current_task, &self.fpu_owner) { - switch_to_fpu_owner(last_stack_pointer, new_stack_pointer.as_usize()) + unsafe { + switch_to_fpu_owner(last_stack_pointer, new_stack_pointer.as_usize()); + } } else { - switch_to_task(last_stack_pointer, new_stack_pointer.as_usize()); + unsafe { + switch_to_task(last_stack_pointer, new_stack_pointer.as_usize()); + } } } } From cd4deda33187c52103a43adcd574e9da89fdf25b Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Mon, 31 Aug 2020 01:13:24 +0200 Subject: [PATCH 2/4] exclude pre_init during tests --- src/arch/x86_64/kernel/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/arch/x86_64/kernel/mod.rs b/src/arch/x86_64/kernel/mod.rs index 84edb127db..fdb91ca993 100644 --- a/src/arch/x86_64/kernel/mod.rs +++ b/src/arch/x86_64/kernel/mod.rs @@ -405,6 +405,7 @@ pub fn print_statistics() { } } +#[cfg(not(test))] #[inline(never)] #[no_mangle] unsafe fn pre_init(boot_info: &'static mut BootInfo) -> ! { From 65a9f21bf6e2d171eaf6419c77f05283d2634e7d Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Mon, 31 Aug 2020 01:17:32 +0200 Subject: [PATCH 3/4] exclude all assembly code during tests --- src/arch/x86_64/kernel/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/arch/x86_64/kernel/mod.rs b/src/arch/x86_64/kernel/mod.rs index fdb91ca993..711759e7fa 100644 --- a/src/arch/x86_64/kernel/mod.rs +++ b/src/arch/x86_64/kernel/mod.rs @@ -50,6 +50,7 @@ use x86::controlregs::{cr0, cr0_write, cr4, Cr0}; #[cfg(not(test))] global_asm!(include_str!("start.s")); +#[cfg(not(test))] global_asm!(include_str!("switch.s")); const SERIAL_PORT_BAUDRATE: u32 = 115_200; From 1a5bb01902048b9ccf5f32c3c8d65f1b505de4f6 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Mon, 31 Aug 2020 01:23:43 +0200 Subject: [PATCH 4/4] add dummy implementations for context switches - only necessary in test environments --- src/arch/mod.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 626f32e70e..a83a4a9511 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -71,6 +71,12 @@ pub use crate::arch::x86_64::kernel::{ get_processor_count, message_output_init, output_message_buf, output_message_byte, }; +#[cfg(test)] +pub fn switch_to_task(_old_stack: *mut usize, _new_stack: usize) {} +#[cfg(test)] +pub fn switch_to_fpu_owner(_old_stack: *mut usize, _new_stack: usize) {} + +#[cfg(not(test))] extern "C" { pub fn switch_to_task(old_stack: *mut usize, new_stack: usize); pub fn switch_to_fpu_owner(old_stack: *mut usize, new_stack: usize);