diff --git a/library/alloc/src/lib.rs b/library/alloc/src/lib.rs index 55e18b0495601..e9dff17ed8ef7 100644 --- a/library/alloc/src/lib.rs +++ b/library/alloc/src/lib.rs @@ -150,6 +150,7 @@ #![feature(slice_ptr_get)] #![feature(slice_ptr_len)] #![feature(slice_range)] +#![feature(stable_wakers)] #![feature(str_internals)] #![feature(strict_provenance)] #![feature(trusted_len)] diff --git a/library/alloc/src/task.rs b/library/alloc/src/task.rs index 9d8e309a978d9..c406f11dfc72f 100644 --- a/library/alloc/src/task.rs +++ b/library/alloc/src/task.rs @@ -121,33 +121,45 @@ impl From> for RawWaker { #[inline(always)] fn raw_waker(waker: Arc) -> RawWaker { // Increment the reference count of the arc to clone it. - unsafe fn clone_waker(waker: *const ()) -> RawWaker { + unsafe extern "C" fn clone_waker( + waker: *const (), + ) -> RawWaker { unsafe { Arc::increment_strong_count(waker as *const W) }; RawWaker::new( waker as *const (), - &RawWakerVTable::new(clone_waker::, wake::, wake_by_ref::, drop_waker::), + &RawWakerVTable::new_with_c_abi( + clone_waker::, + wake::, + wake_by_ref::, + drop_waker::, + ), ) } // Wake by value, moving the Arc into the Wake::wake function - unsafe fn wake(waker: *const ()) { + unsafe extern "C" fn wake(waker: *const ()) { let waker = unsafe { Arc::from_raw(waker as *const W) }; ::wake(waker); } // Wake by reference, wrap the waker in ManuallyDrop to avoid dropping it - unsafe fn wake_by_ref(waker: *const ()) { + unsafe extern "C" fn wake_by_ref(waker: *const ()) { let waker = unsafe { ManuallyDrop::new(Arc::from_raw(waker as *const W)) }; ::wake_by_ref(&waker); } // Decrement the reference count of the Arc on drop - unsafe fn drop_waker(waker: *const ()) { + unsafe extern "C" fn drop_waker(waker: *const ()) { unsafe { Arc::decrement_strong_count(waker as *const W) }; } RawWaker::new( Arc::into_raw(waker) as *const (), - &RawWakerVTable::new(clone_waker::, wake::, wake_by_ref::, drop_waker::), + &RawWakerVTable::new_with_c_abi( + clone_waker::, + wake::, + wake_by_ref::, + drop_waker::, + ), ) } diff --git a/library/core/src/task/wake.rs b/library/core/src/task/wake.rs index 808825326aef5..f9c23c8e1cd22 100644 --- a/library/core/src/task/wake.rs +++ b/library/core/src/task/wake.rs @@ -1,7 +1,7 @@ #![stable(feature = "futures_api", since = "1.36.0")] -use crate::fmt; -use crate::marker::{PhantomData, Unpin}; +use core::fmt; +use core::marker::{PhantomData, Unpin}; /// A `RawWaker` allows the implementor of a task executor to create a [`Waker`] /// which provides customized wakeup behavior. @@ -12,6 +12,7 @@ use crate::marker::{PhantomData, Unpin}; /// that customizes the behavior of the `RawWaker`. #[derive(PartialEq, Debug)] #[stable(feature = "futures_api", since = "1.36.0")] +#[repr(C)] pub struct RawWaker { /// A data pointer, which can be used to store arbitrary data as required /// by the executor. This could be e.g. a type-erased pointer to an `Arc` @@ -61,25 +62,11 @@ impl RawWaker { } } -/// A virtual function pointer table (vtable) that specifies the behavior -/// of a [`RawWaker`]. -/// -/// The pointer passed to all functions inside the vtable is the `data` pointer -/// from the enclosing [`RawWaker`] object. -/// -/// The functions inside this struct are only intended to be called on the `data` -/// pointer of a properly constructed [`RawWaker`] object from inside the -/// [`RawWaker`] implementation. Calling one of the contained functions using -/// any other `data` pointer will cause undefined behavior. -/// -/// These functions must all be thread-safe (even though [`RawWaker`] is -/// \![Send] + \![Sync]) -/// because [`Waker`] is [Send] + [Sync], and thus wakers may be moved to -/// arbitrary threads or invoked by `&` reference. For example, this means that if the -/// `clone` and `drop` functions manage a reference count, they must do so atomically. +const RAW_WAKER_VTABLE_V1_PADDING: usize = 0; #[stable(feature = "futures_api", since = "1.36.0")] #[derive(PartialEq, Copy, Clone, Debug)] -pub struct RawWakerVTable { +#[repr(C)] +struct RawWakerVTableV1 { /// This function will be called when the [`RawWaker`] gets cloned, e.g. when /// the [`Waker`] in which the [`RawWaker`] is stored gets cloned. /// @@ -110,8 +97,118 @@ pub struct RawWakerVTable { /// resources that are associated with this instance of a [`RawWaker`] and /// associated task. drop: unsafe fn(*const ()), + + /// This function guarantees permits calling `clone` with the C calling convention, guaranteeing stability. + clone_adapter: + Option RawWaker, *const ()) -> RawWaker>, + /// This function guarantees permits calling the other function pointers with the C calling convention, guaranteeing stability. + other_adapter: Option, + padding: [*const (); RAW_WAKER_VTABLE_V1_PADDING], // Reserved space for future changes to the v-table, the need for it and amount are debatable. } +#[derive(PartialEq, Copy, Clone, Debug)] +#[repr(C)] +struct RawWakerVTableV2 { + /// This function will be called when the [`RawWaker`] gets cloned, e.g. when + /// the [`Waker`] in which the [`RawWaker`] is stored gets cloned. + /// + /// The implementation of this function must retain all resources that are + /// required for this additional instance of a [`RawWaker`] and associated + /// task. Calling `wake` on the resulting [`RawWaker`] should result in a wakeup + /// of the same task that would have been awoken by the original [`RawWaker`]. + clone: unsafe extern "C" fn(*const ()) -> RawWaker, + + /// This function will be called when `wake` is called on the [`Waker`]. + /// It must wake up the task associated with this [`RawWaker`]. + /// + /// The implementation of this function must make sure to release any + /// resources that are associated with this instance of a [`RawWaker`] and + /// associated task. + wake: unsafe extern "C" fn(*const ()), + + /// This function will be called when `wake_by_ref` is called on the [`Waker`]. + /// It must wake up the task associated with this [`RawWaker`]. + /// + /// This function is similar to `wake`, but must not consume the provided data + /// pointer. + wake_by_ref: unsafe extern "C" fn(*const ()), + + /// This function gets called when a [`Waker`] gets dropped. + /// + /// The implementation of this function must make sure to release any + /// resources that are associated with this instance of a [`RawWaker`] and + /// associated task. + drop: unsafe extern "C" fn(*const ()), + padding: [*const (); 2 + RAW_WAKER_VTABLE_V1_PADDING], // Reserved space for future changes to the v-table, the need for it and amount are debatable. +} + +/// A virtual function pointer table (vtable) that specifies the behavior +/// of a [`RawWaker`]. +/// +/// The pointer passed to all functions inside the vtable is the `data` pointer +/// from the enclosing [`RawWaker`] object. +/// +/// The functions inside this struct are only intended to be called on the `data` +/// pointer of a properly constructed [`RawWaker`] object from inside the +/// [`RawWaker`] implementation. Calling one of the contained functions using +/// any other `data` pointer will cause undefined behavior. +/// +/// These functions must all be thread-safe (even though [`RawWaker`] is +/// \![Send] + \![Sync]) +/// because [`Waker`] is [Send] + [Sync], and thus wakers may be moved to +/// arbitrary threads or invoked by `&` reference. For example, this means that if the +/// `clone` and `drop` functions manage a reference count, they must do so atomically. +#[repr(C)] +#[derive(Clone, Copy)] +#[stable(feature = "futures_api", since = "1.36.0")] +pub union RawWakerVTable { + v1: RawWakerVTableV1, + v2: RawWakerVTableV2, +} +#[stable(feature = "futures_api", since = "1.36.0")] +unsafe impl Send for RawWakerVTable {} +#[stable(feature = "futures_api", since = "1.36.0")] +unsafe impl Sync for RawWakerVTable {} + +#[stable(feature = "futures_api", since = "1.36.0")] +impl PartialEq for RawWakerVTable { + fn eq(&self, other: &Self) -> bool { + // SAFETY: Comparison is just a bunch of pointer-equality: checking exact variants is unnecessary. + unsafe { self.v2 == other.v2 } + } +} + +#[stable(feature = "futures_api", since = "1.36.0")] +impl fmt::Debug for RawWakerVTable { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // SAFETY: Matching on unions is always unsafe. + // The determinant of this union is that `v2`'s padding must always be null pointers, i.e. v1's adapter fields must be `Some(_)` + unsafe { + match self { + RawWakerVTable { v1 } if v1.other_adapter.is_some() => self.v1.fmt(f), + RawWakerVTable { v2 } => v2.fmt(f), + } + } + } +} + +#[allow(improper_ctypes_definitions)] +/// # Safety +/// This function must only be called with function pointers sourced from the same shared object +unsafe extern "C" fn clone_adapter( + clone: unsafe fn(*const ()) -> RawWaker, + data: *const (), +) -> RawWaker { + // SAFETY: The safety constraints are passed up to the caller + unsafe { (clone)(data) } +} +#[allow(improper_ctypes_definitions)] +/// # Safety +/// This function must only be called with function pointers sourced from the same shared object +unsafe extern "C" fn other_adapter(other: unsafe fn(*const ()), data: *const ()) { + // SAFETY: The safety constraints are passed up to the caller + unsafe { (other)(data) } +} impl RawWakerVTable { /// Creates a new `RawWakerVTable` from the provided `clone`, `wake`, /// `wake_by_ref`, and `drop` functions. @@ -159,13 +256,91 @@ impl RawWakerVTable { #[rustc_promotable] #[stable(feature = "futures_api", since = "1.36.0")] #[rustc_const_stable(feature = "futures_api", since = "1.36.0")] + #[deprecated( + since = "TBD", + note = "This constructor makes slower wakers", + suggestion = "new_with_c_abi" + )] pub const fn new( clone: unsafe fn(*const ()) -> RawWaker, wake: unsafe fn(*const ()), wake_by_ref: unsafe fn(*const ()), drop: unsafe fn(*const ()), ) -> Self { - Self { clone, wake, wake_by_ref, drop } + Self { + v1: RawWakerVTableV1 { + clone, + wake, + wake_by_ref, + drop, + clone_adapter: Some(clone_adapter), + other_adapter: Some(other_adapter), + padding: [core::ptr::null(); RAW_WAKER_VTABLE_V1_PADDING], + }, + } + } + + /// Creates a new `RawWakerVTable` from the provided `clone`, `wake`, + /// `wake_by_ref`, and `drop` functions. + /// + /// These functions must all be thread-safe (even though [`RawWaker`] is + /// \![Send] + \![Sync]) + /// because [`Waker`] is [Send] + [Sync], and thus wakers may be moved to + /// arbitrary threads or invoked by `&` reference. For example, this means that if the + /// `clone` and `drop` functions manage a reference count, they must do so atomically. + /// + /// # `clone` + /// + /// This function will be called when the [`RawWaker`] gets cloned, e.g. when + /// the [`Waker`] in which the [`RawWaker`] is stored gets cloned. + /// + /// The implementation of this function must retain all resources that are + /// required for this additional instance of a [`RawWaker`] and associated + /// task. Calling `wake` on the resulting [`RawWaker`] should result in a wakeup + /// of the same task that would have been awoken by the original [`RawWaker`]. + /// + /// # `wake` + /// + /// This function will be called when `wake` is called on the [`Waker`]. + /// It must wake up the task associated with this [`RawWaker`]. + /// + /// The implementation of this function must make sure to release any + /// resources that are associated with this instance of a [`RawWaker`] and + /// associated task. + /// + /// # `wake_by_ref` + /// + /// This function will be called when `wake_by_ref` is called on the [`Waker`]. + /// It must wake up the task associated with this [`RawWaker`]. + /// + /// This function is similar to `wake`, but must not consume the provided data + /// pointer. + /// + /// # `drop` + /// + /// This function gets called when a [`Waker`] gets dropped. + /// + /// The implementation of this function must make sure to release any + /// resources that are associated with this instance of a [`RawWaker`] and + /// associated task. + #[rustc_promotable] + #[unstable(feature = "stable_wakers", issue = "109706")] + #[rustc_const_unstable(feature = "stable_wakers", issue = "109706")] + pub const fn new_with_c_abi( + clone: unsafe extern "C" fn(*const ()) -> RawWaker, + wake: unsafe extern "C" fn(*const ()), + wake_by_ref: unsafe extern "C" fn(*const ()), + drop: unsafe extern "C" fn(*const ()), + ) -> Self { + Self { + v2: RawWakerVTableV2 { + clone, + wake, + wake_by_ref, + drop, + padding: [core::ptr::null(); 2 + RAW_WAKER_VTABLE_V1_PADDING], + }, + } } } @@ -203,7 +378,7 @@ impl<'a> Context<'a> { #[must_use] #[inline] pub const fn waker(&self) -> &'a Waker { - &self.waker + self.waker } } @@ -262,22 +437,31 @@ impl Waker { /// executor’s choice which task to run and the executor may choose to run the /// current task again. /// - /// [`poll()`]: crate::future::Future::poll + /// [`poll()`]: core::future::Future::poll #[inline] #[stable(feature = "futures_api", since = "1.36.0")] pub fn wake(self) { // The actual wakeup call is delegated through a virtual function call // to the implementation which is defined by the executor. - let wake = self.waker.vtable.wake; + let vtable = self.waker.vtable; let data = self.waker.data; // Don't call `drop` -- the waker will be consumed by `wake`. - crate::mem::forget(self); + core::mem::forget(self); // SAFETY: This is safe because `Waker::from_raw` is the only way // to initialize `wake` and `data` requiring the user to acknowledge // that the contract of `RawWaker` is upheld. - unsafe { (wake)(data) }; + // Matching on unions is always unsafe. + // The determinant of this union is that `v2`'s padding must always be null pointers, i.e. v1's adapter fields must be `Some(_)` + unsafe { + match *vtable { + RawWakerVTable { + v1: RawWakerVTableV1 { wake, other_adapter: Some(other_adapter), .. }, + } => (other_adapter)(wake, data), + RawWakerVTable { v2: RawWakerVTableV2 { wake, .. } } => (wake)(data), + } + }; } /// Wake up the task associated with this `Waker` without consuming the `Waker`. @@ -290,9 +474,19 @@ impl Waker { pub fn wake_by_ref(&self) { // The actual wakeup call is delegated through a virtual function call // to the implementation which is defined by the executor. + let RawWaker { data, vtable } = self.waker; // SAFETY: see `wake` - unsafe { (self.waker.vtable.wake_by_ref)(self.waker.data) } + // Matching on unions is always unsafe. + // The determinant of this union is that `v2`'s padding must always be null pointers, i.e. v1's adapter fields must be `Some(_)` + unsafe { + match *vtable { + RawWakerVTable { + v1: RawWakerVTableV1 { wake_by_ref, other_adapter: Some(other_adapter), .. }, + } => (other_adapter)(wake_by_ref, data), + RawWakerVTable { v2: RawWakerVTableV2 { wake_by_ref, .. } } => (wake_by_ref)(data), + } + } } /// Returns `true` if this `Waker` and another `Waker` would awake the same task. @@ -335,11 +529,21 @@ impl Waker { impl Clone for Waker { #[inline] fn clone(&self) -> Self { + let RawWaker { data, vtable } = self.waker; Waker { // SAFETY: This is safe because `Waker::from_raw` is the only way // to initialize `clone` and `data` requiring the user to acknowledge // that the contract of [`RawWaker`] is upheld. - waker: unsafe { (self.waker.vtable.clone)(self.waker.data) }, + // Matching on unions is always unsafe. + // The determinant of this union is that `v2`'s padding must always be null pointers, i.e. v1's adapter fields must be `Some(_)` + waker: unsafe { + match *vtable { + RawWakerVTable { + v1: RawWakerVTableV1 { clone, clone_adapter: Some(clone_adapter), .. }, + } => (clone_adapter)(clone, data), + RawWakerVTable { v2: RawWakerVTableV2 { clone, .. } } => (clone)(data), + } + }, } } } @@ -348,10 +552,20 @@ impl Clone for Waker { impl Drop for Waker { #[inline] fn drop(&mut self) { + let RawWaker { data, vtable } = self.waker; // SAFETY: This is safe because `Waker::from_raw` is the only way // to initialize `drop` and `data` requiring the user to acknowledge // that the contract of `RawWaker` is upheld. - unsafe { (self.waker.vtable.drop)(self.waker.data) } + // Matching on unions is always unsafe. + // The determinant of this union is that `v2`'s padding must always be null pointers, i.e. v1's adapter fields must be `Some(_)` + unsafe { + match *vtable { + RawWakerVTable { + v1: RawWakerVTableV1 { drop, other_adapter: Some(other_adapter), .. }, + } => (other_adapter)(drop, data), + RawWakerVTable { v2: RawWakerVTableV2 { drop, .. } } => (drop)(data), + } + } } }