Skip to content

Commit 05fb58a

Browse files
surbansurban
committed
Run TLS destructors at process exit on all platforms
This calls TLS destructors for the thread that initiates a process exit. This is done by registering a process-wide exit callback. Previously UNIX platforms other than Linux and Apple did not destruct TLS variables on the thread that initiated the process exit (either by returning from main or calling std::process::exit).
1 parent 7db7489 commit 05fb58a

File tree

8 files changed

+286
-38
lines changed

8 files changed

+286
-38
lines changed

‎library/std/src/lib.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -282,6 +282,7 @@
282282
#![feature(allow_internal_unstable)]
283283
#![feature(asm_experimental_arch)]
284284
#![feature(cfg_sanitizer_cfi)]
285+
#![feature(cfg_target_has_atomic)]
285286
#![feature(cfg_target_thread_local)]
286287
#![feature(cfi_encoding)]
287288
#![feature(concat_idents)]

‎library/std/src/sys/thread_local/exit/unix.rs

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
use crate::mem;
2+
3+
pub unsafe fn at_process_exit(cb: unsafe extern "C" fn()) {
4+
// Miri does not support atexit.
5+
#[cfg(not(miri))]
6+
assert_eq!(unsafe { libc::atexit(mem::transmute(cb)) }, 0);
7+
8+
#[cfg(miri)]
9+
let _ = cb;
10+
}

‎library/std/src/sys/thread_local/guard/key.rs

Lines changed: 67 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -3,21 +3,40 @@
33
//! that will run all native TLS destructors in the destructor list.
44
55
use crate::ptr;
6+
use crate::sync::atomic::{AtomicBool, Ordering};
7+
use crate::sys::thread_local::exit::at_process_exit;
68
use crate::sys::thread_local::key::{LazyKey, set};
79

810
#[cfg(target_thread_local)]
911
pub fn enable() {
10-
use crate::sys::thread_local::destructors;
12+
fn enable_thread() {
13+
static DTORS: LazyKey = LazyKey::new(Some(run_thread));
1114

12-
static DTORS: LazyKey = LazyKey::new(Some(run));
15+
// Setting the key value to something other than NULL will result in the
16+
// destructor being run at thread exit.
17+
unsafe {
18+
set(DTORS.force(), ptr::without_provenance_mut(1));
19+
}
20+
21+
unsafe extern "C" fn run_thread(_: *mut u8) {
22+
run()
23+
}
24+
}
1325

14-
// Setting the key value to something other than NULL will result in the
15-
// destructor being run at thread exit.
16-
unsafe {
17-
set(DTORS.force(), ptr::without_provenance_mut(1));
26+
fn enable_process() {
27+
static REGISTERED: AtomicBool = AtomicBool::new(false);
28+
if !REGISTERED.swap(true, Ordering::Relaxed) {
29+
unsafe { at_process_exit(run_process) };
30+
}
31+
32+
unsafe extern "C" fn run_process() {
33+
run()
34+
}
1835
}
1936

20-
unsafe extern "C" fn run(_: *mut u8) {
37+
fn run() {
38+
use crate::sys::thread_local::destructors;
39+
2140
unsafe {
2241
destructors::run();
2342
// On platforms with `__cxa_thread_atexit_impl`, `destructors::run`
@@ -28,33 +47,55 @@ pub fn enable() {
2847
crate::rt::thread_cleanup();
2948
}
3049
}
50+
51+
enable_thread();
52+
enable_process();
3153
}
3254

3355
/// On platforms with key-based TLS, the system runs the destructors for us.
3456
/// We still have to make sure that [`crate::rt::thread_cleanup`] is called,
3557
/// however. This is done by defering the execution of a TLS destructor to
3658
/// the next round of destruction inside the TLS destructors.
59+
///
60+
/// POSIX systems do not run TLS destructors at process exit.
61+
/// Thus we register our own callback to invoke them in that case.
3762
#[cfg(not(target_thread_local))]
3863
pub fn enable() {
39-
const DEFER: *mut u8 = ptr::without_provenance_mut(1);
40-
const RUN: *mut u8 = ptr::without_provenance_mut(2);
41-
42-
static CLEANUP: LazyKey = LazyKey::new(Some(run));
43-
44-
unsafe { set(CLEANUP.force(), DEFER) }
45-
46-
unsafe extern "C" fn run(state: *mut u8) {
47-
if state == DEFER {
48-
// Make sure that this function is run again in the next round of
49-
// TLS destruction. If there is no futher round, there will be leaks,
50-
// but that's okay, `thread_cleanup` is not guaranteed to be called.
51-
unsafe { set(CLEANUP.force(), RUN) }
52-
} else {
53-
debug_assert_eq!(state, RUN);
54-
// If the state is still RUN in the next round of TLS destruction,
55-
// it means that no other TLS destructors defined by this runtime
56-
// have been run, as they would have set the state to DEFER.
57-
crate::rt::thread_cleanup();
64+
fn enable_thread() {
65+
const DEFER: *mut u8 = ptr::without_provenance_mut(1);
66+
const RUN: *mut u8 = ptr::without_provenance_mut(2);
67+
68+
static CLEANUP: LazyKey = LazyKey::new(Some(run_thread));
69+
70+
unsafe { set(CLEANUP.force(), DEFER) }
71+
72+
unsafe extern "C" fn run_thread(state: *mut u8) {
73+
if state == DEFER {
74+
// Make sure that this function is run again in the next round of
75+
// TLS destruction. If there is no futher round, there will be leaks,
76+
// but that's okay, `thread_cleanup` is not guaranteed to be called.
77+
unsafe { set(CLEANUP.force(), RUN) }
78+
} else {
79+
debug_assert_eq!(state, RUN);
80+
// If the state is still RUN in the next round of TLS destruction,
81+
// it means that no other TLS destructors defined by this runtime
82+
// have been run, as they would have set the state to DEFER.
83+
crate::rt::thread_cleanup();
84+
}
85+
}
86+
}
87+
88+
fn enable_process() {
89+
static REGISTERED: AtomicBool = AtomicBool::new(false);
90+
if !REGISTERED.swap(true, Ordering::Relaxed) {
91+
unsafe { at_process_exit(run_process) };
92+
}
93+
94+
unsafe extern "C" fn run_process() {
95+
unsafe { crate::sys::thread_local::key::run_dtors() };
5896
}
5997
}
98+
99+
enable_thread();
100+
enable_process();
60101
}

‎library/std/src/sys/thread_local/guard/statik.rs

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
//! The platform has no threads, so we just need to register
2+
//! a process exit callback.
3+
4+
use crate::cell::Cell;
5+
use crate::sys::thread_local::exit::at_process_exit;
6+
use crate::sys::thread_local::statik::run_dtors;
7+
8+
pub fn enable() {
9+
struct Registered(Cell<bool>);
10+
// SAFETY: the target doesn't have threads.
11+
unsafe impl Sync for Registered {}
12+
13+
static REGISTERED: Registered = Registered(Cell::new(false));
14+
15+
if !REGISTERED.0.get() {
16+
REGISTERED.0.set(true);
17+
unsafe { at_process_exit(run_process) };
18+
}
19+
20+
unsafe extern "C" fn run_process() {
21+
unsafe { run_dtors() };
22+
crate::rt::thread_cleanup();
23+
}
24+
}

‎library/std/src/sys/thread_local/key/racy.rs

Lines changed: 72 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@ pub struct LazyKey {
1717
key: AtomicUsize,
1818
/// Destructor for the TLS value.
1919
dtor: Option<unsafe extern "C" fn(*mut u8)>,
20+
/// Next element of process-wide destructor list.
21+
#[cfg(not(target_thread_local))]
22+
next: atomic::AtomicPtr<LazyKey>,
2023
}
2124

2225
// Define a sentinel value that is likely not to be returned
@@ -32,18 +35,23 @@ const KEY_SENTVAL: usize = libc::PTHREAD_KEYS_MAX + 1;
3235
impl LazyKey {
3336
#[cfg_attr(bootstrap, rustc_const_unstable(feature = "thread_local_internals", issue = "none"))]
3437
pub const fn new(dtor: Option<unsafe extern "C" fn(*mut u8)>) -> LazyKey {
35-
LazyKey { key: atomic::AtomicUsize::new(KEY_SENTVAL), dtor }
38+
LazyKey {
39+
key: atomic::AtomicUsize::new(KEY_SENTVAL),
40+
dtor,
41+
#[cfg(not(target_thread_local))]
42+
next: atomic::AtomicPtr::new(crate::ptr::null_mut()),
43+
}
3644
}
3745

3846
#[inline]
39-
pub fn force(&self) -> super::Key {
47+
pub fn force(&'static self) -> super::Key {
4048
match self.key.load(Ordering::Acquire) {
4149
KEY_SENTVAL => self.lazy_init() as super::Key,
4250
n => n as super::Key,
4351
}
4452
}
4553

46-
fn lazy_init(&self) -> usize {
54+
fn lazy_init(&'static self) -> usize {
4755
// POSIX allows the key created here to be KEY_SENTVAL, but the compare_exchange
4856
// below relies on using KEY_SENTVAL as a sentinel value to check who won the
4957
// race to set the shared TLS key. As far as I know, there is no
@@ -71,7 +79,13 @@ impl LazyKey {
7179
Ordering::Acquire,
7280
) {
7381
// The CAS succeeded, so we've created the actual key
74-
Ok(_) => key as usize,
82+
Ok(_) => {
83+
#[cfg(not(target_thread_local))]
84+
if self.dtor.is_some() {
85+
unsafe { register_dtor(self) };
86+
}
87+
key as usize
88+
}
7589
// If someone beat us to the punch, use their key instead
7690
Err(n) => unsafe {
7791
super::destroy(key);
@@ -80,3 +94,57 @@ impl LazyKey {
8094
}
8195
}
8296
}
97+
98+
/// POSIX does not run TLS destructors on process exit.
99+
/// Thus we keep our own global list for that purpose.
100+
#[cfg(not(target_thread_local))]
101+
static DTORS: atomic::AtomicPtr<LazyKey> = atomic::AtomicPtr::new(crate::ptr::null_mut());
102+
103+
/// Registers destructor to run at process exit.
104+
#[cfg(not(target_thread_local))]
105+
unsafe fn register_dtor(key: &'static LazyKey) {
106+
crate::sys::thread_local::guard::enable();
107+
108+
let this = <*const LazyKey>::cast_mut(key);
109+
// Use acquire ordering to pass along the changes done by the previously
110+
// registered keys when we store the new head with release ordering.
111+
let mut head = DTORS.load(Ordering::Acquire);
112+
loop {
113+
key.next.store(head, Ordering::Relaxed);
114+
match DTORS.compare_exchange_weak(head, this, Ordering::Release, Ordering::Acquire) {
115+
Ok(_) => break,
116+
Err(new) => head = new,
117+
}
118+
}
119+
}
120+
121+
/// Run destructors at process exit.
122+
///
123+
/// SAFETY: This will and must only be run by the destructor callback in [`guard`].
124+
#[cfg(not(target_thread_local))]
125+
pub unsafe fn run_dtors() {
126+
for _ in 0..5 {
127+
let mut any_run = false;
128+
129+
// Use acquire ordering to observe key initialization.
130+
let mut cur = DTORS.load(Ordering::Acquire);
131+
while !cur.is_null() {
132+
let key = unsafe { (*cur).key.load(Ordering::Acquire) };
133+
let dtor = unsafe { (*cur).dtor.unwrap() };
134+
cur = unsafe { (*cur).next.load(Ordering::Relaxed) };
135+
136+
let ptr = unsafe { super::get(key as _) };
137+
if !ptr.is_null() {
138+
unsafe {
139+
super::set(key as _, crate::ptr::null_mut());
140+
dtor(ptr as *mut _);
141+
any_run = true;
142+
}
143+
}
144+
}
145+
146+
if !any_run {
147+
break;
148+
}
149+
}
150+
}

‎library/std/src/sys/thread_local/key/unix.rs

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,25 @@
11
use crate::mem;
22

3+
// For WASI add a few symbols not in upstream `libc` just yet.
4+
#[cfg(all(target_os = "wasi", target_env = "p1", target_feature = "atomics"))]
5+
mod libc {
6+
use crate::ffi;
7+
8+
#[allow(non_camel_case_types)]
9+
pub type pthread_key_t = ffi::c_uint;
10+
11+
extern "C" {
12+
pub fn pthread_key_create(
13+
key: *mut pthread_key_t,
14+
destructor: unsafe extern "C" fn(*mut ffi::c_void),
15+
) -> ffi::c_int;
16+
#[allow(dead_code)]
17+
pub fn pthread_getspecific(key: pthread_key_t) -> *mut ffi::c_void;
18+
pub fn pthread_setspecific(key: pthread_key_t, value: *const ffi::c_void) -> ffi::c_int;
19+
pub fn pthread_key_delete(key: pthread_key_t) -> ffi::c_int;
20+
}
21+
}
22+
323
pub type Key = libc::pthread_key_t;
424

525
#[inline]

‎library/std/src/sys/thread_local/mod.rs

Lines changed: 42 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -85,16 +85,16 @@ pub(crate) mod guard {
8585
} else if #[cfg(target_os = "windows")] {
8686
mod windows;
8787
pub(crate) use windows::enable;
88-
} else if #[cfg(any(
88+
} else if #[cfg(all(
8989
target_family = "wasm",
90-
target_os = "uefi",
91-
target_os = "zkvm",
90+
target_feature = "atomics",
91+
not(all(target_os = "wasi", target_env = "p1"))
9292
))] {
9393
pub(crate) fn enable() {
9494
// FIXME: Right now there is no concept of "thread exit" on
95-
// wasm, but this is likely going to show up at some point in
96-
// the form of an exported symbol that the wasm runtime is going
97-
// to be expected to call. For now we just leak everything, but
95+
// wasm-unknown-unknown, but this is likely going to show up at some
96+
// point in the form of an exported symbol that the wasm runtime is
97+
// going to be expected to call. For now we just leak everything, but
9898
// if such a function starts to exist it will probably need to
9999
// iterate the destructor list with these functions:
100100
#[cfg(all(target_family = "wasm", target_feature = "atomics"))]
@@ -113,6 +113,13 @@ pub(crate) mod guard {
113113
} else if #[cfg(target_os = "solid_asp3")] {
114114
mod solid;
115115
pub(crate) use solid::enable;
116+
} else if #[cfg(any(
117+
all(target_family = "wasm", not(target_feature = "atomics")),
118+
target_os = "uefi",
119+
target_os = "zkvm",
120+
))] {
121+
mod statik;
122+
pub(crate) use statik::enable;
116123
} else {
117124
mod key;
118125
pub(crate) use key::enable;
@@ -135,12 +142,15 @@ pub(crate) mod key {
135142
target_family = "unix",
136143
),
137144
target_os = "teeos",
145+
all(target_os = "wasi", target_env = "p1", target_feature = "atomics"),
138146
))] {
139147
mod racy;
140148
mod unix;
141149
#[cfg(test)]
142150
mod tests;
143151
pub(super) use racy::LazyKey;
152+
#[cfg(not(target_thread_local))]
153+
pub(super) use racy::run_dtors;
144154
pub(super) use unix::{Key, set};
145155
#[cfg(any(not(target_thread_local), test))]
146156
pub(super) use unix::get;
@@ -155,7 +165,7 @@ pub(crate) mod key {
155165
mod sgx;
156166
#[cfg(test)]
157167
mod tests;
158-
pub(super) use racy::LazyKey;
168+
pub(super) use racy::{LazyKey, run_dtors};
159169
pub(super) use sgx::{Key, get, set};
160170
use sgx::{create, destroy};
161171
} else if #[cfg(target_os = "xous")] {
@@ -171,6 +181,31 @@ pub(crate) mod key {
171181
}
172182
}
173183

184+
/// Process exit callback.
185+
///
186+
/// Some platforms (POSIX) do not run TLS destructors at process exit.
187+
/// Thus we need to register an exit callback to run them in that case.
188+
pub(crate) mod exit {
189+
cfg_if::cfg_if! {
190+
if #[cfg(any(
191+
all(
192+
not(target_vendor = "apple"),
193+
not(target_family = "wasm"),
194+
target_family = "unix",
195+
),
196+
target_os = "teeos",
197+
all(target_os = "wasi", target_env = "p1"),
198+
))] {
199+
mod unix;
200+
pub(super) use unix::at_process_exit;
201+
} else if #[cfg(target_family = "wasm")] {
202+
pub unsafe fn at_process_exit(cb: unsafe extern "C" fn()) {
203+
let _ = cb;
204+
}
205+
}
206+
}
207+
}
208+
174209
/// Run a callback in a scenario which must not unwind (such as a `extern "C"
175210
/// fn` declared in a user crate). If the callback unwinds anyway, then
176211
/// `rtabort` with a message about thread local panicking on drop.

0 commit comments

Comments
 (0)