diff --git a/src/bounded.rs b/src/bounded.rs index b98846c..7a7fb71 100644 --- a/src/bounded.rs +++ b/src/bounded.rs @@ -129,7 +129,8 @@ impl Bounded { } } } else if stamp.wrapping_add(self.one_lap) == tail + 1 { - let head = crate::full_fence_for_load(|| self.head.load(Ordering::Relaxed)); + crate::full_fence(); + let head = self.head.load(Ordering::Relaxed); // If the head lags one lap behind the tail as well... if head.wrapping_add(self.one_lap) == tail { @@ -190,7 +191,8 @@ impl Bounded { } } } else if stamp == head { - let tail = crate::full_fence_for_load(|| self.tail.load(Ordering::Relaxed)); + crate::full_fence(); + let tail = self.tail.load(Ordering::Relaxed); // If the tail equals the head, that means the queue is empty. if (tail & !self.mark_bit) == head { diff --git a/src/lib.rs b/src/lib.rs index 3133e6b..d54cee5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -445,7 +445,7 @@ impl fmt::Display for PushError { /// Equivalent to `atomic::fence(Ordering::SeqCst)`, but in some cases faster. #[inline] -fn full_fence_for_load(load_op: impl FnOnce() -> T) -> T { +fn full_fence() { if cfg!(all( any(target_arch = "x86", target_arch = "x86_64"), not(miri) @@ -462,13 +462,11 @@ fn full_fence_for_load(load_op: impl FnOnce() -> T) -> T { // The ideal solution here would be to use inline assembly, but we're instead creating a // temporary atomic variable and compare-and-exchanging its value. No sane compiler to // x86 platforms is going to optimize this away. + atomic::compiler_fence(Ordering::SeqCst); let a = AtomicUsize::new(0); let _ = a.compare_exchange(0, 1, Ordering::SeqCst, Ordering::SeqCst); - // On x86, `lock cmpxchg; mov` is fine. See also https://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html. - load_op() + atomic::compiler_fence(Ordering::SeqCst); } else { - let res = load_op(); atomic::fence(Ordering::SeqCst); - res } } diff --git a/src/unbounded.rs b/src/unbounded.rs index df1b674..c7cba68 100644 --- a/src/unbounded.rs +++ b/src/unbounded.rs @@ -237,7 +237,8 @@ impl Unbounded { let mut new_head = head + (1 << SHIFT); if new_head & MARK_BIT == 0 { - let tail = crate::full_fence_for_load(|| self.tail.index.load(Ordering::Relaxed)); + crate::full_fence(); + let tail = self.tail.index.load(Ordering::Relaxed); // If the tail equals the head, that means the queue is empty. if head >> SHIFT == tail >> SHIFT {