From 035fdfc7c2b0972795f67fd2662b18dfe82d524e Mon Sep 17 00:00:00 2001 From: Thayne McCombs Date: Wed, 13 Mar 2019 00:30:43 -0600 Subject: [PATCH] Add element-wise atomic memory operations WIP --- src/libcore/intrinsics.rs | 27 ++++++ src/librustc_codegen_llvm/builder.rs | 42 +++++++++ src/librustc_codegen_llvm/intrinsic.rs | 86 ++++++++++++++++--- src/librustc_codegen_llvm/llvm/ffi.rs | 22 +++++ src/librustc_codegen_ssa/traits/builder.rs | 27 ++++++ src/librustc_typeck/check/intrinsic.rs | 7 +- src/rustllvm/RustWrapper.cpp | 36 ++++++++ .../run-pass/intrinsics/intrinsic-atomics.rs | 15 ++++ 8 files changed, 245 insertions(+), 17 deletions(-) diff --git a/src/libcore/intrinsics.rs b/src/libcore/intrinsics.rs index 75a33394e3d2f..44ca468d4f10e 100644 --- a/src/libcore/intrinsics.rs +++ b/src/libcore/intrinsics.rs @@ -962,6 +962,33 @@ extern "rust-intrinsic" { /// value is not necessarily valid to be used to actually access memory. pub fn arith_offset(dst: *const T, offset: isize) -> *const T; + /// Equivalent to the appropriate `llvm.memcpy.element.unordered.atomic.p0i8.p0i8.*` intrinsic, with + /// a size of `count` * `size_of::()`, an alignment of + /// `min_align_of::()`, and an element size of `size_of::()`. + /// + /// `size_of::` must be an integer power of two no larger than the + /// target-specific atomic access size limit. + #[cfg(not(stage0))] + pub fn atomic_element_copy_nonoverlapping_memory_unordered(dst: *mut T, src: *const T, count: usize); + + /// Equivalent to the appropriate `llvm.memmove.unordered.atomic.p0i8.p0i8.*` intrinsic, with + /// a size of `count` * `size_of::()`, an alignment of + /// `min_align_of::()`, and an element size of `size_of::()`. + /// + /// `size_of::` must be an integer power of two no larger than the + /// target-specific atomic access size limit. + #[cfg(not(stage0))] + pub fn atomic_element_copy_memory_unordered(dst: *mut T, src: *const T, count: usize); + + /// Equivalent to the appropriate `llvm.memset.unordered.atomic.p0i8.p0i8.*` intrinsic, with + /// a size of `count` * `size_of::()`, an alignment of + /// `min_align_of::()`, and an element size of `size_of::()`. + /// + /// `size_of::` must be an integer power of two no larger than the + /// target-specific atomic access size limit. + #[cfg(not(stage0))] + pub fn atomic_element_set_memory_unordered(dst: *mut T, val: u8, count: usize); + /// Equivalent to the appropriate `llvm.memcpy.p0i8.0i8.*` intrinsic, with /// a size of `count` * `size_of::()` and an alignment of /// `min_align_of::()` diff --git a/src/librustc_codegen_llvm/builder.rs b/src/librustc_codegen_llvm/builder.rs index 123fda1e215ff..6e7b213f59470 100644 --- a/src/librustc_codegen_llvm/builder.rs +++ b/src/librustc_codegen_llvm/builder.rs @@ -966,6 +966,48 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { self.call(llintrinsicfn, &[ptr, fill_byte, size, align, volatile], None); } + fn atomic_element_unordered_memcpy(&mut self, dst: &'ll Value, dst_align: Align, + src: &'ll Value, src_align: Align, + size: &'ll Value, element_size: u32) { + let size = self.intcast(size, self.type_isize(), false); + let dst = self.pointercast(dst, self.type_i8p()); + let src = self.pointercast(src, self.type_i8p()); + unsafe { + llvm::LLVMRustBuildElementUnorderedAtomicMemCpy(self.llbuilder, dst, dst_align.bytes() as c_uint, + src, src_align.bytes() as c_uint, size, element_size); + } + } + + fn atomic_element_unordered_memmove(&mut self, dst: &'ll Value, dst_align: Align, + src: &'ll Value, src_align: Align, + size: &'ll Value, element_size: u32) { + let size = self.intcast(size, self.type_isize(), false); + let dst = self.pointercast(dst, self.type_i8p()); + let src = self.pointercast(src, self.type_i8p()); + let ret_ref = unsafe { + llvm::LLVMRustBuildElementUnorderedAtomicMemMove(self.llbuilder, + dst, dst_align.bytes() as c_uint, + src, src_align.bytes() as c_uint, + size, element_size) + }; + if ret_ref.is_none() { + bug!("llvm.memmove.element.unordered.atomic.* is not supported with LLVM prior to 7.0"); + } + } + + fn atomic_element_unordered_memset(&mut self, ptr: &'ll Value, fill_byte: &'ll Value, + size: &'ll Value, align: Align, element_size: u32) { + let size = self.intcast(size, self.type_isize(), false); + let ptr = self.pointercast(ptr, self.type_i8p()); + let ret_ref = unsafe { + llvm::LLVMRustBuildElementUnorderedAtomicMemSet(self.llbuilder, ptr, fill_byte, + size, align.bytes() as c_uint, element_size) + }; + if ret_ref.is_none() { + bug!("llvm.memset.element.unordered.atomic.* is not supported with LLVM prior to 7.0"); + } + } + fn select( &mut self, cond: &'ll Value, then_val: &'ll Value, diff --git a/src/librustc_codegen_llvm/intrinsic.rs b/src/librustc_codegen_llvm/intrinsic.rs index ceb08f943678b..a1dc096e49292 100644 --- a/src/librustc_codegen_llvm/intrinsic.rs +++ b/src/librustc_codegen_llvm/intrinsic.rs @@ -102,6 +102,13 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { let llret_ty = self.layout_of(ret_ty).llvm_type(self); let result = PlaceRef::new_sized(llresult, fn_ty.ret.layout, fn_ty.ret.layout.align.abi); + let invalid_integer_monomorphization = |ty| { + span_invalid_monomorphization_error(tcx.sess, span, + &format!("invalid monomorphization of `{}` intrinsic: \ + expected basic integer type, found `{}`", name, ty)); + }; + + let simple = get_simple_intrinsic(self, name); let llval = match name { _ if simple.is_some() => { @@ -503,10 +510,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { _ => bug!(), }, None => { - span_invalid_monomorphization_error( - tcx.sess, span, - &format!("invalid monomorphization of `{}` intrinsic: \ - expected basic integer type, found `{}`", name, ty)); + invalid_integer_monomorphization(ty); return; } } @@ -548,6 +552,17 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { Err(()) => return } } + name if name.starts_with("atomic_element_") => { + let ty = substs.type_at(0); + if int_type_width_signed(ty, self).is_some() { + atomic_element_intrinsic(self, name, + substs.type_at(0), + args); + return; + } else { + return invalid_integer_monomorphization(ty); + } + } // This requires that atomic intrinsics follow a specific naming pattern: // "atomic_[_]", and no ordering means SeqCst name if name.starts_with("atomic_") => { @@ -582,12 +597,6 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { _ => self.sess().fatal("Atomic intrinsic not in correct format"), }; - let invalid_monomorphization = |ty| { - span_invalid_monomorphization_error(tcx.sess, span, - &format!("invalid monomorphization of `{}` intrinsic: \ - expected basic integer type, found `{}`", name, ty)); - }; - match split[1] { "cxchg" | "cxchgweak" => { let ty = substs.type_at(0); @@ -610,7 +619,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { self.store(success, dest.llval, dest.align); return; } else { - return invalid_monomorphization(ty); + return invalid_integer_monomorphization(ty); } } @@ -620,7 +629,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { let size = self.size_of(ty); self.atomic_load(args[0].immediate(), order, size) } else { - return invalid_monomorphization(ty); + return invalid_integer_monomorphization(ty); } } @@ -636,7 +645,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { ); return; } else { - return invalid_monomorphization(ty); + return invalid_integer_monomorphization(ty); } } @@ -676,7 +685,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { order ) } else { - return invalid_monomorphization(ty); + return invalid_integer_monomorphization(ty); } } } @@ -754,6 +763,54 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { } } +fn atomic_element_intrinsic( + bx: &mut Builder<'a, 'll, 'tcx>, + name: &str, + ty: Ty<'tcx>, + args: &[OperandRef<'tcx, &'ll Value>], +) { + let (element_size, align) = bx.size_and_align_of(ty); + let element_size = element_size.bytes(); + assert!(element_size <= u32::max_value() as u64); + + let size = bx.mul(bx.const_usize(element_size), args[2].immediate()); + + match name { + "atomic_element_copy_nonoverlapping_memory_unordered" => { + bx.atomic_element_unordered_memcpy( + args[0].immediate(), + align, + args[1].immediate(), + align, + size, + element_size as u32 + ); + } + "atomic_element_copy_memory_unordered" => { + bx.atomic_element_unordered_memmove( + args[0].immediate(), + align, + args[1].immediate(), + align, + size, + element_size as u32 + ); + } + "atomic_element_set_memory_unordered" => { + bx.atomic_element_unordered_memset( + args[0].immediate(), + args[1].immediate(), + size, + align, + element_size as u32 + ); + } + _ => { + bug!("unknown intrinsic '{}'", name); + } + } +} + fn copy_intrinsic( bx: &mut Builder<'a, 'll, 'tcx>, allow_overlap: bool, @@ -777,6 +834,7 @@ fn copy_intrinsic( } } + fn memset_intrinsic( bx: &mut Builder<'a, 'll, 'tcx>, volatile: bool, diff --git a/src/librustc_codegen_llvm/llvm/ffi.rs b/src/librustc_codegen_llvm/llvm/ffi.rs index 2ad6d9c053a20..924358d77ee78 100644 --- a/src/librustc_codegen_llvm/llvm/ffi.rs +++ b/src/librustc_codegen_llvm/llvm/ffi.rs @@ -1154,6 +1154,28 @@ extern "C" { Size: &'a Value, IsVolatile: bool) -> &'a Value; + pub fn LLVMRustBuildElementUnorderedAtomicMemCpy(B: &Builder<'a>, + Dst: &'a Value, + DstAlign: c_uint, + Src: &'a Value, + SrcAlign: c_uint, + Size: &'a Value, + ElementSize: u32) + -> &'a Value; + pub fn LLVMRustBuildElementUnorderedAtomicMemMove(B: &Builder<'a>, + Dst: &'a Value, + DstAlign: c_uint, + Src: &'a Value, + SrcAlign: c_uint, + Size: &'a Value, + ElementSize: u32) -> Option<&'a Value>; + pub fn LLVMRustBuildElementUnorderedAtomicMemSet(B: &Builder<'a>, + Ptr: &'a Value, + Val: &'a Value, + Size: &'a Value, + Align: c_uint, + ElementSize: u32) -> Option<&'a Value>; + pub fn LLVMRustBuildMemMove(B: &Builder<'a>, Dst: &'a Value, DstAlign: c_uint, diff --git a/src/librustc_codegen_ssa/traits/builder.rs b/src/librustc_codegen_ssa/traits/builder.rs index 48142fc9fa9f4..12775492d8237 100644 --- a/src/librustc_codegen_ssa/traits/builder.rs +++ b/src/librustc_codegen_ssa/traits/builder.rs @@ -190,6 +190,33 @@ pub trait BuilderMethods<'a, 'tcx: 'a>: flags: MemFlags, ); + fn atomic_element_unordered_memcpy( + &mut self, + dst: Self::Value, + dst_align: Align, + src: Self::Value, + src_align: Align, + size: Self::Value, + element_size: u32, + ); + fn atomic_element_unordered_memmove( + &mut self, + dst: Self::Value, + dst_align: Align, + src: Self::Value, + src_align: Align, + size: Self::Value, + element_size: u32, + ); + fn atomic_element_unordered_memset( + &mut self, + ptr: Self::Value, + fill_byte: Self::Value, + size: Self::Value, + align: Align, + element_size: u32, + ); + fn select( &mut self, cond: Self::Value, diff --git a/src/librustc_typeck/check/intrinsic.rs b/src/librustc_typeck/check/intrinsic.rs index 40c60caffa42d..2a26098677ec5 100644 --- a/src/librustc_typeck/check/intrinsic.rs +++ b/src/librustc_typeck/check/intrinsic.rs @@ -92,7 +92,7 @@ pub fn check_intrinsic_type<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, }) }; - let (n_tps, inputs, output, unsafety) = if name.starts_with("atomic_") { + let (n_tps, inputs, output, unsafety) = if name.starts_with("atomic_") && ! name.starts_with("atomic_element_") { let split : Vec<&str> = name.split('_').collect(); assert!(split.len() >= 2, "Atomic intrinsic in an incorrect format"); @@ -197,7 +197,8 @@ pub fn check_intrinsic_type<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, ], tcx.mk_unit()) } - "volatile_copy_memory" | "volatile_copy_nonoverlapping_memory" => { + "volatile_copy_memory" | "volatile_copy_nonoverlapping_memory" | + "atomic_element_copy_memory_unordered" | "atomic_element_copy_nonoverlapping_memory_unordered" => { (1, vec![ tcx.mk_ptr(ty::TypeAndMut { @@ -212,7 +213,7 @@ pub fn check_intrinsic_type<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, ], tcx.mk_unit()) } - "write_bytes" | "volatile_set_memory" => { + "write_bytes" | "volatile_set_memory" | "atomic_element_set_memory_unordered" => { (1, vec![ tcx.mk_ptr(ty::TypeAndMut { diff --git a/src/rustllvm/RustWrapper.cpp b/src/rustllvm/RustWrapper.cpp index a00417a362927..5fe8b68ee9547 100644 --- a/src/rustllvm/RustWrapper.cpp +++ b/src/rustllvm/RustWrapper.cpp @@ -1268,6 +1268,42 @@ extern "C" LLVMValueRef LLVMRustBuildMemMove(LLVMBuilderRef B, #endif } +extern "C" LLVMValueRef LLVMRustBuildElementUnorderedAtomicMemCpy(LLVMBuilderRef B, + LLVMValueRef Dst, unsigned DstAlign, + LLVMValueRef Src, unsigned SrcAlign, + LLVMValueRef Size, uint32_t ElementSize) { + return wrap(unwrap(B)->CreateElementUnorderedAtomicMemCpy( + unwrap(Dst), DstAlign, + unwrap(Src), SrcAlign, + unwrap(Size), ElementSize)); +} + +extern "C" LLVMValueRef LLVMRustBuildElementUnorderedAtomicMemMove(LLVMBuilderRef B, + LLVMValueRef Dst, unsigned DstAlign, + LLVMValueRef Src, unsigned SrcAlign, + LLVMValueRef Size, uint32_t ElementSize) { +#if LLVM_VERSION_GE(7, 0) + return wrap(unwrap(B)->CreateElementUnorderedAtomicMemMove( + unwrap(Dst), DstAlign, + unwrap(Src), SrcAlign, + unwrap(Size), ElementSize)); +#else + return nullptr; +#endif +} + +extern "C" LLVMValueRef LLVMRustBuildElementUnorderedAtomicMemSet(LLVMBuilderRef B, + LLVMValueRef Ptr, LLVMValueRef Val, + LLVMValueRef Size, unsigned Align, uint32_t ElementSize) { +#if LLVM_VERSION_GE(7, 0) + return wrap(unwrap(B)->CreateElementUnorderedAtomicMemSet( + unwrap(Ptr), unwrap(Val), + unwrap(Size), Align, ElementSize)); +#else + return nullptr; +#endif +} + extern "C" LLVMValueRef LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMValueRef Fn, LLVMValueRef *Args, unsigned NumArgs, LLVMBasicBlockRef Then, diff --git a/src/test/run-pass/intrinsics/intrinsic-atomics.rs b/src/test/run-pass/intrinsics/intrinsic-atomics.rs index 608cf3dee5230..f2179395f1212 100644 --- a/src/test/run-pass/intrinsics/intrinsic-atomics.rs +++ b/src/test/run-pass/intrinsics/intrinsic-atomics.rs @@ -29,6 +29,10 @@ mod rusti { pub fn atomic_xsub(dst: *mut T, src: T) -> T; pub fn atomic_xsub_acq(dst: *mut T, src: T) -> T; pub fn atomic_xsub_rel(dst: *mut T, src: T) -> T; + + pub fn atomic_element_copy_nonoverlapping_memory_unordered(dst: *mut T, src: *const T, count: usize); + pub fn atomic_element_copy_memory_unordered(dst: *mut T, src: *const T, count: usize); + pub fn atomic_element_set_memory_unordered(dst: *mut T, val: u8, count: usize); } } @@ -99,5 +103,16 @@ pub fn main() { } } assert_eq!(*x, 3); + + let mut mem = [0; 4]; + let src = [1, 2, 3, 4]; + let dst = &mut mem[0] as *mut _; + + rusti::atomic_element_copy_nonoverlapping_memory_unordered(dst, &src[0], 4); + assert_eq!(mem, src); + rusti::atomic_element_copy_memory_unordered(dst, &mem[1], 2); + assert_eq!(mem, [2, 3, 3, 4]); + rusti::atomic_element_set_memory_unordered(dst, 1, 3); + assert_eq!(mem, [1, 1, 1, 4]); } }