diff --git a/src/helpers.rs b/src/helpers.rs index ba12e0a7e3..fe2f33ffd3 100644 --- a/src/helpers.rs +++ b/src/helpers.rs @@ -775,18 +775,3 @@ pub fn isolation_abort_error(name: &str) -> InterpResult<'static> { name, ))) } - -pub fn bool_to_simd_element(b: bool, size: Size) -> Scalar { - // SIMD uses all-1 as pattern for "true" - let val = if b { -1 } else { 0 }; - Scalar::from_int(val, size) -} - -pub fn simd_element_to_bool<'tcx>(elem: ImmTy<'tcx, Tag>) -> InterpResult<'tcx, bool> { - let val = elem.to_scalar()?.to_int(elem.layout.size)?; - Ok(match val { - 0 => false, - -1 => true, - _ => throw_ub_format!("each element of a SIMD mask must be all-0-bits or all-1-bits"), - }) -} diff --git a/src/shims/intrinsics.rs b/src/shims/intrinsics.rs index 726e6b6b96..c344d0ff9c 100644 --- a/src/shims/intrinsics.rs +++ b/src/shims/intrinsics.rs @@ -1,3 +1,4 @@ +use std::convert::TryInto; use std::iter; use log::trace; @@ -5,10 +6,10 @@ use log::trace; use rustc_apfloat::{Float, Round}; use rustc_middle::ty::layout::{HasParamEnv, IntegerExt, LayoutOf}; use rustc_middle::{mir, mir::BinOp, ty, ty::FloatTy}; -use rustc_target::abi::{Align, Integer}; +use rustc_target::abi::{Align, Endian, HasDataLayout, Integer, Size}; use crate::*; -use helpers::{bool_to_simd_element, check_arg_count, simd_element_to_bool}; +use helpers::check_arg_count; pub enum AtomicOp { MirOp(mir::BinOp, bool), @@ -663,6 +664,45 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx this.write_immediate(*val, &dest.into())?; } } + "simd_select_bitmask" => { + let &[ref mask, ref yes, ref no] = check_arg_count(args)?; + let (yes, yes_len) = this.operand_to_simd(yes)?; + let (no, no_len) = this.operand_to_simd(no)?; + let (dest, dest_len) = this.place_to_simd(dest)?; + let bitmask_len = dest_len.max(8); + + assert!(mask.layout.ty.is_integral()); + assert!(bitmask_len <= 64); + assert_eq!(bitmask_len, mask.layout.size.bits()); + assert_eq!(dest_len, yes_len); + assert_eq!(dest_len, no_len); + + let mask: u64 = this + .read_scalar(mask)? + .check_init()? + .to_bits(mask.layout.size)? + .try_into() + .unwrap(); + for i in 0..dest_len { + let mask = + mask & (1 << simd_bitmask_index(i, dest_len, this.data_layout().endian)); + let yes = this.read_immediate(&this.mplace_index(&yes, i)?.into())?; + let no = this.read_immediate(&this.mplace_index(&no, i)?.into())?; + let dest = this.mplace_index(&dest, i)?; + + let val = if mask != 0 { yes } else { no }; + this.write_immediate(*val, &dest.into())?; + } + for i in dest_len..bitmask_len { + // If the mask is "padded", ensure that padding is all-zero. + let mask = mask & (1 << i); + if mask != 0 { + throw_ub_format!( + "a SIMD bitmask less than 8 bits long must be filled with 0s for the remaining bits" + ); + } + } + } #[rustfmt::skip] "simd_cast" | "simd_as" => { let &[ref op] = check_arg_count(args)?; @@ -787,6 +827,24 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx } } } + "simd_bitmask" => { + let &[ref op] = check_arg_count(args)?; + let (op, op_len) = this.operand_to_simd(op)?; + let bitmask_len = op_len.max(8); + + assert!(dest.layout.ty.is_integral()); + assert!(bitmask_len <= 64); + assert_eq!(bitmask_len, dest.layout.size.bits()); + + let mut res = 0u64; + for i in 0..op_len { + let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?; + if simd_element_to_bool(op)? { + res |= 1 << simd_bitmask_index(i, op_len, this.data_layout().endian); + } + } + this.write_int(res, dest)?; + } // Atomic operations "atomic_load" => this.atomic_load(args, dest, AtomicReadOp::SeqCst)?, @@ -1307,3 +1365,26 @@ fn fmin_op<'tcx>( FloatTy::F64 => Scalar::from_f64(left.to_f64()?.min(right.to_f64()?)), }) } + +fn bool_to_simd_element(b: bool, size: Size) -> Scalar { + // SIMD uses all-1 as pattern for "true" + let val = if b { -1 } else { 0 }; + Scalar::from_int(val, size) +} + +fn simd_element_to_bool<'tcx>(elem: ImmTy<'tcx, Tag>) -> InterpResult<'tcx, bool> { + let val = elem.to_scalar()?.to_int(elem.layout.size)?; + Ok(match val { + 0 => false, + -1 => true, + _ => throw_ub_format!("each element of a SIMD mask must be all-0-bits or all-1-bits"), + }) +} + +fn simd_bitmask_index(idx: u64, vec_len: u64, endianess: Endian) -> u64 { + assert!(idx < vec_len); + match endianess { + Endian::Little => idx, + Endian::Big => vec_len - 1 - idx, // reverse order of bits + } +} diff --git a/tests/compile-fail/intrinsics/simd-select-bitmask-invalid.rs b/tests/compile-fail/intrinsics/simd-select-bitmask-invalid.rs new file mode 100644 index 0000000000..ab69072c30 --- /dev/null +++ b/tests/compile-fail/intrinsics/simd-select-bitmask-invalid.rs @@ -0,0 +1,15 @@ +#![feature(platform_intrinsics, repr_simd)] + +extern "platform-intrinsic" { + fn simd_select_bitmask(m: M, yes: T, no: T) -> T; +} + +#[repr(simd)] +#[allow(non_camel_case_types)] +#[derive(Copy, Clone)] +struct i32x2(i32, i32); + +fn main() { unsafe { + let x = i32x2(0, 1); + simd_select_bitmask(0b11111111u8, x, x); //~ERROR bitmask less than 8 bits long must be filled with 0s for the remaining bits +} } diff --git a/tests/compile-fail/intrinsics/simd-select-invalid-bool.rs b/tests/compile-fail/intrinsics/simd-select-invalid-bool.rs new file mode 100644 index 0000000000..98f67cfcd7 --- /dev/null +++ b/tests/compile-fail/intrinsics/simd-select-invalid-bool.rs @@ -0,0 +1,15 @@ +#![feature(platform_intrinsics, repr_simd)] + +extern "platform-intrinsic" { + fn simd_select(m: M, yes: T, no: T) -> T; +} + +#[repr(simd)] +#[allow(non_camel_case_types)] +#[derive(Copy, Clone)] +struct i32x2(i32, i32); + +fn main() { unsafe { + let x = i32x2(0, 1); + simd_select(x, x, x); //~ERROR must be all-0-bits or all-1-bits +} } diff --git a/tests/run-pass/portable-simd.rs b/tests/run-pass/portable-simd.rs index 80b0b4556c..99a64ea370 100644 --- a/tests/run-pass/portable-simd.rs +++ b/tests/run-pass/portable-simd.rs @@ -187,6 +187,21 @@ fn simd_mask() { let intmask = Mask::from_int(i32x4::from_array([0, -1, 0, 0])); assert_eq!(intmask, Mask::from_array([false, true, false, false])); assert_eq!(intmask.to_array(), [false, true, false, false]); + + let values = [ + true, false, false, true, false, false, true, false, true, true, false, false, false, true, + false, true, + ]; + let mask = Mask::::from_array(values); + let bitmask = mask.to_bitmask(); + assert_eq!(bitmask, 0b1010001101001001); + assert_eq!(Mask::::from_bitmask(bitmask), mask); + + let values = [false, false, false, true]; + let mask = Mask::::from_array(values); + let bitmask = mask.to_bitmask(); + // FIXME fails until https://github.com/rust-lang/portable-simd/pull/267 lands: assert_eq!(bitmask, 0b1000); + assert_eq!(Mask::::from_bitmask(bitmask), mask); } fn simd_cast() {