fix overflow error in signed wrapping offset

Author: RalfJung

src/librustc/mir/interpret/pointer.rs

@@ -20,45 +20,48 @@ pub trait PointerArithmetic: layout::HasDataLayout {
         self.data_layout().pointer_size
     }
 
-    //// Trunace the given value to the pointer size; also return whether there was an overflow
+    /// Helper function: truncate given value-"overflowed flag" pair to pointer size and
+    /// update "overflowed flag" if there was an overflow.
+    /// This should be called by all the other methods before returning!
     #[inline]
-    fn truncate_to_ptr(&self, val: u128) -> (u64, bool) {
+    fn truncate_to_ptr(&self, (val, over): (u64, bool)) -> (u64, bool) {
+        let val = val as u128;
         let max_ptr_plus_1 = 1u128 << self.pointer_size().bits();
-        ((val % max_ptr_plus_1) as u64, val >= max_ptr_plus_1)
-    }
-
-    #[inline]
-    fn offset<'tcx>(&self, val: u64, i: u64) -> EvalResult<'tcx, u64> {
-        let (res, over) = self.overflowing_offset(val, i);
-        if over { err!(Overflow(mir::BinOp::Add)) } else { Ok(res) }
+        ((val % max_ptr_plus_1) as u64, over || val >= max_ptr_plus_1)
     }
 
     #[inline]
     fn overflowing_offset(&self, val: u64, i: u64) -> (u64, bool) {
-        let (res, over1) = val.overflowing_add(i);
-        let (res, over2) = self.truncate_to_ptr(u128::from(res));
-        (res, over1 || over2)
-    }
-
-    #[inline]
-    fn signed_offset<'tcx>(&self, val: u64, i: i64) -> EvalResult<'tcx, u64> {
-        let (res, over) = self.overflowing_signed_offset(val, i128::from(i));
-        if over { err!(Overflow(mir::BinOp::Add)) } else { Ok(res) }
+        let res = val.overflowing_add(i);
+        self.truncate_to_ptr(res)
     }
 
     // Overflow checking only works properly on the range from -u64 to +u64.
     #[inline]
     fn overflowing_signed_offset(&self, val: u64, i: i128) -> (u64, bool) {
         // FIXME: is it possible to over/underflow here?
         if i < 0 {
-            // trickery to ensure that i64::min_value() works fine
-            // this formula only works for true negative values, it panics for zero!
+            // Trickery to ensure that i64::min_value() works fine: compute n = -i.
+            // This formula only works for true negative values, it overflows for zero!
             let n = u64::max_value() - (i as u64) + 1;
-            val.overflowing_sub(n)
+            let res = val.overflowing_sub(n);
+            self.truncate_to_ptr(res)
         } else {
             self.overflowing_offset(val, i as u64)
         }
     }
+
+    #[inline]
+    fn offset<'tcx>(&self, val: u64, i: u64) -> EvalResult<'tcx, u64> {
+        let (res, over) = self.overflowing_offset(val, i);
+        if over { err!(Overflow(mir::BinOp::Add)) } else { Ok(res) }
+    }
+
+    #[inline]
+    fn signed_offset<'tcx>(&self, val: u64, i: i64) -> EvalResult<'tcx, u64> {
+        let (res, over) = self.overflowing_signed_offset(val, i128::from(i));
+        if over { err!(Overflow(mir::BinOp::Add)) } else { Ok(res) }
+    }
 }
 
 impl<T: layout::HasDataLayout> PointerArithmetic for T {}