Skip to content

Commit 8d257b9

Browse files
borsbors
committed
Auto merge of rust-lang#117473 - saethlin:codegen-alignment-checks, r=<try>
Move alignment checks to codegen Implementing UB checks entirely in a MIR transform is quite limiting, we don't know for sure what all our types are so we need to make a lot of sacrifices. For example in here we used to emit MIR to compute the alignment mask at runtime, because the pointee type could be generic. Implementing the checks in codegen frees us from that requirement, because we get to deal with monomorphized types. But I don't think we can move these checks entirely into codegen, because inserting the check needs to insert a new terminator into a basic block, which splits the previous basic block into two. We can't add control flow like this in codegen, but we can in MIR. So now the MIR transform just inserts a `TerminatorKind::UbCheck` which is effectively a `Goto` that also reads an `Operand` (because it either goes to the target block or terminates), and codegen expands that new terminator into the actual check. --- Also I'm writing this with the expectation that I implement the niche checks in the same manner, because they have the same problem with polymorphic MIR, possibly worse. r? `@ghost`
2 parents 2c1b65e + 165048a commit 8d257b9

File tree

41 files changed

+252
-255
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+252
-255
lines changed

‎compiler/rustc_borrowck/src/invalidation.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,8 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
205205
| TerminatorKind::UnwindTerminate(_)
206206
| TerminatorKind::Unreachable
207207
| TerminatorKind::FalseEdge { real_target: _, imaginary_target: _ }
208-
| TerminatorKind::FalseUnwind { real_target: _, unwind: _ } => {
208+
| TerminatorKind::FalseUnwind { real_target: _, unwind: _ }
209+
| TerminatorKind::UbCheck { .. } => {
209210
// no data used, thus irrelevant to borrowck
210211
}
211212
}

‎compiler/rustc_borrowck/src/lib.rs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -785,7 +785,8 @@ impl<'cx, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx, R> for MirBorro
785785
| TerminatorKind::Return
786786
| TerminatorKind::CoroutineDrop
787787
| TerminatorKind::FalseEdge { real_target: _, imaginary_target: _ }
788-
| TerminatorKind::FalseUnwind { real_target: _, unwind: _ } => {
788+
| TerminatorKind::FalseUnwind { real_target: _, unwind: _ }
789+
| TerminatorKind::UbCheck { .. } => {
789790
// no data used, thus irrelevant to borrowck
790791
}
791792
}
@@ -835,7 +836,8 @@ impl<'cx, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx, R> for MirBorro
835836
| TerminatorKind::Goto { .. }
836837
| TerminatorKind::SwitchInt { .. }
837838
| TerminatorKind::Unreachable
838-
| TerminatorKind::InlineAsm { .. } => {}
839+
| TerminatorKind::InlineAsm { .. }
840+
| TerminatorKind::UbCheck { .. } => {}
839841
}
840842
}
841843
}

‎compiler/rustc_borrowck/src/type_check/mod.rs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1358,7 +1358,8 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
13581358
| TerminatorKind::Drop { .. }
13591359
| TerminatorKind::FalseEdge { .. }
13601360
| TerminatorKind::FalseUnwind { .. }
1361-
| TerminatorKind::InlineAsm { .. } => {
1361+
| TerminatorKind::InlineAsm { .. }
1362+
| TerminatorKind::UbCheck { .. } => {
13621363
// no checks needed for these
13631364
}
13641365

@@ -1690,6 +1691,9 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
16901691
}
16911692
self.assert_iscleanup_unwind(body, block_data, unwind, is_cleanup);
16921693
}
1694+
TerminatorKind::UbCheck { target, .. } => {
1695+
self.assert_iscleanup(body, block_data, target, is_cleanup)
1696+
}
16931697
}
16941698
}
16951699

‎compiler/rustc_codegen_cranelift/src/base.rs

Lines changed: 34 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -356,18 +356,6 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
356356
source_info.span,
357357
);
358358
}
359-
AssertKind::MisalignedPointerDereference { ref required, ref found } => {
360-
let required = codegen_operand(fx, required).load_scalar(fx);
361-
let found = codegen_operand(fx, found).load_scalar(fx);
362-
let location = fx.get_caller_location(source_info).load_scalar(fx);
363-
364-
codegen_panic_inner(
365-
fx,
366-
rustc_hir::LangItem::PanicMisalignedPointerDereference,
367-
&[required, found, location],
368-
source_info.span,
369-
);
370-
}
371359
_ => {
372360
let msg_str = msg.description();
373361
codegen_panic(fx, msg_str, source_info);
@@ -488,6 +476,40 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
488476
let target_block = fx.get_block(*target);
489477
fx.bcx.ins().jump(target_block, &[]);
490478
}
479+
TerminatorKind::UbCheck { target, kind: UbCheckKind::PointerAlignment { pointer } } => {
480+
let location = fx.get_caller_location(source_info).load_scalar(fx);
481+
let pointer = codegen_operand(fx, pointer);
482+
let pointee_ty = pointer.layout().ty.builtin_deref(true).unwrap();
483+
let pointee_layout = fx.layout_of(pointee_ty.ty);
484+
let pointer = pointer.load_scalar(fx);
485+
486+
// Compute the alignment mask
487+
let align = pointee_layout.align.abi.bytes() as i64;
488+
let mask = fx.bcx.ins().iconst(fx.pointer_type, align - 1);
489+
let required = fx.bcx.ins().iconst(fx.pointer_type, align);
490+
491+
// And the pointer with the mask
492+
let masked = fx.bcx.ins().band(pointer, mask);
493+
494+
// Branch on whether the masked value is zero
495+
let is_zero = fx.bcx.ins().icmp_imm(IntCC::Equal, masked, 0);
496+
497+
// Create a new block that we will jump to if !is_zero
498+
let failure = fx.bcx.create_block();
499+
fx.bcx.set_cold_block(failure);
500+
501+
let target = fx.get_block(*target);
502+
fx.bcx.ins().brif(is_zero, target, &[], failure, &[]);
503+
504+
// Codegen the actual panic
505+
fx.bcx.switch_to_block(failure);
506+
codegen_panic_inner(
507+
fx,
508+
rustc_hir::LangItem::PanicMisalignedPointerDereference,
509+
&[required, pointer, location],
510+
source_info.span,
511+
);
512+
}
491513
};
492514
}
493515
}

‎compiler/rustc_codegen_cranelift/src/constant.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -508,7 +508,8 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
508508
| TerminatorKind::Return
509509
| TerminatorKind::Unreachable
510510
| TerminatorKind::Drop { .. }
511-
| TerminatorKind::Assert { .. } => {}
511+
| TerminatorKind::Assert { .. }
512+
| TerminatorKind::UbCheck { .. } => {}
512513
TerminatorKind::Yield { .. }
513514
| TerminatorKind::CoroutineDrop
514515
| TerminatorKind::FalseEdge { .. }

‎compiler/rustc_codegen_ssa/src/mir/analyze.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -277,7 +277,8 @@ pub fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, CleanupKi
277277
| TerminatorKind::SwitchInt { .. }
278278
| TerminatorKind::Yield { .. }
279279
| TerminatorKind::FalseEdge { .. }
280-
| TerminatorKind::FalseUnwind { .. } => { /* nothing to do */ }
280+
| TerminatorKind::FalseUnwind { .. }
281+
| TerminatorKind::UbCheck { .. } => { /* nothing to do */ }
281282
TerminatorKind::Call { unwind, .. }
282283
| TerminatorKind::InlineAsm { unwind, .. }
283284
| TerminatorKind::Assert { unwind, .. }

‎compiler/rustc_codegen_ssa/src/mir/block.rs

Lines changed: 79 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ use crate::MemFlags;
1212
use rustc_ast as ast;
1313
use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
1414
use rustc_hir::lang_items::LangItem;
15-
use rustc_middle::mir::{self, AssertKind, SwitchTargets, UnwindTerminateReason};
15+
use rustc_middle::mir::{self, AssertKind, SwitchTargets, UbCheckKind, UnwindTerminateReason};
1616
use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, ValidityRequirement};
1717
use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
1818
use rustc_middle::ty::{self, Instance, Ty};
@@ -616,13 +616,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
616616
// and `#[track_caller]` adds an implicit third argument.
617617
(LangItem::PanicBoundsCheck, vec![index, len, location])
618618
}
619-
AssertKind::MisalignedPointerDereference { ref required, ref found } => {
620-
let required = self.codegen_operand(bx, required).immediate();
621-
let found = self.codegen_operand(bx, found).immediate();
622-
// It's `fn panic_misaligned_pointer_dereference(required: usize, found: usize)`,
623-
// and `#[track_caller]` adds an implicit third argument.
624-
(LangItem::PanicMisalignedPointerDereference, vec![required, found, location])
625-
}
626619
_ => {
627620
let msg = bx.const_str(msg.description());
628621
// It's `pub fn panic(expr: &str)`, with the wide reference being passed
@@ -736,6 +729,79 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
736729
}
737730
}
738731

732+
fn codegen_alignment_check(
733+
&mut self,
734+
helper: &TerminatorCodegenHelper<'tcx>,
735+
bx: &mut Bx,
736+
pointer: &mir::Operand<'tcx>,
737+
source_info: mir::SourceInfo,
738+
target: mir::BasicBlock,
739+
) -> MergingSucc {
740+
let span = source_info.span;
741+
let pointer = self.codegen_operand(bx, pointer);
742+
let pointee_ty = pointer.layout.ty.builtin_deref(true).unwrap();
743+
let pointee_layout = bx.layout_of(pointee_ty.ty);
744+
745+
let mk_usize = |v: u64| {
746+
let layout = bx.layout_of(bx.tcx().types.usize);
747+
let rustc_target::abi::Abi::Scalar(abi) = layout.abi else { unreachable!() };
748+
let v = rustc_middle::mir::interpret::Scalar::from_target_usize(v, &bx.tcx());
749+
bx.scalar_to_backend(v, abi, bx.cx().type_isize())
750+
};
751+
752+
let align = pointee_layout.align.abi.bytes();
753+
let mask = mk_usize(align - 1);
754+
let zero = mk_usize(0);
755+
let required = mk_usize(align);
756+
757+
let ptr_imm = match pointer.val {
758+
crate::mir::OperandValue::Immediate(imm) => imm,
759+
crate::mir::OperandValue::Pair(ptr, _) => ptr,
760+
_ => {
761+
unreachable!("{pointer:?}");
762+
}
763+
};
764+
let int_imm = bx.ptrtoint(ptr_imm, bx.cx().type_isize());
765+
766+
let masked = bx.and(int_imm, mask);
767+
768+
let is_zero = bx.icmp(
769+
crate::base::bin_op_to_icmp_predicate(mir::BinOp::Eq.to_hir_binop(), false),
770+
masked,
771+
zero,
772+
);
773+
774+
let lltarget = helper.llbb_with_cleanup(self, target);
775+
let panic_block = bx.append_sibling_block("panic");
776+
777+
bx.cond_br(is_zero, lltarget, panic_block);
778+
779+
bx.switch_to_block(panic_block);
780+
self.set_debug_loc(bx, source_info);
781+
782+
let location = self.get_caller_location(bx, source_info).immediate();
783+
784+
let found = int_imm;
785+
786+
let (lang_item, args) =
787+
(LangItem::PanicMisalignedPointerDereference, vec![required, found, location]);
788+
789+
let (fn_abi, llfn) = common::build_langcall(bx, Some(span), lang_item);
790+
let merging_succ = helper.do_call(
791+
self,
792+
bx,
793+
fn_abi,
794+
llfn,
795+
&args,
796+
None,
797+
mir::UnwindAction::Unreachable,
798+
&[],
799+
false,
800+
);
801+
assert_eq!(merging_succ, MergingSucc::False);
802+
MergingSucc::False
803+
}
804+
739805
fn codegen_call_terminator(
740806
&mut self,
741807
helper: TerminatorCodegenHelper<'tcx>,
@@ -1291,6 +1357,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
12911357
self.instance,
12921358
mergeable_succ(),
12931359
),
1360+
1361+
mir::TerminatorKind::UbCheck {
1362+
target,
1363+
kind: UbCheckKind::PointerAlignment { ref pointer },
1364+
} => self.codegen_alignment_check(&helper, bx, pointer, terminator.source_info, target),
12941365
}
12951366
}
12961367

‎compiler/rustc_const_eval/src/const_eval/machine.rs

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -555,12 +555,6 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
555555
RemainderByZero(op) => RemainderByZero(eval_to_int(op)?),
556556
ResumedAfterReturn(coroutine_kind) => ResumedAfterReturn(*coroutine_kind),
557557
ResumedAfterPanic(coroutine_kind) => ResumedAfterPanic(*coroutine_kind),
558-
MisalignedPointerDereference { ref required, ref found } => {
559-
MisalignedPointerDereference {
560-
required: eval_to_int(required)?,
561-
found: eval_to_int(found)?,
562-
}
563-
}
564558
};
565559
Err(ConstEvalErrKind::AssertFailure(err).into())
566560
}

‎compiler/rustc_const_eval/src/interpret/terminator.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
8585
self.pop_stack_frame(/* unwinding */ false)?
8686
}
8787

88-
Goto { target } => self.go_to_block(target),
88+
Goto { target } | UbCheck { target, .. } => self.go_to_block(target),
8989

9090
SwitchInt { ref discr, ref targets } => {
9191
let discr = self.read_immediate(&self.eval_operand(discr, None)?)?;

‎compiler/rustc_const_eval/src/transform/check_consts/check.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1058,7 +1058,8 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
10581058
| TerminatorKind::UnwindResume
10591059
| TerminatorKind::Return
10601060
| TerminatorKind::SwitchInt { .. }
1061-
| TerminatorKind::Unreachable => {}
1061+
| TerminatorKind::Unreachable
1062+
| TerminatorKind::UbCheck { .. } => {}
10621063
}
10631064
}
10641065
}

‎compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,8 @@ impl<'tcx> Visitor<'tcx> for CheckLiveDrops<'_, 'tcx> {
118118
| mir::TerminatorKind::Return
119119
| mir::TerminatorKind::SwitchInt { .. }
120120
| mir::TerminatorKind::Unreachable
121-
| mir::TerminatorKind::Yield { .. } => {}
121+
| mir::TerminatorKind::Yield { .. }
122+
| mir::TerminatorKind::UbCheck { .. } => {}
122123
}
123124
}
124125
}

‎compiler/rustc_const_eval/src/transform/validate.rs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -393,7 +393,7 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> {
393393

394394
fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) {
395395
match &terminator.kind {
396-
TerminatorKind::Goto { target } => {
396+
TerminatorKind::Goto { target } | TerminatorKind::UbCheck { target, .. } => {
397397
self.check_edge(location, *target, EdgeKind::Normal);
398398
}
399399
TerminatorKind::SwitchInt { targets, discr: _ } => {
@@ -1298,7 +1298,8 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
12981298
| TerminatorKind::UnwindResume
12991299
| TerminatorKind::UnwindTerminate(_)
13001300
| TerminatorKind::Return
1301-
| TerminatorKind::Unreachable => {}
1301+
| TerminatorKind::Unreachable
1302+
| TerminatorKind::UbCheck { .. } => {}
13021303
}
13031304

13041305
self.super_terminator(terminator, location);

‎compiler/rustc_middle/messages.ftl

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,6 @@ middle_assert_divide_by_zero =
1414
1515
middle_assert_gen_resume_after_panic = `gen` fn or block cannot be further iterated on after it panicked
1616
17-
middle_assert_misaligned_ptr_deref =
18-
misaligned pointer dereference: address must be a multiple of {$required} but is {$found}
19-
2017
middle_assert_op_overflow =
2118
attempt to compute `{$left} {$op} {$right}`, which would overflow
2219

‎compiler/rustc_middle/src/mir/pretty.rs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -858,6 +858,9 @@ impl<'tcx> TerminatorKind<'tcx> {
858858
}
859859
write!(fmt, ", options({options:?}))")
860860
}
861+
UbCheck { target: _, kind: UbCheckKind::PointerAlignment { pointer } } => {
862+
write!(fmt, "pointer_alignment_check({pointer:?})")
863+
}
861864
}
862865
}
863866

@@ -866,7 +869,7 @@ impl<'tcx> TerminatorKind<'tcx> {
866869
use self::TerminatorKind::*;
867870
match *self {
868871
Return | UnwindResume | UnwindTerminate(_) | Unreachable | CoroutineDrop => vec![],
869-
Goto { .. } => vec!["".into()],
872+
Goto { .. } | UbCheck { .. } => vec!["".into()],
870873
SwitchInt { ref targets, .. } => targets
871874
.values
872875
.iter()

‎compiler/rustc_middle/src/mir/syntax.rs

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -588,7 +588,9 @@ impl CallSource {
588588
#[derive(Clone, TyEncodable, TyDecodable, Hash, HashStable, PartialEq, TypeFoldable, TypeVisitable)]
589589
pub enum TerminatorKind<'tcx> {
590590
/// Block has one successor; we continue execution there.
591-
Goto { target: BasicBlock },
591+
Goto {
592+
target: BasicBlock,
593+
},
592594

593595
/// Switches based on the computed value.
594596
///
@@ -655,7 +657,12 @@ pub enum TerminatorKind<'tcx> {
655657
/// The `replace` flag indicates whether this terminator was created as part of an assignment.
656658
/// This should only be used for diagnostic purposes, and does not have any operational
657659
/// meaning.
658-
Drop { place: Place<'tcx>, target: BasicBlock, unwind: UnwindAction, replace: bool },
660+
Drop {
661+
place: Place<'tcx>,
662+
target: BasicBlock,
663+
unwind: UnwindAction,
664+
replace: bool,
665+
},
659666

660667
/// Roughly speaking, evaluates the `func` operand and the arguments, and starts execution of
661668
/// the referred to function. The operand types must match the argument types of the function.
@@ -798,6 +805,17 @@ pub enum TerminatorKind<'tcx> {
798805
/// if and only if InlineAsmOptions::MAY_UNWIND is set.
799806
unwind: UnwindAction,
800807
},
808+
809+
UbCheck {
810+
target: BasicBlock,
811+
kind: UbCheckKind<'tcx>,
812+
},
813+
}
814+
815+
#[derive(Clone, TyEncodable, TyDecodable, Hash, HashStable, PartialEq, TypeFoldable, TypeVisitable)]
816+
#[derive(Debug)]
817+
pub enum UbCheckKind<'tcx> {
818+
PointerAlignment { pointer: Operand<'tcx> },
801819
}
802820

803821
impl TerminatorKind<'_> {
@@ -819,6 +837,7 @@ impl TerminatorKind<'_> {
819837
TerminatorKind::FalseEdge { .. } => "FalseEdge",
820838
TerminatorKind::FalseUnwind { .. } => "FalseUnwind",
821839
TerminatorKind::InlineAsm { .. } => "InlineAsm",
840+
TerminatorKind::UbCheck { .. } => "UbCheck",
822841
}
823842
}
824843
}
@@ -885,7 +904,6 @@ pub enum AssertKind<O> {
885904
RemainderByZero(O),
886905
ResumedAfterReturn(CoroutineKind),
887906
ResumedAfterPanic(CoroutineKind),
888-
MisalignedPointerDereference { required: O, found: O },
889907
}
890908

891909
#[derive(Clone, Debug, PartialEq, TyEncodable, TyDecodable, Hash, HashStable)]

0 commit comments

Comments
 (0)