Skip to content

Commit 1019e20

Browse files
DiamondLovesYouDiamondLovesYou
committed
LLVM value address space optimizations: leave values in their original address space where possible.
1 parent e041b4a commit 1019e20

File tree

5 files changed

+53
-36
lines changed

5 files changed

+53
-36
lines changed

‎src/librustc_codegen_llvm/abi.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -652,7 +652,7 @@ impl<'tcx> FnTypeExt<'tcx> for FnType<'tcx, Ty<'tcx>> {
652652
}
653653
PassMode::Cast(cast) => cast.llvm_type(cx),
654654
PassMode::Indirect(..) => {
655-
llargument_tys.push(cx.type_ptr_to_flat(self.ret.memory_ty(cx)));
655+
llargument_tys.push(cx.type_ptr_to_alloca(self.ret.memory_ty(cx)));
656656
cx.type_void()
657657
}
658658
};
@@ -682,7 +682,7 @@ impl<'tcx> FnTypeExt<'tcx> for FnType<'tcx, Ty<'tcx>> {
682682
continue;
683683
}
684684
PassMode::Cast(cast) => cast.llvm_type(cx),
685-
PassMode::Indirect(_, None) => cx.type_ptr_to_flat(arg.memory_ty(cx)),
685+
PassMode::Indirect(_, None) => cx.type_ptr_to_alloca(arg.memory_ty(cx)),
686686
};
687687
llargument_tys.push(llarg_ty);
688688
}

‎src/librustc_codegen_ssa/mir/block.rs

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -263,7 +263,8 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
263263
let llslot = match op.val {
264264
Immediate(_) | Pair(..) => {
265265
let scratch =
266-
PlaceRef::alloca(&mut bx, self.fn_ty.ret.layout, "ret");
266+
PlaceRef::alloca_addr_space(&mut bx, self.fn_ty.ret.layout,
267+
"ret");
267268
op.val.store(&mut bx, scratch);
268269
scratch.llval
269270
}
@@ -791,7 +792,7 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
791792
Immediate(_) | Pair(..) => {
792793
match arg.mode {
793794
PassMode::Indirect(..) | PassMode::Cast(_) => {
794-
let scratch = PlaceRef::alloca(bx, arg.layout, "arg");
795+
let scratch = PlaceRef::alloca_addr_space(bx, arg.layout, "arg");
795796
op.val.store(bx, scratch);
796797
(scratch.llval, scratch.align, true)
797798
}
@@ -806,12 +807,12 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
806807
// think that ATM (Rust 1.16) we only pass temporaries, but we shouldn't
807808
// have scary latent bugs around.
808809

809-
let scratch = PlaceRef::alloca(bx, arg.layout, "arg");
810+
let scratch = PlaceRef::alloca_addr_space(bx, arg.layout, "arg");
810811
base::memcpy_ty(bx, scratch.llval, scratch.align, llval, align,
811812
op.layout, MemFlags::empty());
812813
(scratch.llval, scratch.align, true)
813814
} else {
814-
(llval, align, true)
815+
(bx.flat_addr_cast(llval), align, true)
815816
}
816817
}
817818
};
@@ -883,7 +884,7 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
883884
cx.tcx().mk_mut_ptr(cx.tcx().types.u8),
884885
cx.tcx().types.i32
885886
]));
886-
let slot = PlaceRef::alloca(bx, layout, "personalityslot");
887+
let slot = PlaceRef::alloca_addr_space(bx, layout, "personalityslot");
887888
self.personality_slot = Some(slot);
888889
slot
889890
}
@@ -979,15 +980,15 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
979980
return if fn_ret.is_indirect() {
980981
// Odd, but possible, case, we have an operand temporary,
981982
// but the calling convention has an indirect return.
982-
let tmp = PlaceRef::alloca(bx, fn_ret.layout, "tmp_ret");
983+
let tmp = PlaceRef::alloca_addr_space(bx, fn_ret.layout, "tmp_ret");
983984
tmp.storage_live(bx);
984985
llargs.push(tmp.llval);
985986
ReturnDest::IndirectOperand(tmp, index)
986987
} else if is_intrinsic {
987988
// Currently, intrinsics always need a location to store
988989
// the result. so we create a temporary alloca for the
989990
// result
990-
let tmp = PlaceRef::alloca(bx, fn_ret.layout, "tmp_ret");
991+
let tmp = PlaceRef::alloca_addr_space(bx, fn_ret.layout, "tmp_ret");
991992
tmp.storage_live(bx);
992993
ReturnDest::IndirectOperand(tmp, index)
993994
} else {
@@ -1031,7 +1032,7 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
10311032
LocalRef::Operand(None) => {
10321033
let dst_layout = bx.layout_of(self.monomorphized_place_ty(dst));
10331034
assert!(!dst_layout.ty.has_erasable_regions());
1034-
let place = PlaceRef::alloca(bx, dst_layout, "transmute_temp");
1035+
let place = PlaceRef::alloca_addr_space(bx, dst_layout, "transmute_temp");
10351036
place.storage_live(bx);
10361037
self.codegen_transmute_into(bx, src, place);
10371038
let op = bx.load_operand(place);
@@ -1084,7 +1085,7 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
10841085
DirectOperand(index) => {
10851086
// If there is a cast, we have to store and reload.
10861087
let op = if let PassMode::Cast(_) = ret_ty.mode {
1087-
let tmp = PlaceRef::alloca(bx, ret_ty.layout, "tmp_ret");
1088+
let tmp = PlaceRef::alloca_addr_space(bx, ret_ty.layout, "tmp_ret");
10881089
tmp.storage_live(bx);
10891090
bx.store_arg_ty(&ret_ty, llval, tmp);
10901091
let op = bx.load_operand(tmp);

‎src/librustc_codegen_ssa/mir/mod.rs

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -277,7 +277,7 @@ pub fn codegen_mir<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>>(
277277
// FIXME: add an appropriate debuginfo
278278
LocalRef::UnsizedPlace(indirect_place)
279279
} else {
280-
let place = PlaceRef::alloca(&mut bx, layout, &name.as_str());
280+
let place = PlaceRef::alloca_addr_space(&mut bx, layout, &name.as_str());
281281
if dbg {
282282
let (scope, span) = fx.debug_loc(mir::SourceInfo {
283283
span: decl.source_info.span,
@@ -305,7 +305,8 @@ pub fn codegen_mir<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>>(
305305
);
306306
LocalRef::UnsizedPlace(indirect_place)
307307
} else {
308-
LocalRef::Place(PlaceRef::alloca(&mut bx, layout, &format!("{:?}", local)))
308+
LocalRef::Place(PlaceRef::alloca_addr_space(&mut bx, layout,
309+
&format!("{:?}", local)))
309310
}
310311
} else {
311312
// If this is an immediate local, we do not create an
@@ -468,7 +469,7 @@ fn arg_local_refs<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>>(
468469
_ => bug!("spread argument isn't a tuple?!")
469470
};
470471

471-
let place = PlaceRef::alloca(bx, bx.layout_of(arg_ty), &name);
472+
let place = PlaceRef::alloca_addr_space(bx, bx.layout_of(arg_ty), &name);
472473
for i in 0..tupled_arg_tys.len() {
473474
let arg = &fx.fn_ty.args[idx];
474475
idx += 1;
@@ -559,7 +560,7 @@ fn arg_local_refs<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>>(
559560
indirect_operand.store(bx, tmp);
560561
tmp
561562
} else {
562-
let tmp = PlaceRef::alloca(bx, arg.layout, &name);
563+
let tmp = PlaceRef::alloca_addr_space(bx, arg.layout, &name);
563564
bx.store_fn_arg(arg, &mut llarg_idx, tmp);
564565
tmp
565566
};

‎src/librustc_codegen_ssa/mir/place.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ impl<'a, 'tcx: 'a, V: CodegenObject> PlaceRef<'tcx, V> {
7474
assert!(layout.is_unsized(), "tried to allocate indirect place for sized values");
7575
let ptr_ty = bx.cx().tcx().mk_mut_ptr(layout.ty);
7676
let ptr_layout = bx.cx().layout_of(ptr_ty);
77-
Self::alloca(bx, ptr_layout, name)
77+
Self::alloca_addr_space(bx, ptr_layout, name)
7878
}
7979

8080
pub fn len<Cx: CodegenMethods<'tcx, Value = V>>(

‎src/librustc_codegen_ssa/mir/rvalue.rs

Lines changed: 35 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,36 @@ use super::{FunctionCx, LocalRef};
1818
use super::operand::{OperandRef, OperandValue};
1919
use super::place::PlaceRef;
2020

21+
fn codegen_binop_fixup<'a, 'tcx: 'a, Bx>(bx: &mut Bx,
22+
lhs: Bx::Value,
23+
rhs: Bx::Value)
24+
-> (Bx::Value, Bx::Value)
25+
where Bx: BuilderMethods<'a, 'tcx>,
26+
{
27+
// In case we're in separate addr spaces.
28+
// Can happen when cmp against null_mut, eg.
29+
// `infer-addr-spaces` should propagate.
30+
// But, empirically, `infer-addr-spaces` doesn't.
31+
let fix_null_ty = |val, this_ty, other_ty| {
32+
if bx.cx().const_null(this_ty) == val {
33+
bx.cx().const_null(other_ty)
34+
} else {
35+
val
36+
}
37+
};
38+
let lhs_ty = bx.cx().val_ty(lhs);
39+
let rhs_ty = bx.cx().val_ty(rhs);
40+
let lhs = fix_null_ty(lhs, lhs_ty, rhs_ty);
41+
let rhs = fix_null_ty(rhs, rhs_ty, lhs_ty);
42+
if bx.cx().type_addr_space(lhs_ty).is_some() {
43+
assert!(bx.cx().type_addr_space(rhs_ty).is_some());
44+
(bx.flat_addr_cast(lhs),
45+
bx.flat_addr_cast(rhs))
46+
} else {
47+
(lhs, rhs)
48+
}
49+
}
50+
2151
impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
2252
pub fn codegen_rvalue(
2353
&mut self,
@@ -63,7 +93,8 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
6393
// index into the struct, and this case isn't
6494
// important enough for it.
6595
debug!("codegen_rvalue: creating ugly alloca");
66-
let scratch = PlaceRef::alloca(&mut bx, operand.layout, "__unsize_temp");
96+
let scratch = PlaceRef::alloca_addr_space(&mut bx, operand.layout,
97+
"__unsize_temp");
6798
scratch.storage_live(&mut bx);
6899
operand.val.store(&mut bx, scratch);
69100
base::coerce_unsized_into(&mut bx, scratch, dest);
@@ -89,7 +120,6 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
89120
}
90121
let zero = bx.cx().const_usize(0);
91122
let start = dest.project_index(&mut bx, zero).llval;
92-
let start = bx.flat_addr_cast(start);
93123

94124
if let OperandValue::Immediate(v) = cg_elem.val {
95125
let size = bx.cx().const_usize(dest.layout.size.bytes());
@@ -111,6 +141,7 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
111141

112142
let count = bx.cx().const_usize(count);
113143
let end = dest.project_index(&mut bx, count).llval;
144+
let start = bx.flat_addr_cast(start);
114145
let end = bx.flat_addr_cast(end);
115146

116147
let mut header_bx = bx.build_sibling_block("repeat_loop_header");
@@ -245,7 +276,6 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
245276
// until LLVM removes pointee types.
246277
let lldata = bx.pointercast(lldata,
247278
bx.cx().scalar_pair_element_backend_type(cast, 0, true));
248-
let lldata = bx.flat_addr_cast(lldata);
249279
OperandValue::Pair(lldata, llextra)
250280
}
251281
OperandValue::Immediate(lldata) => {
@@ -618,17 +648,7 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
618648
lhs, rhs
619649
)
620650
} else {
621-
// In case we're in separate addr spaces.
622-
// Can happen when cmp against null_mut, eg.
623-
// `infer-addr-spaces` should propagate.
624-
let lhs_ty = bx.cx().val_ty(rhs);
625-
let (lhs, rhs) = if bx.cx().type_addr_space(lhs_ty).is_some() {
626-
assert!(bx.cx().type_addr_space(bx.cx().val_ty(rhs)).is_some());
627-
(bx.flat_addr_cast(lhs),
628-
bx.flat_addr_cast(rhs))
629-
} else {
630-
(lhs, rhs)
631-
};
651+
let (lhs, rhs) = codegen_binop_fixup(bx, lhs, rhs);
632652
bx.icmp(
633653
base::bin_op_to_icmp_predicate(op.to_hir_binop(), is_signed),
634654
lhs, rhs
@@ -647,12 +667,7 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
647667
rhs_extra: Bx::Value,
648668
_input_ty: Ty<'tcx>,
649669
) -> Bx::Value {
650-
// In case we're in separate addr spaces.
651-
// Can happen when cmp against null_mut, eg.
652-
// `infer-addr-spaces` should propagate.
653-
let lhs_addr = bx.flat_addr_cast(lhs_addr);
654-
let rhs_addr = bx.flat_addr_cast(rhs_addr);
655-
670+
let (lhs_addr, rhs_addr) = codegen_binop_fixup(bx, lhs_addr, rhs_addr);
656671
match op {
657672
mir::BinOp::Eq => {
658673
let lhs = bx.icmp(IntPredicate::IntEQ, lhs_addr, rhs_addr);

0 commit comments

Comments
 (0)