Skip to content

Commit 3172dfa

Browse files
cjgillotcjgillot
committed
Extract inline cost computation.
1 parent 36f4f4a commit 3172dfa

File tree

1 file changed

+134
-112
lines changed

1 file changed

+134
-112
lines changed

‎compiler/rustc_mir_transform/src/inline.rs

Lines changed: 134 additions & 112 deletions
Original file line numberDiff line numberDiff line change
@@ -392,7 +392,9 @@ impl<'tcx> Inliner<'tcx> {
392392
callee_body: &Body<'tcx>,
393393
callee_attrs: &CodegenFnAttrs,
394394
) -> Result<(), &'static str> {
395-
let tcx = self.tcx;
395+
let cost_info = body_cost(self.tcx, self.param_env, callee_body, |ty| {
396+
callsite.callee.subst_mir(self.tcx, &ty)
397+
});
396398

397399
let mut threshold = if callee_attrs.requests_inline() {
398400
self.tcx.sess.opts.unstable_opts.inline_mir_hint_threshold.unwrap_or(100)
@@ -403,123 +405,16 @@ impl<'tcx> Inliner<'tcx> {
403405
// Give a bonus functions with a small number of blocks,
404406
// We normally have two or three blocks for even
405407
// very small functions.
406-
if callee_body.basic_blocks().len() <= 3 {
408+
if cost_info.bbcount <= 3 {
407409
threshold += threshold / 4;
408410
}
409411
debug!(" final inline threshold = {}", threshold);
410412

411-
// FIXME: Give a bonus to functions with only a single caller
412-
let mut first_block = true;
413-
let mut cost = 0;
414-
415-
// Traverse the MIR manually so we can account for the effects of
416-
// inlining on the CFG.
417-
let mut work_list = vec![START_BLOCK];
418-
let mut visited = BitSet::new_empty(callee_body.basic_blocks().len());
419-
while let Some(bb) = work_list.pop() {
420-
if !visited.insert(bb.index()) {
421-
continue;
422-
}
423-
let blk = &callee_body.basic_blocks()[bb];
424-
425-
for stmt in &blk.statements {
426-
// Don't count StorageLive/StorageDead in the inlining cost.
427-
match stmt.kind {
428-
StatementKind::StorageLive(_)
429-
| StatementKind::StorageDead(_)
430-
| StatementKind::Deinit(_)
431-
| StatementKind::Nop => {}
432-
_ => cost += INSTR_COST,
433-
}
434-
}
435-
let term = blk.terminator();
436-
let mut is_drop = false;
437-
match term.kind {
438-
TerminatorKind::Drop { ref place, target, unwind }
439-
| TerminatorKind::DropAndReplace { ref place, target, unwind, .. } => {
440-
is_drop = true;
441-
work_list.push(target);
442-
// If the place doesn't actually need dropping, treat it like
443-
// a regular goto.
444-
let ty = callsite.callee.subst_mir(self.tcx, &place.ty(callee_body, tcx).ty);
445-
if ty.needs_drop(tcx, self.param_env) {
446-
cost += CALL_PENALTY;
447-
if let Some(unwind) = unwind {
448-
cost += LANDINGPAD_PENALTY;
449-
work_list.push(unwind);
450-
}
451-
} else {
452-
cost += INSTR_COST;
453-
}
454-
}
455-
456-
TerminatorKind::Unreachable | TerminatorKind::Call { target: None, .. }
457-
if first_block =>
458-
{
459-
// If the function always diverges, don't inline
460-
// unless the cost is zero
461-
threshold = 0;
462-
}
463-
464-
TerminatorKind::Call { func: Operand::Constant(ref f), cleanup, .. } => {
465-
if let ty::FnDef(def_id, _) =
466-
*callsite.callee.subst_mir(self.tcx, &f.literal.ty()).kind()
467-
{
468-
// Don't give intrinsics the extra penalty for calls
469-
if tcx.is_intrinsic(def_id) {
470-
cost += INSTR_COST;
471-
} else {
472-
cost += CALL_PENALTY;
473-
}
474-
} else {
475-
cost += CALL_PENALTY;
476-
}
477-
if cleanup.is_some() {
478-
cost += LANDINGPAD_PENALTY;
479-
}
480-
}
481-
TerminatorKind::Assert { cleanup, .. } => {
482-
cost += CALL_PENALTY;
483-
484-
if cleanup.is_some() {
485-
cost += LANDINGPAD_PENALTY;
486-
}
487-
}
488-
TerminatorKind::Resume => cost += RESUME_PENALTY,
489-
TerminatorKind::InlineAsm { cleanup, .. } => {
490-
cost += INSTR_COST;
491-
492-
if cleanup.is_some() {
493-
cost += LANDINGPAD_PENALTY;
494-
}
495-
}
496-
_ => cost += INSTR_COST,
497-
}
498-
499-
if !is_drop {
500-
for succ in term.successors() {
501-
work_list.push(succ);
502-
}
503-
}
504-
505-
first_block = false;
413+
if cost_info.diverges {
414+
threshold = 0;
506415
}
507416

508-
// Count up the cost of local variables and temps, if we know the size
509-
// use that, otherwise we use a moderately-large dummy cost.
510-
511-
let ptr_size = tcx.data_layout.pointer_size.bytes();
512-
513-
for v in callee_body.vars_and_temps_iter() {
514-
let ty = callsite.callee.subst_mir(self.tcx, &callee_body.local_decls[v].ty);
515-
// Cost of the var is the size in machine-words, if we know
516-
// it.
517-
if let Some(size) = type_size_of(tcx, self.param_env, ty) {
518-
cost += ((size + ptr_size - 1) / ptr_size) as usize;
519-
} else {
520-
cost += UNKNOWN_SIZE_COST;
521-
}
522-
}
417+
let cost = cost_info.cost;
523418

524419
if let InlineAttr::Always = callee_attrs.inline {
525420
debug!("INLINING {:?} because inline(always) [cost={}]", callsite, cost);
@@ -1012,3 +907,130 @@ impl<'tcx> MutVisitor<'tcx> for Integrator<'_, 'tcx> {
1012907
}
1013908
}
1014909
}
910+
911+
pub struct InlineCostInfo {
912+
pub cost: usize,
913+
pub bbcount: usize,
914+
pub diverges: bool,
915+
}
916+
917+
pub fn body_cost<'tcx>(
918+
tcx: TyCtxt<'tcx>,
919+
param_env: ty::ParamEnv<'tcx>,
920+
callee_body: &Body<'tcx>,
921+
subst_mir: impl Fn(Ty<'tcx>) -> Ty<'tcx>,
922+
) -> InlineCostInfo {
923+
// FIXME: Give a bonus to functions with only a single caller
924+
let mut diverges = false;
925+
let mut first_block = true;
926+
let mut cost = 0;
927+
928+
// Traverse the MIR manually so we can account for the effects of
929+
// inlining on the CFG.
930+
let mut work_list = vec![START_BLOCK];
931+
let mut visited = BitSet::new_empty(callee_body.basic_blocks().len());
932+
while let Some(bb) = work_list.pop() {
933+
if !visited.insert(bb.index()) {
934+
continue;
935+
}
936+
let blk = &callee_body.basic_blocks()[bb];
937+
938+
for stmt in &blk.statements {
939+
// Don't count StorageLive/StorageDead in the inlining cost.
940+
match stmt.kind {
941+
StatementKind::StorageLive(_)
942+
| StatementKind::StorageDead(_)
943+
| StatementKind::Deinit(_)
944+
| StatementKind::Nop => {}
945+
_ => cost += INSTR_COST,
946+
}
947+
}
948+
let term = blk.terminator();
949+
let mut is_drop = false;
950+
match term.kind {
951+
TerminatorKind::Drop { ref place, target, unwind }
952+
| TerminatorKind::DropAndReplace { ref place, target, unwind, .. } => {
953+
is_drop = true;
954+
work_list.push(target);
955+
// If the place doesn't actually need dropping, treat it like
956+
// a regular goto.
957+
let ty = subst_mir(place.ty(callee_body, tcx).ty);
958+
if ty.needs_drop(tcx, param_env) {
959+
cost += CALL_PENALTY;
960+
if let Some(unwind) = unwind {
961+
cost += LANDINGPAD_PENALTY;
962+
work_list.push(unwind);
963+
}
964+
} else {
965+
cost += INSTR_COST;
966+
}
967+
}
968+
969+
TerminatorKind::Unreachable | TerminatorKind::Call { target: None, .. }
970+
if first_block =>
971+
{
972+
// If the function always diverges, don't inline
973+
// unless the cost is zero
974+
diverges = true;
975+
}
976+
977+
TerminatorKind::Call { func: Operand::Constant(ref f), cleanup, .. } => {
978+
if let ty::FnDef(def_id, _) = *subst_mir(f.literal.ty()).kind() {
979+
// Don't give intrinsics the extra penalty for calls
980+
if tcx.is_intrinsic(def_id) {
981+
cost += INSTR_COST;
982+
} else {
983+
cost += CALL_PENALTY;
984+
}
985+
} else {
986+
cost += CALL_PENALTY;
987+
}
988+
if cleanup.is_some() {
989+
cost += LANDINGPAD_PENALTY;
990+
}
991+
}
992+
TerminatorKind::Assert { cleanup, .. } => {
993+
cost += CALL_PENALTY;
994+
995+
if cleanup.is_some() {
996+
cost += LANDINGPAD_PENALTY;
997+
}
998+
}
999+
TerminatorKind::Resume => cost += RESUME_PENALTY,
1000+
TerminatorKind::InlineAsm { cleanup, .. } => {
1001+
cost += INSTR_COST;
1002+
1003+
if cleanup.is_some() {
1004+
cost += LANDINGPAD_PENALTY;
1005+
}
1006+
}
1007+
_ => cost += INSTR_COST,
1008+
}
1009+
1010+
if !is_drop {
1011+
for succ in term.successors() {
1012+
work_list.push(succ);
1013+
}
1014+
}
1015+
1016+
first_block = false;
1017+
}
1018+
1019+
// Count up the cost of local variables and temps, if we know the size
1020+
// use that, otherwise we use a moderately-large dummy cost.
1021+
1022+
let ptr_size = tcx.data_layout.pointer_size.bytes();
1023+
1024+
for v in callee_body.vars_and_temps_iter() {
1025+
let ty = subst_mir(callee_body.local_decls[v].ty);
1026+
// Cost of the var is the size in machine-words, if we know
1027+
// it.
1028+
if let Some(size) = type_size_of(tcx, param_env, ty) {
1029+
cost += ((size + ptr_size - 1) / ptr_size) as usize;
1030+
} else {
1031+
cost += UNKNOWN_SIZE_COST;
1032+
}
1033+
}
1034+
1035+
InlineCostInfo { cost, diverges, bbcount: callee_body.basic_blocks().len() }
1036+
}

0 commit comments

Comments
 (0)