Always consider inlining for trivial functions

Author: saethlin

compiler/rustc_mir_transform/src/inline.rs

@@ -10,7 +10,6 @@ use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
 use rustc_middle::ty::TypeVisitableExt;
 use rustc_middle::ty::{self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt};
-use rustc_session::config::OptLevel;
 use rustc_target::abi::FieldIdx;
 use rustc_target::spec::abi::Abi;
 
@@ -47,12 +46,8 @@ impl<'tcx> MirPass<'tcx> for Inline {
         }
 
         match sess.mir_opt_level() {
-            0 | 1 => false,
-            2 => {
-                (sess.opts.optimize == OptLevel::Default
-                    || sess.opts.optimize == OptLevel::Aggressive)
-                    && sess.opts.incremental == None
-            }
+            0 => false,
+            1 | 2 => sess.opts.incremental.is_none(),
             _ => true,
         }
     }
@@ -471,7 +466,11 @@ impl<'tcx> Inliner<'tcx> {
         if callee_body.basic_blocks.len() <= 3 {
             threshold += threshold / 4;
         }
-        debug!("    final inline threshold = {}", threshold);
+
+        if tcx.sess.mir_opt_level() == 1 {
+            threshold = 2 * INSTR_COST;
+        }
+        debug!("final inline threshold = {}", threshold);
 
         // FIXME: Give a bonus to functions with only a single caller
 
@@ -481,6 +480,7 @@ impl<'tcx> Inliner<'tcx> {
             instance: callsite.callee,
             callee_body,
             cost: 0,
+            threshold,
         };
 
         // Traverse the MIR manually so we can account for the effects of inlining on the CFG.
@@ -493,6 +493,10 @@ impl<'tcx> Inliner<'tcx> {
 
             let blk = &callee_body.basic_blocks[bb];
             checker.visit_basic_block_data(bb, blk);
+            if callee_attrs.inline != InlineAttr::Always && checker.cost > threshold {
+                debug!("checker cost exceeded threshold partway through");
+                return Err("cost above threshold");
+            }
 
             let term = blk.terminator();
             if let TerminatorKind::Drop { ref place, target, unwind, replace: _ } = term.kind {
@@ -805,12 +809,17 @@ struct CostChecker<'b, 'tcx> {
     tcx: TyCtxt<'tcx>,
     param_env: ParamEnv<'tcx>,
     cost: usize,
+    threshold: usize,
     callee_body: &'b Body<'tcx>,
     instance: ty::Instance<'tcx>,
 }
 
 impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
     fn visit_statement(&mut self, statement: &Statement<'tcx>, _: Location) {
+        if self.cost > self.threshold {
+            return;
+        }
+
         // Don't count StorageLive/StorageDead in the inlining cost.
         match statement.kind {
             StatementKind::StorageLive(_)

tests/codegen-units/polymorphization/unused_type_parameters.rs

@@ -1,4 +1,4 @@
-// compile-flags:-Zpolymorphize=on -Zprint-mono-items=lazy -Copt-level=1
+// compile-flags:-Zpolymorphize=on -Zprint-mono-items=lazy -Copt-level=1 -Zmir-opt-level=0
 
 #![crate_type = "rlib"]
 

tests/debuginfo/function-names.rs

@@ -1,7 +1,7 @@
 // Function names are formatted differently in old versions of GDB
 // min-gdb-version: 10.1
 
-// compile-flags:-g
+// compile-flags: -g -Zmir-opt-level=0
 
 // === GDB TESTS ===================================================================================
 

tests/mir-opt/inline/trivial_fn_debug_build.rs

@@ -0,0 +1,22 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+// compile-flags: -Copt-level=0 -Zmir-opt-level=1
+#![crate_type = "lib"]
+
+// Test that we still inline trivial functions even in a debug build
+
+pub struct Thing {
+    inner: u8,
+}
+
+impl Thing {
+    #[inline]
+    pub fn get(&self) -> u8 {
+        self.inner
+    }
+}
+
+// EMIT_MIR trivial_fn_debug_build.wrapper.Inline.diff
+// EMIT_MIR trivial_fn_debug_build.wrapper.PreCodegen.after.mir
+pub fn wrapper(t: &Thing) -> u8 {
+    t.get()
+}

tests/mir-opt/inline/trivial_fn_debug_build.wrapper.Inline.panic-abort.diff

@@ -0,0 +1,24 @@
+- // MIR for `wrapper` before Inline
++ // MIR for `wrapper` after Inline
+  
+  fn wrapper(_1: &Thing) -> u8 {
+      debug t => _1;
+      let mut _0: u8;
+      let mut _2: &Thing;
++     scope 1 (inlined Thing::get) {
++         debug self => _2;
++     }
+  
+      bb0: {
+          StorageLive(_2);
+          _2 = &(*_1);
+-         _0 = Thing::get(move _2) -> [return: bb1, unwind unreachable];
+-     }
+- 
+-     bb1: {
++         _0 = ((*_2).0: u8);
+          StorageDead(_2);
+          return;
+      }
+  }
+  

tests/mir-opt/inline/trivial_fn_debug_build.wrapper.Inline.panic-unwind.diff

@@ -0,0 +1,24 @@
+- // MIR for `wrapper` before Inline
++ // MIR for `wrapper` after Inline
+  
+  fn wrapper(_1: &Thing) -> u8 {
+      debug t => _1;
+      let mut _0: u8;
+      let mut _2: &Thing;
++     scope 1 (inlined Thing::get) {
++         debug self => _2;
++     }
+  
+      bb0: {
+          StorageLive(_2);
+          _2 = &(*_1);
+-         _0 = Thing::get(move _2) -> [return: bb1, unwind continue];
+-     }
+- 
+-     bb1: {
++         _0 = ((*_2).0: u8);
+          StorageDead(_2);
+          return;
+      }
+  }
+  

tests/mir-opt/inline/trivial_fn_debug_build.wrapper.PreCodegen.after.panic-abort.mir

@@ -0,0 +1,14 @@
+// MIR for `wrapper` after PreCodegen
+
+fn wrapper(_1: &Thing) -> u8 {
+    debug t => _1;
+    let mut _0: u8;
+    scope 1 (inlined Thing::get) {
+        debug self => _1;
+    }
+
+    bb0: {
+        _0 = ((*_1).0: u8);
+        return;
+    }
+}

tests/mir-opt/inline/trivial_fn_debug_build.wrapper.PreCodegen.after.panic-unwind.mir

@@ -0,0 +1,14 @@
+// MIR for `wrapper` after PreCodegen
+
+fn wrapper(_1: &Thing) -> u8 {
+    debug t => _1;
+    let mut _0: u8;
+    scope 1 (inlined Thing::get) {
+        debug self => _1;
+    }
+
+    bb0: {
+        _0 = ((*_1).0: u8);
+        return;
+    }
+}

tests/ui-fulldeps/stable-mir/crate-info.rs

@@ -59,7 +59,7 @@ fn test_stable_mir(_tcx: TyCtxt<'_>) -> ControlFlow<()> {
     let foo_bar = get_item(&items, (DefKind::Fn, "foo_bar")).unwrap();
     let body = foo_bar.body();
     assert_eq!(body.locals.len(), 5);
-    assert_eq!(body.blocks.len(), 4);
+    assert_eq!(body.blocks.len(), 3);
     let block = &body.blocks[0];
     match &block.terminator.kind {
         stable_mir::mir::TerminatorKind::Call { .. } => {}

tests/ui/consts/const-eval/index-out-of-bounds-never-type.rs

@@ -1,4 +1,5 @@
 // build-fail
+// compile-flags: -Zmir-opt-level=0
 
 // Regression test for #66975
 #![warn(unconditional_panic)]

tests/ui/consts/const-eval/index-out-of-bounds-never-type.stderr

@@ -1,11 +1,11 @@
 error[E0080]: evaluation of `PrintName::<()>::VOID` failed
-  --> $DIR/index-out-of-bounds-never-type.rs:10:61
+  --> $DIR/index-out-of-bounds-never-type.rs:11:61
    |
 LL |     const VOID: ! = { let x = 0 * std::mem::size_of::<T>(); [][x] };
    |                                                             ^^^^^ index out of bounds: the length is 0 but the index is 0
 
 note: the above error was encountered while instantiating `fn f::<()>`
-  --> $DIR/index-out-of-bounds-never-type.rs:20:5
+  --> $DIR/index-out-of-bounds-never-type.rs:21:5
    |
 LL |     f::<()>();
    |     ^^^^^^^^^

tests/ui/consts/const-eval/issue-50814-2.rs

@@ -1,4 +1,5 @@
 // build-fail
+// compile-flags: -Zmir-opt-level=0
 
 trait C {
     const BOO: usize;

tests/ui/consts/const-eval/issue-50814-2.stderr

@@ -1,17 +1,17 @@
 error[E0080]: evaluation of `<A<()> as Foo<()>>::BAR` failed
-  --> $DIR/issue-50814-2.rs:14:24
+  --> $DIR/issue-50814-2.rs:15:24
    |
 LL |     const BAR: usize = [5, 6, 7][T::BOO];
    |                        ^^^^^^^^^^^^^^^^^ index out of bounds: the length is 3 but the index is 42
 
 note: erroneous constant encountered
-  --> $DIR/issue-50814-2.rs:18:6
+  --> $DIR/issue-50814-2.rs:19:6
    |
 LL |     &<A<T> as Foo<T>>::BAR
    |      ^^^^^^^^^^^^^^^^^^^^^
 
 note: the above error was encountered while instantiating `fn foo::<()>`
-  --> $DIR/issue-50814-2.rs:30:22
+  --> $DIR/issue-50814-2.rs:31:22
    |
 LL |     println!("{:x}", foo::<()>() as *const usize as usize);
    |                      ^^^^^^^^^^^

tests/ui/consts/const-eval/issue-85155.rs

@@ -11,6 +11,7 @@
 // aux-build: post_monomorphization_error.rs
 // build-fail: this is a post-monomorphization error, it passes check runs and requires building
 //             to actually fail.
+// compile-flags: -Zmir-opt-level=0
 
 extern crate post_monomorphization_error;
 

tests/ui/consts/const-eval/issue-85155.stderr

@@ -5,7 +5,7 @@ LL |         let _ = 1 / ((IMM >= MIN && IMM <= MAX) as usize);
    |                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ attempt to divide `1_usize` by zero
 
 note: the above error was encountered while instantiating `fn post_monomorphization_error::stdarch_intrinsic::<2>`
-  --> $DIR/issue-85155.rs:19:5
+  --> $DIR/issue-85155.rs:20:5
    |
 LL |     post_monomorphization_error::stdarch_intrinsic::<2>();
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

tests/ui/generics/post_monomorphization_error_backtrace.rs

@@ -1,4 +1,5 @@
 // build-fail
+// compile-flags: -Zmir-opt-level=0
 
 fn assert_zst<T>() {
     struct F<T>(T);

tests/ui/generics/post_monomorphization_error_backtrace.stderr

@@ -1,27 +1,27 @@
 error[E0080]: evaluation of `assert_zst::F::<u32>::V` failed
-  --> $DIR/post_monomorphization_error_backtrace.rs:6:23
+  --> $DIR/post_monomorphization_error_backtrace.rs:7:23
    |
 LL |         const V: () = assert!(std::mem::size_of::<T>() == 0);
-   |                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program panicked at 'assertion failed: std::mem::size_of::<T>() == 0', $DIR/post_monomorphization_error_backtrace.rs:6:23
+   |                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program panicked at 'assertion failed: std::mem::size_of::<T>() == 0', $DIR/post_monomorphization_error_backtrace.rs:7:23
    |
    = note: this error originates in the macro `assert` (in Nightly builds, run with -Z macro-backtrace for more info)
 
 note: the above error was encountered while instantiating `fn assert_zst::<u32>`
-  --> $DIR/post_monomorphization_error_backtrace.rs:18:5
+  --> $DIR/post_monomorphization_error_backtrace.rs:19:5
    |
 LL |     assert_zst::<U>()
    |     ^^^^^^^^^^^^^^^^^
 
 error[E0080]: evaluation of `assert_zst::F::<i32>::V` failed
-  --> $DIR/post_monomorphization_error_backtrace.rs:6:23
+  --> $DIR/post_monomorphization_error_backtrace.rs:7:23
    |
 LL |         const V: () = assert!(std::mem::size_of::<T>() == 0);
-   |                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program panicked at 'assertion failed: std::mem::size_of::<T>() == 0', $DIR/post_monomorphization_error_backtrace.rs:6:23
+   |                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program panicked at 'assertion failed: std::mem::size_of::<T>() == 0', $DIR/post_monomorphization_error_backtrace.rs:7:23
    |
    = note: this error originates in the macro `assert` (in Nightly builds, run with -Z macro-backtrace for more info)
 
 note: the above error was encountered while instantiating `fn assert_zst::<i32>`
-  --> $DIR/post_monomorphization_error_backtrace.rs:18:5
+  --> $DIR/post_monomorphization_error_backtrace.rs:19:5
    |
 LL |     assert_zst::<U>()
    |     ^^^^^^^^^^^^^^^^^

tests/ui/inline-const/const-expr-generic-err.rs

@@ -1,4 +1,5 @@
 // build-fail
+// compile-flags: -Zmir-opt-level=0
 #![feature(inline_const)]
 
 fn foo<T>() {

tests/ui/inline-const/const-expr-generic-err.stderr

@@ -1,25 +1,25 @@
 error[E0080]: evaluation of `foo::<i32>::{constant#0}` failed
-  --> $DIR/const-expr-generic-err.rs:5:13
+  --> $DIR/const-expr-generic-err.rs:6:13
    |
 LL |     const { assert!(std::mem::size_of::<T>() == 0); }
-   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program panicked at 'assertion failed: std::mem::size_of::<T>() == 0', $DIR/const-expr-generic-err.rs:5:13
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program panicked at 'assertion failed: std::mem::size_of::<T>() == 0', $DIR/const-expr-generic-err.rs:6:13
    |
    = note: this error originates in the macro `assert` (in Nightly builds, run with -Z macro-backtrace for more info)
 
 note: the above error was encountered while instantiating `fn foo::<i32>`
-  --> $DIR/const-expr-generic-err.rs:13:5
+  --> $DIR/const-expr-generic-err.rs:14:5
    |
 LL |     foo::<i32>();
    |     ^^^^^^^^^^^^
 
 error[E0080]: evaluation of `bar::<0>::{constant#0}` failed
-  --> $DIR/const-expr-generic-err.rs:9:13
+  --> $DIR/const-expr-generic-err.rs:10:13
    |
 LL |     const { N - 1 }
    |             ^^^^^ attempt to compute `0_usize - 1_usize`, which would overflow
 
 note: the above error was encountered while instantiating `fn bar::<0>`
-  --> $DIR/const-expr-generic-err.rs:14:5
+  --> $DIR/const-expr-generic-err.rs:15:5
    |
 LL |     bar::<0>();
    |     ^^^^^^^^^^

tests/ui/polymorphization/const_parameters/functions.rs

@@ -1,5 +1,5 @@
 // build-fail
-// compile-flags:-Zpolymorphize=on
+// compile-flags:-Zpolymorphize=on -Zmir-opt-level=0
 #![feature(generic_const_exprs, rustc_attrs)]
 //~^ WARN the feature `generic_const_exprs` is incomplete
 

tests/ui/polymorphization/type_parameters/functions.rs

@@ -1,5 +1,5 @@
 // build-fail
-// compile-flags:-Zpolymorphize=on
+// compile-flags:-Zpolymorphize=on -Zmir-opt-level=0
 #![feature(rustc_attrs)]
 
 // This test checks that the polymorphization analysis correctly detects unused type

tests/ui/type_length_limit.rs

@@ -1,6 +1,6 @@
 // build-fail
 // error-pattern: reached the type-length limit while instantiating
-// compile-flags: -Copt-level=0
+// compile-flags: -Copt-level=0 -Zmir-opt-level=0
 // normalize-stderr-test: ".nll/" -> "/"
 
 // Test that the type length limit can be changed.