Auto merge of #48346 - emilio:pgo, r=alexcrichton

Add basic PGO support.

This PR adds two mutually exclusive options for profile usage and generation using LLVM's instruction profile generation (the same as clang uses), `-C pgo-use` and `-C pgo-gen`.

See each commit for details.

Author: bors

src/libprofiler_builtins/build.rs

@@ -27,6 +27,7 @@ fn main() {
                                    "InstrProfilingFile.c",
                                    "InstrProfilingMerge.c",
                                    "InstrProfilingMergeFile.c",
+                                   "InstrProfilingNameVar.c",
                                    "InstrProfilingPlatformDarwin.c",
                                    "InstrProfilingPlatformLinux.c",
                                    "InstrProfilingPlatformOther.c",
@@ -42,6 +43,8 @@ fn main() {
         cfg.define("strdup", Some("_strdup"));
         cfg.define("open", Some("_open"));
         cfg.define("fdopen", Some("_fdopen"));
+        cfg.define("getpid", Some("_getpid"));
+        cfg.define("fileno", Some("_fileno"));
     } else {
         // Turn off various features of gcc and such, mostly copying
         // compiler-rt's build system already
@@ -50,6 +53,7 @@ fn main() {
         cfg.flag("-fomit-frame-pointer");
         cfg.flag("-ffreestanding");
         cfg.define("VISIBILITY_HIDDEN", None);
+        cfg.define("COMPILER_RT_HAS_UNAME", Some("1"));
     }
 
     for src in profile_sources {

src/librustc/session/config.rs

@@ -1249,6 +1249,14 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
         "extra arguments to prepend to the linker invocation (space separated)"),
     profile: bool = (false, parse_bool, [TRACKED],
                      "insert profiling code"),
+    pgo_gen: Option<String> = (None, parse_opt_string, [TRACKED],
+        "Generate PGO profile data, to a given file, or to the default \
+         location if it's empty."),
+    pgo_use: String = (String::new(), parse_string, [TRACKED],
+        "Use PGO profile data from the given profile file."),
+    disable_instrumentation_preinliner: bool =
+        (false, parse_bool, [TRACKED], "Disable the instrumentation pre-inliner, \
+        useful for profiling / PGO."),
     relro_level: Option<RelroLevel> = (None, parse_relro_level, [TRACKED],
         "choose which RELRO level to use"),
     nll: bool = (false, parse_bool, [UNTRACKED],
@@ -1773,6 +1781,13 @@ pub fn build_session_options_and_crate_config(
         );
     }
 
+    if debugging_opts.pgo_gen.is_some() && !debugging_opts.pgo_use.is_empty() {
+        early_error(
+            error_format,
+            "options `-Z pgo-gen` and `-Z pgo-use` are exclusive",
+        );
+    }
+
     let mut output_types = BTreeMap::new();
     if !debugging_opts.parse_only {
         for list in matches.opt_strs("emit") {
@@ -2886,6 +2901,14 @@ mod tests {
         opts.debugging_opts.tls_model = Some(String::from("tls model"));
         assert!(reference.dep_tracking_hash() != opts.dep_tracking_hash());
 
+        opts = reference.clone();
+        opts.debugging_opts.pgo_gen = Some(String::from("abc"));
+        assert_ne!(reference.dep_tracking_hash(), opts.dep_tracking_hash());
+
+        opts = reference.clone();
+        opts.debugging_opts.pgo_use = String::from("abc");
+        assert_ne!(reference.dep_tracking_hash(), opts.dep_tracking_hash());
+
         opts = reference.clone();
         opts.cg.metadata = vec![String::from("A"), String::from("B")];
         assert!(reference.dep_tracking_hash() != opts.dep_tracking_hash());

src/librustc_llvm/diagnostic.rs

@@ -121,6 +121,7 @@ impl InlineAsmDiagnostic {
 pub enum Diagnostic {
     Optimization(OptimizationDiagnostic),
     InlineAsm(InlineAsmDiagnostic),
+    PGO(DiagnosticInfoRef),
 
     /// LLVM has other types that we do not wrap here.
     UnknownDiagnostic(DiagnosticInfoRef),
@@ -160,6 +161,10 @@ impl Diagnostic {
                 Optimization(OptimizationDiagnostic::unpack(OptimizationFailure, di))
             }
 
+            Dk::PGOProfile => {
+                PGO(di)
+            }
+
             _ => UnknownDiagnostic(di),
         }
     }

src/librustc_llvm/ffi.rs

@@ -322,6 +322,7 @@ pub enum DiagnosticKind {
     OptimizationRemarkAnalysisAliasing,
     OptimizationRemarkOther,
     OptimizationFailure,
+    PGOProfile,
 }
 
 /// LLVMRustArchiveKind
@@ -1646,7 +1647,9 @@ extern "C" {
                                                OptLevel: CodeGenOptLevel,
                                                MergeFunctions: bool,
                                                SLPVectorize: bool,
-                                               LoopVectorize: bool);
+                                               LoopVectorize: bool,
+                                               PGOGenPath: *const c_char,
+                                               PGOUsePath: *const c_char);
     pub fn LLVMRustAddLibraryInfo(PM: PassManagerRef,
                                   M: ModuleRef,
                                   DisableSimplifyLibCalls: bool);
@@ -1741,6 +1744,7 @@ extern "C" {
     pub fn LLVMRustModuleCost(M: ModuleRef) -> u64;
 
     pub fn LLVMRustThinLTOAvailable() -> bool;
+    pub fn LLVMRustPGOAvailable() -> bool;
     pub fn LLVMRustWriteThinBitcodeToFile(PMR: PassManagerRef,
                                           M: ModuleRef,
                                           BC: *const c_char) -> bool;

src/librustc_metadata/creader.rs

@@ -784,7 +784,9 @@ impl<'a> CrateLoader<'a> {
     }
 
     fn inject_profiler_runtime(&mut self) {
-        if self.sess.opts.debugging_opts.profile {
+        if self.sess.opts.debugging_opts.profile ||
+            self.sess.opts.debugging_opts.pgo_gen.is_some()
+        {
             info!("loading profiler");
 
             let symbol = Symbol::intern("profiler_builtins");

src/librustc_trans/attributes.rs

@@ -92,6 +92,11 @@ pub fn set_probestack(cx: &CodegenCx, llfn: ValueRef) {
         _ => {}
     }
 
+    // probestack doesn't play nice either with pgo-gen.
+    if cx.sess().opts.debugging_opts.pgo_gen.is_some() {
+        return;
+    }
+
     // Flag our internal `__rust_probestack` function as the stack probe symbol.
     // This is defined in the `compiler-builtins` crate for each architecture.
     llvm::AddFunctionAttrStringValue(

src/librustc_trans/back/link.rs

@@ -1085,6 +1085,10 @@ fn link_args(cmd: &mut Linker,
         cmd.build_static_executable();
     }
 
+    if sess.opts.debugging_opts.pgo_gen.is_some() {
+        cmd.pgo_gen();
+    }
+
     // FIXME (#2397): At some point we want to rpath our guesses as to
     // where extern libraries might live, based on the
     // addl_lib_search_paths

src/librustc_trans/back/linker.rs

@@ -117,6 +117,7 @@ pub trait Linker {
     fn partial_relro(&mut self);
     fn no_relro(&mut self);
     fn optimize(&mut self);
+    fn pgo_gen(&mut self);
     fn debuginfo(&mut self);
     fn no_default_libraries(&mut self);
     fn build_dylib(&mut self, out_filename: &Path);
@@ -280,6 +281,24 @@ impl<'a> Linker for GccLinker<'a> {
         }
     }
 
+    fn pgo_gen(&mut self) {
+        if !self.sess.target.target.options.linker_is_gnu { return }
+
+        // If we're doing PGO generation stuff and on a GNU-like linker, use the
+        // "-u" flag to properly pull in the profiler runtime bits.
+        //
+        // This is because LLVM otherwise won't add the needed initialization
+        // for us on Linux (though the extra flag should be harmless if it
+        // does).
+        //
+        // See https://reviews.llvm.org/D14033 and https://reviews.llvm.org/D14030.
+        //
+        // Though it may be worth to try to revert those changes upstream, since
+        // the overhead of the initialization should be minor.
+        self.cmd.arg("-u");
+        self.cmd.arg("__llvm_profile_runtime");
+    }
+
     fn debuginfo(&mut self) {
         match self.sess.opts.debuginfo {
             DebugInfoLevel::NoDebugInfo => {
@@ -520,6 +539,10 @@ impl<'a> Linker for MsvcLinker<'a> {
         // Needs more investigation of `/OPT` arguments
     }
 
+    fn pgo_gen(&mut self) {
+        // Nothing needed here.
+    }
+
     fn debuginfo(&mut self) {
         // This will cause the Microsoft linker to generate a PDB file
         // from the CodeView line tables in the object files.
@@ -723,6 +746,10 @@ impl<'a> Linker for EmLinker<'a> {
         self.cmd.args(&["--memory-init-file", "0"]);
     }
 
+    fn pgo_gen(&mut self) {
+        // noop, but maybe we need something like the gnu linker?
+    }
+
     fn debuginfo(&mut self) {
         // Preserve names or generate source maps depending on debug info
         self.cmd.arg(match self.sess.opts.debuginfo {
@@ -888,6 +915,9 @@ impl Linker for WasmLd {
     fn optimize(&mut self) {
     }
 
+    fn pgo_gen(&mut self) {
+    }
+
     fn debuginfo(&mut self) {
     }
 

src/librustc_trans/back/symbol_export.rs

@@ -223,6 +223,20 @@ fn exported_symbols_provider_local<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
         }
     }
 
+    if tcx.sess.opts.debugging_opts.pgo_gen.is_some() {
+        // These are weak symbols that point to the profile version and the
+        // profile name, which need to be treated as exported so LTO doesn't nix
+        // them.
+        const PROFILER_WEAK_SYMBOLS: [&'static str; 2] = [
+            "__llvm_profile_raw_version",
+            "__llvm_profile_filename",
+        ];
+        for sym in &PROFILER_WEAK_SYMBOLS {
+            let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(sym));
+            symbols.push((exported_symbol, SymbolExportLevel::C));
+        }
+    }
+
     if tcx.sess.crate_types.borrow().contains(&config::CrateTypeDylib) {
         let symbol_name = metadata_symbol_name(tcx);
         let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(&symbol_name));

src/librustc_trans/back/write.rs

@@ -240,6 +240,9 @@ pub struct ModuleConfig {
     /// Some(level) to optimize binary size, or None to not affect program size.
     opt_size: Option<llvm::CodeGenOptSize>,
 
+    pgo_gen: Option<String>,
+    pgo_use: String,
+
     // Flags indicating which outputs to produce.
     emit_no_opt_bc: bool,
     emit_bc: bool,
@@ -274,6 +277,9 @@ impl ModuleConfig {
             opt_level: None,
             opt_size: None,
 
+            pgo_gen: None,
+            pgo_use: String::new(),
+
             emit_no_opt_bc: false,
             emit_bc: false,
             emit_bc_compressed: false,
@@ -492,8 +498,13 @@ unsafe extern "C" fn diagnostic_handler(info: DiagnosticInfoRef, user: *mut c_vo
                                                 opt.message));
             }
         }
-
-        _ => (),
+        llvm::diagnostic::PGO(diagnostic_ref) => {
+            let msg = llvm::build_string(|s| {
+                llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s)
+            }).expect("non-UTF8 PGO diagnostic");
+            diag_handler.warn(&msg);
+        }
+        llvm::diagnostic::UnknownDiagnostic(..) => {},
     }
 }
 
@@ -932,6 +943,9 @@ pub fn start_async_translation(tcx: TyCtxt,
         modules_config.passes.push("insert-gcov-profiling".to_owned())
     }
 
+    modules_config.pgo_gen = sess.opts.debugging_opts.pgo_gen.clone();
+    modules_config.pgo_use = sess.opts.debugging_opts.pgo_use.clone();
+
     modules_config.opt_level = Some(get_llvm_opt_level(sess.opts.optimize));
     modules_config.opt_size = Some(get_llvm_opt_size(sess.opts.optimize));
 
@@ -2046,18 +2060,36 @@ pub unsafe fn with_llvm_pmb(llmod: ModuleRef,
                             config: &ModuleConfig,
                             opt_level: llvm::CodeGenOptLevel,
                             f: &mut FnMut(llvm::PassManagerBuilderRef)) {
+    use std::ptr;
+
     // Create the PassManagerBuilder for LLVM. We configure it with
     // reasonable defaults and prepare it to actually populate the pass
     // manager.
     let builder = llvm::LLVMPassManagerBuilderCreate();
     let opt_size = config.opt_size.unwrap_or(llvm::CodeGenOptSizeNone);
     let inline_threshold = config.inline_threshold;
 
-    llvm::LLVMRustConfigurePassManagerBuilder(builder,
-                                              opt_level,
-                                              config.merge_functions,
-                                              config.vectorize_slp,
-                                              config.vectorize_loop);
+    let pgo_gen_path = config.pgo_gen.as_ref().map(|s| {
+        let s = if s.is_empty() { "default_%m.profraw" } else { s };
+        CString::new(s.as_bytes()).unwrap()
+    });
+
+    let pgo_use_path = if config.pgo_use.is_empty() {
+        None
+    } else {
+        Some(CString::new(config.pgo_use.as_bytes()).unwrap())
+    };
+
+    llvm::LLVMRustConfigurePassManagerBuilder(
+        builder,
+        opt_level,
+        config.merge_functions,
+        config.vectorize_slp,
+        config.vectorize_loop,
+        pgo_gen_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()),
+        pgo_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()),
+    );
+
     llvm::LLVMPassManagerBuilderSetSizeLevel(builder, opt_size as u32);
 
     if opt_size != llvm::CodeGenOptSizeNone {

src/librustc_trans/base.rs

@@ -708,6 +708,13 @@ pub fn trans_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
         }
     }
 
+    if (tcx.sess.opts.debugging_opts.pgo_gen.is_some() ||
+        !tcx.sess.opts.debugging_opts.pgo_use.is_empty()) &&
+        unsafe { !llvm::LLVMRustPGOAvailable() }
+    {
+        tcx.sess.fatal("this compiler's LLVM does not support PGO");
+    }
+
     let crate_hash = tcx.crate_hash(LOCAL_CRATE);
     let link_meta = link::build_link_meta(crate_hash);
 

src/librustc_trans/llvm_util.rs

@@ -61,6 +61,9 @@ unsafe fn configure_llvm(sess: &Session) {
         add("rustc"); // fake program name
         if sess.time_llvm_passes() { add("-time-passes"); }
         if sess.print_llvm_passes() { add("-debug-pass=Structure"); }
+        if sess.opts.debugging_opts.disable_instrumentation_preinliner {
+            add("-disable-preinline");
+        }
 
         for arg in &sess.opts.cg.llvm_args {
             add(&(*arg));