diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 3e7d1d54f1284..1f57d85866ff2 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -383,7 +383,7 @@ def __init__(self): self.nix_deps_dir = None self.rustc_commit = None - def download_stage0(self): + def download_toolchain(self, stage0=True, rustc_channel=None): """Fetch the build system for Rust, written in Rust This method will build a cache directory, then it will fetch the @@ -393,43 +393,47 @@ def download_stage0(self): Each downloaded tarball is extracted, after that, the script will move all the content to the right place. """ - rustc_channel = self.rustc_channel + if rustc_channel is None: + rustc_channel = self.rustc_channel rustfmt_channel = self.rustfmt_channel - - if self.rustc().startswith(self.bin_root()) and \ - (not os.path.exists(self.rustc()) or - self.program_out_of_date(self.rustc_stamp(), self.date + str(self.rustc_commit))): - if os.path.exists(self.bin_root()): - shutil.rmtree(self.bin_root()) - download_rustc = self.rustc_commit is not None + bin_root = self.bin_root(stage0) + + key = self.date + if not stage0: + key += str(self.rustc_commit) + if self.rustc(stage0).startswith(bin_root) and \ + (not os.path.exists(self.rustc(stage0)) or + self.program_out_of_date(self.rustc_stamp(stage0), key)): + if os.path.exists(bin_root): + shutil.rmtree(bin_root) tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' filename = "rust-std-{}-{}{}".format( rustc_channel, self.build, tarball_suffix) pattern = "rust-std-{}".format(self.build) - self._download_component_helper(filename, pattern, tarball_suffix, download_rustc) + self._download_component_helper(filename, pattern, tarball_suffix, stage0) filename = "rustc-{}-{}{}".format(rustc_channel, self.build, tarball_suffix) - self._download_component_helper(filename, "rustc", tarball_suffix, download_rustc) + self._download_component_helper(filename, "rustc", tarball_suffix, stage0) filename = "cargo-{}-{}{}".format(rustc_channel, self.build, tarball_suffix) self._download_component_helper(filename, "cargo", tarball_suffix) - if self.rustc_commit is not None: + if not stage0: filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix) self._download_component_helper( - filename, "rustc-dev", tarball_suffix, download_rustc + filename, "rustc-dev", tarball_suffix, stage0 ) - self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root())) - self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root())) - self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root())) - lib_dir = "{}/lib".format(self.bin_root()) + self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root)) + self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root)) + self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root)) + lib_dir = "{}/lib".format(bin_root) for lib in os.listdir(lib_dir): if lib.endswith(".so"): self.fix_bin_or_dylib(os.path.join(lib_dir, lib), rpath_libz=True) - with output(self.rustc_stamp()) as rust_stamp: - rust_stamp.write(self.date + str(self.rustc_commit)) + with output(self.rustc_stamp(stage0)) as rust_stamp: + rust_stamp.write(key) - if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and ( + if self.rustfmt() and self.rustfmt().startswith(bin_root) and ( not os.path.exists(self.rustfmt()) or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel) ): @@ -440,12 +444,13 @@ def download_stage0(self): self._download_component_helper( filename, "rustfmt-preview", tarball_suffix, key=date ) - self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root())) - self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root())) + self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root)) + self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root)) with output(self.rustfmt_stamp()) as rustfmt_stamp: rustfmt_stamp.write(self.rustfmt_channel) - if self.downloading_llvm(): + # Avoid downloading LLVM twice (once for stage0 and once for the master rustc) + if self.downloading_llvm() and stage0: # We want the most recent LLVM submodule update to avoid downloading # LLVM more often than necessary. # @@ -496,27 +501,26 @@ def downloading_llvm(self): or (opt == "if-available" and self.build in supported_platforms) def _download_component_helper( - self, filename, pattern, tarball_suffix, download_rustc=False, key=None + self, filename, pattern, tarball_suffix, stage0=True, key=None ): if key is None: - if download_rustc: - key = self.rustc_commit - else: + if stage0: key = self.date + else: + key = self.rustc_commit cache_dst = os.path.join(self.build_dir, "cache") rustc_cache = os.path.join(cache_dst, key) if not os.path.exists(rustc_cache): os.makedirs(rustc_cache) - if download_rustc: - url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(self.rustc_commit) - else: + if stage0: url = "{}/dist/{}".format(self._download_url, key) + else: + url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(self.rustc_commit) tarball = os.path.join(rustc_cache, filename) if not os.path.exists(tarball): - do_verify = not download_rustc - get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=do_verify) - unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose) + get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=stage0) + unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose) def _download_ci_llvm(self, llvm_sha, llvm_assertions): cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions) @@ -574,10 +578,10 @@ def fix_bin_or_dylib(self, fname, rpath_libz=False): nix_os_msg = "info: you seem to be running NixOS. Attempting to patch" print(nix_os_msg, fname) - # Only build `stage0/.nix-deps` once. + # Only build `.nix-deps` once. nix_deps_dir = self.nix_deps_dir if not nix_deps_dir: - nix_deps_dir = "{}/.nix-deps".format(self.bin_root()) + nix_deps_dir = ".nix-deps" if not os.path.exists(nix_deps_dir): os.makedirs(nix_deps_dir) @@ -635,8 +639,8 @@ def fix_bin_or_dylib(self, fname, rpath_libz=False): print("warning: failed to call patchelf:", reason) return - # Return the stage1 compiler to download, if any. - def maybe_download_rustc(self): + # If `download-rustc` is set, download the most recent commit with CI artifacts + def maybe_download_ci_toolchain(self): # If `download-rustc` is not set, default to rebuilding. if self.get_toml("download-rustc", section="rust") != "true": return None @@ -656,17 +660,23 @@ def maybe_download_rustc(self): if status != 0: print("warning: `download-rustc` is enabled, but there are changes to compiler/") - return commit + if self.verbose: + print("using downloaded stage1 artifacts from CI (commit {})".format(commit)) + self.rustc_commit = commit + # FIXME: support downloading artifacts from the beta channel + self.download_toolchain(False, "nightly") - def rustc_stamp(self): - """Return the path for .rustc-stamp + def rustc_stamp(self, stage0): + """Return the path for .rustc-stamp at the given stage >>> rb = RustBuild() >>> rb.build_dir = "build" - >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp") + >>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp") + True + >>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp") True """ - return os.path.join(self.bin_root(), '.rustc-stamp') + return os.path.join(self.bin_root(stage0), '.rustc-stamp') def rustfmt_stamp(self): """Return the path for .rustfmt-stamp @@ -676,7 +686,7 @@ def rustfmt_stamp(self): >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp") True """ - return os.path.join(self.bin_root(), '.rustfmt-stamp') + return os.path.join(self.bin_root(True), '.rustfmt-stamp') def llvm_stamp(self): """Return the path for .rustfmt-stamp @@ -696,21 +706,27 @@ def program_out_of_date(self, stamp_path, key): with open(stamp_path, 'r') as stamp: return key != stamp.read() - def bin_root(self): - """Return the binary root directory + def bin_root(self, stage0): + """Return the binary root directory for the given stage >>> rb = RustBuild() >>> rb.build_dir = "build" - >>> rb.bin_root() == os.path.join("build", "stage0") + >>> rb.bin_root(True) == os.path.join("build", "stage0") + True + >>> rb.bin_root(False) == os.path.join("build", "ci-rustc") True When the 'build' property is given should be a nested directory: >>> rb.build = "devel" - >>> rb.bin_root() == os.path.join("build", "devel", "stage0") + >>> rb.bin_root(True) == os.path.join("build", "devel", "stage0") True """ - return os.path.join(self.build_dir, self.build, "stage0") + if stage0: + subdir = "stage0" + else: + subdir = "ci-rustc" + return os.path.join(self.build_dir, self.build, subdir) def llvm_root(self): """Return the CI LLVM root directory @@ -773,9 +789,9 @@ def cargo(self): """Return config path for cargo""" return self.program_config('cargo') - def rustc(self): + def rustc(self, stage0): """Return config path for rustc""" - return self.program_config('rustc') + return self.program_config('rustc', stage0) def rustfmt(self): """Return config path for rustfmt""" @@ -783,23 +799,27 @@ def rustfmt(self): return None return self.program_config('rustfmt') - def program_config(self, program): - """Return config path for the given program + def program_config(self, program, stage0=True): + """Return config path for the given program at the given stage >>> rb = RustBuild() >>> rb.config_toml = 'rustc = "rustc"\\n' >>> rb.program_config('rustc') 'rustc' >>> rb.config_toml = '' - >>> cargo_path = rb.program_config('cargo') - >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(), + >>> cargo_path = rb.program_config('cargo', True) + >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True), + ... "bin", "cargo") + True + >>> cargo_path = rb.program_config('cargo', False) + >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False), ... "bin", "cargo") True """ config = self.get_toml(program) if config: return os.path.expanduser(config) - return os.path.join(self.bin_root(), "bin", "{}{}".format( + return os.path.join(self.bin_root(stage0), "bin", "{}{}".format( program, self.exe_suffix())) @staticmethod @@ -854,14 +874,14 @@ def build_bootstrap(self): if "CARGO_BUILD_TARGET" in env: del env["CARGO_BUILD_TARGET"] env["CARGO_TARGET_DIR"] = build_dir - env["RUSTC"] = self.rustc() - env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ + env["RUSTC"] = self.rustc(True) + env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \ (os.pathsep + env["LD_LIBRARY_PATH"]) \ if "LD_LIBRARY_PATH" in env else "" - env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ + env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \ (os.pathsep + env["DYLD_LIBRARY_PATH"]) \ if "DYLD_LIBRARY_PATH" in env else "" - env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ + env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \ (os.pathsep + env["LIBRARY_PATH"]) \ if "LIBRARY_PATH" in env else "" # preserve existing RUSTFLAGS @@ -884,7 +904,7 @@ def build_bootstrap(self): if self.get_toml("deny-warnings", "rust") != "false": env["RUSTFLAGS"] += " -Dwarnings" - env["PATH"] = os.path.join(self.bin_root(), "bin") + \ + env["PATH"] = os.path.join(self.bin_root(True), "bin") + \ os.pathsep + env["PATH"] if not os.path.isfile(self.cargo()): raise Exception("no cargo executable found at `{}`".format( @@ -1135,14 +1155,9 @@ def bootstrap(help_triggered): build.update_submodules() # Fetch/build the bootstrap - build.rustc_commit = build.maybe_download_rustc() - if build.rustc_commit is not None: - if build.verbose: - commit = build.rustc_commit - print("using downloaded stage1 artifacts from CI (commit {})".format(commit)) - # FIXME: support downloading artifacts from the beta channel - build.rustc_channel = "nightly" - build.download_stage0() + build.download_toolchain() + # Download the master compiler if `download-rustc` is set + build.maybe_download_ci_toolchain() sys.stdout.flush() build.ensure_vendored() build.build_bootstrap() diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index 9398f211721b8..8244c7710ab7b 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -65,7 +65,9 @@ impl Step for Std { // These artifacts were already copied (in `impl Step for Sysroot`). // Don't recompile them. - if builder.config.download_rustc { + // NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler, + // so its artifacts can't be reused. + if builder.config.download_rustc && compiler.stage != 0 { return; } @@ -513,7 +515,9 @@ impl Step for Rustc { let compiler = self.compiler; let target = self.target; - if builder.config.download_rustc { + // NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler, + // so its artifacts can't be reused. + if builder.config.download_rustc && compiler.stage != 0 { // Copy the existing artifacts instead of rebuilding them. // NOTE: this path is only taken for tools linking to rustc-dev. builder.ensure(Sysroot { compiler }); @@ -934,14 +938,15 @@ impl Step for Sysroot { t!(fs::create_dir_all(&sysroot)); // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0. - if builder.config.download_rustc { + if builder.config.download_rustc && compiler.stage != 0 { assert_eq!( builder.config.build, compiler.host, "Cross-compiling is not yet supported with `download-rustc`", ); // Copy the compiler into the correct sysroot. - let stage0_dir = builder.config.out.join(&*builder.config.build.triple).join("stage0"); - builder.cp_r(&stage0_dir, &sysroot); + let ci_rustc_dir = + builder.config.out.join(&*builder.config.build.triple).join("ci-rustc"); + builder.cp_r(&ci_rustc_dir, &sysroot); return INTERNER.intern_path(sysroot); } diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index b9b090bb2d2de..471f671fd272a 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -687,51 +687,6 @@ impl Config { set(&mut config.print_step_timings, build.print_step_timings); set(&mut config.print_step_rusage, build.print_step_rusage); - // See https://github.com/rust-lang/compiler-team/issues/326 - config.stage = match config.cmd { - Subcommand::Check { .. } => flags.stage.or(build.check_stage).unwrap_or(0), - Subcommand::Doc { .. } => flags.stage.or(build.doc_stage).unwrap_or(0), - Subcommand::Build { .. } => flags.stage.or(build.build_stage).unwrap_or(1), - Subcommand::Test { .. } => flags.stage.or(build.test_stage).unwrap_or(1), - Subcommand::Bench { .. } => flags.stage.or(build.bench_stage).unwrap_or(2), - Subcommand::Dist { .. } => flags.stage.or(build.dist_stage).unwrap_or(2), - Subcommand::Install { .. } => flags.stage.or(build.install_stage).unwrap_or(2), - // These are all bootstrap tools, which don't depend on the compiler. - // The stage we pass shouldn't matter, but use 0 just in case. - Subcommand::Clean { .. } - | Subcommand::Clippy { .. } - | Subcommand::Fix { .. } - | Subcommand::Run { .. } - | Subcommand::Setup { .. } - | Subcommand::Format { .. } => flags.stage.unwrap_or(0), - }; - - // CI should always run stage 2 builds, unless it specifically states otherwise - #[cfg(not(test))] - if flags.stage.is_none() && crate::CiEnv::current() != crate::CiEnv::None { - match config.cmd { - Subcommand::Test { .. } - | Subcommand::Doc { .. } - | Subcommand::Build { .. } - | Subcommand::Bench { .. } - | Subcommand::Dist { .. } - | Subcommand::Install { .. } => { - assert_eq!( - config.stage, 2, - "x.py should be run with `--stage 2` on CI, but was run with `--stage {}`", - config.stage, - ); - } - Subcommand::Clean { .. } - | Subcommand::Check { .. } - | Subcommand::Clippy { .. } - | Subcommand::Fix { .. } - | Subcommand::Run { .. } - | Subcommand::Setup { .. } - | Subcommand::Format { .. } => {} - } - } - config.verbose = cmp::max(config.verbose, flags.verbose); if let Some(install) = toml.install { @@ -1005,6 +960,59 @@ impl Config { let default = config.channel == "dev"; config.ignore_git = ignore_git.unwrap_or(default); + let download_rustc = config.download_rustc; + // See https://github.com/rust-lang/compiler-team/issues/326 + config.stage = match config.cmd { + Subcommand::Check { .. } => flags.stage.or(build.check_stage).unwrap_or(0), + // `download-rustc` only has a speed-up for stage2 builds. Default to stage2 unless explicitly overridden. + Subcommand::Doc { .. } => { + flags.stage.or(build.doc_stage).unwrap_or(if download_rustc { 2 } else { 0 }) + } + Subcommand::Build { .. } => { + flags.stage.or(build.build_stage).unwrap_or(if download_rustc { 2 } else { 1 }) + } + Subcommand::Test { .. } => { + flags.stage.or(build.test_stage).unwrap_or(if download_rustc { 2 } else { 1 }) + } + Subcommand::Bench { .. } => flags.stage.or(build.bench_stage).unwrap_or(2), + Subcommand::Dist { .. } => flags.stage.or(build.dist_stage).unwrap_or(2), + Subcommand::Install { .. } => flags.stage.or(build.install_stage).unwrap_or(2), + // These are all bootstrap tools, which don't depend on the compiler. + // The stage we pass shouldn't matter, but use 0 just in case. + Subcommand::Clean { .. } + | Subcommand::Clippy { .. } + | Subcommand::Fix { .. } + | Subcommand::Run { .. } + | Subcommand::Setup { .. } + | Subcommand::Format { .. } => flags.stage.unwrap_or(0), + }; + + // CI should always run stage 2 builds, unless it specifically states otherwise + #[cfg(not(test))] + if flags.stage.is_none() && crate::CiEnv::current() != crate::CiEnv::None { + match config.cmd { + Subcommand::Test { .. } + | Subcommand::Doc { .. } + | Subcommand::Build { .. } + | Subcommand::Bench { .. } + | Subcommand::Dist { .. } + | Subcommand::Install { .. } => { + assert_eq!( + config.stage, 2, + "x.py should be run with `--stage 2` on CI, but was run with `--stage {}`", + config.stage, + ); + } + Subcommand::Clean { .. } + | Subcommand::Check { .. } + | Subcommand::Clippy { .. } + | Subcommand::Fix { .. } + | Subcommand::Run { .. } + | Subcommand::Setup { .. } + | Subcommand::Format { .. } => {} + } + } + config } diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs index 3fc3b68fd8681..30bd9c1b4a1ee 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/tool.rs @@ -514,6 +514,19 @@ impl Step for Rustdoc { // rustc compiler it's paired with, so it must be built with the previous stage compiler. let build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build); + // When using `download-rustc` and a stage0 build_compiler, copying rustc doesn't actually + // build stage0 libstd (because the libstd in sysroot has the wrong ABI). Explicitly build + // it. + builder.ensure(compile::Std { compiler: build_compiler, target: target_compiler.host }); + builder.ensure(compile::Rustc { compiler: build_compiler, target: target_compiler.host }); + // NOTE: this implies that `download-rustc` is pretty useless when compiling with the stage0 + // compiler, since you do just as much work. + if !builder.config.dry_run && builder.config.download_rustc && build_compiler.stage == 0 { + println!( + "warning: `download-rustc` does nothing when building stage1 tools; consider using `--stage 2` instead" + ); + } + // The presence of `target_compiler` ensures that the necessary libraries (codegen backends, // compiler libraries, ...) are built. Rustdoc does not require the presence of any // libraries within sysroot_libdir (i.e., rustlib), though doctests may want it (since