From 00e1a90e569e8a76bab96877dac840955cbea9c0 Mon Sep 17 00:00:00 2001 From: Jan David Date: Mon, 20 May 2024 12:56:39 +0200 Subject: [PATCH 01/44] Fix error message when parsing actions --- src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.rs b/src/config.rs index 070ce46..fe046aa 100644 --- a/src/config.rs +++ b/src/config.rs @@ -73,7 +73,7 @@ impl FromStr for Action { match input { "promote-release" => Ok(Action::PromoteRelease), "promote-branches" => Ok(Action::PromoteBranches), - _ => anyhow::bail!("unknown channel: {}", input), + _ => anyhow::bail!("unknown action: {}", input), } } } From 0a64d2565e4d8cf6b78a91707f2ab4f20731fe5b Mon Sep 17 00:00:00 2001 From: Jan David Date: Mon, 20 May 2024 13:17:09 +0200 Subject: [PATCH 02/44] Create action for rustup --- src/config.rs | 10 ++++++++++ src/main.rs | 2 ++ src/rustup.rs | 7 +++++++ 3 files changed, 19 insertions(+) create mode 100644 src/rustup.rs diff --git a/src/config.rs b/src/config.rs index fe046aa..fc3f0fb 100644 --- a/src/config.rs +++ b/src/config.rs @@ -48,6 +48,8 @@ impl std::fmt::Display for Channel { } } +// Allow all variant names to start with `Promote` +#[allow(clippy::enum_variant_names)] #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub(crate) enum Action { /// This is the default action, what we'll do if the environment variable @@ -64,6 +66,13 @@ pub(crate) enum Action { /// * Create a rust-lang/cargo branch for the appropriate beta commit. /// * Post a PR against the newly created beta branch bump src/ci/channel to `beta`. PromoteBranches, + + /// This promotes a new rustup release: + /// + /// * Copy binaries into archives + /// * Copy binaries from dev-static to production + /// * Update dev release number + PromoteRustup, } impl FromStr for Action { @@ -73,6 +82,7 @@ impl FromStr for Action { match input { "promote-release" => Ok(Action::PromoteRelease), "promote-branches" => Ok(Action::PromoteBranches), + "promote-rustup" => Ok(Action::PromoteRustup), _ => anyhow::bail!("unknown action: {}", input), } } diff --git a/src/main.rs b/src/main.rs index 5dd7aea..aed9722 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,6 +8,7 @@ mod discourse; mod fastly; mod github; mod recompress; +mod rustup; mod sign; mod smoke_test; @@ -76,6 +77,7 @@ impl Context { match self.config.action { config::Action::PromoteRelease => self.do_release()?, config::Action::PromoteBranches => self.do_branching()?, + config::Action::PromoteRustup => self.do_rustup()?, } Ok(()) } diff --git a/src/rustup.rs b/src/rustup.rs new file mode 100644 index 0000000..016adae --- /dev/null +++ b/src/rustup.rs @@ -0,0 +1,7 @@ +use crate::Context; + +impl Context { + pub fn do_rustup(&mut self) -> anyhow::Result<()> { + Ok(()) + } +} From 8d4a4476cbeb9ab6b3153df55738921fa2a9b83d Mon Sep 17 00:00:00 2001 From: Jan David Date: Wed, 22 May 2024 16:55:12 +0200 Subject: [PATCH 03/44] Implement release process for rustup --- src/main.rs | 2 +- src/rustup.rs | 139 +++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 138 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index aed9722..ad12f93 100644 --- a/src/main.rs +++ b/src/main.rs @@ -77,7 +77,7 @@ impl Context { match self.config.action { config::Action::PromoteRelease => self.do_release()?, config::Action::PromoteBranches => self.do_branching()?, - config::Action::PromoteRustup => self.do_rustup()?, + config::Action::PromoteRustup => self.promote_rustup()?, } Ok(()) } diff --git a/src/rustup.rs b/src/rustup.rs index 016adae..723ff7c 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -1,7 +1,142 @@ -use crate::Context; +use std::fs; +use std::path::{Path, PathBuf}; + +use anyhow::{anyhow, Error}; + +use crate::config::Channel; +use crate::{run, Context}; impl Context { - pub fn do_rustup(&mut self) -> anyhow::Result<()> { + /// Promote a `rustup` release + /// + /// The [release process] for `rustup` involves copying existing artifacts from one S3 bucket to + /// another, updating the manifest, and archiving the artifacts for long-term storage. + /// + /// `rustup` uses different branches to manage releases. Whenever a commit is pushed to the + /// `stable` branch in [rust-lang/rustup], GitHub Actions workflows build release artifacts and + /// copy them into `s3://dev-static-rust-lang-org/rustup/dist/`. + /// + /// When a new release is done and this method is invoked, it downloads the artifacts from that + /// bucket (which must always be set as the `DOWNLOAD_BUCKET` variable). A copy of the artifacts + /// is archived in `s3://${UPLOAD_BUCKET}/rustup/archive/${version}/`, where `version` is passed + /// to this program as a command-line argument. `UPLOAD_BUCKET` can either be the `dev-static` + /// or the `static` bucket. + /// + /// If the release is for the `stable` channel, the artifacts are also copied to the `dist/` + /// path in the `UPLOAD_BUCKET` bucket. The `dist/` path is used by the `rustup` installer to + /// download the latest release. + /// + /// Then, the `release-stable.toml` manifest is updated with the new version and copied to + /// `s3://${UPLOAD_BUCKET}/rustup/release-stable.toml`. + /// + /// [release process]: https://rust-lang.github.io/rustup/dev-guide/release-process.html + /// [rust-lang/rustup]: https://github.com/rust-lang/rustup + pub fn promote_rustup(&mut self) -> anyhow::Result<()> { + println!("Checking channel..."); + if self.config.channel != Channel::Stable && self.config.channel != Channel::Beta { + return Err(anyhow!( + "promoting rustup is only supported for the stable and beta channels" + )); + } + + // Download the rustup artifacts from S3 + println!("Downloading artifacts from dev-static..."); + let dist_dir = self.download_rustup_artifacts()?; + + // Archive the artifacts + println!("Archiving artifacts..."); + self.archive_rustup_artifacts(&dist_dir)?; + + if self.config.channel == Channel::Stable { + // Promote the artifacts to the release bucket + println!("Promoting artifacts to dist/..."); + self.promote_rustup_artifacts(&dist_dir)?; + } + + // Update the release number + println!("Updating version and manifest..."); + self.update_rustup_release()?; + Ok(()) } + + fn download_rustup_artifacts(&mut self) -> Result { + let dl = self.dl_dir().join("dist"); + // Remove the directory if it exists, otherwise just ignore. + let _ = fs::remove_dir_all(&dl); + fs::create_dir_all(&dl)?; + + run(self + .aws_s3() + .arg("cp") + .arg("--recursive") + .arg("--only-show-errors") + .arg(&self.s3_artifacts_url("dist/")) + .arg(format!("{}/", dl.display())))?; + + Ok(dl) + } + + fn archive_rustup_artifacts(&mut self, dist_dir: &Path) -> Result<(), Error> { + let version = self + .current_version + .as_ref() + .ok_or_else(|| anyhow!("failed to get current version for rustup release"))?; + + let path = format!("archive/{}/", version); + + self.upload_rustup_artifacts(dist_dir, &path) + } + + fn promote_rustup_artifacts(&mut self, dist_dir: &Path) -> Result<(), Error> { + let release_bucket_url = format!( + "s3://{}/{}/{}", + self.config.upload_bucket, + self.config.download_dir, + dist_dir.display(), + ); + + run(self + .aws_s3() + .arg("cp") + .arg("--recursive") + .arg("--only-show-errors") + .arg(format!("{}/", dist_dir.display())) + .arg(&release_bucket_url)) + } + + fn upload_rustup_artifacts(&mut self, dist_dir: &Path, target_path: &str) -> Result<(), Error> { + run(self + .aws_s3() + .arg("cp") + .arg("--recursive") + .arg("--only-show-errors") + .arg(format!("{}/", dist_dir.display())) + .arg(&self.s3_artifacts_url(target_path))) + } + + fn update_rustup_release(&mut self) -> Result<(), Error> { + let version = self + .current_version + .as_ref() + .ok_or_else(|| anyhow!("failed to get current version for rustup release"))?; + + let manifest_path = self.dl_dir().join("release-stable.toml"); + let manifest = format!( + r#" +schema-version = '1' +version = '{}' + "#, + version + ); + + fs::write(&manifest_path, manifest)?; + + run(self + .aws_s3() + .arg("cp") + .arg("--only-show-errors") + .arg(manifest_path) + .arg(&self.s3_artifacts_url("release-stable.toml"))) + } } From d40ff454acab7dc28d30661fde32ab085edcc278 Mon Sep 17 00:00:00 2001 From: Jan David Date: Thu, 27 Jun 2024 12:06:11 +0200 Subject: [PATCH 04/44] Extract channel check for Rustup into function --- src/rustup.rs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/rustup.rs b/src/rustup.rs index 723ff7c..e88d187 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -32,12 +32,8 @@ impl Context { /// [release process]: https://rust-lang.github.io/rustup/dev-guide/release-process.html /// [rust-lang/rustup]: https://github.com/rust-lang/rustup pub fn promote_rustup(&mut self) -> anyhow::Result<()> { - println!("Checking channel..."); - if self.config.channel != Channel::Stable && self.config.channel != Channel::Beta { - return Err(anyhow!( - "promoting rustup is only supported for the stable and beta channels" - )); - } + // Rustup only has beta and stable releases, so we fail fast when trying to promote nightly + self.enforce_rustup_channel()?; // Download the rustup artifacts from S3 println!("Downloading artifacts from dev-static..."); @@ -60,6 +56,18 @@ impl Context { Ok(()) } + fn enforce_rustup_channel(&self) -> anyhow::Result<()> { + println!("Checking channel..."); + + if self.config.channel != Channel::Stable && self.config.channel != Channel::Beta { + return Err(anyhow!( + "promoting rustup is only supported for the stable and beta channels" + )); + } + + Ok(()) + } + fn download_rustup_artifacts(&mut self) -> Result { let dl = self.dl_dir().join("dist"); // Remove the directory if it exists, otherwise just ignore. From b803170598e8f2066e54f240e16c8a4fccdd5cad Mon Sep 17 00:00:00 2001 From: Jan David Date: Thu, 27 Jun 2024 12:07:41 +0200 Subject: [PATCH 05/44] Get latest commit from Rustup's stable branch --- src/rustup.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/rustup.rs b/src/rustup.rs index e88d187..e318a74 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -1,7 +1,7 @@ use std::fs; use std::path::{Path, PathBuf}; -use anyhow::{anyhow, Error}; +use anyhow::{anyhow, Context as AnyhowContext, Error}; use crate::config::Channel; use crate::{run, Context}; @@ -35,6 +35,9 @@ impl Context { // Rustup only has beta and stable releases, so we fail fast when trying to promote nightly self.enforce_rustup_channel()?; + // The latest commit on the `stable` branch is used to determine the version number + let head_sha = self.get_head_sha_for_rustup()?; + // Download the rustup artifacts from S3 println!("Downloading artifacts from dev-static..."); let dist_dir = self.download_rustup_artifacts()?; @@ -68,6 +71,14 @@ impl Context { Ok(()) } + fn get_head_sha_for_rustup(&self) -> anyhow::Result { + self.config + .github() + .context("failed to get HEAD SHA from GitHub - credentials not configured")? + .token("rust-lang/rustup")? + .get_ref("heads/stable") + } + fn download_rustup_artifacts(&mut self) -> Result { let dl = self.dl_dir().join("dist"); // Remove the directory if it exists, otherwise just ignore. From da3164d6bcf1a271e8e7162b8bbe00a91415e28e Mon Sep 17 00:00:00 2001 From: Jan David Date: Thu, 27 Jun 2024 14:22:55 +0200 Subject: [PATCH 06/44] Fetch next Rustup version from GitHub --- src/rustup.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/rustup.rs b/src/rustup.rs index e318a74..8c5dd50 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -2,6 +2,7 @@ use std::fs; use std::path::{Path, PathBuf}; use anyhow::{anyhow, Context as AnyhowContext, Error}; +use serde::Deserialize; use crate::config::Channel; use crate::{run, Context}; @@ -37,6 +38,7 @@ impl Context { // The latest commit on the `stable` branch is used to determine the version number let head_sha = self.get_head_sha_for_rustup()?; + let version = self.get_next_rustup_version(&head_sha)?; // Download the rustup artifacts from S3 println!("Downloading artifacts from dev-static..."); @@ -79,6 +81,26 @@ impl Context { .get_ref("heads/stable") } + fn get_next_rustup_version(&self, sha: &str) -> anyhow::Result { + println!("Getting next Rustup version from Cargo.toml..."); + + #[derive(Debug, Deserialize)] + struct CargoToml { + version: String, + } + + let cargo_toml = self + .config + .github() + .context("failed to get new rustup version from GitHub - credentials not configured")? + .token("rust-lang/rustup")? + .read_file(Some(sha), "Cargo.toml")?; + + let toml: CargoToml = toml::from_str(&cargo_toml.content()?)?; + + Ok(toml.version) + } + fn download_rustup_artifacts(&mut self) -> Result { let dl = self.dl_dir().join("dist"); // Remove the directory if it exists, otherwise just ignore. From 49980a339104f6ee41e6881ba55ff2f555b33cbf Mon Sep 17 00:00:00 2001 From: Jan David Date: Thu, 27 Jun 2024 14:24:59 +0200 Subject: [PATCH 07/44] Download Rustup artifacts for a given commit --- src/rustup.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/rustup.rs b/src/rustup.rs index 8c5dd50..ba53aa0 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -40,9 +40,8 @@ impl Context { let head_sha = self.get_head_sha_for_rustup()?; let version = self.get_next_rustup_version(&head_sha)?; - // Download the rustup artifacts from S3 - println!("Downloading artifacts from dev-static..."); - let dist_dir = self.download_rustup_artifacts()?; + // Download the Rustup artifacts from S3 + let dist_dir = self.download_rustup_artifacts(&head_sha)?; // Archive the artifacts println!("Archiving artifacts..."); @@ -101,7 +100,9 @@ impl Context { Ok(toml.version) } - fn download_rustup_artifacts(&mut self) -> Result { + fn download_rustup_artifacts(&mut self, sha: &str) -> Result { + println!("Downloading artifacts from dev-static..."); + let dl = self.dl_dir().join("dist"); // Remove the directory if it exists, otherwise just ignore. let _ = fs::remove_dir_all(&dl); @@ -112,7 +113,7 @@ impl Context { .arg("cp") .arg("--recursive") .arg("--only-show-errors") - .arg(&self.s3_artifacts_url("dist/")) + .arg(&self.s3_artifacts_url(&format!("builds/{sha}"))) .arg(format!("{}/", dl.display())))?; Ok(dl) From c4cff77d94b64d3d619452379a1e2195bbb88138 Mon Sep 17 00:00:00 2001 From: Jan David Date: Thu, 27 Jun 2024 14:27:23 +0200 Subject: [PATCH 08/44] Pass Rustup version to archive and manifest --- src/rustup.rs | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/src/rustup.rs b/src/rustup.rs index ba53aa0..04aa4fc 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -44,18 +44,15 @@ impl Context { let dist_dir = self.download_rustup_artifacts(&head_sha)?; // Archive the artifacts - println!("Archiving artifacts..."); - self.archive_rustup_artifacts(&dist_dir)?; + self.archive_rustup_artifacts(&dist_dir, &version)?; if self.config.channel == Channel::Stable { // Promote the artifacts to the release bucket - println!("Promoting artifacts to dist/..."); self.promote_rustup_artifacts(&dist_dir)?; } // Update the release number - println!("Updating version and manifest..."); - self.update_rustup_release()?; + self.update_rustup_release(&version)?; Ok(()) } @@ -119,11 +116,8 @@ impl Context { Ok(dl) } - fn archive_rustup_artifacts(&mut self, dist_dir: &Path) -> Result<(), Error> { - let version = self - .current_version - .as_ref() - .ok_or_else(|| anyhow!("failed to get current version for rustup release"))?; + fn archive_rustup_artifacts(&mut self, dist_dir: &Path, version: &str) -> Result<(), Error> { + println!("Archiving artifacts for version {version}..."); let path = format!("archive/{}/", version); @@ -131,6 +125,8 @@ impl Context { } fn promote_rustup_artifacts(&mut self, dist_dir: &Path) -> Result<(), Error> { + println!("Promoting artifacts to dist/..."); + let release_bucket_url = format!( "s3://{}/{}/{}", self.config.upload_bucket, @@ -157,11 +153,8 @@ impl Context { .arg(&self.s3_artifacts_url(target_path))) } - fn update_rustup_release(&mut self) -> Result<(), Error> { - let version = self - .current_version - .as_ref() - .ok_or_else(|| anyhow!("failed to get current version for rustup release"))?; + fn update_rustup_release(&mut self, version: &str) -> Result<(), Error> { + println!("Updating version and manifest..."); let manifest_path = self.dl_dir().join("release-stable.toml"); let manifest = format!( From 7ccff04c8bfdba76c05befea50b528a4adfa8458 Mon Sep 17 00:00:00 2001 From: Jan David Date: Thu, 27 Jun 2024 14:32:02 +0200 Subject: [PATCH 09/44] Update documentation for Rustup release process --- src/rustup.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/rustup.rs b/src/rustup.rs index 04aa4fc..493b6f9 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -15,13 +15,13 @@ impl Context { /// /// `rustup` uses different branches to manage releases. Whenever a commit is pushed to the /// `stable` branch in [rust-lang/rustup], GitHub Actions workflows build release artifacts and - /// copy them into `s3://dev-static-rust-lang-org/rustup/dist/`. + /// copy them into `s3://rustup-builds/builds/${commit-sha}/`. /// - /// When a new release is done and this method is invoked, it downloads the artifacts from that + /// When a new release is cut and this method is invoked, it downloads the artifacts from that /// bucket (which must always be set as the `DOWNLOAD_BUCKET` variable). A copy of the artifacts - /// is archived in `s3://${UPLOAD_BUCKET}/rustup/archive/${version}/`, where `version` is passed - /// to this program as a command-line argument. `UPLOAD_BUCKET` can either be the `dev-static` - /// or the `static` bucket. + /// is archived in `s3://${UPLOAD_BUCKET}/rustup/archive/${version}/`, where `version` is + /// derived from the Cargo.toml file in the `stable` branch. `UPLOAD_BUCKET` can either be the + /// `dev-static` or the `static` bucket. /// /// If the release is for the `stable` channel, the artifacts are also copied to the `dist/` /// path in the `UPLOAD_BUCKET` bucket. The `dist/` path is used by the `rustup` installer to From ede844c60b774613c5660c8b07d83500d5e70fdb Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 24 Sep 2024 10:50:23 +0200 Subject: [PATCH 10/44] Refactor run script to execute different local releases --- local/{run.sh => release.sh} | 0 local/rustup.sh | 8 ++++++++ local/setup.sh | 6 +++--- run.sh | 26 +++++++++++++++++++++----- 4 files changed, 32 insertions(+), 8 deletions(-) rename local/{run.sh => release.sh} (100%) create mode 100755 local/rustup.sh diff --git a/local/run.sh b/local/release.sh similarity index 100% rename from local/run.sh rename to local/release.sh diff --git a/local/rustup.sh b/local/rustup.sh new file mode 100755 index 0000000..f889d54 --- /dev/null +++ b/local/rustup.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# This script is executed at the start of each local release for Rustup, and +# prepares the environment by copying the artifacts built by CI onto the MinIO +# instance. Then, it starts promote-release with the right flags. + +set -euo pipefail +IFS=$'\n\t' diff --git a/local/setup.sh b/local/setup.sh index 2e7e1e0..14b2f45 100755 --- a/local/setup.sh +++ b/local/setup.sh @@ -77,9 +77,9 @@ cat < [commit]" +if [[ "$#" -lt 1 ]]; then + echo "Usage: $0 " + exit 1 +fi +command="$1" + +if [[ "${command}" == "release" ]]; then + if [[ "$#" -lt 2 ]] || [[ "$#" -gt 3 ]]; then + echo "Usage: $0 release [commit]" + exit 1 + fi +fi + +if [[ "${command}" == "rustup" ]]; then + if [[ "$#" -ne 2 ]]; then + echo "Usage: $0 rustup [commit]" exit 1 + fi fi -channel="$1" -override_commit="${2-}" + +channel="$2" +override_commit="${3-}" container_id="$(docker-compose ps -q local)" if [[ "${container_id}" == "" ]]; then @@ -31,4 +47,4 @@ fi cargo build --release # Run the command inside the docker environment. -docker-compose exec -T local /src/local/run.sh "${channel}" "${override_commit}" +docker-compose exec -T local "/src/local/${command}.sh" "${channel}" "${override_commit}" From 1e516af93b00ea66d005d70c2725fe9ff9020650 Mon Sep 17 00:00:00 2001 From: Jan David Date: Wed, 25 Sep 2024 13:11:45 +0200 Subject: [PATCH 11/44] Support other platforms than Linux --- local/Dockerfile | 8 +++++--- local/release.sh | 8 ++++++++ run.sh | 6 ++++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/local/Dockerfile b/local/Dockerfile index 0087840..406b20c 100644 --- a/local/Dockerfile +++ b/local/Dockerfile @@ -16,13 +16,15 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ git \ gnupg \ jq \ + libssl-dev \ + openssl \ + pkg-config \ python3 \ socat # Install rustup while removing the pre-installed stable toolchain. -RUN curl https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init >/tmp/rustup-init && \ - chmod +x /tmp/rustup-init && \ - /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain stable && \ +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ + sh -s -- -y --no-modify-path --profile minimal --default-toolchain stable && \ /root/.cargo/bin/rustup toolchain remove stable ENV PATH=/root/.cargo/bin:$PATH diff --git a/local/release.sh b/local/release.sh index 45f7b98..7564352 100755 --- a/local/release.sh +++ b/local/release.sh @@ -101,6 +101,14 @@ for file in "${DOWNLOAD_STANDALONE[@]}"; do download "${file}" done +# Build the promote-release binary if it hasn't been pre-built +if [[ ! -f "/src/target/release/promote-release" ]]; then + echo "==> building promote-release" + cd /src + cargo build --release + cd .. +fi + echo "==> configuring the environment" # Point to the right GnuPG environment export GNUPGHOME=/persistent/gpg-home diff --git a/run.sh b/run.sh index 217f4ab..2a4e0f3 100755 --- a/run.sh +++ b/run.sh @@ -43,8 +43,10 @@ if [[ "${container_status}" != "running" ]]; then exit 1 fi -# Ensure the release build is done -cargo build --release +# Pre-built the binary if the host and Docker environments match +if [[ "$(uname)" == "Linux" ]]; then + cargo build --release +fi # Run the command inside the docker environment. docker-compose exec -T local "/src/local/${command}.sh" "${channel}" "${override_commit}" From b1dd2a2c21f86cde5a6877d1bb35e0dea9b8ee34 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 12:29:16 +0200 Subject: [PATCH 12/44] Download Rustup files from CDN --- local/rustup.sh | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/local/rustup.sh b/local/rustup.sh index f889d54..1c35379 100755 --- a/local/rustup.sh +++ b/local/rustup.sh @@ -6,3 +6,51 @@ set -euo pipefail IFS=$'\n\t' + +RUSTUP_REPO="https://github.com/rust-lang/rustup" +RUSTUP_DEFAULT_BRANCH="master" + +# S3 bucket from which to download the Rustup artifacts +S3_BUCKET="rustup-builds" + +# CDN from which to download the CI artifacts +DOWNLOAD_BASE="https://rustup-builds.rust-lang.org" + +# The artifacts for the following targets will be downloaded and copied during +# the release process. At least one target is required. +DOWNLOAD_TARGETS=( + "x86_64-unknown-linux-gnu" +) + +# The following files will be downloaded and put into the local MinIO instance. +DOWNLOAD_FILES=( + "rustup-init" + "rustup-init.sha256" + "rustup-setup" + "rustup-setup.sha256" +) + +channel="$1" +override_commit="$2" + +if [[ "${override_commit}" = "" ]]; then + echo "==> detecting the last Rustup commit on the default branch" + commit="$(git ls-remote "${RUSTUP_REPO}" | grep "refs/heads/${RUSTUP_DEFAULT_BRANCH}" | awk '{print($1)}')" +else + echo "=>> using overridden commit ${override_commit}" + commit="${override_commit}" +fi + +for target in "${DOWNLOAD_TARGETS[@]}"; do + if ! mc stat "local/artifacts/builds/${commit}/dist/${target}" >/dev/null 2>&1; then + echo "==> copying ${target} from S3" + + for file in "${DOWNLOAD_FILES[@]}"; do + if curl -Lo /tmp/component "${DOWNLOAD_BASE}/${commit}/dist/${target}/${file}" --fail; then + mc cp /tmp/component "local/artifacts/builds/${commit}/dist/${target}/${file}" >/dev/null + fi + done + else + echo "==> reusing cached ${target} target" + fi +done From b843e516c513e1fd7a0f701a7d549bc472d2684e Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 12:29:49 +0200 Subject: [PATCH 13/44] Build promote-release inside the container if necessary --- local/rustup.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/local/rustup.sh b/local/rustup.sh index 1c35379..e0f0658 100755 --- a/local/rustup.sh +++ b/local/rustup.sh @@ -54,3 +54,11 @@ for target in "${DOWNLOAD_TARGETS[@]}"; do echo "==> reusing cached ${target} target" fi done + +# Build the promote-release binary if it hasn't been pre-built +if [[ ! -f "/src/target/release/promote-release" ]]; then + echo "==> building promote-release" + cd /src + cargo build --release + cd .. +fi From f5c3c5e104364b17b098d54fa09fb6f50ef0917d Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 12:41:39 +0200 Subject: [PATCH 14/44] Get download path from config --- src/rustup.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/rustup.rs b/src/rustup.rs index 493b6f9..0a64d9b 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -105,12 +105,14 @@ impl Context { let _ = fs::remove_dir_all(&dl); fs::create_dir_all(&dl)?; + let download_path = format!("{}/{}", self.config.download_dir, sha); + run(self .aws_s3() .arg("cp") .arg("--recursive") .arg("--only-show-errors") - .arg(&self.s3_artifacts_url(&format!("builds/{sha}"))) + .arg(&self.s3_artifacts_url(&download_path)) .arg(format!("{}/", dl.display())))?; Ok(dl) From a947033cc61807d71d919d731252f7c8bd9b3237 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 12:51:54 +0200 Subject: [PATCH 15/44] Fix upload directory --- src/rustup.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rustup.rs b/src/rustup.rs index 0a64d9b..a0c3e21 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -132,7 +132,7 @@ impl Context { let release_bucket_url = format!( "s3://{}/{}/{}", self.config.upload_bucket, - self.config.download_dir, + self.config.upload_dir, dist_dir.display(), ); From 883b5ede798214ceed0755d550ae5b5cd76fb534 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 13:03:55 +0200 Subject: [PATCH 16/44] Get version from user-provided commit --- src/rustup.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/rustup.rs b/src/rustup.rs index a0c3e21..9a68d12 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -36,8 +36,10 @@ impl Context { // Rustup only has beta and stable releases, so we fail fast when trying to promote nightly self.enforce_rustup_channel()?; - // The latest commit on the `stable` branch is used to determine the version number - let head_sha = self.get_head_sha_for_rustup()?; + // Get the latest commit from the `stable` branch or use the user-provided override + let head_sha = self.get_commit_sha_for_rustup_release()?; + + // The commit on the `stable` branch is used to determine the version number let version = self.get_next_rustup_version(&head_sha)?; // Download the Rustup artifacts from S3 @@ -69,6 +71,13 @@ impl Context { Ok(()) } + fn get_commit_sha_for_rustup_release(&self) -> anyhow::Result { + match &self.config.override_commit { + Some(sha) => Ok(sha.clone()), + None => self.get_head_sha_for_rustup(), + } + } + fn get_head_sha_for_rustup(&self) -> anyhow::Result { self.config .github() From ad86e13e8361ab9750f49c4c9122ae374cac6288 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 15:05:30 +0200 Subject: [PATCH 17/44] Don't require GitHub credentials to fetch Rustup metadata --- src/rustup.rs | 58 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/src/rustup.rs b/src/rustup.rs index 9a68d12..4cbf7f1 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -1,10 +1,12 @@ use std::fs; use std::path::{Path, PathBuf}; -use anyhow::{anyhow, Context as AnyhowContext, Error}; +use anyhow::{anyhow, Error}; +use curl::easy::Easy; use serde::Deserialize; use crate::config::Channel; +use crate::curl_helper::BodyExt; use crate::{run, Context}; impl Context { @@ -79,31 +81,57 @@ impl Context { } fn get_head_sha_for_rustup(&self) -> anyhow::Result { - self.config - .github() - .context("failed to get HEAD SHA from GitHub - credentials not configured")? - .token("rust-lang/rustup")? - .get_ref("heads/stable") + #[derive(Deserialize)] + struct Commit { + sha: String, + } + + let url = format!( + "https://api.github.com/repos/rust-lang/rustup/commits/{}", + self.config.channel + ); + + let mut client = Easy::new(); + client.url(&url)?; + client.useragent("rust-lang/promote-release")?; + + let commit: Commit = client.without_body().send_with_response()?; + + Ok(commit.sha) } fn get_next_rustup_version(&self, sha: &str) -> anyhow::Result { println!("Getting next Rustup version from Cargo.toml..."); - #[derive(Debug, Deserialize)] + #[derive(Deserialize)] + struct Content { + content: String, + } + + #[derive(Deserialize)] struct CargoToml { + package: Package, + } + + #[derive(Deserialize)] + struct Package { version: String, } - let cargo_toml = self - .config - .github() - .context("failed to get new rustup version from GitHub - credentials not configured")? - .token("rust-lang/rustup")? - .read_file(Some(sha), "Cargo.toml")?; + let url = + format!("https://api.github.com/repos/rust-lang/rustup/contents/Cargo.toml?ref={sha}"); + + let mut client = Easy::new(); + client.url(&url)?; + client.useragent("rust-lang/promote-release")?; + + let content: Content = client.without_body().send_with_response()?; + let decoded_content = base64::decode(&content.content.replace('\n', ""))?; + let cargo_toml = String::from_utf8(decoded_content)?; - let toml: CargoToml = toml::from_str(&cargo_toml.content()?)?; + let toml: CargoToml = toml::from_str(&cargo_toml)?; - Ok(toml.version) + Ok(toml.package.version) } fn download_rustup_artifacts(&mut self, sha: &str) -> Result { From 694e3a48924cf983035ea4504c1bbe416dc31d57 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 15:19:47 +0200 Subject: [PATCH 18/44] Create rustup-builds bucket in local environment --- local/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local/setup.sh b/local/setup.sh index 14b2f45..97219b4 100755 --- a/local/setup.sh +++ b/local/setup.sh @@ -11,7 +11,7 @@ MINIO_URL="http://${MINIO_HOST}:${MINIO_PORT}" MINIO_ACCESS_KEY="access_key" MINIO_SECRET_KEY="secret_key" -MINIO_BUCKETS=( "static" "artifacts" ) +MINIO_BUCKETS=( "static" "artifacts" "rustup-builds" ) # Quit immediately when docker-compose receives a Ctrl+C trap exit EXIT From 8132d6928e4071f40b27573faed06ceb9c8afbb6 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 15:20:22 +0200 Subject: [PATCH 19/44] Fix upload destination for Rustup archives --- src/rustup.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/rustup.rs b/src/rustup.rs index 4cbf7f1..757ca20 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -189,7 +189,10 @@ impl Context { .arg("--recursive") .arg("--only-show-errors") .arg(format!("{}/", dist_dir.display())) - .arg(&self.s3_artifacts_url(target_path))) + .arg(format!( + "s3://{}/{}/{}", + self.config.upload_bucket, self.config.upload_dir, target_path + ))) } fn update_rustup_release(&mut self, version: &str) -> Result<(), Error> { From 89f7324e653aa5b4ff91311ce734b41acfc48942 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 15:24:50 +0200 Subject: [PATCH 20/44] Fix upload destination for stable Rustup release --- src/rustup.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/rustup.rs b/src/rustup.rs index 757ca20..7f3d4da 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -167,10 +167,8 @@ impl Context { println!("Promoting artifacts to dist/..."); let release_bucket_url = format!( - "s3://{}/{}/{}", - self.config.upload_bucket, - self.config.upload_dir, - dist_dir.display(), + "s3://{}/{}/dist/", + self.config.upload_bucket, self.config.upload_dir, ); run(self From f82d531655049e6d588b7561b69c708f3b7ef203 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 15:29:13 +0200 Subject: [PATCH 21/44] Fix upload destination for Rustup manifest --- src/rustup.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/rustup.rs b/src/rustup.rs index 7f3d4da..163a6d2 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -212,6 +212,9 @@ version = '{}' .arg("cp") .arg("--only-show-errors") .arg(manifest_path) - .arg(&self.s3_artifacts_url("release-stable.toml"))) + .arg(format!( + "s3://{}/{}/release-stable.toml", + self.config.upload_bucket, self.config.upload_dir + ))) } } From 9b7dba8ee92c94fc493ef588ecd2f3fc773e3443 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 15:35:14 +0200 Subject: [PATCH 22/44] Configure environment to run Rustup releases locally --- local/rustup.sh | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/local/rustup.sh b/local/rustup.sh index e0f0658..5d7ff8c 100755 --- a/local/rustup.sh +++ b/local/rustup.sh @@ -62,3 +62,37 @@ if [[ ! -f "/src/target/release/promote-release" ]]; then cargo build --release cd .. fi + +echo "==> configuring the environment" + +# Release Rustup +export PROMOTE_RELEASE_ACTION="promote-rustup" + +# Point to the right GnuPG environment +export GNUPGHOME=/persistent/gpg-home + +## Environment variables also used in prod releases +export AWS_ACCESS_KEY_ID="access_key" +export AWS_SECRET_ACCESS_KEY="secret_key" +export PROMOTE_RELEASE_CHANNEL="${channel}" +export PROMOTE_RELEASE_CLOUDFRONT_DOC_ID="" +export PROMOTE_RELEASE_CLOUDFRONT_STATIC_ID="" +export PROMOTE_RELEASE_DOWNLOAD_BUCKET="rustup-builds" +export PROMOTE_RELEASE_DOWNLOAD_DIR="builds" +export PROMOTE_RELEASE_GPG_KEY_FILE="" +export PROMOTE_RELEASE_GPG_PASSWORD_FILE="" +export PROMOTE_RELEASE_UPLOAD_ADDR="" +export PROMOTE_RELEASE_UPLOAD_BUCKET="static" +export PROMOTE_RELEASE_UPLOAD_STORAGE_CLASS="STANDARD" +export PROMOTE_RELEASE_UPLOAD_DIR="rustup" + +## Environment variables used only by local releases +export PROMOTE_RELEASE_S3_ENDPOINT_URL="http://minio:9000" + +# Conditional environment variables +if [[ "${override_commit}" != "" ]]; then + export PROMOTE_RELEASE_OVERRIDE_COMMIT="${override_commit}" +fi + +echo "==> starting promote-release" +/src/target/release/promote-release /persistent/release "${channel}" From 5aad90dedf5dc6b9034031b5fc67ee943c993170 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 15:41:48 +0200 Subject: [PATCH 23/44] Set up CI workflows for refactored actions --- .github/workflows/ci.yml | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71178f6..0724cc3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,4 @@ --- - name: CI on: push: @@ -29,7 +28,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - local: + release: name: Local release runs-on: ubuntu-latest @@ -49,7 +48,7 @@ jobs: run: docker-compose up -d - name: Run the local release process for channel ${{ matrix.channel }} - run: ./run.sh ${{ matrix.channel }} + run: ./run.sh release ${{ matrix.channel }} - name: Validate the generated signatures run: docker-compose exec -T local /src/local/check-signature.sh ${{ matrix.channel }} @@ -62,6 +61,33 @@ jobs: env: RUSTUP_DIST_SERVER: http://localhost:9000/static + rustup: + name: Local rustup + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + channel: [stable, beta] + + steps: + - name: Clone the source code + uses: actions/checkout@v3 + + - name: Ensure Rust Stable is up to date + run: rustup self update && rustup update stable + + - name: Start the local environment + run: docker-compose up -d + + - name: Run the local release process for channel ${{ matrix.channel }} + run: ./run.sh rustup ${{ matrix.channel }} + + - name: Update Rustup from the local environment + run: rustup self update + env: + RUSTUP_UPDATE_ROOT: http://localhost:9000/static/rustup + docker: name: Build Docker image runs-on: ubuntu-latest From a52f9753bd2f53a9d230631a30ca0e4f6d5ed840 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 15:43:34 +0200 Subject: [PATCH 24/44] Fix required jobs for deployments --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0724cc3..e9f18d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,7 +117,7 @@ jobs: permissions: id-token: write - needs: [test, local, docker] + needs: [test, release, rustup, docker] if: github.event_name == 'push' && github.repository == 'rust-lang/promote-release' && github.ref == 'refs/heads/master' steps: From ad0c357a235638e9b84b9e4c0df77e713bdbba60 Mon Sep 17 00:00:00 2001 From: Jan David Date: Thu, 3 Oct 2024 11:42:28 +0200 Subject: [PATCH 25/44] Fix unnecessary borrow warnings --- src/rustup.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rustup.rs b/src/rustup.rs index 163a6d2..a2eaaf4 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -126,7 +126,7 @@ impl Context { client.useragent("rust-lang/promote-release")?; let content: Content = client.without_body().send_with_response()?; - let decoded_content = base64::decode(&content.content.replace('\n', ""))?; + let decoded_content = base64::decode(content.content.replace('\n', ""))?; let cargo_toml = String::from_utf8(decoded_content)?; let toml: CargoToml = toml::from_str(&cargo_toml)?; @@ -149,7 +149,7 @@ impl Context { .arg("cp") .arg("--recursive") .arg("--only-show-errors") - .arg(&self.s3_artifacts_url(&download_path)) + .arg(self.s3_artifacts_url(&download_path)) .arg(format!("{}/", dl.display())))?; Ok(dl) From fd3f06d87f511e2ce3bae8ce956e0d49b27acf85 Mon Sep 17 00:00:00 2001 From: Jan David Date: Thu, 3 Oct 2024 11:44:35 +0200 Subject: [PATCH 26/44] Cut beta releases for Rustup from the stable branch --- src/rustup.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/rustup.rs b/src/rustup.rs index a2eaaf4..6c045c9 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -86,13 +86,8 @@ impl Context { sha: String, } - let url = format!( - "https://api.github.com/repos/rust-lang/rustup/commits/{}", - self.config.channel - ); - let mut client = Easy::new(); - client.url(&url)?; + client.url("https://api.github.com/repos/rust-lang/rustup/commits/stable")?; client.useragent("rust-lang/promote-release")?; let commit: Commit = client.without_body().send_with_response()?; From a3a0d897f5280185a3e5dca49433d07e17ad12f9 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 8 Oct 2024 15:48:59 +0200 Subject: [PATCH 27/44] Allow Rustup version to be overridden in tests --- .github/workflows/ci.yml | 3 +++ src/rustup.rs | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05c8dbd..f8966ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,9 @@ jobs: name: Local rustup runs-on: ubuntu-latest + env: + PROMOTE_RELEASE_RUSTUP_OVERRIDE_VERSION: 99.0.0 + strategy: fail-fast: false matrix: diff --git a/src/rustup.rs b/src/rustup.rs index 6c045c9..5c065fb 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -96,6 +96,16 @@ impl Context { } fn get_next_rustup_version(&self, sha: &str) -> anyhow::Result { + // Allow the version to be overridden manually, for example to test the release process + if let Ok(version) = std::env::var("PROMOTE_RELEASE_RUSTUP_OVERRIDE_VERSION") { + println!("Using override version: {}", version); + Ok(version) + } else { + self.get_next_rustup_version_from_github(sha) + } + } + + fn get_next_rustup_version_from_github(&self, sha: &str) -> anyhow::Result { println!("Getting next Rustup version from Cargo.toml..."); #[derive(Deserialize)] From 57d5a20cb3ee617d27de752936a5261e2ba71568 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 7 Jan 2025 14:39:14 +0100 Subject: [PATCH 28/44] Explicitly forward version in local tests --- run.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/run.sh b/run.sh index 15cade6..08413cb 100755 --- a/run.sh +++ b/run.sh @@ -19,7 +19,7 @@ if [[ "${command}" == "release" ]]; then fi if [[ "${command}" == "rustup" ]]; then - if [[ "$#" -ne 2 ]]; then + if [[ "$#" -lt 2 ]] || [[ "$#" -gt 3 ]]; then echo "Usage: $0 rustup [commit]" exit 1 fi @@ -48,5 +48,11 @@ if [[ "$(uname)" == "Linux" ]]; then cargo build --release fi +# If the PROMOTE_RELEASE_RUSTUP_OVERRIDE_VERSION environment variable is set, +# forward it to the Docker environment. +if [[ "$PROMOTE_RELEASE_RUSTUP_OVERRIDE_VERSION" != "" ]]; then + docker compose exec -e "PROMOTE_RELEASE_RUSTUP_OVERRIDE_VERSION=${PROMOTE_RELEASE_RUSTUP_OVERRIDE_VERSION}" -T local "/src/local/${command}.sh" "${channel}" "${override_commit}" +fi + # Run the command inside the docker environment. docker compose exec -T local "/src/local/${command}.sh" "${channel}" "${override_commit}" From b20a0c920b8b215e16605ccd6577acf455d603ef Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 7 Jan 2025 16:40:21 +0100 Subject: [PATCH 29/44] Fix unbound variable warning --- run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/run.sh b/run.sh index 08413cb..66e4529 100755 --- a/run.sh +++ b/run.sh @@ -5,6 +5,8 @@ set -euo pipefail IFS=$'\n\t' +PROMOTE_RELEASE_RUSTUP_OVERRIDE_VERSION="" + if [[ "$#" -lt 1 ]]; then echo "Usage: $0 " exit 1 From 776973f68c74166494c4ee858c5112ad183d32d5 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 7 Jan 2025 16:40:36 +0100 Subject: [PATCH 30/44] Upgrade local container to Ubuntu 24.04 --- local/Dockerfile | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/local/Dockerfile b/local/Dockerfile index 406b20c..6af1897 100644 --- a/local/Dockerfile +++ b/local/Dockerfile @@ -7,10 +7,9 @@ # then copy the binary from it later in the build. FROM quay.io/minio/mc:RELEASE.2023-04-12T02-21-51Z AS mc -FROM ubuntu:22.04 +FROM ubuntu:24.04 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - awscli \ build-essential \ curl \ git \ @@ -20,7 +19,14 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ openssl \ pkg-config \ python3 \ - socat + socat \ + unzip + +# Install the AWS CLI +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-$(arch).zip" -o "awscliv2.zip"; \ + unzip awscliv2.zip; \ + rm awscliv2.zip; \ + ./aws/install # Install rustup while removing the pre-installed stable toolchain. RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ From c22a17eed45a58e28394245525ce8c00b1169ea8 Mon Sep 17 00:00:00 2001 From: Jan David Date: Wed, 8 Jan 2025 12:22:02 +0100 Subject: [PATCH 31/44] Support workspace version for Rustup --- src/rustup.rs | 68 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 20 deletions(-) diff --git a/src/rustup.rs b/src/rustup.rs index 5c065fb..435748b 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -9,6 +9,26 @@ use crate::config::Channel; use crate::curl_helper::BodyExt; use crate::{run, Context}; +#[derive(Deserialize)] +struct Content { + content: String, +} + +#[derive(Deserialize)] +struct CargoToml { + workspace: Workspace, +} + +#[derive(Deserialize)] +struct Workspace { + package: Package, +} + +#[derive(Deserialize)] +struct Package { + version: String, +} + impl Context { /// Promote a `rustup` release /// @@ -108,21 +128,6 @@ impl Context { fn get_next_rustup_version_from_github(&self, sha: &str) -> anyhow::Result { println!("Getting next Rustup version from Cargo.toml..."); - #[derive(Deserialize)] - struct Content { - content: String, - } - - #[derive(Deserialize)] - struct CargoToml { - package: Package, - } - - #[derive(Deserialize)] - struct Package { - version: String, - } - let url = format!("https://api.github.com/repos/rust-lang/rustup/contents/Cargo.toml?ref={sha}"); @@ -131,12 +136,9 @@ impl Context { client.useragent("rust-lang/promote-release")?; let content: Content = client.without_body().send_with_response()?; - let decoded_content = base64::decode(content.content.replace('\n', ""))?; - let cargo_toml = String::from_utf8(decoded_content)?; + let toml = decode_and_deserialize_cargo_toml(&content.content)?; - let toml: CargoToml = toml::from_str(&cargo_toml)?; - - Ok(toml.package.version) + Ok(toml.workspace.package.version) } fn download_rustup_artifacts(&mut self, sha: &str) -> Result { @@ -223,3 +225,29 @@ version = '{}' ))) } } + +fn decode_and_deserialize_cargo_toml(base64_encoded_toml: &str) -> Result { + let decoded_content = base64::decode(base64_encoded_toml.replace('\n', ""))?; + let content_as_string = String::from_utf8(decoded_content)?; + + toml::from_str(&content_as_string).map_err(Into::into) +} + +#[cfg(test)] +mod tests { + use crate::rustup::decode_and_deserialize_cargo_toml; + + #[test] + fn decode_cargo_toml() { + let base64_encoded_toml = base64::encode( + r#" + [workspace.package] + version = "1.2.3" + "#, + ); + + let toml = decode_and_deserialize_cargo_toml(&base64_encoded_toml).unwrap(); + + assert_eq!(toml.workspace.package.version, "1.2.3"); + } +} From 9f4f3ab5c0f9ba530baeb0140595bb7d83a4b03a Mon Sep 17 00:00:00 2001 From: Jan David Date: Wed, 5 Feb 2025 10:56:19 +0100 Subject: [PATCH 32/44] Fix overriding the Rustup version --- .github/workflows/ci.yml | 13 ++++++++----- local/rustup.sh | 9 +++++++-- run.sh | 16 ++++++++-------- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8966ad..d508ab7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: push: branches: - master - pull_request: {} + pull_request: { } jobs: test: @@ -35,7 +35,7 @@ jobs: strategy: fail-fast: false matrix: - channel: [stable, beta, nightly] + channel: [ stable, beta, nightly ] steps: - name: Clone the source code @@ -66,12 +66,12 @@ jobs: runs-on: ubuntu-latest env: - PROMOTE_RELEASE_RUSTUP_OVERRIDE_VERSION: 99.0.0 + RUSTUP_OVERRIDE_VERSION: 99.0.0 strategy: fail-fast: false matrix: - channel: [stable, beta] + channel: [ stable, beta ] steps: - name: Clone the source code @@ -91,6 +91,9 @@ jobs: env: RUSTUP_UPDATE_ROOT: http://localhost:9000/static/rustup + - name: Verify Rustup version + run: rustup --version | grep -q ${{ env.RUSTUP_OVERRIDE_VERSION }} + docker: name: Build Docker image runs-on: ubuntu-latest @@ -120,7 +123,7 @@ jobs: permissions: id-token: write - needs: [test, release, rustup, docker] + needs: [ test, release, rustup, docker ] if: github.event_name == 'push' && github.repository == 'rust-lang/promote-release' && github.ref == 'refs/heads/master' steps: diff --git a/local/rustup.sh b/local/rustup.sh index 5d7ff8c..348d445 100755 --- a/local/rustup.sh +++ b/local/rustup.sh @@ -42,8 +42,8 @@ else fi for target in "${DOWNLOAD_TARGETS[@]}"; do - if ! mc stat "local/artifacts/builds/${commit}/dist/${target}" >/dev/null 2>&1; then - echo "==> copying ${target} from S3" + if ! mc stat "local/artifacts/builds/${commit}/dist/${target}" >/dev/null 2>&1; then + echo "==> copying ${target} from S3" for file in "${DOWNLOAD_FILES[@]}"; do if curl -Lo /tmp/component "${DOWNLOAD_BASE}/${commit}/dist/${target}/${file}" --fail; then @@ -94,5 +94,10 @@ if [[ "${override_commit}" != "" ]]; then export PROMOTE_RELEASE_OVERRIDE_COMMIT="${override_commit}" fi +# Conditionally set a version for the next Rustup release +if [[ "${RUSTUP_OVERRIDE_VERSION:-}" != "" ]]; then + export PROMOTE_RELEASE_RUSTUP_OVERRIDE_VERSION="${RUSTUP_OVERRIDE_VERSION}" +fi + echo "==> starting promote-release" /src/target/release/promote-release /persistent/release "${channel}" diff --git a/run.sh b/run.sh index 66e4529..ea0d963 100755 --- a/run.sh +++ b/run.sh @@ -5,7 +5,7 @@ set -euo pipefail IFS=$'\n\t' -PROMOTE_RELEASE_RUSTUP_OVERRIDE_VERSION="" +RUSTUP_OVERRIDE_VERSION="${RUSTUP_OVERRIDE_VERSION:-}" if [[ "$#" -lt 1 ]]; then echo "Usage: $0 " @@ -50,11 +50,11 @@ if [[ "$(uname)" == "Linux" ]]; then cargo build --release fi -# If the PROMOTE_RELEASE_RUSTUP_OVERRIDE_VERSION environment variable is set, -# forward it to the Docker environment. -if [[ "$PROMOTE_RELEASE_RUSTUP_OVERRIDE_VERSION" != "" ]]; then - docker compose exec -e "PROMOTE_RELEASE_RUSTUP_OVERRIDE_VERSION=${PROMOTE_RELEASE_RUSTUP_OVERRIDE_VERSION}" -T local "/src/local/${command}.sh" "${channel}" "${override_commit}" +if [[ "$RUSTUP_OVERRIDE_VERSION" != "" ]]; then + # If the RUSTUP_OVERRIDE_VERSION environment variable is set, forward it to the Docker environment. + echo "==> running local release with override version ${RUSTUP_OVERRIDE_VERSION}" + docker compose exec -e "RUSTUP_OVERRIDE_VERSION=${RUSTUP_OVERRIDE_VERSION}" -T local "/src/local/${command}.sh" "${channel}" "${override_commit}" +else + # Run the command inside the docker environment. + docker compose exec -T local "/src/local/${command}.sh" "${channel}" "${override_commit}" fi - -# Run the command inside the docker environment. -docker compose exec -T local "/src/local/${command}.sh" "${channel}" "${override_commit}" From e01d9e9926f08c050359c1eeee32337c95aeebce Mon Sep 17 00:00:00 2001 From: Jan David Date: Thu, 6 Feb 2025 12:33:07 +0100 Subject: [PATCH 33/44] Fix paths when downloading Rustup artifacts --- local/rustup.sh | 11 +++++++---- src/rustup.rs | 11 +++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/local/rustup.sh b/local/rustup.sh index 348d445..76707a6 100755 --- a/local/rustup.sh +++ b/local/rustup.sh @@ -8,7 +8,7 @@ set -euo pipefail IFS=$'\n\t' RUSTUP_REPO="https://github.com/rust-lang/rustup" -RUSTUP_DEFAULT_BRANCH="master" +RUSTUP_DEFAULT_BRANCH="stable" # S3 bucket from which to download the Rustup artifacts S3_BUCKET="rustup-builds" @@ -19,6 +19,7 @@ DOWNLOAD_BASE="https://rustup-builds.rust-lang.org" # The artifacts for the following targets will be downloaded and copied during # the release process. At least one target is required. DOWNLOAD_TARGETS=( + "aarch64-unknown-linux-gnu" "x86_64-unknown-linux-gnu" ) @@ -42,12 +43,14 @@ else fi for target in "${DOWNLOAD_TARGETS[@]}"; do - if ! mc stat "local/artifacts/builds/${commit}/dist/${target}" >/dev/null 2>&1; then + if ! mc stat "local/rustup-builds/${commit}/dist/${target}" >/dev/null 2>&1; then echo "==> copying ${target} from S3" for file in "${DOWNLOAD_FILES[@]}"; do + echo "==> copying ${file} from S3" + if curl -Lo /tmp/component "${DOWNLOAD_BASE}/${commit}/dist/${target}/${file}" --fail; then - mc cp /tmp/component "local/artifacts/builds/${commit}/dist/${target}/${file}" >/dev/null + mc cp /tmp/component "local/rustup-builds/${commit}/dist/${target}/${file}" >/dev/null fi done else @@ -78,7 +81,7 @@ export PROMOTE_RELEASE_CHANNEL="${channel}" export PROMOTE_RELEASE_CLOUDFRONT_DOC_ID="" export PROMOTE_RELEASE_CLOUDFRONT_STATIC_ID="" export PROMOTE_RELEASE_DOWNLOAD_BUCKET="rustup-builds" -export PROMOTE_RELEASE_DOWNLOAD_DIR="builds" +export PROMOTE_RELEASE_DOWNLOAD_DIR="" export PROMOTE_RELEASE_GPG_KEY_FILE="" export PROMOTE_RELEASE_GPG_PASSWORD_FILE="" export PROMOTE_RELEASE_UPLOAD_ADDR="" diff --git a/src/rustup.rs b/src/rustup.rs index 435748b..54c752b 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -142,21 +142,24 @@ impl Context { } fn download_rustup_artifacts(&mut self, sha: &str) -> Result { - println!("Downloading artifacts from dev-static..."); + println!( + "Downloading artifacts from {}...", + self.config.download_bucket + ); - let dl = self.dl_dir().join("dist"); + let dl = self.dl_dir().join("rustup"); // Remove the directory if it exists, otherwise just ignore. let _ = fs::remove_dir_all(&dl); fs::create_dir_all(&dl)?; - let download_path = format!("{}/{}", self.config.download_dir, sha); + let artifacts_url = format!("s3://{}/{}", self.config.download_bucket, sha); run(self .aws_s3() .arg("cp") .arg("--recursive") .arg("--only-show-errors") - .arg(self.s3_artifacts_url(&download_path)) + .arg(artifacts_url) .arg(format!("{}/", dl.display())))?; Ok(dl) From 2befc104e5965811a805f2671dc1f182b49e377c Mon Sep 17 00:00:00 2001 From: Jan David Date: Thu, 6 Mar 2025 12:34:28 +0100 Subject: [PATCH 34/44] Do not remove the default toolchain Rustup no longer installs missing toolchains automatically. --- local/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/local/Dockerfile b/local/Dockerfile index 6af1897..453369e 100644 --- a/local/Dockerfile +++ b/local/Dockerfile @@ -30,8 +30,7 @@ RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-$(arch).zip" -o "awscliv # Install rustup while removing the pre-installed stable toolchain. RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ - sh -s -- -y --no-modify-path --profile minimal --default-toolchain stable && \ - /root/.cargo/bin/rustup toolchain remove stable + sh -s -- -y --no-modify-path --profile minimal --default-toolchain stable ENV PATH=/root/.cargo/bin:$PATH COPY --from=mc /usr/bin/mc /usr/local/bin/mc From fbb3508b9af86e18fb07abaa21235518d1c9c808 Mon Sep 17 00:00:00 2001 From: Jan David Date: Thu, 6 Mar 2025 12:45:03 +0100 Subject: [PATCH 35/44] Bump MinIO version The previous, outdated version of MinIO did not work correctly with the latest AWS CLI, resulting in an `invalid argument (invalid/missing checksum)` error. This issue has been fixed in MinIO a while ago (https://github.com/minio/minio/pull/19680). --- docker-compose.yml | 2 +- local/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5b99bbc..cfebd46 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ --- services: minio: - image: quay.io/minio/minio:RELEASE.2023-04-13T03-08-07Z + image: quay.io/minio/minio:RELEASE.2025-02-28T09-55-16Z command: minio server /data ports: - 9000:9000 diff --git a/local/Dockerfile b/local/Dockerfile index 453369e..55a1f56 100644 --- a/local/Dockerfile +++ b/local/Dockerfile @@ -5,7 +5,7 @@ # regularly takes 2 minutes to download 20MB of binary). The only other way # they distribute the CLI is from Docker, so we load their image as a stage and # then copy the binary from it later in the build. -FROM quay.io/minio/mc:RELEASE.2023-04-12T02-21-51Z AS mc +FROM quay.io/minio/mc:RELEASE.2025-02-21T16-00-46Z AS mc FROM ubuntu:24.04 From bbff20e3429f671d91c94c917a0d30dec2b88c4b Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 25 Mar 2025 14:18:53 +0100 Subject: [PATCH 36/44] Always promote a new version New binaries of `rustup` are now uploaded to the `rustup-builds` bucket on S3 and no longer to `dev-static`, which means that they cannot be tested without performing a release to the `beta` channel first. This must include promoting the release and copying the binaries into the `dev-static` bucket. --- src/rustup.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/rustup.rs b/src/rustup.rs index 54c752b..523855d 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -45,9 +45,8 @@ impl Context { /// derived from the Cargo.toml file in the `stable` branch. `UPLOAD_BUCKET` can either be the /// `dev-static` or the `static` bucket. /// - /// If the release is for the `stable` channel, the artifacts are also copied to the `dist/` - /// path in the `UPLOAD_BUCKET` bucket. The `dist/` path is used by the `rustup` installer to - /// download the latest release. + /// The artifacts are also copied to the `dist/` path in the `UPLOAD_BUCKET` bucket, which is + /// used by the `rustup` installer to download the latest release. /// /// Then, the `release-stable.toml` manifest is updated with the new version and copied to /// `s3://${UPLOAD_BUCKET}/rustup/release-stable.toml`. @@ -70,10 +69,8 @@ impl Context { // Archive the artifacts self.archive_rustup_artifacts(&dist_dir, &version)?; - if self.config.channel == Channel::Stable { - // Promote the artifacts to the release bucket - self.promote_rustup_artifacts(&dist_dir)?; - } + // Promote the artifacts to the release bucket + self.promote_rustup_artifacts(&dist_dir)?; // Update the release number self.update_rustup_release(&version)?; From f420838371aed5715ee253b0ecdc2e7c97065755 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 25 Mar 2025 15:10:05 +0100 Subject: [PATCH 37/44] Fix path for rustup artifact upload --- src/rustup.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rustup.rs b/src/rustup.rs index 523855d..011868b 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -183,7 +183,7 @@ impl Context { .arg("cp") .arg("--recursive") .arg("--only-show-errors") - .arg(format!("{}/", dist_dir.display())) + .arg(format!("{}/dist/", dist_dir.display())) .arg(&release_bucket_url)) } From ace9066b6d45e19825aea54ba184296d414c6740 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 25 Mar 2025 15:10:55 +0100 Subject: [PATCH 38/44] Download the rustup installer as well --- local/rustup.sh | 10 ++++++++++ src/rustup.rs | 25 +++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/local/rustup.sh b/local/rustup.sh index 76707a6..85bcf03 100755 --- a/local/rustup.sh +++ b/local/rustup.sh @@ -58,6 +58,16 @@ for target in "${DOWNLOAD_TARGETS[@]}"; do fi done +if ! mc stat "local/rustup-builds/${commit}/rustup-init.sh" >/dev/null 2>&1; then + echo "==> copying rustup-init.sh from S3" + + if curl -Lo /tmp/component "${DOWNLOAD_BASE}/${commit}/rustup-init.sh" --fail; then + mc cp /tmp/component "local/rustup-builds/${commit}/rustup-init.sh" >/dev/null + fi +else + echo "==> reusing cached rustup-init.sh" +fi + # Build the promote-release binary if it hasn't been pre-built if [[ ! -f "/src/target/release/promote-release" ]]; then echo "==> building promote-release" diff --git a/src/rustup.rs b/src/rustup.rs index 011868b..55e5216 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -72,6 +72,9 @@ impl Context { // Promote the artifacts to the release bucket self.promote_rustup_artifacts(&dist_dir)?; + // Update the `rustup` installer + self.update_rustup_installer(&dist_dir)?; + // Update the release number self.update_rustup_release(&version)?; @@ -187,6 +190,28 @@ impl Context { .arg(&release_bucket_url)) } + fn update_rustup_installer(&mut self, dist_dir: &Path) -> Result<(), Error> { + println!("Updating the rustup installer..."); + + let release_bucket_url = format!( + "s3://{}/{}", + self.config.upload_bucket, self.config.upload_dir, + ); + + let destinations = ["rustup-init.sh", "rustup.sh"]; + + for destination in destinations { + run(self + .aws_s3() + .arg("cp") + .arg("--only-show-errors") + .arg(format!("{}/rustup-init.sh", dist_dir.display())) + .arg(format!("{}/{}", release_bucket_url, destination)))?; + } + + Ok(()) + } + fn upload_rustup_artifacts(&mut self, dist_dir: &Path, target_path: &str) -> Result<(), Error> { run(self .aws_s3() From af7e298b6b463848f29d86ea4da6a8eb627fab26 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 25 Mar 2025 15:29:48 +0100 Subject: [PATCH 39/44] Fix archiving of rustup artifacts --- src/rustup.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rustup.rs b/src/rustup.rs index 55e5216..b145dcd 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -170,7 +170,7 @@ impl Context { let path = format!("archive/{}/", version); - self.upload_rustup_artifacts(dist_dir, &path) + self.upload_rustup_artifacts(&dist_dir.join("dist"), &path) } fn promote_rustup_artifacts(&mut self, dist_dir: &Path) -> Result<(), Error> { From c9b96feda1929f03f5447d28e9c4aa7c3576943c Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 25 Mar 2025 15:55:16 +0100 Subject: [PATCH 40/44] Remove override for rustup version The override feature for the rustup version has been removed, since the version seems to be hardcoded in the binary as well. So it was possible to change the path where the artifacts were stored in S3, but using the version to verify that rustup got updated wasn't possible. --- .github/workflows/ci.yml | 18 ++++++++++++++---- local/rustup.sh | 5 ----- run.sh | 12 ++---------- src/rustup.rs | 12 +----------- 4 files changed, 17 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d508ab7..69ff845 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,9 +65,6 @@ jobs: name: Local rustup runs-on: ubuntu-latest - env: - RUSTUP_OVERRIDE_VERSION: 99.0.0 - strategy: fail-fast: false matrix: @@ -80,6 +77,9 @@ jobs: - name: Ensure Rust Stable is up to date run: rustup self update && rustup update stable + - name: Get the current rustup version + run: echo "PREVIOUS_RUSTUP_VERSION=$(rustup --version | cut -d ' ' -f 2)" >> $GITHUB_ENV + - name: Start the local environment run: docker compose up -d @@ -91,8 +91,18 @@ jobs: env: RUSTUP_UPDATE_ROOT: http://localhost:9000/static/rustup + - name: Get the new rustup version + run: echo "NEW_RUSTUP_VERSION=$(rustup --version | cut -d ' ' -f 2)" >> $GITHUB_ENV + - name: Verify Rustup version - run: rustup --version | grep -q ${{ env.RUSTUP_OVERRIDE_VERSION }} + run: | + if [[ "${PREVIOUS_RUSTUP_VERSION}" == "${NEW_RUSTUP_VERSION}" ]]; then + echo "Rustup version did not change" + exit 1 + else + echo "Previous Rustup version: ${PREVIOUS_RUSTUP_VERSION}" + echo "New Rustup version: ${NEW_RUSTUP_VERSION}" + fi docker: name: Build Docker image diff --git a/local/rustup.sh b/local/rustup.sh index 85bcf03..6dd8674 100755 --- a/local/rustup.sh +++ b/local/rustup.sh @@ -107,10 +107,5 @@ if [[ "${override_commit}" != "" ]]; then export PROMOTE_RELEASE_OVERRIDE_COMMIT="${override_commit}" fi -# Conditionally set a version for the next Rustup release -if [[ "${RUSTUP_OVERRIDE_VERSION:-}" != "" ]]; then - export PROMOTE_RELEASE_RUSTUP_OVERRIDE_VERSION="${RUSTUP_OVERRIDE_VERSION}" -fi - echo "==> starting promote-release" /src/target/release/promote-release /persistent/release "${channel}" diff --git a/run.sh b/run.sh index ea0d963..5c807b8 100755 --- a/run.sh +++ b/run.sh @@ -5,8 +5,6 @@ set -euo pipefail IFS=$'\n\t' -RUSTUP_OVERRIDE_VERSION="${RUSTUP_OVERRIDE_VERSION:-}" - if [[ "$#" -lt 1 ]]; then echo "Usage: $0 " exit 1 @@ -50,11 +48,5 @@ if [[ "$(uname)" == "Linux" ]]; then cargo build --release fi -if [[ "$RUSTUP_OVERRIDE_VERSION" != "" ]]; then - # If the RUSTUP_OVERRIDE_VERSION environment variable is set, forward it to the Docker environment. - echo "==> running local release with override version ${RUSTUP_OVERRIDE_VERSION}" - docker compose exec -e "RUSTUP_OVERRIDE_VERSION=${RUSTUP_OVERRIDE_VERSION}" -T local "/src/local/${command}.sh" "${channel}" "${override_commit}" -else - # Run the command inside the docker environment. - docker compose exec -T local "/src/local/${command}.sh" "${channel}" "${override_commit}" -fi +# Run the command inside the docker environment. +docker compose exec -T local "/src/local/${command}.sh" "${channel}" "${override_commit}" diff --git a/src/rustup.rs b/src/rustup.rs index b145dcd..00e71ea 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -61,7 +61,7 @@ impl Context { let head_sha = self.get_commit_sha_for_rustup_release()?; // The commit on the `stable` branch is used to determine the version number - let version = self.get_next_rustup_version(&head_sha)?; + let version = self.get_next_rustup_version_from_github(&head_sha)?; // Download the Rustup artifacts from S3 let dist_dir = self.download_rustup_artifacts(&head_sha)?; @@ -115,16 +115,6 @@ impl Context { Ok(commit.sha) } - fn get_next_rustup_version(&self, sha: &str) -> anyhow::Result { - // Allow the version to be overridden manually, for example to test the release process - if let Ok(version) = std::env::var("PROMOTE_RELEASE_RUSTUP_OVERRIDE_VERSION") { - println!("Using override version: {}", version); - Ok(version) - } else { - self.get_next_rustup_version_from_github(sha) - } - } - fn get_next_rustup_version_from_github(&self, sha: &str) -> anyhow::Result { println!("Getting next Rustup version from Cargo.toml..."); From a5213b0144688606de31ac31cfb690054e46116b Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 25 Mar 2025 16:24:21 +0100 Subject: [PATCH 41/44] Restore override version and check for an update Comparing the Rustup versions before and after the update does not work, since the version in the GitHub repository is not bumped after a release. We're going back to the override version so that we can force rustup to self update, which we can check for without comparing version numbers. This reverts commit c9b96feda1929f03f5447d28e9c4aa7c3576943c. --- .github/workflows/ci.yml | 25 +++++++++---------------- local/rustup.sh | 5 +++++ run.sh | 12 ++++++++++-- src/rustup.rs | 12 +++++++++++- 4 files changed, 35 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69ff845..10203c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,9 @@ jobs: name: Local rustup runs-on: ubuntu-latest + env: + RUSTUP_OVERRIDE_VERSION: 99.0.0 + strategy: fail-fast: false matrix: @@ -77,9 +80,6 @@ jobs: - name: Ensure Rust Stable is up to date run: rustup self update && rustup update stable - - name: Get the current rustup version - run: echo "PREVIOUS_RUSTUP_VERSION=$(rustup --version | cut -d ' ' -f 2)" >> $GITHUB_ENV - - name: Start the local environment run: docker compose up -d @@ -87,22 +87,15 @@ jobs: run: ./run.sh rustup ${{ matrix.channel }} - name: Update Rustup from the local environment - run: rustup self update - env: - RUSTUP_UPDATE_ROOT: http://localhost:9000/static/rustup - - - name: Get the new rustup version - run: echo "NEW_RUSTUP_VERSION=$(rustup --version | cut -d ' ' -f 2)" >> $GITHUB_ENV - - - name: Verify Rustup version run: | - if [[ "${PREVIOUS_RUSTUP_VERSION}" == "${NEW_RUSTUP_VERSION}" ]]; then - echo "Rustup version did not change" - exit 1 + if rustup self update | grep -q "rustup updated"; then + echo "Rustup was updated successfully." else - echo "Previous Rustup version: ${PREVIOUS_RUSTUP_VERSION}" - echo "New Rustup version: ${NEW_RUSTUP_VERSION}" + echo "Rustup was not updated." + exit 1 fi + env: + RUSTUP_UPDATE_ROOT: http://localhost:9000/static/rustup docker: name: Build Docker image diff --git a/local/rustup.sh b/local/rustup.sh index 6dd8674..85bcf03 100755 --- a/local/rustup.sh +++ b/local/rustup.sh @@ -107,5 +107,10 @@ if [[ "${override_commit}" != "" ]]; then export PROMOTE_RELEASE_OVERRIDE_COMMIT="${override_commit}" fi +# Conditionally set a version for the next Rustup release +if [[ "${RUSTUP_OVERRIDE_VERSION:-}" != "" ]]; then + export PROMOTE_RELEASE_RUSTUP_OVERRIDE_VERSION="${RUSTUP_OVERRIDE_VERSION}" +fi + echo "==> starting promote-release" /src/target/release/promote-release /persistent/release "${channel}" diff --git a/run.sh b/run.sh index 5c807b8..ea0d963 100755 --- a/run.sh +++ b/run.sh @@ -5,6 +5,8 @@ set -euo pipefail IFS=$'\n\t' +RUSTUP_OVERRIDE_VERSION="${RUSTUP_OVERRIDE_VERSION:-}" + if [[ "$#" -lt 1 ]]; then echo "Usage: $0 " exit 1 @@ -48,5 +50,11 @@ if [[ "$(uname)" == "Linux" ]]; then cargo build --release fi -# Run the command inside the docker environment. -docker compose exec -T local "/src/local/${command}.sh" "${channel}" "${override_commit}" +if [[ "$RUSTUP_OVERRIDE_VERSION" != "" ]]; then + # If the RUSTUP_OVERRIDE_VERSION environment variable is set, forward it to the Docker environment. + echo "==> running local release with override version ${RUSTUP_OVERRIDE_VERSION}" + docker compose exec -e "RUSTUP_OVERRIDE_VERSION=${RUSTUP_OVERRIDE_VERSION}" -T local "/src/local/${command}.sh" "${channel}" "${override_commit}" +else + # Run the command inside the docker environment. + docker compose exec -T local "/src/local/${command}.sh" "${channel}" "${override_commit}" +fi diff --git a/src/rustup.rs b/src/rustup.rs index 00e71ea..b145dcd 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -61,7 +61,7 @@ impl Context { let head_sha = self.get_commit_sha_for_rustup_release()?; // The commit on the `stable` branch is used to determine the version number - let version = self.get_next_rustup_version_from_github(&head_sha)?; + let version = self.get_next_rustup_version(&head_sha)?; // Download the Rustup artifacts from S3 let dist_dir = self.download_rustup_artifacts(&head_sha)?; @@ -115,6 +115,16 @@ impl Context { Ok(commit.sha) } + fn get_next_rustup_version(&self, sha: &str) -> anyhow::Result { + // Allow the version to be overridden manually, for example to test the release process + if let Ok(version) = std::env::var("PROMOTE_RELEASE_RUSTUP_OVERRIDE_VERSION") { + println!("Using override version: {}", version); + Ok(version) + } else { + self.get_next_rustup_version_from_github(sha) + } + } + fn get_next_rustup_version_from_github(&self, sha: &str) -> anyhow::Result { println!("Getting next Rustup version from Cargo.toml..."); From 9a096ec76220ab98e0a2beecab42692c289bb80e Mon Sep 17 00:00:00 2001 From: Jan David Date: Mon, 31 Mar 2025 15:21:49 +0200 Subject: [PATCH 42/44] Invalidate CDNs when publishing new rustup release After publishing a new rustup release, we are now invalidating both CloudFront and Fastly. This fixes [rust-lang/simpleinfra#415]. [rust-lang/simpleinfra#415]: https://github.com/rust-lang/simpleinfra/issues/415 --- src/rustup.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/rustup.rs b/src/rustup.rs index b145dcd..e3fc567 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -78,6 +78,9 @@ impl Context { // Update the release number self.update_rustup_release(&version)?; + // Invalidate the CDN caches + self.invalidate_rustup()?; + Ok(()) } @@ -249,6 +252,22 @@ version = '{}' self.config.upload_bucket, self.config.upload_dir ))) } + + fn invalidate_rustup(&mut self) -> Result<(), Error> { + println!("Invalidating CDN caches..."); + + let paths = [ + self.config.upload_dir.clone(), + "rustup-init.sh".into(), + "rustup.sh".into(), + "rustup/release-stable.toml".into(), + ]; + + self.invalidate_cloudfront(&self.config.cloudfront_static_id, &paths)?; + self.invalidate_fastly(&paths)?; + + Ok(()) + } } fn decode_and_deserialize_cargo_toml(base64_encoded_toml: &str) -> Result { From acb0303ffd2b36461c0bdfa2f2d1ec04dbaf79c0 Mon Sep 17 00:00:00 2001 From: Jan David Date: Mon, 31 Mar 2025 15:26:37 +0200 Subject: [PATCH 43/44] Fix auto-formatting in GitHub Action --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10203c4..4a6b7c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: push: branches: - master - pull_request: { } + pull_request: {} jobs: test: @@ -35,7 +35,7 @@ jobs: strategy: fail-fast: false matrix: - channel: [ stable, beta, nightly ] + channel: [stable, beta, nightly] steps: - name: Clone the source code @@ -71,7 +71,7 @@ jobs: strategy: fail-fast: false matrix: - channel: [ stable, beta ] + channel: [stable, beta] steps: - name: Clone the source code @@ -126,7 +126,7 @@ jobs: permissions: id-token: write - needs: [ test, release, rustup, docker ] + needs: [test, release, rustup, docker] if: github.event_name == 'push' && github.repository == 'rust-lang/promote-release' && github.ref == 'refs/heads/master' steps: From 1edda5958cf15b7b3512954e53a9b3d43009d217 Mon Sep 17 00:00:00 2001 From: Jan David Date: Mon, 31 Mar 2025 15:28:31 +0200 Subject: [PATCH 44/44] Skip CloudFront validations in loca rustup tests --- local/rustup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/local/rustup.sh b/local/rustup.sh index 85bcf03..038ed11 100755 --- a/local/rustup.sh +++ b/local/rustup.sh @@ -94,6 +94,7 @@ export PROMOTE_RELEASE_DOWNLOAD_BUCKET="rustup-builds" export PROMOTE_RELEASE_DOWNLOAD_DIR="" export PROMOTE_RELEASE_GPG_KEY_FILE="" export PROMOTE_RELEASE_GPG_PASSWORD_FILE="" +export PROMOTE_RELEASE_SKIP_CLOUDFRONT_INVALIDATIONS="yes" export PROMOTE_RELEASE_UPLOAD_ADDR="" export PROMOTE_RELEASE_UPLOAD_BUCKET="static" export PROMOTE_RELEASE_UPLOAD_STORAGE_CLASS="STANDARD"