Fold download crate back into rustup

Author: djc

Cargo.lock

@@ -11,13 +11,19 @@ repository = "https://github.com/rust-lang/rustup"
 build = "build.rs"
 
 [features]
-curl-backend = ["download/curl-backend"]
+curl-backend = ["dep:curl"]
 default = ["curl-backend", "reqwest-native-tls", "reqwest-rustls-tls"]
 
 vendored-openssl = ['openssl/vendored']
 
-reqwest-native-tls = ["download/reqwest-native-tls"]
-reqwest-rustls-tls = ["download/reqwest-rustls-tls"]
+reqwest-native-tls = ["reqwest/native-tls", "dep:reqwest", "dep:env_proxy"]
+reqwest-rustls-tls = [
+  "reqwest/rustls-tls-manual-roots-no-provider",
+  "dep:env_proxy",
+  "dep:reqwest",
+  "dep:rustls",
+  "dep:rustls-platform-verifier",
+]
 
 # Include in the default set to disable self-update and uninstall.
 no-self-update = []
@@ -40,9 +46,10 @@ cfg-if = "1.0"
 chrono = { version = "0.4", default-features = false, features = ["std"] }
 clap = { version = "4", features = ["derive", "wrap_help"] }
 clap_complete = "4"
-download = { path = "download", default-features = false }
+curl = { version = "0.4.44", optional = true }
 effective-limits = "0.5.5"
 enum-map = "2.5.0"
+env_proxy = { version = "0.4.1", optional = true }
 flate2 = "1"
 fs_at.workspace = true
 git-testament = "0.2"
@@ -60,8 +67,11 @@ pulldown-cmark = { version = "0.13", default-features = false }
 rand = "0.9"
 regex = "1"
 remove_dir_all = { version = "1.0.0", features = ["parallel"] }
+reqwest = { version = "0.12", default-features = false, features = ["blocking", "gzip", "socks", "stream"], optional = true }
 retry = { version = "2", default-features = false, features = ["random"] }
 rs_tracing = { version = "1.1", features = ["rs_tracing"] }
+rustls = { version = "0.23", optional = true, default-features = false, features = ["logging", "aws_lc_rs", "tls12"] }
+rustls-platform-verifier = { version = "0.5", optional = true }
 same-file = "1"
 semver = "1.0"
 serde = { version = "1.0", features = ["derive"] }
@@ -115,6 +125,9 @@ version = "0.59"
 
 [dev-dependencies]
 enum-map = "2.5.0"
+http-body-util = "0.1.0"
+hyper = { version = "1.0", default-features = false, features = ["server", "http1"] }
+hyper-util = { version = "0.1.1", features = ["tokio"] }
 platforms.workspace = true
 proptest.workspace = true
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
@@ -126,9 +139,6 @@ platforms.workspace = true
 [lints]
 workspace = true
 
-[workspace]
-members = ["download"]
-
 [workspace.package]
 version = "1.28.1"
 edition = "2024"

Cargo.toml

@@ -50,10 +50,18 @@ target_cargo() {
 
 target_cargo build
 
-download_pkg_test() {
-  features=('--no-default-features' '--features' 'curl-backend,reqwest-native-tls')
+# Machines have 7GB of RAM, and our target/ contents is large enough that
+# thrashing will occur if we build-run-build-run rather than
+# build-build-build-run-run-run. Since this is used solely for non-release
+# artifacts, we try to keep features consistent across the builds, whether for
+# docs/test/runs etc.
+build_test() {
+  cmd="$1"
+  shift
+
+  features=('--features' 'curl-backend,reqwest-native-tls')
   case "$TARGET" in
-    # these platforms aren't supported by ring:
+    # these platforms aren't supported by aws-lc-rs:
     powerpc* ) ;;
     mips* ) ;;
     riscv* ) ;;
@@ -62,23 +70,11 @@ download_pkg_test() {
     * ) features+=('--features' 'reqwest-rustls-tls') ;;
   esac
 
-  cargo "$1" --locked --profile "$BUILD_PROFILE" --target "$TARGET" "${features[@]}" -p download
-}
-
-# Machines have 7GB of RAM, and our target/ contents is large enough that
-# thrashing will occur if we build-run-build-run rather than
-# build-build-build-run-run-run. Since this is used solely for non-release
-# artifacts, we try to keep features consistent across the builds, whether for
-# docs/test/runs etc.
-build_test() {
-  cmd="$1"
-  shift
-  download_pkg_test "${cmd}"
   if [ "build" = "${cmd}" ]; then
-    target_cargo "${cmd}" --workspace --all-targets --features test
+    target_cargo "${cmd}" --workspace --all-targets "${features[@]}" --features test
   else
-    target_cargo "${cmd}" --workspace --features test --tests
-    target_cargo "${cmd}" --doc --workspace --features test
+    target_cargo "${cmd}" --workspace "${features[@]}" --features test --tests
+    target_cargo "${cmd}" --doc --workspace "${features[@]}" --features test
   fi
 }
 

ci/run.bash

@@ -3,11 +3,19 @@
 use std::fs::remove_file;
 use std::path::Path;
 
-use anyhow::Context;
-pub use anyhow::Result;
+#[cfg(any(
+    not(feature = "curl-backend"),
+    not(feature = "reqwest-rustls-tls"),
+    not(feature = "reqwest-native-tls")
+))]
+use anyhow::anyhow;
+use anyhow::{Context, Result};
 use thiserror::Error;
 use url::Url;
 
+#[cfg(test)]
+mod tests;
+
 /// User agent header value for HTTP request.
 /// See: https://github.com/rust-lang/rustup/issues/2860.
 #[cfg(feature = "curl-backend")]
@@ -488,8 +496,6 @@ pub enum DownloadError {
     HttpStatus(u32),
     #[error("file not found")]
     FileNotFound,
-    #[error("download backend '{0}' unavailable")]
-    BackendUnavailable(&'static str),
     #[error("{0}")]
     Message(String),
     #[error(transparent)]

download/Cargo.toml

@@ -21,7 +21,7 @@ mod curl {
     use url::Url;
 
     use super::{serve_file, tmp_dir, write_file};
-    use download::*;
+    use crate::download::{Backend, Event};
 
     #[tokio::test]
     async fn partially_downloaded_file_gets_resumed_from_byte_offset() {
@@ -107,7 +107,7 @@ mod reqwest {
     use url::Url;
 
     use super::{serve_file, tmp_dir, write_file};
-    use download::{Backend, Event, TlsBackend};
+    use crate::download::{Backend, Event, TlsBackend};
 
     static SERIALISE_TESTS: LazyLock<tokio::sync::Mutex<()>> =
         LazyLock::new(|| tokio::sync::Mutex::new(()));

download/src/lib.rs renamed to src/download/mod.rs

@@ -74,6 +74,7 @@ mod command;
 mod config;
 mod diskio;
 pub mod dist;
+mod download;
 pub mod env_var;
 pub mod errors;
 mod fallback_settings;

download/tests/all.rs renamed to src/download/tests.rs

@@ -164,7 +164,7 @@ pub(crate) async fn download_file_with_resume(
     notify_handler: &dyn Fn(Notification<'_>),
     process: &Process,
 ) -> Result<()> {
-    use download::DownloadError as DEK;
+    use crate::download::DownloadError as DEK;
     match download_file_(
         url,
         path,
@@ -213,8 +213,7 @@ async fn download_file_(
     process: &Process,
 ) -> Result<()> {
     #[cfg(any(feature = "reqwest-rustls-tls", feature = "reqwest-native-tls"))]
-    use download::TlsBackend;
-    use download::{Backend, Event};
+    use crate::download::{TlsBackend, Backend, Event};
     use sha2::Digest;
     use std::cell::RefCell;
 
@@ -224,7 +223,7 @@ async fn download_file_(
 
     // This callback will write the download to disk and optionally
     // hash the contents, then forward the notification up the stack
-    let callback: &dyn Fn(Event<'_>) -> download::Result<()> = &|msg| {
+    let callback: &dyn Fn(Event<'_>) -> anyhow::Result<()> = &|msg| {
         if let Event::DownloadDataReceived(data) = msg {
             if let Some(h) = hasher.borrow_mut().as_mut() {
                 h.update(data);