diff --git a/src/cargo/core/features.rs b/src/cargo/core/features.rs index a1a3e0797e8..34cdc898ae7 100644 --- a/src/cargo/core/features.rs +++ b/src/cargo/core/features.rs @@ -408,6 +408,8 @@ features! { // Allow specifying different binary name apart from the crate name (unstable, different_binary_name, "", "reference/unstable.html#different-binary-name"), + + (unstable, expand_env_vars, "", "reference/unstable.html#expand-env-vars"), } pub struct Feature { @@ -656,6 +658,7 @@ unstable_cli_options!( unstable_options: bool = ("Allow the usage of unstable options"), weak_dep_features: bool = ("Allow `dep_name?/feature` feature syntax"), skip_rustdoc_fingerprint: bool = (HIDDEN), + expand_env_vars: bool = ("Expand environment variable references like ${FOO} in certain contexts, such as `path = \"...\"` in dependencies."), ); const STABILIZED_COMPILE_PROGRESS: &str = "The progress bar is now always \ @@ -878,6 +881,7 @@ impl CliUnstable { "extra-link-arg" => stabilized_warn(k, "1.56", STABILIZED_EXTRA_LINK_ARG), "configurable-env" => stabilized_warn(k, "1.56", STABILIZED_CONFIGURABLE_ENV), "future-incompat-report" => self.future_incompat_report = parse_empty(k, v)?, + "expand-env-vars" => self.expand_env_vars = parse_empty(k, v)?, _ => bail!("unknown `-Z` flag specified: {}", k), } diff --git a/src/cargo/util/env_vars.rs b/src/cargo/util/env_vars.rs new file mode 100644 index 00000000000..51a209b04a3 --- /dev/null +++ b/src/cargo/util/env_vars.rs @@ -0,0 +1,251 @@ +//! Expands environment variable references in strings. + +use crate::util::CargoResult; +use std::borrow::Cow; + +/// Expands a string, replacing references to variables with values provided by +/// the caller. +/// +/// This function looks for references to variables, similar to environment +/// variable references in command-line shells or Makefiles, and replaces the +/// references with values. The caller provides a `query` function which gives +/// the values of the variables. +/// +/// The syntax used for variable references is `${name}` or `${name?default}` if +/// a default value is provided. The curly braces are always required; +/// `$FOO` will not be interpreted as a variable reference (and will be copied +/// to the output). +/// +/// If a variable is referenced, then it must have a value (`query` must return +/// `Some`) or the variable reference must provide a default value (using the +/// `...?default` syntax). If `query` returns `None` and the variable reference +/// does not provide a default value, then the expansion of the entire string +/// will fail and the function will return `Err`. +/// +/// Most strings processed by Cargo will not contain variable references. +/// Hence, this function uses `Cow` for its return type; it will return +/// its input string as `Cow::Borrowed` if no variable references were found. +pub fn expand_vars_with<'a, Q>(s: &'a str, query: Q) -> CargoResult> +where + Q: Fn(&str) -> CargoResult>, +{ + let mut rest: &str; + let mut result: String; + if let Some(pos) = s.find('$') { + result = String::with_capacity(s.len() + 50); + result.push_str(&s[..pos]); + rest = &s[pos..]; + } else { + // Most strings do not contain environment variable references. + // We optimize for the case where there is no reference, and + // return the same (borrowed) string. + return Ok(Cow::Borrowed(s)); + }; + + while let Some(pos) = rest.find('$') { + result.push_str(&rest[..pos]); + rest = &rest[pos..]; + let mut chars = rest.chars(); + let c0 = chars.next(); + debug_assert_eq!(c0, Some('$')); // because rest.find() + match chars.next() { + Some('{') => { + // the expected case, which is handled below. + } + Some(c) => { + // We found '$' that was not paired with '{'. + // This is not a variable reference. + // Output the $ and continue. + result.push('$'); + result.push(c); + rest = chars.as_str(); + continue; + } + None => { + // We found '$' at the end of the string. + result.push('$'); + break; + } + } + let name_start = chars.as_str(); + let name: &str; + let default_value: Option<&str>; + // Look for '}' or '?' + loop { + let pos = name_start.len() - chars.as_str().len(); + match chars.next() { + None => { + anyhow::bail!("environment variable reference is missing closing brace.") + } + Some('}') => { + name = &name_start[..pos]; + default_value = None; + break; + } + Some('?') => { + name = &name_start[..pos]; + let default_value_start = chars.as_str(); + loop { + let pos = chars.as_str(); + if let Some(c) = chars.next() { + if c == '}' { + default_value = Some( + &default_value_start[..default_value_start.len() - pos.len()], + ); + break; + } + } else { + anyhow::bail!( + "environment variable reference is missing closing brace." + ); + } + } + break; + } + Some(_) => { + // consume this character (as part of var name) + } + } + } + + if name.is_empty() { + anyhow::bail!("environment variable reference has invalid empty name"); + } + // We now have the environment variable name, and have parsed the end of the + // name reference. + match (query(name)?, default_value) { + (Some(value), _) => result.push_str(&value), + (None, Some(value)) => result.push_str(value), + (None, None) => anyhow::bail!(format!( + "environment variable '{}' is not set and has no default value", + name + )), + } + rest = chars.as_str(); + } + result.push_str(rest); + Ok(Cow::Owned(result)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn basic() { + let query = |name: &str| { + Ok(Some( + match name { + "FOO" => "/foo", + "BAR" => "/bar", + "FOO(ZAP)" => "/foo/zap", + "WINKING_FACE" => "\u{1F609}", + "\u{1F916}" => "ROBOT FACE", + _ => return Ok(None), + } + .to_string(), + )) + }; + + let expand = |s| expand_vars_with(s, query); + + macro_rules! case { + ($input:expr, $expected_output:expr) => {{ + let input = $input; + let expected_output = $expected_output; + match expand(input) { + Ok(output) => { + assert_eq!(output, expected_output, "input = {:?}", input); + } + Err(e) => { + panic!( + "Expected string {:?} to expand successfully, but it failed: {:?}", + input, e + ); + } + } + }}; + } + + macro_rules! err_case { + ($input:expr, $expected_error:expr) => {{ + let input = $input; + let expected_error = $expected_error; + match expand(input) { + Ok(output) => { + panic!("Expected expansion of string {:?} to fail, but it succeeded with value {:?}", input, output); + } + Err(e) => { + let message = e.to_string(); + assert_eq!(message, expected_error, "input = {:?}", input); + } + } + }} + } + + // things without references should not change. + case!("", ""); + case!("identity", "identity"); + + // we require ${...} (braces), so we ignore $FOO. + case!("$FOO/some_package", "$FOO/some_package"); + + // make sure variable references at the beginning, middle, and end + // of a string all work correctly. + case!("${FOO}", "/foo"); + case!("${FOO} one", "/foo one"); + case!("one ${FOO}", "one /foo"); + case!("one ${FOO} two", "one /foo two"); + case!("one ${FOO} two ${BAR} three", "one /foo two /bar three"); + + // variable names can contain most characters, except for '}' or '?' + // (Windows sets "ProgramFiles(x86)", for example.) + case!("${FOO(ZAP)}", "/foo/zap"); + + // variable is set, and has a default (which goes unused) + case!("${FOO?/default}", "/foo"); + + // variable is not set, but does have default + case!("${VAR_NOT_SET?/default}", "/default"); + + // variable is not set and has no default + err_case!( + "${VAR_NOT_SET}", + "environment variable 'VAR_NOT_SET' is not set and has no default value" + ); + + // environment variables with unicode values are ok + case!("${WINKING_FACE}", "\u{1F609}"); + + // strings with unicode in them are ok + case!("\u{1F609}${FOO}", "\u{1F609}/foo"); + + // environment variable names with unicode in them are ok + case!("${\u{1F916}}", "ROBOT FACE"); + + // default values with unicode in them are ok + case!("${VAR_NOT_SET?\u{1F916}}", "\u{1F916}"); + + // invalid names + err_case!( + "${}", + "environment variable reference has invalid empty name" + ); + err_case!( + "${?default}", + "environment variable reference has invalid empty name" + ); + err_case!( + "${", + "environment variable reference is missing closing brace." + ); + err_case!( + "${FOO", + "environment variable reference is missing closing brace." + ); + err_case!( + "${FOO?default", + "environment variable reference is missing closing brace." + ); + } +} diff --git a/src/cargo/util/mod.rs b/src/cargo/util/mod.rs index 4b8604f92fb..b23ff89c848 100644 --- a/src/cargo/util/mod.rs +++ b/src/cargo/util/mod.rs @@ -35,6 +35,7 @@ mod counter; pub mod cpu; mod dependency_queue; pub mod diagnostic_server; +pub mod env_vars; pub mod errors; mod flock; pub mod graph; diff --git a/src/cargo/util/toml/mod.rs b/src/cargo/util/toml/mod.rs index 059c4480377..ca640380191 100644 --- a/src/cargo/util/toml/mod.rs +++ b/src/cargo/util/toml/mod.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::fmt; use std::marker::PhantomData; @@ -251,19 +252,60 @@ impl<'de, P: Deserialize<'de>> de::Deserialize<'de> for TomlDependency

{ } } +/// Expands any environment variables found in `string`. +/// +/// Expanding environment variables is currently an unstable feature. +/// If the feature is not enabled, then we do not expand variable +/// references, so "${FOO}" remains exactly "${FOO}" in the output. +/// +/// This feature may be in one of _three_ modes: +/// * Completely disabled. This is the default (stable) behavior. In this mode, +/// variable references in `string` are ignored (treated like normal text). +/// * `-Z expand_env_vars` was specified on the command-line, but the current +/// manifest does not specify `cargo-features = ["expand_env_vars"]. In this +/// mode, we do look for variable references. If we find _any_ variable +/// reference, then we report an error. This mode is intended only to allow us +/// to do Crater runs, so that we can find any projects that this new +/// feature would break, since they contain strings that match our `${FOO}` syntax. +/// * The current manifest contains `cargo-features = ["expand_env_vars"]`. +/// In this mode, we look for variable references and process them. +/// This is the mode that will eventually be stabilized and become the default. +fn expand_env_var<'a>( + string: &'a str, + config: &Config, + features: &Features, +) -> CargoResult> { + if config.cli_unstable().expand_env_vars || features.is_enabled(Feature::expand_env_vars()) { + let expanded_path = crate::util::env_vars::expand_vars_with(string, |name| { + if features.is_enabled(Feature::expand_env_vars()) { + Ok(std::env::var(name).ok()) + } else { + // This manifest contains a string which looks like a variable + // reference, but the manifest has not enabled the feature. + // Report an error. + anyhow::bail!("this manifest uses environment variable references (e.g. ${FOO}) but has not specified `cargo-features = [\"expand-env-vars\"]`."); + } + })?; + Ok(expanded_path) + } else { + Ok(Cow::Borrowed(string)) + } +} + pub trait ResolveToPath { - fn resolve(&self, config: &Config) -> PathBuf; + fn resolve(&self, config: &Config, features: &Features) -> CargoResult; } impl ResolveToPath for String { - fn resolve(&self, _: &Config) -> PathBuf { - self.into() + fn resolve(&self, config: &Config, features: &Features) -> CargoResult { + let expanded = expand_env_var(self, config, features)?; + Ok(expanded.to_string().into()) } } impl ResolveToPath for ConfigRelativePath { - fn resolve(&self, c: &Config) -> PathBuf { - self.resolve_path(c) + fn resolve(&self, c: &Config, _features: &Features) -> CargoResult { + Ok(self.resolve_path(c)) } } @@ -1887,7 +1929,7 @@ impl DetailedTomlDependency

{ SourceId::for_git(&loc, reference)? } (None, Some(path), _, _) => { - let path = path.resolve(cx.config); + let path = path.resolve(cx.config, cx.features)?; cx.nested_paths.push(path.clone()); // If the source ID for the package we're parsing is a path // source, then we normalize the path here to get rid of diff --git a/src/doc/src/reference/unstable.md b/src/doc/src/reference/unstable.md index 35c877b7e2c..b705bbe6b83 100644 --- a/src/doc/src/reference/unstable.md +++ b/src/doc/src/reference/unstable.md @@ -1328,6 +1328,29 @@ filename = "007bar" path = "src/main.rs" ``` +### expand-env-vars + +The `expand-env-vars` feature allows certain fields within manifests to refer +to environment variables, using `${FOO}` syntax. Currently, the only field that +may refer to environment variables is the `path` field of a dependency. + +For example: + +```toml +cargo-features = ["expand-env-vars"] + +[dependencies] +foo = { path = "${MY_PROJECT_ROOT}/some/long/path/utils" } +``` + +For organizations that manage large code bases, using relative walk-up paths +(e.g. `../../../../long/path/utils`) is not practical; some organizations +forbid using walk-up paths. The `expand-env-vars` feature allows Cargo +to work in such systems. + +The curly braces are required; `$FOO` will not be interpreted as a variable +reference. + ## Stabilized and removed features ### Compile progress diff --git a/tests/testsuite/main.rs b/tests/testsuite/main.rs index 8c30bf929a8..710ed6ba680 100644 --- a/tests/testsuite/main.rs +++ b/tests/testsuite/main.rs @@ -123,6 +123,7 @@ mod tree; mod tree_graph_features; mod unit_graph; mod update; +mod var_refs; mod vendor; mod verify_project; mod version; diff --git a/tests/testsuite/var_refs.rs b/tests/testsuite/var_refs.rs new file mode 100644 index 00000000000..550b17f0c96 --- /dev/null +++ b/tests/testsuite/var_refs.rs @@ -0,0 +1,211 @@ +//! Tests for variable references, e.g. { path = "${FOO}/bar/foo" } + +use cargo_test_support::project; + +#[cargo_test] +fn simple() { + let p = project() + .file( + "Cargo.toml", + r#" + [workspace] + members = ["utils/zoo", "bar"] + "#, + ) + .file( + "utils/zoo/Cargo.toml", + r#" + [package] + name = "zoo" + version = "1.0.0" + edition = "2018" + authors = [] + + [lib] + "#, + ) + .file( + "utils/zoo/src/lib.rs", + r#" + pub fn hello() { println!("Hello, world!"); } + "#, + ) + .file( + "bar/Cargo.toml", + r#" + cargo-features = ["expand-env-vars"] + + [package] + name = "bar" + version = "1.0.0" + edition = "2018" + authors = [] + + [dependencies] + zoo = { path = "${UTILS_ROOT}/zoo" } + "#, + ) + .file( + "bar/src/main.rs", + r#" + fn main() { + zoo::hello(); + } + "#, + ) + .build(); + + p.cargo("build") + .masquerade_as_nightly_cargo() + .cwd("bar") + .env("UTILS_ROOT", "../utils") + .run(); + assert!(p.bin("bar").is_file()); +} + +#[cargo_test] +fn basic_with_default() { + let p = project() + .file( + "Cargo.toml", + r#" + [workspace] + members = ["utils/zoo", "bar"] + "#, + ) + .file( + "utils/zoo/Cargo.toml", + r#" + [package] + name = "zoo" + version = "1.0.0" + edition = "2018" + authors = [] + + [lib] + "#, + ) + .file( + "utils/zoo/src/lib.rs", + r#" + pub fn hello() { println!("Hello, world!"); } + "#, + ) + .file( + "bar/Cargo.toml", + r#" + cargo-features = ["expand-env-vars"] + + [package] + name = "bar" + version = "1.0.0" + edition = "2018" + authors = [] + + [dependencies] + zoo = { path = "${UTILS_ROOT?../utils}/zoo" } + "#, + ) + .file( + "bar/src/main.rs", + r#" + fn main() { + zoo::hello(); + } + "#, + ) + .build(); + + // Note: UTILS_ROOT is not set in the environment. + p.cargo("build") + .masquerade_as_nightly_cargo() + .arg("-Zunstable-options") + .arg("-Zexpand-env-vars") + .cwd("bar") + .run(); + assert!(p.bin("bar").is_file()); +} + +#[cargo_test] +fn missing_features() { + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "1.0.0" + edition = "2018" + + [lib] + + [dependencies] + utils = { path = "${UTILS_ROOT}/utils" } + "#, + ) + .file("src/lib.rs", "") + .build(); + + p.cargo("build") + .masquerade_as_nightly_cargo() + .arg("-Zexpand-env-vars") + .with_status(101) + .with_stderr_contains("[..]this manifest uses environment variable references [..] but has not specified `cargo-features = [\"expand-env-vars\"]`.[..]") + .run(); +} + +#[cargo_test] +fn var_not_set() { + let p = project() + .file( + "Cargo.toml", + r#" + cargo-features = ["expand-env-vars"] + + [package] + name = "foo" + version = "1.0.0" + edition = "2018" + authors = [] + + [dependencies] + bar = { path = "${BAD_VAR}/bar" } + "#, + ) + .file("src/lib.rs", r#""#) + .build(); + + p.cargo("build") + .masquerade_as_nightly_cargo() + .with_status(101) + .with_stderr_contains("[..]environment variable 'BAD_VAR' is not set[..]") + .run(); +} + +#[cargo_test] +fn bad_syntax() { + let p = project() + .file( + "Cargo.toml", + r#" + cargo-features = ["expand-env-vars"] + + [package] + name = "foo" + version = "1.0.0" + edition = "2018" + authors = [] + + [dependencies] + bar = { path = "${BAD_VAR" } + "#, + ) + .file("src/lib.rs", r#""#) + .build(); + + p.cargo("build") + .masquerade_as_nightly_cargo() + .with_status(101) + .with_stderr_contains("[..]environment variable reference is missing closing brace.[..]") + .run(); +}