feat: run source dependencies (#125)

* feat: run source dependencies

* fix: clippy

* fix: update dependencies

* fix: fmt

* fix: taplo

* Add filter globs

* Add `.pixi` to `exclude` instead of `include`

* Fix some of the snapshots

* Update snapshots

* Remove temp file

---------

Co-authored-by: Julian Hofer <[email protected]>

Author: baszalmstra

Cargo.lock

@@ -36,17 +36,41 @@ jsonrpc-stdio-server = "18.0.0"
 jsonrpc-http-server = "18.0.0"
 jsonrpc-core = "18.0.0"
 
-rattler-build = { git = "https://github.com/prefix-dev/rattler-build", rev = "6744398", default-features = false, features = [
+rattler-build = { version = "0.39.0", default-features = false, features = [
   "rustls-tls",
 ] }
 
-rattler_conda_types = { version = "0.30.2", default-features = false }
-rattler_package_streaming = { version = "0.22.27", default-features = false }
-rattler_virtual_packages = { version = "2.0.2", default-features = false }
+rattler_conda_types = { version = "0.32.0", default-features = false }
+rattler_package_streaming = { version = "0.22.35", default-features = false }
+rattler_virtual_packages = { version = "2.0.9", default-features = false }
 
+pixi_build_types = { version = "*" }
+pixi_consts = { version = "*" }
+pixi_manifest = { version = "*" }
+pixi_spec = { version = "*" }
+pixi_build_type_conversions = { version = "*" }
+
+pixi-build-backend = { path = "crates/pixi-build-backend" }
+
+[patch.crates-io]
 pixi_build_types = { git = "https://github.com/prefix-dev/pixi", branch = "main" }
 pixi_consts = { git = "https://github.com/prefix-dev/pixi", branch = "main" }
 pixi_manifest = { git = "https://github.com/prefix-dev/pixi", branch = "main" }
 pixi_spec = { git = "https://github.com/prefix-dev/pixi", branch = "main" }
 pixi_build_type_conversions = { git = "https://github.com/prefix-dev/pixi", branch = "main" }
-pixi-build-backend = { path = "crates/pixi-build-backend" }
+
+#pixi_build_types = { path = "../pixi/crates/pixi_build_types" }
+#pixi_consts = { path = "../pixi/crates/pixi_consts" }
+#pixi_manifest = { path = "../pixi/crates/pixi_manifest" }
+#pixi_spec = { path = "../pixi/crates/pixi_spec" }
+#pixi_build_type_conversions = { path = "../pixi/crates/pixi_build_type_conversions" }
+
+#rattler_cache = { path = "../rattler/crates/rattler_cache" }
+#rattler_conda_types = { path = "../rattler/crates/rattler_conda_types" }
+#rattler_package_streaming = { path = "../rattler/crates/rattler_package_streaming" }
+#rattler_virtual_packages = { path = "../rattler/crates/rattler_virtual_packages" }
+#rattler_repodata_gateway = { path = "../rattler/crates/rattler_repodata_gateway" }
+#simple_spawn_blocking = { path = "../rattler/crates/simple_spawn_blocking" }
+
+rattler-build = { git = "https://github.com/prefix-dev/rattler-build", branch = "main" }
+#rattler-build = { path = "../rattler-build" }

Cargo.toml

@@ -46,15 +46,15 @@ pub struct App {
 pub enum Commands {
     /// Get conda metadata for a recipe.
     GetCondaMetadata {
-        #[clap(env, long, env = "PIXI_PROJECT_MANIFEST", default_value = consts::PROJECT_MANIFEST)]
+        #[clap(env, long, env = "PIXI_PROJECT_MANIFEST", default_value = consts::WORKSPACE_MANIFEST)]
         manifest_path: PathBuf,
 
         #[clap(long)]
         host_platform: Option<Platform>,
     },
     /// Build a conda package.
     CondaBuild {
-        #[clap(env, long, env = "PIXI_PROJECT_MANIFEST", default_value = consts::PROJECT_MANIFEST)]
+        #[clap(env, long, env = "PIXI_PROJECT_MANIFEST", default_value = consts::WORKSPACE_MANIFEST)]
         manifest_path: PathBuf,
     },
     /// Get the capabilities of the backend.

crates/pixi-build-backend/src/cli.rs

@@ -4,5 +4,5 @@ mod requirements;
 mod variants;
 
 pub use configuration::{build_configuration, BuildConfigurationParams};
-pub use requirements::requirements;
+pub use requirements::{requirements, PackageRequirements, SourceRequirements};
 pub use variants::compute_variants;

crates/pixi-build-backend/src/common/mod.rs

@@ -1,25 +1,58 @@
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, HashMap};
 
 use rattler_build::{
     recipe::{parser::Requirements, variable::Variable},
     NormalizedKey,
 };
-use rattler_conda_types::ChannelConfig;
+use serde::Serialize;
 
-use crate::{dependencies::extract_dependencies, traits::Dependencies, ProjectModel, Targets};
+use crate::{
+    dependencies::ExtractedDependencies, traits::Dependencies, PackageSpec, ProjectModel, Targets,
+};
+
+pub struct PackageRequirements<P: ProjectModel> {
+    /// Requirements for rattler-build
+    pub requirements: Requirements,
+
+    /// The source requirements
+    pub source: SourceRequirements<P>,
+}
+
+#[derive(Debug, Serialize)]
+#[serde(bound(
+    serialize = "<<P::Targets as Targets>::Spec as PackageSpec>::SourceSpec: Serialize"
+))]
+pub struct SourceRequirements<P: ProjectModel> {
+    /// Source package specification for build dependencies
+    pub build: HashMap<String, <<P::Targets as Targets>::Spec as PackageSpec>::SourceSpec>,
+
+    /// Source package specification for host dependencies
+    pub host: HashMap<String, <<P::Targets as Targets>::Spec as PackageSpec>::SourceSpec>,
+
+    /// Source package specification for runtime dependencies
+    pub run: HashMap<String, <<P::Targets as Targets>::Spec as PackageSpec>::SourceSpec>,
+}
 
 /// Return requirements for the given project model
 pub fn requirements<P: ProjectModel>(
     dependencies: Dependencies<<P::Targets as Targets>::Spec>,
-    channel_config: &ChannelConfig,
     variant: &BTreeMap<NormalizedKey, Variable>,
-) -> miette::Result<Requirements> {
-    // Extract dependencies into requirements
-    let requirements = Requirements {
-        build: extract_dependencies(channel_config, dependencies.build, variant)?,
-        host: extract_dependencies(channel_config, dependencies.host, variant)?,
-        run: extract_dependencies(channel_config, dependencies.run, variant)?,
-        ..Default::default()
-    };
-    Ok(requirements)
+) -> miette::Result<PackageRequirements<P>> {
+    let build = ExtractedDependencies::from_dependencies(dependencies.build, variant)?;
+    let host = ExtractedDependencies::from_dependencies(dependencies.host, variant)?;
+    let run = ExtractedDependencies::from_dependencies(dependencies.run, variant)?;
+
+    Ok(PackageRequirements {
+        requirements: Requirements {
+            build: build.dependencies,
+            host: host.dependencies,
+            run: run.dependencies,
+            ..Default::default()
+        },
+        source: SourceRequirements {
+            build: build.sources,
+            host: host.sources,
+            run: run.sources,
+        },
+    })
 }

crates/pixi-build-backend/src/common/requirements.rs

@@ -1,6 +1,5 @@
 use std::{
-    collections::BTreeMap,
-    path::{Path, PathBuf},
+    collections::{BTreeMap, HashMap},
     str::FromStr,
 };
 
@@ -10,70 +9,43 @@ use rattler_build::{
     recipe::{parser::Dependency, variable::Variable},
     NormalizedKey,
 };
-use rattler_conda_types::{
-    ChannelConfig, MatchSpec, NamelessMatchSpec, PackageName, ParseStrictness::Strict,
-};
+use rattler_conda_types::{MatchSpec, NamelessMatchSpec, PackageName, ParseStrictness::Strict};
 
 use crate::traits::PackageSpec;
 
 /// A helper struct to extract match specs from a manifest.
+#[derive(Default)]
 pub struct MatchspecExtractor<'a> {
-    channel_config: &'a ChannelConfig,
     variant: Option<&'a BTreeMap<NormalizedKey, Variable>>,
-    ignore_self: bool,
 }
 
-/// Resolves the path relative to `root_dir`. If the path is absolute,
-/// it is returned verbatim.
-///
-/// May return an error if the path is prefixed with `~` and the home
-/// directory is undefined.
-pub(crate) fn resolve_path(path: &Path, root_dir: impl AsRef<Path>) -> Option<PathBuf> {
-    if path.is_absolute() {
-        Some(PathBuf::from(path))
-    } else if let Ok(user_path) = path.strip_prefix("~/") {
-        dirs::home_dir().map(|h| h.join(user_path))
-    } else {
-        Some(root_dir.as_ref().join(path))
-    }
+pub struct ExtractedMatchSpecs<S: PackageSpec> {
+    pub specs: Vec<MatchSpec>,
+    pub sources: HashMap<String, S::SourceSpec>,
 }
 
 impl<'a> MatchspecExtractor<'a> {
-    pub fn new(channel_config: &'a ChannelConfig) -> Self {
-        Self {
-            channel_config,
-            ignore_self: false,
-            variant: None,
-        }
-    }
-
-    /// If `ignore_self` is `true`, the conversion will skip dependencies that
-    /// point to root directory itself.
-    pub fn with_ignore_self(self, ignore_self: bool) -> Self {
-        Self {
-            ignore_self,
-            ..self
-        }
+    pub fn new() -> Self {
+        Self::default()
     }
 
     /// Sets the variant to use for the match specs.
     pub fn with_variant(self, variant: &'a BTreeMap<NormalizedKey, Variable>) -> Self {
         Self {
             variant: Some(variant),
-            ..self
         }
     }
 
     /// Extracts match specs from the given set of dependencies.
     pub fn extract<'b, S>(
         &self,
         dependencies: impl IntoIterator<Item = (&'b pbt::SourcePackageName, &'b S)>,
-    ) -> miette::Result<Vec<MatchSpec>>
+    ) -> miette::Result<ExtractedMatchSpecs<S>>
     where
         S: PackageSpec + 'b,
     {
-        let root_dir = &self.channel_config.root_dir;
         let mut specs = Vec::new();
+        let mut source_specs = HashMap::new();
         for (name, spec) in dependencies.into_iter() {
             let name = PackageName::from_str(name.as_str()).into_diagnostic()?;
             // If we have a variant override, we should use that instead of the spec.
@@ -97,28 +69,45 @@ impl<'a> MatchspecExtractor<'a> {
             }
 
             // Match on supported packages
-            let match_spec = spec.to_match_spec(name, root_dir, self.ignore_self)?;
+            let (match_spec, source_spec) = spec.to_match_spec(name.clone())?;
 
             specs.push(match_spec);
+            if let Some(source_spec) = source_spec {
+                source_specs.insert(name.as_normalized().to_owned(), source_spec);
+            }
         }
 
-        Ok(specs)
+        Ok(ExtractedMatchSpecs {
+            specs,
+            sources: source_specs,
+        })
     }
 }
 
-pub fn extract_dependencies<'a, T>(
-    channel_config: &ChannelConfig,
-    dependencies: impl IntoIterator<Item = (&'a pbt::SourcePackageName, &'a T)>,
-    variant: &BTreeMap<NormalizedKey, Variable>,
-) -> miette::Result<Vec<Dependency>>
-where
-    T: PackageSpec + 'a,
-{
-    Ok(MatchspecExtractor::new(channel_config)
-        .with_ignore_self(true)
-        .with_variant(variant)
-        .extract(dependencies)?
-        .into_iter()
-        .map(Dependency::Spec)
-        .collect())
+pub struct ExtractedDependencies<T: PackageSpec> {
+    pub dependencies: Vec<Dependency>,
+    pub sources: HashMap<String, T::SourceSpec>,
+}
+
+impl<T: PackageSpec> ExtractedDependencies<T> {
+    pub fn from_dependencies<'a>(
+        dependencies: impl IntoIterator<Item = (&'a pbt::SourcePackageName, &'a T)>,
+        variant: &BTreeMap<NormalizedKey, Variable>,
+    ) -> miette::Result<Self>
+    where
+        T: 'a,
+    {
+        let extracted_specs = MatchspecExtractor::new()
+            .with_variant(variant)
+            .extract(dependencies)?;
+
+        Ok(Self {
+            dependencies: extracted_specs
+                .specs
+                .into_iter()
+                .map(Dependency::Spec)
+                .collect(),
+            sources: extracted_specs.sources,
+        })
+    }
 }

crates/pixi-build-backend/src/dependencies.rs

@@ -14,4 +14,4 @@ pub mod traits;
 pub mod utils;
 pub mod variants;
 
-pub use traits::{PackageSpec, ProjectModel, TargetSelector, Targets};
+pub use traits::{PackageSourceSpec, PackageSpec, ProjectModel, TargetSelector, Targets};

crates/pixi-build-backend/src/lib.rs

@@ -14,7 +14,11 @@ use rattler_build::{
         BuildConfiguration, Directories, Output, PackageIdentifier, PackagingSettings,
         PlatformWithVirtualPackages,
     },
-    recipe::{parser::find_outputs_from_src, variable::Variable, Jinja, ParsingError, Recipe},
+    recipe::{
+        parser::{find_outputs_from_src, GlobVec},
+        variable::Variable,
+        Jinja, ParsingError, Recipe,
+    },
     selectors::SelectorConfig,
     system_tools::SystemTools,
     variant_config::{DiscoveredOutput, ParseErrors, VariantConfig},
@@ -151,7 +155,7 @@ impl RattlerBuild {
                 allow_undefined: false,
             };
 
-            let recipe = Recipe::from_node(&discovered_output.node, selector_config.clone())
+            let mut recipe = Recipe::from_node(&discovered_output.node, selector_config.clone())
                 .map_err(|err| {
                     let errs: ParseErrors<_> = err
                         .into_iter()
@@ -161,6 +165,25 @@ impl RattlerBuild {
                     errs
                 })?;
 
+            for source in &mut recipe.source {
+                if let rattler_build::recipe::parser::Source::Path(path_source) = source {
+                    let include = path_source
+                        .filter
+                        .include_globs()
+                        .iter()
+                        .map(|g| g.source())
+                        .collect();
+                    let exclude = path_source
+                        .filter
+                        .exclude_globs()
+                        .iter()
+                        .map(|g| g.source())
+                        .chain([".pixi"])
+                        .collect();
+                    path_source.filter = GlobVec::from_vec(include, Some(exclude));
+                }
+            }
+
             if recipe.build().skip() {
                 eprintln!(
                     "Skipping build for variant: {:#?}",

crates/pixi-build-backend/src/tools.rs

@@ -3,14 +3,14 @@
 //! The main entry point is the [`ProjectModel`] trait which defines the core
 //! interface for a project model.
 //!
-//! Any backend that will deal with Project (from pixi frontend as example) should implement this.
-//!
+//! Any backend that will deal with Project (from pixi frontend as example)
+//! should implement this.
 #![deny(missing_docs)]
 
 pub mod package_spec;
 pub mod project;
 pub mod targets;
 
-pub use package_spec::{AnyVersion, BinarySpecExt, PackageSpec};
+pub use package_spec::{AnyVersion, BinarySpecExt, PackageSourceSpec, PackageSpec};
 pub use project::ProjectModel;
 pub use targets::{Dependencies, TargetSelector, Targets};