From 27c46b25e6ae2a5072cf3997ce2dc7ad5941c9d1 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Fri, 25 Apr 2025 19:08:29 +0200 Subject: [PATCH 01/30] feat: support forking when initializing --- rust/patchable/src/main.rs | 123 +++++++++++++++++++++++++++++++++++-- rust/patchable/src/repo.rs | 2 + 2 files changed, 120 insertions(+), 5 deletions(-) diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index c1adcb53d..ec25db7ba 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -9,8 +9,8 @@ use std::{fs::File, io::Write, path::PathBuf}; use git2::{Oid, Repository}; use serde::{Deserialize, Serialize}; -use snafu::{OptionExt, ResultExt as _, Snafu}; -use tracing_indicatif::IndicatifLayer; +use snafu::{ensure, OptionExt, ResultExt as _, Snafu}; +use tracing_indicatif::{span_ext::IndicatifSpanExt, IndicatifLayer}; use tracing_subscriber::{layer::SubscriberExt as _, util::SubscriberInitExt as _}; #[derive(clap::Parser)] @@ -149,6 +149,11 @@ enum Cmd { /// Refs (such as tags and branches) will be resolved to commit IDs. #[clap(long)] base: String, + + /// Assume a fork exists at stackabletech/ and push the base ref to it. + /// The fork URL will be stored in patchable.toml instead of the original upstream. + #[clap(long)] + forked: bool, }, /// Shows the patch directory for a given product version @@ -197,6 +202,23 @@ pub enum Error { path: PathBuf, }, + #[snafu(display("failed to parse upstream URL {url:?} to extract repository name"))] + ParseUpstreamUrl { url: String }, + #[snafu(display("failed to add temporary fork remote for {url:?}"))] + AddForkRemote { source: git2::Error, url: String }, + #[snafu(display("failed to push commit {commit} (as {refspec}) to fork {url:?}"))] + PushToFork { + source: git2::Error, + url: String, + refspec: String, + commit: Oid, + }, + #[snafu(display("failed to delete remote {name}"))] + DeleteRemote { + source: git2::Error, + name: String, + }, + #[snafu(display("failed to find images repository"))] FindImagesRepo { source: repo::Error }, #[snafu(display("images repository has no work directory"))] @@ -287,7 +309,7 @@ fn main() -> Result<()> { let base_commit = repo::resolve_and_fetch_commitish( &product_repo, &config.base.to_string(), - &config.upstream, + &config.upstream ) .context(FetchBaseCommitSnafu)?; let base_branch = ctx.base_branch(); @@ -397,7 +419,12 @@ fn main() -> Result<()> { ); } - Cmd::Init { pv, upstream, base } => { + Cmd::Init { + pv, + upstream, + base, + forked, + } => { let ctx = ProductVersionContext { pv, images_repo_root, @@ -414,8 +441,94 @@ fn main() -> Result<()> { // --base can be a reference, but patchable.toml should always have a resolved commit id, // so that it cannot be changed under our feet (without us knowing so, anyway...). tracing::info!(?base, "resolving base commit-ish"); - let base_commit = repo::resolve_and_fetch_commitish(&product_repo, &base, &upstream) + + let (base_commit, upstream) = if forked { + // Parse e.g. "https://github.com/apache/druid.git" into "druid" + let repo_name = upstream.split('/').last().map(|repo| repo.trim_end_matches(".git")).context(ParseUpstreamUrlSnafu { url: &upstream })?; + + ensure!(!repo_name.is_empty(), ParseUpstreamUrlSnafu { url: &upstream }); + + let fork_url = format!("https://github.com/stackabletech/{}.git", repo_name); + tracing::info!(%fork_url, "using fork repository"); + + // Fetch from original upstream using a temporary remote name + tracing::info!(upstream = upstream, %base, "fetching base ref from original upstream"); + let base_commit_oid = repo::resolve_and_fetch_commitish( + &product_repo, + &base, + &upstream + ) .context(FetchBaseCommitSnafu)?; + + tracing::info!(commit = %base_commit_oid, "fetched base commit OID"); + + // Add fork remote + let temp_fork_remote = "patchable_fork_push"; + let mut fork_remote = product_repo + .remote(temp_fork_remote, &fork_url) + .context(AddForkRemoteSnafu { url: fork_url.clone() })?; + + // Push the base commit to the fork + tracing::info!(commit = %base_commit_oid, base = base, url = fork_url, "pushing commit to fork"); + let mut callbacks = git2::RemoteCallbacks::new(); + callbacks.credentials(|_url, username_from_url, _allowed_types| { + git2::Cred::credential_helper( + &git2::Config::open_default().unwrap(), // Use default git config + _url, + username_from_url, + ) + }); + + // Add progress tracking for push operation + let span_push = tracing::info_span!("pushing"); + span_push.pb_set_style(&utils::progress_bar_style()); + let _ = span_push.enter(); + let mut quant_push = utils::Quantizer::percent(); + callbacks.push_transfer_progress(move |current, total, _| { + if total > 0 { + quant_push.update_span_progress(current, total, &span_push); + } + }); + + let mut push_options = git2::PushOptions::new(); + push_options.remote_callbacks(callbacks); + + // Check if the reference is a tag or branch by inspecting the git repository + let refspec = { + let tag_ref = format!("refs/tags/{}", base); + let is_tag = product_repo + .find_reference(&tag_ref) + .is_ok(); + + if is_tag { + // It's a tag + format!("{}:refs/tags/{}", base_commit_oid, base) + } else { + // Assume it's a branch as default behavior + format!("{}:refs/heads/{}", base_commit_oid, base) + } + }; + + tracing::info!(refspec = refspec, "constructed push refspec"); + + fork_remote + .push(&[&refspec], Some(&mut push_options)) + .context(PushToForkSnafu { + url: fork_url.clone(), + refspec: &refspec, + commit: base_commit_oid, + })?; + + product_repo.remote_delete(temp_fork_remote) + .context(DeleteRemoteSnafu { name: temp_fork_remote.to_string() })?; + + tracing::info!("successfully pushed base ref to fork"); + + (base_commit_oid, fork_url) + } else { + (repo::resolve_and_fetch_commitish(&product_repo, &base, &upstream).context(FetchBaseCommitSnafu)?, upstream) + }; + tracing::info!(?base, base.commit = ?base_commit, "resolved base commit"); tracing::info!("saving configuration"); diff --git a/rust/patchable/src/repo.rs b/rust/patchable/src/repo.rs index 1b41f726c..05381c73b 100644 --- a/rust/patchable/src/repo.rs +++ b/rust/patchable/src/repo.rs @@ -181,6 +181,8 @@ pub fn resolve_and_fetch_commitish( Some( FetchOptions::new() .update_fetchhead(true) + // download_tags is needed to later determine whether `commitish` is a tag or not + .download_tags(git2::AutotagOption::Auto) .remote_callbacks(callbacks) // TODO: could be 1, CLI option maybe? .depth(0), From b863cb8dc8938e1c1eeae6e48dbaeae06e8a86d1 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Wed, 30 Apr 2025 14:13:31 +0200 Subject: [PATCH 02/30] fix: don't use fetchhead / update libgit / migrate patchable.toml files --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- druid/stackable/patches/30.0.0/patchable.toml | 2 +- druid/stackable/patches/30.0.1/patchable.toml | 2 +- druid/stackable/patches/31.0.1/patchable.toml | 2 +- hadoop/stackable/patches/3.3.4/patchable.toml | 2 +- hadoop/stackable/patches/3.3.6/patchable.toml | 2 +- hadoop/stackable/patches/3.4.0/patchable.toml | 2 +- hadoop/stackable/patches/3.4.1/patchable.toml | 2 +- hbase/stackable/patches/2.4.18/patchable.toml | 2 +- hbase/stackable/patches/2.6.0/patchable.toml | 2 +- hbase/stackable/patches/2.6.1/patchable.toml | 2 +- hive/stackable/patches/3.1.3/patchable.toml | 2 +- hive/stackable/patches/4.0.0/patchable.toml | 2 +- hive/stackable/patches/4.0.1/patchable.toml | 2 +- kafka/stackable/patches/3.7.1/patchable.toml | 2 +- kafka/stackable/patches/3.7.2/patchable.toml | 2 +- kafka/stackable/patches/3.8.0/patchable.toml | 2 +- kafka/stackable/patches/3.9.0/patchable.toml | 2 +- nifi/stackable/patches/1.27.0/patchable.toml | 2 +- nifi/stackable/patches/1.28.1/patchable.toml | 2 +- nifi/stackable/patches/2.2.0/patchable.toml | 2 +- omid/stackable/patches/1.1.0/patchable.toml | 2 +- omid/stackable/patches/1.1.1/patchable.toml | 2 +- omid/stackable/patches/1.1.2/patchable.toml | 2 +- rust/patchable/src/main.rs | 3 +-- rust/patchable/src/repo.rs | 7 +++---- spark-k8s/stackable/patches/3.5.2/patchable.toml | 2 +- spark-k8s/stackable/patches/3.5.5/patchable.toml | 2 +- .../stackable/patches/451/patchable.toml | 2 +- .../stackable/patches/455/patchable.toml | 2 +- .../stackable/patches/470/patchable.toml | 2 +- trino/stackable/patches/451/patchable.toml | 2 +- trino/stackable/patches/455/patchable.toml | 2 +- trino/stackable/patches/470/patchable.toml | 2 +- zookeeper/stackable/patches/3.9.2/patchable.toml | 2 +- zookeeper/stackable/patches/3.9.3/patchable.toml | 2 +- 37 files changed, 42 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 96f7f82bd..2b20caacd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -226,9 +226,9 @@ dependencies = [ [[package]] name = "git2" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fda788993cc341f69012feba8bf45c0ba4f3291fcc08e214b4d5a7332d88aff" +checksum = "5220b8ba44c68a9a7f7a7659e864dd73692e417ef0211bea133c7b74e031eeb9" dependencies = [ "bitflags", "libc", @@ -459,9 +459,9 @@ checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libgit2-sys" -version = "0.18.0+1.9.0" +version = "0.18.1+1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1a117465e7e1597e8febea8bb0c410f1c7fb93b1e1cddf34363f8390367ffec" +checksum = "e1dcb20f84ffcdd825c7a311ae347cce604a6f084a767dec4a4929829645290e" dependencies = [ "cc", "libc", diff --git a/Cargo.toml b/Cargo.toml index 673aa5881..805dab66f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ resolver = "2" [workspace.dependencies] clap = { version = "4.5.27", features = ["derive"] } -git2 = "0.20.0" +git2 = "0.20.1" serde = { version = "1.0.217", features = ["derive"] } snafu = "0.8.5" tempfile = "3.16.0" diff --git a/druid/stackable/patches/30.0.0/patchable.toml b/druid/stackable/patches/30.0.0/patchable.toml index 892b6fab4..332a6c6f0 100644 --- a/druid/stackable/patches/30.0.0/patchable.toml +++ b/druid/stackable/patches/30.0.0/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/druid.git" +upstream = "https://github.com/stackabletech/druid.git" base = "09d36ee324747f1407705c27618b6d415c3fa8a9" diff --git a/druid/stackable/patches/30.0.1/patchable.toml b/druid/stackable/patches/30.0.1/patchable.toml index aad1cde81..3319006d0 100644 --- a/druid/stackable/patches/30.0.1/patchable.toml +++ b/druid/stackable/patches/30.0.1/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/druid.git" +upstream = "https://github.com/stackabletech/druid.git" base = "a30af7a91d528e5c3a90356a5592abc7119191c6" diff --git a/druid/stackable/patches/31.0.1/patchable.toml b/druid/stackable/patches/31.0.1/patchable.toml index 97ae47d66..cff6b5574 100644 --- a/druid/stackable/patches/31.0.1/patchable.toml +++ b/druid/stackable/patches/31.0.1/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/druid.git" +upstream = "https://github.com/stackabletech/druid.git" base = "520482cb9638e452b0553595b4f29bb397a63758" diff --git a/hadoop/stackable/patches/3.3.4/patchable.toml b/hadoop/stackable/patches/3.3.4/patchable.toml index b35894cbc..5d43579e4 100644 --- a/hadoop/stackable/patches/3.3.4/patchable.toml +++ b/hadoop/stackable/patches/3.3.4/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/hadoop.git" +upstream = "https://github.com/stackabletech/hadoop.git" base = "a585a73c3e02ac62350c136643a5e7f6095a3dbb" diff --git a/hadoop/stackable/patches/3.3.6/patchable.toml b/hadoop/stackable/patches/3.3.6/patchable.toml index 26e9adf44..a554a9151 100644 --- a/hadoop/stackable/patches/3.3.6/patchable.toml +++ b/hadoop/stackable/patches/3.3.6/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/hadoop.git" +upstream = "https://github.com/stackabletech/hadoop.git" base = "1be78238728da9266a4f88195058f08fd012bf9c" diff --git a/hadoop/stackable/patches/3.4.0/patchable.toml b/hadoop/stackable/patches/3.4.0/patchable.toml index ef364542d..962010e20 100644 --- a/hadoop/stackable/patches/3.4.0/patchable.toml +++ b/hadoop/stackable/patches/3.4.0/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/hadoop.git" +upstream = "https://github.com/stackabletech/hadoop.git" base = "bd8b77f398f626bb7791783192ee7a5dfaeec760" diff --git a/hadoop/stackable/patches/3.4.1/patchable.toml b/hadoop/stackable/patches/3.4.1/patchable.toml index 6a697d142..9cbca7ccd 100644 --- a/hadoop/stackable/patches/3.4.1/patchable.toml +++ b/hadoop/stackable/patches/3.4.1/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/hadoop.git" +upstream = "https://github.com/stackabletech/hadoop.git" base = "4d7825309348956336b8f06a08322b78422849b1" diff --git a/hbase/stackable/patches/2.4.18/patchable.toml b/hbase/stackable/patches/2.4.18/patchable.toml index e8de3270a..5bb97fac2 100644 --- a/hbase/stackable/patches/2.4.18/patchable.toml +++ b/hbase/stackable/patches/2.4.18/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/hbase.git" +upstream = "https://github.com/stackabletech/hbase.git" base = "a1767f4d76859c0068720a6c1e5cb78282ebfe1e" diff --git a/hbase/stackable/patches/2.6.0/patchable.toml b/hbase/stackable/patches/2.6.0/patchable.toml index 0e7f1956f..3740ace29 100644 --- a/hbase/stackable/patches/2.6.0/patchable.toml +++ b/hbase/stackable/patches/2.6.0/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/hbase.git" +upstream = "https://github.com/stackabletech/hbase.git" base = "de99f8754135ea69adc39da48d2bc2b2710a5366" diff --git a/hbase/stackable/patches/2.6.1/patchable.toml b/hbase/stackable/patches/2.6.1/patchable.toml index 4a7b15c36..9c592290a 100644 --- a/hbase/stackable/patches/2.6.1/patchable.toml +++ b/hbase/stackable/patches/2.6.1/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/hbase.git" +upstream = "https://github.com/stackabletech/hbase.git" base = "7ed50b4dd742269a78875fb32112215f831284ff" diff --git a/hive/stackable/patches/3.1.3/patchable.toml b/hive/stackable/patches/3.1.3/patchable.toml index bf3ab1b59..d6fc31da4 100644 --- a/hive/stackable/patches/3.1.3/patchable.toml +++ b/hive/stackable/patches/3.1.3/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/hive.git" +upstream = "https://github.com/stackabletech/hive.git" base = "4df4d75bf1e16fe0af75aad0b4179c34c07fc975" diff --git a/hive/stackable/patches/4.0.0/patchable.toml b/hive/stackable/patches/4.0.0/patchable.toml index 7e70d2b90..9c6945d34 100644 --- a/hive/stackable/patches/4.0.0/patchable.toml +++ b/hive/stackable/patches/4.0.0/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/hive.git" +upstream = "https://github.com/stackabletech/hive.git" base = "183f8cb41d3dbed961ffd27999876468ff06690c" diff --git a/hive/stackable/patches/4.0.1/patchable.toml b/hive/stackable/patches/4.0.1/patchable.toml index 37091bbe1..12c4b72a5 100644 --- a/hive/stackable/patches/4.0.1/patchable.toml +++ b/hive/stackable/patches/4.0.1/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/hive.git" +upstream = "https://github.com/stackabletech/hive.git" base = "3af4517eb8cfd9407ad34ed78a0b48b57dfaa264" diff --git a/kafka/stackable/patches/3.7.1/patchable.toml b/kafka/stackable/patches/3.7.1/patchable.toml index 55027684d..957ac7760 100644 --- a/kafka/stackable/patches/3.7.1/patchable.toml +++ b/kafka/stackable/patches/3.7.1/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/kafka.git" +upstream = "https://github.com/stackabletech/kafka.git" base = "e2494e6ffb89f8288ed2aeb9b5596c755210bffd" diff --git a/kafka/stackable/patches/3.7.2/patchable.toml b/kafka/stackable/patches/3.7.2/patchable.toml index 250b89189..90b71cc4b 100644 --- a/kafka/stackable/patches/3.7.2/patchable.toml +++ b/kafka/stackable/patches/3.7.2/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/kafka.git" +upstream = "https://github.com/stackabletech/kafka.git" base = "79a8f2b5f44f9d5a6867190d1dfc463d08d60b82" diff --git a/kafka/stackable/patches/3.8.0/patchable.toml b/kafka/stackable/patches/3.8.0/patchable.toml index e24b68512..e90a996c5 100644 --- a/kafka/stackable/patches/3.8.0/patchable.toml +++ b/kafka/stackable/patches/3.8.0/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/kafka.git" +upstream = "https://github.com/stackabletech/kafka.git" base = "771b9576b00ecf5b64ab6e8bedf04156fbdb5cd6" diff --git a/kafka/stackable/patches/3.9.0/patchable.toml b/kafka/stackable/patches/3.9.0/patchable.toml index df24a8ca7..af3c25332 100644 --- a/kafka/stackable/patches/3.9.0/patchable.toml +++ b/kafka/stackable/patches/3.9.0/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/kafka.git" +upstream = "https://github.com/stackabletech/kafka.git" base = "84caaa6e9da06435411510a81fa321d4f99c351f" diff --git a/nifi/stackable/patches/1.27.0/patchable.toml b/nifi/stackable/patches/1.27.0/patchable.toml index 15e4563bc..f84acfd67 100644 --- a/nifi/stackable/patches/1.27.0/patchable.toml +++ b/nifi/stackable/patches/1.27.0/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/nifi.git" +upstream = "https://github.com/stackabletech/nifi.git" base = "e0c4461d90bd4f6e5f2b81765bcff5cd97ed3e18" diff --git a/nifi/stackable/patches/1.28.1/patchable.toml b/nifi/stackable/patches/1.28.1/patchable.toml index 5c9d210be..6fff40a8d 100644 --- a/nifi/stackable/patches/1.28.1/patchable.toml +++ b/nifi/stackable/patches/1.28.1/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/nifi.git" +upstream = "https://github.com/stackabletech/nifi.git" base = "883338fe28883733417d10f6ffa9319e75f5ea06" diff --git a/nifi/stackable/patches/2.2.0/patchable.toml b/nifi/stackable/patches/2.2.0/patchable.toml index 004d95779..306243ed1 100644 --- a/nifi/stackable/patches/2.2.0/patchable.toml +++ b/nifi/stackable/patches/2.2.0/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/nifi.git" +upstream = "https://github.com/stackabletech/nifi.git" base = "b33ffac8aa10992482f7fa54e6cfccc46a5e8e27" diff --git a/omid/stackable/patches/1.1.0/patchable.toml b/omid/stackable/patches/1.1.0/patchable.toml index 7441631bd..0f345e124 100644 --- a/omid/stackable/patches/1.1.0/patchable.toml +++ b/omid/stackable/patches/1.1.0/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/phoenix-omid.git" +upstream = "https://github.com/stackabletech/phoenix-omid.git" base = "3b9e16b7537adbc90a7403507fb8aabd8d1fab0c" diff --git a/omid/stackable/patches/1.1.1/patchable.toml b/omid/stackable/patches/1.1.1/patchable.toml index 00d6fd49f..927e1e710 100644 --- a/omid/stackable/patches/1.1.1/patchable.toml +++ b/omid/stackable/patches/1.1.1/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/phoenix-omid.git" +upstream = "https://github.com/stackabletech/phoenix-omid.git" base = "cd546d58d93f380fec9bf65dbfa618f53493f662" diff --git a/omid/stackable/patches/1.1.2/patchable.toml b/omid/stackable/patches/1.1.2/patchable.toml index 1cc068888..747d96a8f 100644 --- a/omid/stackable/patches/1.1.2/patchable.toml +++ b/omid/stackable/patches/1.1.2/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/phoenix-omid.git" +upstream = "https://github.com/stackabletech/phoenix-omid.git" base = "88812c9e127063f3b3016262f81ea3e8b48ec157" diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index ec25db7ba..7540be651 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -309,7 +309,7 @@ fn main() -> Result<()> { let base_commit = repo::resolve_and_fetch_commitish( &product_repo, &config.base.to_string(), - &config.upstream + &config.upstream, ) .context(FetchBaseCommitSnafu)?; let base_branch = ctx.base_branch(); @@ -501,7 +501,6 @@ fn main() -> Result<()> { .is_ok(); if is_tag { - // It's a tag format!("{}:refs/tags/{}", base_commit_oid, base) } else { // Assume it's a branch as default behavior diff --git a/rust/patchable/src/repo.rs b/rust/patchable/src/repo.rs index 05381c73b..1b7684fb0 100644 --- a/rust/patchable/src/repo.rs +++ b/rust/patchable/src/repo.rs @@ -180,8 +180,8 @@ pub fn resolve_and_fetch_commitish( &[commitish], Some( FetchOptions::new() - .update_fetchhead(true) - // download_tags is needed to later determine whether `commitish` is a tag or not + // Tags need to be present to later determine whether `commitish` is a tag or not + // Patchable needs to know this when initializing a repository with the `--forked` option (to construct the refspec) .download_tags(git2::AutotagOption::Auto) .remote_callbacks(callbacks) // TODO: could be 1, CLI option maybe? @@ -195,8 +195,7 @@ pub fn resolve_and_fetch_commitish( refs: vec![commitish.to_string()], })?; tracing::info!("fetched base commit"); - // FETCH_HEAD is written by Remote::fetch to be the last reference fetched - repo.revparse_single("FETCH_HEAD") + repo.revparse_single(commitish) .and_then(|obj| obj.peel_to_commit()) } Err(err) => Err(err), diff --git a/spark-k8s/stackable/patches/3.5.2/patchable.toml b/spark-k8s/stackable/patches/3.5.2/patchable.toml index a8a860258..9d9cea066 100644 --- a/spark-k8s/stackable/patches/3.5.2/patchable.toml +++ b/spark-k8s/stackable/patches/3.5.2/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/spark.git" +upstream = "https://github.com/stackabletech/spark.git" base = "bb7846dd487f259994fdc69e18e03382e3f64f42" diff --git a/spark-k8s/stackable/patches/3.5.5/patchable.toml b/spark-k8s/stackable/patches/3.5.5/patchable.toml index 1e901f3f3..7edfe76a8 100644 --- a/spark-k8s/stackable/patches/3.5.5/patchable.toml +++ b/spark-k8s/stackable/patches/3.5.5/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/spark.git" +upstream = "https://github.com/stackabletech/spark.git" base = "7c29c664cdc9321205a98a14858aaf8daaa19db2" diff --git a/trino-storage-connector/stackable/patches/451/patchable.toml b/trino-storage-connector/stackable/patches/451/patchable.toml index eed17e752..877a2e351 100644 --- a/trino-storage-connector/stackable/patches/451/patchable.toml +++ b/trino-storage-connector/stackable/patches/451/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/snowlift/trino-storage.git" +upstream = "https://github.com/stackabletech/trino-storage.git" base = "b6e5825bb84a4f1a3f89ff45ea39ce349313f60a" diff --git a/trino-storage-connector/stackable/patches/455/patchable.toml b/trino-storage-connector/stackable/patches/455/patchable.toml index 094a7ef26..42c00d742 100644 --- a/trino-storage-connector/stackable/patches/455/patchable.toml +++ b/trino-storage-connector/stackable/patches/455/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/snowlift/trino-storage.git" +upstream = "https://github.com/stackabletech/trino-storage.git" base = "869a735d8be527117a19150e161ad8ca69317578" diff --git a/trino-storage-connector/stackable/patches/470/patchable.toml b/trino-storage-connector/stackable/patches/470/patchable.toml index d5ec973f5..1c57129a2 100644 --- a/trino-storage-connector/stackable/patches/470/patchable.toml +++ b/trino-storage-connector/stackable/patches/470/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/snowlift/trino-storage.git" +upstream = "https://github.com/stackabletech/trino-storage.git" base = "1b25d617940f14a844a43ee34aa705f7d11fbaf9" diff --git a/trino/stackable/patches/451/patchable.toml b/trino/stackable/patches/451/patchable.toml index 1bb3feba6..e0552160d 100644 --- a/trino/stackable/patches/451/patchable.toml +++ b/trino/stackable/patches/451/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/trinodb/trino.git" +upstream = "https://github.com/stackabletech/trino.git" base = "2c974f7cb1d71e1f9f466941a317190a474fc432" diff --git a/trino/stackable/patches/455/patchable.toml b/trino/stackable/patches/455/patchable.toml index 39118758d..3157fa830 100644 --- a/trino/stackable/patches/455/patchable.toml +++ b/trino/stackable/patches/455/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/trinodb/trino.git" +upstream = "https://github.com/stackabletech/trino.git" base = "e212460ea0aa663f0de9b16fecd480c4ad6490cc" diff --git a/trino/stackable/patches/470/patchable.toml b/trino/stackable/patches/470/patchable.toml index 42b04bed4..c3e7d7a5d 100644 --- a/trino/stackable/patches/470/patchable.toml +++ b/trino/stackable/patches/470/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/trinodb/trino.git" +upstream = "https://github.com/stackabletech/trino.git" base = "05bc059cf0c9263e4ee8be2c1ad69753d0dd4faf" diff --git a/zookeeper/stackable/patches/3.9.2/patchable.toml b/zookeeper/stackable/patches/3.9.2/patchable.toml index f3ebf6062..e25ac35f1 100644 --- a/zookeeper/stackable/patches/3.9.2/patchable.toml +++ b/zookeeper/stackable/patches/3.9.2/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/zookeeper.git" +upstream = "https://github.com/stackabletech/zookeeper.git" base = "e454e8c7283100c7caec6dcae2bc82aaecb63023" diff --git a/zookeeper/stackable/patches/3.9.3/patchable.toml b/zookeeper/stackable/patches/3.9.3/patchable.toml index dd391da25..a8b20bb24 100644 --- a/zookeeper/stackable/patches/3.9.3/patchable.toml +++ b/zookeeper/stackable/patches/3.9.3/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/zookeeper.git" +upstream = "https://github.com/stackabletech/zookeeper.git" base = "c26634f34490bb0ea7a09cc51e05ede3b4e320ee" From 9c85873c7504dee12f048e3336f5f3c529a723bb Mon Sep 17 00:00:00 2001 From: dervoeti Date: Wed, 30 Apr 2025 16:30:56 +0200 Subject: [PATCH 03/30] chore: replace "forking" with "mirroring" --- rust/patchable/src/main.rs | 50 +++++++++++++++++++------------------- rust/patchable/src/repo.rs | 2 +- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index 7540be651..5a670bedd 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -150,10 +150,10 @@ enum Cmd { #[clap(long)] base: String, - /// Assume a fork exists at stackabletech/ and push the base ref to it. - /// The fork URL will be stored in patchable.toml instead of the original upstream. + /// Assume a mirror exists at stackabletech/ and push the base ref to it. + /// The mirror URL will be stored in patchable.toml instead of the original upstream. #[clap(long)] - forked: bool, + mirrored: bool, }, /// Shows the patch directory for a given product version @@ -204,10 +204,10 @@ pub enum Error { #[snafu(display("failed to parse upstream URL {url:?} to extract repository name"))] ParseUpstreamUrl { url: String }, - #[snafu(display("failed to add temporary fork remote for {url:?}"))] - AddForkRemote { source: git2::Error, url: String }, - #[snafu(display("failed to push commit {commit} (as {refspec}) to fork {url:?}"))] - PushToFork { + #[snafu(display("failed to add temporary mirror remote for {url:?}"))] + AddMirrorRemote { source: git2::Error, url: String }, + #[snafu(display("failed to push commit {commit} (as {refspec}) to mirror {url:?}"))] + PushToMirror { source: git2::Error, url: String, refspec: String, @@ -423,7 +423,7 @@ fn main() -> Result<()> { pv, upstream, base, - forked, + mirrored, } => { let ctx = ProductVersionContext { pv, @@ -442,14 +442,14 @@ fn main() -> Result<()> { // so that it cannot be changed under our feet (without us knowing so, anyway...). tracing::info!(?base, "resolving base commit-ish"); - let (base_commit, upstream) = if forked { + let (base_commit, upstream) = if mirrored { // Parse e.g. "https://github.com/apache/druid.git" into "druid" let repo_name = upstream.split('/').last().map(|repo| repo.trim_end_matches(".git")).context(ParseUpstreamUrlSnafu { url: &upstream })?; ensure!(!repo_name.is_empty(), ParseUpstreamUrlSnafu { url: &upstream }); - let fork_url = format!("https://github.com/stackabletech/{}.git", repo_name); - tracing::info!(%fork_url, "using fork repository"); + let mirror_url = format!("https://github.com/stackabletech/{}.git", repo_name); + tracing::info!(%mirror_url, "using mirror repository"); // Fetch from original upstream using a temporary remote name tracing::info!(upstream = upstream, %base, "fetching base ref from original upstream"); @@ -462,14 +462,14 @@ fn main() -> Result<()> { tracing::info!(commit = %base_commit_oid, "fetched base commit OID"); - // Add fork remote - let temp_fork_remote = "patchable_fork_push"; - let mut fork_remote = product_repo - .remote(temp_fork_remote, &fork_url) - .context(AddForkRemoteSnafu { url: fork_url.clone() })?; + // Add mirror remote + let temp_mirror_remote = "patchable_mirror_push"; + let mut mirror_remote = product_repo + .remote(temp_mirror_remote, &mirror_url) + .context(AddMirrorRemoteSnafu { url: mirror_url.clone() })?; - // Push the base commit to the fork - tracing::info!(commit = %base_commit_oid, base = base, url = fork_url, "pushing commit to fork"); + // Push the base commit to the mirror + tracing::info!(commit = %base_commit_oid, base = base, url = mirror_url, "pushing commit to mirror"); let mut callbacks = git2::RemoteCallbacks::new(); callbacks.credentials(|_url, username_from_url, _allowed_types| { git2::Cred::credential_helper( @@ -510,20 +510,20 @@ fn main() -> Result<()> { tracing::info!(refspec = refspec, "constructed push refspec"); - fork_remote + mirror_remote .push(&[&refspec], Some(&mut push_options)) - .context(PushToForkSnafu { - url: fork_url.clone(), + .context(PushToMirrorSnafu { + url: mirror_url.clone(), refspec: &refspec, commit: base_commit_oid, })?; - product_repo.remote_delete(temp_fork_remote) - .context(DeleteRemoteSnafu { name: temp_fork_remote.to_string() })?; + product_repo.remote_delete(temp_mirror_remote) + .context(DeleteRemoteSnafu { name: temp_mirror_remote.to_string() })?; - tracing::info!("successfully pushed base ref to fork"); + tracing::info!("successfully pushed base ref to mirror"); - (base_commit_oid, fork_url) + (base_commit_oid, mirror_url) } else { (repo::resolve_and_fetch_commitish(&product_repo, &base, &upstream).context(FetchBaseCommitSnafu)?, upstream) }; diff --git a/rust/patchable/src/repo.rs b/rust/patchable/src/repo.rs index 1b7684fb0..02f376c91 100644 --- a/rust/patchable/src/repo.rs +++ b/rust/patchable/src/repo.rs @@ -181,7 +181,7 @@ pub fn resolve_and_fetch_commitish( Some( FetchOptions::new() // Tags need to be present to later determine whether `commitish` is a tag or not - // Patchable needs to know this when initializing a repository with the `--forked` option (to construct the refspec) + // Patchable needs to know this when initializing a repository with the `--mirrored` option (to construct the refspec) .download_tags(git2::AutotagOption::Auto) .remote_callbacks(callbacks) // TODO: could be 1, CLI option maybe? From cf27542fcae44d5ebe95289edef42f4f23db23fc Mon Sep 17 00:00:00 2001 From: dervoeti Date: Wed, 30 Apr 2025 18:35:41 +0200 Subject: [PATCH 04/30] feat: keep original upstream / anonymous remote --- druid/stackable/patches/30.0.0/patchable.toml | 3 +- druid/stackable/patches/30.0.1/patchable.toml | 3 +- druid/stackable/patches/31.0.1/patchable.toml | 3 +- hadoop/stackable/patches/3.3.4/patchable.toml | 3 +- hadoop/stackable/patches/3.3.6/patchable.toml | 3 +- hadoop/stackable/patches/3.4.0/patchable.toml | 3 +- hadoop/stackable/patches/3.4.1/patchable.toml | 3 +- .../stackable/patches/1.2.0/patchable.toml | 1 + .../patches/1.3.0-fd5a5fb/patchable.toml | 1 + .../stackable/patches/5.2.1/patchable.toml | 1 + hbase/stackable/patches/2.4.18/patchable.toml | 3 +- hbase/stackable/patches/2.6.0/patchable.toml | 3 +- hbase/stackable/patches/2.6.1/patchable.toml | 3 +- hive/stackable/patches/3.1.3/patchable.toml | 3 +- hive/stackable/patches/4.0.0/patchable.toml | 3 +- hive/stackable/patches/4.0.1/patchable.toml | 3 +- kafka/stackable/patches/3.7.1/patchable.toml | 3 +- kafka/stackable/patches/3.7.2/patchable.toml | 3 +- kafka/stackable/patches/3.8.0/patchable.toml | 3 +- kafka/stackable/patches/3.9.0/patchable.toml | 3 +- nifi/stackable/patches/1.27.0/patchable.toml | 3 +- nifi/stackable/patches/1.28.1/patchable.toml | 3 +- nifi/stackable/patches/2.2.0/patchable.toml | 3 +- omid/stackable/patches/1.1.0/patchable.toml | 3 +- omid/stackable/patches/1.1.1/patchable.toml | 3 +- omid/stackable/patches/1.1.2/patchable.toml | 3 +- .../patches/1.1.3-SNAPSHOT/patchable.toml | 1 + rust/patchable/src/main.rs | 42 ++++++------------- .../stackable/patches/1.0.1/patchable.toml | 1 + .../stackable/patches/3.5.2/patchable.toml | 3 +- .../stackable/patches/3.5.5/patchable.toml | 3 +- .../stackable/patches/451/patchable.toml | 3 +- .../stackable/patches/455/patchable.toml | 3 +- .../stackable/patches/470/patchable.toml | 3 +- trino/stackable/patches/451/patchable.toml | 3 +- trino/stackable/patches/455/patchable.toml | 3 +- trino/stackable/patches/470/patchable.toml | 3 +- .../stackable/patches/3.9.2/patchable.toml | 3 +- .../stackable/patches/3.9.3/patchable.toml | 3 +- 39 files changed, 83 insertions(+), 63 deletions(-) diff --git a/druid/stackable/patches/30.0.0/patchable.toml b/druid/stackable/patches/30.0.0/patchable.toml index 332a6c6f0..7b887616d 100644 --- a/druid/stackable/patches/30.0.0/patchable.toml +++ b/druid/stackable/patches/30.0.0/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/druid.git" +upstream = "https://github.com/apache/druid.git" base = "09d36ee324747f1407705c27618b6d415c3fa8a9" +mirror = "https://github.com/stackabletech/druid.git" diff --git a/druid/stackable/patches/30.0.1/patchable.toml b/druid/stackable/patches/30.0.1/patchable.toml index 3319006d0..bdf04c5b8 100644 --- a/druid/stackable/patches/30.0.1/patchable.toml +++ b/druid/stackable/patches/30.0.1/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/druid.git" +upstream = "https://github.com/apache/druid.git" base = "a30af7a91d528e5c3a90356a5592abc7119191c6" +mirror = "https://github.com/stackabletech/druid.git" diff --git a/druid/stackable/patches/31.0.1/patchable.toml b/druid/stackable/patches/31.0.1/patchable.toml index cff6b5574..818ee9701 100644 --- a/druid/stackable/patches/31.0.1/patchable.toml +++ b/druid/stackable/patches/31.0.1/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/druid.git" +upstream = "https://github.com/apache/druid.git" base = "520482cb9638e452b0553595b4f29bb397a63758" +mirror = "https://github.com/stackabletech/druid.git" diff --git a/hadoop/stackable/patches/3.3.4/patchable.toml b/hadoop/stackable/patches/3.3.4/patchable.toml index 5d43579e4..bfda3d2cf 100644 --- a/hadoop/stackable/patches/3.3.4/patchable.toml +++ b/hadoop/stackable/patches/3.3.4/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/hadoop.git" +upstream = "https://github.com/apache/hadoop.git" base = "a585a73c3e02ac62350c136643a5e7f6095a3dbb" +mirror = "https://github.com/stackabletech/hadoop.git" diff --git a/hadoop/stackable/patches/3.3.6/patchable.toml b/hadoop/stackable/patches/3.3.6/patchable.toml index a554a9151..82226d3a4 100644 --- a/hadoop/stackable/patches/3.3.6/patchable.toml +++ b/hadoop/stackable/patches/3.3.6/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/hadoop.git" +upstream = "https://github.com/apache/hadoop.git" base = "1be78238728da9266a4f88195058f08fd012bf9c" +mirror = "https://github.com/stackabletech/hadoop.git" diff --git a/hadoop/stackable/patches/3.4.0/patchable.toml b/hadoop/stackable/patches/3.4.0/patchable.toml index 962010e20..e0cdc727d 100644 --- a/hadoop/stackable/patches/3.4.0/patchable.toml +++ b/hadoop/stackable/patches/3.4.0/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/hadoop.git" +upstream = "https://github.com/apache/hadoop.git" base = "bd8b77f398f626bb7791783192ee7a5dfaeec760" +mirror = "https://github.com/stackabletech/hadoop.git" diff --git a/hadoop/stackable/patches/3.4.1/patchable.toml b/hadoop/stackable/patches/3.4.1/patchable.toml index 9cbca7ccd..73e216571 100644 --- a/hadoop/stackable/patches/3.4.1/patchable.toml +++ b/hadoop/stackable/patches/3.4.1/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/hadoop.git" +upstream = "https://github.com/apache/hadoop.git" base = "4d7825309348956336b8f06a08322b78422849b1" +mirror = "https://github.com/stackabletech/hadoop.git" diff --git a/hbase/hbase-operator-tools/stackable/patches/1.2.0/patchable.toml b/hbase/hbase-operator-tools/stackable/patches/1.2.0/patchable.toml index 312b144c1..203a00d34 100644 --- a/hbase/hbase-operator-tools/stackable/patches/1.2.0/patchable.toml +++ b/hbase/hbase-operator-tools/stackable/patches/1.2.0/patchable.toml @@ -1,2 +1,3 @@ upstream = "https://github.com/apache/hbase-operator-tools.git" base = "478af00af79f82624264fd2bb447b97fecc8e790" +mirror = "https://github.com/stackabletech/hbase-operator-tools.git" diff --git a/hbase/hbase-operator-tools/stackable/patches/1.3.0-fd5a5fb/patchable.toml b/hbase/hbase-operator-tools/stackable/patches/1.3.0-fd5a5fb/patchable.toml index 165a2494e..290988bad 100644 --- a/hbase/hbase-operator-tools/stackable/patches/1.3.0-fd5a5fb/patchable.toml +++ b/hbase/hbase-operator-tools/stackable/patches/1.3.0-fd5a5fb/patchable.toml @@ -1,2 +1,3 @@ upstream = "https://github.com/apache/hbase-operator-tools.git" base = "fd5a5fb90755949a90c502c76de8313130403fa3" +mirror = "https://github.com/stackabletech/hbase-operator-tools.git" diff --git a/hbase/phoenix/stackable/patches/5.2.1/patchable.toml b/hbase/phoenix/stackable/patches/5.2.1/patchable.toml index fd13de0cb..a46992214 100644 --- a/hbase/phoenix/stackable/patches/5.2.1/patchable.toml +++ b/hbase/phoenix/stackable/patches/5.2.1/patchable.toml @@ -1,2 +1,3 @@ upstream = "https://github.com/apache/phoenix.git" base = "b738d66cb5863b759bb98eaa417b3b5731d41f95" +mirror = "https://github.com/stackabletech/phoenix.git" diff --git a/hbase/stackable/patches/2.4.18/patchable.toml b/hbase/stackable/patches/2.4.18/patchable.toml index 5bb97fac2..b8fcf6638 100644 --- a/hbase/stackable/patches/2.4.18/patchable.toml +++ b/hbase/stackable/patches/2.4.18/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/hbase.git" +upstream = "https://github.com/apache/hbase.git" base = "a1767f4d76859c0068720a6c1e5cb78282ebfe1e" +mirror = "https://github.com/stackabletech/hbase.git" diff --git a/hbase/stackable/patches/2.6.0/patchable.toml b/hbase/stackable/patches/2.6.0/patchable.toml index 3740ace29..c611ac76e 100644 --- a/hbase/stackable/patches/2.6.0/patchable.toml +++ b/hbase/stackable/patches/2.6.0/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/hbase.git" +upstream = "https://github.com/apache/hbase.git" base = "de99f8754135ea69adc39da48d2bc2b2710a5366" +mirror = "https://github.com/stackabletech/hbase.git" diff --git a/hbase/stackable/patches/2.6.1/patchable.toml b/hbase/stackable/patches/2.6.1/patchable.toml index 9c592290a..509985a8d 100644 --- a/hbase/stackable/patches/2.6.1/patchable.toml +++ b/hbase/stackable/patches/2.6.1/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/hbase.git" +upstream = "https://github.com/apache/hbase.git" base = "7ed50b4dd742269a78875fb32112215f831284ff" +mirror = "https://github.com/stackabletech/hbase.git" diff --git a/hive/stackable/patches/3.1.3/patchable.toml b/hive/stackable/patches/3.1.3/patchable.toml index d6fc31da4..51f6afb9f 100644 --- a/hive/stackable/patches/3.1.3/patchable.toml +++ b/hive/stackable/patches/3.1.3/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/hive.git" +upstream = "https://github.com/apache/hive.git" base = "4df4d75bf1e16fe0af75aad0b4179c34c07fc975" +mirror = "https://github.com/stackabletech/hive.git" diff --git a/hive/stackable/patches/4.0.0/patchable.toml b/hive/stackable/patches/4.0.0/patchable.toml index 9c6945d34..2b3bade93 100644 --- a/hive/stackable/patches/4.0.0/patchable.toml +++ b/hive/stackable/patches/4.0.0/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/hive.git" +upstream = "https://github.com/apache/hive.git" base = "183f8cb41d3dbed961ffd27999876468ff06690c" +mirror = "https://github.com/stackabletech/hive.git" diff --git a/hive/stackable/patches/4.0.1/patchable.toml b/hive/stackable/patches/4.0.1/patchable.toml index 12c4b72a5..b6c5ec3f2 100644 --- a/hive/stackable/patches/4.0.1/patchable.toml +++ b/hive/stackable/patches/4.0.1/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/hive.git" +upstream = "https://github.com/apache/hive.git" base = "3af4517eb8cfd9407ad34ed78a0b48b57dfaa264" +mirror = "https://github.com/stackabletech/hive.git" diff --git a/kafka/stackable/patches/3.7.1/patchable.toml b/kafka/stackable/patches/3.7.1/patchable.toml index 957ac7760..587a15c16 100644 --- a/kafka/stackable/patches/3.7.1/patchable.toml +++ b/kafka/stackable/patches/3.7.1/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/kafka.git" +upstream = "https://github.com/apache/kafka.git" base = "e2494e6ffb89f8288ed2aeb9b5596c755210bffd" +mirror = "https://github.com/stackabletech/kafka.git" diff --git a/kafka/stackable/patches/3.7.2/patchable.toml b/kafka/stackable/patches/3.7.2/patchable.toml index 90b71cc4b..1550583ae 100644 --- a/kafka/stackable/patches/3.7.2/patchable.toml +++ b/kafka/stackable/patches/3.7.2/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/kafka.git" +upstream = "https://github.com/apache/kafka.git" base = "79a8f2b5f44f9d5a6867190d1dfc463d08d60b82" +mirror = "https://github.com/stackabletech/kafka.git" diff --git a/kafka/stackable/patches/3.8.0/patchable.toml b/kafka/stackable/patches/3.8.0/patchable.toml index e90a996c5..ce0868779 100644 --- a/kafka/stackable/patches/3.8.0/patchable.toml +++ b/kafka/stackable/patches/3.8.0/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/kafka.git" +upstream = "https://github.com/apache/kafka.git" base = "771b9576b00ecf5b64ab6e8bedf04156fbdb5cd6" +mirror = "https://github.com/stackabletech/kafka.git" diff --git a/kafka/stackable/patches/3.9.0/patchable.toml b/kafka/stackable/patches/3.9.0/patchable.toml index af3c25332..0766f2c0b 100644 --- a/kafka/stackable/patches/3.9.0/patchable.toml +++ b/kafka/stackable/patches/3.9.0/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/kafka.git" +upstream = "https://github.com/apache/kafka.git" base = "84caaa6e9da06435411510a81fa321d4f99c351f" +mirror = "https://github.com/stackabletech/kafka.git" diff --git a/nifi/stackable/patches/1.27.0/patchable.toml b/nifi/stackable/patches/1.27.0/patchable.toml index f84acfd67..bd970aeb2 100644 --- a/nifi/stackable/patches/1.27.0/patchable.toml +++ b/nifi/stackable/patches/1.27.0/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/nifi.git" +upstream = "https://github.com/apache/nifi.git" base = "e0c4461d90bd4f6e5f2b81765bcff5cd97ed3e18" +mirror = "https://github.com/stackabletech/nifi.git" diff --git a/nifi/stackable/patches/1.28.1/patchable.toml b/nifi/stackable/patches/1.28.1/patchable.toml index 6fff40a8d..1ffc5bb22 100644 --- a/nifi/stackable/patches/1.28.1/patchable.toml +++ b/nifi/stackable/patches/1.28.1/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/nifi.git" +upstream = "https://github.com/apache/nifi.git" base = "883338fe28883733417d10f6ffa9319e75f5ea06" +mirror = "https://github.com/stackabletech/nifi.git" diff --git a/nifi/stackable/patches/2.2.0/patchable.toml b/nifi/stackable/patches/2.2.0/patchable.toml index 306243ed1..baf837669 100644 --- a/nifi/stackable/patches/2.2.0/patchable.toml +++ b/nifi/stackable/patches/2.2.0/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/nifi.git" +upstream = "https://github.com/apache/nifi.git" base = "b33ffac8aa10992482f7fa54e6cfccc46a5e8e27" +mirror = "https://github.com/stackabletech/nifi.git" diff --git a/omid/stackable/patches/1.1.0/patchable.toml b/omid/stackable/patches/1.1.0/patchable.toml index 0f345e124..cdc7d3cec 100644 --- a/omid/stackable/patches/1.1.0/patchable.toml +++ b/omid/stackable/patches/1.1.0/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/phoenix-omid.git" +upstream = "https://github.com/apache/phoenix-omid.git" base = "3b9e16b7537adbc90a7403507fb8aabd8d1fab0c" +mirror = "https://github.com/stackabletech/phoenix-omid.git" diff --git a/omid/stackable/patches/1.1.1/patchable.toml b/omid/stackable/patches/1.1.1/patchable.toml index 927e1e710..7e9ab59d6 100644 --- a/omid/stackable/patches/1.1.1/patchable.toml +++ b/omid/stackable/patches/1.1.1/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/phoenix-omid.git" +upstream = "https://github.com/apache/phoenix-omid.git" base = "cd546d58d93f380fec9bf65dbfa618f53493f662" +mirror = "https://github.com/stackabletech/phoenix-omid.git" diff --git a/omid/stackable/patches/1.1.2/patchable.toml b/omid/stackable/patches/1.1.2/patchable.toml index 747d96a8f..08b21ed22 100644 --- a/omid/stackable/patches/1.1.2/patchable.toml +++ b/omid/stackable/patches/1.1.2/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/phoenix-omid.git" +upstream = "https://github.com/apache/phoenix-omid.git" base = "88812c9e127063f3b3016262f81ea3e8b48ec157" +mirror = "https://github.com/stackabletech/phoenix-omid.git" diff --git a/omid/stackable/patches/1.1.3-SNAPSHOT/patchable.toml b/omid/stackable/patches/1.1.3-SNAPSHOT/patchable.toml index 656044995..9fd5987f7 100644 --- a/omid/stackable/patches/1.1.3-SNAPSHOT/patchable.toml +++ b/omid/stackable/patches/1.1.3-SNAPSHOT/patchable.toml @@ -1,2 +1,3 @@ upstream = "https://github.com/apache/phoenix-omid.git" base = "c3e4da626fdb27060fd139a809e057965e52d163" +mirror = "https://github.com/stackabletech/phoenix-omid.git" diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index 5a670bedd..d07916340 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -29,6 +29,7 @@ struct ProductVersionConfig { upstream: String, #[serde(with = "utils::oid_serde")] base: Oid, + mirror: Option, } struct ProductVersionContext { @@ -213,11 +214,6 @@ pub enum Error { refspec: String, commit: Oid, }, - #[snafu(display("failed to delete remote {name}"))] - DeleteRemote { - source: git2::Error, - name: String, - }, #[snafu(display("failed to find images repository"))] FindImagesRepo { source: repo::Error }, @@ -309,7 +305,7 @@ fn main() -> Result<()> { let base_commit = repo::resolve_and_fetch_commitish( &product_repo, &config.base.to_string(), - &config.upstream, + config.mirror.as_deref().unwrap_or(&config.upstream), ) .context(FetchBaseCommitSnafu)?; let base_branch = ctx.base_branch(); @@ -441,8 +437,10 @@ fn main() -> Result<()> { // --base can be a reference, but patchable.toml should always have a resolved commit id, // so that it cannot be changed under our feet (without us knowing so, anyway...). tracing::info!(?base, "resolving base commit-ish"); + let base_commit = repo::resolve_and_fetch_commitish(&product_repo, &base, &upstream).context(FetchBaseCommitSnafu)?; + let mut upstream_mirror = None; - let (base_commit, upstream) = if mirrored { + if mirrored { // Parse e.g. "https://github.com/apache/druid.git" into "druid" let repo_name = upstream.split('/').last().map(|repo| repo.trim_end_matches(".git")).context(ParseUpstreamUrlSnafu { url: &upstream })?; @@ -451,25 +449,13 @@ fn main() -> Result<()> { let mirror_url = format!("https://github.com/stackabletech/{}.git", repo_name); tracing::info!(%mirror_url, "using mirror repository"); - // Fetch from original upstream using a temporary remote name - tracing::info!(upstream = upstream, %base, "fetching base ref from original upstream"); - let base_commit_oid = repo::resolve_and_fetch_commitish( - &product_repo, - &base, - &upstream - ) - .context(FetchBaseCommitSnafu)?; - - tracing::info!(commit = %base_commit_oid, "fetched base commit OID"); - // Add mirror remote - let temp_mirror_remote = "patchable_mirror_push"; let mut mirror_remote = product_repo - .remote(temp_mirror_remote, &mirror_url) + .remote_anonymous(&mirror_url) .context(AddMirrorRemoteSnafu { url: mirror_url.clone() })?; // Push the base commit to the mirror - tracing::info!(commit = %base_commit_oid, base = base, url = mirror_url, "pushing commit to mirror"); + tracing::info!(commit = %base_commit, base = base, url = mirror_url, "pushing commit to mirror"); let mut callbacks = git2::RemoteCallbacks::new(); callbacks.credentials(|_url, username_from_url, _allowed_types| { git2::Cred::credential_helper( @@ -501,10 +487,10 @@ fn main() -> Result<()> { .is_ok(); if is_tag { - format!("{}:refs/tags/{}", base_commit_oid, base) + format!("{}:refs/tags/{}", base_commit, base) } else { // Assume it's a branch as default behavior - format!("{}:refs/heads/{}", base_commit_oid, base) + format!("{}:refs/heads/{}", base_commit, base) } }; @@ -515,17 +501,12 @@ fn main() -> Result<()> { .context(PushToMirrorSnafu { url: mirror_url.clone(), refspec: &refspec, - commit: base_commit_oid, + commit: base_commit, })?; - product_repo.remote_delete(temp_mirror_remote) - .context(DeleteRemoteSnafu { name: temp_mirror_remote.to_string() })?; - tracing::info!("successfully pushed base ref to mirror"); - (base_commit_oid, mirror_url) - } else { - (repo::resolve_and_fetch_commitish(&product_repo, &base, &upstream).context(FetchBaseCommitSnafu)?, upstream) + upstream_mirror = Some(mirror_url); }; tracing::info!(?base, base.commit = ?base_commit, "resolved base commit"); @@ -533,6 +514,7 @@ fn main() -> Result<()> { tracing::info!("saving configuration"); let config = ProductVersionConfig { upstream, + mirror: upstream_mirror, base: base_commit, }; let config_path = ctx.config_path(); diff --git a/spark-k8s/hbase-connectors/stackable/patches/1.0.1/patchable.toml b/spark-k8s/hbase-connectors/stackable/patches/1.0.1/patchable.toml index 74242d678..698d02d46 100644 --- a/spark-k8s/hbase-connectors/stackable/patches/1.0.1/patchable.toml +++ b/spark-k8s/hbase-connectors/stackable/patches/1.0.1/patchable.toml @@ -1,2 +1,3 @@ upstream = "https://github.com/apache/hbase-connectors.git" base = "e5217d13ed729703580ff2d1b02378ada2d94f4d" +mirror = "https://github.com/stackabletech/hbase-connectors.git" diff --git a/spark-k8s/stackable/patches/3.5.2/patchable.toml b/spark-k8s/stackable/patches/3.5.2/patchable.toml index 9d9cea066..f8281c5a5 100644 --- a/spark-k8s/stackable/patches/3.5.2/patchable.toml +++ b/spark-k8s/stackable/patches/3.5.2/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/spark.git" +upstream = "https://github.com/apache/spark.git" base = "bb7846dd487f259994fdc69e18e03382e3f64f42" +mirror = "https://github.com/stackabletech/spark.git" diff --git a/spark-k8s/stackable/patches/3.5.5/patchable.toml b/spark-k8s/stackable/patches/3.5.5/patchable.toml index 7edfe76a8..baf047fcc 100644 --- a/spark-k8s/stackable/patches/3.5.5/patchable.toml +++ b/spark-k8s/stackable/patches/3.5.5/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/spark.git" +upstream = "https://github.com/apache/spark.git" base = "7c29c664cdc9321205a98a14858aaf8daaa19db2" +mirror = "https://github.com/stackabletech/spark.git" diff --git a/trino-storage-connector/stackable/patches/451/patchable.toml b/trino-storage-connector/stackable/patches/451/patchable.toml index 877a2e351..7f579e201 100644 --- a/trino-storage-connector/stackable/patches/451/patchable.toml +++ b/trino-storage-connector/stackable/patches/451/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/trino-storage.git" +upstream = "https://github.com/snowlift/trino-storage.git" base = "b6e5825bb84a4f1a3f89ff45ea39ce349313f60a" +mirror = "https://github.com/stackabletech/trino-storage.git" diff --git a/trino-storage-connector/stackable/patches/455/patchable.toml b/trino-storage-connector/stackable/patches/455/patchable.toml index 42c00d742..dee676dc3 100644 --- a/trino-storage-connector/stackable/patches/455/patchable.toml +++ b/trino-storage-connector/stackable/patches/455/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/trino-storage.git" +upstream = "https://github.com/snowlift/trino-storage.git" base = "869a735d8be527117a19150e161ad8ca69317578" +mirror = "https://github.com/stackabletech/trino-storage.git" diff --git a/trino-storage-connector/stackable/patches/470/patchable.toml b/trino-storage-connector/stackable/patches/470/patchable.toml index 1c57129a2..1fa184a4f 100644 --- a/trino-storage-connector/stackable/patches/470/patchable.toml +++ b/trino-storage-connector/stackable/patches/470/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/trino-storage.git" +upstream = "https://github.com/snowlift/trino-storage.git" base = "1b25d617940f14a844a43ee34aa705f7d11fbaf9" +mirror = "https://github.com/stackabletech/trino-storage.git" diff --git a/trino/stackable/patches/451/patchable.toml b/trino/stackable/patches/451/patchable.toml index e0552160d..67a023b9c 100644 --- a/trino/stackable/patches/451/patchable.toml +++ b/trino/stackable/patches/451/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/trino.git" +upstream = "https://github.com/trinodb/trino.git" base = "2c974f7cb1d71e1f9f466941a317190a474fc432" +mirror = "https://github.com/stackabletech/trino.git" diff --git a/trino/stackable/patches/455/patchable.toml b/trino/stackable/patches/455/patchable.toml index 3157fa830..30473fdc3 100644 --- a/trino/stackable/patches/455/patchable.toml +++ b/trino/stackable/patches/455/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/trino.git" +upstream = "https://github.com/trinodb/trino.git" base = "e212460ea0aa663f0de9b16fecd480c4ad6490cc" +mirror = "https://github.com/stackabletech/trino.git" diff --git a/trino/stackable/patches/470/patchable.toml b/trino/stackable/patches/470/patchable.toml index c3e7d7a5d..6567015ef 100644 --- a/trino/stackable/patches/470/patchable.toml +++ b/trino/stackable/patches/470/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/trino.git" +upstream = "https://github.com/trinodb/trino.git" base = "05bc059cf0c9263e4ee8be2c1ad69753d0dd4faf" +mirror = "https://github.com/stackabletech/trino.git" diff --git a/zookeeper/stackable/patches/3.9.2/patchable.toml b/zookeeper/stackable/patches/3.9.2/patchable.toml index e25ac35f1..f01d683e0 100644 --- a/zookeeper/stackable/patches/3.9.2/patchable.toml +++ b/zookeeper/stackable/patches/3.9.2/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/zookeeper.git" +upstream = "https://github.com/apache/zookeeper.git" base = "e454e8c7283100c7caec6dcae2bc82aaecb63023" +mirror = "https://github.com/stackabletech/zookeeper.git" diff --git a/zookeeper/stackable/patches/3.9.3/patchable.toml b/zookeeper/stackable/patches/3.9.3/patchable.toml index a8b20bb24..4e4e9fccd 100644 --- a/zookeeper/stackable/patches/3.9.3/patchable.toml +++ b/zookeeper/stackable/patches/3.9.3/patchable.toml @@ -1,2 +1,3 @@ -upstream = "https://github.com/stackabletech/zookeeper.git" +upstream = "https://github.com/apache/zookeeper.git" base = "c26634f34490bb0ea7a09cc51e05ede3b4e320ee" +mirror = "https://github.com/stackabletech/zookeeper.git" From 3034966ebf4feb1e465a375804bfd22fb8e7960b Mon Sep 17 00:00:00 2001 From: dervoeti Date: Fri, 2 May 2025 12:53:43 +0200 Subject: [PATCH 05/30] refactor: move progress tracking setup into utils function --- rust/patchable/src/main.rs | 13 +++++++------ rust/patchable/src/repo.rs | 16 ++++++---------- rust/patchable/src/utils.rs | 18 +++++++++++------- 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index d07916340..d3f066f8e 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -10,7 +10,7 @@ use std::{fs::File, io::Write, path::PathBuf}; use git2::{Oid, Repository}; use serde::{Deserialize, Serialize}; use snafu::{ensure, OptionExt, ResultExt as _, Snafu}; -use tracing_indicatif::{span_ext::IndicatifSpanExt, IndicatifLayer}; +use tracing_indicatif::IndicatifLayer; use tracing_subscriber::{layer::SubscriberExt as _, util::SubscriberInitExt as _}; #[derive(clap::Parser)] @@ -466,13 +466,14 @@ fn main() -> Result<()> { }); // Add progress tracking for push operation - let span_push = tracing::info_span!("pushing"); - span_push.pb_set_style(&utils::progress_bar_style()); - let _ = span_push.enter(); - let mut quant_push = utils::Quantizer::percent(); + let (span_push, mut quant_push) = utils::setup_progress_tracking(tracing::info_span!("pushing")); callbacks.push_transfer_progress(move |current, total, _| { if total > 0 { - quant_push.update_span_progress(current, total, &span_push); + quant_push.update_span_progress( + current, + total, + &span_push, + ); } }); diff --git a/rust/patchable/src/repo.rs b/rust/patchable/src/repo.rs index 02f376c91..fe7da752b 100644 --- a/rust/patchable/src/repo.rs +++ b/rust/patchable/src/repo.rs @@ -5,11 +5,10 @@ use git2::{ WorktreeAddOptions, }; use snafu::{ResultExt, Snafu}; -use tracing_indicatif::span_ext::IndicatifSpanExt; use crate::{ error::{self, CommitRef}, - utils::{progress_bar_style, Quantizer}, + utils::setup_progress_tracking, }; #[derive(Debug, Snafu)] @@ -149,15 +148,11 @@ pub fn resolve_and_fetch_commitish( error = &err as &dyn std::error::Error, "base commit not found locally, fetching from upstream" ); - let span_recv = tracing::info_span!("receiving"); - let span_index = tracing::info_span!("indexing"); - span_recv.pb_set_style(&progress_bar_style()); - span_index.pb_set_style(&progress_bar_style()); - let _ = span_recv.enter(); - let _ = span_index.enter(); + + let (span_recv, mut quant_recv) = setup_progress_tracking(tracing::info_span!("receiving")); + let (span_index, mut quant_index) = setup_progress_tracking(tracing::info_span!("indexing")); + let mut callbacks = RemoteCallbacks::new(); - let mut quant_recv = Quantizer::percent(); - let mut quant_index = Quantizer::percent(); callbacks.transfer_progress(move |progress| { quant_recv.update_span_progress( progress.received_objects(), @@ -171,6 +166,7 @@ pub fn resolve_and_fetch_commitish( ); true }); + repo.remote_anonymous(upstream_url) .context(CreateRemoteSnafu { repo, diff --git a/rust/patchable/src/utils.rs b/rust/patchable/src/utils.rs index 9d08666d1..22cca39ba 100644 --- a/rust/patchable/src/utils.rs +++ b/rust/patchable/src/utils.rs @@ -4,13 +4,6 @@ use git2::Repository; use tracing::Span; use tracing_indicatif::{span_ext::IndicatifSpanExt, style::ProgressStyle}; -pub fn progress_bar_style() -> ProgressStyle { - ProgressStyle::with_template( - "{span_child_prefix}{spinner} {span_name}{{{span_fields}}} {wide_msg} {bar:40} {percent:>3}%", - ) - .expect("hard-coded template should be valid") -} - /// Runs a function whenever a `value` changes "enough". /// /// See [`Self::update`], and especially [`Self::update_span_progress`]. @@ -94,3 +87,14 @@ pub mod oid_serde { .and_then(|oid| Oid::from_str(&oid).map_err(::custom)) } } + +/// Sets up progress tracking for Git operations with a progress bar. +pub fn setup_progress_tracking(span: tracing::Span) -> (tracing::Span, Quantizer) { + span.pb_set_style(&ProgressStyle::with_template( + "{span_child_prefix}{spinner} {span_name}{{{span_fields}}} {wide_msg} {bar:40} {percent:>3}%", + ) + .expect("hard-coded template should be valid")); + let _ = span.enter(); + let quantizer = Quantizer::percent(); + (span, quantizer) +} From 42e9cb47761a80e116957e1f95ae59502179006d Mon Sep 17 00:00:00 2001 From: dervoeti Date: Mon, 5 May 2025 15:02:35 +0200 Subject: [PATCH 06/30] feat: always use tags in our mirror repos --- rust/patchable/src/main.rs | 16 +--------------- rust/patchable/src/repo.rs | 7 +++---- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index d3f066f8e..8b507d8af 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -480,21 +480,7 @@ fn main() -> Result<()> { let mut push_options = git2::PushOptions::new(); push_options.remote_callbacks(callbacks); - // Check if the reference is a tag or branch by inspecting the git repository - let refspec = { - let tag_ref = format!("refs/tags/{}", base); - let is_tag = product_repo - .find_reference(&tag_ref) - .is_ok(); - - if is_tag { - format!("{}:refs/tags/{}", base_commit, base) - } else { - // Assume it's a branch as default behavior - format!("{}:refs/heads/{}", base_commit, base) - } - }; - + let refspec = format!("{}:refs/tags/{}", base_commit, base); tracing::info!(refspec = refspec, "constructed push refspec"); mirror_remote diff --git a/rust/patchable/src/repo.rs b/rust/patchable/src/repo.rs index fe7da752b..b57c221d9 100644 --- a/rust/patchable/src/repo.rs +++ b/rust/patchable/src/repo.rs @@ -176,9 +176,7 @@ pub fn resolve_and_fetch_commitish( &[commitish], Some( FetchOptions::new() - // Tags need to be present to later determine whether `commitish` is a tag or not - // Patchable needs to know this when initializing a repository with the `--mirrored` option (to construct the refspec) - .download_tags(git2::AutotagOption::Auto) + .update_fetchhead(true) .remote_callbacks(callbacks) // TODO: could be 1, CLI option maybe? .depth(0), @@ -191,7 +189,8 @@ pub fn resolve_and_fetch_commitish( refs: vec![commitish.to_string()], })?; tracing::info!("fetched base commit"); - repo.revparse_single(commitish) + // FETCH_HEAD is written by Remote::fetch to be the last reference fetched + repo.revparse_single("FETCH_HEAD") .and_then(|obj| obj.peel_to_commit()) } Err(err) => Err(err), From 3a64496b645e00bad276221d83a705952d78acef Mon Sep 17 00:00:00 2001 From: Lukas Krug Date: Mon, 5 May 2025 17:47:15 +0200 Subject: [PATCH 07/30] Update rust/patchable/src/main.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Natalie Klestrup Röijezon --- rust/patchable/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index 8b507d8af..ea7a8b32e 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -152,7 +152,7 @@ enum Cmd { base: String, /// Assume a mirror exists at stackabletech/ and push the base ref to it. - /// The mirror URL will be stored in patchable.toml instead of the original upstream. + /// The mirror URL will be stored in patchable.toml, and used instead of the original upstream. #[clap(long)] mirrored: bool, }, From 1d444db80f5473dc4ba2925a2570fa1b891e407c Mon Sep 17 00:00:00 2001 From: Lukas Krug Date: Mon, 5 May 2025 17:48:03 +0200 Subject: [PATCH 08/30] Update rust/patchable/src/main.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Natalie Klestrup Röijezon --- rust/patchable/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index ea7a8b32e..147f89d5c 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -154,7 +154,7 @@ enum Cmd { /// Assume a mirror exists at stackabletech/ and push the base ref to it. /// The mirror URL will be stored in patchable.toml, and used instead of the original upstream. #[clap(long)] - mirrored: bool, + mirror: bool, }, /// Shows the patch directory for a given product version From 06ca421c4de858907c06809a426182300ae40eb3 Mon Sep 17 00:00:00 2001 From: Lukas Krug Date: Mon, 5 May 2025 17:49:08 +0200 Subject: [PATCH 09/30] Update rust/patchable/src/main.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Natalie Klestrup Röijezon --- rust/patchable/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index 147f89d5c..5d3adcf86 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -446,8 +446,8 @@ fn main() -> Result<()> { ensure!(!repo_name.is_empty(), ParseUpstreamUrlSnafu { url: &upstream }); - let mirror_url = format!("https://github.com/stackabletech/{}.git", repo_name); - tracing::info!(%mirror_url, "using mirror repository"); + let mirror_url = format!("https://github.com/stackabletech/{repo_name}.git"); + tracing::info!(mirror_url, "using mirror repository"); // Add mirror remote let mut mirror_remote = product_repo From 5aa1769726c298a7a49a9191b06bff060b4a4ae9 Mon Sep 17 00:00:00 2001 From: Lukas Krug Date: Mon, 5 May 2025 17:49:24 +0200 Subject: [PATCH 10/30] Update rust/patchable/src/main.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Natalie Klestrup Röijezon --- rust/patchable/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index 5d3adcf86..1de75684c 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -480,7 +480,7 @@ fn main() -> Result<()> { let mut push_options = git2::PushOptions::new(); push_options.remote_callbacks(callbacks); - let refspec = format!("{}:refs/tags/{}", base_commit, base); + let refspec = format!("{base_commit}:refs/tags/{base}"); tracing::info!(refspec = refspec, "constructed push refspec"); mirror_remote From 37b4bf28501e5b90deb9e3084e34f60f8a5db2bd Mon Sep 17 00:00:00 2001 From: Lukas Krug Date: Mon, 5 May 2025 17:49:43 +0200 Subject: [PATCH 11/30] Update rust/patchable/src/main.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Natalie Klestrup Röijezon --- rust/patchable/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index 1de75684c..b72afbaed 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -481,7 +481,7 @@ fn main() -> Result<()> { push_options.remote_callbacks(callbacks); let refspec = format!("{base_commit}:refs/tags/{base}"); - tracing::info!(refspec = refspec, "constructed push refspec"); + tracing::info!(refspec, "constructed push refspec"); mirror_remote .push(&[&refspec], Some(&mut push_options)) From 0db5985ec4609458454b7cc6a57b69153c58a412 Mon Sep 17 00:00:00 2001 From: Lukas Krug Date: Mon, 5 May 2025 17:50:07 +0200 Subject: [PATCH 12/30] Update rust/patchable/src/main.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Natalie Klestrup Röijezon --- rust/patchable/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index b72afbaed..4a6044aa0 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -486,7 +486,7 @@ fn main() -> Result<()> { mirror_remote .push(&[&refspec], Some(&mut push_options)) .context(PushToMirrorSnafu { - url: mirror_url.clone(), + url: &mirror_url, refspec: &refspec, commit: base_commit, })?; From fbf681f2c237aae6f581ee55915ed19a0108e348 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Mon, 5 May 2025 18:50:29 +0200 Subject: [PATCH 13/30] fix: span behaviour / mirror flag --- rust/patchable/src/main.rs | 16 ++++++++-------- rust/patchable/src/repo.rs | 3 +++ rust/patchable/src/utils.rs | 1 - 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index 4a6044aa0..5b8bbd22a 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -9,7 +9,7 @@ use std::{fs::File, io::Write, path::PathBuf}; use git2::{Oid, Repository}; use serde::{Deserialize, Serialize}; -use snafu::{ensure, OptionExt, ResultExt as _, Snafu}; +use snafu::{OptionExt, ResultExt as _, Snafu}; use tracing_indicatif::IndicatifLayer; use tracing_subscriber::{layer::SubscriberExt as _, util::SubscriberInitExt as _}; @@ -419,7 +419,7 @@ fn main() -> Result<()> { pv, upstream, base, - mirrored, + mirror, } => { let ctx = ProductVersionContext { pv, @@ -440,11 +440,9 @@ fn main() -> Result<()> { let base_commit = repo::resolve_and_fetch_commitish(&product_repo, &base, &upstream).context(FetchBaseCommitSnafu)?; let mut upstream_mirror = None; - if mirrored { + if mirror { // Parse e.g. "https://github.com/apache/druid.git" into "druid" - let repo_name = upstream.split('/').last().map(|repo| repo.trim_end_matches(".git")).context(ParseUpstreamUrlSnafu { url: &upstream })?; - - ensure!(!repo_name.is_empty(), ParseUpstreamUrlSnafu { url: &upstream }); + let repo_name = upstream.split('/').last().map(|repo| repo.trim_end_matches(".git")).filter(|name| !name.is_empty()).context(ParseUpstreamUrlSnafu { url: &upstream })?; let mirror_url = format!("https://github.com/stackabletech/{repo_name}.git"); tracing::info!(mirror_url, "using mirror repository"); @@ -457,16 +455,18 @@ fn main() -> Result<()> { // Push the base commit to the mirror tracing::info!(commit = %base_commit, base = base, url = mirror_url, "pushing commit to mirror"); let mut callbacks = git2::RemoteCallbacks::new(); - callbacks.credentials(|_url, username_from_url, _allowed_types| { + callbacks.credentials(|url, username_from_url, _allowed_types| { git2::Cred::credential_helper( &git2::Config::open_default().unwrap(), // Use default git config - _url, + url, username_from_url, ) }); // Add progress tracking for push operation let (span_push, mut quant_push) = utils::setup_progress_tracking(tracing::info_span!("pushing")); + let _ = span_push.enter(); + callbacks.push_transfer_progress(move |current, total, _| { if total > 0 { quant_push.update_span_progress( diff --git a/rust/patchable/src/repo.rs b/rust/patchable/src/repo.rs index b57c221d9..25e1aa6f8 100644 --- a/rust/patchable/src/repo.rs +++ b/rust/patchable/src/repo.rs @@ -152,6 +152,9 @@ pub fn resolve_and_fetch_commitish( let (span_recv, mut quant_recv) = setup_progress_tracking(tracing::info_span!("receiving")); let (span_index, mut quant_index) = setup_progress_tracking(tracing::info_span!("indexing")); + let _ = span_recv.enter(); + let _ = span_index.enter(); + let mut callbacks = RemoteCallbacks::new(); callbacks.transfer_progress(move |progress| { quant_recv.update_span_progress( diff --git a/rust/patchable/src/utils.rs b/rust/patchable/src/utils.rs index 22cca39ba..ac47b3904 100644 --- a/rust/patchable/src/utils.rs +++ b/rust/patchable/src/utils.rs @@ -94,7 +94,6 @@ pub fn setup_progress_tracking(span: tracing::Span) -> (tracing::Span, Quantizer "{span_child_prefix}{spinner} {span_name}{{{span_fields}}} {wide_msg} {bar:40} {percent:>3}%", ) .expect("hard-coded template should be valid")); - let _ = span.enter(); let quantizer = Quantizer::percent(); (span, quantizer) } From 694e5c758db2c6f817243b0454c1b3ca997d616f Mon Sep 17 00:00:00 2001 From: dervoeti Date: Tue, 6 May 2025 20:32:50 +0200 Subject: [PATCH 14/30] feat: separate product level configuration --- rust/patchable/README.md | 28 ++++++-- rust/patchable/src/main.rs | 142 +++++++++++++++++++++++-------------- 2 files changed, 109 insertions(+), 61 deletions(-) diff --git a/rust/patchable/README.md b/rust/patchable/README.md index a22fc5e97..72eec6b91 100644 --- a/rust/patchable/README.md +++ b/rust/patchable/README.md @@ -31,20 +31,36 @@ For more details, run `cargo patchable --help`. ## Configuration -Patchable stores a per-version file in `docker-images//stackable/patches//patchable.toml`. -It currently recognizes the following keys: +Patchable uses a two-level configuration system: +1. A product-level config file at `docker-images//stackable/patches/patchable.toml` +2. A version-level config file at `docker-images//stackable/patches//patchable.toml` + +The product-level config typically contains: - `upstream` - the URL of the upstream repository (such as `https://github.com/apache/druid.git`) -- `base` - the commit hash of the upstream base commit (such as `7cffb81a8e124d5f218f9af16ad685acf5e9c67c`) +- `mirror` - optional URL of a mirror repository (such as `https://github.com/stackabletech/druid.git`) -### Template +The version-level config typically contains: +- `base` - the commit hash of the upstream base commit -Instead of creating this manually, run `patchable init`: +Fields in the version-level config override those in the product-level config if both are specified. + +### Template +If you're adding a completely new product, you need to create the product-level config once: ```toml -cargo patchable init druid 28.0.0 --upstream=https://github.com/apache/druid.git --base=druid-28.0.0 +# docker-images/druid/stackable/patches/patchable.toml +upstream = "https://github.com/apache/druid.git" +mirror = "https://github.com/stackabletech/druid.git" ``` +If you just want to add a new version, simply initiatilize it with patchable: +``` +cargo patchable init druid 28.0.0 --base=druid-28.0.0 --mirror +``` + +Using the `--mirror` flag will push the base ref to the mirror URL specified in the product-level config. + ## Glossary - Images repo/directory - The checkout of stackabletech/docker-images diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index 5b8bbd22a..4877212fd 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -25,29 +25,72 @@ struct ProductVersion { } #[derive(Deserialize, Serialize)] -struct ProductVersionConfig { +struct ProductConfig { upstream: String, + mirror: Option, +} + +#[derive(Deserialize, Serialize)] +struct ProductVersionConfig { + upstream: Option, #[serde(with = "utils::oid_serde")] base: Oid, mirror: Option, } +struct MergedProductVersionConfig { + upstream: String, + base: Oid, + mirror: Option, +} + struct ProductVersionContext { pv: ProductVersion, images_repo_root: PathBuf, } impl ProductVersionContext { - fn load_config(&self) -> Result { - let path = &self.config_path(); + fn load_product_config(&self) -> Result { + let product_config_path = &self.product_config_path(); + tracing::info!( - config.path = ?path, - "loading config" + config.path = ?product_config_path, + "loading product-level config" ); - toml::from_str::( - &std::fs::read_to_string(path).context(LoadConfigSnafu { path })?, + + toml::from_str::(&std::fs::read_to_string(product_config_path).context( + LoadConfigSnafu { + path: product_config_path, + }, + )?) + .context(ParseConfigSnafu { + path: product_config_path, + }) + } + + fn load_version_config(&self) -> Result { + // Load product-level config (required) + let product_config = self.load_product_config()?; + + // Load version-level config (optional) + let version_config_path = &self.version_config_path(); + let loaded_version_config = toml::from_str::( + &std::fs::read_to_string(version_config_path).context(LoadConfigSnafu { + path: version_config_path, + })?, ) - .context(ParseConfigSnafu { path }) + .context(ParseConfigSnafu { + path: version_config_path, + })?; + + // Inherit `upstream` and `mirror` from product-level config if not set in loaded version-level config + Ok(MergedProductVersionConfig { + upstream: loaded_version_config + .upstream + .unwrap_or(product_config.upstream), + base: loaded_version_config.base, + mirror: loaded_version_config.mirror.or(product_config.mirror), + }) } /// The root directory for files related to the product (across all versions). @@ -63,10 +106,15 @@ impl ProductVersionContext { } /// The patchable configuration file for the product version. - fn config_path(&self) -> PathBuf { + fn version_config_path(&self) -> PathBuf { self.patch_dir().join("patchable.toml") } + /// The product-level patchable configuration file + fn product_config_path(&self) -> PathBuf { + self.product_dir().join("stackable/patches/patchable.toml") + } + /// The directory containing all ephemeral data used by patchable for the product (across all versions). /// /// Should be gitignored, and can safely be deleted as long as all relevant versions have been `patchable export`ed. @@ -141,20 +189,11 @@ enum Cmd { #[clap(flatten)] pv: ProductVersion, - /// The upstream URL (such as https://github.com/apache/druid.git) - #[clap(long)] - upstream: String, - /// The upstream commit-ish (such as druid-28.0.0) that the patch series applies to /// /// Refs (such as tags and branches) will be resolved to commit IDs. #[clap(long)] base: String, - - /// Assume a mirror exists at stackabletech/ and push the base ref to it. - /// The mirror URL will be stored in patchable.toml, and used instead of the original upstream. - #[clap(long)] - mirror: bool, }, /// Shows the patch directory for a given product version @@ -203,8 +242,6 @@ pub enum Error { path: PathBuf, }, - #[snafu(display("failed to parse upstream URL {url:?} to extract repository name"))] - ParseUpstreamUrl { url: String }, #[snafu(display("failed to add temporary mirror remote for {url:?}"))] AddMirrorRemote { source: git2::Error, url: String }, #[snafu(display("failed to push commit {commit} (as {refspec}) to mirror {url:?}"))] @@ -297,7 +334,7 @@ fn main() -> Result<()> { pv, images_repo_root, }; - let config = ctx.load_config()?; + let config = ctx.load_version_config()?; let product_repo_root = ctx.product_repo(); let product_repo = repo::ensure_bare_repo(&product_repo_root) .context(OpenProductRepoForCheckoutSnafu)?; @@ -305,7 +342,10 @@ fn main() -> Result<()> { let base_commit = repo::resolve_and_fetch_commitish( &product_repo, &config.base.to_string(), - config.mirror.as_deref().unwrap_or(&config.upstream), + config + .mirror + .as_deref() + .unwrap_or(&config.upstream), ) .context(FetchBaseCommitSnafu)?; let base_branch = ctx.base_branch(); @@ -366,7 +406,7 @@ fn main() -> Result<()> { pv, images_repo_root, }; - let config = ctx.load_config()?; + let config = ctx.load_version_config()?; let product_worktree_root = ctx.worktree_root(); tracing::info!( @@ -415,12 +455,7 @@ fn main() -> Result<()> { ); } - Cmd::Init { - pv, - upstream, - base, - mirror, - } => { + Cmd::Init { pv, base } => { let ctx = ProductVersionContext { pv, images_repo_root, @@ -434,77 +469,74 @@ fn main() -> Result<()> { .in_scope(|| repo::ensure_bare_repo(&product_repo_root)) .context(OpenProductRepoForCheckoutSnafu)?; + let config = ctx.load_product_config()?; + let upstream = config.upstream; + // --base can be a reference, but patchable.toml should always have a resolved commit id, // so that it cannot be changed under our feet (without us knowing so, anyway...). tracing::info!(?base, "resolving base commit-ish"); - let base_commit = repo::resolve_and_fetch_commitish(&product_repo, &base, &upstream).context(FetchBaseCommitSnafu)?; - let mut upstream_mirror = None; - - if mirror { - // Parse e.g. "https://github.com/apache/druid.git" into "druid" - let repo_name = upstream.split('/').last().map(|repo| repo.trim_end_matches(".git")).filter(|name| !name.is_empty()).context(ParseUpstreamUrlSnafu { url: &upstream })?; - - let mirror_url = format!("https://github.com/stackabletech/{repo_name}.git"); - tracing::info!(mirror_url, "using mirror repository"); + let base_commit = repo::resolve_and_fetch_commitish(&product_repo, &base, &upstream) + .context(FetchBaseCommitSnafu)?; + if let Some(mirror_url) = config.mirror { // Add mirror remote - let mut mirror_remote = product_repo - .remote_anonymous(&mirror_url) - .context(AddMirrorRemoteSnafu { url: mirror_url.clone() })?; + let mut mirror_remote = + product_repo + .remote_anonymous(&mirror_url) + .context(AddMirrorRemoteSnafu { + url: mirror_url.clone(), + })?; // Push the base commit to the mirror tracing::info!(commit = %base_commit, base = base, url = mirror_url, "pushing commit to mirror"); let mut callbacks = git2::RemoteCallbacks::new(); callbacks.credentials(|url, username_from_url, _allowed_types| { git2::Cred::credential_helper( - &git2::Config::open_default().unwrap(), // Use default git config + &git2::Config::open_default() + .expect("failed to open default Git configuration"), // Use default git config, url, username_from_url, ) }); // Add progress tracking for push operation - let (span_push, mut quant_push) = utils::setup_progress_tracking(tracing::info_span!("pushing")); + let (span_push, mut quant_push) = + utils::setup_progress_tracking(tracing::info_span!("pushing")); let _ = span_push.enter(); callbacks.push_transfer_progress(move |current, total, _| { if total > 0 { - quant_push.update_span_progress( - current, - total, - &span_push, - ); + quant_push.update_span_progress(current, total, &span_push); } }); let mut push_options = git2::PushOptions::new(); push_options.remote_callbacks(callbacks); + // Always push the commit as a Git tag named like the value of `base` let refspec = format!("{base_commit}:refs/tags/{base}"); tracing::info!(refspec, "constructed push refspec"); mirror_remote .push(&[&refspec], Some(&mut push_options)) .context(PushToMirrorSnafu { - url: &mirror_url, + url: mirror_url.clone(), refspec: &refspec, commit: base_commit, })?; tracing::info!("successfully pushed base ref to mirror"); - - upstream_mirror = Some(mirror_url); }; tracing::info!(?base, base.commit = ?base_commit, "resolved base commit"); - tracing::info!("saving configuration"); + tracing::info!("saving version-level configuration"); let config = ProductVersionConfig { - upstream, - mirror: upstream_mirror, + upstream: None, + mirror: None, base: base_commit, }; - let config_path = ctx.config_path(); + let config_path = ctx.version_config_path(); if let Some(config_dir) = config_path.parent() { std::fs::create_dir_all(config_dir) .context(CreatePatchDirSnafu { path: config_dir })?; From 43f33c1d856208b499084d1ba4ee05eba9ea74ac Mon Sep 17 00:00:00 2001 From: dervoeti Date: Tue, 6 May 2025 21:47:20 +0200 Subject: [PATCH 15/30] refactor: separate product and version config --- rust/patchable/src/main.rs | 62 +++++++++++--------------------------- 1 file changed, 18 insertions(+), 44 deletions(-) diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index 4877212fd..47522c9d8 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -32,16 +32,8 @@ struct ProductConfig { #[derive(Deserialize, Serialize)] struct ProductVersionConfig { - upstream: Option, #[serde(with = "utils::oid_serde")] base: Oid, - mirror: Option, -} - -struct MergedProductVersionConfig { - upstream: String, - base: Oid, - mirror: Option, } struct ProductVersionContext { @@ -50,47 +42,30 @@ struct ProductVersionContext { } impl ProductVersionContext { - fn load_product_config(&self) -> Result { - let product_config_path = &self.product_config_path(); - + fn load_config Deserialize<'de>>(&self, path: &PathBuf) -> Result { tracing::info!( - config.path = ?product_config_path, - "loading product-level config" + config.path = ?path, + "loading config" ); - toml::from_str::(&std::fs::read_to_string(product_config_path).context( + toml::from_str::(&std::fs::read_to_string(path).context( LoadConfigSnafu { - path: product_config_path, + path, }, )?) .context(ParseConfigSnafu { - path: product_config_path, + path, }) } - fn load_version_config(&self) -> Result { - // Load product-level config (required) - let product_config = self.load_product_config()?; + fn load_product_config(&self) -> Result { + let path = self.product_config_path(); + self.load_config(&path) + } - // Load version-level config (optional) - let version_config_path = &self.version_config_path(); - let loaded_version_config = toml::from_str::( - &std::fs::read_to_string(version_config_path).context(LoadConfigSnafu { - path: version_config_path, - })?, - ) - .context(ParseConfigSnafu { - path: version_config_path, - })?; - - // Inherit `upstream` and `mirror` from product-level config if not set in loaded version-level config - Ok(MergedProductVersionConfig { - upstream: loaded_version_config - .upstream - .unwrap_or(product_config.upstream), - base: loaded_version_config.base, - mirror: loaded_version_config.mirror.or(product_config.mirror), - }) + fn load_version_config(&self) -> Result { + let path = self.version_config_path(); + self.load_config(&path) } /// The root directory for files related to the product (across all versions). @@ -334,18 +309,19 @@ fn main() -> Result<()> { pv, images_repo_root, }; - let config = ctx.load_version_config()?; + let product_config = ctx.load_product_config()?; + let version_config = ctx.load_version_config()?; let product_repo_root = ctx.product_repo(); let product_repo = repo::ensure_bare_repo(&product_repo_root) .context(OpenProductRepoForCheckoutSnafu)?; let base_commit = repo::resolve_and_fetch_commitish( &product_repo, - &config.base.to_string(), - config + &version_config.base.to_string(), + product_config .mirror .as_deref() - .unwrap_or(&config.upstream), + .unwrap_or(&product_config.upstream), ) .context(FetchBaseCommitSnafu)?; let base_branch = ctx.base_branch(); @@ -532,8 +508,6 @@ fn main() -> Result<()> { tracing::info!("saving version-level configuration"); let config = ProductVersionConfig { - upstream: None, - mirror: None, base: base_commit, }; let config_path = ctx.version_config_path(); From 2adc5e135f48501a7d625f63a279536d197a2124 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Tue, 6 May 2025 22:07:12 +0200 Subject: [PATCH 16/30] fix: adjust README --- rust/patchable/README.md | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/rust/patchable/README.md b/rust/patchable/README.md index 72eec6b91..3f7a65a27 100644 --- a/rust/patchable/README.md +++ b/rust/patchable/README.md @@ -36,15 +36,13 @@ Patchable uses a two-level configuration system: 1. A product-level config file at `docker-images//stackable/patches/patchable.toml` 2. A version-level config file at `docker-images//stackable/patches//patchable.toml` -The product-level config typically contains: +The product-level config contains: - `upstream` - the URL of the upstream repository (such as `https://github.com/apache/druid.git`) - `mirror` - optional URL of a mirror repository (such as `https://github.com/stackabletech/druid.git`) -The version-level config typically contains: +The version-level config contains: - `base` - the commit hash of the upstream base commit -Fields in the version-level config override those in the product-level config if both are specified. - ### Template If you're adding a completely new product, you need to create the product-level config once: @@ -54,13 +52,11 @@ upstream = "https://github.com/apache/druid.git" mirror = "https://github.com/stackabletech/druid.git" ``` -If you just want to add a new version, simply initiatilize it with patchable: +If you just want to add a new version, initiatilize the version-level config with patchable: ``` cargo patchable init druid 28.0.0 --base=druid-28.0.0 --mirror ``` -Using the `--mirror` flag will push the base ref to the mirror URL specified in the product-level config. - ## Glossary - Images repo/directory - The checkout of stackabletech/docker-images From 27a9096f8ea99d5cfd1bdef9238cbb610ddc50c0 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Wed, 7 May 2025 11:00:25 +0200 Subject: [PATCH 17/30] chore: formatting / unnecessary clone --- rust/patchable/src/main.rs | 16 ++++------------ rust/patchable/src/repo.rs | 1 - 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index 47522c9d8..a14407d15 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -48,14 +48,8 @@ impl ProductVersionContext { "loading config" ); - toml::from_str::(&std::fs::read_to_string(path).context( - LoadConfigSnafu { - path, - }, - )?) - .context(ParseConfigSnafu { - path, - }) + toml::from_str::(&std::fs::read_to_string(path).context(LoadConfigSnafu { path })?) + .context(ParseConfigSnafu { path }) } fn load_product_config(&self) -> Result { @@ -496,7 +490,7 @@ fn main() -> Result<()> { mirror_remote .push(&[&refspec], Some(&mut push_options)) .context(PushToMirrorSnafu { - url: mirror_url.clone(), + url: mirror_url, refspec: &refspec, commit: base_commit, })?; @@ -507,9 +501,7 @@ fn main() -> Result<()> { tracing::info!(?base, base.commit = ?base_commit, "resolved base commit"); tracing::info!("saving version-level configuration"); - let config = ProductVersionConfig { - base: base_commit, - }; + let config = ProductVersionConfig { base: base_commit }; let config_path = ctx.version_config_path(); if let Some(config_dir) = config_path.parent() { std::fs::create_dir_all(config_dir) diff --git a/rust/patchable/src/repo.rs b/rust/patchable/src/repo.rs index 25e1aa6f8..e4aa500e5 100644 --- a/rust/patchable/src/repo.rs +++ b/rust/patchable/src/repo.rs @@ -169,7 +169,6 @@ pub fn resolve_and_fetch_commitish( ); true }); - repo.remote_anonymous(upstream_url) .context(CreateRemoteSnafu { repo, From 081eec1b616f8bf9169d4fbc12a96765ae93bc7e Mon Sep 17 00:00:00 2001 From: dervoeti Date: Wed, 7 May 2025 11:02:14 +0200 Subject: [PATCH 18/30] fix: tracing --- rust/patchable/src/main.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index a14407d15..c5b972ee6 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -447,6 +447,7 @@ fn main() -> Result<()> { tracing::info!(?base, "resolving base commit-ish"); let base_commit = repo::resolve_and_fetch_commitish(&product_repo, &base, &upstream) .context(FetchBaseCommitSnafu)?; + tracing::info!(?base, base.commit = ?base_commit, "resolved base commit"); if let Some(mirror_url) = config.mirror { // Add mirror remote @@ -498,8 +499,6 @@ fn main() -> Result<()> { tracing::info!("successfully pushed base ref to mirror"); }; - tracing::info!(?base, base.commit = ?base_commit, "resolved base commit"); - tracing::info!("saving version-level configuration"); let config = ProductVersionConfig { base: base_commit }; let config_path = ctx.version_config_path(); From ea3fb745c7a241f6d2801f417ab1731d79643d09 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Wed, 7 May 2025 11:03:57 +0200 Subject: [PATCH 19/30] fix: readme adjustment --- rust/patchable/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/patchable/README.md b/rust/patchable/README.md index 3f7a65a27..7c2427750 100644 --- a/rust/patchable/README.md +++ b/rust/patchable/README.md @@ -54,7 +54,7 @@ mirror = "https://github.com/stackabletech/druid.git" If you just want to add a new version, initiatilize the version-level config with patchable: ``` -cargo patchable init druid 28.0.0 --base=druid-28.0.0 --mirror +cargo patchable init druid 28.0.0 --base=druid-28.0.0 ``` ## Glossary From 51b26eda86c6f43a420b2030cf94f0ba5442e3ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 8 May 2025 11:16:32 +0200 Subject: [PATCH 20/30] Make mirroring optional again --- rust/patchable/src/main.rs | 59 ++++++++++++++++++++++++++++++++------ rust/patchable/src/repo.rs | 6 ++-- 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index c5b972ee6..07b0559c1 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -24,14 +24,35 @@ struct ProductVersion { version: String, } +/// Configuration that applies to all versions of a product, located at $ROOT/$product/stackable/patches/patchable.toml +/// +/// Must be created by hand (for now). #[derive(Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] struct ProductConfig { + /// The upstream repository URL upstream: String, - mirror: Option, + + /// The repository that commits are mirrored to by `init --mirror`, typically `https://github.com/stackabletech/$product.git` + /// + /// This value is _not_ used by `checkout`, that uses [`ProductVersionConfig::mirror`] instead. + /// `init --mirror` copies this value into [`ProductVersionConfig::mirror`]. + default_mirror: Option, } +/// Configuration that applies to an individual version of a product, located at $ROOT/$product/stackable/patches/$version/patchable.toml +/// +/// Typically created by `patchable init`. #[derive(Deserialize, Serialize)] struct ProductVersionConfig { + /// The mirror repository that this version can be fetched from + /// + /// Copied from [`ProductConfig::default_mirror`] by `init --mirror`. + mirror: Option, + + /// The upstream base commit for this version + /// + /// Must be a commit ID, not a generic commitish (branch, tag, or anotherother ref), those are resolved by `init`. #[serde(with = "utils::oid_serde")] base: Oid, } @@ -163,6 +184,9 @@ enum Cmd { /// Refs (such as tags and branches) will be resolved to commit IDs. #[clap(long)] base: String, + + #[clap(long)] + mirror: bool, }, /// Shows the patch directory for a given product version @@ -211,6 +235,10 @@ pub enum Error { path: PathBuf, }, + #[snafu(display( + "mirroring requested, but default-mirror is not configured in product configuration" + ))] + InitMirrorNotConfigured, #[snafu(display("failed to add temporary mirror remote for {url:?}"))] AddMirrorRemote { source: git2::Error, url: String }, #[snafu(display("failed to push commit {commit} (as {refspec}) to mirror {url:?}"))] @@ -312,10 +340,10 @@ fn main() -> Result<()> { let base_commit = repo::resolve_and_fetch_commitish( &product_repo, &version_config.base.to_string(), - product_config - .mirror - .as_deref() - .unwrap_or(&product_config.upstream), + version_config.mirror.as_deref().unwrap_or_else(|| { + tracing::warn!("this product version is not mirrored, re-init it with --mirror before merging it"); + &product_config.upstream + }), ) .context(FetchBaseCommitSnafu)?; let base_branch = ctx.base_branch(); @@ -425,7 +453,7 @@ fn main() -> Result<()> { ); } - Cmd::Init { pv, base } => { + Cmd::Init { pv, base, mirror } => { let ctx = ProductVersionContext { pv, images_repo_root, @@ -449,7 +477,11 @@ fn main() -> Result<()> { .context(FetchBaseCommitSnafu)?; tracing::info!(?base, base.commit = ?base_commit, "resolved base commit"); - if let Some(mirror_url) = config.mirror { + let mirror_url = if mirror { + let mirror_url = config + .default_mirror + .context(InitMirrorNotConfiguredSnafu)?; + // Add mirror remote let mut mirror_remote = product_repo @@ -491,16 +523,25 @@ fn main() -> Result<()> { mirror_remote .push(&[&refspec], Some(&mut push_options)) .context(PushToMirrorSnafu { - url: mirror_url, + url: &mirror_url, refspec: &refspec, commit: base_commit, })?; tracing::info!("successfully pushed base ref to mirror"); + Some(mirror_url) + } else { + tracing::warn!( + "this version is not mirrored, re-run with --mirror before merging into main" + ); + None }; tracing::info!("saving version-level configuration"); - let config = ProductVersionConfig { base: base_commit }; + let config = ProductVersionConfig { + base: base_commit, + mirror: mirror_url, + }; let config_path = ctx.version_config_path(); if let Some(config_dir) = config_path.parent() { std::fs::create_dir_all(config_dir) diff --git a/rust/patchable/src/repo.rs b/rust/patchable/src/repo.rs index e4aa500e5..006023103 100644 --- a/rust/patchable/src/repo.rs +++ b/rust/patchable/src/repo.rs @@ -149,8 +149,10 @@ pub fn resolve_and_fetch_commitish( "base commit not found locally, fetching from upstream" ); - let (span_recv, mut quant_recv) = setup_progress_tracking(tracing::info_span!("receiving")); - let (span_index, mut quant_index) = setup_progress_tracking(tracing::info_span!("indexing")); + let (span_recv, mut quant_recv) = + setup_progress_tracking(tracing::info_span!("receiving")); + let (span_index, mut quant_index) = + setup_progress_tracking(tracing::info_span!("indexing")); let _ = span_recv.enter(); let _ = span_index.enter(); From dd32e8aa5588d1ce6e258c4a6917ffac59d536cd Mon Sep 17 00:00:00 2001 From: dervoeti Date: Thu, 8 May 2025 13:07:01 +0200 Subject: [PATCH 21/30] feat: ssh support --- Cargo.lock | 1 + Cargo.toml | 1 + rust/patchable/Cargo.toml | 1 + rust/patchable/src/main.rs | 61 ++++++++++++++++++++++++++----------- rust/patchable/src/repo.rs | 6 ++-- rust/patchable/src/utils.rs | 45 +++++++++++++++++++++++++++ 6 files changed, 94 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2b20caacd..2f99d0bc2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -596,6 +596,7 @@ dependencies = [ "tracing", "tracing-indicatif", "tracing-subscriber", + "url", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 805dab66f..a064df4be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,3 +13,4 @@ toml = "0.8.19" tracing = "0.1.41" tracing-indicatif = "0.3.9" tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } +url = "2.5.4" diff --git a/rust/patchable/Cargo.toml b/rust/patchable/Cargo.toml index cb72ed092..0f8fb7ea3 100644 --- a/rust/patchable/Cargo.toml +++ b/rust/patchable/Cargo.toml @@ -14,3 +14,4 @@ toml.workspace = true tracing.workspace = true tracing-indicatif.workspace = true tracing-subscriber.workspace = true +url.workspace = true diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index 07b0559c1..98a342ed1 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -13,6 +13,8 @@ use snafu::{OptionExt, ResultExt as _, Snafu}; use tracing_indicatif::IndicatifLayer; use tracing_subscriber::{layer::SubscriberExt as _, util::SubscriberInitExt as _}; +use crate::utils::setup_git_credentials; + #[derive(clap::Parser)] struct ProductVersion { /// The product name slug (such as druid) @@ -164,6 +166,10 @@ enum Cmd { /// Check out the base commit, without applying patches #[clap(long)] base_only: bool, + + /// Use SSH for git operations + #[clap(long)] + ssh: bool, }, /// Export the patches from the source tree at docker-images//patchable-work/worktree/ @@ -185,8 +191,13 @@ enum Cmd { #[clap(long)] base: String, + /// Mirror the product version to the default mirror repository #[clap(long)] mirror: bool, + + /// Use SSH for git operations + #[clap(long)] + ssh: bool, }, /// Shows the patch directory for a given product version @@ -235,6 +246,9 @@ pub enum Error { path: PathBuf, }, + #[snafu(display("failed to rewrite URL for SSH: {source}"))] + UrlRewrite { source: utils::UrlRewriteError }, + #[snafu(display( "mirroring requested, but default-mirror is not configured in product configuration" ))] @@ -326,7 +340,7 @@ fn main() -> Result<()> { } }; match opts.cmd { - Cmd::Checkout { pv, base_only } => { + Cmd::Checkout { pv, base_only, ssh } => { let ctx = ProductVersionContext { pv, images_repo_root, @@ -337,13 +351,20 @@ fn main() -> Result<()> { let product_repo = repo::ensure_bare_repo(&product_repo_root) .context(OpenProductRepoForCheckoutSnafu)?; + let mut upstream = version_config.mirror.unwrap_or_else(|| { + tracing::warn!("this product version is not mirrored, re-init it with --mirror before merging it"); + product_config.upstream + }); + + if ssh { + upstream = + utils::rewrite_git_https_url_to_ssh(&upstream).context(UrlRewriteSnafu)?; + } + let base_commit = repo::resolve_and_fetch_commitish( &product_repo, &version_config.base.to_string(), - version_config.mirror.as_deref().unwrap_or_else(|| { - tracing::warn!("this product version is not mirrored, re-init it with --mirror before merging it"); - &product_config.upstream - }), + &upstream, ) .context(FetchBaseCommitSnafu)?; let base_branch = ctx.base_branch(); @@ -453,7 +474,12 @@ fn main() -> Result<()> { ); } - Cmd::Init { pv, base, mirror } => { + Cmd::Init { + pv, + base, + mirror, + ssh, + } => { let ctx = ProductVersionContext { pv, images_repo_root, @@ -468,7 +494,11 @@ fn main() -> Result<()> { .context(OpenProductRepoForCheckoutSnafu)?; let config = ctx.load_product_config()?; - let upstream = config.upstream; + let upstream = if ssh { + utils::rewrite_git_https_url_to_ssh(&config.upstream).context(UrlRewriteSnafu)? + } else { + config.upstream + }; // --base can be a reference, but patchable.toml should always have a resolved commit id, // so that it cannot be changed under our feet (without us knowing so, anyway...). @@ -478,10 +508,13 @@ fn main() -> Result<()> { tracing::info!(?base, base.commit = ?base_commit, "resolved base commit"); let mirror_url = if mirror { - let mirror_url = config + let mut mirror_url = config .default_mirror .context(InitMirrorNotConfiguredSnafu)?; - + if ssh { + mirror_url = + utils::rewrite_git_https_url_to_ssh(&mirror_url).context(UrlRewriteSnafu)? + }; // Add mirror remote let mut mirror_remote = product_repo @@ -492,15 +525,7 @@ fn main() -> Result<()> { // Push the base commit to the mirror tracing::info!(commit = %base_commit, base = base, url = mirror_url, "pushing commit to mirror"); - let mut callbacks = git2::RemoteCallbacks::new(); - callbacks.credentials(|url, username_from_url, _allowed_types| { - git2::Cred::credential_helper( - &git2::Config::open_default() - .expect("failed to open default Git configuration"), // Use default git config, - url, - username_from_url, - ) - }); + let mut callbacks = setup_git_credentials(); // Add progress tracking for push operation let (span_push, mut quant_push) = diff --git a/rust/patchable/src/repo.rs b/rust/patchable/src/repo.rs index 006023103..962dc03d9 100644 --- a/rust/patchable/src/repo.rs +++ b/rust/patchable/src/repo.rs @@ -1,14 +1,14 @@ use std::path::{self, Path, PathBuf}; use git2::{ - FetchOptions, ObjectType, Oid, RemoteCallbacks, Repository, RepositoryInitOptions, + FetchOptions, ObjectType, Oid, Repository, RepositoryInitOptions, WorktreeAddOptions, }; use snafu::{ResultExt, Snafu}; use crate::{ error::{self, CommitRef}, - utils::setup_progress_tracking, + utils::{setup_git_credentials, setup_progress_tracking}, }; #[derive(Debug, Snafu)] @@ -157,7 +157,7 @@ pub fn resolve_and_fetch_commitish( let _ = span_recv.enter(); let _ = span_index.enter(); - let mut callbacks = RemoteCallbacks::new(); + let mut callbacks = setup_git_credentials(); callbacks.transfer_progress(move |progress| { quant_recv.update_span_progress( progress.received_objects(), diff --git a/rust/patchable/src/utils.rs b/rust/patchable/src/utils.rs index ac47b3904..a9c711915 100644 --- a/rust/patchable/src/utils.rs +++ b/rust/patchable/src/utils.rs @@ -1,8 +1,36 @@ use std::path::Path; use git2::Repository; +use snafu::{OptionExt as _, ResultExt as _, Snafu}; use tracing::Span; use tracing_indicatif::{span_ext::IndicatifSpanExt, style::ProgressStyle}; +use url::Url; + +/// Errors that can occur during URL rewriting. +#[derive(Debug, Snafu)] +pub enum UrlRewriteError { + #[snafu(display("Failed to parse URL {url:?}: {source}"))] + ParseUrl { source: url::ParseError, url: String }, + #[snafu(display("URL {url:?} has no host component"))] + NoHostInUrl { url: String }, +} + +/// Rewrites a given URL to an SSH-style Git URL +/// For example, `https://github.com/user/repo.git` becomes `git@github.com:user/repo.git`. +pub fn rewrite_git_https_url_to_ssh( + original_url: &str, +) -> Result { + let parsed_url = Url::parse(original_url).context(ParseUrlSnafu { + url: original_url, + })?; + + let host = parsed_url.host_str().context(NoHostInUrlSnafu { + url: original_url, + })?; + let path = parsed_url.path().trim_start_matches('/'); + + Ok(format!("git@{}:{}", host, path)) +} /// Runs a function whenever a `value` changes "enough". /// @@ -97,3 +125,20 @@ pub fn setup_progress_tracking(span: tracing::Span) -> (tracing::Span, Quantizer let quantizer = Quantizer::percent(); (span, quantizer) } + +/// Basic configuration of credentials for Git operations. +pub fn setup_git_credentials<'a>() -> git2::RemoteCallbacks<'a> { + let mut callbacks = git2::RemoteCallbacks::new(); + callbacks.credentials(|url, username_from_url, allowed_types| { + if allowed_types.contains(git2::CredentialType::SSH_KEY) { + git2::Cred::ssh_key_from_agent(username_from_url.unwrap_or("git")) + } else { + git2::Cred::credential_helper( + &git2::Config::open_default().expect("failed to open default Git configuration"), + url, + username_from_url, + ) + } + }); + callbacks +} From 9923c9dc84e252e9ef653941c6716af0cff4177a Mon Sep 17 00:00:00 2001 From: dervoeti Date: Thu, 8 May 2025 15:52:25 +0200 Subject: [PATCH 22/30] feat: remove url crate --- Cargo.lock | 1 - Cargo.toml | 1 - rust/patchable/Cargo.toml | 1 - rust/patchable/src/utils.rs | 30 ++++++++++-------------------- 4 files changed, 10 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2f99d0bc2..2b20caacd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -596,7 +596,6 @@ dependencies = [ "tracing", "tracing-indicatif", "tracing-subscriber", - "url", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index a064df4be..805dab66f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,4 +13,3 @@ toml = "0.8.19" tracing = "0.1.41" tracing-indicatif = "0.3.9" tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } -url = "2.5.4" diff --git a/rust/patchable/Cargo.toml b/rust/patchable/Cargo.toml index 0f8fb7ea3..cb72ed092 100644 --- a/rust/patchable/Cargo.toml +++ b/rust/patchable/Cargo.toml @@ -14,4 +14,3 @@ toml.workspace = true tracing.workspace = true tracing-indicatif.workspace = true tracing-subscriber.workspace = true -url.workspace = true diff --git a/rust/patchable/src/utils.rs b/rust/patchable/src/utils.rs index a9c711915..0acff7456 100644 --- a/rust/patchable/src/utils.rs +++ b/rust/patchable/src/utils.rs @@ -1,35 +1,25 @@ use std::path::Path; use git2::Repository; -use snafu::{OptionExt as _, ResultExt as _, Snafu}; +use snafu::{OptionExt as _, Snafu}; use tracing::Span; use tracing_indicatif::{span_ext::IndicatifSpanExt, style::ProgressStyle}; -use url::Url; -/// Errors that can occur during URL rewriting. #[derive(Debug, Snafu)] pub enum UrlRewriteError { - #[snafu(display("Failed to parse URL {url:?}: {source}"))] - ParseUrl { source: url::ParseError, url: String }, - #[snafu(display("URL {url:?} has no host component"))] - NoHostInUrl { url: String }, + #[snafu(display("URL does not have https schema: {}", url))] + NoHttpsSchema { url: String }, } /// Rewrites a given URL to an SSH-style Git URL /// For example, `https://github.com/user/repo.git` becomes `git@github.com:user/repo.git`. -pub fn rewrite_git_https_url_to_ssh( - original_url: &str, -) -> Result { - let parsed_url = Url::parse(original_url).context(ParseUrlSnafu { - url: original_url, - })?; - - let host = parsed_url.host_str().context(NoHostInUrlSnafu { - url: original_url, - })?; - let path = parsed_url.path().trim_start_matches('/'); - - Ok(format!("git@{}:{}", host, path)) +pub fn rewrite_git_https_url_to_ssh(original_url: &str) -> Result { + let schemaless = original_url + .strip_prefix("https://") + .context(NoHttpsSchemaSnafu { + url: original_url.to_string(), + })?; + Ok(format!("ssh://git@{schemaless}")) } /// Runs a function whenever a `value` changes "enough". From b101cd4b52ed9ef9856cd479981cce2b00c8f641 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Thu, 8 May 2025 16:00:00 +0200 Subject: [PATCH 23/30] docs: update README --- rust/patchable/README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/rust/patchable/README.md b/rust/patchable/README.md index 7c2427750..107591b39 100644 --- a/rust/patchable/README.md +++ b/rust/patchable/README.md @@ -38,10 +38,11 @@ Patchable uses a two-level configuration system: The product-level config contains: - `upstream` - the URL of the upstream repository (such as `https://github.com/apache/druid.git`) -- `mirror` - optional URL of a mirror repository (such as `https://github.com/stackabletech/druid.git`) +- `default_mirror` - optional: default URL of a mirror repository (such as `https://github.com/stackabletech/druid.git`) The version-level config contains: - `base` - the commit hash of the upstream base commit +- `mirror` - optional: URL of the mirror repository for this version, if mirroring is enabled ### Template @@ -52,11 +53,14 @@ upstream = "https://github.com/apache/druid.git" mirror = "https://github.com/stackabletech/druid.git" ``` -If you just want to add a new version, initiatilize the version-level config with patchable: -``` -cargo patchable init druid 28.0.0 --base=druid-28.0.0 +If you just want to add a new version, initialize the version-level config with patchable: +```sh +cargo patchable init druid 28.0.0 --base=druid-28.0.0 --mirror ``` +This will initialize the version-level config with the base commit hash and the default mirror URL from the product-level config. +You can optionally provide the `--ssh` flag to use SSH instead of HTTPS for Git operations. + ## Glossary - Images repo/directory - The checkout of stackabletech/docker-images From b30dfe67960e2193aaf3f463a6e37fbfaee5da6b Mon Sep 17 00:00:00 2001 From: Lukas Krug Date: Thu, 8 May 2025 16:24:59 +0200 Subject: [PATCH 24/30] Update rust/patchable/src/main.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Natalie Klestrup Röijezon --- rust/patchable/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index 98a342ed1..65983daf9 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -246,7 +246,7 @@ pub enum Error { path: PathBuf, }, - #[snafu(display("failed to rewrite URL for SSH: {source}"))] + #[snafu(display("failed to rewrite URL for SSH"))] UrlRewrite { source: utils::UrlRewriteError }, #[snafu(display( From 84c6ce6c9d1ec5d9702dbde955d83e3f4a9cccdc Mon Sep 17 00:00:00 2001 From: dervoeti Date: Thu, 8 May 2025 16:38:50 +0200 Subject: [PATCH 25/30] feat: product-level configs for patchable --- druid/stackable/patches/patchable.toml | 2 ++ hadoop/stackable/patches/patchable.toml | 2 ++ hbase/stackable/patches/patchable.toml | 2 ++ hive/stackable/patches/patchable.toml | 2 ++ kafka/stackable/patches/patchable.toml | 2 ++ nifi/stackable/patches/patchable.toml | 2 ++ omid/stackable/patches/patchable.toml | 2 ++ spark-k8s/stackable/patches/patchable.toml | 2 ++ trino-storage-connector/stackable/patches/patchable.toml | 2 ++ trino/stackable/patches/patchable.toml | 2 ++ zookeeper/stackable/patches/patchable.toml | 2 ++ 11 files changed, 22 insertions(+) create mode 100644 druid/stackable/patches/patchable.toml create mode 100644 hadoop/stackable/patches/patchable.toml create mode 100644 hbase/stackable/patches/patchable.toml create mode 100644 hive/stackable/patches/patchable.toml create mode 100644 kafka/stackable/patches/patchable.toml create mode 100644 nifi/stackable/patches/patchable.toml create mode 100644 omid/stackable/patches/patchable.toml create mode 100644 spark-k8s/stackable/patches/patchable.toml create mode 100644 trino-storage-connector/stackable/patches/patchable.toml create mode 100644 trino/stackable/patches/patchable.toml create mode 100644 zookeeper/stackable/patches/patchable.toml diff --git a/druid/stackable/patches/patchable.toml b/druid/stackable/patches/patchable.toml new file mode 100644 index 000000000..7b7139a39 --- /dev/null +++ b/druid/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/apache/druid.git" +default-mirror = "https://github.com/stackabletech/druid.git" diff --git a/hadoop/stackable/patches/patchable.toml b/hadoop/stackable/patches/patchable.toml new file mode 100644 index 000000000..79e086ca7 --- /dev/null +++ b/hadoop/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/apache/hadoop.git" +default-mirror = "https://github.com/stackabletech/hadoop.git" diff --git a/hbase/stackable/patches/patchable.toml b/hbase/stackable/patches/patchable.toml new file mode 100644 index 000000000..2a84f6d70 --- /dev/null +++ b/hbase/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/apache/hbase.git" +default-mirror = "https://github.com/stackabletech/hbase.git" diff --git a/hive/stackable/patches/patchable.toml b/hive/stackable/patches/patchable.toml new file mode 100644 index 000000000..b193b451c --- /dev/null +++ b/hive/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/apache/hive.git" +default-mirror = "https://github.com/stackabletech/hive.git" diff --git a/kafka/stackable/patches/patchable.toml b/kafka/stackable/patches/patchable.toml new file mode 100644 index 000000000..df9a32e54 --- /dev/null +++ b/kafka/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/apache/kafka.git" +default-mirror = "https://github.com/stackabletech/kafka.git" diff --git a/nifi/stackable/patches/patchable.toml b/nifi/stackable/patches/patchable.toml new file mode 100644 index 000000000..0867b53cc --- /dev/null +++ b/nifi/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/apache/nifi.git" +default-mirror = "https://github.com/stackabletech/nifi.git" diff --git a/omid/stackable/patches/patchable.toml b/omid/stackable/patches/patchable.toml new file mode 100644 index 000000000..1c4d3c150 --- /dev/null +++ b/omid/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/apache/phoenix-omid.git" +default-mirror = "https://github.com/stackabletech/phoenix-omid.git" diff --git a/spark-k8s/stackable/patches/patchable.toml b/spark-k8s/stackable/patches/patchable.toml new file mode 100644 index 000000000..5ff7ed87a --- /dev/null +++ b/spark-k8s/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/apache/spark.git" +default-mirror = "https://github.com/stackabletech/spark.git" diff --git a/trino-storage-connector/stackable/patches/patchable.toml b/trino-storage-connector/stackable/patches/patchable.toml new file mode 100644 index 000000000..0c169d4e8 --- /dev/null +++ b/trino-storage-connector/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/snowlift/trino-storage.git" +default-mirror = "https://github.com/stackabletech/trino-storage.git" diff --git a/trino/stackable/patches/patchable.toml b/trino/stackable/patches/patchable.toml new file mode 100644 index 000000000..3404f802c --- /dev/null +++ b/trino/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/trinodb/trino.git" +default-mirror = "https://github.com/stackabletech/trino.git" diff --git a/zookeeper/stackable/patches/patchable.toml b/zookeeper/stackable/patches/patchable.toml new file mode 100644 index 000000000..16f696fe3 --- /dev/null +++ b/zookeeper/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/apache/zookeeper.git" +default-mirror = "https://github.com/stackabletech/zookeeper.git" From 9604e63007a19a1f5a9152752d160839f67144bb Mon Sep 17 00:00:00 2001 From: dervoeti Date: Thu, 8 May 2025 16:45:00 +0200 Subject: [PATCH 26/30] docs: fixed README.md for markdownlint --- rust/patchable/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rust/patchable/README.md b/rust/patchable/README.md index 107591b39..76c90b1fd 100644 --- a/rust/patchable/README.md +++ b/rust/patchable/README.md @@ -37,16 +37,19 @@ Patchable uses a two-level configuration system: 2. A version-level config file at `docker-images//stackable/patches//patchable.toml` The product-level config contains: + - `upstream` - the URL of the upstream repository (such as `https://github.com/apache/druid.git`) - `default_mirror` - optional: default URL of a mirror repository (such as `https://github.com/stackabletech/druid.git`) The version-level config contains: + - `base` - the commit hash of the upstream base commit - `mirror` - optional: URL of the mirror repository for this version, if mirroring is enabled ### Template If you're adding a completely new product, you need to create the product-level config once: + ```toml # docker-images/druid/stackable/patches/patchable.toml upstream = "https://github.com/apache/druid.git" @@ -54,6 +57,7 @@ mirror = "https://github.com/stackabletech/druid.git" ``` If you just want to add a new version, initialize the version-level config with patchable: + ```sh cargo patchable init druid 28.0.0 --base=druid-28.0.0 --mirror ``` From 155a162a70bf934f4721263c407f08eef5264404 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Fri, 9 May 2025 11:20:11 +0200 Subject: [PATCH 27/30] chore: removed upstream from version configs / added missing product-level configs --- druid/stackable/patches/30.0.0/patchable.toml | 1 - druid/stackable/patches/30.0.1/patchable.toml | 1 - druid/stackable/patches/31.0.1/patchable.toml | 1 - hadoop/stackable/patches/3.3.4/patchable.toml | 1 - hadoop/stackable/patches/3.3.6/patchable.toml | 1 - hadoop/stackable/patches/3.4.0/patchable.toml | 1 - hadoop/stackable/patches/3.4.1/patchable.toml | 1 - .../hbase-operator-tools/stackable/patches/1.2.0/patchable.toml | 1 - .../stackable/patches/1.3.0-fd5a5fb/patchable.toml | 1 - hbase/hbase-operator-tools/stackable/patches/patchable.toml | 2 ++ hbase/phoenix/stackable/patches/5.2.1/patchable.toml | 1 - hbase/phoenix/stackable/patches/patchable.toml | 2 ++ hbase/stackable/patches/2.4.18/patchable.toml | 1 - hbase/stackable/patches/2.6.0/patchable.toml | 1 - hbase/stackable/patches/2.6.1/patchable.toml | 1 - hive/stackable/patches/3.1.3/patchable.toml | 1 - hive/stackable/patches/4.0.0/patchable.toml | 1 - hive/stackable/patches/4.0.1/patchable.toml | 1 - kafka/stackable/patches/3.7.1/patchable.toml | 1 - kafka/stackable/patches/3.7.2/patchable.toml | 1 - kafka/stackable/patches/3.8.0/patchable.toml | 1 - kafka/stackable/patches/3.9.0/patchable.toml | 1 - nifi/stackable/patches/1.27.0/patchable.toml | 1 - nifi/stackable/patches/1.28.1/patchable.toml | 1 - nifi/stackable/patches/2.2.0/patchable.toml | 1 - omid/stackable/patches/1.1.0/patchable.toml | 1 - omid/stackable/patches/1.1.1/patchable.toml | 1 - omid/stackable/patches/1.1.2/patchable.toml | 1 - omid/stackable/patches/1.1.3-SNAPSHOT/patchable.toml | 1 - .../hbase-connectors/stackable/patches/1.0.1/patchable.toml | 1 - spark-k8s/hbase-connectors/stackable/patches/patchable.toml | 2 ++ spark-k8s/stackable/patches/3.5.2/patchable.toml | 1 - spark-k8s/stackable/patches/3.5.5/patchable.toml | 1 - trino-storage-connector/stackable/patches/451/patchable.toml | 1 - trino-storage-connector/stackable/patches/455/patchable.toml | 1 - trino-storage-connector/stackable/patches/470/patchable.toml | 1 - trino/stackable/patches/451/patchable.toml | 1 - trino/stackable/patches/455/patchable.toml | 1 - trino/stackable/patches/470/patchable.toml | 1 - zookeeper/stackable/patches/3.9.3/patchable.toml | 1 - 40 files changed, 6 insertions(+), 37 deletions(-) create mode 100644 hbase/hbase-operator-tools/stackable/patches/patchable.toml create mode 100644 hbase/phoenix/stackable/patches/patchable.toml create mode 100644 spark-k8s/hbase-connectors/stackable/patches/patchable.toml diff --git a/druid/stackable/patches/30.0.0/patchable.toml b/druid/stackable/patches/30.0.0/patchable.toml index 7b887616d..53c505db4 100644 --- a/druid/stackable/patches/30.0.0/patchable.toml +++ b/druid/stackable/patches/30.0.0/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/druid.git" base = "09d36ee324747f1407705c27618b6d415c3fa8a9" mirror = "https://github.com/stackabletech/druid.git" diff --git a/druid/stackable/patches/30.0.1/patchable.toml b/druid/stackable/patches/30.0.1/patchable.toml index bdf04c5b8..2f31eb1e9 100644 --- a/druid/stackable/patches/30.0.1/patchable.toml +++ b/druid/stackable/patches/30.0.1/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/druid.git" base = "a30af7a91d528e5c3a90356a5592abc7119191c6" mirror = "https://github.com/stackabletech/druid.git" diff --git a/druid/stackable/patches/31.0.1/patchable.toml b/druid/stackable/patches/31.0.1/patchable.toml index 818ee9701..63a03b2f9 100644 --- a/druid/stackable/patches/31.0.1/patchable.toml +++ b/druid/stackable/patches/31.0.1/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/druid.git" base = "520482cb9638e452b0553595b4f29bb397a63758" mirror = "https://github.com/stackabletech/druid.git" diff --git a/hadoop/stackable/patches/3.3.4/patchable.toml b/hadoop/stackable/patches/3.3.4/patchable.toml index bfda3d2cf..a7fece048 100644 --- a/hadoop/stackable/patches/3.3.4/patchable.toml +++ b/hadoop/stackable/patches/3.3.4/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/hadoop.git" base = "a585a73c3e02ac62350c136643a5e7f6095a3dbb" mirror = "https://github.com/stackabletech/hadoop.git" diff --git a/hadoop/stackable/patches/3.3.6/patchable.toml b/hadoop/stackable/patches/3.3.6/patchable.toml index 82226d3a4..54002b2ca 100644 --- a/hadoop/stackable/patches/3.3.6/patchable.toml +++ b/hadoop/stackable/patches/3.3.6/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/hadoop.git" base = "1be78238728da9266a4f88195058f08fd012bf9c" mirror = "https://github.com/stackabletech/hadoop.git" diff --git a/hadoop/stackable/patches/3.4.0/patchable.toml b/hadoop/stackable/patches/3.4.0/patchable.toml index e0cdc727d..038c64315 100644 --- a/hadoop/stackable/patches/3.4.0/patchable.toml +++ b/hadoop/stackable/patches/3.4.0/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/hadoop.git" base = "bd8b77f398f626bb7791783192ee7a5dfaeec760" mirror = "https://github.com/stackabletech/hadoop.git" diff --git a/hadoop/stackable/patches/3.4.1/patchable.toml b/hadoop/stackable/patches/3.4.1/patchable.toml index 73e216571..fd6df6895 100644 --- a/hadoop/stackable/patches/3.4.1/patchable.toml +++ b/hadoop/stackable/patches/3.4.1/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/hadoop.git" base = "4d7825309348956336b8f06a08322b78422849b1" mirror = "https://github.com/stackabletech/hadoop.git" diff --git a/hbase/hbase-operator-tools/stackable/patches/1.2.0/patchable.toml b/hbase/hbase-operator-tools/stackable/patches/1.2.0/patchable.toml index 203a00d34..a6296d2f7 100644 --- a/hbase/hbase-operator-tools/stackable/patches/1.2.0/patchable.toml +++ b/hbase/hbase-operator-tools/stackable/patches/1.2.0/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/hbase-operator-tools.git" base = "478af00af79f82624264fd2bb447b97fecc8e790" mirror = "https://github.com/stackabletech/hbase-operator-tools.git" diff --git a/hbase/hbase-operator-tools/stackable/patches/1.3.0-fd5a5fb/patchable.toml b/hbase/hbase-operator-tools/stackable/patches/1.3.0-fd5a5fb/patchable.toml index 290988bad..e6f7c576d 100644 --- a/hbase/hbase-operator-tools/stackable/patches/1.3.0-fd5a5fb/patchable.toml +++ b/hbase/hbase-operator-tools/stackable/patches/1.3.0-fd5a5fb/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/hbase-operator-tools.git" base = "fd5a5fb90755949a90c502c76de8313130403fa3" mirror = "https://github.com/stackabletech/hbase-operator-tools.git" diff --git a/hbase/hbase-operator-tools/stackable/patches/patchable.toml b/hbase/hbase-operator-tools/stackable/patches/patchable.toml new file mode 100644 index 000000000..ce4c62faf --- /dev/null +++ b/hbase/hbase-operator-tools/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/apache/hbase-operator-tools.git" +default-mirror = "https://github.com/stackabletech/hbase-operator-tools.git" diff --git a/hbase/phoenix/stackable/patches/5.2.1/patchable.toml b/hbase/phoenix/stackable/patches/5.2.1/patchable.toml index a46992214..a0b11000b 100644 --- a/hbase/phoenix/stackable/patches/5.2.1/patchable.toml +++ b/hbase/phoenix/stackable/patches/5.2.1/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/phoenix.git" base = "b738d66cb5863b759bb98eaa417b3b5731d41f95" mirror = "https://github.com/stackabletech/phoenix.git" diff --git a/hbase/phoenix/stackable/patches/patchable.toml b/hbase/phoenix/stackable/patches/patchable.toml new file mode 100644 index 000000000..6dd31259f --- /dev/null +++ b/hbase/phoenix/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/apache/phoenix.git" +default-mirror = "https://github.com/stackabletech/phoenix.git" diff --git a/hbase/stackable/patches/2.4.18/patchable.toml b/hbase/stackable/patches/2.4.18/patchable.toml index b8fcf6638..20dfd9e95 100644 --- a/hbase/stackable/patches/2.4.18/patchable.toml +++ b/hbase/stackable/patches/2.4.18/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/hbase.git" base = "a1767f4d76859c0068720a6c1e5cb78282ebfe1e" mirror = "https://github.com/stackabletech/hbase.git" diff --git a/hbase/stackable/patches/2.6.0/patchable.toml b/hbase/stackable/patches/2.6.0/patchable.toml index c611ac76e..c914eb82b 100644 --- a/hbase/stackable/patches/2.6.0/patchable.toml +++ b/hbase/stackable/patches/2.6.0/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/hbase.git" base = "de99f8754135ea69adc39da48d2bc2b2710a5366" mirror = "https://github.com/stackabletech/hbase.git" diff --git a/hbase/stackable/patches/2.6.1/patchable.toml b/hbase/stackable/patches/2.6.1/patchable.toml index 509985a8d..c1c9eacbf 100644 --- a/hbase/stackable/patches/2.6.1/patchable.toml +++ b/hbase/stackable/patches/2.6.1/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/hbase.git" base = "7ed50b4dd742269a78875fb32112215f831284ff" mirror = "https://github.com/stackabletech/hbase.git" diff --git a/hive/stackable/patches/3.1.3/patchable.toml b/hive/stackable/patches/3.1.3/patchable.toml index 51f6afb9f..b44ca05a9 100644 --- a/hive/stackable/patches/3.1.3/patchable.toml +++ b/hive/stackable/patches/3.1.3/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/hive.git" base = "4df4d75bf1e16fe0af75aad0b4179c34c07fc975" mirror = "https://github.com/stackabletech/hive.git" diff --git a/hive/stackable/patches/4.0.0/patchable.toml b/hive/stackable/patches/4.0.0/patchable.toml index 2b3bade93..f8bd6ce3f 100644 --- a/hive/stackable/patches/4.0.0/patchable.toml +++ b/hive/stackable/patches/4.0.0/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/hive.git" base = "183f8cb41d3dbed961ffd27999876468ff06690c" mirror = "https://github.com/stackabletech/hive.git" diff --git a/hive/stackable/patches/4.0.1/patchable.toml b/hive/stackable/patches/4.0.1/patchable.toml index b6c5ec3f2..807793caf 100644 --- a/hive/stackable/patches/4.0.1/patchable.toml +++ b/hive/stackable/patches/4.0.1/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/hive.git" base = "3af4517eb8cfd9407ad34ed78a0b48b57dfaa264" mirror = "https://github.com/stackabletech/hive.git" diff --git a/kafka/stackable/patches/3.7.1/patchable.toml b/kafka/stackable/patches/3.7.1/patchable.toml index 587a15c16..742bc6db8 100644 --- a/kafka/stackable/patches/3.7.1/patchable.toml +++ b/kafka/stackable/patches/3.7.1/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/kafka.git" base = "e2494e6ffb89f8288ed2aeb9b5596c755210bffd" mirror = "https://github.com/stackabletech/kafka.git" diff --git a/kafka/stackable/patches/3.7.2/patchable.toml b/kafka/stackable/patches/3.7.2/patchable.toml index 1550583ae..d1f02d916 100644 --- a/kafka/stackable/patches/3.7.2/patchable.toml +++ b/kafka/stackable/patches/3.7.2/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/kafka.git" base = "79a8f2b5f44f9d5a6867190d1dfc463d08d60b82" mirror = "https://github.com/stackabletech/kafka.git" diff --git a/kafka/stackable/patches/3.8.0/patchable.toml b/kafka/stackable/patches/3.8.0/patchable.toml index ce0868779..80b106d44 100644 --- a/kafka/stackable/patches/3.8.0/patchable.toml +++ b/kafka/stackable/patches/3.8.0/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/kafka.git" base = "771b9576b00ecf5b64ab6e8bedf04156fbdb5cd6" mirror = "https://github.com/stackabletech/kafka.git" diff --git a/kafka/stackable/patches/3.9.0/patchable.toml b/kafka/stackable/patches/3.9.0/patchable.toml index 0766f2c0b..ebdd7dfb1 100644 --- a/kafka/stackable/patches/3.9.0/patchable.toml +++ b/kafka/stackable/patches/3.9.0/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/kafka.git" base = "84caaa6e9da06435411510a81fa321d4f99c351f" mirror = "https://github.com/stackabletech/kafka.git" diff --git a/nifi/stackable/patches/1.27.0/patchable.toml b/nifi/stackable/patches/1.27.0/patchable.toml index bd970aeb2..107e4c1c3 100644 --- a/nifi/stackable/patches/1.27.0/patchable.toml +++ b/nifi/stackable/patches/1.27.0/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/nifi.git" base = "e0c4461d90bd4f6e5f2b81765bcff5cd97ed3e18" mirror = "https://github.com/stackabletech/nifi.git" diff --git a/nifi/stackable/patches/1.28.1/patchable.toml b/nifi/stackable/patches/1.28.1/patchable.toml index 1ffc5bb22..b252ecf5a 100644 --- a/nifi/stackable/patches/1.28.1/patchable.toml +++ b/nifi/stackable/patches/1.28.1/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/nifi.git" base = "883338fe28883733417d10f6ffa9319e75f5ea06" mirror = "https://github.com/stackabletech/nifi.git" diff --git a/nifi/stackable/patches/2.2.0/patchable.toml b/nifi/stackable/patches/2.2.0/patchable.toml index baf837669..cca3aee5b 100644 --- a/nifi/stackable/patches/2.2.0/patchable.toml +++ b/nifi/stackable/patches/2.2.0/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/nifi.git" base = "b33ffac8aa10992482f7fa54e6cfccc46a5e8e27" mirror = "https://github.com/stackabletech/nifi.git" diff --git a/omid/stackable/patches/1.1.0/patchable.toml b/omid/stackable/patches/1.1.0/patchable.toml index cdc7d3cec..9fc718c99 100644 --- a/omid/stackable/patches/1.1.0/patchable.toml +++ b/omid/stackable/patches/1.1.0/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/phoenix-omid.git" base = "3b9e16b7537adbc90a7403507fb8aabd8d1fab0c" mirror = "https://github.com/stackabletech/phoenix-omid.git" diff --git a/omid/stackable/patches/1.1.1/patchable.toml b/omid/stackable/patches/1.1.1/patchable.toml index 7e9ab59d6..5f9cf06a6 100644 --- a/omid/stackable/patches/1.1.1/patchable.toml +++ b/omid/stackable/patches/1.1.1/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/phoenix-omid.git" base = "cd546d58d93f380fec9bf65dbfa618f53493f662" mirror = "https://github.com/stackabletech/phoenix-omid.git" diff --git a/omid/stackable/patches/1.1.2/patchable.toml b/omid/stackable/patches/1.1.2/patchable.toml index 08b21ed22..418cf0637 100644 --- a/omid/stackable/patches/1.1.2/patchable.toml +++ b/omid/stackable/patches/1.1.2/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/phoenix-omid.git" base = "88812c9e127063f3b3016262f81ea3e8b48ec157" mirror = "https://github.com/stackabletech/phoenix-omid.git" diff --git a/omid/stackable/patches/1.1.3-SNAPSHOT/patchable.toml b/omid/stackable/patches/1.1.3-SNAPSHOT/patchable.toml index 9fd5987f7..b69b3f86c 100644 --- a/omid/stackable/patches/1.1.3-SNAPSHOT/patchable.toml +++ b/omid/stackable/patches/1.1.3-SNAPSHOT/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/phoenix-omid.git" base = "c3e4da626fdb27060fd139a809e057965e52d163" mirror = "https://github.com/stackabletech/phoenix-omid.git" diff --git a/spark-k8s/hbase-connectors/stackable/patches/1.0.1/patchable.toml b/spark-k8s/hbase-connectors/stackable/patches/1.0.1/patchable.toml index 698d02d46..6d8c2c8db 100644 --- a/spark-k8s/hbase-connectors/stackable/patches/1.0.1/patchable.toml +++ b/spark-k8s/hbase-connectors/stackable/patches/1.0.1/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/hbase-connectors.git" base = "e5217d13ed729703580ff2d1b02378ada2d94f4d" mirror = "https://github.com/stackabletech/hbase-connectors.git" diff --git a/spark-k8s/hbase-connectors/stackable/patches/patchable.toml b/spark-k8s/hbase-connectors/stackable/patches/patchable.toml new file mode 100644 index 000000000..1124d7c9d --- /dev/null +++ b/spark-k8s/hbase-connectors/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/apache/hbase-connectors.git" +default-mirror = "https://github.com/stackabletech/hbase-connectors.git" diff --git a/spark-k8s/stackable/patches/3.5.2/patchable.toml b/spark-k8s/stackable/patches/3.5.2/patchable.toml index f8281c5a5..98aa7ed62 100644 --- a/spark-k8s/stackable/patches/3.5.2/patchable.toml +++ b/spark-k8s/stackable/patches/3.5.2/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/spark.git" base = "bb7846dd487f259994fdc69e18e03382e3f64f42" mirror = "https://github.com/stackabletech/spark.git" diff --git a/spark-k8s/stackable/patches/3.5.5/patchable.toml b/spark-k8s/stackable/patches/3.5.5/patchable.toml index baf047fcc..fbb635ebc 100644 --- a/spark-k8s/stackable/patches/3.5.5/patchable.toml +++ b/spark-k8s/stackable/patches/3.5.5/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/spark.git" base = "7c29c664cdc9321205a98a14858aaf8daaa19db2" mirror = "https://github.com/stackabletech/spark.git" diff --git a/trino-storage-connector/stackable/patches/451/patchable.toml b/trino-storage-connector/stackable/patches/451/patchable.toml index 7f579e201..938a7d526 100644 --- a/trino-storage-connector/stackable/patches/451/patchable.toml +++ b/trino-storage-connector/stackable/patches/451/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/snowlift/trino-storage.git" base = "b6e5825bb84a4f1a3f89ff45ea39ce349313f60a" mirror = "https://github.com/stackabletech/trino-storage.git" diff --git a/trino-storage-connector/stackable/patches/455/patchable.toml b/trino-storage-connector/stackable/patches/455/patchable.toml index dee676dc3..a48fac780 100644 --- a/trino-storage-connector/stackable/patches/455/patchable.toml +++ b/trino-storage-connector/stackable/patches/455/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/snowlift/trino-storage.git" base = "869a735d8be527117a19150e161ad8ca69317578" mirror = "https://github.com/stackabletech/trino-storage.git" diff --git a/trino-storage-connector/stackable/patches/470/patchable.toml b/trino-storage-connector/stackable/patches/470/patchable.toml index 1fa184a4f..039f4a39d 100644 --- a/trino-storage-connector/stackable/patches/470/patchable.toml +++ b/trino-storage-connector/stackable/patches/470/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/snowlift/trino-storage.git" base = "1b25d617940f14a844a43ee34aa705f7d11fbaf9" mirror = "https://github.com/stackabletech/trino-storage.git" diff --git a/trino/stackable/patches/451/patchable.toml b/trino/stackable/patches/451/patchable.toml index 67a023b9c..3a968d254 100644 --- a/trino/stackable/patches/451/patchable.toml +++ b/trino/stackable/patches/451/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/trinodb/trino.git" base = "2c974f7cb1d71e1f9f466941a317190a474fc432" mirror = "https://github.com/stackabletech/trino.git" diff --git a/trino/stackable/patches/455/patchable.toml b/trino/stackable/patches/455/patchable.toml index 30473fdc3..4566cdee5 100644 --- a/trino/stackable/patches/455/patchable.toml +++ b/trino/stackable/patches/455/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/trinodb/trino.git" base = "e212460ea0aa663f0de9b16fecd480c4ad6490cc" mirror = "https://github.com/stackabletech/trino.git" diff --git a/trino/stackable/patches/470/patchable.toml b/trino/stackable/patches/470/patchable.toml index 6567015ef..640675696 100644 --- a/trino/stackable/patches/470/patchable.toml +++ b/trino/stackable/patches/470/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/trinodb/trino.git" base = "05bc059cf0c9263e4ee8be2c1ad69753d0dd4faf" mirror = "https://github.com/stackabletech/trino.git" diff --git a/zookeeper/stackable/patches/3.9.3/patchable.toml b/zookeeper/stackable/patches/3.9.3/patchable.toml index 4e4e9fccd..bcad3c061 100644 --- a/zookeeper/stackable/patches/3.9.3/patchable.toml +++ b/zookeeper/stackable/patches/3.9.3/patchable.toml @@ -1,3 +1,2 @@ -upstream = "https://github.com/apache/zookeeper.git" base = "c26634f34490bb0ea7a09cc51e05ede3b4e320ee" mirror = "https://github.com/stackabletech/zookeeper.git" From 61486086d383ebeaf4f15f34290fbf36c2cde0c7 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Fri, 9 May 2025 12:11:35 +0200 Subject: [PATCH 28/30] fix: copy complete patches directories to include patchable product config --- druid/Dockerfile | 2 +- hadoop/Dockerfile | 2 +- hbase/Dockerfile | 6 +++--- hive/Dockerfile | 2 +- kafka/Dockerfile | 2 +- nifi/Dockerfile | 2 +- omid/Dockerfile | 2 +- spark-k8s/Dockerfile | 4 ++-- trino-storage-connector/Dockerfile | 2 +- trino/Dockerfile | 2 +- zookeeper/Dockerfile | 2 +- 11 files changed, 14 insertions(+), 14 deletions(-) diff --git a/druid/Dockerfile b/druid/Dockerfile index a45d6b02a..56d783cf1 100644 --- a/druid/Dockerfile +++ b/druid/Dockerfile @@ -32,7 +32,7 @@ EOF USER ${STACKABLE_USER_UID} WORKDIR /stackable -COPY --chown=${STACKABLE_USER_UID}:0 druid/stackable/patches/${PRODUCT} /stackable/src/druid/stackable/patches/${PRODUCT} +COPY --chown=${STACKABLE_USER_UID}:0 druid/stackable/patches /stackable/src/druid/stackable/patches # Cache mounts are owned by root by default # We need to explicitly give the uid to use which is hardcoded to "1000" in stackable-base diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index cdbc8caa8..8f200e5cb 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -51,7 +51,7 @@ ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/j EOF WORKDIR /build -COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/patches/${PRODUCT} /build/src/hadoop/stackable/patches/${PRODUCT} +COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/patches /build/src/hadoop/stackable/patches COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/fuse_dfs_wrapper /build COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/jmx /stackable/jmx # Hadoop Pipes requires libtirpc to build, whose headers are not packaged in RedHat UBI, so skip building this module diff --git a/hbase/Dockerfile b/hbase/Dockerfile index ae10f8054..df514128f 100644 --- a/hbase/Dockerfile +++ b/hbase/Dockerfile @@ -26,7 +26,7 @@ COPY hbase/licenses /licenses USER ${STACKABLE_USER_UID} WORKDIR /stackable -COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/patches/${PRODUCT} /stackable/src/hbase/stackable/patches/${PRODUCT} +COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/patches /stackable/src/hbase/stackable/patches COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/jmx/config${JMX_EXPORTER} /stackable/jmx # Cache mounts are owned by root by default @@ -142,7 +142,7 @@ ARG DELETE_CACHES="true" # so that they are not expanded. Disabling ShellCheck rules in a Dockerfile # does not work, so please ignore the according warning (SC2016). COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/bin/hbck2.env /stackable/bin/ -COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-operator-tools/stackable/patches/${HBASE_OPERATOR_TOOLS} /stackable/src/hbase-operator-tools/stackable/patches/${HBASE_OPERATOR_TOOLS} +COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-operator-tools/stackable/patches /stackable/src/hbase-operator-tools/stackable/patches COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/bin/hbase-entrypoint.sh /stackable/bin/ USER ${STACKABLE_USER_UID} @@ -241,7 +241,7 @@ ARG STACKABLE_USER_UID # This can be used to speed up builds when disk space is of no concern. ARG DELETE_CACHES="true" -COPY --chown=${STACKABLE_USER_UID}:0 hbase/phoenix/stackable/patches/${PHOENIX} /stackable/src/phoenix/stackable/patches/${PHOENIX} +COPY --chown=${STACKABLE_USER_UID}:0 hbase/phoenix/stackable/patches /stackable/src/phoenix/stackable/patches USER ${STACKABLE_USER_UID} WORKDIR /stackable diff --git a/hive/Dockerfile b/hive/Dockerfile index 65c7209b5..b73aab817 100644 --- a/hive/Dockerfile +++ b/hive/Dockerfile @@ -23,7 +23,7 @@ ARG STACKABLE_USER_UID ARG DELETE_CACHES="true" # Copy patches into the builder -COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/patches/${PRODUCT} /stackable/src/hive/stackable/patches/${PRODUCT} +COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/patches /stackable/src/hive/stackable/patches # It is useful to see which version of Hadoop is used at a glance # Therefore the use of the full name here # TODO: Do we really need all of Hadoop in here? diff --git a/kafka/Dockerfile b/kafka/Dockerfile index ff8f7a5d4..b2cf879e4 100644 --- a/kafka/Dockerfile +++ b/kafka/Dockerfile @@ -15,7 +15,7 @@ USER ${STACKABLE_USER_UID} WORKDIR /stackable COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/jmx/ /stackable/jmx/ -COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/patches/${PRODUCT} /stackable/src/kafka/stackable/patches/${PRODUCT} +COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/patches /stackable/src/kafka/stackable/patches RUN < Date: Fri, 9 May 2025 12:24:44 +0200 Subject: [PATCH 29/30] Revert "fix: copy complete patches directories to include patchable product config" This reverts commit 61486086d383ebeaf4f15f34290fbf36c2cde0c7. --- druid/Dockerfile | 2 +- hadoop/Dockerfile | 2 +- hbase/Dockerfile | 6 +++--- hive/Dockerfile | 2 +- kafka/Dockerfile | 2 +- nifi/Dockerfile | 2 +- omid/Dockerfile | 2 +- spark-k8s/Dockerfile | 4 ++-- trino-storage-connector/Dockerfile | 2 +- trino/Dockerfile | 2 +- zookeeper/Dockerfile | 2 +- 11 files changed, 14 insertions(+), 14 deletions(-) diff --git a/druid/Dockerfile b/druid/Dockerfile index 56d783cf1..a45d6b02a 100644 --- a/druid/Dockerfile +++ b/druid/Dockerfile @@ -32,7 +32,7 @@ EOF USER ${STACKABLE_USER_UID} WORKDIR /stackable -COPY --chown=${STACKABLE_USER_UID}:0 druid/stackable/patches /stackable/src/druid/stackable/patches +COPY --chown=${STACKABLE_USER_UID}:0 druid/stackable/patches/${PRODUCT} /stackable/src/druid/stackable/patches/${PRODUCT} # Cache mounts are owned by root by default # We need to explicitly give the uid to use which is hardcoded to "1000" in stackable-base diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index 8f200e5cb..cdbc8caa8 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -51,7 +51,7 @@ ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/j EOF WORKDIR /build -COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/patches /build/src/hadoop/stackable/patches +COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/patches/${PRODUCT} /build/src/hadoop/stackable/patches/${PRODUCT} COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/fuse_dfs_wrapper /build COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/jmx /stackable/jmx # Hadoop Pipes requires libtirpc to build, whose headers are not packaged in RedHat UBI, so skip building this module diff --git a/hbase/Dockerfile b/hbase/Dockerfile index df514128f..ae10f8054 100644 --- a/hbase/Dockerfile +++ b/hbase/Dockerfile @@ -26,7 +26,7 @@ COPY hbase/licenses /licenses USER ${STACKABLE_USER_UID} WORKDIR /stackable -COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/patches /stackable/src/hbase/stackable/patches +COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/patches/${PRODUCT} /stackable/src/hbase/stackable/patches/${PRODUCT} COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/jmx/config${JMX_EXPORTER} /stackable/jmx # Cache mounts are owned by root by default @@ -142,7 +142,7 @@ ARG DELETE_CACHES="true" # so that they are not expanded. Disabling ShellCheck rules in a Dockerfile # does not work, so please ignore the according warning (SC2016). COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/bin/hbck2.env /stackable/bin/ -COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-operator-tools/stackable/patches /stackable/src/hbase-operator-tools/stackable/patches +COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-operator-tools/stackable/patches/${HBASE_OPERATOR_TOOLS} /stackable/src/hbase-operator-tools/stackable/patches/${HBASE_OPERATOR_TOOLS} COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/bin/hbase-entrypoint.sh /stackable/bin/ USER ${STACKABLE_USER_UID} @@ -241,7 +241,7 @@ ARG STACKABLE_USER_UID # This can be used to speed up builds when disk space is of no concern. ARG DELETE_CACHES="true" -COPY --chown=${STACKABLE_USER_UID}:0 hbase/phoenix/stackable/patches /stackable/src/phoenix/stackable/patches +COPY --chown=${STACKABLE_USER_UID}:0 hbase/phoenix/stackable/patches/${PHOENIX} /stackable/src/phoenix/stackable/patches/${PHOENIX} USER ${STACKABLE_USER_UID} WORKDIR /stackable diff --git a/hive/Dockerfile b/hive/Dockerfile index b73aab817..65c7209b5 100644 --- a/hive/Dockerfile +++ b/hive/Dockerfile @@ -23,7 +23,7 @@ ARG STACKABLE_USER_UID ARG DELETE_CACHES="true" # Copy patches into the builder -COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/patches /stackable/src/hive/stackable/patches +COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/patches/${PRODUCT} /stackable/src/hive/stackable/patches/${PRODUCT} # It is useful to see which version of Hadoop is used at a glance # Therefore the use of the full name here # TODO: Do we really need all of Hadoop in here? diff --git a/kafka/Dockerfile b/kafka/Dockerfile index b2cf879e4..ff8f7a5d4 100644 --- a/kafka/Dockerfile +++ b/kafka/Dockerfile @@ -15,7 +15,7 @@ USER ${STACKABLE_USER_UID} WORKDIR /stackable COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/jmx/ /stackable/jmx/ -COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/patches /stackable/src/kafka/stackable/patches +COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/patches/${PRODUCT} /stackable/src/kafka/stackable/patches/${PRODUCT} RUN < Date: Fri, 9 May 2025 12:33:01 +0200 Subject: [PATCH 30/30] fix: copy patchable product config as well --- druid/Dockerfile | 1 + hadoop/Dockerfile | 1 + hbase/Dockerfile | 3 +++ hive/Dockerfile | 1 + kafka/Dockerfile | 1 + nifi/Dockerfile | 1 + omid/Dockerfile | 1 + spark-k8s/Dockerfile | 2 ++ trino-storage-connector/Dockerfile | 1 + trino/Dockerfile | 1 + zookeeper/Dockerfile | 1 + 11 files changed, 14 insertions(+) diff --git a/druid/Dockerfile b/druid/Dockerfile index a45d6b02a..7acb8f3f1 100644 --- a/druid/Dockerfile +++ b/druid/Dockerfile @@ -32,6 +32,7 @@ EOF USER ${STACKABLE_USER_UID} WORKDIR /stackable +COPY --chown=${STACKABLE_USER_UID}:0 druid/stackable/patches/patchable.toml /stackable/src/druid/stackable/patches/patchable.toml COPY --chown=${STACKABLE_USER_UID}:0 druid/stackable/patches/${PRODUCT} /stackable/src/druid/stackable/patches/${PRODUCT} # Cache mounts are owned by root by default diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index cdbc8caa8..990425c5f 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -51,6 +51,7 @@ ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/j EOF WORKDIR /build +COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/patches/patchable.toml /build/src/hadoop/stackable/patches/patchable.toml COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/patches/${PRODUCT} /build/src/hadoop/stackable/patches/${PRODUCT} COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/fuse_dfs_wrapper /build COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/jmx /stackable/jmx diff --git a/hbase/Dockerfile b/hbase/Dockerfile index ae10f8054..4a0ba9dd5 100644 --- a/hbase/Dockerfile +++ b/hbase/Dockerfile @@ -26,6 +26,7 @@ COPY hbase/licenses /licenses USER ${STACKABLE_USER_UID} WORKDIR /stackable +COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/patches/patchable.toml /stackable/src/hbase/stackable/patches/patchable.toml COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/patches/${PRODUCT} /stackable/src/hbase/stackable/patches/${PRODUCT} COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/jmx/config${JMX_EXPORTER} /stackable/jmx @@ -142,6 +143,7 @@ ARG DELETE_CACHES="true" # so that they are not expanded. Disabling ShellCheck rules in a Dockerfile # does not work, so please ignore the according warning (SC2016). COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/bin/hbck2.env /stackable/bin/ +COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-operator-tools/stackable/patches/patchable.toml /stackable/src/hbase-operator-tools/stackable/patches/patchable.toml COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-operator-tools/stackable/patches/${HBASE_OPERATOR_TOOLS} /stackable/src/hbase-operator-tools/stackable/patches/${HBASE_OPERATOR_TOOLS} COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/bin/hbase-entrypoint.sh /stackable/bin/ @@ -241,6 +243,7 @@ ARG STACKABLE_USER_UID # This can be used to speed up builds when disk space is of no concern. ARG DELETE_CACHES="true" +COPY --chown=${STACKABLE_USER_UID}:0 hbase/phoenix/stackable/patches/patchable.toml /stackable/src/phoenix/stackable/patches/patchable.toml COPY --chown=${STACKABLE_USER_UID}:0 hbase/phoenix/stackable/patches/${PHOENIX} /stackable/src/phoenix/stackable/patches/${PHOENIX} USER ${STACKABLE_USER_UID} WORKDIR /stackable diff --git a/hive/Dockerfile b/hive/Dockerfile index 65c7209b5..e04987743 100644 --- a/hive/Dockerfile +++ b/hive/Dockerfile @@ -23,6 +23,7 @@ ARG STACKABLE_USER_UID ARG DELETE_CACHES="true" # Copy patches into the builder +COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/patches/patchable.toml /stackable/src/hive/stackable/patches/patchable.toml COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/patches/${PRODUCT} /stackable/src/hive/stackable/patches/${PRODUCT} # It is useful to see which version of Hadoop is used at a glance # Therefore the use of the full name here diff --git a/kafka/Dockerfile b/kafka/Dockerfile index ff8f7a5d4..df64d10cb 100644 --- a/kafka/Dockerfile +++ b/kafka/Dockerfile @@ -15,6 +15,7 @@ USER ${STACKABLE_USER_UID} WORKDIR /stackable COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/jmx/ /stackable/jmx/ +COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/patches/patchable.toml /stackable/src/kafka/stackable/patches/patchable.toml COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/patches/${PRODUCT} /stackable/src/kafka/stackable/patches/${PRODUCT} RUN <