feat: patchable init mirroring

Cargo.toml

@@ -4,7 +4,7 @@ resolver = "2"
 
 [workspace.dependencies]
 clap = { version = "4.5.27", features = ["derive"] }
-git2 = "0.20.0"
+git2 = "0.20.1"
 serde = { version = "1.0.217", features = ["derive"] }
 snafu = "0.8.5"
 tempfile = "3.16.0"

druid/Dockerfile

@@ -32,6 +32,7 @@ EOF
 USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
+COPY --chown=${STACKABLE_USER_UID}:0 druid/stackable/patches/patchable.toml /stackable/src/druid/stackable/patches/patchable.toml
 COPY --chown=${STACKABLE_USER_UID}:0 druid/stackable/patches/${PRODUCT} /stackable/src/druid/stackable/patches/${PRODUCT}
 
 # Cache mounts are owned by root by default

druid/stackable/patches/30.0.0/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/druid.git"
 base = "09d36ee324747f1407705c27618b6d415c3fa8a9"
+mirror = "https://github.com/stackabletech/druid.git"

druid/stackable/patches/30.0.1/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/druid.git"
 base = "a30af7a91d528e5c3a90356a5592abc7119191c6"
+mirror = "https://github.com/stackabletech/druid.git"

druid/stackable/patches/31.0.1/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/druid.git"
 base = "520482cb9638e452b0553595b4f29bb397a63758"
+mirror = "https://github.com/stackabletech/druid.git"

druid/stackable/patches/patchable.toml

@@ -0,0 +1,2 @@
+upstream = "https://github.com/apache/druid.git"
+default-mirror = "https://github.com/stackabletech/druid.git"

hadoop/Dockerfile

@@ -51,6 +51,7 @@ ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/j
 EOF
 
 WORKDIR /build
+COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/patches/patchable.toml /build/src/hadoop/stackable/patches/patchable.toml
 COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/patches/${PRODUCT} /build/src/hadoop/stackable/patches/${PRODUCT}
 COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/fuse_dfs_wrapper /build
 COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/jmx /stackable/jmx

hadoop/stackable/patches/3.3.4/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/hadoop.git"
 base = "a585a73c3e02ac62350c136643a5e7f6095a3dbb"
+mirror = "https://github.com/stackabletech/hadoop.git"

hadoop/stackable/patches/3.3.6/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/hadoop.git"
 base = "1be78238728da9266a4f88195058f08fd012bf9c"
+mirror = "https://github.com/stackabletech/hadoop.git"

hadoop/stackable/patches/3.4.0/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/hadoop.git"
 base = "bd8b77f398f626bb7791783192ee7a5dfaeec760"
+mirror = "https://github.com/stackabletech/hadoop.git"

hadoop/stackable/patches/3.4.1/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/hadoop.git"
 base = "4d7825309348956336b8f06a08322b78422849b1"
+mirror = "https://github.com/stackabletech/hadoop.git"

hadoop/stackable/patches/patchable.toml

@@ -0,0 +1,2 @@
+upstream = "https://github.com/apache/hadoop.git"
+default-mirror = "https://github.com/stackabletech/hadoop.git"

hbase/Dockerfile

@@ -26,6 +26,7 @@ COPY hbase/licenses /licenses
 USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
+COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/patches/patchable.toml /stackable/src/hbase/stackable/patches/patchable.toml
 COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/patches/${PRODUCT} /stackable/src/hbase/stackable/patches/${PRODUCT}
 COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/jmx/config${JMX_EXPORTER} /stackable/jmx
 
@@ -142,6 +143,7 @@ ARG DELETE_CACHES="true"
 # so that they are not expanded. Disabling ShellCheck rules in a Dockerfile
 # does not work, so please ignore the according warning (SC2016).
 COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/bin/hbck2.env /stackable/bin/
+COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-operator-tools/stackable/patches/patchable.toml /stackable/src/hbase-operator-tools/stackable/patches/patchable.toml
 COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-operator-tools/stackable/patches/${HBASE_OPERATOR_TOOLS} /stackable/src/hbase-operator-tools/stackable/patches/${HBASE_OPERATOR_TOOLS}
 COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/bin/hbase-entrypoint.sh /stackable/bin/
 
@@ -241,6 +243,7 @@ ARG STACKABLE_USER_UID
 # This can be used to speed up builds when disk space is of no concern.
 ARG DELETE_CACHES="true"
 
+COPY --chown=${STACKABLE_USER_UID}:0 hbase/phoenix/stackable/patches/patchable.toml /stackable/src/phoenix/stackable/patches/patchable.toml
 COPY --chown=${STACKABLE_USER_UID}:0 hbase/phoenix/stackable/patches/${PHOENIX} /stackable/src/phoenix/stackable/patches/${PHOENIX}
 USER ${STACKABLE_USER_UID}
 WORKDIR /stackable

hbase/hbase-operator-tools/stackable/patches/1.2.0/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/hbase-operator-tools.git"
 base = "478af00af79f82624264fd2bb447b97fecc8e790"
+mirror = "https://github.com/stackabletech/hbase-operator-tools.git"

hbase/hbase-operator-tools/stackable/patches/1.3.0-fd5a5fb/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/hbase-operator-tools.git"
 base = "fd5a5fb90755949a90c502c76de8313130403fa3"
+mirror = "https://github.com/stackabletech/hbase-operator-tools.git"

hbase/hbase-operator-tools/stackable/patches/patchable.toml

@@ -0,0 +1,2 @@
+upstream = "https://github.com/apache/hbase-operator-tools.git"
+default-mirror = "https://github.com/stackabletech/hbase-operator-tools.git"

hbase/phoenix/stackable/patches/5.2.1/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/phoenix.git"
 base = "b738d66cb5863b759bb98eaa417b3b5731d41f95"
+mirror = "https://github.com/stackabletech/phoenix.git"

hbase/phoenix/stackable/patches/patchable.toml

@@ -0,0 +1,2 @@
+upstream = "https://github.com/apache/phoenix.git"
+default-mirror = "https://github.com/stackabletech/phoenix.git"

hbase/stackable/patches/2.4.18/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/hbase.git"
 base = "a1767f4d76859c0068720a6c1e5cb78282ebfe1e"
+mirror = "https://github.com/stackabletech/hbase.git"

hbase/stackable/patches/2.6.0/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/hbase.git"
 base = "de99f8754135ea69adc39da48d2bc2b2710a5366"
+mirror = "https://github.com/stackabletech/hbase.git"

hbase/stackable/patches/2.6.1/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/hbase.git"
 base = "7ed50b4dd742269a78875fb32112215f831284ff"
+mirror = "https://github.com/stackabletech/hbase.git"

hbase/stackable/patches/patchable.toml

@@ -0,0 +1,2 @@
+upstream = "https://github.com/apache/hbase.git"
+default-mirror = "https://github.com/stackabletech/hbase.git"

hive/Dockerfile

@@ -23,6 +23,7 @@ ARG STACKABLE_USER_UID
 ARG DELETE_CACHES="true"
 
 # Copy patches into the builder
+COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/patches/patchable.toml /stackable/src/hive/stackable/patches/patchable.toml
 COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/patches/${PRODUCT} /stackable/src/hive/stackable/patches/${PRODUCT}
 # It is useful to see which version of Hadoop is used at a glance
 # Therefore the use of the full name here

hive/stackable/patches/3.1.3/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/hive.git"
 base = "4df4d75bf1e16fe0af75aad0b4179c34c07fc975"
+mirror = "https://github.com/stackabletech/hive.git"

hive/stackable/patches/4.0.0/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/hive.git"
 base = "183f8cb41d3dbed961ffd27999876468ff06690c"
+mirror = "https://github.com/stackabletech/hive.git"

hive/stackable/patches/4.0.1/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/hive.git"
 base = "3af4517eb8cfd9407ad34ed78a0b48b57dfaa264"
+mirror = "https://github.com/stackabletech/hive.git"

hive/stackable/patches/patchable.toml

@@ -0,0 +1,2 @@
+upstream = "https://github.com/apache/hive.git"
+default-mirror = "https://github.com/stackabletech/hive.git"

kafka/Dockerfile

@@ -15,6 +15,7 @@ USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
 COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/jmx/ /stackable/jmx/
+COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/patches/patchable.toml /stackable/src/kafka/stackable/patches/patchable.toml
 COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/patches/${PRODUCT} /stackable/src/kafka/stackable/patches/${PRODUCT}
 
 RUN <<EOF

kafka/stackable/patches/3.7.1/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/kafka.git"
 base = "e2494e6ffb89f8288ed2aeb9b5596c755210bffd"
+mirror = "https://github.com/stackabletech/kafka.git"

kafka/stackable/patches/3.7.2/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/kafka.git"
 base = "79a8f2b5f44f9d5a6867190d1dfc463d08d60b82"
+mirror = "https://github.com/stackabletech/kafka.git"

kafka/stackable/patches/3.8.0/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/kafka.git"
 base = "771b9576b00ecf5b64ab6e8bedf04156fbdb5cd6"
+mirror = "https://github.com/stackabletech/kafka.git"

kafka/stackable/patches/3.9.0/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/kafka.git"
 base = "84caaa6e9da06435411510a81fa321d4f99c351f"
+mirror = "https://github.com/stackabletech/kafka.git"

kafka/stackable/patches/patchable.toml

@@ -0,0 +1,2 @@
+upstream = "https://github.com/apache/kafka.git"
+default-mirror = "https://github.com/stackabletech/kafka.git"

nifi/Dockerfile

@@ -30,6 +30,7 @@ EOF
 USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
+COPY --chown=${STACKABLE_USER_UID}:0 nifi/stackable/patches/patchable.toml /stackable/src/nifi/stackable/patches/patchable.toml
 COPY --chown=${STACKABLE_USER_UID}:0 nifi/stackable/patches/${PRODUCT} /stackable/src/nifi/stackable/patches/${PRODUCT}
 
 RUN <<EOF

nifi/stackable/patches/1.27.0/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/nifi.git"
 base = "e0c4461d90bd4f6e5f2b81765bcff5cd97ed3e18"
+mirror = "https://github.com/stackabletech/nifi.git"

nifi/stackable/patches/1.28.1/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/nifi.git"
 base = "883338fe28883733417d10f6ffa9319e75f5ea06"
+mirror = "https://github.com/stackabletech/nifi.git"

nifi/stackable/patches/2.2.0/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/nifi.git"
 base = "b33ffac8aa10992482f7fa54e6cfccc46a5e8e27"
+mirror = "https://github.com/stackabletech/nifi.git"

nifi/stackable/patches/patchable.toml

@@ -0,0 +1,2 @@
+upstream = "https://github.com/apache/nifi.git"
+default-mirror = "https://github.com/stackabletech/nifi.git"

omid/Dockerfile

@@ -21,6 +21,7 @@ EOF
 USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
+COPY --chown=${STACKABLE_USER_UID}:0 omid/stackable/patches/patchable.toml /stackable/src/omid/stackable/patches/patchable.toml
 COPY --chown=${STACKABLE_USER_UID}:0 omid/stackable/patches/${PRODUCT} /stackable/src/omid/stackable/patches/${PRODUCT}
 
 RUN --mount=type=cache,id=maven-omid-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository <<EOF

omid/stackable/patches/1.1.0/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/phoenix-omid.git"
 base = "3b9e16b7537adbc90a7403507fb8aabd8d1fab0c"
+mirror = "https://github.com/stackabletech/phoenix-omid.git"

omid/stackable/patches/1.1.1/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/phoenix-omid.git"
 base = "cd546d58d93f380fec9bf65dbfa618f53493f662"
+mirror = "https://github.com/stackabletech/phoenix-omid.git"

omid/stackable/patches/1.1.2/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/phoenix-omid.git"
 base = "88812c9e127063f3b3016262f81ea3e8b48ec157"
+mirror = "https://github.com/stackabletech/phoenix-omid.git"

omid/stackable/patches/1.1.3-SNAPSHOT/patchable.toml

@@ -1,2 +1,2 @@
-upstream = "https://github.com/apache/phoenix-omid.git"
 base = "c3e4da626fdb27060fd139a809e057965e52d163"
+mirror = "https://github.com/stackabletech/phoenix-omid.git"

omid/stackable/patches/patchable.toml

@@ -0,0 +1,2 @@
+upstream = "https://github.com/apache/phoenix-omid.git"
+default-mirror = "https://github.com/stackabletech/phoenix-omid.git"

rust/patchable/README.md

@@ -31,20 +31,40 @@ For more details, run `cargo patchable --help`.
 
 ## Configuration
 
-Patchable stores a per-version file in `docker-images/<PRODUCT>/stackable/patches/<VERSION>/patchable.toml`.
-It currently recognizes the following keys:
+Patchable uses a two-level configuration system:
+
+1. A product-level config file at `docker-images/<PRODUCT>/stackable/patches/patchable.toml`
+2. A version-level config file at `docker-images/<PRODUCT>/stackable/patches/<VERSION>/patchable.toml`
+
+The product-level config contains:
 
 - `upstream` - the URL of the upstream repository (such as `https://github.com/apache/druid.git`)
-- `base` - the commit hash of the upstream base commit (such as `7cffb81a8e124d5f218f9af16ad685acf5e9c67c`)
+- `default_mirror` - optional: default URL of a mirror repository (such as `https://github.com/stackabletech/druid.git`)
+
+The version-level config contains:
+
+- `base` - the commit hash of the upstream base commit
+- `mirror` - optional: URL of the mirror repository for this version, if mirroring is enabled
 
 ### Template
 
-Instead of creating this manually, run `patchable init`:
+If you're adding a completely new product, you need to create the product-level config once:
 
 ```toml
-cargo patchable init druid 28.0.0 --upstream=https://github.com/apache/druid.git --base=druid-28.0.0
+# docker-images/druid/stackable/patches/patchable.toml
+upstream = "https://github.com/apache/druid.git"
+mirror = "https://github.com/stackabletech/druid.git"
 ```
 
+If you just want to add a new version, initialize the version-level config with patchable:
+
+```sh
+cargo patchable init druid 28.0.0 --base=druid-28.0.0 --mirror
+```
+
+This will initialize the version-level config with the base commit hash and the default mirror URL from the product-level config.
+You can optionally provide the `--ssh` flag to use SSH instead of HTTPS for Git operations.
+
 ## Glossary
 
 - Images repo/directory - The checkout of stackabletech/docker-images