chore: support for Spark 3.5.6 (#580)

* chore: bump spark version to 3.5.6

* update changelog

* Update docs/modules/spark-k8s/partials/supported-versions.adoc

Co-authored-by: Nick <[email protected]>

* fix image name

* update test artefact

* fix md lint

---------

Co-authored-by: Nick <[email protected]>

Author: razvan

CHANGELOG.md

@@ -12,6 +12,7 @@ All notable changes to this project will be documented in this file.
   - Use `--file-log-rotation-period` (or `FILE_LOG_ROTATION_PERIOD`) to configure the frequency of rotation.
   - Use `--console-log-format` (or `CONSOLE_LOG_FORMAT`) to set the format to `plain` (default) or `json`.
 - Expose history and connect services via listener classes ([#562]).
+- Support for Spark 3.5.6 ([#580]).
 
 ### Changed
 
@@ -56,6 +57,7 @@ All notable changes to this project will be documented in this file.
 [#572]: https://github.com/stackabletech/spark-k8s-operator/pull/572
 [#573]: https://github.com/stackabletech/spark-k8s-operator/pull/573
 [#574]: https://github.com/stackabletech/spark-k8s-operator/pull/574
+[#580]: https://github.com/stackabletech/spark-k8s-operator/pull/580
 [#575]: https://github.com/stackabletech/spark-k8s-operator/pull/575
 
 ## [25.3.0] - 2025-03-21

docs/modules/spark-k8s/examples/example-history-app.yaml

@@ -5,7 +5,7 @@ metadata:
   name: spark-pi-s3-1
 spec:
   sparkImage:
-    productVersion: 3.5.5
+    productVersion: 3.5.6
     pullPolicy: IfNotPresent
   mode: cluster
   mainClass: org.apache.spark.examples.SparkPi

docs/modules/spark-k8s/examples/example-history-server.yaml

@@ -5,7 +5,7 @@ metadata:
   name: spark-history
 spec:
   image:
-    productVersion: 3.5.5
+    productVersion: 3.5.6
   logFileDirectory:  # <1>
     s3:
       prefix: eventlogs/  # <2>

docs/modules/spark-k8s/examples/example-spark-connect.yaml

@@ -5,7 +5,7 @@ metadata:
   name: spark-connect # <1>
 spec:
   image:
-    productVersion: "3.5.5" # <2>
+    productVersion: "3.5.6" # <2>
     pullPolicy: IfNotPresent
   args:
     - "--package org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1" # <3>

docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml

@@ -6,7 +6,7 @@ metadata:
   namespace: default
 spec:
   sparkImage:
-    productVersion: 3.5.5
+    productVersion: 3.5.6
   mode: cluster
   mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny-tlc-report-1.1.0.jar # <3>
   mainClass: tech.stackable.demo.spark.NYTLCReport

docs/modules/spark-k8s/examples/example-sparkapp-image.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   image: oci.stackable.tech/stackable/ny-tlc-report:0.2.0 # <1>
   sparkImage:
-    productVersion: 3.5.5
+    productVersion: 3.5.6
   mode: cluster
   mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py # <2>
   args:

docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml

@@ -6,7 +6,7 @@ metadata:
   namespace: default
 spec:
   sparkImage:
-    productVersion: 3.5.5
+    productVersion: 3.5.6
   mode: cluster
   mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny-tlc-report-1.0-SNAPSHOT.jar # <1>
   mainClass: org.example.App # <2>

docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml

@@ -5,7 +5,7 @@ metadata:
   name: example-sparkapp-s3-private
 spec:
   sparkImage:
-    productVersion: 3.5.5
+    productVersion: 3.5.6
   mode: cluster
   mainApplicationFile: s3a://my-bucket/spark-examples.jar # <1>
   mainClass: org.apache.spark.examples.SparkPi # <2>

docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml

@@ -6,7 +6,7 @@ metadata:
   namespace: default
 spec:
   sparkImage:
-    productVersion: 3.5.5
+    productVersion: 3.5.6
   mode: cluster
   mainApplicationFile: local:///stackable/spark/examples/src/main/python/streaming/hdfs_wordcount.py
   args:

docs/modules/spark-k8s/examples/getting_started/getting_started.sh

@@ -56,7 +56,7 @@ metadata:
   namespace: default
 spec:
   sparkImage:
-    productVersion: 3.5.5
+    productVersion: 3.5.6
   mode: cluster
   mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py
   driver:

docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2

@@ -56,7 +56,7 @@ metadata:
   namespace: default
 spec:
   sparkImage:
-    productVersion: 3.5.5
+    productVersion: 3.5.6
   mode: cluster
   mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py
   driver:

docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc

@@ -58,7 +58,7 @@ Below is an example of a custom image that includes a JDBC driver:
 
 [source, Dockerfile]
 ----
-FROM oci.stackable.tech/sdp/spark-k8s:3.5.5-stackable25.3.0 # <1>
+FROM oci.stackable.tech/sdp/spark-k8s:3.5.6-stackable25.3.0 # <1>
 
 RUN curl --fail -o /stackable/spark/jars/postgresql-42.6.0.jar "https://jdbc.postgresql.org/download/postgresql-42.6.0.jar"
 ----
@@ -75,8 +75,8 @@ metadata:
   name: spark-jdbc
 spec:
   sparkImage:
-    custom: "oci.stackable.tech/sandbox/spark-k8s:3.5.5-stackable0.0.0-dev" # <1>
-    productVersion: "3.5.5" # <2>
+    custom: "oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-dev" # <1>
+    productVersion: "3.5.6" # <2>
     pullPolicy: IfNotPresent # <3>
 ...
 ----

docs/modules/spark-k8s/partials/supported-versions.adoc

@@ -3,4 +3,5 @@
 // Stackable Platform documentation.
 // Please sort the versions in descending order (newest first)
 
-- 3.5.5 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 17) (LTS)
+- 3.5.5 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 17) (Deprecated)
+- 3.5.6 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 17) (LTS)

examples/README-examples.md

@@ -50,10 +50,10 @@ Several resources are needed in this store. These can be loaded like this:
 
 ````text
 kubectl exec minio-mc-0 -- sh -c 'mc alias set test-minio http://test-minio:9000/'
-kubectl cp examples/ny-tlc-report-1.1.0-3.5.5.jar  minio-mc-0:/tmp
+kubectl cp examples/ny-tlc-report-1.1.0-3.5.6.jar  minio-mc-0:/tmp
 kubectl cp apps/ny_tlc_report.py  minio-mc-0:/tmp
 kubectl cp examples/yellow_tripdata_2021-07.csv  minio-mc-0:/tmp
-kubectl exec minio-mc-0 -- mc cp /tmp/ny-tlc-report-1.1.0-3.5.5.jar test-minio/my-bucket
+kubectl exec minio-mc-0 -- mc cp /tmp/ny-tlc-report-1.1.0-3.5.6.jar test-minio/my-bucket
 kubectl exec minio-mc-0 -- mc cp /tmp/ny_tlc_report.py test-minio/my-bucket
 kubectl exec minio-mc-0 -- mc cp /tmp/yellow_tripdata_2021-07.csv test-minio/my-bucket
 ````

examples/ny-tlc-report-external-dependencies.yaml

@@ -6,7 +6,7 @@ metadata:
   namespace: default
 spec:
   sparkImage:
-    productVersion: 3.5.5
+    productVersion: 3.5.6
     pullPolicy: IfNotPresent
   mode: cluster
   mainApplicationFile: s3a://my-bucket/ny_tlc_report.py

examples/ny-tlc-report-image.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   # everything under /jobs will be copied to /stackable/spark/jobs
   image: oci.stackable.tech/stackable/ny-tlc-report:0.2.0
-  sparkImage: oci.stackable.tech/sdp/spark-k8s:3.5.5-stackable0.0.0-dev
+  sparkImage: oci.stackable.tech/sdp/spark-k8s:3.5.6-stackable0.0.0-dev
   sparkImagePullPolicy: IfNotPresent
   mode: cluster
   mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py

examples/ny-tlc-report.yaml

@@ -13,9 +13,9 @@ metadata:
   name: spark-ny-cm
 spec:
   sparkImage:
-    productVersion: 3.5.5
+    productVersion: 3.5.6
   mode: cluster
-  mainApplicationFile: s3a://my-bucket/ny-tlc-report-1.1.0-3.5.5.jar
+  mainApplicationFile: s3a://my-bucket/ny-tlc-report-1.1.0-3.5.6.jar
   mainClass: tech.stackable.demo.spark.NYTLCReport
   volumes:
     - name: cm-job-arguments

rust/operator-binary/src/crd/affinity.rs

@@ -47,7 +47,7 @@ mod test {
           name: spark-history
         spec:
           image:
-            productVersion: 3.5.5
+            productVersion: 3.5.6
           logFileDirectory:
             s3:
               prefix: eventlogs/

rust/operator-binary/src/crd/history.rs

@@ -425,7 +425,7 @@ mod test {
           name: spark-history
         spec:
           image:
-            productVersion: 3.5.5
+            productVersion: 3.5.6
           logFileDirectory:
             s3:
               prefix: eventlogs/

rust/operator-binary/src/history/config/jvm.rs

@@ -67,7 +67,7 @@ mod tests {
           name: spark-history
         spec:
           image:
-            productVersion: 3.5.5
+            productVersion: 3.5.6
           logFileDirectory:
             s3:
               prefix: eventlogs/
@@ -100,7 +100,7 @@ mod tests {
           name: spark-history
         spec:
           image:
-            productVersion: 3.5.5
+            productVersion: 3.5.6
           logFileDirectory:
             s3:
               prefix: eventlogs/

tests/templates/kuttl/spark-connect/20-run-connect-client.yaml.j2

@@ -60,7 +60,7 @@ spec:
 {% if test_scenario['values']['spark-connect-client'].find(",") > 0 %}
           image: "{{ test_scenario['values']['spark-connect-client'].split(',')[1] }}"
 {% else %}
-          image: oci.stackable.tech/sdp/spark-connect-client:{{ test_scenario['values']['spark-connect-client'] }}-stackable0.0.0-dev
+          image: oci.stackable.tech/stackable/spark-connect-client:{{ test_scenario['values']['spark-connect-client'] }}-stackable0.0.0-dev
 {% endif %}
           imagePullPolicy: IfNotPresent
           command:

tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.2.jar renamed to tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.6.jar

@@ -6,21 +6,22 @@ dimensions:
   - name: spark
     values:
       - 3.5.5
+      - 3.5.6
       # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version
       # as in the example below.
-      # - 3.5.5,oci.stackable.tech/sandbox/spark-k8s:3.5.5-stackable0.0.0-dev
+      # - 3.5.6,oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-dev
   - name: spark-delta-lake
     values:
-      - 3.5.5
-      # - 3.5.5,oci.stackable.tech/sandbox/spark-k8s:3.5.5-stackable0.0.0-dev
+      - 3.5.6
+      # - 3.5.6,oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-dev
   - name: spark-connect
     values:
-      - 3.5.5
-      # - 3.5.5,oci.stackable.tech/sandbox/spark-k8s:3.5.5-stackable0.0.0-dev
+      - 3.5.6
+      # - 3.5.6,oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-dev
   - name: spark-connect-client
     values:
-      - 3.5.5
-      # - 3.5.5,oci.stackable.tech/sandbox/spark-k8s:3.5.5-stackable0.0.0-dev
+      - 3.5.6
+      # - 3.5.6,oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-dev
   - name: hbase
     values:
       - 2.6.2