Switch gpu-operator-validator to distroless base image

Signed-off-by: Christopher Desiniotis <[email protected]>

Author: cdesiniotis

validator/Dockerfile

@@ -51,22 +51,16 @@ FROM ${CUDA_SAMPLE_IMAGE} AS sample-builder
 RUN mkdir /artifacts
 RUN cp /cuda-samples/vectorAdd /artifacts/vectorAdd
 
-FROM nvcr.io/nvidia/cuda:12.8.1-base-ubi9
+# The C/C++ distroless image is used as a base since the CUDA vectorAdd \
+# sample application depends on C/C++ libraries.
+FROM nvcr.io/nvidia/distroless/cc:v3.1.7-dev
 
-# Remove CUDA libs(compat etc) in favor of libs installed by the NVIDIA driver
-RUN dnf remove -y cuda-*
+USER 0:0
 
-RUN dnf install -y \
-        kmod \
-        pciutils && \
-    rm -rf /var/cache/yum/*
-
-RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
 COPY --from=build /artifacts/nvidia-validator  /usr/bin/nvidia-validator
 COPY --from=sample-builder /artifacts/vectorAdd /usr/bin/vectorAdd
-RUN mkdir -p /var/nvidia/manifests
-COPY --from=build /artifacts/plugin-workload-validation.yaml /var/nvidia/manifests
-COPY --from=build /artifacts/cuda-workload-validation.yaml /var/nvidia/manifests
+COPY --from=build /artifacts/plugin-workload-validation.yaml /var/nvidia/manifests/plugin-workload-validation.yaml
+COPY --from=build /artifacts/cuda-workload-validation.yaml /var/nvidia/manifests/cuda-workload-validation.yaml
 
 ENV NVIDIA_DISABLE_REQUIRE="true"
 ENV NVIDIA_VISIBLE_DEVICES=void
@@ -83,12 +77,3 @@ LABEL name="NVIDIA Validator for the GPU Operator"
 LABEL summary="NVIDIA Validator for the GPU Operator"
 LABEL description="See summary"
 LABEL vsc-ref=${GIT_COMMIT}
-
-# Install / upgrade packages here that are required to resolve CVEs
-ARG CVE_UPDATES
-RUN if [ -n "${CVE_UPDATES}" ]; then \
-        dnf update -y ${CVE_UPDATES} && \
-        rm -rf /var/cache/yum/*; \
-    fi
-
-ENTRYPOINT ["/bin/bash"]

validator/main.go

@@ -616,6 +616,7 @@ func runCommandWithWait(command string, args []string, sleepSeconds int, silent
 		fmt.Printf("running command %s with args %v\n", command, args)
 		err := cmd.Run()
 		if err != nil {
+			log.Warningf("error running command: %v", err)
 			fmt.Printf("command failed, retrying after %d seconds\n", sleepSeconds)
 			time.Sleep(time.Duration(sleepSeconds) * time.Second)
 			continue
@@ -649,7 +650,7 @@ func setEnvVar(envvars []string, key, value string) []string {
 // For driver container installs, check existence of .driver-ctr-ready to confirm running driver
 // container has completed and is in Ready state.
 func assertDriverContainerReady(silent bool) error {
-	command := "bash"
+	command := "sh"
 	args := []string{"-c", "stat /run/nvidia/validations/.driver-ctr-ready"}
 
 	if withWaitFlag {
@@ -932,7 +933,7 @@ func (n *NvidiaFs) validate() error {
 
 func (n *NvidiaFs) runValidation(silent bool) error {
 	// check for nvidia_fs module to be loaded
-	command := "bash"
+	command := "sh"
 	args := []string{"-c", "lsmod | grep nvidia_fs"}
 
 	if withWaitFlag {
@@ -1067,7 +1068,7 @@ func (m *MOFED) validate() error {
 
 func (m *MOFED) runValidation(silent bool) error {
 	// check for mlx5_core module to be loaded
-	command := "bash"
+	command := "sh"
 	args := []string{"-c", "lsmod | grep mlx5_core"}
 
 	// If MOFED container is running then use readiness flag set by the driver container instead
@@ -1632,7 +1633,7 @@ func (c *CCManager) setKubeClient(kubeClient kubernetes.Interface) {
 
 // Check that the ccManager container is ready after applying required ccMode
 func assertCCManagerContainerReady(silent, withWaitFlag bool) error {
-	command := "bash"
+	command := "sh"
 	args := []string{"-c", "stat /run/nvidia/validations/.cc-manager-ctr-ready"}
 
 	if withWaitFlag {