From 42286330ef9ddbc84863562b8eaddaec8dcc18f5 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Mon, 2 Jun 2025 17:08:28 +0530 Subject: [PATCH 1/5] fix image references and increase leniency of the pod anti affinity rule --- deploy/k8s/controller-deployment.yaml | 20 +++++++++++--------- deploy/k8s/node-daemonset.yaml | 2 +- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/deploy/k8s/controller-deployment.yaml b/deploy/k8s/controller-deployment.yaml index 44adffc..423a229 100644 --- a/deploy/k8s/controller-deployment.yaml +++ b/deploy/k8s/controller-deployment.yaml @@ -25,14 +25,16 @@ spec: serviceAccountName: cloudstack-csi-controller affinity: podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: "app.kubernetes.io/name" - operator: In - values: - - cloudstack-csi-controller - topologyKey: "kubernetes.io/hostname" + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app.kubernetes.io/name" + operator: In + values: + - cloudstack-csi-controller + topologyKey: "kubernetes.io/hostname" nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -59,7 +61,7 @@ spec: containers: - name: cloudstack-csi-controller - image: cloudstack-csi-driver + image: ghcr.io/shapeblue/cloudstack-csi-driver:master imagePullPolicy: Always args: - "controller" diff --git a/deploy/k8s/node-daemonset.yaml b/deploy/k8s/node-daemonset.yaml index 665312b..1b92092 100644 --- a/deploy/k8s/node-daemonset.yaml +++ b/deploy/k8s/node-daemonset.yaml @@ -36,7 +36,7 @@ spec: containers: - name: cloudstack-csi-node - image: cloudstack-csi-driver + image: ghcr.io/shapeblue/cloudstack-csi-driver:master imagePullPolicy: IfNotPresent args: - "node" From 285cd971b1ac381a4cf34c73b676bae8d9ff3a3d Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Tue, 3 Jun 2025 11:03:53 +0530 Subject: [PATCH 2/5] add logs to identify device path for xenserver --- pkg/mount/mount.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/mount/mount.go b/pkg/mount/mount.go index 6e45aaf..db7679e 100644 --- a/pkg/mount/mount.go +++ b/pkg/mount/mount.go @@ -113,8 +113,10 @@ func (m *mounter) GetDevicePath(ctx context.Context, volumeID string) (string, e func (m *mounter) getDevicePathBySerialID(volumeID string) (string, error) { sourcePathPrefixes := []string{"virtio-", "scsi-", "scsi-0QEMU_QEMU_HARDDISK_"} serial := diskUUIDToSerial(volumeID) + fmt.Println("serial", serial) for _, prefix := range sourcePathPrefixes { source := filepath.Join(diskIDPath, prefix+serial) + fmt.Println("source", source) _, err := os.Stat(source) if err == nil { return source, nil From 7fe3d59a5b88fc6b4b9fd3c026837bbd18b37577 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Tue, 3 Jun 2025 14:34:58 +0530 Subject: [PATCH 3/5] add more logs for xen debug --- pkg/mount/mount.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pkg/mount/mount.go b/pkg/mount/mount.go index db7679e..8feb797 100644 --- a/pkg/mount/mount.go +++ b/pkg/mount/mount.go @@ -80,15 +80,17 @@ func (m *mounter) GetBlockSizeBytes(devicePath string) (int64, error) { func (m *mounter) GetDevicePath(ctx context.Context, volumeID string) (string, error) { backoff := wait.Backoff{ - Duration: 1 * time.Second, - Factor: 1.1, - Steps: 15, + Duration: 2 * time.Second, + Factor: 1.5, + Steps: 20, } var devicePath string err := wait.ExponentialBackoffWithContext(ctx, backoff, func(context.Context) (bool, error) { path, err := m.getDevicePathBySerialID(volumeID) + fmt.Println("path", path) if err != nil { + fmt.Println("err", err) return false, err } if path != "" { @@ -96,6 +98,7 @@ func (m *mounter) GetDevicePath(ctx context.Context, volumeID string) (string, e return true, nil } + fmt.Println("probeVolume") m.probeVolume(ctx) return false, nil @@ -113,15 +116,17 @@ func (m *mounter) GetDevicePath(ctx context.Context, volumeID string) (string, e func (m *mounter) getDevicePathBySerialID(volumeID string) (string, error) { sourcePathPrefixes := []string{"virtio-", "scsi-", "scsi-0QEMU_QEMU_HARDDISK_"} serial := diskUUIDToSerial(volumeID) - fmt.Println("serial", serial) + fmt.Println("Searching for device with serial: %s", serial) for _, prefix := range sourcePathPrefixes { source := filepath.Join(diskIDPath, prefix+serial) fmt.Println("source", source) + fmt.Println("Checking path: %s", source) _, err := os.Stat(source) if err == nil { return source, nil } if !os.IsNotExist(err) { + fmt.Println("Not found: %s", err.Error()) return "", err } } From d804dfbb2b8502e4066bf350d591099a8ce30341 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Tue, 3 Jun 2025 16:16:53 +0530 Subject: [PATCH 4/5] logic to find device for xenserver and update dockerfile to include udevadm in PATH --- cmd/cloudstack-csi-driver/Dockerfile | 4 +- deploy/k8s/node-daemonset.yaml | 6 ++ pkg/mount/mount.go | 104 +++++++++++++++++++++++++-- 3 files changed, 109 insertions(+), 5 deletions(-) diff --git a/cmd/cloudstack-csi-driver/Dockerfile b/cmd/cloudstack-csi-driver/Dockerfile index 4c260e8..268331b 100644 --- a/cmd/cloudstack-csi-driver/Dockerfile +++ b/cmd/cloudstack-csi-driver/Dockerfile @@ -14,7 +14,9 @@ RUN apk add --no-cache \ # blkid, mount and umount are required by k8s.io/mount-utils \ blkid \ mount \ - umount + umount \ + # Provides udevadm for device management + udev COPY ./bin/cloudstack-csi-driver /cloudstack-csi-driver ENTRYPOINT ["/cloudstack-csi-driver"] \ No newline at end of file diff --git a/deploy/k8s/node-daemonset.yaml b/deploy/k8s/node-daemonset.yaml index 1b92092..1d6ec05 100644 --- a/deploy/k8s/node-daemonset.yaml +++ b/deploy/k8s/node-daemonset.yaml @@ -64,6 +64,8 @@ spec: mountPath: /dev - name: cloud-init-dir mountPath: /run/cloud-init/ + - name: sys-dir + mountPath: /sys # Comment the above 2 lines and uncomment the next 2 lines for Ignition support # - name: ignition-dir # mountPath: /run/metadata @@ -177,6 +179,10 @@ spec: hostPath: path: /run/cloud-init/ type: Directory + - name: sys-dir + hostPath: + path: /sys + type: Directory # Comment the above 4 lines and uncomment the next 4 lines for Ignition support # - name: ignition-dir # hostPath: diff --git a/pkg/mount/mount.go b/pkg/mount/mount.go index 8feb797..52f6ee6 100644 --- a/pkg/mount/mount.go +++ b/pkg/mount/mount.go @@ -114,19 +114,35 @@ func (m *mounter) GetDevicePath(ctx context.Context, volumeID string) (string, e } func (m *mounter) getDevicePathBySerialID(volumeID string) (string, error) { + // First try XenServer device paths + for i := 'b'; i <= 'z'; i++ { + devicePath := fmt.Sprintf("/dev/xvd%c", i) + fmt.Printf("Checking XenServer device path: %s\n", devicePath) + + if _, err := os.Stat(devicePath); err == nil { + isBlock, err := m.IsBlockDevice(devicePath) + if err == nil && isBlock { + if m.verifyXenServerDevice(devicePath, volumeID) { + fmt.Printf("Found and verified XenServer device: %s\n", devicePath) + return devicePath, nil + } + } + } + } + + // Fall back to standard device paths sourcePathPrefixes := []string{"virtio-", "scsi-", "scsi-0QEMU_QEMU_HARDDISK_"} serial := diskUUIDToSerial(volumeID) - fmt.Println("Searching for device with serial: %s", serial) + fmt.Printf("Searching for device with serial: %s\n", serial) for _, prefix := range sourcePathPrefixes { source := filepath.Join(diskIDPath, prefix+serial) - fmt.Println("source", source) - fmt.Println("Checking path: %s", source) + fmt.Printf("Checking path: %s\n", source) _, err := os.Stat(source) if err == nil { return source, nil } if !os.IsNotExist(err) { - fmt.Println("Not found: %s", err.Error()) + fmt.Printf("Not found: %s\n", err.Error()) return "", err } } @@ -134,6 +150,86 @@ func (m *mounter) getDevicePathBySerialID(volumeID string) (string, error) { return "", nil } +func (m *mounter) verifyXenServerDevice(devicePath string, volumeID string) bool { + size, err := m.GetBlockSizeBytes(devicePath) + if err != nil { + fmt.Printf("Failed to get device size: %v\n", err) + return false + } + fmt.Printf("Device size: %d bytes\n", size) + + mounted, err := m.isDeviceMounted(devicePath) + if err != nil { + fmt.Printf("Failed to check if device is mounted: %v\n", err) + return false + } + if mounted { + fmt.Printf("Device is already mounted: %s\n", devicePath) + return false + } + + inUse, err := m.isDeviceInUse(devicePath) + if err != nil { + fmt.Printf("Failed to check if device is in use: %v\n", err) + return false + } + if inUse { + fmt.Printf("Device is in use: %s\n", devicePath) + return false + } + + props, err := m.getDeviceProperties(devicePath) + if err != nil { + fmt.Printf("Failed to get device properties: %v\n", err) + return false + } + fmt.Printf("Device properties: %v\n", props) + + return true +} + +func (m *mounter) isDeviceMounted(devicePath string) (bool, error) { + output, err := m.Exec.Command("grep", devicePath, "/proc/mounts").Output() + if err != nil { + if strings.Contains(err.Error(), "exit status 1") { + return false, nil + } + return false, err + } + return len(output) > 0, nil +} + +func (m *mounter) isDeviceInUse(devicePath string) (bool, error) { + output, err := m.Exec.Command("lsof", devicePath).Output() + if err != nil { + if strings.Contains(err.Error(), "exit status 1") { + return false, nil + } + return false, err + } + return len(output) > 0, nil +} + +func (m *mounter) getDeviceProperties(devicePath string) (map[string]string, error) { + output, err := m.Exec.Command("udevadm", "info", "--query=property", devicePath).Output() + if err != nil { + return nil, err + } + + props := make(map[string]string) + for _, line := range strings.Split(string(output), "\n") { + if line == "" { + continue + } + parts := strings.Split(line, "=") + if len(parts) == 2 { + props[parts[0]] = parts[1] + } + } + + return props, nil +} + func (m *mounter) probeVolume(ctx context.Context) { logger := klog.FromContext(ctx) logger.V(2).Info("Scanning SCSI host") From b990746fcd1441e079f15157794a0af61b491c18 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Tue, 3 Jun 2025 17:02:09 +0530 Subject: [PATCH 5/5] logic to find device for xenserver and update dockerfile to include udevadm in PATH --- pkg/mount/mount.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/pkg/mount/mount.go b/pkg/mount/mount.go index 52f6ee6..5b71ec1 100644 --- a/pkg/mount/mount.go +++ b/pkg/mount/mount.go @@ -168,16 +168,6 @@ func (m *mounter) verifyXenServerDevice(devicePath string, volumeID string) bool return false } - inUse, err := m.isDeviceInUse(devicePath) - if err != nil { - fmt.Printf("Failed to check if device is in use: %v\n", err) - return false - } - if inUse { - fmt.Printf("Device is in use: %s\n", devicePath) - return false - } - props, err := m.getDeviceProperties(devicePath) if err != nil { fmt.Printf("Failed to get device properties: %v\n", err)