🐛 supervisor: create ClusterModule per MachineDeployment and re-reconcile VirtualMachineResourceSetPolicy to update VirtualMachineSetResourcePolicy

apis/vmware/v1beta1/vspheremachine_types.go

@@ -23,6 +23,12 @@ import (
 	"sigs.k8s.io/cluster-api/errors"
 )
 
+const (
+	// TopologyKeyESXIHost is a topology key usable for soft Machine Anti Affinity
+	// based on ESXi hosts for VSphereMachines.
+	TopologyKeyESXIHost = "node.cluster.x-k8s.io/esxi-host"
+)
+
 // VSphereMachineVolume defines a PVC attachment.
 type VSphereMachineVolume struct {
 	// Name is suffix used to name this PVC as: VSphereMachine.Name + "-" + Name
@@ -91,6 +97,10 @@ type VSphereMachineSpec struct {
 	// NamingStrategy allows configuring the naming strategy used when calculating the name of the VirtualMachine.
 	// +optional
 	NamingStrategy *VirtualMachineNamingStrategy `json:"namingStrategy,omitempty"`
+
+	// affinity defines scheduling rules for the VSphereMachine.
+	// +optional
+	Affinity *VSphereMachineAffinity `json:"affinity,omitempty"`
 }
 
 // VirtualMachineNamingStrategy defines the naming strategy for the VirtualMachines.
@@ -113,6 +123,60 @@ type VirtualMachineNamingStrategy struct {
 	Template *string `json:"template,omitempty"`
 }
 
+// VSphereMachineAffinity is a group of affinity scheduling rules.
+type VSphereMachineAffinity struct {
+	// machineDeploymentMachineAntiAffinity describes VSphereMachine anti-affinity
+	// rules to other machinedeployment machines (workers).
+	// +optional
+	MachineDeploymentMachineAntiAffinity *VSphereMachineMachineDeploymentMachineAntiAffinity `json:"machineDeploymentMachineAntiAffinity,omitempty"`
+}
+
+// VSphereMachineMachineDeploymentMachineAntiAffinity defines anti-affinity rules
+// to other machinedeployment machines (workers).
+type VSphereMachineMachineDeploymentMachineAntiAffinity struct {
+	// preferredDuringSchedulingPreferredDuringExecution defines rules to schedule the
+	// VSphereMachine to a ESXi host that satisfies the anti-affinity expressions
+	// specified by this field, but may choose an ESXi host that violates these if they
+	// cannot be fulfilled.
+	// If not defined, anti-affinity will be applied by grouping all worker machines of
+	// a cluster. This is equal to defining preferredDuringSchedulingPreferredDuringExecution
+	// with a topologyKey of `node.cluster.x-k8s.io/esxi-host` and matchLabelKeys set to
+	// `[cluster.x-k8s.io/cluster-name]`.
+	// +optional
+	// +kubebuilder:validation:MinItems=1
+	// +kubebuilder:validation:MaxItems=1
+	PreferredDuringSchedulingPreferredDuringExecution []TopologyMachineDeploymentAffinityTerm `json:"preferredDuringSchedulingPreferredDuringExecution,omitempty"`
+}
+
+// TopologyMachineDeploymentAffinityTerm defines a set of machinedeployments (namely
+// those matching the labelSelector) that this VSphereMachine should be co-located
+// (affinity) or not co-located (anti-affinity) with, where co-located is defined as
+// scheduled on a ESXi host whose value of topologyKey matches that of any ESXi hosts
+// on which a machine of the set of machinedeployments is running.
+type TopologyMachineDeploymentAffinityTerm struct {
+	// topologyKey identifies the matcher for ESXi hosts to apply for affinity or anti-affinity to.
+	// The only valid value is `node.cluster.x-k8s.io/esxi-host` which means co-locating (affinity)
+	// or not co-locating (anti-affinity) on ESXi hosts.
+	// +required
+	// +kubebuilder:validation:MinLength=1
+	// +kubebuilder:validation:MaxLength=256
+	// +kubebuilder:validation:Enum=node.cluster.x-k8s.io/esxi-host
+	TopologyKey string `json:"topologyKey"`
+
+	// matchLabelKeys is a set of machinedeployment label keys to select which machinedeployment's
+	// machines will be taken into consideration for the incoming machine's (anti-) affinity.
+	// Valid values for (anti-) affinity are leaving it empty for disabling,
+	// `[cluster.x-k8s.io/cluster-name]` for all worker nodes belonging to the same cluster
+	// or `[cluster.x-k8s.io/cluster-name, cluster.x-k8s.io/deployment-name]` for all worker
+	// nodes of the same machinedeployment.-
+	// +optional
+	// +kubebuilder:validation:MinItems=0
+	// +kubebuilder:validation:MaxItems=2
+	// +kubebuilder:validation:items:MaxLength=256
+	// +kubebuilder:validation:items:Pattern=`^cluster\.x-k8s\.io\/(deployment|cluster)-name$`
+	MatchLabelKeys []string `json:"matchLabelKeys,omitempty"`
+}
+
 // VSphereMachineStatus defines the observed state of VSphereMachine.
 type VSphereMachineStatus struct {
 	// Ready is true when the provider resource is ready.

config/manager/manager.yaml

@@ -21,7 +21,7 @@ spec:
         - "--diagnostics-address=${CAPI_DIAGNOSTICS_ADDRESS:=:8443}"
         - "--insecure-diagnostics=${CAPI_INSECURE_DIAGNOSTICS:=false}"
         - --v=4
-        - "--feature-gates=NodeAntiAffinity=${EXP_NODE_ANTI_AFFINITY:=false},NamespaceScopedZones=${EXP_NAMESPACE_SCOPED_ZONES:=false},PriorityQueue=${EXP_PRIORITY_QUEUE:=false}"
+        - "--feature-gates=NodeAntiAffinity=${EXP_NODE_ANTI_AFFINITY:=false},NamespaceScopedZones=${EXP_NAMESPACE_SCOPED_ZONES:=false},PriorityQueue=${EXP_PRIORITY_QUEUE:=false},WorkerAntiAffinity=${EXP_WORKER_ANTI_AFFINITY:=false}"
         image: controller:latest
         imagePullPolicy: IfNotPresent
         name: manager

config/supervisor/crd/bases/vmware.infrastructure.cluster.x-k8s.io_vspheremachines.yaml

@@ -54,6 +54,65 @@ spec:
           spec:
             description: VSphereMachineSpec defines the desired state of VSphereMachine.
             properties:
+              affinity:
+                description: affinity defines scheduling rules for the VSphereMachine.
+                properties:
+                  machineDeploymentMachineAntiAffinity:
+                    description: |-
+                      machineDeploymentMachineAntiAffinity describes VSphereMachine anti-affinity
+                      rules to other machinedeployment machines (workers).
+                    properties:
+                      preferredDuringSchedulingPreferredDuringExecution:
+                        description: |-
+                          preferredDuringSchedulingPreferredDuringExecution defines rules to schedule the
+                          VSphereMachine to a ESXi host that satisfies the anti-affinity expressions
+                          specified by this field, but may choose an ESXi host that violates these if they
+                          cannot be fulfilled.
+                          If not defined, anti-affinity will be applied by grouping all worker machines of
+                          a cluster. This is equal to defining preferredDuringSchedulingPreferredDuringExecution
+                          with a topologyKey of `node.cluster.x-k8s.io/esxi-host` and matchLabelKeys set to
+                          `[cluster.x-k8s.io/cluster-name]`.
+                        items:
+                          description: |-
+                            TopologyMachineDeploymentAffinityTerm defines a set of machinedeployments (namely
+                            those matching the labelSelector) that this VSphereMachine should be co-located
+                            (affinity) or not co-located (anti-affinity) with, where co-located is defined as
+                            scheduled on a ESXi host whose value of topologyKey matches that of any ESXi hosts
+                            on which a machine of the set of machinedeployments is running.
+                          properties:
+                            matchLabelKeys:
+                              description: |-
+                                matchLabelKeys is a set of machinedeployment label keys to select which machinedeployment's
+                                machines will be taken into consideration for the incoming machine's (anti-) affinity.
+                                Valid values for (anti-) affinity are leaving it empty for disabling,
+                                `[cluster.x-k8s.io/cluster-name]` for all worker nodes belonging to the same cluster
+                                or `[cluster.x-k8s.io/cluster-name, cluster.x-k8s.io/deployment-name]` for all worker
+                                nodes of the same machinedeployment.-
+                              items:
+                                maxLength: 256
+                                pattern: ^cluster\.x-k8s\.io\/(deployment|cluster)-name$
+                                type: string
+                              maxItems: 2
+                              minItems: 0
+                              type: array
+                            topologyKey:
+                              description: |-
+                                topologyKey identifies the matcher for ESXi hosts to apply for affinity or anti-affinity to.
+                                The only valid value is `node.cluster.x-k8s.io/esxi-host` which means co-locating (affinity)
+                                or not co-locating (anti-affinity) on ESXi hosts.
+                              enum:
+                              - node.cluster.x-k8s.io/esxi-host
+                              maxLength: 256
+                              minLength: 1
+                              type: string
+                          required:
+                          - topologyKey
+                          type: object
+                        maxItems: 1
+                        minItems: 1
+                        type: array
+                    type: object
+                type: object
               className:
                 description: |-
                   ClassName is the name of the class used when specifying the underlying

config/supervisor/crd/bases/vmware.infrastructure.cluster.x-k8s.io_vspheremachinetemplates.yaml

@@ -50,6 +50,65 @@ spec:
                     description: Spec is the specification of the desired behavior
                       of the machine.
                     properties:
+                      affinity:
+                        description: affinity defines scheduling rules for the VSphereMachine.
+                        properties:
+                          machineDeploymentMachineAntiAffinity:
+                            description: |-
+                              machineDeploymentMachineAntiAffinity describes VSphereMachine anti-affinity
+                              rules to other machinedeployment machines (workers).
+                            properties:
+                              preferredDuringSchedulingPreferredDuringExecution:
+                                description: |-
+                                  preferredDuringSchedulingPreferredDuringExecution defines rules to schedule the
+                                  VSphereMachine to a ESXi host that satisfies the anti-affinity expressions
+                                  specified by this field, but may choose an ESXi host that violates these if they
+                                  cannot be fulfilled.
+                                  If not defined, anti-affinity will be applied by grouping all worker machines of
+                                  a cluster. This is equal to defining preferredDuringSchedulingPreferredDuringExecution
+                                  with a topologyKey of `node.cluster.x-k8s.io/esxi-host` and matchLabelKeys set to
+                                  `[cluster.x-k8s.io/cluster-name]`.
+                                items:
+                                  description: |-
+                                    TopologyMachineDeploymentAffinityTerm defines a set of machinedeployments (namely
+                                    those matching the labelSelector) that this VSphereMachine should be co-located
+                                    (affinity) or not co-located (anti-affinity) with, where co-located is defined as
+                                    scheduled on a ESXi host whose value of topologyKey matches that of any ESXi hosts
+                                    on which a machine of the set of machinedeployments is running.
+                                  properties:
+                                    matchLabelKeys:
+                                      description: |-
+                                        matchLabelKeys is a set of machinedeployment label keys to select which machinedeployment's
+                                        machines will be taken into consideration for the incoming machine's (anti-) affinity.
+                                        Valid values for (anti-) affinity are leaving it empty for disabling,
+                                        `[cluster.x-k8s.io/cluster-name]` for all worker nodes belonging to the same cluster
+                                        or `[cluster.x-k8s.io/cluster-name, cluster.x-k8s.io/deployment-name]` for all worker
+                                        nodes of the same machinedeployment.-
+                                      items:
+                                        maxLength: 256
+                                        pattern: ^cluster\.x-k8s\.io\/(deployment|cluster)-name$
+                                        type: string
+                                      maxItems: 2
+                                      minItems: 0
+                                      type: array
+                                    topologyKey:
+                                      description: |-
+                                        topologyKey identifies the matcher for ESXi hosts to apply for affinity or anti-affinity to.
+                                        The only valid value is `node.cluster.x-k8s.io/esxi-host` which means co-locating (affinity)
+                                        or not co-locating (anti-affinity) on ESXi hosts.
+                                      enum:
+                                      - node.cluster.x-k8s.io/esxi-host
+                                      maxLength: 256
+                                      minLength: 1
+                                      type: string
+                                  required:
+                                  - topologyKey
+                                  type: object
+                                maxItems: 1
+                                minItems: 1
+                                type: array
+                            type: object
+                        type: object
                       className:
                         description: |-
                           ClassName is the name of the class used when specifying the underlying

controllers/vmware/vspherecluster_reconciler.go

@@ -72,6 +72,7 @@ type ClusterReconciler struct {
 // +kubebuilder:rbac:groups=netoperator.vmware.com,resources=networks,verbs=get;list;watch
 // +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;update;create;delete
 // +kubebuilder:rbac:groups="",resources=persistentvolumeclaims/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machines,verbs=get;list;watch
 
 func (r *ClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) {
 	log := ctrl.LoggerFrom(ctx)
@@ -243,7 +244,7 @@ func (r *ClusterReconciler) reconcileNormal(ctx context.Context, clusterCtx *vmw
 	// Reconcile ResourcePolicy before we create the machines. If the ResourcePolicy is not reconciled before we create the Node VMs,
 	// it will be handled by vm operator by relocating the VMs to the ResourcePool and Folder specified by the ResourcePolicy.
 	// Reconciling the ResourcePolicy early potentially saves us the extra relocate operation.
-	resourcePolicyName, err := r.ResourcePolicyService.ReconcileResourcePolicy(ctx, clusterCtx)
+	resourcePolicyName, err := r.ResourcePolicyService.ReconcileResourcePolicy(ctx, clusterCtx.Cluster, clusterCtx.VSphereCluster)
 	if err != nil {
 		conditions.MarkFalse(clusterCtx.VSphereCluster, vmwarev1.ResourcePolicyReadyCondition, vmwarev1.ResourcePolicyCreationFailedReason, clusterv1.ConditionSeverityWarning, err.Error())
 		v1beta2conditions.Set(clusterCtx.VSphereCluster, metav1.Condition{
@@ -436,14 +437,8 @@ func (r *ClusterReconciler) VSphereMachineToCluster(ctx context.Context, o clien
 	log = log.WithValues("VSphereMachine", klog.KObj(vsphereMachine))
 	ctx = ctrl.LoggerInto(ctx, log)
 
-	if !util.IsControlPlaneMachine(vsphereMachine) {
-		log.V(6).Info("Skipping VSphereCluster reconcile as Machine is not a control plane Machine")
-		return nil
-	}
-
-	// Only currently interested in updating Cluster from VSphereMachines with IP addresses
-	if vsphereMachine.Status.IPAddr == "" {
-		log.V(6).Info("Skipping VSphereCluster reconcile as Machine does not have an IP address")
+	if util.IsControlPlaneMachine(vsphereMachine) && vsphereMachine.Status.IPAddr == "" {
+		log.V(6).Info("Skipping VSphereCluster reconcile because the control plane machine does not have an IP address")
 		return nil
 	}