Skip to content

Commit 81afbe3

Browse files
dergeberldergeberl
authored
Add possibility to use multiple loadbalancerClasses and use service.spec.loadBalancerClass (#217)
* Add possibility to use multiple loadbalancerClasses and use service.spec.loadBalancerClass * update docs and dev setup * add code suggestions from PR review
1 parent aceca0a commit 81afbe3

File tree

12 files changed

+245
-27
lines changed

12 files changed

+245
-27
lines changed

‎README.md

+1
Original file line numberDiff line numberDiff line change
@@ -252,6 +252,7 @@ metadata:
252252
# Reference the name of the SSH key provided to OpenStack for debugging .
253253
yawol.stackit.cloud/debugsshkey: "OS-keyName"
254254
# Allows filtering services in cloud-controller.
255+
# Deprecated: Use service.spec.loadBalancerClass instead.
255256
yawol.stackit.cloud/className: "test"
256257
# Specify the number of LoadBalancer machines to deploy (default 1).
257258
yawol.stackit.cloud/replicas: "3"

‎api/v1beta1/loadbalancer_types.go

+1
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ const (
3232
// ServiceDebugSSHKey set an sshkey
3333
ServiceDebugSSHKey = "yawol.stackit.cloud/debugsshkey"
3434
// ServiceClassName for filtering services in cloud-controller
35+
// Deprecated: use .spec.loadBalancerClass instead
3536
ServiceClassName = "yawol.stackit.cloud/className"
3637
// ServiceReplicas for setting loadbalancer replicas in cloud-controller
3738
ServiceReplicas = "yawol.stackit.cloud/replicas"

‎cmd/yawol-cloud-controller/main.go

+29-5
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"flag"
66
"os"
77
"strconv"
8+
"strings"
89
"time"
910

1011
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
@@ -18,6 +19,7 @@ import (
1819
yawolv1beta1 "github.com/stackitcloud/yawol/api/v1beta1"
1920
"github.com/stackitcloud/yawol/controllers/yawol-cloud-controller/controlcontroller"
2021
"github.com/stackitcloud/yawol/controllers/yawol-cloud-controller/targetcontroller"
22+
"github.com/stackitcloud/yawol/internal/helper"
2123
"go.uber.org/zap/zapcore"
2224
"k8s.io/apimachinery/pkg/runtime"
2325
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -33,6 +35,8 @@ var (
3335
setupLog = ctrl.Log.WithName("setup")
3436
)
3537

38+
type loadbalancerClassNames []string
39+
3640
const (
3741
// Namespace in for LoadBalancer CRs
3842
EnvClusterNamespace = "CLUSTER_NAMESPACE"
@@ -72,7 +76,8 @@ func main() {
7276
var targetEnableLeaderElection bool
7377
var targetKubeconfig string
7478
var controlKubeconfig string
75-
var className string
79+
var classNames loadbalancerClassNames
80+
var emptyClassName bool
7681
// settings for leases
7782
var leasesDurationInt int
7883
var leasesRenewDeadlineInt int
@@ -94,9 +99,12 @@ func main() {
9499
"K8s credentials for watching the Service resources.")
95100
flag.StringVar(&controlKubeconfig, "control-kubeconfig", "",
96101
"K8s credentials for deploying the LoadBalancer resources.")
97-
flag.StringVar(&className, "classname", "",
98-
"Only listen to Services with the given className. "+
99-
"Default is empty and listen to all services with out className annotation")
102+
flag.Var(&classNames, "classname",
103+
"Only listen to Services with the given className. Can be set multiple times. "+
104+
"If no classname is set it will defaults to "+helper.DefaultLoadbalancerClass+" "+
105+
"and services without class. See also --empty-classname.")
106+
flag.BoolVar(&emptyClassName, "empty-classname", true,
107+
"Listen to services without a loadBalancerClass. Default is true.")
100108
flag.IntVar(&leasesDurationInt, "leases-duration", 60,
101109
"Is the time in seconds a non-leader will wait until forcing to acquire leadership.")
102110
flag.IntVar(&leasesRenewDeadlineInt, "leases-renew-deadline", 50,
@@ -113,6 +121,13 @@ func main() {
113121
opts.BindFlags(flag.CommandLine)
114122
flag.Parse()
115123

124+
if len(classNames) == 0 {
125+
classNames = append(classNames, helper.DefaultLoadbalancerClass)
126+
}
127+
if emptyClassName {
128+
classNames = append(classNames, "")
129+
}
130+
116131
leasesDuration = time.Duration(leasesDurationInt) * time.Second
117132
leasesRenewDeadline = time.Duration(leasesRenewDeadlineInt) * time.Second
118133
leasesRetryPeriod = time.Duration(leasesRetryPeriodInt) * time.Second
@@ -178,7 +193,7 @@ func main() {
178193
Log: ctrl.Log.WithName("controller").WithName("Service"),
179194
Scheme: targetMgr.GetScheme(),
180195
Recorder: targetMgr.GetEventRecorderFor("yawol-cloud-controller"),
181-
ClassName: className,
196+
ClassNames: classNames,
182197
}).SetupWithManager(targetMgr); err != nil {
183198
setupLog.Error(err, "unable to create controller", "controller", "Service")
184199
os.Exit(1)
@@ -352,3 +367,12 @@ func getInfrastructureDefaultsFromEnvOrDie() targetcontroller.InfrastructureDefa
352367
InternalLB: pointer.Bool(internalLb),
353368
}
354369
}
370+
371+
func (i *loadbalancerClassNames) String() string {
372+
return strings.Join(*i, ",")
373+
}
374+
375+
func (i *loadbalancerClassNames) Set(value string) error {
376+
*i = append(*i, value)
377+
return nil
378+
}

‎controllers/yawol-cloud-controller/targetcontroller/service_controller.go

+3-12
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ type ServiceReconciler struct {
4141
Log logr.Logger
4242
Scheme *runtime.Scheme
4343
Recorder record.EventRecorder
44-
ClassName string
44+
ClassNames []string
4545
}
4646

4747
// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch
@@ -57,14 +57,9 @@ func (r *ServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
5757
return ctrl.Result{}, client.IgnoreNotFound(err)
5858
}
5959

60-
className, ok := svc.Annotations[yawolv1beta1.ServiceClassName]
61-
if !ok {
62-
className = ""
63-
}
64-
6560
infraDefaults := GetMergedInfrastructureDetails(r.InfrastructureDefaults, svc)
6661

67-
if className != r.ClassName {
62+
if !helper.CheckLoadBalancerClasses(svc, r.ClassNames) {
6863
r.Log.WithValues("service", req.NamespacedName).Info("service and controller classname does not match")
6964
if err := r.ControlClient.Get(ctx, types.NamespacedName{
7065
Namespace: *infraDefaults.Namespace,
@@ -560,11 +555,7 @@ func (r *ServiceReconciler) deletionRoutine(
560555
}
561556

562557
if apierrors.IsNotFound(err) {
563-
className, ok := svc.Annotations[yawolv1beta1.ServiceClassName]
564-
if !ok {
565-
className = ""
566-
}
567-
if r.ClassName != className {
558+
if !helper.CheckLoadBalancerClasses(svc, r.ClassNames) {
568559
return ctrl.Result{}, nil
569560
}
570561

‎controllers/yawol-cloud-controller/targetcontroller/service_controller_test.go

+136-7
Original file line numberDiff line numberDiff line change
@@ -472,11 +472,11 @@ var _ = Describe("Check loadbalancer reconcile", Serial, Ordered, func() {
472472
}, time.Second*5, time.Millisecond*500).Should(Succeed())
473473
})
474474

475-
It("create service with wrong className", func() {
475+
It("create service with wrong className in annotation", func() {
476476
By("create service")
477477
service := v1.Service{
478478
ObjectMeta: metav1.ObjectMeta{
479-
Name: "service-test8",
479+
Name: "class-name-service-test1",
480480
Namespace: "default",
481481
Annotations: map[string]string{
482482
yawolv1beta1.ServiceClassName: "foo",
@@ -498,19 +498,19 @@ var _ = Describe("Check loadbalancer reconcile", Serial, Ordered, func() {
498498

499499
By("check for LB creation")
500500
Consistently(func() error {
501-
err := k8sClient.Get(ctx, types.NamespacedName{Name: "default--service-test8", Namespace: "default"}, &lb)
501+
err := k8sClient.Get(ctx, types.NamespacedName{Name: "default--class-name-service-test1", Namespace: "default"}, &lb)
502502
if err != nil {
503503
return client.IgnoreNotFound(err)
504504
}
505505
return helper.ErrInvalidClassname
506506
}, time.Second*5, time.Millisecond*500).Should(Succeed())
507507
})
508508

509-
It("create service with correct classname", func() {
509+
It("create service with correct classname in annotation", func() {
510510
By("create service")
511511
service := v1.Service{
512512
ObjectMeta: metav1.ObjectMeta{
513-
Name: "service-test15",
513+
Name: "class-name-service-test2",
514514
Namespace: "default",
515515
Annotations: map[string]string{
516516
yawolv1beta1.ServiceClassName: "",
@@ -533,7 +533,7 @@ var _ = Describe("Check loadbalancer reconcile", Serial, Ordered, func() {
533533
By("check creation of LB")
534534
Eventually(func() error {
535535
err := k8sClient.Get(ctx, types.NamespacedName{
536-
Name: "default--service-test15",
536+
Name: "default--class-name-service-test2",
537537
Namespace: "default",
538538
}, &lb)
539539
return err
@@ -547,7 +547,136 @@ var _ = Describe("Check loadbalancer reconcile", Serial, Ordered, func() {
547547
return err
548548
}
549549
for _, event := range eventList.Items {
550-
if event.InvolvedObject.Name == "service-test15" &&
550+
if event.InvolvedObject.Name == "class-name-service-test2" &&
551+
event.InvolvedObject.Kind == "Service" &&
552+
strings.Contains(event.Message, "LoadBalancer is in creation") {
553+
return nil
554+
}
555+
}
556+
return helper.ErrNoEventFound
557+
}, time.Second*5, time.Millisecond*500).Should(Succeed())
558+
})
559+
560+
It("create service with wrong className in spec", func() {
561+
By("create service")
562+
service := v1.Service{
563+
ObjectMeta: metav1.ObjectMeta{
564+
Name: "class-name-service-test3",
565+
Namespace: "default",
566+
},
567+
Spec: v1.ServiceSpec{
568+
LoadBalancerClass: pointer.String("foo"),
569+
Ports: []v1.ServicePort{
570+
{
571+
Name: "port1",
572+
Protocol: v1.ProtocolTCP,
573+
Port: 65030,
574+
TargetPort: intstr.IntOrString{IntVal: 12345},
575+
NodePort: 30133,
576+
},
577+
},
578+
Type: "LoadBalancer",
579+
}}
580+
Expect(k8sClient.Create(ctx, &service)).Should(Succeed())
581+
582+
By("check for LB creation")
583+
Consistently(func() error {
584+
err := k8sClient.Get(ctx, types.NamespacedName{Name: "default--class-name-service-test1", Namespace: "default"}, &lb)
585+
if err != nil {
586+
return client.IgnoreNotFound(err)
587+
}
588+
return helper.ErrInvalidClassname
589+
}, time.Second*5, time.Millisecond*500).Should(Succeed())
590+
})
591+
592+
It("create service with correct classname in spec", func() {
593+
By("create service")
594+
service := v1.Service{
595+
ObjectMeta: metav1.ObjectMeta{
596+
Name: "class-name-service-test4",
597+
Namespace: "default",
598+
},
599+
Spec: v1.ServiceSpec{
600+
LoadBalancerClass: pointer.String(helper.DefaultLoadbalancerClass),
601+
Ports: []v1.ServicePort{
602+
{
603+
Name: "port1",
604+
Protocol: v1.ProtocolTCP,
605+
Port: 12345,
606+
TargetPort: intstr.IntOrString{IntVal: 12345},
607+
NodePort: 30335,
608+
},
609+
},
610+
Type: "LoadBalancer",
611+
}}
612+
Expect(k8sClient.Create(ctx, &service)).Should(Succeed())
613+
614+
By("check creation of LB")
615+
Eventually(func() error {
616+
err := k8sClient.Get(ctx, types.NamespacedName{
617+
Name: "default--class-name-service-test4",
618+
Namespace: "default",
619+
}, &lb)
620+
return err
621+
}, time.Second*5, time.Millisecond*500).Should(Succeed())
622+
623+
By("Check Event for creation")
624+
Eventually(func() error {
625+
eventList := v1.EventList{}
626+
err := k8sClient.List(ctx, &eventList)
627+
if err != nil {
628+
return err
629+
}
630+
for _, event := range eventList.Items {
631+
if event.InvolvedObject.Name == "class-name-service-test4" &&
632+
event.InvolvedObject.Kind == "Service" &&
633+
strings.Contains(event.Message, "LoadBalancer is in creation") {
634+
return nil
635+
}
636+
}
637+
return helper.ErrNoEventFound
638+
}, time.Second*5, time.Millisecond*500).Should(Succeed())
639+
})
640+
641+
It("create service without classname", func() {
642+
By("create service")
643+
service := v1.Service{
644+
ObjectMeta: metav1.ObjectMeta{
645+
Name: "class-name-service-test5",
646+
Namespace: "default",
647+
},
648+
Spec: v1.ServiceSpec{
649+
Ports: []v1.ServicePort{
650+
{
651+
Name: "port1",
652+
Protocol: v1.ProtocolTCP,
653+
Port: 12345,
654+
TargetPort: intstr.IntOrString{IntVal: 12345},
655+
NodePort: 30360,
656+
},
657+
},
658+
Type: "LoadBalancer",
659+
}}
660+
Expect(k8sClient.Create(ctx, &service)).Should(Succeed())
661+
662+
By("check creation of LB")
663+
Eventually(func() error {
664+
err := k8sClient.Get(ctx, types.NamespacedName{
665+
Name: "default--class-name-service-test5",
666+
Namespace: "default",
667+
}, &lb)
668+
return err
669+
}, time.Second*5, time.Millisecond*500).Should(Succeed())
670+
671+
By("Check Event for creation")
672+
Eventually(func() error {
673+
eventList := v1.EventList{}
674+
err := k8sClient.List(ctx, &eventList)
675+
if err != nil {
676+
return err
677+
}
678+
for _, event := range eventList.Items {
679+
if event.InvolvedObject.Name == "class-name-service-test5" &&
551680
event.InvolvedObject.Kind == "Service" &&
552681
strings.Contains(event.Message, "LoadBalancer is in creation") {
553682
return nil

‎controllers/yawol-cloud-controller/targetcontroller/suite_test.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"sigs.k8s.io/controller-runtime/pkg/log/zap"
1818

1919
yawolv1beta1 "github.com/stackitcloud/yawol/api/v1beta1"
20+
"github.com/stackitcloud/yawol/internal/helper"
2021
// +kubebuilder:scaffold:imports
2122
)
2223

@@ -101,7 +102,7 @@ var _ = BeforeSuite(func() {
101102
Log: ctrl.Log.WithName("controllers").WithName("Service"),
102103
Scheme: k8sManager.GetScheme(),
103104
Recorder: k8sManager.GetEventRecorderFor("Loadbalancer"),
104-
ClassName: "",
105+
ClassNames: []string{"", helper.DefaultLoadbalancerClass},
105106
}).SetupWithManager(k8sManager)
106107
Expect(err).ToNot(HaveOccurred())
107108

‎docs/development.md

-1
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,6 @@ The controllers are using the default kubeconfig ($KUBECONFIG, InCluster or
104104
# or
105105
kubectl create deployment --image nginx:latest nginx --replicas 1
106106
kubectl expose deployment --port 80 --type LoadBalancer nginx --name loadbalancer
107-
kubectl annotate service loadbalancer yawol.stackit.cloud/className=test # annotation needs to match the value of the `classname` flag from `run-ycc.sh`
108107
```
109108

110109
2. Check if the yawol-cloud-controller created a new `LoadBalancer` object

‎example-setup/yawol-cloud-controller/service.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ metadata:
1515
# yawol.stackit.cloud/tcpProxyProtocol: "false"
1616
# yawol.stackit.cloud/tcpProxyProtocolPortsFilter: ""
1717
spec:
18+
loadBalancerClass: "test"
1819
type: LoadBalancer
1920
selector:
2021
app: nginx

‎internal/helper/const.go

+1
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,5 @@ const (
1212
LoadBalancerKind = "LoadBalancer"
1313
VRRPInstanceName = "ENVOY"
1414
HasKeepalivedMaster = "HasKeepalivedMaster"
15+
DefaultLoadbalancerClass = "stackit.cloud/yawol"
1516
)

‎internal/helper/service.go

+17
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
coreV1 "k8s.io/api/core/v1"
1414
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1515
"k8s.io/apimachinery/pkg/types"
16+
"k8s.io/apimachinery/pkg/util/sets"
1617
"k8s.io/client-go/tools/record"
1718
"sigs.k8s.io/controller-runtime/pkg/client"
1819
)
@@ -164,6 +165,22 @@ func GetLoadBalancerNameFromService(service *coreV1.Service) string {
164165
return service.Namespace + "--" + service.Name
165166
}
166167

168+
func getLoadBalancerClass(service *coreV1.Service) string {
169+
if className := service.Annotations[yawolv1beta1.ServiceClassName]; className != "" {
170+
return className
171+
}
172+
173+
if service.Spec.LoadBalancerClass != nil {
174+
return *service.Spec.LoadBalancerClass
175+
}
176+
177+
return ""
178+
}
179+
180+
func CheckLoadBalancerClasses(service *coreV1.Service, validClasses []string) bool {
181+
return sets.New(validClasses...).Has(getLoadBalancerClass(service))
182+
}
183+
167184
// ValidateService checks if the service is valid
168185
func ValidateService(svc *coreV1.Service) error {
169186
for _, port := range svc.Spec.Ports {

0 commit comments

Comments
 (0)