cleanup: remove kube-rbac-proxy (#321)

* cleanup: remove kube-rbac-proxy

* remove kube-rbac-proxy and update helm config

Author: nayihz

chart/templates/deployment.yaml

@@ -39,6 +39,9 @@ spec:
           periodSeconds: 20
         name: manager
         ports:
+        - containerPort: 8443
+          name: metrics
+          protocol: TCP
         - containerPort: 9443
           name: webhook-server
           protocol: TCP
@@ -56,21 +59,6 @@ spec:
         - mountPath: /tmp/k8s-webhook-server/serving-certs
           name: cert
           readOnly: true
-      - args: {{- toYaml .Values.controllerManager.kubeRbacProxy.args | nindent 8 }}
-        env:
-        - name: KUBERNETES_CLUSTER_DOMAIN
-          value: {{ quote .Values.kubernetesClusterDomain }}
-        image: {{ .Values.controllerManager.kubeRbacProxy.image.repository }}:{{ .Values.controllerManager.kubeRbacProxy.image.tag
-          | default .Chart.AppVersion }}
-        name: kube-rbac-proxy
-        ports:
-        - containerPort: 8443
-          name: https
-          protocol: TCP
-        resources: {{- toYaml .Values.controllerManager.kubeRbacProxy.resources | nindent
-          10 }}
-        securityContext: {{- toYaml .Values.controllerManager.kubeRbacProxy.containerSecurityContext
-          | nindent 10 }}
       securityContext: {{- toYaml .Values.controllerManager.podSecurityContext | nindent
         8 }}
       serviceAccountName: {{ include "chart.fullname" . }}-controller-manager

chart/templates/metrics-reader-rbac.yaml

@@ -3,12 +3,26 @@ kind: ClusterRole
 metadata:
   name: {{ include "chart.fullname" . }}-metrics-reader
   labels:
-    app.kubernetes.io/component: kube-rbac-proxy
     app.kubernetes.io/created-by: llmaz
     app.kubernetes.io/part-of: llmaz
   {{- include "chart.labels" . | nindent 4 }}
 rules:
 - nonResourceURLs:
   - /metrics
   verbs:
-  - get
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "chart.fullname" . }}-metrics-reader-rolebinding
+  labels:
+  {{- include "chart.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: '{{ include "chart.fullname" . }}-metrics-reader'
+subjects:
+- kind: ServiceAccount
+  name: '{{ include "chart.fullname" . }}-controller-manager'
+  namespace: '{{ .Release.Namespace }}'

chart/templates/metrics-service.yaml

@@ -3,10 +3,8 @@ kind: Service
 metadata:
   name: {{ include "chart.fullname" . }}-controller-manager-metrics-service
   labels:
-    app.kubernetes.io/component: kube-rbac-proxy
     app.kubernetes.io/created-by: llmaz
     app.kubernetes.io/part-of: llmaz
-    control-plane: controller-manager
   {{- include "chart.labels" . | nindent 4 }}
 spec:
   type: {{ .Values.metricsService.type }}

chart/templates/proxy-rbac.yaml

@@ -3,7 +3,6 @@ kind: ClusterRole
 metadata:
   name: {{ include "chart.fullname" . }}-proxy-role
   labels:
-    app.kubernetes.io/component: kube-rbac-proxy
     app.kubernetes.io/created-by: llmaz
     app.kubernetes.io/part-of: llmaz
   {{- include "chart.labels" . | nindent 4 }}
@@ -26,7 +25,6 @@ kind: ClusterRoleBinding
 metadata:
   name: {{ include "chart.fullname" . }}-proxy-rolebinding
   labels:
-    app.kubernetes.io/component: kube-rbac-proxy
     app.kubernetes.io/created-by: llmaz
     app.kubernetes.io/part-of: llmaz
   {{- include "chart.labels" . | nindent 4 }}

chart/values.yaml

@@ -1,29 +1,8 @@
 controllerManager:
-  kubeRbacProxy:
-    args:
-    - --secure-listen-address=0.0.0.0:8443
-    - --upstream=http://127.0.0.1:8080/
-    - --logtostderr=true
-    - --v=0
-    containerSecurityContext:
-      allowPrivilegeEscalation: false
-      capabilities:
-        drop:
-        - ALL
-    image:
-      repository: gcr.io/kubebuilder/kube-rbac-proxy
-      tag: v0.15.0
-    resources:
-      limits:
-        cpu: 500m
-        memory: 128Mi
-      requests:
-        cpu: 5m
-        memory: 64Mi
   manager:
     args:
     - --health-probe-bind-address=:8081
-    - --metrics-bind-address=127.0.0.1:8080
+    - --metrics-bind-address=:8443
     - --leader-elect
     - --namespace=llmaz-system
     containerSecurityContext:
@@ -33,7 +12,7 @@ controllerManager:
         - ALL
     image:
       repository: inftyai/llmaz
-      tag: v0.1.1
+      tag: main
     resources:
       limits:
         cpu: 500m
@@ -52,7 +31,7 @@ metricsService:
   - name: https
     port: 8443
     protocol: TCP
-    targetPort: https
+    targetPort: 8443
   type: ClusterIP
 webhookService:
   ports:

cmd/main.go

@@ -30,6 +30,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	lws "sigs.k8s.io/lws/api/leaderworkerset/v1"
 
@@ -77,9 +78,18 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
+	// Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server.
+	// More info:
+	// - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/server
+	// - https://book.kubebuilder.io/reference/metrics.html
+	metricsServerOptions := metricsserver.Options{
+		BindAddress:    metricsAddr,
+		SecureServing:  true,
+		FilterProvider: filters.WithAuthenticationAndAuthorization,
+	}
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:                 scheme,
-		Metrics:                metricsserver.Options{BindAddress: metricsAddr},
+		Metrics:                metricsServerOptions,
 		HealthProbeBindAddress: probeAddr,
 		LeaderElection:         enableLeaderElection,
 		LeaderElectionID:       "fbb36db9.llmaz.io",

config/default/kustomization.yaml

@@ -29,17 +29,15 @@ resources:
 #- ../certmanager
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
 #- ../prometheus
+# [METRICS] Expose the controller manager metrics service.
+- manager_metrics_service.yaml
 
 patches:
-# Protect the /metrics endpoint by putting it behind auth.
-# If you want your controller-manager to expose the /metrics
-# endpoint w/o any authn/z, please comment the following line.
-- path: manager_auth_proxy_patch.yaml
-
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml
 - path: manager_webhook_patch.yaml
-
+# Expose port used by the metrics service
+- path: manager_metrics_patch.yaml
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'.
 # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks.
 # 'CERTMANAGER' needs to be enabled to use ca injection

config/default/manager_auth_proxy_patch.yaml

@@ -0,0 +1,15 @@
+# This patch exposes 8443 port used by metrics service
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+        - name: manager
+          ports:
+          - containerPort: 8443
+            name: metrics
+            protocol: TCP

config/default/manager_metrics_patch.yaml

@@ -2,20 +2,17 @@ apiVersion: v1
 kind: Service
 metadata:
   labels:
-    control-plane: controller-manager
     app.kubernetes.io/name: service
     app.kubernetes.io/instance: controller-manager-metrics-service
-    app.kubernetes.io/component: kube-rbac-proxy
     app.kubernetes.io/created-by: llmaz
     app.kubernetes.io/part-of: llmaz
-    app.kubernetes.io/managed-by: kustomize
   name: controller-manager-metrics-service
   namespace: system
 spec:
   ports:
   - name: https
     port: 8443
     protocol: TCP
-    targetPort: https
+    targetPort: 8443
   selector:
     control-plane: controller-manager

config/rbac/auth_proxy_service.yaml renamed to config/default/manager_metrics_service.yaml

@@ -69,7 +69,10 @@ spec:
       - command:
         - /manager
         args:
-        - --leader-elect
+          - --health-probe-bind-address=:8081
+          - --metrics-bind-address=:8443
+          - --leader-elect
+          - --namespace=llmaz-system
         image: controller:latest
         name: manager
         securityContext:

config/manager/manager.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metrics-reader-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metrics-reader
+subjects:
+  - kind: ServiceAccount
+    name: controller-manager
+    namespace: system

config/rbac/auth_proxy_client_binding.yaml

@@ -4,7 +4,6 @@ metadata:
   labels:
     app.kubernetes.io/name: clusterrole
     app.kubernetes.io/instance: metrics-reader
-    app.kubernetes.io/component: kube-rbac-proxy
     app.kubernetes.io/created-by: llmaz
     app.kubernetes.io/part-of: llmaz
     app.kubernetes.io/managed-by: kustomize

config/rbac/auth_proxy_client_clusterrole.yaml

@@ -4,7 +4,6 @@ metadata:
   labels:
     app.kubernetes.io/name: clusterrole
     app.kubernetes.io/instance: proxy-role
-    app.kubernetes.io/component: kube-rbac-proxy
     app.kubernetes.io/created-by: llmaz
     app.kubernetes.io/part-of: llmaz
     app.kubernetes.io/managed-by: kustomize

config/rbac/auth_proxy_role.yaml

@@ -4,7 +4,6 @@ metadata:
   labels:
     app.kubernetes.io/name: clusterrolebinding
     app.kubernetes.io/instance: proxy-rolebinding
-    app.kubernetes.io/component: kube-rbac-proxy
     app.kubernetes.io/created-by: llmaz
     app.kubernetes.io/part-of: llmaz
     app.kubernetes.io/managed-by: kustomize

config/rbac/auth_proxy_role_binding.yaml

@@ -9,10 +9,7 @@ resources:
 - role_binding.yaml
 - leader_election_role.yaml
 - leader_election_role_binding.yaml
-# Comment the following 4 lines if you want to disable
-# the auth proxy (https://github.com/brancz/kube-rbac-proxy)
-# which protects your /metrics endpoint.
-- auth_proxy_service.yaml
 - auth_proxy_role.yaml
 - auth_proxy_role_binding.yaml
 - auth_proxy_client_clusterrole.yaml
+- auth_proxy_client_binding.yaml