Skip to content

Commit 7e5cdf9

Browse files
hdurand0710oktalz
hdurand0710
authored and
oktalz
committed
MEDIUM: add ingress.class annotation to TCP CR
1 parent ecf8a20 commit 7e5cdf9

14 files changed

+219
-10
lines changed

‎deploy/tests/e2e/crd-tcp/config/tcp-cr-add-services.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ apiVersion: ingress.v1.haproxy.org/v1
22
kind: TCP
33
metadata:
44
name: tcp-1
5+
annotations:
6+
ingress.class: haproxy
57
spec:
68
- name: tcp-http-echo-80
79
frontend:

‎deploy/tests/e2e/crd-tcp/config/tcp-cr-backend-switching-rule-acls.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ apiVersion: ingress.v1.haproxy.org/v1
22
kind: TCP
33
metadata:
44
name: tcp-1
5+
annotations:
6+
ingress.class: haproxy
57
spec:
68
- name: tcp-test
79
frontend:

‎deploy/tests/e2e/crd-tcp/config/tcp-cr-backend-switching-rule.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ apiVersion: ingress.v1.haproxy.org/v1
22
kind: TCP
33
metadata:
44
name: tcp-1
5+
annotations:
6+
ingress.class: haproxy
57
spec:
68
- name: tcp-test
79
frontend:

‎deploy/tests/e2e/crd-tcp/config/tcp-cr-full.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ apiVersion: ingress.v1.haproxy.org/v1
22
kind: TCP
33
metadata:
44
name: tcp-1
5+
annotations:
6+
ingress.class: haproxy
57
spec:
68
- name: tcp-test
79
frontend:

‎deploy/tests/e2e/crd-tcp/config/tcp-cr-no-ingress-class.yaml

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
apiVersion: ingress.v1.haproxy.org/v1
2+
kind: TCP
3+
metadata:
4+
name: tcp-1
5+
spec:
6+
- name: tcp-http-echo-80
7+
frontend:
8+
name: fe-http-echo-80
9+
tcplog: true
10+
log_format: "%{+Q}o %t %s"
11+
binds:
12+
- name: v4
13+
port: 32766
14+
- name: v4v6
15+
address: "::"
16+
port: 32766
17+
v4v6: true
18+
service:
19+
name: "http-echo"
20+
port: 80
21+
services:
22+
- name: "http-echo-2"
23+
port: 443
24+
- name: "http-echo-2"
25+
port: 80

‎deploy/tests/e2e/crd-tcp/config/tcp-cr-ssl.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ apiVersion: ingress.v1.haproxy.org/v1
22
kind: TCP
33
metadata:
44
name: tcp-1
5+
annotations:
6+
ingress.class: haproxy
57
spec:
68
- name: tcp-http-echo-443
79
frontend:

‎deploy/tests/e2e/crd-tcp/config/tcp-cr.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ apiVersion: ingress.v1.haproxy.org/v1
22
kind: TCP
33
metadata:
44
name: tcp-1
5+
annotations:
6+
ingress.class: haproxy
57
spec:
68
- name: tcp-http-echo-80
79
frontend:

‎deploy/tests/e2e/crd-tcp/cr_tcp_no_ingress_class_test.go

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
// Copyright 2019 HAProxy Technologies LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
//go:build e2e_sequential
16+
17+
package crdtcp
18+
19+
import (
20+
"strings"
21+
"testing"
22+
23+
parser "github.com/haproxytech/config-parser/v5"
24+
"github.com/haproxytech/config-parser/v5/options"
25+
"github.com/haproxytech/kubernetes-ingress/deploy/tests/e2e"
26+
"github.com/stretchr/testify/suite"
27+
)
28+
29+
type TCPSuiteNoIngressClass struct {
30+
CRDTCPSuite
31+
}
32+
33+
func TestTCPSuiteNoIngressClasss(t *testing.T) {
34+
suite.Run(t, new(TCPSuiteNoIngressClass))
35+
}
36+
37+
// Expected configuration:
38+
// frontend tcpcr_e2e-tests-crd-tcp_fe-http-echo-80
39+
// backend e2e-tests-crd-tcp_http-echo-2_http ## from service/http-echo-2 (port 80)
40+
// backend e2e-tests-crd-tcp_http-echo-2_https ## from service/http-echo-2 (port 443)
41+
// backend e2e-tests-crd-tcp_http-echo_http ## from service/http-echo (port 80)
42+
// SHOULD NOT be created
43+
44+
func (suite *TCPSuiteNoIngressClass) Test_CRD_TCP_No_Ingress_Class() {
45+
suite.Run("TCP CR Additional Services", func() {
46+
var err error
47+
suite.Require().NoError(suite.test.Apply("config/tcp-cr-no-ingress-class.yaml", suite.test.GetNS(), nil))
48+
client2, err := e2e.NewHTTPClient(suite.tmplData.Host2, 32766)
49+
suite.Require().Eventually(func() bool {
50+
_, _, err := client2.Do()
51+
return err != nil // should return an error!
52+
}, e2e.WaitDuration, e2e.TickDuration)
53+
54+
// Get updated config and check it
55+
cfg, err := suite.test.GetIngressControllerFile("/etc/haproxy/haproxy.cfg")
56+
suite.NoError(err, "Could not get Haproxy config")
57+
reader := strings.NewReader(cfg)
58+
p, err := parser.New(options.Reader(reader))
59+
suite.NoError(err, "Could not get Haproxy config parser")
60+
61+
_, err = p.Get(parser.Frontends, "tcpcr_e2e-tests-crd-tcp_fe-http-echo-80", "bind")
62+
suite.Require().Equal(err.Error(), "section missing")
63+
_, err = p.Get(parser.Backends, "e2e-tests-crd-tcp_http-echo-2_http", "mode")
64+
suite.Require().Equal(err.Error(), "section missing")
65+
_, err = p.Get(parser.Backends, "e2e-tests-crd-tcp_http-echo-2_https", "mode")
66+
suite.Require().Equal(err.Error(), "section missing")
67+
_, err = p.Get(parser.Backends, "e2e-tests-crd-tcp_http-echo_http", "mode")
68+
suite.Require().Equal(err.Error(), "section missing")
69+
})
70+
}

‎documentation/custom-resource-tcp.md

+30
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ kind: TCP
3535
metadata:
3636
name: tcp-1
3737
namespace: test
38+
annotations:
39+
ingress.class: haproxy
3840
spec:
3941
- name: tcp-http-echo-8443
4042
frontend:
@@ -103,6 +105,30 @@ Note that in the TCP CR :
103105

104106
Except the frontend keyword `default_backend`, all other lines are not automatically generated but are in a flexible way handled by the `frontend` section in the TCP CR.
105107

108+
## ingress.class
109+
110+
Starting `3.1`, the TCP Custom Resource managed by the Ingress Controller can be filtered using the `ingress.class` annotation.
111+
It behaves the same way as `Ingress`:
112+
113+
| ingress.class controller flag | TCP CR ingress.class annotation | Behavior |
114+
|------------------|---------------------|-----------------|
115+
| '' (not set) | * (any value) | TCP CR managed by IC |
116+
| \<igclass\> | Same value as controller | TCP CR managed by IC |
117+
| \<igclass\> | Value different from controller | TCP CR not managed by IC, frontend and backend deleted if existing |
118+
| \<igclass\> | '' (empty, not set)| if controller `empty-ingress-class` flag is set, TCP CR managed by IC, otherwise ignored (and frontend and backend are deleted)|
119+
120+
121+
### Migration 3.0 to 3.1: action required regarding ingress.class annotation
122+
123+
If some TCP CRs were deployed with Ingress Controller version <= v3.0, and the Ingress Controller has a `ingress.class` flag for the controller, the TCP CRs need to have the same value for the `ingress.class` annotation in the TCP CR.
124+
125+
If the annotation is not set, the corresponding backends and frontends in the haproxy configuration would be deleted:
126+
- except if the controller `empty-ingress-class` flag is set (same behavior as for `Ingress`).
127+
128+
The setting of the `ingress.class` to the TCP CRs should be done **prior to the upgrade to** `v3.1`. It will not be used in v3.0 but needs to be there starting v3.1.
129+
130+
131+
106132
## Pod and Service definitions
107133

108134
with the following Kubernetes Service and Pod manifests:
@@ -258,6 +284,8 @@ kind: TCP
258284
metadata:
259285
name: tcp-2
260286
namespace: test
287+
annotations:
288+
ingress.class: haproxy
261289
spec:
262290
- name: tcp-http-echo-test2-8443
263291
frontend:
@@ -297,6 +325,8 @@ kind: TCP
297325
metadata:
298326
name: tcp-1
299327
namespace: test
328+
annotations:
329+
ingress.class: haproxy
300330
spec:
301331
- name: tcp-http-echo-443
302332
frontend:

‎documentation/tcp-cr-full-example/tcp-cr-full.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ apiVersion: ingress.v1.haproxy.org/v1
22
kind: TCP
33
metadata:
44
name: tcp-1
5+
annotations:
6+
ingress.class: haproxy
57
spec:
68
- name: tcp-test
79
frontend:

‎pkg/controller/handler.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ func (c *HAProxyController) initHandlers() {
5252
&handler.PatternFiles{},
5353
annotations.ConfigSnippetHandler{},
5454
c.updateStatusManager,
55-
handler.TCPCustomResource{},
55+
handler.NewTCPCustomResource(c.osArgs.IngressClass, c.osArgs.EmptyIngressClass),
5656
}
5757

5858
defer func() { c.updateHandlers = append(c.updateHandlers, handler.Refresh{}) }()

‎pkg/handler/tcp-cr.go

+66-1
Original file line numberDiff line numberDiff line change
@@ -42,19 +42,62 @@ import (
4242

4343
const tcpServicePrefix = "tcpcr"
4444

45-
type TCPCustomResource struct{}
45+
type TCPCustomResource struct {
46+
controllerIngressClass string
47+
allowEmptyIngressClass bool
48+
}
4649

4750
type tcpcontext struct {
4851
k store.K8s
4952
namespace string
5053
h haproxy.HAProxy
5154
}
5255

56+
// var syncIngressClassLog sync.Once
57+
func NewTCPCustomResource(controllerIngressClass string, allowEmptyIngressClass bool) TCPCustomResource {
58+
return TCPCustomResource{
59+
controllerIngressClass: controllerIngressClass,
60+
allowEmptyIngressClass: allowEmptyIngressClass,
61+
}
62+
}
63+
64+
// func logTCPMigration30To31Warning() {
65+
// logger := utils.GetLogger()
66+
// // For 3.0, (WARNING)
67+
// // Starting from 3.1, if ingress.class is set for controller, you will need to set the ingress.class annotation in the TCP CRD
68+
// // - Setting the ingress.class annotation in the TCP CRD in 3.0 is highly recommended before migration to 3.1
69+
// // - empty-ingress-class controller option will also impact TCP CRD starting 3.1
70+
// logger.Warning("Using TCP CRD without ingress.class annotation will work only in 3.0")
71+
// logger.Warning("If you are using TCP CRDS without ingress.class annotation and ingress.class is set for the controller,an action is required before migrating to 3.1")
72+
// logger.Warning("Please read https://github.com/haproxytech/kubernetes-ingress/blob/master/documentation/custom-resource-tcp.md for more information")
73+
// }
74+
5375
func (handler TCPCustomResource) Update(k store.K8s, h haproxy.HAProxy, a annotations.Annotations) (err error) {
5476
var errs utils.Errors
5577

5678
for _, ns := range k.Namespaces {
5779
for _, tcpCR := range ns.CRs.TCPsPerCR {
80+
//----------------------------------
81+
// ingress.class migration
82+
// To log in 3.0
83+
// Not in 3.1
84+
// syncIngressClassLog.Do(func() {
85+
// logTCPMigration30To31Warning()
86+
// })
87+
88+
// >= v3.1 for ingress.class
89+
// Not in 3.0
90+
supported := handler.isSupportedIngressClass(tcpCR)
91+
if !supported {
92+
for _, atcp := range tcpCR.Items {
93+
owner := atcp.Owner()
94+
k.FrontendRC.RemoveOwner(owner)
95+
}
96+
continue
97+
}
98+
// end ingress.class migration
99+
//----------------------------
100+
58101
// Cleanup will done after Haproxy config transaction succeeds
59102
if tcpCR.Status == store.DELETED {
60103
continue
@@ -302,3 +345,25 @@ func (handler TCPCustomResource) reconcileAdditionalBackends(ctx tcpcontext, ser
302345
}
303346
return errors.Result()
304347
}
348+
349+
func (handler TCPCustomResource) isSupportedIngressClass(tcps *store.TCPs) bool {
350+
var supported bool
351+
tcpIgClassAnn := tcps.IngressClass
352+
353+
switch handler.controllerIngressClass {
354+
case "", tcpIgClassAnn:
355+
supported = true
356+
default: // mismatch osArgs.Ingress and TCP IngressClass annotation
357+
if tcpIgClassAnn == "" {
358+
supported = handler.allowEmptyIngressClass
359+
if !supported {
360+
utils.GetLogger().Warningf("[SKIP] TCP %s/%s ingress.class annotation='%s' does not match with controller ingress.class flag '%s' and controller flag 'empty-ingress-class' is false",
361+
tcps.Namespace, tcps.Name, tcpIgClassAnn, handler.controllerIngressClass)
362+
}
363+
} else {
364+
utils.GetLogger().Warningf("[SKIP] TCP %s/%s ingress.class annotation='%s' does not match with controller ingress.class flag '%s'",
365+
tcps.Namespace, tcps.Name, tcpIgClassAnn, handler.controllerIngressClass)
366+
}
367+
}
368+
return supported
369+
}

‎pkg/k8s/cr-tcp.go

+5-4
Original file line numberDiff line numberDiff line change
@@ -71,10 +71,11 @@ func convertToStoreTCP(k8sData interface{}, status store.Status) *store.TCPs {
7171
return nil
7272
}
7373
storeTCP := store.TCPs{
74-
Status: status,
75-
Namespace: data.GetNamespace(),
76-
Name: data.GetName(),
77-
Items: make([]*store.TCPResource, 0),
74+
Status: status,
75+
Namespace: data.GetNamespace(),
76+
IngressClass: data.Annotations["ingress.class"],
77+
Name: data.GetName(),
78+
Items: make([]*store.TCPResource, 0),
7879
}
7980
for _, tcp := range data.Spec {
8081
storeTCP.Items = append(storeTCP.Items, &store.TCPResource{

‎pkg/store/types-tcp-cr.go

+8-4
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,11 @@ type TCPResource struct {
4545
type TCPResourceList []*TCPResource
4646

4747
type TCPs struct {
48-
Status Status `json:"status,omitempty"`
49-
Namespace string `json:"namespace,omitempty"`
50-
Name string `json:"name,omitempty"`
51-
Items TCPResourceList `json:"items"`
48+
Status Status `json:"status,omitempty"`
49+
IngressClass string `json:"ingress_class,omitempty"`
50+
Namespace string `json:"namespace,omitempty"`
51+
Name string `json:"name,omitempty"`
52+
Items TCPResourceList `json:"items"`
5253
}
5354

5455
func (a *TCPs) Equal(b *TCPs, opt ...models.Options) bool {
@@ -64,6 +65,9 @@ func (a *TCPs) Equal(b *TCPs, opt ...models.Options) bool {
6465
if a.Namespace != b.Namespace {
6566
return false
6667
}
68+
if a.IngressClass != b.IngressClass {
69+
return false
70+
}
6771
// Always ordered before being added into the store, so no need to order here
6872
a.Items.Order()
6973
b.Items.Order()

0 commit comments

Comments
 (0)