Skip to content

Commit 76d177f

Browse files
seans3seans3
committed
Initial e2e test migration from in-tree to gcp cloud provider
1 parent 1f529ad commit 76d177f

File tree

10 files changed

+1660
-2
lines changed

10 files changed

+1660
-2
lines changed

Diff for: ‎BUILD

+1
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,4 @@ gazelle(
1919
)
2020

2121
# gazelle:exclude crd
22+
# gazelle:exclude test/e2e

Diff for: ‎cluster/gce/util.sh

+1
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,7 @@ function copy-to-staging() {
253253
fi
254254
fi
255255

256+
rm -f "${tar}.sha512"
256257
echo "${hash}" > "${tar}.sha512"
257258
gsutil -m -q -h "Cache-Control:private, max-age=0" cp "${tar}" "${tar}.sha512" "${staging_path}"
258259
gsutil -m acl ch -g all:R "${gs_url}" "${gs_url}.sha512" >/dev/null 2>&1 || true

Diff for: ‎test/boskos.sh

+26-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,24 @@
11
#!/usr/bin/env bash
22

3+
# Copyright 2024 The Kubernetes Authors.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# Source this file to be able to acquire a boskos project using
18+
# the acquire_project function. Must be run in prow, since acquiring
19+
# a boskos project expects the owner to be the prow job name (JOB_NAME).
20+
# Can set a custom boskos url by passing the BOSKOS_URL env var.
21+
322
set -o errexit
423
set -o nounset
524
set -o pipefail
@@ -9,13 +28,18 @@ if [[ -z "${BOSKOS_URL:-}" ]]; then
928
BOSKOS_URL="http://boskos.test-pods.svc.cluster.local"
1029
fi
1130

12-
# acquires a project from boskos
31+
# Acquires a "gce" project from boskos. Returns project by setting/exporting PROJECT env var.
32+
# Parameter: JOB_NAME is an env var set by prow. Parameter: BOSKOS_URL is an env var set above
33+
# by either passing in the env var, or using the default url. Starts/runs a heartbeat with
34+
# the returned bosko project. Returns an error if unable to acquire the boskos project.
1335
acquire_project() {
1436
local project=""
1537
local project_type="gce-project"
1638

1739
boskos_response=$(curl -X POST "${BOSKOS_URL}/acquire?type=${project_type}&state=free&dest=busy&owner=${JOB_NAME}")
18-
40+
echo
41+
echo "DEBUG--Boskos Response: ${boskos_response}"
42+
echo
1943
if project=$(echo "${boskos_response}" | jq -r '.name'); then
2044
echo "Using GCP project: ${project}"
2145
PROJECT="${project}"

Diff for: ‎test/e2e/firewall.go

+74
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
package e2e
17+
18+
import (
19+
"context"
20+
"time"
21+
22+
. "github.com/onsi/ginkgo/v2"
23+
v1 "k8s.io/api/core/v1"
24+
clientset "k8s.io/client-go/kubernetes"
25+
"k8s.io/kubernetes/pkg/cluster/ports"
26+
kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config"
27+
"k8s.io/kubernetes/test/e2e/framework"
28+
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
29+
)
30+
31+
const firewallTestTCPTimeout = time.Duration(1 * time.Second)
32+
33+
var _ = Describe("[cloud-provider-gcp-e2e] Firewall Rules", func() {
34+
f := framework.NewDefaultFramework("firewall-rules")
35+
36+
var cs clientset.Interface
37+
BeforeEach(func() {
38+
cs = f.ClientSet
39+
})
40+
41+
AfterEach(func() {
42+
// After each test
43+
})
44+
45+
// Firewall Test
46+
f.It("control plane should not expose well-known ports", func(ctx context.Context) {
47+
nodes, err := e2enode.GetReadySchedulableNodes(ctx, cs)
48+
framework.ExpectNoError(err)
49+
50+
By("Checking well known ports on master and nodes are not exposed externally")
51+
nodeAddr := e2enode.FirstAddress(nodes, v1.NodeExternalIP)
52+
if nodeAddr != "" {
53+
assertNotReachableHTTPTimeout(nodeAddr, "/", ports.KubeletPort, firewallTestTCPTimeout, false)
54+
assertNotReachableHTTPTimeout(nodeAddr, "/", ports.KubeletReadOnlyPort, firewallTestTCPTimeout, false)
55+
assertNotReachableHTTPTimeout(nodeAddr, "/", ports.ProxyStatusPort, firewallTestTCPTimeout, false)
56+
}
57+
58+
controlPlaneAddresses := framework.GetControlPlaneAddresses(ctx, cs)
59+
for _, instanceAddress := range controlPlaneAddresses {
60+
assertNotReachableHTTPTimeout(instanceAddress, "/healthz", ports.KubeControllerManagerPort, firewallTestTCPTimeout, true)
61+
assertNotReachableHTTPTimeout(instanceAddress, "/healthz", kubeschedulerconfig.DefaultKubeSchedulerPort, firewallTestTCPTimeout, true)
62+
}
63+
})
64+
})
65+
66+
func assertNotReachableHTTPTimeout(ip, path string, port int, timeout time.Duration, enableHTTPS bool) {
67+
result := PokeHTTP(ip, port, path, &HTTPPokeParams{Timeout: timeout, EnableHTTPS: enableHTTPS})
68+
if result.Status == HTTPError {
69+
framework.Failf("Unexpected error checking for reachability of %s:%d: %v", ip, port, result.Error)
70+
}
71+
if result.Code != 0 {
72+
framework.Failf("Was unexpectedly able to reach %s:%d", ip, port)
73+
}
74+
}

Diff for: ‎test/e2e/gce.go

+118
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package e2e
18+
19+
import (
20+
"fmt"
21+
"math/rand"
22+
23+
gcecloud "k8s.io/cloud-provider-gcp/providers/gce"
24+
"k8s.io/kubernetes/test/e2e/framework"
25+
)
26+
27+
// Run when the "gce" provider is registered in "init()".
28+
func factory() (framework.ProviderInterface, error) {
29+
framework.Logf("Fetching cloud provider for %q\r", framework.TestContext.Provider)
30+
zone := framework.TestContext.CloudConfig.Zone
31+
region := framework.TestContext.CloudConfig.Region
32+
allowedZones := framework.TestContext.CloudConfig.Zones
33+
34+
// ensure users don't specify a zone outside of the requested zones
35+
if len(zone) > 0 && len(allowedZones) > 0 {
36+
var found bool
37+
for _, allowedZone := range allowedZones {
38+
if zone == allowedZone {
39+
found = true
40+
break
41+
}
42+
}
43+
if !found {
44+
return nil, fmt.Errorf("the provided zone %q must be included in the list of allowed zones %v", zone, allowedZones)
45+
}
46+
}
47+
48+
var err error
49+
if region == "" {
50+
region, err = gcecloud.GetGCERegion(zone)
51+
if err != nil {
52+
return nil, fmt.Errorf("error parsing GCE/GKE region from zone %q: %w", zone, err)
53+
}
54+
}
55+
managedZones := []string{} // Manage all zones in the region
56+
if !framework.TestContext.CloudConfig.MultiZone {
57+
managedZones = []string{zone}
58+
}
59+
if len(allowedZones) > 0 {
60+
managedZones = allowedZones
61+
}
62+
63+
gceCloud, err := gcecloud.CreateGCECloud(&gcecloud.CloudConfig{
64+
APIEndpoint: framework.TestContext.CloudConfig.APIEndpoint,
65+
ProjectID: framework.TestContext.CloudConfig.ProjectID,
66+
Region: region,
67+
Zone: zone,
68+
ManagedZones: managedZones,
69+
NetworkName: "", // TODO: Change this to use framework.TestContext.CloudConfig.Network?
70+
SubnetworkName: "",
71+
NodeTags: nil,
72+
NodeInstancePrefix: "",
73+
TokenSource: nil,
74+
UseMetadataServer: false,
75+
AlphaFeatureGate: gcecloud.NewAlphaFeatureGate([]string{}),
76+
})
77+
78+
if err != nil {
79+
return nil, fmt.Errorf("Error building GCE/GKE provider: %w", err)
80+
}
81+
82+
// Arbitrarily pick one of the zones we have nodes in, looking at prepopulated zones first.
83+
if framework.TestContext.CloudConfig.Zone == "" && len(managedZones) > 0 {
84+
framework.TestContext.CloudConfig.Zone = managedZones[rand.Intn(len(managedZones))]
85+
}
86+
if framework.TestContext.CloudConfig.Zone == "" && framework.TestContext.CloudConfig.MultiZone {
87+
zones, err := gceCloud.GetAllZonesFromCloudProvider()
88+
if err != nil {
89+
return nil, err
90+
}
91+
92+
framework.TestContext.CloudConfig.Zone, _ = zones.PopAny()
93+
}
94+
95+
return NewProvider(gceCloud), nil
96+
}
97+
98+
// Provider is a structure to handle GCE clouds for e2e testing
99+
type Provider struct {
100+
framework.NullProvider
101+
gceCloud *gcecloud.Cloud
102+
}
103+
104+
// NewProvider returns a cloud provider interface for GCE
105+
func NewProvider(gceCloud *gcecloud.Cloud) framework.ProviderInterface {
106+
return &Provider{
107+
gceCloud: gceCloud,
108+
}
109+
}
110+
111+
// GetGCECloud returns GCE cloud provider
112+
func GetGCECloud() (*gcecloud.Cloud, error) {
113+
p, ok := framework.TestContext.CloudConfig.Provider.(*Provider)
114+
if !ok {
115+
return nil, fmt.Errorf("failed to convert CloudConfig.Provider to GCE provider: %#v", framework.TestContext.CloudConfig.Provider)
116+
}
117+
return p.gceCloud, nil
118+
}

Diff for: ‎test/e2e/go.mod

+138
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
module k8s.io/cloud-provider-gcp/tests/e2e
2+
3+
go 1.22.0
4+
5+
toolchain go1.22.1
6+
7+
require (
8+
github.com/onsi/ginkgo/v2 v2.15.0
9+
github.com/onsi/gomega v1.31.0
10+
google.golang.org/api v0.151.0
11+
k8s.io/api v0.30.0
12+
k8s.io/apimachinery v0.30.0
13+
k8s.io/client-go v0.30.0
14+
k8s.io/cloud-provider-gcp/providers v0.0.0-00010101000000-000000000000
15+
k8s.io/kubernetes v1.30.0
16+
k8s.io/pod-security-admission v0.30.0
17+
)
18+
19+
require (
20+
cloud.google.com/go/compute v1.23.1 // indirect
21+
cloud.google.com/go/compute/metadata v0.2.3 // indirect
22+
github.com/GoogleCloudPlatform/k8s-cloud-provider v1.25.0 // indirect
23+
github.com/NYTimes/gziphandler v1.1.1 // indirect
24+
github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect
25+
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
26+
github.com/beorn7/perks v1.0.1 // indirect
27+
github.com/blang/semver/v4 v4.0.0 // indirect
28+
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
29+
github.com/cespare/xxhash/v2 v2.2.0 // indirect
30+
github.com/coreos/go-semver v0.3.1 // indirect
31+
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
32+
github.com/davecgh/go-spew v1.1.1 // indirect
33+
github.com/distribution/reference v0.5.0 // indirect
34+
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
35+
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
36+
github.com/felixge/httpsnoop v1.0.3 // indirect
37+
github.com/fsnotify/fsnotify v1.7.0 // indirect
38+
github.com/go-logr/logr v1.4.1 // indirect
39+
github.com/go-logr/stdr v1.2.2 // indirect
40+
github.com/go-openapi/jsonpointer v0.19.6 // indirect
41+
github.com/go-openapi/jsonreference v0.20.2 // indirect
42+
github.com/go-openapi/swag v0.22.3 // indirect
43+
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
44+
github.com/gogo/protobuf v1.3.2 // indirect
45+
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
46+
github.com/golang/protobuf v1.5.4 // indirect
47+
github.com/google/cel-go v0.17.8 // indirect
48+
github.com/google/gnostic-models v0.6.8 // indirect
49+
github.com/google/go-cmp v0.6.0 // indirect
50+
github.com/google/gofuzz v1.2.0 // indirect
51+
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
52+
github.com/google/s2a-go v0.1.7 // indirect
53+
github.com/google/uuid v1.4.0 // indirect
54+
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
55+
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
56+
github.com/gorilla/websocket v1.5.0 // indirect
57+
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
58+
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
59+
github.com/imdario/mergo v0.3.6 // indirect
60+
github.com/inconshreveable/mousetrap v1.1.0 // indirect
61+
github.com/josharian/intern v1.0.0 // indirect
62+
github.com/json-iterator/go v1.1.12 // indirect
63+
github.com/mailru/easyjson v0.7.7 // indirect
64+
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
65+
github.com/moby/spdystream v0.2.0 // indirect
66+
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
67+
github.com/modern-go/reflect2 v1.0.2 // indirect
68+
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
69+
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
70+
github.com/opencontainers/go-digest v1.0.0 // indirect
71+
github.com/pkg/errors v0.9.1 // indirect
72+
github.com/prometheus/client_golang v1.16.0 // indirect
73+
github.com/prometheus/client_model v0.4.0 // indirect
74+
github.com/prometheus/common v0.44.0 // indirect
75+
github.com/prometheus/procfs v0.10.1 // indirect
76+
github.com/spf13/cobra v1.7.0 // indirect
77+
github.com/spf13/pflag v1.0.5 // indirect
78+
github.com/stoewer/go-strcase v1.2.0 // indirect
79+
go.etcd.io/etcd/api/v3 v3.5.10 // indirect
80+
go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect
81+
go.etcd.io/etcd/client/v3 v3.5.10 // indirect
82+
go.opencensus.io v0.24.0 // indirect
83+
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 // indirect
84+
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 // indirect
85+
go.opentelemetry.io/otel v1.19.0 // indirect
86+
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect
87+
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 // indirect
88+
go.opentelemetry.io/otel/metric v1.19.0 // indirect
89+
go.opentelemetry.io/otel/sdk v1.19.0 // indirect
90+
go.opentelemetry.io/otel/trace v1.19.0 // indirect
91+
go.opentelemetry.io/proto/otlp v1.0.0 // indirect
92+
go.uber.org/multierr v1.11.0 // indirect
93+
go.uber.org/zap v1.26.0 // indirect
94+
golang.org/x/crypto v0.21.0 // indirect
95+
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
96+
golang.org/x/net v0.23.0 // indirect
97+
golang.org/x/oauth2 v0.13.0 // indirect
98+
golang.org/x/sync v0.6.0 // indirect
99+
golang.org/x/sys v0.18.0 // indirect
100+
golang.org/x/term v0.18.0 // indirect
101+
golang.org/x/text v0.14.0 // indirect
102+
golang.org/x/time v0.3.0 // indirect
103+
golang.org/x/tools v0.18.0 // indirect
104+
google.golang.org/appengine v1.6.7 // indirect
105+
google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect
106+
google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect
107+
google.golang.org/genproto/googleapis/rpc v0.0.0-20231030173426-d783a09b4405 // indirect
108+
google.golang.org/grpc v1.59.0 // indirect
109+
google.golang.org/protobuf v1.33.0 // indirect
110+
gopkg.in/gcfg.v1 v1.2.3 // indirect
111+
gopkg.in/inf.v0 v0.9.1 // indirect
112+
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
113+
gopkg.in/warnings.v0 v0.1.2 // indirect
114+
gopkg.in/yaml.v2 v2.4.0 // indirect
115+
gopkg.in/yaml.v3 v3.0.1 // indirect
116+
k8s.io/apiextensions-apiserver v0.0.0 // indirect
117+
k8s.io/apiserver v0.30.0 // indirect
118+
k8s.io/cloud-provider v0.30.0 // indirect
119+
k8s.io/component-base v0.30.0 // indirect
120+
k8s.io/component-helpers v0.30.0 // indirect
121+
k8s.io/controller-manager v0.30.0 // indirect
122+
k8s.io/klog/v2 v2.120.1 // indirect
123+
k8s.io/kms v0.30.0 // indirect
124+
k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
125+
k8s.io/kubectl v0.0.0 // indirect
126+
k8s.io/kubelet v0.30.0 // indirect
127+
k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
128+
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 // indirect
129+
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
130+
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
131+
sigs.k8s.io/yaml v1.3.0 // indirect
132+
)
133+
134+
replace (
135+
k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.30.0
136+
k8s.io/cloud-provider-gcp/providers => ../../providers
137+
k8s.io/kubectl => k8s.io/kubectl v0.30.0
138+
)

0 commit comments

Comments
 (0)