From 65fb20a60e577cd5d37b338fc07d5448dcbbb4aa Mon Sep 17 00:00:00 2001 From: "David Muto (pseudomuto)" Date: Tue, 24 May 2022 18:09:55 +0000 Subject: [PATCH 1/2] Run only upgrades e2e test for nightlies Our nightlies have been (sometimes) timing out for a while now. Currently we run create, update, and decommission e2e tests for the nightly CI runs for GKE and OpenShift. This PR reduces that list to just the upgrades test. This involves creating a cluster, updating the version, and decommissioning it, which should be sufficient for us. We still run each of the e2e tests in CI, so this doesn't represent a reduction in overall test coverage, just limits what's being tested in the nightly run. --- Makefile | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Makefile b/Makefile index 511bdd3dc..a7021fe34 100644 --- a/Makefile +++ b/Makefile @@ -114,8 +114,6 @@ test/e2e/k3d-%: test/e2e/testrunner-eks: KUBECONFIG=$(TMPDIR)/$(CLUSTER_NAME)-eks.kubeconfig.yaml bazel-bin/hack/bin/kubectl create -f hack/eks-storageclass.yaml bazel test --stamp //e2e/upgrades/... --action_env=KUBECONFIG=$(TMPDIR)/$(CLUSTER_NAME)-eks.kubeconfig.yaml - bazel test --stamp //e2e/create/... --action_env=KUBECONFIG=$(TMPDIR)/$(CLUSTER_NAME)-eks.kubeconfig.yaml - bazel test --stamp //e2e/decommission/... --action_env=KUBECONFIG=$(TMPDIR)/$(CLUSTER_NAME)-eks.kubeconfig.yaml # Use this target to run e2e tests with a eks cluster. # This target uses kubetest2 to start a eks k8s cluster and runs the e2e tests @@ -143,9 +141,6 @@ test/e2e/testrunner-gke: #bazel run --stamp --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 \ # //manifests:install_operator.apply bazel test --stamp //e2e/upgrades/... - bazel test --stamp //e2e/create/... - bazel test --stamp --test_arg=--pvc=true //e2e/pvcresize/... - bazel test --stamp //e2e/decommission/... # Use this target to run e2e tests with a gke cluster. # This target uses kubetest2 to start a gke k8s cluster and runs the e2e tests @@ -171,8 +166,6 @@ test/e2e/gke: .PHONY: test/e2e/testrunner-openshift test/e2e/testrunner-openshift: bazel test --stamp //e2e/upgrades/... --action_env=KUBECONFIG=$(HOME)/openshift-$(CLUSTER_NAME)/auth/kubeconfig - bazel test --stamp //e2e/create/... --action_env=KUBECONFIG=$(HOME)/openshift-$(CLUSTER_NAME)/auth/kubeconfig - bazel test --stamp //e2e/decommission/... --action_env=KUBECONFIG=$(HOME)/openshift-$(CLUSTER_NAME)/auth/kubeconfig # Use this target to run e2e tests with a openshift cluster. # This target uses kubetest2 to start a openshift cluster and runs the e2e tests From ff8901d3edba8c2e9c2961075f91712f3fa50016 Mon Sep 17 00:00:00 2001 From: "David Muto (pseudomuto)" Date: Tue, 24 May 2022 19:14:39 +0000 Subject: [PATCH 2/2] Bump kubetest2 binaries --- hack/bin/BUILD.bazel | 40 +++++---------------- hack/bin/deps.bzl | 67 +++++++++++------------------------- pkg/resource/BUILD.bazel | 2 +- pkg/testutil/env/BUILD.bazel | 1 + pkg/testutil/env/env.go | 27 +++++++++++---- 5 files changed, 52 insertions(+), 85 deletions(-) diff --git a/hack/bin/BUILD.bazel b/hack/bin/BUILD.bazel index 9fc2bc241..ccd99c271 100644 --- a/hack/bin/BUILD.bazel +++ b/hack/bin/BUILD.bazel @@ -143,38 +143,16 @@ genrule( genrule( name = "fetch_kubetest2", srcs = select({ - ":m1": ["@kubetest2_darwin//file"], - ":darwin": ["@kubetest2_darwin//file"], - ":k8": ["@kubetest2_linux//file"], + ":m1": ["@kubetest2_darwin//:files"], + ":darwin": ["@kubetest2_darwin//:files"], + ":k8": ["@kubetest2_linux//:files"], }), - outs = ["kubetest2"], - cmd = "cp $(SRCS) $@", - visibility = ["//visibility:public"], -) - -# fetch_kubetest2_gke rules to fetch the binary for kubetest2_gke used on e2e tests that are saved on a google cloud bucket -genrule( - name = "fetch_kubetest2_gke", - srcs = select({ - ":m1": ["@kubetest2_gke_darwin//file"], - ":darwin": ["@kubetest2_gke_darwin//file"], - ":k8": ["@kubetest2_gke_linux//file"], - }), - outs = ["kubetest2-gke"], - cmd = "cp $(SRCS) $@", - visibility = ["//visibility:public"], -) - -# fetch_kubetest2_exe`rules to fetch the binary for kubetest2_exe used on e2e tests that are saved on a google cloud bucket -genrule( - name = "fetch_kubetest2_exe", - srcs = select({ - ":m1": ["@kubetest2_exe_darwin//file"], - ":darwin": ["@kubetest2_exe_darwin//file"], - ":k8": ["@kubetest2_exe_linux//file"], - }), - outs = ["kubetest2-tester-exec"], - cmd = "cp $(SRCS) $@", + outs = [ + "kubetest2", + "kubetest2-gke", + "kubetest2-tester-exec", + ], + cmd = "cp $(SRCS) $(@D)", visibility = ["//visibility:public"], ) diff --git a/hack/bin/deps.bzl b/hack/bin/deps.bzl index 45bdebd20..8d8e5abf4 100644 --- a/hack/bin/deps.bzl +++ b/hack/bin/deps.bzl @@ -64,8 +64,6 @@ def install(): install_k3d() install_kubetest2() install_kubetest2_aws() - install_kubetest2_exe() - install_kubetest2_gke() install_kustomize() install_oc() install_operator_sdk() @@ -266,54 +264,31 @@ filegroup( ## Fetch kubetest2 binary used during e2e tests def install_kubetest2(): - # install kubetest2 binary - http_file( - name = "kubetest2_darwin", - executable = 1, - sha256 = "5b20aadd05eca47dead180a7c8296d75e81c184aabf182d4a41ef96597db543d", - urls = ["https://storage.googleapis.com/crdb-bazel-artifacts/osx/kubetest2"], - ) + url = "https://github.com/pseudomuto/kubetest2/releases/download/v0.1.0/kubetest2_0.1.0_{}_amd64.tar.gz" + content = """ +filegroup( + name = "files", + srcs = [ + "kubetest2", + "kubetest2-gke", + "kubetest2-tester-exec", + ], + visibility = ["//visibility:public"], +) +""" - http_file( + http_archive( name = "kubetest2_linux", - executable = 1, - sha256 = "7f0b05654fa43ca1c607db297b5f3a775f65eea90355bb6b10137a7fffff5e1a", - urls = ["https://storage.googleapis.com/crdb-bazel-artifacts/linux/kubetest2"], + sha256 = "04319fb9cef1a5370548c47a0c65fa8d4f85a1b6d4417b318a93125d72590d1b", + urls = [url.format("linux")], + build_file_content = content, ) -## Fetch kubetest2-gke binary used during e2e tests -def install_kubetest2_gke(): - # install kubetest2-gke binary - # TODO osx support - http_file( - name = "kubetest2_gke_darwin", - executable = 1, - sha256 = "a1cbe02f61931dbe6c8d1662442f42cb538c81e4ec8cdd40f548f0e05cbd55a7", - urls = ["https://storage.googleapis.com/crdb-bazel-artifacts/osx/kubetest2-gke"], - ) - - http_file( - name = "kubetest2_gke_linux", - executable = 1, - sha256 = "9ac658234efc7f59968888662dd2d21908587789f6b812392ac5b6766b17c0b4", - urls = ["https://storage.googleapis.com/crdb-bazel-artifacts/linux/kubetest2-gke"], - ) -## Fetch kubetest2-tester-exe binary used during e2e tests -def install_kubetest2_exe(): - # install kubetest2-exe binary - # TODO osx support - http_file( - name = "kubetest2_exe_darwin", - executable = 1, - sha256 = "818690cb55590440e163b18dd139c8a8714df9480f869bafe19eb344047cf37c", - urls = ["https://storage.googleapis.com/crdb-bazel-artifacts/osx/kubetest2-tester-exec"], - ) - - http_file( - name = "kubetest2_exe_linux", - executable = 1, - sha256 = "4483f40f48b98e8a6aa41f58bfdf1f2787066a4e1ad1343e4281892aa1326736", - urls = ["https://storage.googleapis.com/crdb-bazel-artifacts/linux/kubetest2-tester-exec"], + http_archive( + name = "kubetest2_darwin", + sha256 = "972d639e9548f296c8470c3418630baa4176aabc2b2e5d47d62b9eb8ee2b91cf", + urls = [url.format("darwin")], + build_file_content = content, ) ## Fetch operator-sdk used on generating csv diff --git a/pkg/resource/BUILD.bazel b/pkg/resource/BUILD.bazel index f9d459125..34d3b85e1 100644 --- a/pkg/resource/BUILD.bazel +++ b/pkg/resource/BUILD.bazel @@ -57,8 +57,8 @@ go_library( go_test( name = "go_default_test", srcs = [ - "cluster_test.go", "certificate_test.go", + "cluster_test.go", "discovery_service_test.go", "pod_distruption_budget_test.go", "public_service_test.go", diff --git a/pkg/testutil/env/BUILD.bazel b/pkg/testutil/env/BUILD.bazel index 5adb680d5..3af829c90 100644 --- a/pkg/testutil/env/BUILD.bazel +++ b/pkg/testutil/env/BUILD.bazel @@ -24,6 +24,7 @@ go_library( "//apis/v1alpha1:go_default_library", "//pkg/client/clientset/versioned:go_default_library", "//pkg/kube:go_default_library", + "@com_github_cenkalti_backoff//:go_default_library", "@com_github_cockroachdb_errors//:go_default_library", "@com_github_stretchr_testify//require:go_default_library", "@io_k8s_api//core/v1:go_default_library", diff --git a/pkg/testutil/env/env.go b/pkg/testutil/env/env.go index bb606d6bf..c202dcb2a 100644 --- a/pkg/testutil/env/env.go +++ b/pkg/testutil/env/env.go @@ -20,12 +20,14 @@ import ( "flag" "fmt" "os" + "time" + "github.com/cenkalti/backoff" api "github.com/cockroachdb/cockroach-operator/apis/v1alpha1" customClient "github.com/cockroachdb/cockroach-operator/pkg/client/clientset/versioned" "github.com/cockroachdb/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - apiruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" @@ -40,7 +42,7 @@ import ( // environments. var testBinaries = flag.String("binaries", "hack/bin", "") -func NewEnv(builder apiruntime.SchemeBuilder) *Env { +func NewEnv(builder runtime.SchemeBuilder) *Env { flag.Parse() // ensure hack/bin is added to the path and KUBEBUILDER_ASSETS @@ -48,7 +50,7 @@ func NewEnv(builder apiruntime.SchemeBuilder) *Env { os.Setenv("KUBEBUILDER_ASSETS", p) PrependToPath(p) - scheme := apiruntime.NewScheme() + scheme := runtime.NewScheme() if err := kscheme.AddToScheme(scheme); err != nil { panic(err) @@ -76,7 +78,7 @@ func NewEnv(builder apiruntime.SchemeBuilder) *Env { type Env struct { envtest.Environment - Scheme *apiruntime.Scheme + Scheme *runtime.Scheme } func (env *Env) Start() *ActiveEnv { @@ -129,7 +131,7 @@ func (k k8s) namespaceableResource(gvr schema.GroupVersionResource) dynamic.Name type ActiveEnv struct { *k8s - scheme *apiruntime.Scheme + scheme *runtime.Scheme resources []schema.GroupVersionResource } @@ -139,8 +141,19 @@ func CreateActiveEnvForTest() *Env { } func loadResources(k *k8s) ([]schema.GroupVersionResource, error) { - lists, err := k.DiscoveryClient.ServerPreferredResources() - if err != nil { + b := backoff.NewExponentialBackOff() + b.MaxElapsedTime = 60 * time.Second + lists := make([]*metav1.APIResourceList, 0) + + // GKE was having issues fetching api resources. Giving it some time + // (tested via time.Sleep(10*time.Second)) seemed to resolve this issue. + // Rather than a sleep call, we should retry until it's ready or ultimately + // fails. + if err := backoff.Retry(func() error { + var err error + lists, err = k.DiscoveryClient.ServerPreferredResources() + return err + }, b); err != nil { return nil, errors.Wrap(err, "failed to fetch preferred server resource") }