diff --git a/CHANGELOG.md b/CHANGELOG.md index 61b7575e5..62408c63f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). # [Unreleased](https://github.com/cockroachdb/cockroach-operator/compare/v2.8.0...master) +## Added + +* Add support for enabling Enterprise Encryption at Rest + # [v2.8.0](https://github.com/cockroachdb/cockroach-operator/compare/v2.7.0...v2.8.0) ## Added diff --git a/DEVELOPER.md b/DEVELOPER.md index 4557768ef..eed09cb27 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -116,7 +116,7 @@ The examples directory contains various examples, for example you can run `kubec First you need to create a new Kubernetes Custom Resource, in this example we will use the example/example.yaml file. ```bash -kubectl create -f example/example.yaml +kubectl create -f examples/example.yaml ``` When the database is up and running run the following command to get the first pod that is creasted. diff --git a/apis/v1alpha1/cluster_types.go b/apis/v1alpha1/cluster_types.go index b7d27ae0c..c0ff226f8 100644 --- a/apis/v1alpha1/cluster_types.go +++ b/apis/v1alpha1/cluster_types.go @@ -145,6 +145,26 @@ type CrdbClusterSpec struct { // Default: false // +optional AutomountServiceAccountToken bool `json:"automountServiceAccountToken,omitempty"` + // (Optional) EncryptionEnabled determines if enterprise encryption is enabled for your CockroachDB Cluster + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Enterprise Encryption Enabled",xDescriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch" + // +optional + EncryptionEnabled bool `json:"encryptionEnabled,omitempty"` + // (Optional) The secret with the encryption store keys + // The naming of files is expected as (key) for the active store key + // and (old-key) for the previous store key. + // Default: "" + // +optional + EncryptionStoreKeySecret string `json:"encryptionStoreKeySecret,omitempty"` + // (Optional) Whether the active encryption type is plaintext. This is only applicable + // if encryptionEnabled is set to true + // Default: "" + // +optional + EncryptionTypePlain bool `json:"encryptionTypePlain,omitempty"` + // (Optional) Whether the previous encryption type is plaintext. This is only applicable + // if encryptionEnabled is set to true + // Default: "" + // +optional + OldEncryptionTypePlain bool `json:"oldEncryptionTypePlain,omitempty"` } // +k8s:openapi-gen=true diff --git a/config/crd/bases/crdb.cockroachlabs.com_crdbclusters.yaml b/config/crd/bases/crdb.cockroachlabs.com_crdbclusters.yaml index 457c9e965..8354c4f1e 100644 --- a/config/crd/bases/crdb.cockroachlabs.com_crdbclusters.yaml +++ b/config/crd/bases/crdb.cockroachlabs.com_crdbclusters.yaml @@ -851,6 +851,20 @@ spec: resize without restarting the entire cluster Default: false' type: boolean type: object + encryptionEnabled: + description: (Optional) EncryptionEnabled determines if enterprise + encryption is enabled for your CockroachDB Cluster + type: boolean + encryptionStoreKeySecret: + description: '(Optional) The secret with the encryption store keys + The naming of files is expected as (key) for the active store key + and (old-key) for the previous store key. Default: ""' + type: string + encryptionTypePlain: + description: '(Optional) Whether the active encryption type is plaintext. + This is only applicable if encryptionEnabled is set to true Default: + ""' + type: boolean grpcPort: description: '(Optional) The database port (`--port` CLI parameter when starting the service) Default: 26258' @@ -1024,6 +1038,11 @@ spec: format: int32 minimum: 3 type: integer + oldEncryptionTypePlain: + description: '(Optional) Whether the previous encryption type is plaintext. + This is only applicable if encryptionEnabled is set to true Default: + ""' + type: boolean podEnvVariables: description: '(Optional) PodEnvVariables is a slice of environment variables that are added to the pods Default: (empty list)' diff --git a/examples/example-with-encryption.yaml b/examples/example-with-encryption.yaml new file mode 100644 index 000000000..efd5e757e --- /dev/null +++ b/examples/example-with-encryption.yaml @@ -0,0 +1,72 @@ +# Copyright 2022 The Cockroach Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated, do not edit. Please edit this file instead: config/templates/example.yaml.in +# + +apiVersion: crdb.cockroachlabs.com/v1alpha1 +kind: CrdbCluster +metadata: + # this translates to the name of the statefulset that is created + name: cockroachdb +spec: + dataStore: + pvc: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "10Gi" + volumeMode: Filesystem + resources: + requests: + # This is intentionally low to make it work on local k3d clusters. + cpu: 500m + memory: 2Gi + limits: + cpu: 2 + memory: 8Gi + tlsEnabled: true + encryptionEnabled: true + encryptionStoreKeySecret: store-encryption-key + encryptionTypePlain: false + oldEncryptionTypePlain: true + image: + name: cockroachdb/cockroach:v22.1.7 + # nodes refers to the number of crdb pods that are created + # via the statefulset + nodes: 3 + additionalLabels: + crdb: is-cool + # affinity is a new API field that is behind a feature gate that is + # disabled by default. To enable please see the operator.yaml file. + + # The affinity field will accept any podSpec affinity rule. + # affinity: + # podAntiAffinity: + # preferredDuringSchedulingIgnoredDuringExecution: + # - weight: 100 + # podAffinityTerm: + # labelSelector: + # matchExpressions: + # - key: app.kubernetes.io/instance + # operator: In + # values: + # - cockroachdb + # topologyKey: kubernetes.io/hostname + + # nodeSelectors used to match against + # nodeSelector: + # worker-pool-name: crdb-workers diff --git a/pkg/resource/cluster.go b/pkg/resource/cluster.go index 8a5f6c263..45c312c8f 100644 --- a/pkg/resource/cluster.go +++ b/pkg/resource/cluster.go @@ -199,7 +199,7 @@ func (cluster Cluster) LookupSupportedVersion(version string) (string, bool) { return "", false } -//GetVersionAnnotation gets the current version of the cluster retrieved by version checker action +// GetVersionAnnotation gets the current version of the cluster retrieved by version checker action func (cluster Cluster) GetVersionAnnotation() string { return cluster.getAnnotation(CrdbVersionAnnotation) } @@ -303,6 +303,14 @@ func (cluster Cluster) ClientTLSSecretName() string { return fmt.Sprintf("%s-root", cluster.Name()) } + +func (cluster Cluster) EncryptionKeySecretName() string { + if cluster.Spec().EncryptionStoreKeySecret != "" { + return cluster.Spec().EncryptionStoreKeySecret + } + return "store-encryption-key" +} + func (cluster Cluster) CASecretName() string { return fmt.Sprintf("%s-ca", cluster.Name()) } diff --git a/pkg/resource/statefulset.go b/pkg/resource/statefulset.go index 6f8d780e1..e6422661e 100644 --- a/pkg/resource/statefulset.go +++ b/pkg/resource/statefulset.go @@ -42,6 +42,9 @@ const ( dataDirName = "datadir" dataDirMountPath = "/cockroach/cockroach-data/" + encryptionKeyDirName = "encryptionkeys" + encryptionKeyDirMountPath = "/cockroach/encryption-keys/" + certsDirName = "certs" certCpCmd = ">- cp -p /cockroach/cockroach-certs-prestage/..data/* /cockroach/cockroach-certs/ && chmod 700 /cockroach/cockroach-certs/*.key && chown 1000581000:1000581000 /cockroach/cockroach-certs/*.key" emptyDirName = "emptydir" @@ -98,6 +101,51 @@ func (b StatefulSetBuilder) Build(obj client.Object) error { return err } + if b.Spec().EncryptionEnabled { + if err := addStoreKeysVolumeMountOnInitContiners(DbContainerName, &ss.Spec.Template.Spec); err != nil { + return err + } + if err := addStoreKeysVolumeMount(DbContainerName, &ss.Spec.Template.Spec); err != nil { + return err + } + items := make([]corev1.KeyToPath, 0) + if b.Spec().EncryptionStoreKeySecret != "" { + if !b.Spec().EncryptionTypePlain { + items = append(items, corev1.KeyToPath{ + Key: "key", + Path: "key", + Mode: ptr.Int32(400), + }) + } + + if !b.Spec().OldEncryptionTypePlain { + items = append(items, corev1.KeyToPath{ + Key: "old-key", + Path: "old-key", + Mode: ptr.Int32(400), + }) + } + } + ss.Spec.Template.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: encryptionKeyDirName, + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + DefaultMode: ptr.Int32(400), + Sources: []corev1.VolumeProjection{ + { + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: b.encryptionKeySecretName(), + }, + Items: items, + }, + }, + }, + }, + }, + }) + } + if b.Spec().TLSEnabled { if err := addCertsVolumeMountOnInitContiners(DbContainerName, &ss.Spec.Template.Spec); err != nil { return err @@ -350,6 +398,14 @@ func (b StatefulSetBuilder) clientTLSSecretName() string { return b.Spec().ClientTLSSecret } +func (b StatefulSetBuilder) encryptionKeySecretName() string { + if b.Spec().EncryptionStoreKeySecret == "" { + return b.Cluster.EncryptionKeySecretName() + } + + return b.Spec().EncryptionStoreKeySecret +} + func (b StatefulSetBuilder) commandArgs() []string { exec := "exec " + strings.Join(b.dbArgs(), " ") return []string{"/bin/bash", "-ecx", exec} @@ -386,6 +442,18 @@ func (b StatefulSetBuilder) dbArgs() []string { aa = append(aa, "--max-sql-memory $(expr $MEMORY_LIMIT_MIB / 4)MiB") } + if b.Spec().EncryptionEnabled { + key := "plain" + oldKey := "plain" + if !b.Spec().EncryptionTypePlain { + key = encryptionKeyDirMountPath + "key" + } + if !b.Spec().OldEncryptionTypePlain { + oldKey = encryptionKeyDirMountPath + "old-key" + } + aa = append(aa, fmt.Sprintf("--enterprise-encryption=path=%s,key=%s,old-key=%s", dataDirMountPath, key, oldKey)) + } + aa = append(aa, b.Spec().AdditionalArgs...) needsDefaultJoin := true @@ -412,6 +480,53 @@ func (b StatefulSetBuilder) joinStr() string { return strings.Join(seeds, ",") } + +func addStoreKeysVolumeMountOnInitContiners(container string, spec *corev1.PodSpec) error { + found := false + initContainer := fmt.Sprintf("%s-init", container) + for i := range spec.InitContainers { + c := &spec.InitContainers[i] + if c.Name == initContainer { + found = true + + c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{ + Name: encryptionKeyDirName, + MountPath: encryptionKeyDirMountPath, + }) + + break + } + } + + if !found { + return fmt.Errorf("failed to find container %s to attach volume", container) + } + + return nil +} + +func addStoreKeysVolumeMount(container string, spec *corev1.PodSpec) error { + found := false + for i := range spec.Containers { + c := &spec.Containers[i] + if c.Name == container { + found = true + + c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{ + Name: encryptionKeyDirName, + MountPath: encryptionKeyDirMountPath, + }) + break + } + } + + if !found { + return fmt.Errorf("failed to find container %s to attach volume", container) + } + + return nil +} + func addCertsVolumeMountOnInitContiners(container string, spec *corev1.PodSpec) error { found := false initContainer := fmt.Sprintf("%s-init", container) diff --git a/pkg/resource/testdata/TestStatefulSetBuilder/insecure_statefulset_with_encryption.golden b/pkg/resource/testdata/TestStatefulSetBuilder/insecure_statefulset_with_encryption.golden new file mode 100644 index 000000000..4d63a14d3 --- /dev/null +++ b/pkg/resource/testdata/TestStatefulSetBuilder/insecure_statefulset_with_encryption.golden @@ -0,0 +1,114 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + annotations: + crdb.io/containerimage: "" + crdb.io/version: "" + creationTimestamp: null + name: test-cluster +spec: + podManagementPolicy: Parallel + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: database + app.kubernetes.io/instance: test-cluster + app.kubernetes.io/name: cockroachdb + serviceName: test-cluster + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: database + app.kubernetes.io/instance: test-cluster + app.kubernetes.io/name: cockroachdb + spec: + automountServiceAccountToken: false + containers: + - command: + - /bin/bash + - -ecx + - 'exec /cockroach/cockroach.sh start --advertise-host=$(POD_NAME).test-cluster.test-ns + --insecure --http-port=8080 --sql-addr=:26257 --listen-addr=:26258 --log="{sinks: + {stderr: {channels: [OPS, HEALTH], redact: true}}}" --cache $(expr $MEMORY_LIMIT_MIB + / 4)MiB --max-sql-memory $(expr $MEMORY_LIMIT_MIB / 4)MiB + --enterprise-encryption=path=/cockroach/cockroach-data/,key=store-encryption-key,old-key=plain + --join=test-cluster-0.test-cluster.test-ns:26258' + env: + - name: COCKROACH_CHANNEL + value: kubernetes-operator-gke + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: GOMAXPROCS + valueFrom: + resourceFieldRef: + divisor: "1" + resource: limits.cpu + - name: MEMORY_LIMIT_MIB + valueFrom: + resourceFieldRef: + divisor: 1Mi + resource: limits.memory + image: cockroachdb/cockroach:v21.1.0 + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - sh + - -c + - /cockroach/cockroach node drain --insecure || exit 0 + name: db + ports: + - containerPort: 26258 + name: grpc + protocol: TCP + - containerPort: 8080 + name: http + protocol: TCP + - containerPort: 26257 + name: sql + protocol: TCP + readinessProbe: + failureThreshold: 2 + httpGet: + path: /health?ready=1 + port: http + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + resources: {} + volumeMounts: + - mountPath: /cockroach/cockroach-data/ + name: datadir + securityContext: + fsGroup: 1000581000 + runAsUser: 1000581000 + serviceAccountName: test-cluster-sa + terminationGracePeriodSeconds: 300 + volumes: + - name: datadir + persistentVolumeClaim: + claimName: "" + updateStrategy: + rollingUpdate: {} + volumeClaimTemplates: + - metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: database + app.kubernetes.io/instance: test-cluster + app.kubernetes.io/name: cockroachdb + name: datadir + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + volumeMode: Filesystem + status: {} +status: + replicas: 0 diff --git a/pkg/resource/testdata/TestStatefulSetBuilder/insecure_statefulset_with_encryption_in.yaml b/pkg/resource/testdata/TestStatefulSetBuilder/insecure_statefulset_with_encryption_in.yaml new file mode 100644 index 000000000..aabefbaa7 --- /dev/null +++ b/pkg/resource/testdata/TestStatefulSetBuilder/insecure_statefulset_with_encryption_in.yaml @@ -0,0 +1,43 @@ +# Copyright 2022 The Cockroach Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: crdb.cockroachlabs.com/v1alpha1 +kind: CrdbCluster +metadata: + creationTimestamp: null + name: test-cluster + namespace: test-ns +spec: + dataStore: + pvc: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1Gi" + volumeMode: Filesystem + grpcPort: 26258 + httpPort: 8080 + encryptionEnabled: true + encryptionStoreKeySecret: store-encryption-key + encryptionTypePlain: false + oldEncryptionTypePlain: true + image: + name: cockroachdb/cockroach:v21.1.0 + nodes: 1 + topology: + zones: + - locality: "" +status: {}