Invalidate registries cache upon configuration change to allow dynamic config updates without restart

Signed-off-by: dtfranz <[email protected]>

Author: dtfranz

cmd/manager/main.go

@@ -102,6 +102,7 @@ func main() {
 		systemNamespace           string
 		caCertDir                 string
 		globalPullSecret          string
+		registriesConfig          string
 	)
 	flag.StringVar(&metricsAddr, "metrics-bind-address", "", "The address for the metrics endpoint. Requires tls-cert and tls-key. (Default: ':8443')")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
@@ -115,6 +116,7 @@ func main() {
 	flag.BoolVar(&operatorControllerVersion, "version", false, "Prints operator-controller version information")
 	flag.StringVar(&systemNamespace, "system-namespace", "", "Configures the namespace that gets used to deploy system resources.")
 	flag.StringVar(&globalPullSecret, "global-pull-secret", "", "The <namespace>/<name> of the global pull secret that is going to be used to pull bundle images.")
+	flag.StringVar(&registriesConfig, "registries-config", "/etc/containers/registries.conf", "The local file path of the registries configuration file")
 
 	klog.InitFlags(flag.CommandLine)
 
@@ -294,8 +296,9 @@ func main() {
 		BaseCachePath: filepath.Join(cachePath, "unpack"),
 		SourceContextFunc: func(logger logr.Logger) (*types.SystemContext, error) {
 			srcContext := &types.SystemContext{
-				DockerCertPath: caCertDir,
-				OCICertPath:    caCertDir,
+				DockerCertPath:           caCertDir,
+				OCICertPath:              caCertDir,
+				SystemRegistriesConfPath: registriesConfig,
 			}
 			if _, err := os.Stat(authFilePath); err == nil && globalPullSecretKey != nil {
 				logger.Info("using available authentication information for pulling image")

internal/rukpak/source/containers_image.go

@@ -3,6 +3,7 @@ package source
 import (
 	"archive/tar"
 	"context"
+	"crypto/sha256"
 	"errors"
 	"fmt"
 	"io"
@@ -17,6 +18,7 @@ import (
 	"github.com/containers/image/v5/oci/layout"
 	"github.com/containers/image/v5/pkg/blobinfocache/none"
 	"github.com/containers/image/v5/pkg/compression"
+	"github.com/containers/image/v5/pkg/sysregistriesv2"
 	"github.com/containers/image/v5/signature"
 	"github.com/containers/image/v5/types"
 	"github.com/go-logr/logr"
@@ -26,8 +28,9 @@ import (
 )
 
 type ContainersImageRegistry struct {
-	BaseCachePath     string
-	SourceContextFunc func(logger logr.Logger) (*types.SystemContext, error)
+	BaseCachePath      string
+	SourceContextFunc  func(logger logr.Logger) (*types.SystemContext, error)
+	registriesConfHash [32]byte
 }
 
 func (i *ContainersImageRegistry) Unpack(ctx context.Context, bundle *BundleSource) (*Result, error) {
@@ -45,6 +48,22 @@ func (i *ContainersImageRegistry) Unpack(ctx context.Context, bundle *BundleSour
 	if err != nil {
 		return nil, err
 	}
+
+	//////////////////////////////////////////////////////
+	//
+	// Verify that registries configuration is up-to-date
+	//
+	//////////////////////////////////////////////////////
+	data, err := os.ReadFile(srcCtx.SystemRegistriesConfPath)
+	if err != nil {
+		return nil, fmt.Errorf("could not read registries configuration file, error: %w", err)
+	}
+	registriesConfHash := sha256.Sum256(data)
+	if registriesConfHash != i.registriesConfHash {
+		i.registriesConfHash = registriesConfHash
+		sysregistriesv2.InvalidateCache()
+	}
+
 	//////////////////////////////////////////////////////
 	//
 	// Resolve a canonical reference for the image.
@@ -250,6 +269,11 @@ func (i *ContainersImageRegistry) unpackImage(ctx context.Context, unpackPath st
 	if err != nil {
 		return fmt.Errorf("error creating image source: %w", err)
 	}
+	defer func() {
+		if err := layoutSrc.Close(); err != nil {
+			panic(err)
+		}
+	}()
 
 	if err := os.MkdirAll(unpackPath, 0700); err != nil {
 		return fmt.Errorf("error creating unpack directory: %w", err)

test/e2e/cluster_extension_install_test.go

@@ -358,6 +358,82 @@ func TestClusterExtensionInstallRegistry(t *testing.T) {
 	}
 }
 
+func TestClusterExtensionInstallRegistryDynamic(t *testing.T) {
+	// NOTE: Like 'TestClusterExtensionInstallRegistry', this test also requires extra configuration in /etc/containers/registries.conf
+	packageName := "dynamic"
+
+	t.Log("When a cluster extension is installed from a catalog")
+	t.Log("When the extension bundle format is registry+v1")
+
+	clusterExtension, extensionCatalog, sa, ns := testInit(t)
+	defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
+	defer getArtifactsOutput(t)
+
+	clusterExtension.Spec = ocv1.ClusterExtensionSpec{
+		Source: ocv1.SourceConfig{
+			SourceType: "Catalog",
+			Catalog: &ocv1.CatalogSource{
+				PackageName: packageName,
+				Selector: &metav1.LabelSelector{
+					MatchLabels: map[string]string{"olm.operatorframework.io/metadata.name": extensionCatalog.Name},
+				},
+			},
+		},
+		Namespace: ns.Name,
+		ServiceAccount: ocv1.ServiceAccountReference{
+			Name: sa.Name,
+		},
+	}
+	t.Log("It updates the registries.conf file contents")
+	cm := corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "e2e-registries-conf",
+			Namespace: "olmv1-system",
+		},
+		Data: map[string]string{
+			"registries.conf": `[[registry]]
+prefix = "dynamic-registry.operator-controller-e2e.svc.cluster.local:5000"
+location = "docker-registry.operator-controller-e2e.svc.cluster.local:5000"`,
+		},
+	}
+	require.NoError(t, c.Update(context.Background(), &cm))
+
+	t.Log("It resolves the specified package with correct bundle path")
+	t.Log("By creating the ClusterExtension resource")
+	require.NoError(t, c.Create(context.Background(), clusterExtension))
+
+	t.Log("By eventually reporting a successful resolution and bundle path")
+	require.EventuallyWithT(t, func(ct *assert.CollectT) {
+		assert.NoError(ct, c.Get(context.Background(), types.NamespacedName{Name: clusterExtension.Name}, clusterExtension))
+	}, 2*time.Minute, pollInterval)
+
+	// Give the check 2 minutes instead of the typical 1 for the pod's
+	// files to update from the configmap change.
+	// The theoretical max time is the kubelet sync period of 1 minute +
+	// ConfigMap cache TTL of 1 minute = 2 minutes
+	t.Log("By eventually reporting progressing as True")
+	require.EventuallyWithT(t, func(ct *assert.CollectT) {
+		assert.NoError(ct, c.Get(context.Background(), types.NamespacedName{Name: clusterExtension.Name}, clusterExtension))
+		cond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1.TypeProgressing)
+		if assert.NotNil(ct, cond) {
+			assert.Equal(ct, metav1.ConditionTrue, cond.Status)
+			assert.Equal(ct, ocv1.ReasonSucceeded, cond.Reason)
+		}
+	}, 2*time.Minute, pollInterval)
+
+	t.Log("By eventually installing the package successfully")
+	require.EventuallyWithT(t, func(ct *assert.CollectT) {
+		assert.NoError(ct, c.Get(context.Background(), types.NamespacedName{Name: clusterExtension.Name}, clusterExtension))
+		cond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1.TypeInstalled)
+		if assert.NotNil(ct, cond) {
+			assert.Equal(ct, metav1.ConditionTrue, cond.Status)
+			assert.Equal(ct, ocv1.ReasonSucceeded, cond.Reason)
+			assert.Contains(ct, cond.Message, "Installed bundle")
+			assert.NotEmpty(ct, clusterExtension.Status.Install.Bundle)
+		}
+	}, pollDuration, pollInterval)
+}
+
 func TestClusterExtensionInstallRegistryMultipleBundles(t *testing.T) {
 	t.Log("When a cluster extension is installed from a catalog")
 

testdata/images/catalogs/test-catalog/v1/configs/catalog.yaml

@@ -69,3 +69,23 @@ properties:
     value:
       packageName: test-mirrored
       version: 1.2.0
+---
+schema: olm.package
+name: dynamic
+defaultChannel: beta
+---
+schema: olm.channel
+name: beta
+package: dynamic
+entries:
+  - name: dynamic-operator.1.2.0
+---
+schema: olm.bundle
+name: dynamic-operator.1.2.0
+package: dynamic
+image: dynamic-registry.operator-controller-e2e.svc.cluster.local:5000/bundles/registry-v1/test-operator:v1.0.0
+properties:
+  - type: olm.package
+    value:
+      packageName: dynamic
+      version: 1.2.0