Invalidate registries cache upon configuration change to allow dynamic config updates without restart

Signed-off-by: dtfranz <[email protected]>

Author: dtfranz

cmd/manager/main.go

@@ -291,7 +291,8 @@ func main() {
 	}
 
 	unpacker := &source.ContainersImageRegistry{
-		BaseCachePath: filepath.Join(cachePath, "unpack"),
+		BaseCachePath:           filepath.Join(cachePath, "unpack"),
+		RegistriesConfTimestamp: time.Now(),
 		SourceContextFunc: func(logger logr.Logger) (*types.SystemContext, error) {
 			srcContext := &types.SystemContext{
 				DockerCertPath: caCertDir,
@@ -306,7 +307,8 @@ func main() {
 				return nil, fmt.Errorf("could not stat auth file, error: %w", err)
 			}
 			return srcContext, nil
-		}}
+		},
+	}
 
 	clusterExtensionFinalizers := crfinalizer.NewFinalizers()
 	if err := clusterExtensionFinalizers.Register(controllers.ClusterExtensionCleanupUnpackCacheFinalizer, finalizers.FinalizerFunc(func(ctx context.Context, obj client.Object) (crfinalizer.Result, error) {

internal/rukpak/source/containers_image.go

@@ -6,8 +6,12 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
+	"strings"
+	"sync"
+	"time"
 
 	"github.com/containerd/containerd/archive"
 	"github.com/containers/image/v5/copy"
@@ -17,6 +21,7 @@ import (
 	"github.com/containers/image/v5/oci/layout"
 	"github.com/containers/image/v5/pkg/blobinfocache/none"
 	"github.com/containers/image/v5/pkg/compression"
+	"github.com/containers/image/v5/pkg/sysregistriesv2"
 	"github.com/containers/image/v5/signature"
 	"github.com/containers/image/v5/types"
 	"github.com/go-logr/logr"
@@ -25,9 +30,17 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
+const (
+	// Pattern match for symlinks mounted via configmap: YYYY_MM_DD_hh_mm_ss.s
+	mountedSymlinkTimestampFormat = "2006_01_02_15_04_05.999999999"
+	containersConfigFolder        = "/etc/containers/"
+)
+
 type ContainersImageRegistry struct {
-	BaseCachePath     string
-	SourceContextFunc func(logger logr.Logger) (*types.SystemContext, error)
+	BaseCachePath           string
+	SourceContextFunc       func(logger logr.Logger) (*types.SystemContext, error)
+	RegistriesConfTimestamp time.Time
+	checkConfigLock         sync.RWMutex
 }
 
 func (i *ContainersImageRegistry) Unpack(ctx context.Context, bundle *BundleSource) (*Result, error) {
@@ -45,6 +58,17 @@ func (i *ContainersImageRegistry) Unpack(ctx context.Context, bundle *BundleSour
 	if err != nil {
 		return nil, err
 	}
+
+	//////////////////////////////////////////////////////
+	//
+	// Verify that registries configuration is up-to-date
+	//
+	//////////////////////////////////////////////////////
+	err = i.checkUpdatedConfiguration()
+	if err != nil {
+		return nil, fmt.Errorf("error checking registries configuration: %w", err)
+	}
+
 	//////////////////////////////////////////////////////
 	//
 	// Resolve a canonical reference for the image.
@@ -171,6 +195,35 @@ func (i *ContainersImageRegistry) Cleanup(_ context.Context, bundle *BundleSourc
 	return deleteRecursive(i.bundlePath(bundle.Name))
 }
 
+func (i *ContainersImageRegistry) checkUpdatedConfiguration() error {
+	i.checkConfigLock.Lock()
+	defer i.checkConfigLock.Unlock()
+	dirEntries, err := os.ReadDir(containersConfigFolder)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		// Ignore not found errors
+		return fmt.Errorf("could not read registries configuration directory, error: %w", err)
+	}
+	for _, entry := range dirEntries {
+		if entry.Type().IsRegular() || !strings.HasPrefix(entry.Name(), "..") {
+			// Skip 'normal' entries
+			continue
+		}
+		// Trim the leading '..' for symlinks
+		trimmed := strings.TrimPrefix(entry.Name(), "..")
+		t, err := time.Parse(mountedSymlinkTimestampFormat, trimmed)
+		if err != nil {
+			continue
+		} else if !t.Equal(i.RegistriesConfTimestamp) {
+			// We've found the timestamped symlink and it's been updated
+			i.RegistriesConfTimestamp = t
+			sysregistriesv2.InvalidateCache()
+			return nil
+		}
+	}
+	// No issue if the file was not found - likely we just don't have mounted registries configuration.
+	return nil
+}
+
 func (i *ContainersImageRegistry) bundlePath(bundleName string) string {
 	return filepath.Join(i.BaseCachePath, bundleName)
 }
@@ -250,6 +303,11 @@ func (i *ContainersImageRegistry) unpackImage(ctx context.Context, unpackPath st
 	if err != nil {
 		return fmt.Errorf("error creating image source: %w", err)
 	}
+	defer func() {
+		if err := layoutSrc.Close(); err != nil {
+			panic(err)
+		}
+	}()
 
 	if err := os.MkdirAll(unpackPath, 0700); err != nil {
 		return fmt.Errorf("error creating unpack directory: %w", err)

test/e2e/cluster_extension_install_test.go

@@ -358,6 +358,82 @@ func TestClusterExtensionInstallRegistry(t *testing.T) {
 	}
 }
 
+func TestClusterExtensionInstallRegistryDynamic(t *testing.T) {
+	// NOTE: Like 'TestClusterExtensionInstallRegistry', this test also requires extra configuration in /etc/containers/registries.conf
+	packageName := "dynamic"
+
+	t.Log("When a cluster extension is installed from a catalog")
+	t.Log("When the extension bundle format is registry+v1")
+
+	clusterExtension, extensionCatalog, sa, ns := testInit(t)
+	defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
+	defer getArtifactsOutput(t)
+
+	clusterExtension.Spec = ocv1.ClusterExtensionSpec{
+		Source: ocv1.SourceConfig{
+			SourceType: "Catalog",
+			Catalog: &ocv1.CatalogSource{
+				PackageName: packageName,
+				Selector: &metav1.LabelSelector{
+					MatchLabels: map[string]string{"olm.operatorframework.io/metadata.name": extensionCatalog.Name},
+				},
+			},
+		},
+		Namespace: ns.Name,
+		ServiceAccount: ocv1.ServiceAccountReference{
+			Name: sa.Name,
+		},
+	}
+	t.Log("It updates the registries.conf file contents")
+	cm := corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "e2e-registries-conf",
+			Namespace: "olmv1-system",
+		},
+		Data: map[string]string{
+			"registries.conf": `[[registry]]
+prefix = "dynamic-registry.operator-controller-e2e.svc.cluster.local:5000"
+location = "docker-registry.operator-controller-e2e.svc.cluster.local:5000"`,
+		},
+	}
+	require.NoError(t, c.Update(context.Background(), &cm))
+
+	t.Log("It resolves the specified package with correct bundle path")
+	t.Log("By creating the ClusterExtension resource")
+	require.NoError(t, c.Create(context.Background(), clusterExtension))
+
+	t.Log("By eventually reporting a successful resolution and bundle path")
+	require.EventuallyWithT(t, func(ct *assert.CollectT) {
+		assert.NoError(ct, c.Get(context.Background(), types.NamespacedName{Name: clusterExtension.Name}, clusterExtension))
+	}, 2*time.Minute, pollInterval)
+
+	// Give the check 2 minutes instead of the typical 1 for the pod's
+	// files to update from the configmap change.
+	// The theoretical max time is the kubelet sync period of 1 minute +
+	// ConfigMap cache TTL of 1 minute = 2 minutes
+	t.Log("By eventually reporting progressing as True")
+	require.EventuallyWithT(t, func(ct *assert.CollectT) {
+		assert.NoError(ct, c.Get(context.Background(), types.NamespacedName{Name: clusterExtension.Name}, clusterExtension))
+		cond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1.TypeProgressing)
+		if assert.NotNil(ct, cond) {
+			assert.Equal(ct, metav1.ConditionTrue, cond.Status)
+			assert.Equal(ct, ocv1.ReasonSucceeded, cond.Reason)
+		}
+	}, 2*time.Minute, pollInterval)
+
+	t.Log("By eventually installing the package successfully")
+	require.EventuallyWithT(t, func(ct *assert.CollectT) {
+		assert.NoError(ct, c.Get(context.Background(), types.NamespacedName{Name: clusterExtension.Name}, clusterExtension))
+		cond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1.TypeInstalled)
+		if assert.NotNil(ct, cond) {
+			assert.Equal(ct, metav1.ConditionTrue, cond.Status)
+			assert.Equal(ct, ocv1.ReasonSucceeded, cond.Reason)
+			assert.Contains(ct, cond.Message, "Installed bundle")
+			assert.NotEmpty(ct, clusterExtension.Status.Install.Bundle)
+		}
+	}, pollDuration, pollInterval)
+}
+
 func TestClusterExtensionInstallRegistryMultipleBundles(t *testing.T) {
 	t.Log("When a cluster extension is installed from a catalog")
 

testdata/images/catalogs/test-catalog/v1/configs/catalog.yaml

@@ -69,3 +69,23 @@ properties:
     value:
       packageName: test-mirrored
       version: 1.2.0
+---
+schema: olm.package
+name: dynamic
+defaultChannel: beta
+---
+schema: olm.channel
+name: beta
+package: dynamic
+entries:
+  - name: dynamic-operator.1.2.0
+---
+schema: olm.bundle
+name: dynamic-operator.1.2.0
+package: dynamic
+image: dynamic-registry.operator-controller-e2e.svc.cluster.local:5000/bundles/registry-v1/test-operator:v1.0.0
+properties:
+  - type: olm.package
+    value:
+      packageName: dynamic
+      version: 1.2.0