fix: ClusterPromotion: react to referenced ConfigMap/Secret changes

config/default/manager_pull_policy.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       initContainers:
       - name: initialization
-        imagePullPolicy: 
+        imagePullPolicy: IfNotPresent 
       containers:
       - name: controller
-        imagePullPolicy: 
+        imagePullPolicy: IfNotPresent

config/rbac/role.yaml

@@ -8,10 +8,13 @@ rules:
   - ""
   resources:
   - configmaps
+  - secrets
   verbs:
+  - create
   - delete
   - get
   - list
+  - update
   - watch
 - apiGroups:
   - ""
@@ -20,14 +23,6 @@ rules:
   verbs:
   - get
   - list
-- apiGroups:
-  - ""
-  resources:
-  - secrets
-  verbs:
-  - get
-  - list
-  - watch
 - apiGroups:
   - '*'
   resources:

controllers/clusterpromotion_controller.go

@@ -19,7 +19,9 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"sync"
 
+	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/rest"
@@ -29,6 +31,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
@@ -37,6 +40,7 @@ import (
 	configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1"
 	"github.com/projectsveltos/addon-controller/pkg/scope"
 	logs "github.com/projectsveltos/libsveltos/lib/logsettings"
+	libsveltosset "github.com/projectsveltos/libsveltos/lib/set"
 )
 
 const (
@@ -84,12 +88,18 @@ type ClusterPromotionReconciler struct {
 	Scheme               *runtime.Scheme
 	eventRecorder        events.EventRecorder
 	ConcurrentReconciles int
+
+	PolicyMux    sync.Mutex                                    // use a Mutex to update ReferenceMap as MaxConcurrentReconciles is higher than one
+	ReferenceMap map[corev1.ObjectReference]*libsveltosset.Set // key: Referenced object; value: set of all ClusterPromotions referencing the resource
 }
 
 // +kubebuilder:rbac:groups=config.projectsveltos.io,resources=clusterpromotions,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=config.projectsveltos.io,resources=clusterpromotions/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=config.projectsveltos.io,resources=clusterpromotions/finalizers,verbs=update
 // +kubebuilder:rbac:groups=config.projectsveltos.io,resources=clusterprofiles,verbs=get;list;watch;create;update;patch;delete
+// ClusterPromotion creates a copy of the referenced ConfigMaps/Secrets for each stage
+// +kubebuilder:rbac:groups="",resources=configmaps,verbs=create;update
+// +kubebuilder:rbac:groups="",resources=secrets,verbs=create;update;delete
 
 func (r *ClusterPromotionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) {
 	logger := ctrl.LoggerFrom(ctx)
@@ -121,8 +131,10 @@ func (r *ClusterPromotionReconciler) Reconcile(ctx context.Context, req ctrl.Req
 
 	if !clusterPromotion.DeletionTimestamp.IsZero() {
 		licenseManagerInstance.RemoveClusterPromotion(req.Namespace, req.Name)
+		r.cleanMaps(clusterPromotion)
 	} else {
 		licenseManagerInstance.AddClusterPromotion(clusterPromotion)
+		r.updateMaps(clusterPromotion)
 	}
 
 	// Always close the scope when exiting this function so we can persist any ClusterPromotion
@@ -212,6 +224,18 @@ func (r *ClusterPromotionReconciler) SetupWithManager(mgr ctrl.Manager) error {
 				DependenciesHashChangedPredicate{},
 			),
 		)).
+		Watches(&corev1.ConfigMap{},
+			handler.EnqueueRequestsFromMapFunc(r.requeueClusterPromotionForReference),
+			builder.WithPredicates(
+				ConfigMapPredicates(mgr.GetLogger().WithValues("predicate", "configmappredicate")),
+			),
+		).
+		Watches(&corev1.Secret{},
+			handler.EnqueueRequestsFromMapFunc(r.requeueClusterPromotionForReference),
+			builder.WithPredicates(
+				SecretPredicates(mgr.GetLogger().WithValues("predicate", "secretpredicate")),
+			),
+		).
 		WithOptions(controller.Options{
 			MaxConcurrentReconciles: r.ConcurrentReconciles,
 		}).

controllers/clusterpromotion_controller_test.go

@@ -26,10 +26,12 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1"
 	"github.com/projectsveltos/addon-controller/controllers"
+	libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1"
 )
 
 const (
@@ -130,4 +132,60 @@ var _ = Describe("ClusterPromotionController", func() {
 		Expect(c.List(ctx, clusterProfiles, listOptions...)).To(Succeed())
 		Expect(len(clusterProfiles.Items)).To(BeZero())
 	})
+
+	It("requeueClusterPromotionForReference enqueues the ClusterPromotions referencing the ConfigMap/Secret", func() {
+		namespace := randomString()
+		configMap := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: randomString()}}
+		secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: randomString()}}
+
+		policyRefPromotion := &configv1beta1.ClusterPromotion{
+			ObjectMeta: metav1.ObjectMeta{Name: randomString()},
+			Spec: configv1beta1.ClusterPromotionSpec{ProfileSpec: configv1beta1.ProfileSpec{
+				PolicyRefs: []configv1beta1.PolicyRef{
+					{Namespace: namespace, Name: configMap.Name, Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind)},
+				},
+			}},
+		}
+		valuesFromPromotion := &configv1beta1.ClusterPromotion{
+			ObjectMeta: metav1.ObjectMeta{Name: randomString()},
+			Spec: configv1beta1.ClusterPromotionSpec{ProfileSpec: configv1beta1.ProfileSpec{
+				HelmCharts: []configv1beta1.HelmChart{{
+					ValuesFrom: []configv1beta1.ValueFrom{
+						{Namespace: namespace, Name: secret.Name, Kind: string(libsveltosv1beta1.SecretReferencedResourceKind)},
+					},
+				}},
+			}},
+		}
+		// Same name as the ConfigMap but it is a Secret: must not match
+		otherKindPromotion := &configv1beta1.ClusterPromotion{
+			ObjectMeta: metav1.ObjectMeta{Name: randomString()},
+			Spec: configv1beta1.ClusterPromotionSpec{ProfileSpec: configv1beta1.ProfileSpec{
+				PatchesFrom: []configv1beta1.ValueFrom{
+					{Namespace: namespace, Name: configMap.Name, Kind: string(libsveltosv1beta1.SecretReferencedResourceKind)},
+				},
+			}},
+		}
+
+		reconciler := controllers.ClusterPromotionReconciler{}
+		controllers.ClusterPromotionUpdateMaps(&reconciler, policyRefPromotion)
+		controllers.ClusterPromotionUpdateMaps(&reconciler, valuesFromPromotion)
+		controllers.ClusterPromotionUpdateMaps(&reconciler, otherKindPromotion)
+
+		requests := controllers.RequeueClusterPromotionForReference(&reconciler, context.TODO(), configMap)
+		Expect(requests).To(HaveLen(1))
+		Expect(requests[0].Name).To(Equal(policyRefPromotion.Name))
+
+		requests = controllers.RequeueClusterPromotionForReference(&reconciler, context.TODO(), secret)
+		Expect(requests).To(HaveLen(1))
+		Expect(requests[0].Name).To(Equal(valuesFromPromotion.Name))
+
+		By("updating the ClusterPromotion so it does not reference the ConfigMap anymore")
+		policyRefPromotion.Spec.ProfileSpec.PolicyRefs = nil
+		controllers.ClusterPromotionUpdateMaps(&reconciler, policyRefPromotion)
+		Expect(controllers.RequeueClusterPromotionForReference(&reconciler, context.TODO(), configMap)).To(BeEmpty())
+
+		By("cleaning the maps for a deleted ClusterPromotion")
+		controllers.ClusterPromotionCleanMaps(&reconciler, valuesFromPromotion)
+		Expect(controllers.RequeueClusterPromotionForReference(&reconciler, context.TODO(), secret)).To(BeEmpty())
+	})
 })

controllers/clusterpromotion_transformations.go

@@ -0,0 +1,172 @@
+/*
+Copyright 2025. projectsveltos.io. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1"
+	libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1"
+	logs "github.com/projectsveltos/libsveltos/lib/logsettings"
+	libsveltosset "github.com/projectsveltos/libsveltos/lib/set"
+)
+
+// requeueClusterPromotionForReference is a handler.ToRequestsFunc to be used to enqueue
+// ClusterPromotion instances referencing the ConfigMap/Secret that changed. A ClusterPromotion
+// snapshots referenced resources per stage, so it needs to know when the original changes.
+func (r *ClusterPromotionReconciler) requeueClusterPromotionForReference(
+	ctx context.Context, o client.Object,
+) []reconcile.Request {
+
+	logger := ctrl.LoggerFrom(ctx).WithValues("reference", o.GetName())
+
+	var kind string
+	switch o.(type) {
+	case *corev1.ConfigMap:
+		kind = string(libsveltosv1beta1.ConfigMapReferencedResourceKind)
+	case *corev1.Secret:
+		kind = string(libsveltosv1beta1.SecretReferencedResourceKind)
+	default:
+		return nil
+	}
+
+	key := corev1.ObjectReference{
+		APIVersion: corev1.SchemeGroupVersion.String(),
+		Kind:       kind,
+		Namespace:  o.GetNamespace(),
+		Name:       o.GetName(),
+	}
+
+	r.PolicyMux.Lock()
+	defer r.PolicyMux.Unlock()
+
+	consumers := r.getReferenceMapForEntry(&key).Items()
+	requests := make([]reconcile.Request, len(consumers))
+	for i := range consumers {
+		logger.V(logs.LogDebug).Info(fmt.Sprintf("requeue consumer: %s", consumers[i]))
+		requests[i] = reconcile.Request{
+			NamespacedName: client.ObjectKey{Name: consumers[i].Name},
+		}
+	}
+
+	return requests
+}
+
+// updateMaps records the ConfigMaps/Secrets referenced by the ClusterPromotion, replacing
+// what was recorded before.
+func (r *ClusterPromotionReconciler) updateMaps(clusterPromotion *configv1beta1.ClusterPromotion) {
+	r.PolicyMux.Lock()
+	defer r.PolicyMux.Unlock()
+
+	r.eraseFromReferenceMap(clusterPromotion)
+
+	clusterPromotionInfo := getClusterPromotionReference(clusterPromotion)
+	references := getClusterPromotionReferences(&clusterPromotion.Spec.ProfileSpec)
+	for i := range references {
+		r.getReferenceMapForEntry(&references[i]).Insert(clusterPromotionInfo)
+	}
+}
+
+// cleanMaps removes the ClusterPromotion from the recorded references
+func (r *ClusterPromotionReconciler) cleanMaps(clusterPromotion *configv1beta1.ClusterPromotion) {
+	r.PolicyMux.Lock()
+	defer r.PolicyMux.Unlock()
+
+	r.eraseFromReferenceMap(clusterPromotion)
+}
+
+// eraseFromReferenceMap must be called with PolicyMux held
+func (r *ClusterPromotionReconciler) eraseFromReferenceMap(clusterPromotion *configv1beta1.ClusterPromotion) {
+	clusterPromotionInfo := getClusterPromotionReference(clusterPromotion)
+	for k, l := range r.ReferenceMap {
+		l.Erase(clusterPromotionInfo)
+		if l.Len() == 0 {
+			delete(r.ReferenceMap, k)
+		}
+	}
+}
+
+// getReferenceMapForEntry must be called with PolicyMux held
+func (r *ClusterPromotionReconciler) getReferenceMapForEntry(entry *corev1.ObjectReference) *libsveltosset.Set {
+	if r.ReferenceMap == nil {
+		r.ReferenceMap = make(map[corev1.ObjectReference]*libsveltosset.Set)
+	}
+
+	s := r.ReferenceMap[*entry]
+	if s == nil {
+		s = &libsveltosset.Set{}
+		r.ReferenceMap[*entry] = s
+	}
+	return s
+}
+
+func getClusterPromotionReference(clusterPromotion *configv1beta1.ClusterPromotion) *corev1.ObjectReference {
+	return &corev1.ObjectReference{
+		APIVersion: configv1beta1.GroupVersion.String(),
+		Kind:       configv1beta1.ClusterPromotionKind,
+		Name:       clusterPromotion.Name,
+	}
+}
+
+// getClusterPromotionReferences returns the ConfigMaps/Secrets referenced by the ProfileSpec
+// (policyRefs, kustomizationRefs, valuesFrom, patchesFrom)
+func getClusterPromotionReferences(spec *configv1beta1.ProfileSpec) []corev1.ObjectReference {
+	references := make([]corev1.ObjectReference, 0)
+
+	add := func(kind, namespace, name string) {
+		if kind != string(libsveltosv1beta1.ConfigMapReferencedResourceKind) &&
+			kind != string(libsveltosv1beta1.SecretReferencedResourceKind) {
+
+			return
+		}
+		references = append(references, corev1.ObjectReference{
+			APIVersion: corev1.SchemeGroupVersion.String(),
+			Kind:       kind,
+			Namespace:  namespace,
+			Name:       name,
+		})
+	}
+
+	addValuesFrom := func(valuesFrom []configv1beta1.ValueFrom) {
+		for i := range valuesFrom {
+			add(valuesFrom[i].Kind, valuesFrom[i].Namespace, valuesFrom[i].Name)
+		}
+	}
+
+	for i := range spec.PolicyRefs {
+		add(spec.PolicyRefs[i].Kind, spec.PolicyRefs[i].Namespace, spec.PolicyRefs[i].Name)
+	}
+
+	for i := range spec.KustomizationRefs {
+		add(spec.KustomizationRefs[i].Kind, spec.KustomizationRefs[i].Namespace, spec.KustomizationRefs[i].Name)
+		addValuesFrom(spec.KustomizationRefs[i].ValuesFrom)
+	}
+
+	for i := range spec.HelmCharts {
+		addValuesFrom(spec.HelmCharts[i].ValuesFrom)
+	}
+
+	addValuesFrom(spec.PatchesFrom)
+
+	return references
+}

controllers/export_test.go

@@ -88,11 +88,14 @@ var (
 	ResetFeatureStatus                   = (*ClusterSummaryReconciler).resetFeatureStatus
 	PrepareForDeployment                 = (*ClusterSummaryReconciler).prepareForDeployment
 
-	ConvertResultStatus               = (*ClusterSummaryReconciler).convertResultStatus
-	RequeueClusterSummaryForReference = (*ClusterSummaryReconciler).requeueClusterSummaryForReference
-	RequeueClusterSummaryForCluster   = (*ClusterSummaryReconciler).requeueClusterSummaryForCluster
-	HandleDeployerError               = (*ClusterSummaryReconciler).handleDeployerError
-	GetFeatureSummaryForFeatureID     = getFeatureSummaryForFeatureID
+	ConvertResultStatus                 = (*ClusterSummaryReconciler).convertResultStatus
+	RequeueClusterSummaryForReference   = (*ClusterSummaryReconciler).requeueClusterSummaryForReference
+	RequeueClusterPromotionForReference = (*ClusterPromotionReconciler).requeueClusterPromotionForReference
+	ClusterPromotionUpdateMaps          = (*ClusterPromotionReconciler).updateMaps
+	ClusterPromotionCleanMaps           = (*ClusterPromotionReconciler).cleanMaps
+	RequeueClusterSummaryForCluster     = (*ClusterSummaryReconciler).requeueClusterSummaryForCluster
+	HandleDeployerError                 = (*ClusterSummaryReconciler).handleDeployerError
+	GetFeatureSummaryForFeatureID       = getFeatureSummaryForFeatureID
 
 	GetPatchesFrom = getPatchesFrom
 )

manifest/deployment-agentless.yaml

@@ -44,6 +44,7 @@ spec:
             fieldRef:
               fieldPath: metadata.namespace
         image: docker.io/projectsveltos/addon-controller:main
+        imagePullPolicy: IfNotPresent
         livenessProbe:
           failureThreshold: 3
           httpGet:
@@ -102,6 +103,7 @@ spec:
         - name: IS_INITIALIZATION
           value: "true"
         image: docker.io/projectsveltos/addon-controller:main
+        imagePullPolicy: IfNotPresent
         name: initialization
         securityContext:
           allowPrivilegeEscalation: false