diff --git a/config/default/manager_pull_policy.yaml b/config/default/manager_pull_policy.yaml index b59dfcb3..0d1a618e 100644 --- a/config/default/manager_pull_policy.yaml +++ b/config/default/manager_pull_policy.yaml @@ -8,7 +8,7 @@ spec: spec: initContainers: - name: initialization - imagePullPolicy: + imagePullPolicy: IfNotPresent containers: - name: controller - imagePullPolicy: + imagePullPolicy: IfNotPresent diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 3be04947..7dd01cb0 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -8,10 +8,13 @@ rules: - "" resources: - configmaps + - secrets verbs: + - create - delete - get - list + - update - watch - apiGroups: - "" @@ -20,14 +23,6 @@ rules: verbs: - get - list -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch - apiGroups: - '*' resources: diff --git a/controllers/clusterpromotion_controller.go b/controllers/clusterpromotion_controller.go index 17138a15..86ad2752 100644 --- a/controllers/clusterpromotion_controller.go +++ b/controllers/clusterpromotion_controller.go @@ -19,7 +19,9 @@ package controllers import ( "context" "fmt" + "sync" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" @@ -29,6 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -37,6 +40,7 @@ import ( configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/pkg/scope" logs "github.com/projectsveltos/libsveltos/lib/logsettings" + libsveltosset "github.com/projectsveltos/libsveltos/lib/set" ) const ( @@ -84,12 +88,18 @@ type ClusterPromotionReconciler struct { Scheme *runtime.Scheme eventRecorder events.EventRecorder ConcurrentReconciles int + + PolicyMux sync.Mutex // use a Mutex to update ReferenceMap as MaxConcurrentReconciles is higher than one + ReferenceMap map[corev1.ObjectReference]*libsveltosset.Set // key: Referenced object; value: set of all ClusterPromotions referencing the resource } // +kubebuilder:rbac:groups=config.projectsveltos.io,resources=clusterpromotions,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=config.projectsveltos.io,resources=clusterpromotions/status,verbs=get;update;patch // +kubebuilder:rbac:groups=config.projectsveltos.io,resources=clusterpromotions/finalizers,verbs=update // +kubebuilder:rbac:groups=config.projectsveltos.io,resources=clusterprofiles,verbs=get;list;watch;create;update;patch;delete +// ClusterPromotion creates a copy of the referenced ConfigMaps/Secrets for each stage +// +kubebuilder:rbac:groups="",resources=configmaps,verbs=create;update +// +kubebuilder:rbac:groups="",resources=secrets,verbs=create;update;delete func (r *ClusterPromotionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) { logger := ctrl.LoggerFrom(ctx) @@ -121,8 +131,10 @@ func (r *ClusterPromotionReconciler) Reconcile(ctx context.Context, req ctrl.Req if !clusterPromotion.DeletionTimestamp.IsZero() { licenseManagerInstance.RemoveClusterPromotion(req.Namespace, req.Name) + r.cleanMaps(clusterPromotion) } else { licenseManagerInstance.AddClusterPromotion(clusterPromotion) + r.updateMaps(clusterPromotion) } // Always close the scope when exiting this function so we can persist any ClusterPromotion @@ -212,6 +224,18 @@ func (r *ClusterPromotionReconciler) SetupWithManager(mgr ctrl.Manager) error { DependenciesHashChangedPredicate{}, ), )). + Watches(&corev1.ConfigMap{}, + handler.EnqueueRequestsFromMapFunc(r.requeueClusterPromotionForReference), + builder.WithPredicates( + ConfigMapPredicates(mgr.GetLogger().WithValues("predicate", "configmappredicate")), + ), + ). + Watches(&corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(r.requeueClusterPromotionForReference), + builder.WithPredicates( + SecretPredicates(mgr.GetLogger().WithValues("predicate", "secretpredicate")), + ), + ). WithOptions(controller.Options{ MaxConcurrentReconciles: r.ConcurrentReconciles, }). diff --git a/controllers/clusterpromotion_controller_test.go b/controllers/clusterpromotion_controller_test.go index b9c16819..5d6b22fc 100644 --- a/controllers/clusterpromotion_controller_test.go +++ b/controllers/clusterpromotion_controller_test.go @@ -26,10 +26,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" + libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" ) const ( @@ -130,4 +132,60 @@ var _ = Describe("ClusterPromotionController", func() { Expect(c.List(ctx, clusterProfiles, listOptions...)).To(Succeed()) Expect(len(clusterProfiles.Items)).To(BeZero()) }) + + It("requeueClusterPromotionForReference enqueues the ClusterPromotions referencing the ConfigMap/Secret", func() { + namespace := randomString() + configMap := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: randomString()}} + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: randomString()}} + + policyRefPromotion := &configv1beta1.ClusterPromotion{ + ObjectMeta: metav1.ObjectMeta{Name: randomString()}, + Spec: configv1beta1.ClusterPromotionSpec{ProfileSpec: configv1beta1.ProfileSpec{ + PolicyRefs: []configv1beta1.PolicyRef{ + {Namespace: namespace, Name: configMap.Name, Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind)}, + }, + }}, + } + valuesFromPromotion := &configv1beta1.ClusterPromotion{ + ObjectMeta: metav1.ObjectMeta{Name: randomString()}, + Spec: configv1beta1.ClusterPromotionSpec{ProfileSpec: configv1beta1.ProfileSpec{ + HelmCharts: []configv1beta1.HelmChart{{ + ValuesFrom: []configv1beta1.ValueFrom{ + {Namespace: namespace, Name: secret.Name, Kind: string(libsveltosv1beta1.SecretReferencedResourceKind)}, + }, + }}, + }}, + } + // Same name as the ConfigMap but it is a Secret: must not match + otherKindPromotion := &configv1beta1.ClusterPromotion{ + ObjectMeta: metav1.ObjectMeta{Name: randomString()}, + Spec: configv1beta1.ClusterPromotionSpec{ProfileSpec: configv1beta1.ProfileSpec{ + PatchesFrom: []configv1beta1.ValueFrom{ + {Namespace: namespace, Name: configMap.Name, Kind: string(libsveltosv1beta1.SecretReferencedResourceKind)}, + }, + }}, + } + + reconciler := controllers.ClusterPromotionReconciler{} + controllers.ClusterPromotionUpdateMaps(&reconciler, policyRefPromotion) + controllers.ClusterPromotionUpdateMaps(&reconciler, valuesFromPromotion) + controllers.ClusterPromotionUpdateMaps(&reconciler, otherKindPromotion) + + requests := controllers.RequeueClusterPromotionForReference(&reconciler, context.TODO(), configMap) + Expect(requests).To(HaveLen(1)) + Expect(requests[0].Name).To(Equal(policyRefPromotion.Name)) + + requests = controllers.RequeueClusterPromotionForReference(&reconciler, context.TODO(), secret) + Expect(requests).To(HaveLen(1)) + Expect(requests[0].Name).To(Equal(valuesFromPromotion.Name)) + + By("updating the ClusterPromotion so it does not reference the ConfigMap anymore") + policyRefPromotion.Spec.ProfileSpec.PolicyRefs = nil + controllers.ClusterPromotionUpdateMaps(&reconciler, policyRefPromotion) + Expect(controllers.RequeueClusterPromotionForReference(&reconciler, context.TODO(), configMap)).To(BeEmpty()) + + By("cleaning the maps for a deleted ClusterPromotion") + controllers.ClusterPromotionCleanMaps(&reconciler, valuesFromPromotion) + Expect(controllers.RequeueClusterPromotionForReference(&reconciler, context.TODO(), secret)).To(BeEmpty()) + }) }) diff --git a/controllers/clusterpromotion_transformations.go b/controllers/clusterpromotion_transformations.go new file mode 100644 index 00000000..d2088588 --- /dev/null +++ b/controllers/clusterpromotion_transformations.go @@ -0,0 +1,172 @@ +/* +Copyright 2025. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" + libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + logs "github.com/projectsveltos/libsveltos/lib/logsettings" + libsveltosset "github.com/projectsveltos/libsveltos/lib/set" +) + +// requeueClusterPromotionForReference is a handler.ToRequestsFunc to be used to enqueue +// ClusterPromotion instances referencing the ConfigMap/Secret that changed. A ClusterPromotion +// snapshots referenced resources per stage, so it needs to know when the original changes. +func (r *ClusterPromotionReconciler) requeueClusterPromotionForReference( + ctx context.Context, o client.Object, +) []reconcile.Request { + + logger := ctrl.LoggerFrom(ctx).WithValues("reference", o.GetName()) + + var kind string + switch o.(type) { + case *corev1.ConfigMap: + kind = string(libsveltosv1beta1.ConfigMapReferencedResourceKind) + case *corev1.Secret: + kind = string(libsveltosv1beta1.SecretReferencedResourceKind) + default: + return nil + } + + key := corev1.ObjectReference{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: kind, + Namespace: o.GetNamespace(), + Name: o.GetName(), + } + + r.PolicyMux.Lock() + defer r.PolicyMux.Unlock() + + consumers := r.getReferenceMapForEntry(&key).Items() + requests := make([]reconcile.Request, len(consumers)) + for i := range consumers { + logger.V(logs.LogDebug).Info(fmt.Sprintf("requeue consumer: %s", consumers[i])) + requests[i] = reconcile.Request{ + NamespacedName: client.ObjectKey{Name: consumers[i].Name}, + } + } + + return requests +} + +// updateMaps records the ConfigMaps/Secrets referenced by the ClusterPromotion, replacing +// what was recorded before. +func (r *ClusterPromotionReconciler) updateMaps(clusterPromotion *configv1beta1.ClusterPromotion) { + r.PolicyMux.Lock() + defer r.PolicyMux.Unlock() + + r.eraseFromReferenceMap(clusterPromotion) + + clusterPromotionInfo := getClusterPromotionReference(clusterPromotion) + references := getClusterPromotionReferences(&clusterPromotion.Spec.ProfileSpec) + for i := range references { + r.getReferenceMapForEntry(&references[i]).Insert(clusterPromotionInfo) + } +} + +// cleanMaps removes the ClusterPromotion from the recorded references +func (r *ClusterPromotionReconciler) cleanMaps(clusterPromotion *configv1beta1.ClusterPromotion) { + r.PolicyMux.Lock() + defer r.PolicyMux.Unlock() + + r.eraseFromReferenceMap(clusterPromotion) +} + +// eraseFromReferenceMap must be called with PolicyMux held +func (r *ClusterPromotionReconciler) eraseFromReferenceMap(clusterPromotion *configv1beta1.ClusterPromotion) { + clusterPromotionInfo := getClusterPromotionReference(clusterPromotion) + for k, l := range r.ReferenceMap { + l.Erase(clusterPromotionInfo) + if l.Len() == 0 { + delete(r.ReferenceMap, k) + } + } +} + +// getReferenceMapForEntry must be called with PolicyMux held +func (r *ClusterPromotionReconciler) getReferenceMapForEntry(entry *corev1.ObjectReference) *libsveltosset.Set { + if r.ReferenceMap == nil { + r.ReferenceMap = make(map[corev1.ObjectReference]*libsveltosset.Set) + } + + s := r.ReferenceMap[*entry] + if s == nil { + s = &libsveltosset.Set{} + r.ReferenceMap[*entry] = s + } + return s +} + +func getClusterPromotionReference(clusterPromotion *configv1beta1.ClusterPromotion) *corev1.ObjectReference { + return &corev1.ObjectReference{ + APIVersion: configv1beta1.GroupVersion.String(), + Kind: configv1beta1.ClusterPromotionKind, + Name: clusterPromotion.Name, + } +} + +// getClusterPromotionReferences returns the ConfigMaps/Secrets referenced by the ProfileSpec +// (policyRefs, kustomizationRefs, valuesFrom, patchesFrom) +func getClusterPromotionReferences(spec *configv1beta1.ProfileSpec) []corev1.ObjectReference { + references := make([]corev1.ObjectReference, 0) + + add := func(kind, namespace, name string) { + if kind != string(libsveltosv1beta1.ConfigMapReferencedResourceKind) && + kind != string(libsveltosv1beta1.SecretReferencedResourceKind) { + + return + } + references = append(references, corev1.ObjectReference{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: kind, + Namespace: namespace, + Name: name, + }) + } + + addValuesFrom := func(valuesFrom []configv1beta1.ValueFrom) { + for i := range valuesFrom { + add(valuesFrom[i].Kind, valuesFrom[i].Namespace, valuesFrom[i].Name) + } + } + + for i := range spec.PolicyRefs { + add(spec.PolicyRefs[i].Kind, spec.PolicyRefs[i].Namespace, spec.PolicyRefs[i].Name) + } + + for i := range spec.KustomizationRefs { + add(spec.KustomizationRefs[i].Kind, spec.KustomizationRefs[i].Namespace, spec.KustomizationRefs[i].Name) + addValuesFrom(spec.KustomizationRefs[i].ValuesFrom) + } + + for i := range spec.HelmCharts { + addValuesFrom(spec.HelmCharts[i].ValuesFrom) + } + + addValuesFrom(spec.PatchesFrom) + + return references +} diff --git a/controllers/export_test.go b/controllers/export_test.go index 4d1c64d5..81ad433a 100644 --- a/controllers/export_test.go +++ b/controllers/export_test.go @@ -88,11 +88,14 @@ var ( ResetFeatureStatus = (*ClusterSummaryReconciler).resetFeatureStatus PrepareForDeployment = (*ClusterSummaryReconciler).prepareForDeployment - ConvertResultStatus = (*ClusterSummaryReconciler).convertResultStatus - RequeueClusterSummaryForReference = (*ClusterSummaryReconciler).requeueClusterSummaryForReference - RequeueClusterSummaryForCluster = (*ClusterSummaryReconciler).requeueClusterSummaryForCluster - HandleDeployerError = (*ClusterSummaryReconciler).handleDeployerError - GetFeatureSummaryForFeatureID = getFeatureSummaryForFeatureID + ConvertResultStatus = (*ClusterSummaryReconciler).convertResultStatus + RequeueClusterSummaryForReference = (*ClusterSummaryReconciler).requeueClusterSummaryForReference + RequeueClusterPromotionForReference = (*ClusterPromotionReconciler).requeueClusterPromotionForReference + ClusterPromotionUpdateMaps = (*ClusterPromotionReconciler).updateMaps + ClusterPromotionCleanMaps = (*ClusterPromotionReconciler).cleanMaps + RequeueClusterSummaryForCluster = (*ClusterSummaryReconciler).requeueClusterSummaryForCluster + HandleDeployerError = (*ClusterSummaryReconciler).handleDeployerError + GetFeatureSummaryForFeatureID = getFeatureSummaryForFeatureID GetPatchesFrom = getPatchesFrom ) diff --git a/manifest/deployment-agentless.yaml b/manifest/deployment-agentless.yaml index 5504187b..7df7f11d 100644 --- a/manifest/deployment-agentless.yaml +++ b/manifest/deployment-agentless.yaml @@ -44,6 +44,7 @@ spec: fieldRef: fieldPath: metadata.namespace image: docker.io/projectsveltos/addon-controller:main + imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 httpGet: @@ -102,6 +103,7 @@ spec: - name: IS_INITIALIZATION value: "true" image: docker.io/projectsveltos/addon-controller:main + imagePullPolicy: IfNotPresent name: initialization securityContext: allowPrivilegeEscalation: false diff --git a/manifest/deployment-shard.yaml b/manifest/deployment-shard.yaml index c17ef18d..9d263a79 100644 --- a/manifest/deployment-shard.yaml +++ b/manifest/deployment-shard.yaml @@ -44,6 +44,7 @@ spec: fieldRef: fieldPath: metadata.namespace image: docker.io/projectsveltos/addon-controller:main + imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 httpGet: @@ -102,6 +103,7 @@ spec: - name: IS_INITIALIZATION value: "true" image: docker.io/projectsveltos/addon-controller:main + imagePullPolicy: IfNotPresent name: initialization securityContext: allowPrivilegeEscalation: false diff --git a/manifest/manifest.yaml b/manifest/manifest.yaml index 48204394..444078e5 100644 --- a/manifest/manifest.yaml +++ b/manifest/manifest.yaml @@ -11686,10 +11686,13 @@ rules: - "" resources: - configmaps + - secrets verbs: + - create - delete - get - list + - update - watch - apiGroups: - "" @@ -11698,14 +11701,6 @@ rules: verbs: - get - list -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch - apiGroups: - '*' resources: @@ -12092,6 +12087,7 @@ spec: fieldRef: fieldPath: metadata.namespace image: docker.io/projectsveltos/addon-controller:main + imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 httpGet: @@ -12150,6 +12146,7 @@ spec: - name: IS_INITIALIZATION value: "true" image: docker.io/projectsveltos/addon-controller:main + imagePullPolicy: IfNotPresent name: initialization securityContext: allowPrivilegeEscalation: false diff --git a/pkg/app/app.go b/pkg/app/app.go index b2f55b24..5b9e518a 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -674,6 +674,8 @@ func startControllersAndWatchers(ctx context.Context, mgr manager.Manager) { Scheme: mgr.GetScheme(), Config: mgr.GetConfig(), ConcurrentReconciles: concurrentReconciles, + ReferenceMap: make(map[corev1.ObjectReference]*libsveltosset.Set), + PolicyMux: sync.Mutex{}, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "ClusterPromotion") os.Exit(1) diff --git a/test/fv/promotion_configmap_test.go b/test/fv/promotion_configmap_test.go new file mode 100644 index 00000000..17cdacb3 --- /dev/null +++ b/test/fv/promotion_configmap_test.go @@ -0,0 +1,268 @@ +/* +Copyright 2026. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fv_test + +import ( + "bytes" + "context" + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" + "github.com/projectsveltos/addon-controller/lib/clusterops" + libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" +) + +var _ = Describe("Stage Promotions with referenced ConfigMap", func() { + const ( + namePrefix = "promotion-cm-" + + // A change to the referenced ConfigMap restarts the promotion. Each stage is checked + // every couple of minutes so allow for a whole promotion. + restartTimeout = 10 * time.Minute + + payloadPolicy = `apiVersion: v1 +kind: ConfigMap +metadata: + name: %s + namespace: %s +data: + version: %s` + ) + + // getStageCopyName returns the name of the ConfigMap ClusterPromotion creates for a stage + getStageCopyName := func(clusterPromotion *configv1beta1.ClusterPromotion, stage *configv1beta1.Stage, + configMap *corev1.ConfigMap) string { + + return fmt.Sprintf("%s-%s-%s", clusterPromotion.Name, stage.Name, configMap.Name) + } + + getClusterPromotion := func(name string) *configv1beta1.ClusterPromotion { + clusterPromotion := &configv1beta1.ClusterPromotion{} + ExpectWithOffset(1, k8sClient.Get(context.TODO(), types.NamespacedName{Name: name}, + clusterPromotion)).To(Succeed()) + return clusterPromotion + } + + // getStageProfile returns the ClusterProfile ClusterPromotion created for a stage + getStageProfile := func(clusterPromotion *configv1beta1.ClusterPromotion, stage *configv1beta1.Stage, + ) (*configv1beta1.ClusterProfile, error) { + + clusterProfile := &configv1beta1.ClusterProfile{} + err := k8sClient.Get(context.TODO(), + types.NamespacedName{Name: fmt.Sprintf("%s-%s", clusterPromotion.Name, stage.Name)}, clusterProfile) + return clusterProfile, err + } + + // getCopyContent returns the policy stored in the ConfigMap copy + getCopyContent := func(namespace, name string) (string, error) { + copyConfigMap := &corev1.ConfigMap{} + err := k8sClient.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: name}, copyConfigMap) + if err != nil { + return "", err + } + return copyConfigMap.Data["policy0.yaml"], nil + } + + // getStageAppliedTime returns the time the stage was successfully applied, nil if it was not + getStageAppliedTime := func(clusterPromotion *configv1beta1.ClusterPromotion, stageName string) *metav1.Time { + for i := range clusterPromotion.Status.Stages { + if clusterPromotion.Status.Stages[i].Name == stageName { + return clusterPromotion.Status.Stages[i].LastSuccessfulAppliedTime + } + } + return nil + } + + It("Restarts the promotion when the referenced ConfigMap changes", Label("Enterprise"), func() { + payloadName := namePrefix + randomString() + payloadNamespace := defaultNamespace + + Byf("Create the ConfigMap ClusterPromotion references") + configMap := createConfigMapWithPolicy(defaultNamespace, namePrefix+randomString(), + fmt.Sprintf(payloadPolicy, payloadName, payloadNamespace, "v1")) + Expect(k8sClient.Create(context.TODO(), configMap)).To(Succeed()) + + // No trigger: the promotion moves to the next stage as soon as a stage is provisioned + staging := configv1beta1.Stage{ + Name: stagingValue, + ClusterSelector: libsveltosv1beta1.Selector{ + LabelSelector: metav1.LabelSelector{MatchLabels: map[string]string{key: value}}, + }, + } + production := configv1beta1.Stage{ + Name: productionValue, + ClusterSelector: libsveltosv1beta1.Selector{ + LabelSelector: metav1.LabelSelector{MatchLabels: map[string]string{key: productionValue}}, + }, + } + + Byf("Create a ClusterPromotion with two stages") + clusterPromotion := &configv1beta1.ClusterPromotion{ + ObjectMeta: metav1.ObjectMeta{Name: namePrefix + randomString()}, + Spec: configv1beta1.ClusterPromotionSpec{ + ProfileSpec: configv1beta1.ProfileSpec{ + PolicyRefs: []configv1beta1.PolicyRef{ + { + Namespace: configMap.Namespace, + Name: configMap.Name, + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + }, + }, + }, + Stages: []configv1beta1.Stage{staging, production}, + }, + } + Expect(k8sClient.Create(context.TODO(), clusterPromotion)).To(Succeed()) + + stages := []*configv1beta1.Stage{&staging, &production} + + Byf("Verify all stages are provisioned") + Eventually(func() bool { + current := getClusterPromotion(clusterPromotion.Name) + for _, stage := range stages { + if getStageAppliedTime(current, stage.Name) == nil { + return false + } + } + return true + }, restartTimeout, pollingInterval).Should(BeTrue()) + + Byf("Verify each stage has its own ClusterProfile referencing its own copy of the ConfigMap") + for _, stage := range stages { + copyName := getStageCopyName(clusterPromotion, stage, configMap) + + clusterProfile, err := getStageProfile(clusterPromotion, stage) + Expect(err).To(BeNil()) + Expect(clusterProfile.Spec.PolicyRefs).To(HaveLen(1)) + Expect(clusterProfile.Spec.PolicyRefs[0].Name).To(Equal(copyName)) + Expect(clusterProfile.Spec.PolicyRefs[0].Namespace).To(Equal(configMap.Namespace)) + + content, err := getCopyContent(configMap.Namespace, copyName) + Expect(err).To(BeNil()) + Expect(content).To(Equal(configMap.Data["policy0.yaml"])) + } + + Byf("Verify the resource is deployed in the cluster matching stage %s", staging.Name) + stagingProfile, err := getStageProfile(clusterPromotion, &staging) + Expect(err).To(BeNil()) + clusterSummary := verifyClusterSummary(clusterops.ClusterProfileLabelName, stagingProfile.Name, + &stagingProfile.Spec, kindWorkloadCluster.GetNamespace(), kindWorkloadCluster.GetName(), getClusterType()) + verifyFeatureStatusIsProvisioned(kindWorkloadCluster.GetNamespace(), clusterSummary.Name, + libsveltosv1beta1.FeatureResources) + + workloadClient, err := getKindWorkloadClusterKubeconfig() + Expect(err).To(BeNil()) + Eventually(func() string { + deployed := &corev1.ConfigMap{} + err := workloadClient.Get(context.TODO(), + types.NamespacedName{Namespace: payloadNamespace, Name: payloadName}, deployed) + if err != nil { + return "" + } + return deployed.Data["version"] + }, timeout, pollingInterval).Should(Equal("v1")) + + appliedTimes := map[string]metav1.Time{} + current := getClusterPromotion(clusterPromotion.Name) + for _, stage := range stages { + appliedTimes[stage.Name] = *getStageAppliedTime(current, stage.Name) + } + profileSpecHash := current.Status.ProfileSpecHash + + Byf("Change the referenced ConfigMap") + Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Namespace: configMap.Namespace, + Name: configMap.Name}, configMap)).To(Succeed()) + configMap = updateConfigMapWithPolicy(configMap, fmt.Sprintf(payloadPolicy, payloadName, payloadNamespace, "v2")) + Expect(k8sClient.Update(context.TODO(), configMap)).To(Succeed()) + + Byf("Verify the promotion starts over: ClusterPromotion detects the change") + Eventually(func() bool { + current := getClusterPromotion(clusterPromotion.Name) + return !bytes.Equal(current.Status.ProfileSpecHash, profileSpecHash) + }, timeout, pollingInterval).Should(BeTrue()) + + Byf("Verify every stage is provisioned again, in a new run") + Eventually(func() bool { + current := getClusterPromotion(clusterPromotion.Name) + for _, stage := range stages { + appliedTime := getStageAppliedTime(current, stage.Name) + if appliedTime == nil || !appliedTime.After(appliedTimes[stage.Name].Time) { + return false + } + } + return true + }, restartTimeout, pollingInterval).Should(BeTrue()) + + Byf("Verify each stage copy has the new content") + for _, stage := range stages { + Eventually(func() string { + content, err := getCopyContent(configMap.Namespace, getStageCopyName(clusterPromotion, stage, configMap)) + if err != nil { + return "" + } + return content + }, timeout, pollingInterval).Should(Equal(configMap.Data["policy0.yaml"])) + } + + Byf("Verify the new content is deployed in the cluster matching stage %s", staging.Name) + Eventually(func() string { + deployed := &corev1.ConfigMap{} + err := workloadClient.Get(context.TODO(), + types.NamespacedName{Namespace: payloadNamespace, Name: payloadName}, deployed) + if err != nil { + return "" + } + return deployed.Data["version"] + }, timeout, pollingInterval).Should(Equal("v2")) + + Byf("Deleting ClusterPromotion %s", clusterPromotion.Name) + Expect(k8sClient.Delete(context.TODO(), getClusterPromotion(clusterPromotion.Name))).To(Succeed()) + + Byf("Verify ClusterProfiles and the ConfigMap copies are deleted") + Eventually(func() bool { + for _, stage := range stages { + profile, err := getStageProfile(clusterPromotion, stage) + if err == nil && profile.DeletionTimestamp.IsZero() { + return false + } + if err != nil && !apierrors.IsNotFound(err) { + return false + } + + _, err = getCopyContent(configMap.Namespace, getStageCopyName(clusterPromotion, stage, configMap)) + if err == nil || !apierrors.IsNotFound(err) { + return false + } + } + return true + }, timeout, pollingInterval).Should(BeTrue()) + + Byf("Verify the original ConfigMap is not deleted") + Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Namespace: configMap.Namespace, + Name: configMap.Name}, &corev1.ConfigMap{})).To(Succeed()) + Expect(k8sClient.Delete(context.TODO(), configMap)).To(Succeed()) + }) +}) diff --git a/test/fv/promotion_test.go b/test/fv/promotion_test.go index 74559fe7..74cb0308 100644 --- a/test/fv/promotion_test.go +++ b/test/fv/promotion_test.go @@ -109,7 +109,7 @@ end` clusterLabels := map[string]string{key: value} const two = 2 stage1 := configv1beta1.Stage{ - Name: "staging", + Name: stagingValue, ClusterSelector: libsveltosv1beta1.Selector{ LabelSelector: metav1.LabelSelector{ MatchLabels: clusterLabels, diff --git a/test/fv/test_constants_test.go b/test/fv/test_constants_test.go index e9726631..3ab9792b 100644 --- a/test/fv/test_constants_test.go +++ b/test/fv/test_constants_test.go @@ -151,6 +151,7 @@ const ( clusterNameKey = "cluster-name" // Annotation values for profile + stagingValue = "staging" productionValue = "production" promotionNameAnnotation = "config.projectsveltos.io/promotionname" promotionVerifAnnotation = "config.projectsveltos.io/promotion-verification"