diff --git a/ctrltest/gc.go b/ctrltest/gc.go index c403f4d..bb6a2fc 100644 --- a/ctrltest/gc.go +++ b/ctrltest/gc.go @@ -4,6 +4,7 @@ package ctrltest import ( "context" + "reflect" "slices" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -11,6 +12,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -24,12 +26,14 @@ type ownerKey struct { } // ownerResolution is what resolveOwner found: found is false for an owner -// genuinely absent, true with uid set for one that exists. The zero value is -// never looked at directly; resolveOwner's second return says whether this is -// meaningful at all. +// genuinely absent, true with uid set for one that exists, and +// deletingDependents true for one that exists and is waiting on foreground +// deletion of its dependents. The zero value is never looked at directly; +// resolveOwner's second return says whether this is meaningful at all. type ownerResolution struct { - found bool - uid types.UID + found bool + uid types.UID + deletingDependents bool } // tickGC acts as Kubernetes' garbage collector over DataPlaneSimOptions's @@ -51,12 +55,21 @@ func (s *DataPlaneSim) tickGC(ctx context.Context) { if err != nil { return } + s.sweepGC(ctx, objs) +} - // Foreground and orphan finalizers first, and in place on objs: an - // orphan dependent has its owner reference stripped from the very - // client.Object held in objs, so the background pass below sees the - // post-strip state rather than reaching for the same owner again through - // a reference that no longer exists in the cluster. +// sweepGC is one garbage collection pass over objs, a List taken at some +// earlier point. Everything it decides from objs is a candidate only: by the +// time it acts, an owner may have been recreated under the same name, a +// dependent deleted and recreated, or a dependent adopted by a new owner, so +// every write re-reads its target first and is guarded by a precondition. +// See collect. +func (s *DataPlaneSim) sweepGC(ctx context.Context, objs []client.Object) { + // Foreground and orphan finalizers first. Neither updates objs, so the + // background pass below can still see a reference an orphan strip just + // removed, pointing at an owner whose finalizer just came off. That is + // safe only because collect judges each candidate from a live read, where + // the stripped reference is gone. for _, obj := range objs { dt := obj.GetDeletionTimestamp() if dt.IsZero() { @@ -84,7 +97,7 @@ func (s *DataPlaneSim) tickGC(ctx context.Context) { } } if dangling { - _ = s.c.Delete(ctx, obj) + s.collect(ctx, obj) } } } @@ -117,9 +130,20 @@ func (s *DataPlaneSim) listWatched(ctx context.Context) ([]client.Object, error) } // foregroundDeleteDependents is one sweep of foreground deletion for owner, -// which has a deletionTimestamp and the foregroundDeletion finalizer: delete +// which has a deletionTimestamp and the foregroundDeletion finalizer: collect // every dependent of owner found among objs, and once none remain, remove the -// finalizer so the API server can finish deleting owner itself. +// finalizer so the API server can finish deleting owner itself. collect +// deletes a dependent whose only owners are gone or deleting, and strips the +// reference to owner from one that still has a live owner, which is what +// lets the finalizer come off without deleting an object someone else still +// owns. +// +// Two upstream refinements are not modelled. Every dependent is waited on, +// whether or not its reference sets blockOwnerDeletion, so a non-blocking +// dependent held up by a finalizer of its own also holds owner. And deletes +// carry no propagation policy, where upstream deletes a dependent that has +// dependents of its own in the foreground, so owner can be released before +// its grandchildren are gone. // // Deletes issued this sweep usually have not landed by the time remaining is // counted, so the finalizer typically comes off one sweep later, once a fresh @@ -138,7 +162,7 @@ func (s *DataPlaneSim) foregroundDeleteDependents( continue } remaining++ - _ = s.c.Delete(ctx, dep) + s.collect(ctx, dep) } if remaining == 0 { s.removeFinalizer(ctx, owner, metav1.FinalizerDeleteDependents) @@ -147,51 +171,209 @@ func (s *DataPlaneSim) foregroundDeleteDependents( // orphanDependents is one sweep of orphan deletion for owner, which has a // deletionTimestamp and the orphan finalizer: strip owner's reference from -// every dependent found among objs, then remove the finalizer. Unlike -// foreground deletion this always finishes in one sweep, since stripping a -// reference is an action this sim takes directly rather than one it waits on -// another object's deletion to complete. +// every dependent found among objs, then remove the finalizer. The finalizer +// comes off only once every strip has landed or found its dependent gone; one +// that failed, a conflict included, leaves it for the next sweep, because +// removing it early lets owner go while a dependent still names it, and the +// background pass would then collect the very dependent orphaning exists to +// keep. +// +// objs only says which dependents to look at. Each strip re-reads its +// dependent and patches under that read's resourceVersion, so it never lands +// on a recreated object of the same name or overwrites an owner reference +// added since: upstream orphanDependents in +// pkg/controller/garbagecollector/garbagecollector.go does the same with a +// UID precondition, falling back to deleteOwnerRefJSONMergePatch in patch.go, +// which carries the resourceVersion of a fresh read. func (s *DataPlaneSim) orphanDependents( ctx context.Context, owner client.Object, objs []client.Object, ) { + done := true for _, dep := range objs { if !referencesUID(dep, owner.GetUID()) { continue } - base, ok := dep.DeepCopyObject().(client.Object) - if !ok { + live, err := s.getLive(ctx, dep) + switch { + case apierrors.IsNotFound(err): + continue + case err != nil: + done = false continue } - kept := make([]metav1.OwnerReference, 0, len(dep.GetOwnerReferences())) - for _, ref := range dep.GetOwnerReferences() { - if ref.UID != owner.GetUID() { - kept = append(kept, ref) - } + if !referencesUID(live, owner.GetUID()) { + continue + } + err = s.stripOwnerRefs(ctx, live, func(uid types.UID) bool { return uid == owner.GetUID() }) + if err != nil && !apierrors.IsNotFound(err) { + done = false } - dep.SetOwnerReferences(kept) - _ = s.c.Patch(ctx, dep, client.StrategicMergeFrom(base)) } - s.removeFinalizer(ctx, owner, metav1.FinalizerOrphanDependents) + if done { + s.removeFinalizer(ctx, owner, metav1.FinalizerOrphanDependents) + } } -// removeFinalizer patches finalizer off obj with a strategic merge patch, so -// a finalizer another writer added to obj in the meantime survives: the same -// reasoning tickPods already applies to pod conditions. -func (s *DataPlaneSim) removeFinalizer(ctx context.Context, obj client.Object, finalizer string) { - base, ok := obj.DeepCopyObject().(client.Object) - if !ok { +// removeFinalizer takes finalizer off owner, the object the sweep listed, as +// upstream's removeFinalizer (pkg/controller/garbagecollector/operations.go) +// does: read it fresh, and patch the finalizer list under that read's +// resourceVersion, retrying on conflict from a new read. A JSON merge patch +// replaces the whole list, so the lock is what keeps a concurrent change to +// the list; the retry is what keeps an owner under constant unrelated writes +// from holding its finalizer forever. The UID check keeps the patch off a +// recreated owner of the same name, which may itself be waiting on +// dependents this sweep's List never saw. A merge patch rather than a +// strategic one because the API server refuses strategic merge on custom +// resources. +func (s *DataPlaneSim) removeFinalizer(ctx context.Context, owner client.Object, finalizer string) { + _ = retry.RetryOnConflict(retry.DefaultRetry, func() error { + live, err := s.getLive(ctx, owner) + if err != nil { + return client.IgnoreNotFound(err) + } + if live.GetUID() != owner.GetUID() || !slices.Contains(live.GetFinalizers(), finalizer) { + return nil + } + base := live.DeepCopy() + live.SetFinalizers(slices.DeleteFunc(slices.Clone(live.GetFinalizers()), + func(f string) bool { return f == finalizer })) + return s.c.Patch( + ctx, + live, + client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{}), + ) + }) +} + +// collect is upstream's attemptToDeleteItem +// (pkg/controller/garbagecollector/garbagecollector.go) for one dependent the +// sweep's List marked as a candidate. listed is only a name: the List may be +// arbitrarily stale, and acting on it directly is how a sim deletes a +// recreated object that reuses the name, or an object that has since gained +// a live owner. So, as upstream does: +// +// - Read the dependent live, and decide from that read alone. +// - Classify each of its owners afresh, with no sweep cache: solid (exists +// with the referenced UID and is not deleting its dependents), dangling +// (absent, or the name now holds a different UID), or waiting (exists +// and is under foreground deletion). An owner this sim cannot resolve +// (an unmapped kind, a failed lookup) ends the attempt with nothing +// done, so a guess never deletes or strips anything. +// - With any solid owner, strip the references to dangling and waiting +// owners instead of deleting. +// - Otherwise delete, preconditioned on the UID and resourceVersion of the +// live read, through deleteLive. +// +// Errors are dropped rather than returned: every failure here, a conflict +// included, means "decide again next sweep", which the next List does. +func (s *DataPlaneSim) collect(ctx context.Context, listed client.Object) { + live, err := s.getLive(ctx, listed) + if err != nil { return } - kept := make([]string, 0, len(obj.GetFinalizers())) - for _, f := range obj.GetFinalizers() { - if f != finalizer { - kept = append(kept, f) + refs := live.GetOwnerReferences() + if len(refs) == 0 { + return + } + var solid bool + notSolid := map[types.UID]bool{} + for _, ref := range refs { + res, resolvable := s.resolveOwner(ctx, nil, ref, live.GetNamespace()) + switch { + case !resolvable: + // Neither solid nor dangling is known, so do nothing and let a + // later sweep decide, as upstream's classifyReferences does by + // returning the lookup error. Treating it as solid would strip a + // foreground owner's reference on the strength of an owner never + // verified to exist. + return + case !res.found || res.uid != ref.UID, res.deletingDependents: + notSolid[ref.UID] = true + default: + solid = true } } - obj.SetFinalizers(kept) - _ = s.c.Patch(ctx, obj, client.StrategicMergeFrom(base)) + if solid { + if len(notSolid) > 0 { + _ = s.stripOwnerRefs(ctx, live, func(uid types.UID) bool { return notSolid[uid] }) + } + return + } + s.deleteLive(ctx, live) +} + +// getLive reads obj's metadata straight from the API server into a fresh +// object. Metadata only, because that is all garbage collection decides on, +// and fresh rather than into a copy of obj, because decoding over a +// populated object keeps any field the response omits: an owner reference +// list emptied since obj was listed would survive the read. +func (s *DataPlaneSim) getLive( + ctx context.Context, + obj client.Object, +) (*metav1.PartialObjectMetadata, error) { + gvk, err := s.c.GroupVersionKindFor(obj) + if err != nil { + return nil, err + } + live := &metav1.PartialObjectMetadata{} + live.SetGroupVersionKind(gvk) + if err := s.c.Get(ctx, client.ObjectKeyFromObject(obj), live); err != nil { + return nil, err + } + return live, nil +} + +// deleteLive is upstream's deleteObject (operations.go): delete live under +// both its UID and its resourceVersion. The UID keeps the delete off a +// recreated object of the same name; the resourceVersion keeps it off an +// object whose owners changed after live was read. +// +// A resourceVersion conflict alone would starve collection of any object +// written to often, so, again as upstream does, a conflict is followed by one +// more read, and if the object is the same incarnation with exactly the owner +// references live was judged on, the delete is retried under the UID alone. +// Anything else (NotFound, a different UID, changed owners, any other error) +// is left to the next sweep to judge from scratch. +func (s *DataPlaneSim) deleteLive(ctx context.Context, live *metav1.PartialObjectMetadata) { + uid, rv := live.GetUID(), live.GetResourceVersion() + err := s.c.Delete(ctx, live, client.Preconditions{UID: &uid, ResourceVersion: &rv}) + if !apierrors.IsConflict(err) { + return + } + again, err := s.getLive(ctx, live) + if err != nil || again.GetUID() != uid || again.GetResourceVersion() == rv || + !reflect.DeepEqual(again.GetOwnerReferences(), live.GetOwnerReferences()) { + return + } + _ = s.c.Delete(ctx, again, client.Preconditions{UID: &uid}) +} + +// stripOwnerRefs removes every owner reference on live whose UID drop +// matches, with a JSON merge patch under live's resourceVersion: the patch +// writes the whole remaining list, so the lock is what stops it erasing a +// reference added since live was read, and what keeps it off a recreated +// object of the same name. A merge patch rather than a strategic one because +// the API server refuses strategic merge on custom resources. +func (s *DataPlaneSim) stripOwnerRefs( + ctx context.Context, + live *metav1.PartialObjectMetadata, + drop func(types.UID) bool, +) error { + base := live.DeepCopy() + kept := make([]metav1.OwnerReference, 0, len(live.GetOwnerReferences())) + for _, ref := range live.GetOwnerReferences() { + if !drop(ref.UID) { + kept = append(kept, ref) + } + } + live.SetOwnerReferences(kept) + return s.c.Patch( + ctx, + live, + client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{}), + ) } // referencesUID reports whether obj carries an owner reference to uid. @@ -227,7 +409,8 @@ func (s *DataPlaneSim) ownerDangling( } // resolveOwner looks up the object ref names, caching the result in cache for -// the rest of the sweep. The second return is false when the kind cannot be +// the rest of the sweep, or with a nil cache, reading it fresh and caching +// nothing. The second return is false when the kind cannot be // resolved at all (a malformed APIVersion, or one the RESTMapper does not // map), as distinct from a successful lookup that found nothing. func (s *DataPlaneSim) resolveOwner( @@ -255,6 +438,11 @@ func (s *DataPlaneSim) resolveOwner( if cached, ok := cache[key]; ok { return *cached, true } + remember := func(res ownerResolution) { + if cache != nil { + cache[key] = &res + } + } pom := &metav1.PartialObjectMetadata{ TypeMeta: metav1.TypeMeta{APIVersion: ref.APIVersion, Kind: ref.Kind}, @@ -263,7 +451,7 @@ func (s *DataPlaneSim) resolveOwner( switch { case apierrors.IsNotFound(err): res := ownerResolution{found: false} - cache[key] = &res + remember(res) return res, true case err != nil: // Transient or unexpected: not cached, so the next sweep tries again @@ -271,8 +459,13 @@ func (s *DataPlaneSim) resolveOwner( // one. return ownerResolution{}, false default: - res := ownerResolution{found: true, uid: pom.UID} - cache[key] = &res + res := ownerResolution{ + found: true, + uid: pom.UID, + deletingDependents: !pom.DeletionTimestamp.IsZero() && + slices.Contains(pom.Finalizers, metav1.FinalizerDeleteDependents), + } + remember(res) return res, true } } diff --git a/ctrltest/gc_stale_test.go b/ctrltest/gc_stale_test.go new file mode 100644 index 0000000..81f9b74 --- /dev/null +++ b/ctrltest/gc_stale_test.go @@ -0,0 +1,479 @@ +// SPDX-License-Identifier: Apache-2.0 + +package ctrltest + +import ( + "context" + "slices" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +// These tests drive garbage collection from a List taken before the cluster +// moved on, which is the window a sweep under load really acts in: an owner +// recreated under the same name, a dependent recreated or adopted, a write +// landing between the sim's read and its delete. They run against envtest +// rather than the fake client because the fake enforces neither a UID +// precondition on Delete nor the foreground and orphan finalizers the API +// server adds. +// +// Dependents are Secrets and owners ServiceAccounts, two kinds the suite's own +// sim does not watch, so nothing but the sim each test builds and steps by +// hand ever collects a dependent or clears an owner's finalizer here. An +// owner of a watched kind would race that sim for its finalizers. + +// staleGC is a garbage-collecting sim stepped by hand, over one test's +// namespace, whose reads of a dependent's live metadata can be intercepted. +type staleGC struct { + *C + ctx context.Context + sim *DataPlaneSim + + // afterLiveGet, keyed by "Kind/name", runs after the sim's nth live + // read of that object returns (n counting from 1), so a test can change + // the cluster in the window between that read and the write it decides. + afterLiveGet map[string]map[int]func() + liveGets map[string]int +} + +func newStaleGC(t *testing.T) *staleGC { + t.Helper() + g := &staleGC{ + C: shared.Case(t), + ctx: context.Background(), + afterLiveGet: map[string]map[int]func(){}, + liveGets: map[string]int{}, + } + cl := interceptor.NewClient(shared.Client, interceptor.Funcs{ + Get: func( + ctx context.Context, w client.WithWatch, key client.ObjectKey, + obj client.Object, opts ...client.GetOption, + ) error { + err := w.Get(ctx, key, obj, opts...) + pom, ok := obj.(*metav1.PartialObjectMetadata) + if !ok || key.Namespace != g.NS { + return err + } + id := pom.Kind + "/" + key.Name + g.liveGets[id]++ + if hook := g.afterLiveGet[id][g.liveGets[id]]; hook != nil { + hook() + } + return err + }, + }) + g.sim = NewDataPlaneSimWithOptions(cl, time.Hour, DataPlaneSimOptions{ + GarbageCollection: true, + WatchedKinds: []client.ObjectList{&corev1.SecretList{}, &corev1.ServiceAccountList{}}, + RESTMapper: shared.Client.RESTMapper(), + }) + return g +} + +// after registers fn to run after the sim's nth live read of dependent name. +func (g *staleGC) after(name string, n int, fn func()) { g.afterKind("Secret", name, n, fn) } + +// afterKind is after for an object of any kind. +func (g *staleGC) afterKind(kind, name string, n int, fn func()) { + id := kind + "/" + name + if g.afterLiveGet[id] == nil { + g.afterLiveGet[id] = map[int]func(){} + } + g.afterLiveGet[id][n] = fn +} + +// list is the sweep's List, narrowed to this test's namespace so a sweep +// never acts on another test's objects. +func (g *staleGC) list() []client.Object { + g.Helper() + objs, err := g.sim.listWatched(g.ctx) + g.NoError(err, "list watched kinds") + return slices.DeleteFunc(objs, func(o client.Object) bool { return o.GetNamespace() != g.NS }) +} + +func (g *staleGC) sweep(objs []client.Object) { g.sim.sweepGC(g.ctx, objs) } + +func (g *staleGC) owner(name string) *corev1.ServiceAccount { + g.Helper() + sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: g.NS}} + g.NoError(shared.Client.Create(g.ctx, sa), "create owner %s", name) + return sa +} + +func (g *staleGC) dependent(name string, owners ...metav1.OwnerReference) *corev1.Secret { + g.Helper() + s := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: g.NS, OwnerReferences: owners, + }} + g.NoError(shared.Client.Create(g.ctx, s), "create dependent %s", name) + return s +} + +// recreate deletes dependent name and creates a new one in its place, with a +// new UID, owned by owners. +func (g *staleGC) recreate(name string, owners ...metav1.OwnerReference) *corev1.Secret { + g.Helper() + old := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: g.NS}} + g.NoError(shared.Client.Delete(g.ctx, old), "delete dependent %s", name) + return g.dependent(name, owners...) +} + +func (g *staleGC) deleteOwner(sa *corev1.ServiceAccount, policy metav1.DeletionPropagation) { + g.Helper() + err := shared.Client.Delete(g.ctx, sa, client.PropagationPolicy(policy)) + g.NoError(err, "delete owner %s", sa.Name) +} + +// get returns dependent name, or nil if it is gone. +func (g *staleGC) get(name string) *corev1.Secret { + g.Helper() + s := &corev1.Secret{} + err := shared.Client.Get(g.ctx, client.ObjectKey{Namespace: g.NS, Name: name}, s) + if apierrors.IsNotFound(err) { + return nil + } + g.NoError(err, "get dependent %s", name) + return s +} + +func (g *staleGC) getOwner(name string) *corev1.ServiceAccount { + g.Helper() + sa := &corev1.ServiceAccount{} + err := shared.Client.Get(g.ctx, client.ObjectKey{Namespace: g.NS, Name: name}, sa) + if apierrors.IsNotFound(err) { + return nil + } + g.NoError(err, "get owner %s", name) + return sa +} + +func (g *staleGC) relabel(name string) { + g.Helper() + s := g.get(name) + g.NotNil(s, "relabel: dependent %s is gone", name) + base := s.DeepCopy() + s.Labels = map[string]string{"bump": s.ResourceVersion} + g.NoError(shared.Client.Patch(g.ctx, s, client.MergeFrom(base)), "relabel %s", name) +} + +func saRef(sa *corev1.ServiceAccount) metav1.OwnerReference { + return metav1.OwnerReference{ + APIVersion: "v1", + Kind: "ServiceAccount", + Name: sa.Name, + UID: sa.UID, + } +} + +func ownerUIDs(s *corev1.Secret) []types.UID { + var out []types.UID + for _, r := range s.OwnerReferences { + out = append(out, r.UID) + } + return out +} + +// TestGCStaleListSparesARecreatedDependent is an owner and its dependent +// both deleted and recreated under the same names after the sweep's List: the +// dependent the List saw is gone, and the one now holding its name belongs to +// a live owner. Deleting by the listed name would take the new one. +func TestGCStaleListSparesARecreatedDependent(t *testing.T) { + g := newStaleGC(t) + shard := g.owner("shard") + g.dependent("pool-0", saRef(shard)) + g.dependent("orphan", saRef(shard)) + g.deleteOwner(shard, metav1.DeletePropagationBackground) + + objs := g.list() + shard2 := g.owner("shard") + fresh := g.recreate("pool-0", saRef(shard2)) + + g.sweep(objs) + + got := g.get("pool-0") + g.NotNil(got, "the recreated pool-0, owned by the live shard, was collected") + g.Eq(fresh.UID, got.UID, "pool-0 is not the recreated incarnation") + g.Nil(g.get("orphan"), "a dependent whose only owner is gone was not collected") +} + +// TestGCStaleListSparesADependentAdoptedSinceTheList is the same dependent, +// same UID, listed while its only owner was dead and adopted by a live owner +// before the sweep acts. A UID precondition still matches it, so only judging +// its owners from a fresh read saves it. The reference to the dead owner is +// stripped, as the real collector does for a dependent with a live owner. +func TestGCStaleListSparesADependentAdoptedSinceTheList(t *testing.T) { + g := newStaleGC(t) + shard := g.owner("shard") + data := g.dependent("data", saRef(shard)) + g.dependent("orphan", saRef(shard)) + g.deleteOwner(shard, metav1.DeletePropagationBackground) + + objs := g.list() + anchor := g.owner("anchor") + base := data.DeepCopy() + data.OwnerReferences = append(data.OwnerReferences, saRef(anchor)) + g.NoError(shared.Client.Patch(g.ctx, data, client.MergeFrom(base)), "adopt data") + + g.sweep(objs) + + got := g.get("data") + g.NotNil(got, "data, adopted by a live owner after the List, was collected") + g.Eq(data.UID, got.UID, "data is not the same incarnation") + g.ElementsMatch([]types.UID{anchor.UID}, ownerUIDs(got), "owners after the sweep") + g.Nil(g.get("orphan"), "a dependent whose only owner is gone was not collected") +} + +// TestGCDeleteIsPreconditionedOnTheLiveRead covers the window after the sim +// has read a dependent and judged it collectable: the dependent is recreated +// before the delete lands. Only a precondition on the read's UID and +// resourceVersion keeps the delete off the new object. +func TestGCDeleteIsPreconditionedOnTheLiveRead(t *testing.T) { + g := newStaleGC(t) + shard := g.owner("shard") + g.dependent("pool-0", saRef(shard)) + g.deleteOwner(shard, metav1.DeletePropagationBackground) + + var fresh *corev1.Secret + g.after("pool-0", 1, func() { fresh = g.recreate("pool-0") }) + + g.sweep(g.list()) + + got := g.get("pool-0") + g.NotNil(got, "pool-0, recreated between the sim's read and its delete, was deleted") + g.Eq(fresh.UID, got.UID, "pool-0 is not the recreated incarnation") +} + +// TestGCRetriesADeleteThatConflictedOnUnrelatedWrites is the liveness side of +// the resourceVersion precondition: an orphan written to between the sim's +// read and its delete, without its owners changing, is still collected in +// the same sweep, so an object under constant unrelated writes is never +// starved. +func TestGCRetriesADeleteThatConflictedOnUnrelatedWrites(t *testing.T) { + g := newStaleGC(t) + shard := g.owner("shard") + g.dependent("busy", saRef(shard)) + g.deleteOwner(shard, metav1.DeletePropagationBackground) + + g.after("busy", 1, func() { g.relabel("busy") }) + + g.sweep(g.list()) + + g.Nil(g.get("busy"), "an orphan with an unrelated write in flight was not collected") +} + +// TestGCRetryIsStillPreconditionedOnUID is the retry after a conflict landing +// on a recreated object: the retry drops the resourceVersion, but not the UID. +func TestGCRetryIsStillPreconditionedOnUID(t *testing.T) { + g := newStaleGC(t) + shard := g.owner("shard") + g.dependent("busy", saRef(shard)) + g.deleteOwner(shard, metav1.DeletePropagationBackground) + + var fresh *corev1.Secret + g.after("busy", 1, func() { g.relabel("busy") }) + g.after("busy", 2, func() { fresh = g.recreate("busy") }) + + g.sweep(g.list()) + + got := g.get("busy") + g.NotNil(got, "busy, recreated before the retried delete, was deleted") + g.Eq(fresh.UID, got.UID, "busy is not the recreated incarnation") +} + +// TestGCForegroundStripsADependentWithAnotherLiveOwner pins foreground +// deletion against a dependent that something else still owns: it loses its +// reference to the deleting owner rather than being deleted, a dependent with +// no other owner is deleted, and the owner's finalizer comes off once neither +// references it. +func TestGCForegroundStripsADependentWithAnotherLiveOwner(t *testing.T) { + g := newStaleGC(t) + shard := g.owner("shard") + anchor := g.owner("anchor") + g.dependent("shared", saRef(shard), saRef(anchor)) + g.dependent("solo", saRef(shard)) + g.deleteOwner(shard, metav1.DeletePropagationForeground) + + g.sweep(g.list()) + + got := g.get("shared") + g.NotNil(got, "a dependent with a live second owner was deleted by foreground deletion") + g.ElementsMatch([]types.UID{anchor.UID}, ownerUIDs(got), "owners after the sweep") + g.Nil(g.get("solo"), "a dependent of a foreground-deleted owner was not deleted") + + g.sweep(g.list()) + g.Nil(g.getOwner("shard"), "foreground owner still present once no dependent references it") +} + +// TestGCOrphanLeavesTheFinalizerUntilEveryStripLands is an owner reference +// added to a dependent between the sim's read of it and its strip of the +// orphaning owner. The strip must not erase the new reference, and the +// owner's orphan finalizer must stay until a strip lands: removed early, the +// owner goes while the dependent still names it. +func TestGCOrphanLeavesTheFinalizerUntilEveryStripLands(t *testing.T) { + g := newStaleGC(t) + shard := g.owner("shard") + anchor := g.owner("anchor") + kept := g.dependent("kept", saRef(shard)) + g.deleteOwner(shard, metav1.DeletePropagationOrphan) + + g.after("kept", 1, func() { + cur := g.get("kept") + base := cur.DeepCopy() + cur.OwnerReferences = append(cur.OwnerReferences, saRef(anchor)) + g.NoError(shared.Client.Patch(g.ctx, cur, client.MergeFrom(base)), "adopt kept") + }) + + g.sweep(g.list()) + + got := g.get("kept") + g.NotNil(got, "an orphaned dependent was deleted") + g.Contains( + ownerUIDs(got), + anchor.UID, + "the strip erased an owner reference added after the read", + ) + owner := g.getOwner("shard") + g.NotNil(owner, "orphan finalizer came off although the strip did not land") + + g.sweep(g.list()) + + got = g.get("kept") + g.NotNil(got, "an orphaned dependent was deleted") + g.Eq(kept.UID, got.UID, "kept is not the same incarnation") + g.ElementsMatch([]types.UID{anchor.UID}, ownerUIDs(got), "owners after orphaning") + g.Nil(g.getOwner("shard"), "orphaning owner still present after its dependent was stripped") +} + +// TestGCFinalizerRemovalSparesARecreatedOwner is a foreground-deleting owner +// that finished deleting and was recreated, under the same name, and is +// itself being foreground-deleted with a dependent still alive, all after the +// sweep's List. The listed owner has no dependents left, so the sweep removes +// its finalizer; landing on the new owner, that would let it go while its +// dependent still exists. +func TestGCFinalizerRemovalSparesARecreatedOwner(t *testing.T) { + g := newStaleGC(t) + shard := g.owner("shard") + g.deleteOwner(shard, metav1.DeletePropagationForeground) + + objs := g.list() + cur := g.getOwner("shard") + base := cur.DeepCopy() + cur.Finalizers = nil + g.NoError(shared.Client.Patch(g.ctx, cur, client.MergeFrom(base)), "finish deleting shard") + g.Nil(g.getOwner("shard"), "shard still present after its last finalizer came off") + shard2 := g.owner("shard") + g.dependent("child", saRef(shard2)) + g.deleteOwner(shard2, metav1.DeletePropagationForeground) + + g.sweep(objs) + + got := g.getOwner("shard") + g.NotNil(got, "the recreated shard was released while its dependent still exists") + g.Eq(shard2.UID, got.UID, "shard is not the recreated incarnation") + g.NotNil(g.get("child"), "child was collected from a List that never saw it") + + g.sweep(g.list()) + g.Nil(g.get("child"), "a dependent of a foreground-deleted owner was not deleted") + g.sweep(g.list()) + g.Nil(g.getOwner("shard"), "foreground owner still present once no dependent references it") +} + +// TestGCStaleListSparesADependentReleasedSinceTheList is a dependent listed +// with only a dead owner whose owner references were all removed before the +// sweep acts. With no owners left it is nobody's dependent, and garbage +// collection has nothing to say about it. +func TestGCStaleListSparesADependentReleasedSinceTheList(t *testing.T) { + g := newStaleGC(t) + shard := g.owner("shard") + data := g.dependent("data", saRef(shard)) + g.deleteOwner(shard, metav1.DeletePropagationBackground) + + objs := g.list() + base := data.DeepCopy() + data.OwnerReferences = nil + g.NoError(shared.Client.Patch(g.ctx, data, client.MergeFrom(base)), "release data") + + g.sweep(objs) + + g.NotNil(g.get("data"), "data, released by every owner after the List, was collected") +} + +// TestGCSparesADependentAdoptedAfterTheLiveRead is adoption inside the window +// between the sim's live read of a dependent and its delete: same UID, so only +// the resourceVersion precondition catches it, and the conflict retry must +// see the changed owners and stand down rather than retry under the UID alone. +func TestGCSparesADependentAdoptedAfterTheLiveRead(t *testing.T) { + g := newStaleGC(t) + shard := g.owner("shard") + anchor := g.owner("anchor") + data := g.dependent("data", saRef(shard)) + g.deleteOwner(shard, metav1.DeletePropagationBackground) + + g.after("data", 1, func() { + cur := g.get("data") + base := cur.DeepCopy() + cur.OwnerReferences = append(cur.OwnerReferences, saRef(anchor)) + g.NoError(shared.Client.Patch(g.ctx, cur, client.MergeFrom(base)), "adopt data") + }) + + g.sweep(g.list()) + + got := g.get("data") + g.NotNil(got, "data, adopted between the sim's read and its delete, was deleted") + g.Eq(data.UID, got.UID, "data is not the same incarnation") +} + +// TestGCFinalizerRemovalRetriesOnUnrelatedWrites is a foreground owner with +// no dependents left, written to between the sim's read of it and its +// finalizer patch. The patch conflicts, and the retry from a fresh read must +// still release the owner in the same sweep, so an owner under constant +// writes is never held. +func TestGCFinalizerRemovalRetriesOnUnrelatedWrites(t *testing.T) { + g := newStaleGC(t) + shard := g.owner("shard") + g.deleteOwner(shard, metav1.DeletePropagationForeground) + + g.afterKind("ServiceAccount", "shard", 1, func() { + cur := g.getOwner("shard") + base := cur.DeepCopy() + cur.Labels = map[string]string{"bump": cur.ResourceVersion} + g.NoError(shared.Client.Patch(g.ctx, cur, client.MergeFrom(base)), "relabel shard") + }) + + g.sweep(g.list()) + + g.Nil(g.getOwner("shard"), "foreground owner held by an unrelated write in flight") +} + +// TestGCForegroundLeavesADependentWithAnUnresolvableOwner is a dependent of a +// foreground-deleting owner whose other owner is of a kind the RESTMapper +// does not know. Whether that owner is live is unknown, so the dependent is +// neither deleted nor stripped of its reference to the deleting owner: the +// real collector retries the lookup rather than guess, and the owner waits. +func TestGCForegroundLeavesADependentWithAnUnresolvableOwner(t *testing.T) { + g := newStaleGC(t) + shard := g.owner("shard") + unknown := metav1.OwnerReference{ + APIVersion: "example.com/v1", Kind: "Widget", Name: "w", UID: types.UID("widget-uid"), + } + g.dependent("data", saRef(shard), unknown) + g.deleteOwner(shard, metav1.DeletePropagationForeground) + + g.sweep(g.list()) + + got := g.get("data") + g.NotNil(got, "a dependent with an unresolvable owner was deleted") + g.Contains( + ownerUIDs(got), + shard.UID, + "the reference to the deleting owner was stripped on a guess", + ) + g.NotNil(g.getOwner("shard"), "foreground owner released while a dependent still references it") +} diff --git a/go.sum b/go.sum index a4ba920..a9d85a3 100644 --- a/go.sum +++ b/go.sum @@ -24,6 +24,8 @@ github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+r github.com/fxamacker/cbor/v2 v2.9.1/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v1.0.0 h1:kR9tHqY0CtZaOPVFm622dPVNhrvYpwr4uCxgL3h1H8s= @@ -130,6 +132,18 @@ go.etcd.io/etcd/client/pkg/v3 v3.7.2 h1:SVtlR7tiSVAYOQ4nWPIyFXb4RMgEcnzeAG9RQ8Mo go.etcd.io/etcd/client/pkg/v3 v3.7.2/go.mod h1:HsSux/B3ahgyw/D5+d4YbZqicOi0mEbuxm6lIUdjAoI= go.etcd.io/etcd/client/v3 v3.7.2 h1:Z66GqDQDI7zPDfVSsIBqGSK4mJYLtv8ESwXa4mPf+wY= go.etcd.io/etcd/client/v3 v3.7.2/go.mod h1:x03t1qMs4tGZirCDJlMuzPBJdQffXJImIyEjLhNBCsY= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -160,6 +174,8 @@ golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk=