From 907c2d60d9a107eac1de647e14427443269f5e9a Mon Sep 17 00:00:00 2001 From: Brent Graveland Date: Thu, 5 Nov 2026 11:30:12 -0600 Subject: [PATCH 1/2] fix(ctrltest): judge garbage collection from a live read and precondition every write The simulated garbage collector acted on a List taken at the start of the sweep: it deleted dependents by name with no preconditions, so a dependent recreated under the same name after the List (new UID), or the same dependent adopted by a live owner since (same UID), was deleted anyway. Orphan strips and finalizer removal were strategic merge patches of the listed copy with no resourceVersion. Each candidate now goes through collect, which follows upstream's attemptToDeleteItem: read the dependent's metadata live, classify each owner from a fresh lookup (solid, dangling, or waiting on foreground deletion), strip the dangling and waiting references if a solid owner remains, and otherwise delete under the live read's UID and resourceVersion. A conflict is followed by one re-read, and the delete is retried under the UID alone only if the object is the same incarnation with unchanged owner references, as upstream's deleteObject does, so an object under constant unrelated writes is still collected. An owner the sim cannot resolve ends the attempt with nothing done rather than counting as solid. Orphan strips and finalizer removal are JSON merge patches under a resourceVersion lock from a fresh read (strategic merge is refused for custom resources); finalizer removal checks the owner's UID and retries on conflict, like upstream's removeFinalizer. The orphan finalizer now comes off only after every strip has landed, so the comment saying orphaning always finishes in one sweep is removed, as is the one claiming strips update the listed objects in place, which they no longer do. The new tests drive a hand-stepped sim against envtest from a stale List, with hooks between the sim's read and its write. Signed-off-by: Brent Graveland --- ctrltest/gc.go | 281 ++++++++++++++++++---- ctrltest/gc_stale_test.go | 479 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 716 insertions(+), 44 deletions(-) create mode 100644 ctrltest/gc_stale_test.go diff --git a/ctrltest/gc.go b/ctrltest/gc.go index c403f4d..bb6a2fc 100644 --- a/ctrltest/gc.go +++ b/ctrltest/gc.go @@ -4,6 +4,7 @@ package ctrltest import ( "context" + "reflect" "slices" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -11,6 +12,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -24,12 +26,14 @@ type ownerKey struct { } // ownerResolution is what resolveOwner found: found is false for an owner -// genuinely absent, true with uid set for one that exists. The zero value is -// never looked at directly; resolveOwner's second return says whether this is -// meaningful at all. +// genuinely absent, true with uid set for one that exists, and +// deletingDependents true for one that exists and is waiting on foreground +// deletion of its dependents. The zero value is never looked at directly; +// resolveOwner's second return says whether this is meaningful at all. type ownerResolution struct { - found bool - uid types.UID + found bool + uid types.UID + deletingDependents bool } // tickGC acts as Kubernetes' garbage collector over DataPlaneSimOptions's @@ -51,12 +55,21 @@ func (s *DataPlaneSim) tickGC(ctx context.Context) { if err != nil { return } + s.sweepGC(ctx, objs) +} - // Foreground and orphan finalizers first, and in place on objs: an - // orphan dependent has its owner reference stripped from the very - // client.Object held in objs, so the background pass below sees the - // post-strip state rather than reaching for the same owner again through - // a reference that no longer exists in the cluster. +// sweepGC is one garbage collection pass over objs, a List taken at some +// earlier point. Everything it decides from objs is a candidate only: by the +// time it acts, an owner may have been recreated under the same name, a +// dependent deleted and recreated, or a dependent adopted by a new owner, so +// every write re-reads its target first and is guarded by a precondition. +// See collect. +func (s *DataPlaneSim) sweepGC(ctx context.Context, objs []client.Object) { + // Foreground and orphan finalizers first. Neither updates objs, so the + // background pass below can still see a reference an orphan strip just + // removed, pointing at an owner whose finalizer just came off. That is + // safe only because collect judges each candidate from a live read, where + // the stripped reference is gone. for _, obj := range objs { dt := obj.GetDeletionTimestamp() if dt.IsZero() { @@ -84,7 +97,7 @@ func (s *DataPlaneSim) tickGC(ctx context.Context) { } } if dangling { - _ = s.c.Delete(ctx, obj) + s.collect(ctx, obj) } } } @@ -117,9 +130,20 @@ func (s *DataPlaneSim) listWatched(ctx context.Context) ([]client.Object, error) } // foregroundDeleteDependents is one sweep of foreground deletion for owner, -// which has a deletionTimestamp and the foregroundDeletion finalizer: delete +// which has a deletionTimestamp and the foregroundDeletion finalizer: collect // every dependent of owner found among objs, and once none remain, remove the -// finalizer so the API server can finish deleting owner itself. +// finalizer so the API server can finish deleting owner itself. collect +// deletes a dependent whose only owners are gone or deleting, and strips the +// reference to owner from one that still has a live owner, which is what +// lets the finalizer come off without deleting an object someone else still +// owns. +// +// Two upstream refinements are not modelled. Every dependent is waited on, +// whether or not its reference sets blockOwnerDeletion, so a non-blocking +// dependent held up by a finalizer of its own also holds owner. And deletes +// carry no propagation policy, where upstream deletes a dependent that has +// dependents of its own in the foreground, so owner can be released before +// its grandchildren are gone. // // Deletes issued this sweep usually have not landed by the time remaining is // counted, so the finalizer typically comes off one sweep later, once a fresh @@ -138,7 +162,7 @@ func (s *DataPlaneSim) foregroundDeleteDependents( continue } remaining++ - _ = s.c.Delete(ctx, dep) + s.collect(ctx, dep) } if remaining == 0 { s.removeFinalizer(ctx, owner, metav1.FinalizerDeleteDependents) @@ -147,51 +171,209 @@ func (s *DataPlaneSim) foregroundDeleteDependents( // orphanDependents is one sweep of orphan deletion for owner, which has a // deletionTimestamp and the orphan finalizer: strip owner's reference from -// every dependent found among objs, then remove the finalizer. Unlike -// foreground deletion this always finishes in one sweep, since stripping a -// reference is an action this sim takes directly rather than one it waits on -// another object's deletion to complete. +// every dependent found among objs, then remove the finalizer. The finalizer +// comes off only once every strip has landed or found its dependent gone; one +// that failed, a conflict included, leaves it for the next sweep, because +// removing it early lets owner go while a dependent still names it, and the +// background pass would then collect the very dependent orphaning exists to +// keep. +// +// objs only says which dependents to look at. Each strip re-reads its +// dependent and patches under that read's resourceVersion, so it never lands +// on a recreated object of the same name or overwrites an owner reference +// added since: upstream orphanDependents in +// pkg/controller/garbagecollector/garbagecollector.go does the same with a +// UID precondition, falling back to deleteOwnerRefJSONMergePatch in patch.go, +// which carries the resourceVersion of a fresh read. func (s *DataPlaneSim) orphanDependents( ctx context.Context, owner client.Object, objs []client.Object, ) { + done := true for _, dep := range objs { if !referencesUID(dep, owner.GetUID()) { continue } - base, ok := dep.DeepCopyObject().(client.Object) - if !ok { + live, err := s.getLive(ctx, dep) + switch { + case apierrors.IsNotFound(err): + continue + case err != nil: + done = false continue } - kept := make([]metav1.OwnerReference, 0, len(dep.GetOwnerReferences())) - for _, ref := range dep.GetOwnerReferences() { - if ref.UID != owner.GetUID() { - kept = append(kept, ref) - } + if !referencesUID(live, owner.GetUID()) { + continue + } + err = s.stripOwnerRefs(ctx, live, func(uid types.UID) bool { return uid == owner.GetUID() }) + if err != nil && !apierrors.IsNotFound(err) { + done = false } - dep.SetOwnerReferences(kept) - _ = s.c.Patch(ctx, dep, client.StrategicMergeFrom(base)) } - s.removeFinalizer(ctx, owner, metav1.FinalizerOrphanDependents) + if done { + s.removeFinalizer(ctx, owner, metav1.FinalizerOrphanDependents) + } } -// removeFinalizer patches finalizer off obj with a strategic merge patch, so -// a finalizer another writer added to obj in the meantime survives: the same -// reasoning tickPods already applies to pod conditions. -func (s *DataPlaneSim) removeFinalizer(ctx context.Context, obj client.Object, finalizer string) { - base, ok := obj.DeepCopyObject().(client.Object) - if !ok { +// removeFinalizer takes finalizer off owner, the object the sweep listed, as +// upstream's removeFinalizer (pkg/controller/garbagecollector/operations.go) +// does: read it fresh, and patch the finalizer list under that read's +// resourceVersion, retrying on conflict from a new read. A JSON merge patch +// replaces the whole list, so the lock is what keeps a concurrent change to +// the list; the retry is what keeps an owner under constant unrelated writes +// from holding its finalizer forever. The UID check keeps the patch off a +// recreated owner of the same name, which may itself be waiting on +// dependents this sweep's List never saw. A merge patch rather than a +// strategic one because the API server refuses strategic merge on custom +// resources. +func (s *DataPlaneSim) removeFinalizer(ctx context.Context, owner client.Object, finalizer string) { + _ = retry.RetryOnConflict(retry.DefaultRetry, func() error { + live, err := s.getLive(ctx, owner) + if err != nil { + return client.IgnoreNotFound(err) + } + if live.GetUID() != owner.GetUID() || !slices.Contains(live.GetFinalizers(), finalizer) { + return nil + } + base := live.DeepCopy() + live.SetFinalizers(slices.DeleteFunc(slices.Clone(live.GetFinalizers()), + func(f string) bool { return f == finalizer })) + return s.c.Patch( + ctx, + live, + client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{}), + ) + }) +} + +// collect is upstream's attemptToDeleteItem +// (pkg/controller/garbagecollector/garbagecollector.go) for one dependent the +// sweep's List marked as a candidate. listed is only a name: the List may be +// arbitrarily stale, and acting on it directly is how a sim deletes a +// recreated object that reuses the name, or an object that has since gained +// a live owner. So, as upstream does: +// +// - Read the dependent live, and decide from that read alone. +// - Classify each of its owners afresh, with no sweep cache: solid (exists +// with the referenced UID and is not deleting its dependents), dangling +// (absent, or the name now holds a different UID), or waiting (exists +// and is under foreground deletion). An owner this sim cannot resolve +// (an unmapped kind, a failed lookup) ends the attempt with nothing +// done, so a guess never deletes or strips anything. +// - With any solid owner, strip the references to dangling and waiting +// owners instead of deleting. +// - Otherwise delete, preconditioned on the UID and resourceVersion of the +// live read, through deleteLive. +// +// Errors are dropped rather than returned: every failure here, a conflict +// included, means "decide again next sweep", which the next List does. +func (s *DataPlaneSim) collect(ctx context.Context, listed client.Object) { + live, err := s.getLive(ctx, listed) + if err != nil { return } - kept := make([]string, 0, len(obj.GetFinalizers())) - for _, f := range obj.GetFinalizers() { - if f != finalizer { - kept = append(kept, f) + refs := live.GetOwnerReferences() + if len(refs) == 0 { + return + } + var solid bool + notSolid := map[types.UID]bool{} + for _, ref := range refs { + res, resolvable := s.resolveOwner(ctx, nil, ref, live.GetNamespace()) + switch { + case !resolvable: + // Neither solid nor dangling is known, so do nothing and let a + // later sweep decide, as upstream's classifyReferences does by + // returning the lookup error. Treating it as solid would strip a + // foreground owner's reference on the strength of an owner never + // verified to exist. + return + case !res.found || res.uid != ref.UID, res.deletingDependents: + notSolid[ref.UID] = true + default: + solid = true } } - obj.SetFinalizers(kept) - _ = s.c.Patch(ctx, obj, client.StrategicMergeFrom(base)) + if solid { + if len(notSolid) > 0 { + _ = s.stripOwnerRefs(ctx, live, func(uid types.UID) bool { return notSolid[uid] }) + } + return + } + s.deleteLive(ctx, live) +} + +// getLive reads obj's metadata straight from the API server into a fresh +// object. Metadata only, because that is all garbage collection decides on, +// and fresh rather than into a copy of obj, because decoding over a +// populated object keeps any field the response omits: an owner reference +// list emptied since obj was listed would survive the read. +func (s *DataPlaneSim) getLive( + ctx context.Context, + obj client.Object, +) (*metav1.PartialObjectMetadata, error) { + gvk, err := s.c.GroupVersionKindFor(obj) + if err != nil { + return nil, err + } + live := &metav1.PartialObjectMetadata{} + live.SetGroupVersionKind(gvk) + if err := s.c.Get(ctx, client.ObjectKeyFromObject(obj), live); err != nil { + return nil, err + } + return live, nil +} + +// deleteLive is upstream's deleteObject (operations.go): delete live under +// both its UID and its resourceVersion. The UID keeps the delete off a +// recreated object of the same name; the resourceVersion keeps it off an +// object whose owners changed after live was read. +// +// A resourceVersion conflict alone would starve collection of any object +// written to often, so, again as upstream does, a conflict is followed by one +// more read, and if the object is the same incarnation with exactly the owner +// references live was judged on, the delete is retried under the UID alone. +// Anything else (NotFound, a different UID, changed owners, any other error) +// is left to the next sweep to judge from scratch. +func (s *DataPlaneSim) deleteLive(ctx context.Context, live *metav1.PartialObjectMetadata) { + uid, rv := live.GetUID(), live.GetResourceVersion() + err := s.c.Delete(ctx, live, client.Preconditions{UID: &uid, ResourceVersion: &rv}) + if !apierrors.IsConflict(err) { + return + } + again, err := s.getLive(ctx, live) + if err != nil || again.GetUID() != uid || again.GetResourceVersion() == rv || + !reflect.DeepEqual(again.GetOwnerReferences(), live.GetOwnerReferences()) { + return + } + _ = s.c.Delete(ctx, again, client.Preconditions{UID: &uid}) +} + +// stripOwnerRefs removes every owner reference on live whose UID drop +// matches, with a JSON merge patch under live's resourceVersion: the patch +// writes the whole remaining list, so the lock is what stops it erasing a +// reference added since live was read, and what keeps it off a recreated +// object of the same name. A merge patch rather than a strategic one because +// the API server refuses strategic merge on custom resources. +func (s *DataPlaneSim) stripOwnerRefs( + ctx context.Context, + live *metav1.PartialObjectMetadata, + drop func(types.UID) bool, +) error { + base := live.DeepCopy() + kept := make([]metav1.OwnerReference, 0, len(live.GetOwnerReferences())) + for _, ref := range live.GetOwnerReferences() { + if !drop(ref.UID) { + kept = append(kept, ref) + } + } + live.SetOwnerReferences(kept) + return s.c.Patch( + ctx, + live, + client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{}), + ) } // referencesUID reports whether obj carries an owner reference to uid. @@ -227,7 +409,8 @@ func (s *DataPlaneSim) ownerDangling( } // resolveOwner looks up the object ref names, caching the result in cache for -// the rest of the sweep. The second return is false when the kind cannot be +// the rest of the sweep, or with a nil cache, reading it fresh and caching +// nothing. The second return is false when the kind cannot be // resolved at all (a malformed APIVersion, or one the RESTMapper does not // map), as distinct from a successful lookup that found nothing. func (s *DataPlaneSim) resolveOwner( @@ -255,6 +438,11 @@ func (s *DataPlaneSim) resolveOwner( if cached, ok := cache[key]; ok { return *cached, true } + remember := func(res ownerResolution) { + if cache != nil { + cache[key] = &res + } + } pom := &metav1.PartialObjectMetadata{ TypeMeta: metav1.TypeMeta{APIVersion: ref.APIVersion, Kind: ref.Kind}, @@ -263,7 +451,7 @@ func (s *DataPlaneSim) resolveOwner( switch { case apierrors.IsNotFound(err): res := ownerResolution{found: false} - cache[key] = &res + remember(res) return res, true case err != nil: // Transient or unexpected: not cached, so the next sweep tries again @@ -271,8 +459,13 @@ func (s *DataPlaneSim) resolveOwner( // one. return ownerResolution{}, false default: - res := ownerResolution{found: true, uid: pom.UID} - cache[key] = &res + res := ownerResolution{ + found: true, + uid: pom.UID, + deletingDependents: !pom.DeletionTimestamp.IsZero() && + slices.Contains(pom.Finalizers, metav1.FinalizerDeleteDependents), + } + remember(res) return res, true } } diff --git a/ctrltest/gc_stale_test.go b/ctrltest/gc_stale_test.go new file mode 100644 index 0000000..81f9b74 --- /dev/null +++ b/ctrltest/gc_stale_test.go @@ -0,0 +1,479 @@ +// SPDX-License-Identifier: Apache-2.0 + +package ctrltest + +import ( + "context" + "slices" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +// These tests drive garbage collection from a List taken before the cluster +// moved on, which is the window a sweep under load really acts in: an owner +// recreated under the same name, a dependent recreated or adopted, a write +// landing between the sim's read and its delete. They run against envtest +// rather than the fake client because the fake enforces neither a UID +// precondition on Delete nor the foreground and orphan finalizers the API +// server adds. +// +// Dependents are Secrets and owners ServiceAccounts, two kinds the suite's own +// sim does not watch, so nothing but the sim each test builds and steps by +// hand ever collects a dependent or clears an owner's finalizer here. An +// owner of a watched kind would race that sim for its finalizers. + +// staleGC is a garbage-collecting sim stepped by hand, over one test's +// namespace, whose reads of a dependent's live metadata can be intercepted. +type staleGC struct { + *C + ctx context.Context + sim *DataPlaneSim + + // afterLiveGet, keyed by "Kind/name", runs after the sim's nth live + // read of that object returns (n counting from 1), so a test can change + // the cluster in the window between that read and the write it decides. + afterLiveGet map[string]map[int]func() + liveGets map[string]int +} + +func newStaleGC(t *testing.T) *staleGC { + t.Helper() + g := &staleGC{ + C: shared.Case(t), + ctx: context.Background(), + afterLiveGet: map[string]map[int]func(){}, + liveGets: map[string]int{}, + } + cl := interceptor.NewClient(shared.Client, interceptor.Funcs{ + Get: func( + ctx context.Context, w client.WithWatch, key client.ObjectKey, + obj client.Object, opts ...client.GetOption, + ) error { + err := w.Get(ctx, key, obj, opts...) + pom, ok := obj.(*metav1.PartialObjectMetadata) + if !ok || key.Namespace != g.NS { + return err + } + id := pom.Kind + "/" + key.Name + g.liveGets[id]++ + if hook := g.afterLiveGet[id][g.liveGets[id]]; hook != nil { + hook() + } + return err + }, + }) + g.sim = NewDataPlaneSimWithOptions(cl, time.Hour, DataPlaneSimOptions{ + GarbageCollection: true, + WatchedKinds: []client.ObjectList{&corev1.SecretList{}, &corev1.ServiceAccountList{}}, + RESTMapper: shared.Client.RESTMapper(), + }) + return g +} + +// after registers fn to run after the sim's nth live read of dependent name. +func (g *staleGC) after(name string, n int, fn func()) { g.afterKind("Secret", name, n, fn) } + +// afterKind is after for an object of any kind. +func (g *staleGC) afterKind(kind, name string, n int, fn func()) { + id := kind + "/" + name + if g.afterLiveGet[id] == nil { + g.afterLiveGet[id] = map[int]func(){} + } + g.afterLiveGet[id][n] = fn +} + +// list is the sweep's List, narrowed to this test's namespace so a sweep +// never acts on another test's objects. +func (g *staleGC) list() []client.Object { + g.Helper() + objs, err := g.sim.listWatched(g.ctx) + g.NoError(err, "list watched kinds") + return slices.DeleteFunc(objs, func(o client.Object) bool { return o.GetNamespace() != g.NS }) +} + +func (g *staleGC) sweep(objs []client.Object) { g.sim.sweepGC(g.ctx, objs) } + +func (g *staleGC) owner(name string) *corev1.ServiceAccount { + g.Helper() + sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: g.NS}} + g.NoError(shared.Client.Create(g.ctx, sa), "create owner %s", name) + return sa +} + +func (g *staleGC) dependent(name string, owners ...metav1.OwnerReference) *corev1.Secret { + g.Helper() + s := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: g.NS, OwnerReferences: owners, + }} + g.NoError(shared.Client.Create(g.ctx, s), "create dependent %s", name) + return s +} + +// recreate deletes dependent name and creates a new one in its place, with a +// new UID, owned by owners. +func (g *staleGC) recreate(name string, owners ...metav1.OwnerReference) *corev1.Secret { + g.Helper() + old := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: g.NS}} + g.NoError(shared.Client.Delete(g.ctx, old), "delete dependent %s", name) + return g.dependent(name, owners...) +} + +func (g *staleGC) deleteOwner(sa *corev1.ServiceAccount, policy metav1.DeletionPropagation) { + g.Helper() + err := shared.Client.Delete(g.ctx, sa, client.PropagationPolicy(policy)) + g.NoError(err, "delete owner %s", sa.Name) +} + +// get returns dependent name, or nil if it is gone. +func (g *staleGC) get(name string) *corev1.Secret { + g.Helper() + s := &corev1.Secret{} + err := shared.Client.Get(g.ctx, client.ObjectKey{Namespace: g.NS, Name: name}, s) + if apierrors.IsNotFound(err) { + return nil + } + g.NoError(err, "get dependent %s", name) + return s +} + +func (g *staleGC) getOwner(name string) *corev1.ServiceAccount { + g.Helper() + sa := &corev1.ServiceAccount{} + err := shared.Client.Get(g.ctx, client.ObjectKey{Namespace: g.NS, Name: name}, sa) + if apierrors.IsNotFound(err) { + return nil + } + g.NoError(err, "get owner %s", name) + return sa +} + +func (g *staleGC) relabel(name string) { + g.Helper() + s := g.get(name) + g.NotNil(s, "relabel: dependent %s is gone", name) + base := s.DeepCopy() + s.Labels = map[string]string{"bump": s.ResourceVersion} + g.NoError(shared.Client.Patch(g.ctx, s, client.MergeFrom(base)), "relabel %s", name) +} + +func saRef(sa *corev1.ServiceAccount) metav1.OwnerReference { + return metav1.OwnerReference{ + APIVersion: "v1", + Kind: "ServiceAccount", + Name: sa.Name, + UID: sa.UID, + } +} + +func ownerUIDs(s *corev1.Secret) []types.UID { + var out []types.UID + for _, r := range s.OwnerReferences { + out = append(out, r.UID) + } + return out +} + +// TestGCStaleListSparesARecreatedDependent is an owner and its dependent +// both deleted and recreated under the same names after the sweep's List: the +// dependent the List saw is gone, and the one now holding its name belongs to +// a live owner. Deleting by the listed name would take the new one. +func TestGCStaleListSparesARecreatedDependent(t *testing.T) { + g := newStaleGC(t) + shard := g.owner("shard") + g.dependent("pool-0", saRef(shard)) + g.dependent("orphan", saRef(shard)) + g.deleteOwner(shard, metav1.DeletePropagationBackground) + + objs := g.list() + shard2 := g.owner("shard") + fresh := g.recreate("pool-0", saRef(shard2)) + + g.sweep(objs) + + got := g.get("pool-0") + g.NotNil(got, "the recreated pool-0, owned by the live shard, was collected") + g.Eq(fresh.UID, got.UID, "pool-0 is not the recreated incarnation") + g.Nil(g.get("orphan"), "a dependent whose only owner is gone was not collected") +} + +// TestGCStaleListSparesADependentAdoptedSinceTheList is the same dependent, +// same UID, listed while its only owner was dead and adopted by a live owner +// before the sweep acts. A UID precondition still matches it, so only judging +// its owners from a fresh read saves it. The reference to the dead owner is +// stripped, as the real collector does for a dependent with a live owner. +func TestGCStaleListSparesADependentAdoptedSinceTheList(t *testing.T) { + g := newStaleGC(t) + shard := g.owner("shard") + data := g.dependent("data", saRef(shard)) + g.dependent("orphan", saRef(shard)) + g.deleteOwner(shard, metav1.DeletePropagationBackground) + + objs := g.list() + anchor := g.owner("anchor") + base := data.DeepCopy() + data.OwnerReferences = append(data.OwnerReferences, saRef(anchor)) + g.NoError(shared.Client.Patch(g.ctx, data, client.MergeFrom(base)), "adopt data") + + g.sweep(objs) + + got := g.get("data") + g.NotNil(got, "data, adopted by a live owner after the List, was collected") + g.Eq(data.UID, got.UID, "data is not the same incarnation") + g.ElementsMatch([]types.UID{anchor.UID}, ownerUIDs(got), "owners after the sweep") + g.Nil(g.get("orphan"), "a dependent whose only owner is gone was not collected") +} + +// TestGCDeleteIsPreconditionedOnTheLiveRead covers the window after the sim +// has read a dependent and judged it collectable: the dependent is recreated +// before the delete lands. Only a precondition on the read's UID and +// resourceVersion keeps the delete off the new object. +func TestGCDeleteIsPreconditionedOnTheLiveRead(t *testing.T) { + g := newStaleGC(t) + shard := g.owner("shard") + g.dependent("pool-0", saRef(shard)) + g.deleteOwner(shard, metav1.DeletePropagationBackground) + + var fresh *corev1.Secret + g.after("pool-0", 1, func() { fresh = g.recreate("pool-0") }) + + g.sweep(g.list()) + + got := g.get("pool-0") + g.NotNil(got, "pool-0, recreated between the sim's read and its delete, was deleted") + g.Eq(fresh.UID, got.UID, "pool-0 is not the recreated incarnation") +} + +// TestGCRetriesADeleteThatConflictedOnUnrelatedWrites is the liveness side of +// the resourceVersion precondition: an orphan written to between the sim's +// read and its delete, without its owners changing, is still collected in +// the same sweep, so an object under constant unrelated writes is never +// starved. +func TestGCRetriesADeleteThatConflictedOnUnrelatedWrites(t *testing.T) { + g := newStaleGC(t) + shard := g.owner("shard") + g.dependent("busy", saRef(shard)) + g.deleteOwner(shard, metav1.DeletePropagationBackground) + + g.after("busy", 1, func() { g.relabel("busy") }) + + g.sweep(g.list()) + + g.Nil(g.get("busy"), "an orphan with an unrelated write in flight was not collected") +} + +// TestGCRetryIsStillPreconditionedOnUID is the retry after a conflict landing +// on a recreated object: the retry drops the resourceVersion, but not the UID. +func TestGCRetryIsStillPreconditionedOnUID(t *testing.T) { + g := newStaleGC(t) + shard := g.owner("shard") + g.dependent("busy", saRef(shard)) + g.deleteOwner(shard, metav1.DeletePropagationBackground) + + var fresh *corev1.Secret + g.after("busy", 1, func() { g.relabel("busy") }) + g.after("busy", 2, func() { fresh = g.recreate("busy") }) + + g.sweep(g.list()) + + got := g.get("busy") + g.NotNil(got, "busy, recreated before the retried delete, was deleted") + g.Eq(fresh.UID, got.UID, "busy is not the recreated incarnation") +} + +// TestGCForegroundStripsADependentWithAnotherLiveOwner pins foreground +// deletion against a dependent that something else still owns: it loses its +// reference to the deleting owner rather than being deleted, a dependent with +// no other owner is deleted, and the owner's finalizer comes off once neither +// references it. +func TestGCForegroundStripsADependentWithAnotherLiveOwner(t *testing.T) { + g := newStaleGC(t) + shard := g.owner("shard") + anchor := g.owner("anchor") + g.dependent("shared", saRef(shard), saRef(anchor)) + g.dependent("solo", saRef(shard)) + g.deleteOwner(shard, metav1.DeletePropagationForeground) + + g.sweep(g.list()) + + got := g.get("shared") + g.NotNil(got, "a dependent with a live second owner was deleted by foreground deletion") + g.ElementsMatch([]types.UID{anchor.UID}, ownerUIDs(got), "owners after the sweep") + g.Nil(g.get("solo"), "a dependent of a foreground-deleted owner was not deleted") + + g.sweep(g.list()) + g.Nil(g.getOwner("shard"), "foreground owner still present once no dependent references it") +} + +// TestGCOrphanLeavesTheFinalizerUntilEveryStripLands is an owner reference +// added to a dependent between the sim's read of it and its strip of the +// orphaning owner. The strip must not erase the new reference, and the +// owner's orphan finalizer must stay until a strip lands: removed early, the +// owner goes while the dependent still names it. +func TestGCOrphanLeavesTheFinalizerUntilEveryStripLands(t *testing.T) { + g := newStaleGC(t) + shard := g.owner("shard") + anchor := g.owner("anchor") + kept := g.dependent("kept", saRef(shard)) + g.deleteOwner(shard, metav1.DeletePropagationOrphan) + + g.after("kept", 1, func() { + cur := g.get("kept") + base := cur.DeepCopy() + cur.OwnerReferences = append(cur.OwnerReferences, saRef(anchor)) + g.NoError(shared.Client.Patch(g.ctx, cur, client.MergeFrom(base)), "adopt kept") + }) + + g.sweep(g.list()) + + got := g.get("kept") + g.NotNil(got, "an orphaned dependent was deleted") + g.Contains( + ownerUIDs(got), + anchor.UID, + "the strip erased an owner reference added after the read", + ) + owner := g.getOwner("shard") + g.NotNil(owner, "orphan finalizer came off although the strip did not land") + + g.sweep(g.list()) + + got = g.get("kept") + g.NotNil(got, "an orphaned dependent was deleted") + g.Eq(kept.UID, got.UID, "kept is not the same incarnation") + g.ElementsMatch([]types.UID{anchor.UID}, ownerUIDs(got), "owners after orphaning") + g.Nil(g.getOwner("shard"), "orphaning owner still present after its dependent was stripped") +} + +// TestGCFinalizerRemovalSparesARecreatedOwner is a foreground-deleting owner +// that finished deleting and was recreated, under the same name, and is +// itself being foreground-deleted with a dependent still alive, all after the +// sweep's List. The listed owner has no dependents left, so the sweep removes +// its finalizer; landing on the new owner, that would let it go while its +// dependent still exists. +func TestGCFinalizerRemovalSparesARecreatedOwner(t *testing.T) { + g := newStaleGC(t) + shard := g.owner("shard") + g.deleteOwner(shard, metav1.DeletePropagationForeground) + + objs := g.list() + cur := g.getOwner("shard") + base := cur.DeepCopy() + cur.Finalizers = nil + g.NoError(shared.Client.Patch(g.ctx, cur, client.MergeFrom(base)), "finish deleting shard") + g.Nil(g.getOwner("shard"), "shard still present after its last finalizer came off") + shard2 := g.owner("shard") + g.dependent("child", saRef(shard2)) + g.deleteOwner(shard2, metav1.DeletePropagationForeground) + + g.sweep(objs) + + got := g.getOwner("shard") + g.NotNil(got, "the recreated shard was released while its dependent still exists") + g.Eq(shard2.UID, got.UID, "shard is not the recreated incarnation") + g.NotNil(g.get("child"), "child was collected from a List that never saw it") + + g.sweep(g.list()) + g.Nil(g.get("child"), "a dependent of a foreground-deleted owner was not deleted") + g.sweep(g.list()) + g.Nil(g.getOwner("shard"), "foreground owner still present once no dependent references it") +} + +// TestGCStaleListSparesADependentReleasedSinceTheList is a dependent listed +// with only a dead owner whose owner references were all removed before the +// sweep acts. With no owners left it is nobody's dependent, and garbage +// collection has nothing to say about it. +func TestGCStaleListSparesADependentReleasedSinceTheList(t *testing.T) { + g := newStaleGC(t) + shard := g.owner("shard") + data := g.dependent("data", saRef(shard)) + g.deleteOwner(shard, metav1.DeletePropagationBackground) + + objs := g.list() + base := data.DeepCopy() + data.OwnerReferences = nil + g.NoError(shared.Client.Patch(g.ctx, data, client.MergeFrom(base)), "release data") + + g.sweep(objs) + + g.NotNil(g.get("data"), "data, released by every owner after the List, was collected") +} + +// TestGCSparesADependentAdoptedAfterTheLiveRead is adoption inside the window +// between the sim's live read of a dependent and its delete: same UID, so only +// the resourceVersion precondition catches it, and the conflict retry must +// see the changed owners and stand down rather than retry under the UID alone. +func TestGCSparesADependentAdoptedAfterTheLiveRead(t *testing.T) { + g := newStaleGC(t) + shard := g.owner("shard") + anchor := g.owner("anchor") + data := g.dependent("data", saRef(shard)) + g.deleteOwner(shard, metav1.DeletePropagationBackground) + + g.after("data", 1, func() { + cur := g.get("data") + base := cur.DeepCopy() + cur.OwnerReferences = append(cur.OwnerReferences, saRef(anchor)) + g.NoError(shared.Client.Patch(g.ctx, cur, client.MergeFrom(base)), "adopt data") + }) + + g.sweep(g.list()) + + got := g.get("data") + g.NotNil(got, "data, adopted between the sim's read and its delete, was deleted") + g.Eq(data.UID, got.UID, "data is not the same incarnation") +} + +// TestGCFinalizerRemovalRetriesOnUnrelatedWrites is a foreground owner with +// no dependents left, written to between the sim's read of it and its +// finalizer patch. The patch conflicts, and the retry from a fresh read must +// still release the owner in the same sweep, so an owner under constant +// writes is never held. +func TestGCFinalizerRemovalRetriesOnUnrelatedWrites(t *testing.T) { + g := newStaleGC(t) + shard := g.owner("shard") + g.deleteOwner(shard, metav1.DeletePropagationForeground) + + g.afterKind("ServiceAccount", "shard", 1, func() { + cur := g.getOwner("shard") + base := cur.DeepCopy() + cur.Labels = map[string]string{"bump": cur.ResourceVersion} + g.NoError(shared.Client.Patch(g.ctx, cur, client.MergeFrom(base)), "relabel shard") + }) + + g.sweep(g.list()) + + g.Nil(g.getOwner("shard"), "foreground owner held by an unrelated write in flight") +} + +// TestGCForegroundLeavesADependentWithAnUnresolvableOwner is a dependent of a +// foreground-deleting owner whose other owner is of a kind the RESTMapper +// does not know. Whether that owner is live is unknown, so the dependent is +// neither deleted nor stripped of its reference to the deleting owner: the +// real collector retries the lookup rather than guess, and the owner waits. +func TestGCForegroundLeavesADependentWithAnUnresolvableOwner(t *testing.T) { + g := newStaleGC(t) + shard := g.owner("shard") + unknown := metav1.OwnerReference{ + APIVersion: "example.com/v1", Kind: "Widget", Name: "w", UID: types.UID("widget-uid"), + } + g.dependent("data", saRef(shard), unknown) + g.deleteOwner(shard, metav1.DeletePropagationForeground) + + g.sweep(g.list()) + + got := g.get("data") + g.NotNil(got, "a dependent with an unresolvable owner was deleted") + g.Contains( + ownerUIDs(got), + shard.UID, + "the reference to the deleting owner was stripped on a guess", + ) + g.NotNil(g.getOwner("shard"), "foreground owner released while a dependent still references it") +} From b6d734378479d70e68b7a69abd2a765601cdf8d7 Mon Sep 17 00:00:00 2001 From: Brent Graveland Date: Thu, 5 Nov 2026 11:37:37 -0600 Subject: [PATCH 2/2] chore(deps): tidy go.sum go mod tidy adds checksums for the otel, stdr and gonum modules in the build graph, which the Lint job's tidy check requires. Signed-off-by: Brent Graveland --- go.sum | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/go.sum b/go.sum index a4ba920..a9d85a3 100644 --- a/go.sum +++ b/go.sum @@ -24,6 +24,8 @@ github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+r github.com/fxamacker/cbor/v2 v2.9.1/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v1.0.0 h1:kR9tHqY0CtZaOPVFm622dPVNhrvYpwr4uCxgL3h1H8s= @@ -130,6 +132,18 @@ go.etcd.io/etcd/client/pkg/v3 v3.7.2 h1:SVtlR7tiSVAYOQ4nWPIyFXb4RMgEcnzeAG9RQ8Mo go.etcd.io/etcd/client/pkg/v3 v3.7.2/go.mod h1:HsSux/B3ahgyw/D5+d4YbZqicOi0mEbuxm6lIUdjAoI= go.etcd.io/etcd/client/v3 v3.7.2 h1:Z66GqDQDI7zPDfVSsIBqGSK4mJYLtv8ESwXa4mPf+wY= go.etcd.io/etcd/client/v3 v3.7.2/go.mod h1:x03t1qMs4tGZirCDJlMuzPBJdQffXJImIyEjLhNBCsY= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -160,6 +174,8 @@ golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk=