From 410fc29f52529d739c731ce0ce82f7db9b33764b Mon Sep 17 00:00:00 2001 From: ostefani Date: Wed, 18 Nov 2026 03:14:59 -0800 Subject: [PATCH 1/4] fix(test): measure closure drain ingest p99 correctly - use nearest-rank p99 and report the maximum separately - report request timing around observed drain completion - include relay round-trip timing for the slowest samples --- internal/server/server_test.go | 91 +++++++++++++++++++++++++++------- 1 file changed, 73 insertions(+), 18 deletions(-) diff --git a/internal/server/server_test.go b/internal/server/server_test.go index db16d72..892be75 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -968,22 +968,53 @@ func briefingDeps(t *testing.T, h *harness) briefing.Deps { } } +type relayTimingTransport struct { + base http.RoundTripper + elapsed time.Duration +} + +func (t *relayTimingTransport) RoundTrip(req *http.Request) (*http.Response, error) { + start := time.Now() + res, err := t.base.RoundTrip(req) + t.elapsed += time.Since(start) + return res, err +} + func TestClosureDrainBudgets(t *testing.T) { h := newHarness(t, func(c *config.Config) { c.Mode = config.ModeProxy }) now := time.Now() seeded := seedClosedGroups(t, h, 256) + // Only the serial ingest loop uses this client. Keep timing with + // each request so a slow ingest can be matched to its relay call. + base := h.srv.proxy.Transport + if base == nil { + base = http.DefaultTransport + } + relay := &relayTimingTransport{base: base} + h.srv.proxy.Transport = relay + ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + deps := briefingDeps(t, h) runDone := make(chan error, 1) - go func() { runDone <- briefing.Run(ctx, briefingDeps(t, h)) }() start := time.Now() + go func() { runDone <- briefing.Run(ctx, deps) }() + defer func() { + cancel() + if err := <-runDone; err != nil { + t.Errorf("briefing worker: %v", err) + } + }() - // Background ingest at 50 ev/s on its own goroutine while the main - // goroutine polls the seeded groups: every request lands while the - // worker holds the shared writer. + // Pace requests at up to 50/s and keep sampling after the drain, + // which can finish well before the 100-request workload ends. + type sample struct { + started time.Time + total time.Duration + relay time.Duration + } type ingestResult struct { - durs []time.Duration + samples []sample lost int saturated int } @@ -994,9 +1025,10 @@ func TestClosureDrainBudgets(t *testing.T) { var res ingestResult for i := 0; i < 100; i++ { <-tick.C + relay.elapsed = 0 reqStart := time.Now() rec := h.signedPost(t, uniqueID(), "host-bg", "host.appeared", now) - res.durs = append(res.durs, time.Since(reqStart)) + res.samples = append(res.samples, sample{reqStart, time.Since(reqStart), relay.elapsed}) switch rec.Code { case http.StatusAccepted: case http.StatusServiceUnavailable: @@ -1007,23 +1039,46 @@ func TestClosureDrainBudgets(t *testing.T) { } ingestDone <- res }() - for !seededDone(t, h, seeded) { - if time.Since(start) > 5*time.Second { - t.Fatal("256 seeded groups not briefed within 5s") - } + + for time.Since(start) < 5*time.Second && !seededDone(t, h, seeded) { time.Sleep(10 * time.Millisecond) } - drainElapsed := time.Since(start) - t.Logf("256-group drain took %v (budget 5s)", drainElapsed) + drainObserved := time.Now() res := <-ingestDone - sort.Slice(res.durs, func(a, b int) bool { return res.durs[a] < res.durs[b] }) - p99 := res.durs[int(float64(len(res.durs))*0.99)] - t.Logf("ingest p99 during drain over %d requests: %v (budget 50ms)", len(res.durs), p99) + if elapsed := drainObserved.Sub(start); elapsed > 5*time.Second { + t.Fatalf("256 seeded groups not observed done within 5s (polled for %v)", elapsed) + } + t.Logf("256-group drain observed in %v (budget 5s)", drainObserved.Sub(start)) + var during, after []sample + for _, req := range res.samples { + // Polling gives an upper bound on when the drain completed. + if req.started.Before(drainObserved) { + during = append(during, req) + } else { + after = append(after, req) + } + } + report := func(label string, samples []sample) time.Duration { + if len(samples) == 0 { + t.Logf("%s: 0 requests", label) + return 0 + } + sort.Slice(samples, func(a, b int) bool { return samples[a].total < samples[b].total }) + // Nearest rank, converted from a one-based rank to a slice index. + p99 := samples[(99*len(samples)+99)/100-1] + max := samples[len(samples)-1] + t.Logf("%s: %d requests, p99 %v (relay %v), max %v (relay %v)", + label, len(samples), p99.total, p99.relay, max.total, max.relay) + return p99.total + } + p99 := report("all proxy ingest", res.samples) + report("started before drain completion was observed", during) + report("started after drain completion was observed", after) if p99 > 50*time.Millisecond { - t.Fatalf("ingest p99 %v exceeds 50ms during drain", p99) + t.Errorf("proxy ingest p99 %v exceeds 50ms", p99) } if res.saturated != 0 || res.lost != 0 { - t.Fatalf("ingest loss during drain: saturated=%d lost=%d", res.saturated, res.lost) + t.Errorf("ingest loss: saturated=%d lost=%d", res.saturated, res.lost) } var mem runtime.MemStats runtime.GC() From 0d4fa98aca17dd92ce7388543fb47eb72a458ed8 Mon Sep 17 00:00:00 2001 From: ostefani Date: Wed, 18 Nov 2026 03:22:03 -0800 Subject: [PATCH 2/4] fix(test): measure closure drain from committed briefings - use the final seeded briefing notification to mark drain completion - attribute relay response-header timing to each event - bound briefing worker cleanup and preserve the 50 ms ingest gate --- internal/server/server_test.go | 115 +++++++++++++++++++++------------ 1 file changed, 75 insertions(+), 40 deletions(-) diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 892be75..fe79384 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -2,6 +2,7 @@ package server import ( "bytes" + "cmp" "context" "crypto/hmac" "crypto/sha256" @@ -17,6 +18,7 @@ import ( "os" "path/filepath" "runtime" + "slices" "sort" "strconv" "strings" @@ -940,17 +942,6 @@ func seedClosedGroups(t *testing.T, h *harness, n int) []string { return ids } -func seededDone(t *testing.T, h *harness, ids []string) bool { - t.Helper() - for _, id := range ids { - st, err := h.srv.events.GroupStatus(context.Background(), id) - if err != nil || st != "done" { - return false - } - } - return true -} - func briefingDeps(t *testing.T, h *harness) briefing.Deps { t.Helper() return briefing.Deps{ @@ -970,41 +961,74 @@ func briefingDeps(t *testing.T, h *harness) briefing.Deps { type relayTimingTransport struct { base http.RoundTripper - elapsed time.Duration + mu sync.Mutex + byEvent map[string]time.Duration } func (t *relayTimingTransport) RoundTrip(req *http.Request) (*http.Response, error) { start := time.Now() res, err := t.base.RoundTrip(req) - t.elapsed += time.Since(start) + elapsed := time.Since(start) + t.mu.Lock() + t.byEvent[req.Header.Get("X-SubnetLens-Event-ID")] += elapsed + t.mu.Unlock() return res, err } +func (t *relayTimingTransport) take(eventID string) time.Duration { + t.mu.Lock() + defer t.mu.Unlock() + elapsed := t.byEvent[eventID] + delete(t.byEvent, eventID) + return elapsed +} + func TestClosureDrainBudgets(t *testing.T) { h := newHarness(t, func(c *config.Config) { c.Mode = config.ModeProxy }) now := time.Now() seeded := seedClosedGroups(t, h, 256) - // Only the serial ingest loop uses this client. Keep timing with - // each request so a slow ingest can be matched to its relay call. + // Install before starting either goroutine. The test uses 100 distinct + // event IDs and consumes each timing after the request returns. base := h.srv.proxy.Transport if base == nil { base = http.DefaultTransport } - relay := &relayTimingTransport{base: base} + relay := &relayTimingTransport{base: base, byEvent: make(map[string]time.Duration)} h.srv.proxy.Transport = relay - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(t.Context()) deps := briefingDeps(t, h) + pending := make(map[string]struct{}, len(seeded)) + for _, id := range seeded { + pending[id] = struct{}{} + } + drainDone := make(chan time.Time, 1) + deps.OnBriefing = func(b store.Briefing) { + if _, ok := pending[b.GroupID]; ok { + delete(pending, b.GroupID) + if len(pending) == 0 { + drainDone <- time.Now() + } + } + } + runDone := make(chan error, 1) - start := time.Now() - go func() { runDone <- briefing.Run(ctx, deps) }() - defer func() { + t.Cleanup(func() { cancel() - if err := <-runDone; err != nil { - t.Errorf("briefing worker: %v", err) + select { + case err := <-runDone: + if err != nil { + t.Errorf("briefing worker: %v", err) + } + case <-time.After(5 * time.Second): + t.Error("briefing worker did not stop after cancellation") } - }() + }) + + const drainBudget = 5 * time.Second + start := time.Now() + go func() { runDone <- briefing.Run(ctx, deps) }() // Pace requests at up to 50/s and keep sampling after the drain, // which can finish well before the 100-request workload ends. @@ -1025,10 +1049,11 @@ func TestClosureDrainBudgets(t *testing.T) { var res ingestResult for i := 0; i < 100; i++ { <-tick.C - relay.elapsed = 0 reqStart := time.Now() - rec := h.signedPost(t, uniqueID(), "host-bg", "host.appeared", now) - res.samples = append(res.samples, sample{reqStart, time.Since(reqStart), relay.elapsed}) + id := uniqueID() + rec := h.signedPost(t, id, "host-bg", "host.appeared", now) + elapsed := time.Since(reqStart) + res.samples = append(res.samples, sample{started: reqStart, total: elapsed, relay: relay.take(id)}) switch rec.Code { case http.StatusAccepted: case http.StatusServiceUnavailable: @@ -1040,19 +1065,27 @@ func TestClosureDrainBudgets(t *testing.T) { ingestDone <- res }() - for time.Since(start) < 5*time.Second && !seededDone(t, h, seeded) { - time.Sleep(10 * time.Millisecond) + var drainCommitted time.Time + timer := time.NewTimer(time.Until(start.Add(drainBudget))) + defer timer.Stop() + select { + case drainCommitted = <-drainDone: + case <-timer.C: + // The observer may have published before the deadline even if + // both channels became ready before this goroutine was scheduled. + select { + case drainCommitted = <-drainDone: + default: + } } - drainObserved := time.Now() res := <-ingestDone - if elapsed := drainObserved.Sub(start); elapsed > 5*time.Second { - t.Fatalf("256 seeded groups not observed done within 5s (polled for %v)", elapsed) + if drainCommitted.IsZero() || drainCommitted.Sub(start) > drainBudget { + t.Fatalf("%d seeded groups did not finish within %v", len(seeded), drainBudget) } - t.Logf("256-group drain observed in %v (budget 5s)", drainObserved.Sub(start)) + t.Logf("%d-group drain completed in %v (budget %v)", len(seeded), drainCommitted.Sub(start), drainBudget) var during, after []sample for _, req := range res.samples { - // Polling gives an upper bound on when the drain completed. - if req.started.Before(drainObserved) { + if req.started.Before(drainCommitted) { during = append(during, req) } else { after = append(after, req) @@ -1063,17 +1096,19 @@ func TestClosureDrainBudgets(t *testing.T) { t.Logf("%s: 0 requests", label) return 0 } - sort.Slice(samples, func(a, b int) bool { return samples[a].total < samples[b].total }) + slices.SortFunc(samples, func(a, b sample) int { return cmp.Compare(a.total, b.total) }) // Nearest rank, converted from a one-based rank to a slice index. p99 := samples[(99*len(samples)+99)/100-1] - max := samples[len(samples)-1] - t.Logf("%s: %d requests, p99 %v (relay %v), max %v (relay %v)", - label, len(samples), p99.total, p99.relay, max.total, max.relay) + slowest := samples[len(samples)-1] + t.Logf("%s: %d requests, p99 %v (relay to headers %v), max %v (relay to headers %v)", + label, len(samples), p99.total, p99.relay, slowest.total, slowest.relay) return p99.total } + // The drain can finish in a few ticker intervals. Keep the gate + // over the full workload and report the small drain cohort separately. p99 := report("all proxy ingest", res.samples) - report("started before drain completion was observed", during) - report("started after drain completion was observed", after) + report("started before the final seeded briefing committed", during) + report("started after the final seeded briefing committed", after) if p99 > 50*time.Millisecond { t.Errorf("proxy ingest p99 %v exceeds 50ms", p99) } From 0a507687cd341f24d91a93e24d1219f535ce8ec5 Mon Sep 17 00:00:00 2001 From: ostefani Date: Wed, 18 Nov 2026 03:28:46 -0800 Subject: [PATCH 3/4] fix(test): detect missing relay timings in closure drain checks - require an event-correlated relay timing for accepted proxy requests - use the test context to cancel the briefing worker before cleanup - identify the full workload in p99 failures --- internal/server/server_test.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/internal/server/server_test.go b/internal/server/server_test.go index fe79384..7222d78 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -975,12 +975,12 @@ func (t *relayTimingTransport) RoundTrip(req *http.Request) (*http.Response, err return res, err } -func (t *relayTimingTransport) take(eventID string) time.Duration { +func (t *relayTimingTransport) take(eventID string) (time.Duration, bool) { t.mu.Lock() defer t.mu.Unlock() - elapsed := t.byEvent[eventID] + elapsed, ok := t.byEvent[eventID] delete(t.byEvent, eventID) - return elapsed + return elapsed, ok } func TestClosureDrainBudgets(t *testing.T) { @@ -997,7 +997,7 @@ func TestClosureDrainBudgets(t *testing.T) { relay := &relayTimingTransport{base: base, byEvent: make(map[string]time.Duration)} h.srv.proxy.Transport = relay - ctx, cancel := context.WithCancel(t.Context()) + ctx := t.Context() deps := briefingDeps(t, h) pending := make(map[string]struct{}, len(seeded)) for _, id := range seeded { @@ -1015,7 +1015,6 @@ func TestClosureDrainBudgets(t *testing.T) { runDone := make(chan error, 1) t.Cleanup(func() { - cancel() select { case err := <-runDone: if err != nil { @@ -1053,9 +1052,13 @@ func TestClosureDrainBudgets(t *testing.T) { id := uniqueID() rec := h.signedPost(t, id, "host-bg", "host.appeared", now) elapsed := time.Since(reqStart) - res.samples = append(res.samples, sample{started: reqStart, total: elapsed, relay: relay.take(id)}) + relayElapsed, forwarded := relay.take(id) + res.samples = append(res.samples, sample{started: reqStart, total: elapsed, relay: relayElapsed}) switch rec.Code { case http.StatusAccepted: + if !forwarded { + t.Errorf("accepted proxy request %s has no relay timing for its event ID", id) + } case http.StatusServiceUnavailable: res.saturated++ default: @@ -1110,7 +1113,7 @@ func TestClosureDrainBudgets(t *testing.T) { report("started before the final seeded briefing committed", during) report("started after the final seeded briefing committed", after) if p99 > 50*time.Millisecond { - t.Errorf("proxy ingest p99 %v exceeds 50ms", p99) + t.Errorf("proxy ingest p99 across all %d requests was %v (budget 50ms)", len(res.samples), p99) } if res.saturated != 0 || res.lost != 0 { t.Errorf("ingest loss: saturated=%d lost=%d", res.saturated, res.lost) From e60760b65517f59a50c430f17c0b4fbea5da035e Mon Sep 17 00:00:00 2001 From: ostefani Date: Wed, 18 Nov 2026 03:33:58 -0800 Subject: [PATCH 4/4] fix(test): aggregate missing relay timings in drain checks - report the count and first affected event for missing relay timings - document worker callback ownership and test cancellation order --- internal/server/server_test.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 7222d78..dceda2b 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -997,6 +997,7 @@ func TestClosureDrainBudgets(t *testing.T) { relay := &relayTimingTransport{base: base, byEvent: make(map[string]time.Duration)} h.srv.proxy.Transport = relay + // The test cancels this context before cleanup waits for the worker. ctx := t.Context() deps := briefingDeps(t, h) pending := make(map[string]struct{}, len(seeded)) @@ -1004,6 +1005,7 @@ func TestClosureDrainBudgets(t *testing.T) { pending[id] = struct{}{} } drainDone := make(chan time.Time, 1) + // OnBriefing runs on the worker goroutine, which owns pending. deps.OnBriefing = func(b store.Briefing) { if _, ok := pending[b.GroupID]; ok { delete(pending, b.GroupID) @@ -1037,9 +1039,11 @@ func TestClosureDrainBudgets(t *testing.T) { relay time.Duration } type ingestResult struct { - samples []sample - lost int - saturated int + samples []sample + lost int + saturated int + missingRelay int + firstMissingRelay string } ingestDone := make(chan ingestResult, 1) go func() { @@ -1057,7 +1061,10 @@ func TestClosureDrainBudgets(t *testing.T) { switch rec.Code { case http.StatusAccepted: if !forwarded { - t.Errorf("accepted proxy request %s has no relay timing for its event ID", id) + if res.missingRelay == 0 { + res.firstMissingRelay = id + } + res.missingRelay++ } case http.StatusServiceUnavailable: res.saturated++ @@ -1082,6 +1089,9 @@ func TestClosureDrainBudgets(t *testing.T) { } } res := <-ingestDone + if res.missingRelay > 0 { + t.Errorf("%d accepted proxy requests missing relay timing (first event ID: %s)", res.missingRelay, res.firstMissingRelay) + } if drainCommitted.IsZero() || drainCommitted.Sub(start) > drainBudget { t.Fatalf("%d seeded groups did not finish within %v", len(seeded), drainBudget) }