From 27c827cdb6a6cdfe833a9bcfc1515f2c11a8f6fd Mon Sep 17 00:00:00 2001 From: Francisco Delmar Kurpiel Date: Tue, 27 Oct 2026 18:34:16 +0100 Subject: [PATCH 1/8] Dequeue closed PRs in processPR instead of suspending them --- internal/mergequeue/queue.go | 9 ++++++ internal/mergequeue/queue_processpr.go | 5 ++- internal/mergequeue/queue_processpr_test.go | 13 ++++++++ internal/mergequeue/queue_test.go | 35 +++++++++++++++++++++ 4 files changed, 61 insertions(+), 1 deletion(-) diff --git a/internal/mergequeue/queue.go b/internal/mergequeue/queue.go index c47e951..f5e575c 100644 --- a/internal/mergequeue/queue.go +++ b/internal/mergequeue/queue.go @@ -586,6 +586,15 @@ func (q *queue) processPR(ctx context.Context, pr *PullRequest, task Task) { logger.Info("pull request suspended", logfields.Reason(reason)) + case ActionDequeue: + if _, err := q.Dequeue(pr.Number, false); err != nil { + logger.Error("dequeuing PR failed", zap.Error(err)) + continue + } + + metrics.DequeueOpsInc(&q.baseBranch.BranchID) + logger.Info("pull request removed from merge queue", logfields.Reason(reason)) + case ActionCreatePullRequestComment: // use a shorter timeout to prevent that this optional operation blocks for long ctx, cancelFunc := context.WithTimeout(ctx, postCommentTimeout) diff --git a/internal/mergequeue/queue_processpr.go b/internal/mergequeue/queue_processpr.go index b24601d..c8baf72 100644 --- a/internal/mergequeue/queue_processpr.go +++ b/internal/mergequeue/queue_processpr.go @@ -20,6 +20,7 @@ const ( ActionUndefined Action = iota ActionNone ActionSuspend + ActionDequeue ActionCreatePullRequestComment ActionUpdateStateUnchanged ActionCreateSuccessfulGithubStatus @@ -45,6 +46,8 @@ func (a Action) String() string { return "none" case ActionSuspend: return "suspend" + case ActionDequeue: + return "dequeue" case ActionCreatePullRequestComment: return "create pull request comment" case ActionUpdateStateUnchanged: @@ -95,7 +98,7 @@ func (q *queue) evalPRAction(ctx context.Context, logger *zap.Logger, pr *PullRe if err != nil { if errors.Is(err, ErrPullRequestIsClosed) { return &requiredActions{ - Actions: []Action{ActionSuspend}, + Actions: []Action{ActionDequeue}, Reason: reasonPullRequestClosed, }, nil } diff --git a/internal/mergequeue/queue_processpr_test.go b/internal/mergequeue/queue_processpr_test.go index f3944ec..6adb5e8 100644 --- a/internal/mergequeue/queue_processpr_test.go +++ b/internal/mergequeue/queue_processpr_test.go @@ -191,6 +191,19 @@ func TestEvalPRAction_PendingCIJobs(t *testing.T) { assert.Contains(t, reqActions.Actions, ActionCreateSuccessfulGithubStatus) } +func TestEvalPRAction_DequeueActionOnClosedPR(t *testing.T) { + prt := initEvalPRActionTest(t) + prt.GhClt.EXPECT(). + UpdateBranch(gomock.Any(), gomock.Eq(repoOwner), gomock.Eq(repo), gomock.Eq(prNR)). + Return(nil, githubclt.ErrPullRequestIsClosed). + Times(1) + + reqActions, err := prt.Q.evalPRAction(context.Background(), prt.L, prt.PR) + require.NoError(t, err) + require.NotNil(t, reqActions) + require.Equal(t, []Action{ActionDequeue}, reqActions.Actions) +} + func TestCITriggerNeeded(t *testing.T) { tcs := []struct { name string diff --git a/internal/mergequeue/queue_test.go b/internal/mergequeue/queue_test.go index 829f681..3f74b5c 100644 --- a/internal/mergequeue/queue_test.go +++ b/internal/mergequeue/queue_test.go @@ -46,6 +46,41 @@ func TestUpdatePR_DoesNotCallBaseBranchUpdateIfPRIsNotApproved(t *testing.T) { q.processPR(context.Background(), pr, TaskNone) } +func TestProcessPR_DequeuesClosedPRWithoutSettingPendingStatus(t *testing.T) { + t.Cleanup(zap.ReplaceGlobals(zaptest.NewLogger(t).Named(t.Name()))) + + mockctrl := gomock.NewController(t) + ghClient := mocks.NewMockGithubClient(mockctrl) + + bb, err := NewBaseBranch(repoOwner, repo, "main") + require.NoError(t, err) + q := newQueue(bb, zap.L(), ghClient, retry.NewRetryer(), nil, queueHeadLabel) + t.Cleanup(q.Stop) + + pr, err := NewPullRequest(prNR, prBranch, "fho", "test pr", "") + require.NoError(t, err) + + _, added := q.active.InsertIfNotExist(pr.Number, pr) + require.True(t, added) + + mockReadyForMergeStatus( + ghClient, pr.Number, + githubclt.ReviewDecisionApproved, githubclt.CIStatusSuccess, + ).Times(1) + ghClient.EXPECT(). + UpdateBranch(gomock.Any(), gomock.Eq(repoOwner), gomock.Eq(repo), gomock.Eq(pr.Number)). + Return(nil, githubclt.ErrPullRequestIsClosed). + Times(1) + mockSuccessfulGithubRemoveLabelQueueHeadCall(ghClient, pr.Number).Times(1) + ghClient.EXPECT(). + CreateHeadCommitStatus(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Times(0) + + q.processPR(context.Background(), pr, TaskNone) + assert.Equal(t, 0, q.activeLen()) + assert.Equal(t, 0, q.suspendedLen()) +} + func TestUpdatePRWithBaseReturnsChangedWhenScheduled(t *testing.T) { t.Cleanup(zap.ReplaceGlobals(zaptest.NewLogger(t).Named(t.Name()))) From 810b3efb971578b438a1636c10bf5925afcdf876 Mon Sep 17 00:00:00 2001 From: Francisco Delmar Kurpiel Date: Tue, 27 Oct 2026 18:42:14 +0100 Subject: [PATCH 2/8] Fix inverted error check and nil dereference in processCheckRunEvent --- internal/mergequeue/coordinator.go | 27 ++++++++-------- internal/mergequeue/coordinator_test.go | 41 +++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/internal/mergequeue/coordinator.go b/internal/mergequeue/coordinator.go index 702a6c2..a1c8aa2 100644 --- a/internal/mergequeue/coordinator.go +++ b/internal/mergequeue/coordinator.go @@ -777,31 +777,32 @@ func (a *Coordinator) processCheckRunEvent(ctx context.Context, logger *zap.Logg baseBranch := pr.GetBase().GetRef() prNumber := pr.GetNumber() - logger = logger.With( + prLogger := logger.With( logfields.BaseBranch(baseBranch), logfields.PullRequest(prNumber), ) bb, err := NewBaseBranch(owner, repo, baseBranch) if err != nil { - logger.Warn( + prLogger.Warn( "ignoring check run event for pull-request, incomplete base branch information", zap.Error(err), ) + continue + } - err := a.SetPRStaleSinceIfNewer(ctx, bb, pr.GetNumber(), time.Now()) - if err != nil { - if errors.Is(err, ErrNotFound) { - logger.Debug( - "pr not queued, can not update stale timestamp", - ) - continue - } - logger.Error( - "updating stale timestamp failed", - zap.Error(err), + err = a.SetPRStaleSinceIfNewer(ctx, bb, prNumber, time.Now()) + if err != nil { + if errors.Is(err, ErrNotFound) { + prLogger.Debug( + "pr not queued, can not update stale timestamp", ) + continue } + prLogger.Error( + "updating stale timestamp failed", + zap.Error(err), + ) } } diff --git a/internal/mergequeue/coordinator_test.go b/internal/mergequeue/coordinator_test.go index c946df1..76daef5 100644 --- a/internal/mergequeue/coordinator_test.go +++ b/internal/mergequeue/coordinator_test.go @@ -2162,3 +2162,44 @@ func TestCIJobsRetriedWhenTriggerFails(t *testing.T) { waitForQueueUpdateRunsGreaterThan(t, queue, 2) assert.EqualValues(t, 2, ciBuildCalls.Load(), "CI jobs must not be triggered again after succeeding") } + +func TestCheckRunEventUpdatesStaleTimestampAndHandlesMissingBaseBranch(t *testing.T) { + l := forwardLogsToTestLogger(t) + + evChan := make(chan *github_prov.Event, 1) + t.Cleanup(func() { close(evChan) }) + mockctrl := gomock.NewController(t) + ghClient := mocks.NewMockGithubClient(mockctrl) + ciClient := mocks.NewMockJenkinsClient(mockctrl) + + autoupdater := newAutoupdater( + ghClient, + ciClient, + evChan, + []Repository{{OwnerLogin: repoOwner, RepositoryName: repo}}, + true, + nil, + ) + + bb, err := NewBaseBranch(repoOwner, repo, "master") + require.NoError(t, err) + + pr, err := NewPullRequest(1, "pr_branch1", "me", "test pr", "") + require.NoError(t, err) + initialTime := time.Now().Add(-time.Hour) + pr.SetStateUnchangedSince(initialTime) + + q := newQueue(bb, l, ghClient, retry.NewRetryer(), autoupdater.CI, queueHeadLabel) + t.Cleanup(q.Stop) + _, added := q.active.InsertIfNotExist(pr.Number, pr) + require.True(t, added) + autoupdater.queues[bb.BranchID] = q + + ev := newCheckRunEvent("stale", "", "pr_branch1") + ev.CheckRun.PullRequests = append(ev.CheckRun.PullRequests, newBasicPullRequest(2, "", "pr_branch2")) + + require.NotPanics(t, func() { + autoupdater.processCheckRunEvent(context.Background(), l, ev) + }) + assert.True(t, pr.GetStateUnchangedSince().After(initialTime), "expected check_run event to update stateUnchangedSince timestamp") +} From c8f91c33c422c134beeb65e13db8789672c499da Mon Sep 17 00:00:00 2001 From: Francisco Delmar Kurpiel Date: Tue, 27 Oct 2026 19:01:19 +0100 Subject: [PATCH 3/8] Log ignored check_run/status events at debug level and fix SetPRStaleSinceIfNewerByBranch --- internal/mergequeue/coordinator.go | 11 +++-- internal/mergequeue/coordinator_test.go | 54 +++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/internal/mergequeue/coordinator.go b/internal/mergequeue/coordinator.go index a1c8aa2..4a5e35e 100644 --- a/internal/mergequeue/coordinator.go +++ b/internal/mergequeue/coordinator.go @@ -768,7 +768,7 @@ func (a *Coordinator) processCheckRunEvent(ctx context.Context, logger *zap.Logg logger.Debug("event received") if len(branches) == 0 { - logger.Info("ignoring event, pull request or branches fields is empty") + logger.Debug("ignoring event, pull request or branches fields is empty") return } @@ -849,7 +849,7 @@ func (a *Coordinator) processStatusEvent(ctx context.Context, logger *zap.Logger logger.Debug("event received") if len(branches) == 0 { - logger.Info("ignorning event, pull request or branches field is empty") + logger.Debug("ignoring event, pull request or branches field is empty") return } @@ -986,7 +986,12 @@ func (a *Coordinator) SetPRStaleSinceIfNewerByBranch( continue } - missing = q.SetPRStaleSinceIfNewerByBranch(branchNames, updatedAt) + notFoundInQueue := q.SetPRStaleSinceIfNewerByBranch(branchNames, updatedAt) + for b := range missing { + if !notFoundInQueue.Contains(b) { + delete(missing, b) + } + } } return missing.Slice() diff --git a/internal/mergequeue/coordinator_test.go b/internal/mergequeue/coordinator_test.go index 76daef5..2c15dee 100644 --- a/internal/mergequeue/coordinator_test.go +++ b/internal/mergequeue/coordinator_test.go @@ -2203,3 +2203,57 @@ func TestCheckRunEventUpdatesStaleTimestampAndHandlesMissingBaseBranch(t *testin }) assert.True(t, pr.GetStateUnchangedSince().After(initialTime), "expected check_run event to update stateUnchangedSince timestamp") } + +func TestSetPRStaleSinceIfNewerByBranchMultipleQueues(t *testing.T) { + l := forwardLogsToTestLogger(t) + + evChan := make(chan *github_prov.Event, 1) + t.Cleanup(func() { close(evChan) }) + mockctrl := gomock.NewController(t) + ghClient := mocks.NewMockGithubClient(mockctrl) + ciClient := mocks.NewMockJenkinsClient(mockctrl) + + autoupdater := newAutoupdater( + ghClient, + ciClient, + evChan, + []Repository{{OwnerLogin: repoOwner, RepositoryName: repo}}, + true, + nil, + ) + + bbMaster, err := NewBaseBranch(repoOwner, repo, "master") + require.NoError(t, err) + bbRelease, err := NewBaseBranch(repoOwner, repo, "release") + require.NoError(t, err) + + pr1, err := NewPullRequest(1, "pr_branch1", "me", "test pr 1", "") + require.NoError(t, err) + pr2, err := NewPullRequest(2, "pr_branch2", "me", "test pr 2", "") + require.NoError(t, err) + + qMaster := newQueue(bbMaster, l, ghClient, retry.NewRetryer(), autoupdater.CI, queueHeadLabel) + t.Cleanup(qMaster.Stop) + _, added := qMaster.active.InsertIfNotExist(pr1.Number, pr1) + require.True(t, added) + autoupdater.queues[bbMaster.BranchID] = qMaster + + qRelease := newQueue(bbRelease, l, ghClient, retry.NewRetryer(), autoupdater.CI, queueHeadLabel) + t.Cleanup(qRelease.Stop) + _, added = qRelease.active.InsertIfNotExist(pr2.Number, pr2) + require.True(t, added) + autoupdater.queues[bbRelease.BranchID] = qRelease + + updatedAt := time.Now().Add(time.Minute) + notFound := autoupdater.SetPRStaleSinceIfNewerByBranch( + context.Background(), + repoOwner, + repo, + []string{"pr_branch1", "pr_branch2", "unqueued_branch"}, + updatedAt, + ) + + assert.Equal(t, []string{"unqueued_branch"}, notFound) + assert.Equal(t, updatedAt, pr1.GetStateUnchangedSince()) + assert.Equal(t, updatedAt, pr2.GetStateUnchangedSince()) +} From bd5bd27b005954e410a8cede9b32655ff9b0ae42 Mon Sep 17 00:00:00 2001 From: Francisco Delmar Kurpiel Date: Tue, 27 Oct 2026 19:06:17 +0100 Subject: [PATCH 4/8] mergequeue: avoid accumulating PR log fields across sync loop --- internal/mergequeue/sync.go | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/internal/mergequeue/sync.go b/internal/mergequeue/sync.go index 9b382f4..4bb84bc 100644 --- a/internal/mergequeue/sync.go +++ b/internal/mergequeue/sync.go @@ -71,24 +71,26 @@ func (a *Coordinator) sync(ctx context.Context, owner, repo string) error { stats.Seen++ actions := a.evaluateActions(ghPR) - if len(actions) > 0 { - stats.PRsOutOfSync++ + if len(actions) == 0 { + continue } - for _, action := range actions { - pr, err := NewPullRequest(ghPR.Number, ghPR.Branch, ghPR.Author, ghPR.Title, ghPR.Link) - if err != nil { - return fmt.Errorf("pr information retrieved from github is incomplete: %w", err) - } - pr.SetAssignees(ghPR.Assignees) + stats.PRsOutOfSync++ + + pr, err := NewPullRequest(ghPR.Number, ghPR.Branch, ghPR.Author, ghPR.Title, ghPR.Link) + if err != nil { + return fmt.Errorf("pr information retrieved from github is incomplete: %w", err) + } + pr.SetAssignees(ghPR.Assignees) - logger = logger.With(pr.LogFields...) + prLogger := logger.With(pr.LogFields...) + for _, action := range actions { switch action { case unlabel: err := a.removeLabel(ctx, owner, repo, pr) if err != nil { - logger.Warn( + prLogger.Warn( "removing pull request label failed", zap.Error(err), ) @@ -99,19 +101,19 @@ func (a *Coordinator) sync(ctx context.Context, owner, repo string) error { case enqueue: err := a.enqueuePR(ctx, owner, repo, ghPR.BaseBranch, pr) if errors.Is(err, ErrAlreadyExists) { - logger.Debug("queue in-sync, pr is already enqueued") + prLogger.Debug("queue in-sync, pr is already enqueued") break } if err != nil { stats.Failures++ - logger.Warn("adding pr to queue failed", + prLogger.Warn("adding pr to queue failed", zap.Error(err), ) break } stats.Enqueued++ - logger.Info("enqueued pr") + prLogger.Info("enqueued pr") case resetStatusState: err := createCommitStatus(ctx, @@ -120,14 +122,14 @@ func (a *Coordinator) sync(ctx context.Context, owner, repo string) error { githubclt.StatusStatePending, ) if err != nil { - logger.Warn("setting github status state failed", zap.Error(err)) + prLogger.Warn("setting github status state failed", zap.Error(err)) continue } stats.StatusStateReset++ default: - logger.DPanic( + prLogger.DPanic( "evaluateActions() returned unexpected value", zap.Int("value", int(action)), ) From d736e86bff0420e1fe9a61313df5883e32ccaf8b Mon Sep 17 00:00:00 2001 From: Francisco Delmar Kurpiel Date: Tue, 27 Oct 2026 19:21:11 +0100 Subject: [PATCH 5/8] mergequeue: deduplicate AddLabel calls and fix error handling in label helpers --- internal/mergequeue/coordinator_test.go | 2 +- internal/mergequeue/pullrequest.go | 3 ++ internal/mergequeue/queue.go | 18 +++++++ internal/mergequeue/queue_test.go | 70 +++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 1 deletion(-) diff --git a/internal/mergequeue/coordinator_test.go b/internal/mergequeue/coordinator_test.go index 2c15dee..7289b42 100644 --- a/internal/mergequeue/coordinator_test.go +++ b/internal/mergequeue/coordinator_test.go @@ -1884,7 +1884,7 @@ func TestPauseResumeQueue(t *testing.T) { require.NoError(t, err) mockSuccessfulGithubUpdateBranchCall(ghClient, pr.Number, false).Times(4) - mockSuccessfulGithubAddLabelQueueHeadCall(ghClient, pr.Number).Times(4) + mockSuccessfulGithubAddLabelQueueHeadCall(ghClient, pr.Number).Times(1) mockReadyForMergeStatus( ghClient, pr.Number, githubclt.ReviewDecisionApproved, githubclt.CIStatusExpected, diff --git a/internal/mergequeue/pullrequest.go b/internal/mergequeue/pullrequest.go index cc9de5e..54506ec 100644 --- a/internal/mergequeue/pullrequest.go +++ b/internal/mergequeue/pullrequest.go @@ -52,6 +52,9 @@ type PullRequest struct { GithubStatusLastSetState ReportedStatusState GithubStatusLock sync.Mutex + + hasHeadLabel bool + headLabelLock sync.Mutex } type ReportedStatusState struct { diff --git a/internal/mergequeue/queue.go b/internal/mergequeue/queue.go index f5e575c..d3aa6ff 100644 --- a/internal/mergequeue/queue.go +++ b/internal/mergequeue/queue.go @@ -765,6 +765,15 @@ func (q *queue) prsByBranch(branchNames set.Set[string]) ( } func (q *queue) prAddQueueHeadLabel(ctx context.Context, pr *PullRequest) { + pr.headLabelLock.Lock() + defer pr.headLabelLock.Unlock() + + if pr.hasHeadLabel { + q.logger.Debug("skipping adding queue head label, label was already added", + logfields.NewWith(pr.LogFields, zap.String("github_label", q.headLabel))...) + return + } + ctx, cancelFunc := context.WithTimeout(ctx, operationTimeout) defer cancelFunc() err := q.retryer.Run(ctx, func(ctx context.Context) error { @@ -784,13 +793,21 @@ func (q *queue) prAddQueueHeadLabel(ctx context.Context, pr *PullRequest) { zap.Error(err), zap.String("github_label", q.headLabel), )...) + return } + pr.hasHeadLabel = true + q.logger.Info("queue head label was added to pr", logfields.NewWith(pr.LogFields, zap.String("github_label", q.headLabel))...) } func (q *queue) prRemoveQueueHeadLabel(ctx context.Context, logReason string, pr *PullRequest) { + pr.headLabelLock.Lock() + defer pr.headLabelLock.Unlock() + + pr.hasHeadLabel = false + ctx, cancelFunc := context.WithTimeout(ctx, operationTimeout) defer cancelFunc() err := q.retryer.Run(ctx, func(ctx context.Context) error { @@ -807,6 +824,7 @@ func (q *queue) prRemoveQueueHeadLabel(ctx context.Context, logReason string, pr zap.String("github_label", q.headLabel), logfields.Reason(logReason), }, pr.LogFields...)...) + return } q.logger.Info("queue head label was removed from pr", append([]zapcore.Field{ diff --git a/internal/mergequeue/queue_test.go b/internal/mergequeue/queue_test.go index 3f74b5c..8216233 100644 --- a/internal/mergequeue/queue_test.go +++ b/internal/mergequeue/queue_test.go @@ -1,7 +1,9 @@ package mergequeue import ( + "bytes" "context" + "errors" "testing" "time" @@ -9,6 +11,7 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" "go.uber.org/zap" + "go.uber.org/zap/zapcore" "go.uber.org/zap/zaptest" "github.com/simplesurance/directorius/internal/githubclt" @@ -257,3 +260,70 @@ func TestOrderBefore_OlderEnqueuedAtTimeFirst(t *testing.T) { require.Equal(t, 1, orderBefore(pr1, pr2)) } + +func TestQueueHeadLabelAddedOnlyOnceAndRetriedOnError(t *testing.T) { + var logBuf bytes.Buffer + core := zapcore.NewCore( + zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()), + zapcore.AddSync(&logBuf), + zapcore.InfoLevel, + ) + logger := zap.New(core) + + mockctrl := gomock.NewController(t) + ghClient := mocks.NewMockGithubClient(mockctrl) + + bb, err := NewBaseBranch(repoOwner, repo, "main") + require.NoError(t, err) + q := newQueue(bb, logger, ghClient, retry.NewRetryer(), nil, queueHeadLabel) + t.Cleanup(q.Stop) + + pr, err := NewPullRequest(prNR, prBranch, "fho", "test pr", "") + require.NoError(t, err) + + gomock.InOrder( + ghClient.EXPECT(). + AddLabel(gomock.Any(), gomock.Eq(repoOwner), gomock.Eq(repo), gomock.Eq(pr.Number), gomock.Eq(queueHeadLabel)). + Return(errors.New("transient github error")). + Times(1), + ghClient.EXPECT(). + AddLabel(gomock.Any(), gomock.Eq(repoOwner), gomock.Eq(repo), gomock.Eq(pr.Number), gomock.Eq(queueHeadLabel)). + Return(nil). + Times(1), + ghClient.EXPECT(). + RemoveLabel(gomock.Any(), gomock.Eq(repoOwner), gomock.Eq(repo), gomock.Eq(pr.Number), gomock.Eq(queueHeadLabel)). + Return(errors.New("transient remove error")). + Times(1), + ghClient.EXPECT(). + AddLabel(gomock.Any(), gomock.Eq(repoOwner), gomock.Eq(repo), gomock.Eq(pr.Number), gomock.Eq(queueHeadLabel)). + Return(nil). + Times(1), + ) + + // 1. First attempt fails: should log Warn and NOT log "queue head label was added to pr". + q.prAddQueueHeadLabel(context.Background(), pr) + assert.Contains(t, logBuf.String(), "adding label to PR failed") + assert.NotContains(t, logBuf.String(), "queue head label was added to pr") + logBuf.Reset() + + // 2. Second attempt succeeds: should call AddLabel and log Info once. + q.prAddQueueHeadLabel(context.Background(), pr) + assert.Contains(t, logBuf.String(), "queue head label was added to pr") + logBuf.Reset() + + // 3. Third call while label is already set: should skip calling GitHub AddLabel. + q.prAddQueueHeadLabel(context.Background(), pr) + assert.NotContains(t, logBuf.String(), "queue head label was added to pr") + logBuf.Reset() + + // 4. RemoveLabel fails: should log Warn, NOT log "queue head label was removed from pr", + // and reset hasHeadLabel so a subsequent AddLabel is not skipped. + q.prRemoveQueueHeadLabel(context.Background(), "dequeue", pr) + assert.Contains(t, logBuf.String(), "removing label from PR failed") + assert.NotContains(t, logBuf.String(), "queue head label was removed from pr") + logBuf.Reset() + + // 5. Subsequent AddLabel after removal calls GitHub again. + q.prAddQueueHeadLabel(context.Background(), pr) + assert.Contains(t, logBuf.String(), "queue head label was added to pr") +} From 146fc1a41657950e5364c258930c7b9843c5199e Mon Sep 17 00:00:00 2001 From: Francisco Delmar Kurpiel Date: Tue, 27 Oct 2026 19:35:34 +0100 Subject: [PATCH 6/8] mergequeue: preserve queue head position during startup sync --- internal/mergequeue/coordinator_test.go | 87 +++++++++++++++++++++++++ internal/mergequeue/sync.go | 46 ++++++++++++- 2 files changed, 131 insertions(+), 2 deletions(-) diff --git a/internal/mergequeue/coordinator_test.go b/internal/mergequeue/coordinator_test.go index 7289b42..eeb5f3f 100644 --- a/internal/mergequeue/coordinator_test.go +++ b/internal/mergequeue/coordinator_test.go @@ -2257,3 +2257,90 @@ func TestSetPRStaleSinceIfNewerByBranchMultipleQueues(t *testing.T) { assert.Equal(t, updatedAt, pr1.GetStateUnchangedSince()) assert.Equal(t, updatedAt, pr2.GetStateUnchangedSince()) } + +func TestInitialSyncPreservesQueueHeadAndOrdersByNumber(t *testing.T) { + t.Cleanup(zap.ReplaceGlobals(zaptest.NewLogger(t).Named(t.Name()))) + + evChan := make(chan *github_prov.Event, 1) + defer close(evChan) + + mockctrl := gomock.NewController(t) + ghClient := mocks.NewMockGithubClient(mockctrl) + ciClient := mocks.NewMockJenkinsClient(mockctrl) + + mockSuccessfulGithubUpdateBranchCall(ghClient, 30, false).AnyTimes() + mockReadyForMergeStatus( + ghClient, 30, + githubclt.ReviewDecisionApproved, githubclt.CIStatusExpected, + ).AnyTimes() + mockCreateCommitStatusSuccessful(ghClient).Times(1) + ciClient. + EXPECT(). + Build(gomock.Any(), gomock.Any()). + AnyTimes() + mockGetBuildFromQueueItemID(ciClient).AnyTimes() + + // Simulate paginated updated-asc results where PR 20 is yielded on page 1, + // PR 10 on page 2, and PR 30 (the previous queue head carrying queueHeadLabel) + // on the last page. + syncPRRequestsRet := []*githubclt.PR{ + { + Number: 20, + AutoMergeEnabled: true, + Branch: "pr20", + BaseBranch: "main", + }, + { + Number: 10, + AutoMergeEnabled: true, + Branch: "pr10", + BaseBranch: "main", + }, + { + Number: 30, + AutoMergeEnabled: true, + Branch: "pr30", + BaseBranch: "main", + Labels: []string{queueHeadLabel}, + }, + } + + ghClient. + EXPECT(). + ListPRs(gomock.Any(), gomock.Eq(repoOwner), gomock.Eq(repo)). + Return( + func(yield func(*githubclt.PR, error) bool) { + for _, item := range syncPRRequestsRet { + if !yield(item, nil) { + return + } + } + }, + ).Times(1) + + autoupdater := newAutoupdater( + ghClient, + ciClient, + evChan, + []Repository{{OwnerLogin: repoOwner, RepositoryName: repo}}, + true, + nil, + ) + autoupdater.CI.Jobs = map[string]*jenkins.JobTemplate{ + ciJobExpectedName: {RelURL: "here"}, + } + + err := autoupdater.InitSync(context.Background()) + require.NoError(t, err) + t.Cleanup(autoupdater.Stop) + + q := autoupdater.getQueue(&BranchID{Repository: repo, RepositoryOwner: repoOwner, Branch: "main"}) + require.NotNil(t, q) + waitForQueueUpdateRunsGreaterThan(t, q, 0) + + activePRs, _ := q.asSlices() + require.Len(t, activePRs, 3) + assert.Equal(t, 30, activePRs[0].Number, "PR carrying head label before sync must remain at queue head") + assert.Equal(t, 10, activePRs[1].Number, "remaining PRs must be ordered by ascending PR number") + assert.Equal(t, 20, activePRs[2].Number, "remaining PRs must be ordered by ascending PR number") +} diff --git a/internal/mergequeue/sync.go b/internal/mergequeue/sync.go index 4bb84bc..86063b0 100644 --- a/internal/mergequeue/sync.go +++ b/internal/mergequeue/sync.go @@ -1,10 +1,12 @@ package mergequeue import ( + "cmp" "context" "errors" "fmt" "iter" + "slices" "time" "go.uber.org/zap" @@ -22,13 +24,23 @@ const ( resetStatusState ) +type syncItem struct { + ghPR *githubclt.PR + actions []syncAction +} + +func (s syncItem) isQueuedHead() bool { + return slices.Contains(s.actions, enqueue) && slices.Contains(s.actions, unlabel) +} + // InitSync does an initial synchronization of the autoupdater queues with the // pull request state at GitHub. // It must be run **before** Autoupdater is started. // Pull request information is queried from github. // If a PR meets a condition to be enqueued for auto-updates it is enqueued. // If it meets a condition for not being automatically updated, it is dequeued. -// If a PR has the [a.HeadLabel] it is removed, if it has a non-pending github +// A PR that has [a.HeadLabel] and is enqueued is placed at the head of its +// queue; otherwise [a.HeadLabel] is removed. If a PR has a non-pending github // status from directorius it is set to pending. func (a *Coordinator) InitSync(ctx context.Context) error { for repo := range a.MonitoredRepositories { @@ -52,6 +64,7 @@ func (a *Coordinator) sync(ctx context.Context, owner, repo string) error { defer stop() stats := syncStat{StartTime: time.Now()} + var items []syncItem hasMore := true for { var ghPR *githubclt.PR @@ -76,6 +89,26 @@ func (a *Coordinator) sync(ctx context.Context, owner, repo string) error { } stats.PRsOutOfSync++ + items = append(items, syncItem{ghPR: ghPR, actions: actions}) + } + + slices.SortFunc(items, func(x, y syncItem) int { + xHead := x.isQueuedHead() + yHead := y.isQueuedHead() + if xHead != yHead { + if xHead { + return -1 + } + return 1 + } + + return cmp.Compare(x.ghPR.Number, y.ghPR.Number) + }) + + headPreserved := map[string]bool{} + + for _, item := range items { + ghPR := item.ghPR pr, err := NewPullRequest(ghPR.Number, ghPR.Branch, ghPR.Author, ghPR.Title, ghPR.Link) if err != nil { @@ -83,11 +116,20 @@ func (a *Coordinator) sync(ctx context.Context, owner, repo string) error { } pr.SetAssignees(ghPR.Assignees) + preserveHeadLabel := item.isQueuedHead() && !headPreserved[ghPR.BaseBranch] + if preserveHeadLabel { + headPreserved[ghPR.BaseBranch] = true + pr.hasHeadLabel = true + } + prLogger := logger.With(pr.LogFields...) - for _, action := range actions { + for _, action := range item.actions { switch action { case unlabel: + if preserveHeadLabel { + continue + } err := a.removeLabel(ctx, owner, repo, pr) if err != nil { prLogger.Warn( From e03b7b947e84b1cdaf538d8827df291e5091b3b6 Mon Sep 17 00:00:00 2001 From: Francisco Delmar Kurpiel Date: Tue, 27 Oct 2026 19:48:48 +0100 Subject: [PATCH 7/8] mergequeue: log canceled processPR evaluation at debug level --- internal/mergequeue/queue.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/internal/mergequeue/queue.go b/internal/mergequeue/queue.go index d3aa6ff..69e02e4 100644 --- a/internal/mergequeue/queue.go +++ b/internal/mergequeue/queue.go @@ -548,15 +548,19 @@ func (q *queue) processPR(ctx context.Context, pr *PullRequest, task Task) { actions, err := q.evalPRAction(ctx, logger, pr) if err != nil { + if errors.Is(err, context.Canceled) { + logger.Debug( + "evaluating status of first pull request in the queue canceled", + zap.Error(err), + ) + return + } + logger.Error( "evaluating status of first pull request in the queue failed", zap.Error(err), ) - if errors.Is(err, context.Canceled) { - return - } - if err := q.suspend(pr.Number); err != nil { logger.Error("suspending PR failed", zap.Error(err)) return From b66d381e38ceab0a3a1789fa3146ab8c64d0a352 Mon Sep 17 00:00:00 2001 From: Francisco Delmar Kurpiel Date: Tue, 27 Oct 2026 20:21:42 +0100 Subject: [PATCH 8/8] mergequeue: address review feedback on sync and dequeue handling --- internal/mergequeue/coordinator.go | 15 ++++++++++++--- internal/mergequeue/coordinator_test.go | 11 ++++------- internal/mergequeue/httplistdata.go | 3 +++ internal/mergequeue/queue.go | 5 ++++- internal/mergequeue/queue_processpr.go | 2 ++ internal/mergequeue/sync.go | 24 ++++++++++++++++++++---- 6 files changed, 45 insertions(+), 15 deletions(-) diff --git a/internal/mergequeue/coordinator.go b/internal/mergequeue/coordinator.go index 4a5e35e..8cb3315 100644 --- a/internal/mergequeue/coordinator.go +++ b/internal/mergequeue/coordinator.go @@ -168,7 +168,13 @@ func (a *Coordinator) eventLoop() { case <-periodicTrigger.C: a.queuesLock.Lock() - for _, q := range a.queues { + for branchID, q := range a.queues { + if q.IsEmpty() { + q.Wait() + delete(a.queues, branchID) + a.Logger.With(q.baseBranch.Logfields...).Debug("removed empty queue for base branch") + continue + } q.ScheduleProcessPR(context.Background(), TaskNone) a.Logger.Debug("periodic run scheduled", q.baseBranch.Logfields...) } @@ -949,11 +955,14 @@ func (a *Coordinator) Dequeue(_ context.Context, baseBranch *BaseBranch, prNumbe pr, err := q.Dequeue(prNumber, setPendingStatusState) if err != nil { + if q.IsEmpty() { + q.Wait() + delete(a.queues, baseBranch.BranchID) + a.Logger.With(baseBranch.Logfields...).Debug("removed empty queue for base branch") + } return nil, fmt.Errorf("removing pr from merge queue failed: %w", err) } - metrics.DequeueOpsInc(&baseBranch.BranchID) - if q.IsEmpty() { q.Wait() delete(a.queues, baseBranch.BranchID) diff --git a/internal/mergequeue/coordinator_test.go b/internal/mergequeue/coordinator_test.go index eeb5f3f..925c5ff 100644 --- a/internal/mergequeue/coordinator_test.go +++ b/internal/mergequeue/coordinator_test.go @@ -2280,20 +2280,17 @@ func TestInitialSyncPreservesQueueHeadAndOrdersByNumber(t *testing.T) { AnyTimes() mockGetBuildFromQueueItemID(ciClient).AnyTimes() - // Simulate paginated updated-asc results where PR 20 is yielded on page 1, - // PR 10 on page 2, and PR 30 (the previous queue head carrying queueHeadLabel) - // on the last page. syncPRRequestsRet := []*githubclt.PR{ { - Number: 20, + Number: 10, AutoMergeEnabled: true, - Branch: "pr20", + Branch: "pr10", BaseBranch: "main", }, { - Number: 10, + Number: 20, AutoMergeEnabled: true, - Branch: "pr10", + Branch: "pr20", BaseBranch: "main", }, { diff --git a/internal/mergequeue/httplistdata.go b/internal/mergequeue/httplistdata.go index 335df55..731758a 100644 --- a/internal/mergequeue/httplistdata.go +++ b/internal/mergequeue/httplistdata.go @@ -51,6 +51,9 @@ func (a *Coordinator) httpListData() *pagestypes.ListData { } activePRs, suspendedPRs := queue.asSlices() + if len(activePRs) == 0 && len(suspendedPRs) == 0 { + continue + } // sort them PRs to show list in a stable order in the UI slices.SortFunc(suspendedPRs, orderBefore) diff --git a/internal/mergequeue/queue.go b/internal/mergequeue/queue.go index 69e02e4..b811998 100644 --- a/internal/mergequeue/queue.go +++ b/internal/mergequeue/queue.go @@ -312,6 +312,7 @@ func (q *queue) Dequeue(prNumber int, setPendingStatusState bool) (*PullRequest, logger.Debug("pull request removed from suspend queue") pr.SetStateUnchangedSince(time.Time{}) + metrics.DequeueOpsInc(&q.baseBranch.BranchID) return pr, nil } else if !errors.Is(err, ErrNotFound) { @@ -325,6 +326,7 @@ func (q *queue) Dequeue(prNumber int, setPendingStatusState bool) (*PullRequest, return nil, ErrNotFound } + metrics.DequeueOpsInc(&q.baseBranch.BranchID) q.cancelActionForPR(prNumber) removed.SetStateUnchangedSince(time.Time{}) @@ -583,6 +585,7 @@ func (q *queue) processPR(ctx context.Context, pr *PullRequest, task Task) { pr.SetStateUnchangedSinceIfNewer(time.Now()) case ActionSuspend: + // q.suspend calls q.cancelActionForPR, which cancels ctx. if err := q.suspend(pr.Number); err != nil { logger.Error("suspending PR failed", zap.Error(err)) continue @@ -591,12 +594,12 @@ func (q *queue) processPR(ctx context.Context, pr *PullRequest, task Task) { logger.Info("pull request suspended", logfields.Reason(reason)) case ActionDequeue: + // q.Dequeue calls q.cancelActionForPR, which cancels ctx. if _, err := q.Dequeue(pr.Number, false); err != nil { logger.Error("dequeuing PR failed", zap.Error(err)) continue } - metrics.DequeueOpsInc(&q.baseBranch.BranchID) logger.Info("pull request removed from merge queue", logfields.Reason(reason)) case ActionCreatePullRequestComment: diff --git a/internal/mergequeue/queue_processpr.go b/internal/mergequeue/queue_processpr.go index c8baf72..b6c52c8 100644 --- a/internal/mergequeue/queue_processpr.go +++ b/internal/mergequeue/queue_processpr.go @@ -97,6 +97,8 @@ func (q *queue) evalPRAction(ctx context.Context, logger *zap.Logger, pr *PullRe branchChanged, updateHeadCommit, err := q.updatePRWithBase(ctx, pr) if err != nil { if errors.Is(err, ErrPullRequestIsClosed) { + // ActionDequeue cancels the running processPR context via + // cancelActionForPR, so no context-using actions may follow it. return &requiredActions{ Actions: []Action{ActionDequeue}, Reason: reasonPullRequestClosed, diff --git a/internal/mergequeue/sync.go b/internal/mergequeue/sync.go index 86063b0..cc144f5 100644 --- a/internal/mergequeue/sync.go +++ b/internal/mergequeue/sync.go @@ -1,7 +1,6 @@ package mergequeue import ( - "cmp" "context" "errors" "fmt" @@ -92,7 +91,7 @@ func (a *Coordinator) sync(ctx context.Context, owner, repo string) error { items = append(items, syncItem{ghPR: ghPR, actions: actions}) } - slices.SortFunc(items, func(x, y syncItem) int { + slices.SortStableFunc(items, func(x, y syncItem) int { xHead := x.isQueuedHead() yHead := y.isQueuedHead() if xHead != yHead { @@ -102,7 +101,7 @@ func (a *Coordinator) sync(ctx context.Context, owner, repo string) error { return 1 } - return cmp.Compare(x.ghPR.Number, y.ghPR.Number) + return 0 }) headPreserved := map[string]bool{} @@ -118,7 +117,6 @@ func (a *Coordinator) sync(ctx context.Context, owner, repo string) error { preserveHeadLabel := item.isQueuedHead() && !headPreserved[ghPR.BaseBranch] if preserveHeadLabel { - headPreserved[ghPR.BaseBranch] = true pr.hasHeadLabel = true } @@ -143,6 +141,9 @@ func (a *Coordinator) sync(ctx context.Context, owner, repo string) error { case enqueue: err := a.enqueuePR(ctx, owner, repo, ghPR.BaseBranch, pr) if errors.Is(err, ErrAlreadyExists) { + if preserveHeadLabel { + headPreserved[ghPR.BaseBranch] = true + } prLogger.Debug("queue in-sync, pr is already enqueued") break } @@ -151,9 +152,24 @@ func (a *Coordinator) sync(ctx context.Context, owner, repo string) error { prLogger.Warn("adding pr to queue failed", zap.Error(err), ) + if preserveHeadLabel { + preserveHeadLabel = false + pr.hasHeadLabel = false + if rmErr := a.removeLabel(ctx, owner, repo, pr); rmErr != nil { + prLogger.Warn( + "removing pull request label failed", + zap.Error(rmErr), + ) + } else { + stats.Unlabeled++ + } + } break } + if preserveHeadLabel { + headPreserved[ghPR.BaseBranch] = true + } stats.Enqueued++ prLogger.Info("enqueued pr")