From bc46b5cd3f330892dc842494e8bdec87b5e9bb3b Mon Sep 17 00:00:00 2001 From: Francisco Delmar Kurpiel Date: Mon, 9 Nov 2026 11:58:34 +0100 Subject: [PATCH] mergequeue: dequeue PRs when trigger condition is no longer met GitHub webhook events that remove a pull request from the merge queue (auto_merge_disabled, unlabeled, closed) can be missed. Previously,ReadyForMerge only queried review and CI status, so a pull request whose auto_merge_disabled or unlabeled event was dropped remained in the queue and was repeatedly moved back to the active queue on base-branch pushes. Extend ReadyForMergeStatus to include Closed, AutoMergeEnabled, and Labels from the existing GraphQL query, and verify queue eligibility in both evalPRAction and resumeIfPRMergeStatusPositive before updating branches, adding the head label, triggering CI, or resuming suspended PRs. --- internal/githubclt/readyformergestatus.go | 51 ++ .../githubclt/readyformergestatus_test.go | 114 ++++ internal/mergequeue/coordinator.go | 40 ++ internal/mergequeue/coordinator_test.go | 17 +- internal/mergequeue/drygithubclient.go | 16 +- internal/mergequeue/enqueuetrigger.go | 73 +++ internal/mergequeue/queue.go | 105 +++- .../mergequeue/queue_enqueuetrigger_test.go | 556 ++++++++++++++++++ internal/mergequeue/queue_processpr.go | 22 +- internal/mergequeue/queue_processpr_test.go | 4 +- 10 files changed, 989 insertions(+), 9 deletions(-) create mode 100644 internal/mergequeue/enqueuetrigger.go create mode 100644 internal/mergequeue/queue_enqueuetrigger_test.go diff --git a/internal/githubclt/readyformergestatus.go b/internal/githubclt/readyformergestatus.go index 668ccc9..5f2ae8a 100644 --- a/internal/githubclt/readyformergestatus.go +++ b/internal/githubclt/readyformergestatus.go @@ -3,8 +3,10 @@ package githubclt import ( "context" "fmt" + "time" "github.com/shurcooL/githubv4" + "go.uber.org/zap" ) // CIStatus is the result or current status of a CI job. @@ -39,14 +41,32 @@ type CIJobStatus struct { // ReadyForMergeStatus represent the information deciding if a pull request is // ready to be merged. It contains the review decision and CI job results. +// It also contains the information that determines if the pull request is +// still eligible to be in a merge queue: if it is open, if auto-merge is +// enabled and its labels. type ReadyForMergeStatus struct { ReviewDecision ReviewDecision CIStatus CIStatus Statuses []*CIJobStatus Commit string + + // Closed is true if the pull request is closed or merged. + Closed bool + // AutoMergeEnabled is true if auto-merge is enabled for the pull + // request. + AutoMergeEnabled bool + // Labels contains the names of the labels of the pull request. + // At most the first [maxLabels] labels are retrieved. + Labels []string } +// maxLabels is the max. number of pull request labels that are retrieved. +// It must be kept in sync with the labels(first: N) argument in the query. +const maxLabels = 100 + // ReadyForMerge returns the [review decision] and [status check rollup] for a PR. +// It additionally returns if the PR is closed, if auto-merge is enabled and the +// labels of the PR. // // The status of checks that are not required is ignored. // If a required check failed the returned [ReadyForMerge.CIStatus] is @@ -73,6 +93,10 @@ func (clt *Client) ReadyForMerge(ctx context.Context, owner, repo string, prNumb CIStatus: overallCIStatus(queryResult.StatusCheckRollupState, statuses), Statuses: statuses, Commit: queryResult.Commit, + + Closed: queryResult.Closed, + AutoMergeEnabled: queryResult.AutoMergeEnabled, + Labels: queryResult.Labels, }, nil } @@ -244,6 +268,9 @@ type queryCIStatusResult struct { CheckRuns []*queryCheckStatus StatusContext []*queryStatusContext Commit string + Closed bool + AutoMergeEnabled bool + Labels []string } func (clt *Client) reviewAndCIStatus(ctx context.Context, owner, repo string, prNumber int) (*queryCIStatusResult, error) { @@ -251,6 +278,17 @@ func (clt *Client) reviewAndCIStatus(ctx context.Context, owner, repo string, pr Repository struct { PullRequest struct { ReviewDecision githubv4.PullRequestReviewDecision + // Closed is also true for merged pull requests. + Closed bool + AutoMergeRequest struct { + EnabledAt time.Time + } + Labels struct { + PageInfo PageInfo + Nodes []struct { + Name string + } + } `graphql:"labels(first: 100)"` BaseRef struct { BranchProtectionRule struct { @@ -336,6 +374,19 @@ func (clt *Client) reviewAndCIStatus(ctx context.Context, owner, repo string, pr result.StatusCheckRollupState = commitsNode.StatusCheckRollup.State result.RequiredStatusCheckContexts = q.Repository.PullRequest.BaseRef.BranchProtectionRule.RequiredStatusCheckContexts result.Commit = prHEADCommitID + result.Closed = q.Repository.PullRequest.Closed + result.AutoMergeEnabled = !q.Repository.PullRequest.AutoMergeRequest.EnabledAt.IsZero() + result.Labels = make([]string, 0, len(q.Repository.PullRequest.Labels.Nodes)) + for _, label := range q.Repository.PullRequest.Labels.Nodes { + result.Labels = append(result.Labels, label.Name) + } + + if q.Repository.PullRequest.Labels.PageInfo.HasNextPage { + clt.logger.Warn("incomplete pr labels retrieved, more label pages are available but only the first page was fetched, pagination is not implemented", + zap.Int("github.pull_request", prNumber), + zap.Int("github.labels_fetched", maxLabels), + ) + } return &result, nil } diff --git a/internal/githubclt/readyformergestatus_test.go b/internal/githubclt/readyformergestatus_test.go index 61a257a..a1332f8 100644 --- a/internal/githubclt/readyformergestatus_test.go +++ b/internal/githubclt/readyformergestatus_test.go @@ -1,10 +1,15 @@ package githubclt import ( + "context" + "io" + "net/http" + "net/http/httptest" "testing" "github.com/shurcooL/githubv4" "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" ) func TestOverallCIStatus_optionalFailedChecksAreIgnored(t *testing.T) { @@ -142,3 +147,112 @@ func TestOverallCIStatus_Expected(t *testing.T) { require.Equal(t, CIStatusExpected, status) } + +// readyForMergeResponse returns the response of the GraphQL query of +// reviewAndCIStatus for a PR that has a single successful required check. +// prFields contains additional pullRequest fields in JSON format. +func readyForMergeResponse(prFields string) string { + return `{"data":{"repository":{"pullRequest":{ + "reviewDecision":"APPROVED", + ` + prFields + `, + "baseRef":{"branchProtectionRule":{"requiredStatusCheckContexts":["build"]}}, + "commits":{"nodes":[{"commit":{ + "oid":"abc123", + "statusCheckRollup":{ + "state":"SUCCESS", + "contexts":{ + "pageInfo":{"endCursor":"","hasNextPage":false}, + "edges":[{"node":{ + "name":"build", + "conclusion":"SUCCESS", + "status":"COMPLETED", + "detailsUrl":"http://localhost/job/build/1" + }}] + } + } + }}]} + }}}}` +} + +func newReadyForMergeTestClient(t *testing.T, response string) (clt *Client, requestBody func() string) { + t.Helper() + + var body string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, err := io.ReadAll(r.Body) + if err != nil { + t.Errorf("reading request body failed: %v", err) + } + body = string(b) + + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, response) + })) + t.Cleanup(srv.Close) + + return &Client{ + logger: zaptest.NewLogger(t), + graphQLClt: githubv4.NewEnterpriseClient(srv.URL, srv.Client()), + }, func() string { return body } +} + +func TestReadyForMerge_ReturnsAutoMergeLabelsAndOpenState(t *testing.T) { + clt, requestBody := newReadyForMergeTestClient(t, readyForMergeResponse(` + "closed":false, + "autoMergeRequest":{"enabledAt":"2026-01-02T03:04:05Z"}, + "labels":{ + "pageInfo":{"endCursor":"","hasNextPage":false}, + "nodes":[{"name":"queue-add"},{"name":"bug"}] + }`, + )) + + status, err := clt.ReadyForMerge(context.Background(), "owner", "repo", 1) + require.NoError(t, err) + + require.False(t, status.Closed) + require.True(t, status.AutoMergeEnabled) + require.Equal(t, []string{"queue-add", "bug"}, status.Labels) + require.Equal(t, ReviewDecisionApproved, status.ReviewDecision) + require.Equal(t, CIStatusSuccess, status.CIStatus) + require.Equal(t, "abc123", status.Commit) + + // the fields must be part of the query + for _, field := range []string{"closed", "autoMergeRequest{enabledAt}", "labels(first: 100)"} { + require.Contains(t, requestBody(), field) + } +} + +func TestReadyForMerge_AutoMergeDisabledAndNoLabels(t *testing.T) { + // GitHub returns null as autoMergeRequest if auto-merge is not enabled + clt, _ := newReadyForMergeTestClient(t, readyForMergeResponse(` + "closed":false, + "autoMergeRequest":null, + "labels":{ + "pageInfo":{"endCursor":"","hasNextPage":false}, + "nodes":[] + }`, + )) + + status, err := clt.ReadyForMerge(context.Background(), "owner", "repo", 1) + require.NoError(t, err) + + require.False(t, status.Closed) + require.False(t, status.AutoMergeEnabled) + require.Empty(t, status.Labels) +} + +func TestReadyForMerge_ClosedPR(t *testing.T) { + clt, _ := newReadyForMergeTestClient(t, readyForMergeResponse(` + "closed":true, + "autoMergeRequest":null, + "labels":{ + "pageInfo":{"endCursor":"","hasNextPage":false}, + "nodes":[] + }`, + )) + + status, err := clt.ReadyForMerge(context.Background(), "owner", "repo", 1) + require.NoError(t, err) + + require.True(t, status.Closed) +} diff --git a/internal/mergequeue/coordinator.go b/internal/mergequeue/coordinator.go index 8cb3315..495c3d7 100644 --- a/internal/mergequeue/coordinator.go +++ b/internal/mergequeue/coordinator.go @@ -60,6 +60,13 @@ type Coordinator struct { // shutdownChan can be closed to communicate to the event loop go-routine to terminate. shutdownChan chan struct{} + // emptiedQueues receives the ID of queues that dequeued their last pull + // request on their own, because it was not eligible to be queued + // anymore. The event loop removes them. + // Queues can not remove themselves, they run in the context of their own + // action pool, that the removal has to wait for. + emptiedQueues chan BranchID + // processedEventCnt counts the number of events that were processed. // It is currently only used in testcase to delay checks until a send // event was processed. @@ -83,6 +90,7 @@ func NewCoordinator(cfg Config) *Coordinator { wg: sync.WaitGroup{}, periodicTriggerIntv: defPeriodicTriggerInterval, shutdownChan: make(chan struct{}, 1), + emptiedQueues: make(chan BranchID, 16), } cfg.setDefaults() @@ -166,6 +174,9 @@ func (a *Coordinator) eventLoop() { a.processEvent(context.Background(), event) + case branchID := <-a.emptiedQueues: + a.removeQueueIfEmpty(branchID) + case <-periodicTrigger.C: a.queuesLock.Lock() for branchID, q := range a.queues { @@ -187,6 +198,33 @@ func (a *Coordinator) eventLoop() { } } +// removeQueueIfEmpty removes the queue for the branch, if it is empty. +// It must not be called from a queue operation, it waits for the queue's +// running operations to finish. +func (a *Coordinator) removeQueueIfEmpty(branchID BranchID) { + a.queuesLock.Lock() + defer a.queuesLock.Unlock() + + q, exist := a.queues[branchID] + if !exist || !q.IsEmpty() { + return + } + + q.Wait() + delete(a.queues, branchID) + a.Logger.With(q.baseBranch.Logfields...).Debug("removed empty queue for base branch") +} + +// queueEmptied is called by a queue when it became empty because it dequeued +// a pull request on its own. It does not block, if the notification can not be +// delivered the periodic run of the event loop removes the empty queue. +func (a *Coordinator) queueEmptied(branchID BranchID) { + select { + case a.emptiedQueues <- branchID: + default: + } +} + // processEvent processes GitHub webhook events. // // Events for repositories that are not listed in @@ -921,6 +959,8 @@ func (a *Coordinator) Enqueue(_ context.Context, baseBranch *BaseBranch, pr *Pul a.Retryer, a.CI, a.HeadLabel, + withEnqueueTrigger(a.TriggerOnAutomerge, a.TriggerLabels), + withOnEmpty(func() { a.queueEmptied(baseBranch.BranchID) }), ) a.queues[baseBranch.BranchID] = q diff --git a/internal/mergequeue/coordinator_test.go b/internal/mergequeue/coordinator_test.go index 925c5ff..bb34670 100644 --- a/internal/mergequeue/coordinator_test.go +++ b/internal/mergequeue/coordinator_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "slices" "sync" "sync/atomic" "testing" @@ -140,11 +141,22 @@ type mergeStatusMock struct { mu sync.Mutex } +// mockedTriggerLabel is the trigger label that tests configure for the +// coordinator. mockReadyForMergeStatus returns it as label of the pull request. +const mockedTriggerLabel = "queue-add" + +// mockReadyForMergeStatus mocks ReadyForMerge to return an open pull request +// that has auto-merge enabled and the mockedTriggerLabel, so that it is +// eligible to be in a queue regardless of the configured trigger. +// To simulate that it is not eligible anymore, change AutoMergeEnabled and +// Labels of the returned mock while holding its mutex. func mockReadyForMergeStatus(clt *mocks.MockGithubClient, prNumber int, reviewDecision githubclt.ReviewDecision, checkState githubclt.CIStatus) *mergeStatusMock { res := &mergeStatusMock{ ReadyForMergeStatus: githubclt.ReadyForMergeStatus{ - ReviewDecision: reviewDecision, - CIStatus: checkState, + ReviewDecision: reviewDecision, + CIStatus: checkState, + AutoMergeEnabled: true, + Labels: []string{mockedTriggerLabel}, Statuses: []*githubclt.CIJobStatus{{ Name: ciJobExpectedName, Status: checkState, @@ -163,6 +175,7 @@ func mockReadyForMergeStatus(clt *mocks.MockGithubClient, prNumber int, reviewDe // return a copy of the object to avoid race conditions status := res.ReadyForMergeStatus + status.Labels = slices.Clone(res.Labels) status.Statuses = make([]*githubclt.CIJobStatus, len(res.Statuses)) for i, s := range res.Statuses { sCopy := *s diff --git a/internal/mergequeue/drygithubclient.go b/internal/mergequeue/drygithubclient.go index 7f49db9..4ebf20b 100644 --- a/internal/mergequeue/drygithubclient.go +++ b/internal/mergequeue/drygithubclient.go @@ -32,13 +32,27 @@ func (c *DryGithubClient) UpdateBranch(context.Context, string, string, int) (*g return &githubclt.UpdateBranchResult{HeadCommitID: dryGitHubClientHeadCommitID}, nil } -func (c *DryGithubClient) ReadyForMerge(context.Context, string, string, int) (*githubclt.ReadyForMergeStatus, error) { +// ReadyForMerge simulates that the pull request is approved and all checks are +// successful. +// The information about whether the pull request is closed, has auto-merge +// enabled and its labels is retrieved from GitHub. The queue uses them to decide +// if the pull request must stay queued, simulating them would dequeue it. +func (c *DryGithubClient) ReadyForMerge(ctx context.Context, owner, repo string, prNumber int) (*githubclt.ReadyForMergeStatus, error) { + status, err := c.clt.ReadyForMerge(ctx, owner, repo, prNumber) + if err != nil { + return nil, err + } + c.logger.Info("simulated fetching ready for merge status, pr is approved, all checks successful") return &githubclt.ReadyForMergeStatus{ ReviewDecision: githubclt.ReviewDecisionApproved, CIStatus: githubclt.CIStatusSuccess, Commit: dryGitHubClientHeadCommitID, + + Closed: status.Closed, + AutoMergeEnabled: status.AutoMergeEnabled, + Labels: status.Labels, }, nil } diff --git a/internal/mergequeue/enqueuetrigger.go b/internal/mergequeue/enqueuetrigger.go new file mode 100644 index 0000000..14d8a7d --- /dev/null +++ b/internal/mergequeue/enqueuetrigger.go @@ -0,0 +1,73 @@ +package mergequeue + +import ( + "slices" + + "github.com/simplesurance/directorius/internal/githubclt" + "github.com/simplesurance/directorius/internal/set" +) + +const ( + reasonPullRequestClosed = "pull request is closed" + reasonTriggerNotMet = "pull request does not meet the queue trigger condition, " + + "auto-merge is not enabled and it has no trigger label" +) + +// enqueueTrigger defines the conditions of which at least one must be met for +// a pull request to be in a merge queue. +// It mirrors [Config.TriggerOnAutomerge] and [Config.TriggerLabels]. +type enqueueTrigger struct { + // OnAutomerge is true if pull requests with enabled auto-merge belong + // in the queue. + OnAutomerge bool + // Labels are labels that cause pull requests to belong in the queue. + Labels set.Set[string] +} + +// isConfigured returns true if at least one trigger condition is defined. +// The configuration of the application requires it. A queue without a +// configured trigger does not verify the condition. +func (t enqueueTrigger) isConfigured() bool { + return t.OnAutomerge || len(t.Labels) > 0 +} + +// isMet returns true if a pull request with the given auto-merge state and +// labels fulfills one of the trigger conditions. +// If no trigger is configured it always returns true. +func (t enqueueTrigger) isMet(autoMergeEnabled bool, labels []string) bool { + if !t.isConfigured() { + return true + } + + if t.OnAutomerge && autoMergeEnabled { + return true + } + + return slices.ContainsFunc(labels, t.Labels.Contains) +} + +// dequeueRequest describes why a pull request must be removed from the queue. +type dequeueRequest struct { + Reason string + // ResetStatus is true if the github commit status of the pull request + // must be reset to pending when it is dequeued. + // It is only needed for open pull requests. + ResetStatus bool +} + +// dequeueRequired checks if the pull request, that status describes, is still +// eligible to be in the queue. +// It is not eligible when it is closed or none of the trigger conditions is +// met anymore. GitHub webhook events that communicate this can be lost. +// If it is not eligible, a dequeueRequest is returned, otherwise nil. +func (q *queue) dequeueRequired(status *githubclt.ReadyForMergeStatus) *dequeueRequest { + if status.Closed { + return &dequeueRequest{Reason: reasonPullRequestClosed} + } + + if !q.trigger.isMet(status.AutoMergeEnabled, status.Labels) { + return &dequeueRequest{Reason: reasonTriggerNotMet, ResetStatus: true} + } + + return nil +} diff --git a/internal/mergequeue/queue.go b/internal/mergequeue/queue.go index b811998..5921eb6 100644 --- a/internal/mergequeue/queue.go +++ b/internal/mergequeue/queue.go @@ -101,16 +101,54 @@ type queue struct { // pull-request in the active queue headLabel string + // trigger defines the conditions of which one must be met, for a pull + // request to stay in the queue. + trigger enqueueTrigger + // onEmpty is optional. It is called, without blocking, when the queue + // became empty because the queue itself dequeued a pull request that + // was not eligible anymore. It is called from the [queue.actionPool], + // it must not wait for it. + onEmpty func() + // paused can be set to true to skip processPR operations. paused atomic.Bool } +type queueOption func(*queue) + +// withEnqueueTrigger configures the conditions that a pull request must +// fulfill to stay in the queue. If none is met, the pull request is dequeued +// the next time its status is evaluated. +// If neither onAutomerge is true nor labels is non-empty, the condition is not +// verified. +func withEnqueueTrigger(onAutomerge bool, labels set.Set[string]) queueOption { + return func(q *queue) { + q.trigger = enqueueTrigger{OnAutomerge: onAutomerge, Labels: labels} + } +} + +// withOnEmpty configures a function that is called when the queue dequeued a +// pull request on its own and became empty. +func withOnEmpty(fn func()) queueOption { + return func(q *queue) { + q.onEmpty = fn + } +} + type runningOperation struct { pr int cancelFunc context.CancelFunc } -func newQueue(base *BaseBranch, logger *zap.Logger, ghClient GithubClient, retryer *retry.Retryer, ci *CI, headLabel string) *queue { +func newQueue( + base *BaseBranch, + logger *zap.Logger, + ghClient GithubClient, + retryer *retry.Retryer, + ci *CI, + headLabel string, + opts ...queueOption, +) *queue { q := queue{ baseBranch: *base, active: orderedmap.New[int, *PullRequest](orderBefore), @@ -125,6 +163,10 @@ func newQueue(base *BaseBranch, logger *zap.Logger, ghClient GithubClient, retry ci: ci, } + for _, opt := range opts { + opt(&q) + } + if qm, err := newQueueMetrics(base.BranchID); err == nil { q.metrics = qm } else { @@ -178,6 +220,13 @@ func (q *queue) IsEmpty() bool { return q.active.Len() == 0 && len(q.suspended) == 0 } +// notifyIfEmpty calls the onEmpty function if it is set and the queue is empty. +func (q *queue) notifyIfEmpty() { + if q.onEmpty != nil && q.IsEmpty() { + q.onEmpty() + } +} + func (q *queue) _activePullRequests() []*PullRequest { return q.active.AsSlice() } @@ -508,6 +557,11 @@ func (q *queue) isPRStale(pr *PullRequest) bool { // and the actions to undertake. // The PR must be the first in the active queue otherwise nothing is done. // +// If the PR is closed or does not meet the queue trigger condition anymore +// (auto-merge was disabled, the trigger label was removed), it is dequeued. +// GitHub webhook events that report it can be lost, therefore the state at +// GitHub is verified. +// // If the base-branch contains changes that are not in the pull request branch, // updating it, by merging the base-branch into the PR branch, is schedule via // the GitHub API. If updating is not possible because a merge-conflict exist @@ -595,12 +649,22 @@ func (q *queue) processPR(ctx context.Context, pr *PullRequest, task Task) { case ActionDequeue: // q.Dequeue calls q.cancelActionForPR, which cancels ctx. - if _, err := q.Dequeue(pr.Number, false); err != nil { + if _, err := q.Dequeue(pr.Number, actions.ResetStatusOnDequeue); err != nil { + if errors.Is(err, ErrNotFound) { + // A webhook event dequeued the PR while its + // status was evaluated, it is not a failure. + logger.Debug("pull request was already removed from the merge queue", + zap.Error(err), logfields.Reason(reason)) + q.notifyIfEmpty() + continue + } + logger.Error("dequeuing PR failed", zap.Error(err)) continue } logger.Info("pull request removed from merge queue", logfields.Reason(reason)) + q.notifyIfEmpty() case ActionCreatePullRequestComment: // use a shorter timeout to prevent that this optional operation blocks for long @@ -750,6 +814,9 @@ func (q *queue) prReadyForMergeStatus(ctx context.Context, pr *PullRequest) (*gi logfields.ReviewDecision(string(status.ReviewDecision)), logfields.CIStatusSummary(string(status.CIStatus)), zap.Any("github.ci_statuses", status.Statuses), + zap.Bool("github.pull_request_closed", status.Closed), + zap.Bool("github.auto_merge_enabled", status.AutoMergeEnabled), + zap.Strings("github.labels", status.Labels), }, loggingFields...)..., ) @@ -980,8 +1047,18 @@ func (q *queue) _suspendedPRsbyBranch(branches set.Set[string]) ( return result, notFound } +// isSuspended returns true if the pull request with the given number is in the +// suspend queue. +func (q *queue) isSuspended(prNumber int) bool { + q.lock.Lock() + defer q.lock.Unlock() + + _, exist := q.suspended[prNumber] + return exist +} + func (q *queue) resumeIfPRMergeStatusPositive(ctx context.Context, logger *zap.Logger, pr *PullRequest) error { - if _, exist := q.suspended[pr.Number]; !exist { + if !q.isSuspended(pr.Number) { return ErrNotFound } @@ -990,6 +1067,28 @@ func (q *queue) resumeIfPRMergeStatusPositive(ctx context.Context, logger *zap.L return fmt.Errorf("retrieving ready for merge status failed: %w", err) } + // The webhook event that removes a PR from the queue (auto_merge_disabled, + // unlabeled, closed) can be lost. Check it here, otherwise the PR would + // be moved to the active queue and block it, although GitHub will + // never merge it. + if req := q.dequeueRequired(status); req != nil { + if _, err := q.Dequeue(pr.Number, req.ResetStatus); err != nil { + if errors.Is(err, ErrNotFound) { + // A webhook event dequeued the PR while the status + // was retrieved, it is not a failure. + q.notifyIfEmpty() + return ErrNotFound + } + + return fmt.Errorf("removing pull request from merge queue failed: %w", err) + } + + logger.Info("pull request removed from merge queue", logfields.Reason(req.Reason)) + q.notifyIfEmpty() + + return nil + } + if status.ReviewDecision != githubclt.ReviewDecisionApproved { logger.Info("updates for pr is not resumed, reviewdecision is not positive") return nil diff --git a/internal/mergequeue/queue_enqueuetrigger_test.go b/internal/mergequeue/queue_enqueuetrigger_test.go new file mode 100644 index 0000000..106211d --- /dev/null +++ b/internal/mergequeue/queue_enqueuetrigger_test.go @@ -0,0 +1,556 @@ +package mergequeue + +import ( + "context" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/simplesurance/directorius/internal/githubclt" + "github.com/simplesurance/directorius/internal/mergequeue/mocks" + github_prov "github.com/simplesurance/directorius/internal/provider/github" + "github.com/simplesurance/directorius/internal/set" +) + +const triggerLabel = "queue-add" + +// approvedStatus returns the ready for merge status of an open, approved pull +// request with successful CI jobs. +func approvedStatus(autoMergeEnabled bool, labels ...string) *githubclt.ReadyForMergeStatus { + return &githubclt.ReadyForMergeStatus{ + ReviewDecision: githubclt.ReviewDecisionApproved, + CIStatus: githubclt.CIStatusSuccess, + Commit: dryGitHubClientHeadCommitID, + AutoMergeEnabled: autoMergeEnabled, + Labels: labels, + } +} + +// mockReadyForMergeStatusFunc replaces the ReadyForMerge mock, the returned +// status is created by fn for the passed pull request number. +// The mock controller must have been created with overridable expectations. +func mockReadyForMergeStatusFunc(clt *mocks.MockGithubClient, fn func(prNumber int) *githubclt.ReadyForMergeStatus) { + clt.EXPECT(). + ReadyForMerge(gomock.Any(), gomock.Eq(repoOwner), gomock.Eq(repo), gomock.Any()). + DoAndReturn(func(_ context.Context, _, _ string, prNumber int) (*githubclt.ReadyForMergeStatus, error) { + return fn(prNumber), nil + }). + AnyTimes() +} + +func mockReadyForMergeStatusConst(clt *mocks.MockGithubClient, status *githubclt.ReadyForMergeStatus) { + mockReadyForMergeStatusFunc(clt, func(int) *githubclt.ReadyForMergeStatus { + s := *status + return &s + }) +} + +// mockNoUpdateBranchCalls fails the test when UpdateBranch is called. +func mockNoUpdateBranchCalls(clt *mocks.MockGithubClient) { + clt.EXPECT(). + UpdateBranch(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Times(0) +} + +func TestEnqueueTrigger_IsMet(t *testing.T) { + tcs := []struct { + name string + trigger enqueueTrigger + autoMergeEnabled bool + labels []string + want bool + }{ + { + name: "no trigger configured, condition is not verified", + trigger: enqueueTrigger{}, + want: true, + }, + { + name: "no trigger configured, empty label set", + trigger: enqueueTrigger{Labels: set.Set[string]{}}, + want: true, + }, + { + name: "auto-merge trigger, auto-merge enabled", + trigger: enqueueTrigger{OnAutomerge: true}, + autoMergeEnabled: true, + want: true, + }, + { + name: "auto-merge trigger, auto-merge disabled", + trigger: enqueueTrigger{OnAutomerge: true}, + labels: []string{triggerLabel}, + want: false, + }, + { + name: "label trigger, label set", + trigger: enqueueTrigger{Labels: set.From([]string{triggerLabel})}, + labels: []string{"bug", triggerLabel}, + want: true, + }, + { + name: "label trigger, label missing", + trigger: enqueueTrigger{Labels: set.From([]string{triggerLabel})}, + labels: []string{"bug"}, + want: false, + }, + { + name: "label trigger, auto-merge enabled is irrelevant", + trigger: enqueueTrigger{Labels: set.From([]string{triggerLabel})}, + autoMergeEnabled: true, + want: false, + }, + { + name: "both triggers, only auto-merge", + trigger: enqueueTrigger{OnAutomerge: true, Labels: set.From([]string{triggerLabel})}, + autoMergeEnabled: true, + want: true, + }, + { + name: "both triggers, only label", + trigger: enqueueTrigger{OnAutomerge: true, Labels: set.From([]string{triggerLabel})}, + labels: []string{triggerLabel}, + want: true, + }, + { + name: "both triggers, none met", + trigger: enqueueTrigger{OnAutomerge: true, Labels: set.From([]string{triggerLabel})}, + labels: []string{"bug"}, + want: false, + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, tc.trigger.isMet(tc.autoMergeEnabled, tc.labels)) + }) + } +} + +type triggerTestCase struct { + name string + onAutomerge bool + labels []string + status *githubclt.ReadyForMergeStatus +} + +func TestEvalPRAction_DequeuesPRThatDoesNotMeetTriggerAnymore(t *testing.T) { + tcs := []triggerTestCase{ + { + name: "auto-merge disabled", + onAutomerge: true, + status: approvedStatus(false), + }, + { + name: "auto-merge disabled, unrelated label", + onAutomerge: true, + status: approvedStatus(false, "bug"), + }, + { + name: "trigger label removed", + labels: []string{triggerLabel}, + status: approvedStatus(false, "bug"), + }, + { + name: "trigger label removed, auto-merge is enabled but not a trigger", + labels: []string{triggerLabel}, + status: approvedStatus(true), + }, + { + name: "both triggers configured, none met", + onAutomerge: true, + labels: []string{triggerLabel}, + status: approvedStatus(false, "bug"), + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + prt := initEvalPRActionTest(t, withEnqueueTrigger(tc.onAutomerge, set.From(tc.labels))) + mockReadyForMergeStatusConst(prt.GhClt, tc.status) + // the branch must not be updated, it would trigger CI + mockNoUpdateBranchCalls(prt.GhClt) + + reqActions, err := prt.Q.evalPRAction(context.Background(), prt.L, prt.PR) + require.NoError(t, err) + require.NotNil(t, reqActions) + + assert.Equal(t, []Action{ActionDequeue}, reqActions.Actions) + assert.Equal(t, reasonTriggerNotMet, reqActions.Reason) + assert.True(t, reqActions.ResetStatusOnDequeue, "github status of an open pr must be reset") + }) + } +} + +func TestEvalPRAction_DoesNotDequeuePRThatMeetsTrigger(t *testing.T) { + tcs := []triggerTestCase{ + { + name: "auto-merge enabled", + onAutomerge: true, + status: approvedStatus(true), + }, + { + name: "trigger label set", + labels: []string{triggerLabel}, + status: approvedStatus(false, "bug", triggerLabel), + }, + { + name: "both triggers configured, only auto-merge", + onAutomerge: true, + labels: []string{triggerLabel}, + status: approvedStatus(true), + }, + { + name: "both triggers configured, only label", + onAutomerge: true, + labels: []string{triggerLabel}, + status: approvedStatus(false, triggerLabel), + }, + { + name: "no trigger configured", + status: approvedStatus(false), + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + prt := initEvalPRActionTest(t, withEnqueueTrigger(tc.onAutomerge, set.From(tc.labels))) + mockReadyForMergeStatusConst(prt.GhClt, tc.status) + + reqActions, err := prt.Q.evalPRAction(context.Background(), prt.L, prt.PR) + require.NoError(t, err) + require.NotNil(t, reqActions) + + assert.NotContains(t, reqActions.Actions, ActionDequeue) + assert.Equal(t, []Action{ActionCreateSuccessfulGithubStatus, ActionNone}, reqActions.Actions) + }) + } +} + +func TestEvalPRAction_DequeuesClosedPR(t *testing.T) { + tcs := []triggerTestCase{ + { + name: "trigger is met", + onAutomerge: true, + status: approvedStatus(true), + }, + { + name: "trigger is not met", + onAutomerge: true, + status: approvedStatus(false), + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + prt := initEvalPRActionTest(t, withEnqueueTrigger(tc.onAutomerge, set.From(tc.labels))) + tc.status.Closed = true + mockReadyForMergeStatusConst(prt.GhClt, tc.status) + mockNoUpdateBranchCalls(prt.GhClt) + + reqActions, err := prt.Q.evalPRAction(context.Background(), prt.L, prt.PR) + require.NoError(t, err) + require.NotNil(t, reqActions) + + assert.Equal(t, []Action{ActionDequeue}, reqActions.Actions) + assert.Equal(t, reasonPullRequestClosed, reqActions.Reason) + assert.False(t, reqActions.ResetStatusOnDequeue, "github status of a closed pr must not be reset") + }) + } +} + +func TestProcessPR_DequeuesFirstPRThatDoesNotMeetTriggerAnymore(t *testing.T) { + var emptied atomic.Int32 + + prt := initEvalPRActionTest(t, + withEnqueueTrigger(true, nil), + withOnEmpty(func() { emptied.Add(1) }), + ) + mockReadyForMergeStatusConst(prt.GhClt, approvedStatus(false)) + mockNoUpdateBranchCalls(prt.GhClt) + // unexpected AddLabel and CreateCommitStatus calls fail the test. + mockSuccessfulGithubRemoveLabelQueueHeadCall(prt.GhClt, prNR).Times(1) + mockCreateHeadCommitStatusPendingPRNr(prt.GhClt, prNR).Times(1) + + require.NoError(t, prt.Q.Enqueue(prt.PR)) + + require.Eventually(t, + func() bool { return emptied.Load() == 1 }, + condWaitTimeout, condCheckInterval, + "queue was not reported as empty after the pr was dequeued", + ) + assert.True(t, prt.Q.IsEmpty()) +} + +func TestProcessPR_DequeuesClosedFirstPRWithoutResettingStatus(t *testing.T) { + var emptied atomic.Int32 + + prt := initEvalPRActionTest(t, + withEnqueueTrigger(true, nil), + withOnEmpty(func() { emptied.Add(1) }), + ) + status := approvedStatus(true) + status.Closed = true + mockReadyForMergeStatusConst(prt.GhClt, status) + mockNoUpdateBranchCalls(prt.GhClt) + // unexpected CreateHeadCommitStatus calls fail the test. + mockSuccessfulGithubRemoveLabelQueueHeadCall(prt.GhClt, prNR).Times(1) + + require.NoError(t, prt.Q.Enqueue(prt.PR)) + + require.Eventually(t, + func() bool { return emptied.Load() == 1 }, + condWaitTimeout, condCheckInterval, + "queue was not reported as empty after the pr was dequeued", + ) + assert.True(t, prt.Q.IsEmpty()) +} + +func TestProcessPR_ProcessesNextPRAfterFirstDoesNotMeetTriggerAnymore(t *testing.T) { + const nextPRNr = prNR + 1 + + var emptied atomic.Int32 + var updateBranchCallsFirstPR, updateBranchCallsNextPR atomic.Int32 + + prt := initEvalPRActionTest(t, + withEnqueueTrigger(true, nil), + withOnEmpty(func() { emptied.Add(1) }), + ) + mockReadyForMergeStatusFunc(prt.GhClt, func(prNumber int) *githubclt.ReadyForMergeStatus { + // the auto-merge disabled event for the first pr was lost + return approvedStatus(prNumber != prNR) + }) + prt.GhClt.EXPECT(). + UpdateBranch(gomock.Any(), gomock.Eq(repoOwner), gomock.Eq(repo), gomock.Any()). + DoAndReturn(func(_ context.Context, _, _ string, prNumber int) (*githubclt.UpdateBranchResult, error) { + if prNumber == prNR { + updateBranchCallsFirstPR.Add(1) + } else { + updateBranchCallsNextPR.Add(1) + } + return &githubclt.UpdateBranchResult{HeadCommitID: dryGitHubClientHeadCommitID}, nil + }). + AnyTimes() + mockSuccessfulGithubRemoveLabelQueueHeadCall(prt.GhClt, prNR).Times(1) + mockCreateHeadCommitStatusPendingPRNr(prt.GhClt, prNR).Times(1) + mockSuccessfulGithubAddLabelQueueHeadCall(prt.GhClt, nextPRNr).AnyTimes() + mockCreateCommitStatusSuccessful(prt.GhClt).AnyTimes() + + nextPR, err := NewPullRequest(nextPRNr, "next_branch", "me", "next pr", "") + require.NoError(t, err) + + require.NoError(t, prt.Q.Enqueue(prt.PR)) + require.NoError(t, prt.Q.Enqueue(nextPR)) + + require.Eventually(t, + func() bool { + first := prt.Q.FirstActive() + return first != nil && first.Number == nextPRNr && updateBranchCallsNextPR.Load() > 0 + }, + condWaitTimeout, condCheckInterval, + "the pr after the dequeued one was not processed", + ) + + assert.Equal(t, 1, prt.Q.activeLen()) + assert.Zero(t, updateBranchCallsFirstPR.Load(), "branch of the dequeued pr was updated") + assert.Zero(t, emptied.Load(), "queue is not empty") +} + +func suspendPR(q *queue, pr *PullRequest) { + q.lock.Lock() + defer q.lock.Unlock() + q.suspended[pr.Number] = pr +} + +func TestResumeIfPRMergeStatusPositive_DequeuesPRThatDoesNotMeetTriggerAnymore(t *testing.T) { + var emptied atomic.Int32 + + // A suspended PR had its label already removed and its status reset when + // it was suspended, calls for it would fail the test. + prt := initEvalPRActionTest(t, + withEnqueueTrigger(false, set.From([]string{triggerLabel})), + withOnEmpty(func() { emptied.Add(1) }), + ) + mockReadyForMergeStatusConst(prt.GhClt, approvedStatus(false, "bug")) + suspendPR(prt.Q, prt.PR) + + err := prt.Q.resumeIfPRMergeStatusPositive(context.Background(), prt.L, prt.PR) + require.NoError(t, err) + + assert.Zero(t, prt.Q.suspendedLen(), "pr is still in suspend queue") + assert.Zero(t, prt.Q.activeLen(), "pr was moved to active queue") + assert.Equal(t, int32(1), emptied.Load()) +} + +func TestResumeIfPRMergeStatusPositive_DequeuesClosedPR(t *testing.T) { + var emptied atomic.Int32 + + prt := initEvalPRActionTest(t, + withEnqueueTrigger(true, nil), + withOnEmpty(func() { emptied.Add(1) }), + ) + status := approvedStatus(true) + status.Closed = true + mockReadyForMergeStatusConst(prt.GhClt, status) + suspendPR(prt.Q, prt.PR) + + err := prt.Q.resumeIfPRMergeStatusPositive(context.Background(), prt.L, prt.PR) + require.NoError(t, err) + + assert.Zero(t, prt.Q.suspendedLen(), "pr is still in suspend queue") + assert.Zero(t, prt.Q.activeLen(), "pr was moved to active queue") + assert.Equal(t, int32(1), emptied.Load()) +} + +func TestResumeIfPRMergeStatusPositive_KeepsPRThatMeetsTrigger(t *testing.T) { + var emptied atomic.Int32 + + prt := initEvalPRActionTest(t, + withEnqueueTrigger(true, nil), + withOnEmpty(func() { emptied.Add(1) }), + ) + status := approvedStatus(true) + status.ReviewDecision = githubclt.ReviewDecisionReviewRequired + mockReadyForMergeStatusConst(prt.GhClt, status) + suspendPR(prt.Q, prt.PR) + + err := prt.Q.resumeIfPRMergeStatusPositive(context.Background(), prt.L, prt.PR) + require.NoError(t, err) + + assert.Equal(t, 1, prt.Q.suspendedLen(), "pr was removed from suspend queue") + assert.Zero(t, emptied.Load()) +} + +// TestMissedAutoMergeDisabledEventDequeuesPRAndRemovesQueue verifies that a +// queued PR for that GitHub never delivered an auto_merge_disabled event, does +// not block the queue forever. +// When the base branch changes, the PR is removed from the queue, the queue head +// label is removed and the empty queue is removed from the coordinator. +func TestMissedAutoMergeDisabledEventDequeuesPRAndRemovesQueue(t *testing.T) { + const prNumber = 1 + const prBranch = "pr_branch" + + evChan := make(chan *github_prov.Event, 1) + defer close(evChan) + + mockctrl := gomock.NewController(t) + ghClient := mocks.NewMockGithubClient(mockctrl) + ciClient := mocks.NewMockJenkinsClient(mockctrl) + forwardLogsToTestLogger(t) + + // Called once, when the PR is enqueued. When the PR is processed + // after it is ineligible, the branch must not be updated. + mockSuccessfulGithubUpdateBranchCall(ghClient, prNumber, true).Times(1) + status := mockReadyForMergeStatus( + ghClient, prNumber, + githubclt.ReviewDecisionApproved, githubclt.CIStatusExpected, + ) + status.AnyTimes() + mockSuccessfulGithubRemoveLabelQueueHeadCall(ghClient, prNumber).Times(1) + mockCreateHeadCommitStatusPendingPRNr(ghClient, prNumber).Times(1) + + autoupdater := newAutoupdater( + ghClient, + ciClient, + evChan, + []Repository{{OwnerLogin: repoOwner, RepositoryName: repo}}, + true, + nil, + ) + autoupdater.Start() + t.Cleanup(autoupdater.Stop) + + baseBranch, err := NewBaseBranch(repoOwner, repo, "main") + require.NoError(t, err) + + evChan <- &github_prov.Event{Event: newPullRequestAutomergeEnabledEvent(prNumber, prBranch, baseBranch.Branch)} + waitForProcessedEventCnt(t, autoupdater, 1) + + q := autoupdater.getQueue(&baseBranch.BranchID) + require.NotNil(t, q) + require.Equal(t, 1, q.activeLen()) + // wait until the PR was processed once, it is the first in the queue + waitForQueueUpdateRunsGreaterThan(t, q, 0) + + // auto-merge is disabled on GitHub but the auto_merge_disabled event is lost + status.mu.Lock() + status.AutoMergeEnabled = false + status.mu.Unlock() + + evChan <- &github_prov.Event{Event: newPushEvent(baseBranch.Branch)} + waitForProcessedEventCnt(t, autoupdater, 2) + + require.Eventually(t, + func() bool { return autoupdater.getQueue(&baseBranch.BranchID) == nil }, + condWaitTimeout, condCheckInterval, + "empty queue was not removed after the pr was dequeued", + ) +} + +// A webhook event can dequeue a PR while its status is retrieved from GitHub. +// The queue must handle it as no failure and report that it became empty. +// The PR is dequeued by the event, not by the queue itself, therefore onEmpty +// can only be called by the code under test. +func TestResumeIfPRMergeStatusPositive_PRDequeuedConcurrently(t *testing.T) { + var emptied atomic.Int32 + + prt := initEvalPRActionTest(t, + withEnqueueTrigger(true, nil), + withOnEmpty(func() { emptied.Add(1) }), + ) + suspendPR(prt.Q, prt.PR) + + prt.GhClt.EXPECT(). + ReadyForMerge(gomock.Any(), gomock.Eq(repoOwner), gomock.Eq(repo), gomock.Eq(prNR)). + DoAndReturn(func(context.Context, string, string, int) (*githubclt.ReadyForMergeStatus, error) { + _, err := prt.Q.Dequeue(prNR, true) + require.NoError(t, err) + + return approvedStatus(false), nil + }). + Times(1) + + err := prt.Q.resumeIfPRMergeStatusPositive(context.Background(), prt.L, prt.PR) + require.ErrorIs(t, err, ErrNotFound) + + assert.Equal(t, int32(1), emptied.Load()) +} + +func TestProcessPR_PRDequeuedConcurrently(t *testing.T) { + var emptied atomic.Int32 + + prt := initEvalPRActionTest(t, + withEnqueueTrigger(true, nil), + withOnEmpty(func() { emptied.Add(1) }), + ) + mockNoUpdateBranchCalls(prt.GhClt) + + // The label is removed and the status reset once, by the event. The + // queue must not repeat it when it finds out that the PR is gone. + mockSuccessfulGithubRemoveLabelQueueHeadCall(prt.GhClt, prNR).Times(1) + mockCreateHeadCommitStatusPendingPRNr(prt.GhClt, prNR).Times(1) + + prt.GhClt.EXPECT(). + ReadyForMerge(gomock.Any(), gomock.Eq(repoOwner), gomock.Eq(repo), gomock.Eq(prNR)). + DoAndReturn(func(context.Context, string, string, int) (*githubclt.ReadyForMergeStatus, error) { + _, err := prt.Q.Dequeue(prNR, true) + require.NoError(t, err) + + return approvedStatus(false), nil + }). + Times(1) + + // Insert without scheduling processPR, it is run directly. + prt.Q.lock.Lock() + _, added := prt.Q.active.InsertIfNotExist(prt.PR.Number, prt.PR) + prt.Q.lock.Unlock() + require.True(t, added) + + prt.Q.processPR(context.Background(), prt.PR, TaskNone) + + assert.True(t, prt.Q.IsEmpty()) + assert.Equal(t, int32(1), emptied.Load()) +} diff --git a/internal/mergequeue/queue_processpr.go b/internal/mergequeue/queue_processpr.go index b6c52c8..d5cb6dc 100644 --- a/internal/mergequeue/queue_processpr.go +++ b/internal/mergequeue/queue_processpr.go @@ -34,6 +34,11 @@ type requiredActions struct { Reason string HeadCommitID string + // ResetStatusOnDequeue is only used when ActionDequeue is part of + // Actions. If it is true, the github commit status is reset to pending + // when the pull request is dequeued. + ResetStatusOnDequeue bool + // ExpectedCIRuns is only set when ActionTriggerCIJobs is part of Actions. ExpectedCIRuns []string } @@ -68,7 +73,6 @@ func (a Action) String() string { func (q *queue) evalPRAction(ctx context.Context, logger *zap.Logger, pr *PullRequest) (*requiredActions, error) { const ( reasonNotApproved = "not approved" - reasonPullRequestClosed = "pull request is closed" reasonMergeConflict = "update with base branch failed: merge conflict" reasonBranchUpdatedWithBase = "pull request branch has been updated with base branch" reasonIsStale = "pull request is stale" @@ -84,6 +88,22 @@ func (q *queue) evalPRAction(ctx context.Context, logger *zap.Logger, pr *PullRe return nil, fmt.Errorf("retrieving ready to merge status from github failed: %w", err) } + // The webhook event that removes a PR from the queue (auto_merge_disabled, + // unlabeled, closed) can be lost. Check it here, otherwise the PR blocks + // the head of the queue, although GitHub will never merge it. + // This must be done before anything else, to prevent that the branch of the + // PR is updated, it is labeled or CI jobs are triggered for it. + if req := q.dequeueRequired(status); req != nil { + // ActionDequeue cancels the running processPR context via + // cancelActionForPR, so no context-using actions may follow it. + return &requiredActions{ + Actions: []Action{ActionDequeue}, + Reason: req.Reason, + HeadCommitID: status.Commit, + ResetStatusOnDequeue: req.ResetStatus, + }, nil + } + if status.ReviewDecision != githubclt.ReviewDecisionApproved { return &requiredActions{ Actions: []Action{ActionSuspend}, diff --git a/internal/mergequeue/queue_processpr_test.go b/internal/mergequeue/queue_processpr_test.go index 6adb5e8..e5b3e4d 100644 --- a/internal/mergequeue/queue_processpr_test.go +++ b/internal/mergequeue/queue_processpr_test.go @@ -22,7 +22,7 @@ type evalPRTest struct { GhClt *mocks.MockGithubClient } -func initEvalPRActionTest(t *testing.T) *evalPRTest { +func initEvalPRActionTest(t *testing.T, queueOpts ...queueOption) *evalPRTest { l := forwardLogsToTestLogger(t) mockctrl := gomock.NewController(t, gomock.WithOverridableExpectations()) @@ -39,7 +39,7 @@ func initEvalPRActionTest(t *testing.T) *evalPRTest { bb, err := NewBaseBranch(repoOwner, repo, "main") require.NoError(t, err) ci := &CI{Client: ciClient, logger: l} - q := newQueue(bb, l, ghClient, retry.NewRetryer(), ci, queueHeadLabel) + q := newQueue(bb, l, ghClient, retry.NewRetryer(), ci, queueHeadLabel, queueOpts...) t.Cleanup(q.Stop) pr, err := NewPullRequest(prNR, prBranch, "me", "test pr", "")