diff --git a/internal/githubclt/readyformergestatus.go b/internal/githubclt/readyformergestatus.go index 668ccc9..5f2ae8a 100644 --- a/internal/githubclt/readyformergestatus.go +++ b/internal/githubclt/readyformergestatus.go @@ -3,8 +3,10 @@ package githubclt import ( "context" "fmt" + "time" "github.com/shurcooL/githubv4" + "go.uber.org/zap" ) // CIStatus is the result or current status of a CI job. @@ -39,14 +41,32 @@ type CIJobStatus struct { // ReadyForMergeStatus represent the information deciding if a pull request is // ready to be merged. It contains the review decision and CI job results. +// It also contains the information that determines if the pull request is +// still eligible to be in a merge queue: if it is open, if auto-merge is +// enabled and its labels. type ReadyForMergeStatus struct { ReviewDecision ReviewDecision CIStatus CIStatus Statuses []*CIJobStatus Commit string + + // Closed is true if the pull request is closed or merged. + Closed bool + // AutoMergeEnabled is true if auto-merge is enabled for the pull + // request. + AutoMergeEnabled bool + // Labels contains the names of the labels of the pull request. + // At most the first [maxLabels] labels are retrieved. + Labels []string } +// maxLabels is the max. number of pull request labels that are retrieved. +// It must be kept in sync with the labels(first: N) argument in the query. +const maxLabels = 100 + // ReadyForMerge returns the [review decision] and [status check rollup] for a PR. +// It additionally returns if the PR is closed, if auto-merge is enabled and the +// labels of the PR. // // The status of checks that are not required is ignored. // If a required check failed the returned [ReadyForMerge.CIStatus] is @@ -73,6 +93,10 @@ func (clt *Client) ReadyForMerge(ctx context.Context, owner, repo string, prNumb CIStatus: overallCIStatus(queryResult.StatusCheckRollupState, statuses), Statuses: statuses, Commit: queryResult.Commit, + + Closed: queryResult.Closed, + AutoMergeEnabled: queryResult.AutoMergeEnabled, + Labels: queryResult.Labels, }, nil } @@ -244,6 +268,9 @@ type queryCIStatusResult struct { CheckRuns []*queryCheckStatus StatusContext []*queryStatusContext Commit string + Closed bool + AutoMergeEnabled bool + Labels []string } func (clt *Client) reviewAndCIStatus(ctx context.Context, owner, repo string, prNumber int) (*queryCIStatusResult, error) { @@ -251,6 +278,17 @@ func (clt *Client) reviewAndCIStatus(ctx context.Context, owner, repo string, pr Repository struct { PullRequest struct { ReviewDecision githubv4.PullRequestReviewDecision + // Closed is also true for merged pull requests. + Closed bool + AutoMergeRequest struct { + EnabledAt time.Time + } + Labels struct { + PageInfo PageInfo + Nodes []struct { + Name string + } + } `graphql:"labels(first: 100)"` BaseRef struct { BranchProtectionRule struct { @@ -336,6 +374,19 @@ func (clt *Client) reviewAndCIStatus(ctx context.Context, owner, repo string, pr result.StatusCheckRollupState = commitsNode.StatusCheckRollup.State result.RequiredStatusCheckContexts = q.Repository.PullRequest.BaseRef.BranchProtectionRule.RequiredStatusCheckContexts result.Commit = prHEADCommitID + result.Closed = q.Repository.PullRequest.Closed + result.AutoMergeEnabled = !q.Repository.PullRequest.AutoMergeRequest.EnabledAt.IsZero() + result.Labels = make([]string, 0, len(q.Repository.PullRequest.Labels.Nodes)) + for _, label := range q.Repository.PullRequest.Labels.Nodes { + result.Labels = append(result.Labels, label.Name) + } + + if q.Repository.PullRequest.Labels.PageInfo.HasNextPage { + clt.logger.Warn("incomplete pr labels retrieved, more label pages are available but only the first page was fetched, pagination is not implemented", + zap.Int("github.pull_request", prNumber), + zap.Int("github.labels_fetched", maxLabels), + ) + } return &result, nil } diff --git a/internal/githubclt/readyformergestatus_test.go b/internal/githubclt/readyformergestatus_test.go index 61a257a..a1332f8 100644 --- a/internal/githubclt/readyformergestatus_test.go +++ b/internal/githubclt/readyformergestatus_test.go @@ -1,10 +1,15 @@ package githubclt import ( + "context" + "io" + "net/http" + "net/http/httptest" "testing" "github.com/shurcooL/githubv4" "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" ) func TestOverallCIStatus_optionalFailedChecksAreIgnored(t *testing.T) { @@ -142,3 +147,112 @@ func TestOverallCIStatus_Expected(t *testing.T) { require.Equal(t, CIStatusExpected, status) } + +// readyForMergeResponse returns the response of the GraphQL query of +// reviewAndCIStatus for a PR that has a single successful required check. +// prFields contains additional pullRequest fields in JSON format. +func readyForMergeResponse(prFields string) string { + return `{"data":{"repository":{"pullRequest":{ + "reviewDecision":"APPROVED", + ` + prFields + `, + "baseRef":{"branchProtectionRule":{"requiredStatusCheckContexts":["build"]}}, + "commits":{"nodes":[{"commit":{ + "oid":"abc123", + "statusCheckRollup":{ + "state":"SUCCESS", + "contexts":{ + "pageInfo":{"endCursor":"","hasNextPage":false}, + "edges":[{"node":{ + "name":"build", + "conclusion":"SUCCESS", + "status":"COMPLETED", + "detailsUrl":"http://localhost/job/build/1" + }}] + } + } + }}]} + }}}}` +} + +func newReadyForMergeTestClient(t *testing.T, response string) (clt *Client, requestBody func() string) { + t.Helper() + + var body string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, err := io.ReadAll(r.Body) + if err != nil { + t.Errorf("reading request body failed: %v", err) + } + body = string(b) + + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, response) + })) + t.Cleanup(srv.Close) + + return &Client{ + logger: zaptest.NewLogger(t), + graphQLClt: githubv4.NewEnterpriseClient(srv.URL, srv.Client()), + }, func() string { return body } +} + +func TestReadyForMerge_ReturnsAutoMergeLabelsAndOpenState(t *testing.T) { + clt, requestBody := newReadyForMergeTestClient(t, readyForMergeResponse(` + "closed":false, + "autoMergeRequest":{"enabledAt":"2026-01-02T03:04:05Z"}, + "labels":{ + "pageInfo":{"endCursor":"","hasNextPage":false}, + "nodes":[{"name":"queue-add"},{"name":"bug"}] + }`, + )) + + status, err := clt.ReadyForMerge(context.Background(), "owner", "repo", 1) + require.NoError(t, err) + + require.False(t, status.Closed) + require.True(t, status.AutoMergeEnabled) + require.Equal(t, []string{"queue-add", "bug"}, status.Labels) + require.Equal(t, ReviewDecisionApproved, status.ReviewDecision) + require.Equal(t, CIStatusSuccess, status.CIStatus) + require.Equal(t, "abc123", status.Commit) + + // the fields must be part of the query + for _, field := range []string{"closed", "autoMergeRequest{enabledAt}", "labels(first: 100)"} { + require.Contains(t, requestBody(), field) + } +} + +func TestReadyForMerge_AutoMergeDisabledAndNoLabels(t *testing.T) { + // GitHub returns null as autoMergeRequest if auto-merge is not enabled + clt, _ := newReadyForMergeTestClient(t, readyForMergeResponse(` + "closed":false, + "autoMergeRequest":null, + "labels":{ + "pageInfo":{"endCursor":"","hasNextPage":false}, + "nodes":[] + }`, + )) + + status, err := clt.ReadyForMerge(context.Background(), "owner", "repo", 1) + require.NoError(t, err) + + require.False(t, status.Closed) + require.False(t, status.AutoMergeEnabled) + require.Empty(t, status.Labels) +} + +func TestReadyForMerge_ClosedPR(t *testing.T) { + clt, _ := newReadyForMergeTestClient(t, readyForMergeResponse(` + "closed":true, + "autoMergeRequest":null, + "labels":{ + "pageInfo":{"endCursor":"","hasNextPage":false}, + "nodes":[] + }`, + )) + + status, err := clt.ReadyForMerge(context.Background(), "owner", "repo", 1) + require.NoError(t, err) + + require.True(t, status.Closed) +} diff --git a/internal/mergequeue/coordinator.go b/internal/mergequeue/coordinator.go index 8cb3315..495c3d7 100644 --- a/internal/mergequeue/coordinator.go +++ b/internal/mergequeue/coordinator.go @@ -60,6 +60,13 @@ type Coordinator struct { // shutdownChan can be closed to communicate to the event loop go-routine to terminate. shutdownChan chan struct{} + // emptiedQueues receives the ID of queues that dequeued their last pull + // request on their own, because it was not eligible to be queued + // anymore. The event loop removes them. + // Queues can not remove themselves, they run in the context of their own + // action pool, that the removal has to wait for. + emptiedQueues chan BranchID + // processedEventCnt counts the number of events that were processed. // It is currently only used in testcase to delay checks until a send // event was processed. @@ -83,6 +90,7 @@ func NewCoordinator(cfg Config) *Coordinator { wg: sync.WaitGroup{}, periodicTriggerIntv: defPeriodicTriggerInterval, shutdownChan: make(chan struct{}, 1), + emptiedQueues: make(chan BranchID, 16), } cfg.setDefaults() @@ -166,6 +174,9 @@ func (a *Coordinator) eventLoop() { a.processEvent(context.Background(), event) + case branchID := <-a.emptiedQueues: + a.removeQueueIfEmpty(branchID) + case <-periodicTrigger.C: a.queuesLock.Lock() for branchID, q := range a.queues { @@ -187,6 +198,33 @@ func (a *Coordinator) eventLoop() { } } +// removeQueueIfEmpty removes the queue for the branch, if it is empty. +// It must not be called from a queue operation, it waits for the queue's +// running operations to finish. +func (a *Coordinator) removeQueueIfEmpty(branchID BranchID) { + a.queuesLock.Lock() + defer a.queuesLock.Unlock() + + q, exist := a.queues[branchID] + if !exist || !q.IsEmpty() { + return + } + + q.Wait() + delete(a.queues, branchID) + a.Logger.With(q.baseBranch.Logfields...).Debug("removed empty queue for base branch") +} + +// queueEmptied is called by a queue when it became empty because it dequeued +// a pull request on its own. It does not block, if the notification can not be +// delivered the periodic run of the event loop removes the empty queue. +func (a *Coordinator) queueEmptied(branchID BranchID) { + select { + case a.emptiedQueues <- branchID: + default: + } +} + // processEvent processes GitHub webhook events. // // Events for repositories that are not listed in @@ -921,6 +959,8 @@ func (a *Coordinator) Enqueue(_ context.Context, baseBranch *BaseBranch, pr *Pul a.Retryer, a.CI, a.HeadLabel, + withEnqueueTrigger(a.TriggerOnAutomerge, a.TriggerLabels), + withOnEmpty(func() { a.queueEmptied(baseBranch.BranchID) }), ) a.queues[baseBranch.BranchID] = q diff --git a/internal/mergequeue/coordinator_test.go b/internal/mergequeue/coordinator_test.go index 925c5ff..bb34670 100644 --- a/internal/mergequeue/coordinator_test.go +++ b/internal/mergequeue/coordinator_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "slices" "sync" "sync/atomic" "testing" @@ -140,11 +141,22 @@ type mergeStatusMock struct { mu sync.Mutex } +// mockedTriggerLabel is the trigger label that tests configure for the +// coordinator. mockReadyForMergeStatus returns it as label of the pull request. +const mockedTriggerLabel = "queue-add" + +// mockReadyForMergeStatus mocks ReadyForMerge to return an open pull request +// that has auto-merge enabled and the mockedTriggerLabel, so that it is +// eligible to be in a queue regardless of the configured trigger. +// To simulate that it is not eligible anymore, change AutoMergeEnabled and +// Labels of the returned mock while holding its mutex. func mockReadyForMergeStatus(clt *mocks.MockGithubClient, prNumber int, reviewDecision githubclt.ReviewDecision, checkState githubclt.CIStatus) *mergeStatusMock { res := &mergeStatusMock{ ReadyForMergeStatus: githubclt.ReadyForMergeStatus{ - ReviewDecision: reviewDecision, - CIStatus: checkState, + ReviewDecision: reviewDecision, + CIStatus: checkState, + AutoMergeEnabled: true, + Labels: []string{mockedTriggerLabel}, Statuses: []*githubclt.CIJobStatus{{ Name: ciJobExpectedName, Status: checkState, @@ -163,6 +175,7 @@ func mockReadyForMergeStatus(clt *mocks.MockGithubClient, prNumber int, reviewDe // return a copy of the object to avoid race conditions status := res.ReadyForMergeStatus + status.Labels = slices.Clone(res.Labels) status.Statuses = make([]*githubclt.CIJobStatus, len(res.Statuses)) for i, s := range res.Statuses { sCopy := *s diff --git a/internal/mergequeue/drygithubclient.go b/internal/mergequeue/drygithubclient.go index 7f49db9..4ebf20b 100644 --- a/internal/mergequeue/drygithubclient.go +++ b/internal/mergequeue/drygithubclient.go @@ -32,13 +32,27 @@ func (c *DryGithubClient) UpdateBranch(context.Context, string, string, int) (*g return &githubclt.UpdateBranchResult{HeadCommitID: dryGitHubClientHeadCommitID}, nil } -func (c *DryGithubClient) ReadyForMerge(context.Context, string, string, int) (*githubclt.ReadyForMergeStatus, error) { +// ReadyForMerge simulates that the pull request is approved and all checks are +// successful. +// The information about whether the pull request is closed, has auto-merge +// enabled and its labels is retrieved from GitHub. The queue uses them to decide +// if the pull request must stay queued, simulating them would dequeue it. +func (c *DryGithubClient) ReadyForMerge(ctx context.Context, owner, repo string, prNumber int) (*githubclt.ReadyForMergeStatus, error) { + status, err := c.clt.ReadyForMerge(ctx, owner, repo, prNumber) + if err != nil { + return nil, err + } + c.logger.Info("simulated fetching ready for merge status, pr is approved, all checks successful") return &githubclt.ReadyForMergeStatus{ ReviewDecision: githubclt.ReviewDecisionApproved, CIStatus: githubclt.CIStatusSuccess, Commit: dryGitHubClientHeadCommitID, + + Closed: status.Closed, + AutoMergeEnabled: status.AutoMergeEnabled, + Labels: status.Labels, }, nil } diff --git a/internal/mergequeue/enqueuetrigger.go b/internal/mergequeue/enqueuetrigger.go new file mode 100644 index 0000000..14d8a7d --- /dev/null +++ b/internal/mergequeue/enqueuetrigger.go @@ -0,0 +1,73 @@ +package mergequeue + +import ( + "slices" + + "github.com/simplesurance/directorius/internal/githubclt" + "github.com/simplesurance/directorius/internal/set" +) + +const ( + reasonPullRequestClosed = "pull request is closed" + reasonTriggerNotMet = "pull request does not meet the queue trigger condition, " + + "auto-merge is not enabled and it has no trigger label" +) + +// enqueueTrigger defines the conditions of which at least one must be met for +// a pull request to be in a merge queue. +// It mirrors [Config.TriggerOnAutomerge] and [Config.TriggerLabels]. +type enqueueTrigger struct { + // OnAutomerge is true if pull requests with enabled auto-merge belong + // in the queue. + OnAutomerge bool + // Labels are labels that cause pull requests to belong in the queue. + Labels set.Set[string] +} + +// isConfigured returns true if at least one trigger condition is defined. +// The configuration of the application requires it. A queue without a +// configured trigger does not verify the condition. +func (t enqueueTrigger) isConfigured() bool { + return t.OnAutomerge || len(t.Labels) > 0 +} + +// isMet returns true if a pull request with the given auto-merge state and +// labels fulfills one of the trigger conditions. +// If no trigger is configured it always returns true. +func (t enqueueTrigger) isMet(autoMergeEnabled bool, labels []string) bool { + if !t.isConfigured() { + return true + } + + if t.OnAutomerge && autoMergeEnabled { + return true + } + + return slices.ContainsFunc(labels, t.Labels.Contains) +} + +// dequeueRequest describes why a pull request must be removed from the queue. +type dequeueRequest struct { + Reason string + // ResetStatus is true if the github commit status of the pull request + // must be reset to pending when it is dequeued. + // It is only needed for open pull requests. + ResetStatus bool +} + +// dequeueRequired checks if the pull request, that status describes, is still +// eligible to be in the queue. +// It is not eligible when it is closed or none of the trigger conditions is +// met anymore. GitHub webhook events that communicate this can be lost. +// If it is not eligible, a dequeueRequest is returned, otherwise nil. +func (q *queue) dequeueRequired(status *githubclt.ReadyForMergeStatus) *dequeueRequest { + if status.Closed { + return &dequeueRequest{Reason: reasonPullRequestClosed} + } + + if !q.trigger.isMet(status.AutoMergeEnabled, status.Labels) { + return &dequeueRequest{Reason: reasonTriggerNotMet, ResetStatus: true} + } + + return nil +} diff --git a/internal/mergequeue/queue.go b/internal/mergequeue/queue.go index b811998..5921eb6 100644 --- a/internal/mergequeue/queue.go +++ b/internal/mergequeue/queue.go @@ -101,16 +101,54 @@ type queue struct { // pull-request in the active queue headLabel string + // trigger defines the conditions of which one must be met, for a pull + // request to stay in the queue. + trigger enqueueTrigger + // onEmpty is optional. It is called, without blocking, when the queue + // became empty because the queue itself dequeued a pull request that + // was not eligible anymore. It is called from the [queue.actionPool], + // it must not wait for it. + onEmpty func() + // paused can be set to true to skip processPR operations. paused atomic.Bool } +type queueOption func(*queue) + +// withEnqueueTrigger configures the conditions that a pull request must +// fulfill to stay in the queue. If none is met, the pull request is dequeued +// the next time its status is evaluated. +// If neither onAutomerge is true nor labels is non-empty, the condition is not +// verified. +func withEnqueueTrigger(onAutomerge bool, labels set.Set[string]) queueOption { + return func(q *queue) { + q.trigger = enqueueTrigger{OnAutomerge: onAutomerge, Labels: labels} + } +} + +// withOnEmpty configures a function that is called when the queue dequeued a +// pull request on its own and became empty. +func withOnEmpty(fn func()) queueOption { + return func(q *queue) { + q.onEmpty = fn + } +} + type runningOperation struct { pr int cancelFunc context.CancelFunc } -func newQueue(base *BaseBranch, logger *zap.Logger, ghClient GithubClient, retryer *retry.Retryer, ci *CI, headLabel string) *queue { +func newQueue( + base *BaseBranch, + logger *zap.Logger, + ghClient GithubClient, + retryer *retry.Retryer, + ci *CI, + headLabel string, + opts ...queueOption, +) *queue { q := queue{ baseBranch: *base, active: orderedmap.New[int, *PullRequest](orderBefore), @@ -125,6 +163,10 @@ func newQueue(base *BaseBranch, logger *zap.Logger, ghClient GithubClient, retry ci: ci, } + for _, opt := range opts { + opt(&q) + } + if qm, err := newQueueMetrics(base.BranchID); err == nil { q.metrics = qm } else { @@ -178,6 +220,13 @@ func (q *queue) IsEmpty() bool { return q.active.Len() == 0 && len(q.suspended) == 0 } +// notifyIfEmpty calls the onEmpty function if it is set and the queue is empty. +func (q *queue) notifyIfEmpty() { + if q.onEmpty != nil && q.IsEmpty() { + q.onEmpty() + } +} + func (q *queue) _activePullRequests() []*PullRequest { return q.active.AsSlice() } @@ -508,6 +557,11 @@ func (q *queue) isPRStale(pr *PullRequest) bool { // and the actions to undertake. // The PR must be the first in the active queue otherwise nothing is done. // +// If the PR is closed or does not meet the queue trigger condition anymore +// (auto-merge was disabled, the trigger label was removed), it is dequeued. +// GitHub webhook events that report it can be lost, therefore the state at +// GitHub is verified. +// // If the base-branch contains changes that are not in the pull request branch, // updating it, by merging the base-branch into the PR branch, is schedule via // the GitHub API. If updating is not possible because a merge-conflict exist @@ -595,12 +649,22 @@ func (q *queue) processPR(ctx context.Context, pr *PullRequest, task Task) { case ActionDequeue: // q.Dequeue calls q.cancelActionForPR, which cancels ctx. - if _, err := q.Dequeue(pr.Number, false); err != nil { + if _, err := q.Dequeue(pr.Number, actions.ResetStatusOnDequeue); err != nil { + if errors.Is(err, ErrNotFound) { + // A webhook event dequeued the PR while its + // status was evaluated, it is not a failure. + logger.Debug("pull request was already removed from the merge queue", + zap.Error(err), logfields.Reason(reason)) + q.notifyIfEmpty() + continue + } + logger.Error("dequeuing PR failed", zap.Error(err)) continue } logger.Info("pull request removed from merge queue", logfields.Reason(reason)) + q.notifyIfEmpty() case ActionCreatePullRequestComment: // use a shorter timeout to prevent that this optional operation blocks for long @@ -750,6 +814,9 @@ func (q *queue) prReadyForMergeStatus(ctx context.Context, pr *PullRequest) (*gi logfields.ReviewDecision(string(status.ReviewDecision)), logfields.CIStatusSummary(string(status.CIStatus)), zap.Any("github.ci_statuses", status.Statuses), + zap.Bool("github.pull_request_closed", status.Closed), + zap.Bool("github.auto_merge_enabled", status.AutoMergeEnabled), + zap.Strings("github.labels", status.Labels), }, loggingFields...)..., ) @@ -980,8 +1047,18 @@ func (q *queue) _suspendedPRsbyBranch(branches set.Set[string]) ( return result, notFound } +// isSuspended returns true if the pull request with the given number is in the +// suspend queue. +func (q *queue) isSuspended(prNumber int) bool { + q.lock.Lock() + defer q.lock.Unlock() + + _, exist := q.suspended[prNumber] + return exist +} + func (q *queue) resumeIfPRMergeStatusPositive(ctx context.Context, logger *zap.Logger, pr *PullRequest) error { - if _, exist := q.suspended[pr.Number]; !exist { + if !q.isSuspended(pr.Number) { return ErrNotFound } @@ -990,6 +1067,28 @@ func (q *queue) resumeIfPRMergeStatusPositive(ctx context.Context, logger *zap.L return fmt.Errorf("retrieving ready for merge status failed: %w", err) } + // The webhook event that removes a PR from the queue (auto_merge_disabled, + // unlabeled, closed) can be lost. Check it here, otherwise the PR would + // be moved to the active queue and block it, although GitHub will + // never merge it. + if req := q.dequeueRequired(status); req != nil { + if _, err := q.Dequeue(pr.Number, req.ResetStatus); err != nil { + if errors.Is(err, ErrNotFound) { + // A webhook event dequeued the PR while the status + // was retrieved, it is not a failure. + q.notifyIfEmpty() + return ErrNotFound + } + + return fmt.Errorf("removing pull request from merge queue failed: %w", err) + } + + logger.Info("pull request removed from merge queue", logfields.Reason(req.Reason)) + q.notifyIfEmpty() + + return nil + } + if status.ReviewDecision != githubclt.ReviewDecisionApproved { logger.Info("updates for pr is not resumed, reviewdecision is not positive") return nil diff --git a/internal/mergequeue/queue_enqueuetrigger_test.go b/internal/mergequeue/queue_enqueuetrigger_test.go new file mode 100644 index 0000000..106211d --- /dev/null +++ b/internal/mergequeue/queue_enqueuetrigger_test.go @@ -0,0 +1,556 @@ +package mergequeue + +import ( + "context" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/simplesurance/directorius/internal/githubclt" + "github.com/simplesurance/directorius/internal/mergequeue/mocks" + github_prov "github.com/simplesurance/directorius/internal/provider/github" + "github.com/simplesurance/directorius/internal/set" +) + +const triggerLabel = "queue-add" + +// approvedStatus returns the ready for merge status of an open, approved pull +// request with successful CI jobs. +func approvedStatus(autoMergeEnabled bool, labels ...string) *githubclt.ReadyForMergeStatus { + return &githubclt.ReadyForMergeStatus{ + ReviewDecision: githubclt.ReviewDecisionApproved, + CIStatus: githubclt.CIStatusSuccess, + Commit: dryGitHubClientHeadCommitID, + AutoMergeEnabled: autoMergeEnabled, + Labels: labels, + } +} + +// mockReadyForMergeStatusFunc replaces the ReadyForMerge mock, the returned +// status is created by fn for the passed pull request number. +// The mock controller must have been created with overridable expectations. +func mockReadyForMergeStatusFunc(clt *mocks.MockGithubClient, fn func(prNumber int) *githubclt.ReadyForMergeStatus) { + clt.EXPECT(). + ReadyForMerge(gomock.Any(), gomock.Eq(repoOwner), gomock.Eq(repo), gomock.Any()). + DoAndReturn(func(_ context.Context, _, _ string, prNumber int) (*githubclt.ReadyForMergeStatus, error) { + return fn(prNumber), nil + }). + AnyTimes() +} + +func mockReadyForMergeStatusConst(clt *mocks.MockGithubClient, status *githubclt.ReadyForMergeStatus) { + mockReadyForMergeStatusFunc(clt, func(int) *githubclt.ReadyForMergeStatus { + s := *status + return &s + }) +} + +// mockNoUpdateBranchCalls fails the test when UpdateBranch is called. +func mockNoUpdateBranchCalls(clt *mocks.MockGithubClient) { + clt.EXPECT(). + UpdateBranch(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Times(0) +} + +func TestEnqueueTrigger_IsMet(t *testing.T) { + tcs := []struct { + name string + trigger enqueueTrigger + autoMergeEnabled bool + labels []string + want bool + }{ + { + name: "no trigger configured, condition is not verified", + trigger: enqueueTrigger{}, + want: true, + }, + { + name: "no trigger configured, empty label set", + trigger: enqueueTrigger{Labels: set.Set[string]{}}, + want: true, + }, + { + name: "auto-merge trigger, auto-merge enabled", + trigger: enqueueTrigger{OnAutomerge: true}, + autoMergeEnabled: true, + want: true, + }, + { + name: "auto-merge trigger, auto-merge disabled", + trigger: enqueueTrigger{OnAutomerge: true}, + labels: []string{triggerLabel}, + want: false, + }, + { + name: "label trigger, label set", + trigger: enqueueTrigger{Labels: set.From([]string{triggerLabel})}, + labels: []string{"bug", triggerLabel}, + want: true, + }, + { + name: "label trigger, label missing", + trigger: enqueueTrigger{Labels: set.From([]string{triggerLabel})}, + labels: []string{"bug"}, + want: false, + }, + { + name: "label trigger, auto-merge enabled is irrelevant", + trigger: enqueueTrigger{Labels: set.From([]string{triggerLabel})}, + autoMergeEnabled: true, + want: false, + }, + { + name: "both triggers, only auto-merge", + trigger: enqueueTrigger{OnAutomerge: true, Labels: set.From([]string{triggerLabel})}, + autoMergeEnabled: true, + want: true, + }, + { + name: "both triggers, only label", + trigger: enqueueTrigger{OnAutomerge: true, Labels: set.From([]string{triggerLabel})}, + labels: []string{triggerLabel}, + want: true, + }, + { + name: "both triggers, none met", + trigger: enqueueTrigger{OnAutomerge: true, Labels: set.From([]string{triggerLabel})}, + labels: []string{"bug"}, + want: false, + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, tc.trigger.isMet(tc.autoMergeEnabled, tc.labels)) + }) + } +} + +type triggerTestCase struct { + name string + onAutomerge bool + labels []string + status *githubclt.ReadyForMergeStatus +} + +func TestEvalPRAction_DequeuesPRThatDoesNotMeetTriggerAnymore(t *testing.T) { + tcs := []triggerTestCase{ + { + name: "auto-merge disabled", + onAutomerge: true, + status: approvedStatus(false), + }, + { + name: "auto-merge disabled, unrelated label", + onAutomerge: true, + status: approvedStatus(false, "bug"), + }, + { + name: "trigger label removed", + labels: []string{triggerLabel}, + status: approvedStatus(false, "bug"), + }, + { + name: "trigger label removed, auto-merge is enabled but not a trigger", + labels: []string{triggerLabel}, + status: approvedStatus(true), + }, + { + name: "both triggers configured, none met", + onAutomerge: true, + labels: []string{triggerLabel}, + status: approvedStatus(false, "bug"), + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + prt := initEvalPRActionTest(t, withEnqueueTrigger(tc.onAutomerge, set.From(tc.labels))) + mockReadyForMergeStatusConst(prt.GhClt, tc.status) + // the branch must not be updated, it would trigger CI + mockNoUpdateBranchCalls(prt.GhClt) + + reqActions, err := prt.Q.evalPRAction(context.Background(), prt.L, prt.PR) + require.NoError(t, err) + require.NotNil(t, reqActions) + + assert.Equal(t, []Action{ActionDequeue}, reqActions.Actions) + assert.Equal(t, reasonTriggerNotMet, reqActions.Reason) + assert.True(t, reqActions.ResetStatusOnDequeue, "github status of an open pr must be reset") + }) + } +} + +func TestEvalPRAction_DoesNotDequeuePRThatMeetsTrigger(t *testing.T) { + tcs := []triggerTestCase{ + { + name: "auto-merge enabled", + onAutomerge: true, + status: approvedStatus(true), + }, + { + name: "trigger label set", + labels: []string{triggerLabel}, + status: approvedStatus(false, "bug", triggerLabel), + }, + { + name: "both triggers configured, only auto-merge", + onAutomerge: true, + labels: []string{triggerLabel}, + status: approvedStatus(true), + }, + { + name: "both triggers configured, only label", + onAutomerge: true, + labels: []string{triggerLabel}, + status: approvedStatus(false, triggerLabel), + }, + { + name: "no trigger configured", + status: approvedStatus(false), + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + prt := initEvalPRActionTest(t, withEnqueueTrigger(tc.onAutomerge, set.From(tc.labels))) + mockReadyForMergeStatusConst(prt.GhClt, tc.status) + + reqActions, err := prt.Q.evalPRAction(context.Background(), prt.L, prt.PR) + require.NoError(t, err) + require.NotNil(t, reqActions) + + assert.NotContains(t, reqActions.Actions, ActionDequeue) + assert.Equal(t, []Action{ActionCreateSuccessfulGithubStatus, ActionNone}, reqActions.Actions) + }) + } +} + +func TestEvalPRAction_DequeuesClosedPR(t *testing.T) { + tcs := []triggerTestCase{ + { + name: "trigger is met", + onAutomerge: true, + status: approvedStatus(true), + }, + { + name: "trigger is not met", + onAutomerge: true, + status: approvedStatus(false), + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + prt := initEvalPRActionTest(t, withEnqueueTrigger(tc.onAutomerge, set.From(tc.labels))) + tc.status.Closed = true + mockReadyForMergeStatusConst(prt.GhClt, tc.status) + mockNoUpdateBranchCalls(prt.GhClt) + + reqActions, err := prt.Q.evalPRAction(context.Background(), prt.L, prt.PR) + require.NoError(t, err) + require.NotNil(t, reqActions) + + assert.Equal(t, []Action{ActionDequeue}, reqActions.Actions) + assert.Equal(t, reasonPullRequestClosed, reqActions.Reason) + assert.False(t, reqActions.ResetStatusOnDequeue, "github status of a closed pr must not be reset") + }) + } +} + +func TestProcessPR_DequeuesFirstPRThatDoesNotMeetTriggerAnymore(t *testing.T) { + var emptied atomic.Int32 + + prt := initEvalPRActionTest(t, + withEnqueueTrigger(true, nil), + withOnEmpty(func() { emptied.Add(1) }), + ) + mockReadyForMergeStatusConst(prt.GhClt, approvedStatus(false)) + mockNoUpdateBranchCalls(prt.GhClt) + // unexpected AddLabel and CreateCommitStatus calls fail the test. + mockSuccessfulGithubRemoveLabelQueueHeadCall(prt.GhClt, prNR).Times(1) + mockCreateHeadCommitStatusPendingPRNr(prt.GhClt, prNR).Times(1) + + require.NoError(t, prt.Q.Enqueue(prt.PR)) + + require.Eventually(t, + func() bool { return emptied.Load() == 1 }, + condWaitTimeout, condCheckInterval, + "queue was not reported as empty after the pr was dequeued", + ) + assert.True(t, prt.Q.IsEmpty()) +} + +func TestProcessPR_DequeuesClosedFirstPRWithoutResettingStatus(t *testing.T) { + var emptied atomic.Int32 + + prt := initEvalPRActionTest(t, + withEnqueueTrigger(true, nil), + withOnEmpty(func() { emptied.Add(1) }), + ) + status := approvedStatus(true) + status.Closed = true + mockReadyForMergeStatusConst(prt.GhClt, status) + mockNoUpdateBranchCalls(prt.GhClt) + // unexpected CreateHeadCommitStatus calls fail the test. + mockSuccessfulGithubRemoveLabelQueueHeadCall(prt.GhClt, prNR).Times(1) + + require.NoError(t, prt.Q.Enqueue(prt.PR)) + + require.Eventually(t, + func() bool { return emptied.Load() == 1 }, + condWaitTimeout, condCheckInterval, + "queue was not reported as empty after the pr was dequeued", + ) + assert.True(t, prt.Q.IsEmpty()) +} + +func TestProcessPR_ProcessesNextPRAfterFirstDoesNotMeetTriggerAnymore(t *testing.T) { + const nextPRNr = prNR + 1 + + var emptied atomic.Int32 + var updateBranchCallsFirstPR, updateBranchCallsNextPR atomic.Int32 + + prt := initEvalPRActionTest(t, + withEnqueueTrigger(true, nil), + withOnEmpty(func() { emptied.Add(1) }), + ) + mockReadyForMergeStatusFunc(prt.GhClt, func(prNumber int) *githubclt.ReadyForMergeStatus { + // the auto-merge disabled event for the first pr was lost + return approvedStatus(prNumber != prNR) + }) + prt.GhClt.EXPECT(). + UpdateBranch(gomock.Any(), gomock.Eq(repoOwner), gomock.Eq(repo), gomock.Any()). + DoAndReturn(func(_ context.Context, _, _ string, prNumber int) (*githubclt.UpdateBranchResult, error) { + if prNumber == prNR { + updateBranchCallsFirstPR.Add(1) + } else { + updateBranchCallsNextPR.Add(1) + } + return &githubclt.UpdateBranchResult{HeadCommitID: dryGitHubClientHeadCommitID}, nil + }). + AnyTimes() + mockSuccessfulGithubRemoveLabelQueueHeadCall(prt.GhClt, prNR).Times(1) + mockCreateHeadCommitStatusPendingPRNr(prt.GhClt, prNR).Times(1) + mockSuccessfulGithubAddLabelQueueHeadCall(prt.GhClt, nextPRNr).AnyTimes() + mockCreateCommitStatusSuccessful(prt.GhClt).AnyTimes() + + nextPR, err := NewPullRequest(nextPRNr, "next_branch", "me", "next pr", "") + require.NoError(t, err) + + require.NoError(t, prt.Q.Enqueue(prt.PR)) + require.NoError(t, prt.Q.Enqueue(nextPR)) + + require.Eventually(t, + func() bool { + first := prt.Q.FirstActive() + return first != nil && first.Number == nextPRNr && updateBranchCallsNextPR.Load() > 0 + }, + condWaitTimeout, condCheckInterval, + "the pr after the dequeued one was not processed", + ) + + assert.Equal(t, 1, prt.Q.activeLen()) + assert.Zero(t, updateBranchCallsFirstPR.Load(), "branch of the dequeued pr was updated") + assert.Zero(t, emptied.Load(), "queue is not empty") +} + +func suspendPR(q *queue, pr *PullRequest) { + q.lock.Lock() + defer q.lock.Unlock() + q.suspended[pr.Number] = pr +} + +func TestResumeIfPRMergeStatusPositive_DequeuesPRThatDoesNotMeetTriggerAnymore(t *testing.T) { + var emptied atomic.Int32 + + // A suspended PR had its label already removed and its status reset when + // it was suspended, calls for it would fail the test. + prt := initEvalPRActionTest(t, + withEnqueueTrigger(false, set.From([]string{triggerLabel})), + withOnEmpty(func() { emptied.Add(1) }), + ) + mockReadyForMergeStatusConst(prt.GhClt, approvedStatus(false, "bug")) + suspendPR(prt.Q, prt.PR) + + err := prt.Q.resumeIfPRMergeStatusPositive(context.Background(), prt.L, prt.PR) + require.NoError(t, err) + + assert.Zero(t, prt.Q.suspendedLen(), "pr is still in suspend queue") + assert.Zero(t, prt.Q.activeLen(), "pr was moved to active queue") + assert.Equal(t, int32(1), emptied.Load()) +} + +func TestResumeIfPRMergeStatusPositive_DequeuesClosedPR(t *testing.T) { + var emptied atomic.Int32 + + prt := initEvalPRActionTest(t, + withEnqueueTrigger(true, nil), + withOnEmpty(func() { emptied.Add(1) }), + ) + status := approvedStatus(true) + status.Closed = true + mockReadyForMergeStatusConst(prt.GhClt, status) + suspendPR(prt.Q, prt.PR) + + err := prt.Q.resumeIfPRMergeStatusPositive(context.Background(), prt.L, prt.PR) + require.NoError(t, err) + + assert.Zero(t, prt.Q.suspendedLen(), "pr is still in suspend queue") + assert.Zero(t, prt.Q.activeLen(), "pr was moved to active queue") + assert.Equal(t, int32(1), emptied.Load()) +} + +func TestResumeIfPRMergeStatusPositive_KeepsPRThatMeetsTrigger(t *testing.T) { + var emptied atomic.Int32 + + prt := initEvalPRActionTest(t, + withEnqueueTrigger(true, nil), + withOnEmpty(func() { emptied.Add(1) }), + ) + status := approvedStatus(true) + status.ReviewDecision = githubclt.ReviewDecisionReviewRequired + mockReadyForMergeStatusConst(prt.GhClt, status) + suspendPR(prt.Q, prt.PR) + + err := prt.Q.resumeIfPRMergeStatusPositive(context.Background(), prt.L, prt.PR) + require.NoError(t, err) + + assert.Equal(t, 1, prt.Q.suspendedLen(), "pr was removed from suspend queue") + assert.Zero(t, emptied.Load()) +} + +// TestMissedAutoMergeDisabledEventDequeuesPRAndRemovesQueue verifies that a +// queued PR for that GitHub never delivered an auto_merge_disabled event, does +// not block the queue forever. +// When the base branch changes, the PR is removed from the queue, the queue head +// label is removed and the empty queue is removed from the coordinator. +func TestMissedAutoMergeDisabledEventDequeuesPRAndRemovesQueue(t *testing.T) { + const prNumber = 1 + const prBranch = "pr_branch" + + evChan := make(chan *github_prov.Event, 1) + defer close(evChan) + + mockctrl := gomock.NewController(t) + ghClient := mocks.NewMockGithubClient(mockctrl) + ciClient := mocks.NewMockJenkinsClient(mockctrl) + forwardLogsToTestLogger(t) + + // Called once, when the PR is enqueued. When the PR is processed + // after it is ineligible, the branch must not be updated. + mockSuccessfulGithubUpdateBranchCall(ghClient, prNumber, true).Times(1) + status := mockReadyForMergeStatus( + ghClient, prNumber, + githubclt.ReviewDecisionApproved, githubclt.CIStatusExpected, + ) + status.AnyTimes() + mockSuccessfulGithubRemoveLabelQueueHeadCall(ghClient, prNumber).Times(1) + mockCreateHeadCommitStatusPendingPRNr(ghClient, prNumber).Times(1) + + autoupdater := newAutoupdater( + ghClient, + ciClient, + evChan, + []Repository{{OwnerLogin: repoOwner, RepositoryName: repo}}, + true, + nil, + ) + autoupdater.Start() + t.Cleanup(autoupdater.Stop) + + baseBranch, err := NewBaseBranch(repoOwner, repo, "main") + require.NoError(t, err) + + evChan <- &github_prov.Event{Event: newPullRequestAutomergeEnabledEvent(prNumber, prBranch, baseBranch.Branch)} + waitForProcessedEventCnt(t, autoupdater, 1) + + q := autoupdater.getQueue(&baseBranch.BranchID) + require.NotNil(t, q) + require.Equal(t, 1, q.activeLen()) + // wait until the PR was processed once, it is the first in the queue + waitForQueueUpdateRunsGreaterThan(t, q, 0) + + // auto-merge is disabled on GitHub but the auto_merge_disabled event is lost + status.mu.Lock() + status.AutoMergeEnabled = false + status.mu.Unlock() + + evChan <- &github_prov.Event{Event: newPushEvent(baseBranch.Branch)} + waitForProcessedEventCnt(t, autoupdater, 2) + + require.Eventually(t, + func() bool { return autoupdater.getQueue(&baseBranch.BranchID) == nil }, + condWaitTimeout, condCheckInterval, + "empty queue was not removed after the pr was dequeued", + ) +} + +// A webhook event can dequeue a PR while its status is retrieved from GitHub. +// The queue must handle it as no failure and report that it became empty. +// The PR is dequeued by the event, not by the queue itself, therefore onEmpty +// can only be called by the code under test. +func TestResumeIfPRMergeStatusPositive_PRDequeuedConcurrently(t *testing.T) { + var emptied atomic.Int32 + + prt := initEvalPRActionTest(t, + withEnqueueTrigger(true, nil), + withOnEmpty(func() { emptied.Add(1) }), + ) + suspendPR(prt.Q, prt.PR) + + prt.GhClt.EXPECT(). + ReadyForMerge(gomock.Any(), gomock.Eq(repoOwner), gomock.Eq(repo), gomock.Eq(prNR)). + DoAndReturn(func(context.Context, string, string, int) (*githubclt.ReadyForMergeStatus, error) { + _, err := prt.Q.Dequeue(prNR, true) + require.NoError(t, err) + + return approvedStatus(false), nil + }). + Times(1) + + err := prt.Q.resumeIfPRMergeStatusPositive(context.Background(), prt.L, prt.PR) + require.ErrorIs(t, err, ErrNotFound) + + assert.Equal(t, int32(1), emptied.Load()) +} + +func TestProcessPR_PRDequeuedConcurrently(t *testing.T) { + var emptied atomic.Int32 + + prt := initEvalPRActionTest(t, + withEnqueueTrigger(true, nil), + withOnEmpty(func() { emptied.Add(1) }), + ) + mockNoUpdateBranchCalls(prt.GhClt) + + // The label is removed and the status reset once, by the event. The + // queue must not repeat it when it finds out that the PR is gone. + mockSuccessfulGithubRemoveLabelQueueHeadCall(prt.GhClt, prNR).Times(1) + mockCreateHeadCommitStatusPendingPRNr(prt.GhClt, prNR).Times(1) + + prt.GhClt.EXPECT(). + ReadyForMerge(gomock.Any(), gomock.Eq(repoOwner), gomock.Eq(repo), gomock.Eq(prNR)). + DoAndReturn(func(context.Context, string, string, int) (*githubclt.ReadyForMergeStatus, error) { + _, err := prt.Q.Dequeue(prNR, true) + require.NoError(t, err) + + return approvedStatus(false), nil + }). + Times(1) + + // Insert without scheduling processPR, it is run directly. + prt.Q.lock.Lock() + _, added := prt.Q.active.InsertIfNotExist(prt.PR.Number, prt.PR) + prt.Q.lock.Unlock() + require.True(t, added) + + prt.Q.processPR(context.Background(), prt.PR, TaskNone) + + assert.True(t, prt.Q.IsEmpty()) + assert.Equal(t, int32(1), emptied.Load()) +} diff --git a/internal/mergequeue/queue_processpr.go b/internal/mergequeue/queue_processpr.go index b6c52c8..d5cb6dc 100644 --- a/internal/mergequeue/queue_processpr.go +++ b/internal/mergequeue/queue_processpr.go @@ -34,6 +34,11 @@ type requiredActions struct { Reason string HeadCommitID string + // ResetStatusOnDequeue is only used when ActionDequeue is part of + // Actions. If it is true, the github commit status is reset to pending + // when the pull request is dequeued. + ResetStatusOnDequeue bool + // ExpectedCIRuns is only set when ActionTriggerCIJobs is part of Actions. ExpectedCIRuns []string } @@ -68,7 +73,6 @@ func (a Action) String() string { func (q *queue) evalPRAction(ctx context.Context, logger *zap.Logger, pr *PullRequest) (*requiredActions, error) { const ( reasonNotApproved = "not approved" - reasonPullRequestClosed = "pull request is closed" reasonMergeConflict = "update with base branch failed: merge conflict" reasonBranchUpdatedWithBase = "pull request branch has been updated with base branch" reasonIsStale = "pull request is stale" @@ -84,6 +88,22 @@ func (q *queue) evalPRAction(ctx context.Context, logger *zap.Logger, pr *PullRe return nil, fmt.Errorf("retrieving ready to merge status from github failed: %w", err) } + // The webhook event that removes a PR from the queue (auto_merge_disabled, + // unlabeled, closed) can be lost. Check it here, otherwise the PR blocks + // the head of the queue, although GitHub will never merge it. + // This must be done before anything else, to prevent that the branch of the + // PR is updated, it is labeled or CI jobs are triggered for it. + if req := q.dequeueRequired(status); req != nil { + // ActionDequeue cancels the running processPR context via + // cancelActionForPR, so no context-using actions may follow it. + return &requiredActions{ + Actions: []Action{ActionDequeue}, + Reason: req.Reason, + HeadCommitID: status.Commit, + ResetStatusOnDequeue: req.ResetStatus, + }, nil + } + if status.ReviewDecision != githubclt.ReviewDecisionApproved { return &requiredActions{ Actions: []Action{ActionSuspend}, diff --git a/internal/mergequeue/queue_processpr_test.go b/internal/mergequeue/queue_processpr_test.go index 6adb5e8..e5b3e4d 100644 --- a/internal/mergequeue/queue_processpr_test.go +++ b/internal/mergequeue/queue_processpr_test.go @@ -22,7 +22,7 @@ type evalPRTest struct { GhClt *mocks.MockGithubClient } -func initEvalPRActionTest(t *testing.T) *evalPRTest { +func initEvalPRActionTest(t *testing.T, queueOpts ...queueOption) *evalPRTest { l := forwardLogsToTestLogger(t) mockctrl := gomock.NewController(t, gomock.WithOverridableExpectations()) @@ -39,7 +39,7 @@ func initEvalPRActionTest(t *testing.T) *evalPRTest { bb, err := NewBaseBranch(repoOwner, repo, "main") require.NoError(t, err) ci := &CI{Client: ciClient, logger: l} - q := newQueue(bb, l, ghClient, retry.NewRetryer(), ci, queueHeadLabel) + q := newQueue(bb, l, ghClient, retry.NewRetryer(), ci, queueHeadLabel, queueOpts...) t.Cleanup(q.Stop) pr, err := NewPullRequest(prNR, prBranch, "me", "test pr", "")