From 4bce6ffb02bd8ffac646e56a4dc299ea5cadf9ce Mon Sep 17 00:00:00 2001 From: Kerem Goksel Date: Tue, 17 Nov 2026 20:53:59 -0800 Subject: [PATCH 1/2] test(orchestrator): reproduce stale redelivery in validate and landconflictsignal Record what the two controllers do today when a message arrives for a request a batch has already claimed. Unit: landconflictsignal writes Validated over Batched and republishes to the batch topic on a landable result, and terminates the request on a not-landable one. validate re-runs the validator and either terminates the request or republishes the land-conflict check. E2E: with the request's build gated, a replayed SUCCEEDED result moves the request from Batched back to Validated while its batch is still speculating. A replayed FAILED result fails the request while its batch goes on to succeed, a cancel does not reach the batch, and a resubmission of the same URI is admitted and lands. --- .../landconflictsignal_test.go | 88 ++++++++++++ .../controller/validate/validate_test.go | 113 +++++++++++++++ test/e2e/submitqueue/BUILD.bazel | 1 + test/e2e/submitqueue/harness_test.go | 57 ++++++++ test/e2e/submitqueue/queues.yaml | 2 + test/e2e/submitqueue/suite_test.go | 134 ++++++++++++++++++ 6 files changed, 395 insertions(+) diff --git a/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal_test.go b/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal_test.go index 9947651e2..1ec161fbf 100644 --- a/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal_test.go +++ b/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal_test.go @@ -238,3 +238,91 @@ func TestProcess_HaltedRequestSkips(t *testing.T) { msg := entityqueue.NewMessage(testRequestID, resultPayload(t, res), testQueue, nil) require.NoError(t, controller.Process(context.Background(), newDelivery(ctrl, msg))) } + +// A result redelivered after the request has moved past validation. The +// expectations record what the controller does today: a SUCCEEDED result writes +// Validated over the later state and hands the request to the batch stage +// again, and a FAILED one terminates the request out from under its batch. +func TestProcess_SignalForAdvancedRequest(t *testing.T) { + tests := []struct { + name string + state entity.RequestState + outcome runwaypb.Outcome + wantState entity.RequestState + wantStatus entity.RequestStatus + wantTopics []string + }{ + { + name: "batched, landable", + state: entity.RequestStateBatched, + outcome: runwaypb.Outcome_SUCCEEDED, + wantState: entity.RequestStateValidated, + wantStatus: entity.RequestStatusValidated, + wantTopics: []string{"log", "batch"}, + }, + { + name: "batched, not landable", + state: entity.RequestStateBatched, + outcome: runwaypb.Outcome_FAILED, + wantState: entity.RequestStateError, + wantStatus: entity.RequestStatusError, + wantTopics: []string{"log"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + + request := entity.Request{ID: testRequestID, Queue: testQueue, State: tt.state, Version: 3} + var written []entity.Request + reqStore := storagemock.NewMockRequestStore(ctrl) + reqStore.EXPECT().Get(gomock.Any(), testRequestID).Return(request, nil).AnyTimes() + reqStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(3), int32(4)).DoAndReturn( + func(_ context.Context, updated entity.Request, _, _ int32) error { + written = append(written, updated) + return nil + }, + ).AnyTimes() + + store := orchstoragemock.NewMockStorage(ctrl) + store.EXPECT().GetRequestStore().Return(reqStore).AnyTimes() + + var gotTopics []string + var gotPayloads [][]byte + pub := queuemock.NewMockPublisher(ctrl) + pub.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, topic string, msg entityqueue.Message) error { + gotTopics = append(gotTopics, topic) + gotPayloads = append(gotPayloads, msg.Payload) + return nil + }, + ).AnyTimes() + q := queuemock.NewMockQueue(ctrl) + q.EXPECT().Publisher().Return(pub).AnyTimes() + registry, err := consumer.NewTopicRegistry( + []consumer.TopicConfig{ + {Key: topickey.TopicKeyBatch, Name: "batch", Queue: q}, + {Key: topickey.TopicKeyLog, Name: "log", Queue: q}, + }, + ) + require.NoError(t, err) + + controller := NewController(zaptest.NewLogger(t).Sugar(), tally.NoopScope, staticStorageFactory{store: store}, registry, + runwaymq.TopicKeyMergeConflictCheckSignal, "orchestrator-landconflictsignal") + + res := runwaymq.MergeResult{Id: testRequestID, Outcome: tt.outcome, Reason: "replayed"} + msg := entityqueue.NewMessage(testRequestID, resultPayload(t, res), testQueue, nil) + require.NoError(t, controller.Process(context.Background(), newDelivery(ctrl, msg))) + + require.Len(t, written, 1) + assert.Equal(t, tt.wantState, written[0].State) + + require.Equal(t, tt.wantTopics, gotTopics) + logEntry, err := sqmq.UnmarshalRequestLog(gotPayloads[0]) + require.NoError(t, err) + assert.Equal(t, tt.wantStatus, logEntry.Status) + assert.Equal(t, int32(4), logEntry.RequestVersion) + }) + } +} diff --git a/submitqueue/orchestrator/controller/validate/validate_test.go b/submitqueue/orchestrator/controller/validate/validate_test.go index 07f543b7e..43667de82 100644 --- a/submitqueue/orchestrator/controller/validate/validate_test.go +++ b/submitqueue/orchestrator/controller/validate/validate_test.go @@ -891,3 +891,116 @@ func TestController_Process_CustomValidatorFailure_TerminationPublishFails(t *te require.Error(t, err) assert.ErrorContains(t, err, "failed to terminate rejected request") } + +// A validate message redelivered after the request has moved past validation. +// The expectations record what the controller does today: it reports +// "validating" again, re-runs the validator, and then either fails the request +// out from under its batch or asks runway for the land-conflict check again. +func TestController_Process_RedeliveryForAdvancedRequest(t *testing.T) { + tests := []struct { + name string + state entity.RequestState + validatorErr error + wantWrittenStates []entity.RequestState + wantLogStatuses []entity.RequestStatus + wantCheckPublishes int + }{ + { + name: "batched, validator rejects", + state: entity.RequestStateBatched, + validatorErr: fmt.Errorf("%w: policy changed", validator.ErrRejected), + wantWrittenStates: []entity.RequestState{entity.RequestStateError}, + wantLogStatuses: []entity.RequestStatus{entity.RequestStatusValidating, entity.RequestStatusError}, + }, + { + name: "batched, validator passes", + state: entity.RequestStateBatched, + wantLogStatuses: []entity.RequestStatus{entity.RequestStatusValidating}, + wantCheckPublishes: 1, + }, + { + name: "validated, validator passes", + state: entity.RequestStateValidated, + wantLogStatuses: []entity.RequestStatus{entity.RequestStatusValidating}, + wantCheckPublishes: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + + request := entity.Request{ + ID: "123", + Queue: "test-queue", + Change: change.Change{URIs: []string{"github://uber/service/pull/456/abcdef0123456789abcdef0123456789abcdef01"}}, + LandStrategy: mergestrategy.MergeStrategyRebase, + State: tt.state, + Version: 3, + } + store, mockReqStore := newMockStorage(ctrl, request) + store.EXPECT().GetChangeStore().Return(newMockChangeStore(ctrl)).AnyTimes() + var writtenStates []entity.RequestState + mockReqStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(3), int32(4)).DoAndReturn( + func(_ context.Context, updated entity.Request, _, _ int32) error { + writtenStates = append(writtenStates, updated.State) + return nil + }, + ).AnyTimes() + + var logStatuses []entity.RequestStatus + checkPublishes := 0 + mockPub := queuemock.NewMockPublisher(ctrl) + mockPub.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, topic string, msg entityqueue.Message) error { + if topic == "merge-conflict-check" { + checkPublishes++ + return nil + } + entry, err := sqmq.UnmarshalRequestLog(msg.Payload) + require.NoError(t, err) + logStatuses = append(logStatuses, entry.Status) + return nil + }, + ).AnyTimes() + mockQ := queuemock.NewMockQueue(ctrl) + mockQ.EXPECT().Publisher().Return(mockPub).AnyTimes() + registry, err := consumer.NewTopicRegistry( + []consumer.TopicConfig{ + {Key: runwaymq.TopicKeyMergeConflictCheck, Name: "merge-conflict-check", Queue: mockQ}, + {Key: topickey.TopicKeyLog, Name: "log", Queue: mockQ}, + }, + ) + require.NoError(t, err) + + cpFactory := changeprovidermock.NewMockFactory(ctrl) + cpFactory.EXPECT().For(gomock.Any()).Return(&mockChangeProvider{}, nil).AnyTimes() + + validatorCalls := 0 + mockValidator := validatormock.NewMockValidator(ctrl) + mockValidator.EXPECT().Validate(gomock.Any(), gomock.Any()).DoAndReturn( + func(context.Context, entity.Request) error { + validatorCalls++ + return tt.validatorErr + }, + ).AnyTimes() + mockValidatorFactory := validatormock.NewMockFactory(ctrl) + mockValidatorFactory.EXPECT().For(gomock.Any()).Return(mockValidator, nil).AnyTimes() + + controller := NewController(zaptest.NewLogger(t).Sugar(), tally.NoopScope, staticStorageFactory{store: store}, registry, cpFactory, mockValidatorFactory, runwaymq.TopicKeyMergeConflictCheck, topickey.TopicKeyValidate, "orchestrator-validate") + + msg := entityqueue.NewMessage(request.ID, requestIDPayload(t, request.ID), request.Queue, nil) + msg.Tenant = request.Queue + delivery := consumermock.NewMockDelivery(ctrl) + delivery.EXPECT().Message().Return(msg).AnyTimes() + delivery.EXPECT().Attempt().Return(1).AnyTimes() + + require.NoError(t, controller.Process(context.Background(), delivery)) + + assert.Equal(t, 1, validatorCalls) + assert.Equal(t, tt.wantWrittenStates, writtenStates) + assert.Equal(t, tt.wantLogStatuses, logStatuses) + assert.Equal(t, tt.wantCheckPublishes, checkPublishes) + }) + } +} diff --git a/test/e2e/submitqueue/BUILD.bazel b/test/e2e/submitqueue/BUILD.bazel index 0734554b3..403e238a5 100644 --- a/test/e2e/submitqueue/BUILD.bazel +++ b/test/e2e/submitqueue/BUILD.bazel @@ -58,6 +58,7 @@ go_test( "//api/base/change/protopb:go_default_library", "//api/base/mergestrategy/protopb:go_default_library", "//api/runway/messagequeue:go_default_library", + "//api/runway/messagequeue/protopb:go_default_library", "//api/submitqueue/gateway/protopb:go_default_library", "//api/submitqueue/orchestrator/protopb:go_default_library", "//platform/base/change:go_default_library", diff --git a/test/e2e/submitqueue/harness_test.go b/test/e2e/submitqueue/harness_test.go index 48ba9ca68..0ae1fd158 100644 --- a/test/e2e/submitqueue/harness_test.go +++ b/test/e2e/submitqueue/harness_test.go @@ -34,6 +34,8 @@ import ( "github.com/uber-go/tally" changepb "github.com/uber/submitqueue/api/base/change/protopb" mergestrategypb "github.com/uber/submitqueue/api/base/mergestrategy/protopb" + runwaymq "github.com/uber/submitqueue/api/runway/messagequeue" + runwaypb "github.com/uber/submitqueue/api/runway/messagequeue/protopb" gatewaypb "github.com/uber/submitqueue/api/submitqueue/gateway/protopb" entityqueue "github.com/uber/submitqueue/platform/base/messagequeue" "github.com/uber/submitqueue/platform/consumer" @@ -310,6 +312,61 @@ func (s *E2EIntegrationSuite) publishIDMessage(queueName string, topic consumer. return messageID } +// redeliverLandConflictSignal publishes a second land-conflict result for the +// request, which is what the orchestrator sees when the signal delivery is +// retried after its ack was lost. The message ID is fresh for the reason given +// on redeliverBatchMessage. +func (s *E2EIntegrationSuite) redeliverLandConflictSignal(req request, outcome runwaypb.Outcome, reason string) { + t := s.T() + t.Helper() + + queue, err := queuemysql.NewQueue(queuemysql.Params{ + DB: s.queueDB, + Logger: zap.NewNop(), + MetricsScope: tally.NoopScope, + Tenants: []string{req.queue}, + }) + require.NoError(t, err, "failed to open the queue for a manual publish") + defer func() { require.NoError(t, queue.Close()) }() + + topic := runwaymq.TopicKeyMergeConflictCheckSignal + registry, err := consumer.NewTopicRegistry([]consumer.TopicConfig{ + {Key: topic, Name: topic.String(), Queue: queue}, + }) + require.NoError(t, err) + + payload, err := runwaymq.Marshal(&runwaymq.MergeResult{Id: req.sqid, QueueName: req.queue, Outcome: outcome, Reason: reason}) + require.NoError(t, err) + require.NoError(t, publish.Message(entityqueue.WithQueueName(s.ctx, req.queue), registry, topic, publish.MessageParams{ + Tenant: req.queue, + ID: publish.UniqueID(req.sqid), + Payload: payload, + PartitionKey: req.queue, + }), "failed to publish %s message", topic) +} + +// queueMessageIDs returns the IDs of every row the queue still holds for the +// topic, in offset order. A publish that deduplicated wrote no row, so the list +// shows whether a republish reached the topic. +func (s *E2EIntegrationSuite) queueMessageIDs(queueName string, topic consumer.TopicKey) []string { + t := s.T() + t.Helper() + + rows, err := s.queueDB.QueryContext(s.ctx, + "SELECT id FROM queue_messages WHERE tenant = ? AND topic = ? ORDER BY `offset`", queueName, topic.String()) + require.NoError(t, err, "failed to read queue rows for %s", topic) + defer rows.Close() + + ids := []string{} + for rows.Next() { + var id string + require.NoError(t, rows.Scan(&id)) + ids = append(ids, id) + } + require.NoError(t, rows.Err()) + return ids +} + // batchState reads a batch's current state from the operating store. func (s *E2EIntegrationSuite) batchState(queue, batchID string) entity.BatchState { t := s.T() diff --git a/test/e2e/submitqueue/queues.yaml b/test/e2e/submitqueue/queues.yaml index 7ee2ad5c5..a87530a51 100644 --- a/test/e2e/submitqueue/queues.yaml +++ b/test/e2e/submitqueue/queues.yaml @@ -9,6 +9,8 @@ queues: - name: e2e-partial-coverage-queue - name: e2e-land-wakeup-queue - name: e2e-redelivery-queue + - name: e2e-stale-validation-queue + - name: e2e-stale-not-landable-queue - name: e2e-strand-queue - name: e2e-conflict-error-queue - name: e2e-respeculate-queue diff --git a/test/e2e/submitqueue/suite_test.go b/test/e2e/submitqueue/suite_test.go index c7bc2f182..136803e10 100644 --- a/test/e2e/submitqueue/suite_test.go +++ b/test/e2e/submitqueue/suite_test.go @@ -39,10 +39,14 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/uber-go/tally" + changepb "github.com/uber/submitqueue/api/base/change/protopb" + mergestrategypb "github.com/uber/submitqueue/api/base/mergestrategy/protopb" runwaymq "github.com/uber/submitqueue/api/runway/messagequeue" + runwaypb "github.com/uber/submitqueue/api/runway/messagequeue/protopb" gatewaypb "github.com/uber/submitqueue/api/submitqueue/gateway/protopb" orchestratorpb "github.com/uber/submitqueue/api/submitqueue/orchestrator/protopb" consumergatefile "github.com/uber/submitqueue/platform/extension/consumergate/file" + "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" yamlqueueconfig "github.com/uber/submitqueue/submitqueue/extension/queueconfig/yaml" storagemysql "github.com/uber/submitqueue/submitqueue/orchestrator/extension/storage/mysql" @@ -780,6 +784,136 @@ func (s *E2EIntegrationSuite) TestBatchRedelivery_DoesNotEnrolTheRequestTwice() s.awaitStatus(settle, entity.RequestStatusLanded) } +// A validate message and a SUCCEEDED land-conflict result replayed after the +// request has been claimed by a batch. The expectations record what the +// pipeline does today: the replayed result writes Validated over Batched while +// the batch that owns the request is still running. +// +// The build gate holds the request in Batched for the replays, as in +// TestBatchRedelivery_DoesNotEnrolTheRequestTwice. The second request is the +// settle signal for the replayed validate: validate, the conflict check and its +// signal are each consumed in order per queue, so its batch existing proves +// anything the replay set off for the first request has already been handled. +func (s *E2EIntegrationSuite) TestStaleValidationMessages_RewindABatchedRequest() { + t := s.T() + + const queue = "e2e-stale-validation-queue" + const gateGroup = "orchestrator" + const heldBatch = "1" + + s.closeGate(queue, gateGroup, heldBatch, "e2e: hold the build so the request stays batched for the replays") + defer s.openGate(queue, gateGroup, heldBatch) + + req := s.land(queue, "github://github.example.com/uber/e2e-stale-validation/pull/1/abcdef0123456789abcdef0123456789abcdef01") + require.Equal(t, heldBatch, s.awaitBatchID(req), "the first batch of a fresh queue must be batch/1") + s.awaitStatus(req, entity.RequestStatusSpeculating) + require.Equal(t, entity.RequestStateBatched, s.terminalState(req)) + + s.publishIDMessage(queue, topickey.TopicKeyValidate, req.sqid) + settle := s.land(queue, "github://github.example.com/uber/e2e-stale-validation/pull/2/1234567890abcdef1234567890abcdef12345678") + s.awaitBatchID(settle) + s.log.Logf("STALE-REPRO after replayed validate: request %s is %q", req.sqid, s.terminalState(req)) + + s.redeliverLandConflictSignal(req, runwaypb.Outcome_SUCCEEDED, "") + pollUntil(persistPollInterval, func() bool { + got := s.terminalState(req) + s.log.Logf("STALE-REPRO after replayed signal: request %s is %q", req.sqid, got) + return got == entity.RequestStateValidated + }) + assert.Equal(t, entity.BatchStateSpeculating, s.batchState(queue, heldBatch), + "the batch that owns the request is still running") + s.log.Logf("STALE-REPRO batches for request %s: %v", req.sqid, s.batchIDsFor(req)) + // The request ID is the message ID on both topics, so one row per request + // means the republish from each replay deduplicated. + batchRows := s.queueMessageIDs(queue, topickey.TopicKeyBatch) + checkRows := s.queueMessageIDs(queue, runwaymq.TopicKeyMergeConflictCheck) + s.log.Logf("STALE-REPRO queue rows: batch=%v merge-conflict-check=%v signal=%v validate=%v", batchRows, checkRows, + s.queueMessageIDs(queue, runwaymq.TopicKeyMergeConflictCheckSignal), s.queueMessageIDs(queue, topickey.TopicKeyValidate)) + assert.Equal(t, []string{req.sqid, settle.sqid}, batchRows, "the republished batch message deduplicated") + assert.Equal(t, []string{req.sqid, settle.sqid}, checkRows, "the republished land-conflict check deduplicated") + + s.openGate(queue, gateGroup, heldBatch) + s.awaitStatus(req, entity.RequestStatusLanded) + s.awaitStatus(settle, entity.RequestStatusLanded) + s.log.Logf("STALE-REPRO timeline for request %s: %v", req.sqid, s.timeline(req)) +} + +// A not-landable land-conflict result replayed after the request has been +// claimed by a batch. The expectations record what the pipeline does today: the +// request is failed while its batch runs on, the batch still merges the change, +// the user can no longer cancel it, and the same change is admitted again. +// +// The build gate holds the request in Batched for the replay, as in +// TestBatchRedelivery_DoesNotEnrolTheRequestTwice. +func (s *E2EIntegrationSuite) TestStaleNotLandableSignal_FailsABatchedRequest() { + t := s.T() + + const queue = "e2e-stale-not-landable-queue" + const gateGroup = "orchestrator" + const heldBatch = "1" + const uri = "github://github.example.com/uber/e2e-stale-not-landable/pull/1/abcdef0123456789abcdef0123456789abcdef01" + + s.closeGate(queue, gateGroup, heldBatch, "e2e: hold the build so the request stays batched for the replay") + defer s.openGate(queue, gateGroup, heldBatch) + + req := s.land(queue, uri) + require.Equal(t, heldBatch, s.awaitBatchID(req), "the first batch of a fresh queue must be batch/1") + s.awaitStatus(req, entity.RequestStatusSpeculating) + require.Equal(t, entity.RequestStateBatched, s.terminalState(req)) + + s.redeliverLandConflictSignal(req, runwaypb.Outcome_FAILED, "replayed conflict") + pollUntil(persistPollInterval, func() bool { + got := s.terminalState(req) + s.log.Logf("STALE-FAILED after replayed signal: request %s is %q", req.sqid, got) + return got == entity.RequestStateError + }) + s.awaitStatus(req, entity.RequestStatusError) + s.log.Logf("STALE-FAILED request %s: status=%q last_error=%q batch %s is %q", + req.sqid, s.mustStatus(req), s.lastError(req), heldBatch, s.batchState(queue, heldBatch)) + assert.Equal(t, entity.BatchStateSpeculating, s.batchState(queue, heldBatch), + "the batch that owns the request is still running") + + cancelResp, cancelErr := s.gatewayClient.Cancel(s.ctx, &gatewaypb.CancelRequest{Sqid: req.sqid, Queue: queue, Reason: "e2e stale not-landable"}) + s.log.Logf("STALE-FAILED cancel while batch %s is live: response=%v error=%v", heldBatch, cancelResp, cancelErr) + + // The resubmission is behind the cancel on the wire, and its own batch + // serializes behind the held one, so it is observed while that is live. + resubmitResp, resubmitErr := s.gatewayClient.Land(s.ctx, &gatewaypb.LandRequest{ + Queue: queue, + Change: &changepb.Change{Uris: []string{uri}}, + Strategy: mergestrategypb.Strategy_REBASE, + }) + s.log.Logf("STALE-FAILED resubmission of the same URI: response=%v error=%v", resubmitResp, resubmitErr) + require.NoError(t, resubmitErr) + resubmit := request{queue: queue, sqid: resubmitResp.Sqid} + var resubmitStatus entity.RequestStatus + pollUntil(persistPollInterval, func() bool { + resubmitStatus, _ = s.currentStatus(resubmit) + s.log.Logf("STALE-FAILED resubmission %s is %q", resubmit.sqid, resubmitStatus) + switch resubmitStatus { + case entity.RequestStatusUnknown, entity.RequestStatusAccepted, entity.RequestStatusStarted, + entity.RequestStatusValidating, entity.RequestStatusValidated, entity.RequestStatusBatching: + return false + } + return true + }) + s.log.Logf("STALE-FAILED resubmission %s: status=%q last_error=%q batches=%v; batch %s is %q after the cancel", + resubmit.sqid, resubmitStatus, s.lastError(resubmit), s.batchIDsFor(resubmit), heldBatch, s.batchState(queue, heldBatch)) + assert.False(t, isTerminalStatus(resubmitStatus), "the same change is admitted while the first batch is live") + assert.Equal(t, entity.BatchStateSpeculating, s.batchState(queue, heldBatch), "the cancel did not reach the live batch") + + s.openGate(queue, gateGroup, heldBatch) + pollUntil(persistPollInterval, func() bool { return s.batchState(queue, heldBatch).IsTerminal() }) + assert.Equal(t, entity.BatchStateSucceeded, s.batchState(queue, heldBatch), "the batch merges the change of a request already failed") + resubmitFinal := s.awaitTerminal(resubmit) + + s.log.Logf("STALE-FAILED final: request %s state=%q status=%q timeline=%v; batch %s is %q", + req.sqid, s.terminalState(req), s.mustStatus(req), s.timeline(req), heldBatch, s.batchState(queue, heldBatch)) + s.log.Logf("STALE-FAILED final: resubmission %s state=%q status=%q timeline=%v batches=%v", + resubmit.sqid, s.terminalState(resubmit), resubmitFinal, s.timeline(resubmit), s.batchIDsFor(resubmit)) + assert.Equal(t, entity.RequestStateError, s.terminalState(req), "the request stays failed although its batch merged") +} + // A batch left in Created is dependency-eligible, so everything created after // it serializes behind it — and nothing will name it on the speculate topic // again. Without a run that looks for such batches the queue wedges here with From 955270c1ca64e97a1d7a16c95c307321ccb7b7dc Mon Sep 17 00:00:00 2001 From: Kerem Goksel Date: Tue, 17 Nov 2026 20:53:59 -0800 Subject: [PATCH 2/2] fix(orchestrator): skip validate and landconflictsignal for requests past validation A redelivered validate message or land-conflict result could reach a request that a batch had already claimed. validate re-ran the validator and could terminate the request to Error; landconflictsignal wrote Validated over Batched, or terminated the request on a not-landable result. From Batched onward the batch owns the request's outcome, so both controllers now ack such deliveries without writing or publishing. landconflictsignal still lets Validated through: a retry lands there when the state write succeeded and the hand-off to batch did not. --- .../landconflictsignal/landconflictsignal.go | 13 ++ .../landconflictsignal_test.go | 103 ++++++++------- .../controller/validate/validate.go | 12 ++ .../controller/validate/validate_test.go | 90 +++----------- test/e2e/submitqueue/harness_test.go | 22 ---- test/e2e/submitqueue/suite_test.go | 117 +++++------------- 6 files changed, 121 insertions(+), 236 deletions(-) diff --git a/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal.go b/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal.go index 267aa00ca..867a85ec6 100644 --- a/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal.go +++ b/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal.go @@ -128,6 +128,19 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er return nil } + // A result is only the request's outcome while validation still owns it. + // Validated is let through because a retry lands there after the state write + // succeeded and the hand-off to batch did not; past that the batch owns the + // request, and a replayed result must neither rewind nor terminate it. + if request.State != entity.RequestStateStarted && request.State != entity.RequestStateValidated { + metrics.NamedCounter(c.metricsScope, opName, "skipped_stale", 1, metrics.TagsFromContext(ctx)...) + c.logger.Infow("skipping landconflict signal for request past validation", + "request_id", request.ID, + "state", string(request.State), + ) + return nil + } + if result.Outcome != runwaypb.Outcome_SUCCEEDED { metrics.NamedCounter(c.metricsScope, opName, "not_landable", 1, metrics.TagsFromContext(ctx)...) c.logger.Infow("request not landable", diff --git a/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal_test.go b/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal_test.go index 1ec161fbf..5db01e369 100644 --- a/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal_test.go +++ b/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal_test.go @@ -239,65 +239,32 @@ func TestProcess_HaltedRequestSkips(t *testing.T) { require.NoError(t, controller.Process(context.Background(), newDelivery(ctrl, msg))) } -// A result redelivered after the request has moved past validation. The -// expectations record what the controller does today: a SUCCEEDED result writes -// Validated over the later state and hands the request to the batch stage -// again, and a FAILED one terminates the request out from under its batch. -func TestProcess_SignalForAdvancedRequest(t *testing.T) { +// A result redelivered after the request has moved past validation is acked +// without a state write or a publish: the batch owns the request by then. +func TestProcess_SignalForAdvancedRequestIsSkipped(t *testing.T) { tests := []struct { - name string - state entity.RequestState - outcome runwaypb.Outcome - wantState entity.RequestState - wantStatus entity.RequestStatus - wantTopics []string + name string + state entity.RequestState + outcome runwaypb.Outcome }{ - { - name: "batched, landable", - state: entity.RequestStateBatched, - outcome: runwaypb.Outcome_SUCCEEDED, - wantState: entity.RequestStateValidated, - wantStatus: entity.RequestStatusValidated, - wantTopics: []string{"log", "batch"}, - }, - { - name: "batched, not landable", - state: entity.RequestStateBatched, - outcome: runwaypb.Outcome_FAILED, - wantState: entity.RequestStateError, - wantStatus: entity.RequestStatusError, - wantTopics: []string{"log"}, - }, + {name: "batched, landable", state: entity.RequestStateBatched, outcome: runwaypb.Outcome_SUCCEEDED}, + {name: "batched, not landable", state: entity.RequestStateBatched, outcome: runwaypb.Outcome_FAILED}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ctrl := gomock.NewController(t) + // No Update expectation: gomock fails the test on any state write. request := entity.Request{ID: testRequestID, Queue: testQueue, State: tt.state, Version: 3} - var written []entity.Request reqStore := storagemock.NewMockRequestStore(ctrl) reqStore.EXPECT().Get(gomock.Any(), testRequestID).Return(request, nil).AnyTimes() - reqStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(3), int32(4)).DoAndReturn( - func(_ context.Context, updated entity.Request, _, _ int32) error { - written = append(written, updated) - return nil - }, - ).AnyTimes() store := orchstoragemock.NewMockStorage(ctrl) store.EXPECT().GetRequestStore().Return(reqStore).AnyTimes() - var gotTopics []string - var gotPayloads [][]byte + // No Publish expectation either. pub := queuemock.NewMockPublisher(ctrl) - pub.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( - func(_ context.Context, topic string, msg entityqueue.Message) error { - gotTopics = append(gotTopics, topic) - gotPayloads = append(gotPayloads, msg.Payload) - return nil - }, - ).AnyTimes() q := queuemock.NewMockQueue(ctrl) q.EXPECT().Publisher().Return(pub).AnyTimes() registry, err := consumer.NewTopicRegistry( @@ -314,15 +281,47 @@ func TestProcess_SignalForAdvancedRequest(t *testing.T) { res := runwaymq.MergeResult{Id: testRequestID, Outcome: tt.outcome, Reason: "replayed"} msg := entityqueue.NewMessage(testRequestID, resultPayload(t, res), testQueue, nil) require.NoError(t, controller.Process(context.Background(), newDelivery(ctrl, msg))) - - require.Len(t, written, 1) - assert.Equal(t, tt.wantState, written[0].State) - - require.Equal(t, tt.wantTopics, gotTopics) - logEntry, err := sqmq.UnmarshalRequestLog(gotPayloads[0]) - require.NoError(t, err) - assert.Equal(t, tt.wantStatus, logEntry.Status) - assert.Equal(t, int32(4), logEntry.RequestVersion) }) } } + +// A retry after the Validated write succeeded and the hand-off failed finds the +// request in Validated and must still reach the batch topic. +func TestProcess_LandableRetryFromValidatedRepublishesToBatch(t *testing.T) { + ctrl := gomock.NewController(t) + + request := entity.Request{ID: testRequestID, Queue: testQueue, State: entity.RequestStateValidated, Version: 2} + reqStore := storagemock.NewMockRequestStore(ctrl) + reqStore.EXPECT().Get(gomock.Any(), testRequestID).Return(request, nil) + reqStore.EXPECT().Update(gomock.Any(), request, int32(2), int32(3)).Return(nil) + + store := orchstoragemock.NewMockStorage(ctrl) + store.EXPECT().GetRequestStore().Return(reqStore).AnyTimes() + + var gotTopics []string + pub := queuemock.NewMockPublisher(ctrl) + pub.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, topic string, _ entityqueue.Message) error { + gotTopics = append(gotTopics, topic) + return nil + }, + ).AnyTimes() + q := queuemock.NewMockQueue(ctrl) + q.EXPECT().Publisher().Return(pub).AnyTimes() + registry, err := consumer.NewTopicRegistry( + []consumer.TopicConfig{ + {Key: topickey.TopicKeyBatch, Name: "batch", Queue: q}, + {Key: topickey.TopicKeyLog, Name: "log", Queue: q}, + }, + ) + require.NoError(t, err) + + controller := NewController(zaptest.NewLogger(t).Sugar(), tally.NoopScope, staticStorageFactory{store: store}, registry, + runwaymq.TopicKeyMergeConflictCheckSignal, "orchestrator-landconflictsignal") + + res := runwaymq.MergeResult{Id: testRequestID, Outcome: runwaypb.Outcome_SUCCEEDED} + msg := entityqueue.NewMessage(testRequestID, resultPayload(t, res), testQueue, nil) + require.NoError(t, controller.Process(context.Background(), newDelivery(ctrl, msg))) + + assert.Equal(t, []string{"log", "batch"}, gotTopics) +} diff --git a/submitqueue/orchestrator/controller/validate/validate.go b/submitqueue/orchestrator/controller/validate/validate.go index 808073c35..239bc1000 100644 --- a/submitqueue/orchestrator/controller/validate/validate.go +++ b/submitqueue/orchestrator/controller/validate/validate.go @@ -148,6 +148,18 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er return nil } + // Validation runs once, on a request nothing downstream has picked up yet. + // A redelivery that arrives after the conflict check came back would + // otherwise re-judge a request its batch now owns. + if request.State != entity.RequestStateStarted { + coremetrics.NamedCounter(c.metricsScope, "process", "skipped_stale", 1, coremetrics.TagsFromContext(ctx)...) + c.logger.Infow("skipping validate for request past validation", + "request_id", request.ID, + "state", string(request.State), + ) + return nil + } + // Report that validation has begun. This stage is not instantaneous — the // land-conflict check below is an async round trip to runway — so without // this the request reads "started" for the whole of it. No occurrence: a diff --git a/submitqueue/orchestrator/controller/validate/validate_test.go b/submitqueue/orchestrator/controller/validate/validate_test.go index 43667de82..fbfc0f98b 100644 --- a/submitqueue/orchestrator/controller/validate/validate_test.go +++ b/submitqueue/orchestrator/controller/validate/validate_test.go @@ -892,42 +892,15 @@ func TestController_Process_CustomValidatorFailure_TerminationPublishFails(t *te assert.ErrorContains(t, err, "failed to terminate rejected request") } -// A validate message redelivered after the request has moved past validation. -// The expectations record what the controller does today: it reports -// "validating" again, re-runs the validator, and then either fails the request -// out from under its batch or asks runway for the land-conflict check again. -func TestController_Process_RedeliveryForAdvancedRequest(t *testing.T) { - tests := []struct { - name string - state entity.RequestState - validatorErr error - wantWrittenStates []entity.RequestState - wantLogStatuses []entity.RequestStatus - wantCheckPublishes int - }{ - { - name: "batched, validator rejects", - state: entity.RequestStateBatched, - validatorErr: fmt.Errorf("%w: policy changed", validator.ErrRejected), - wantWrittenStates: []entity.RequestState{entity.RequestStateError}, - wantLogStatuses: []entity.RequestStatus{entity.RequestStatusValidating, entity.RequestStatusError}, - }, - { - name: "batched, validator passes", - state: entity.RequestStateBatched, - wantLogStatuses: []entity.RequestStatus{entity.RequestStatusValidating}, - wantCheckPublishes: 1, - }, - { - name: "validated, validator passes", - state: entity.RequestStateValidated, - wantLogStatuses: []entity.RequestStatus{entity.RequestStatusValidating}, - wantCheckPublishes: 1, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { +// A validate message redelivered after the request has moved past validation +// is acked without touching it: no "validating" log, no validator call, no +// state write, and no second land-conflict check. +func TestController_Process_RedeliveryForAdvancedRequestIsSkipped(t *testing.T) { + for _, state := range []entity.RequestState{ + entity.RequestStateValidated, + entity.RequestStateBatched, + } { + t.Run(string(state), func(t *testing.T) { ctrl := gomock.NewController(t) request := entity.Request{ @@ -935,34 +908,15 @@ func TestController_Process_RedeliveryForAdvancedRequest(t *testing.T) { Queue: "test-queue", Change: change.Change{URIs: []string{"github://uber/service/pull/456/abcdef0123456789abcdef0123456789abcdef01"}}, LandStrategy: mergestrategy.MergeStrategyRebase, - State: tt.state, + State: state, Version: 3, } - store, mockReqStore := newMockStorage(ctrl, request) - store.EXPECT().GetChangeStore().Return(newMockChangeStore(ctrl)).AnyTimes() - var writtenStates []entity.RequestState - mockReqStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(3), int32(4)).DoAndReturn( - func(_ context.Context, updated entity.Request, _, _ int32) error { - writtenStates = append(writtenStates, updated.State) - return nil - }, - ).AnyTimes() + // No Update expectation: gomock fails the test on any state write. + store, _ := newMockStorage(ctrl, request) + // No EXPECTs on the change store, publisher, or validator either. + store.EXPECT().GetChangeStore().Return(storagemock.NewMockChangeStore(ctrl)).AnyTimes() - var logStatuses []entity.RequestStatus - checkPublishes := 0 mockPub := queuemock.NewMockPublisher(ctrl) - mockPub.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( - func(_ context.Context, topic string, msg entityqueue.Message) error { - if topic == "merge-conflict-check" { - checkPublishes++ - return nil - } - entry, err := sqmq.UnmarshalRequestLog(msg.Payload) - require.NoError(t, err) - logStatuses = append(logStatuses, entry.Status) - return nil - }, - ).AnyTimes() mockQ := queuemock.NewMockQueue(ctrl) mockQ.EXPECT().Publisher().Return(mockPub).AnyTimes() registry, err := consumer.NewTopicRegistry( @@ -975,17 +929,8 @@ func TestController_Process_RedeliveryForAdvancedRequest(t *testing.T) { cpFactory := changeprovidermock.NewMockFactory(ctrl) cpFactory.EXPECT().For(gomock.Any()).Return(&mockChangeProvider{}, nil).AnyTimes() - - validatorCalls := 0 - mockValidator := validatormock.NewMockValidator(ctrl) - mockValidator.EXPECT().Validate(gomock.Any(), gomock.Any()).DoAndReturn( - func(context.Context, entity.Request) error { - validatorCalls++ - return tt.validatorErr - }, - ).AnyTimes() mockValidatorFactory := validatormock.NewMockFactory(ctrl) - mockValidatorFactory.EXPECT().For(gomock.Any()).Return(mockValidator, nil).AnyTimes() + mockValidatorFactory.EXPECT().For(gomock.Any()).Return(validatormock.NewMockValidator(ctrl), nil).AnyTimes() controller := NewController(zaptest.NewLogger(t).Sugar(), tally.NoopScope, staticStorageFactory{store: store}, registry, cpFactory, mockValidatorFactory, runwaymq.TopicKeyMergeConflictCheck, topickey.TopicKeyValidate, "orchestrator-validate") @@ -996,11 +941,6 @@ func TestController_Process_RedeliveryForAdvancedRequest(t *testing.T) { delivery.EXPECT().Attempt().Return(1).AnyTimes() require.NoError(t, controller.Process(context.Background(), delivery)) - - assert.Equal(t, 1, validatorCalls) - assert.Equal(t, tt.wantWrittenStates, writtenStates) - assert.Equal(t, tt.wantLogStatuses, logStatuses) - assert.Equal(t, tt.wantCheckPublishes, checkPublishes) }) } } diff --git a/test/e2e/submitqueue/harness_test.go b/test/e2e/submitqueue/harness_test.go index 0ae1fd158..8be1aeef7 100644 --- a/test/e2e/submitqueue/harness_test.go +++ b/test/e2e/submitqueue/harness_test.go @@ -345,28 +345,6 @@ func (s *E2EIntegrationSuite) redeliverLandConflictSignal(req request, outcome r }), "failed to publish %s message", topic) } -// queueMessageIDs returns the IDs of every row the queue still holds for the -// topic, in offset order. A publish that deduplicated wrote no row, so the list -// shows whether a republish reached the topic. -func (s *E2EIntegrationSuite) queueMessageIDs(queueName string, topic consumer.TopicKey) []string { - t := s.T() - t.Helper() - - rows, err := s.queueDB.QueryContext(s.ctx, - "SELECT id FROM queue_messages WHERE tenant = ? AND topic = ? ORDER BY `offset`", queueName, topic.String()) - require.NoError(t, err, "failed to read queue rows for %s", topic) - defer rows.Close() - - ids := []string{} - for rows.Next() { - var id string - require.NoError(t, rows.Scan(&id)) - ids = append(ids, id) - } - require.NoError(t, rows.Err()) - return ids -} - // batchState reads a batch's current state from the operating store. func (s *E2EIntegrationSuite) batchState(queue, batchID string) entity.BatchState { t := s.T() diff --git a/test/e2e/submitqueue/suite_test.go b/test/e2e/submitqueue/suite_test.go index 136803e10..2957fd37b 100644 --- a/test/e2e/submitqueue/suite_test.go +++ b/test/e2e/submitqueue/suite_test.go @@ -39,8 +39,6 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/uber-go/tally" - changepb "github.com/uber/submitqueue/api/base/change/protopb" - mergestrategypb "github.com/uber/submitqueue/api/base/mergestrategy/protopb" runwaymq "github.com/uber/submitqueue/api/runway/messagequeue" runwaypb "github.com/uber/submitqueue/api/runway/messagequeue/protopb" gatewaypb "github.com/uber/submitqueue/api/submitqueue/gateway/protopb" @@ -785,16 +783,15 @@ func (s *E2EIntegrationSuite) TestBatchRedelivery_DoesNotEnrolTheRequestTwice() } // A validate message and a SUCCEEDED land-conflict result replayed after the -// request has been claimed by a batch. The expectations record what the -// pipeline does today: the replayed result writes Validated over Batched while -// the batch that owns the request is still running. +// request has been claimed by a batch must leave it alone: from Batched onward +// the batch owns the request, so neither replay may write its state. // // The build gate holds the request in Batched for the replays, as in -// TestBatchRedelivery_DoesNotEnrolTheRequestTwice. The second request is the -// settle signal for the replayed validate: validate, the conflict check and its -// signal are each consumed in order per queue, so its batch existing proves -// anything the replay set off for the first request has already been handled. -func (s *E2EIntegrationSuite) TestStaleValidationMessages_RewindABatchedRequest() { +// TestBatchRedelivery_DoesNotEnrolTheRequestTwice. A request landed after each +// replay is its settle signal: validate, the conflict check and its signal are +// each consumed in order per queue, so the later request reaching a batch +// proves the replay ahead of it has already been handled. +func (s *E2EIntegrationSuite) TestStaleValidationMessages_LeaveABatchedRequestAlone() { t := s.T() const queue = "e2e-stale-validation-queue" @@ -810,108 +807,54 @@ func (s *E2EIntegrationSuite) TestStaleValidationMessages_RewindABatchedRequest( require.Equal(t, entity.RequestStateBatched, s.terminalState(req)) s.publishIDMessage(queue, topickey.TopicKeyValidate, req.sqid) - settle := s.land(queue, "github://github.example.com/uber/e2e-stale-validation/pull/2/1234567890abcdef1234567890abcdef12345678") - s.awaitBatchID(settle) - s.log.Logf("STALE-REPRO after replayed validate: request %s is %q", req.sqid, s.terminalState(req)) + afterValidate := s.land(queue, "github://github.example.com/uber/e2e-stale-validation/pull/2/1234567890abcdef1234567890abcdef12345678") + s.awaitBatchID(afterValidate) + assert.Equal(t, entity.RequestStateBatched, s.terminalState(req), "a replayed validate must not move a batched request") s.redeliverLandConflictSignal(req, runwaypb.Outcome_SUCCEEDED, "") - pollUntil(persistPollInterval, func() bool { - got := s.terminalState(req) - s.log.Logf("STALE-REPRO after replayed signal: request %s is %q", req.sqid, got) - return got == entity.RequestStateValidated - }) - assert.Equal(t, entity.BatchStateSpeculating, s.batchState(queue, heldBatch), - "the batch that owns the request is still running") - s.log.Logf("STALE-REPRO batches for request %s: %v", req.sqid, s.batchIDsFor(req)) - // The request ID is the message ID on both topics, so one row per request - // means the republish from each replay deduplicated. - batchRows := s.queueMessageIDs(queue, topickey.TopicKeyBatch) - checkRows := s.queueMessageIDs(queue, runwaymq.TopicKeyMergeConflictCheck) - s.log.Logf("STALE-REPRO queue rows: batch=%v merge-conflict-check=%v signal=%v validate=%v", batchRows, checkRows, - s.queueMessageIDs(queue, runwaymq.TopicKeyMergeConflictCheckSignal), s.queueMessageIDs(queue, topickey.TopicKeyValidate)) - assert.Equal(t, []string{req.sqid, settle.sqid}, batchRows, "the republished batch message deduplicated") - assert.Equal(t, []string{req.sqid, settle.sqid}, checkRows, "the republished land-conflict check deduplicated") + afterSignal := s.land(queue, "github://github.example.com/uber/e2e-stale-validation/pull/3/fedcba9876543210fedcba9876543210fedcba98") + s.awaitBatchID(afterSignal) + assert.Equal(t, entity.RequestStateBatched, s.terminalState(req), "a replayed result must not rewind a batched request") + assert.Equal(t, []string{heldBatch}, s.batchIDsFor(req)) s.openGate(queue, gateGroup, heldBatch) s.awaitStatus(req, entity.RequestStatusLanded) - s.awaitStatus(settle, entity.RequestStatusLanded) - s.log.Logf("STALE-REPRO timeline for request %s: %v", req.sqid, s.timeline(req)) + s.awaitStatus(afterValidate, entity.RequestStatusLanded) + s.awaitStatus(afterSignal, entity.RequestStatusLanded) + s.assertStatusCount(req, entity.RequestStatusValidated, 1) } // A not-landable land-conflict result replayed after the request has been -// claimed by a batch. The expectations record what the pipeline does today: the -// request is failed while its batch runs on, the batch still merges the change, -// the user can no longer cancel it, and the same change is admitted again. +// claimed by a batch must not fail it: the batch owns the outcome, and a +// request failed underneath it would read as an error for a change that the +// batch then merges. // -// The build gate holds the request in Batched for the replay, as in -// TestBatchRedelivery_DoesNotEnrolTheRequestTwice. -func (s *E2EIntegrationSuite) TestStaleNotLandableSignal_FailsABatchedRequest() { +// The gate and the settle request work as in the test above. +func (s *E2EIntegrationSuite) TestStaleNotLandableSignal_LeavesABatchedRequestAlone() { t := s.T() const queue = "e2e-stale-not-landable-queue" const gateGroup = "orchestrator" const heldBatch = "1" - const uri = "github://github.example.com/uber/e2e-stale-not-landable/pull/1/abcdef0123456789abcdef0123456789abcdef01" s.closeGate(queue, gateGroup, heldBatch, "e2e: hold the build so the request stays batched for the replay") defer s.openGate(queue, gateGroup, heldBatch) - req := s.land(queue, uri) + req := s.land(queue, "github://github.example.com/uber/e2e-stale-not-landable/pull/1/abcdef0123456789abcdef0123456789abcdef01") require.Equal(t, heldBatch, s.awaitBatchID(req), "the first batch of a fresh queue must be batch/1") s.awaitStatus(req, entity.RequestStatusSpeculating) require.Equal(t, entity.RequestStateBatched, s.terminalState(req)) s.redeliverLandConflictSignal(req, runwaypb.Outcome_FAILED, "replayed conflict") - pollUntil(persistPollInterval, func() bool { - got := s.terminalState(req) - s.log.Logf("STALE-FAILED after replayed signal: request %s is %q", req.sqid, got) - return got == entity.RequestStateError - }) - s.awaitStatus(req, entity.RequestStatusError) - s.log.Logf("STALE-FAILED request %s: status=%q last_error=%q batch %s is %q", - req.sqid, s.mustStatus(req), s.lastError(req), heldBatch, s.batchState(queue, heldBatch)) - assert.Equal(t, entity.BatchStateSpeculating, s.batchState(queue, heldBatch), - "the batch that owns the request is still running") - - cancelResp, cancelErr := s.gatewayClient.Cancel(s.ctx, &gatewaypb.CancelRequest{Sqid: req.sqid, Queue: queue, Reason: "e2e stale not-landable"}) - s.log.Logf("STALE-FAILED cancel while batch %s is live: response=%v error=%v", heldBatch, cancelResp, cancelErr) - - // The resubmission is behind the cancel on the wire, and its own batch - // serializes behind the held one, so it is observed while that is live. - resubmitResp, resubmitErr := s.gatewayClient.Land(s.ctx, &gatewaypb.LandRequest{ - Queue: queue, - Change: &changepb.Change{Uris: []string{uri}}, - Strategy: mergestrategypb.Strategy_REBASE, - }) - s.log.Logf("STALE-FAILED resubmission of the same URI: response=%v error=%v", resubmitResp, resubmitErr) - require.NoError(t, resubmitErr) - resubmit := request{queue: queue, sqid: resubmitResp.Sqid} - var resubmitStatus entity.RequestStatus - pollUntil(persistPollInterval, func() bool { - resubmitStatus, _ = s.currentStatus(resubmit) - s.log.Logf("STALE-FAILED resubmission %s is %q", resubmit.sqid, resubmitStatus) - switch resubmitStatus { - case entity.RequestStatusUnknown, entity.RequestStatusAccepted, entity.RequestStatusStarted, - entity.RequestStatusValidating, entity.RequestStatusValidated, entity.RequestStatusBatching: - return false - } - return true - }) - s.log.Logf("STALE-FAILED resubmission %s: status=%q last_error=%q batches=%v; batch %s is %q after the cancel", - resubmit.sqid, resubmitStatus, s.lastError(resubmit), s.batchIDsFor(resubmit), heldBatch, s.batchState(queue, heldBatch)) - assert.False(t, isTerminalStatus(resubmitStatus), "the same change is admitted while the first batch is live") - assert.Equal(t, entity.BatchStateSpeculating, s.batchState(queue, heldBatch), "the cancel did not reach the live batch") + settle := s.land(queue, "github://github.example.com/uber/e2e-stale-not-landable/pull/2/1234567890abcdef1234567890abcdef12345678") + s.awaitBatchID(settle) + assert.Equal(t, entity.RequestStateBatched, s.terminalState(req), "a replayed not-landable result must not fail a batched request") s.openGate(queue, gateGroup, heldBatch) - pollUntil(persistPollInterval, func() bool { return s.batchState(queue, heldBatch).IsTerminal() }) - assert.Equal(t, entity.BatchStateSucceeded, s.batchState(queue, heldBatch), "the batch merges the change of a request already failed") - resubmitFinal := s.awaitTerminal(resubmit) - - s.log.Logf("STALE-FAILED final: request %s state=%q status=%q timeline=%v; batch %s is %q", - req.sqid, s.terminalState(req), s.mustStatus(req), s.timeline(req), heldBatch, s.batchState(queue, heldBatch)) - s.log.Logf("STALE-FAILED final: resubmission %s state=%q status=%q timeline=%v batches=%v", - resubmit.sqid, s.terminalState(resubmit), resubmitFinal, s.timeline(resubmit), s.batchIDsFor(resubmit)) - assert.Equal(t, entity.RequestStateError, s.terminalState(req), "the request stays failed although its batch merged") + s.awaitStatus(req, entity.RequestStatusLanded) + s.awaitStatus(settle, entity.RequestStatusLanded) + assert.Equal(t, entity.RequestStateLanded, s.terminalState(req)) + s.assertStatusesNever(req, entity.RequestStatusError) } // A batch left in Created is dependency-eligible, so everything created after