diff --git a/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal.go b/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal.go index 267aa00ca..867a85ec6 100644 --- a/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal.go +++ b/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal.go @@ -128,6 +128,19 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er return nil } + // A result is only the request's outcome while validation still owns it. + // Validated is let through because a retry lands there after the state write + // succeeded and the hand-off to batch did not; past that the batch owns the + // request, and a replayed result must neither rewind nor terminate it. + if request.State != entity.RequestStateStarted && request.State != entity.RequestStateValidated { + metrics.NamedCounter(c.metricsScope, opName, "skipped_stale", 1, metrics.TagsFromContext(ctx)...) + c.logger.Infow("skipping landconflict signal for request past validation", + "request_id", request.ID, + "state", string(request.State), + ) + return nil + } + if result.Outcome != runwaypb.Outcome_SUCCEEDED { metrics.NamedCounter(c.metricsScope, opName, "not_landable", 1, metrics.TagsFromContext(ctx)...) c.logger.Infow("request not landable", diff --git a/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal_test.go b/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal_test.go index 9947651e2..5db01e369 100644 --- a/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal_test.go +++ b/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal_test.go @@ -238,3 +238,90 @@ func TestProcess_HaltedRequestSkips(t *testing.T) { msg := entityqueue.NewMessage(testRequestID, resultPayload(t, res), testQueue, nil) require.NoError(t, controller.Process(context.Background(), newDelivery(ctrl, msg))) } + +// A result redelivered after the request has moved past validation is acked +// without a state write or a publish: the batch owns the request by then. +func TestProcess_SignalForAdvancedRequestIsSkipped(t *testing.T) { + tests := []struct { + name string + state entity.RequestState + outcome runwaypb.Outcome + }{ + {name: "batched, landable", state: entity.RequestStateBatched, outcome: runwaypb.Outcome_SUCCEEDED}, + {name: "batched, not landable", state: entity.RequestStateBatched, outcome: runwaypb.Outcome_FAILED}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + + // No Update expectation: gomock fails the test on any state write. + request := entity.Request{ID: testRequestID, Queue: testQueue, State: tt.state, Version: 3} + reqStore := storagemock.NewMockRequestStore(ctrl) + reqStore.EXPECT().Get(gomock.Any(), testRequestID).Return(request, nil).AnyTimes() + + store := orchstoragemock.NewMockStorage(ctrl) + store.EXPECT().GetRequestStore().Return(reqStore).AnyTimes() + + // No Publish expectation either. + pub := queuemock.NewMockPublisher(ctrl) + q := queuemock.NewMockQueue(ctrl) + q.EXPECT().Publisher().Return(pub).AnyTimes() + registry, err := consumer.NewTopicRegistry( + []consumer.TopicConfig{ + {Key: topickey.TopicKeyBatch, Name: "batch", Queue: q}, + {Key: topickey.TopicKeyLog, Name: "log", Queue: q}, + }, + ) + require.NoError(t, err) + + controller := NewController(zaptest.NewLogger(t).Sugar(), tally.NoopScope, staticStorageFactory{store: store}, registry, + runwaymq.TopicKeyMergeConflictCheckSignal, "orchestrator-landconflictsignal") + + res := runwaymq.MergeResult{Id: testRequestID, Outcome: tt.outcome, Reason: "replayed"} + msg := entityqueue.NewMessage(testRequestID, resultPayload(t, res), testQueue, nil) + require.NoError(t, controller.Process(context.Background(), newDelivery(ctrl, msg))) + }) + } +} + +// A retry after the Validated write succeeded and the hand-off failed finds the +// request in Validated and must still reach the batch topic. +func TestProcess_LandableRetryFromValidatedRepublishesToBatch(t *testing.T) { + ctrl := gomock.NewController(t) + + request := entity.Request{ID: testRequestID, Queue: testQueue, State: entity.RequestStateValidated, Version: 2} + reqStore := storagemock.NewMockRequestStore(ctrl) + reqStore.EXPECT().Get(gomock.Any(), testRequestID).Return(request, nil) + reqStore.EXPECT().Update(gomock.Any(), request, int32(2), int32(3)).Return(nil) + + store := orchstoragemock.NewMockStorage(ctrl) + store.EXPECT().GetRequestStore().Return(reqStore).AnyTimes() + + var gotTopics []string + pub := queuemock.NewMockPublisher(ctrl) + pub.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, topic string, _ entityqueue.Message) error { + gotTopics = append(gotTopics, topic) + return nil + }, + ).AnyTimes() + q := queuemock.NewMockQueue(ctrl) + q.EXPECT().Publisher().Return(pub).AnyTimes() + registry, err := consumer.NewTopicRegistry( + []consumer.TopicConfig{ + {Key: topickey.TopicKeyBatch, Name: "batch", Queue: q}, + {Key: topickey.TopicKeyLog, Name: "log", Queue: q}, + }, + ) + require.NoError(t, err) + + controller := NewController(zaptest.NewLogger(t).Sugar(), tally.NoopScope, staticStorageFactory{store: store}, registry, + runwaymq.TopicKeyMergeConflictCheckSignal, "orchestrator-landconflictsignal") + + res := runwaymq.MergeResult{Id: testRequestID, Outcome: runwaypb.Outcome_SUCCEEDED} + msg := entityqueue.NewMessage(testRequestID, resultPayload(t, res), testQueue, nil) + require.NoError(t, controller.Process(context.Background(), newDelivery(ctrl, msg))) + + assert.Equal(t, []string{"log", "batch"}, gotTopics) +} diff --git a/submitqueue/orchestrator/controller/validate/validate.go b/submitqueue/orchestrator/controller/validate/validate.go index 808073c35..239bc1000 100644 --- a/submitqueue/orchestrator/controller/validate/validate.go +++ b/submitqueue/orchestrator/controller/validate/validate.go @@ -148,6 +148,18 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er return nil } + // Validation runs once, on a request nothing downstream has picked up yet. + // A redelivery that arrives after the conflict check came back would + // otherwise re-judge a request its batch now owns. + if request.State != entity.RequestStateStarted { + coremetrics.NamedCounter(c.metricsScope, "process", "skipped_stale", 1, coremetrics.TagsFromContext(ctx)...) + c.logger.Infow("skipping validate for request past validation", + "request_id", request.ID, + "state", string(request.State), + ) + return nil + } + // Report that validation has begun. This stage is not instantaneous — the // land-conflict check below is an async round trip to runway — so without // this the request reads "started" for the whole of it. No occurrence: a diff --git a/submitqueue/orchestrator/controller/validate/validate_test.go b/submitqueue/orchestrator/controller/validate/validate_test.go index 07f543b7e..fbfc0f98b 100644 --- a/submitqueue/orchestrator/controller/validate/validate_test.go +++ b/submitqueue/orchestrator/controller/validate/validate_test.go @@ -891,3 +891,56 @@ func TestController_Process_CustomValidatorFailure_TerminationPublishFails(t *te require.Error(t, err) assert.ErrorContains(t, err, "failed to terminate rejected request") } + +// A validate message redelivered after the request has moved past validation +// is acked without touching it: no "validating" log, no validator call, no +// state write, and no second land-conflict check. +func TestController_Process_RedeliveryForAdvancedRequestIsSkipped(t *testing.T) { + for _, state := range []entity.RequestState{ + entity.RequestStateValidated, + entity.RequestStateBatched, + } { + t.Run(string(state), func(t *testing.T) { + ctrl := gomock.NewController(t) + + request := entity.Request{ + ID: "123", + Queue: "test-queue", + Change: change.Change{URIs: []string{"github://uber/service/pull/456/abcdef0123456789abcdef0123456789abcdef01"}}, + LandStrategy: mergestrategy.MergeStrategyRebase, + State: state, + Version: 3, + } + // No Update expectation: gomock fails the test on any state write. + store, _ := newMockStorage(ctrl, request) + // No EXPECTs on the change store, publisher, or validator either. + store.EXPECT().GetChangeStore().Return(storagemock.NewMockChangeStore(ctrl)).AnyTimes() + + mockPub := queuemock.NewMockPublisher(ctrl) + mockQ := queuemock.NewMockQueue(ctrl) + mockQ.EXPECT().Publisher().Return(mockPub).AnyTimes() + registry, err := consumer.NewTopicRegistry( + []consumer.TopicConfig{ + {Key: runwaymq.TopicKeyMergeConflictCheck, Name: "merge-conflict-check", Queue: mockQ}, + {Key: topickey.TopicKeyLog, Name: "log", Queue: mockQ}, + }, + ) + require.NoError(t, err) + + cpFactory := changeprovidermock.NewMockFactory(ctrl) + cpFactory.EXPECT().For(gomock.Any()).Return(&mockChangeProvider{}, nil).AnyTimes() + mockValidatorFactory := validatormock.NewMockFactory(ctrl) + mockValidatorFactory.EXPECT().For(gomock.Any()).Return(validatormock.NewMockValidator(ctrl), nil).AnyTimes() + + controller := NewController(zaptest.NewLogger(t).Sugar(), tally.NoopScope, staticStorageFactory{store: store}, registry, cpFactory, mockValidatorFactory, runwaymq.TopicKeyMergeConflictCheck, topickey.TopicKeyValidate, "orchestrator-validate") + + msg := entityqueue.NewMessage(request.ID, requestIDPayload(t, request.ID), request.Queue, nil) + msg.Tenant = request.Queue + delivery := consumermock.NewMockDelivery(ctrl) + delivery.EXPECT().Message().Return(msg).AnyTimes() + delivery.EXPECT().Attempt().Return(1).AnyTimes() + + require.NoError(t, controller.Process(context.Background(), delivery)) + }) + } +} diff --git a/test/e2e/submitqueue/BUILD.bazel b/test/e2e/submitqueue/BUILD.bazel index 0734554b3..403e238a5 100644 --- a/test/e2e/submitqueue/BUILD.bazel +++ b/test/e2e/submitqueue/BUILD.bazel @@ -58,6 +58,7 @@ go_test( "//api/base/change/protopb:go_default_library", "//api/base/mergestrategy/protopb:go_default_library", "//api/runway/messagequeue:go_default_library", + "//api/runway/messagequeue/protopb:go_default_library", "//api/submitqueue/gateway/protopb:go_default_library", "//api/submitqueue/orchestrator/protopb:go_default_library", "//platform/base/change:go_default_library", diff --git a/test/e2e/submitqueue/harness_test.go b/test/e2e/submitqueue/harness_test.go index 48ba9ca68..8be1aeef7 100644 --- a/test/e2e/submitqueue/harness_test.go +++ b/test/e2e/submitqueue/harness_test.go @@ -34,6 +34,8 @@ import ( "github.com/uber-go/tally" changepb "github.com/uber/submitqueue/api/base/change/protopb" mergestrategypb "github.com/uber/submitqueue/api/base/mergestrategy/protopb" + runwaymq "github.com/uber/submitqueue/api/runway/messagequeue" + runwaypb "github.com/uber/submitqueue/api/runway/messagequeue/protopb" gatewaypb "github.com/uber/submitqueue/api/submitqueue/gateway/protopb" entityqueue "github.com/uber/submitqueue/platform/base/messagequeue" "github.com/uber/submitqueue/platform/consumer" @@ -310,6 +312,39 @@ func (s *E2EIntegrationSuite) publishIDMessage(queueName string, topic consumer. return messageID } +// redeliverLandConflictSignal publishes a second land-conflict result for the +// request, which is what the orchestrator sees when the signal delivery is +// retried after its ack was lost. The message ID is fresh for the reason given +// on redeliverBatchMessage. +func (s *E2EIntegrationSuite) redeliverLandConflictSignal(req request, outcome runwaypb.Outcome, reason string) { + t := s.T() + t.Helper() + + queue, err := queuemysql.NewQueue(queuemysql.Params{ + DB: s.queueDB, + Logger: zap.NewNop(), + MetricsScope: tally.NoopScope, + Tenants: []string{req.queue}, + }) + require.NoError(t, err, "failed to open the queue for a manual publish") + defer func() { require.NoError(t, queue.Close()) }() + + topic := runwaymq.TopicKeyMergeConflictCheckSignal + registry, err := consumer.NewTopicRegistry([]consumer.TopicConfig{ + {Key: topic, Name: topic.String(), Queue: queue}, + }) + require.NoError(t, err) + + payload, err := runwaymq.Marshal(&runwaymq.MergeResult{Id: req.sqid, QueueName: req.queue, Outcome: outcome, Reason: reason}) + require.NoError(t, err) + require.NoError(t, publish.Message(entityqueue.WithQueueName(s.ctx, req.queue), registry, topic, publish.MessageParams{ + Tenant: req.queue, + ID: publish.UniqueID(req.sqid), + Payload: payload, + PartitionKey: req.queue, + }), "failed to publish %s message", topic) +} + // batchState reads a batch's current state from the operating store. func (s *E2EIntegrationSuite) batchState(queue, batchID string) entity.BatchState { t := s.T() diff --git a/test/e2e/submitqueue/queues.yaml b/test/e2e/submitqueue/queues.yaml index 7ee2ad5c5..a87530a51 100644 --- a/test/e2e/submitqueue/queues.yaml +++ b/test/e2e/submitqueue/queues.yaml @@ -9,6 +9,8 @@ queues: - name: e2e-partial-coverage-queue - name: e2e-land-wakeup-queue - name: e2e-redelivery-queue + - name: e2e-stale-validation-queue + - name: e2e-stale-not-landable-queue - name: e2e-strand-queue - name: e2e-conflict-error-queue - name: e2e-respeculate-queue diff --git a/test/e2e/submitqueue/suite_test.go b/test/e2e/submitqueue/suite_test.go index c7bc2f182..2957fd37b 100644 --- a/test/e2e/submitqueue/suite_test.go +++ b/test/e2e/submitqueue/suite_test.go @@ -40,9 +40,11 @@ import ( "github.com/stretchr/testify/suite" "github.com/uber-go/tally" runwaymq "github.com/uber/submitqueue/api/runway/messagequeue" + runwaypb "github.com/uber/submitqueue/api/runway/messagequeue/protopb" gatewaypb "github.com/uber/submitqueue/api/submitqueue/gateway/protopb" orchestratorpb "github.com/uber/submitqueue/api/submitqueue/orchestrator/protopb" consumergatefile "github.com/uber/submitqueue/platform/extension/consumergate/file" + "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" yamlqueueconfig "github.com/uber/submitqueue/submitqueue/extension/queueconfig/yaml" storagemysql "github.com/uber/submitqueue/submitqueue/orchestrator/extension/storage/mysql" @@ -780,6 +782,81 @@ func (s *E2EIntegrationSuite) TestBatchRedelivery_DoesNotEnrolTheRequestTwice() s.awaitStatus(settle, entity.RequestStatusLanded) } +// A validate message and a SUCCEEDED land-conflict result replayed after the +// request has been claimed by a batch must leave it alone: from Batched onward +// the batch owns the request, so neither replay may write its state. +// +// The build gate holds the request in Batched for the replays, as in +// TestBatchRedelivery_DoesNotEnrolTheRequestTwice. A request landed after each +// replay is its settle signal: validate, the conflict check and its signal are +// each consumed in order per queue, so the later request reaching a batch +// proves the replay ahead of it has already been handled. +func (s *E2EIntegrationSuite) TestStaleValidationMessages_LeaveABatchedRequestAlone() { + t := s.T() + + const queue = "e2e-stale-validation-queue" + const gateGroup = "orchestrator" + const heldBatch = "1" + + s.closeGate(queue, gateGroup, heldBatch, "e2e: hold the build so the request stays batched for the replays") + defer s.openGate(queue, gateGroup, heldBatch) + + req := s.land(queue, "github://github.example.com/uber/e2e-stale-validation/pull/1/abcdef0123456789abcdef0123456789abcdef01") + require.Equal(t, heldBatch, s.awaitBatchID(req), "the first batch of a fresh queue must be batch/1") + s.awaitStatus(req, entity.RequestStatusSpeculating) + require.Equal(t, entity.RequestStateBatched, s.terminalState(req)) + + s.publishIDMessage(queue, topickey.TopicKeyValidate, req.sqid) + afterValidate := s.land(queue, "github://github.example.com/uber/e2e-stale-validation/pull/2/1234567890abcdef1234567890abcdef12345678") + s.awaitBatchID(afterValidate) + assert.Equal(t, entity.RequestStateBatched, s.terminalState(req), "a replayed validate must not move a batched request") + + s.redeliverLandConflictSignal(req, runwaypb.Outcome_SUCCEEDED, "") + afterSignal := s.land(queue, "github://github.example.com/uber/e2e-stale-validation/pull/3/fedcba9876543210fedcba9876543210fedcba98") + s.awaitBatchID(afterSignal) + assert.Equal(t, entity.RequestStateBatched, s.terminalState(req), "a replayed result must not rewind a batched request") + assert.Equal(t, []string{heldBatch}, s.batchIDsFor(req)) + + s.openGate(queue, gateGroup, heldBatch) + s.awaitStatus(req, entity.RequestStatusLanded) + s.awaitStatus(afterValidate, entity.RequestStatusLanded) + s.awaitStatus(afterSignal, entity.RequestStatusLanded) + s.assertStatusCount(req, entity.RequestStatusValidated, 1) +} + +// A not-landable land-conflict result replayed after the request has been +// claimed by a batch must not fail it: the batch owns the outcome, and a +// request failed underneath it would read as an error for a change that the +// batch then merges. +// +// The gate and the settle request work as in the test above. +func (s *E2EIntegrationSuite) TestStaleNotLandableSignal_LeavesABatchedRequestAlone() { + t := s.T() + + const queue = "e2e-stale-not-landable-queue" + const gateGroup = "orchestrator" + const heldBatch = "1" + + s.closeGate(queue, gateGroup, heldBatch, "e2e: hold the build so the request stays batched for the replay") + defer s.openGate(queue, gateGroup, heldBatch) + + req := s.land(queue, "github://github.example.com/uber/e2e-stale-not-landable/pull/1/abcdef0123456789abcdef0123456789abcdef01") + require.Equal(t, heldBatch, s.awaitBatchID(req), "the first batch of a fresh queue must be batch/1") + s.awaitStatus(req, entity.RequestStatusSpeculating) + require.Equal(t, entity.RequestStateBatched, s.terminalState(req)) + + s.redeliverLandConflictSignal(req, runwaypb.Outcome_FAILED, "replayed conflict") + settle := s.land(queue, "github://github.example.com/uber/e2e-stale-not-landable/pull/2/1234567890abcdef1234567890abcdef12345678") + s.awaitBatchID(settle) + assert.Equal(t, entity.RequestStateBatched, s.terminalState(req), "a replayed not-landable result must not fail a batched request") + + s.openGate(queue, gateGroup, heldBatch) + s.awaitStatus(req, entity.RequestStatusLanded) + s.awaitStatus(settle, entity.RequestStatusLanded) + assert.Equal(t, entity.RequestStateLanded, s.terminalState(req)) + s.assertStatusesNever(req, entity.RequestStatusError) +} + // A batch left in Created is dependency-eligible, so everything created after // it serializes behind it — and nothing will name it on the speculate topic // again. Without a run that looks for such batches the queue wedges here with