Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,19 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er
return nil
}

// A result is only the request's outcome while validation still owns it.
// Validated is let through because a retry lands there after the state write
// succeeded and the hand-off to batch did not; past that the batch owns the
// request, and a replayed result must neither rewind nor terminate it.
if request.State != entity.RequestStateStarted && request.State != entity.RequestStateValidated {
metrics.NamedCounter(c.metricsScope, opName, "skipped_stale", 1, metrics.TagsFromContext(ctx)...)
c.logger.Infow("skipping landconflict signal for request past validation",
"request_id", request.ID,
"state", string(request.State),
)
return nil
}

if result.Outcome != runwaypb.Outcome_SUCCEEDED {
metrics.NamedCounter(c.metricsScope, opName, "not_landable", 1, metrics.TagsFromContext(ctx)...)
c.logger.Infow("request not landable",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -238,3 +238,90 @@ func TestProcess_HaltedRequestSkips(t *testing.T) {
msg := entityqueue.NewMessage(testRequestID, resultPayload(t, res), testQueue, nil)
require.NoError(t, controller.Process(context.Background(), newDelivery(ctrl, msg)))
}

// A result redelivered after the request has moved past validation is acked
// without a state write or a publish: the batch owns the request by then.
func TestProcess_SignalForAdvancedRequestIsSkipped(t *testing.T) {
tests := []struct {
name string
state entity.RequestState
outcome runwaypb.Outcome
}{
{name: "batched, landable", state: entity.RequestStateBatched, outcome: runwaypb.Outcome_SUCCEEDED},
{name: "batched, not landable", state: entity.RequestStateBatched, outcome: runwaypb.Outcome_FAILED},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctrl := gomock.NewController(t)

// No Update expectation: gomock fails the test on any state write.
request := entity.Request{ID: testRequestID, Queue: testQueue, State: tt.state, Version: 3}
reqStore := storagemock.NewMockRequestStore(ctrl)
reqStore.EXPECT().Get(gomock.Any(), testRequestID).Return(request, nil).AnyTimes()

store := orchstoragemock.NewMockStorage(ctrl)
store.EXPECT().GetRequestStore().Return(reqStore).AnyTimes()

// No Publish expectation either.
pub := queuemock.NewMockPublisher(ctrl)
q := queuemock.NewMockQueue(ctrl)
q.EXPECT().Publisher().Return(pub).AnyTimes()
registry, err := consumer.NewTopicRegistry(
[]consumer.TopicConfig{
{Key: topickey.TopicKeyBatch, Name: "batch", Queue: q},
{Key: topickey.TopicKeyLog, Name: "log", Queue: q},
},
)
require.NoError(t, err)

controller := NewController(zaptest.NewLogger(t).Sugar(), tally.NoopScope, staticStorageFactory{store: store}, registry,
runwaymq.TopicKeyMergeConflictCheckSignal, "orchestrator-landconflictsignal")

res := runwaymq.MergeResult{Id: testRequestID, Outcome: tt.outcome, Reason: "replayed"}
msg := entityqueue.NewMessage(testRequestID, resultPayload(t, res), testQueue, nil)
require.NoError(t, controller.Process(context.Background(), newDelivery(ctrl, msg)))
})
}
}

// A retry after the Validated write succeeded and the hand-off failed finds the
// request in Validated and must still reach the batch topic.
func TestProcess_LandableRetryFromValidatedRepublishesToBatch(t *testing.T) {
ctrl := gomock.NewController(t)

request := entity.Request{ID: testRequestID, Queue: testQueue, State: entity.RequestStateValidated, Version: 2}
reqStore := storagemock.NewMockRequestStore(ctrl)
reqStore.EXPECT().Get(gomock.Any(), testRequestID).Return(request, nil)
reqStore.EXPECT().Update(gomock.Any(), request, int32(2), int32(3)).Return(nil)

store := orchstoragemock.NewMockStorage(ctrl)
store.EXPECT().GetRequestStore().Return(reqStore).AnyTimes()

var gotTopics []string
pub := queuemock.NewMockPublisher(ctrl)
pub.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(
func(_ context.Context, topic string, _ entityqueue.Message) error {
gotTopics = append(gotTopics, topic)
return nil
},
).AnyTimes()
q := queuemock.NewMockQueue(ctrl)
q.EXPECT().Publisher().Return(pub).AnyTimes()
registry, err := consumer.NewTopicRegistry(
[]consumer.TopicConfig{
{Key: topickey.TopicKeyBatch, Name: "batch", Queue: q},
{Key: topickey.TopicKeyLog, Name: "log", Queue: q},
},
)
require.NoError(t, err)

controller := NewController(zaptest.NewLogger(t).Sugar(), tally.NoopScope, staticStorageFactory{store: store}, registry,
runwaymq.TopicKeyMergeConflictCheckSignal, "orchestrator-landconflictsignal")

res := runwaymq.MergeResult{Id: testRequestID, Outcome: runwaypb.Outcome_SUCCEEDED}
msg := entityqueue.NewMessage(testRequestID, resultPayload(t, res), testQueue, nil)
require.NoError(t, controller.Process(context.Background(), newDelivery(ctrl, msg)))

assert.Equal(t, []string{"log", "batch"}, gotTopics)
}
12 changes: 12 additions & 0 deletions submitqueue/orchestrator/controller/validate/validate.go
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,18 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er
return nil
}

// Validation runs once, on a request nothing downstream has picked up yet.
// A redelivery that arrives after the conflict check came back would
// otherwise re-judge a request its batch now owns.
if request.State != entity.RequestStateStarted {
coremetrics.NamedCounter(c.metricsScope, "process", "skipped_stale", 1, coremetrics.TagsFromContext(ctx)...)
c.logger.Infow("skipping validate for request past validation",
"request_id", request.ID,
"state", string(request.State),
)
return nil
}

// Report that validation has begun. This stage is not instantaneous — the
// land-conflict check below is an async round trip to runway — so without
// this the request reads "started" for the whole of it. No occurrence: a
Expand Down
53 changes: 53 additions & 0 deletions submitqueue/orchestrator/controller/validate/validate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -891,3 +891,56 @@ func TestController_Process_CustomValidatorFailure_TerminationPublishFails(t *te
require.Error(t, err)
assert.ErrorContains(t, err, "failed to terminate rejected request")
}

// A validate message redelivered after the request has moved past validation
// is acked without touching it: no "validating" log, no validator call, no
// state write, and no second land-conflict check.
func TestController_Process_RedeliveryForAdvancedRequestIsSkipped(t *testing.T) {
for _, state := range []entity.RequestState{
entity.RequestStateValidated,
entity.RequestStateBatched,
} {
t.Run(string(state), func(t *testing.T) {
ctrl := gomock.NewController(t)

request := entity.Request{
ID: "123",
Queue: "test-queue",
Change: change.Change{URIs: []string{"github://uber/service/pull/456/abcdef0123456789abcdef0123456789abcdef01"}},
LandStrategy: mergestrategy.MergeStrategyRebase,
State: state,
Version: 3,
}
// No Update expectation: gomock fails the test on any state write.
store, _ := newMockStorage(ctrl, request)
// No EXPECTs on the change store, publisher, or validator either.
store.EXPECT().GetChangeStore().Return(storagemock.NewMockChangeStore(ctrl)).AnyTimes()

mockPub := queuemock.NewMockPublisher(ctrl)
mockQ := queuemock.NewMockQueue(ctrl)
mockQ.EXPECT().Publisher().Return(mockPub).AnyTimes()
registry, err := consumer.NewTopicRegistry(
[]consumer.TopicConfig{
{Key: runwaymq.TopicKeyMergeConflictCheck, Name: "merge-conflict-check", Queue: mockQ},
{Key: topickey.TopicKeyLog, Name: "log", Queue: mockQ},
},
)
require.NoError(t, err)

cpFactory := changeprovidermock.NewMockFactory(ctrl)
cpFactory.EXPECT().For(gomock.Any()).Return(&mockChangeProvider{}, nil).AnyTimes()
mockValidatorFactory := validatormock.NewMockFactory(ctrl)
mockValidatorFactory.EXPECT().For(gomock.Any()).Return(validatormock.NewMockValidator(ctrl), nil).AnyTimes()

controller := NewController(zaptest.NewLogger(t).Sugar(), tally.NoopScope, staticStorageFactory{store: store}, registry, cpFactory, mockValidatorFactory, runwaymq.TopicKeyMergeConflictCheck, topickey.TopicKeyValidate, "orchestrator-validate")

msg := entityqueue.NewMessage(request.ID, requestIDPayload(t, request.ID), request.Queue, nil)
msg.Tenant = request.Queue
delivery := consumermock.NewMockDelivery(ctrl)
delivery.EXPECT().Message().Return(msg).AnyTimes()
delivery.EXPECT().Attempt().Return(1).AnyTimes()

require.NoError(t, controller.Process(context.Background(), delivery))
})
}
}
1 change: 1 addition & 0 deletions test/e2e/submitqueue/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ go_test(
"//api/base/change/protopb:go_default_library",
"//api/base/mergestrategy/protopb:go_default_library",
"//api/runway/messagequeue:go_default_library",
"//api/runway/messagequeue/protopb:go_default_library",
"//api/submitqueue/gateway/protopb:go_default_library",
"//api/submitqueue/orchestrator/protopb:go_default_library",
"//platform/base/change:go_default_library",
Expand Down
35 changes: 35 additions & 0 deletions test/e2e/submitqueue/harness_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ import (
"github.com/uber-go/tally"
changepb "github.com/uber/submitqueue/api/base/change/protopb"
mergestrategypb "github.com/uber/submitqueue/api/base/mergestrategy/protopb"
runwaymq "github.com/uber/submitqueue/api/runway/messagequeue"
runwaypb "github.com/uber/submitqueue/api/runway/messagequeue/protopb"
gatewaypb "github.com/uber/submitqueue/api/submitqueue/gateway/protopb"
entityqueue "github.com/uber/submitqueue/platform/base/messagequeue"
"github.com/uber/submitqueue/platform/consumer"
Expand Down Expand Up @@ -310,6 +312,39 @@ func (s *E2EIntegrationSuite) publishIDMessage(queueName string, topic consumer.
return messageID
}

// redeliverLandConflictSignal publishes a second land-conflict result for the
// request, which is what the orchestrator sees when the signal delivery is
// retried after its ack was lost. The message ID is fresh for the reason given
// on redeliverBatchMessage.
func (s *E2EIntegrationSuite) redeliverLandConflictSignal(req request, outcome runwaypb.Outcome, reason string) {
t := s.T()
t.Helper()

queue, err := queuemysql.NewQueue(queuemysql.Params{
DB: s.queueDB,
Logger: zap.NewNop(),
MetricsScope: tally.NoopScope,
Tenants: []string{req.queue},
})
require.NoError(t, err, "failed to open the queue for a manual publish")
defer func() { require.NoError(t, queue.Close()) }()

topic := runwaymq.TopicKeyMergeConflictCheckSignal
registry, err := consumer.NewTopicRegistry([]consumer.TopicConfig{
{Key: topic, Name: topic.String(), Queue: queue},
})
require.NoError(t, err)

payload, err := runwaymq.Marshal(&runwaymq.MergeResult{Id: req.sqid, QueueName: req.queue, Outcome: outcome, Reason: reason})
require.NoError(t, err)
require.NoError(t, publish.Message(entityqueue.WithQueueName(s.ctx, req.queue), registry, topic, publish.MessageParams{
Tenant: req.queue,
ID: publish.UniqueID(req.sqid),
Payload: payload,
PartitionKey: req.queue,
}), "failed to publish %s message", topic)
}

// batchState reads a batch's current state from the operating store.
func (s *E2EIntegrationSuite) batchState(queue, batchID string) entity.BatchState {
t := s.T()
Expand Down
2 changes: 2 additions & 0 deletions test/e2e/submitqueue/queues.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ queues:
- name: e2e-partial-coverage-queue
- name: e2e-land-wakeup-queue
- name: e2e-redelivery-queue
- name: e2e-stale-validation-queue
- name: e2e-stale-not-landable-queue
- name: e2e-strand-queue
- name: e2e-conflict-error-queue
- name: e2e-respeculate-queue
Expand Down
77 changes: 77 additions & 0 deletions test/e2e/submitqueue/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,11 @@ import (
"github.com/stretchr/testify/suite"
"github.com/uber-go/tally"
runwaymq "github.com/uber/submitqueue/api/runway/messagequeue"
runwaypb "github.com/uber/submitqueue/api/runway/messagequeue/protopb"
gatewaypb "github.com/uber/submitqueue/api/submitqueue/gateway/protopb"
orchestratorpb "github.com/uber/submitqueue/api/submitqueue/orchestrator/protopb"
consumergatefile "github.com/uber/submitqueue/platform/extension/consumergate/file"
"github.com/uber/submitqueue/submitqueue/core/topickey"
"github.com/uber/submitqueue/submitqueue/entity"
yamlqueueconfig "github.com/uber/submitqueue/submitqueue/extension/queueconfig/yaml"
storagemysql "github.com/uber/submitqueue/submitqueue/orchestrator/extension/storage/mysql"
Expand Down Expand Up @@ -780,6 +782,81 @@ func (s *E2EIntegrationSuite) TestBatchRedelivery_DoesNotEnrolTheRequestTwice()
s.awaitStatus(settle, entity.RequestStatusLanded)
}

// A validate message and a SUCCEEDED land-conflict result replayed after the
// request has been claimed by a batch must leave it alone: from Batched onward
// the batch owns the request, so neither replay may write its state.
//
// The build gate holds the request in Batched for the replays, as in
// TestBatchRedelivery_DoesNotEnrolTheRequestTwice. A request landed after each
// replay is its settle signal: validate, the conflict check and its signal are
// each consumed in order per queue, so the later request reaching a batch
// proves the replay ahead of it has already been handled.
func (s *E2EIntegrationSuite) TestStaleValidationMessages_LeaveABatchedRequestAlone() {
t := s.T()

const queue = "e2e-stale-validation-queue"
const gateGroup = "orchestrator"
const heldBatch = "1"

s.closeGate(queue, gateGroup, heldBatch, "e2e: hold the build so the request stays batched for the replays")
defer s.openGate(queue, gateGroup, heldBatch)

req := s.land(queue, "github://github.example.com/uber/e2e-stale-validation/pull/1/abcdef0123456789abcdef0123456789abcdef01")
require.Equal(t, heldBatch, s.awaitBatchID(req), "the first batch of a fresh queue must be batch/1")
s.awaitStatus(req, entity.RequestStatusSpeculating)
require.Equal(t, entity.RequestStateBatched, s.terminalState(req))

s.publishIDMessage(queue, topickey.TopicKeyValidate, req.sqid)
afterValidate := s.land(queue, "github://github.example.com/uber/e2e-stale-validation/pull/2/1234567890abcdef1234567890abcdef12345678")
s.awaitBatchID(afterValidate)
assert.Equal(t, entity.RequestStateBatched, s.terminalState(req), "a replayed validate must not move a batched request")

s.redeliverLandConflictSignal(req, runwaypb.Outcome_SUCCEEDED, "")
afterSignal := s.land(queue, "github://github.example.com/uber/e2e-stale-validation/pull/3/fedcba9876543210fedcba9876543210fedcba98")
s.awaitBatchID(afterSignal)
assert.Equal(t, entity.RequestStateBatched, s.terminalState(req), "a replayed result must not rewind a batched request")
assert.Equal(t, []string{heldBatch}, s.batchIDsFor(req))

s.openGate(queue, gateGroup, heldBatch)
s.awaitStatus(req, entity.RequestStatusLanded)
s.awaitStatus(afterValidate, entity.RequestStatusLanded)
s.awaitStatus(afterSignal, entity.RequestStatusLanded)
s.assertStatusCount(req, entity.RequestStatusValidated, 1)
}

// A not-landable land-conflict result replayed after the request has been
// claimed by a batch must not fail it: the batch owns the outcome, and a
// request failed underneath it would read as an error for a change that the
// batch then merges.
//
// The gate and the settle request work as in the test above.
func (s *E2EIntegrationSuite) TestStaleNotLandableSignal_LeavesABatchedRequestAlone() {
t := s.T()

const queue = "e2e-stale-not-landable-queue"
const gateGroup = "orchestrator"
const heldBatch = "1"

s.closeGate(queue, gateGroup, heldBatch, "e2e: hold the build so the request stays batched for the replay")
defer s.openGate(queue, gateGroup, heldBatch)

req := s.land(queue, "github://github.example.com/uber/e2e-stale-not-landable/pull/1/abcdef0123456789abcdef0123456789abcdef01")
require.Equal(t, heldBatch, s.awaitBatchID(req), "the first batch of a fresh queue must be batch/1")
s.awaitStatus(req, entity.RequestStatusSpeculating)
require.Equal(t, entity.RequestStateBatched, s.terminalState(req))

s.redeliverLandConflictSignal(req, runwaypb.Outcome_FAILED, "replayed conflict")
settle := s.land(queue, "github://github.example.com/uber/e2e-stale-not-landable/pull/2/1234567890abcdef1234567890abcdef12345678")
s.awaitBatchID(settle)
assert.Equal(t, entity.RequestStateBatched, s.terminalState(req), "a replayed not-landable result must not fail a batched request")

s.openGate(queue, gateGroup, heldBatch)
s.awaitStatus(req, entity.RequestStatusLanded)
s.awaitStatus(settle, entity.RequestStatusLanded)
assert.Equal(t, entity.RequestStateLanded, s.terminalState(req))
s.assertStatusesNever(req, entity.RequestStatusError)
}

// A batch left in Created is dependency-eligible, so everything created after
// it serializes behind it — and nothing will name it on the speculate topic
// again. Without a run that looks for such batches the queue wedges here with
Expand Down