diff --git a/doc/rfc/stovepipe/steps/buildanalysis.md b/doc/rfc/stovepipe/steps/buildanalysis.md index 31c270007..df3852bb0 100644 --- a/doc/rfc/stovepipe/steps/buildanalysis.md +++ b/doc/rfc/stovepipe/steps/buildanalysis.md @@ -18,6 +18,6 @@ Status-polling DLQ recovery does not publish a build-completion notice. The anal ## Analysis errors and dead letters -Analysis errors are returned to the normal consumer whether the Build is running or terminal; errors do not use the two-minute polling hold. The reference service uses backend error classification and the extended retry configuration: a maximum of 10 attempts, with retry delays of 5, 10, 20, 40, then 60 seconds, capped at 60 seconds. A successful hold resets failure accounting. Non-retryable errors and exhausted retries send the message to the analysis DLQ, so final analysis is attempted rather than guaranteed to succeed. +Analysis errors are returned to the normal consumer whether the Build is running or terminal; errors do not use the configured polling hold. The reference service uses backend error classification and the extended retry configuration: a maximum of 10 attempts, with retry delays of 5, 10, 20, 40, then 60 seconds, capped at 60 seconds. A successful hold resets failure accounting. Non-retryable errors and exhausted retries send the message to the analysis DLQ, so final analysis is attempted rather than guaranteed to succeed. The analysis DLQ uses the existing DLQ consumer, with its always-retryable processor and unlimited-retry subscription. The recovery controller records incomplete analysis in Request history without rerunning detection or changing the validation outcome. Only state reads and history persistence retry; unusable messages are logged and acknowledged. See [DLQ recovery](../../../../stovepipe/controller/buildanalysis/README.md#dlq-recovery) for the history identity and error-handling rules. diff --git a/doc/rfc/stovepipe/steps/buildsignal.md b/doc/rfc/stovepipe/steps/buildsignal.md index bb3bf1719..7e8aab897 100644 --- a/doc/rfc/stovepipe/steps/buildsignal.md +++ b/doc/rfc/stovepipe/steps/buildsignal.md @@ -115,10 +115,10 @@ Two-tier cadence, matching SubmitQueue: | Status | Delay | Rationale | |---|---|---| -| `accepted` | `PollDelayAcceptedMs` (default 5000ms) | Queued by the runner, not started — poll infrequently. | -| `running` | `PollDelayRunningMs` (default 2000ms) | Actively executing — poll more often. | +| `accepted` | `BuildSignal.AcceptedPollDelayMs` (default 5000ms) | Queued by the runner, not started — poll infrequently. | +| `running` | `BuildSignal.RunningPollDelayMs` (default 2000ms) | Actively executing — poll more often. | -Package-level `var`s (not `const`s) so tests can shorten them; the server always uses the defaults. Future: move these behind a `queueconfig`-style extension so operators can tune cadence per queue without a code change (the same TODO SubmitQueue's buildsignal carries). +Each delivery resolves these settings through the queue configuration store. Hosts apply a service baseline, deployment-wide settings, and per-queue overrides before supplying immutable configurations. The reference server embeds its baseline YAML and layers an optional configuration file over it. See the [configuration guide](../../../../service/stovepipe/server/README.md). ## Error classification diff --git a/service/stovepipe/server/BUILD.bazel b/service/stovepipe/server/BUILD.bazel index bc840af3b..812ec120b 100644 --- a/service/stovepipe/server/BUILD.bazel +++ b/service/stovepipe/server/BUILD.bazel @@ -104,11 +104,13 @@ go_test( "//stovepipe/entity:go_default_library", "//stovepipe/extension/buildissue/noop:go_default_library", "//stovepipe/extension/queueconfig:go_default_library", + "//stovepipe/extension/queueconfig/mock:go_default_library", "//stovepipe/extension/storage:go_default_library", "//stovepipe/extension/storage/mock:go_default_library", "@com_github_stretchr_testify//assert:go_default_library", "@com_github_stretchr_testify//require:go_default_library", "@com_github_uber_go_tally//:go_default_library", + "@org_golang_google_protobuf//proto:go_default_library", "@org_uber_go_mock//gomock:go_default_library", "@org_uber_go_zap//zaptest:go_default_library", ], diff --git a/service/stovepipe/server/README.md b/service/stovepipe/server/README.md index 32873526d..0dbfdd8ef 100644 --- a/service/stovepipe/server/README.md +++ b/service/stovepipe/server/README.md @@ -15,5 +15,3 @@ Deployment-wide stage settings from the optional file override the service basel Every queue override must name a queue in `MQ_TENANTS`; queues without overrides inherit the deployment configuration. Queue registration is separate from stage settings. The static store only serves registered queues. The baseline retains a concurrency limit of one, a five-second admission wait, five-second accepted-build polling, two-second running-build polling, two-minute build analysis polling, and thirty-second issue refresh and recovery waits. These values are defined only in the baseline YAML. - -Process-stage admission consumes this configuration. Other stage controllers retain their existing intervals. diff --git a/service/stovepipe/server/main.go b/service/stovepipe/server/main.go index c879f5fed..1811606a3 100644 --- a/service/stovepipe/server/main.go +++ b/service/stovepipe/server/main.go @@ -459,6 +459,7 @@ func registerPrimaryControllers( logger, scope, store, + configs, materializer, brf, registry, @@ -474,6 +475,7 @@ func registerPrimaryControllers( logger, scope, store, + configs, materializer, detectorFactory, registry, @@ -489,6 +491,7 @@ func registerPrimaryControllers( logger, scope, store, + configs, materializer, registry, stovepipemq.TopicKeyIssueTrack, diff --git a/service/stovepipe/server/main_test.go b/service/stovepipe/server/main_test.go index 5c88ac2ca..5f014520d 100644 --- a/service/stovepipe/server/main_test.go +++ b/service/stovepipe/server/main_test.go @@ -35,10 +35,12 @@ import ( "github.com/uber/submitqueue/stovepipe/entity" buildissuenoop "github.com/uber/submitqueue/stovepipe/extension/buildissue/noop" "github.com/uber/submitqueue/stovepipe/extension/queueconfig" + queueconfigmock "github.com/uber/submitqueue/stovepipe/extension/queueconfig/mock" "github.com/uber/submitqueue/stovepipe/extension/storage" storagemock "github.com/uber/submitqueue/stovepipe/extension/storage/mock" "go.uber.org/mock/gomock" "go.uber.org/zap/zaptest" + "google.golang.org/protobuf/proto" ) // recordingConsumer captures what the host registers instead of subscribing. @@ -393,3 +395,59 @@ func TestBuildAnalysisRegistration(t *testing.T) { assert.True(t, cfg.DLQ.Enabled) assert.Equal(t, "_dlq", cfg.DLQ.TopicSuffix) } + +func TestStageControllersUseHostConfiguration(t *testing.T) { + const queue = "configured-queue" + + for _, tt := range []struct { + name string + topic consumer.TopicKey + notification proto.Message + partition string + }{ + { + "build signal", + stovepipemq.TopicKeyBuildSignal, + &stovepipemq.BuildSignal{Id: "build", QueueName: queue}, + "build", + }, + { + "build analysis", + stovepipemq.TopicKeyBuildAnalysis, + &stovepipemq.BuildAnalysis{Id: "build", QueueName: queue}, + "build", + }, + { + "issue tracking", + stovepipemq.TopicKeyIssueTrack, + &stovepipemq.IssueTrack{RequestId: "1", QueueName: queue}, + queue, + }, + } { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + configs := queueconfigmock.NewMockStore(ctrl) + configs.EXPECT().Get(gomock.Any(), queue).Return(entity.QueueConfig{}, assert.AnError) + + _, controllers, _ := registeredControllersWithConfig(t, configs) + + payload, err := stovepipemq.Marshal(tt.notification) + require.NoError(t, err) + + message := entityqueue.NewMessage("notification", payload, tt.partition, nil) + message.Tenant = queue + + delivery := consumermock.NewMockDelivery(ctrl) + delivery.EXPECT().Message().Return(message) + + for _, controller := range controllers { + if controller.TopicKey() == tt.topic { + require.ErrorIs(t, controller.Process(context.Background(), delivery), assert.AnError) + return + } + } + + require.Fail(t, "stage controller was not registered") + }) + } +} diff --git a/stovepipe/controller/build/BUILD.bazel b/stovepipe/controller/build/BUILD.bazel index 540880e2a..1422ecd23 100644 --- a/stovepipe/controller/build/BUILD.bazel +++ b/stovepipe/controller/build/BUILD.bazel @@ -48,6 +48,7 @@ go_test( "//stovepipe/extension/buildissue/mock:go_default_library", "//stovepipe/extension/buildrunner:go_default_library", "//stovepipe/extension/buildrunner/mock:go_default_library", + "//stovepipe/extension/queueconfig/mock:go_default_library", "//stovepipe/extension/storage:go_default_library", "//stovepipe/extension/storage/mock:go_default_library", "@com_github_stretchr_testify//assert:go_default_library", diff --git a/stovepipe/controller/build/build_analysis_test.go b/stovepipe/controller/build/build_analysis_test.go index be01646cc..7356e7925 100644 --- a/stovepipe/controller/build/build_analysis_test.go +++ b/stovepipe/controller/build/build_analysis_test.go @@ -31,6 +31,7 @@ import ( "github.com/uber/submitqueue/stovepipe/extension/buildissue" buildissuemock "github.com/uber/submitqueue/stovepipe/extension/buildissue/mock" "github.com/uber/submitqueue/stovepipe/extension/buildrunner" + queueconfigmock "github.com/uber/submitqueue/stovepipe/extension/queueconfig/mock" "github.com/uber/submitqueue/stovepipe/extension/storage" storagemock "github.com/uber/submitqueue/stovepipe/extension/storage/mock" "go.uber.org/mock/gomock" @@ -142,11 +143,33 @@ func TestBuildAnalysisPipeline(t *testing.T) { detector.EXPECT().Detect(gomock.Any(), entity.BuildID{ID: testBuildID}).Return([]entity.BuildIssue{a, b}, nil), ) - analysisController := buildanalysis.NewController(zap.NewNop().Sugar(), tally.NoopScope, - staticStorageFactory{store: m.store}, requestlog.NewMaterializer(tally.NoopScope), detectorFactory, buildController.registry, stovepipemq.TopicKeyBuildAnalysis, "analysis") - statusController := buildsignal.NewController(zap.NewNop().Sugar(), tally.NoopScope, - staticStorageFactory{store: m.store}, m.materializer, m.runnerFactory, buildController.registry, - stovepipemq.TopicKeyBuildSignal, "signal") + configs := queueconfigmock.NewMockStore(ctrl) + cfg := entity.QueueConfig{Name: testQueue} + cfg.Stages.BuildAnalysis.PollDelayMs = 137 + configs.EXPECT().Get(gomock.Any(), testQueue).Return(cfg, nil).AnyTimes() + + analysisController := buildanalysis.NewController( + zap.NewNop().Sugar(), + tally.NoopScope, + staticStorageFactory{store: m.store}, + configs, + requestlog.NewMaterializer(tally.NoopScope), + detectorFactory, + buildController.registry, + stovepipemq.TopicKeyBuildAnalysis, + "analysis", + ) + statusController := buildsignal.NewController( + zap.NewNop().Sugar(), + tally.NoopScope, + staticStorageFactory{store: m.store}, + configs, + m.materializer, + m.runnerFactory, + buildController.registry, + stovepipemq.TopicKeyBuildSignal, + "signal", + ) delivered := func(msg entityqueue.Message) *consumermock.MockDelivery { d := consumermock.NewMockDelivery(ctrl) @@ -159,13 +182,13 @@ func TestBuildAnalysisPipeline(t *testing.T) { require.Len(t, published["buildsignal"], 1) firstPoll := delivered(published["buildanalysis"][0]) - firstPoll.EXPECT().Hold(buildanalysis.PollDelayMs) + firstPoll.EXPECT().Hold(cfg.Stages.BuildAnalysis.PollDelayMs) require.NoError(t, analysisController.Process(ctx, firstPoll)) assert.True(t, persisted[a]) assert.False(t, persisted[b]) secondPoll := delivered(published["buildanalysis"][0]) - secondPoll.EXPECT().Hold(buildanalysis.PollDelayMs) + secondPoll.EXPECT().Hold(cfg.Stages.BuildAnalysis.PollDelayMs) require.NoError(t, analysisController.Process(ctx, secondPoll)) assert.True(t, persisted[b]) diff --git a/stovepipe/controller/buildanalysis/BUILD.bazel b/stovepipe/controller/buildanalysis/BUILD.bazel index 4c63ee231..51d5c6675 100644 --- a/stovepipe/controller/buildanalysis/BUILD.bazel +++ b/stovepipe/controller/buildanalysis/BUILD.bazel @@ -18,6 +18,7 @@ go_library( "//stovepipe/core/requestlog:go_default_library", "//stovepipe/entity:go_default_library", "//stovepipe/extension/buildissue:go_default_library", + "//stovepipe/extension/queueconfig:go_default_library", "//stovepipe/extension/storage:go_default_library", "@com_github_uber_go_tally//:go_default_library", "@org_uber_go_zap//:go_default_library", @@ -46,6 +47,7 @@ go_test( "//stovepipe/entity:go_default_library", "//stovepipe/extension/buildissue:go_default_library", "//stovepipe/extension/buildissue/mock:go_default_library", + "//stovepipe/extension/queueconfig/mock:go_default_library", "//stovepipe/extension/storage:go_default_library", "//stovepipe/extension/storage/mock:go_default_library", "@com_github_stretchr_testify//assert:go_default_library", diff --git a/stovepipe/controller/buildanalysis/README.md b/stovepipe/controller/buildanalysis/README.md index e0c0df08d..dcd09088a 100644 --- a/stovepipe/controller/buildanalysis/README.md +++ b/stovepipe/controller/buildanalysis/README.md @@ -18,7 +18,7 @@ All analysis deliveries for a build share its build-ID partition and consumer gr ## Polling and completion -Every delivery reloads the Build and its Request before detection. After a successful pass, a nonterminal Build holds the delivery for two minutes; a terminal Build completes after persistence and completion history once its Request outcome is known, or the Request has ended without one. A pass that begins with a nonterminal Build always holds, even if the Build finishes during detection, so another delivery analyzes it after observing terminal status. More than one delivery may analyze the same terminal Build; build-issue persistence remains idempotent. +Every delivery reloads the Build and its Request before detection. After a successful pass, a nonterminal Build holds the delivery for the configured polling interval (two minutes by default); a terminal Build completes after persistence and completion history once its Request outcome is known, or the Request has ended without one. A pass that begins with a nonterminal Build always holds, even if the Build finishes during detection, so another delivery analyzes it after observing terminal status. More than one delivery may analyze the same terminal Build; build-issue persistence remains idempotent. If the Request is terminal but the loaded Build is nonterminal, analysis reloads the Build before deciding to stop. The Build may have finished between the initial Build read and the Request read; reloading avoids skipping its final analysis. If it is still nonterminal, analysis retains abandonment and stops without detection or a hold. This can occur when the status-polling DLQ handler fails the Request without learning the actual build result, leaving the Build at its last observed status. @@ -26,7 +26,7 @@ Status polling and updates remain the responsibility of buildsignal. Analysis ne ## Errors -Detection, persistence, configuration, state-loading, and message-validation errors are returned to the consumer for its normal retry or DLQ policy, whether the Build is running or terminal. Analysis errors do not use the two-minute polling hold. The consumer classifies errors and applies its configured retry limits and backoff; a final analysis can still fail and dead-letter. +Detection, persistence, configuration, state-loading, and message-validation errors are returned to the consumer for its normal retry or DLQ policy, whether the Build is running or terminal. Analysis errors do not use the configured polling hold. The consumer classifies errors and applies its configured retry limits and backoff; a final analysis can still fail and dead-letter. ## DLQ recovery diff --git a/stovepipe/controller/buildanalysis/buildanalysis.go b/stovepipe/controller/buildanalysis/buildanalysis.go index f9269491f..3f5be048f 100644 --- a/stovepipe/controller/buildanalysis/buildanalysis.go +++ b/stovepipe/controller/buildanalysis/buildanalysis.go @@ -32,17 +32,15 @@ import ( "github.com/uber/submitqueue/stovepipe/core/requestlog" "github.com/uber/submitqueue/stovepipe/entity" "github.com/uber/submitqueue/stovepipe/extension/buildissue" + "github.com/uber/submitqueue/stovepipe/extension/queueconfig" "github.com/uber/submitqueue/stovepipe/extension/storage" "go.uber.org/zap" ) -// PollDelayMs is the delay in milliseconds between successful polling passes. -// Two minutes limits provider traffic during long-running builds. -const PollDelayMs int64 = 120000 - // Controller accumulates build issues while builds run and after completion. // Terminal analysis must persist all build issues and its completion event before acknowledging. type Controller struct { + queueConfigs queueconfig.Store logger *zap.SugaredLogger metricsScope tally.Scope stores storage.Factory @@ -57,6 +55,7 @@ func NewController( logger *zap.SugaredLogger, scope tally.Scope, stores storage.Factory, + queueConfigs queueconfig.Store, materializer requestlog.Materializer, detectorFactory buildissue.DetectorFactory, registry consumer.TopicRegistry, @@ -64,6 +63,7 @@ func NewController( consumerGroup string, ) *Controller { return &Controller{ + queueConfigs: queueConfigs, logger: logger.Named("buildanalysis_controller"), metricsScope: scope.SubScope("buildanalysis_controller"), stores: stores, @@ -98,6 +98,11 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) (r return fmt.Errorf("build analysis must be partitioned by build %q", analysis.GetId()) } + cfg, err := c.queueConfigs.Get(ctx, analysis.GetQueueName()) + if err != nil { + return fmt.Errorf("load queue configuration: %w", err) + } + store, err := c.stores.For(storage.Config{QueueName: analysis.GetQueueName()}) if err != nil { return fmt.Errorf("failed to resolve storage for queue %q: %w", analysis.GetQueueName(), err) @@ -207,7 +212,7 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) (r } if hold { - delivery.Hold(PollDelayMs) + delivery.Hold(cfg.Stages.BuildAnalysis.PollDelayMs) } c.logger.Debugw("build analysis pass complete", diff --git a/stovepipe/controller/buildanalysis/buildanalysis_test.go b/stovepipe/controller/buildanalysis/buildanalysis_test.go index 096436378..cd3a054ed 100644 --- a/stovepipe/controller/buildanalysis/buildanalysis_test.go +++ b/stovepipe/controller/buildanalysis/buildanalysis_test.go @@ -33,12 +33,15 @@ import ( "github.com/uber/submitqueue/stovepipe/entity" "github.com/uber/submitqueue/stovepipe/extension/buildissue" buildissuemock "github.com/uber/submitqueue/stovepipe/extension/buildissue/mock" + queueconfigmock "github.com/uber/submitqueue/stovepipe/extension/queueconfig/mock" "github.com/uber/submitqueue/stovepipe/extension/storage" storagemock "github.com/uber/submitqueue/stovepipe/extension/storage/mock" "go.uber.org/mock/gomock" "go.uber.org/zap" ) +const testPollDelayMs int64 = 137 + const ( testQueue = "validation/main" testBuildID = "build/42" @@ -132,8 +135,24 @@ func newTestController(t *testing.T) (*Controller, analysisMocks) { history.EXPECT().Get(gomock.Any(), gomock.Any(), "event/build_analysis_abandoned/"+testBuildID).DoAndReturn(readHistory).AnyTimes() m.historyRead = history.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(readHistory).AnyTimes() - c := NewController(zap.NewNop().Sugar(), m.scope, m.stores, m.materializer, m.detectorFactory, m.registry, - stovepipemq.TopicKeyBuildAnalysis, "stovepipe-buildanalysis") + configs := queueconfigmock.NewMockStore(ctrl) + + cfg := entity.QueueConfig{Name: testQueue} + cfg.Stages.BuildAnalysis.PollDelayMs = testPollDelayMs + + configs.EXPECT().Get(gomock.Any(), testQueue).Return(cfg, nil).AnyTimes() + + c := NewController( + zap.NewNop().Sugar(), + m.scope, + m.stores, + configs, + m.materializer, + m.detectorFactory, + m.registry, + stovepipemq.TopicKeyBuildAnalysis, + "stovepipe-buildanalysis", + ) return c, m } @@ -200,7 +219,7 @@ func TestProcessPersistsBeforeHoldingOrCompleting(t *testing.T) { if status.IsTerminal() { m.historyCreate.After(last) } else { - d.EXPECT().Hold(PollDelayMs).After(last) + d.EXPECT().Hold(testPollDelayMs).After(last) } require.NoError(t, c.Process(context.Background(), d)) @@ -224,7 +243,7 @@ func TestProcessAcceptsRepeatedAndTaskWideObservations(t *testing.T) { Return(fmt.Errorf("already recorded: %w", storage.ErrAlreadyExists)).After(last) d := m.delivery(t) - d.EXPECT().Hold(PollDelayMs).After(last) + d.EXPECT().Hold(testPollDelayMs).After(last) require.NoError(t, c.Process(context.Background(), d)) } @@ -256,7 +275,7 @@ func TestProcessRetriesPartialPersistenceAndReloadsStatus(t *testing.T) { assert.True(t, errs.IsRetryable(err)) } else { if !pass.status.IsTerminal() { - d.EXPECT().Hold(PollDelayMs).After(last) + d.EXPECT().Hold(testPollDelayMs).After(last) } require.NoError(t, c.Process(context.Background(), d)) @@ -284,7 +303,7 @@ func TestProcessRunsAgainWhenBuildCompletesDuringDetection(t *testing.T) { ) first := m.delivery(t) - first.EXPECT().Hold(PollDelayMs) + first.EXPECT().Hold(testPollDelayMs) require.NoError(t, c.Process(context.Background(), first)) @@ -568,7 +587,7 @@ func TestProcessWaitsForWinningBuildBeforeFinalizing(t *testing.T) { d := m.delivery(t) if outcome != "another build wins" { if outcome == "pending" { - d.EXPECT().Hold(PollDelayMs).After(detected) + d.EXPECT().Hold(testPollDelayMs).After(detected) } } @@ -650,6 +669,7 @@ func TestProjectAnalysisPublicationRetries(t *testing.T) { c, m := newTestController(t) *m.publishFailure = assert.AnError m.expectPass(entity.BuildStatusFailed, nil, nil) + require.ErrorIs(t, c.Process(context.Background(), m.delivery(t)), assert.AnError) require.Len(t, m.history, 1) *m.publishFailure = nil @@ -673,3 +693,15 @@ func TestAbandonedAnalysisDoesNotResumeAfterOutcome(t *testing.T) { require.NoError(t, c.Process(context.Background(), m.delivery(t))) require.Empty(t, *m.published) } + +func TestProcessReturnsConfigurationError(t *testing.T) { + c, m := newTestController(t) + + configs := queueconfigmock.NewMockStore(m.ctrl) + + configs.EXPECT().Get(gomock.Any(), testQueue).Return(entity.QueueConfig{}, assert.AnError) + + c.queueConfigs = configs + + require.ErrorIs(t, c.Process(context.Background(), m.delivery(t)), assert.AnError) +} diff --git a/stovepipe/controller/buildanalysis/history_test.go b/stovepipe/controller/buildanalysis/history_test.go index 1ddcdb1af..caada2681 100644 --- a/stovepipe/controller/buildanalysis/history_test.go +++ b/stovepipe/controller/buildanalysis/history_test.go @@ -81,7 +81,7 @@ func TestCompletedAnalysisHistoryRepairsOnRetry(t *testing.T) { } d := m.delivery(t) if waiting { - d.EXPECT().Hold(PollDelayMs).After(last) + d.EXPECT().Hold(testPollDelayMs).After(last) } err := c.Process(context.Background(), d) if historyFailure { diff --git a/stovepipe/controller/buildsignal/BUILD.bazel b/stovepipe/controller/buildsignal/BUILD.bazel index 77ac7b005..96e65a7ea 100644 --- a/stovepipe/controller/buildsignal/BUILD.bazel +++ b/stovepipe/controller/buildsignal/BUILD.bazel @@ -16,6 +16,7 @@ go_library( "//stovepipe/core/requestlog:go_default_library", "//stovepipe/entity:go_default_library", "//stovepipe/extension/buildrunner:go_default_library", + "//stovepipe/extension/queueconfig:go_default_library", "//stovepipe/extension/storage:go_default_library", "@com_github_uber_go_tally//:go_default_library", "@org_uber_go_zap//:go_default_library", @@ -41,6 +42,7 @@ go_test( "//stovepipe/entity:go_default_library", "//stovepipe/extension/buildrunner:go_default_library", "//stovepipe/extension/buildrunner/mock:go_default_library", + "//stovepipe/extension/queueconfig/mock:go_default_library", "//stovepipe/extension/storage:go_default_library", "//stovepipe/extension/storage/mock:go_default_library", "@com_github_stretchr_testify//assert:go_default_library", diff --git a/stovepipe/controller/buildsignal/buildsignal.go b/stovepipe/controller/buildsignal/buildsignal.go index d4ccadcb9..debb63217 100644 --- a/stovepipe/controller/buildsignal/buildsignal.go +++ b/stovepipe/controller/buildsignal/buildsignal.go @@ -37,30 +37,18 @@ import ( "github.com/uber/submitqueue/stovepipe/core/requestlog" "github.com/uber/submitqueue/stovepipe/entity" "github.com/uber/submitqueue/stovepipe/extension/buildrunner" + "github.com/uber/submitqueue/stovepipe/extension/queueconfig" "github.com/uber/submitqueue/stovepipe/extension/storage" "go.uber.org/zap" ) -// Poll delays for non-terminal statuses. Vars (not consts) so tests can -// shorten them; the server always uses the defaults. -// -// TODO: move these behind a queueconfig-style extension so operators can -// tune poll cadence per queue without a code change. -var ( - // PollDelayAcceptedMs is the delay before the next Status call while the - // build is queued by the runner but has not started executing. - PollDelayAcceptedMs int64 = 5000 - // PollDelayRunningMs is the delay before the next Status call while the - // build is executing. - PollDelayRunningMs int64 = 2000 -) - // Controller consumes BuildSignal messages, polls the build-runner toward a // terminal status, persists the result, and either holds the delivery until // the next poll or releases the queue's build slot, projects the outcome onto // the request, and queues terminal analysis. Implements // consumer.Controller. type Controller struct { + queueConfigs queueconfig.Store logger *zap.SugaredLogger metricsScope tally.Scope stores storage.Factory @@ -79,6 +67,7 @@ func NewController( logger *zap.SugaredLogger, scope tally.Scope, stores storage.Factory, + queueConfigs queueconfig.Store, materializer requestlog.Materializer, buildRunners buildrunner.Factory, registry consumer.TopicRegistry, @@ -86,6 +75,7 @@ func NewController( consumerGroup string, ) *Controller { return &Controller{ + queueConfigs: queueConfigs, logger: logger.Named("buildsignal_controller"), metricsScope: scope.SubScope("buildsignal_controller"), stores: stores, @@ -115,6 +105,12 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er if err := entityqueue.ValidatePayloadQueue(msg, sig.GetQueueName()); err != nil { return fmt.Errorf("invalid message identity: %w", err) } + + cfg, err := c.queueConfigs.Get(ctx, sig.GetQueueName()) + if err != nil { + return fmt.Errorf("load queue configuration: %w", err) + } + store, err := c.stores.For(storage.Config{QueueName: sig.GetQueueName()}) if err != nil { metrics.NamedCounter(c.metricsScope, _opName, "storage_resolve_errors", 1, metrics.TagsFromContext(ctx)...) @@ -209,7 +205,7 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er // Not terminal yet: hold the delivery so this same message redelivers // after the poll delay — the partition (keyed by build id) sleeps with it, // and the redelivery does not count toward the retry limit. - delayMs := pollDelay(effective) + delayMs := pollDelay(effective, cfg.Stages.BuildSignal) delivery.Hold(delayMs) c.logger.Debugw("holding for next build status poll", "build_id", build.ID, @@ -458,13 +454,13 @@ func (c *Controller) loadRequest(ctx context.Context, store storage.Storage, id } // pollDelay returns the delay before the next Status call for a non-terminal status. -func pollDelay(status entity.BuildStatus) int64 { +func pollDelay(status entity.BuildStatus, cfg entity.BuildSignalConfig) int64 { switch status { case entity.BuildStatusRunning: - return PollDelayRunningMs + return cfg.RunningPollDelayMs default: // Accepted and any unknown non-terminal status. - return PollDelayAcceptedMs + return cfg.AcceptedPollDelayMs } } diff --git a/stovepipe/controller/buildsignal/buildsignal_test.go b/stovepipe/controller/buildsignal/buildsignal_test.go index 6676889fb..5d862a655 100644 --- a/stovepipe/controller/buildsignal/buildsignal_test.go +++ b/stovepipe/controller/buildsignal/buildsignal_test.go @@ -22,6 +22,7 @@ import ( "testing" "github.com/uber/submitqueue/platform/publish" + queueconfigmock "github.com/uber/submitqueue/stovepipe/extension/queueconfig/mock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -45,6 +46,11 @@ import ( "go.uber.org/zap" ) +const ( + testRunningPollDelayMs int64 = 139 + testAcceptedPollDelayMs int64 = 149 +) + const ( testQueue = "monorepo/main" testID = "7" @@ -106,7 +112,26 @@ func newController(t *testing.T, ctrl *gomock.Controller) (*Controller, buildsig }) require.NoError(t, err) - c := NewController(zap.NewNop().Sugar(), scope, staticStorageFactory{store: m.store}, m.materializer, m.runnerFactory, registry, stovepipemq.TopicKeyBuildSignal, "stovepipe-buildsignal") + configs := queueconfigmock.NewMockStore(ctrl) + + cfg := entity.QueueConfig{Name: testQueue} + cfg.Stages.BuildSignal.AcceptedPollDelayMs = testAcceptedPollDelayMs + cfg.Stages.BuildSignal.RunningPollDelayMs = testRunningPollDelayMs + + configs.EXPECT().Get(gomock.Any(), testQueue).Return(cfg, nil).AnyTimes() + + c := NewController( + zap.NewNop().Sugar(), + scope, + staticStorageFactory{store: m.store}, + configs, + m.materializer, + m.runnerFactory, + registry, + stovepipemq.TopicKeyBuildSignal, + "stovepipe-buildsignal", + ) + return c, m } @@ -324,9 +349,20 @@ func TestProcess(t *testing.T) { m.runner.EXPECT().Status(gomock.Any(), entity.BuildID{ID: testBuildID}).Return(entity.BuildStatus(""), nil, errors.New("runner unavailable")) }, }, + + { + name: "accepted status uses its configured polling interval", + wantHoldMs: testAcceptedPollDelayMs, + setup: func(m buildsignalMocks) { + m.buildStore.EXPECT().Get(gomock.Any(), testBuildID).Return(build(entity.BuildStatusAccepted, 1), nil) + m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateProcessing), nil) + m.runnerFactory.EXPECT().For(buildrunner.Config{QueueName: testQueue}).Return(m.runner, nil) + m.runner.EXPECT().Status(gomock.Any(), entity.BuildID{ID: testBuildID}).Return(entity.BuildStatusAccepted, nil, nil) + }, + }, { name: "unchanged status skips write and holds for next poll", - wantHoldMs: PollDelayRunningMs, + wantHoldMs: testRunningPollDelayMs, setup: func(m buildsignalMocks) { m.buildStore.EXPECT().Get(gomock.Any(), testBuildID).Return(build(entity.BuildStatusRunning, 2), nil) m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateProcessing), nil) @@ -336,7 +372,7 @@ func TestProcess(t *testing.T) { }, { name: "status transition persists and holds for next poll", - wantHoldMs: PollDelayRunningMs, + wantHoldMs: testRunningPollDelayMs, setup: func(m buildsignalMocks) { m.buildStore.EXPECT().Get(gomock.Any(), testBuildID).Return(build(entity.BuildStatusAccepted, 1), nil) m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateProcessing), nil) @@ -708,7 +744,7 @@ func TestNonterminalPollDoesNotPublishTerminalAnalysis(t *testing.T) { m.runner.EXPECT().Status(gomock.Any(), entity.BuildID{ID: testBuildID}).Return(entity.BuildStatusRunning, nil, nil) d := delivery(t, ctrl, buildSignalPayload(t, testBuildID)) - d.EXPECT().Hold(PollDelayRunningMs) + d.EXPECT().Hold(testRunningPollDelayMs) require.NoError(t, c.Process(queueContext(), d)) } @@ -802,3 +838,16 @@ func TestProcessBookmarkUsesWinningRequestOutcome(t *testing.T) { }) } } + +func TestProcessReturnsConfigurationError(t *testing.T) { + ctrl := gomock.NewController(t) + c, _ := newController(t, ctrl) + + configs := queueconfigmock.NewMockStore(ctrl) + + configs.EXPECT().Get(gomock.Any(), testQueue).Return(entity.QueueConfig{}, assert.AnError) + + c.queueConfigs = configs + + require.ErrorIs(t, c.Process(queueContext(), delivery(t, ctrl, buildSignalPayload(t, testBuildID))), assert.AnError) +} diff --git a/stovepipe/controller/issuetrack/BUILD.bazel b/stovepipe/controller/issuetrack/BUILD.bazel index 6f3008b4d..d5642705e 100644 --- a/stovepipe/controller/issuetrack/BUILD.bazel +++ b/stovepipe/controller/issuetrack/BUILD.bazel @@ -23,6 +23,7 @@ go_library( "//stovepipe/core/messagequeue:go_default_library", "//stovepipe/core/requestlog:go_default_library", "//stovepipe/entity:go_default_library", + "//stovepipe/extension/queueconfig:go_default_library", "//stovepipe/extension/storage:go_default_library", "@com_github_uber_go_tally//:go_default_library", "@org_uber_go_zap//:go_default_library", @@ -54,6 +55,7 @@ go_test( "//stovepipe/core/requestlog:go_default_library", "//stovepipe/core/requestlog/mock:go_default_library", "//stovepipe/entity:go_default_library", + "//stovepipe/extension/queueconfig/mock:go_default_library", "//stovepipe/extension/storage:go_default_library", "//stovepipe/extension/storage/mock:go_default_library", "@com_github_stretchr_testify//assert:go_default_library", diff --git a/stovepipe/controller/issuetrack/README.md b/stovepipe/controller/issuetrack/README.md index d6c1f08b9..0e2c88754 100644 --- a/stovepipe/controller/issuetrack/README.md +++ b/stovepipe/controller/issuetrack/README.md @@ -8,7 +8,7 @@ Admission registers one request-scoped tracking job before committing the reques The tracking subscription must fetch only the first unacknowledged message in each queue partition. The reference host uses the shared `SingleMessageSubscriptionConfig`, whose message batch size is one; reconciliation still processes up to 100 issue keys per delivery. A larger message batch can prefetch later requests before the current job establishes its hold. Build execution and per-build detection remain concurrent. -A request's job stays at the queue head through ordinary refreshes. Successful refreshes hold for 30 seconds; unfinished batches request immediate redelivery. The current request cannot be replaced until it is finalized or safely abandoned. A later request that reaches processing during recovery waits for that boundary rather than retrying the failed request's remaining work. +A request's job stays at the queue head through ordinary refreshes. Successful refreshes hold for the configured wait (30 seconds by default); unfinished batches request immediate redelivery. The current request cannot be replaced until it is finalized or safely abandoned. A later request that reaches processing during recovery waits for that boundary rather than retrying the failed request's remaining work. Terminal readiness comes from the retained `build_analysis_completed` event, written only after the complete terminal issue set has been saved. `build_analysis_abandoned` explicitly ends incomplete analysis without allowing absence-based closure. These are point reads by the request and deterministic event ID, not scans of history. A completion event takes precedence if duplicate analysis deliveries retained both events. Neither CI status nor an empty issue set establishes readiness. The request, build, issue set, and analysis history must remain available until tracking finishes. @@ -82,7 +82,7 @@ A finalized checkpoint retains `issues_reconciled` in its Request's history befo Retryable errors return to normal consumer retry, and the next delivery resumes the cursor before advancing. Abandoned analysis and requests that end without a build outcome cannot complete tracking, so they return non-retryable errors and go directly to the DLQ. The DLQ controller owns all abandonment cleanup and stops the failed pass rather than rerunning its remaining batches: -1. Conditionally enter `abandoning`, retaining the source, notification, and cursor. Normal deliveries hold for 30 seconds while this phase is present; they cannot perform cleanup or restart issue application. +1. Conditionally enter `abandoning`, retaining the source, notification, and cursor. Normal deliveries hold for the configured wait (30 seconds by default) while this phase is present; they cannot perform cleanup or restart issue application. 2. If there is a cursor, read its issue and conditionally advance progress after that read. Advance the issue's version without changing its status or transition metadata. This invalidates a paused worker's earlier row guard. If the row has not been created yet, finish that one opening instead: the normal worker can only have saved such a cursor after finding a reported build issue. Creating it prevents the delayed create from succeeding. Closed issue rows are never deleted. 3. Repair the cursor's opening or closing history, summary, and open-issue lookup from the retained row. Record an `issue_tracking_abandoned` Request-history event keyed by the failed notification, with its build ID. 4. Conditionally enter `abandoned`. Keep the source-order boundary and notification identity, so the same delivery and older builds cannot restart the abandoned work. diff --git a/stovepipe/controller/issuetrack/dlq.go b/stovepipe/controller/issuetrack/dlq.go index 1176f65ec..825f73dbb 100644 --- a/stovepipe/controller/issuetrack/dlq.go +++ b/stovepipe/controller/issuetrack/dlq.go @@ -36,9 +36,6 @@ import ( "go.uber.org/zap" ) -// Avoid spinning the primary consumer while recovery retries storage or history writes. -const _recoveryDelayMs int64 = 30000 - // DLQController abandons incomplete issue tracking while retaining accepted issue changes. type DLQController struct { logger *zap.SugaredLogger diff --git a/stovepipe/controller/issuetrack/fixture_test.go b/stovepipe/controller/issuetrack/fixture_test.go index cea62f881..2956ee72d 100644 --- a/stovepipe/controller/issuetrack/fixture_test.go +++ b/stovepipe/controller/issuetrack/fixture_test.go @@ -29,6 +29,7 @@ import ( "github.com/uber/submitqueue/stovepipe/core/messagequeue" "github.com/uber/submitqueue/stovepipe/core/requestlog" "github.com/uber/submitqueue/stovepipe/entity" + queueconfigmock "github.com/uber/submitqueue/stovepipe/extension/queueconfig/mock" "github.com/uber/submitqueue/stovepipe/extension/storage" storagemock "github.com/uber/submitqueue/stovepipe/extension/storage/mock" "go.uber.org/mock/gomock" @@ -37,6 +38,11 @@ import ( const testQueue = "validation/main" +const ( + _analysisWaitMs int64 = 151 + _recoveryDelayMs int64 = 157 +) + type logKey struct { issueKey build string @@ -432,7 +438,25 @@ func newFixture(t *testing.T) *fixture { } return log, nil }).AnyTimes() - f.controller = NewController(zap.NewNop().Sugar(), tally.NoopScope, f.factory, f.materializer, registry, messagequeue.TopicKeyIssueTrack, "test") + + configs := queueconfigmock.NewMockStore(f.mocks) + + cfg := entity.QueueConfig{Name: testQueue} + cfg.Stages.IssueTrack.RefreshDelayMs = _analysisWaitMs + cfg.Stages.IssueTrack.RecoveryWaitDelayMs = _recoveryDelayMs + + configs.EXPECT().Get(gomock.Any(), testQueue).Return(cfg, nil).AnyTimes() + + f.controller = NewController( + zap.NewNop().Sugar(), + tally.NoopScope, + f.factory, + configs, + f.materializer, + registry, + messagequeue.TopicKeyIssueTrack, + "test", + ) f.controller.nowMs = func() int64 { return 1000 } return f } @@ -490,3 +514,15 @@ func (f *fixture) seedIssue(task, unit string, status entity.IssueStatus) { f.openIssues[issueKey{task, unit}] = entity.OpenIssue{TaskID: task, UnitID: unit, FirstSeenBuildID: "older"} } } + +func TestProcessReturnsConfigurationError(t *testing.T) { + f := newFixture(t) + + configs := queueconfigmock.NewMockStore(f.mocks) + + configs.EXPECT().Get(gomock.Any(), testQueue).Return(entity.QueueConfig{}, assert.AnError) + + f.controller.queueConfigs = configs + + require.ErrorIs(t, f.deliver(f.message("older", false), false), assert.AnError) +} diff --git a/stovepipe/controller/issuetrack/issuetrack.go b/stovepipe/controller/issuetrack/issuetrack.go index 29f812485..f1e8f9027 100644 --- a/stovepipe/controller/issuetrack/issuetrack.go +++ b/stovepipe/controller/issuetrack/issuetrack.go @@ -36,20 +36,21 @@ import ( stovepipemq "github.com/uber/submitqueue/stovepipe/core/messagequeue" "github.com/uber/submitqueue/stovepipe/core/requestlog" "github.com/uber/submitqueue/stovepipe/entity" + "github.com/uber/submitqueue/stovepipe/extension/queueconfig" "github.com/uber/submitqueue/stovepipe/extension/storage" "go.uber.org/zap" ) const ( // Each delivery processes at most this many issue keys, including a revisited cursor and closed records. - _issueBatchSize = 100 - _analysisWaitMs int64 = 30000 + _issueBatchSize = 100 ) type issueKey struct{ task, unit string } // Controller retains issue transitions and their queue-scoped progress. type Controller struct { + queueConfigs queueconfig.Store logger *zap.SugaredLogger metricsScope tally.Scope stores storage.Factory @@ -61,8 +62,18 @@ type Controller struct { nowMs func() int64 } -func NewController(logger *zap.SugaredLogger, scope tally.Scope, stores storage.Factory, materializer requestlog.Materializer, registry consumer.TopicRegistry, topicKey consumer.TopicKey, consumerGroup string) *Controller { +func NewController( + logger *zap.SugaredLogger, + scope tally.Scope, + stores storage.Factory, + queueConfigs queueconfig.Store, + materializer requestlog.Materializer, + registry consumer.TopicRegistry, + topicKey consumer.TopicKey, + consumerGroup string, +) *Controller { return &Controller{ + queueConfigs: queueConfigs, logger: logger.Named("issuetrack_controller"), metricsScope: scope.SubScope("issuetrack_controller"), stores: stores, @@ -94,14 +105,28 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er if msg.PartitionKey != queue { return fmt.Errorf("issue tracking must be partitioned by queue %q", queue) } + + cfg, err := c.queueConfigs.Get(ctx, queue) + if err != nil { + return fmt.Errorf("load queue configuration: %w", err) + } + store, err := c.stores.For(storage.Config{QueueName: queue}) if err != nil { return fmt.Errorf("resolve issue tracking storage: %w", err) } - return c.trackRequest(ctx, delivery, store, notification, msg.ID) + + return c.trackRequest(ctx, delivery, store, notification, msg.ID, cfg.Stages.IssueTrack) } -func (c *Controller) trackRequest(ctx context.Context, delivery consumer.Delivery, store storage.Storage, notification *stovepipemq.IssueTrack, notificationID string) error { +func (c *Controller) trackRequest( + ctx context.Context, + delivery consumer.Delivery, + store storage.Storage, + notification *stovepipemq.IssueTrack, + notificationID string, + cfg entity.IssueTrackConfig, +) error { queue := notification.GetQueueName() tracking := store.GetIssueTrackingStore() for { @@ -138,7 +163,7 @@ func (c *Controller) trackRequest(ctx context.Context, delivery consumer.Deliver } switch progress.Phase { case entity.IssueTrackingPhaseAbandoning: - delivery.Hold(_recoveryDelayMs) + delivery.Hold(cfg.RecoveryWaitDelayMs) return nil case entity.IssueTrackingPhaseFinalized, entity.IssueTrackingPhaseAbandoned: if order == 0 { @@ -146,7 +171,7 @@ func (c *Controller) trackRequest(ctx context.Context, delivery consumer.Deliver } default: if order > 0 { - delivery.Hold(_analysisWaitMs) + delivery.Hold(cfg.RefreshDelayMs) return nil } } @@ -161,7 +186,7 @@ func (c *Controller) trackRequest(ctx context.Context, delivery consumer.Deliver return nil } if !request.State.IsTerminal() { - delivery.Hold(_analysisWaitMs) + delivery.Hold(cfg.RefreshDelayMs) return nil } } @@ -247,7 +272,7 @@ func (c *Controller) trackRequest(ctx context.Context, delivery consumer.Deliver case entity.IssueTrackingPhaseRefreshing, entity.IssueTrackingPhaseFinalizing: delivery.Hold(0) case entity.IssueTrackingPhaseRefreshed: - delivery.Hold(_analysisWaitMs) + delivery.Hold(cfg.RefreshDelayMs) } return nil }