From bed50b0c513e9c856f2d7c6510d2d0d50bb01a21 Mon Sep 17 00:00:00 2001 From: "prath.shenoy" Date: Mon, 16 Nov 2026 22:01:05 +0000 Subject: [PATCH] feat(stovepipe): Load layered configuration --- doc/rfc/stovepipe/steps/process.md | 4 +- service/stovepipe/README.md | 4 + service/stovepipe/server/BUILD.bazel | 5 +- service/stovepipe/server/README.md | 19 ++ service/stovepipe/server/config/BUILD.bazel | 34 +++ service/stovepipe/server/config/base.yaml | 14 ++ .../stovepipe/server/config/build_analysis.go | 37 +++ .../stovepipe/server/config/build_signal.go | 45 ++++ service/stovepipe/server/config/config.go | 120 ++++++++++ .../stovepipe/server/config/config_test.go | 214 ++++++++++++++++++ service/stovepipe/server/config/duration.go | 33 +++ .../stovepipe/server/config/issue_track.go | 45 ++++ service/stovepipe/server/config/process.go | 40 ++++ service/stovepipe/server/config/stages.go | 60 +++++ service/stovepipe/server/main.go | 127 +++++++++-- service/stovepipe/server/main_test.go | 91 +++++++- stovepipe/controller/process/BUILD.bazel | 2 +- stovepipe/controller/process/process.go | 4 +- stovepipe/controller/process/process_test.go | 9 +- stovepipe/entity/queue_config.go | 30 ++- stovepipe/entity/stage_config_test.go | 12 + stovepipe/extension/queueconfig/README.md | 22 +- .../extension/queueconfig/default/default.go | 55 ----- .../queueconfig/default/default_test.go | 49 ---- .../extension/queueconfig/queueconfig.go | 2 +- .../{default => static}/BUILD.bazel | 7 +- .../extension/queueconfig/static/static.go | 103 +++++++++ .../queueconfig/static/static_test.go | 129 +++++++++++ 28 files changed, 1153 insertions(+), 163 deletions(-) create mode 100644 service/stovepipe/server/README.md create mode 100644 service/stovepipe/server/config/BUILD.bazel create mode 100644 service/stovepipe/server/config/base.yaml create mode 100644 service/stovepipe/server/config/build_analysis.go create mode 100644 service/stovepipe/server/config/build_signal.go create mode 100644 service/stovepipe/server/config/config.go create mode 100644 service/stovepipe/server/config/config_test.go create mode 100644 service/stovepipe/server/config/duration.go create mode 100644 service/stovepipe/server/config/issue_track.go create mode 100644 service/stovepipe/server/config/process.go create mode 100644 service/stovepipe/server/config/stages.go delete mode 100644 stovepipe/extension/queueconfig/default/default.go delete mode 100644 stovepipe/extension/queueconfig/default/default_test.go rename stovepipe/extension/queueconfig/{default => static}/BUILD.bazel (83%) create mode 100644 stovepipe/extension/queueconfig/static/static.go create mode 100644 stovepipe/extension/queueconfig/static/static_test.go diff --git a/doc/rfc/stovepipe/steps/process.md b/doc/rfc/stovepipe/steps/process.md index 24e9fffa9..f0fa87b88 100644 --- a/doc/rfc/stovepipe/steps/process.md +++ b/doc/rfc/stovepipe/steps/process.md @@ -60,13 +60,13 @@ Strategy and baseline are persisted on the Request and are **immutable**: a rede Validation is expensive and shares a baseline, so heads arriving while an earlier one runs `build → buildsignal → record` must not all start builds at once. -**Runtime state** lives on the Queue row; the **concurrency cap** does not — it is deployment configuration, resolved at gate-check time the same way SubmitQueue separates `storage` from `queueconfig` (see [submitqueue/extension/queueconfig/README.md](../../../../submitqueue/extension/queueconfig/README.md)): pipeline stages read mutable state from the store and read knobs like `max_concurrent` from a config `Store` (or a wiring default for MVP). Config is not written by ingest/process/record and does not need optimistic locking. +**Runtime state** lives on the Queue row; the **concurrency cap** does not — it is deployment configuration, resolved at gate-check time the same way SubmitQueue separates `storage` from `queueconfig` (see [submitqueue/extension/queueconfig/README.md](../../../../submitqueue/extension/queueconfig/README.md)): pipeline stages read mutable state from the store and read knobs like `max_concurrent` from a config `Store`. Config is not written by ingest/process/record and does not need optimistic locking. | Source | Field | Meaning | |---|---|---| | Queue row | `last_green_uri` | Bookmark `buildsignal` advances on a successful build; empty until first green. | | Queue row | `in_flight_count` | Requests past `process` and not yet terminal. `process` increments on admit; `record` (or DLQ reconciliation) decrements on terminal. | -| Queue config | `max_concurrent` | Cap on concurrent in-flight validations. **Default 1** (global wiring default for MVP; per-queue override when a Stovepipe `queueconfig` extension lands). | +| Queue config | `max_concurrent` | Cap on concurrent in-flight validations. **Default 1** in the reference service baseline; deployment-wide and per-queue overrides live under the process stage configuration. | A slot is held from admit until the build goes terminal (`process → build → buildsignal`), not just while `process` runs. It is released when the Request reaches **any** terminal state and `in_flight_count` is decremented — `buildsignal` recording the build's outcome, success *or* failure, or the DLQ reconciler forcing a terminal `failed` (see [integrity](#in_flight_count-integrity)). A build *failure* frees the slot just like a success; only a Request that never terminates keeps its slot. diff --git a/service/stovepipe/README.md b/service/stovepipe/README.md index 3aace589e..546dc3655 100644 --- a/service/stovepipe/README.md +++ b/service/stovepipe/README.md @@ -53,6 +53,10 @@ The Stovepipe controllers live under [`stovepipe/controller/`](../../stovepipe/c | `PORT` | no | gRPC listen address | `:8083` | | `HOSTNAME` | no | Subscriber name for the queue consumers | `stovepipe-` | +## Queue and stage configuration + +The server accepts an optional `-config` YAML file for deployment-wide queue and stage settings and per-queue overrides. See the [configuration guide](server/README.md) for settings, validation. The service always loads its embedded baseline; the optional file overrides it. + ## Running ### Docker Compose (recommended) diff --git a/service/stovepipe/server/BUILD.bazel b/service/stovepipe/server/BUILD.bazel index 8b6b60ef2..bc840af3b 100644 --- a/service/stovepipe/server/BUILD.bazel +++ b/service/stovepipe/server/BUILD.bazel @@ -23,6 +23,7 @@ go_library( "//platform/extension/messagequeue/mysql:go_default_library", "//platform/hook:go_default_library", "//service/messagequeue:go_default_library", + "//service/stovepipe/server/config:go_default_library", "//service/stovepipe/server/handler:go_default_library", "//stovepipe/controller:go_default_library", "//stovepipe/controller/build:go_default_library", @@ -44,7 +45,7 @@ go_library( "//stovepipe/extension/fixer/noop:go_default_library", "//stovepipe/extension/projectissues/noop:go_default_library", "//stovepipe/extension/projectstatus/noop:go_default_library", - "//stovepipe/extension/queueconfig/default:go_default_library", + "//stovepipe/extension/queueconfig:go_default_library", "//stovepipe/extension/sourcecontrol:go_default_library", "//stovepipe/extension/sourcecontrol/fake:go_default_library", "//stovepipe/extension/storage:go_default_library", @@ -96,11 +97,13 @@ go_test( "//platform/consumer/mock:go_default_library", "//platform/extension/messagequeue:go_default_library", "//platform/extension/messagequeue/mock:go_default_library", + "//service/stovepipe/server/config:go_default_library", "//stovepipe/controller/dlq:go_default_library", "//stovepipe/core/messagequeue:go_default_library", "//stovepipe/core/requestlog:go_default_library", "//stovepipe/entity:go_default_library", "//stovepipe/extension/buildissue/noop:go_default_library", + "//stovepipe/extension/queueconfig:go_default_library", "//stovepipe/extension/storage:go_default_library", "//stovepipe/extension/storage/mock:go_default_library", "@com_github_stretchr_testify//assert:go_default_library", diff --git a/service/stovepipe/server/README.md b/service/stovepipe/server/README.md new file mode 100644 index 000000000..32873526d --- /dev/null +++ b/service/stovepipe/server/README.md @@ -0,0 +1,19 @@ +# Stovepipe service configuration + +The server embeds `config/base.yaml` as its operational baseline and accepts an optional `-config` YAML file layered over it. Omitting the optional file still loads the baseline; there are no hidden Go fallback values. Invalid configuration fails startup before consumers start. Effective settings are logged at startup; updates require a restart. + +## Stage settings + +The `queueconfig.stages` section contains typed stage blocks. `process` owns `maxConcurrent` and `gateWaitDelay`. `buildSignal` owns `acceptedPollDelay` and `runningPollDelay`; `buildAnalysis` owns `pollDelay`; `issueTrack` owns `refreshDelay` and `recoveryWaitDelay`. Timing and non-timing settings belong to the stage that owns their behavior. + +Each stage has its own YAML adapter, shared override application, and validation. The shared library uses integer milliseconds; host adapters accept duration strings such as `2m`, `30s`, or `250ms`, restricted to whole milliseconds. Partial layers are merged before semantic validation. Omitted fields inherit; explicit zero is not omission. The current limits and intervals must be positive. Unknown fields, duplicate keys, and multiple YAML documents are rejected. + +## Layering and queues + +Deployment-wide stage settings from the optional file override the service baseline. Entries under `queueconfig.queues`, keyed by queue name, contain their own `stages` blocks and override deployment settings. When multiple files specify the same queue, its fields are merged rather than replacing its entire configuration. Queue layers retain file order and are applied after deployment-wide settings. + +Every queue override must name a queue in `MQ_TENANTS`; queues without overrides inherit the deployment configuration. Queue registration is separate from stage settings. The static store only serves registered queues. + +The baseline retains a concurrency limit of one, a five-second admission wait, five-second accepted-build polling, two-second running-build polling, two-minute build analysis polling, and thirty-second issue refresh and recovery waits. These values are defined only in the baseline YAML. + +Process-stage admission consumes this configuration. Other stage controllers retain their existing intervals. diff --git a/service/stovepipe/server/config/BUILD.bazel b/service/stovepipe/server/config/BUILD.bazel new file mode 100644 index 000000000..54b8d59ec --- /dev/null +++ b/service/stovepipe/server/config/BUILD.bazel @@ -0,0 +1,34 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "build_analysis.go", + "build_signal.go", + "config.go", + "duration.go", + "issue_track.go", + "process.go", + "stages.go", + ], + embedsrcs = ["base.yaml"], + importpath = "github.com/uber/submitqueue/service/stovepipe/server/config", + visibility = ["//service/stovepipe/server:__pkg__"], + deps = [ + "//stovepipe/entity:go_default_library", + "//stovepipe/extension/queueconfig:go_default_library", + "//stovepipe/extension/queueconfig/static:go_default_library", + "@in_gopkg_yaml_v3//:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["config_test.go"], + embed = [":go_default_library"], + deps = [ + "//stovepipe/extension/queueconfig:go_default_library", + "@com_github_stretchr_testify//assert:go_default_library", + "@com_github_stretchr_testify//require:go_default_library", + ], +) diff --git a/service/stovepipe/server/config/base.yaml b/service/stovepipe/server/config/base.yaml new file mode 100644 index 000000000..1bfa7013d --- /dev/null +++ b/service/stovepipe/server/config/base.yaml @@ -0,0 +1,14 @@ +queueconfig: + stages: + process: + maxConcurrent: 1 + gateWaitDelay: 5s + buildSignal: + acceptedPollDelay: 5s + runningPollDelay: 2s + buildAnalysis: + pollDelay: 2m + issueTrack: + refreshDelay: 30s + recoveryWaitDelay: 30s + queues: {} diff --git a/service/stovepipe/server/config/build_analysis.go b/service/stovepipe/server/config/build_analysis.go new file mode 100644 index 000000000..4f0441942 --- /dev/null +++ b/service/stovepipe/server/config/build_analysis.go @@ -0,0 +1,37 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + "time" + + "github.com/uber/submitqueue/stovepipe/extension/queueconfig" +) + +type buildAnalysisSettings struct { + PollDelay *time.Duration `yaml:"pollDelay"` +} + +func (s buildAnalysisSettings) overrides() (queueconfig.BuildAnalysisOverrides, error) { + pollDelay, err := durationMilliseconds(s.PollDelay) + if err != nil { + return queueconfig.BuildAnalysisOverrides{}, fmt.Errorf("pollDelay: %w", err) + } + + return queueconfig.BuildAnalysisOverrides{ + PollDelayMs: pollDelay, + }, nil +} diff --git a/service/stovepipe/server/config/build_signal.go b/service/stovepipe/server/config/build_signal.go new file mode 100644 index 000000000..9d431ed72 --- /dev/null +++ b/service/stovepipe/server/config/build_signal.go @@ -0,0 +1,45 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + "time" + + "github.com/uber/submitqueue/stovepipe/extension/queueconfig" +) + +type buildSignalSettings struct { + AcceptedPollDelay *time.Duration `yaml:"acceptedPollDelay"` + + RunningPollDelay *time.Duration `yaml:"runningPollDelay"` +} + +func (s buildSignalSettings) overrides() (queueconfig.BuildSignalOverrides, error) { + acceptedPollDelay, err := durationMilliseconds(s.AcceptedPollDelay) + if err != nil { + return queueconfig.BuildSignalOverrides{}, fmt.Errorf("acceptedPollDelay: %w", err) + } + + runningPollDelay, err := durationMilliseconds(s.RunningPollDelay) + if err != nil { + return queueconfig.BuildSignalOverrides{}, fmt.Errorf("runningPollDelay: %w", err) + } + + return queueconfig.BuildSignalOverrides{ + AcceptedPollDelayMs: acceptedPollDelay, + RunningPollDelayMs: runningPollDelay, + }, nil +} diff --git a/service/stovepipe/server/config/config.go b/service/stovepipe/server/config/config.go new file mode 100644 index 000000000..1a1ca2f29 --- /dev/null +++ b/service/stovepipe/server/config/config.go @@ -0,0 +1,120 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package config loads the reference service's layered startup configuration. +package config + +import ( + "bytes" + _ "embed" + "fmt" + "io" + "os" + + "github.com/uber/submitqueue/stovepipe/entity" + "github.com/uber/submitqueue/stovepipe/extension/queueconfig" + "github.com/uber/submitqueue/stovepipe/extension/queueconfig/static" + "gopkg.in/yaml.v3" +) + +//go:embed base.yaml +var baseYAML []byte + +type document struct { + QueueConfig queueSection `yaml:"queueconfig"` +} + +type queueSection struct { + queueSettings `yaml:",inline"` + + Queues map[string]queueSettings `yaml:"queues"` +} + +type queueSettings struct { + Stages stageSettings `yaml:"stages"` +} + +// Load layers an optional file over the embedded service baseline. +// Queue-specific settings override deployment settings. Updates require a restart. +func Load(path string, names []string) (queueconfig.Store, error) { + base, err := decodeDocument(bytes.NewReader(baseYAML)) + if err != nil { + return nil, fmt.Errorf("decode service baseline: %w", err) + } + + layers := []document{base} + + if path != "" { + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("open configuration %q: %w", path, err) + } + defer file.Close() + + layer, err := decodeDocument(file) + if err != nil { + return nil, fmt.Errorf("decode configuration %q: %w", path, err) + } + + layers = append(layers, layer) + } + + return newQueueConfigStore(names, layers...) +} + +func decodeDocument(reader io.Reader) (document, error) { + decoder := yaml.NewDecoder(reader) + decoder.KnownFields(true) + + var cfg document + if err := decoder.Decode(&cfg); err != nil { + return document{}, err + } + + var extra any + if err := decoder.Decode(&extra); err != io.EOF { + if err != nil { + return document{}, err + } + + return document{}, fmt.Errorf("configuration must contain exactly one YAML document") + } + + return cfg, nil +} + +func newQueueConfigStore(names []string, layers ...document) (queueconfig.Store, error) { + var deployment entity.StageConfig + overrides := make(map[string][]queueconfig.Overrides) + + for _, layer := range layers { + common, err := layer.QueueConfig.Stages.overrides() + if err != nil { + return nil, fmt.Errorf("deployment configuration: %w", err) + } + + deployment = common.Apply(deployment) + + for name, settings := range layer.QueueConfig.Queues { + override, err := settings.Stages.overrides() + if err != nil { + return nil, fmt.Errorf("queue %q configuration: %w", name, err) + } + + overrides[name] = append(overrides[name], override) + } + } + + return static.NewStore(names, deployment, overrides) +} diff --git a/service/stovepipe/server/config/config_test.go b/service/stovepipe/server/config/config_test.go new file mode 100644 index 000000000..a8e68ee98 --- /dev/null +++ b/service/stovepipe/server/config/config_test.go @@ -0,0 +1,214 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/uber/submitqueue/stovepipe/extension/queueconfig" +) + +func loadTestLayer(input string, names []string) (queueconfig.Store, error) { + base, err := decodeDocument(bytes.NewReader(baseYAML)) + if err != nil { + return nil, err + } + + layer, err := decodeDocument(strings.NewReader(input)) + if err != nil { + return nil, err + } + + return newQueueConfigStore(names, base, layer) +} + +func TestStageAndQueueOverrides(t *testing.T) { + store, err := loadTestLayer(` +queueconfig: + stages: + process: + maxConcurrent: 3 + gateWaitDelay: 123ms + buildSignal: + acceptedPollDelay: 234ms + runningPollDelay: 345ms + buildAnalysis: + pollDelay: 456ms + issueTrack: + refreshDelay: 567ms + recoveryWaitDelay: 678ms + queues: + special: + stages: + process: + maxConcurrent: 4 + buildAnalysis: + pollDelay: 2s +`, []string{"special", "ordinary"}) + require.NoError(t, err) + + ordinary, err := store.Get(context.Background(), "ordinary") + require.NoError(t, err) + + special, err := store.Get(context.Background(), "special") + require.NoError(t, err) + + assert.Equal(t, int32(3), ordinary.Stages.Process.MaxConcurrent) + assert.Equal(t, int64(123), ordinary.Stages.Process.GateWaitDelayMs) + assert.Equal(t, int64(234), ordinary.Stages.BuildSignal.AcceptedPollDelayMs) + assert.Equal(t, int64(345), ordinary.Stages.BuildSignal.RunningPollDelayMs) + assert.Equal(t, int64(456), ordinary.Stages.BuildAnalysis.PollDelayMs) + assert.Equal(t, int64(567), ordinary.Stages.IssueTrack.RefreshDelayMs) + assert.Equal(t, int64(678), ordinary.Stages.IssueTrack.RecoveryWaitDelayMs) + + assert.Equal(t, int32(4), special.Stages.Process.MaxConcurrent) + assert.Equal(t, ordinary.Stages.Process.GateWaitDelayMs, special.Stages.Process.GateWaitDelayMs) + assert.Equal(t, int64(2000), special.Stages.BuildAnalysis.PollDelayMs) + assert.Equal(t, ordinary.Stages.BuildSignal, special.Stages.BuildSignal) + assert.Equal(t, ordinary.Stages.IssueTrack, special.Stages.IssueTrack) +} + +func TestLayeredQueueSettingsAreMergedBeforeValidation(t *testing.T) { + base, err := decodeDocument(bytes.NewReader(baseYAML)) + require.NoError(t, err) + + first, err := decodeDocument(strings.NewReader(` +queueconfig: + stages: + buildSignal: + runningPollDelay: 0s + queues: + special: + stages: + process: + maxConcurrent: 3 + buildAnalysis: + pollDelay: 0s +`)) + require.NoError(t, err) + + second, err := decodeDocument(strings.NewReader(` +queueconfig: + stages: + buildSignal: + runningPollDelay: 555ms + queues: + special: + stages: + process: + gateWaitDelay: 321ms + buildAnalysis: + pollDelay: 100ms +`)) + require.NoError(t, err) + + store, err := newQueueConfigStore([]string{"special"}, base, first, second) + require.NoError(t, err) + + cfg, err := store.Get(context.Background(), "special") + require.NoError(t, err) + + assert.Equal(t, int32(3), cfg.Stages.Process.MaxConcurrent) + assert.Equal(t, int64(321), cfg.Stages.Process.GateWaitDelayMs) + assert.Equal(t, int64(100), cfg.Stages.BuildAnalysis.PollDelayMs) + assert.Equal(t, int64(555), cfg.Stages.BuildSignal.RunningPollDelayMs) +} + +func TestRejectsInvalidConfiguration(t *testing.T) { + for _, tt := range []struct { + name string + input string + }{ + {"empty", ""}, + {"unknown section", "queueconfg: {}"}, + {"unknown stage", "queueconfig: {stages: {unknown: {}}}"}, + {"unknown process field", "queueconfig: {stages: {process: {maxConcurent: 2}}}"}, + {"unknown signal field", "queueconfig: {stages: {buildSignal: {pollDelay: 1s}}}"}, + {"unknown analysis field", "queueconfig: {stages: {buildAnalysis: {polDelay: 1s}}}"}, + {"unknown tracking field", "queueconfig: {stages: {issueTrack: {pollDelay: 1s}}}"}, + {"unknown queue", "queueconfig: {queues: {unknown: {}}}"}, + {"duplicate key", "queueconfig: {stages: {process: {maxConcurrent: 1, maxConcurrent: 2}}}"}, + {"zero concurrency", "queueconfig: {stages: {process: {maxConcurrent: 0}}}"}, + {"zero wait", "queueconfig: {stages: {buildAnalysis: {pollDelay: 0s}}}"}, + {"negative wait", "queueconfig: {stages: {issueTrack: {refreshDelay: -1s}}}"}, + {"invalid duration", "queueconfig: {stages: {process: {gateWaitDelay: soon}}}"}, + {"submillisecond", "queueconfig: {stages: {buildSignal: {acceptedPollDelay: 1us}}}"}, + {"fractional millisecond", "queueconfig: {stages: {issueTrack: {recoveryWaitDelay: 1500us}}}"}, + {"invalid queue override", "queueconfig: {queues: {known: {stages: {process: {gateWaitDelay: 0s}}}}}"}, + {"second document", "{}\n---\n{}"}, + {"malformed second document", "{}\n---\n["}, + } { + t.Run(tt.name, func(t *testing.T) { + _, err := loadTestLayer(tt.input, []string{"known"}) + require.Error(t, err) + }) + } +} + +func TestLoad(t *testing.T) { + ctx := context.Background() + + t.Run("embedded baseline and explicit queue registry", func(t *testing.T) { + store, err := Load("", []string{"known"}) + require.NoError(t, err) + + cfg, err := store.Get(ctx, "known") + require.NoError(t, err) + require.NoError(t, cfg.Validate()) + + _, err = store.Get(ctx, "unknown") + require.ErrorIs(t, err, queueconfig.ErrNotFound) + }) + + t.Run("optional file overlays rather than replaces baseline", func(t *testing.T) { + baseline, err := Load("", []string{"known"}) + require.NoError(t, err) + + want, err := baseline.Get(ctx, "known") + require.NoError(t, err) + want.Stages.Process.GateWaitDelayMs = 321 + + path := filepath.Join(t.TempDir(), "stovepipe.yaml") + input := "queueconfig: {stages: {process: {gateWaitDelay: 321ms}}}" + require.NoError(t, os.WriteFile(path, []byte(input), 0600)) + + store, err := Load(path, []string{"known"}) + require.NoError(t, err) + + got, err := store.Get(ctx, "known") + require.NoError(t, err) + assert.Equal(t, want, got) + }) + + t.Run("missing explicit file fails", func(t *testing.T) { + _, err := Load(filepath.Join(t.TempDir(), "missing"), []string{"known"}) + require.ErrorIs(t, err, os.ErrNotExist) + }) + + t.Run("incomplete baseline is not silently defaulted", func(t *testing.T) { + partial, err := decodeDocument(strings.NewReader("queueconfig: {stages: {process: {maxConcurrent: 1}}}")) + require.NoError(t, err) + + _, err = newQueueConfigStore(nil, partial) + require.Error(t, err) + }) +} diff --git a/service/stovepipe/server/config/duration.go b/service/stovepipe/server/config/duration.go new file mode 100644 index 000000000..ca9bd6470 --- /dev/null +++ b/service/stovepipe/server/config/duration.go @@ -0,0 +1,33 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + "time" +) + +func durationMilliseconds(value *time.Duration) (*int64, error) { + if value == nil { + return nil, nil + } + + if *value%time.Millisecond != 0 { + return nil, fmt.Errorf("duration must use whole milliseconds") + } + + result := value.Milliseconds() + return &result, nil +} diff --git a/service/stovepipe/server/config/issue_track.go b/service/stovepipe/server/config/issue_track.go new file mode 100644 index 000000000..d4c1f5f4f --- /dev/null +++ b/service/stovepipe/server/config/issue_track.go @@ -0,0 +1,45 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + "time" + + "github.com/uber/submitqueue/stovepipe/extension/queueconfig" +) + +type issueTrackSettings struct { + RefreshDelay *time.Duration `yaml:"refreshDelay"` + + RecoveryWaitDelay *time.Duration `yaml:"recoveryWaitDelay"` +} + +func (s issueTrackSettings) overrides() (queueconfig.IssueTrackOverrides, error) { + refreshDelay, err := durationMilliseconds(s.RefreshDelay) + if err != nil { + return queueconfig.IssueTrackOverrides{}, fmt.Errorf("refreshDelay: %w", err) + } + + recoveryWaitDelay, err := durationMilliseconds(s.RecoveryWaitDelay) + if err != nil { + return queueconfig.IssueTrackOverrides{}, fmt.Errorf("recoveryWaitDelay: %w", err) + } + + return queueconfig.IssueTrackOverrides{ + RefreshDelayMs: refreshDelay, + RecoveryWaitDelayMs: recoveryWaitDelay, + }, nil +} diff --git a/service/stovepipe/server/config/process.go b/service/stovepipe/server/config/process.go new file mode 100644 index 000000000..c61cce0ce --- /dev/null +++ b/service/stovepipe/server/config/process.go @@ -0,0 +1,40 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + "time" + + "github.com/uber/submitqueue/stovepipe/extension/queueconfig" +) + +type processSettings struct { + MaxConcurrent *int32 `yaml:"maxConcurrent"` + + GateWaitDelay *time.Duration `yaml:"gateWaitDelay"` +} + +func (s processSettings) overrides() (queueconfig.ProcessOverrides, error) { + gateWaitDelay, err := durationMilliseconds(s.GateWaitDelay) + if err != nil { + return queueconfig.ProcessOverrides{}, fmt.Errorf("gateWaitDelay: %w", err) + } + + return queueconfig.ProcessOverrides{ + MaxConcurrent: s.MaxConcurrent, + GateWaitDelayMs: gateWaitDelay, + }, nil +} diff --git a/service/stovepipe/server/config/stages.go b/service/stovepipe/server/config/stages.go new file mode 100644 index 000000000..01c8668a6 --- /dev/null +++ b/service/stovepipe/server/config/stages.go @@ -0,0 +1,60 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + + "github.com/uber/submitqueue/stovepipe/extension/queueconfig" +) + +type stageSettings struct { + Process processSettings `yaml:"process"` + + BuildSignal buildSignalSettings `yaml:"buildSignal"` + + BuildAnalysis buildAnalysisSettings `yaml:"buildAnalysis"` + + IssueTrack issueTrackSettings `yaml:"issueTrack"` +} + +func (s stageSettings) overrides() (queueconfig.Overrides, error) { + process, err := s.Process.overrides() + if err != nil { + return queueconfig.Overrides{}, fmt.Errorf("process: %w", err) + } + + buildSignal, err := s.BuildSignal.overrides() + if err != nil { + return queueconfig.Overrides{}, fmt.Errorf("buildSignal: %w", err) + } + + buildAnalysis, err := s.BuildAnalysis.overrides() + if err != nil { + return queueconfig.Overrides{}, fmt.Errorf("buildAnalysis: %w", err) + } + + issueTrack, err := s.IssueTrack.overrides() + if err != nil { + return queueconfig.Overrides{}, fmt.Errorf("issueTrack: %w", err) + } + + return queueconfig.Overrides{ + Process: process, + BuildSignal: buildSignal, + BuildAnalysis: buildAnalysis, + IssueTrack: issueTrack, + }, nil +} diff --git a/service/stovepipe/server/main.go b/service/stovepipe/server/main.go index 0a9a0da0c..c879f5fed 100644 --- a/service/stovepipe/server/main.go +++ b/service/stovepipe/server/main.go @@ -18,6 +18,7 @@ import ( "context" "database/sql" "errors" + "flag" "fmt" "net" "os" @@ -45,6 +46,7 @@ import ( queueMySQL "github.com/uber/submitqueue/platform/extension/messagequeue/mysql" platformhook "github.com/uber/submitqueue/platform/hook" servicemq "github.com/uber/submitqueue/service/messagequeue" + serverconfig "github.com/uber/submitqueue/service/stovepipe/server/config" "github.com/uber/submitqueue/service/stovepipe/server/handler" "github.com/uber/submitqueue/stovepipe/controller" "github.com/uber/submitqueue/stovepipe/controller/build" @@ -66,7 +68,7 @@ import ( fixernoop "github.com/uber/submitqueue/stovepipe/extension/fixer/noop" projectissuesnoop "github.com/uber/submitqueue/stovepipe/extension/projectissues/noop" projectstatusnoop "github.com/uber/submitqueue/stovepipe/extension/projectstatus/noop" - queueconfigdefault "github.com/uber/submitqueue/stovepipe/extension/queueconfig/default" + "github.com/uber/submitqueue/stovepipe/extension/queueconfig" "github.com/uber/submitqueue/stovepipe/extension/sourcecontrol" sourcecontrolfake "github.com/uber/submitqueue/stovepipe/extension/sourcecontrol/fake" "github.com/uber/submitqueue/stovepipe/extension/storage" @@ -114,8 +116,10 @@ func (hookResolver) For(*basehook.HookEvent) []hookext.Hook { } func main() { + configPath := flag.String("config", "", "path to the Stovepipe YAML configuration") + flag.Parse() code := 0 - if err := run(); err != nil { + if err := run(*configPath); err != nil { if errors.Is(err, context.Canceled) { fmt.Println("Stovepipe server stopped by signal") @@ -131,7 +135,7 @@ func main() { os.Exit(code) } -func run() error { +func run(configPath string) error { // Set up signal handling early so retry loops can respond to SIGTERM ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer cancel() @@ -143,6 +147,23 @@ func run() error { } defer logger.Sync() + tenants, err := servicemq.ParseRequiredTenants(os.Getenv("MQ_TENANTS")) + if err != nil { + return fmt.Errorf("failed to configure queue subscribers: %w", err) + } + + configs, err := serverconfig.Load(configPath, tenants) + if err != nil { + return fmt.Errorf("load queue configurations: %w", err) + } + + resolved, err := configs.List(ctx) + if err != nil { + return fmt.Errorf("list queue configurations: %w", err) + } + + logger.Info("queue configurations loaded", zap.Any("queues", resolved)) + // Initialize metrics scope scope := tally.NewTestScope("stovepipe", nil) metricsStopCh := make(chan interface{}, 1) @@ -205,11 +226,6 @@ func run() error { return fmt.Errorf("failed to configure queue database pool: %w", err) } - tenants, err := servicemq.ParseRequiredTenants(os.Getenv("MQ_TENANTS")) - if err != nil { - return fmt.Errorf("failed to configure queue subscribers: %w", err) - } - mysqlQueue, err := queueMySQL.NewQueue(queueMySQL.Params{ DB: queueDB, Logger: logger, @@ -263,7 +279,20 @@ func run() error { storageFty := storageFactory{backend: store} materializer := requestlog.NewMaterializer(scope) - primaryCount, err := registerPrimaryControllers(primaryConsumer, logger.Sugar(), scope, storageFty, materializer, registry, sourceControl, brf, hookResolver{}, detectorFactory, repairExecutors) + primaryCount, err := registerPrimaryControllers( + primaryConsumer, + logger.Sugar(), + scope, + storageFty, + materializer, + configs, + registry, + sourceControl, + brf, + hookResolver{}, + detectorFactory, + repairExecutors, + ) if err != nil { return err } @@ -385,6 +414,7 @@ func registerPrimaryControllers( scope tally.Scope, store storage.Factory, materializer requestlog.Materializer, + configs queueconfig.Store, registry consumer.TopicRegistry, sourceControl sourcecontrol.Factory, brf buildrunner.Factory, @@ -399,7 +429,7 @@ func registerPrimaryControllers( scope, store, materializer, - queueconfigdefault.NewStore(), + configs, sourceControl, registry, stovepipemq.TopicKeyProcess, @@ -410,43 +440,106 @@ func registerPrimaryControllers( } count++ - buildController := build.NewController(logger, scope, store, materializer, brf, registry, stovepipemq.TopicKeyBuild, "stovepipe-build") + buildController := build.NewController( + logger, + scope, + store, + materializer, + brf, + registry, + stovepipemq.TopicKeyBuild, + "stovepipe-build", + ) if err := c.Register(buildController); err != nil { return count, fmt.Errorf("failed to register build controller: %w", err) } count++ - buildSignalController := buildsignal.NewController(logger, scope, store, materializer, brf, registry, stovepipemq.TopicKeyBuildSignal, "stovepipe-buildsignal") + buildSignalController := buildsignal.NewController( + logger, + scope, + store, + materializer, + brf, + registry, + stovepipemq.TopicKeyBuildSignal, + "stovepipe-buildsignal", + ) if err := c.Register(buildSignalController); err != nil { return count, fmt.Errorf("failed to register buildsignal controller: %w", err) } count++ - analysisController := buildanalysis.NewController(logger, scope, store, materializer, detectorFactory, registry, stovepipemq.TopicKeyBuildAnalysis, "stovepipe-buildanalysis") + analysisController := buildanalysis.NewController( + logger, + scope, + store, + materializer, + detectorFactory, + registry, + stovepipemq.TopicKeyBuildAnalysis, + "stovepipe-buildanalysis", + ) if err := c.Register(analysisController); err != nil { return count, fmt.Errorf("failed to register buildanalysis controller: %w", err) } count++ - issueTrackController := issuetrack.NewController(logger, scope, store, materializer, registry, stovepipemq.TopicKeyIssueTrack, "stovepipe-issuetrack") + issueTrackController := issuetrack.NewController( + logger, + scope, + store, + materializer, + registry, + stovepipemq.TopicKeyIssueTrack, + "stovepipe-issuetrack", + ) if err := c.Register(issueTrackController); err != nil { return count, fmt.Errorf("failed to register issuetrack controller: %w", err) } count++ - fixController := fix.NewController(logger, scope, store, materializer, repairExecutors, registry, stovepipemq.TopicKeyFix, "stovepipe-fix") + fixController := fix.NewController( + logger, + scope, + store, + materializer, + repairExecutors, + registry, + stovepipemq.TopicKeyFix, + "stovepipe-fix", + ) if err := c.Register(fixController); err != nil { return count, fmt.Errorf("failed to register fix controller: %w", err) } count++ - projectController := projectanalysis.NewController(logger, scope, store, materializer, projectstatusnoop.New(), projectissuesnoop.New(), registry, stovepipemq.TopicKeyProjectAnalysis, "stovepipe-projectanalysis") + projectController := projectanalysis.NewController( + logger, + scope, + store, + materializer, + projectstatusnoop.New(), + projectissuesnoop.New(), + registry, + stovepipemq.TopicKeyProjectAnalysis, + "stovepipe-projectanalysis", + ) if err := c.Register(projectController); err != nil { return count, fmt.Errorf("failed to register projectanalysis controller: %w", err) } count++ - recordController := record.NewController(logger, scope, store, materializer, sourceControl, registry, stovepipemq.TopicKeyRecord, "stovepipe-record") + recordController := record.NewController( + logger, + scope, + store, + materializer, + sourceControl, + registry, + stovepipemq.TopicKeyRecord, + "stovepipe-record", + ) if err := c.Register(recordController); err != nil { return count, fmt.Errorf("failed to register record controller: %w", err) } diff --git a/service/stovepipe/server/main_test.go b/service/stovepipe/server/main_test.go index b5d1733e4..5c88ac2ca 100644 --- a/service/stovepipe/server/main_test.go +++ b/service/stovepipe/server/main_test.go @@ -28,11 +28,13 @@ import ( consumermock "github.com/uber/submitqueue/platform/consumer/mock" extqueue "github.com/uber/submitqueue/platform/extension/messagequeue" queuemock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" + serverconfig "github.com/uber/submitqueue/service/stovepipe/server/config" "github.com/uber/submitqueue/stovepipe/controller/dlq" stovepipemq "github.com/uber/submitqueue/stovepipe/core/messagequeue" "github.com/uber/submitqueue/stovepipe/core/requestlog" "github.com/uber/submitqueue/stovepipe/entity" buildissuenoop "github.com/uber/submitqueue/stovepipe/extension/buildissue/noop" + "github.com/uber/submitqueue/stovepipe/extension/queueconfig" "github.com/uber/submitqueue/stovepipe/extension/storage" storagemock "github.com/uber/submitqueue/stovepipe/extension/storage/mock" "go.uber.org/mock/gomock" @@ -53,11 +55,25 @@ func (c *recordingConsumer) Start(context.Context) error { return nil } func (c *recordingConsumer) Stop(int64) error { return nil } +func testQueueConfigs(t *testing.T) queueconfig.Store { + t.Helper() + + configs, err := serverconfig.Load("", nil) + require.NoError(t, err) + + return configs +} + // registeredControllers runs the host's registration exactly as run() does and // returns the registry it registers against, the primary controllers, and the // DLQ controllers. func registeredControllers(t *testing.T) (consumer.TopicRegistry, []consumer.Controller, []consumer.Controller) { t.Helper() + return registeredControllersWithConfig(t, testQueueConfigs(t)) +} + +func registeredControllersWithConfig(t *testing.T, configs queueconfig.Store) (consumer.TopicRegistry, []consumer.Controller, []consumer.Controller) { + t.Helper() registry, err := newTopicRegistry(nil, "subscriber") require.NoError(t, err) @@ -67,8 +83,20 @@ func registeredControllers(t *testing.T) (consumer.TopicRegistry, []consumer.Con primary := &recordingConsumer{} deadLetter := &recordingConsumer{} - _, err = registerPrimaryControllers(primary, logger, tally.NoopScope, store, requestlog.NewMaterializer(tally.NoopScope), registry, - fakeSourceControlFactory{}, fakeBuildRunnerFactory{}, hookResolver{}, buildissuenoop.New(), noopRepairExecutorFactory{}) + _, err = registerPrimaryControllers( + primary, + logger, + tally.NoopScope, + store, + requestlog.NewMaterializer(tally.NoopScope), + configs, + registry, + fakeSourceControlFactory{}, + fakeBuildRunnerFactory{}, + hookResolver{}, + buildissuenoop.New(), + noopRepairExecutorFactory{}, + ) require.NoError(t, err) _, err = registerDLQControllers(deadLetter, logger, tally.NoopScope, store, requestlog.NewMaterializer(tally.NoopScope), registry, noopRepairExecutorFactory{}) @@ -214,9 +242,20 @@ func TestRegisteredFixControllerRetainsPerExecutorDisposition(t *testing.T) { registry, err := newTopicRegistry(queue, "subscriber") require.NoError(t, err) primary := &recordingConsumer{} - _, err = registerPrimaryControllers(primary, zaptest.NewLogger(t).Sugar(), tally.NoopScope, stores, - requestlog.NewMaterializer(tally.NoopScope), registry, fakeSourceControlFactory{}, fakeBuildRunnerFactory{}, - hookResolver{}, buildissuenoop.New(), noopRepairExecutorFactory{}) + _, err = registerPrimaryControllers( + primary, + zaptest.NewLogger(t).Sugar(), + tally.NoopScope, + stores, + requestlog.NewMaterializer(tally.NoopScope), + testQueueConfigs(t), + registry, + fakeSourceControlFactory{}, + fakeBuildRunnerFactory{}, + hookResolver{}, + buildissuenoop.New(), + noopRepairExecutorFactory{}, + ) require.NoError(t, err) var fixController consumer.Controller for _, controller := range primary.controllers { @@ -278,13 +317,43 @@ func TestPipelineSubscriptionRetryPolicies(t *testing.T) { consumerGroup string wantRetry extqueue.RetryConfig }{ - {name: "process", topicKey: stovepipemq.TopicKeyProcess, consumerGroup: "stovepipe-process", wantRetry: defaultRetry}, - {name: "build", topicKey: stovepipemq.TopicKeyBuild, consumerGroup: "stovepipe-build", wantRetry: defaultRetry}, - {name: "buildsignal", topicKey: stovepipemq.TopicKeyBuildSignal, consumerGroup: "stovepipe-buildsignal", wantRetry: extendedRetry}, - {name: "buildanalysis", topicKey: stovepipemq.TopicKeyBuildAnalysis, consumerGroup: "stovepipe-buildanalysis", wantRetry: extendedRetry}, - {name: "issuetrack", topicKey: stovepipemq.TopicKeyIssueTrack, consumerGroup: "stovepipe-issuetrack", wantRetry: extendedRetry}, + { + name: "process", + topicKey: stovepipemq.TopicKeyProcess, + consumerGroup: "stovepipe-process", + wantRetry: defaultRetry, + }, + { + name: "build", + topicKey: stovepipemq.TopicKeyBuild, + consumerGroup: "stovepipe-build", + wantRetry: defaultRetry, + }, + { + name: "buildsignal", + topicKey: stovepipemq.TopicKeyBuildSignal, + consumerGroup: "stovepipe-buildsignal", + wantRetry: extendedRetry, + }, + { + name: "buildanalysis", + topicKey: stovepipemq.TopicKeyBuildAnalysis, + consumerGroup: "stovepipe-buildanalysis", + wantRetry: extendedRetry, + }, + { + name: "issuetrack", + topicKey: stovepipemq.TopicKeyIssueTrack, + consumerGroup: "stovepipe-issuetrack", + wantRetry: extendedRetry, + }, {name: "fix", topicKey: stovepipemq.TopicKeyFix, consumerGroup: "stovepipe-fix", wantRetry: extendedRetry}, - {name: "record", topicKey: stovepipemq.TopicKeyRecord, consumerGroup: "stovepipe-record", wantRetry: extendedRetry}, + { + name: "record", + topicKey: stovepipemq.TopicKeyRecord, + consumerGroup: "stovepipe-record", + wantRetry: extendedRetry, + }, {name: "hook", topicKey: basehook.TopicKeyHook, consumerGroup: "stovepipe-hook", wantRetry: defaultRetry}, } diff --git a/stovepipe/controller/process/BUILD.bazel b/stovepipe/controller/process/BUILD.bazel index 27fbe8d7d..321fce15c 100644 --- a/stovepipe/controller/process/BUILD.bazel +++ b/stovepipe/controller/process/BUILD.bazel @@ -43,7 +43,7 @@ go_test( "//stovepipe/core/requestlog:go_default_library", "//stovepipe/core/requestlog/mock:go_default_library", "//stovepipe/entity:go_default_library", - "//stovepipe/extension/queueconfig/default:go_default_library", + "//stovepipe/extension/queueconfig/mock:go_default_library", "//stovepipe/extension/sourcecontrol:go_default_library", "//stovepipe/extension/sourcecontrol/mock:go_default_library", "//stovepipe/extension/storage:go_default_library", diff --git a/stovepipe/controller/process/process.go b/stovepipe/controller/process/process.go index 10edcf83c..b39436209 100644 --- a/stovepipe/controller/process/process.go +++ b/stovepipe/controller/process/process.go @@ -229,8 +229,8 @@ func (c *Controller) coalesce(ctx context.Context, store storage.Storage, reques // admitLatestHead holds the delivery while the build-slot gate is closed; otherwise it marks the // request processing, counts its build slot, and publishes it to build. func (c *Controller) admitLatestHead(ctx context.Context, store storage.Storage, delivery consumer.Delivery, request entity.Request, queueRow entity.Queue, cfg entity.QueueConfig) error { - if queueRow.InFlightCount >= cfg.MaxConcurrent { - return c.holdForBuildSlot(ctx, delivery, request, queueRow.InFlightCount, cfg.GateWaitDelayMs) + if queueRow.InFlightCount >= cfg.Stages.Process.MaxConcurrent { + return c.holdForBuildSlot(ctx, delivery, request, queueRow.InFlightCount, cfg.Stages.Process.GateWaitDelayMs) } strategy, baseURI, err := c.deriveBuildStrategy(ctx, queueRow, request) diff --git a/stovepipe/controller/process/process_test.go b/stovepipe/controller/process/process_test.go index ef82e7b14..3fceac6e2 100644 --- a/stovepipe/controller/process/process_test.go +++ b/stovepipe/controller/process/process_test.go @@ -35,7 +35,7 @@ import ( "github.com/uber/submitqueue/stovepipe/core/requestlog" requestlogmock "github.com/uber/submitqueue/stovepipe/core/requestlog/mock" "github.com/uber/submitqueue/stovepipe/entity" - queueconfigdefault "github.com/uber/submitqueue/stovepipe/extension/queueconfig/default" + queueconfigmock "github.com/uber/submitqueue/stovepipe/extension/queueconfig/mock" "github.com/uber/submitqueue/stovepipe/extension/sourcecontrol" sourcecontrolmock "github.com/uber/submitqueue/stovepipe/extension/sourcecontrol/mock" "github.com/uber/submitqueue/stovepipe/extension/storage" @@ -104,12 +104,17 @@ func newControllerWithScope(t *testing.T, ctrl *gomock.Controller, scope tally.S }) require.NoError(t, err) + configs := queueconfigmock.NewMockStore(ctrl) + cfg := entity.QueueConfig{Name: testQueue} + cfg.Stages.Process = entity.ProcessConfig{MaxConcurrent: 1, GateWaitDelayMs: 5000} + configs.EXPECT().Get(gomock.Any(), gomock.Any()).Return(cfg, nil).AnyTimes() + c := NewController( zap.NewNop().Sugar(), scope, staticStorageFactory{store: m.store}, m.materializer, - queueconfigdefault.NewStore(), + configs, m.sourceFactory, registry, stovepipemq.TopicKeyProcess, diff --git a/stovepipe/entity/queue_config.go b/stovepipe/entity/queue_config.go index b6681cd83..05464f91e 100644 --- a/stovepipe/entity/queue_config.go +++ b/stovepipe/entity/queue_config.go @@ -1,4 +1,4 @@ -// Copyright (c) 2025 Uber Technologies, Inc. +// Copyright (c) 2026 Uber Technologies, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,16 +14,26 @@ package entity -// QueueConfig holds deployment configuration for a Stovepipe validation queue. -// Mutable runtime state (latest head, in-flight count) lives on the Queue row; -// knobs such as max_concurrent are resolved here at gate-check time. Immutable -// after load. +import "fmt" + +// QueueConfig is an immutable, resolved configuration for a validation queue. type QueueConfig struct { // Name uniquely identifies this queue within the system. - // Referenced by Request.Queue. Name string `json:"name" yaml:"name"` - // MaxConcurrent is the cap on concurrent in-flight validations for the queue. - MaxConcurrent int32 `json:"max_concurrent" yaml:"max_concurrent"` - // GateWaitDelayMs is the redelivery delay while the latest head waits for a slot. - GateWaitDelayMs int64 `json:"gate_wait_delay_ms" yaml:"gate_wait_delay_ms"` + + // Stages contains configuration grouped by pipeline stage. + Stages StageConfig `json:"stages" yaml:"stages"` +} + +// Validate checks queue identity and stage configuration constraints. +func (c QueueConfig) Validate() error { + if c.Name == "" { + return fmt.Errorf("queue name must not be empty") + } + + if err := c.Stages.Validate(); err != nil { + return fmt.Errorf("queue %q: %w", c.Name, err) + } + + return nil } diff --git a/stovepipe/entity/stage_config_test.go b/stovepipe/entity/stage_config_test.go index 0125e27c1..826c4c425 100644 --- a/stovepipe/entity/stage_config_test.go +++ b/stovepipe/entity/stage_config_test.go @@ -50,3 +50,15 @@ func TestStageConfigValidatesEveryStage(t *testing.T) { }) } } + +func TestQueueConfigValidate(t *testing.T) { + cfg := QueueConfig{Name: "known", Stages: testStageConfig()} + require.NoError(t, cfg.Validate()) + + cfg.Name = "" + require.Error(t, cfg.Validate()) + + cfg.Name = "known" + cfg.Stages.BuildAnalysis = BuildAnalysisConfig{} + require.Error(t, cfg.Validate()) +} diff --git a/stovepipe/extension/queueconfig/README.md b/stovepipe/extension/queueconfig/README.md index 40f88558d..b9a78c68f 100644 --- a/stovepipe/extension/queueconfig/README.md +++ b/stovepipe/extension/queueconfig/README.md @@ -1,17 +1,21 @@ -# Queue Config Extension +# Queue configuration -Vendor-agnostic interface for providing Stovepipe queue configurations. +Queue configuration is deployment policy, separate from mutable queue state in storage. The store interface supplies immutable, fully resolved settings by queue name; it does not imply database persistence. -Pipeline stages read mutable runtime state from storage and read knobs such as `max_concurrent` from a config `Store` at call time, matching the SubmitQueue split documented in [submitqueue/extension/queueconfig/README.md](../../../submitqueue/extension/queueconfig/README.md). +## Stage ownership -## Interfaces +Configuration is grouped by stage: process, build signal, build analysis, and issue tracking. Each stage owns its resolved entity, its validation, and its optional overrides in stage-specific files. A stage's override application copies the supplied baseline and replaces only explicitly supplied fields. The aggregate composes stages; it does not inspect their individual settings. -`Store` provides queue configurations by name via `Get` and `List`. See `queueconfig.go`. +The process stage owns concurrency and admission waits. The other current stage settings control build-status polling, build-issue analysis polling, and issue-tracking refresh and recovery waits. Stage configuration can contain both timing and non-timing settings. Immediate batch continuation and consumer error-retry policies remain separate. -## Entities +## Resolution -Queue configuration entity lives in `stovepipe/entity/queue_config.go` and carries deployment knobs (`max_concurrent`, `gate_wait_delay_ms`) separate from the mutable `Queue` row. +Hosts supply a complete deployment baseline and ordered per-queue override layers. The static store requires a valid deployment baseline, applies every layer for each named queue, then validates the final queue configuration. Partial override layers are not validated in isolation. Nil fields inherit; explicit values follow the owning stage's constraints. The current concurrency limits and millisecond intervals must be positive. -## Implementations +The static store rejects duplicate names and overrides for unknown queues, and returns independent configuration values. There is no permissive fallback for unregistered queues. -`default` returns the global wiring defaults for any non-empty queue name until a file- or service-backed store lands. +## Host integration + +The shared library does not parse configuration files or supply hardcoded operational defaults. Hosts own loading and dependency injection and can use different configuration frameworks. The reference host embeds a service baseline and optionally layers another YAML file over it. Queue-specific layers take precedence over deployment-wide settings. Changes to startup configuration require a restart. + +Adding a setting to an existing stage requires updating that stage's entity, override application, validation, and host adapter. Required values must be supplied by the host baseline. It does not require adding field-level logic to the aggregate or store. diff --git a/stovepipe/extension/queueconfig/default/default.go b/stovepipe/extension/queueconfig/default/default.go deleted file mode 100644 index 5fb696710..000000000 --- a/stovepipe/extension/queueconfig/default/default.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package defaultconfig provides a queueconfig.Store that returns global wiring -// defaults for any non-empty queue name. Replace with a YAML- or service-backed -// store when per-queue overrides are needed. -package defaultconfig - -import ( - "context" - - "github.com/uber/submitqueue/stovepipe/entity" - "github.com/uber/submitqueue/stovepipe/extension/queueconfig" -) - -const ( - _defaultMaxConcurrent = 1 - _defaultGateWaitDelayMs = 5000 -) - -// Store is a queueconfig.Store that returns the same defaults for every queue. -type Store struct{} - -// NewStore returns a Store backed by global wiring defaults. -func NewStore() Store { - return Store{} -} - -// Get returns the default configuration for any non-empty queue name. -func (Store) Get(_ context.Context, name string) (entity.QueueConfig, error) { - if name == "" { - return entity.QueueConfig{}, queueconfig.ErrNotFound - } - return entity.QueueConfig{ - Name: name, - MaxConcurrent: _defaultMaxConcurrent, - GateWaitDelayMs: _defaultGateWaitDelayMs, - }, nil -} - -// List returns an empty slice; this store does not enumerate known queues. -func (Store) List(context.Context) ([]entity.QueueConfig, error) { - return nil, nil -} diff --git a/stovepipe/extension/queueconfig/default/default_test.go b/stovepipe/extension/queueconfig/default/default_test.go deleted file mode 100644 index 490ff35a7..000000000 --- a/stovepipe/extension/queueconfig/default/default_test.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package defaultconfig - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/uber/submitqueue/stovepipe/extension/queueconfig" -) - -func TestStore_Get(t *testing.T) { - store := NewStore() - - t.Run("returns defaults for named queue", func(t *testing.T) { - cfg, err := store.Get(context.Background(), "monorepo/main") - require.NoError(t, err) - assert.Equal(t, "monorepo/main", cfg.Name) - assert.Equal(t, int32(1), cfg.MaxConcurrent) - assert.Equal(t, int64(5000), cfg.GateWaitDelayMs) - }) - - t.Run("empty name is not found", func(t *testing.T) { - _, err := store.Get(context.Background(), "") - require.ErrorIs(t, err, queueconfig.ErrNotFound) - }) -} - -func TestStore_List(t *testing.T) { - store := NewStore() - - cfg, err := store.List(context.Background()) - require.NoError(t, err) - assert.Empty(t, cfg) -} diff --git a/stovepipe/extension/queueconfig/queueconfig.go b/stovepipe/extension/queueconfig/queueconfig.go index 437d9a2ba..b03b2d883 100644 --- a/stovepipe/extension/queueconfig/queueconfig.go +++ b/stovepipe/extension/queueconfig/queueconfig.go @@ -29,7 +29,7 @@ var ErrNotFound = errors.New("queue config not found") // Store loads and provides queue configurations. // Implementations may read from YAML files, databases, remote services, etc. type Store interface { - // Get returns the configuration for a named queue. + // Get returns a fully resolved configuration for a named queue. // Returns ErrNotFound if no configuration exists for the given name. Get(ctx context.Context, name string) (entity.QueueConfig, error) diff --git a/stovepipe/extension/queueconfig/default/BUILD.bazel b/stovepipe/extension/queueconfig/static/BUILD.bazel similarity index 83% rename from stovepipe/extension/queueconfig/default/BUILD.bazel rename to stovepipe/extension/queueconfig/static/BUILD.bazel index f2c8fc3dc..25bff0b0a 100644 --- a/stovepipe/extension/queueconfig/default/BUILD.bazel +++ b/stovepipe/extension/queueconfig/static/BUILD.bazel @@ -2,8 +2,8 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", - srcs = ["default.go"], - importpath = "github.com/uber/submitqueue/stovepipe/extension/queueconfig/default", + srcs = ["static.go"], + importpath = "github.com/uber/submitqueue/stovepipe/extension/queueconfig/static", visibility = ["//visibility:public"], deps = [ "//stovepipe/entity:go_default_library", @@ -13,9 +13,10 @@ go_library( go_test( name = "go_default_test", - srcs = ["default_test.go"], + srcs = ["static_test.go"], embed = [":go_default_library"], deps = [ + "//stovepipe/entity:go_default_library", "//stovepipe/extension/queueconfig:go_default_library", "@com_github_stretchr_testify//assert:go_default_library", "@com_github_stretchr_testify//require:go_default_library", diff --git a/stovepipe/extension/queueconfig/static/static.go b/stovepipe/extension/queueconfig/static/static.go new file mode 100644 index 000000000..d5b3bca3d --- /dev/null +++ b/stovepipe/extension/queueconfig/static/static.go @@ -0,0 +1,103 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package static provides immutable, resolved queue configuration. +package static + +import ( + "context" + "fmt" + "sort" + + "github.com/uber/submitqueue/stovepipe/entity" + "github.com/uber/submitqueue/stovepipe/extension/queueconfig" +) + +// Store contains configurations for an explicit set of queues. +// Get and List return independent values and are safe for concurrent use. +type Store struct { + queues map[string]entity.QueueConfig + names []string +} + +// NewStore applies ordered per-queue overrides to a complete deployment configuration. +// Unknown override names, duplicate queue names and invalid settings are rejected. +func NewStore( + names []string, + deployment entity.StageConfig, + overrides map[string][]queueconfig.Overrides, +) (Store, error) { + if err := deployment.Validate(); err != nil { + return Store{}, fmt.Errorf("deployment configuration: %w", err) + } + + store := Store{queues: make(map[string]entity.QueueConfig, len(names))} + + for _, name := range names { + if _, exists := store.queues[name]; exists { + return Store{}, fmt.Errorf("duplicate queue %q", name) + } + + stages := deployment + + for _, layer := range overrides[name] { + stages = layer.Apply(stages) + } + + cfg := entity.QueueConfig{ + Name: name, + Stages: stages, + } + + if err := cfg.Validate(); err != nil { + return Store{}, err + } + + store.queues[name] = cfg + store.names = append(store.names, name) + } + + for name := range overrides { + if _, exists := store.queues[name]; !exists { + return Store{}, fmt.Errorf("override for unknown queue %q: %w", name, queueconfig.ErrNotFound) + } + } + + sort.Strings(store.names) + + return store, nil +} + +// Get returns ErrNotFound for queues outside the configured set. +func (s Store) Get(_ context.Context, name string) (entity.QueueConfig, error) { + cfg, ok := s.queues[name] + if !ok { + return entity.QueueConfig{}, queueconfig.ErrNotFound + } + + return cfg, nil +} + +// List returns configurations ordered by queue name. +func (s Store) List(context.Context) ([]entity.QueueConfig, error) { + configs := make([]entity.QueueConfig, 0, len(s.names)) + + for _, name := range s.names { + configs = append(configs, s.queues[name]) + } + + return configs, nil +} + +var _ queueconfig.Store = Store{} diff --git a/stovepipe/extension/queueconfig/static/static_test.go b/stovepipe/extension/queueconfig/static/static_test.go new file mode 100644 index 000000000..9b48f9e92 --- /dev/null +++ b/stovepipe/extension/queueconfig/static/static_test.go @@ -0,0 +1,129 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package static + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/uber/submitqueue/stovepipe/entity" + "github.com/uber/submitqueue/stovepipe/extension/queueconfig" +) + +func testStages() entity.StageConfig { + return entity.StageConfig{ + Process: entity.ProcessConfig{MaxConcurrent: 2, GateWaitDelayMs: 11}, + BuildSignal: entity.BuildSignalConfig{AcceptedPollDelayMs: 12, RunningPollDelayMs: 13}, + BuildAnalysis: entity.BuildAnalysisConfig{PollDelayMs: 14}, + IssueTrack: entity.IssueTrackConfig{RefreshDelayMs: 15, RecoveryWaitDelayMs: 16}, + } +} + +func TestStoreInheritanceAndIsolation(t *testing.T) { + ctx := context.Background() + deployment := testStages() + poll := int64(123) + overrides := map[string][]queueconfig.Overrides{ + "special": {{BuildAnalysis: queueconfig.BuildAnalysisOverrides{PollDelayMs: &poll}}}, + } + names := []string{"special", "ordinary"} + + store, err := NewStore(names, deployment, overrides) + require.NoError(t, err) + + deployment.Process.MaxConcurrent = 99 + poll = 0 + names[0] = "mutated" + delete(overrides, "special") + + special, err := store.Get(ctx, "special") + require.NoError(t, err) + + ordinary, err := store.Get(ctx, "ordinary") + require.NoError(t, err) + + assert.Equal(t, int64(123), special.Stages.BuildAnalysis.PollDelayMs) + assert.Equal(t, testStages(), ordinary.Stages) + assert.Equal(t, ordinary.Stages.Process, special.Stages.Process) + assert.Equal(t, ordinary.Stages.BuildSignal, special.Stages.BuildSignal) + assert.Equal(t, ordinary.Stages.IssueTrack, special.Stages.IssueTrack) + + configs, err := store.List(ctx) + require.NoError(t, err) + require.Len(t, configs, 2) + assert.Equal(t, []string{"ordinary", "special"}, []string{configs[0].Name, configs[1].Name}) + + configs[1].Stages.BuildAnalysis.PollDelayMs = 0 + again, err := store.Get(ctx, "special") + require.NoError(t, err) + assert.Equal(t, special, again) + + for _, name := range []string{"", "unknown"} { + _, err := store.Get(ctx, name) + require.ErrorIs(t, err, queueconfig.ErrNotFound) + } +} + +func TestStoreValidatesAfterAllQueueLayers(t *testing.T) { + zero, final := int64(0), int64(250) + overrides := map[string][]queueconfig.Overrides{ + "known": { + {BuildAnalysis: queueconfig.BuildAnalysisOverrides{PollDelayMs: &zero}}, + {BuildAnalysis: queueconfig.BuildAnalysisOverrides{PollDelayMs: &final}}, + }, + } + + store, err := NewStore([]string{"known"}, testStages(), overrides) + require.NoError(t, err) + + cfg, err := store.Get(context.Background(), "known") + require.NoError(t, err) + assert.Equal(t, final, cfg.Stages.BuildAnalysis.PollDelayMs) + + overrides["known"] = overrides["known"][:1] + _, err = NewStore([]string{"known"}, testStages(), overrides) + require.Error(t, err) +} + +func TestStoreRejectsInvalidRegistry(t *testing.T) { + for _, tt := range []struct { + name string + names []string + overrides map[string][]queueconfig.Overrides + }{ + {name: "empty name", names: []string{""}}, + {name: "duplicate name", names: []string{"a", "a"}}, + {name: "unknown override", names: []string{"a"}, overrides: map[string][]queueconfig.Overrides{"unknown": nil}}, + } { + t.Run(tt.name, func(t *testing.T) { + _, err := NewStore(tt.names, testStages(), tt.overrides) + require.Error(t, err) + }) + } +} + +func TestStoreRequiresValidDeploymentEvenWithoutQueues(t *testing.T) { + _, err := NewStore(nil, entity.StageConfig{}, nil) + require.Error(t, err) + + store, err := NewStore(nil, testStages(), nil) + require.NoError(t, err) + + queues, err := store.List(context.Background()) + require.NoError(t, err) + assert.Empty(t, queues) +}