diff --git a/submitqueue/core/hookevent/BUILD.bazel b/submitqueue/core/hookevent/BUILD.bazel new file mode 100644 index 000000000..ea1932ddf --- /dev/null +++ b/submitqueue/core/hookevent/BUILD.bazel @@ -0,0 +1,25 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["hookevent.go"], + importpath = "github.com/uber/submitqueue/submitqueue/core/hookevent", + visibility = ["//visibility:public"], + deps = [ + "//api/base/hook:go_default_library", + "//submitqueue/entity:go_default_library", + "@org_golang_google_protobuf//types/known/structpb:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["hookevent_test.go"], + embed = [":go_default_library"], + deps = [ + "//api/base/hook:go_default_library", + "//submitqueue/entity:go_default_library", + "@com_github_stretchr_testify//assert:go_default_library", + "@com_github_stretchr_testify//require:go_default_library", + ], +) diff --git a/submitqueue/core/hookevent/hookevent.go b/submitqueue/core/hookevent/hookevent.go new file mode 100644 index 000000000..e7345dbc8 --- /dev/null +++ b/submitqueue/core/hookevent/hookevent.go @@ -0,0 +1,126 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package hookevent holds SubmitQueue's hook events: the source it reports, the +// types it publishes, and a constructor per type. Centralizing them keeps the +// wire contract in one place, so a publisher names an event rather than +// assembling one. +// +// The events follow a request's public status, the one its summary shows: one +// when the request is received, and one when it lands or fails. Cancellation +// has no event yet because no hook needs one. Deriving them from the summary +// rather than from pipeline transitions keeps every hook in step with what a +// user sees, including a request whose start failed before the pipeline saved +// it. +// +// Each status announces one event per change of the request, so a hook that +// retries one change's failed side effect never repeats the others'. Payloads +// carry the queue, the request id, and that change's URI. The gateway +// publishes these events and the orchestrator's hook stage consumes them, and a +// hook there cannot read the gateway's store, so the payload carries what a +// hook needs to reach the change. A request's URIs never change, so they cannot +// go stale on redelivery. See doc/rfc/hook-framework.md. +// +// Known gaps, accepted for now: +// - request.received is published inside Land, an RPC. If Land records +// accepted but the publish fails, no later status announces the receipt. +// - A finished status is announced again whenever its log is redelivered or +// resent, so a failed publish repairs itself; a copy that arrives after the +// queue forgot the id reaches hooks twice. +// - Order holds within a request's partition, but a receipt waiting to retry +// does not hold back the finish, so a hook can see the finish first. +// - The queue drops a repeated event id only until the first copy is consumed +// and collected, seconds later. +package hookevent + +import ( + "crypto/sha256" + "fmt" + + basehook "github.com/uber/submitqueue/api/base/hook" + "github.com/uber/submitqueue/submitqueue/entity" + "google.golang.org/protobuf/types/known/structpb" +) + +// Source is reported on every hook event SubmitQueue publishes. A consumer +// serving several domains matches on it to recognize SubmitQueue's events. +const Source = "submitqueue" + +// Type is the event type a consumer filters on. +type Type string + +const ( + // TypeRequestReceived announces that the request named in the payload has + // left the internal accepting status and is visible to users. It carries no + // verdict: the request has not been validated yet. + TypeRequestReceived Type = "request.received" + // TypeRequestLanded announces that the request named in the payload landed. + TypeRequestLanded Type = "request.landed" + // TypeRequestFailed announces that the request named in the payload ended in + // error. The request's summary carries the reason. + TypeRequestFailed Type = "request.failed" +) + +// finishedTypeByStatus maps each finished status that has an event to its type. +var finishedTypeByStatus = map[entity.RequestStatus]Type{ + entity.RequestStatusLanded: TypeRequestLanded, + entity.RequestStatusError: TypeRequestFailed, +} + +// NewRequestReceived builds the events announcing that summary's request was +// received, one per change. A request is received once, so the ids carry version +// 0 and every republish for the same request mints the same ids. +func NewRequestReceived(summary entity.RequestSummary) []*basehook.HookEvent { + return newRequestEvents(TypeRequestReceived, summary, 0, summary.ReceivedAtMs) +} + +// NewRequestFinished builds the events announcing summary's finished status, one +// per change, and none for a status that has no event. The ids carry +// summary.RequestVersion, so a status the summary restates mints the same ids and +// a later finished status the summary moves to mints new ones. +func NewRequestFinished(summary entity.RequestSummary) []*basehook.HookEvent { + eventType, ok := finishedTypeByStatus[summary.Status] + if !ok { + return nil + } + return newRequestEvents(eventType, summary, summary.RequestVersion, summary.StatusTimestampMs) +} + +func newRequestEvents(eventType Type, summary entity.RequestSummary, version int32, timestampMs int64) []*basehook.HookEvent { + events := make([]*basehook.HookEvent, 0, len(summary.ChangeURIs)) + for _, uri := range summary.ChangeURIs { + events = append(events, &basehook.HookEvent{ + Id: basehook.NewEventID(Source, string(eventType), changeSubject(summary, uri), version), + Source: Source, + Type: string(eventType), + TimestampMs: timestampMs, + Version: version, + Payload: &structpb.Struct{Fields: map[string]*structpb.Value{ + "queue": structpb.NewStringValue(summary.Queue), + "request_id": structpb.NewStringValue(summary.RequestID), + "change_uri": structpb.NewStringValue(uri), + }}, + }) + } + return events +} + +// changeSubject identifies one change of a request across queues. Request IDs +// are only unique within a queue, and queue names and URIs contain the id +// separator, so every part is length-prefixed and hashed into one separator-free +// component. +func changeSubject(summary entity.RequestSummary, uri string) string { + identity := fmt.Sprintf("%d:%s%d:%s%d:%s", len(summary.Queue), summary.Queue, len(summary.RequestID), summary.RequestID, len(uri), uri) + return fmt.Sprintf("%x", sha256.Sum256([]byte(identity))) +} diff --git a/submitqueue/core/hookevent/hookevent_test.go b/submitqueue/core/hookevent/hookevent_test.go new file mode 100644 index 000000000..363fbfd74 --- /dev/null +++ b/submitqueue/core/hookevent/hookevent_test.go @@ -0,0 +1,146 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hookevent + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + basehook "github.com/uber/submitqueue/api/base/hook" + "github.com/uber/submitqueue/submitqueue/entity" +) + +func TestNewRequestReceived(t *testing.T) { + tests := []struct { + name string + summary entity.RequestSummary + }{ + { + name: "one change in a queue with a slash", + summary: entity.RequestSummary{RequestID: "1149", Queue: "acme/monorepo", ChangeURIs: []string{"github://github.com/acme/monorepo/pull/42/abc123"}, Status: entity.RequestStatusAccepted, ReceivedAtMs: 100}, + }, + { + name: "a stack, received under a later status", + summary: entity.RequestSummary{RequestID: "7", Queue: "demo-queue", ChangeURIs: []string{"uri/1", "uri/2"}, Status: entity.RequestStatusStarted, RequestVersion: 1, ReceivedAtMs: 200}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + events := NewRequestReceived(tt.summary) + + require.Len(t, events, len(tt.summary.ChangeURIs)) + for i, event := range events { + uri := tt.summary.ChangeURIs[i] + require.NoError(t, basehook.Validate(event)) + assert.Equal(t, Source, event.GetSource()) + assert.Equal(t, string(TypeRequestReceived), event.GetType()) + assert.Zero(t, event.GetVersion()) + assert.Equal(t, tt.summary.ReceivedAtMs, event.GetTimestampMs()) + assert.Equal(t, tt.summary.Queue, event.GetPayload().GetFields()["queue"].GetStringValue()) + assert.Equal(t, tt.summary.RequestID, event.GetPayload().GetFields()["request_id"].GetStringValue()) + assert.Equal(t, uri, event.GetPayload().GetFields()["change_uri"].GetStringValue()) + assert.Equal(t, basehook.NewEventID(Source, string(TypeRequestReceived), changeSubject(tt.summary, uri), 0), event.GetId()) + } + + later := tt.summary + later.Status, later.RequestVersion = entity.RequestStatusLanded, 5 + assert.Equal(t, eventIDs(events), eventIDs(NewRequestReceived(later)), "a request is received once, whatever its status now") + }) + } +} + +func TestNewRequestFinished(t *testing.T) { + tests := []struct { + name string + status entity.RequestStatus + version int32 + wantType Type + wantEvent bool + }{ + {name: "landed", status: entity.RequestStatusLanded, version: 5, wantType: TypeRequestLanded, wantEvent: true}, + {name: "failed", status: entity.RequestStatusError, version: 3, wantType: TypeRequestFailed, wantEvent: true}, + {name: "failed before the pipeline saved the request", status: entity.RequestStatusError, version: 0, wantType: TypeRequestFailed, wantEvent: true}, + {name: "cancelled has no event", status: entity.RequestStatusCancelled, version: 4}, + {name: "unfinished status", status: entity.RequestStatusBatched, version: 2}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + summary := entity.RequestSummary{RequestID: "1149", Queue: "acme/monorepo", ChangeURIs: []string{"uri/1", "uri/2"}, Status: tt.status, RequestVersion: tt.version, StatusTimestampMs: 300} + + events := NewRequestFinished(summary) + + if !tt.wantEvent { + assert.Empty(t, events) + return + } + require.Len(t, events, len(summary.ChangeURIs)) + for i, event := range events { + uri := summary.ChangeURIs[i] + require.NoError(t, basehook.Validate(event)) + assert.Equal(t, string(tt.wantType), event.GetType()) + assert.Equal(t, tt.version, event.GetVersion()) + assert.Equal(t, summary.StatusTimestampMs, event.GetTimestampMs()) + assert.Equal(t, uri, event.GetPayload().GetFields()["change_uri"].GetStringValue()) + assert.Equal(t, basehook.NewEventID(Source, string(tt.wantType), changeSubject(summary, uri), tt.version), event.GetId()) + } + }) + } +} + +func TestChangeSubject(t *testing.T) { + tests := []struct { + name string + a, b entity.RequestSummary + aURI string + bURI string + }{ + { + name: "same id in different queues", + a: entity.RequestSummary{RequestID: "1", Queue: "queue-a"}, + b: entity.RequestSummary{RequestID: "1", Queue: "queue-b"}, + aURI: "uri/1", bURI: "uri/1", + }, + { + name: "boundary between queue and id moves", + a: entity.RequestSummary{RequestID: "23", Queue: "q1"}, + b: entity.RequestSummary{RequestID: "3", Queue: "q12"}, + aURI: "uri/1", bURI: "uri/1", + }, + { + name: "different changes of one request", + a: entity.RequestSummary{RequestID: "1", Queue: "q"}, + b: entity.RequestSummary{RequestID: "1", Queue: "q"}, + aURI: "uri/1", bURI: "uri/2", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.NotEqual(t, changeSubject(tt.a, tt.aURI), changeSubject(tt.b, tt.bURI)) + assert.NotContains(t, changeSubject(tt.a, tt.aURI), "/") + }) + } +} + +func eventIDs(events []*basehook.HookEvent) []string { + ids := make([]string, 0, len(events)) + for _, event := range events { + ids = append(ids, event.GetId()) + } + return ids +} diff --git a/submitqueue/gateway/controller/BUILD.bazel b/submitqueue/gateway/controller/BUILD.bazel index 3281e0c2c..d0d88da7c 100644 --- a/submitqueue/gateway/controller/BUILD.bazel +++ b/submitqueue/gateway/controller/BUILD.bazel @@ -51,6 +51,7 @@ go_test( ], embed = [":go_default_library"], deps = [ + "//api/base/hook:go_default_library", "//api/submitqueue/gateway/protopb:go_default_library", "//platform/base/change:go_default_library", "//platform/base/mergestrategy:go_default_library", diff --git a/submitqueue/gateway/controller/land_test.go b/submitqueue/gateway/controller/land_test.go index 6f31bf913..b9172d6bf 100644 --- a/submitqueue/gateway/controller/land_test.go +++ b/submitqueue/gateway/controller/land_test.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/uber-go/tally" + basehook "github.com/uber/submitqueue/api/base/hook" "github.com/uber/submitqueue/platform/base/change" "github.com/uber/submitqueue/platform/base/mergestrategy" entityqueue "github.com/uber/submitqueue/platform/base/messagequeue" @@ -573,6 +574,37 @@ func TestLand_ReturnsErrorWhenPublishFails(t *testing.T) { require.Error(t, err) } +func TestLand_ReceiptHookPublishFailureLeavesTheLandIntact(t *testing.T) { + ctrl := gomock.NewController(t) + cnt := countermock.NewMockCounter(ctrl) + cnt.EXPECT().Next(gomock.Any(), gomock.Any()).Return(int64(999), nil) + fixture := newControllerStorageFixture(ctrl) + publisher := queuemock.NewMockPublisher(ctrl) + publisher.EXPECT().Publish(gomock.Any(), "start", gomock.Any()).Return(nil) + publisher.EXPECT().Publish(gomock.Any(), "hook", gomock.Any()).Return(fmt.Errorf("queue unavailable")) + queue := queuemock.NewMockQueue(ctrl) + queue.EXPECT().Publisher().Return(publisher).AnyTimes() + registry, err := consumer.NewTopicRegistry([]consumer.TopicConfig{ + {Key: topickey.TopicKeyStart, Name: "start", Queue: queue}, + {Key: basehook.TopicKeyHook, Name: "hook", Queue: queue}, + }) + require.NoError(t, err) + scope := tally.NewTestScope("", nil) + materializer := requestcore.NewMaterializer(fixture.newFactory(ctrl), requestcore.WithMetricsScope(scope), requestcore.WithHookPublishing(registry)) + controller := NewLandController(zap.NewNop().Sugar(), scope, staticCounterFactory{counter: cnt}, fixture.newFactory(ctrl), materializer, noopQueueConfigStore(ctrl), queuepausenoop.New(), registry) + + result, err := controller.Land(context.Background(), testLandRequest("test-queue")) + + require.NoError(t, err) + assert.Equal(t, "999", result.ID) + assert.Equal(t, entity.RequestStatusAccepted, fixture.summaries["999"].Status) + counters := scope.Snapshot().Counters() + assert.Contains(t, counters, "request_hook.publish_errors+event_type=request.received,queue=test-queue") + for key := range counters { + assert.NotContains(t, key, "accepted_log_failure", "the accepted log was written; only the hook event was lost") + } +} + func TestLand_ReturnsSqidWhenAcceptedLogFailsAfterPublish(t *testing.T) { ctrl := gomock.NewController(t) cnt := countermock.NewMockCounter(ctrl) diff --git a/submitqueue/gateway/core/request/BUILD.bazel b/submitqueue/gateway/core/request/BUILD.bazel index 4776a6963..e5e3428db 100644 --- a/submitqueue/gateway/core/request/BUILD.bazel +++ b/submitqueue/gateway/core/request/BUILD.bazel @@ -10,7 +10,11 @@ go_library( importpath = "github.com/uber/submitqueue/submitqueue/gateway/core/request", visibility = ["//visibility:public"], deps = [ + "//api/base/hook:go_default_library", + "//platform/consumer:go_default_library", + "//platform/hook:go_default_library", "//platform/metrics:go_default_library", + "//submitqueue/core/hookevent:go_default_library", "//submitqueue/entity:go_default_library", "//submitqueue/extension/storage:go_default_library", "//submitqueue/gateway/extension/storage:go_default_library", @@ -27,6 +31,11 @@ go_test( ], embed = [":go_default_library"], deps = [ + "//api/base/hook:go_default_library", + "//platform/base/messagequeue:go_default_library", + "//platform/consumer:go_default_library", + "//platform/extension/messagequeue/mock:go_default_library", + "//submitqueue/core/hookevent:go_default_library", "//submitqueue/entity:go_default_library", "//submitqueue/extension/storage:go_default_library", "//submitqueue/extension/storage/mock:go_default_library", diff --git a/submitqueue/gateway/core/request/materializer.go b/submitqueue/gateway/core/request/materializer.go index 273bb68b4..f2dbd35ec 100644 --- a/submitqueue/gateway/core/request/materializer.go +++ b/submitqueue/gateway/core/request/materializer.go @@ -22,7 +22,11 @@ import ( "time" "github.com/uber-go/tally" + basehook "github.com/uber/submitqueue/api/base/hook" + "github.com/uber/submitqueue/platform/consumer" + platformhook "github.com/uber/submitqueue/platform/hook" "github.com/uber/submitqueue/platform/metrics" + "github.com/uber/submitqueue/submitqueue/core/hookevent" "github.com/uber/submitqueue/submitqueue/entity" basestorage "github.com/uber/submitqueue/submitqueue/extension/storage" gwstorage "github.com/uber/submitqueue/submitqueue/gateway/extension/storage" @@ -33,8 +37,10 @@ import ( // Every store it touches is queue-scoped, so each call resolves the aggregate once // from the queue carried on the log being persisted. type Materializer struct { - stores gwstorage.Factory - metricsScope tally.Scope + stores gwstorage.Factory + metricsScope tally.Scope + hookRegistry consumer.TopicRegistry + publishesHooks bool } // MaterializerOption configures a Materializer. @@ -46,6 +52,15 @@ func WithMetricsScope(scope tally.Scope) MaterializerOption { return func(m *Materializer) { m.metricsScope = scope } } +// WithHookPublishing publishes request hook events (see submitqueue/core/hookevent) +// to registry's hook topic. Without it the materializer publishes none. +func WithHookPublishing(registry consumer.TopicRegistry) MaterializerOption { + return func(m *Materializer) { + m.hookRegistry = registry + m.publishesHooks = true + } +} + // NewMaterializer creates a request read-model materializer. func NewMaterializer(stores gwstorage.Factory, opts ...MaterializerOption) *Materializer { m := &Materializer{stores: stores, metricsScope: tally.NoopScope} @@ -76,6 +91,7 @@ func (m *Materializer) PersistLog(ctx context.Context, log entity.RequestLog) er return fmt.Errorf("failed to get request summary request_id=%s: %w", log.RequestID, err) } + beforeUpdate, updatedSummary := summary, false if logWins(log, summary) { oldVersion := summary.Version newVersion := oldVersion + 1 @@ -94,7 +110,7 @@ func (m *Materializer) PersistLog(ctx context.Context, log entity.RequestLog) er } updated.Version = newVersion m.recordLifecycleTransition(summary, updated) - summary = updated + summary, updatedSummary = updated, true } if summary.Status == entity.RequestStatusAccepting { @@ -106,7 +122,7 @@ func (m *Materializer) PersistLog(ctx context.Context, log entity.RequestLog) er if err := ensureRequestReceiptMapping(ctx, stores.GetRequestReceiptStore(), summary); err != nil { return err } - return nil + return m.publishRequestHookEvents(ctx, log, beforeUpdate, summary, updatedSummary) } } @@ -138,6 +154,50 @@ func (m *Materializer) recordLifecycleTransition(before, after entity.RequestSum queueTag, statusTag).RecordDuration(elapsed) } +// publishRequestHookEvents announces what log did to the request: the receipt +// when this delivery's update took the summary out of accepting, and the finish +// whenever log restates a finished summary. +// +// The receipt is announced once. Nothing redelivers the accepting step, so a +// failed receipt publish is only counted: returning it would retry nothing, and +// on Land it would read as a failed accepted log. The finish is announced on +// every delivery of its log, so a failed publish is returned and the log's +// redelivery repairs it. +func (m *Materializer) publishRequestHookEvents(ctx context.Context, log entity.RequestLog, before, after entity.RequestSummary, updated bool) error { + if !m.publishesHooks { + return nil + } + if updated && before.Status == entity.RequestStatusAccepting && after.Status != entity.RequestStatusAccepting { + for _, event := range hookevent.NewRequestReceived(after) { + _ = m.publishRequestHookEvent(ctx, after, event) + } + } + if !logRestatesSummary(log, after) { + return nil + } + for _, event := range hookevent.NewRequestFinished(after) { + if err := m.publishRequestHookEvent(ctx, after, event); err != nil { + return err + } + } + return nil +} + +func (m *Materializer) publishRequestHookEvent(ctx context.Context, summary entity.RequestSummary, event *basehook.HookEvent) error { + if err := platformhook.Publish(ctx, m.hookRegistry, summary.Queue, event, summary.RequestID); err != nil { + metrics.NamedCounter(m.metricsScope, "request_hook", "publish_errors", 1, + metrics.NewTag("queue", summary.Queue), metrics.NewTag("event_type", event.GetType())) + return fmt.Errorf("failed to publish %s hook event request_id=%s: %w", event.GetType(), summary.RequestID, err) + } + return nil +} + +// logRestatesSummary reports whether log carries the status and request version +// summary shows, whether log just set them or repeats them. +func logRestatesSummary(log entity.RequestLog, summary entity.RequestSummary) bool { + return log.Type == entity.RequestLogTypeStatus && log.Status == summary.Status && log.RequestVersion == summary.RequestVersion +} + func isTerminalStatus(status entity.RequestStatus) bool { return entity.IsRequestStateTerminal(entity.RequestState(status)) } diff --git a/submitqueue/gateway/core/request/materializer_test.go b/submitqueue/gateway/core/request/materializer_test.go index e35ccab36..e2d9333d9 100644 --- a/submitqueue/gateway/core/request/materializer_test.go +++ b/submitqueue/gateway/core/request/materializer_test.go @@ -21,6 +21,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + basehook "github.com/uber/submitqueue/api/base/hook" + entityqueue "github.com/uber/submitqueue/platform/base/messagequeue" + "github.com/uber/submitqueue/platform/consumer" + queuemock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" + "github.com/uber/submitqueue/submitqueue/core/hookevent" "github.com/uber/submitqueue/submitqueue/entity" "github.com/uber/submitqueue/submitqueue/extension/storage" "go.uber.org/mock/gomock" @@ -162,6 +167,246 @@ func TestMaterializer_RecordsLifecycleMetrics(t *testing.T) { } } +func TestMaterializer_PublishesRequestHookEvents(t *testing.T) { + accepting := testRequestSummary() + accepted := accepting + accepted.Status, accepted.StatusTimestampMs, accepted.Version = entity.RequestStatusAccepted, 15, 2 + started := accepted + started.Status, started.RequestVersion, started.StatusTimestampMs, started.Version = entity.RequestStatusStarted, 1, 16, 3 + landed := started + landed.Status, landed.RequestVersion, landed.StatusTimestampMs, landed.Version = entity.RequestStatusLanded, 2, 20, 4 + statusLog := func(status entity.RequestStatus, version int32, timestampMs int64) entity.RequestLog { + return entity.RequestLog{Queue: "q", RequestID: "1", Type: entity.RequestLogTypeStatus, Status: status, RequestVersion: version, TimestampMs: timestampMs} + } + type published struct { + eventType hookevent.Type + version int32 + changeURI string + } + perChange := func(eventType hookevent.Type, version int32) []published { + return []published{{eventType, version, "uri/1"}, {eventType, version, "uri/2"}} + } + + tests := []struct { + name string + before entity.RequestSummary + log entity.RequestLog + wantUpdate bool + wantPublished []published + }{ + { + name: "accepted log announces the receipt for every change", + before: accepting, + log: statusLog(entity.RequestStatusAccepted, 0, 15), + wantUpdate: true, + wantPublished: perChange(hookevent.TypeRequestReceived, 0), + }, + { + name: "a later status standing in for a lost accepted log still announces the receipt", + before: accepting, + log: statusLog(entity.RequestStatusStarted, 1, 16), + wantUpdate: true, + wantPublished: perChange(hookevent.TypeRequestReceived, 0), + }, + { + name: "landed log announces the finish", + before: started, + log: statusLog(entity.RequestStatusLanded, 2, 20), + wantUpdate: true, + wantPublished: perChange(hookevent.TypeRequestLanded, 2), + }, + { + name: "failure before the pipeline saved the request announces every receipt, then every finish", + before: accepting, + log: statusLog(entity.RequestStatusError, 0, 20), + wantUpdate: true, + wantPublished: append(perChange(hookevent.TypeRequestReceived, 0), perChange(hookevent.TypeRequestFailed, 0)...), + }, + { + name: "a later finished version announces again", + before: landed, + log: statusLog(entity.RequestStatusError, 3, 30), + wantUpdate: true, + wantPublished: perChange(hookevent.TypeRequestFailed, 3), + }, + { + name: "a redelivered finished log announces the finish again, repairing a failed publish", + before: landed, + log: statusLog(entity.RequestStatusLanded, 2, 20), + wantPublished: perChange(hookevent.TypeRequestLanded, 2), + }, + { + name: "a resent finished log announces the finish again", + before: landed, + log: statusLog(entity.RequestStatusLanded, 2, 30), + wantUpdate: true, + wantPublished: perChange(hookevent.TypeRequestLanded, 2), + }, + { + name: "cancellation has no event", + before: started, + log: statusLog(entity.RequestStatusCancelled, 2, 20), + wantUpdate: true, + }, + { + name: "an unfinished status after the receipt has no event", + before: accepted, + log: statusLog(entity.RequestStatusStarted, 1, 16), + wantUpdate: true, + }, + { + name: "a stale log after the finish has no event", + before: landed, + log: statusLog(entity.RequestStatusStarted, 1, 16), + }, + { + name: "a lower finished version after the finish has no event", + before: landed, + log: statusLog(entity.RequestStatusError, 1, 40), + }, + { + name: "a redelivered accepted log has no event, even if its first publish failed", + before: accepted, + log: statusLog(entity.RequestStatusAccepted, 0, 15), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + var messages []entityqueue.Message + registry := newHookRegistry(t, ctrl, func(msg entityqueue.Message) error { + messages = append(messages, msg) + return nil + }) + f := newMaterializerReceiptFixture(ctrl, WithHookPublishing(registry)) + f.logs.EXPECT().Insert(gomock.Any(), tt.log).Return(nil) + f.summaries.EXPECT().Get(gomock.Any(), tt.before.RequestID).Return(tt.before, nil) + if tt.wantUpdate { + f.summaries.EXPECT().Update(gomock.Any(), gomock.Any(), tt.before.Version, tt.before.Version+1).Return(nil) + } + f.uris.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + f.receipts.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + + require.NoError(t, f.materializer.PersistLog(context.Background(), tt.log)) + + got := []published{} + for _, msg := range messages { + event := &basehook.HookEvent{} + require.NoError(t, basehook.Unmarshal(msg.Payload, event)) + assert.Equal(t, event.GetId(), msg.ID, "the event id is the queue dedupe key") + assert.Equal(t, tt.before.RequestID, msg.PartitionKey) + assert.Equal(t, tt.before.Queue, msg.Tenant) + got = append(got, published{hookevent.Type(event.GetType()), event.GetVersion(), event.GetPayload().GetFields()["change_uri"].GetStringValue()}) + } + if tt.wantPublished == nil { + tt.wantPublished = []published{} + } + assert.Equal(t, tt.wantPublished, got) + }) + } +} + +func TestMaterializer_HookPublishFailures(t *testing.T) { + accepting := testRequestSummary() + started := accepting + started.Status, started.RequestVersion, started.StatusTimestampMs, started.Version = entity.RequestStatusStarted, 1, 16, 3 + + tests := []struct { + name string + before entity.RequestSummary + log entity.RequestLog + wantErr bool + wantCounter string + wantCount int64 + }{ + { + name: "a failed receipt is counted but does not fail the log, which could not repeat it", + before: accepting, + log: entity.RequestLog{Queue: "q", RequestID: "1", Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusAccepted, TimestampMs: 20}, + wantCounter: "request_hook.publish_errors+event_type=request.received,queue=q", + wantCount: 2, + }, + { + name: "a failed finish fails the log, whose redelivery announces it again", + before: started, + log: entity.RequestLog{Queue: "q", RequestID: "1", Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusLanded, RequestVersion: 2, TimestampMs: 20}, + wantErr: true, + wantCounter: "request_hook.publish_errors+event_type=request.landed,queue=q", + wantCount: 1, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + publishErr := errors.New("queue unavailable") + registry := newHookRegistry(t, ctrl, func(entityqueue.Message) error { return publishErr }) + f := newMaterializerReceiptFixture(ctrl, WithHookPublishing(registry)) + f.logs.EXPECT().Insert(gomock.Any(), tt.log).Return(nil) + f.summaries.EXPECT().Get(gomock.Any(), tt.before.RequestID).Return(tt.before, nil) + f.summaries.EXPECT().Update(gomock.Any(), gomock.Any(), tt.before.Version, tt.before.Version+1).Return(nil) + f.uris.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil).Times(len(tt.before.ChangeURIs)) + f.receipts.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil) + + err := f.materializer.PersistLog(context.Background(), tt.log) + + if tt.wantErr { + require.ErrorIs(t, err, publishErr) + } else { + require.NoError(t, err) + } + counter, ok := f.metrics.Snapshot().Counters()[tt.wantCounter] + require.True(t, ok) + assert.Equal(t, tt.wantCount, counter.Value()) + }) + } +} + +func TestMaterializer_PublishesTheWinningTransitionAcrossAVersionConflict(t *testing.T) { + ctrl := gomock.NewController(t) + var published []string + registry := newHookRegistry(t, ctrl, func(msg entityqueue.Message) error { + published = append(published, msg.ID) + return nil + }) + f := newMaterializerReceiptFixture(ctrl, WithHookPublishing(registry)) + current := testRequestSummary() + concurrent := current + concurrent.Status, concurrent.RequestVersion, concurrent.StatusTimestampMs, concurrent.Version = entity.RequestStatusStarted, 1, 15, 2 + log := entity.RequestLog{Queue: current.Queue, RequestID: current.RequestID, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusLanded, RequestVersion: 2, TimestampMs: 20} + gomock.InOrder( + f.logs.EXPECT().Insert(gomock.Any(), log).Return(nil), + f.summaries.EXPECT().Get(gomock.Any(), current.RequestID).Return(current, nil), + f.summaries.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).Return(storage.ErrVersionMismatch), + f.summaries.EXPECT().Get(gomock.Any(), current.RequestID).Return(concurrent, nil), + f.summaries.EXPECT().Update(gomock.Any(), gomock.Any(), int32(2), int32(3)).Return(nil), + ) + f.uris.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + f.receipts.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + + require.NoError(t, f.materializer.PersistLog(context.Background(), log)) + + landed := concurrent + landed.Status, landed.RequestVersion = entity.RequestStatusLanded, 2 + want := []string{} + for _, event := range hookevent.NewRequestFinished(landed) { + want = append(want, event.GetId()) + } + assert.Equal(t, want, published, "the concurrent writer already announced the receipt; this update only finishes the request") +} + +// newHookRegistry binds the hook topic to a publisher that hands every message to publish. +func newHookRegistry(t *testing.T, ctrl *gomock.Controller, publish func(entityqueue.Message) error) consumer.TopicRegistry { + publisher := queuemock.NewMockPublisher(ctrl) + publisher.EXPECT().Publish(gomock.Any(), "hook", gomock.Any()).DoAndReturn( + func(_ context.Context, _ string, msg entityqueue.Message) error { return publish(msg) }, + ).AnyTimes() + queue := queuemock.NewMockQueue(ctrl) + queue.EXPECT().Publisher().Return(publisher).AnyTimes() + registry, err := consumer.NewTopicRegistry([]consumer.TopicConfig{{Key: basehook.TopicKeyHook, Name: "hook", Queue: queue}}) + require.NoError(t, err) + return registry +} + func TestMaterializer_PersistenceFailures(t *testing.T) { writeErr := errors.New("storage unavailable") tests := []struct { diff --git a/submitqueue/gateway/core/request/request_receipt_test.go b/submitqueue/gateway/core/request/request_receipt_test.go index a35dd05ec..2d4e00053 100644 --- a/submitqueue/gateway/core/request/request_receipt_test.go +++ b/submitqueue/gateway/core/request/request_receipt_test.go @@ -37,7 +37,7 @@ type materializerReceiptFixture struct { metrics tally.TestScope } -func newMaterializerReceiptFixture(ctrl *gomock.Controller) materializerReceiptFixture { +func newMaterializerReceiptFixture(ctrl *gomock.Controller, opts ...MaterializerOption) materializerReceiptFixture { f := materializerReceiptFixture{ summaries: storagemock.NewMockRequestSummaryStore(ctrl), uris: storagemock.NewMockRequestURIStore(ctrl), logs: storagemock.NewMockRequestLogStore(ctrl), @@ -51,7 +51,7 @@ func newMaterializerReceiptFixture(ctrl *gomock.Controller) materializerReceiptF stores.EXPECT().GetRequestReceiptStore().Return(f.receipts).AnyTimes() factory := gwstoragemock.NewMockFactory(ctrl) factory.EXPECT().For(gomock.Any()).Return(stores, nil).AnyTimes() - f.materializer = NewMaterializer(factory, WithMetricsScope(f.metrics)) + f.materializer = NewMaterializer(factory, append([]MaterializerOption{WithMetricsScope(f.metrics)}, opts...)...) return f }