From 5540e3fa45c312859458f127204504dba16e8ff1 Mon Sep 17 00:00:00 2001 From: Chenghan Ying <125171961+roychying@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:43:03 -0800 Subject: [PATCH 1/2] feat(submitqueue): Announce received and finished requests from the gateway's request materializer --- submitqueue/core/hookevent/BUILD.bazel | 25 ++ submitqueue/core/hookevent/hookevent.go | 126 +++++++++ submitqueue/core/hookevent/hookevent_test.go | 146 +++++++++++ submitqueue/gateway/controller/BUILD.bazel | 1 + submitqueue/gateway/controller/land_test.go | 32 +++ submitqueue/gateway/core/request/BUILD.bazel | 9 + .../gateway/core/request/materializer.go | 68 ++++- .../gateway/core/request/materializer_test.go | 245 ++++++++++++++++++ .../core/request/request_receipt_test.go | 4 +- 9 files changed, 650 insertions(+), 6 deletions(-) create mode 100644 submitqueue/core/hookevent/BUILD.bazel create mode 100644 submitqueue/core/hookevent/hookevent.go create mode 100644 submitqueue/core/hookevent/hookevent_test.go diff --git a/submitqueue/core/hookevent/BUILD.bazel b/submitqueue/core/hookevent/BUILD.bazel new file mode 100644 index 000000000..ea1932ddf --- /dev/null +++ b/submitqueue/core/hookevent/BUILD.bazel @@ -0,0 +1,25 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["hookevent.go"], + importpath = "github.com/uber/submitqueue/submitqueue/core/hookevent", + visibility = ["//visibility:public"], + deps = [ + "//api/base/hook:go_default_library", + "//submitqueue/entity:go_default_library", + "@org_golang_google_protobuf//types/known/structpb:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["hookevent_test.go"], + embed = [":go_default_library"], + deps = [ + "//api/base/hook:go_default_library", + "//submitqueue/entity:go_default_library", + "@com_github_stretchr_testify//assert:go_default_library", + "@com_github_stretchr_testify//require:go_default_library", + ], +) diff --git a/submitqueue/core/hookevent/hookevent.go b/submitqueue/core/hookevent/hookevent.go new file mode 100644 index 000000000..e7345dbc8 --- /dev/null +++ b/submitqueue/core/hookevent/hookevent.go @@ -0,0 +1,126 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package hookevent holds SubmitQueue's hook events: the source it reports, the +// types it publishes, and a constructor per type. Centralizing them keeps the +// wire contract in one place, so a publisher names an event rather than +// assembling one. +// +// The events follow a request's public status, the one its summary shows: one +// when the request is received, and one when it lands or fails. Cancellation +// has no event yet because no hook needs one. Deriving them from the summary +// rather than from pipeline transitions keeps every hook in step with what a +// user sees, including a request whose start failed before the pipeline saved +// it. +// +// Each status announces one event per change of the request, so a hook that +// retries one change's failed side effect never repeats the others'. Payloads +// carry the queue, the request id, and that change's URI. The gateway +// publishes these events and the orchestrator's hook stage consumes them, and a +// hook there cannot read the gateway's store, so the payload carries what a +// hook needs to reach the change. A request's URIs never change, so they cannot +// go stale on redelivery. See doc/rfc/hook-framework.md. +// +// Known gaps, accepted for now: +// - request.received is published inside Land, an RPC. If Land records +// accepted but the publish fails, no later status announces the receipt. +// - A finished status is announced again whenever its log is redelivered or +// resent, so a failed publish repairs itself; a copy that arrives after the +// queue forgot the id reaches hooks twice. +// - Order holds within a request's partition, but a receipt waiting to retry +// does not hold back the finish, so a hook can see the finish first. +// - The queue drops a repeated event id only until the first copy is consumed +// and collected, seconds later. +package hookevent + +import ( + "crypto/sha256" + "fmt" + + basehook "github.com/uber/submitqueue/api/base/hook" + "github.com/uber/submitqueue/submitqueue/entity" + "google.golang.org/protobuf/types/known/structpb" +) + +// Source is reported on every hook event SubmitQueue publishes. A consumer +// serving several domains matches on it to recognize SubmitQueue's events. +const Source = "submitqueue" + +// Type is the event type a consumer filters on. +type Type string + +const ( + // TypeRequestReceived announces that the request named in the payload has + // left the internal accepting status and is visible to users. It carries no + // verdict: the request has not been validated yet. + TypeRequestReceived Type = "request.received" + // TypeRequestLanded announces that the request named in the payload landed. + TypeRequestLanded Type = "request.landed" + // TypeRequestFailed announces that the request named in the payload ended in + // error. The request's summary carries the reason. + TypeRequestFailed Type = "request.failed" +) + +// finishedTypeByStatus maps each finished status that has an event to its type. +var finishedTypeByStatus = map[entity.RequestStatus]Type{ + entity.RequestStatusLanded: TypeRequestLanded, + entity.RequestStatusError: TypeRequestFailed, +} + +// NewRequestReceived builds the events announcing that summary's request was +// received, one per change. A request is received once, so the ids carry version +// 0 and every republish for the same request mints the same ids. +func NewRequestReceived(summary entity.RequestSummary) []*basehook.HookEvent { + return newRequestEvents(TypeRequestReceived, summary, 0, summary.ReceivedAtMs) +} + +// NewRequestFinished builds the events announcing summary's finished status, one +// per change, and none for a status that has no event. The ids carry +// summary.RequestVersion, so a status the summary restates mints the same ids and +// a later finished status the summary moves to mints new ones. +func NewRequestFinished(summary entity.RequestSummary) []*basehook.HookEvent { + eventType, ok := finishedTypeByStatus[summary.Status] + if !ok { + return nil + } + return newRequestEvents(eventType, summary, summary.RequestVersion, summary.StatusTimestampMs) +} + +func newRequestEvents(eventType Type, summary entity.RequestSummary, version int32, timestampMs int64) []*basehook.HookEvent { + events := make([]*basehook.HookEvent, 0, len(summary.ChangeURIs)) + for _, uri := range summary.ChangeURIs { + events = append(events, &basehook.HookEvent{ + Id: basehook.NewEventID(Source, string(eventType), changeSubject(summary, uri), version), + Source: Source, + Type: string(eventType), + TimestampMs: timestampMs, + Version: version, + Payload: &structpb.Struct{Fields: map[string]*structpb.Value{ + "queue": structpb.NewStringValue(summary.Queue), + "request_id": structpb.NewStringValue(summary.RequestID), + "change_uri": structpb.NewStringValue(uri), + }}, + }) + } + return events +} + +// changeSubject identifies one change of a request across queues. Request IDs +// are only unique within a queue, and queue names and URIs contain the id +// separator, so every part is length-prefixed and hashed into one separator-free +// component. +func changeSubject(summary entity.RequestSummary, uri string) string { + identity := fmt.Sprintf("%d:%s%d:%s%d:%s", len(summary.Queue), summary.Queue, len(summary.RequestID), summary.RequestID, len(uri), uri) + return fmt.Sprintf("%x", sha256.Sum256([]byte(identity))) +} diff --git a/submitqueue/core/hookevent/hookevent_test.go b/submitqueue/core/hookevent/hookevent_test.go new file mode 100644 index 000000000..363fbfd74 --- /dev/null +++ b/submitqueue/core/hookevent/hookevent_test.go @@ -0,0 +1,146 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hookevent + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + basehook "github.com/uber/submitqueue/api/base/hook" + "github.com/uber/submitqueue/submitqueue/entity" +) + +func TestNewRequestReceived(t *testing.T) { + tests := []struct { + name string + summary entity.RequestSummary + }{ + { + name: "one change in a queue with a slash", + summary: entity.RequestSummary{RequestID: "1149", Queue: "acme/monorepo", ChangeURIs: []string{"github://github.com/acme/monorepo/pull/42/abc123"}, Status: entity.RequestStatusAccepted, ReceivedAtMs: 100}, + }, + { + name: "a stack, received under a later status", + summary: entity.RequestSummary{RequestID: "7", Queue: "demo-queue", ChangeURIs: []string{"uri/1", "uri/2"}, Status: entity.RequestStatusStarted, RequestVersion: 1, ReceivedAtMs: 200}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + events := NewRequestReceived(tt.summary) + + require.Len(t, events, len(tt.summary.ChangeURIs)) + for i, event := range events { + uri := tt.summary.ChangeURIs[i] + require.NoError(t, basehook.Validate(event)) + assert.Equal(t, Source, event.GetSource()) + assert.Equal(t, string(TypeRequestReceived), event.GetType()) + assert.Zero(t, event.GetVersion()) + assert.Equal(t, tt.summary.ReceivedAtMs, event.GetTimestampMs()) + assert.Equal(t, tt.summary.Queue, event.GetPayload().GetFields()["queue"].GetStringValue()) + assert.Equal(t, tt.summary.RequestID, event.GetPayload().GetFields()["request_id"].GetStringValue()) + assert.Equal(t, uri, event.GetPayload().GetFields()["change_uri"].GetStringValue()) + assert.Equal(t, basehook.NewEventID(Source, string(TypeRequestReceived), changeSubject(tt.summary, uri), 0), event.GetId()) + } + + later := tt.summary + later.Status, later.RequestVersion = entity.RequestStatusLanded, 5 + assert.Equal(t, eventIDs(events), eventIDs(NewRequestReceived(later)), "a request is received once, whatever its status now") + }) + } +} + +func TestNewRequestFinished(t *testing.T) { + tests := []struct { + name string + status entity.RequestStatus + version int32 + wantType Type + wantEvent bool + }{ + {name: "landed", status: entity.RequestStatusLanded, version: 5, wantType: TypeRequestLanded, wantEvent: true}, + {name: "failed", status: entity.RequestStatusError, version: 3, wantType: TypeRequestFailed, wantEvent: true}, + {name: "failed before the pipeline saved the request", status: entity.RequestStatusError, version: 0, wantType: TypeRequestFailed, wantEvent: true}, + {name: "cancelled has no event", status: entity.RequestStatusCancelled, version: 4}, + {name: "unfinished status", status: entity.RequestStatusBatched, version: 2}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + summary := entity.RequestSummary{RequestID: "1149", Queue: "acme/monorepo", ChangeURIs: []string{"uri/1", "uri/2"}, Status: tt.status, RequestVersion: tt.version, StatusTimestampMs: 300} + + events := NewRequestFinished(summary) + + if !tt.wantEvent { + assert.Empty(t, events) + return + } + require.Len(t, events, len(summary.ChangeURIs)) + for i, event := range events { + uri := summary.ChangeURIs[i] + require.NoError(t, basehook.Validate(event)) + assert.Equal(t, string(tt.wantType), event.GetType()) + assert.Equal(t, tt.version, event.GetVersion()) + assert.Equal(t, summary.StatusTimestampMs, event.GetTimestampMs()) + assert.Equal(t, uri, event.GetPayload().GetFields()["change_uri"].GetStringValue()) + assert.Equal(t, basehook.NewEventID(Source, string(tt.wantType), changeSubject(summary, uri), tt.version), event.GetId()) + } + }) + } +} + +func TestChangeSubject(t *testing.T) { + tests := []struct { + name string + a, b entity.RequestSummary + aURI string + bURI string + }{ + { + name: "same id in different queues", + a: entity.RequestSummary{RequestID: "1", Queue: "queue-a"}, + b: entity.RequestSummary{RequestID: "1", Queue: "queue-b"}, + aURI: "uri/1", bURI: "uri/1", + }, + { + name: "boundary between queue and id moves", + a: entity.RequestSummary{RequestID: "23", Queue: "q1"}, + b: entity.RequestSummary{RequestID: "3", Queue: "q12"}, + aURI: "uri/1", bURI: "uri/1", + }, + { + name: "different changes of one request", + a: entity.RequestSummary{RequestID: "1", Queue: "q"}, + b: entity.RequestSummary{RequestID: "1", Queue: "q"}, + aURI: "uri/1", bURI: "uri/2", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.NotEqual(t, changeSubject(tt.a, tt.aURI), changeSubject(tt.b, tt.bURI)) + assert.NotContains(t, changeSubject(tt.a, tt.aURI), "/") + }) + } +} + +func eventIDs(events []*basehook.HookEvent) []string { + ids := make([]string, 0, len(events)) + for _, event := range events { + ids = append(ids, event.GetId()) + } + return ids +} diff --git a/submitqueue/gateway/controller/BUILD.bazel b/submitqueue/gateway/controller/BUILD.bazel index 3281e0c2c..d0d88da7c 100644 --- a/submitqueue/gateway/controller/BUILD.bazel +++ b/submitqueue/gateway/controller/BUILD.bazel @@ -51,6 +51,7 @@ go_test( ], embed = [":go_default_library"], deps = [ + "//api/base/hook:go_default_library", "//api/submitqueue/gateway/protopb:go_default_library", "//platform/base/change:go_default_library", "//platform/base/mergestrategy:go_default_library", diff --git a/submitqueue/gateway/controller/land_test.go b/submitqueue/gateway/controller/land_test.go index 6f31bf913..b9172d6bf 100644 --- a/submitqueue/gateway/controller/land_test.go +++ b/submitqueue/gateway/controller/land_test.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/uber-go/tally" + basehook "github.com/uber/submitqueue/api/base/hook" "github.com/uber/submitqueue/platform/base/change" "github.com/uber/submitqueue/platform/base/mergestrategy" entityqueue "github.com/uber/submitqueue/platform/base/messagequeue" @@ -573,6 +574,37 @@ func TestLand_ReturnsErrorWhenPublishFails(t *testing.T) { require.Error(t, err) } +func TestLand_ReceiptHookPublishFailureLeavesTheLandIntact(t *testing.T) { + ctrl := gomock.NewController(t) + cnt := countermock.NewMockCounter(ctrl) + cnt.EXPECT().Next(gomock.Any(), gomock.Any()).Return(int64(999), nil) + fixture := newControllerStorageFixture(ctrl) + publisher := queuemock.NewMockPublisher(ctrl) + publisher.EXPECT().Publish(gomock.Any(), "start", gomock.Any()).Return(nil) + publisher.EXPECT().Publish(gomock.Any(), "hook", gomock.Any()).Return(fmt.Errorf("queue unavailable")) + queue := queuemock.NewMockQueue(ctrl) + queue.EXPECT().Publisher().Return(publisher).AnyTimes() + registry, err := consumer.NewTopicRegistry([]consumer.TopicConfig{ + {Key: topickey.TopicKeyStart, Name: "start", Queue: queue}, + {Key: basehook.TopicKeyHook, Name: "hook", Queue: queue}, + }) + require.NoError(t, err) + scope := tally.NewTestScope("", nil) + materializer := requestcore.NewMaterializer(fixture.newFactory(ctrl), requestcore.WithMetricsScope(scope), requestcore.WithHookPublishing(registry)) + controller := NewLandController(zap.NewNop().Sugar(), scope, staticCounterFactory{counter: cnt}, fixture.newFactory(ctrl), materializer, noopQueueConfigStore(ctrl), queuepausenoop.New(), registry) + + result, err := controller.Land(context.Background(), testLandRequest("test-queue")) + + require.NoError(t, err) + assert.Equal(t, "999", result.ID) + assert.Equal(t, entity.RequestStatusAccepted, fixture.summaries["999"].Status) + counters := scope.Snapshot().Counters() + assert.Contains(t, counters, "request_hook.publish_errors+event_type=request.received,queue=test-queue") + for key := range counters { + assert.NotContains(t, key, "accepted_log_failure", "the accepted log was written; only the hook event was lost") + } +} + func TestLand_ReturnsSqidWhenAcceptedLogFailsAfterPublish(t *testing.T) { ctrl := gomock.NewController(t) cnt := countermock.NewMockCounter(ctrl) diff --git a/submitqueue/gateway/core/request/BUILD.bazel b/submitqueue/gateway/core/request/BUILD.bazel index 4776a6963..e5e3428db 100644 --- a/submitqueue/gateway/core/request/BUILD.bazel +++ b/submitqueue/gateway/core/request/BUILD.bazel @@ -10,7 +10,11 @@ go_library( importpath = "github.com/uber/submitqueue/submitqueue/gateway/core/request", visibility = ["//visibility:public"], deps = [ + "//api/base/hook:go_default_library", + "//platform/consumer:go_default_library", + "//platform/hook:go_default_library", "//platform/metrics:go_default_library", + "//submitqueue/core/hookevent:go_default_library", "//submitqueue/entity:go_default_library", "//submitqueue/extension/storage:go_default_library", "//submitqueue/gateway/extension/storage:go_default_library", @@ -27,6 +31,11 @@ go_test( ], embed = [":go_default_library"], deps = [ + "//api/base/hook:go_default_library", + "//platform/base/messagequeue:go_default_library", + "//platform/consumer:go_default_library", + "//platform/extension/messagequeue/mock:go_default_library", + "//submitqueue/core/hookevent:go_default_library", "//submitqueue/entity:go_default_library", "//submitqueue/extension/storage:go_default_library", "//submitqueue/extension/storage/mock:go_default_library", diff --git a/submitqueue/gateway/core/request/materializer.go b/submitqueue/gateway/core/request/materializer.go index 273bb68b4..f2dbd35ec 100644 --- a/submitqueue/gateway/core/request/materializer.go +++ b/submitqueue/gateway/core/request/materializer.go @@ -22,7 +22,11 @@ import ( "time" "github.com/uber-go/tally" + basehook "github.com/uber/submitqueue/api/base/hook" + "github.com/uber/submitqueue/platform/consumer" + platformhook "github.com/uber/submitqueue/platform/hook" "github.com/uber/submitqueue/platform/metrics" + "github.com/uber/submitqueue/submitqueue/core/hookevent" "github.com/uber/submitqueue/submitqueue/entity" basestorage "github.com/uber/submitqueue/submitqueue/extension/storage" gwstorage "github.com/uber/submitqueue/submitqueue/gateway/extension/storage" @@ -33,8 +37,10 @@ import ( // Every store it touches is queue-scoped, so each call resolves the aggregate once // from the queue carried on the log being persisted. type Materializer struct { - stores gwstorage.Factory - metricsScope tally.Scope + stores gwstorage.Factory + metricsScope tally.Scope + hookRegistry consumer.TopicRegistry + publishesHooks bool } // MaterializerOption configures a Materializer. @@ -46,6 +52,15 @@ func WithMetricsScope(scope tally.Scope) MaterializerOption { return func(m *Materializer) { m.metricsScope = scope } } +// WithHookPublishing publishes request hook events (see submitqueue/core/hookevent) +// to registry's hook topic. Without it the materializer publishes none. +func WithHookPublishing(registry consumer.TopicRegistry) MaterializerOption { + return func(m *Materializer) { + m.hookRegistry = registry + m.publishesHooks = true + } +} + // NewMaterializer creates a request read-model materializer. func NewMaterializer(stores gwstorage.Factory, opts ...MaterializerOption) *Materializer { m := &Materializer{stores: stores, metricsScope: tally.NoopScope} @@ -76,6 +91,7 @@ func (m *Materializer) PersistLog(ctx context.Context, log entity.RequestLog) er return fmt.Errorf("failed to get request summary request_id=%s: %w", log.RequestID, err) } + beforeUpdate, updatedSummary := summary, false if logWins(log, summary) { oldVersion := summary.Version newVersion := oldVersion + 1 @@ -94,7 +110,7 @@ func (m *Materializer) PersistLog(ctx context.Context, log entity.RequestLog) er } updated.Version = newVersion m.recordLifecycleTransition(summary, updated) - summary = updated + summary, updatedSummary = updated, true } if summary.Status == entity.RequestStatusAccepting { @@ -106,7 +122,7 @@ func (m *Materializer) PersistLog(ctx context.Context, log entity.RequestLog) er if err := ensureRequestReceiptMapping(ctx, stores.GetRequestReceiptStore(), summary); err != nil { return err } - return nil + return m.publishRequestHookEvents(ctx, log, beforeUpdate, summary, updatedSummary) } } @@ -138,6 +154,50 @@ func (m *Materializer) recordLifecycleTransition(before, after entity.RequestSum queueTag, statusTag).RecordDuration(elapsed) } +// publishRequestHookEvents announces what log did to the request: the receipt +// when this delivery's update took the summary out of accepting, and the finish +// whenever log restates a finished summary. +// +// The receipt is announced once. Nothing redelivers the accepting step, so a +// failed receipt publish is only counted: returning it would retry nothing, and +// on Land it would read as a failed accepted log. The finish is announced on +// every delivery of its log, so a failed publish is returned and the log's +// redelivery repairs it. +func (m *Materializer) publishRequestHookEvents(ctx context.Context, log entity.RequestLog, before, after entity.RequestSummary, updated bool) error { + if !m.publishesHooks { + return nil + } + if updated && before.Status == entity.RequestStatusAccepting && after.Status != entity.RequestStatusAccepting { + for _, event := range hookevent.NewRequestReceived(after) { + _ = m.publishRequestHookEvent(ctx, after, event) + } + } + if !logRestatesSummary(log, after) { + return nil + } + for _, event := range hookevent.NewRequestFinished(after) { + if err := m.publishRequestHookEvent(ctx, after, event); err != nil { + return err + } + } + return nil +} + +func (m *Materializer) publishRequestHookEvent(ctx context.Context, summary entity.RequestSummary, event *basehook.HookEvent) error { + if err := platformhook.Publish(ctx, m.hookRegistry, summary.Queue, event, summary.RequestID); err != nil { + metrics.NamedCounter(m.metricsScope, "request_hook", "publish_errors", 1, + metrics.NewTag("queue", summary.Queue), metrics.NewTag("event_type", event.GetType())) + return fmt.Errorf("failed to publish %s hook event request_id=%s: %w", event.GetType(), summary.RequestID, err) + } + return nil +} + +// logRestatesSummary reports whether log carries the status and request version +// summary shows, whether log just set them or repeats them. +func logRestatesSummary(log entity.RequestLog, summary entity.RequestSummary) bool { + return log.Type == entity.RequestLogTypeStatus && log.Status == summary.Status && log.RequestVersion == summary.RequestVersion +} + func isTerminalStatus(status entity.RequestStatus) bool { return entity.IsRequestStateTerminal(entity.RequestState(status)) } diff --git a/submitqueue/gateway/core/request/materializer_test.go b/submitqueue/gateway/core/request/materializer_test.go index e35ccab36..e2d9333d9 100644 --- a/submitqueue/gateway/core/request/materializer_test.go +++ b/submitqueue/gateway/core/request/materializer_test.go @@ -21,6 +21,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + basehook "github.com/uber/submitqueue/api/base/hook" + entityqueue "github.com/uber/submitqueue/platform/base/messagequeue" + "github.com/uber/submitqueue/platform/consumer" + queuemock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" + "github.com/uber/submitqueue/submitqueue/core/hookevent" "github.com/uber/submitqueue/submitqueue/entity" "github.com/uber/submitqueue/submitqueue/extension/storage" "go.uber.org/mock/gomock" @@ -162,6 +167,246 @@ func TestMaterializer_RecordsLifecycleMetrics(t *testing.T) { } } +func TestMaterializer_PublishesRequestHookEvents(t *testing.T) { + accepting := testRequestSummary() + accepted := accepting + accepted.Status, accepted.StatusTimestampMs, accepted.Version = entity.RequestStatusAccepted, 15, 2 + started := accepted + started.Status, started.RequestVersion, started.StatusTimestampMs, started.Version = entity.RequestStatusStarted, 1, 16, 3 + landed := started + landed.Status, landed.RequestVersion, landed.StatusTimestampMs, landed.Version = entity.RequestStatusLanded, 2, 20, 4 + statusLog := func(status entity.RequestStatus, version int32, timestampMs int64) entity.RequestLog { + return entity.RequestLog{Queue: "q", RequestID: "1", Type: entity.RequestLogTypeStatus, Status: status, RequestVersion: version, TimestampMs: timestampMs} + } + type published struct { + eventType hookevent.Type + version int32 + changeURI string + } + perChange := func(eventType hookevent.Type, version int32) []published { + return []published{{eventType, version, "uri/1"}, {eventType, version, "uri/2"}} + } + + tests := []struct { + name string + before entity.RequestSummary + log entity.RequestLog + wantUpdate bool + wantPublished []published + }{ + { + name: "accepted log announces the receipt for every change", + before: accepting, + log: statusLog(entity.RequestStatusAccepted, 0, 15), + wantUpdate: true, + wantPublished: perChange(hookevent.TypeRequestReceived, 0), + }, + { + name: "a later status standing in for a lost accepted log still announces the receipt", + before: accepting, + log: statusLog(entity.RequestStatusStarted, 1, 16), + wantUpdate: true, + wantPublished: perChange(hookevent.TypeRequestReceived, 0), + }, + { + name: "landed log announces the finish", + before: started, + log: statusLog(entity.RequestStatusLanded, 2, 20), + wantUpdate: true, + wantPublished: perChange(hookevent.TypeRequestLanded, 2), + }, + { + name: "failure before the pipeline saved the request announces every receipt, then every finish", + before: accepting, + log: statusLog(entity.RequestStatusError, 0, 20), + wantUpdate: true, + wantPublished: append(perChange(hookevent.TypeRequestReceived, 0), perChange(hookevent.TypeRequestFailed, 0)...), + }, + { + name: "a later finished version announces again", + before: landed, + log: statusLog(entity.RequestStatusError, 3, 30), + wantUpdate: true, + wantPublished: perChange(hookevent.TypeRequestFailed, 3), + }, + { + name: "a redelivered finished log announces the finish again, repairing a failed publish", + before: landed, + log: statusLog(entity.RequestStatusLanded, 2, 20), + wantPublished: perChange(hookevent.TypeRequestLanded, 2), + }, + { + name: "a resent finished log announces the finish again", + before: landed, + log: statusLog(entity.RequestStatusLanded, 2, 30), + wantUpdate: true, + wantPublished: perChange(hookevent.TypeRequestLanded, 2), + }, + { + name: "cancellation has no event", + before: started, + log: statusLog(entity.RequestStatusCancelled, 2, 20), + wantUpdate: true, + }, + { + name: "an unfinished status after the receipt has no event", + before: accepted, + log: statusLog(entity.RequestStatusStarted, 1, 16), + wantUpdate: true, + }, + { + name: "a stale log after the finish has no event", + before: landed, + log: statusLog(entity.RequestStatusStarted, 1, 16), + }, + { + name: "a lower finished version after the finish has no event", + before: landed, + log: statusLog(entity.RequestStatusError, 1, 40), + }, + { + name: "a redelivered accepted log has no event, even if its first publish failed", + before: accepted, + log: statusLog(entity.RequestStatusAccepted, 0, 15), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + var messages []entityqueue.Message + registry := newHookRegistry(t, ctrl, func(msg entityqueue.Message) error { + messages = append(messages, msg) + return nil + }) + f := newMaterializerReceiptFixture(ctrl, WithHookPublishing(registry)) + f.logs.EXPECT().Insert(gomock.Any(), tt.log).Return(nil) + f.summaries.EXPECT().Get(gomock.Any(), tt.before.RequestID).Return(tt.before, nil) + if tt.wantUpdate { + f.summaries.EXPECT().Update(gomock.Any(), gomock.Any(), tt.before.Version, tt.before.Version+1).Return(nil) + } + f.uris.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + f.receipts.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + + require.NoError(t, f.materializer.PersistLog(context.Background(), tt.log)) + + got := []published{} + for _, msg := range messages { + event := &basehook.HookEvent{} + require.NoError(t, basehook.Unmarshal(msg.Payload, event)) + assert.Equal(t, event.GetId(), msg.ID, "the event id is the queue dedupe key") + assert.Equal(t, tt.before.RequestID, msg.PartitionKey) + assert.Equal(t, tt.before.Queue, msg.Tenant) + got = append(got, published{hookevent.Type(event.GetType()), event.GetVersion(), event.GetPayload().GetFields()["change_uri"].GetStringValue()}) + } + if tt.wantPublished == nil { + tt.wantPublished = []published{} + } + assert.Equal(t, tt.wantPublished, got) + }) + } +} + +func TestMaterializer_HookPublishFailures(t *testing.T) { + accepting := testRequestSummary() + started := accepting + started.Status, started.RequestVersion, started.StatusTimestampMs, started.Version = entity.RequestStatusStarted, 1, 16, 3 + + tests := []struct { + name string + before entity.RequestSummary + log entity.RequestLog + wantErr bool + wantCounter string + wantCount int64 + }{ + { + name: "a failed receipt is counted but does not fail the log, which could not repeat it", + before: accepting, + log: entity.RequestLog{Queue: "q", RequestID: "1", Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusAccepted, TimestampMs: 20}, + wantCounter: "request_hook.publish_errors+event_type=request.received,queue=q", + wantCount: 2, + }, + { + name: "a failed finish fails the log, whose redelivery announces it again", + before: started, + log: entity.RequestLog{Queue: "q", RequestID: "1", Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusLanded, RequestVersion: 2, TimestampMs: 20}, + wantErr: true, + wantCounter: "request_hook.publish_errors+event_type=request.landed,queue=q", + wantCount: 1, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + publishErr := errors.New("queue unavailable") + registry := newHookRegistry(t, ctrl, func(entityqueue.Message) error { return publishErr }) + f := newMaterializerReceiptFixture(ctrl, WithHookPublishing(registry)) + f.logs.EXPECT().Insert(gomock.Any(), tt.log).Return(nil) + f.summaries.EXPECT().Get(gomock.Any(), tt.before.RequestID).Return(tt.before, nil) + f.summaries.EXPECT().Update(gomock.Any(), gomock.Any(), tt.before.Version, tt.before.Version+1).Return(nil) + f.uris.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil).Times(len(tt.before.ChangeURIs)) + f.receipts.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil) + + err := f.materializer.PersistLog(context.Background(), tt.log) + + if tt.wantErr { + require.ErrorIs(t, err, publishErr) + } else { + require.NoError(t, err) + } + counter, ok := f.metrics.Snapshot().Counters()[tt.wantCounter] + require.True(t, ok) + assert.Equal(t, tt.wantCount, counter.Value()) + }) + } +} + +func TestMaterializer_PublishesTheWinningTransitionAcrossAVersionConflict(t *testing.T) { + ctrl := gomock.NewController(t) + var published []string + registry := newHookRegistry(t, ctrl, func(msg entityqueue.Message) error { + published = append(published, msg.ID) + return nil + }) + f := newMaterializerReceiptFixture(ctrl, WithHookPublishing(registry)) + current := testRequestSummary() + concurrent := current + concurrent.Status, concurrent.RequestVersion, concurrent.StatusTimestampMs, concurrent.Version = entity.RequestStatusStarted, 1, 15, 2 + log := entity.RequestLog{Queue: current.Queue, RequestID: current.RequestID, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusLanded, RequestVersion: 2, TimestampMs: 20} + gomock.InOrder( + f.logs.EXPECT().Insert(gomock.Any(), log).Return(nil), + f.summaries.EXPECT().Get(gomock.Any(), current.RequestID).Return(current, nil), + f.summaries.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).Return(storage.ErrVersionMismatch), + f.summaries.EXPECT().Get(gomock.Any(), current.RequestID).Return(concurrent, nil), + f.summaries.EXPECT().Update(gomock.Any(), gomock.Any(), int32(2), int32(3)).Return(nil), + ) + f.uris.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + f.receipts.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + + require.NoError(t, f.materializer.PersistLog(context.Background(), log)) + + landed := concurrent + landed.Status, landed.RequestVersion = entity.RequestStatusLanded, 2 + want := []string{} + for _, event := range hookevent.NewRequestFinished(landed) { + want = append(want, event.GetId()) + } + assert.Equal(t, want, published, "the concurrent writer already announced the receipt; this update only finishes the request") +} + +// newHookRegistry binds the hook topic to a publisher that hands every message to publish. +func newHookRegistry(t *testing.T, ctrl *gomock.Controller, publish func(entityqueue.Message) error) consumer.TopicRegistry { + publisher := queuemock.NewMockPublisher(ctrl) + publisher.EXPECT().Publish(gomock.Any(), "hook", gomock.Any()).DoAndReturn( + func(_ context.Context, _ string, msg entityqueue.Message) error { return publish(msg) }, + ).AnyTimes() + queue := queuemock.NewMockQueue(ctrl) + queue.EXPECT().Publisher().Return(publisher).AnyTimes() + registry, err := consumer.NewTopicRegistry([]consumer.TopicConfig{{Key: basehook.TopicKeyHook, Name: "hook", Queue: queue}}) + require.NoError(t, err) + return registry +} + func TestMaterializer_PersistenceFailures(t *testing.T) { writeErr := errors.New("storage unavailable") tests := []struct { diff --git a/submitqueue/gateway/core/request/request_receipt_test.go b/submitqueue/gateway/core/request/request_receipt_test.go index a35dd05ec..2d4e00053 100644 --- a/submitqueue/gateway/core/request/request_receipt_test.go +++ b/submitqueue/gateway/core/request/request_receipt_test.go @@ -37,7 +37,7 @@ type materializerReceiptFixture struct { metrics tally.TestScope } -func newMaterializerReceiptFixture(ctrl *gomock.Controller) materializerReceiptFixture { +func newMaterializerReceiptFixture(ctrl *gomock.Controller, opts ...MaterializerOption) materializerReceiptFixture { f := materializerReceiptFixture{ summaries: storagemock.NewMockRequestSummaryStore(ctrl), uris: storagemock.NewMockRequestURIStore(ctrl), logs: storagemock.NewMockRequestLogStore(ctrl), @@ -51,7 +51,7 @@ func newMaterializerReceiptFixture(ctrl *gomock.Controller) materializerReceiptF stores.EXPECT().GetRequestReceiptStore().Return(f.receipts).AnyTimes() factory := gwstoragemock.NewMockFactory(ctrl) factory.EXPECT().For(gomock.Any()).Return(stores, nil).AnyTimes() - f.materializer = NewMaterializer(factory, WithMetricsScope(f.metrics)) + f.materializer = NewMaterializer(factory, append([]MaterializerOption{WithMetricsScope(f.metrics)}, opts...)...) return f } From 417fb9b01bcda96d66a425f033e95cc828f8cb03 Mon Sep 17 00:00:00 2001 From: Chenghan Ying <125171961+roychying@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:51:53 -0800 Subject: [PATCH 2/2] feat(submitqueue): Publish request hook events from the gateway to the orchestrator's hook stage --- doc/rfc/hook-framework.md | 3 ++- doc/rfc/submitqueue/workflow.md | 4 ++-- .../submitqueue/gateway/server/BUILD.bazel | 4 ++++ .../gateway/server/log_consumer.go | 3 +++ .../gateway/server/log_consumer_test.go | 22 +++++++++++++++++++ service/submitqueue/gateway/server/main.go | 4 ++-- submitqueue/gateway/README.md | 4 ++++ 7 files changed, 39 insertions(+), 5 deletions(-) diff --git a/doc/rfc/hook-framework.md b/doc/rfc/hook-framework.md index 186add8d2..3d655f2ae 100644 --- a/doc/rfc/hook-framework.md +++ b/doc/rfc/hook-framework.md @@ -4,7 +4,7 @@ Fire-and-forget side effects for pipeline lifecycle events: one shared event con ## Status -Implemented. The contract is `api/base/hook`, the dispatcher and DLQ reconciler are `platform/hook`, and the extension is `platform/extension/hook`. Stovepipe's `process` and `record` publish repository-scoped hook events, and `service/stovepipe/server` registers the stage. The SubmitQueue orchestrator pipeline registers a `submitqueue-hook` stage; no orchestrator controller publishes a hook event, and `service/submitqueue/orchestrator/server` resolves every event to noop. A deployment's real integrations are whatever its resolver returns. +Implemented. The contract is `api/base/hook`, the dispatcher and DLQ reconciler are `platform/hook`, and the extension is `platform/extension/hook`. Stovepipe's `process` and `record` publish repository-scoped hook events, and `service/stovepipe/server` registers the stage. The SubmitQueue orchestrator pipeline registers a `submitqueue-hook` stage. The SubmitQueue gateway's request materializer publishes `request.received`, `request.landed`, and `request.failed` (`submitqueue/core/hookevent`) to it; no orchestrator controller publishes a hook event, and `service/submitqueue/orchestrator/server` resolves every event to noop. A deployment's real integrations are whatever its resolver returns. ## Problem @@ -52,6 +52,7 @@ Delivery promise: - Partition key = subject id; per-subject order only. - Replay finding the target state already written → republish (idempotent). Beyond it → superseded; the event may be lost. Named, accepted gap. - RPC-caused transitions are at-most-once; needing the guarantee means publishing from the first queue-driven stage. +- SubmitQueue request events follow the gateway's request summary, the status users see, rather than pipeline transitions, so a hook agrees with the UI even for a request whose start failed before the orchestrator saved it. Each status announces one event per change, so a hook retrying one change's failure never repeats another's. A finish is announced on every delivery of a log that restates it, which keeps the replay rule above: a failed publish fails the delivery and its redelivery republishes. `request.received` is published inside `Land`, which makes it RPC-caused and at-most-once; a later status standing in for a lost `accepted` log announces it instead. The events cross from the gateway to the orchestrator's hook stage, so their payload carries the change's URI, which never changes. - Opt-in per host via topic-key registration; a registered host never skips, so off and loss are distinguishable. ### Contract diff --git a/doc/rfc/submitqueue/workflow.md b/doc/rfc/submitqueue/workflow.md index a050b2357..2a40b16e8 100644 --- a/doc/rfc/submitqueue/workflow.md +++ b/doc/rfc/submitqueue/workflow.md @@ -6,7 +6,7 @@ Two cycles re-enter speculation. `speculate` dispatches funded paths to `build`; Terminal request states are not `conclude`'s alone. `cancel` completes a request that has not been enrolled in a batch. `landconflictsignal` fails a request Runway reports as conflicted. `conclude` maps a terminal batch onto its member requests. The DLQ reconcilers force a failed terminal state when a consumed stage cannot finish. -The same `Stages` table registers a `submitqueue-hook` stage, outside the order of the flow above. It consumes hook events and invokes the wired hooks, and it publishes nothing onward. No orchestrator controller publishes a hook event; `service/submitqueue/orchestrator/server` resolves every event to noop. The comment on that row in `pipeline.go` is the contract: any stage can publish there. +The same `Stages` table registers a `submitqueue-hook` stage, outside the order of the flow above. It consumes hook events and invokes the wired hooks, and it publishes nothing onward. The gateway's request materializer publishes request events there (see the [gateway README](../../../submitqueue/gateway/README.md)); no orchestrator controller does, and `service/submitqueue/orchestrator/server` resolves every event to noop. The comment on that row in `pipeline.go` is the contract: any stage can publish there. ## Diagram @@ -131,7 +131,7 @@ class node_storagecontract,node_queuestorage,node_requestlog,node_messagequeue t class node_submitter toneIndigo ``` -`hook` is drawn with the other stages because the topology registers it. Nothing in the orchestrator publishes to it, so the diagram has no edge into that node. +`hook` is drawn with the other stages because the topology registers it. Only the gateway's request materializer publishes to it, so the diagram has no edge into that node from an orchestrator stage. ## Per-controller summary diff --git a/service/submitqueue/gateway/server/BUILD.bazel b/service/submitqueue/gateway/server/BUILD.bazel index 5e1709930..cb2682eb6 100644 --- a/service/submitqueue/gateway/server/BUILD.bazel +++ b/service/submitqueue/gateway/server/BUILD.bazel @@ -19,6 +19,7 @@ go_library( importpath = "github.com/uber/submitqueue/service/submitqueue/gateway/server", visibility = ["//visibility:private"], deps = [ + "//api/base/hook:go_default_library", "//api/submitqueue/gateway/protopb:go_default_library", "//platform/consumer:go_default_library", "//platform/errs:go_default_library", @@ -93,6 +94,7 @@ go_test( ], embed = [":gateway_lib"], # keep deps = [ + "//api/base/hook:go_default_library", "//api/submitqueue/gateway/protopb:go_default_library", "//platform/base/failure:go_default_library", "//platform/base/messagequeue:go_default_library", @@ -103,12 +105,14 @@ go_test( "//platform/extension/messagequeue/mock:go_default_library", "//platform/lifecycle:go_default_library", "//submitqueue/core/messagequeue:go_default_library", + "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", "//submitqueue/extension/queueconfig/mock:go_default_library", "//submitqueue/gateway/controller:go_default_library", "//submitqueue/gateway/core/request:go_default_library", "//submitqueue/gateway/extension/storage:go_default_library", "//submitqueue/gateway/extension/storage/mock:go_default_library", + "//submitqueue/orchestrator:go_default_library", "@com_github_stretchr_testify//assert:go_default_library", "@com_github_stretchr_testify//require:go_default_library", "@com_github_uber_go_tally//:go_default_library", diff --git a/service/submitqueue/gateway/server/log_consumer.go b/service/submitqueue/gateway/server/log_consumer.go index b52e23a68..263af1a36 100644 --- a/service/submitqueue/gateway/server/log_consumer.go +++ b/service/submitqueue/gateway/server/log_consumer.go @@ -20,6 +20,7 @@ import ( "time" "github.com/uber-go/tally" + basehook "github.com/uber/submitqueue/api/base/hook" "github.com/uber/submitqueue/platform/consumer" "github.com/uber/submitqueue/platform/errs" genericerrs "github.com/uber/submitqueue/platform/errs/generic" @@ -47,6 +48,8 @@ func newGatewayTopicRegistry(queue extqueue.Queue, subscriberName string) (consu {Key: topickey.TopicKeyStart, Name: "start", Queue: queue}, {Key: topickey.TopicKeyCancel, Name: "cancel", Queue: queue}, {Key: topickey.TopicKeyLog, Name: "log", Queue: queue, Subscription: primary}, + // Publish-only: the orchestrator pipeline's hook stage consumes it under this name. + {Key: basehook.TopicKeyHook, Name: "submitqueue-hook", Queue: queue}, { Key: logDLQTopicKey, Name: "log" + logDLQSuffix, Queue: queue, Subscription: extqueue.DLQSubscriptionConfig(subscriberName, logDLQConsumerGroup), diff --git a/service/submitqueue/gateway/server/log_consumer_test.go b/service/submitqueue/gateway/server/log_consumer_test.go index ad52e957e..e54869839 100644 --- a/service/submitqueue/gateway/server/log_consumer_test.go +++ b/service/submitqueue/gateway/server/log_consumer_test.go @@ -22,6 +22,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/uber-go/tally" + basehook "github.com/uber/submitqueue/api/base/hook" "github.com/uber/submitqueue/platform/base/failure" entityqueue "github.com/uber/submitqueue/platform/base/messagequeue" "github.com/uber/submitqueue/platform/consumer" @@ -30,10 +31,12 @@ import ( queuemock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" "github.com/uber/submitqueue/platform/lifecycle" sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" + "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" requestcore "github.com/uber/submitqueue/submitqueue/gateway/core/request" gwstorage "github.com/uber/submitqueue/submitqueue/gateway/extension/storage" gwstoragemock "github.com/uber/submitqueue/submitqueue/gateway/extension/storage/mock" + "github.com/uber/submitqueue/submitqueue/orchestrator" "go.uber.org/mock/gomock" "go.uber.org/zap" "google.golang.org/grpc" @@ -112,6 +115,25 @@ func TestLogConsumers_RetainFailedRecovery(t *testing.T) { } } +func TestGatewayTopicRegistry_MatchesOrchestratorStages(t *testing.T) { + registry, err := newGatewayTopicRegistry(queuemock.NewMockQueue(gomock.NewController(t)), "worker") + require.NoError(t, err) + stageNames := map[consumer.TopicKey]string{} + for _, stage := range orchestrator.Stages { + stageNames[stage.Key] = stage.Name + } + + for _, key := range []consumer.TopicKey{topickey.TopicKeyStart, topickey.TopicKeyCancel, basehook.TopicKeyHook} { + t.Run(string(key), func(t *testing.T) { + want, ok := stageNames[key] + require.True(t, ok, "the orchestrator consumes every topic the gateway publishes to") + got, ok := registry.TopicName(key) + require.True(t, ok) + assert.Equal(t, want, got) + }) + } +} + type lifecycleTestConsumer struct { consumer.Consumer name string diff --git a/service/submitqueue/gateway/server/main.go b/service/submitqueue/gateway/server/main.go index eca086794..43ce33fd8 100644 --- a/service/submitqueue/gateway/server/main.go +++ b/service/submitqueue/gateway/server/main.go @@ -334,7 +334,7 @@ func run() error { // Land creates receipt projections, request-summary and List controllers // read materialized views, and request-history controllers read retained // logs. Normal log-topic persistence and terminal DLQ repair both use the - // materializer. + // materializer, which also publishes request hook events. store, err := mysqlstorage.NewStorage(appDB, scope.SubScope("storage")) if err != nil { return fmt.Errorf("failed to create storage: %w", err) @@ -343,7 +343,7 @@ func run() error { // Create controllers and wrap them for gRPC. Every store is queue-scoped and // resolves through the factory adapter; land/cancel/log share one materializer. storageFty := storageFactory{backend: store} - materializer := requestcore.NewMaterializer(storageFty, requestcore.WithMetricsScope(scope)) + materializer := requestcore.NewMaterializer(storageFty, requestcore.WithMetricsScope(scope), requestcore.WithHookPublishing(registry)) pingController := controller.NewPingController(logger, scope) landController := controller.NewLandController(logger.Sugar(), scope, cnt, storageFty, materializer, queueConfigs, queuePauses, registry) cancelController := controller.NewCancelController(logger.Sugar(), scope, storageFty, materializer, registry) diff --git a/submitqueue/gateway/README.md b/submitqueue/gateway/README.md index 24eefced5..259d7f9f9 100644 --- a/submitqueue/gateway/README.md +++ b/submitqueue/gateway/README.md @@ -25,6 +25,10 @@ The gateway owns the request log read model and is the only service that reads i The materializer appends every audit event, selects the current authoritative winner, and repairs the queue projection. The normal orchestrator pipeline does not read or write the request-log store directly. +## Request hook events + +The materializer publishes SubmitQueue's request hook events (`submitqueue/core/hookevent`) to the `submitqueue-hook` topic, which the orchestrator pipeline's hook stage consumes, one event per change of the request. The update that takes the summary out of `accepting` announces `request.received`; that happens inside `Land`, so a failed receipt publish is only counted on `request_hook.publish_errors`. Every delivery of a log that restates a `landed` or `error` summary announces `request.landed` or `request.failed`, so a failed publish is returned and the log's redelivery repairs it; a copy that arrives after the queue has forgotten the event id reaches hooks twice. Events therefore follow the summary users see, including a request whose start failed before the orchestrator saved it, and a stale or unfinished log publishes nothing. + ## Final log recovery The primary `gateway-log` consumer sends failed deliveries to `log_dlq` after a non-retryable error or exhaustion of its finite retry budget. The gateway's `gateway-log-dlq` consumer replays retained entries through the same log controller and materializer, with unlimited attempts, capped backoff, and no second-level DLQ. Valid log deliveries are acknowledged only after history, summary, URI mappings, and receipt projection succeed. Replays may append equivalent history rows; terminal-summary precedence and mapping repair are unchanged.