Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions submitqueue/core/hookevent/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
load("@rules_go//go:def.bzl", "go_library", "go_test")

go_library(
name = "go_default_library",
srcs = ["hookevent.go"],
importpath = "github.com/uber/submitqueue/submitqueue/core/hookevent",
visibility = ["//visibility:public"],
deps = [
"//api/base/hook:go_default_library",
"//submitqueue/entity:go_default_library",
"@org_golang_google_protobuf//types/known/structpb:go_default_library",
],
)

go_test(
name = "go_default_test",
srcs = ["hookevent_test.go"],
embed = [":go_default_library"],
deps = [
"//api/base/hook:go_default_library",
"//submitqueue/entity:go_default_library",
"@com_github_stretchr_testify//assert:go_default_library",
"@com_github_stretchr_testify//require:go_default_library",
],
)
126 changes: 126 additions & 0 deletions submitqueue/core/hookevent/hookevent.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
// Copyright (c) 2026 Uber Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package hookevent holds SubmitQueue's hook events: the source it reports, the
// types it publishes, and a constructor per type. Centralizing them keeps the
// wire contract in one place, so a publisher names an event rather than
// assembling one.
//
// The events follow a request's public status, the one its summary shows: one
// when the request is received, and one when it lands or fails. Cancellation
// has no event yet because no hook needs one. Deriving them from the summary
// rather than from pipeline transitions keeps every hook in step with what a
// user sees, including a request whose start failed before the pipeline saved
// it.
//
// Each status announces one event per change of the request, so a hook that
// retries one change's failed side effect never repeats the others'. Payloads
// carry the queue, the request id, and that change's URI. The gateway
// publishes these events and the orchestrator's hook stage consumes them, and a
// hook there cannot read the gateway's store, so the payload carries what a
// hook needs to reach the change. A request's URIs never change, so they cannot
// go stale on redelivery. See doc/rfc/hook-framework.md.
//
// Known gaps, accepted for now:
// - request.received is published inside Land, an RPC. If Land records
// accepted but the publish fails, no later status announces the receipt.
// - A finished status is announced again whenever its log is redelivered or
// resent, so a failed publish repairs itself; a copy that arrives after the
// queue forgot the id reaches hooks twice.
// - Order holds within a request's partition, but a receipt waiting to retry
// does not hold back the finish, so a hook can see the finish first.
// - The queue drops a repeated event id only until the first copy is consumed
// and collected, seconds later.
package hookevent

import (
"crypto/sha256"
"fmt"

basehook "github.com/uber/submitqueue/api/base/hook"
"github.com/uber/submitqueue/submitqueue/entity"
"google.golang.org/protobuf/types/known/structpb"
)

// Source is reported on every hook event SubmitQueue publishes. A consumer
// serving several domains matches on it to recognize SubmitQueue's events.
const Source = "submitqueue"

// Type is the event type a consumer filters on.
type Type string

const (
// TypeRequestReceived announces that the request named in the payload has
// left the internal accepting status and is visible to users. It carries no
// verdict: the request has not been validated yet.
TypeRequestReceived Type = "request.received"
// TypeRequestLanded announces that the request named in the payload landed.
TypeRequestLanded Type = "request.landed"
// TypeRequestFailed announces that the request named in the payload ended in
// error. The request's summary carries the reason.
TypeRequestFailed Type = "request.failed"
)

// finishedTypeByStatus maps each finished status that has an event to its type.
var finishedTypeByStatus = map[entity.RequestStatus]Type{
entity.RequestStatusLanded: TypeRequestLanded,
entity.RequestStatusError: TypeRequestFailed,
}

// NewRequestReceived builds the events announcing that summary's request was
// received, one per change. A request is received once, so the ids carry version
// 0 and every republish for the same request mints the same ids.
func NewRequestReceived(summary entity.RequestSummary) []*basehook.HookEvent {
return newRequestEvents(TypeRequestReceived, summary, 0, summary.ReceivedAtMs)
}

// NewRequestFinished builds the events announcing summary's finished status, one
// per change, and none for a status that has no event. The ids carry
// summary.RequestVersion, so a status the summary restates mints the same ids and
// a later finished status the summary moves to mints new ones.
func NewRequestFinished(summary entity.RequestSummary) []*basehook.HookEvent {
eventType, ok := finishedTypeByStatus[summary.Status]
if !ok {
return nil
}
return newRequestEvents(eventType, summary, summary.RequestVersion, summary.StatusTimestampMs)
}

func newRequestEvents(eventType Type, summary entity.RequestSummary, version int32, timestampMs int64) []*basehook.HookEvent {
events := make([]*basehook.HookEvent, 0, len(summary.ChangeURIs))
for _, uri := range summary.ChangeURIs {
events = append(events, &basehook.HookEvent{
Id: basehook.NewEventID(Source, string(eventType), changeSubject(summary, uri), version),
Source: Source,
Type: string(eventType),
TimestampMs: timestampMs,
Version: version,
Payload: &structpb.Struct{Fields: map[string]*structpb.Value{
"queue": structpb.NewStringValue(summary.Queue),
"request_id": structpb.NewStringValue(summary.RequestID),
"change_uri": structpb.NewStringValue(uri),
}},
})
}
return events
}

// changeSubject identifies one change of a request across queues. Request IDs
// are only unique within a queue, and queue names and URIs contain the id
// separator, so every part is length-prefixed and hashed into one separator-free
// component.
func changeSubject(summary entity.RequestSummary, uri string) string {
identity := fmt.Sprintf("%d:%s%d:%s%d:%s", len(summary.Queue), summary.Queue, len(summary.RequestID), summary.RequestID, len(uri), uri)
return fmt.Sprintf("%x", sha256.Sum256([]byte(identity)))
}
146 changes: 146 additions & 0 deletions submitqueue/core/hookevent/hookevent_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
// Copyright (c) 2026 Uber Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hookevent

import (
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
basehook "github.com/uber/submitqueue/api/base/hook"
"github.com/uber/submitqueue/submitqueue/entity"
)

func TestNewRequestReceived(t *testing.T) {
tests := []struct {
name string
summary entity.RequestSummary
}{
{
name: "one change in a queue with a slash",
summary: entity.RequestSummary{RequestID: "1149", Queue: "acme/monorepo", ChangeURIs: []string{"github://github.com/acme/monorepo/pull/42/abc123"}, Status: entity.RequestStatusAccepted, ReceivedAtMs: 100},
},
{
name: "a stack, received under a later status",
summary: entity.RequestSummary{RequestID: "7", Queue: "demo-queue", ChangeURIs: []string{"uri/1", "uri/2"}, Status: entity.RequestStatusStarted, RequestVersion: 1, ReceivedAtMs: 200},
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
events := NewRequestReceived(tt.summary)

require.Len(t, events, len(tt.summary.ChangeURIs))
for i, event := range events {
uri := tt.summary.ChangeURIs[i]
require.NoError(t, basehook.Validate(event))
assert.Equal(t, Source, event.GetSource())
assert.Equal(t, string(TypeRequestReceived), event.GetType())
assert.Zero(t, event.GetVersion())
assert.Equal(t, tt.summary.ReceivedAtMs, event.GetTimestampMs())
assert.Equal(t, tt.summary.Queue, event.GetPayload().GetFields()["queue"].GetStringValue())
assert.Equal(t, tt.summary.RequestID, event.GetPayload().GetFields()["request_id"].GetStringValue())
assert.Equal(t, uri, event.GetPayload().GetFields()["change_uri"].GetStringValue())
assert.Equal(t, basehook.NewEventID(Source, string(TypeRequestReceived), changeSubject(tt.summary, uri), 0), event.GetId())
}

later := tt.summary
later.Status, later.RequestVersion = entity.RequestStatusLanded, 5
assert.Equal(t, eventIDs(events), eventIDs(NewRequestReceived(later)), "a request is received once, whatever its status now")
})
}
}

func TestNewRequestFinished(t *testing.T) {
tests := []struct {
name string
status entity.RequestStatus
version int32
wantType Type
wantEvent bool
}{
{name: "landed", status: entity.RequestStatusLanded, version: 5, wantType: TypeRequestLanded, wantEvent: true},
{name: "failed", status: entity.RequestStatusError, version: 3, wantType: TypeRequestFailed, wantEvent: true},
{name: "failed before the pipeline saved the request", status: entity.RequestStatusError, version: 0, wantType: TypeRequestFailed, wantEvent: true},
{name: "cancelled has no event", status: entity.RequestStatusCancelled, version: 4},
{name: "unfinished status", status: entity.RequestStatusBatched, version: 2},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
summary := entity.RequestSummary{RequestID: "1149", Queue: "acme/monorepo", ChangeURIs: []string{"uri/1", "uri/2"}, Status: tt.status, RequestVersion: tt.version, StatusTimestampMs: 300}

events := NewRequestFinished(summary)

if !tt.wantEvent {
assert.Empty(t, events)
return
}
require.Len(t, events, len(summary.ChangeURIs))
for i, event := range events {
uri := summary.ChangeURIs[i]
require.NoError(t, basehook.Validate(event))
assert.Equal(t, string(tt.wantType), event.GetType())
assert.Equal(t, tt.version, event.GetVersion())
assert.Equal(t, summary.StatusTimestampMs, event.GetTimestampMs())
assert.Equal(t, uri, event.GetPayload().GetFields()["change_uri"].GetStringValue())
assert.Equal(t, basehook.NewEventID(Source, string(tt.wantType), changeSubject(summary, uri), tt.version), event.GetId())
}
})
}
}

func TestChangeSubject(t *testing.T) {
tests := []struct {
name string
a, b entity.RequestSummary
aURI string
bURI string
}{
{
name: "same id in different queues",
a: entity.RequestSummary{RequestID: "1", Queue: "queue-a"},
b: entity.RequestSummary{RequestID: "1", Queue: "queue-b"},
aURI: "uri/1", bURI: "uri/1",
},
{
name: "boundary between queue and id moves",
a: entity.RequestSummary{RequestID: "23", Queue: "q1"},
b: entity.RequestSummary{RequestID: "3", Queue: "q12"},
aURI: "uri/1", bURI: "uri/1",
},
{
name: "different changes of one request",
a: entity.RequestSummary{RequestID: "1", Queue: "q"},
b: entity.RequestSummary{RequestID: "1", Queue: "q"},
aURI: "uri/1", bURI: "uri/2",
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.NotEqual(t, changeSubject(tt.a, tt.aURI), changeSubject(tt.b, tt.bURI))
assert.NotContains(t, changeSubject(tt.a, tt.aURI), "/")
})
}
}

func eventIDs(events []*basehook.HookEvent) []string {
ids := make([]string, 0, len(events))
for _, event := range events {
ids = append(ids, event.GetId())
}
return ids
}
1 change: 1 addition & 0 deletions submitqueue/gateway/controller/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ go_test(
],
embed = [":go_default_library"],
deps = [
"//api/base/hook:go_default_library",
"//api/submitqueue/gateway/protopb:go_default_library",
"//platform/base/change:go_default_library",
"//platform/base/mergestrategy:go_default_library",
Expand Down
32 changes: 32 additions & 0 deletions submitqueue/gateway/controller/land_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/uber-go/tally"
basehook "github.com/uber/submitqueue/api/base/hook"
"github.com/uber/submitqueue/platform/base/change"
"github.com/uber/submitqueue/platform/base/mergestrategy"
entityqueue "github.com/uber/submitqueue/platform/base/messagequeue"
Expand Down Expand Up @@ -573,6 +574,37 @@ func TestLand_ReturnsErrorWhenPublishFails(t *testing.T) {
require.Error(t, err)
}

func TestLand_ReceiptHookPublishFailureLeavesTheLandIntact(t *testing.T) {
ctrl := gomock.NewController(t)
cnt := countermock.NewMockCounter(ctrl)
cnt.EXPECT().Next(gomock.Any(), gomock.Any()).Return(int64(999), nil)
fixture := newControllerStorageFixture(ctrl)
publisher := queuemock.NewMockPublisher(ctrl)
publisher.EXPECT().Publish(gomock.Any(), "start", gomock.Any()).Return(nil)
publisher.EXPECT().Publish(gomock.Any(), "hook", gomock.Any()).Return(fmt.Errorf("queue unavailable"))
queue := queuemock.NewMockQueue(ctrl)
queue.EXPECT().Publisher().Return(publisher).AnyTimes()
registry, err := consumer.NewTopicRegistry([]consumer.TopicConfig{
{Key: topickey.TopicKeyStart, Name: "start", Queue: queue},
{Key: basehook.TopicKeyHook, Name: "hook", Queue: queue},
})
require.NoError(t, err)
scope := tally.NewTestScope("", nil)
materializer := requestcore.NewMaterializer(fixture.newFactory(ctrl), requestcore.WithMetricsScope(scope), requestcore.WithHookPublishing(registry))
controller := NewLandController(zap.NewNop().Sugar(), scope, staticCounterFactory{counter: cnt}, fixture.newFactory(ctrl), materializer, noopQueueConfigStore(ctrl), queuepausenoop.New(), registry)

result, err := controller.Land(context.Background(), testLandRequest("test-queue"))

require.NoError(t, err)
assert.Equal(t, "999", result.ID)
assert.Equal(t, entity.RequestStatusAccepted, fixture.summaries["999"].Status)
counters := scope.Snapshot().Counters()
assert.Contains(t, counters, "request_hook.publish_errors+event_type=request.received,queue=test-queue")
for key := range counters {
assert.NotContains(t, key, "accepted_log_failure", "the accepted log was written; only the hook event was lost")
}
}

func TestLand_ReturnsSqidWhenAcceptedLogFailsAfterPublish(t *testing.T) {
ctrl := gomock.NewController(t)
cnt := countermock.NewMockCounter(ctrl)
Expand Down
9 changes: 9 additions & 0 deletions submitqueue/gateway/core/request/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,11 @@ go_library(
importpath = "github.com/uber/submitqueue/submitqueue/gateway/core/request",
visibility = ["//visibility:public"],
deps = [
"//api/base/hook:go_default_library",
"//platform/consumer:go_default_library",
"//platform/hook:go_default_library",
"//platform/metrics:go_default_library",
"//submitqueue/core/hookevent:go_default_library",
"//submitqueue/entity:go_default_library",
"//submitqueue/extension/storage:go_default_library",
"//submitqueue/gateway/extension/storage:go_default_library",
Expand All @@ -27,6 +31,11 @@ go_test(
],
embed = [":go_default_library"],
deps = [
"//api/base/hook:go_default_library",
"//platform/base/messagequeue:go_default_library",
"//platform/consumer:go_default_library",
"//platform/extension/messagequeue/mock:go_default_library",
"//submitqueue/core/hookevent:go_default_library",
"//submitqueue/entity:go_default_library",
"//submitqueue/extension/storage:go_default_library",
"//submitqueue/extension/storage/mock:go_default_library",
Expand Down
Loading
Loading