diff --git a/VERSION b/VERSION index 66c4c22..7ee7020 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.9 +1.0.10 diff --git a/docs/metrics.md b/docs/metrics.md index 2c49a04..945f9af 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -169,13 +169,12 @@ Other useful starting conditions: `/healthz` checks liveness. Paid orchestrator `/readyz` requires writable payment storage, chain access, an active recipient, valid pricing, and worker completion. -Checks run at heartbeat cadence even with empty queues. Payment chain metadata -refreshes every five seconds. An observation is refused at thirty seconds old, -if the head timestamp is stale, or after thirty seconds without protocol L1 -progress. A successful RPC response with an unchanged head does not renew its -age. One provider cannot prove absolute L1 lag. Other dependency checks and -worker completion expire after `max(60 seconds, 3 × HeartbeatInterval)`; -queue stalls and individual ticket errors remain separate signals. +Checks run at heartbeat cadence even with empty queues. Chain data refreshes +every five seconds. On Arbitrum, L2 observations expire after 30 seconds and L1 +height must advance within 60 seconds. One provider cannot prove absolute L1 +lag. Other dependency checks and worker completion expire after +`max(60 seconds, 3 × HeartbeatInterval)`. Queue stalls and individual ticket +errors remain separate signals. Off-chain orchestrator/discovery readiness reports listener health. Signer readiness requires chain access, positive deposit/total reserve, no withdrawal through the next round, fresh pricing, and healthy enabled outbox diff --git a/eth/payment_observer.go b/eth/payment_observer.go index fea3b3b..f0884c3 100644 --- a/eth/payment_observer.go +++ b/eth/payment_observer.go @@ -19,6 +19,9 @@ import ( var ErrObservationUnavailable = errors.New("chain observation unavailable") +// Arbitrum's L1 height can stay flat across many fresh L2 heads. +const l1ProgressMaxAge = 60 * time.Second + // PaymentObserver publishes complete canonical observations. Gaps and reorgs // rebuild state at the new hash instead of replaying uncertain event history. type PaymentObserver struct { @@ -31,6 +34,7 @@ type PaymentObserver struct { header *miniHeader generation uint64 requireL1 bool + startedOnL1 bool contact, progress time.Time l1Progress time.Time l1HighWater *big.Int @@ -162,6 +166,10 @@ func (o *PaymentObserver) validate(s ChainSnapshot) error { func (o *PaymentObserver) viewRejection(s ChainSnapshot, l1Progress time.Time) string { now := o.now() + progressMaxAge := o.maxAge + if o.startedOnL1 { + progressMaxAge = l1ProgressMaxAge + } switch { case o.lifecycle != nil && o.lifecycle.Err() != nil: return "chain_observer_stopped" @@ -173,7 +181,7 @@ func (o *PaymentObserver) viewRejection(s ChainSnapshot, l1Progress time.Time) s return "chain_view_stale" case now.Sub(s.blockTime) >= o.maxAge: return "chain_head_stale" - case now.Sub(l1Progress) >= o.maxAge: + case now.Sub(l1Progress) >= progressMaxAge: return "chain_l1_progress_stale" default: return "" @@ -278,6 +286,10 @@ func (o *PaymentObserver) refresh(ctx context.Context) error { if o.viewRejection(s, l1Progress) != "" { return ErrObservationUnavailable } + if prior == nil { + // Tie the deadline to the clock that established the progress baseline. + o.startedOnL1 = head.L1BlockNumber != nil + } for key, entry := range o.payers { if _, affected := changed[key.payer]; reset || affected { entry.version++ diff --git a/eth/payment_observer_test.go b/eth/payment_observer_test.go index 8d66378..efb2044 100644 --- a/eth/payment_observer_test.go +++ b/eth/payment_observer_test.go @@ -169,16 +169,24 @@ func TestObserverRejectedHeadersInvalidateConflictingIdentity(t *testing.T) { } func TestObserverL1ProgressDeadline(t *testing.T) { - for _, clocks := range [][]uint64{{50, 50, 50}, {49, 50, 49}} { + elapsed := []time.Duration{10, 20, 30, 40, 50, 59, 60} + for _, tc := range []struct { + requireL1 bool + clocks [7]uint64 + }{ + {true, [7]uint64{50, 50, 50, 50, 50, 50, 50}}, + {false, [7]uint64{49, 50, 49, 50, 49, 50, 49}}, + } { f, o, now := newTestObserver(t) + o.requireL1 = tc.requireL1 require.NoError(t, o.refresh(t.Context())) progress, accepted := o.Status().L1ProgressAt, o.Status().ObservedAt - for step, clock := range clocks { - *now = now.Add(10 * time.Second) + for step, clock := range tc.clocks { + *now = progress.Add(elapsed[step] * time.Second) advanceObserver(f, *now) f.Update(func(s *test.ChainState) { s.L1Block = clock }) err := o.refresh(t.Context()) - if step < 2 { + if step < len(tc.clocks)-1 { require.NoError(t, err) accepted = o.Status().ObservedAt } else { @@ -196,6 +204,34 @@ func TestObserverL1ProgressDeadline(t *testing.T) { } } +func TestObserverNativeProgressDeadline(t *testing.T) { + for _, omitL1 := range []bool{true, false} { + f, o, now := newTestObserver(t) + o.requireL1 = false + f.Update(func(s *test.ChainState) { s.Number, s.OmitL1 = 100, true }) + require.NoError(t, o.refresh(t.Context())) + + *now = now.Add(29 * time.Second) + f.Update(func(s *test.ChainState) { + s.Hash, s.Timestamp = common.HexToHash("0xbeef"), uint64(now.Unix()) + s.L1Block, s.OmitL1 = 50, omitL1 + }) + require.NoError(t, o.refresh(t.Context())) + require.True(t, o.Status().Healthy) + + *now = now.Add(time.Second) + require.ErrorIs(t, o.refresh(t.Context()), ErrObservationUnavailable) + require.Equal(t, "chain_l1_progress_stale", o.Status().RecipientRejection) + + if omitL1 { + *now = now.Add(5 * time.Second) + advanceObserver(f, *now) + f.Update(func(s *test.ChainState) { s.L1Block, s.OmitL1 = 50, false }) + require.ErrorIs(t, o.refresh(t.Context()), ErrObservationUnavailable) + } + } +} + func TestObserverExpiredCandidateIsNotPublished(t *testing.T) { for _, phase := range []string{"RoundsManager.currentRound", "eth_getLogs"} { t.Run(phase, func(t *testing.T) {