Two control planes send a notification once

Each kept its own hour of quiet per event, so a budget spent on a team
both control planes served was announced by each. An event is now claimed
in the store before it is sent (notify_sent, migration 0043; one statement,
so two asking at once cannot both win), and only the claimer sends it. A
store that cannot be asked sends it anyway.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-10 13:36:08 +02:00
1 parent 8d7b96621e
commit 39bb8ee557
9 files changed
+179 -2

No files matched your search

+36
View File
@@ -110,6 +110,23 @@ type Notifier struct {
mu sync.Mutex
sent map[string]time.Time
wg sync.WaitGroup
claim Claimer
}
// Claimer is where control planes agree which of them sends an event: each
// raises the same ones — a budget is spent on every control plane serving
// its requests — and only the one that claims an event sends it.
type Claimer interface {
ClaimNotice(ctx context.Context, key string, at time.Time, window time.Duration) (bool, error)
}
// SetClaimer has events claimed before they are sent. Set before the first
// event; nil leaves each process to send its own.
func (n *Notifier) SetClaimer(c Claimer) {
if n != nil {
n.claim = c
}
}
type hook struct {
@@ -199,6 +216,9 @@ func (n *Notifier) run(ctx context.Context) {
if !ok {
return
}
if !n.claimed(ctx, e) {
continue
}
for _, h := range n.hooks {
if len(h.Events) > 0 && !slices.Contains(h.Events, e.Kind) {
continue
@@ -295,3 +315,19 @@ func (n *Notifier) send(ctx context.Context, h hook, e Event, attempts int) erro
}
}
}
// claimed reports whether this process is the one to send an event. A store
// that cannot be asked sends it anyway: told twice beats not told.
func (n *Notifier) claimed(ctx context.Context, e Event) bool {
if n.claim == nil {
return true
}
cctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
ok, err := n.claim.ClaimNotice(cctx, e.Kind+"\x00"+e.Subject, e.At, dedupe)
if err != nil {
n.log.Warn("could not ask the store whether a notification was sent; sending it", "kind", e.Kind, "err", err)
return true
}
return ok
}
+55
View File
@@ -6,6 +6,7 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
@@ -167,3 +168,57 @@ func TestTestSaysHowEachAnsweredWithoutTheSecretPath(t *testing.T) {
}
}
}
// sharedClaims is a store two control planes share.
type sharedClaims struct {
mu sync.Mutex
last map[string]time.Time
err error
}
func (s *sharedClaims) ClaimNotice(_ context.Context, key string, at time.Time, window time.Duration) (bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.err != nil {
return false, s.err
}
if prev, ok := s.last[key]; ok && at.Sub(prev) < window {
return false, nil
}
s.last[key] = at
return true, nil
}
func TestTwoControlPlanesSendAnEventOnce(t *testing.T) {
// Each control plane serving a team's requests sees its budget spent,
// and each kept its own hour of quiet: two behind a balancer sent every
// budget event twice.
var g got
srv := httptest.NewServer(g.handler(0))
defer srv.Close()
claims := &sharedClaims{last: map[string]time.Time{}}
a := newNotifier(t, Config{Host: "ctl1", Webhooks: []Webhook{{URL: srv.URL}}})
b := newNotifier(t, Config{Host: "ctl2", Webhooks: []Webhook{{URL: srv.URL}}})
a.SetClaimer(claims)
b.SetClaimer(claims)
a.Notify(BudgetSpent, "team:t1", "spent", nil)
b.Notify(BudgetSpent, "team:t1", "spent", nil)
b.Notify(BudgetSpent, "team:t2", "spent", nil)
a.Wait()
b.Wait()
if len(g.calls) != 2 {
t.Errorf("%d sent, want one for each team", len(g.calls))
}
// A store that cannot be asked: sent, rather than lost.
var g2 got
srv2 := httptest.NewServer(g2.handler(0))
defer srv2.Close()
c := newNotifier(t, Config{Webhooks: []Webhook{{URL: srv2.URL}}})
c.SetClaimer(&sharedClaims{err: errors.New("store down")})
c.Notify(NodeLost, "node3", "lost", nil)
c.Wait()
if len(g2.calls) != 1 {
t.Errorf("with the store down, %d sent, want 1", len(g2.calls))
}
}