From 39bb8ee55735ac2d7a7ac917573567f45fb88c1c Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sat, 10 Oct 2026 13:36:08 +0200 Subject: [PATCH] Two control planes send a notification once Each kept its own hour of quiet per event, so a budget spent on a team both control planes served was announced by each. An event is now claimed in the store before it is sent (notify_sent, migration 0043; one statement, so two asking at once cannot both win), and only the claimer sends it. A store that cannot be asked sends it anyway. Co-Authored-By: Claude Opus 5.5 (1M context) --- BACKLOG.md | 5 +- CHANGELOG.md | 4 ++ internal/cli/server.go | 4 ++ internal/notify/notify.go | 36 ++++++++++++ internal/notify/notify_test.go | 55 +++++++++++++++++++ .../migrations/postgres/0043_notify_sent.sql | 8 +++ .../migrations/sqlite/0043_notify_sent.sql | 7 +++ internal/store/notify.go | 23 ++++++++ internal/store/notify_test.go | 39 +++++++++++++ 9 files changed, 179 insertions(+), 2 deletions(-) create mode 100644 internal/store/migrations/postgres/0043_notify_sent.sql create mode 100644 internal/store/migrations/sqlite/0043_notify_sent.sql create mode 100644 internal/store/notify.go create mode 100644 internal/store/notify_test.go diff --git a/BACKLOG.md b/BACKLOG.md index 1239d981..b3860089 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -404,8 +404,9 @@ portal is not done — the page shows; the console and ctl change. lost, budget spent, update available; signed JSON or Slack; retries, hourly de-duplication), `ctl notify test`, Alertmanager receiver templates. Left: `server init` and the console do not yet ask for a - receiver — the templates are filled in by hand; each control plane sends - budget events on its own, so two behind a balancer may both send one. + receiver — the templates are filled in by hand. Events are claimed in + the store before they are sent (2026-10-10), so two control planes send + each once. **Alerting out of the box (S).** `deploy/monitoring/alertmanager.yml` ships `receiver: none`; add Slack/email/webhook/PagerDuty receiver templates driven by `server init` or the console; llmgw-native webhooks diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d7f7e50..99d6d8ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,10 @@ APIs change between minors without a deprecation period. so with two control planes a callback that reached the other one was refused. The key is now a secret in the store, the same on every control plane. +- **Notifications sent once by two control planes.** Each kept its own + hour of quiet per event, so a budget spent on a team both served was + announced twice. The control plane that claims an event in the store + (migration 0043) sends it; with the store unreachable it is sent anyway. - The usage sums and exports count pictures (`images`), as the request records did; pictures made before are added to the sums. diff --git a/internal/cli/server.go b/internal/cli/server.go index 97895692..78cc8863 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -210,6 +210,10 @@ func newServerCmd() *cobra.Command { if err != nil { return err } + if st := rt.Store(); st != nil { + // Each control plane raises the same events; one sends each. + notifier.SetClaimer(st) + } rt.SetNotifier(notifier) if gate != nil && notifier != nil { gate.SetJudgeAlarm(func(model string, err error) { diff --git a/internal/notify/notify.go b/internal/notify/notify.go index 72c148bb..444d4789 100644 --- a/internal/notify/notify.go +++ b/internal/notify/notify.go @@ -110,6 +110,23 @@ type Notifier struct { mu sync.Mutex sent map[string]time.Time wg sync.WaitGroup + + claim Claimer +} + +// Claimer is where control planes agree which of them sends an event: each +// raises the same ones — a budget is spent on every control plane serving +// its requests — and only the one that claims an event sends it. +type Claimer interface { + ClaimNotice(ctx context.Context, key string, at time.Time, window time.Duration) (bool, error) +} + +// SetClaimer has events claimed before they are sent. Set before the first +// event; nil leaves each process to send its own. +func (n *Notifier) SetClaimer(c Claimer) { + if n != nil { + n.claim = c + } } type hook struct { @@ -199,6 +216,9 @@ func (n *Notifier) run(ctx context.Context) { if !ok { return } + if !n.claimed(ctx, e) { + continue + } for _, h := range n.hooks { if len(h.Events) > 0 && !slices.Contains(h.Events, e.Kind) { continue @@ -295,3 +315,19 @@ func (n *Notifier) send(ctx context.Context, h hook, e Event, attempts int) erro } } } + +// claimed reports whether this process is the one to send an event. A store +// that cannot be asked sends it anyway: told twice beats not told. +func (n *Notifier) claimed(ctx context.Context, e Event) bool { + if n.claim == nil { + return true + } + cctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + ok, err := n.claim.ClaimNotice(cctx, e.Kind+"\x00"+e.Subject, e.At, dedupe) + if err != nil { + n.log.Warn("could not ask the store whether a notification was sent; sending it", "kind", e.Kind, "err", err) + return true + } + return ok +} diff --git a/internal/notify/notify_test.go b/internal/notify/notify_test.go index d87d2c6c..371a5d5d 100644 --- a/internal/notify/notify_test.go +++ b/internal/notify/notify_test.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "io" "net/http" "net/http/httptest" @@ -167,3 +168,57 @@ func TestTestSaysHowEachAnsweredWithoutTheSecretPath(t *testing.T) { } } } + +// sharedClaims is a store two control planes share. +type sharedClaims struct { + mu sync.Mutex + last map[string]time.Time + err error +} + +func (s *sharedClaims) ClaimNotice(_ context.Context, key string, at time.Time, window time.Duration) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.err != nil { + return false, s.err + } + if prev, ok := s.last[key]; ok && at.Sub(prev) < window { + return false, nil + } + s.last[key] = at + return true, nil +} + +func TestTwoControlPlanesSendAnEventOnce(t *testing.T) { + // Each control plane serving a team's requests sees its budget spent, + // and each kept its own hour of quiet: two behind a balancer sent every + // budget event twice. + var g got + srv := httptest.NewServer(g.handler(0)) + defer srv.Close() + claims := &sharedClaims{last: map[string]time.Time{}} + a := newNotifier(t, Config{Host: "ctl1", Webhooks: []Webhook{{URL: srv.URL}}}) + b := newNotifier(t, Config{Host: "ctl2", Webhooks: []Webhook{{URL: srv.URL}}}) + a.SetClaimer(claims) + b.SetClaimer(claims) + a.Notify(BudgetSpent, "team:t1", "spent", nil) + b.Notify(BudgetSpent, "team:t1", "spent", nil) + b.Notify(BudgetSpent, "team:t2", "spent", nil) + a.Wait() + b.Wait() + if len(g.calls) != 2 { + t.Errorf("%d sent, want one for each team", len(g.calls)) + } + + // A store that cannot be asked: sent, rather than lost. + var g2 got + srv2 := httptest.NewServer(g2.handler(0)) + defer srv2.Close() + c := newNotifier(t, Config{Webhooks: []Webhook{{URL: srv2.URL}}}) + c.SetClaimer(&sharedClaims{err: errors.New("store down")}) + c.Notify(NodeLost, "node3", "lost", nil) + c.Wait() + if len(g2.calls) != 1 { + t.Errorf("with the store down, %d sent, want 1", len(g2.calls)) + } +} diff --git a/internal/store/migrations/postgres/0043_notify_sent.sql b/internal/store/migrations/postgres/0043_notify_sent.sql new file mode 100644 index 00000000..f2a3f2d5 --- /dev/null +++ b/internal/store/migrations/postgres/0043_notify_sent.sql @@ -0,0 +1,8 @@ +-- Ported from the SQLite migration of the same number. +-- When each notification was last sent, by its kind and subject: every +-- control plane raises the same events — a budget spent is seen by each one +-- serving its requests — and the one that claims an event here sends it. +CREATE TABLE notify_sent ( + key TEXT PRIMARY KEY, + sent_at BIGINT NOT NULL +); diff --git a/internal/store/migrations/sqlite/0043_notify_sent.sql b/internal/store/migrations/sqlite/0043_notify_sent.sql new file mode 100644 index 00000000..890c0e08 --- /dev/null +++ b/internal/store/migrations/sqlite/0043_notify_sent.sql @@ -0,0 +1,7 @@ +-- When each notification was last sent, by its kind and subject: every +-- control plane raises the same events — a budget spent is seen by each one +-- serving its requests — and the one that claims an event here sends it. +CREATE TABLE notify_sent ( + key TEXT PRIMARY KEY, + sent_at BIGINT NOT NULL +); diff --git a/internal/store/notify.go b/internal/store/notify.go new file mode 100644 index 00000000..974090f9 --- /dev/null +++ b/internal/store/notify.go @@ -0,0 +1,23 @@ +package store + +import ( + "context" + "time" +) + +// ClaimNotice claims the sending of a notification: true when nobody has +// sent the same one within the window, and from now on, nobody else may +// until it has passed. One statement, so two control planes asking at once +// cannot both be told yes. +func (s *DB) ClaimNotice(ctx context.Context, key string, at time.Time, window time.Duration) (bool, error) { + now := at.UnixMilli() + res, err := s.db.ExecContext(ctx, + `INSERT INTO notify_sent (key, sent_at) VALUES (?, ?) + ON CONFLICT (key) DO UPDATE SET sent_at = excluded.sent_at WHERE notify_sent.sent_at < ?`, + key, now, now-window.Milliseconds()) + if err != nil { + return false, err + } + n, err := res.RowsAffected() + return n == 1, err +} diff --git a/internal/store/notify_test.go b/internal/store/notify_test.go new file mode 100644 index 00000000..e84fc29c --- /dev/null +++ b/internal/store/notify_test.go @@ -0,0 +1,39 @@ +package store + +import ( + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestANoticeIsClaimedByOneOfThoseAskingAtOnce(t *testing.T) { + db := capacityStore(t) + at := time.Date(2026, 10, 10, 9, 0, 0, 0, time.UTC) + var yes atomic.Int64 + var wg sync.WaitGroup + for range 8 { + wg.Go(func() { + ok, err := db.ClaimNotice(t.Context(), "budget_spent\x00team:t1", at, time.Hour) + if err != nil { + t.Error(err) + } + if ok { + yes.Add(1) + } + }) + } + wg.Wait() + if yes.Load() != 1 { + t.Fatalf("%d of eight claimed the same notice", yes.Load()) + } + if ok, _ := db.ClaimNotice(t.Context(), "budget_spent\x00team:t1", at.Add(59*time.Minute), time.Hour); ok { + t.Error("claimed again within the hour") + } + if ok, _ := db.ClaimNotice(t.Context(), "budget_spent\x00team:t2", at, time.Hour); !ok { + t.Error("another subject could not be claimed") + } + if ok, _ := db.ClaimNotice(t.Context(), "budget_spent\x00team:t1", at.Add(61*time.Minute), time.Hour); !ok { + t.Error("not claimable once the hour had passed") + } +}