Two control planes send a notification once

Each kept its own hour of quiet per event, so a budget spent on a team
both control planes served was announced by each. An event is now claimed
in the store before it is sent (notify_sent, migration 0043; one statement,
so two asking at once cannot both win), and only the claimer sends it. A
store that cannot be asked sends it anyway.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-10 13:36:08 +02:00
1 parent 8d7b96621e
commit 39bb8ee557
9 files changed
+179 -2

No files matched your search

+3 -2
View File
@@ -404,8 +404,9 @@ portal is not done — the page shows; the console and ctl change.
lost, budget spent, update available; signed JSON or Slack; retries,
hourly de-duplication), `ctl notify test`, Alertmanager receiver
templates. Left: `server init` and the console do not yet ask for a
receiver — the templates are filled in by hand; each control plane sends
budget events on its own, so two behind a balancer may both send one.
receiver — the templates are filled in by hand. Events are claimed in
the store before they are sent (2026-10-10), so two control planes send
each once.
**Alerting out of the box (S).** `deploy/monitoring/alertmanager.yml`
ships `receiver: none`; add Slack/email/webhook/PagerDuty receiver
templates driven by `server init` or the console; llmgw-native webhooks
+4
View File
@@ -39,6 +39,10 @@ APIs change between minors without a deprecation period.
so with two control planes a callback that reached the other one was
refused. The key is now a secret in the store, the same on every control
plane.
- **Notifications sent once by two control planes.** Each kept its own
hour of quiet per event, so a budget spent on a team both served was
announced twice. The control plane that claims an event in the store
(migration 0043) sends it; with the store unreachable it is sent anyway.
- The usage sums and exports count pictures (`images`), as the request
records did; pictures made before are added to the sums.
+4
View File
@@ -210,6 +210,10 @@ func newServerCmd() *cobra.Command {
if err != nil {
return err
}
if st := rt.Store(); st != nil {
// Each control plane raises the same events; one sends each.
notifier.SetClaimer(st)
}
rt.SetNotifier(notifier)
if gate != nil && notifier != nil {
gate.SetJudgeAlarm(func(model string, err error) {
+36
View File
@@ -110,6 +110,23 @@ type Notifier struct {
mu sync.Mutex
sent map[string]time.Time
wg sync.WaitGroup
claim Claimer
}
// Claimer is where control planes agree which of them sends an event: each
// raises the same ones — a budget is spent on every control plane serving
// its requests — and only the one that claims an event sends it.
type Claimer interface {
ClaimNotice(ctx context.Context, key string, at time.Time, window time.Duration) (bool, error)
}
// SetClaimer has events claimed before they are sent. Set before the first
// event; nil leaves each process to send its own.
func (n *Notifier) SetClaimer(c Claimer) {
if n != nil {
n.claim = c
}
}
type hook struct {
@@ -199,6 +216,9 @@ func (n *Notifier) run(ctx context.Context) {
if !ok {
return
}
if !n.claimed(ctx, e) {
continue
}
for _, h := range n.hooks {
if len(h.Events) > 0 && !slices.Contains(h.Events, e.Kind) {
continue
@@ -295,3 +315,19 @@ func (n *Notifier) send(ctx context.Context, h hook, e Event, attempts int) erro
}
}
}
// claimed reports whether this process is the one to send an event. A store
// that cannot be asked sends it anyway: told twice beats not told.
func (n *Notifier) claimed(ctx context.Context, e Event) bool {
if n.claim == nil {
return true
}
cctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
ok, err := n.claim.ClaimNotice(cctx, e.Kind+"\x00"+e.Subject, e.At, dedupe)
if err != nil {
n.log.Warn("could not ask the store whether a notification was sent; sending it", "kind", e.Kind, "err", err)
return true
}
return ok
}
+55
View File
@@ -6,6 +6,7 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
@@ -167,3 +168,57 @@ func TestTestSaysHowEachAnsweredWithoutTheSecretPath(t *testing.T) {
}
}
}
// sharedClaims is a store two control planes share.
type sharedClaims struct {
mu sync.Mutex
last map[string]time.Time
err error
}
func (s *sharedClaims) ClaimNotice(_ context.Context, key string, at time.Time, window time.Duration) (bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.err != nil {
return false, s.err
}
if prev, ok := s.last[key]; ok && at.Sub(prev) < window {
return false, nil
}
s.last[key] = at
return true, nil
}
func TestTwoControlPlanesSendAnEventOnce(t *testing.T) {
// Each control plane serving a team's requests sees its budget spent,
// and each kept its own hour of quiet: two behind a balancer sent every
// budget event twice.
var g got
srv := httptest.NewServer(g.handler(0))
defer srv.Close()
claims := &sharedClaims{last: map[string]time.Time{}}
a := newNotifier(t, Config{Host: "ctl1", Webhooks: []Webhook{{URL: srv.URL}}})
b := newNotifier(t, Config{Host: "ctl2", Webhooks: []Webhook{{URL: srv.URL}}})
a.SetClaimer(claims)
b.SetClaimer(claims)
a.Notify(BudgetSpent, "team:t1", "spent", nil)
b.Notify(BudgetSpent, "team:t1", "spent", nil)
b.Notify(BudgetSpent, "team:t2", "spent", nil)
a.Wait()
b.Wait()
if len(g.calls) != 2 {
t.Errorf("%d sent, want one for each team", len(g.calls))
}
// A store that cannot be asked: sent, rather than lost.
var g2 got
srv2 := httptest.NewServer(g2.handler(0))
defer srv2.Close()
c := newNotifier(t, Config{Webhooks: []Webhook{{URL: srv2.URL}}})
c.SetClaimer(&sharedClaims{err: errors.New("store down")})
c.Notify(NodeLost, "node3", "lost", nil)
c.Wait()
if len(g2.calls) != 1 {
t.Errorf("with the store down, %d sent, want 1", len(g2.calls))
}
}
@@ -0,0 +1,8 @@
-- Ported from the SQLite migration of the same number.
-- When each notification was last sent, by its kind and subject: every
-- control plane raises the same events — a budget spent is seen by each one
-- serving its requests — and the one that claims an event here sends it.
CREATE TABLE notify_sent (
key TEXT PRIMARY KEY,
sent_at BIGINT NOT NULL
);
@@ -0,0 +1,7 @@
-- When each notification was last sent, by its kind and subject: every
-- control plane raises the same events — a budget spent is seen by each one
-- serving its requests — and the one that claims an event here sends it.
CREATE TABLE notify_sent (
key TEXT PRIMARY KEY,
sent_at BIGINT NOT NULL
);
+23
View File
@@ -0,0 +1,23 @@
package store
import (
"context"
"time"
)
// ClaimNotice claims the sending of a notification: true when nobody has
// sent the same one within the window, and from now on, nobody else may
// until it has passed. One statement, so two control planes asking at once
// cannot both be told yes.
func (s *DB) ClaimNotice(ctx context.Context, key string, at time.Time, window time.Duration) (bool, error) {
now := at.UnixMilli()
res, err := s.db.ExecContext(ctx,
`INSERT INTO notify_sent (key, sent_at) VALUES (?, ?)
ON CONFLICT (key) DO UPDATE SET sent_at = excluded.sent_at WHERE notify_sent.sent_at < ?`,
key, now, now-window.Milliseconds())
if err != nil {
return false, err
}
n, err := res.RowsAffected()
return n == 1, err
}
+39
View File
@@ -0,0 +1,39 @@
package store
import (
"sync"
"sync/atomic"
"testing"
"time"
)
func TestANoticeIsClaimedByOneOfThoseAskingAtOnce(t *testing.T) {
db := capacityStore(t)
at := time.Date(2026, 10, 10, 9, 0, 0, 0, time.UTC)
var yes atomic.Int64
var wg sync.WaitGroup
for range 8 {
wg.Go(func() {
ok, err := db.ClaimNotice(t.Context(), "budget_spent\x00team:t1", at, time.Hour)
if err != nil {
t.Error(err)
}
if ok {
yes.Add(1)
}
})
}
wg.Wait()
if yes.Load() != 1 {
t.Fatalf("%d of eight claimed the same notice", yes.Load())
}
if ok, _ := db.ClaimNotice(t.Context(), "budget_spent\x00team:t1", at.Add(59*time.Minute), time.Hour); ok {
t.Error("claimed again within the hour")
}
if ok, _ := db.ClaimNotice(t.Context(), "budget_spent\x00team:t2", at, time.Hour); !ok {
t.Error("another subject could not be claimed")
}
if ok, _ := db.ClaimNotice(t.Context(), "budget_spent\x00team:t1", at.Add(61*time.Minute), time.Hour); !ok {
t.Error("not claimable once the hour had passed")
}
}