Two control planes send a notification once
Each kept its own hour of quiet per event, so a budget spent on a team both control planes served was announced by each. An event is now claimed in the store before it is sent (notify_sent, migration 0043; one statement, so two asking at once cannot both win), and only the claimer sends it. A store that cannot be asked sends it anyway. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
8d7b96621e
commit
39bb8ee557
9 files changed
+179
-2
No files matched your search
+3
-2
@@ -404,8 +404,9 @@ portal is not done — the page shows; the console and ctl change.
|
||||
lost, budget spent, update available; signed JSON or Slack; retries,
|
||||
hourly de-duplication), `ctl notify test`, Alertmanager receiver
|
||||
templates. Left: `server init` and the console do not yet ask for a
|
||||
receiver — the templates are filled in by hand; each control plane sends
|
||||
budget events on its own, so two behind a balancer may both send one.
|
||||
receiver — the templates are filled in by hand. Events are claimed in
|
||||
the store before they are sent (2026-10-10), so two control planes send
|
||||
each once.
|
||||
**Alerting out of the box (S).** `deploy/monitoring/alertmanager.yml`
|
||||
ships `receiver: none`; add Slack/email/webhook/PagerDuty receiver
|
||||
templates driven by `server init` or the console; llmgw-native webhooks
|
||||
|
||||
@@ -39,6 +39,10 @@ APIs change between minors without a deprecation period.
|
||||
so with two control planes a callback that reached the other one was
|
||||
refused. The key is now a secret in the store, the same on every control
|
||||
plane.
|
||||
- **Notifications sent once by two control planes.** Each kept its own
|
||||
hour of quiet per event, so a budget spent on a team both served was
|
||||
announced twice. The control plane that claims an event in the store
|
||||
(migration 0043) sends it; with the store unreachable it is sent anyway.
|
||||
- The usage sums and exports count pictures (`images`), as the request
|
||||
records did; pictures made before are added to the sums.
|
||||
|
||||
|
||||
@@ -210,6 +210,10 @@ func newServerCmd() *cobra.Command {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if st := rt.Store(); st != nil {
|
||||
// Each control plane raises the same events; one sends each.
|
||||
notifier.SetClaimer(st)
|
||||
}
|
||||
rt.SetNotifier(notifier)
|
||||
if gate != nil && notifier != nil {
|
||||
gate.SetJudgeAlarm(func(model string, err error) {
|
||||
|
||||
@@ -110,6 +110,23 @@ type Notifier struct {
|
||||
mu sync.Mutex
|
||||
sent map[string]time.Time
|
||||
wg sync.WaitGroup
|
||||
|
||||
claim Claimer
|
||||
}
|
||||
|
||||
// Claimer is where control planes agree which of them sends an event: each
|
||||
// raises the same ones — a budget is spent on every control plane serving
|
||||
// its requests — and only the one that claims an event sends it.
|
||||
type Claimer interface {
|
||||
ClaimNotice(ctx context.Context, key string, at time.Time, window time.Duration) (bool, error)
|
||||
}
|
||||
|
||||
// SetClaimer has events claimed before they are sent. Set before the first
|
||||
// event; nil leaves each process to send its own.
|
||||
func (n *Notifier) SetClaimer(c Claimer) {
|
||||
if n != nil {
|
||||
n.claim = c
|
||||
}
|
||||
}
|
||||
|
||||
type hook struct {
|
||||
@@ -199,6 +216,9 @@ func (n *Notifier) run(ctx context.Context) {
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if !n.claimed(ctx, e) {
|
||||
continue
|
||||
}
|
||||
for _, h := range n.hooks {
|
||||
if len(h.Events) > 0 && !slices.Contains(h.Events, e.Kind) {
|
||||
continue
|
||||
@@ -295,3 +315,19 @@ func (n *Notifier) send(ctx context.Context, h hook, e Event, attempts int) erro
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// claimed reports whether this process is the one to send an event. A store
|
||||
// that cannot be asked sends it anyway: told twice beats not told.
|
||||
func (n *Notifier) claimed(ctx context.Context, e Event) bool {
|
||||
if n.claim == nil {
|
||||
return true
|
||||
}
|
||||
cctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
ok, err := n.claim.ClaimNotice(cctx, e.Kind+"\x00"+e.Subject, e.At, dedupe)
|
||||
if err != nil {
|
||||
n.log.Warn("could not ask the store whether a notification was sent; sending it", "kind", e.Kind, "err", err)
|
||||
return true
|
||||
}
|
||||
return ok
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -167,3 +168,57 @@ func TestTestSaysHowEachAnsweredWithoutTheSecretPath(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sharedClaims is a store two control planes share.
|
||||
type sharedClaims struct {
|
||||
mu sync.Mutex
|
||||
last map[string]time.Time
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *sharedClaims) ClaimNotice(_ context.Context, key string, at time.Time, window time.Duration) (bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.err != nil {
|
||||
return false, s.err
|
||||
}
|
||||
if prev, ok := s.last[key]; ok && at.Sub(prev) < window {
|
||||
return false, nil
|
||||
}
|
||||
s.last[key] = at
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func TestTwoControlPlanesSendAnEventOnce(t *testing.T) {
|
||||
// Each control plane serving a team's requests sees its budget spent,
|
||||
// and each kept its own hour of quiet: two behind a balancer sent every
|
||||
// budget event twice.
|
||||
var g got
|
||||
srv := httptest.NewServer(g.handler(0))
|
||||
defer srv.Close()
|
||||
claims := &sharedClaims{last: map[string]time.Time{}}
|
||||
a := newNotifier(t, Config{Host: "ctl1", Webhooks: []Webhook{{URL: srv.URL}}})
|
||||
b := newNotifier(t, Config{Host: "ctl2", Webhooks: []Webhook{{URL: srv.URL}}})
|
||||
a.SetClaimer(claims)
|
||||
b.SetClaimer(claims)
|
||||
a.Notify(BudgetSpent, "team:t1", "spent", nil)
|
||||
b.Notify(BudgetSpent, "team:t1", "spent", nil)
|
||||
b.Notify(BudgetSpent, "team:t2", "spent", nil)
|
||||
a.Wait()
|
||||
b.Wait()
|
||||
if len(g.calls) != 2 {
|
||||
t.Errorf("%d sent, want one for each team", len(g.calls))
|
||||
}
|
||||
|
||||
// A store that cannot be asked: sent, rather than lost.
|
||||
var g2 got
|
||||
srv2 := httptest.NewServer(g2.handler(0))
|
||||
defer srv2.Close()
|
||||
c := newNotifier(t, Config{Webhooks: []Webhook{{URL: srv2.URL}}})
|
||||
c.SetClaimer(&sharedClaims{err: errors.New("store down")})
|
||||
c.Notify(NodeLost, "node3", "lost", nil)
|
||||
c.Wait()
|
||||
if len(g2.calls) != 1 {
|
||||
t.Errorf("with the store down, %d sent, want 1", len(g2.calls))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
-- Ported from the SQLite migration of the same number.
|
||||
-- When each notification was last sent, by its kind and subject: every
|
||||
-- control plane raises the same events — a budget spent is seen by each one
|
||||
-- serving its requests — and the one that claims an event here sends it.
|
||||
CREATE TABLE notify_sent (
|
||||
key TEXT PRIMARY KEY,
|
||||
sent_at BIGINT NOT NULL
|
||||
);
|
||||
@@ -0,0 +1,7 @@
|
||||
-- When each notification was last sent, by its kind and subject: every
|
||||
-- control plane raises the same events — a budget spent is seen by each one
|
||||
-- serving its requests — and the one that claims an event here sends it.
|
||||
CREATE TABLE notify_sent (
|
||||
key TEXT PRIMARY KEY,
|
||||
sent_at BIGINT NOT NULL
|
||||
);
|
||||
@@ -0,0 +1,23 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ClaimNotice claims the sending of a notification: true when nobody has
|
||||
// sent the same one within the window, and from now on, nobody else may
|
||||
// until it has passed. One statement, so two control planes asking at once
|
||||
// cannot both be told yes.
|
||||
func (s *DB) ClaimNotice(ctx context.Context, key string, at time.Time, window time.Duration) (bool, error) {
|
||||
now := at.UnixMilli()
|
||||
res, err := s.db.ExecContext(ctx,
|
||||
`INSERT INTO notify_sent (key, sent_at) VALUES (?, ?)
|
||||
ON CONFLICT (key) DO UPDATE SET sent_at = excluded.sent_at WHERE notify_sent.sent_at < ?`,
|
||||
key, now, now-window.Milliseconds())
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
n, err := res.RowsAffected()
|
||||
return n == 1, err
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestANoticeIsClaimedByOneOfThoseAskingAtOnce(t *testing.T) {
|
||||
db := capacityStore(t)
|
||||
at := time.Date(2026, 10, 10, 9, 0, 0, 0, time.UTC)
|
||||
var yes atomic.Int64
|
||||
var wg sync.WaitGroup
|
||||
for range 8 {
|
||||
wg.Go(func() {
|
||||
ok, err := db.ClaimNotice(t.Context(), "budget_spent\x00team:t1", at, time.Hour)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if ok {
|
||||
yes.Add(1)
|
||||
}
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
if yes.Load() != 1 {
|
||||
t.Fatalf("%d of eight claimed the same notice", yes.Load())
|
||||
}
|
||||
if ok, _ := db.ClaimNotice(t.Context(), "budget_spent\x00team:t1", at.Add(59*time.Minute), time.Hour); ok {
|
||||
t.Error("claimed again within the hour")
|
||||
}
|
||||
if ok, _ := db.ClaimNotice(t.Context(), "budget_spent\x00team:t2", at, time.Hour); !ok {
|
||||
t.Error("another subject could not be claimed")
|
||||
}
|
||||
if ok, _ := db.ClaimNotice(t.Context(), "budget_spent\x00team:t1", at.Add(61*time.Minute), time.Hour); !ok {
|
||||
t.Error("not claimable once the hour had passed")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user