Standbys that answer the client API themselves, behind router.active_active
CI / test (push) Successful in 9m35s

A standby follows the leader's Ready replicas over the agent port, every
two seconds, and reports back the replicas its router used and the
on-demand models it was asked for. No engine keys travel: every control
plane derives them from the cluster secret. Budgets are raised to the
store's every 30 s. Unheard from the leader for a minute, a standby
passes /v1 on as before. Off by default.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-01 13:47:47 +01:00
1 parent f664592948
commit 8b7eefef16
22 files changed
+936 -15

No files matched your search

+33 -5
View File
@@ -78,10 +78,41 @@ func (a *Accountant) RestoreBudgets(ctx context.Context, s store.Store, period s
if start.IsZero() {
return nil
}
rows, err := s.SpendSinceByScope(ctx, start)
spend, keys, err := storedSpend(ctx, s, start)
if err != nil {
return fmt.Errorf("restoring budgets: %w", err)
}
a.budgets.Restore(period, spend)
a.log.Info("restored budgets from the store",
"keys", keys, "counters", len(spend), "since", start)
return nil
}
// CatchUpBudgets raises each counter to what the store has recorded, where
// that is more. Run every so often by a router that is one of several taking
// requests: the others' spend reaches it only through the store. Never
// lowers a counter, which leads the store by whatever this process has not
// written yet.
func (a *Accountant) CatchUpBudgets(ctx context.Context, s store.Store, period store.Period) error {
start := PeriodStart(period, time.Now())
if start.IsZero() {
return nil
}
spend, _, err := storedSpend(ctx, s, start)
if err != nil {
return fmt.Errorf("reading budgets: %w", err)
}
a.budgets.Raise(period, spend)
return nil
}
// storedSpend is the spend since start the store has recorded, by counter,
// and how many keys it came from.
func storedSpend(ctx context.Context, s store.Store, start time.Time) (map[string]int64, int, error) {
rows, err := s.SpendSinceByScope(ctx, start)
if err != nil {
return nil, 0, err
}
// Every scope a key sits under, not just the key. A budget owned by a team
// is one counter fed by every key beneath it, and restoring only per-key
// totals would bring the control plane back up believing the team had spent
@@ -113,10 +144,7 @@ func (a *Accountant) RestoreBudgets(ctx context.Context, s store.Store, period s
}
}
}
a.budgets.Restore(period, spend)
a.log.Info("restored budgets from the store",
"keys", len(rows), "counters", len(spend), "since", start)
return nil
return spend, len(rows), nil
}
// Reservation is what Check returns: the estimate it charged, so Record can
+16
View File
@@ -216,6 +216,22 @@ func TestBudgetSurvivesARestart(t *testing.T) {
}
}
func TestCatchingUpRaisesButNeverLowers(t *testing.T) {
// Another router's spend arrives through the store; this one's own, not
// written yet, must not be wiped by a store that is behind.
b := NewBudgets()
b.Add("mine", store.PeriodMonth, 900)
b.Add("shared", store.PeriodMonth, 100)
b.Raise(store.PeriodMonth, map[string]int64{"mine": 400, "shared": 700, "theirs": 50})
for key, want := range map[string]int64{"mine": 900, "shared": 700, "theirs": 50} {
if got := b.Spent(key, store.PeriodMonth); got != want {
t.Errorf("%s: spent %d, want %d", key, got, want)
}
}
}
func TestBudgetsResetAtThePeriodBoundary(t *testing.T) {
b := NewBudgets()
now := time.Date(2026, 1, 31, 23, 59, 0, 0, time.UTC)
+13
View File
@@ -61,6 +61,19 @@ func (b *Budgets) Restore(period store.Period, spend map[string]int64) {
}
}
// Raise lifts counters to the spend given, where it is more.
func (b *Budgets) Raise(period store.Period, spend map[string]int64) {
start := PeriodStart(period, b.now())
b.mu.Lock()
defer b.mu.Unlock()
for key, micros := range spend {
if c := b.currentLocked(key, start); micros > c.micros {
c.micros = micros
b.spend[key] = c
}
}
}
// Spent returns a key's spend in the current period, resetting it if the period
// has rolled over.
func (b *Budgets) Spent(keyID string, period store.Period) int64 {