Standbys that answer the client API themselves, behind router.active_active
CI / test (push) Successful in 9m35s
CI / test (push) Successful in 9m35s
A standby follows the leader's Ready replicas over the agent port, every two seconds, and reports back the replicas its router used and the on-demand models it was asked for. No engine keys travel: every control plane derives them from the cluster secret. Budgets are raised to the store's every 30 s. Unheard from the leader for a minute, a standby passes /v1 on as before. Off by default. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
f664592948
commit
8b7eefef16
22 files changed
+936
-15
No files matched your search
@@ -78,10 +78,41 @@ func (a *Accountant) RestoreBudgets(ctx context.Context, s store.Store, period s
|
||||
if start.IsZero() {
|
||||
return nil
|
||||
}
|
||||
rows, err := s.SpendSinceByScope(ctx, start)
|
||||
spend, keys, err := storedSpend(ctx, s, start)
|
||||
if err != nil {
|
||||
return fmt.Errorf("restoring budgets: %w", err)
|
||||
}
|
||||
a.budgets.Restore(period, spend)
|
||||
a.log.Info("restored budgets from the store",
|
||||
"keys", keys, "counters", len(spend), "since", start)
|
||||
return nil
|
||||
}
|
||||
|
||||
// CatchUpBudgets raises each counter to what the store has recorded, where
|
||||
// that is more. Run every so often by a router that is one of several taking
|
||||
// requests: the others' spend reaches it only through the store. Never
|
||||
// lowers a counter, which leads the store by whatever this process has not
|
||||
// written yet.
|
||||
func (a *Accountant) CatchUpBudgets(ctx context.Context, s store.Store, period store.Period) error {
|
||||
start := PeriodStart(period, time.Now())
|
||||
if start.IsZero() {
|
||||
return nil
|
||||
}
|
||||
spend, _, err := storedSpend(ctx, s, start)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading budgets: %w", err)
|
||||
}
|
||||
a.budgets.Raise(period, spend)
|
||||
return nil
|
||||
}
|
||||
|
||||
// storedSpend is the spend since start the store has recorded, by counter,
|
||||
// and how many keys it came from.
|
||||
func storedSpend(ctx context.Context, s store.Store, start time.Time) (map[string]int64, int, error) {
|
||||
rows, err := s.SpendSinceByScope(ctx, start)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
// Every scope a key sits under, not just the key. A budget owned by a team
|
||||
// is one counter fed by every key beneath it, and restoring only per-key
|
||||
// totals would bring the control plane back up believing the team had spent
|
||||
@@ -113,10 +144,7 @@ func (a *Accountant) RestoreBudgets(ctx context.Context, s store.Store, period s
|
||||
}
|
||||
}
|
||||
}
|
||||
a.budgets.Restore(period, spend)
|
||||
a.log.Info("restored budgets from the store",
|
||||
"keys", len(rows), "counters", len(spend), "since", start)
|
||||
return nil
|
||||
return spend, len(rows), nil
|
||||
}
|
||||
|
||||
// Reservation is what Check returns: the estimate it charged, so Record can
|
||||
|
||||
@@ -216,6 +216,22 @@ func TestBudgetSurvivesARestart(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCatchingUpRaisesButNeverLowers(t *testing.T) {
|
||||
// Another router's spend arrives through the store; this one's own, not
|
||||
// written yet, must not be wiped by a store that is behind.
|
||||
b := NewBudgets()
|
||||
b.Add("mine", store.PeriodMonth, 900)
|
||||
b.Add("shared", store.PeriodMonth, 100)
|
||||
|
||||
b.Raise(store.PeriodMonth, map[string]int64{"mine": 400, "shared": 700, "theirs": 50})
|
||||
|
||||
for key, want := range map[string]int64{"mine": 900, "shared": 700, "theirs": 50} {
|
||||
if got := b.Spent(key, store.PeriodMonth); got != want {
|
||||
t.Errorf("%s: spent %d, want %d", key, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBudgetsResetAtThePeriodBoundary(t *testing.T) {
|
||||
b := NewBudgets()
|
||||
now := time.Date(2026, 1, 31, 23, 59, 0, 0, time.UTC)
|
||||
|
||||
@@ -61,6 +61,19 @@ func (b *Budgets) Restore(period store.Period, spend map[string]int64) {
|
||||
}
|
||||
}
|
||||
|
||||
// Raise lifts counters to the spend given, where it is more.
|
||||
func (b *Budgets) Raise(period store.Period, spend map[string]int64) {
|
||||
start := PeriodStart(period, b.now())
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
for key, micros := range spend {
|
||||
if c := b.currentLocked(key, start); micros > c.micros {
|
||||
c.micros = micros
|
||||
b.spend[key] = c
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Spent returns a key's spend in the current period, resetting it if the period
|
||||
// has rolled over.
|
||||
func (b *Budgets) Spent(keyID string, period store.Period) int64 {
|
||||
|
||||
Reference in new issue
Block a user