Rate limits counted across the control planes with router.active_active
CI / test (push) Successful in 9m30s

Each router's per-counter request and token counts, this minute's and the
last, ride the standby's two-second exchange with the leader. The leader
answers each standby with everyone else's sum, its own included, and puts
the standbys' sum in force on its own router; each limiter adds the
others' counts to its sliding window.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-01 17:30:08 +01:00
1 parent 95fb5eeb6e
commit 540b518fde
17 files changed
+494 -28

No files matched your search

+15 -1
View File
@@ -15,6 +15,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/router"
)
// How often a standby answering requests itself hears from the leader, and
@@ -28,6 +29,9 @@ const (
standbyTrust = time.Minute
)
// The router is what counts against limits, and shares the counts.
var _ control.RateSharer = (*router.Router)(nil)
// standbyRoutes is a standby's copy of the leader's Ready replicas, when
// standbys answer requests themselves (router.active_active). Each tick it
// reports what its router used and is answered with the replicas, in one
@@ -164,6 +168,12 @@ func (s *standbyRoutes) exchange(ctx context.Context) error {
}
s.mu.Unlock()
// What the leader and the other standbys counted, so this router's
// limits are the cluster's.
if rs, ok := s.usage.(control.RateSharer); ok && rt.Rates != nil {
rs.SetOtherRates(*rt.Rates)
}
// A model added or changed through the leader is in the store: the
// catalog first, so the replicas published next resolve against it.
if stale && s.refresh != nil {
@@ -208,7 +218,11 @@ func (s *standbyRoutes) activity() (control.Activity, map[string]time.Time) {
}
s.mu.Lock()
defer s.mu.Unlock()
act := control.Activity{LastUsed: map[string]time.Time{}}
act := control.Activity{From: s.fwd.self, LastUsed: map[string]time.Time{}}
if rs, ok := s.usage.(control.RateSharer); ok {
r := rs.Rates()
act.Rates = &r
}
for id, at := range used {
if at.After(s.sent[id]) {
act.LastUsed[id] = at
+33
View File
@@ -14,6 +14,7 @@ import (
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
)
@@ -51,6 +52,38 @@ type fixedUsage map[string]time.Time
func (u fixedUsage) LastUsed() map[string]time.Time { return u }
// sharingUsage is a router whose limits are counted with the others'.
type sharingUsage struct {
fixedUsage
own accounting.Rates
others *accounting.Rates
}
func (u *sharingUsage) Rates() accounting.Rates { return u.own }
func (u *sharingUsage) SetOtherRates(r accounting.Rates) { u.others = &r }
// A standby reports its counts under its name and puts what the leader says
// the rest of the cluster counted in force on its router.
func TestAStandbysLimitsAreTheClusters(t *testing.T) {
theirs := accounting.NewRates(time.Now())
theirs.Tokens.Current = map[string]int64{"key-1": 900}
leader := &fakeLeaderRouting{routing: control.Routing{Rates: &theirs}}
c := newForwardClusterWith(t, leader)
u := &sharingUsage{own: accounting.NewRates(time.Now())}
u.own.Requests.Current = map[string]int64{"key-1": 4}
s := newStandbyRoutes(c.standby, u, nil, nil, slog.New(slog.DiscardHandler))
if err := s.exchange(t.Context()); err != nil {
t.Fatal(err)
}
act := leader.last()
if act.From != "ctl-b" || act.Rates == nil || act.Rates.Requests.Current["key-1"] != 4 {
t.Errorf("reported %+v", act)
}
if u.others == nil || u.others.Tokens.Current["key-1"] != 900 {
t.Errorf("the router was told %+v", u.others)
}
}
func TestAStandbyFollowsTheLeadersReplicas(t *testing.T) {
leader := &fakeLeaderRouting{routing: control.Routing{Catalog: "a/1",
Replicas: []control.RoutedReplica{{ID: "llama-r0", Model: "llama", Endpoint: "http://10.0.0.5:20001"}}}}