Rate limits counted across the control planes with router.active_active
CI / test (push) Successful in 9m30s
CI / test (push) Successful in 9m30s
Each router's per-counter request and token counts, this minute's and the last, ride the standby's two-second exchange with the leader. The leader answers each standby with everyone else's sum, its own included, and puts the standbys' sum in force on its own router; each limiter adds the others' counts to its sliding window. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
95fb5eeb6e
commit
540b518fde
17 files changed
+494
-28
No files matched your search
@@ -15,6 +15,7 @@ import (
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router"
|
||||
)
|
||||
|
||||
// How often a standby answering requests itself hears from the leader, and
|
||||
@@ -28,6 +29,9 @@ const (
|
||||
standbyTrust = time.Minute
|
||||
)
|
||||
|
||||
// The router is what counts against limits, and shares the counts.
|
||||
var _ control.RateSharer = (*router.Router)(nil)
|
||||
|
||||
// standbyRoutes is a standby's copy of the leader's Ready replicas, when
|
||||
// standbys answer requests themselves (router.active_active). Each tick it
|
||||
// reports what its router used and is answered with the replicas, in one
|
||||
@@ -164,6 +168,12 @@ func (s *standbyRoutes) exchange(ctx context.Context) error {
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
// What the leader and the other standbys counted, so this router's
|
||||
// limits are the cluster's.
|
||||
if rs, ok := s.usage.(control.RateSharer); ok && rt.Rates != nil {
|
||||
rs.SetOtherRates(*rt.Rates)
|
||||
}
|
||||
|
||||
// A model added or changed through the leader is in the store: the
|
||||
// catalog first, so the replicas published next resolve against it.
|
||||
if stale && s.refresh != nil {
|
||||
@@ -208,7 +218,11 @@ func (s *standbyRoutes) activity() (control.Activity, map[string]time.Time) {
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
act := control.Activity{LastUsed: map[string]time.Time{}}
|
||||
act := control.Activity{From: s.fwd.self, LastUsed: map[string]time.Time{}}
|
||||
if rs, ok := s.usage.(control.RateSharer); ok {
|
||||
r := rs.Rates()
|
||||
act.Rates = &r
|
||||
}
|
||||
for id, at := range used {
|
||||
if at.After(s.sent[id]) {
|
||||
act.LastUsed[id] = at
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
)
|
||||
|
||||
@@ -51,6 +52,38 @@ type fixedUsage map[string]time.Time
|
||||
|
||||
func (u fixedUsage) LastUsed() map[string]time.Time { return u }
|
||||
|
||||
// sharingUsage is a router whose limits are counted with the others'.
|
||||
type sharingUsage struct {
|
||||
fixedUsage
|
||||
own accounting.Rates
|
||||
others *accounting.Rates
|
||||
}
|
||||
|
||||
func (u *sharingUsage) Rates() accounting.Rates { return u.own }
|
||||
func (u *sharingUsage) SetOtherRates(r accounting.Rates) { u.others = &r }
|
||||
|
||||
// A standby reports its counts under its name and puts what the leader says
|
||||
// the rest of the cluster counted in force on its router.
|
||||
func TestAStandbysLimitsAreTheClusters(t *testing.T) {
|
||||
theirs := accounting.NewRates(time.Now())
|
||||
theirs.Tokens.Current = map[string]int64{"key-1": 900}
|
||||
leader := &fakeLeaderRouting{routing: control.Routing{Rates: &theirs}}
|
||||
c := newForwardClusterWith(t, leader)
|
||||
u := &sharingUsage{own: accounting.NewRates(time.Now())}
|
||||
u.own.Requests.Current = map[string]int64{"key-1": 4}
|
||||
s := newStandbyRoutes(c.standby, u, nil, nil, slog.New(slog.DiscardHandler))
|
||||
if err := s.exchange(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
act := leader.last()
|
||||
if act.From != "ctl-b" || act.Rates == nil || act.Rates.Requests.Current["key-1"] != 4 {
|
||||
t.Errorf("reported %+v", act)
|
||||
}
|
||||
if u.others == nil || u.others.Tokens.Current["key-1"] != 900 {
|
||||
t.Errorf("the router was told %+v", u.others)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStandbyFollowsTheLeadersReplicas(t *testing.T) {
|
||||
leader := &fakeLeaderRouting{routing: control.Routing{Catalog: "a/1",
|
||||
Replicas: []control.RoutedReplica{{ID: "llama-r0", Model: "llama", Endpoint: "http://10.0.0.5:20001"}}}}
|
||||
|
||||
Reference in new issue
Block a user