Rate limits counted across the control planes with router.active_active
CI / test (push) Successful in 9m30s
CI / test (push) Successful in 9m30s
Each router's per-counter request and token counts, this minute's and the last, ride the standby's two-second exchange with the leader. The leader answers each standby with everyone else's sum, its own included, and puts the standbys' sum in force on its own router; each limiter adds the others' counts to its sliding window. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
95fb5eeb6e
commit
540b518fde
17 files changed
+494
-28
No files matched your search
@@ -2,7 +2,10 @@ package control
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
|
||||
)
|
||||
|
||||
// RoutedReplica is a Ready replica as a router needs it: where it is, what it
|
||||
@@ -26,6 +29,10 @@ type Routing struct {
|
||||
// added, scaled or given a canary is in the store, and the standby reads
|
||||
// it again from there.
|
||||
Catalog string `json:"catalog"`
|
||||
// Rates are what every other control plane's router has counted against
|
||||
// each limit, the leader's among them: added to the asker's own, they
|
||||
// make a key's limits the cluster's.
|
||||
Rates *accounting.Rates `json:"rates,omitempty"`
|
||||
}
|
||||
|
||||
// Activity is what a standby's router reports back: when it last sent each
|
||||
@@ -33,8 +40,66 @@ type Routing struct {
|
||||
// evicts and loads by these, and without them a replica busy behind a
|
||||
// standby looks idle to the one machine that decides.
|
||||
type Activity struct {
|
||||
// From names the standby, so its counts replace its own last ones and
|
||||
// are not given back to it.
|
||||
From string `json:"from,omitempty"`
|
||||
LastUsed map[string]time.Time `json:"last_used,omitempty"`
|
||||
Want []string `json:"want,omitempty"`
|
||||
// Rates are its router's own counts against each limit.
|
||||
Rates *accounting.Rates `json:"rates,omitempty"`
|
||||
}
|
||||
|
||||
// RateSharer is a router whose limits can be counted with other routers'.
|
||||
type RateSharer interface {
|
||||
Rates() accounting.Rates
|
||||
SetOtherRates(accounting.Rates)
|
||||
}
|
||||
|
||||
// ratesFresh is how long a standby's counts are believed after it last
|
||||
// reported: a standby gone quiet stops counting against anyone's limits
|
||||
// shortly after it stops taking requests.
|
||||
const ratesFresh = 10 * time.Second
|
||||
|
||||
// sharedRates are the standbys' last counts, by name.
|
||||
type sharedRates struct {
|
||||
mu sync.Mutex
|
||||
by map[string]rateReport
|
||||
seen bool // whether any standby has ever reported
|
||||
}
|
||||
|
||||
type rateReport struct {
|
||||
rates accounting.Rates
|
||||
at time.Time
|
||||
}
|
||||
|
||||
// put keeps a standby's latest counts.
|
||||
func (s *sharedRates) put(from string, r accounting.Rates, now time.Time) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.by == nil {
|
||||
s.by = map[string]rateReport{}
|
||||
}
|
||||
s.by[from] = rateReport{rates: r, at: now}
|
||||
s.seen = true
|
||||
}
|
||||
|
||||
// sum adds every fresh standby's counts but except's; stale ones are
|
||||
// forgotten. ok is false when no standby has ever reported, and there is
|
||||
// nothing to put in force.
|
||||
func (s *sharedRates) sum(except string, now time.Time) (accounting.Rates, bool) {
|
||||
out := accounting.NewRates(now)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for name, r := range s.by {
|
||||
if now.Sub(r.at) > ratesFresh {
|
||||
delete(s.by, name)
|
||||
continue
|
||||
}
|
||||
if name != except {
|
||||
out.Add(r.rates)
|
||||
}
|
||||
}
|
||||
return out, s.seen
|
||||
}
|
||||
|
||||
// ReplicaFeed is a standby's copy of the leader's Ready replicas. ok is false
|
||||
@@ -48,17 +113,44 @@ type ReplicaFeed interface {
|
||||
func (c *Controller) SetReplicaFeed(f ReplicaFeed) { c.feed = f }
|
||||
|
||||
// Routing is this control plane's Ready replicas, for a standby to route to.
|
||||
func (c *Controller) Routing() Routing {
|
||||
func (c *Controller) Routing() Routing { return c.RoutingFor("") }
|
||||
|
||||
// RoutingFor is Routing for the standby named from, with what every other
|
||||
// router has counted against each limit.
|
||||
func (c *Controller) RoutingFor(from string) Routing {
|
||||
out := Routing{Replicas: c.ready(), Catalog: c.CatalogRev()}
|
||||
if rs, ok := c.sink.(RateSharer); ok {
|
||||
others, _ := c.rates.sum(from, time.Now())
|
||||
others.Add(rs.Rates())
|
||||
out.Rates = &others
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ready is the reconciler's Ready replicas, as a router needs them.
|
||||
func (c *Controller) ready() []RoutedReplica {
|
||||
reps := c.rec.ReadyReplicas()
|
||||
out := Routing{Replicas: make([]RoutedReplica, 0, len(reps)), Catalog: c.CatalogRev()}
|
||||
out := make([]RoutedReplica, 0, len(reps))
|
||||
for _, r := range reps {
|
||||
out.Replicas = append(out.Replicas, RoutedReplica{
|
||||
out = append(out, RoutedReplica{
|
||||
ID: r.ID, Model: r.Model, Profile: r.Profile, Pool: r.Pool, Endpoint: r.Endpoint, Weight: r.Weight,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// shareRates puts the standbys' counts in force on this control plane's own
|
||||
// router, while it leads. Once a tick, beside publishing.
|
||||
func (c *Controller) shareRates() {
|
||||
rs, ok := c.sink.(RateSharer)
|
||||
if !ok || (c.lead != nil && !c.lead.IsLeader()) {
|
||||
return
|
||||
}
|
||||
if others, seen := c.rates.sum("", time.Now()); seen {
|
||||
rs.SetOtherRates(others)
|
||||
}
|
||||
}
|
||||
|
||||
// CatalogRev names the catalog in force here. The process's owner id with a
|
||||
// counter: a new leader, or a restarted one, is a different catalog as far as
|
||||
// any standby knows.
|
||||
@@ -72,6 +164,9 @@ func (c *Controller) CatalogRev() string {
|
||||
|
||||
// RecordActivity takes in what a standby's router did since its last report.
|
||||
func (c *Controller) RecordActivity(a Activity) {
|
||||
if a.From != "" && a.Rates != nil {
|
||||
c.rates.put(a.From, *a.Rates, time.Now())
|
||||
}
|
||||
c.rec.RecordUsage(a.LastUsed)
|
||||
for _, m := range a.Want {
|
||||
c.rec.Want(m)
|
||||
@@ -87,5 +182,5 @@ func (c *Controller) routable() []RoutedReplica {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return c.Routing().Replicas
|
||||
return c.ready()
|
||||
}
|
||||
Reference in new issue
Block a user