Standbys that answer the client API themselves, behind router.active_active
A standby follows the leader's Ready replicas over the agent port, every two seconds, and reports back the replicas its router used and the on-demand models it was asked for. No engine keys travel: every control plane derives them from the cluster secret. Budgets are raised to the store's every 30 s. Unheard from the leader for a minute, a standby passes /v1 on as before. Off by default. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
f664592948
commit
8b7eefef16
22 files changed
+936
-15
No files matched your search
@@ -0,0 +1,91 @@
|
||||
package control
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RoutedReplica is a Ready replica as a router needs it: where it is, what it
|
||||
// serves and how much it may take. Not its key — every control plane derives
|
||||
// that from the replica's id and the cluster's secret, so none travels.
|
||||
type RoutedReplica struct {
|
||||
ID string `json:"id"`
|
||||
Model string `json:"model"`
|
||||
Profile string `json:"profile"`
|
||||
Pool string `json:"pool"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
Weight float64 `json:"weight"`
|
||||
}
|
||||
|
||||
// Routing is what a standby that answers requests itself reads from the
|
||||
// leader: the replicas it may send to, and which catalog they belong to.
|
||||
type Routing struct {
|
||||
Replicas []RoutedReplica `json:"replicas"`
|
||||
// Catalog changes whenever the leader puts a catalog in force. Only
|
||||
// compared with the last one a standby saw, never with its own: a model
|
||||
// added, scaled or given a canary is in the store, and the standby reads
|
||||
// it again from there.
|
||||
Catalog string `json:"catalog"`
|
||||
}
|
||||
|
||||
// Activity is what a standby's router reports back: when it last sent each
|
||||
// replica a request, and the on-demand models it was asked for. The leader
|
||||
// evicts and loads by these, and without them a replica busy behind a
|
||||
// standby looks idle to the one machine that decides.
|
||||
type Activity struct {
|
||||
LastUsed map[string]time.Time `json:"last_used,omitempty"`
|
||||
Want []string `json:"want,omitempty"`
|
||||
}
|
||||
|
||||
// ReplicaFeed is a standby's copy of the leader's Ready replicas. ok is false
|
||||
// when it has none worth using — standbys do not answer requests, or the
|
||||
// leader has not been heard from lately — and the standby routes nothing.
|
||||
type ReplicaFeed interface {
|
||||
Ready() (reps []RoutedReplica, ok bool)
|
||||
}
|
||||
|
||||
// SetReplicaFeed lets a standby route to the leader's replicas.
|
||||
func (c *Controller) SetReplicaFeed(f ReplicaFeed) { c.feed = f }
|
||||
|
||||
// Routing is this control plane's Ready replicas, for a standby to route to.
|
||||
func (c *Controller) Routing() Routing {
|
||||
reps := c.rec.ReadyReplicas()
|
||||
out := Routing{Replicas: make([]RoutedReplica, 0, len(reps)), Catalog: c.CatalogRev()}
|
||||
for _, r := range reps {
|
||||
out.Replicas = append(out.Replicas, RoutedReplica{
|
||||
ID: r.ID, Model: r.Model, Profile: r.Profile, Pool: r.Pool, Endpoint: r.Endpoint, Weight: r.Weight,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// CatalogRev names the catalog in force here. The process's owner id with a
|
||||
// counter: a new leader, or a restarted one, is a different catalog as far as
|
||||
// any standby knows.
|
||||
func (c *Controller) CatalogRev() string {
|
||||
owner := ""
|
||||
if c.lead != nil {
|
||||
owner = c.lead.Owner()
|
||||
}
|
||||
return owner + "/" + strconv.FormatUint(c.catRev.Load(), 10)
|
||||
}
|
||||
|
||||
// RecordActivity takes in what a standby's router did since its last report.
|
||||
func (c *Controller) RecordActivity(a Activity) {
|
||||
c.rec.RecordUsage(a.LastUsed)
|
||||
for _, m := range a.Want {
|
||||
c.rec.Want(m)
|
||||
}
|
||||
}
|
||||
|
||||
// routable is what this process's router may send to: the replicas the
|
||||
// reconciler holds Ready, or, on a standby following the leader, the leader's.
|
||||
func (c *Controller) routable() []RoutedReplica {
|
||||
if c.feed != nil && c.lead != nil && !c.lead.IsLeader() {
|
||||
if reps, ok := c.feed.Ready(); ok {
|
||||
return reps
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return c.Routing().Replicas
|
||||
}
|
||||
Reference in new issue
Block a user