Standbys that answer the client API themselves, behind router.active_active
CI / test (push) Successful in 9m35s
CI / test (push) Successful in 9m35s
A standby follows the leader's Ready replicas over the agent port, every two seconds, and reports back the replicas its router used and the on-demand models it was asked for. No engine keys travel: every control plane derives them from the cluster secret. Budgets are raised to the store's every 30 s. Unheard from the leader for a minute, a standby passes /v1 on as before. Off by default. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
f664592948
commit
8b7eefef16
22 files changed
+936
-15
No files matched your search
@@ -182,6 +182,7 @@ func (a *API) Register(mux Mux) {
|
||||
a.registerPresets(mux)
|
||||
mux.HandleFunc("GET /api/v1/models", a.listModels)
|
||||
mux.HandleFunc("GET /api/v1/replicas", a.listReplicas)
|
||||
a.registerRouting(mux)
|
||||
// Listing is a read and deleting is a separate verb, so a mistyped
|
||||
// command cannot remove anything.
|
||||
mux.HandleFunc("GET /api/v1/weights/orphans", a.listOrphans)
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
llmgw "gitea.mynodes.xyz/ilker/llmgw"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bench"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/federation"
|
||||
@@ -194,6 +195,15 @@ var Routes = []Route{
|
||||
{Method: "GET", Path: "/api/v1/replicas", Summary: "Every replica, with its shards",
|
||||
Query: []Param{{Name: "model", Description: "Only this model's replicas."}},
|
||||
Response: []ReplicaView{}},
|
||||
{Method: "GET", Path: "/api/v1/routing", Summary: "The Ready replicas a standby routes to",
|
||||
Description: "Read by a standby control plane when router.active_active is on: each replica's address, " +
|
||||
"model, profile, pool and weight, without its key, and which catalog is in force.",
|
||||
Response: control.Routing{}},
|
||||
{Method: "POST", Path: "/api/v1/routing/activity", Summary: "A standby's router reports what it used",
|
||||
Description: "When it last sent each replica a request, and the on-demand models it was asked for, " +
|
||||
"so the leader does not evict a replica busy behind a standby. Answered with the routing, " +
|
||||
"so one request a tick does both.",
|
||||
Request: control.Activity{}, Response: control.Routing{}},
|
||||
{Method: "GET", Path: "/api/v1/explain-placement", Summary: "Why a model could not be placed",
|
||||
Query: []Param{{Name: "model", Required: true, Description: "The model's name or an alias."}},
|
||||
Response: ExplainView{}},
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
)
|
||||
|
||||
// registerRouting is what a standby answering requests itself reads from the
|
||||
// leader, and reports back to it. Reached by the standby through the agent
|
||||
// port with its control-plane certificate, like the rest of the forwarded API.
|
||||
func (a *API) registerRouting(mux Mux) {
|
||||
mux.HandleFunc("GET /api/v1/routing", func(w http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(w, http.StatusOK, a.ctl.Routing())
|
||||
})
|
||||
mux.HandleFunc("POST /api/v1/routing/activity", func(w http.ResponseWriter, r *http.Request) {
|
||||
var act control.Activity
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 8<<20)).Decode(&act); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
a.ctl.RecordActivity(act)
|
||||
writeJSON(w, http.StatusOK, a.ctl.Routing())
|
||||
})
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
@@ -95,6 +96,13 @@ type Controller struct {
|
||||
mu sync.Mutex
|
||||
last string // fingerprint of the last published list
|
||||
|
||||
// feed is where a standby's router learns the leader's Ready replicas,
|
||||
// when standbys answer requests themselves; nil when they do not.
|
||||
feed ReplicaFeed
|
||||
// catRev changes with every catalog put in force here, so a standby
|
||||
// following this one knows to read the catalog again.
|
||||
catRev atomic.Uint64
|
||||
|
||||
stop chan struct{}
|
||||
done chan struct{}
|
||||
once sync.Once
|
||||
@@ -193,6 +201,7 @@ type LoadSource interface {
|
||||
// killed.
|
||||
func (c *Controller) ReplaceCatalog(ctx context.Context, next *catalog.Catalog) {
|
||||
c.cat.Store(next)
|
||||
c.catRev.Add(1)
|
||||
if cs, ok := c.sink.(CatalogSink); ok {
|
||||
cs.SetCatalog(next)
|
||||
}
|
||||
@@ -345,7 +354,7 @@ func (c *Controller) Publish(ctx context.Context) {
|
||||
|
||||
// endpoints converts Ready replicas into the router's view.
|
||||
func (c *Controller) endpoints() []endpoint.Info {
|
||||
ready := c.rec.ReadyReplicas()
|
||||
ready := c.routable()
|
||||
out := make([]endpoint.Info, 0, len(ready))
|
||||
|
||||
for _, rep := range ready {
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
package control
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RoutedReplica is a Ready replica as a router needs it: where it is, what it
|
||||
// serves and how much it may take. Not its key — every control plane derives
|
||||
// that from the replica's id and the cluster's secret, so none travels.
|
||||
type RoutedReplica struct {
|
||||
ID string `json:"id"`
|
||||
Model string `json:"model"`
|
||||
Profile string `json:"profile"`
|
||||
Pool string `json:"pool"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
Weight float64 `json:"weight"`
|
||||
}
|
||||
|
||||
// Routing is what a standby that answers requests itself reads from the
|
||||
// leader: the replicas it may send to, and which catalog they belong to.
|
||||
type Routing struct {
|
||||
Replicas []RoutedReplica `json:"replicas"`
|
||||
// Catalog changes whenever the leader puts a catalog in force. Only
|
||||
// compared with the last one a standby saw, never with its own: a model
|
||||
// added, scaled or given a canary is in the store, and the standby reads
|
||||
// it again from there.
|
||||
Catalog string `json:"catalog"`
|
||||
}
|
||||
|
||||
// Activity is what a standby's router reports back: when it last sent each
|
||||
// replica a request, and the on-demand models it was asked for. The leader
|
||||
// evicts and loads by these, and without them a replica busy behind a
|
||||
// standby looks idle to the one machine that decides.
|
||||
type Activity struct {
|
||||
LastUsed map[string]time.Time `json:"last_used,omitempty"`
|
||||
Want []string `json:"want,omitempty"`
|
||||
}
|
||||
|
||||
// ReplicaFeed is a standby's copy of the leader's Ready replicas. ok is false
|
||||
// when it has none worth using — standbys do not answer requests, or the
|
||||
// leader has not been heard from lately — and the standby routes nothing.
|
||||
type ReplicaFeed interface {
|
||||
Ready() (reps []RoutedReplica, ok bool)
|
||||
}
|
||||
|
||||
// SetReplicaFeed lets a standby route to the leader's replicas.
|
||||
func (c *Controller) SetReplicaFeed(f ReplicaFeed) { c.feed = f }
|
||||
|
||||
// Routing is this control plane's Ready replicas, for a standby to route to.
|
||||
func (c *Controller) Routing() Routing {
|
||||
reps := c.rec.ReadyReplicas()
|
||||
out := Routing{Replicas: make([]RoutedReplica, 0, len(reps)), Catalog: c.CatalogRev()}
|
||||
for _, r := range reps {
|
||||
out.Replicas = append(out.Replicas, RoutedReplica{
|
||||
ID: r.ID, Model: r.Model, Profile: r.Profile, Pool: r.Pool, Endpoint: r.Endpoint, Weight: r.Weight,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// CatalogRev names the catalog in force here. The process's owner id with a
|
||||
// counter: a new leader, or a restarted one, is a different catalog as far as
|
||||
// any standby knows.
|
||||
func (c *Controller) CatalogRev() string {
|
||||
owner := ""
|
||||
if c.lead != nil {
|
||||
owner = c.lead.Owner()
|
||||
}
|
||||
return owner + "/" + strconv.FormatUint(c.catRev.Load(), 10)
|
||||
}
|
||||
|
||||
// RecordActivity takes in what a standby's router did since its last report.
|
||||
func (c *Controller) RecordActivity(a Activity) {
|
||||
c.rec.RecordUsage(a.LastUsed)
|
||||
for _, m := range a.Want {
|
||||
c.rec.Want(m)
|
||||
}
|
||||
}
|
||||
|
||||
// routable is what this process's router may send to: the replicas the
|
||||
// reconciler holds Ready, or, on a standby following the leader, the leader's.
|
||||
func (c *Controller) routable() []RoutedReplica {
|
||||
if c.feed != nil && c.lead != nil && !c.lead.IsLeader() {
|
||||
if reps, ok := c.feed.Ready(); ok {
|
||||
return reps
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return c.Routing().Replicas
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package control
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
)
|
||||
|
||||
type fixedFeed struct {
|
||||
reps []RoutedReplica
|
||||
ok bool
|
||||
}
|
||||
|
||||
func (f fixedFeed) Ready() ([]RoutedReplica, bool) { return f.reps, f.ok }
|
||||
|
||||
func routingController(t *testing.T, sink *recordSink) *Controller {
|
||||
t.Helper()
|
||||
cat, err := catalog.New([]catalog.Model{{Name: "llama", Source: "/models/llama", Aliases: []string{"ll"},
|
||||
Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||
Profiles: []catalog.Profile{{Name: "one", GPUClass: "h100-80g", Engine: "vllm", Priority: 3}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := DefaultConfig()
|
||||
cfg.Reconciler.EngineKey = EngineKeys([]byte("secret"))
|
||||
return New(cfg, catalog.NewLive(cat), noDispatch{}, sink, slog.New(slog.DiscardHandler))
|
||||
}
|
||||
|
||||
// standby is a leadership that does not lead: one without a store always
|
||||
// would.
|
||||
func standby() *Leadership {
|
||||
l := NewLeadership(nil, 0, nil)
|
||||
l.leader.Store(false)
|
||||
return l
|
||||
}
|
||||
|
||||
// A standby following the leader routes to the leader's replicas, each with
|
||||
// the key it derives itself and the engine and priority its own catalog
|
||||
// gives the profile; a replica of a model its catalog lacks is left out.
|
||||
func TestAStandbyRoutesToTheLeadersReplicas(t *testing.T) {
|
||||
sink := &recordSink{}
|
||||
c := routingController(t, sink)
|
||||
c.SetLeadership(standby())
|
||||
c.SetReplicaFeed(fixedFeed{ok: true, reps: []RoutedReplica{
|
||||
{ID: "llama-r0", Model: "llama", Profile: "one", Pool: "a", Endpoint: "http://10.0.0.5:20001/", Weight: 2},
|
||||
{ID: "gone-r0", Model: "gone", Profile: "x", Endpoint: "http://10.0.0.6:20001"},
|
||||
}})
|
||||
c.Publish(t.Context())
|
||||
|
||||
if len(sink.infos) != 1 {
|
||||
t.Fatalf("published %+v", sink.infos)
|
||||
}
|
||||
e := sink.infos[0]
|
||||
if e.ID != "llama-r0" || e.URL != "http://10.0.0.5:20001" || e.Pool != "a" || e.Weight != 2 ||
|
||||
e.Engine != "vllm" || e.Priority != 3 || len(e.Models) != 2 || e.Models[1] != "ll" {
|
||||
t.Errorf("endpoint %+v", e)
|
||||
}
|
||||
if e.APIKey == "" || e.APIKey != EngineKeys([]byte("secret"))("llama-r0") {
|
||||
t.Errorf("the key is not the one the leader gave the engine: %q", e.APIKey)
|
||||
}
|
||||
}
|
||||
|
||||
// One that has not heard from the leader lately routes to nothing, and the
|
||||
// leader never reads a feed: its replicas are its reconciler's.
|
||||
func TestAStandbyCutOffRoutesNothingAndTheLeaderIgnoresTheFeed(t *testing.T) {
|
||||
sink := &recordSink{}
|
||||
c := routingController(t, sink)
|
||||
l := standby()
|
||||
c.SetLeadership(l)
|
||||
c.SetReplicaFeed(fixedFeed{ok: false, reps: []RoutedReplica{{ID: "llama-r0", Model: "llama", Profile: "one"}}})
|
||||
if got := c.endpoints(); len(got) != 0 {
|
||||
t.Errorf("a standby that has not heard from the leader routes to %+v", got)
|
||||
}
|
||||
|
||||
l.leader.Store(true)
|
||||
c.SetReplicaFeed(fixedFeed{ok: true, reps: []RoutedReplica{{ID: "llama-r0", Model: "llama", Profile: "one"}}})
|
||||
if got := c.endpoints(); len(got) != 0 {
|
||||
t.Errorf("the leader routed to a feed rather than its own replicas: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalog's revision moves with every catalog put in force, so a
|
||||
// standby knows to read it again.
|
||||
func TestTheCatalogRevisionMovesWithTheCatalog(t *testing.T) {
|
||||
c := routingController(t, &recordSink{})
|
||||
before := c.Routing().Catalog
|
||||
c.ReplaceCatalog(t.Context(), c.cat.Load())
|
||||
if after := c.Routing().Catalog; after == before {
|
||||
t.Errorf("the revision stayed %q", after)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user