Standbys that answer the client API themselves, behind router.active_active
CI / test (push) Successful in 9m35s

A standby follows the leader's Ready replicas over the agent port, every
two seconds, and reports back the replicas its router used and the
on-demand models it was asked for. No engine keys travel: every control
plane derives them from the cluster secret. Budgets are raised to the
store's every 30 s. Unheard from the leader for a minute, a standby
passes /v1 on as before. Off by default.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-01 13:47:47 +01:00
1 parent f664592948
commit 8b7eefef16
22 files changed
+936 -15

No files matched your search

+1
View File
@@ -182,6 +182,7 @@ func (a *API) Register(mux Mux) {
a.registerPresets(mux)
mux.HandleFunc("GET /api/v1/models", a.listModels)
mux.HandleFunc("GET /api/v1/replicas", a.listReplicas)
a.registerRouting(mux)
// Listing is a read and deleting is a separate verb, so a mistyped
// command cannot remove anything.
mux.HandleFunc("GET /api/v1/weights/orphans", a.listOrphans)
+10
View File
@@ -6,6 +6,7 @@ import (
llmgw "gitea.mynodes.xyz/ilker/llmgw"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/federation"
@@ -194,6 +195,15 @@ var Routes = []Route{
{Method: "GET", Path: "/api/v1/replicas", Summary: "Every replica, with its shards",
Query: []Param{{Name: "model", Description: "Only this model's replicas."}},
Response: []ReplicaView{}},
{Method: "GET", Path: "/api/v1/routing", Summary: "The Ready replicas a standby routes to",
Description: "Read by a standby control plane when router.active_active is on: each replica's address, " +
"model, profile, pool and weight, without its key, and which catalog is in force.",
Response: control.Routing{}},
{Method: "POST", Path: "/api/v1/routing/activity", Summary: "A standby's router reports what it used",
Description: "When it last sent each replica a request, and the on-demand models it was asked for, " +
"so the leader does not evict a replica busy behind a standby. Answered with the routing, " +
"so one request a tick does both.",
Request: control.Activity{}, Response: control.Routing{}},
{Method: "GET", Path: "/api/v1/explain-placement", Summary: "Why a model could not be placed",
Query: []Param{{Name: "model", Required: true, Description: "The model's name or an alias."}},
Response: ExplainView{}},
+27
View File
@@ -0,0 +1,27 @@
package adminapi
import (
"encoding/json"
"io"
"net/http"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
)
// registerRouting is what a standby answering requests itself reads from the
// leader, and reports back to it. Reached by the standby through the agent
// port with its control-plane certificate, like the rest of the forwarded API.
func (a *API) registerRouting(mux Mux) {
mux.HandleFunc("GET /api/v1/routing", func(w http.ResponseWriter, _ *http.Request) {
writeJSON(w, http.StatusOK, a.ctl.Routing())
})
mux.HandleFunc("POST /api/v1/routing/activity", func(w http.ResponseWriter, r *http.Request) {
var act control.Activity
if err := json.NewDecoder(io.LimitReader(r.Body, 8<<20)).Decode(&act); err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
a.ctl.RecordActivity(act)
writeJSON(w, http.StatusOK, a.ctl.Routing())
})
}
+10 -1
View File
@@ -12,6 +12,7 @@ import (
"os"
"strings"
"sync"
"sync/atomic"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
@@ -95,6 +96,13 @@ type Controller struct {
mu sync.Mutex
last string // fingerprint of the last published list
// feed is where a standby's router learns the leader's Ready replicas,
// when standbys answer requests themselves; nil when they do not.
feed ReplicaFeed
// catRev changes with every catalog put in force here, so a standby
// following this one knows to read the catalog again.
catRev atomic.Uint64
stop chan struct{}
done chan struct{}
once sync.Once
@@ -193,6 +201,7 @@ type LoadSource interface {
// killed.
func (c *Controller) ReplaceCatalog(ctx context.Context, next *catalog.Catalog) {
c.cat.Store(next)
c.catRev.Add(1)
if cs, ok := c.sink.(CatalogSink); ok {
cs.SetCatalog(next)
}
@@ -345,7 +354,7 @@ func (c *Controller) Publish(ctx context.Context) {
// endpoints converts Ready replicas into the router's view.
func (c *Controller) endpoints() []endpoint.Info {
ready := c.rec.ReadyReplicas()
ready := c.routable()
out := make([]endpoint.Info, 0, len(ready))
for _, rep := range ready {
+91
View File
@@ -0,0 +1,91 @@
package control
import (
"strconv"
"time"
)
// RoutedReplica is a Ready replica as a router needs it: where it is, what it
// serves and how much it may take. Not its key — every control plane derives
// that from the replica's id and the cluster's secret, so none travels.
type RoutedReplica struct {
ID string `json:"id"`
Model string `json:"model"`
Profile string `json:"profile"`
Pool string `json:"pool"`
Endpoint string `json:"endpoint"`
Weight float64 `json:"weight"`
}
// Routing is what a standby that answers requests itself reads from the
// leader: the replicas it may send to, and which catalog they belong to.
type Routing struct {
Replicas []RoutedReplica `json:"replicas"`
// Catalog changes whenever the leader puts a catalog in force. Only
// compared with the last one a standby saw, never with its own: a model
// added, scaled or given a canary is in the store, and the standby reads
// it again from there.
Catalog string `json:"catalog"`
}
// Activity is what a standby's router reports back: when it last sent each
// replica a request, and the on-demand models it was asked for. The leader
// evicts and loads by these, and without them a replica busy behind a
// standby looks idle to the one machine that decides.
type Activity struct {
LastUsed map[string]time.Time `json:"last_used,omitempty"`
Want []string `json:"want,omitempty"`
}
// ReplicaFeed is a standby's copy of the leader's Ready replicas. ok is false
// when it has none worth using — standbys do not answer requests, or the
// leader has not been heard from lately — and the standby routes nothing.
type ReplicaFeed interface {
Ready() (reps []RoutedReplica, ok bool)
}
// SetReplicaFeed lets a standby route to the leader's replicas.
func (c *Controller) SetReplicaFeed(f ReplicaFeed) { c.feed = f }
// Routing is this control plane's Ready replicas, for a standby to route to.
func (c *Controller) Routing() Routing {
reps := c.rec.ReadyReplicas()
out := Routing{Replicas: make([]RoutedReplica, 0, len(reps)), Catalog: c.CatalogRev()}
for _, r := range reps {
out.Replicas = append(out.Replicas, RoutedReplica{
ID: r.ID, Model: r.Model, Profile: r.Profile, Pool: r.Pool, Endpoint: r.Endpoint, Weight: r.Weight,
})
}
return out
}
// CatalogRev names the catalog in force here. The process's owner id with a
// counter: a new leader, or a restarted one, is a different catalog as far as
// any standby knows.
func (c *Controller) CatalogRev() string {
owner := ""
if c.lead != nil {
owner = c.lead.Owner()
}
return owner + "/" + strconv.FormatUint(c.catRev.Load(), 10)
}
// RecordActivity takes in what a standby's router did since its last report.
func (c *Controller) RecordActivity(a Activity) {
c.rec.RecordUsage(a.LastUsed)
for _, m := range a.Want {
c.rec.Want(m)
}
}
// routable is what this process's router may send to: the replicas the
// reconciler holds Ready, or, on a standby following the leader, the leader's.
func (c *Controller) routable() []RoutedReplica {
if c.feed != nil && c.lead != nil && !c.lead.IsLeader() {
if reps, ok := c.feed.Ready(); ok {
return reps
}
return nil
}
return c.Routing().Replicas
}
+92
View File
@@ -0,0 +1,92 @@
package control
import (
"log/slog"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
type fixedFeed struct {
reps []RoutedReplica
ok bool
}
func (f fixedFeed) Ready() ([]RoutedReplica, bool) { return f.reps, f.ok }
func routingController(t *testing.T, sink *recordSink) *Controller {
t.Helper()
cat, err := catalog.New([]catalog.Model{{Name: "llama", Source: "/models/llama", Aliases: []string{"ll"},
Scaling: catalog.Scaling{Min: 1, Max: 1},
Profiles: []catalog.Profile{{Name: "one", GPUClass: "h100-80g", Engine: "vllm", Priority: 3}}}})
if err != nil {
t.Fatal(err)
}
cfg := DefaultConfig()
cfg.Reconciler.EngineKey = EngineKeys([]byte("secret"))
return New(cfg, catalog.NewLive(cat), noDispatch{}, sink, slog.New(slog.DiscardHandler))
}
// standby is a leadership that does not lead: one without a store always
// would.
func standby() *Leadership {
l := NewLeadership(nil, 0, nil)
l.leader.Store(false)
return l
}
// A standby following the leader routes to the leader's replicas, each with
// the key it derives itself and the engine and priority its own catalog
// gives the profile; a replica of a model its catalog lacks is left out.
func TestAStandbyRoutesToTheLeadersReplicas(t *testing.T) {
sink := &recordSink{}
c := routingController(t, sink)
c.SetLeadership(standby())
c.SetReplicaFeed(fixedFeed{ok: true, reps: []RoutedReplica{
{ID: "llama-r0", Model: "llama", Profile: "one", Pool: "a", Endpoint: "http://10.0.0.5:20001/", Weight: 2},
{ID: "gone-r0", Model: "gone", Profile: "x", Endpoint: "http://10.0.0.6:20001"},
}})
c.Publish(t.Context())
if len(sink.infos) != 1 {
t.Fatalf("published %+v", sink.infos)
}
e := sink.infos[0]
if e.ID != "llama-r0" || e.URL != "http://10.0.0.5:20001" || e.Pool != "a" || e.Weight != 2 ||
e.Engine != "vllm" || e.Priority != 3 || len(e.Models) != 2 || e.Models[1] != "ll" {
t.Errorf("endpoint %+v", e)
}
if e.APIKey == "" || e.APIKey != EngineKeys([]byte("secret"))("llama-r0") {
t.Errorf("the key is not the one the leader gave the engine: %q", e.APIKey)
}
}
// One that has not heard from the leader lately routes to nothing, and the
// leader never reads a feed: its replicas are its reconciler's.
func TestAStandbyCutOffRoutesNothingAndTheLeaderIgnoresTheFeed(t *testing.T) {
sink := &recordSink{}
c := routingController(t, sink)
l := standby()
c.SetLeadership(l)
c.SetReplicaFeed(fixedFeed{ok: false, reps: []RoutedReplica{{ID: "llama-r0", Model: "llama", Profile: "one"}}})
if got := c.endpoints(); len(got) != 0 {
t.Errorf("a standby that has not heard from the leader routes to %+v", got)
}
l.leader.Store(true)
c.SetReplicaFeed(fixedFeed{ok: true, reps: []RoutedReplica{{ID: "llama-r0", Model: "llama", Profile: "one"}}})
if got := c.endpoints(); len(got) != 0 {
t.Errorf("the leader routed to a feed rather than its own replicas: %+v", got)
}
}
// The catalog's revision moves with every catalog put in force, so a
// standby knows to read it again.
func TestTheCatalogRevisionMovesWithTheCatalog(t *testing.T) {
c := routingController(t, &recordSink{})
before := c.Routing().Catalog
c.ReplaceCatalog(t.Context(), c.cat.Load())
if after := c.Routing().Catalog; after == before {
t.Errorf("the revision stayed %q", after)
}
}