Standbys that answer the client API themselves, behind router.active_active
CI / test (push) Successful in 9m35s
CI / test (push) Successful in 9m35s
A standby follows the leader's Ready replicas over the agent port, every two seconds, and reports back the replicas its router used and the on-demand models it was asked for. No engine keys travel: every control plane derives them from the cluster secret. Budgets are raised to the store's every 30 s. Unheard from the leader for a minute, a standby passes /v1 on as before. Off by default. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
f664592948
commit
8b7eefef16
22 files changed
+936
-15
No files matched your search
@@ -0,0 +1,92 @@
|
||||
package control
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
)
|
||||
|
||||
type fixedFeed struct {
|
||||
reps []RoutedReplica
|
||||
ok bool
|
||||
}
|
||||
|
||||
func (f fixedFeed) Ready() ([]RoutedReplica, bool) { return f.reps, f.ok }
|
||||
|
||||
func routingController(t *testing.T, sink *recordSink) *Controller {
|
||||
t.Helper()
|
||||
cat, err := catalog.New([]catalog.Model{{Name: "llama", Source: "/models/llama", Aliases: []string{"ll"},
|
||||
Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||
Profiles: []catalog.Profile{{Name: "one", GPUClass: "h100-80g", Engine: "vllm", Priority: 3}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := DefaultConfig()
|
||||
cfg.Reconciler.EngineKey = EngineKeys([]byte("secret"))
|
||||
return New(cfg, catalog.NewLive(cat), noDispatch{}, sink, slog.New(slog.DiscardHandler))
|
||||
}
|
||||
|
||||
// standby is a leadership that does not lead: one without a store always
|
||||
// would.
|
||||
func standby() *Leadership {
|
||||
l := NewLeadership(nil, 0, nil)
|
||||
l.leader.Store(false)
|
||||
return l
|
||||
}
|
||||
|
||||
// A standby following the leader routes to the leader's replicas, each with
|
||||
// the key it derives itself and the engine and priority its own catalog
|
||||
// gives the profile; a replica of a model its catalog lacks is left out.
|
||||
func TestAStandbyRoutesToTheLeadersReplicas(t *testing.T) {
|
||||
sink := &recordSink{}
|
||||
c := routingController(t, sink)
|
||||
c.SetLeadership(standby())
|
||||
c.SetReplicaFeed(fixedFeed{ok: true, reps: []RoutedReplica{
|
||||
{ID: "llama-r0", Model: "llama", Profile: "one", Pool: "a", Endpoint: "http://10.0.0.5:20001/", Weight: 2},
|
||||
{ID: "gone-r0", Model: "gone", Profile: "x", Endpoint: "http://10.0.0.6:20001"},
|
||||
}})
|
||||
c.Publish(t.Context())
|
||||
|
||||
if len(sink.infos) != 1 {
|
||||
t.Fatalf("published %+v", sink.infos)
|
||||
}
|
||||
e := sink.infos[0]
|
||||
if e.ID != "llama-r0" || e.URL != "http://10.0.0.5:20001" || e.Pool != "a" || e.Weight != 2 ||
|
||||
e.Engine != "vllm" || e.Priority != 3 || len(e.Models) != 2 || e.Models[1] != "ll" {
|
||||
t.Errorf("endpoint %+v", e)
|
||||
}
|
||||
if e.APIKey == "" || e.APIKey != EngineKeys([]byte("secret"))("llama-r0") {
|
||||
t.Errorf("the key is not the one the leader gave the engine: %q", e.APIKey)
|
||||
}
|
||||
}
|
||||
|
||||
// One that has not heard from the leader lately routes to nothing, and the
|
||||
// leader never reads a feed: its replicas are its reconciler's.
|
||||
func TestAStandbyCutOffRoutesNothingAndTheLeaderIgnoresTheFeed(t *testing.T) {
|
||||
sink := &recordSink{}
|
||||
c := routingController(t, sink)
|
||||
l := standby()
|
||||
c.SetLeadership(l)
|
||||
c.SetReplicaFeed(fixedFeed{ok: false, reps: []RoutedReplica{{ID: "llama-r0", Model: "llama", Profile: "one"}}})
|
||||
if got := c.endpoints(); len(got) != 0 {
|
||||
t.Errorf("a standby that has not heard from the leader routes to %+v", got)
|
||||
}
|
||||
|
||||
l.leader.Store(true)
|
||||
c.SetReplicaFeed(fixedFeed{ok: true, reps: []RoutedReplica{{ID: "llama-r0", Model: "llama", Profile: "one"}}})
|
||||
if got := c.endpoints(); len(got) != 0 {
|
||||
t.Errorf("the leader routed to a feed rather than its own replicas: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalog's revision moves with every catalog put in force, so a
|
||||
// standby knows to read it again.
|
||||
func TestTheCatalogRevisionMovesWithTheCatalog(t *testing.T) {
|
||||
c := routingController(t, &recordSink{})
|
||||
before := c.Routing().Catalog
|
||||
c.ReplaceCatalog(t.Context(), c.cat.Load())
|
||||
if after := c.Routing().Catalog; after == before {
|
||||
t.Errorf("the revision stayed %q", after)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user