Files
llmgw/internal/control/reconciler/preempt_test.go
T
Ilker ManapandClaude Opus 5.5 ac24e204a6
Release / build (push) Successful in 1m7s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m46s
Add capacity classes and preempt lower ones to place a model
A model's class (critical, standard, opportunistic; unset follows its
scaling) decides what may be stopped to make room for it: a lower class
for a higher, and within a class only what another grew into under load,
for a model below its minimum. Victims are tried on a copy of the
inventory and stopped only if they make the room. Preempted replicas are
drained with a short grace and their model is held back until the room is
used. ctl model resources and class move under ctl model.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 18:02:59 +02:00

238 lines
7.7 KiB
Go

package reconciler
import (
"log/slog"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// newFleetHarness is newHarness with a catalog of several models.
func newFleetHarness(t *testing.T, models []catalog.Model, nodes ...inventory.Node) *harness {
t.Helper()
cat, err := catalog.New(models)
if err != nil {
t.Fatal(err)
}
inv := inventory.NewStore()
c := newClock()
for _, n := range nodes {
inv.Upsert(n)
}
disp := &fakeDispatcher{}
cfg := DefaultConfig()
cfg.UnreachableAfter = 15 * time.Second
cfg.LostAfter = 60 * time.Second
r := New(cfg, catalog.NewLive(cat), inv, disp, slog.New(slog.DiscardHandler))
r.SetClock(c.now)
c.advance(cfg.SettleFor)
for _, n := range nodes {
touchAt(inv, n.ID, c.now())
}
return &harness{r: r, inv: inv, disp: disp, clock: c}
}
// gpuModel runs on one whole 8-GPU node.
func gpuModel(name, class string, min int) catalog.Model {
return catalog.Model{Name: name, Source: "/models/" + name, Class: class,
Scaling: catalog.Scaling{Min: min, Max: min},
Profiles: []catalog.Profile{{Name: "tp8", Engine: "vllm", GPUClass: "h100-80g", TP: 8}}}
}
// settle runs passes until nothing changes, marking what starts ready and
// letting preempted replicas run out their grace.
func (h *harness) settle(t *testing.T, passes int) {
t.Helper()
for range passes {
h.r.Reconcile(t.Context())
h.markReady()
h.advance(h.r.cfg.PreemptGrace + time.Second)
}
}
func replicasOf(h *harness, model string) (live int) {
for _, rep := range h.r.Replicas() {
if rep.Model == model && rep.State.Counts() {
live++
}
}
return live
}
// fill places the standard model, then ages it past MinLifetime so it may be
// taken from, and only then adds the model that wants its node.
func fillThenWant(t *testing.T, want catalog.Model, have catalog.Model, nodes ...inventory.Node) *harness {
t.Helper()
h := newFleetHarness(t, []catalog.Model{have, withMin(want, 0)}, nodes...)
h.settle(t, 3)
if got := replicasOf(h, have.Name); got != have.Scaling.Min {
t.Fatalf("%s has %d replicas before anything wanted its room", have.Name, got)
}
h.advance(h.r.cfg.MinLifetime + time.Minute)
cat, err := catalog.New([]catalog.Model{have, want})
if err != nil {
t.Fatal(err)
}
h.r.cat.Store(cat)
return h
}
func withMin(m catalog.Model, n int) catalog.Model {
m.Scaling = catalog.Scaling{Min: n, Max: n}
return m
}
func TestACriticalModelTakesAStandardModelsNode(t *testing.T) {
h := fillThenWant(t, gpuModel("vital", catalog.ClassCritical, 1), gpuModel("chat", "", 1), gpuNode("1", 8))
h.r.Reconcile(t.Context())
var draining int
for _, rep := range h.r.Replicas() {
if rep.Model == "chat" && rep.State == Draining && rep.PreemptedFor == "vital" {
draining++
}
}
if draining != 1 {
t.Fatalf("the standard replica was not drained for the critical one: %+v", h.r.Replicas())
}
if hs := health(h, "chat"); hs.HeldFor != "vital" || hs.Preempted != 1 {
t.Errorf("the victim's health does not say it gave way: %+v", hs)
}
h.settle(t, 4)
if replicasOf(h, "vital") != 1 {
t.Fatalf("the critical model was not placed once room was made: %+v", h.r.Replicas())
}
// Held back until the critical model had its replica, then short again
// because there is no room: it must not have taken the node back — not
// even for a moment, which would load a model only to preempt it again.
if replicasOf(h, "chat") != 0 {
t.Error("the standard model took the node back")
}
started := 0
for _, cmd := range h.disp.startedShards() {
if strings.HasPrefix(cmd.ReplicaID, "chat-") {
started++
}
}
if started != 1 {
t.Errorf("the standard model started %d replicas; it should not have tried again", started)
}
}
func TestAStandardModelDoesNotTakeAnothersMinimum(t *testing.T) {
h := fillThenWant(t, gpuModel("second", "", 1), gpuModel("first", "", 1), gpuNode("1", 8))
h.settle(t, 3)
if replicasOf(h, "first") != 1 || replicasOf(h, "second") != 0 {
t.Errorf("a model of the same class took another's minimum: %+v", h.r.Replicas())
}
if hs := health(h, "second"); !hs.Unplaceable {
t.Error("the model left without room should say it cannot be placed")
}
}
func TestWithinAClassOnlyWhatWasGrownIntoIsTaken(t *testing.T) {
grown := gpuModel("busy", "", 1)
grown.Scaling = catalog.Scaling{Min: 1, Max: 2}
h := newFleetHarness(t, []catalog.Model{grown, withMin(gpuModel("new", "", 1), 0)}, gpuNode("1", 8), gpuNode("2", 8))
h.settle(t, 2)
// Load grows it to its max: the second replica is above its minimum.
h.r.mu.Lock()
h.r.extra["busy"] = 1
h.r.mu.Unlock()
h.settle(t, 3)
if replicasOf(h, "busy") != 2 {
t.Fatalf("setup: busy did not grow to 2: %+v", h.r.Replicas())
}
h.advance(h.r.cfg.MinLifetime + time.Minute)
cat, _ := catalog.New([]catalog.Model{grown, gpuModel("new", "", 1)})
h.r.cat.Store(cat)
h.settle(t, 5)
if replicasOf(h, "new") != 1 || replicasOf(h, "busy") != 1 {
t.Errorf("the grown replica should give way to a minimum: busy %d, new %d",
replicasOf(h, "busy"), replicasOf(h, "new"))
}
}
func TestNothingIsStoppedThatWouldNotMakeRoom(t *testing.T) {
// Two nodes, each half used by an opportunistic model; the critical one
// needs a whole node. Stopping one frees half a node, which is no use —
// and both together are on two nodes, which is no use either.
half := catalog.Model{Name: "half", Source: "/m", Class: catalog.ClassOpportunistic,
Scaling: catalog.Scaling{Min: 2, Max: 2},
Profiles: []catalog.Profile{{Name: "tp4", Engine: "vllm", GPUClass: "h100-80g", TP: 4}}}
five := func(id string) inventory.Node { return gpuNode(id, 5) }
h := fillThenWant(t, gpuModel("vital", catalog.ClassCritical, 1), half, five("1"), five("2"))
h.r.Reconcile(t.Context())
for _, rep := range h.r.Replicas() {
if rep.PreemptedFor != "" || rep.State == Draining {
t.Fatalf("a replica was stopped without making room: %+v", rep)
}
}
}
func TestACriticalMinimumIsNeverTaken(t *testing.T) {
h := fillThenWant(t, gpuModel("also-vital", catalog.ClassCritical, 1), gpuModel("vital", catalog.ClassCritical, 1), gpuNode("1", 8))
h.settle(t, 3)
if replicasOf(h, "vital") != 1 {
t.Error("a critical model's minimum was taken")
}
}
func TestAReplicaThatJustStartedIsLeftAlone(t *testing.T) {
h := newFleetHarness(t, []catalog.Model{gpuModel("chat", "", 1), withMin(gpuModel("vital", catalog.ClassCritical, 1), 0)}, gpuNode("1", 8))
h.settle(t, 2)
cat, _ := catalog.New([]catalog.Model{gpuModel("chat", "", 1), gpuModel("vital", catalog.ClassCritical, 1)})
h.r.cat.Store(cat)
h.r.Reconcile(t.Context())
for _, rep := range h.r.Replicas() {
if rep.State == Draining {
t.Fatal("a replica that had only just loaded was preempted")
}
}
}
func health(h *harness, model string) ModelHealth {
for _, x := range h.r.ModelHealth() {
if x.Model == model {
return x
}
}
return ModelHealth{}
}
func TestAHoldLastsUntilTheRoomIsUsedOrItExpires(t *testing.T) {
h := fillThenWant(t, gpuModel("vital", catalog.ClassCritical, 1), gpuModel("chat", "", 1), gpuNode("1", 8))
h.r.Reconcile(t.Context())
if !h.r.isHeld("chat") {
t.Fatal("the model that gave way is not held")
}
if h.r.isHeld("vital") {
t.Error("the model room was made for is held")
}
// Released as soon as the model it was given to has what it asked for.
h.settle(t, 3)
if replicasOf(h, "vital") != 1 {
t.Fatalf("setup: vital was not placed: %+v", h.r.Replicas())
}
if h.r.isHeld("chat") {
t.Error("the hold outlasted its purpose")
}
// And, whatever happens, after PreemptHold.
h.r.mu.Lock()
h.r.held["other"] = hold{Until: h.r.now().Add(time.Minute), For: "never-placed"}
h.r.mu.Unlock()
if !h.r.isHeld("other") {
t.Fatal("a fresh hold is not in force")
}
h.advance(2 * time.Minute)
if h.r.isHeld("other") {
t.Error("a hold outlived its time")
}
}