diff --git a/CHANGELOG.md b/CHANGELOG.md index 7aa273d..a59cc6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,7 +72,10 @@ these rules: the same within the class, then lower classes' minimums; the longest unused first within each. - The set is tried on a copy of the inventory first, at most three replicas, - and nothing is stopped unless it makes the room. + and nothing is stopped unless it makes the room. Of a set that does, each + replica whose stopping turns out not to be needed is left running, so an + idle replica tried first on the wrong node is not stopped with the one that + made the room. A replica stopped this way is drained and given 30 seconds to finish what it is serving. Its model is held back for up to five minutes, or until the model diff --git a/internal/control/reconciler/preempt.go b/internal/control/reconciler/preempt.go index f8cea55..155ff92 100644 --- a/internal/control/reconciler/preempt.go +++ b/internal/control/reconciler/preempt.go @@ -54,28 +54,46 @@ func (r *Reconciler) preemptFor(ctx context.Context, m catalog.Model, avoid []st if len(cands) == 0 { return false } - trial := r.inv.Clone() - sched := placement.New(trial) + // Candidates in order until the model fits; then every one whose + // stopping turns out not to be needed is kept after all. Without that + // second step an idle replica on the wrong node, tried first, would be + // stopped along with the one that actually made the room. var chosen []victim for _, v := range cands { if len(chosen) == maxVictims { break } - trial.ReleaseReplica(v.ID) chosen = append(chosen, v) - res := sched.Place(r.orderedProfiles(m), avoid) - if res.Placement == nil && len(avoid) > 0 { - res = sched.Place(r.orderedProfiles(m), nil) - } - if res.Placement == nil { + if !r.fitsWithout(m, avoid, chosen) { continue } + for i := len(chosen) - 1; i >= 0 && len(chosen) > 1; i-- { + fewer := append(append([]victim(nil), chosen[:i]...), chosen[i+1:]...) + if r.fitsWithout(m, avoid, fewer) { + chosen = fewer + } + } r.preempt(ctx, m, chosen) return true } return false } +// fitsWithout reports whether m could be placed with these replicas gone, +// tried on a copy of the inventory. +func (r *Reconciler) fitsWithout(m catalog.Model, avoid []string, gone []victim) bool { + trial := r.inv.Clone() + for _, v := range gone { + trial.ReleaseReplica(v.ID) + } + sched := placement.New(trial) + res := sched.Place(r.orderedProfiles(m), avoid) + if res.Placement == nil && len(avoid) > 0 { + res = sched.Place(r.orderedProfiles(m), nil) + } + return res.Placement != nil +} + // preempt drains the chosen replicas and holds their models back. func (r *Reconciler) preempt(ctx context.Context, m catalog.Model, chosen []victim) { now := r.now() diff --git a/internal/control/reconciler/preempt_test.go b/internal/control/reconciler/preempt_test.go index 87cf858..d7eb2a8 100644 --- a/internal/control/reconciler/preempt_test.go +++ b/internal/control/reconciler/preempt_test.go @@ -235,3 +235,50 @@ func TestAHoldLastsUntilTheRoomIsUsedOrItExpires(t *testing.T) { t.Error("a hold outlived its time") } } + +func TestOnlyTheReplicasThatMakeTheRoomAreStopped(t *testing.T) { + // Two half-used nodes. "idle" on node 1 is on demand and long unused, so + // it is first in line; but the critical model needs node 2's other half, + // which only stopping "chat" frees. + idle := catalog.Model{Name: "idle", Source: "/m", Scaling: catalog.Scaling{Min: 0, Max: 1}, + Profiles: []catalog.Profile{{Name: "tp4", Engine: "vllm", GPUClass: "h100-80g", TP: 4}}} + chat := catalog.Model{Name: "chat", Source: "/m", Scaling: catalog.Scaling{Min: 1, Max: 1}, + Profiles: []catalog.Profile{{Name: "tp6", Engine: "vllm", GPUClass: "h100-80g", TP: 6}}} + vital := catalog.Model{Name: "vital", Source: "/m", Class: catalog.ClassCritical, + Scaling: catalog.Scaling{Min: 1, Max: 1}, + Profiles: []catalog.Profile{{Name: "tp8", Engine: "vllm", GPUClass: "h100-80g", TP: 8}}} + h := newFleetHarness(t, []catalog.Model{idle, chat}, gpuNode("1", 6), gpuNode("2", 8)) + h.r.Want("idle") + h.settle(t, 4) + var idleNode, chatNode string + for _, rep := range h.r.Replicas() { + switch rep.Model { + case "idle": + idleNode = rep.Shards[0].NodeID + case "chat": + chatNode = rep.Shards[0].NodeID + } + } + if idleNode == "" || chatNode != "2" { + t.Fatalf("setup: idle on %q, chat on %q; chat must hold node 2: %+v", idleNode, chatNode, h.r.Replicas()) + } + h.r.mu.Lock() + for _, rep := range h.r.replicas { + rep.LastUsedAt = h.r.now().Add(-time.Hour) + } + h.r.mu.Unlock() + h.advance(h.r.cfg.MinLifetime + time.Minute) + cat, _ := catalog.New([]catalog.Model{idle, chat, vital}) + h.r.cat.Store(cat) + + h.r.Reconcile(t.Context()) + var stopped []string + for _, rep := range h.r.Replicas() { + if rep.PreemptedFor == "vital" { + stopped = append(stopped, rep.Model) + } + } + if len(stopped) != 1 || stopped[0] != "chat" { + t.Errorf("stopped %v; only chat's replica makes room for a whole node", stopped) + } +}