From 0e73f371282dda6613c2c0b752d01035bc097323 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 30 Sep 2026 18:21:58 +0200 Subject: [PATCH] Stop only the replicas that make the room Candidates were added in order until the model fitted, and every one added was stopped, including those tried first that freed room on the wrong node. A set that works is now pruned to the replicas it needs. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 5 ++- internal/control/reconciler/preempt.go | 34 +++++++++++---- internal/control/reconciler/preempt_test.go | 47 +++++++++++++++++++++ 3 files changed, 77 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7aa273d..a59cc6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,7 +72,10 @@ these rules: the same within the class, then lower classes' minimums; the longest unused first within each. - The set is tried on a copy of the inventory first, at most three replicas, - and nothing is stopped unless it makes the room. + and nothing is stopped unless it makes the room. Of a set that does, each + replica whose stopping turns out not to be needed is left running, so an + idle replica tried first on the wrong node is not stopped with the one that + made the room. A replica stopped this way is drained and given 30 seconds to finish what it is serving. Its model is held back for up to five minutes, or until the model diff --git a/internal/control/reconciler/preempt.go b/internal/control/reconciler/preempt.go index f8cea55..155ff92 100644 --- a/internal/control/reconciler/preempt.go +++ b/internal/control/reconciler/preempt.go @@ -54,28 +54,46 @@ func (r *Reconciler) preemptFor(ctx context.Context, m catalog.Model, avoid []st if len(cands) == 0 { return false } - trial := r.inv.Clone() - sched := placement.New(trial) + // Candidates in order until the model fits; then every one whose + // stopping turns out not to be needed is kept after all. Without that + // second step an idle replica on the wrong node, tried first, would be + // stopped along with the one that actually made the room. var chosen []victim for _, v := range cands { if len(chosen) == maxVictims { break } - trial.ReleaseReplica(v.ID) chosen = append(chosen, v) - res := sched.Place(r.orderedProfiles(m), avoid) - if res.Placement == nil && len(avoid) > 0 { - res = sched.Place(r.orderedProfiles(m), nil) - } - if res.Placement == nil { + if !r.fitsWithout(m, avoid, chosen) { continue } + for i := len(chosen) - 1; i >= 0 && len(chosen) > 1; i-- { + fewer := append(append([]victim(nil), chosen[:i]...), chosen[i+1:]...) + if r.fitsWithout(m, avoid, fewer) { + chosen = fewer + } + } r.preempt(ctx, m, chosen) return true } return false } +// fitsWithout reports whether m could be placed with these replicas gone, +// tried on a copy of the inventory. +func (r *Reconciler) fitsWithout(m catalog.Model, avoid []string, gone []victim) bool { + trial := r.inv.Clone() + for _, v := range gone { + trial.ReleaseReplica(v.ID) + } + sched := placement.New(trial) + res := sched.Place(r.orderedProfiles(m), avoid) + if res.Placement == nil && len(avoid) > 0 { + res = sched.Place(r.orderedProfiles(m), nil) + } + return res.Placement != nil +} + // preempt drains the chosen replicas and holds their models back. func (r *Reconciler) preempt(ctx context.Context, m catalog.Model, chosen []victim) { now := r.now() diff --git a/internal/control/reconciler/preempt_test.go b/internal/control/reconciler/preempt_test.go index 87cf858..d7eb2a8 100644 --- a/internal/control/reconciler/preempt_test.go +++ b/internal/control/reconciler/preempt_test.go @@ -235,3 +235,50 @@ func TestAHoldLastsUntilTheRoomIsUsedOrItExpires(t *testing.T) { t.Error("a hold outlived its time") } } + +func TestOnlyTheReplicasThatMakeTheRoomAreStopped(t *testing.T) { + // Two half-used nodes. "idle" on node 1 is on demand and long unused, so + // it is first in line; but the critical model needs node 2's other half, + // which only stopping "chat" frees. + idle := catalog.Model{Name: "idle", Source: "/m", Scaling: catalog.Scaling{Min: 0, Max: 1}, + Profiles: []catalog.Profile{{Name: "tp4", Engine: "vllm", GPUClass: "h100-80g", TP: 4}}} + chat := catalog.Model{Name: "chat", Source: "/m", Scaling: catalog.Scaling{Min: 1, Max: 1}, + Profiles: []catalog.Profile{{Name: "tp6", Engine: "vllm", GPUClass: "h100-80g", TP: 6}}} + vital := catalog.Model{Name: "vital", Source: "/m", Class: catalog.ClassCritical, + Scaling: catalog.Scaling{Min: 1, Max: 1}, + Profiles: []catalog.Profile{{Name: "tp8", Engine: "vllm", GPUClass: "h100-80g", TP: 8}}} + h := newFleetHarness(t, []catalog.Model{idle, chat}, gpuNode("1", 6), gpuNode("2", 8)) + h.r.Want("idle") + h.settle(t, 4) + var idleNode, chatNode string + for _, rep := range h.r.Replicas() { + switch rep.Model { + case "idle": + idleNode = rep.Shards[0].NodeID + case "chat": + chatNode = rep.Shards[0].NodeID + } + } + if idleNode == "" || chatNode != "2" { + t.Fatalf("setup: idle on %q, chat on %q; chat must hold node 2: %+v", idleNode, chatNode, h.r.Replicas()) + } + h.r.mu.Lock() + for _, rep := range h.r.replicas { + rep.LastUsedAt = h.r.now().Add(-time.Hour) + } + h.r.mu.Unlock() + h.advance(h.r.cfg.MinLifetime + time.Minute) + cat, _ := catalog.New([]catalog.Model{idle, chat, vital}) + h.r.cat.Store(cat) + + h.r.Reconcile(t.Context()) + var stopped []string + for _, rep := range h.r.Replicas() { + if rep.PreemptedFor == "vital" { + stopped = append(stopped, rep.Model) + } + } + if len(stopped) != 1 || stopped[0] != "chat" { + t.Errorf("stopped %v; only chat's replica makes room for a whole node", stopped) + } +}