Stop only the replicas that make the room

Candidates were added in order until the model fitted, and every one added
was stopped, including those tried first that freed room on the wrong node.
A set that works is now pruned to the replicas it needs.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 18:21:58 +02:00
1 parent 78e41460f9
commit 0e73f37128
3 files changed
+77 -9

No files matched your search

+4 -1
View File
@@ -72,7 +72,10 @@ these rules:
the same within the class, then lower classes' minimums; the longest unused
first within each.
- The set is tried on a copy of the inventory first, at most three replicas,
and nothing is stopped unless it makes the room.
and nothing is stopped unless it makes the room. Of a set that does, each
replica whose stopping turns out not to be needed is left running, so an
idle replica tried first on the wrong node is not stopped with the one that
made the room.
A replica stopped this way is drained and given 30 seconds to finish what it
is serving. Its model is held back for up to five minutes, or until the model
+26 -8
View File
@@ -54,28 +54,46 @@ func (r *Reconciler) preemptFor(ctx context.Context, m catalog.Model, avoid []st
if len(cands) == 0 {
return false
}
trial := r.inv.Clone()
sched := placement.New(trial)
// Candidates in order until the model fits; then every one whose
// stopping turns out not to be needed is kept after all. Without that
// second step an idle replica on the wrong node, tried first, would be
// stopped along with the one that actually made the room.
var chosen []victim
for _, v := range cands {
if len(chosen) == maxVictims {
break
}
trial.ReleaseReplica(v.ID)
chosen = append(chosen, v)
res := sched.Place(r.orderedProfiles(m), avoid)
if res.Placement == nil && len(avoid) > 0 {
res = sched.Place(r.orderedProfiles(m), nil)
}
if res.Placement == nil {
if !r.fitsWithout(m, avoid, chosen) {
continue
}
for i := len(chosen) - 1; i >= 0 && len(chosen) > 1; i-- {
fewer := append(append([]victim(nil), chosen[:i]...), chosen[i+1:]...)
if r.fitsWithout(m, avoid, fewer) {
chosen = fewer
}
}
r.preempt(ctx, m, chosen)
return true
}
return false
}
// fitsWithout reports whether m could be placed with these replicas gone,
// tried on a copy of the inventory.
func (r *Reconciler) fitsWithout(m catalog.Model, avoid []string, gone []victim) bool {
trial := r.inv.Clone()
for _, v := range gone {
trial.ReleaseReplica(v.ID)
}
sched := placement.New(trial)
res := sched.Place(r.orderedProfiles(m), avoid)
if res.Placement == nil && len(avoid) > 0 {
res = sched.Place(r.orderedProfiles(m), nil)
}
return res.Placement != nil
}
// preempt drains the chosen replicas and holds their models back.
func (r *Reconciler) preempt(ctx context.Context, m catalog.Model, chosen []victim) {
now := r.now()
@@ -235,3 +235,50 @@ func TestAHoldLastsUntilTheRoomIsUsedOrItExpires(t *testing.T) {
t.Error("a hold outlived its time")
}
}
func TestOnlyTheReplicasThatMakeTheRoomAreStopped(t *testing.T) {
// Two half-used nodes. "idle" on node 1 is on demand and long unused, so
// it is first in line; but the critical model needs node 2's other half,
// which only stopping "chat" frees.
idle := catalog.Model{Name: "idle", Source: "/m", Scaling: catalog.Scaling{Min: 0, Max: 1},
Profiles: []catalog.Profile{{Name: "tp4", Engine: "vllm", GPUClass: "h100-80g", TP: 4}}}
chat := catalog.Model{Name: "chat", Source: "/m", Scaling: catalog.Scaling{Min: 1, Max: 1},
Profiles: []catalog.Profile{{Name: "tp6", Engine: "vllm", GPUClass: "h100-80g", TP: 6}}}
vital := catalog.Model{Name: "vital", Source: "/m", Class: catalog.ClassCritical,
Scaling: catalog.Scaling{Min: 1, Max: 1},
Profiles: []catalog.Profile{{Name: "tp8", Engine: "vllm", GPUClass: "h100-80g", TP: 8}}}
h := newFleetHarness(t, []catalog.Model{idle, chat}, gpuNode("1", 6), gpuNode("2", 8))
h.r.Want("idle")
h.settle(t, 4)
var idleNode, chatNode string
for _, rep := range h.r.Replicas() {
switch rep.Model {
case "idle":
idleNode = rep.Shards[0].NodeID
case "chat":
chatNode = rep.Shards[0].NodeID
}
}
if idleNode == "" || chatNode != "2" {
t.Fatalf("setup: idle on %q, chat on %q; chat must hold node 2: %+v", idleNode, chatNode, h.r.Replicas())
}
h.r.mu.Lock()
for _, rep := range h.r.replicas {
rep.LastUsedAt = h.r.now().Add(-time.Hour)
}
h.r.mu.Unlock()
h.advance(h.r.cfg.MinLifetime + time.Minute)
cat, _ := catalog.New([]catalog.Model{idle, chat, vital})
h.r.cat.Store(cat)
h.r.Reconcile(t.Context())
var stopped []string
for _, rep := range h.r.Replicas() {
if rep.PreemptedFor == "vital" {
stopped = append(stopped, rep.Model)
}
}
if len(stopped) != 1 || stopped[0] != "chat" {
t.Errorf("stopped %v; only chat's replica makes room for a whole node", stopped)
}
}