Stop only the replicas that make the room
Candidates were added in order until the model fitted, and every one added was stopped, including those tried first that freed room on the wrong node. A set that works is now pruned to the replicas it needs. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
78e41460f9
commit
0e73f37128
3 files changed
+77
-9
No files matched your search
+4
-1
@@ -72,7 +72,10 @@ these rules:
|
||||
the same within the class, then lower classes' minimums; the longest unused
|
||||
first within each.
|
||||
- The set is tried on a copy of the inventory first, at most three replicas,
|
||||
and nothing is stopped unless it makes the room.
|
||||
and nothing is stopped unless it makes the room. Of a set that does, each
|
||||
replica whose stopping turns out not to be needed is left running, so an
|
||||
idle replica tried first on the wrong node is not stopped with the one that
|
||||
made the room.
|
||||
|
||||
A replica stopped this way is drained and given 30 seconds to finish what it
|
||||
is serving. Its model is held back for up to five minutes, or until the model
|
||||
|
||||
@@ -54,28 +54,46 @@ func (r *Reconciler) preemptFor(ctx context.Context, m catalog.Model, avoid []st
|
||||
if len(cands) == 0 {
|
||||
return false
|
||||
}
|
||||
trial := r.inv.Clone()
|
||||
sched := placement.New(trial)
|
||||
// Candidates in order until the model fits; then every one whose
|
||||
// stopping turns out not to be needed is kept after all. Without that
|
||||
// second step an idle replica on the wrong node, tried first, would be
|
||||
// stopped along with the one that actually made the room.
|
||||
var chosen []victim
|
||||
for _, v := range cands {
|
||||
if len(chosen) == maxVictims {
|
||||
break
|
||||
}
|
||||
trial.ReleaseReplica(v.ID)
|
||||
chosen = append(chosen, v)
|
||||
res := sched.Place(r.orderedProfiles(m), avoid)
|
||||
if res.Placement == nil && len(avoid) > 0 {
|
||||
res = sched.Place(r.orderedProfiles(m), nil)
|
||||
}
|
||||
if res.Placement == nil {
|
||||
if !r.fitsWithout(m, avoid, chosen) {
|
||||
continue
|
||||
}
|
||||
for i := len(chosen) - 1; i >= 0 && len(chosen) > 1; i-- {
|
||||
fewer := append(append([]victim(nil), chosen[:i]...), chosen[i+1:]...)
|
||||
if r.fitsWithout(m, avoid, fewer) {
|
||||
chosen = fewer
|
||||
}
|
||||
}
|
||||
r.preempt(ctx, m, chosen)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// fitsWithout reports whether m could be placed with these replicas gone,
|
||||
// tried on a copy of the inventory.
|
||||
func (r *Reconciler) fitsWithout(m catalog.Model, avoid []string, gone []victim) bool {
|
||||
trial := r.inv.Clone()
|
||||
for _, v := range gone {
|
||||
trial.ReleaseReplica(v.ID)
|
||||
}
|
||||
sched := placement.New(trial)
|
||||
res := sched.Place(r.orderedProfiles(m), avoid)
|
||||
if res.Placement == nil && len(avoid) > 0 {
|
||||
res = sched.Place(r.orderedProfiles(m), nil)
|
||||
}
|
||||
return res.Placement != nil
|
||||
}
|
||||
|
||||
// preempt drains the chosen replicas and holds their models back.
|
||||
func (r *Reconciler) preempt(ctx context.Context, m catalog.Model, chosen []victim) {
|
||||
now := r.now()
|
||||
|
||||
@@ -235,3 +235,50 @@ func TestAHoldLastsUntilTheRoomIsUsedOrItExpires(t *testing.T) {
|
||||
t.Error("a hold outlived its time")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyTheReplicasThatMakeTheRoomAreStopped(t *testing.T) {
|
||||
// Two half-used nodes. "idle" on node 1 is on demand and long unused, so
|
||||
// it is first in line; but the critical model needs node 2's other half,
|
||||
// which only stopping "chat" frees.
|
||||
idle := catalog.Model{Name: "idle", Source: "/m", Scaling: catalog.Scaling{Min: 0, Max: 1},
|
||||
Profiles: []catalog.Profile{{Name: "tp4", Engine: "vllm", GPUClass: "h100-80g", TP: 4}}}
|
||||
chat := catalog.Model{Name: "chat", Source: "/m", Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||
Profiles: []catalog.Profile{{Name: "tp6", Engine: "vllm", GPUClass: "h100-80g", TP: 6}}}
|
||||
vital := catalog.Model{Name: "vital", Source: "/m", Class: catalog.ClassCritical,
|
||||
Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||
Profiles: []catalog.Profile{{Name: "tp8", Engine: "vllm", GPUClass: "h100-80g", TP: 8}}}
|
||||
h := newFleetHarness(t, []catalog.Model{idle, chat}, gpuNode("1", 6), gpuNode("2", 8))
|
||||
h.r.Want("idle")
|
||||
h.settle(t, 4)
|
||||
var idleNode, chatNode string
|
||||
for _, rep := range h.r.Replicas() {
|
||||
switch rep.Model {
|
||||
case "idle":
|
||||
idleNode = rep.Shards[0].NodeID
|
||||
case "chat":
|
||||
chatNode = rep.Shards[0].NodeID
|
||||
}
|
||||
}
|
||||
if idleNode == "" || chatNode != "2" {
|
||||
t.Fatalf("setup: idle on %q, chat on %q; chat must hold node 2: %+v", idleNode, chatNode, h.r.Replicas())
|
||||
}
|
||||
h.r.mu.Lock()
|
||||
for _, rep := range h.r.replicas {
|
||||
rep.LastUsedAt = h.r.now().Add(-time.Hour)
|
||||
}
|
||||
h.r.mu.Unlock()
|
||||
h.advance(h.r.cfg.MinLifetime + time.Minute)
|
||||
cat, _ := catalog.New([]catalog.Model{idle, chat, vital})
|
||||
h.r.cat.Store(cat)
|
||||
|
||||
h.r.Reconcile(t.Context())
|
||||
var stopped []string
|
||||
for _, rep := range h.r.Replicas() {
|
||||
if rep.PreemptedFor == "vital" {
|
||||
stopped = append(stopped, rep.Model)
|
||||
}
|
||||
}
|
||||
if len(stopped) != 1 || stopped[0] != "chat" {
|
||||
t.Errorf("stopped %v; only chat's replica makes room for a whole node", stopped)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user