Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e7a8105aa4 | ||
|
|
d0e9813ce9 | ||
|
|
0e73f37128 |
No files matched your search
+5
-2
@@ -70,9 +70,12 @@ these rules:
|
||||
loaded, nor one spanning several nodes.
|
||||
- Idle on-demand replicas go first, then what lower classes grew into, then
|
||||
the same within the class, then lower classes' minimums; the longest unused
|
||||
first within each.
|
||||
first within each, and of equally idle ones the smallest.
|
||||
- The set is tried on a copy of the inventory first, at most three replicas,
|
||||
and nothing is stopped unless it makes the room.
|
||||
and nothing is stopped unless it makes the room. Of a set that does, each
|
||||
replica whose stopping turns out not to be needed is left running, so an
|
||||
idle replica tried first on the wrong node is not stopped with the one that
|
||||
made the room.
|
||||
|
||||
A replica stopped this way is drained and given 30 seconds to finish what it
|
||||
is serving. Its model is held back for up to five minutes, or until the model
|
||||
|
||||
@@ -29,6 +29,9 @@ type victim struct {
|
||||
AboveMin bool
|
||||
Tier int
|
||||
Idle time.Duration
|
||||
// Size is what stopping it frees, for choosing the smaller of two
|
||||
// otherwise equal victims.
|
||||
Size quota.Amount
|
||||
}
|
||||
|
||||
// preemptFor stops replicas of lower classes when a model cannot otherwise be
|
||||
@@ -54,28 +57,46 @@ func (r *Reconciler) preemptFor(ctx context.Context, m catalog.Model, avoid []st
|
||||
if len(cands) == 0 {
|
||||
return false
|
||||
}
|
||||
trial := r.inv.Clone()
|
||||
sched := placement.New(trial)
|
||||
// Candidates in order until the model fits; then every one whose
|
||||
// stopping turns out not to be needed is kept after all. Without that
|
||||
// second step an idle replica on the wrong node, tried first, would be
|
||||
// stopped along with the one that actually made the room.
|
||||
var chosen []victim
|
||||
for _, v := range cands {
|
||||
if len(chosen) == maxVictims {
|
||||
break
|
||||
}
|
||||
trial.ReleaseReplica(v.ID)
|
||||
chosen = append(chosen, v)
|
||||
res := sched.Place(r.orderedProfiles(m), avoid)
|
||||
if res.Placement == nil && len(avoid) > 0 {
|
||||
res = sched.Place(r.orderedProfiles(m), nil)
|
||||
}
|
||||
if res.Placement == nil {
|
||||
if !r.fitsWithout(m, avoid, chosen) {
|
||||
continue
|
||||
}
|
||||
for i := len(chosen) - 1; i >= 0 && len(chosen) > 1; i-- {
|
||||
fewer := append(append([]victim(nil), chosen[:i]...), chosen[i+1:]...)
|
||||
if r.fitsWithout(m, avoid, fewer) {
|
||||
chosen = fewer
|
||||
}
|
||||
}
|
||||
r.preempt(ctx, m, chosen)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// fitsWithout reports whether m could be placed with these replicas gone,
|
||||
// tried on a copy of the inventory.
|
||||
func (r *Reconciler) fitsWithout(m catalog.Model, avoid []string, gone []victim) bool {
|
||||
trial := r.inv.Clone()
|
||||
for _, v := range gone {
|
||||
trial.ReleaseReplica(v.ID)
|
||||
}
|
||||
sched := placement.New(trial)
|
||||
res := sched.Place(r.orderedProfiles(m), avoid)
|
||||
if res.Placement == nil && len(avoid) > 0 {
|
||||
res = sched.Place(r.orderedProfiles(m), nil)
|
||||
}
|
||||
return res.Placement != nil
|
||||
}
|
||||
|
||||
// preempt drains the chosen replicas and holds their models back.
|
||||
func (r *Reconciler) preempt(ctx context.Context, m catalog.Model, chosen []victim) {
|
||||
now := r.now()
|
||||
@@ -245,15 +266,32 @@ func (r *Reconciler) victims(m catalog.Model) []victim {
|
||||
if len(nodesOf(rep)) > 1 {
|
||||
continue // see makeRoomFor: a multi-node replica is its own decision
|
||||
}
|
||||
prof, _ := other.ProfileByName(rep.Profile)
|
||||
var size quota.Amount
|
||||
for _, sh := range rep.Shards {
|
||||
size = size.Plus(quota.Amount{GPUs: len(sh.GPUIndices), CPUCores: cpuCores(prof), MemoryGB: prof.MinMemoryGB})
|
||||
}
|
||||
out = append(out, victim{ID: rep.ID, Model: model, Class: class,
|
||||
AboveMin: aboveMin, Tier: tier, Idle: idleFor(rep, now)})
|
||||
AboveMin: aboveMin, Tier: tier, Idle: idleFor(rep, now), Size: size})
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool {
|
||||
if out[i].Tier != out[j].Tier {
|
||||
return out[i].Tier < out[j].Tier
|
||||
a, b := out[i], out[j]
|
||||
if a.Tier != b.Tier {
|
||||
return a.Tier < b.Tier
|
||||
}
|
||||
return out[i].Idle > out[j].Idle
|
||||
if a.Idle != b.Idle {
|
||||
return a.Idle > b.Idle
|
||||
}
|
||||
// The smaller first: the least served is lost if it is enough, and
|
||||
// the pruning after keeps a larger one only when it is needed.
|
||||
if a.Size.GPUs != b.Size.GPUs {
|
||||
return a.Size.GPUs < b.Size.GPUs
|
||||
}
|
||||
if a.Size.MemoryGB != b.Size.MemoryGB {
|
||||
return a.Size.MemoryGB < b.Size.MemoryGB
|
||||
}
|
||||
return a.ID < b.ID
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -235,3 +235,80 @@ func TestAHoldLastsUntilTheRoomIsUsedOrItExpires(t *testing.T) {
|
||||
t.Error("a hold outlived its time")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyTheReplicasThatMakeTheRoomAreStopped(t *testing.T) {
|
||||
// Two half-used nodes. "idle" on node 1 is on demand and long unused, so
|
||||
// it is first in line; but the critical model needs node 2's other half,
|
||||
// which only stopping "chat" frees.
|
||||
idle := catalog.Model{Name: "idle", Source: "/m", Scaling: catalog.Scaling{Min: 0, Max: 1},
|
||||
Profiles: []catalog.Profile{{Name: "tp4", Engine: "vllm", GPUClass: "h100-80g", TP: 4}}}
|
||||
chat := catalog.Model{Name: "chat", Source: "/m", Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||
Profiles: []catalog.Profile{{Name: "tp6", Engine: "vllm", GPUClass: "h100-80g", TP: 6}}}
|
||||
vital := catalog.Model{Name: "vital", Source: "/m", Class: catalog.ClassCritical,
|
||||
Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||
Profiles: []catalog.Profile{{Name: "tp8", Engine: "vllm", GPUClass: "h100-80g", TP: 8}}}
|
||||
h := newFleetHarness(t, []catalog.Model{idle, chat}, gpuNode("1", 6), gpuNode("2", 8))
|
||||
h.r.Want("idle")
|
||||
h.settle(t, 4)
|
||||
var idleNode, chatNode string
|
||||
for _, rep := range h.r.Replicas() {
|
||||
switch rep.Model {
|
||||
case "idle":
|
||||
idleNode = rep.Shards[0].NodeID
|
||||
case "chat":
|
||||
chatNode = rep.Shards[0].NodeID
|
||||
}
|
||||
}
|
||||
if idleNode == "" || chatNode != "2" {
|
||||
t.Fatalf("setup: idle on %q, chat on %q; chat must hold node 2: %+v", idleNode, chatNode, h.r.Replicas())
|
||||
}
|
||||
h.r.mu.Lock()
|
||||
for _, rep := range h.r.replicas {
|
||||
rep.LastUsedAt = h.r.now().Add(-time.Hour)
|
||||
}
|
||||
h.r.mu.Unlock()
|
||||
h.advance(h.r.cfg.MinLifetime + time.Minute)
|
||||
cat, _ := catalog.New([]catalog.Model{idle, chat, vital})
|
||||
h.r.cat.Store(cat)
|
||||
|
||||
h.r.Reconcile(t.Context())
|
||||
var stopped []string
|
||||
for _, rep := range h.r.Replicas() {
|
||||
if rep.PreemptedFor == "vital" {
|
||||
stopped = append(stopped, rep.Model)
|
||||
}
|
||||
}
|
||||
if len(stopped) != 1 || stopped[0] != "chat" {
|
||||
t.Errorf("stopped %v; only chat's replica makes room for a whole node", stopped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOfTwoThatWouldEachMakeRoomTheSmallerIsStopped(t *testing.T) {
|
||||
cpu := inventory.Node{ID: "a", MgmtAddr: "10.0.0.1", State: inventory.Ready, MemoryTotalMB: 31 * 1024, CPUCount: 8}
|
||||
big := catalog.Model{Name: "big", Source: "/m", Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 12, CPUCores: 4}}}
|
||||
small := catalog.Model{Name: "small", Source: "/m", Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 2, CPUCores: 1}}}
|
||||
vital := catalog.Model{Name: "vital", Source: "/m", Class: catalog.ClassCritical,
|
||||
Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 15, CPUCores: 4}}}
|
||||
// 8 cores: big 4 + small 1 leaves 3, and vital needs 4. Stopping either
|
||||
// is enough; stopping big loses far more.
|
||||
for range 5 { // the choice must not depend on map order
|
||||
h := newFleetHarness(t, []catalog.Model{big, small}, cpu)
|
||||
h.settle(t, 3)
|
||||
h.advance(h.r.cfg.MinLifetime + time.Minute)
|
||||
cat, _ := catalog.New([]catalog.Model{big, small, vital})
|
||||
h.r.cat.Store(cat)
|
||||
h.r.Reconcile(t.Context())
|
||||
var stopped []string
|
||||
for _, rep := range h.r.Replicas() {
|
||||
if rep.PreemptedFor == "vital" {
|
||||
stopped = append(stopped, rep.Model)
|
||||
}
|
||||
}
|
||||
if len(stopped) != 1 || stopped[0] != "small" {
|
||||
t.Fatalf("stopped %v; the small replica alone makes room", stopped)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -755,10 +755,10 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
|
||||
r.mu.Lock()
|
||||
rep, ok := r.replicas[id]
|
||||
var expired bool
|
||||
grace, preemptedFor := r.cfg.DrainGrace, ""
|
||||
if ok {
|
||||
grace := r.cfg.DrainGrace
|
||||
if rep.PreemptedFor != "" {
|
||||
grace = r.cfg.PreemptGrace
|
||||
grace, preemptedFor = r.cfg.PreemptGrace, rep.PreemptedFor
|
||||
}
|
||||
expired = !rep.DrainingAt.IsZero() && r.now().Sub(rep.DrainingAt) > grace
|
||||
}
|
||||
@@ -768,11 +768,15 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
|
||||
case !ok:
|
||||
case replaced:
|
||||
r.stopReplica(ctx, id)
|
||||
case expired && preemptedFor != "":
|
||||
r.log.Info("stopping a preempted replica", "replica", id,
|
||||
"for", preemptedFor, "after", grace)
|
||||
r.stopReplica(ctx, id)
|
||||
case expired:
|
||||
// The replacement never arrived — often because nowhere else can
|
||||
// host it. Draining forever would leave the node undrainable.
|
||||
r.log.Warn("stopping a drained replica without a replacement",
|
||||
"replica", id, "after", r.cfg.DrainGrace)
|
||||
"replica", id, "after", grace)
|
||||
r.stopReplica(ctx, id)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user