diff --git a/CHANGELOG.md b/CHANGELOG.md index a59cc6f..b19fa4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,7 +70,7 @@ these rules: loaded, nor one spanning several nodes. - Idle on-demand replicas go first, then what lower classes grew into, then the same within the class, then lower classes' minimums; the longest unused - first within each. + first within each, and of equally idle ones the smallest. - The set is tried on a copy of the inventory first, at most three replicas, and nothing is stopped unless it makes the room. Of a set that does, each replica whose stopping turns out not to be needed is left running, so an diff --git a/internal/control/reconciler/preempt.go b/internal/control/reconciler/preempt.go index 155ff92..497f183 100644 --- a/internal/control/reconciler/preempt.go +++ b/internal/control/reconciler/preempt.go @@ -29,6 +29,9 @@ type victim struct { AboveMin bool Tier int Idle time.Duration + // Size is what stopping it frees, for choosing the smaller of two + // otherwise equal victims. + Size quota.Amount } // preemptFor stops replicas of lower classes when a model cannot otherwise be @@ -263,15 +266,32 @@ func (r *Reconciler) victims(m catalog.Model) []victim { if len(nodesOf(rep)) > 1 { continue // see makeRoomFor: a multi-node replica is its own decision } + prof, _ := other.ProfileByName(rep.Profile) + var size quota.Amount + for _, sh := range rep.Shards { + size = size.Plus(quota.Amount{GPUs: len(sh.GPUIndices), CPUCores: cpuCores(prof), MemoryGB: prof.MinMemoryGB}) + } out = append(out, victim{ID: rep.ID, Model: model, Class: class, - AboveMin: aboveMin, Tier: tier, Idle: idleFor(rep, now)}) + AboveMin: aboveMin, Tier: tier, Idle: idleFor(rep, now), Size: size}) } } sort.SliceStable(out, func(i, j int) bool { - if out[i].Tier != out[j].Tier { - return out[i].Tier < out[j].Tier + a, b := out[i], out[j] + if a.Tier != b.Tier { + return a.Tier < b.Tier } - return out[i].Idle > out[j].Idle + if a.Idle != b.Idle { + return a.Idle > b.Idle + } + // The smaller first: the least served is lost if it is enough, and + // the pruning after keeps a larger one only when it is needed. + if a.Size.GPUs != b.Size.GPUs { + return a.Size.GPUs < b.Size.GPUs + } + if a.Size.MemoryGB != b.Size.MemoryGB { + return a.Size.MemoryGB < b.Size.MemoryGB + } + return a.ID < b.ID }) return out } diff --git a/internal/control/reconciler/preempt_test.go b/internal/control/reconciler/preempt_test.go index d7eb2a8..f095480 100644 --- a/internal/control/reconciler/preempt_test.go +++ b/internal/control/reconciler/preempt_test.go @@ -282,3 +282,33 @@ func TestOnlyTheReplicasThatMakeTheRoomAreStopped(t *testing.T) { t.Errorf("stopped %v; only chat's replica makes room for a whole node", stopped) } } + +func TestOfTwoThatWouldEachMakeRoomTheSmallerIsStopped(t *testing.T) { + cpu := inventory.Node{ID: "a", MgmtAddr: "10.0.0.1", State: inventory.Ready, MemoryTotalMB: 31 * 1024, CPUCount: 8} + big := catalog.Model{Name: "big", Source: "/m", Scaling: catalog.Scaling{Min: 1, Max: 1}, + Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 12, CPUCores: 4}}} + small := catalog.Model{Name: "small", Source: "/m", Scaling: catalog.Scaling{Min: 1, Max: 1}, + Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 2, CPUCores: 1}}} + vital := catalog.Model{Name: "vital", Source: "/m", Class: catalog.ClassCritical, + Scaling: catalog.Scaling{Min: 1, Max: 1}, + Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 15, CPUCores: 4}}} + // 8 cores: big 4 + small 1 leaves 3, and vital needs 4. Stopping either + // is enough; stopping big loses far more. + for range 5 { // the choice must not depend on map order + h := newFleetHarness(t, []catalog.Model{big, small}, cpu) + h.settle(t, 3) + h.advance(h.r.cfg.MinLifetime + time.Minute) + cat, _ := catalog.New([]catalog.Model{big, small, vital}) + h.r.cat.Store(cat) + h.r.Reconcile(t.Context()) + var stopped []string + for _, rep := range h.r.Replicas() { + if rep.PreemptedFor == "vital" { + stopped = append(stopped, rep.Model) + } + } + if len(stopped) != 1 || stopped[0] != "small" { + t.Fatalf("stopped %v; the small replica alone makes room", stopped) + } + } +}