diff --git a/CHANGELOG.md b/CHANGELOG.md index f49cd574..cf877128 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -86,6 +86,11 @@ APIs change between minors without a deprecation period. - A picture in a prompt was counted as its base64 by the context check, so one 1280x720 PNG was "138 thousand tokens" and a vision model refused it; a picture or a sound now counts as about a thousand. +- An on-demand model of a higher class was evicted, idle, for a lower-class + one two minutes into a lull; it keeps the card through its idle timeout, + and a model waiting on it is told so. +- A model removed from the catalog stayed wanted, so added again it loaded + unasked. ## v0.14.3 — 2026-10-10 diff --git a/internal/control/reconciler/evict.go b/internal/control/reconciler/evict.go index 212607ab..0bbbb15d 100644 --- a/internal/control/reconciler/evict.go +++ b/internal/control/reconciler/evict.go @@ -90,6 +90,11 @@ func (r *Reconciler) evictable(wantedBy string) []evictionCandidate { cat := r.cat.Load() now := r.now() + wantRank := -1 + if w, ok := cat.Resolve(wantedBy); ok { + wantRank = catalog.ClassRank(w.EffectiveClass()) + } + r.mu.Lock() defer r.mu.Unlock() @@ -102,6 +107,12 @@ func (r *Reconciler) evictable(wantedBy string) []evictionCandidate { if !ok || !other.Scaling.OnDemand() { continue } + // Not for a model of a lower class: the one put first keeps the + // card through its idle timeout, rather than losing it two minutes + // into a lull to whatever was asked for next. + if catalog.ClassRank(other.EffectiveClass()) > wantRank { + continue + } // Unknown is not idle. A replica the router has said nothing about — // after a restart, that is all of them — must not be evicted on the // strength of a fact nobody has reported. diff --git a/internal/control/reconciler/health.go b/internal/control/reconciler/health.go index 38ea71d3..dbdb5dc6 100644 --- a/internal/control/reconciler/health.go +++ b/internal/control/reconciler/health.go @@ -201,13 +201,17 @@ func (r *Reconciler) WaitFor(model string) (heldBy string, eta time.Duration, ok } h := holder{v: victim{ID: rep.ID, Model: rep.Model}} if !rep.LastUsedAt.IsZero() { - evict := rep.LastUsedAt.Add(r.cfg.EvictionGrace) - if !rep.ReadyAt.IsZero() && rep.ReadyAt.Add(r.cfg.MinLifetime).After(evict) { - evict = rep.ReadyAt.Add(r.cfg.MinLifetime) - } - h.release = evict - if idle := rep.LastUsedAt.Add(r.idleTimeout(other)); idle.Before(evict) { - h.release = idle + h.release = rep.LastUsedAt.Add(r.idleTimeout(other)) + // Evicted sooner, when it is no higher a class than the model + // waiting; one put first is waited for to its idle timeout. + if catalog.ClassRank(other.EffectiveClass()) <= catalog.ClassRank(m.EffectiveClass()) { + evict := rep.LastUsedAt.Add(r.cfg.EvictionGrace) + if !rep.ReadyAt.IsZero() && rep.ReadyAt.Add(r.cfg.MinLifetime).After(evict) { + evict = rep.ReadyAt.Add(r.cfg.MinLifetime) + } + if evict.Before(h.release) { + h.release = evict + } } } holders = append(holders, h) diff --git a/internal/control/reconciler/reconciler_test.go b/internal/control/reconciler/reconciler_test.go index 0467828d..56ef40f2 100644 --- a/internal/control/reconciler/reconciler_test.go +++ b/internal/control/reconciler/reconciler_test.go @@ -1337,6 +1337,48 @@ func TestAModelRemovedWhileRunningIsStopped(t *testing.T) { } } +func TestAModelPutFirstIsNotEvictedForALowerClassOne(t *testing.T) { + // The VLM is put first: a picture or 3D model asked for in a lull must + // wait for its idle timeout, not take the card two minutes in. + h := newHarness(t, testModel(1), gpuNode("1", 4)) + cat := twoOnDemandModels(t) + ms := cat.Models() + for i := range ms { + if ms[i].Name == "llama" { + ms[i].Class = catalog.ClassStandard + } else { + ms[i].Class = catalog.ClassOpportunistic + } + } + built, err := catalog.New(ms) + if err != nil { + t.Fatal(err) + } + h.r.cat.Store(built) + h.r.Want("llama") + h.r.Reconcile(t.Context()) + h.markReady() + reps := h.r.Replicas() + h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()}) + h.advance(DefaultConfig().MinLifetime + DefaultConfig().EvictionGrace + time.Minute) + + if got := h.r.evictable("second"); len(got) != 0 { + t.Errorf("a standard model was offered to an opportunistic one: %+v", got) + } + h.r.Want("second") + h.r.Reconcile(t.Context()) + // It waits for the idle timeout (an hour here), and is told so. + _, eta, ok := h.r.WaitFor("second") + want := time.Hour - (DefaultConfig().MinLifetime + DefaultConfig().EvictionGrace + time.Minute) + if !ok || eta != want { + t.Errorf("WaitFor: %s %v, want %s", eta, ok, want) + } + // The other way, it is offered. + if got := h.r.evictable("llama"); len(got) != 0 { + t.Errorf("nothing of second runs, yet %+v was offered", got) + } +} + func TestAnOnDemandModelRemovedAndAddedAgainIsNotLoadedUnasked(t *testing.T) { // The VLM, removed and added again to give it a licence, came back // loading on its own: still on the wanted list from before its removal.