From fa6e68c8e86d2017834b26b47e6e8d497cb37149 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 11 Oct 2026 20:54:19 +0200 Subject: [PATCH] A model put first is not evicted, idle, for a lower class The VLM is to come first on the one card. Its class makes it take the card from flux, triposr or sf3d when asked for, but idle eviction ignored class: a 3D job two minutes into a lull would have taken the card from it. Idle eviction offers only replicas of no higher class than the model wanting room, and the wait such a model is told runs to the holder's idle timeout. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 5 +++ internal/control/reconciler/evict.go | 11 +++++ internal/control/reconciler/health.go | 18 ++++---- .../control/reconciler/reconciler_test.go | 42 +++++++++++++++++++ 4 files changed, 69 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f49cd574..cf877128 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -86,6 +86,11 @@ APIs change between minors without a deprecation period. - A picture in a prompt was counted as its base64 by the context check, so one 1280x720 PNG was "138 thousand tokens" and a vision model refused it; a picture or a sound now counts as about a thousand. +- An on-demand model of a higher class was evicted, idle, for a lower-class + one two minutes into a lull; it keeps the card through its idle timeout, + and a model waiting on it is told so. +- A model removed from the catalog stayed wanted, so added again it loaded + unasked. ## v0.14.3 — 2026-10-10 diff --git a/internal/control/reconciler/evict.go b/internal/control/reconciler/evict.go index 212607ab..0bbbb15d 100644 --- a/internal/control/reconciler/evict.go +++ b/internal/control/reconciler/evict.go @@ -90,6 +90,11 @@ func (r *Reconciler) evictable(wantedBy string) []evictionCandidate { cat := r.cat.Load() now := r.now() + wantRank := -1 + if w, ok := cat.Resolve(wantedBy); ok { + wantRank = catalog.ClassRank(w.EffectiveClass()) + } + r.mu.Lock() defer r.mu.Unlock() @@ -102,6 +107,12 @@ func (r *Reconciler) evictable(wantedBy string) []evictionCandidate { if !ok || !other.Scaling.OnDemand() { continue } + // Not for a model of a lower class: the one put first keeps the + // card through its idle timeout, rather than losing it two minutes + // into a lull to whatever was asked for next. + if catalog.ClassRank(other.EffectiveClass()) > wantRank { + continue + } // Unknown is not idle. A replica the router has said nothing about — // after a restart, that is all of them — must not be evicted on the // strength of a fact nobody has reported. diff --git a/internal/control/reconciler/health.go b/internal/control/reconciler/health.go index 38ea71d3..dbdb5dc6 100644 --- a/internal/control/reconciler/health.go +++ b/internal/control/reconciler/health.go @@ -201,13 +201,17 @@ func (r *Reconciler) WaitFor(model string) (heldBy string, eta time.Duration, ok } h := holder{v: victim{ID: rep.ID, Model: rep.Model}} if !rep.LastUsedAt.IsZero() { - evict := rep.LastUsedAt.Add(r.cfg.EvictionGrace) - if !rep.ReadyAt.IsZero() && rep.ReadyAt.Add(r.cfg.MinLifetime).After(evict) { - evict = rep.ReadyAt.Add(r.cfg.MinLifetime) - } - h.release = evict - if idle := rep.LastUsedAt.Add(r.idleTimeout(other)); idle.Before(evict) { - h.release = idle + h.release = rep.LastUsedAt.Add(r.idleTimeout(other)) + // Evicted sooner, when it is no higher a class than the model + // waiting; one put first is waited for to its idle timeout. + if catalog.ClassRank(other.EffectiveClass()) <= catalog.ClassRank(m.EffectiveClass()) { + evict := rep.LastUsedAt.Add(r.cfg.EvictionGrace) + if !rep.ReadyAt.IsZero() && rep.ReadyAt.Add(r.cfg.MinLifetime).After(evict) { + evict = rep.ReadyAt.Add(r.cfg.MinLifetime) + } + if evict.Before(h.release) { + h.release = evict + } } } holders = append(holders, h) diff --git a/internal/control/reconciler/reconciler_test.go b/internal/control/reconciler/reconciler_test.go index 0467828d..56ef40f2 100644 --- a/internal/control/reconciler/reconciler_test.go +++ b/internal/control/reconciler/reconciler_test.go @@ -1337,6 +1337,48 @@ func TestAModelRemovedWhileRunningIsStopped(t *testing.T) { } } +func TestAModelPutFirstIsNotEvictedForALowerClassOne(t *testing.T) { + // The VLM is put first: a picture or 3D model asked for in a lull must + // wait for its idle timeout, not take the card two minutes in. + h := newHarness(t, testModel(1), gpuNode("1", 4)) + cat := twoOnDemandModels(t) + ms := cat.Models() + for i := range ms { + if ms[i].Name == "llama" { + ms[i].Class = catalog.ClassStandard + } else { + ms[i].Class = catalog.ClassOpportunistic + } + } + built, err := catalog.New(ms) + if err != nil { + t.Fatal(err) + } + h.r.cat.Store(built) + h.r.Want("llama") + h.r.Reconcile(t.Context()) + h.markReady() + reps := h.r.Replicas() + h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()}) + h.advance(DefaultConfig().MinLifetime + DefaultConfig().EvictionGrace + time.Minute) + + if got := h.r.evictable("second"); len(got) != 0 { + t.Errorf("a standard model was offered to an opportunistic one: %+v", got) + } + h.r.Want("second") + h.r.Reconcile(t.Context()) + // It waits for the idle timeout (an hour here), and is told so. + _, eta, ok := h.r.WaitFor("second") + want := time.Hour - (DefaultConfig().MinLifetime + DefaultConfig().EvictionGrace + time.Minute) + if !ok || eta != want { + t.Errorf("WaitFor: %s %v, want %s", eta, ok, want) + } + // The other way, it is offered. + if got := h.r.evictable("llama"); len(got) != 0 { + t.Errorf("nothing of second runs, yet %+v was offered", got) + } +} + func TestAnOnDemandModelRemovedAndAddedAgainIsNotLoadedUnasked(t *testing.T) { // The VLM, removed and added again to give it a licence, came back // loading on its own: still on the wanted list from before its removal.