How long an on-demand model holds a card is a live setting

On one card shared by flux and sf3d, a mesh asked for after a picture
waited out the picture model's five minutes of protection and two of idle.
on_demand.evict_after_idle and on_demand.min_lifetime set those for the
cluster, in force on the leader as they change; unset keeps 2m and 5m.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-14 08:58:38 +02:00
1 parent a9a2360f44
commit 1e7086e13e
13 files changed
+139 -7

No files matched your search

+33 -2
View File
@@ -120,7 +120,7 @@ func (r *Reconciler) evictable(wantedBy string) []evictionCandidate {
continue
}
idle := now.Sub(rep.LastUsedAt)
if idle < r.cfg.EvictionGrace {
if idle < r.evictionGrace() {
continue
}
// A replica that has only just started is left alone however idle it
@@ -128,7 +128,7 @@ func (r *Reconciler) evictable(wantedBy string) []evictionCandidate {
// otherwise take turns evicting each other, paying a model load each
// time — and a load is expensive enough that waiting is strictly
// better than winning.
if !rep.ReadyAt.IsZero() && now.Sub(rep.ReadyAt) < r.cfg.MinLifetime {
if !rep.ReadyAt.IsZero() && now.Sub(rep.ReadyAt) < r.minLifetime() {
continue
}
nodes := map[string]struct{}{}
@@ -156,3 +156,34 @@ func (r *Reconciler) EvictionReason(model string) string {
}
return "nothing idle enough to stop"
}
// evictionGrace is how long an on-demand replica must go unasked-for before
// another model may have its card: the cluster's setting, else the
// configuration's.
func (r *Reconciler) evictionGrace() time.Duration {
if d := time.Duration(r.evictGrace.Load()); d > 0 {
return d
}
return r.cfg.EvictionGrace
}
// minLifetime is how long a replica is safe from eviction once it is up.
func (r *Reconciler) minLifetime() time.Duration {
if d := time.Duration(r.minLife.Load()); d > 0 {
return d
}
return r.cfg.MinLifetime
}
// SetEvictionTimes puts the cluster's on_demand settings in force: how long
// an idle on-demand replica keeps its card, and how long a new one is safe.
// Zero keeps the configuration's.
//
// On one card shared by several models these are the wait between them: a
// 3D job after a picture waited out five minutes of a picture model it no
// longer needed.
func (r *Reconciler) SetEvictionTimes(afterIdle, minLifetime time.Duration) {
r.evictGrace.Store(int64(afterIdle))
r.minLife.Store(int64(minLifetime))
r.Notify()
}
+3 -3
View File
@@ -205,9 +205,9 @@ func (r *Reconciler) WaitFor(model string) (heldBy string, eta time.Duration, ok
// Evicted sooner, when it is no higher a class than the model
// waiting; one put first is waited for to its idle timeout.
if catalog.ClassRank(other.EffectiveClass()) <= catalog.ClassRank(m.EffectiveClass()) {
evict := rep.LastUsedAt.Add(r.cfg.EvictionGrace)
if !rep.ReadyAt.IsZero() && rep.ReadyAt.Add(r.cfg.MinLifetime).After(evict) {
evict = rep.ReadyAt.Add(r.cfg.MinLifetime)
evict := rep.LastUsedAt.Add(r.evictionGrace())
if !rep.ReadyAt.IsZero() && rep.ReadyAt.Add(r.minLifetime()).After(evict) {
evict = rep.ReadyAt.Add(r.minLifetime())
}
if evict.Before(h.release) {
h.release = evict
+2 -2
View File
@@ -249,7 +249,7 @@ func (r *Reconciler) victims(m catalog.Model) []victim {
aboveMin := i < above
tier := -1
switch {
case rank < myRank && other.Scaling.OnDemand() && idleFor(rep, now) >= r.cfg.EvictionGrace:
case rank < myRank && other.Scaling.OnDemand() && idleFor(rep, now) >= r.evictionGrace():
tier = 0
case rank < myRank && aboveMin:
tier = 2
@@ -275,7 +275,7 @@ func (r *Reconciler) victims(m catalog.Model) []victim {
}
// Just started: stopping it throws away a model load that has
// only now been paid for.
if !rep.ReadyAt.IsZero() && now.Sub(rep.ReadyAt) < r.cfg.MinLifetime {
if !rep.ReadyAt.IsZero() && now.Sub(rep.ReadyAt) < r.minLifetime() {
continue
}
if len(nodesOf(rep)) > 1 {
@@ -264,6 +264,9 @@ type Reconciler struct {
// because it is state about this moment and not about what the model is.
// A model nobody has wanted has no entry and is simply not running.
wanted map[string]time.Time
// evictGrace and minLife are the cluster's on_demand settings, as
// durations; zero keeps cfg's.
evictGrace, minLife atomic.Int64
// coldStart is how long each model's last replica took from placed to
// ready, for telling a request waiting on it about how long it waits.
coldStart map[string]time.Duration
@@ -1763,6 +1763,31 @@ func mustResolve(t *testing.T, h *harness, name string) catalog.Model {
return m
}
func TestTheClustersOnDemandSettingsShortenTheWaitForACard(t *testing.T) {
// One card, a picture model and a 3D model: a mesh after a picture
// waited out five minutes of the picture model's protection.
h := newHarness(t, testModel(1), gpuNode("1", 4))
h.r.cat.Store(twoOnDemandModels(t))
h.r.Want("llama")
h.r.Reconcile(t.Context())
h.markReady()
reps := h.r.Replicas()
h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()})
h.advance(90 * time.Second)
if got := h.r.evictable("second"); len(got) != 0 {
t.Fatalf("offered at 90 s with the defaults: %+v", got)
}
h.r.SetEvictionTimes(30*time.Second, time.Minute)
if got := h.r.evictable("second"); len(got) != 1 {
t.Errorf("not offered at 90 s with 30 s idle and 1 min lifetime: %+v", got)
}
// Zero goes back to the configuration's.
h.r.SetEvictionTimes(0, 0)
if got := h.r.evictable("second"); len(got) != 0 {
t.Errorf("offered after the settings were cleared: %+v", got)
}
}
func TestAFreshlyStartedReplicaIsNotEvicted(t *testing.T) {
// What stops two models on a cluster with room for one from taking turns
// evicting each other, paying a model load each time. A load is expensive