How long an on-demand model holds a card is a live setting
On one card shared by flux and sf3d, a mesh asked for after a picture waited out the picture model's five minutes of protection and two of idle. on_demand.evict_after_idle and on_demand.min_lifetime set those for the cluster, in force on the leader as they change; unset keeps 2m and 5m. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
a9a2360f44
commit
1e7086e13e
13 files changed
+139
-7
No files matched your search
@@ -120,7 +120,7 @@ func (r *Reconciler) evictable(wantedBy string) []evictionCandidate {
|
||||
continue
|
||||
}
|
||||
idle := now.Sub(rep.LastUsedAt)
|
||||
if idle < r.cfg.EvictionGrace {
|
||||
if idle < r.evictionGrace() {
|
||||
continue
|
||||
}
|
||||
// A replica that has only just started is left alone however idle it
|
||||
@@ -128,7 +128,7 @@ func (r *Reconciler) evictable(wantedBy string) []evictionCandidate {
|
||||
// otherwise take turns evicting each other, paying a model load each
|
||||
// time — and a load is expensive enough that waiting is strictly
|
||||
// better than winning.
|
||||
if !rep.ReadyAt.IsZero() && now.Sub(rep.ReadyAt) < r.cfg.MinLifetime {
|
||||
if !rep.ReadyAt.IsZero() && now.Sub(rep.ReadyAt) < r.minLifetime() {
|
||||
continue
|
||||
}
|
||||
nodes := map[string]struct{}{}
|
||||
@@ -156,3 +156,34 @@ func (r *Reconciler) EvictionReason(model string) string {
|
||||
}
|
||||
return "nothing idle enough to stop"
|
||||
}
|
||||
|
||||
// evictionGrace is how long an on-demand replica must go unasked-for before
|
||||
// another model may have its card: the cluster's setting, else the
|
||||
// configuration's.
|
||||
func (r *Reconciler) evictionGrace() time.Duration {
|
||||
if d := time.Duration(r.evictGrace.Load()); d > 0 {
|
||||
return d
|
||||
}
|
||||
return r.cfg.EvictionGrace
|
||||
}
|
||||
|
||||
// minLifetime is how long a replica is safe from eviction once it is up.
|
||||
func (r *Reconciler) minLifetime() time.Duration {
|
||||
if d := time.Duration(r.minLife.Load()); d > 0 {
|
||||
return d
|
||||
}
|
||||
return r.cfg.MinLifetime
|
||||
}
|
||||
|
||||
// SetEvictionTimes puts the cluster's on_demand settings in force: how long
|
||||
// an idle on-demand replica keeps its card, and how long a new one is safe.
|
||||
// Zero keeps the configuration's.
|
||||
//
|
||||
// On one card shared by several models these are the wait between them: a
|
||||
// 3D job after a picture waited out five minutes of a picture model it no
|
||||
// longer needed.
|
||||
func (r *Reconciler) SetEvictionTimes(afterIdle, minLifetime time.Duration) {
|
||||
r.evictGrace.Store(int64(afterIdle))
|
||||
r.minLife.Store(int64(minLifetime))
|
||||
r.Notify()
|
||||
}
|
||||
@@ -205,9 +205,9 @@ func (r *Reconciler) WaitFor(model string) (heldBy string, eta time.Duration, ok
|
||||
// Evicted sooner, when it is no higher a class than the model
|
||||
// waiting; one put first is waited for to its idle timeout.
|
||||
if catalog.ClassRank(other.EffectiveClass()) <= catalog.ClassRank(m.EffectiveClass()) {
|
||||
evict := rep.LastUsedAt.Add(r.cfg.EvictionGrace)
|
||||
if !rep.ReadyAt.IsZero() && rep.ReadyAt.Add(r.cfg.MinLifetime).After(evict) {
|
||||
evict = rep.ReadyAt.Add(r.cfg.MinLifetime)
|
||||
evict := rep.LastUsedAt.Add(r.evictionGrace())
|
||||
if !rep.ReadyAt.IsZero() && rep.ReadyAt.Add(r.minLifetime()).After(evict) {
|
||||
evict = rep.ReadyAt.Add(r.minLifetime())
|
||||
}
|
||||
if evict.Before(h.release) {
|
||||
h.release = evict
|
||||
|
||||
@@ -249,7 +249,7 @@ func (r *Reconciler) victims(m catalog.Model) []victim {
|
||||
aboveMin := i < above
|
||||
tier := -1
|
||||
switch {
|
||||
case rank < myRank && other.Scaling.OnDemand() && idleFor(rep, now) >= r.cfg.EvictionGrace:
|
||||
case rank < myRank && other.Scaling.OnDemand() && idleFor(rep, now) >= r.evictionGrace():
|
||||
tier = 0
|
||||
case rank < myRank && aboveMin:
|
||||
tier = 2
|
||||
@@ -275,7 +275,7 @@ func (r *Reconciler) victims(m catalog.Model) []victim {
|
||||
}
|
||||
// Just started: stopping it throws away a model load that has
|
||||
// only now been paid for.
|
||||
if !rep.ReadyAt.IsZero() && now.Sub(rep.ReadyAt) < r.cfg.MinLifetime {
|
||||
if !rep.ReadyAt.IsZero() && now.Sub(rep.ReadyAt) < r.minLifetime() {
|
||||
continue
|
||||
}
|
||||
if len(nodesOf(rep)) > 1 {
|
||||
|
||||
@@ -264,6 +264,9 @@ type Reconciler struct {
|
||||
// because it is state about this moment and not about what the model is.
|
||||
// A model nobody has wanted has no entry and is simply not running.
|
||||
wanted map[string]time.Time
|
||||
// evictGrace and minLife are the cluster's on_demand settings, as
|
||||
// durations; zero keeps cfg's.
|
||||
evictGrace, minLife atomic.Int64
|
||||
// coldStart is how long each model's last replica took from placed to
|
||||
// ready, for telling a request waiting on it about how long it waits.
|
||||
coldStart map[string]time.Duration
|
||||
|
||||
@@ -1763,6 +1763,31 @@ func mustResolve(t *testing.T, h *harness, name string) catalog.Model {
|
||||
return m
|
||||
}
|
||||
|
||||
func TestTheClustersOnDemandSettingsShortenTheWaitForACard(t *testing.T) {
|
||||
// One card, a picture model and a 3D model: a mesh after a picture
|
||||
// waited out five minutes of the picture model's protection.
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 4))
|
||||
h.r.cat.Store(twoOnDemandModels(t))
|
||||
h.r.Want("llama")
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
reps := h.r.Replicas()
|
||||
h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()})
|
||||
h.advance(90 * time.Second)
|
||||
if got := h.r.evictable("second"); len(got) != 0 {
|
||||
t.Fatalf("offered at 90 s with the defaults: %+v", got)
|
||||
}
|
||||
h.r.SetEvictionTimes(30*time.Second, time.Minute)
|
||||
if got := h.r.evictable("second"); len(got) != 1 {
|
||||
t.Errorf("not offered at 90 s with 30 s idle and 1 min lifetime: %+v", got)
|
||||
}
|
||||
// Zero goes back to the configuration's.
|
||||
h.r.SetEvictionTimes(0, 0)
|
||||
if got := h.r.evictable("second"); len(got) != 0 {
|
||||
t.Errorf("offered after the settings were cleared: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFreshlyStartedReplicaIsNotEvicted(t *testing.T) {
|
||||
// What stops two models on a cluster with room for one from taking turns
|
||||
// evicting each other, paying a model load each time. A load is expensive
|
||||
|
||||
Reference in new issue
Block a user