On one card shared by flux and sf3d, a mesh asked for after a picture waited out the picture model's five minutes of protection and two of idle. on_demand.evict_after_idle and on_demand.min_lifetime set those for the cluster, in force on the leader as they change; unset keeps 2m and 5m. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
190 lines
6.3 KiB
Go
190 lines
6.3 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sort"
|
|
"time"
|
|
|
|
"go.opentelemetry.io/otel/attribute"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
)
|
|
|
|
// evictionCandidate is a replica that could be stopped to make room.
|
|
type evictionCandidate struct {
|
|
ID string
|
|
Model string
|
|
Idle time.Duration
|
|
GPUs int
|
|
// Nodes is how many machines the replica spans. More than one is left
|
|
// alone — see makeRoomFor.
|
|
Nodes int
|
|
}
|
|
|
|
// makeRoomFor stops an idle replica so a wanted model can be placed.
|
|
//
|
|
// Without this, on-demand loading works right up to the moment the cluster is
|
|
// full and then stops meaning anything: the model somebody just asked for
|
|
// cannot be placed, and the cards are held by a model nobody has asked for in
|
|
// an hour. That is the exact situation the feature exists to resolve, and it is
|
|
// the only situation in which it matters.
|
|
//
|
|
// Deliberately conservative, on three counts.
|
|
//
|
|
// Only idle replicas of on-demand models. A model at min 1 is a promise that it
|
|
// stays up, and quietly breaking that to serve a different request would make
|
|
// every guarantee in the catalog conditional on load.
|
|
//
|
|
// Only single-node replicas. Evicting a multi-node replica frees several
|
|
// machines at once and takes tens of minutes to rebuild, which is a decision
|
|
// worth making deliberately rather than as a side effect of one request.
|
|
//
|
|
// One at a time. Freeing everything that qualifies would empty a cluster to
|
|
// satisfy a request that may only have needed one node, and the next pass can
|
|
// always free more.
|
|
func (r *Reconciler) makeRoomFor(ctx context.Context, m catalog.Model) bool {
|
|
candidates := r.evictable(m.Name)
|
|
if len(candidates) == 0 {
|
|
return false
|
|
}
|
|
|
|
// The longest idle first. It is the replica least likely to be wanted
|
|
// again, and the one whose KV cache is worth the least.
|
|
best := candidates[0]
|
|
r.log.Info("evicting an idle replica to make room",
|
|
"stopping", best.ID, "idle_model", best.Model,
|
|
"idle", best.Idle.Round(time.Second), "gpus", best.GPUs, "for", m.Name)
|
|
ctx, span := startSpan(ctx, "evict",
|
|
attribute.String("llmgw.for", m.Name),
|
|
attribute.String("llmgw.replica", best.ID),
|
|
attribute.String("llmgw.model", best.Model),
|
|
attribute.Float64("llmgw.idle_seconds", best.Idle.Seconds()),
|
|
attribute.Int("llmgw.gpus", best.GPUs),
|
|
attribute.Int("llmgw.candidates", len(candidates)))
|
|
defer span.End()
|
|
r.stopReplica(ctx, best.ID, "evicted, idle, to make room for "+m.Name)
|
|
r.forgetWantIfGone(best.Model, best.ID)
|
|
return true
|
|
}
|
|
|
|
// forgetWantIfGone takes an evicted model off the wanted list when that was
|
|
// its last replica. Still wanted, it was placed again on the next pass —
|
|
// ahead of the model it made room for whenever it came first in the catalog —
|
|
// and the card went back to the model nobody was asking for. Asked again, it
|
|
// is wanted again.
|
|
func (r *Reconciler) forgetWantIfGone(model, stopped string) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
for _, rep := range r.replicas {
|
|
if rep.Model == model && rep.ID != stopped && rep.State.Counts() {
|
|
return
|
|
}
|
|
}
|
|
delete(r.wanted, model)
|
|
}
|
|
|
|
// evictable lists replicas that could be stopped for a model that needs room,
|
|
// longest idle first.
|
|
func (r *Reconciler) evictable(wantedBy string) []evictionCandidate {
|
|
cat := r.cat.Load()
|
|
now := r.now()
|
|
|
|
wantRank := -1
|
|
if w, ok := cat.Resolve(wantedBy); ok {
|
|
wantRank = catalog.ClassRank(w.EffectiveClass())
|
|
}
|
|
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
|
|
var out []evictionCandidate
|
|
for _, rep := range r.replicas {
|
|
if rep.Model == wantedBy || !rep.State.Counts() {
|
|
continue
|
|
}
|
|
other, ok := cat.Resolve(rep.Model)
|
|
if !ok || !other.Scaling.OnDemand() {
|
|
continue
|
|
}
|
|
// Not for a model of a lower class: the one put first keeps the
|
|
// card through its idle timeout, rather than losing it two minutes
|
|
// into a lull to whatever was asked for next.
|
|
if catalog.ClassRank(other.EffectiveClass()) > wantRank {
|
|
continue
|
|
}
|
|
// Unknown is not idle. A replica the router has said nothing about —
|
|
// after a restart, that is all of them — must not be evicted on the
|
|
// strength of a fact nobody has reported.
|
|
if rep.LastUsedAt.IsZero() {
|
|
continue
|
|
}
|
|
idle := now.Sub(rep.LastUsedAt)
|
|
if idle < r.evictionGrace() {
|
|
continue
|
|
}
|
|
// A replica that has only just started is left alone however idle it
|
|
// looks. Two on-demand models on a cluster with room for one would
|
|
// otherwise take turns evicting each other, paying a model load each
|
|
// time — and a load is expensive enough that waiting is strictly
|
|
// better than winning.
|
|
if !rep.ReadyAt.IsZero() && now.Sub(rep.ReadyAt) < r.minLifetime() {
|
|
continue
|
|
}
|
|
nodes := map[string]struct{}{}
|
|
gpus := 0
|
|
for _, sh := range rep.Shards {
|
|
nodes[sh.NodeID] = struct{}{}
|
|
gpus += len(sh.GPUIndices)
|
|
}
|
|
if len(nodes) > 1 {
|
|
continue
|
|
}
|
|
out = append(out, evictionCandidate{
|
|
ID: rep.ID, Model: rep.Model, Idle: idle, GPUs: gpus, Nodes: len(nodes),
|
|
})
|
|
}
|
|
|
|
sort.Slice(out, func(i, j int) bool { return out[i].Idle > out[j].Idle })
|
|
return out
|
|
}
|
|
|
|
// EvictionReason explains why room could not be made, for placement failures.
|
|
func (r *Reconciler) EvictionReason(model string) string {
|
|
if c := r.evictable(model); len(c) > 0 {
|
|
return fmt.Sprintf("%d idle replica(s) could be stopped to make room", len(c))
|
|
}
|
|
return "nothing idle enough to stop"
|
|
}
|
|
|
|
// evictionGrace is how long an on-demand replica must go unasked-for before
|
|
// another model may have its card: the cluster's setting, else the
|
|
// configuration's.
|
|
func (r *Reconciler) evictionGrace() time.Duration {
|
|
if d := time.Duration(r.evictGrace.Load()); d > 0 {
|
|
return d
|
|
}
|
|
return r.cfg.EvictionGrace
|
|
}
|
|
|
|
// minLifetime is how long a replica is safe from eviction once it is up.
|
|
func (r *Reconciler) minLifetime() time.Duration {
|
|
if d := time.Duration(r.minLife.Load()); d > 0 {
|
|
return d
|
|
}
|
|
return r.cfg.MinLifetime
|
|
}
|
|
|
|
// SetEvictionTimes puts the cluster's on_demand settings in force: how long
|
|
// an idle on-demand replica keeps its card, and how long a new one is safe.
|
|
// Zero keeps the configuration's.
|
|
//
|
|
// On one card shared by several models these are the wait between them: a
|
|
// 3D job after a picture waited out five minutes of a picture model it no
|
|
// longer needed.
|
|
func (r *Reconciler) SetEvictionTimes(afterIdle, minLifetime time.Duration) {
|
|
r.evictGrace.Store(int64(afterIdle))
|
|
r.minLife.Store(int64(minLifetime))
|
|
r.Notify()
|
|
}
|