Files
llmgw/internal/control/reconciler/evict.go
T
Ilker ManapandClaude Opus 5.5 fa6e68c8e8 A model put first is not evicted, idle, for a lower class
The VLM is to come first on the one card. Its class makes it take the card
from flux, triposr or sf3d when asked for, but idle eviction ignored class:
a 3D job two minutes into a lull would have taken the card from it. Idle
eviction offers only replicas of no higher class than the model wanting
room, and the wait such a model is told runs to the holder's idle timeout.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-13 10:15:14 +02:00

159 lines
5.2 KiB
Go

package reconciler
import (
"context"
"fmt"
"sort"
"time"
"go.opentelemetry.io/otel/attribute"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
// evictionCandidate is a replica that could be stopped to make room.
type evictionCandidate struct {
ID string
Model string
Idle time.Duration
GPUs int
// Nodes is how many machines the replica spans. More than one is left
// alone — see makeRoomFor.
Nodes int
}
// makeRoomFor stops an idle replica so a wanted model can be placed.
//
// Without this, on-demand loading works right up to the moment the cluster is
// full and then stops meaning anything: the model somebody just asked for
// cannot be placed, and the cards are held by a model nobody has asked for in
// an hour. That is the exact situation the feature exists to resolve, and it is
// the only situation in which it matters.
//
// Deliberately conservative, on three counts.
//
// Only idle replicas of on-demand models. A model at min 1 is a promise that it
// stays up, and quietly breaking that to serve a different request would make
// every guarantee in the catalog conditional on load.
//
// Only single-node replicas. Evicting a multi-node replica frees several
// machines at once and takes tens of minutes to rebuild, which is a decision
// worth making deliberately rather than as a side effect of one request.
//
// One at a time. Freeing everything that qualifies would empty a cluster to
// satisfy a request that may only have needed one node, and the next pass can
// always free more.
func (r *Reconciler) makeRoomFor(ctx context.Context, m catalog.Model) bool {
candidates := r.evictable(m.Name)
if len(candidates) == 0 {
return false
}
// The longest idle first. It is the replica least likely to be wanted
// again, and the one whose KV cache is worth the least.
best := candidates[0]
r.log.Info("evicting an idle replica to make room",
"stopping", best.ID, "idle_model", best.Model,
"idle", best.Idle.Round(time.Second), "gpus", best.GPUs, "for", m.Name)
ctx, span := startSpan(ctx, "evict",
attribute.String("llmgw.for", m.Name),
attribute.String("llmgw.replica", best.ID),
attribute.String("llmgw.model", best.Model),
attribute.Float64("llmgw.idle_seconds", best.Idle.Seconds()),
attribute.Int("llmgw.gpus", best.GPUs),
attribute.Int("llmgw.candidates", len(candidates)))
defer span.End()
r.stopReplica(ctx, best.ID, "evicted, idle, to make room for "+m.Name)
r.forgetWantIfGone(best.Model, best.ID)
return true
}
// forgetWantIfGone takes an evicted model off the wanted list when that was
// its last replica. Still wanted, it was placed again on the next pass —
// ahead of the model it made room for whenever it came first in the catalog —
// and the card went back to the model nobody was asking for. Asked again, it
// is wanted again.
func (r *Reconciler) forgetWantIfGone(model, stopped string) {
r.mu.Lock()
defer r.mu.Unlock()
for _, rep := range r.replicas {
if rep.Model == model && rep.ID != stopped && rep.State.Counts() {
return
}
}
delete(r.wanted, model)
}
// evictable lists replicas that could be stopped for a model that needs room,
// longest idle first.
func (r *Reconciler) evictable(wantedBy string) []evictionCandidate {
cat := r.cat.Load()
now := r.now()
wantRank := -1
if w, ok := cat.Resolve(wantedBy); ok {
wantRank = catalog.ClassRank(w.EffectiveClass())
}
r.mu.Lock()
defer r.mu.Unlock()
var out []evictionCandidate
for _, rep := range r.replicas {
if rep.Model == wantedBy || !rep.State.Counts() {
continue
}
other, ok := cat.Resolve(rep.Model)
if !ok || !other.Scaling.OnDemand() {
continue
}
// Not for a model of a lower class: the one put first keeps the
// card through its idle timeout, rather than losing it two minutes
// into a lull to whatever was asked for next.
if catalog.ClassRank(other.EffectiveClass()) > wantRank {
continue
}
// Unknown is not idle. A replica the router has said nothing about —
// after a restart, that is all of them — must not be evicted on the
// strength of a fact nobody has reported.
if rep.LastUsedAt.IsZero() {
continue
}
idle := now.Sub(rep.LastUsedAt)
if idle < r.cfg.EvictionGrace {
continue
}
// A replica that has only just started is left alone however idle it
// looks. Two on-demand models on a cluster with room for one would
// otherwise take turns evicting each other, paying a model load each
// time — and a load is expensive enough that waiting is strictly
// better than winning.
if !rep.ReadyAt.IsZero() && now.Sub(rep.ReadyAt) < r.cfg.MinLifetime {
continue
}
nodes := map[string]struct{}{}
gpus := 0
for _, sh := range rep.Shards {
nodes[sh.NodeID] = struct{}{}
gpus += len(sh.GPUIndices)
}
if len(nodes) > 1 {
continue
}
out = append(out, evictionCandidate{
ID: rep.ID, Model: rep.Model, Idle: idle, GPUs: gpus, Nodes: len(nodes),
})
}
sort.Slice(out, func(i, j int) bool { return out[i].Idle > out[j].Idle })
return out
}
// EvictionReason explains why room could not be made, for placement failures.
func (r *Reconciler) EvictionReason(model string) string {
if c := r.evictable(model); len(c) > 0 {
return fmt.Sprintf("%d idle replica(s) could be stopped to make room", len(c))
}
return "nothing idle enough to stop"
}