flux evicted to make room for triposr was placed again two seconds later: still on the wanted list, and first in the catalog, it took back the card it had just given up, and the 3D job went on waiting. Its last replica evicted, it leaves the list until it is asked for again. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
148 lines
4.9 KiB
Go
148 lines
4.9 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sort"
|
|
"time"
|
|
|
|
"go.opentelemetry.io/otel/attribute"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
)
|
|
|
|
// evictionCandidate is a replica that could be stopped to make room.
|
|
type evictionCandidate struct {
|
|
ID string
|
|
Model string
|
|
Idle time.Duration
|
|
GPUs int
|
|
// Nodes is how many machines the replica spans. More than one is left
|
|
// alone — see makeRoomFor.
|
|
Nodes int
|
|
}
|
|
|
|
// makeRoomFor stops an idle replica so a wanted model can be placed.
|
|
//
|
|
// Without this, on-demand loading works right up to the moment the cluster is
|
|
// full and then stops meaning anything: the model somebody just asked for
|
|
// cannot be placed, and the cards are held by a model nobody has asked for in
|
|
// an hour. That is the exact situation the feature exists to resolve, and it is
|
|
// the only situation in which it matters.
|
|
//
|
|
// Deliberately conservative, on three counts.
|
|
//
|
|
// Only idle replicas of on-demand models. A model at min 1 is a promise that it
|
|
// stays up, and quietly breaking that to serve a different request would make
|
|
// every guarantee in the catalog conditional on load.
|
|
//
|
|
// Only single-node replicas. Evicting a multi-node replica frees several
|
|
// machines at once and takes tens of minutes to rebuild, which is a decision
|
|
// worth making deliberately rather than as a side effect of one request.
|
|
//
|
|
// One at a time. Freeing everything that qualifies would empty a cluster to
|
|
// satisfy a request that may only have needed one node, and the next pass can
|
|
// always free more.
|
|
func (r *Reconciler) makeRoomFor(ctx context.Context, m catalog.Model) bool {
|
|
candidates := r.evictable(m.Name)
|
|
if len(candidates) == 0 {
|
|
return false
|
|
}
|
|
|
|
// The longest idle first. It is the replica least likely to be wanted
|
|
// again, and the one whose KV cache is worth the least.
|
|
best := candidates[0]
|
|
r.log.Info("evicting an idle replica to make room",
|
|
"stopping", best.ID, "idle_model", best.Model,
|
|
"idle", best.Idle.Round(time.Second), "gpus", best.GPUs, "for", m.Name)
|
|
ctx, span := startSpan(ctx, "evict",
|
|
attribute.String("llmgw.for", m.Name),
|
|
attribute.String("llmgw.replica", best.ID),
|
|
attribute.String("llmgw.model", best.Model),
|
|
attribute.Float64("llmgw.idle_seconds", best.Idle.Seconds()),
|
|
attribute.Int("llmgw.gpus", best.GPUs),
|
|
attribute.Int("llmgw.candidates", len(candidates)))
|
|
defer span.End()
|
|
r.stopReplica(ctx, best.ID, "evicted, idle, to make room for "+m.Name)
|
|
r.forgetWantIfGone(best.Model, best.ID)
|
|
return true
|
|
}
|
|
|
|
// forgetWantIfGone takes an evicted model off the wanted list when that was
|
|
// its last replica. Still wanted, it was placed again on the next pass —
|
|
// ahead of the model it made room for whenever it came first in the catalog —
|
|
// and the card went back to the model nobody was asking for. Asked again, it
|
|
// is wanted again.
|
|
func (r *Reconciler) forgetWantIfGone(model, stopped string) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
for _, rep := range r.replicas {
|
|
if rep.Model == model && rep.ID != stopped && rep.State.Counts() {
|
|
return
|
|
}
|
|
}
|
|
delete(r.wanted, model)
|
|
}
|
|
|
|
// evictable lists replicas that could be stopped for a model that needs room,
|
|
// longest idle first.
|
|
func (r *Reconciler) evictable(wantedBy string) []evictionCandidate {
|
|
cat := r.cat.Load()
|
|
now := r.now()
|
|
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
|
|
var out []evictionCandidate
|
|
for _, rep := range r.replicas {
|
|
if rep.Model == wantedBy || !rep.State.Counts() {
|
|
continue
|
|
}
|
|
other, ok := cat.Resolve(rep.Model)
|
|
if !ok || !other.Scaling.OnDemand() {
|
|
continue
|
|
}
|
|
// Unknown is not idle. A replica the router has said nothing about —
|
|
// after a restart, that is all of them — must not be evicted on the
|
|
// strength of a fact nobody has reported.
|
|
if rep.LastUsedAt.IsZero() {
|
|
continue
|
|
}
|
|
idle := now.Sub(rep.LastUsedAt)
|
|
if idle < r.cfg.EvictionGrace {
|
|
continue
|
|
}
|
|
// A replica that has only just started is left alone however idle it
|
|
// looks. Two on-demand models on a cluster with room for one would
|
|
// otherwise take turns evicting each other, paying a model load each
|
|
// time — and a load is expensive enough that waiting is strictly
|
|
// better than winning.
|
|
if !rep.ReadyAt.IsZero() && now.Sub(rep.ReadyAt) < r.cfg.MinLifetime {
|
|
continue
|
|
}
|
|
nodes := map[string]struct{}{}
|
|
gpus := 0
|
|
for _, sh := range rep.Shards {
|
|
nodes[sh.NodeID] = struct{}{}
|
|
gpus += len(sh.GPUIndices)
|
|
}
|
|
if len(nodes) > 1 {
|
|
continue
|
|
}
|
|
out = append(out, evictionCandidate{
|
|
ID: rep.ID, Model: rep.Model, Idle: idle, GPUs: gpus, Nodes: len(nodes),
|
|
})
|
|
}
|
|
|
|
sort.Slice(out, func(i, j int) bool { return out[i].Idle > out[j].Idle })
|
|
return out
|
|
}
|
|
|
|
// EvictionReason explains why room could not be made, for placement failures.
|
|
func (r *Reconciler) EvictionReason(model string) string {
|
|
if c := r.evictable(model); len(c) > 0 {
|
|
return fmt.Sprintf("%d idle replica(s) could be stopped to make room", len(c))
|
|
}
|
|
return "nothing idle enough to stop"
|
|
}
|