flux and sf3d on one card handed it to each other, each starting ComfyUI and loading its weights from nothing (BACKLOG H3, placement side). An environment of two or more models now gets a hidden catalog entry, comfyui-env-<name>: on its first member's profiles with the most memory any member needs, on demand for the longest idle timeout, carrying every member's weights, repositories and the environment's custom nodes. The members get no replicas of their own; asking for one wants the entry. The agent fetches each member's weights; the gate links all of them, refusing two weights files of one name, links each member's Hugging Face repository, and answers /v1/models for every member. The control plane offers the replica under the members' names with each one's workflow and checkpoint, which the router sends by the model asked for. A member's sibling never evicts the replica serving it, and the members' load counts as the entry's. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
192 lines
6.4 KiB
Go
192 lines
6.4 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sort"
|
|
"time"
|
|
|
|
"go.opentelemetry.io/otel/attribute"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
)
|
|
|
|
// evictionCandidate is a replica that could be stopped to make room.
|
|
type evictionCandidate struct {
|
|
ID string
|
|
Model string
|
|
Idle time.Duration
|
|
GPUs int
|
|
// Nodes is how many machines the replica spans. More than one is left
|
|
// alone — see makeRoomFor.
|
|
Nodes int
|
|
}
|
|
|
|
// makeRoomFor stops an idle replica so a wanted model can be placed.
|
|
//
|
|
// Without this, on-demand loading works right up to the moment the cluster is
|
|
// full and then stops meaning anything: the model somebody just asked for
|
|
// cannot be placed, and the cards are held by a model nobody has asked for in
|
|
// an hour. That is the exact situation the feature exists to resolve, and it is
|
|
// the only situation in which it matters.
|
|
//
|
|
// Deliberately conservative, on three counts.
|
|
//
|
|
// Only idle replicas of on-demand models. A model at min 1 is a promise that it
|
|
// stays up, and quietly breaking that to serve a different request would make
|
|
// every guarantee in the catalog conditional on load.
|
|
//
|
|
// Only single-node replicas. Evicting a multi-node replica frees several
|
|
// machines at once and takes tens of minutes to rebuild, which is a decision
|
|
// worth making deliberately rather than as a side effect of one request.
|
|
//
|
|
// One at a time. Freeing everything that qualifies would empty a cluster to
|
|
// satisfy a request that may only have needed one node, and the next pass can
|
|
// always free more.
|
|
func (r *Reconciler) makeRoomFor(ctx context.Context, m catalog.Model) bool {
|
|
candidates := r.evictable(m.Name)
|
|
if len(candidates) == 0 {
|
|
return false
|
|
}
|
|
|
|
// The longest idle first. It is the replica least likely to be wanted
|
|
// again, and the one whose KV cache is worth the least.
|
|
best := candidates[0]
|
|
r.log.Info("evicting an idle replica to make room",
|
|
"stopping", best.ID, "idle_model", best.Model,
|
|
"idle", best.Idle.Round(time.Second), "gpus", best.GPUs, "for", m.Name)
|
|
ctx, span := startSpan(ctx, "evict",
|
|
attribute.String("llmgw.for", m.Name),
|
|
attribute.String("llmgw.replica", best.ID),
|
|
attribute.String("llmgw.model", best.Model),
|
|
attribute.Float64("llmgw.idle_seconds", best.Idle.Seconds()),
|
|
attribute.Int("llmgw.gpus", best.GPUs),
|
|
attribute.Int("llmgw.candidates", len(candidates)))
|
|
defer span.End()
|
|
r.stopReplica(ctx, best.ID, "evicted, idle, to make room for "+m.Name)
|
|
r.forgetWantIfGone(best.Model, best.ID)
|
|
return true
|
|
}
|
|
|
|
// forgetWantIfGone takes an evicted model off the wanted list when that was
|
|
// its last replica. Still wanted, it was placed again on the next pass —
|
|
// ahead of the model it made room for whenever it came first in the catalog —
|
|
// and the card went back to the model nobody was asking for. Asked again, it
|
|
// is wanted again.
|
|
func (r *Reconciler) forgetWantIfGone(model, stopped string) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
for _, rep := range r.replicas {
|
|
if rep.Model == model && rep.ID != stopped && rep.State.Counts() {
|
|
return
|
|
}
|
|
}
|
|
delete(r.wanted, model)
|
|
}
|
|
|
|
// evictable lists replicas that could be stopped for a model that needs room,
|
|
// longest idle first.
|
|
func (r *Reconciler) evictable(wantedBy string) []evictionCandidate {
|
|
cat := r.cat.Load()
|
|
// A member never evicts the replica that serves it.
|
|
wantedBy = r.hostOf(wantedBy)
|
|
now := r.now()
|
|
|
|
wantRank := -1
|
|
if w, ok := cat.Resolve(wantedBy); ok {
|
|
wantRank = catalog.ClassRank(w.EffectiveClass())
|
|
}
|
|
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
|
|
var out []evictionCandidate
|
|
for _, rep := range r.replicas {
|
|
if rep.Model == wantedBy || !rep.State.Counts() {
|
|
continue
|
|
}
|
|
other, ok := cat.Resolve(rep.Model)
|
|
if !ok || !other.Scaling.OnDemand() {
|
|
continue
|
|
}
|
|
// Not for a model of a lower class: the one put first keeps the
|
|
// card through its idle timeout, rather than losing it two minutes
|
|
// into a lull to whatever was asked for next.
|
|
if catalog.ClassRank(other.EffectiveClass()) > wantRank {
|
|
continue
|
|
}
|
|
// Unknown is not idle. A replica the router has said nothing about —
|
|
// after a restart, that is all of them — must not be evicted on the
|
|
// strength of a fact nobody has reported.
|
|
if rep.LastUsedAt.IsZero() {
|
|
continue
|
|
}
|
|
idle := now.Sub(rep.LastUsedAt)
|
|
if idle < r.evictionGrace() {
|
|
continue
|
|
}
|
|
// A replica that has only just started is left alone however idle it
|
|
// looks. Two on-demand models on a cluster with room for one would
|
|
// otherwise take turns evicting each other, paying a model load each
|
|
// time — and a load is expensive enough that waiting is strictly
|
|
// better than winning.
|
|
if !rep.ReadyAt.IsZero() && now.Sub(rep.ReadyAt) < r.minLifetime() {
|
|
continue
|
|
}
|
|
nodes := map[string]struct{}{}
|
|
gpus := 0
|
|
for _, sh := range rep.Shards {
|
|
nodes[sh.NodeID] = struct{}{}
|
|
gpus += len(sh.GPUIndices)
|
|
}
|
|
if len(nodes) > 1 {
|
|
continue
|
|
}
|
|
out = append(out, evictionCandidate{
|
|
ID: rep.ID, Model: rep.Model, Idle: idle, GPUs: gpus, Nodes: len(nodes),
|
|
})
|
|
}
|
|
|
|
sort.Slice(out, func(i, j int) bool { return out[i].Idle > out[j].Idle })
|
|
return out
|
|
}
|
|
|
|
// EvictionReason explains why room could not be made, for placement failures.
|
|
func (r *Reconciler) EvictionReason(model string) string {
|
|
if c := r.evictable(model); len(c) > 0 {
|
|
return fmt.Sprintf("%d idle replica(s) could be stopped to make room", len(c))
|
|
}
|
|
return "nothing idle enough to stop"
|
|
}
|
|
|
|
// evictionGrace is how long an on-demand replica must go unasked-for before
|
|
// another model may have its card: the cluster's setting, else the
|
|
// configuration's.
|
|
func (r *Reconciler) evictionGrace() time.Duration {
|
|
if d := time.Duration(r.evictGrace.Load()); d > 0 {
|
|
return d
|
|
}
|
|
return r.cfg.EvictionGrace
|
|
}
|
|
|
|
// minLifetime is how long a replica is safe from eviction once it is up.
|
|
func (r *Reconciler) minLifetime() time.Duration {
|
|
if d := time.Duration(r.minLife.Load()); d > 0 {
|
|
return d
|
|
}
|
|
return r.cfg.MinLifetime
|
|
}
|
|
|
|
// SetEvictionTimes puts the cluster's on_demand settings in force: how long
|
|
// an idle on-demand replica keeps its card, and how long a new one is safe.
|
|
// Zero keeps the configuration's.
|
|
//
|
|
// On one card shared by several models these are the wait between them: a
|
|
// 3D job after a picture waited out five minutes of a picture model it no
|
|
// longer needed.
|
|
func (r *Reconciler) SetEvictionTimes(afterIdle, minLifetime time.Duration) {
|
|
r.evictGrace.Store(int64(afterIdle))
|
|
r.minLife.Store(int64(minLifetime))
|
|
r.Notify()
|
|
}
|