flux and sf3d on one card handed it to each other, each starting ComfyUI and loading its weights from nothing (BACKLOG H3, placement side). An environment of two or more models now gets a hidden catalog entry, comfyui-env-<name>: on its first member's profiles with the most memory any member needs, on demand for the longest idle timeout, carrying every member's weights, repositories and the environment's custom nodes. The members get no replicas of their own; asking for one wants the entry. The agent fetches each member's weights; the gate links all of them, refusing two weights files of one name, links each member's Hugging Face repository, and answers /v1/models for every member. The control plane offers the replica under the members' names with each one's workflow and checkpoint, which the router sends by the model asked for. A member's sibling never evicts the replica serving it, and the members' load counts as the entry's. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
306 lines
9.6 KiB
Go
306 lines
9.6 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
)
|
|
|
|
// RecordUsage stamps replicas with when each was last sent a request.
|
|
//
|
|
// Replicas absent from the map are left alone rather than zeroed: a router that
|
|
// has just restarted knows nothing about what happened before it, and treating
|
|
// that as "never used" would make every replica look abandoned at once —
|
|
// exactly when a fleet is least able to afford acting on that.
|
|
func (r *Reconciler) RecordUsage(lastUsed map[string]time.Time) {
|
|
if len(lastUsed) == 0 {
|
|
return
|
|
}
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
for id, at := range lastUsed {
|
|
rep, ok := r.replicas[id]
|
|
if !ok {
|
|
continue
|
|
}
|
|
// Only forwards. Two routers reporting, or a stale snapshot arriving
|
|
// late, must not move a replica's last use backwards.
|
|
if at.After(rep.LastUsedAt) {
|
|
rep.LastUsedAt = at
|
|
}
|
|
}
|
|
r.noteUsage(lastUsed)
|
|
}
|
|
|
|
// IdleFor reports how long a replica has gone without a request, and whether
|
|
// that is known at all.
|
|
//
|
|
// The second return is not a detail. "Idle for an hour" and "nobody has told me
|
|
// yet" are different states, and anything that acts on idleness — eviction,
|
|
// most obviously — has to refuse to act on the second.
|
|
func (r *Reconciler) IdleFor(id string) (time.Duration, bool) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
rep, ok := r.replicas[id]
|
|
if !ok || rep.LastUsedAt.IsZero() {
|
|
return 0, false
|
|
}
|
|
return r.now().Sub(rep.LastUsedAt), true
|
|
}
|
|
|
|
// Want marks an on-demand model as asked for, so the next pass places it.
|
|
//
|
|
// Called by whatever fields the request — the router, in practice. It returns
|
|
// whether the model is one this applies to, so a caller can tell "being loaded"
|
|
// from "there is genuinely nothing serving this", which are different answers
|
|
// to give a client.
|
|
func (r *Reconciler) Want(model string) bool {
|
|
m, ok := r.cat.Load().Resolve(r.hostOf(model))
|
|
if !ok || !m.Scaling.OnDemand() {
|
|
return false
|
|
}
|
|
r.mu.Lock()
|
|
r.wanted[m.Name] = r.now()
|
|
// A placement that failed before this request says nothing about now:
|
|
// forgotten, so the pass this triggers decides afresh and Unservable
|
|
// does not answer from a stale result.
|
|
if res, ok := r.lastExplain[m.Name]; ok && res.Placement == nil {
|
|
delete(r.lastExplain, m.Name)
|
|
}
|
|
r.mu.Unlock()
|
|
// Immediately, not on the next tick. Someone is waiting on this request,
|
|
// and five seconds of ticker is five seconds they spend staring at nothing.
|
|
r.Notify()
|
|
return true
|
|
}
|
|
|
|
// desiredFor is how many replicas a model should have right now.
|
|
//
|
|
// For an ordinary model that is its configured minimum. For an on-demand one it
|
|
// is one while somebody is asking and none once they stop, which is the whole
|
|
// feature: a cluster with more models than GPUs can hold every model in its
|
|
// catalog and only the wanted ones in its cards.
|
|
func (r *Reconciler) desiredFor(m catalog.Model) int {
|
|
// Stopped by the operators: nothing runs, whatever the catalog asks.
|
|
if r.stopped.Load() {
|
|
return 0
|
|
}
|
|
base := r.baseFor(m)
|
|
if base == 0 {
|
|
return 0
|
|
}
|
|
r.mu.Lock()
|
|
extra := r.extra[m.Name]
|
|
r.mu.Unlock()
|
|
return min(base+extra, max(r.maxFor(m), base))
|
|
}
|
|
|
|
// minFor is a model's minimum now: its schedule's window's, or its own
|
|
// outside every window.
|
|
func (r *Reconciler) minFor(m catalog.Model) int {
|
|
if w, ok := m.Scaling.Window(r.localNow()); ok {
|
|
return w.Min
|
|
}
|
|
return m.Scaling.Min
|
|
}
|
|
|
|
// maxFor is the ceiling load scaling reaches now.
|
|
func (r *Reconciler) maxFor(m catalog.Model) int {
|
|
if w, ok := m.Scaling.Window(r.localNow()); ok && w.Max > 0 {
|
|
return w.Max
|
|
}
|
|
return m.Scaling.Max
|
|
}
|
|
|
|
// localNow is now in the time zone schedules are written in.
|
|
func (r *Reconciler) localNow() time.Time {
|
|
if r.cfg.Location != nil {
|
|
return r.now().In(r.cfg.Location)
|
|
}
|
|
return r.now()
|
|
}
|
|
|
|
// baseFor is what a model runs without load: its minimum, or for an on-demand
|
|
// model one while somebody is asking and none once they stop. A schedule's
|
|
// window keeps its minimum running however quiet it is.
|
|
func (r *Reconciler) baseFor(m catalog.Model) int {
|
|
if m.IsExternal() {
|
|
return 0 // served elsewhere: nothing to place, whatever its scaling says
|
|
}
|
|
if _, shared := r.cat.Load().SharedBy(m.Name); shared {
|
|
return 0 // its environment's replica serves it
|
|
}
|
|
if lo := r.minFor(m); !m.Scaling.OnDemand() || lo > 0 {
|
|
return lo
|
|
}
|
|
r.mu.Lock()
|
|
last, asked := r.wanted[m.Name]
|
|
r.mu.Unlock()
|
|
if !asked {
|
|
return 0
|
|
}
|
|
if r.now().Sub(last) > r.idleTimeout(m) {
|
|
return 0
|
|
}
|
|
return 1
|
|
}
|
|
|
|
// idleTimeout is how long an on-demand model stays up unasked-for.
|
|
func (r *Reconciler) idleTimeout(m catalog.Model) time.Duration {
|
|
if m.Scaling.IdleTimeout > 0 {
|
|
return m.Scaling.IdleTimeout
|
|
}
|
|
return r.cfg.IdleTimeout
|
|
}
|
|
|
|
// noteUsage keeps an on-demand model alive while it is being used.
|
|
//
|
|
// Without this a model would be evicted exactly IdleTimeout after the request
|
|
// that loaded it, however busy it had been since — the timer would measure
|
|
// time since the first request rather than the last.
|
|
func (r *Reconciler) noteUsage(lastUsed map[string]time.Time) {
|
|
byModel := map[string]time.Time{}
|
|
for id, at := range lastUsed {
|
|
rep, ok := r.replicas[id]
|
|
if !ok {
|
|
continue
|
|
}
|
|
if at.After(byModel[rep.Model]) {
|
|
byModel[rep.Model] = at
|
|
}
|
|
}
|
|
for model, at := range byModel {
|
|
if cur, ok := r.wanted[model]; ok && at.After(cur) {
|
|
r.wanted[model] = at
|
|
}
|
|
}
|
|
}
|
|
|
|
// Load is what a model's replicas are doing, as the router sees it.
|
|
type Load struct {
|
|
Replicas int
|
|
Running int
|
|
Waiting int
|
|
}
|
|
|
|
// RecordLoad adds or takes away replicas between a model's min and max as its
|
|
// load says.
|
|
//
|
|
// Max was a ceiling nothing reached: every model ran exactly its minimum
|
|
// however long its queue. Now requests waiting at the engines — every slot of
|
|
// every replica busy — held for ScaleUpAfter add one replica, and a quiet spell
|
|
// of ScaleDownAfter takes one of those back. One step at a time, and not while
|
|
// the last one is still starting: on a small CPU fleet each replica is minutes
|
|
// of weights and gigabytes of memory, and a spike that is over by the time it
|
|
// loads should not leave three behind.
|
|
//
|
|
// With a benchmark stored for the profile a model runs on, its measured
|
|
// capacity is used too: more requests per replica than were measured to meet
|
|
// the SLO is busy, even with the engines' queues empty — an engine will take
|
|
// requests into its slots long after latency has gone — and a replica is only
|
|
// taken back when the rest would still be under scaleDownHeadroom of what they
|
|
// were measured to hold, so scaling down does not land on the edge.
|
|
func (r *Reconciler) RecordLoad(loads map[string]Load) {
|
|
now := r.now()
|
|
// A shared replica's load is its members' together.
|
|
for name, l := range loads {
|
|
if host := r.hostOf(name); host != name {
|
|
h := loads[host]
|
|
h.Replicas = max(h.Replicas, l.Replicas)
|
|
h.Running += l.Running
|
|
h.Waiting += l.Waiting
|
|
loads[host] = h
|
|
delete(loads, name)
|
|
}
|
|
}
|
|
// The profile each model's replicas run on, for its measurement; read
|
|
// before the loop, which holds the lock.
|
|
running := map[string]string{}
|
|
r.mu.Lock()
|
|
for _, rep := range r.replicas {
|
|
if rep.State.Counts() {
|
|
running[rep.Model] = rep.Profile
|
|
}
|
|
}
|
|
r.mu.Unlock()
|
|
measured := map[string]Measured{}
|
|
for model, profile := range running {
|
|
if m, ok := r.measured(model, profile); ok {
|
|
measured[model] = m
|
|
}
|
|
}
|
|
for _, m := range r.cat.Load().Models() {
|
|
base := r.baseFor(m)
|
|
r.mu.Lock()
|
|
ceiling := r.maxFor(m)
|
|
if base == 0 || ceiling <= base {
|
|
delete(r.extra, m.Name)
|
|
delete(r.busySince, m.Name)
|
|
delete(r.quietSince, m.Name)
|
|
r.mu.Unlock()
|
|
continue
|
|
}
|
|
starting := false
|
|
for _, rep := range r.replicas {
|
|
if rep.Model == m.Name && (rep.State == Placing || rep.State == Starting) {
|
|
starting = true
|
|
}
|
|
}
|
|
l := loads[m.Name]
|
|
perReplica := float64(l.Waiting) / float64(max(l.Replicas, 1))
|
|
extra := r.extra[m.Name]
|
|
meas, hasMeas := measured[m.Name]
|
|
overMeasured := hasMeas &&
|
|
float64(l.Running+l.Waiting) > float64(meas.MaxConcurrency*max(l.Replicas, 1))
|
|
fitsInFewer := !hasMeas ||
|
|
float64(l.Running) <= scaleDownHeadroom*float64(meas.MaxConcurrency*max(l.Replicas-1, 0))
|
|
switch {
|
|
case perReplica >= r.cfg.ScaleUpWaiting || overMeasured:
|
|
delete(r.quietSince, m.Name)
|
|
since, ok := r.busySince[m.Name]
|
|
if !ok {
|
|
r.busySince[m.Name] = now
|
|
break
|
|
}
|
|
if now.Sub(since) >= r.cfg.ScaleUpAfter && base+extra < ceiling && !starting {
|
|
r.extra[m.Name] = extra + 1
|
|
r.busySince[m.Name] = now
|
|
r.log.Info("scaling up for load", "model", m.Name, "replicas", base+extra+1,
|
|
"waiting_per_replica", perReplica, "in_flight", l.Running+l.Waiting,
|
|
"measured_per_replica", meas.MaxConcurrency, "for", now.Sub(since).Round(time.Second))
|
|
}
|
|
case l.Waiting == 0 && extra > 0 && fitsInFewer:
|
|
delete(r.busySince, m.Name)
|
|
since, ok := r.quietSince[m.Name]
|
|
if !ok {
|
|
r.quietSince[m.Name] = now
|
|
break
|
|
}
|
|
if now.Sub(since) >= r.cfg.ScaleDownAfter {
|
|
r.extra[m.Name] = extra - 1
|
|
r.quietSince[m.Name] = now
|
|
r.log.Info("scaling down after a quiet spell", "model", m.Name,
|
|
"replicas", base+extra-1, "quiet_for", now.Sub(since).Round(time.Second))
|
|
}
|
|
default:
|
|
delete(r.busySince, m.Name)
|
|
delete(r.quietSince, m.Name)
|
|
}
|
|
r.mu.Unlock()
|
|
}
|
|
}
|
|
|
|
// scaleDownHeadroom is how full, of their measured capacity, the replicas left
|
|
// after a scale-down may be. Below one so that the load that was there a
|
|
// minute ago does not put them straight back over.
|
|
const scaleDownHeadroom = 0.7
|
|
|
|
// hostOf is the model whose replicas serve name: its environment's hidden
|
|
// entry when it shares one, else itself.
|
|
func (r *Reconciler) hostOf(name string) string {
|
|
if host, ok := r.cat.Load().SharedBy(name); ok {
|
|
return host
|
|
}
|
|
return name
|
|
}
|