A catalog model with external {url, model, api_key_file} and no profiles
is published as an endpoint by the control plane, the key read from its
file each time; the router rewrites the request's model to the provider's
name and sends its key. It is never scraped nor stale, and nothing is
placed for it. Named in fallbacks, it serves when the cluster has no room.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
252 lines
8.1 KiB
Go
252 lines
8.1 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
)
|
|
|
|
// RecordUsage stamps replicas with when each was last sent a request.
|
|
//
|
|
// Replicas absent from the map are left alone rather than zeroed: a router that
|
|
// has just restarted knows nothing about what happened before it, and treating
|
|
// that as "never used" would make every replica look abandoned at once —
|
|
// exactly when a fleet is least able to afford acting on that.
|
|
func (r *Reconciler) RecordUsage(lastUsed map[string]time.Time) {
|
|
if len(lastUsed) == 0 {
|
|
return
|
|
}
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
for id, at := range lastUsed {
|
|
rep, ok := r.replicas[id]
|
|
if !ok {
|
|
continue
|
|
}
|
|
// Only forwards. Two routers reporting, or a stale snapshot arriving
|
|
// late, must not move a replica's last use backwards.
|
|
if at.After(rep.LastUsedAt) {
|
|
rep.LastUsedAt = at
|
|
}
|
|
}
|
|
r.noteUsage(lastUsed)
|
|
}
|
|
|
|
// IdleFor reports how long a replica has gone without a request, and whether
|
|
// that is known at all.
|
|
//
|
|
// The second return is not a detail. "Idle for an hour" and "nobody has told me
|
|
// yet" are different states, and anything that acts on idleness — eviction,
|
|
// most obviously — has to refuse to act on the second.
|
|
func (r *Reconciler) IdleFor(id string) (time.Duration, bool) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
rep, ok := r.replicas[id]
|
|
if !ok || rep.LastUsedAt.IsZero() {
|
|
return 0, false
|
|
}
|
|
return r.now().Sub(rep.LastUsedAt), true
|
|
}
|
|
|
|
// Want marks an on-demand model as asked for, so the next pass places it.
|
|
//
|
|
// Called by whatever fields the request — the router, in practice. It returns
|
|
// whether the model is one this applies to, so a caller can tell "being loaded"
|
|
// from "there is genuinely nothing serving this", which are different answers
|
|
// to give a client.
|
|
func (r *Reconciler) Want(model string) bool {
|
|
m, ok := r.cat.Load().Resolve(model)
|
|
if !ok || !m.Scaling.OnDemand() {
|
|
return false
|
|
}
|
|
r.mu.Lock()
|
|
r.wanted[m.Name] = r.now()
|
|
// A placement that failed before this request says nothing about now:
|
|
// forgotten, so the pass this triggers decides afresh and Unservable
|
|
// does not answer from a stale result.
|
|
if res, ok := r.lastExplain[m.Name]; ok && res.Placement == nil {
|
|
delete(r.lastExplain, m.Name)
|
|
}
|
|
r.mu.Unlock()
|
|
// Immediately, not on the next tick. Someone is waiting on this request,
|
|
// and five seconds of ticker is five seconds they spend staring at nothing.
|
|
r.Notify()
|
|
return true
|
|
}
|
|
|
|
// desiredFor is how many replicas a model should have right now.
|
|
//
|
|
// For an ordinary model that is its configured minimum. For an on-demand one it
|
|
// is one while somebody is asking and none once they stop, which is the whole
|
|
// feature: a cluster with more models than GPUs can hold every model in its
|
|
// catalog and only the wanted ones in its cards.
|
|
func (r *Reconciler) desiredFor(m catalog.Model) int {
|
|
base := r.baseFor(m)
|
|
if base == 0 {
|
|
return 0
|
|
}
|
|
r.mu.Lock()
|
|
extra := r.extra[m.Name]
|
|
r.mu.Unlock()
|
|
return min(base+extra, max(m.Scaling.Max, base))
|
|
}
|
|
|
|
// baseFor is what a model runs without load: its minimum, or for an on-demand
|
|
// model one while somebody is asking and none once they stop.
|
|
func (r *Reconciler) baseFor(m catalog.Model) int {
|
|
if m.IsExternal() {
|
|
return 0 // served elsewhere: nothing to place, whatever its scaling says
|
|
}
|
|
if !m.Scaling.OnDemand() {
|
|
return m.Scaling.Min
|
|
}
|
|
r.mu.Lock()
|
|
last, asked := r.wanted[m.Name]
|
|
r.mu.Unlock()
|
|
if !asked {
|
|
return 0
|
|
}
|
|
if r.now().Sub(last) > r.idleTimeout(m) {
|
|
return 0
|
|
}
|
|
return 1
|
|
}
|
|
|
|
// idleTimeout is how long an on-demand model stays up unasked-for.
|
|
func (r *Reconciler) idleTimeout(m catalog.Model) time.Duration {
|
|
if m.Scaling.IdleTimeout > 0 {
|
|
return m.Scaling.IdleTimeout
|
|
}
|
|
return r.cfg.IdleTimeout
|
|
}
|
|
|
|
// noteUsage keeps an on-demand model alive while it is being used.
|
|
//
|
|
// Without this a model would be evicted exactly IdleTimeout after the request
|
|
// that loaded it, however busy it had been since — the timer would measure
|
|
// time since the first request rather than the last.
|
|
func (r *Reconciler) noteUsage(lastUsed map[string]time.Time) {
|
|
byModel := map[string]time.Time{}
|
|
for id, at := range lastUsed {
|
|
rep, ok := r.replicas[id]
|
|
if !ok {
|
|
continue
|
|
}
|
|
if at.After(byModel[rep.Model]) {
|
|
byModel[rep.Model] = at
|
|
}
|
|
}
|
|
for model, at := range byModel {
|
|
if cur, ok := r.wanted[model]; ok && at.After(cur) {
|
|
r.wanted[model] = at
|
|
}
|
|
}
|
|
}
|
|
|
|
// Load is what a model's replicas are doing, as the router sees it.
|
|
type Load struct {
|
|
Replicas int
|
|
Running int
|
|
Waiting int
|
|
}
|
|
|
|
// RecordLoad adds or takes away replicas between a model's min and max as its
|
|
// load says.
|
|
//
|
|
// Max was a ceiling nothing reached: every model ran exactly its minimum
|
|
// however long its queue. Now requests waiting at the engines — every slot of
|
|
// every replica busy — held for ScaleUpAfter add one replica, and a quiet spell
|
|
// of ScaleDownAfter takes one of those back. One step at a time, and not while
|
|
// the last one is still starting: on a small CPU fleet each replica is minutes
|
|
// of weights and gigabytes of memory, and a spike that is over by the time it
|
|
// loads should not leave three behind.
|
|
//
|
|
// With a benchmark stored for the profile a model runs on, its measured
|
|
// capacity is used too: more requests per replica than were measured to meet
|
|
// the SLO is busy, even with the engines' queues empty — an engine will take
|
|
// requests into its slots long after latency has gone — and a replica is only
|
|
// taken back when the rest would still be under scaleDownHeadroom of what they
|
|
// were measured to hold, so scaling down does not land on the edge.
|
|
func (r *Reconciler) RecordLoad(loads map[string]Load) {
|
|
now := r.now()
|
|
// The profile each model's replicas run on, for its measurement; read
|
|
// before the loop, which holds the lock.
|
|
running := map[string]string{}
|
|
r.mu.Lock()
|
|
for _, rep := range r.replicas {
|
|
if rep.State.Counts() {
|
|
running[rep.Model] = rep.Profile
|
|
}
|
|
}
|
|
r.mu.Unlock()
|
|
measured := map[string]Measured{}
|
|
for model, profile := range running {
|
|
if m, ok := r.measured(model, profile); ok {
|
|
measured[model] = m
|
|
}
|
|
}
|
|
for _, m := range r.cat.Load().Models() {
|
|
base := r.baseFor(m)
|
|
r.mu.Lock()
|
|
if base == 0 || m.Scaling.Max <= base {
|
|
delete(r.extra, m.Name)
|
|
delete(r.busySince, m.Name)
|
|
delete(r.quietSince, m.Name)
|
|
r.mu.Unlock()
|
|
continue
|
|
}
|
|
starting := false
|
|
for _, rep := range r.replicas {
|
|
if rep.Model == m.Name && (rep.State == Placing || rep.State == Starting) {
|
|
starting = true
|
|
}
|
|
}
|
|
l := loads[m.Name]
|
|
perReplica := float64(l.Waiting) / float64(max(l.Replicas, 1))
|
|
extra := r.extra[m.Name]
|
|
meas, hasMeas := measured[m.Name]
|
|
overMeasured := hasMeas &&
|
|
float64(l.Running+l.Waiting) > float64(meas.MaxConcurrency*max(l.Replicas, 1))
|
|
fitsInFewer := !hasMeas ||
|
|
float64(l.Running) <= scaleDownHeadroom*float64(meas.MaxConcurrency*max(l.Replicas-1, 0))
|
|
switch {
|
|
case perReplica >= r.cfg.ScaleUpWaiting || overMeasured:
|
|
delete(r.quietSince, m.Name)
|
|
since, ok := r.busySince[m.Name]
|
|
if !ok {
|
|
r.busySince[m.Name] = now
|
|
break
|
|
}
|
|
if now.Sub(since) >= r.cfg.ScaleUpAfter && base+extra < m.Scaling.Max && !starting {
|
|
r.extra[m.Name] = extra + 1
|
|
r.busySince[m.Name] = now
|
|
r.log.Info("scaling up for load", "model", m.Name, "replicas", base+extra+1,
|
|
"waiting_per_replica", perReplica, "in_flight", l.Running+l.Waiting,
|
|
"measured_per_replica", meas.MaxConcurrency, "for", now.Sub(since).Round(time.Second))
|
|
}
|
|
case l.Waiting == 0 && extra > 0 && fitsInFewer:
|
|
delete(r.busySince, m.Name)
|
|
since, ok := r.quietSince[m.Name]
|
|
if !ok {
|
|
r.quietSince[m.Name] = now
|
|
break
|
|
}
|
|
if now.Sub(since) >= r.cfg.ScaleDownAfter {
|
|
r.extra[m.Name] = extra - 1
|
|
r.quietSince[m.Name] = now
|
|
r.log.Info("scaling down after a quiet spell", "model", m.Name,
|
|
"replicas", base+extra-1, "quiet_for", now.Sub(since).Round(time.Second))
|
|
}
|
|
default:
|
|
delete(r.busySince, m.Name)
|
|
delete(r.quietSince, m.Name)
|
|
}
|
|
r.mu.Unlock()
|
|
}
|
|
}
|
|
|
|
// scaleDownHeadroom is how full, of their measured capacity, the replicas left
|
|
// after a scale-down may be. Below one so that the load that was there a
|
|
// minute ago does not put them straight back over.
|
|
const scaleDownHeadroom = 0.7
|