Files
llmgw/internal/control/reconciler/demand.go
T
Ilker ManapandClaude Opus 5.5 62657f4c2a
Release / build (push) Successful in 4m11s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 4m57s
Release v0.10.4: CPU threads, tracing, and the rest of the review
CPU replicas start on half the machine's CPUs, not all of them: two busy
llama.cpp servers each spinning on all 8 vCPUs fell to 0.8 tok/s apiece,
where at 4 threads they keep 5.5. llamacpp.threads sets it per node.

Inference requests can be traced over OTLP/HTTP (tracing.endpoint), with
admission, schedule and upstream spans and traceparent sent to the engine.

Security: onboarding no longer puts the cluster root in the system trust
store and removes it where it was; the mirror's mutual-TLS API can be
limited to named certificates (--api-client); init.log no longer keeps the
administrator's password; a warning when auth is off on a public listen;
the recorded actor is sanitised and sudo-aware; Ray's dashboard is off, and
the remaining engine gaps are documented.

reconciler.go and agent.go are split by topic, the status mapping of each
failure is tested, and the plan's checklist is brought up to date.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 04:00:35 +02:00

215 lines
6.5 KiB
Go

package reconciler
import (
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
// RecordUsage stamps replicas with when each was last sent a request.
//
// Replicas absent from the map are left alone rather than zeroed: a router that
// has just restarted knows nothing about what happened before it, and treating
// that as "never used" would make every replica look abandoned at once —
// exactly when a fleet is least able to afford acting on that.
func (r *Reconciler) RecordUsage(lastUsed map[string]time.Time) {
if len(lastUsed) == 0 {
return
}
r.mu.Lock()
defer r.mu.Unlock()
for id, at := range lastUsed {
rep, ok := r.replicas[id]
if !ok {
continue
}
// Only forwards. Two routers reporting, or a stale snapshot arriving
// late, must not move a replica's last use backwards.
if at.After(rep.LastUsedAt) {
rep.LastUsedAt = at
}
}
r.noteUsage(lastUsed)
}
// IdleFor reports how long a replica has gone without a request, and whether
// that is known at all.
//
// The second return is not a detail. "Idle for an hour" and "nobody has told me
// yet" are different states, and anything that acts on idleness — eviction,
// most obviously — has to refuse to act on the second.
func (r *Reconciler) IdleFor(id string) (time.Duration, bool) {
r.mu.Lock()
defer r.mu.Unlock()
rep, ok := r.replicas[id]
if !ok || rep.LastUsedAt.IsZero() {
return 0, false
}
return r.now().Sub(rep.LastUsedAt), true
}
// Want marks an on-demand model as asked for, so the next pass places it.
//
// Called by whatever fields the request — the router, in practice. It returns
// whether the model is one this applies to, so a caller can tell "being loaded"
// from "there is genuinely nothing serving this", which are different answers
// to give a client.
func (r *Reconciler) Want(model string) bool {
m, ok := r.cat.Load().Resolve(model)
if !ok || !m.Scaling.OnDemand() {
return false
}
r.mu.Lock()
r.wanted[m.Name] = r.now()
// A placement that failed before this request says nothing about now:
// forgotten, so the pass this triggers decides afresh and Unservable
// does not answer from a stale result.
if res, ok := r.lastExplain[m.Name]; ok && res.Placement == nil {
delete(r.lastExplain, m.Name)
}
r.mu.Unlock()
// Immediately, not on the next tick. Someone is waiting on this request,
// and five seconds of ticker is five seconds they spend staring at nothing.
r.Notify()
return true
}
// desiredFor is how many replicas a model should have right now.
//
// For an ordinary model that is its configured minimum. For an on-demand one it
// is one while somebody is asking and none once they stop, which is the whole
// feature: a cluster with more models than GPUs can hold every model in its
// catalog and only the wanted ones in its cards.
func (r *Reconciler) desiredFor(m catalog.Model) int {
base := r.baseFor(m)
if base == 0 {
return 0
}
r.mu.Lock()
extra := r.extra[m.Name]
r.mu.Unlock()
return min(base+extra, max(m.Scaling.Max, base))
}
// baseFor is what a model runs without load: its minimum, or for an on-demand
// model one while somebody is asking and none once they stop.
func (r *Reconciler) baseFor(m catalog.Model) int {
if !m.Scaling.OnDemand() {
return m.Scaling.Min
}
r.mu.Lock()
last, asked := r.wanted[m.Name]
r.mu.Unlock()
if !asked {
return 0
}
if r.now().Sub(last) > r.idleTimeout(m) {
return 0
}
return 1
}
// idleTimeout is how long an on-demand model stays up unasked-for.
func (r *Reconciler) idleTimeout(m catalog.Model) time.Duration {
if m.Scaling.IdleTimeout > 0 {
return m.Scaling.IdleTimeout
}
return r.cfg.IdleTimeout
}
// noteUsage keeps an on-demand model alive while it is being used.
//
// Without this a model would be evicted exactly IdleTimeout after the request
// that loaded it, however busy it had been since — the timer would measure
// time since the first request rather than the last.
func (r *Reconciler) noteUsage(lastUsed map[string]time.Time) {
byModel := map[string]time.Time{}
for id, at := range lastUsed {
rep, ok := r.replicas[id]
if !ok {
continue
}
if at.After(byModel[rep.Model]) {
byModel[rep.Model] = at
}
}
for model, at := range byModel {
if cur, ok := r.wanted[model]; ok && at.After(cur) {
r.wanted[model] = at
}
}
}
// Load is what a model's replicas are doing, as the router sees it.
type Load struct {
Replicas int
Running int
Waiting int
}
// RecordLoad adds or takes away replicas between a model's min and max as its
// load says.
//
// Max was a ceiling nothing reached: every model ran exactly its minimum
// however long its queue. Now requests waiting at the engines — every slot of
// every replica busy — held for ScaleUpAfter add one replica, and a quiet spell
// of ScaleDownAfter takes one of those back. One step at a time, and not while
// the last one is still starting: on a small CPU fleet each replica is minutes
// of weights and gigabytes of memory, and a spike that is over by the time it
// loads should not leave three behind.
func (r *Reconciler) RecordLoad(loads map[string]Load) {
now := r.now()
for _, m := range r.cat.Load().Models() {
base := r.baseFor(m)
r.mu.Lock()
if base == 0 || m.Scaling.Max <= base {
delete(r.extra, m.Name)
delete(r.busySince, m.Name)
delete(r.quietSince, m.Name)
r.mu.Unlock()
continue
}
starting := false
for _, rep := range r.replicas {
if rep.Model == m.Name && (rep.State == Placing || rep.State == Starting) {
starting = true
}
}
l := loads[m.Name]
perReplica := float64(l.Waiting) / float64(max(l.Replicas, 1))
extra := r.extra[m.Name]
switch {
case perReplica >= r.cfg.ScaleUpWaiting:
delete(r.quietSince, m.Name)
since, ok := r.busySince[m.Name]
if !ok {
r.busySince[m.Name] = now
break
}
if now.Sub(since) >= r.cfg.ScaleUpAfter && base+extra < m.Scaling.Max && !starting {
r.extra[m.Name] = extra + 1
r.busySince[m.Name] = now
r.log.Info("scaling up for load", "model", m.Name, "replicas", base+extra+1,
"waiting_per_replica", perReplica, "for", now.Sub(since).Round(time.Second))
}
case l.Waiting == 0 && extra > 0:
delete(r.busySince, m.Name)
since, ok := r.quietSince[m.Name]
if !ok {
r.quietSince[m.Name] = now
break
}
if now.Sub(since) >= r.cfg.ScaleDownAfter {
r.extra[m.Name] = extra - 1
r.quietSince[m.Name] = now
r.log.Info("scaling down after a quiet spell", "model", m.Name,
"replicas", base+extra-1, "quiet_for", now.Sub(since).Round(time.Second))
}
default:
delete(r.busySince, m.Name)
delete(r.quietSince, m.Name)
}
r.mu.Unlock()
}
}