Files
llmgw/internal/control/reconciler/health.go
T
Ilker ManapandClaude Opus 5.5 fa6e68c8e8
CI / test (push) Failing after 7m59s
A model put first is not evicted, idle, for a lower class
The VLM is to come first on the one card. Its class makes it take the card
from flux, triposr or sf3d when asked for, but idle eviction ignored class:
a 3D job two minutes into a lull would have taken the card from it. Idle
eviction offers only replicas of no higher class than the model wanting
room, and the wait such a model is told runs to the holder's idle timeout.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-13 10:15:14 +02:00

262 lines
7.7 KiB
Go

package reconciler
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
// parkRecord is why a model stopped being retried.
type parkRecord struct {
Since time.Time
Reason string
Failures int
// fp is the model's definition when it was parked; a different one
// unparks it.
fp string
}
type placementNote struct {
text string
at time.Time
}
// modelFingerprint identifies a model's definition, so a change to it can be
// told from a retry of the same thing.
func modelFingerprint(m catalog.Model) string {
b, _ := json.Marshal(m)
sum := sha256.Sum256(b)
return hex.EncodeToString(sum[:8])
}
// placementLogEvery is how often an unchanged placement failure is repeated.
const placementLogEvery = 10 * time.Minute
// notePlacementFailure logs why a model could not be placed, when the reason
// is new or has not been said for a while. Said on every pass it was 250
// identical lines in two days for one model, burying everything else.
func (r *Reconciler) notePlacementFailure(model, why string) {
r.mu.Lock()
last := r.placementLogged[model]
say := last.text != why || r.now().Sub(last.at) >= placementLogEvery
if say {
r.placementLogged[model] = placementNote{text: why, at: r.now()}
}
r.mu.Unlock()
if say {
r.log.Warn("no placement for model", "model", model, "reasons", why)
}
}
// Retry clears a parked model and its backoff, so it is placed on the next
// pass.
func (r *Reconciler) Retry(model string) bool {
r.mu.Lock()
_, was := r.parked[model]
delete(r.parked, model)
r.attempts[model] = 0
delete(r.nextAttempt, model)
r.mu.Unlock()
r.Notify()
return was
}
// ModelHealth is how one model stands, for status and metrics.
type ModelHealth struct {
Model string
// Desired is how many replicas it should have now; 0 for a stopped model
// or an on-demand one nobody is asking for.
Desired int
Ready int
// Failures counts replicas that failed in a row.
Failures int
// Parked is set when Failures reached ParkAfter; Reason is the last error.
Parked bool
ParkedSince time.Time
Reason string
// Unplaceable is set when it is short of replicas and the last attempt to
// place one found nowhere; Placement says why.
Unplaceable bool
Placement string
// Class is the capacity class it is treated as. Preempted counts the
// replicas it has lost to models of a higher class since the control
// plane started; HeldFor names the model it is giving room to now.
Class string
Preempted int
HeldFor string
}
// ModelHealth reports every model in the catalog.
func (r *Reconciler) ModelHealth() []ModelHealth {
models := r.cat.Load().Models()
desired := map[string]int{}
for _, m := range models {
desired[m.Name] = r.desiredFor(m)
}
r.mu.Lock()
defer r.mu.Unlock()
ready, live := map[string]int{}, map[string]int{}
for _, rep := range r.replicas {
if rep.State == Ready {
ready[rep.Model]++
}
if rep.State.Counts() {
live[rep.Model]++
}
}
out := make([]ModelHealth, 0, len(models))
for _, m := range models {
h := ModelHealth{Model: m.Name, Desired: desired[m.Name], Ready: ready[m.Name],
Failures: r.attempts[m.Name], Class: m.EffectiveClass(), Preempted: r.preempted[m.Name]}
if hd, ok := r.held[m.Name]; ok && r.now().Before(hd.Until) {
h.HeldFor = hd.For
}
if p, ok := r.parked[m.Name]; ok {
h.Parked, h.ParkedSince, h.Reason = true, p.Since, p.Reason
}
if res, ok := r.lastExplain[m.Name]; ok && res.Placement == nil && live[m.Name] < h.Desired {
h.Unplaceable, h.Placement = true, res.Explain()
}
out = append(out, h)
}
return out
}
// Unservable says why a model cannot be served at all, so a request for it is
// told now rather than asked to retry something that will never come up.
func (r *Reconciler) Unservable(model string) (string, bool) {
m, ok := r.cat.Load().Resolve(model)
if !ok {
return "", false
}
cat := r.cat.Load()
r.mu.Lock()
if p, ok := r.parked[m.Name]; ok {
r.mu.Unlock()
return fmt.Sprintf("%s failed to start %d times in a row and is not being retried: %s",
m.Name, p.Failures, p.Reason), true
}
var asleepLater []victim
for _, rep := range r.replicas {
if !rep.State.Counts() {
continue
}
if rep.Model == m.Name {
r.mu.Unlock()
return "", false // something is on its way
}
if other, ok := cat.Resolve(rep.Model); ok && other.Scaling.OnDemand() && len(nodesOf(rep)) == 1 {
asleepLater = append(asleepLater, victim{ID: rep.ID, Model: rep.Model})
}
}
res, ok := r.lastExplain[m.Name]
r.mu.Unlock()
if !ok || res.Placement != nil {
return "", false
}
// The cards are held by models that let go of them when idle, which
// makeRoomFor hastens once they have been up a while. That is a wait, not
// a never: a 3D model asked for straight after a picture, on a cluster
// with one card, waits for the picture model to step aside.
if m.Scaling.OnDemand() && len(asleepLater) > 0 && r.fitsWithout(m, nil, asleepLater) {
return "", false
}
return m.Name + " cannot be placed on any node: " + firstLine(res.Explain()), true
}
// WaitFor says which model holds the cards an on-demand model is waiting
// for, and about how long until it serves: until the holder lets go — evicted
// once idle and old enough, or stopped at its own idle timeout — and then its
// own start, as long as its last one took. ok is false when it is not waiting
// on another model; eta is zero when when the holder lets go is not known.
func (r *Reconciler) WaitFor(model string) (heldBy string, eta time.Duration, ok bool) {
cat := r.cat.Load()
m, found := cat.Resolve(model)
if !found || !m.Scaling.OnDemand() {
return "", 0, false
}
type holder struct {
v victim
release time.Time
}
now := r.now()
var holders []holder
r.mu.Lock()
for _, rep := range r.replicas {
if !rep.State.Counts() || len(nodesOf(rep)) != 1 {
continue
}
if rep.Model == m.Name {
r.mu.Unlock()
return "", 0, false // its own replica is on its way
}
other, found := cat.Resolve(rep.Model)
if !found || !other.Scaling.OnDemand() {
continue
}
h := holder{v: victim{ID: rep.ID, Model: rep.Model}}
if !rep.LastUsedAt.IsZero() {
h.release = rep.LastUsedAt.Add(r.idleTimeout(other))
// Evicted sooner, when it is no higher a class than the model
// waiting; one put first is waited for to its idle timeout.
if catalog.ClassRank(other.EffectiveClass()) <= catalog.ClassRank(m.EffectiveClass()) {
evict := rep.LastUsedAt.Add(r.cfg.EvictionGrace)
if !rep.ReadyAt.IsZero() && rep.ReadyAt.Add(r.cfg.MinLifetime).After(evict) {
evict = rep.ReadyAt.Add(r.cfg.MinLifetime)
}
if evict.Before(h.release) {
h.release = evict
}
}
}
holders = append(holders, h)
}
cold := r.coldStart[m.Name]
r.mu.Unlock()
// The one whose going alone makes room, letting go soonest; failing
// that, all of them, and the last to let go.
var best *holder
for i := range holders {
h := &holders[i]
if !r.fitsWithout(m, nil, []victim{h.v}) {
continue
}
if best == nil || (!h.release.IsZero() && (best.release.IsZero() || h.release.Before(best.release))) {
best = h
}
}
if best == nil {
all := make([]victim, len(holders))
for i, h := range holders {
all[i] = h.v
}
if len(all) == 0 || !r.fitsWithout(m, nil, all) {
return "", 0, false
}
for i := range holders {
h := &holders[i]
if best == nil || h.release.IsZero() || (!best.release.IsZero() && h.release.After(best.release)) {
best = h
}
}
}
if best.release.IsZero() {
return best.v.Model, 0, true
}
return best.v.Model, max(best.release.Sub(now), 0) + cold, true
}
func firstLine(s string) string {
s = strings.TrimSpace(s)
if i := strings.IndexByte(s, '\n'); i >= 0 {
return strings.TrimSpace(s[:i]) + " (and others; see llmgw ctl explain-placement)"
}
return s
}