Files
llmgw/internal/control/reconciler/health.go
T
Ilker ManapandClaude Opus 5.5 d864438272
CI / test (push) Successful in 9m31s
A model that failed to start is placed elsewhere next
The same inputs place a replica on the same node and card, so an on-demand
model that failed to load there was started there again after every
backoff until it was parked. The reconciler remembers where a model's
replicas failed and places the next away from them when anything else
fits (Scheduler.Away); with nowhere else, there again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-10 09:22:11 +01:00

263 lines
7.8 KiB
Go

package reconciler
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
// parkRecord is why a model stopped being retried.
type parkRecord struct {
Since time.Time
Reason string
Failures int
// fp is the model's definition when it was parked; a different one
// unparks it.
fp string
}
type placementNote struct {
text string
at time.Time
}
// modelFingerprint identifies a model's definition, so a change to it can be
// told from a retry of the same thing.
func modelFingerprint(m catalog.Model) string {
b, _ := json.Marshal(m)
sum := sha256.Sum256(b)
return hex.EncodeToString(sum[:8])
}
// placementLogEvery is how often an unchanged placement failure is repeated.
const placementLogEvery = 10 * time.Minute
// notePlacementFailure logs why a model could not be placed, when the reason
// is new or has not been said for a while. Said on every pass it was 250
// identical lines in two days for one model, burying everything else.
func (r *Reconciler) notePlacementFailure(model, why string) {
r.mu.Lock()
last := r.placementLogged[model]
say := last.text != why || r.now().Sub(last.at) >= placementLogEvery
if say {
r.placementLogged[model] = placementNote{text: why, at: r.now()}
}
r.mu.Unlock()
if say {
r.log.Warn("no placement for model", "model", model, "reasons", why)
}
}
// Retry clears a parked model and its backoff, so it is placed on the next
// pass.
func (r *Reconciler) Retry(model string) bool {
r.mu.Lock()
_, was := r.parked[model]
delete(r.parked, model)
r.attempts[model] = 0
delete(r.nextAttempt, model)
delete(r.failedAt, model)
r.mu.Unlock()
r.Notify()
return was
}
// ModelHealth is how one model stands, for status and metrics.
type ModelHealth struct {
Model string
// Desired is how many replicas it should have now; 0 for a stopped model
// or an on-demand one nobody is asking for.
Desired int
Ready int
// Failures counts replicas that failed in a row.
Failures int
// Parked is set when Failures reached ParkAfter; Reason is the last error.
Parked bool
ParkedSince time.Time
Reason string
// Unplaceable is set when it is short of replicas and the last attempt to
// place one found nowhere; Placement says why.
Unplaceable bool
Placement string
// Class is the capacity class it is treated as. Preempted counts the
// replicas it has lost to models of a higher class since the control
// plane started; HeldFor names the model it is giving room to now.
Class string
Preempted int
HeldFor string
}
// ModelHealth reports every model in the catalog.
func (r *Reconciler) ModelHealth() []ModelHealth {
models := r.cat.Load().Models()
desired := map[string]int{}
for _, m := range models {
desired[m.Name] = r.desiredFor(m)
}
r.mu.Lock()
defer r.mu.Unlock()
ready, live := map[string]int{}, map[string]int{}
for _, rep := range r.replicas {
if rep.State == Ready {
ready[rep.Model]++
}
if rep.State.Counts() {
live[rep.Model]++
}
}
out := make([]ModelHealth, 0, len(models))
for _, m := range models {
h := ModelHealth{Model: m.Name, Desired: desired[m.Name], Ready: ready[m.Name],
Failures: r.attempts[m.Name], Class: m.EffectiveClass(), Preempted: r.preempted[m.Name]}
if hd, ok := r.held[m.Name]; ok && r.now().Before(hd.Until) {
h.HeldFor = hd.For
}
if p, ok := r.parked[m.Name]; ok {
h.Parked, h.ParkedSince, h.Reason = true, p.Since, p.Reason
}
if res, ok := r.lastExplain[m.Name]; ok && res.Placement == nil && live[m.Name] < h.Desired {
h.Unplaceable, h.Placement = true, res.Explain()
}
out = append(out, h)
}
return out
}
// Unservable says why a model cannot be served at all, so a request for it is
// told now rather than asked to retry something that will never come up.
func (r *Reconciler) Unservable(model string) (string, bool) {
m, ok := r.cat.Load().Resolve(r.hostOf(model))
if !ok {
return "", false
}
cat := r.cat.Load()
r.mu.Lock()
if p, ok := r.parked[m.Name]; ok {
r.mu.Unlock()
return fmt.Sprintf("%s failed to start %d times in a row and is not being retried: %s",
m.Name, p.Failures, p.Reason), true
}
var asleepLater []victim
for _, rep := range r.replicas {
if !rep.State.Counts() {
continue
}
if rep.Model == m.Name {
r.mu.Unlock()
return "", false // something is on its way
}
if other, ok := cat.Resolve(rep.Model); ok && other.Scaling.OnDemand() && len(nodesOf(rep)) == 1 {
asleepLater = append(asleepLater, victim{ID: rep.ID, Model: rep.Model})
}
}
res, ok := r.lastExplain[m.Name]
r.mu.Unlock()
if !ok || res.Placement != nil {
return "", false
}
// The cards are held by models that let go of them when idle, which
// makeRoomFor hastens once they have been up a while. That is a wait, not
// a never: a 3D model asked for straight after a picture, on a cluster
// with one card, waits for the picture model to step aside.
if m.Scaling.OnDemand() && len(asleepLater) > 0 && r.fitsWithout(m, nil, asleepLater) {
return "", false
}
return m.Name + " cannot be placed on any node: " + firstLine(res.Explain()), true
}
// WaitFor says which model holds the cards an on-demand model is waiting
// for, and about how long until it serves: until the holder lets go — evicted
// once idle and old enough, or stopped at its own idle timeout — and then its
// own start, as long as its last one took. ok is false when it is not waiting
// on another model; eta is zero when when the holder lets go is not known.
func (r *Reconciler) WaitFor(model string) (heldBy string, eta time.Duration, ok bool) {
cat := r.cat.Load()
m, found := cat.Resolve(r.hostOf(model))
if !found || !m.Scaling.OnDemand() {
return "", 0, false
}
type holder struct {
v victim
release time.Time
}
now := r.now()
var holders []holder
r.mu.Lock()
for _, rep := range r.replicas {
if !rep.State.Counts() || len(nodesOf(rep)) != 1 {
continue
}
if rep.Model == m.Name {
r.mu.Unlock()
return "", 0, false // its own replica is on its way
}
other, found := cat.Resolve(rep.Model)
if !found || !other.Scaling.OnDemand() {
continue
}
h := holder{v: victim{ID: rep.ID, Model: rep.Model}}
if !rep.LastUsedAt.IsZero() {
h.release = rep.LastUsedAt.Add(r.idleTimeout(other))
// Evicted sooner, when it is no higher a class than the model
// waiting; one put first is waited for to its idle timeout.
if catalog.ClassRank(other.EffectiveClass()) <= catalog.ClassRank(m.EffectiveClass()) {
evict := rep.LastUsedAt.Add(r.evictionGrace())
if !rep.ReadyAt.IsZero() && rep.ReadyAt.Add(r.minLifetime()).After(evict) {
evict = rep.ReadyAt.Add(r.minLifetime())
}
if evict.Before(h.release) {
h.release = evict
}
}
}
holders = append(holders, h)
}
cold := r.coldStart[m.Name]
r.mu.Unlock()
// The one whose going alone makes room, letting go soonest; failing
// that, all of them, and the last to let go.
var best *holder
for i := range holders {
h := &holders[i]
if !r.fitsWithout(m, nil, []victim{h.v}) {
continue
}
if best == nil || (!h.release.IsZero() && (best.release.IsZero() || h.release.Before(best.release))) {
best = h
}
}
if best == nil {
all := make([]victim, len(holders))
for i, h := range holders {
all[i] = h.v
}
if len(all) == 0 || !r.fitsWithout(m, nil, all) {
return "", 0, false
}
for i := range holders {
h := &holders[i]
if best == nil || h.release.IsZero() || (!best.release.IsZero() && h.release.After(best.release)) {
best = h
}
}
}
if best.release.IsZero() {
return best.v.Model, 0, true
}
return best.v.Model, max(best.release.Sub(now), 0) + cold, true
}
func firstLine(s string) string {
s = strings.TrimSpace(s)
if i := strings.IndexByte(s, '\n'); i >= 0 {
return strings.TrimSpace(s[:i]) + " (and others; see llmgw ctl explain-placement)"
}
return s
}