Files
llmgw/internal/control/reconciler/health.go
T
Ilker ManapandClaude Opus 5.5 aa531b1053 ComfyUI models of one environment share one replica
flux and sf3d on one card handed it to each other, each starting ComfyUI
and loading its weights from nothing (BACKLOG H3, placement side). An
environment of two or more models now gets a hidden catalog entry,
comfyui-env-<name>: on its first member's profiles with the most memory
any member needs, on demand for the longest idle timeout, carrying every
member's weights, repositories and the environment's custom nodes. The
members get no replicas of their own; asking for one wants the entry.

The agent fetches each member's weights; the gate links all of them,
refusing two weights files of one name, links each member's Hugging Face
repository, and answers /v1/models for every member. The control plane
offers the replica under the members' names with each one's workflow and
checkpoint, which the router sends by the model asked for. A member's
sibling never evicts the replica serving it, and the members' load counts
as the entry's.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-14 13:55:34 +02:00

262 lines
7.8 KiB
Go

package reconciler
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
// parkRecord is why a model stopped being retried.
type parkRecord struct {
Since time.Time
Reason string
Failures int
// fp is the model's definition when it was parked; a different one
// unparks it.
fp string
}
type placementNote struct {
text string
at time.Time
}
// modelFingerprint identifies a model's definition, so a change to it can be
// told from a retry of the same thing.
func modelFingerprint(m catalog.Model) string {
b, _ := json.Marshal(m)
sum := sha256.Sum256(b)
return hex.EncodeToString(sum[:8])
}
// placementLogEvery is how often an unchanged placement failure is repeated.
const placementLogEvery = 10 * time.Minute
// notePlacementFailure logs why a model could not be placed, when the reason
// is new or has not been said for a while. Said on every pass it was 250
// identical lines in two days for one model, burying everything else.
func (r *Reconciler) notePlacementFailure(model, why string) {
r.mu.Lock()
last := r.placementLogged[model]
say := last.text != why || r.now().Sub(last.at) >= placementLogEvery
if say {
r.placementLogged[model] = placementNote{text: why, at: r.now()}
}
r.mu.Unlock()
if say {
r.log.Warn("no placement for model", "model", model, "reasons", why)
}
}
// Retry clears a parked model and its backoff, so it is placed on the next
// pass.
func (r *Reconciler) Retry(model string) bool {
r.mu.Lock()
_, was := r.parked[model]
delete(r.parked, model)
r.attempts[model] = 0
delete(r.nextAttempt, model)
r.mu.Unlock()
r.Notify()
return was
}
// ModelHealth is how one model stands, for status and metrics.
type ModelHealth struct {
Model string
// Desired is how many replicas it should have now; 0 for a stopped model
// or an on-demand one nobody is asking for.
Desired int
Ready int
// Failures counts replicas that failed in a row.
Failures int
// Parked is set when Failures reached ParkAfter; Reason is the last error.
Parked bool
ParkedSince time.Time
Reason string
// Unplaceable is set when it is short of replicas and the last attempt to
// place one found nowhere; Placement says why.
Unplaceable bool
Placement string
// Class is the capacity class it is treated as. Preempted counts the
// replicas it has lost to models of a higher class since the control
// plane started; HeldFor names the model it is giving room to now.
Class string
Preempted int
HeldFor string
}
// ModelHealth reports every model in the catalog.
func (r *Reconciler) ModelHealth() []ModelHealth {
models := r.cat.Load().Models()
desired := map[string]int{}
for _, m := range models {
desired[m.Name] = r.desiredFor(m)
}
r.mu.Lock()
defer r.mu.Unlock()
ready, live := map[string]int{}, map[string]int{}
for _, rep := range r.replicas {
if rep.State == Ready {
ready[rep.Model]++
}
if rep.State.Counts() {
live[rep.Model]++
}
}
out := make([]ModelHealth, 0, len(models))
for _, m := range models {
h := ModelHealth{Model: m.Name, Desired: desired[m.Name], Ready: ready[m.Name],
Failures: r.attempts[m.Name], Class: m.EffectiveClass(), Preempted: r.preempted[m.Name]}
if hd, ok := r.held[m.Name]; ok && r.now().Before(hd.Until) {
h.HeldFor = hd.For
}
if p, ok := r.parked[m.Name]; ok {
h.Parked, h.ParkedSince, h.Reason = true, p.Since, p.Reason
}
if res, ok := r.lastExplain[m.Name]; ok && res.Placement == nil && live[m.Name] < h.Desired {
h.Unplaceable, h.Placement = true, res.Explain()
}
out = append(out, h)
}
return out
}
// Unservable says why a model cannot be served at all, so a request for it is
// told now rather than asked to retry something that will never come up.
func (r *Reconciler) Unservable(model string) (string, bool) {
m, ok := r.cat.Load().Resolve(r.hostOf(model))
if !ok {
return "", false
}
cat := r.cat.Load()
r.mu.Lock()
if p, ok := r.parked[m.Name]; ok {
r.mu.Unlock()
return fmt.Sprintf("%s failed to start %d times in a row and is not being retried: %s",
m.Name, p.Failures, p.Reason), true
}
var asleepLater []victim
for _, rep := range r.replicas {
if !rep.State.Counts() {
continue
}
if rep.Model == m.Name {
r.mu.Unlock()
return "", false // something is on its way
}
if other, ok := cat.Resolve(rep.Model); ok && other.Scaling.OnDemand() && len(nodesOf(rep)) == 1 {
asleepLater = append(asleepLater, victim{ID: rep.ID, Model: rep.Model})
}
}
res, ok := r.lastExplain[m.Name]
r.mu.Unlock()
if !ok || res.Placement != nil {
return "", false
}
// The cards are held by models that let go of them when idle, which
// makeRoomFor hastens once they have been up a while. That is a wait, not
// a never: a 3D model asked for straight after a picture, on a cluster
// with one card, waits for the picture model to step aside.
if m.Scaling.OnDemand() && len(asleepLater) > 0 && r.fitsWithout(m, nil, asleepLater) {
return "", false
}
return m.Name + " cannot be placed on any node: " + firstLine(res.Explain()), true
}
// WaitFor says which model holds the cards an on-demand model is waiting
// for, and about how long until it serves: until the holder lets go — evicted
// once idle and old enough, or stopped at its own idle timeout — and then its
// own start, as long as its last one took. ok is false when it is not waiting
// on another model; eta is zero when when the holder lets go is not known.
func (r *Reconciler) WaitFor(model string) (heldBy string, eta time.Duration, ok bool) {
cat := r.cat.Load()
m, found := cat.Resolve(r.hostOf(model))
if !found || !m.Scaling.OnDemand() {
return "", 0, false
}
type holder struct {
v victim
release time.Time
}
now := r.now()
var holders []holder
r.mu.Lock()
for _, rep := range r.replicas {
if !rep.State.Counts() || len(nodesOf(rep)) != 1 {
continue
}
if rep.Model == m.Name {
r.mu.Unlock()
return "", 0, false // its own replica is on its way
}
other, found := cat.Resolve(rep.Model)
if !found || !other.Scaling.OnDemand() {
continue
}
h := holder{v: victim{ID: rep.ID, Model: rep.Model}}
if !rep.LastUsedAt.IsZero() {
h.release = rep.LastUsedAt.Add(r.idleTimeout(other))
// Evicted sooner, when it is no higher a class than the model
// waiting; one put first is waited for to its idle timeout.
if catalog.ClassRank(other.EffectiveClass()) <= catalog.ClassRank(m.EffectiveClass()) {
evict := rep.LastUsedAt.Add(r.evictionGrace())
if !rep.ReadyAt.IsZero() && rep.ReadyAt.Add(r.minLifetime()).After(evict) {
evict = rep.ReadyAt.Add(r.minLifetime())
}
if evict.Before(h.release) {
h.release = evict
}
}
}
holders = append(holders, h)
}
cold := r.coldStart[m.Name]
r.mu.Unlock()
// The one whose going alone makes room, letting go soonest; failing
// that, all of them, and the last to let go.
var best *holder
for i := range holders {
h := &holders[i]
if !r.fitsWithout(m, nil, []victim{h.v}) {
continue
}
if best == nil || (!h.release.IsZero() && (best.release.IsZero() || h.release.Before(best.release))) {
best = h
}
}
if best == nil {
all := make([]victim, len(holders))
for i, h := range holders {
all[i] = h.v
}
if len(all) == 0 || !r.fitsWithout(m, nil, all) {
return "", 0, false
}
for i := range holders {
h := &holders[i]
if best == nil || h.release.IsZero() || (!best.release.IsZero() && h.release.After(best.release)) {
best = h
}
}
}
if best.release.IsZero() {
return best.v.Model, 0, true
}
return best.v.Model, max(best.release.Sub(now), 0) + cold, true
}
func firstLine(s string) string {
s = strings.TrimSpace(s)
if i := strings.IndexByte(s, '\n'); i >= 0 {
return strings.TrimSpace(s[:i]) + " (and others; see llmgw ctl explain-placement)"
}
return s
}