The VLM is to come first on the one card. Its class makes it take the card from flux, triposr or sf3d when asked for, but idle eviction ignored class: a 3D job two minutes into a lull would have taken the card from it. Idle eviction offers only replicas of no higher class than the model wanting room, and the wait such a model is told runs to the holder's idle timeout. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
262 lines
7.7 KiB
Go
262 lines
7.7 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
)
|
|
|
|
// parkRecord is why a model stopped being retried.
|
|
type parkRecord struct {
|
|
Since time.Time
|
|
Reason string
|
|
Failures int
|
|
// fp is the model's definition when it was parked; a different one
|
|
// unparks it.
|
|
fp string
|
|
}
|
|
|
|
type placementNote struct {
|
|
text string
|
|
at time.Time
|
|
}
|
|
|
|
// modelFingerprint identifies a model's definition, so a change to it can be
|
|
// told from a retry of the same thing.
|
|
func modelFingerprint(m catalog.Model) string {
|
|
b, _ := json.Marshal(m)
|
|
sum := sha256.Sum256(b)
|
|
return hex.EncodeToString(sum[:8])
|
|
}
|
|
|
|
// placementLogEvery is how often an unchanged placement failure is repeated.
|
|
const placementLogEvery = 10 * time.Minute
|
|
|
|
// notePlacementFailure logs why a model could not be placed, when the reason
|
|
// is new or has not been said for a while. Said on every pass it was 250
|
|
// identical lines in two days for one model, burying everything else.
|
|
func (r *Reconciler) notePlacementFailure(model, why string) {
|
|
r.mu.Lock()
|
|
last := r.placementLogged[model]
|
|
say := last.text != why || r.now().Sub(last.at) >= placementLogEvery
|
|
if say {
|
|
r.placementLogged[model] = placementNote{text: why, at: r.now()}
|
|
}
|
|
r.mu.Unlock()
|
|
if say {
|
|
r.log.Warn("no placement for model", "model", model, "reasons", why)
|
|
}
|
|
}
|
|
|
|
// Retry clears a parked model and its backoff, so it is placed on the next
|
|
// pass.
|
|
func (r *Reconciler) Retry(model string) bool {
|
|
r.mu.Lock()
|
|
_, was := r.parked[model]
|
|
delete(r.parked, model)
|
|
r.attempts[model] = 0
|
|
delete(r.nextAttempt, model)
|
|
r.mu.Unlock()
|
|
r.Notify()
|
|
return was
|
|
}
|
|
|
|
// ModelHealth is how one model stands, for status and metrics.
|
|
type ModelHealth struct {
|
|
Model string
|
|
// Desired is how many replicas it should have now; 0 for a stopped model
|
|
// or an on-demand one nobody is asking for.
|
|
Desired int
|
|
Ready int
|
|
// Failures counts replicas that failed in a row.
|
|
Failures int
|
|
// Parked is set when Failures reached ParkAfter; Reason is the last error.
|
|
Parked bool
|
|
ParkedSince time.Time
|
|
Reason string
|
|
// Unplaceable is set when it is short of replicas and the last attempt to
|
|
// place one found nowhere; Placement says why.
|
|
Unplaceable bool
|
|
Placement string
|
|
// Class is the capacity class it is treated as. Preempted counts the
|
|
// replicas it has lost to models of a higher class since the control
|
|
// plane started; HeldFor names the model it is giving room to now.
|
|
Class string
|
|
Preempted int
|
|
HeldFor string
|
|
}
|
|
|
|
// ModelHealth reports every model in the catalog.
|
|
func (r *Reconciler) ModelHealth() []ModelHealth {
|
|
models := r.cat.Load().Models()
|
|
desired := map[string]int{}
|
|
for _, m := range models {
|
|
desired[m.Name] = r.desiredFor(m)
|
|
}
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
ready, live := map[string]int{}, map[string]int{}
|
|
for _, rep := range r.replicas {
|
|
if rep.State == Ready {
|
|
ready[rep.Model]++
|
|
}
|
|
if rep.State.Counts() {
|
|
live[rep.Model]++
|
|
}
|
|
}
|
|
out := make([]ModelHealth, 0, len(models))
|
|
for _, m := range models {
|
|
h := ModelHealth{Model: m.Name, Desired: desired[m.Name], Ready: ready[m.Name],
|
|
Failures: r.attempts[m.Name], Class: m.EffectiveClass(), Preempted: r.preempted[m.Name]}
|
|
if hd, ok := r.held[m.Name]; ok && r.now().Before(hd.Until) {
|
|
h.HeldFor = hd.For
|
|
}
|
|
if p, ok := r.parked[m.Name]; ok {
|
|
h.Parked, h.ParkedSince, h.Reason = true, p.Since, p.Reason
|
|
}
|
|
if res, ok := r.lastExplain[m.Name]; ok && res.Placement == nil && live[m.Name] < h.Desired {
|
|
h.Unplaceable, h.Placement = true, res.Explain()
|
|
}
|
|
out = append(out, h)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Unservable says why a model cannot be served at all, so a request for it is
|
|
// told now rather than asked to retry something that will never come up.
|
|
func (r *Reconciler) Unservable(model string) (string, bool) {
|
|
m, ok := r.cat.Load().Resolve(model)
|
|
if !ok {
|
|
return "", false
|
|
}
|
|
cat := r.cat.Load()
|
|
r.mu.Lock()
|
|
if p, ok := r.parked[m.Name]; ok {
|
|
r.mu.Unlock()
|
|
return fmt.Sprintf("%s failed to start %d times in a row and is not being retried: %s",
|
|
m.Name, p.Failures, p.Reason), true
|
|
}
|
|
var asleepLater []victim
|
|
for _, rep := range r.replicas {
|
|
if !rep.State.Counts() {
|
|
continue
|
|
}
|
|
if rep.Model == m.Name {
|
|
r.mu.Unlock()
|
|
return "", false // something is on its way
|
|
}
|
|
if other, ok := cat.Resolve(rep.Model); ok && other.Scaling.OnDemand() && len(nodesOf(rep)) == 1 {
|
|
asleepLater = append(asleepLater, victim{ID: rep.ID, Model: rep.Model})
|
|
}
|
|
}
|
|
res, ok := r.lastExplain[m.Name]
|
|
r.mu.Unlock()
|
|
if !ok || res.Placement != nil {
|
|
return "", false
|
|
}
|
|
// The cards are held by models that let go of them when idle, which
|
|
// makeRoomFor hastens once they have been up a while. That is a wait, not
|
|
// a never: a 3D model asked for straight after a picture, on a cluster
|
|
// with one card, waits for the picture model to step aside.
|
|
if m.Scaling.OnDemand() && len(asleepLater) > 0 && r.fitsWithout(m, nil, asleepLater) {
|
|
return "", false
|
|
}
|
|
return m.Name + " cannot be placed on any node: " + firstLine(res.Explain()), true
|
|
}
|
|
|
|
// WaitFor says which model holds the cards an on-demand model is waiting
|
|
// for, and about how long until it serves: until the holder lets go — evicted
|
|
// once idle and old enough, or stopped at its own idle timeout — and then its
|
|
// own start, as long as its last one took. ok is false when it is not waiting
|
|
// on another model; eta is zero when when the holder lets go is not known.
|
|
func (r *Reconciler) WaitFor(model string) (heldBy string, eta time.Duration, ok bool) {
|
|
cat := r.cat.Load()
|
|
m, found := cat.Resolve(model)
|
|
if !found || !m.Scaling.OnDemand() {
|
|
return "", 0, false
|
|
}
|
|
type holder struct {
|
|
v victim
|
|
release time.Time
|
|
}
|
|
now := r.now()
|
|
var holders []holder
|
|
r.mu.Lock()
|
|
for _, rep := range r.replicas {
|
|
if !rep.State.Counts() || len(nodesOf(rep)) != 1 {
|
|
continue
|
|
}
|
|
if rep.Model == m.Name {
|
|
r.mu.Unlock()
|
|
return "", 0, false // its own replica is on its way
|
|
}
|
|
other, found := cat.Resolve(rep.Model)
|
|
if !found || !other.Scaling.OnDemand() {
|
|
continue
|
|
}
|
|
h := holder{v: victim{ID: rep.ID, Model: rep.Model}}
|
|
if !rep.LastUsedAt.IsZero() {
|
|
h.release = rep.LastUsedAt.Add(r.idleTimeout(other))
|
|
// Evicted sooner, when it is no higher a class than the model
|
|
// waiting; one put first is waited for to its idle timeout.
|
|
if catalog.ClassRank(other.EffectiveClass()) <= catalog.ClassRank(m.EffectiveClass()) {
|
|
evict := rep.LastUsedAt.Add(r.cfg.EvictionGrace)
|
|
if !rep.ReadyAt.IsZero() && rep.ReadyAt.Add(r.cfg.MinLifetime).After(evict) {
|
|
evict = rep.ReadyAt.Add(r.cfg.MinLifetime)
|
|
}
|
|
if evict.Before(h.release) {
|
|
h.release = evict
|
|
}
|
|
}
|
|
}
|
|
holders = append(holders, h)
|
|
}
|
|
cold := r.coldStart[m.Name]
|
|
r.mu.Unlock()
|
|
|
|
// The one whose going alone makes room, letting go soonest; failing
|
|
// that, all of them, and the last to let go.
|
|
var best *holder
|
|
for i := range holders {
|
|
h := &holders[i]
|
|
if !r.fitsWithout(m, nil, []victim{h.v}) {
|
|
continue
|
|
}
|
|
if best == nil || (!h.release.IsZero() && (best.release.IsZero() || h.release.Before(best.release))) {
|
|
best = h
|
|
}
|
|
}
|
|
if best == nil {
|
|
all := make([]victim, len(holders))
|
|
for i, h := range holders {
|
|
all[i] = h.v
|
|
}
|
|
if len(all) == 0 || !r.fitsWithout(m, nil, all) {
|
|
return "", 0, false
|
|
}
|
|
for i := range holders {
|
|
h := &holders[i]
|
|
if best == nil || h.release.IsZero() || (!best.release.IsZero() && h.release.After(best.release)) {
|
|
best = h
|
|
}
|
|
}
|
|
}
|
|
if best.release.IsZero() {
|
|
return best.v.Model, 0, true
|
|
}
|
|
return best.v.Model, max(best.release.Sub(now), 0) + cold, true
|
|
}
|
|
|
|
func firstLine(s string) string {
|
|
s = strings.TrimSpace(s)
|
|
if i := strings.IndexByte(s, '\n'); i >= 0 {
|
|
return strings.TrimSpace(s[:i]) + " (and others; see llmgw ctl explain-placement)"
|
|
}
|
|
return s
|
|
}
|