CI / test (push) Failing after 7m59s
The VLM is to come first on the one card. Its class makes it take the card from flux, triposr or sf3d when asked for, but idle eviction ignored class: a 3D job two minutes into a lull would have taken the card from it. Idle eviction offers only replicas of no higher class than the model wanting room, and the wait such a model is told runs to the holder's idle timeout. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
262 lines
7.7 KiB
Go
262 lines
7.7 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
)
|
|
|
|
// parkRecord is why a model stopped being retried.
|
|
type parkRecord struct {
|
|
Since time.Time
|
|
Reason string
|
|
Failures int
|
|
// fp is the model's definition when it was parked; a different one
|
|
// unparks it.
|
|
fp string
|
|
}
|
|
|
|
type placementNote struct {
|
|
text string
|
|
at time.Time
|
|
}
|
|
|
|
// modelFingerprint identifies a model's definition, so a change to it can be
|
|
// told from a retry of the same thing.
|
|
func modelFingerprint(m catalog.Model) string {
|
|
b, _ := json.Marshal(m)
|
|
sum := sha256.Sum256(b)
|
|
return hex.EncodeToString(sum[:8])
|
|
}
|
|
|
|
// placementLogEvery is how often an unchanged placement failure is repeated.
|
|
const placementLogEvery = 10 * time.Minute
|
|
|
|
// notePlacementFailure logs why a model could not be placed, when the reason
|
|
// is new or has not been said for a while. Said on every pass it was 250
|
|
// identical lines in two days for one model, burying everything else.
|
|
func (r *Reconciler) notePlacementFailure(model, why string) {
|
|
r.mu.Lock()
|
|
last := r.placementLogged[model]
|
|
say := last.text != why || r.now().Sub(last.at) >= placementLogEvery
|
|
if say {
|
|
r.placementLogged[model] = placementNote{text: why, at: r.now()}
|
|
}
|
|
r.mu.Unlock()
|
|
if say {
|
|
r.log.Warn("no placement for model", "model", model, "reasons", why)
|
|
}
|
|
}
|
|
|
|
// Retry clears a parked model and its backoff, so it is placed on the next
|
|
// pass.
|
|
func (r *Reconciler) Retry(model string) bool {
|
|
r.mu.Lock()
|
|
_, was := r.parked[model]
|
|
delete(r.parked, model)
|
|
r.attempts[model] = 0
|
|
delete(r.nextAttempt, model)
|
|
r.mu.Unlock()
|
|
r.Notify()
|
|
return was
|
|
}
|
|
|
|
// ModelHealth is how one model stands, for status and metrics.
|
|
type ModelHealth struct {
|
|
Model string
|
|
// Desired is how many replicas it should have now; 0 for a stopped model
|
|
// or an on-demand one nobody is asking for.
|
|
Desired int
|
|
Ready int
|
|
// Failures counts replicas that failed in a row.
|
|
Failures int
|
|
// Parked is set when Failures reached ParkAfter; Reason is the last error.
|
|
Parked bool
|
|
ParkedSince time.Time
|
|
Reason string
|
|
// Unplaceable is set when it is short of replicas and the last attempt to
|
|
// place one found nowhere; Placement says why.
|
|
Unplaceable bool
|
|
Placement string
|
|
// Class is the capacity class it is treated as. Preempted counts the
|
|
// replicas it has lost to models of a higher class since the control
|
|
// plane started; HeldFor names the model it is giving room to now.
|
|
Class string
|
|
Preempted int
|
|
HeldFor string
|
|
}
|
|
|
|
// ModelHealth reports every model in the catalog.
|
|
func (r *Reconciler) ModelHealth() []ModelHealth {
|
|
models := r.cat.Load().Models()
|
|
desired := map[string]int{}
|
|
for _, m := range models {
|
|
desired[m.Name] = r.desiredFor(m)
|
|
}
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
ready, live := map[string]int{}, map[string]int{}
|
|
for _, rep := range r.replicas {
|
|
if rep.State == Ready {
|
|
ready[rep.Model]++
|
|
}
|
|
if rep.State.Counts() {
|
|
live[rep.Model]++
|
|
}
|
|
}
|
|
out := make([]ModelHealth, 0, len(models))
|
|
for _, m := range models {
|
|
h := ModelHealth{Model: m.Name, Desired: desired[m.Name], Ready: ready[m.Name],
|
|
Failures: r.attempts[m.Name], Class: m.EffectiveClass(), Preempted: r.preempted[m.Name]}
|
|
if hd, ok := r.held[m.Name]; ok && r.now().Before(hd.Until) {
|
|
h.HeldFor = hd.For
|
|
}
|
|
if p, ok := r.parked[m.Name]; ok {
|
|
h.Parked, h.ParkedSince, h.Reason = true, p.Since, p.Reason
|
|
}
|
|
if res, ok := r.lastExplain[m.Name]; ok && res.Placement == nil && live[m.Name] < h.Desired {
|
|
h.Unplaceable, h.Placement = true, res.Explain()
|
|
}
|
|
out = append(out, h)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Unservable says why a model cannot be served at all, so a request for it is
|
|
// told now rather than asked to retry something that will never come up.
|
|
func (r *Reconciler) Unservable(model string) (string, bool) {
|
|
m, ok := r.cat.Load().Resolve(model)
|
|
if !ok {
|
|
return "", false
|
|
}
|
|
cat := r.cat.Load()
|
|
r.mu.Lock()
|
|
if p, ok := r.parked[m.Name]; ok {
|
|
r.mu.Unlock()
|
|
return fmt.Sprintf("%s failed to start %d times in a row and is not being retried: %s",
|
|
m.Name, p.Failures, p.Reason), true
|
|
}
|
|
var asleepLater []victim
|
|
for _, rep := range r.replicas {
|
|
if !rep.State.Counts() {
|
|
continue
|
|
}
|
|
if rep.Model == m.Name {
|
|
r.mu.Unlock()
|
|
return "", false // something is on its way
|
|
}
|
|
if other, ok := cat.Resolve(rep.Model); ok && other.Scaling.OnDemand() && len(nodesOf(rep)) == 1 {
|
|
asleepLater = append(asleepLater, victim{ID: rep.ID, Model: rep.Model})
|
|
}
|
|
}
|
|
res, ok := r.lastExplain[m.Name]
|
|
r.mu.Unlock()
|
|
if !ok || res.Placement != nil {
|
|
return "", false
|
|
}
|
|
// The cards are held by models that let go of them when idle, which
|
|
// makeRoomFor hastens once they have been up a while. That is a wait, not
|
|
// a never: a 3D model asked for straight after a picture, on a cluster
|
|
// with one card, waits for the picture model to step aside.
|
|
if m.Scaling.OnDemand() && len(asleepLater) > 0 && r.fitsWithout(m, nil, asleepLater) {
|
|
return "", false
|
|
}
|
|
return m.Name + " cannot be placed on any node: " + firstLine(res.Explain()), true
|
|
}
|
|
|
|
// WaitFor says which model holds the cards an on-demand model is waiting
|
|
// for, and about how long until it serves: until the holder lets go — evicted
|
|
// once idle and old enough, or stopped at its own idle timeout — and then its
|
|
// own start, as long as its last one took. ok is false when it is not waiting
|
|
// on another model; eta is zero when when the holder lets go is not known.
|
|
func (r *Reconciler) WaitFor(model string) (heldBy string, eta time.Duration, ok bool) {
|
|
cat := r.cat.Load()
|
|
m, found := cat.Resolve(model)
|
|
if !found || !m.Scaling.OnDemand() {
|
|
return "", 0, false
|
|
}
|
|
type holder struct {
|
|
v victim
|
|
release time.Time
|
|
}
|
|
now := r.now()
|
|
var holders []holder
|
|
r.mu.Lock()
|
|
for _, rep := range r.replicas {
|
|
if !rep.State.Counts() || len(nodesOf(rep)) != 1 {
|
|
continue
|
|
}
|
|
if rep.Model == m.Name {
|
|
r.mu.Unlock()
|
|
return "", 0, false // its own replica is on its way
|
|
}
|
|
other, found := cat.Resolve(rep.Model)
|
|
if !found || !other.Scaling.OnDemand() {
|
|
continue
|
|
}
|
|
h := holder{v: victim{ID: rep.ID, Model: rep.Model}}
|
|
if !rep.LastUsedAt.IsZero() {
|
|
h.release = rep.LastUsedAt.Add(r.idleTimeout(other))
|
|
// Evicted sooner, when it is no higher a class than the model
|
|
// waiting; one put first is waited for to its idle timeout.
|
|
if catalog.ClassRank(other.EffectiveClass()) <= catalog.ClassRank(m.EffectiveClass()) {
|
|
evict := rep.LastUsedAt.Add(r.cfg.EvictionGrace)
|
|
if !rep.ReadyAt.IsZero() && rep.ReadyAt.Add(r.cfg.MinLifetime).After(evict) {
|
|
evict = rep.ReadyAt.Add(r.cfg.MinLifetime)
|
|
}
|
|
if evict.Before(h.release) {
|
|
h.release = evict
|
|
}
|
|
}
|
|
}
|
|
holders = append(holders, h)
|
|
}
|
|
cold := r.coldStart[m.Name]
|
|
r.mu.Unlock()
|
|
|
|
// The one whose going alone makes room, letting go soonest; failing
|
|
// that, all of them, and the last to let go.
|
|
var best *holder
|
|
for i := range holders {
|
|
h := &holders[i]
|
|
if !r.fitsWithout(m, nil, []victim{h.v}) {
|
|
continue
|
|
}
|
|
if best == nil || (!h.release.IsZero() && (best.release.IsZero() || h.release.Before(best.release))) {
|
|
best = h
|
|
}
|
|
}
|
|
if best == nil {
|
|
all := make([]victim, len(holders))
|
|
for i, h := range holders {
|
|
all[i] = h.v
|
|
}
|
|
if len(all) == 0 || !r.fitsWithout(m, nil, all) {
|
|
return "", 0, false
|
|
}
|
|
for i := range holders {
|
|
h := &holders[i]
|
|
if best == nil || h.release.IsZero() || (!best.release.IsZero() && h.release.After(best.release)) {
|
|
best = h
|
|
}
|
|
}
|
|
}
|
|
if best.release.IsZero() {
|
|
return best.v.Model, 0, true
|
|
}
|
|
return best.v.Model, max(best.release.Sub(now), 0) + cold, true
|
|
}
|
|
|
|
func firstLine(s string) string {
|
|
s = strings.TrimSpace(s)
|
|
if i := strings.IndexByte(s, '\n'); i >= 0 {
|
|
return strings.TrimSpace(s[:i]) + " (and others; see llmgw ctl explain-placement)"
|
|
}
|
|
return s
|
|
}
|