A model's class (critical, standard, opportunistic; unset follows its scaling) decides what may be stopped to make room for it: a lower class for a higher, and within a class only what another grew into under load, for a model below its minimum. Victims are tried on a copy of the inventory and stopped only if they make the room. Preempted replicas are drained with a short grace and their model is held back until the room is used. ctl model resources and class move under ctl model. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
161 lines
4.6 KiB
Go
161 lines
4.6 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
)
|
|
|
|
// parkRecord is why a model stopped being retried.
|
|
type parkRecord struct {
|
|
Since time.Time
|
|
Reason string
|
|
Failures int
|
|
// fp is the model's definition when it was parked; a different one
|
|
// unparks it.
|
|
fp string
|
|
}
|
|
|
|
type placementNote struct {
|
|
text string
|
|
at time.Time
|
|
}
|
|
|
|
// modelFingerprint identifies a model's definition, so a change to it can be
|
|
// told from a retry of the same thing.
|
|
func modelFingerprint(m catalog.Model) string {
|
|
b, _ := json.Marshal(m)
|
|
sum := sha256.Sum256(b)
|
|
return hex.EncodeToString(sum[:8])
|
|
}
|
|
|
|
// placementLogEvery is how often an unchanged placement failure is repeated.
|
|
const placementLogEvery = 10 * time.Minute
|
|
|
|
// notePlacementFailure logs why a model could not be placed, when the reason
|
|
// is new or has not been said for a while. Said on every pass it was 250
|
|
// identical lines in two days for one model, burying everything else.
|
|
func (r *Reconciler) notePlacementFailure(model, why string) {
|
|
r.mu.Lock()
|
|
last := r.placementLogged[model]
|
|
say := last.text != why || r.now().Sub(last.at) >= placementLogEvery
|
|
if say {
|
|
r.placementLogged[model] = placementNote{text: why, at: r.now()}
|
|
}
|
|
r.mu.Unlock()
|
|
if say {
|
|
r.log.Warn("no placement for model", "model", model, "reasons", why)
|
|
}
|
|
}
|
|
|
|
// Retry clears a parked model and its backoff, so it is placed on the next
|
|
// pass.
|
|
func (r *Reconciler) Retry(model string) bool {
|
|
r.mu.Lock()
|
|
_, was := r.parked[model]
|
|
delete(r.parked, model)
|
|
r.attempts[model] = 0
|
|
delete(r.nextAttempt, model)
|
|
r.mu.Unlock()
|
|
r.Notify()
|
|
return was
|
|
}
|
|
|
|
// ModelHealth is how one model stands, for status and metrics.
|
|
type ModelHealth struct {
|
|
Model string
|
|
// Desired is how many replicas it should have now; 0 for a stopped model
|
|
// or an on-demand one nobody is asking for.
|
|
Desired int
|
|
Ready int
|
|
// Failures counts replicas that failed in a row.
|
|
Failures int
|
|
// Parked is set when Failures reached ParkAfter; Reason is the last error.
|
|
Parked bool
|
|
ParkedSince time.Time
|
|
Reason string
|
|
// Unplaceable is set when it is short of replicas and the last attempt to
|
|
// place one found nowhere; Placement says why.
|
|
Unplaceable bool
|
|
Placement string
|
|
// Class is the capacity class it is treated as. Preempted counts the
|
|
// replicas it has lost to models of a higher class since the control
|
|
// plane started; HeldFor names the model it is giving room to now.
|
|
Class string
|
|
Preempted int
|
|
HeldFor string
|
|
}
|
|
|
|
// ModelHealth reports every model in the catalog.
|
|
func (r *Reconciler) ModelHealth() []ModelHealth {
|
|
models := r.cat.Load().Models()
|
|
desired := map[string]int{}
|
|
for _, m := range models {
|
|
desired[m.Name] = r.desiredFor(m)
|
|
}
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
ready, live := map[string]int{}, map[string]int{}
|
|
for _, rep := range r.replicas {
|
|
if rep.State == Ready {
|
|
ready[rep.Model]++
|
|
}
|
|
if rep.State.Counts() {
|
|
live[rep.Model]++
|
|
}
|
|
}
|
|
out := make([]ModelHealth, 0, len(models))
|
|
for _, m := range models {
|
|
h := ModelHealth{Model: m.Name, Desired: desired[m.Name], Ready: ready[m.Name],
|
|
Failures: r.attempts[m.Name], Class: m.EffectiveClass(), Preempted: r.preempted[m.Name]}
|
|
if hd, ok := r.held[m.Name]; ok && r.now().Before(hd.Until) {
|
|
h.HeldFor = hd.For
|
|
}
|
|
if p, ok := r.parked[m.Name]; ok {
|
|
h.Parked, h.ParkedSince, h.Reason = true, p.Since, p.Reason
|
|
}
|
|
if res, ok := r.lastExplain[m.Name]; ok && res.Placement == nil && live[m.Name] < h.Desired {
|
|
h.Unplaceable, h.Placement = true, res.Explain()
|
|
}
|
|
out = append(out, h)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Unservable says why a model cannot be served at all, so a request for it is
|
|
// told now rather than asked to retry something that will never come up.
|
|
func (r *Reconciler) Unservable(model string) (string, bool) {
|
|
m, ok := r.cat.Load().Resolve(model)
|
|
if !ok {
|
|
return "", false
|
|
}
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
if p, ok := r.parked[m.Name]; ok {
|
|
return fmt.Sprintf("%s failed to start %d times in a row and is not being retried: %s",
|
|
m.Name, p.Failures, p.Reason), true
|
|
}
|
|
for _, rep := range r.replicas {
|
|
if rep.Model == m.Name && rep.State.Counts() {
|
|
return "", false // something is on its way
|
|
}
|
|
}
|
|
if res, ok := r.lastExplain[m.Name]; ok && res.Placement == nil {
|
|
return m.Name + " cannot be placed on any node: " + firstLine(res.Explain()), true
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
func firstLine(s string) string {
|
|
s = strings.TrimSpace(s)
|
|
if i := strings.IndexByte(s, '\n'); i >= 0 {
|
|
return strings.TrimSpace(s[:i]) + " (and others; see llmgw ctl explain-placement)"
|
|
}
|
|
return s
|
|
}
|