package reconciler import ( "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "strings" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" ) // parkRecord is why a model stopped being retried. type parkRecord struct { Since time.Time Reason string Failures int // fp is the model's definition when it was parked; a different one // unparks it. fp string } type placementNote struct { text string at time.Time } // modelFingerprint identifies a model's definition, so a change to it can be // told from a retry of the same thing. func modelFingerprint(m catalog.Model) string { b, _ := json.Marshal(m) sum := sha256.Sum256(b) return hex.EncodeToString(sum[:8]) } // placementLogEvery is how often an unchanged placement failure is repeated. const placementLogEvery = 10 * time.Minute // notePlacementFailure logs why a model could not be placed, when the reason // is new or has not been said for a while. Said on every pass it was 250 // identical lines in two days for one model, burying everything else. func (r *Reconciler) notePlacementFailure(model, why string) { r.mu.Lock() last := r.placementLogged[model] say := last.text != why || r.now().Sub(last.at) >= placementLogEvery if say { r.placementLogged[model] = placementNote{text: why, at: r.now()} } r.mu.Unlock() if say { r.log.Warn("no placement for model", "model", model, "reasons", why) } } // Retry clears a parked model and its backoff, so it is placed on the next // pass. func (r *Reconciler) Retry(model string) bool { r.mu.Lock() _, was := r.parked[model] delete(r.parked, model) r.attempts[model] = 0 delete(r.nextAttempt, model) r.mu.Unlock() r.Notify() return was } // ModelHealth is how one model stands, for status and metrics. type ModelHealth struct { Model string // Desired is how many replicas it should have now; 0 for a stopped model // or an on-demand one nobody is asking for. Desired int Ready int // Failures counts replicas that failed in a row. Failures int // Parked is set when Failures reached ParkAfter; Reason is the last error. Parked bool ParkedSince time.Time Reason string // Unplaceable is set when it is short of replicas and the last attempt to // place one found nowhere; Placement says why. Unplaceable bool Placement string // Class is the capacity class it is treated as. Preempted counts the // replicas it has lost to models of a higher class since the control // plane started; HeldFor names the model it is giving room to now. Class string Preempted int HeldFor string } // ModelHealth reports every model in the catalog. func (r *Reconciler) ModelHealth() []ModelHealth { models := r.cat.Load().Models() desired := map[string]int{} for _, m := range models { desired[m.Name] = r.desiredFor(m) } r.mu.Lock() defer r.mu.Unlock() ready, live := map[string]int{}, map[string]int{} for _, rep := range r.replicas { if rep.State == Ready { ready[rep.Model]++ } if rep.State.Counts() { live[rep.Model]++ } } out := make([]ModelHealth, 0, len(models)) for _, m := range models { h := ModelHealth{Model: m.Name, Desired: desired[m.Name], Ready: ready[m.Name], Failures: r.attempts[m.Name], Class: m.EffectiveClass(), Preempted: r.preempted[m.Name]} if hd, ok := r.held[m.Name]; ok && r.now().Before(hd.Until) { h.HeldFor = hd.For } if p, ok := r.parked[m.Name]; ok { h.Parked, h.ParkedSince, h.Reason = true, p.Since, p.Reason } if res, ok := r.lastExplain[m.Name]; ok && res.Placement == nil && live[m.Name] < h.Desired { h.Unplaceable, h.Placement = true, res.Explain() } out = append(out, h) } return out } // Unservable says why a model cannot be served at all, so a request for it is // told now rather than asked to retry something that will never come up. func (r *Reconciler) Unservable(model string) (string, bool) { m, ok := r.cat.Load().Resolve(model) if !ok { return "", false } r.mu.Lock() defer r.mu.Unlock() if p, ok := r.parked[m.Name]; ok { return fmt.Sprintf("%s failed to start %d times in a row and is not being retried: %s", m.Name, p.Failures, p.Reason), true } for _, rep := range r.replicas { if rep.Model == m.Name && rep.State.Counts() { return "", false // something is on its way } } if res, ok := r.lastExplain[m.Name]; ok && res.Placement == nil { return m.Name + " cannot be placed on any node: " + firstLine(res.Explain()), true } return "", false } func firstLine(s string) string { s = strings.TrimSpace(s) if i := strings.IndexByte(s, '\n'); i >= 0 { return strings.TrimSpace(s[:i]) + " (and others; see llmgw ctl explain-placement)" } return s }