Park a model that fails every time it starts, and say so everywhere
On the live cluster kev-0.5b — an architecture the nodes' llama.cpp did not know — was started 155 times in eight hours, failing the same way each time, and phi-4, which fitted on no node, logged "no placement" 250 times in two days. Neither showed up anywhere an operator looked: the backoff never gave up, no metric said a model was failing or unplaceable, and the one alert for a model with nothing serving could not fire, because its series was published only while a replica was ready. After five failures in a row a model is parked: not placed again until its definition changes — a new source or profile — or `llmgw ctl retry` is run. Status, the model list and the admin API show it with its last error. The placement warning is logged when its reason changes or every ten minutes. A request for a parked or unplaceable model is answered at once with why, rather than "loading; retry shortly", which a client retries forever. Each request still clears the previous placement result and triggers a fresh attempt, so a model that fits once room is made is not refused on a stale answer; the router waits for that attempt, not for its full activation window. Metrics now cover model health (desired, ready at 0 for what should run, consecutive start failures, parked, unplaceable), engine installs per node, leadership, and certificate expiry read from disk at each scrape, with alerts for each. `ctl status` counts a model against what it should have now, so an on-demand model that was asked for and fitted nowhere no longer reads as satisfied. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
5e1b6ecea0
commit
37a80a9b44
17 files changed
+740
-24
No files matched your search
@@ -78,6 +78,7 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
|
||||
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
|
||||
mux.HandleFunc("POST /api/v1/restart", a.restart)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel)
|
||||
}
|
||||
|
||||
// --- responses ---
|
||||
@@ -233,6 +234,18 @@ type ModelView struct {
|
||||
Max int `json:"max_replicas"`
|
||||
Ready int `json:"ready_replicas"`
|
||||
Total int `json:"total_replicas"`
|
||||
// Desired is how many it should have now: Min, or for an on-demand
|
||||
// model one while it is being asked for and none otherwise.
|
||||
Desired int `json:"desired_replicas"`
|
||||
// Failures counts replicas that failed to start in a row.
|
||||
Failures int `json:"failures,omitempty"`
|
||||
// Parked is set once Failures reached the limit; the model is not retried
|
||||
// until its definition changes or `ctl retry` is run. Reason is the last
|
||||
// error.
|
||||
Parked bool `json:"parked,omitempty"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
// Unplaceable is set when it is short and the last attempt found no node.
|
||||
Unplaceable bool `json:"unplaceable,omitempty"`
|
||||
Fallbacks []string `json:"fallbacks,omitempty"`
|
||||
Profiles []string `json:"profiles"`
|
||||
}
|
||||
@@ -442,13 +455,20 @@ func (a *API) Models() []ModelView {
|
||||
}
|
||||
}
|
||||
|
||||
health := map[string]reconciler.ModelHealth{}
|
||||
for _, h := range a.ctl.Reconciler().ModelHealth() {
|
||||
health[h.Model] = h
|
||||
}
|
||||
out := make([]ModelView, 0)
|
||||
for _, m := range a.ctl.Catalog().Models() {
|
||||
h := health[m.Name]
|
||||
v := ModelView{
|
||||
Name: m.Name, Description: m.Description, Aliases: m.Aliases, Source: m.Source,
|
||||
Min: m.Scaling.Min, Max: m.Scaling.Max,
|
||||
Ready: ready[m.Name], Total: total[m.Name],
|
||||
Fallbacks: m.Fallbacks,
|
||||
Desired: h.Desired, Failures: h.Failures,
|
||||
Parked: h.Parked, Reason: h.Reason, Unplaceable: h.Unplaceable,
|
||||
}
|
||||
for _, p := range m.Profiles {
|
||||
v.Profiles = append(v.Profiles, p.Name)
|
||||
@@ -1085,3 +1105,13 @@ func (a *API) restart(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, RestartResult{Queued: ids})
|
||||
}
|
||||
|
||||
func (a *API) retryModel(w http.ResponseWriter, r *http.Request) {
|
||||
name := r.PathValue("name")
|
||||
if _, ok := a.ctl.Catalog().Resolve(name); !ok {
|
||||
writeError(w, http.StatusNotFound, "no model called "+name)
|
||||
return
|
||||
}
|
||||
was := a.ctl.Reconciler().Retry(name)
|
||||
writeJSON(w, http.StatusOK, map[string]bool{"was_parked": was})
|
||||
}
|
||||
@@ -25,6 +25,9 @@ type Summary struct {
|
||||
// Unplaceable names models the scheduler currently cannot satisfy. This is
|
||||
// the line that turns "something is wrong" into a place to look.
|
||||
Unplaceable []string `json:"unplaceable,omitempty"`
|
||||
// Parked are models that failed every time they started and are no
|
||||
// longer retried, with the last error.
|
||||
Parked []ParkedModel `json:"parked,omitempty"`
|
||||
// IdleReplicas are serving replicas nothing has asked for in a while, which
|
||||
// on a fleet with more models than GPUs is the first place to find room.
|
||||
IdleReplicas []IdleReplica `json:"idle_replicas,omitempty"`
|
||||
@@ -201,17 +204,30 @@ func (a *API) Summary() Summary {
|
||||
|
||||
for _, m := range a.Models() {
|
||||
s.Models.Total++
|
||||
if m.Ready >= m.Min {
|
||||
if m.Parked {
|
||||
s.Parked = append(s.Parked, ParkedModel{Model: m.Name, Failures: m.Failures, Reason: m.Reason})
|
||||
}
|
||||
// Against what it should have now, not its minimum: an on-demand
|
||||
// model's minimum is 0, so one that was asked for and fitted nowhere
|
||||
// used to count as satisfied.
|
||||
if m.Ready >= m.Desired && !m.Parked {
|
||||
s.Models.Satisfied++
|
||||
continue
|
||||
}
|
||||
s.Models.Short++
|
||||
// Short of replicas and unable to place more is the actionable case;
|
||||
// short and still starting is just a cluster doing its job.
|
||||
if m.Total <= m.Ready {
|
||||
if m.Unplaceable || (m.Total <= m.Ready && !m.Parked) {
|
||||
s.Unplaceable = append(s.Unplaceable, m.Name)
|
||||
}
|
||||
}
|
||||
sort.Strings(s.Unplaceable)
|
||||
return s
|
||||
}
|
||||
|
||||
// ParkedModel is a model no longer retried, and why.
|
||||
type ParkedModel struct {
|
||||
Model string `json:"model"`
|
||||
Failures int `json:"failures"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
@@ -171,6 +171,9 @@ func (c *Controller) Nodes() []inventory.Node { return c.inv.Nodes() }
|
||||
func (c *Controller) FreeGPUs(nodeID string) []int { return c.inv.FreeGPUs(nodeID) }
|
||||
func (c *Controller) Replicas() []reconciler.Replica { return c.rec.Replicas() }
|
||||
|
||||
// ModelHealth is how each model in the catalog stands.
|
||||
func (c *Controller) ModelHealth() []reconciler.ModelHealth { return c.rec.ModelHealth() }
|
||||
|
||||
// Reconciler exposes the reconciliation loop, for status reports and ctl.
|
||||
func (c *Controller) Reconciler() *reconciler.Reconciler { return c.rec }
|
||||
|
||||
|
||||
@@ -39,9 +39,25 @@ type Source interface {
|
||||
// derived from state that already exists, so computing them when asked is both
|
||||
// simpler and fresher than maintaining a parallel copy that can drift. Nothing
|
||||
// runs between scrapes.
|
||||
// HealthSource adds what the reconciler knows about each model and where
|
||||
// engine installs stand, and whether this control plane is the one placing
|
||||
// work. Asked for by type assertion, so a source without it — a test's —
|
||||
// publishes what it did before.
|
||||
type HealthSource interface {
|
||||
ModelHealth() []reconciler.ModelHealth
|
||||
IsLeader() bool
|
||||
}
|
||||
|
||||
type Collector struct {
|
||||
src Source
|
||||
|
||||
modelDesired *prometheus.Desc
|
||||
modelFailures *prometheus.Desc
|
||||
modelParked *prometheus.Desc
|
||||
modelUnplaceable *prometheus.Desc
|
||||
engineInstall *prometheus.Desc
|
||||
leader *prometheus.Desc
|
||||
|
||||
nodeUp *prometheus.Desc
|
||||
nodeGPUs *prometheus.Desc
|
||||
nodeGPUsFree *prometheus.Desc
|
||||
@@ -137,7 +153,26 @@ func New(src Source) *Collector {
|
||||
replicaState: prometheus.NewDesc("llmgw_replicas",
|
||||
"Replicas by model and state.", []string{"model", "state"}, nil),
|
||||
replicaReady: prometheus.NewDesc("llmgw_model_replicas_ready",
|
||||
"Replicas of a model that are serving.", []string{"model"}, nil),
|
||||
"Replicas of a model that are serving. Published, as 0 when none are, for every "+
|
||||
"model that should be running; a stopped model, or an on-demand one nobody is "+
|
||||
"asking for, publishes nothing.", []string{"model"}, nil),
|
||||
modelDesired: prometheus.NewDesc("llmgw_model_replicas_desired",
|
||||
"Replicas a model should have now: its minimum, or for an on-demand model 1 while "+
|
||||
"it is being asked for.", []string{"model"}, nil),
|
||||
modelFailures: prometheus.NewDesc("llmgw_model_start_failures",
|
||||
"Replicas of a model that failed to start in a row. Resets when one comes up.",
|
||||
[]string{"model"}, nil),
|
||||
modelParked: prometheus.NewDesc("llmgw_model_parked",
|
||||
"1 when a model failed so often in a row that it is no longer retried.",
|
||||
[]string{"model"}, nil),
|
||||
modelUnplaceable: prometheus.NewDesc("llmgw_model_unplaceable",
|
||||
"1 when a model is short of replicas and the last attempt found no node to place one on.",
|
||||
[]string{"model"}, nil),
|
||||
engineInstall: prometheus.NewDesc("llmgw_node_engine_install",
|
||||
"Where installing an engine stands on a node: 1 for the current state.",
|
||||
[]string{"node", "engine", "version", "state"}, nil),
|
||||
leader: prometheus.NewDesc("llmgw_control_plane_leader",
|
||||
"1 when this control plane is the one placing work, 0 when it is a standby.", nil, nil),
|
||||
replicaIdle: prometheus.NewDesc("llmgw_replica_idle_seconds",
|
||||
"Seconds since a replica was last sent a request. Absent when nothing has been reported, "+
|
||||
"which is not the same as idle.", []string{"model", "replica"}, nil),
|
||||
@@ -201,6 +236,8 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
|
||||
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy,
|
||||
c.gpuNVLink, c.gpuECC,
|
||||
c.replicaState, c.replicaReady, c.replicaIdle,
|
||||
c.modelDesired, c.modelFailures, c.modelParked, c.modelUnplaceable,
|
||||
c.engineInstall, c.leader,
|
||||
} {
|
||||
ch <- d
|
||||
}
|
||||
@@ -387,8 +424,43 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
|
||||
ch <- prometheus.MustNewConstMetric(c.replicaState, prometheus.GaugeValue,
|
||||
float64(v), k[0], k[1])
|
||||
}
|
||||
for model, v := range ready {
|
||||
ch <- prometheus.MustNewConstMetric(c.replicaReady, prometheus.GaugeValue,
|
||||
float64(v), model)
|
||||
hs, ok := c.src.(HealthSource)
|
||||
if !ok {
|
||||
for model, v := range ready {
|
||||
ch <- prometheus.MustNewConstMetric(c.replicaReady, prometheus.GaugeValue,
|
||||
float64(v), model)
|
||||
}
|
||||
return
|
||||
}
|
||||
for _, n := range c.src.Nodes() {
|
||||
for _, p := range n.Provision {
|
||||
ch <- prometheus.MustNewConstMetric(c.engineInstall, prometheus.GaugeValue, 1,
|
||||
n.ID, p.Engine, p.Version, p.State)
|
||||
}
|
||||
}
|
||||
lead := 0.0
|
||||
if hs.IsLeader() {
|
||||
lead = 1
|
||||
}
|
||||
ch <- prometheus.MustNewConstMetric(c.leader, prometheus.GaugeValue, lead)
|
||||
for _, h := range hs.ModelHealth() {
|
||||
// Ready is published at zero for what should be running — the one
|
||||
// value ModelHasNothingServing exists to catch, and which, emitted
|
||||
// only for replicas that were Ready, it never saw.
|
||||
if h.Desired > 0 || ready[h.Model] > 0 {
|
||||
ch <- prometheus.MustNewConstMetric(c.replicaReady, prometheus.GaugeValue,
|
||||
float64(ready[h.Model]), h.Model)
|
||||
}
|
||||
ch <- prometheus.MustNewConstMetric(c.modelDesired, prometheus.GaugeValue, float64(h.Desired), h.Model)
|
||||
ch <- prometheus.MustNewConstMetric(c.modelFailures, prometheus.GaugeValue, float64(h.Failures), h.Model)
|
||||
ch <- prometheus.MustNewConstMetric(c.modelParked, prometheus.GaugeValue, b2f(h.Parked), h.Model)
|
||||
ch <- prometheus.MustNewConstMetric(c.modelUnplaceable, prometheus.GaugeValue, b2f(h.Unplaceable), h.Model)
|
||||
}
|
||||
}
|
||||
|
||||
func b2f(b bool) float64 {
|
||||
if b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -196,3 +196,35 @@ func TestEveryIBPortIsReportedWithItsCounters(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type healthSource struct {
|
||||
fakeSource
|
||||
health []reconciler.ModelHealth
|
||||
}
|
||||
|
||||
func (h healthSource) ModelHealth() []reconciler.ModelHealth { return h.health }
|
||||
func (h healthSource) IsLeader() bool { return true }
|
||||
|
||||
func TestAModelThatShouldRunPublishesZeroReplicasReady(t *testing.T) {
|
||||
// ModelHasNothingServing alerts on llmgw_model_replicas_ready == 0, and
|
||||
// the series was only ever published while a replica was ready — so the
|
||||
// alert could not fire for the case it was written for.
|
||||
src := healthSource{health: []reconciler.ModelHealth{
|
||||
{Model: "wanted", Desired: 1, Failures: 5, Parked: true, Reason: "boom"},
|
||||
{Model: "stopped", Desired: 0},
|
||||
}}
|
||||
out := gather(t, src)
|
||||
for _, want := range []string{
|
||||
"llmgw_model_replicas_ready model=wanted -> 0",
|
||||
"llmgw_model_parked model=wanted -> 1",
|
||||
"llmgw_model_start_failures model=wanted -> 5",
|
||||
"llmgw_control_plane_leader -> 1",
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("missing %s", want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(out, "llmgw_model_replicas_ready model=stopped") {
|
||||
t.Error("a stopped model publishes a ready count, so the alert would fire for it")
|
||||
}
|
||||
}
|
||||
@@ -3,11 +3,15 @@ package reconciler
|
||||
import (
|
||||
"context"
|
||||
crypterand "crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math/rand/v2"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -66,6 +70,15 @@ type Config struct {
|
||||
// Without a bound, draining a node whose model cannot be placed anywhere
|
||||
// else would never finish.
|
||||
DrainGrace time.Duration
|
||||
// ParkAfter is how many failures in a row stop a model being retried.
|
||||
//
|
||||
// Backoff alone never gives up: it grows to MaxBackoff and keeps trying,
|
||||
// and a model whose weights its engine cannot load — an architecture the
|
||||
// installed llama.cpp does not know — was started 155 times over eight
|
||||
// hours on the live cluster, each time failing the same way, with nothing
|
||||
// to show for it but the log. A parked model is not placed again until its
|
||||
// definition changes or an operator runs `llmgw ctl retry`.
|
||||
ParkAfter int
|
||||
// PortRange is the per-node port range for engine HTTP and dist-init.
|
||||
PortRange [2]int
|
||||
// MgmtInterface is the ethernet interface used for NCCL bootstrap.
|
||||
@@ -126,6 +139,7 @@ func DefaultConfig() Config {
|
||||
EvictionGrace: 2 * time.Minute,
|
||||
MinLifetime: 5 * time.Minute,
|
||||
DrainGrace: 5 * time.Minute,
|
||||
ParkAfter: 5,
|
||||
PortRange: [2]int{20000, 20999},
|
||||
MgmtInterface: "eth0",
|
||||
}
|
||||
@@ -165,6 +179,9 @@ func (c *Config) normalize() {
|
||||
if c.DrainGrace <= 0 {
|
||||
c.DrainGrace = d.DrainGrace
|
||||
}
|
||||
if c.ParkAfter <= 0 {
|
||||
c.ParkAfter = d.ParkAfter
|
||||
}
|
||||
if c.PortRange[1] <= c.PortRange[0] {
|
||||
c.PortRange = d.PortRange
|
||||
}
|
||||
@@ -209,6 +226,11 @@ type Reconciler struct {
|
||||
// restarted and not yet begun. One is replaced at a time: replacing them
|
||||
// all at once would need the memory of every replica twice over.
|
||||
restartQueue map[string][]string
|
||||
// parked holds models that failed ParkAfter times in a row.
|
||||
parked map[string]parkRecord
|
||||
// placementLogged remembers the last "no placement" said per model, so it
|
||||
// is said when it changes rather than on every five-second pass.
|
||||
placementLogged map[string]placementNote
|
||||
|
||||
wake chan struct{}
|
||||
stop chan struct{}
|
||||
@@ -230,19 +252,21 @@ func New(cfg Config, cat *catalog.Live, inv *inventory.Store, disp Dispatcher, l
|
||||
}
|
||||
return &Reconciler{
|
||||
cfg: cfg, cat: cat, inv: inv, disp: disp, log: log,
|
||||
sched: placement.New(inv),
|
||||
now: time.Now,
|
||||
replicas: map[string]*Replica{},
|
||||
nextAttempt: map[string]time.Time{},
|
||||
attempts: map[string]int{},
|
||||
reservedPorts: map[int]struct{}{},
|
||||
wanted: map[string]time.Time{},
|
||||
restartQueue: map[string][]string{},
|
||||
lastExplain: map[string]placement.Result{},
|
||||
wake: make(chan struct{}, 1),
|
||||
stop: make(chan struct{}),
|
||||
done: make(chan struct{}),
|
||||
startedAt: time.Now(),
|
||||
sched: placement.New(inv),
|
||||
now: time.Now,
|
||||
replicas: map[string]*Replica{},
|
||||
nextAttempt: map[string]time.Time{},
|
||||
attempts: map[string]int{},
|
||||
reservedPorts: map[int]struct{}{},
|
||||
wanted: map[string]time.Time{},
|
||||
restartQueue: map[string][]string{},
|
||||
parked: map[string]parkRecord{},
|
||||
placementLogged: map[string]placementNote{},
|
||||
lastExplain: map[string]placement.Result{},
|
||||
wake: make(chan struct{}, 1),
|
||||
stop: make(chan struct{}),
|
||||
done: make(chan struct{}),
|
||||
startedAt: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -464,6 +488,12 @@ func (r *Reconciler) Want(model string) bool {
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.wanted[m.Name] = r.now()
|
||||
// A placement that failed before this request says nothing about now:
|
||||
// forgotten, so the pass this triggers decides afresh and Unservable
|
||||
// does not answer from a stale result.
|
||||
if res, ok := r.lastExplain[m.Name]; ok && res.Placement == nil {
|
||||
delete(r.lastExplain, m.Name)
|
||||
}
|
||||
r.mu.Unlock()
|
||||
// Immediately, not on the next tick. Someone is waiting on this request,
|
||||
// and five seconds of ticker is five seconds they spend staring at nothing.
|
||||
@@ -669,6 +699,17 @@ func (r *Reconciler) reapFailed(ctx context.Context) {
|
||||
// replica is about to disappear, and the backoff has to survive it.
|
||||
r.attempts[rep.Model]++
|
||||
r.nextAttempt[rep.Model] = r.now().Add(r.backoff(r.attempts[rep.Model]))
|
||||
if n := r.attempts[rep.Model]; n >= r.cfg.ParkAfter {
|
||||
if _, already := r.parked[rep.Model]; !already {
|
||||
m, _ := r.cat.Load().Resolve(rep.Model)
|
||||
r.parked[rep.Model] = parkRecord{
|
||||
Since: r.now(), Reason: rep.Err, Failures: n, fp: modelFingerprint(m),
|
||||
}
|
||||
r.log.Error("model parked: it failed every time it started, and is not retried "+
|
||||
"until its definition changes or `llmgw ctl retry` is run",
|
||||
"model", rep.Model, "failures", n, "last_error", rep.Err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
r.mu.Unlock()
|
||||
@@ -746,10 +787,22 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
|
||||
}
|
||||
}
|
||||
blockedUntil := r.nextAttempt[m.Name]
|
||||
park, isParked := r.parked[m.Name]
|
||||
if isParked && park.fp != modelFingerprint(m) {
|
||||
// Somebody changed the model — a new source, an edited profile — so
|
||||
// the reason it kept failing may be gone. Worth one more streak.
|
||||
delete(r.parked, m.Name)
|
||||
r.attempts[m.Name] = 0
|
||||
delete(r.nextAttempt, m.Name)
|
||||
isParked, blockedUntil = false, time.Time{}
|
||||
r.log.Info("model unparked: its definition changed", "model", m.Name)
|
||||
}
|
||||
r.mu.Unlock()
|
||||
|
||||
desired := r.desiredFor(m)
|
||||
switch {
|
||||
case len(liveIDs) < desired && isParked:
|
||||
// Not placed. Its failure is on every status the operator reads.
|
||||
case len(liveIDs) < desired:
|
||||
if r.now().Before(blockedUntil) {
|
||||
// Still in backoff. Rebuilding immediately would hot-loop on a
|
||||
@@ -925,7 +978,7 @@ func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []strin
|
||||
if m.Scaling.OnDemand() && r.makeRoomFor(ctx, m) {
|
||||
return
|
||||
}
|
||||
r.log.Warn("no placement for model", "model", m.Name, "reasons", res.Explain())
|
||||
r.notePlacementFailure(m.Name, res.Explain())
|
||||
return
|
||||
}
|
||||
if err := r.startReplica(ctx, m, res.Placement); err != nil {
|
||||
@@ -1523,3 +1576,142 @@ func (r *Reconciler) engineKey(replicaID string) string {
|
||||
}
|
||||
return r.cfg.EngineKey(replicaID)
|
||||
}
|
||||
|
||||
// parkRecord is why a model stopped being retried.
|
||||
type parkRecord struct {
|
||||
Since time.Time
|
||||
Reason string
|
||||
Failures int
|
||||
// fp is the model's definition when it was parked; a different one
|
||||
// unparks it.
|
||||
fp string
|
||||
}
|
||||
|
||||
type placementNote struct {
|
||||
text string
|
||||
at time.Time
|
||||
}
|
||||
|
||||
// modelFingerprint identifies a model's definition, so a change to it can be
|
||||
// told from a retry of the same thing.
|
||||
func modelFingerprint(m catalog.Model) string {
|
||||
b, _ := json.Marshal(m)
|
||||
sum := sha256.Sum256(b)
|
||||
return hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
||||
// placementLogEvery is how often an unchanged placement failure is repeated.
|
||||
const placementLogEvery = 10 * time.Minute
|
||||
|
||||
// notePlacementFailure logs why a model could not be placed, when the reason
|
||||
// is new or has not been said for a while. Said on every pass it was 250
|
||||
// identical lines in two days for one model, burying everything else.
|
||||
func (r *Reconciler) notePlacementFailure(model, why string) {
|
||||
r.mu.Lock()
|
||||
last := r.placementLogged[model]
|
||||
say := last.text != why || r.now().Sub(last.at) >= placementLogEvery
|
||||
if say {
|
||||
r.placementLogged[model] = placementNote{text: why, at: r.now()}
|
||||
}
|
||||
r.mu.Unlock()
|
||||
if say {
|
||||
r.log.Warn("no placement for model", "model", model, "reasons", why)
|
||||
}
|
||||
}
|
||||
|
||||
// Retry clears a parked model and its backoff, so it is placed on the next
|
||||
// pass.
|
||||
func (r *Reconciler) Retry(model string) bool {
|
||||
r.mu.Lock()
|
||||
_, was := r.parked[model]
|
||||
delete(r.parked, model)
|
||||
r.attempts[model] = 0
|
||||
delete(r.nextAttempt, model)
|
||||
r.mu.Unlock()
|
||||
r.Notify()
|
||||
return was
|
||||
}
|
||||
|
||||
// ModelHealth is how one model stands, for status and metrics.
|
||||
type ModelHealth struct {
|
||||
Model string
|
||||
// Desired is how many replicas it should have now; 0 for a stopped model
|
||||
// or an on-demand one nobody is asking for.
|
||||
Desired int
|
||||
Ready int
|
||||
// Failures counts replicas that failed in a row.
|
||||
Failures int
|
||||
// Parked is set when Failures reached ParkAfter; Reason is the last error.
|
||||
Parked bool
|
||||
ParkedSince time.Time
|
||||
Reason string
|
||||
// Unplaceable is set when it is short of replicas and the last attempt to
|
||||
// place one found nowhere; Placement says why.
|
||||
Unplaceable bool
|
||||
Placement string
|
||||
}
|
||||
|
||||
// ModelHealth reports every model in the catalog.
|
||||
func (r *Reconciler) ModelHealth() []ModelHealth {
|
||||
models := r.cat.Load().Models()
|
||||
desired := map[string]int{}
|
||||
for _, m := range models {
|
||||
desired[m.Name] = r.desiredFor(m)
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
ready, live := map[string]int{}, map[string]int{}
|
||||
for _, rep := range r.replicas {
|
||||
if rep.State == Ready {
|
||||
ready[rep.Model]++
|
||||
}
|
||||
if rep.State.Counts() {
|
||||
live[rep.Model]++
|
||||
}
|
||||
}
|
||||
out := make([]ModelHealth, 0, len(models))
|
||||
for _, m := range models {
|
||||
h := ModelHealth{Model: m.Name, Desired: desired[m.Name], Ready: ready[m.Name],
|
||||
Failures: r.attempts[m.Name]}
|
||||
if p, ok := r.parked[m.Name]; ok {
|
||||
h.Parked, h.ParkedSince, h.Reason = true, p.Since, p.Reason
|
||||
}
|
||||
if res, ok := r.lastExplain[m.Name]; ok && res.Placement == nil && live[m.Name] < h.Desired {
|
||||
h.Unplaceable, h.Placement = true, res.Explain()
|
||||
}
|
||||
out = append(out, h)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Unservable says why a model cannot be served at all, so a request for it is
|
||||
// told now rather than asked to retry something that will never come up.
|
||||
func (r *Reconciler) Unservable(model string) (string, bool) {
|
||||
m, ok := r.cat.Load().Resolve(model)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if p, ok := r.parked[m.Name]; ok {
|
||||
return fmt.Sprintf("%s failed to start %d times in a row and is not being retried: %s",
|
||||
m.Name, p.Failures, p.Reason), true
|
||||
}
|
||||
for _, rep := range r.replicas {
|
||||
if rep.Model == m.Name && rep.State.Counts() {
|
||||
return "", false // something is on its way
|
||||
}
|
||||
}
|
||||
if res, ok := r.lastExplain[m.Name]; ok && res.Placement == nil {
|
||||
return m.Name + " cannot be placed on any node: " + firstLine(res.Explain()), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if i := strings.IndexByte(s, '\n'); i >= 0 {
|
||||
return strings.TrimSpace(s[:i]) + " (and others; see llmgw ctl explain-placement)"
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package reconciler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
@@ -1836,3 +1837,98 @@ func TestReplicasOfAModelAreRestartedOneAtATime(t *testing.T) {
|
||||
t.Error("an unknown replica was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// failRepeatedly starts and fails the model's replica n times over.
|
||||
func failRepeatedly(t *testing.T, h *harness, n int, why string) {
|
||||
t.Helper()
|
||||
for i := 0; i < n; i++ {
|
||||
h.disp.reset()
|
||||
h.r.Reconcile(t.Context())
|
||||
started := h.disp.startedShards()
|
||||
if len(started) == 0 {
|
||||
return // parked, or still in backoff
|
||||
}
|
||||
s := started[0]
|
||||
h.r.OnShardStatus(s.ReplicaID, s.ShardID, supervisor.Failed, why)
|
||||
h.r.Reconcile(t.Context())
|
||||
h.advance(DefaultConfig().MaxBackoff + time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModelThatAlwaysFailsIsParkedNotRetriedForever(t *testing.T) {
|
||||
// kev-0.5b: an architecture the node's llama.cpp did not know, started
|
||||
// 155 times in eight hours, the same error every time.
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
failRepeatedly(t, h, DefaultConfig().ParkAfter, "unknown model architecture: 'ggmlc'")
|
||||
|
||||
h.disp.reset()
|
||||
h.advance(DefaultConfig().MaxBackoff + time.Second)
|
||||
h.r.Reconcile(t.Context())
|
||||
if n := len(h.disp.startedShards()); n != 0 {
|
||||
t.Fatalf("a parked model was started again (%d shards)", n)
|
||||
}
|
||||
var mh ModelHealth
|
||||
for _, x := range h.r.ModelHealth() {
|
||||
if x.Model == "llama" {
|
||||
mh = x
|
||||
}
|
||||
}
|
||||
if !mh.Parked || !strings.Contains(mh.Reason, "ggmlc") || mh.Failures < DefaultConfig().ParkAfter {
|
||||
t.Errorf("health %+v", mh)
|
||||
}
|
||||
if why, no := h.r.Unservable("llama"); !no || !strings.Contains(why, "ggmlc") {
|
||||
t.Errorf("a request would not be told why: %q %v", why, no)
|
||||
}
|
||||
|
||||
// Retried by the operator once fixed.
|
||||
if !h.r.Retry("llama") {
|
||||
t.Error("Retry did not report the model as parked")
|
||||
}
|
||||
h.r.Reconcile(t.Context())
|
||||
if len(h.disp.startedShards()) == 0 {
|
||||
t.Error("a retried model was not placed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChangingAParkedModelsDefinitionTriesItAgain(t *testing.T) {
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
failRepeatedly(t, h, DefaultConfig().ParkAfter, "boom")
|
||||
|
||||
fixed := testModel(1)
|
||||
fixed.Source = "/models/llama-fixed"
|
||||
cat, err := catalog.New([]catalog.Model{fixed})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.r.cat.Store(cat)
|
||||
h.disp.reset()
|
||||
h.r.Reconcile(t.Context())
|
||||
if len(h.disp.startedShards()) == 0 {
|
||||
t.Error("a model whose definition changed stayed parked")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPlacementFailureIsLoggedWhenItChangesNotEveryPass(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 1)) // tp4 needs 4 cards
|
||||
h.r.log = slog.New(slog.NewTextHandler(&buf, nil))
|
||||
for i := 0; i < 20; i++ {
|
||||
h.r.Reconcile(t.Context())
|
||||
h.advance(5 * time.Second)
|
||||
}
|
||||
if n := strings.Count(buf.String(), "no placement for model"); n != 1 {
|
||||
t.Errorf("logged %d times over 20 passes", n)
|
||||
}
|
||||
h.advance(placementLogEvery)
|
||||
h.r.Reconcile(t.Context())
|
||||
if n := strings.Count(buf.String(), "no placement for model"); n != 2 {
|
||||
t.Errorf("not repeated after %s: %d", placementLogEvery, n)
|
||||
}
|
||||
var mh ModelHealth
|
||||
for _, x := range h.r.ModelHealth() {
|
||||
mh = x
|
||||
}
|
||||
if !mh.Unplaceable || mh.Desired != 1 {
|
||||
t.Errorf("health %+v", mh)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user