Park a model that fails every time it starts, and say so everywhere
On the live cluster kev-0.5b — an architecture the nodes' llama.cpp did not know — was started 155 times in eight hours, failing the same way each time, and phi-4, which fitted on no node, logged "no placement" 250 times in two days. Neither showed up anywhere an operator looked: the backoff never gave up, no metric said a model was failing or unplaceable, and the one alert for a model with nothing serving could not fire, because its series was published only while a replica was ready. After five failures in a row a model is parked: not placed again until its definition changes — a new source or profile — or `llmgw ctl retry` is run. Status, the model list and the admin API show it with its last error. The placement warning is logged when its reason changes or every ten minutes. A request for a parked or unplaceable model is answered at once with why, rather than "loading; retry shortly", which a client retries forever. Each request still clears the previous placement result and triggers a fresh attempt, so a model that fits once room is made is not refused on a stale answer; the router waits for that attempt, not for its full activation window. Metrics now cover model health (desired, ready at 0 for what should run, consecutive start failures, parked, unplaceable), engine installs per node, leadership, and certificate expiry read from disk at each scrape, with alerts for each. `ctl status` counts a model against what it should have now, so an on-demand model that was asked for and fitted nowhere no longer reads as satisfied. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
5e1b6ecea0
commit
37a80a9b44
17 files changed
+740
-24
No files matched your search
@@ -78,6 +78,7 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
|
||||
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
|
||||
mux.HandleFunc("POST /api/v1/restart", a.restart)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel)
|
||||
}
|
||||
|
||||
// --- responses ---
|
||||
@@ -233,6 +234,18 @@ type ModelView struct {
|
||||
Max int `json:"max_replicas"`
|
||||
Ready int `json:"ready_replicas"`
|
||||
Total int `json:"total_replicas"`
|
||||
// Desired is how many it should have now: Min, or for an on-demand
|
||||
// model one while it is being asked for and none otherwise.
|
||||
Desired int `json:"desired_replicas"`
|
||||
// Failures counts replicas that failed to start in a row.
|
||||
Failures int `json:"failures,omitempty"`
|
||||
// Parked is set once Failures reached the limit; the model is not retried
|
||||
// until its definition changes or `ctl retry` is run. Reason is the last
|
||||
// error.
|
||||
Parked bool `json:"parked,omitempty"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
// Unplaceable is set when it is short and the last attempt found no node.
|
||||
Unplaceable bool `json:"unplaceable,omitempty"`
|
||||
Fallbacks []string `json:"fallbacks,omitempty"`
|
||||
Profiles []string `json:"profiles"`
|
||||
}
|
||||
@@ -442,13 +455,20 @@ func (a *API) Models() []ModelView {
|
||||
}
|
||||
}
|
||||
|
||||
health := map[string]reconciler.ModelHealth{}
|
||||
for _, h := range a.ctl.Reconciler().ModelHealth() {
|
||||
health[h.Model] = h
|
||||
}
|
||||
out := make([]ModelView, 0)
|
||||
for _, m := range a.ctl.Catalog().Models() {
|
||||
h := health[m.Name]
|
||||
v := ModelView{
|
||||
Name: m.Name, Description: m.Description, Aliases: m.Aliases, Source: m.Source,
|
||||
Min: m.Scaling.Min, Max: m.Scaling.Max,
|
||||
Ready: ready[m.Name], Total: total[m.Name],
|
||||
Fallbacks: m.Fallbacks,
|
||||
Desired: h.Desired, Failures: h.Failures,
|
||||
Parked: h.Parked, Reason: h.Reason, Unplaceable: h.Unplaceable,
|
||||
}
|
||||
for _, p := range m.Profiles {
|
||||
v.Profiles = append(v.Profiles, p.Name)
|
||||
@@ -1085,3 +1105,13 @@ func (a *API) restart(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, RestartResult{Queued: ids})
|
||||
}
|
||||
|
||||
func (a *API) retryModel(w http.ResponseWriter, r *http.Request) {
|
||||
name := r.PathValue("name")
|
||||
if _, ok := a.ctl.Catalog().Resolve(name); !ok {
|
||||
writeError(w, http.StatusNotFound, "no model called "+name)
|
||||
return
|
||||
}
|
||||
was := a.ctl.Reconciler().Retry(name)
|
||||
writeJSON(w, http.StatusOK, map[string]bool{"was_parked": was})
|
||||
}
|
||||
@@ -25,6 +25,9 @@ type Summary struct {
|
||||
// Unplaceable names models the scheduler currently cannot satisfy. This is
|
||||
// the line that turns "something is wrong" into a place to look.
|
||||
Unplaceable []string `json:"unplaceable,omitempty"`
|
||||
// Parked are models that failed every time they started and are no
|
||||
// longer retried, with the last error.
|
||||
Parked []ParkedModel `json:"parked,omitempty"`
|
||||
// IdleReplicas are serving replicas nothing has asked for in a while, which
|
||||
// on a fleet with more models than GPUs is the first place to find room.
|
||||
IdleReplicas []IdleReplica `json:"idle_replicas,omitempty"`
|
||||
@@ -201,17 +204,30 @@ func (a *API) Summary() Summary {
|
||||
|
||||
for _, m := range a.Models() {
|
||||
s.Models.Total++
|
||||
if m.Ready >= m.Min {
|
||||
if m.Parked {
|
||||
s.Parked = append(s.Parked, ParkedModel{Model: m.Name, Failures: m.Failures, Reason: m.Reason})
|
||||
}
|
||||
// Against what it should have now, not its minimum: an on-demand
|
||||
// model's minimum is 0, so one that was asked for and fitted nowhere
|
||||
// used to count as satisfied.
|
||||
if m.Ready >= m.Desired && !m.Parked {
|
||||
s.Models.Satisfied++
|
||||
continue
|
||||
}
|
||||
s.Models.Short++
|
||||
// Short of replicas and unable to place more is the actionable case;
|
||||
// short and still starting is just a cluster doing its job.
|
||||
if m.Total <= m.Ready {
|
||||
if m.Unplaceable || (m.Total <= m.Ready && !m.Parked) {
|
||||
s.Unplaceable = append(s.Unplaceable, m.Name)
|
||||
}
|
||||
}
|
||||
sort.Strings(s.Unplaceable)
|
||||
return s
|
||||
}
|
||||
|
||||
// ParkedModel is a model no longer retried, and why.
|
||||
type ParkedModel struct {
|
||||
Model string `json:"model"`
|
||||
Failures int `json:"failures"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
Reference in new issue
Block a user