Park a model that fails every time it starts, and say so everywhere
On the live cluster kev-0.5b — an architecture the nodes' llama.cpp did not know — was started 155 times in eight hours, failing the same way each time, and phi-4, which fitted on no node, logged "no placement" 250 times in two days. Neither showed up anywhere an operator looked: the backoff never gave up, no metric said a model was failing or unplaceable, and the one alert for a model with nothing serving could not fire, because its series was published only while a replica was ready. After five failures in a row a model is parked: not placed again until its definition changes — a new source or profile — or `llmgw ctl retry` is run. Status, the model list and the admin API show it with its last error. The placement warning is logged when its reason changes or every ten minutes. A request for a parked or unplaceable model is answered at once with why, rather than "loading; retry shortly", which a client retries forever. Each request still clears the previous placement result and triggers a fresh attempt, so a model that fits once room is made is not refused on a stale answer; the router waits for that attempt, not for its full activation window. Metrics now cover model health (desired, ready at 0 for what should run, consecutive start failures, parked, unplaceable), engine installs per node, leadership, and certificate expiry read from disk at each scrape, with alerts for each. `ctl status` counts a model against what it should have now, so an on-demand model that was asked for and fitted nowhere no longer reads as satisfied. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
5e1b6ecea0
commit
37a80a9b44
17 files changed
+740
-24
No files matched your search
@@ -139,6 +139,7 @@ func newCtlCmd() *cobra.Command {
|
||||
newModelCmd(client),
|
||||
newScaleCmd(client),
|
||||
newRestartCmd(client),
|
||||
newRetryCmd(client),
|
||||
newEngineCheckCmd(),
|
||||
newToolCheckCmd(),
|
||||
newWeightsCmd(client, &asJSON),
|
||||
@@ -248,6 +249,10 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
// actually served, never what is merely catalogued.
|
||||
func modelStatus(m adminapi.ModelView) string {
|
||||
switch {
|
||||
case m.Parked:
|
||||
return "parked"
|
||||
case m.Unplaceable:
|
||||
return "cannot place"
|
||||
case m.Min == 0 && m.Total == 0:
|
||||
return "stopped"
|
||||
case m.Ready >= m.Min && m.Min > 0:
|
||||
|
||||
@@ -46,3 +46,30 @@ func newRestartCmd(client func() *ctlClient) *cobra.Command {
|
||||
c.Flags().BoolVar(&all, "all", false, "every ready replica in the cluster")
|
||||
return c
|
||||
}
|
||||
|
||||
func newRetryCmd(client func() *ctlClient) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "retry MODEL",
|
||||
Short: "Try a parked model again, and clear its backoff",
|
||||
Long: "A model whose replicas fail every time they start is parked after five\n" +
|
||||
"failures in a row: it is not placed again until its definition changes\n" +
|
||||
"or this is run. Run it once the cause — the weights, the engine, the\n" +
|
||||
"profile — has been dealt with; `llmgw ctl status` lists parked models with\n" +
|
||||
"their last error.",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
var res struct {
|
||||
WasParked bool `json:"was_parked"`
|
||||
}
|
||||
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/retry", []byte(`{}`), &res); err != nil {
|
||||
return err
|
||||
}
|
||||
if res.WasParked {
|
||||
fmt.Printf("%s unparked; it is placed on the next pass\n", args[0])
|
||||
} else {
|
||||
fmt.Printf("%s was not parked; its backoff is cleared\n", args[0])
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -36,6 +36,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router"
|
||||
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/web"
|
||||
)
|
||||
@@ -131,6 +132,11 @@ func newServerCmd() *cobra.Command {
|
||||
// On the same /metrics as the traffic figures: correlating
|
||||
// "the cluster was full" with "requests were queueing" should
|
||||
// not need two systems.
|
||||
if err := rt.RegisterMetrics(tlsconf.ExpiryCollector(map[string]string{
|
||||
"agent-server": cfg.AgentTLS.Cert, "agent-ca": cfg.AgentTLS.CA,
|
||||
})); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := rt.RegisterMetrics(fleetmetrics.New(ctl)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -94,6 +94,14 @@ func printSummary(cmd *cobra.Command, s adminapi.Summary) {
|
||||
fmt.Fprintf(out, "\nMODELS %d of %d have the replicas they asked for\n",
|
||||
s.Models.Satisfied, s.Models.Total)
|
||||
|
||||
if len(s.Parked) > 0 {
|
||||
fmt.Fprintln(out, "\nPARKED — failed every time they started; not retried until fixed")
|
||||
for _, p := range s.Parked {
|
||||
fmt.Fprintf(out, " %s (%d failures): %s\n", p.Model, p.Failures, p.Reason)
|
||||
}
|
||||
fmt.Fprintln(out, "\n Once fixed: llmgw ctl retry <model>")
|
||||
}
|
||||
|
||||
if len(s.Unplaceable) > 0 {
|
||||
fmt.Fprintln(out, "\nNOT PLACED")
|
||||
for _, m := range s.Unplaceable {
|
||||
|
||||
Reference in new issue
Block a user