Ilker ManapandClaude Opus 5.5
37a80a9b44
On the live cluster kev-0.5b — an architecture the nodes' llama.cpp did not know — was started 155 times in eight hours, failing the same way each time, and phi-4, which fitted on no node, logged "no placement" 250 times in two days. Neither showed up anywhere an operator looked: the backoff never gave up, no metric said a model was failing or unplaceable, and the one alert for a model with nothing serving could not fire, because its series was published only while a replica was ready. After five failures in a row a model is parked: not placed again until its definition changes — a new source or profile — or `llmgw ctl retry` is run. Status, the model list and the admin API show it with its last error. The placement warning is logged when its reason changes or every ten minutes. A request for a parked or unplaceable model is answered at once with why, rather than "loading; retry shortly", which a client retries forever. Each request still clears the previous placement result and triggers a fresh attempt, so a model that fits once room is made is not refused on a stale answer; the router waits for that attempt, not for its full activation window. Metrics now cover model health (desired, ready at 0 for what should run, consecutive start failures, parked, unplaceable), engine installs per node, leadership, and certificate expiry read from disk at each scrape, with alerts for each. `ctl status` counts a model against what it should have now, so an on-demand model that was asked for and fitted nowhere no longer reads as satisfied. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
76 lines
2.8 KiB
Go
76 lines
2.8 KiB
Go
package cli
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
|
|
"github.com/spf13/cobra"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
)
|
|
|
|
func newRestartCmd(client func() *ctlClient) *cobra.Command {
|
|
var (
|
|
models []string
|
|
all bool
|
|
)
|
|
c := &cobra.Command{
|
|
Use: "restart [REPLICA...]",
|
|
Short: "Replace replicas with new ones, starting each replacement before stopping the old",
|
|
Long: "How a running engine picks up what only applies at start: an engine key,\n" +
|
|
"an edited profile, a new engine version.\n\n" +
|
|
"Each replacement is placed beside the replica it replaces, and the old one\n" +
|
|
"keeps serving until the new one is ready. Replicas of one model are\n" +
|
|
"replaced one at a time. Where nothing else has room for a second copy —\n" +
|
|
"the usual case on a small CPU fleet — the old one is stopped after the\n" +
|
|
"drain grace period (5 minutes by default) and the new one starts in its\n" +
|
|
"place; a model with only that replica is unserved while it loads.\n\n" +
|
|
" llmgw ctl restart qwen2.5-7b-3f2a91c0\n" +
|
|
" llmgw ctl restart --model qwen2.5-7b\n" +
|
|
" llmgw ctl restart --all",
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
if len(args) == 0 && len(models) == 0 && !all {
|
|
return errors.New("name replicas, --model, or --all")
|
|
}
|
|
body, _ := json.Marshal(adminapi.RestartRequest{Replicas: args, Models: models, All: all})
|
|
var res adminapi.RestartResult
|
|
if err := client().post(cmd.Context(), "/api/v1/restart", body, &res); err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("%d replica(s) queued for restart; watch with: llmgw ctl replicas\n", len(res.Queued))
|
|
return nil
|
|
},
|
|
}
|
|
c.Flags().StringArrayVar(&models, "model", nil, "every replica of this model; repeatable")
|
|
c.Flags().BoolVar(&all, "all", false, "every ready replica in the cluster")
|
|
return c
|
|
}
|
|
|
|
func newRetryCmd(client func() *ctlClient) *cobra.Command {
|
|
return &cobra.Command{
|
|
Use: "retry MODEL",
|
|
Short: "Try a parked model again, and clear its backoff",
|
|
Long: "A model whose replicas fail every time they start is parked after five\n" +
|
|
"failures in a row: it is not placed again until its definition changes\n" +
|
|
"or this is run. Run it once the cause — the weights, the engine, the\n" +
|
|
"profile — has been dealt with; `llmgw ctl status` lists parked models with\n" +
|
|
"their last error.",
|
|
Args: cobra.ExactArgs(1),
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
var res struct {
|
|
WasParked bool `json:"was_parked"`
|
|
}
|
|
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/retry", []byte(`{}`), &res); err != nil {
|
|
return err
|
|
}
|
|
if res.WasParked {
|
|
fmt.Printf("%s unparked; it is placed on the next pass\n", args[0])
|
|
} else {
|
|
fmt.Printf("%s was not parked; its backoff is cleared\n", args[0])
|
|
}
|
|
return nil
|
|
},
|
|
}
|
|
}
|