From 37a80a9b44b1b04e775611f1a003b83b8f4e23c3 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 30 Sep 2026 01:21:57 +0200 Subject: [PATCH] Park a model that fails every time it starts, and say so everywhere MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On the live cluster kev-0.5b — an architecture the nodes' llama.cpp did not know — was started 155 times in eight hours, failing the same way each time, and phi-4, which fitted on no node, logged "no placement" 250 times in two days. Neither showed up anywhere an operator looked: the backoff never gave up, no metric said a model was failing or unplaceable, and the one alert for a model with nothing serving could not fire, because its series was published only while a replica was ready. After five failures in a row a model is parked: not placed again until its definition changes — a new source or profile — or `llmgw ctl retry` is run. Status, the model list and the admin API show it with its last error. The placement warning is logged when its reason changes or every ten minutes. A request for a parked or unplaceable model is answered at once with why, rather than "loading; retry shortly", which a client retries forever. Each request still clears the previous placement result and triggers a fresh attempt, so a model that fits once room is made is not refused on a stale answer; the router waits for that attempt, not for its full activation window. Metrics now cover model health (desired, ready at 0 for what should run, consecutive start failures, parked, unplaceable), engine installs per node, leadership, and certificate expiry read from disk at each scrape, with alerts for each. `ctl status` counts a model against what it should have now, so an on-demand model that was asked for and fitted nowhere no longer reads as satisfied. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 31 +++ deploy/monitoring/llmgw-rules.yml | 84 ++++++- internal/cli/ctl.go | 5 + internal/cli/ctl_restart.go | 27 +++ internal/cli/server.go | 6 + internal/cli/status.go | 8 + internal/control/adminapi/api.go | 30 +++ internal/control/adminapi/summary.go | 20 +- internal/control/control.go | 3 + internal/control/fleetmetrics/fleetmetrics.go | 80 ++++++- .../control/fleetmetrics/fleetmetrics_test.go | 32 +++ internal/control/reconciler/reconciler.go | 220 ++++++++++++++++-- .../control/reconciler/reconciler_test.go | 96 ++++++++ internal/router/activate.go | 23 ++ internal/router/router.go | 5 + internal/router/router_test.go | 36 +++ internal/tlsconf/expiry.go | 58 +++++ 17 files changed, 740 insertions(+), 24 deletions(-) create mode 100644 internal/tlsconf/expiry.go diff --git a/CHANGELOG.md b/CHANGELOG.md index a57fb1f..d81d80e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,37 @@ grace period and the new one starts in its place. This is how running engines pick up the keys introduced in v0.10.3, an edited profile, or a new engine version. +### A model that keeps failing is parked, and everybody is told + +A model whose replicas failed at every start was retried forever: kev-0.5b, +whose architecture the nodes' llama.cpp did not know, was started 155 times in +eight hours. After five failures in a row a model is now parked — not placed +again until its definition changes or `llmgw ctl retry MODEL` is run — and +`ctl status`, `ctl models` and the admin API say so with the last error. + +A request for a model that will not come up — parked, or fitting on no node — +is answered at once with the reason, instead of "the model is loading; retry +shortly", which a client would retry forever. Each request still makes the +control plane try placing it afresh, so a model that fits once room is made is +not refused on an old answer. "no placement for model" is logged when its +reason changes or every ten minutes, not on every pass: it was 250 identical +lines in two days for one model. + +### Metrics and alerts for what goes wrong + +New series: `llmgw_model_replicas_desired`, `llmgw_model_start_failures`, +`llmgw_model_parked`, `llmgw_model_unplaceable`, `llmgw_node_engine_install`, +`llmgw_control_plane_leader` and `llmgw_certificate_expiry_timestamp_seconds`. +`llmgw_model_replicas_ready` is now published at 0 for every model that should +be running; it was published only while a replica was ready, so +ModelHasNothingServing could never fire. New alerts: ModelParked, +ModelFailingToStart, ModelCannotBePlaced, EngineInstallFailed, +ControlPlaneLeadership, CertificateExpiresSoon and CertificateExpiresVerySoon. + +`ctl status` counts a model against the replicas it should have now rather +than its minimum, so an on-demand model that was asked for and fitted nowhere +no longer reads as satisfied. + ### Fixes - `ctl` reported a change the API had queued (202 Accepted) as an error, as diff --git a/deploy/monitoring/llmgw-rules.yml b/deploy/monitoring/llmgw-rules.yml index a35afc3..128f11e 100644 --- a/deploy/monitoring/llmgw-rules.yml +++ b/deploy/monitoring/llmgw-rules.yml @@ -42,10 +42,86 @@ groups: annotations: summary: "{{ $labels.model }} has no replica ready" description: >- - The model is placed and nothing is serving it. Ten minutes, because - a large model on CPU legitimately takes minutes to load. A model - that is deliberately stopped publishes no series at all, so this - cannot fire for one nobody asked for. + The model should be running and nothing is serving it. Ten minutes, + because a large model on CPU legitimately takes minutes to load. The + series is published at 0 for every model that should be running — + before v0.10.4 it was published only while a replica was ready, so + this could not fire at all. A model deliberately stopped, or an + on-demand one nobody is asking for, publishes nothing. + + - alert: ModelParked + expr: llmgw_model_parked == 1 + labels: { severity: critical } + annotations: + summary: "{{ $labels.model }} failed every time it started and is no longer retried" + description: >- + Five replicas in a row failed at start. Before this existed a model + whose weights its engine could not load was started 155 times over + eight hours. `llmgw ctl status` has the last error; once it is dealt + with, `llmgw ctl retry `. Changing the model's definition + also retries it. + + - alert: ModelFailingToStart + expr: llmgw_model_start_failures >= 2 + for: 5m + labels: { severity: warning } + annotations: + summary: "{{ $labels.model }} has failed to start {{ $value }} times in a row" + description: >- + On its way to being parked. The engine's own words are in + `llmgw ctl logs --source ` for the next attempt, or in + the control plane's log for the last one. + + - alert: ModelCannotBePlaced + expr: llmgw_model_unplaceable == 1 + for: 10m + labels: { severity: warning } + annotations: + summary: "{{ $labels.model }} is wanted and fits on no node" + description: >- + Short of replicas, and the last attempt found nowhere with the + memory, cards or engine it needs. `llmgw ctl explain-placement + ` says what each node lacks. For an on-demand model, requests + for it are told this rather than asked to retry. + + - alert: EngineInstallFailed + expr: llmgw_node_engine_install{state="failed"} == 1 + for: 15m + labels: { severity: warning } + annotations: + summary: "{{ $labels.engine }} {{ $labels.version }} will not install on {{ $labels.node }}" + description: >- + The node takes no model needing it until it does. The agent retries + on its own; `llmgw ctl nodes` and the node's logs on the cluster + page say why it failed. + + - alert: ControlPlaneLeadership + expr: sum(llmgw_control_plane_leader) != 1 + for: 2m + labels: { severity: critical } + annotations: + summary: "{{ $value }} control planes believe they are placing work" + description: >- + None means nothing is placed or replaced; two means both are, each + into GPUs the other believes free. The lease is in the store: check + it is reachable from every control plane. + + - alert: CertificateExpiresSoon + expr: (llmgw_certificate_expiry_timestamp_seconds - time()) < 30 * 86400 + labels: { severity: warning } + annotations: + summary: "{{ $labels.cert }} ({{ $labels.subject }}) expires in {{ $value | humanizeDuration }}" + description: >- + Certificates last a year and nothing renews them. When the agent + server's runs out every agent refuses to connect at once. Reissue + with `llmgw ctl cert` and restart the control plane. + + - alert: CertificateExpiresVerySoon + expr: (llmgw_certificate_expiry_timestamp_seconds - time()) < 7 * 86400 + labels: { severity: critical } + annotations: + summary: "{{ $labels.cert }} ({{ $labels.subject }}) expires in {{ $value | humanizeDuration }}" + description: See CertificateExpiresSoon; a week is not long. - alert: PoolSaturated expr: llmgw_pool_saturation > 1 diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index c75fcd9..868fdc3 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -139,6 +139,7 @@ func newCtlCmd() *cobra.Command { newModelCmd(client), newScaleCmd(client), newRestartCmd(client), + newRetryCmd(client), newEngineCheckCmd(), newToolCheckCmd(), newWeightsCmd(client, &asJSON), @@ -248,6 +249,10 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { // actually served, never what is merely catalogued. func modelStatus(m adminapi.ModelView) string { switch { + case m.Parked: + return "parked" + case m.Unplaceable: + return "cannot place" case m.Min == 0 && m.Total == 0: return "stopped" case m.Ready >= m.Min && m.Min > 0: diff --git a/internal/cli/ctl_restart.go b/internal/cli/ctl_restart.go index 3b5febd..bf6cb93 100644 --- a/internal/cli/ctl_restart.go +++ b/internal/cli/ctl_restart.go @@ -46,3 +46,30 @@ func newRestartCmd(client func() *ctlClient) *cobra.Command { c.Flags().BoolVar(&all, "all", false, "every ready replica in the cluster") return c } + +func newRetryCmd(client func() *ctlClient) *cobra.Command { + return &cobra.Command{ + Use: "retry MODEL", + Short: "Try a parked model again, and clear its backoff", + Long: "A model whose replicas fail every time they start is parked after five\n" + + "failures in a row: it is not placed again until its definition changes\n" + + "or this is run. Run it once the cause — the weights, the engine, the\n" + + "profile — has been dealt with; `llmgw ctl status` lists parked models with\n" + + "their last error.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + var res struct { + WasParked bool `json:"was_parked"` + } + if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/retry", []byte(`{}`), &res); err != nil { + return err + } + if res.WasParked { + fmt.Printf("%s unparked; it is placed on the next pass\n", args[0]) + } else { + fmt.Printf("%s was not parked; its backoff is cleared\n", args[0]) + } + return nil + }, + } +} diff --git a/internal/cli/server.go b/internal/cli/server.go index 51e62b4..2c6884d 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -36,6 +36,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/promq" "gitea.mynodes.xyz/ilker/llmgw/internal/router" storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store" + "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" "gitea.mynodes.xyz/ilker/llmgw/internal/version" "gitea.mynodes.xyz/ilker/llmgw/internal/web" ) @@ -131,6 +132,11 @@ func newServerCmd() *cobra.Command { // On the same /metrics as the traffic figures: correlating // "the cluster was full" with "requests were queueing" should // not need two systems. + if err := rt.RegisterMetrics(tlsconf.ExpiryCollector(map[string]string{ + "agent-server": cfg.AgentTLS.Cert, "agent-ca": cfg.AgentTLS.CA, + })); err != nil { + return err + } if err := rt.RegisterMetrics(fleetmetrics.New(ctl)); err != nil { return err } diff --git a/internal/cli/status.go b/internal/cli/status.go index 82beee3..acb35ff 100644 --- a/internal/cli/status.go +++ b/internal/cli/status.go @@ -94,6 +94,14 @@ func printSummary(cmd *cobra.Command, s adminapi.Summary) { fmt.Fprintf(out, "\nMODELS %d of %d have the replicas they asked for\n", s.Models.Satisfied, s.Models.Total) + if len(s.Parked) > 0 { + fmt.Fprintln(out, "\nPARKED — failed every time they started; not retried until fixed") + for _, p := range s.Parked { + fmt.Fprintf(out, " %s (%d failures): %s\n", p.Model, p.Failures, p.Reason) + } + fmt.Fprintln(out, "\n Once fixed: llmgw ctl retry ") + } + if len(s.Unplaceable) > 0 { fmt.Fprintln(out, "\nNOT PLACED") for _, m := range s.Unplaceable { diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 447e09a..3742e0b 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -78,6 +78,7 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest) mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs) mux.HandleFunc("POST /api/v1/restart", a.restart) + mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel) } // --- responses --- @@ -233,6 +234,18 @@ type ModelView struct { Max int `json:"max_replicas"` Ready int `json:"ready_replicas"` Total int `json:"total_replicas"` + // Desired is how many it should have now: Min, or for an on-demand + // model one while it is being asked for and none otherwise. + Desired int `json:"desired_replicas"` + // Failures counts replicas that failed to start in a row. + Failures int `json:"failures,omitempty"` + // Parked is set once Failures reached the limit; the model is not retried + // until its definition changes or `ctl retry` is run. Reason is the last + // error. + Parked bool `json:"parked,omitempty"` + Reason string `json:"reason,omitempty"` + // Unplaceable is set when it is short and the last attempt found no node. + Unplaceable bool `json:"unplaceable,omitempty"` Fallbacks []string `json:"fallbacks,omitempty"` Profiles []string `json:"profiles"` } @@ -442,13 +455,20 @@ func (a *API) Models() []ModelView { } } + health := map[string]reconciler.ModelHealth{} + for _, h := range a.ctl.Reconciler().ModelHealth() { + health[h.Model] = h + } out := make([]ModelView, 0) for _, m := range a.ctl.Catalog().Models() { + h := health[m.Name] v := ModelView{ Name: m.Name, Description: m.Description, Aliases: m.Aliases, Source: m.Source, Min: m.Scaling.Min, Max: m.Scaling.Max, Ready: ready[m.Name], Total: total[m.Name], Fallbacks: m.Fallbacks, + Desired: h.Desired, Failures: h.Failures, + Parked: h.Parked, Reason: h.Reason, Unplaceable: h.Unplaceable, } for _, p := range m.Profiles { v.Profiles = append(v.Profiles, p.Name) @@ -1085,3 +1105,13 @@ func (a *API) restart(w http.ResponseWriter, r *http.Request) { } writeJSON(w, http.StatusAccepted, RestartResult{Queued: ids}) } + +func (a *API) retryModel(w http.ResponseWriter, r *http.Request) { + name := r.PathValue("name") + if _, ok := a.ctl.Catalog().Resolve(name); !ok { + writeError(w, http.StatusNotFound, "no model called "+name) + return + } + was := a.ctl.Reconciler().Retry(name) + writeJSON(w, http.StatusOK, map[string]bool{"was_parked": was}) +} diff --git a/internal/control/adminapi/summary.go b/internal/control/adminapi/summary.go index b3d23c1..897568c 100644 --- a/internal/control/adminapi/summary.go +++ b/internal/control/adminapi/summary.go @@ -25,6 +25,9 @@ type Summary struct { // Unplaceable names models the scheduler currently cannot satisfy. This is // the line that turns "something is wrong" into a place to look. Unplaceable []string `json:"unplaceable,omitempty"` + // Parked are models that failed every time they started and are no + // longer retried, with the last error. + Parked []ParkedModel `json:"parked,omitempty"` // IdleReplicas are serving replicas nothing has asked for in a while, which // on a fleet with more models than GPUs is the first place to find room. IdleReplicas []IdleReplica `json:"idle_replicas,omitempty"` @@ -201,17 +204,30 @@ func (a *API) Summary() Summary { for _, m := range a.Models() { s.Models.Total++ - if m.Ready >= m.Min { + if m.Parked { + s.Parked = append(s.Parked, ParkedModel{Model: m.Name, Failures: m.Failures, Reason: m.Reason}) + } + // Against what it should have now, not its minimum: an on-demand + // model's minimum is 0, so one that was asked for and fitted nowhere + // used to count as satisfied. + if m.Ready >= m.Desired && !m.Parked { s.Models.Satisfied++ continue } s.Models.Short++ // Short of replicas and unable to place more is the actionable case; // short and still starting is just a cluster doing its job. - if m.Total <= m.Ready { + if m.Unplaceable || (m.Total <= m.Ready && !m.Parked) { s.Unplaceable = append(s.Unplaceable, m.Name) } } sort.Strings(s.Unplaceable) return s } + +// ParkedModel is a model no longer retried, and why. +type ParkedModel struct { + Model string `json:"model"` + Failures int `json:"failures"` + Reason string `json:"reason"` +} diff --git a/internal/control/control.go b/internal/control/control.go index 73f62d9..38727b2 100644 --- a/internal/control/control.go +++ b/internal/control/control.go @@ -171,6 +171,9 @@ func (c *Controller) Nodes() []inventory.Node { return c.inv.Nodes() } func (c *Controller) FreeGPUs(nodeID string) []int { return c.inv.FreeGPUs(nodeID) } func (c *Controller) Replicas() []reconciler.Replica { return c.rec.Replicas() } +// ModelHealth is how each model in the catalog stands. +func (c *Controller) ModelHealth() []reconciler.ModelHealth { return c.rec.ModelHealth() } + // Reconciler exposes the reconciliation loop, for status reports and ctl. func (c *Controller) Reconciler() *reconciler.Reconciler { return c.rec } diff --git a/internal/control/fleetmetrics/fleetmetrics.go b/internal/control/fleetmetrics/fleetmetrics.go index 660df99..ecaf5b6 100644 --- a/internal/control/fleetmetrics/fleetmetrics.go +++ b/internal/control/fleetmetrics/fleetmetrics.go @@ -39,9 +39,25 @@ type Source interface { // derived from state that already exists, so computing them when asked is both // simpler and fresher than maintaining a parallel copy that can drift. Nothing // runs between scrapes. +// HealthSource adds what the reconciler knows about each model and where +// engine installs stand, and whether this control plane is the one placing +// work. Asked for by type assertion, so a source without it — a test's — +// publishes what it did before. +type HealthSource interface { + ModelHealth() []reconciler.ModelHealth + IsLeader() bool +} + type Collector struct { src Source + modelDesired *prometheus.Desc + modelFailures *prometheus.Desc + modelParked *prometheus.Desc + modelUnplaceable *prometheus.Desc + engineInstall *prometheus.Desc + leader *prometheus.Desc + nodeUp *prometheus.Desc nodeGPUs *prometheus.Desc nodeGPUsFree *prometheus.Desc @@ -137,7 +153,26 @@ func New(src Source) *Collector { replicaState: prometheus.NewDesc("llmgw_replicas", "Replicas by model and state.", []string{"model", "state"}, nil), replicaReady: prometheus.NewDesc("llmgw_model_replicas_ready", - "Replicas of a model that are serving.", []string{"model"}, nil), + "Replicas of a model that are serving. Published, as 0 when none are, for every "+ + "model that should be running; a stopped model, or an on-demand one nobody is "+ + "asking for, publishes nothing.", []string{"model"}, nil), + modelDesired: prometheus.NewDesc("llmgw_model_replicas_desired", + "Replicas a model should have now: its minimum, or for an on-demand model 1 while "+ + "it is being asked for.", []string{"model"}, nil), + modelFailures: prometheus.NewDesc("llmgw_model_start_failures", + "Replicas of a model that failed to start in a row. Resets when one comes up.", + []string{"model"}, nil), + modelParked: prometheus.NewDesc("llmgw_model_parked", + "1 when a model failed so often in a row that it is no longer retried.", + []string{"model"}, nil), + modelUnplaceable: prometheus.NewDesc("llmgw_model_unplaceable", + "1 when a model is short of replicas and the last attempt found no node to place one on.", + []string{"model"}, nil), + engineInstall: prometheus.NewDesc("llmgw_node_engine_install", + "Where installing an engine stands on a node: 1 for the current state.", + []string{"node", "engine", "version", "state"}, nil), + leader: prometheus.NewDesc("llmgw_control_plane_leader", + "1 when this control plane is the one placing work, 0 when it is a standby.", nil, nil), replicaIdle: prometheus.NewDesc("llmgw_replica_idle_seconds", "Seconds since a replica was last sent a request. Absent when nothing has been reported, "+ "which is not the same as idle.", []string{"model", "replica"}, nil), @@ -201,6 +236,8 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) { c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy, c.gpuNVLink, c.gpuECC, c.replicaState, c.replicaReady, c.replicaIdle, + c.modelDesired, c.modelFailures, c.modelParked, c.modelUnplaceable, + c.engineInstall, c.leader, } { ch <- d } @@ -387,8 +424,43 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric(c.replicaState, prometheus.GaugeValue, float64(v), k[0], k[1]) } - for model, v := range ready { - ch <- prometheus.MustNewConstMetric(c.replicaReady, prometheus.GaugeValue, - float64(v), model) + hs, ok := c.src.(HealthSource) + if !ok { + for model, v := range ready { + ch <- prometheus.MustNewConstMetric(c.replicaReady, prometheus.GaugeValue, + float64(v), model) + } + return + } + for _, n := range c.src.Nodes() { + for _, p := range n.Provision { + ch <- prometheus.MustNewConstMetric(c.engineInstall, prometheus.GaugeValue, 1, + n.ID, p.Engine, p.Version, p.State) + } + } + lead := 0.0 + if hs.IsLeader() { + lead = 1 + } + ch <- prometheus.MustNewConstMetric(c.leader, prometheus.GaugeValue, lead) + for _, h := range hs.ModelHealth() { + // Ready is published at zero for what should be running — the one + // value ModelHasNothingServing exists to catch, and which, emitted + // only for replicas that were Ready, it never saw. + if h.Desired > 0 || ready[h.Model] > 0 { + ch <- prometheus.MustNewConstMetric(c.replicaReady, prometheus.GaugeValue, + float64(ready[h.Model]), h.Model) + } + ch <- prometheus.MustNewConstMetric(c.modelDesired, prometheus.GaugeValue, float64(h.Desired), h.Model) + ch <- prometheus.MustNewConstMetric(c.modelFailures, prometheus.GaugeValue, float64(h.Failures), h.Model) + ch <- prometheus.MustNewConstMetric(c.modelParked, prometheus.GaugeValue, b2f(h.Parked), h.Model) + ch <- prometheus.MustNewConstMetric(c.modelUnplaceable, prometheus.GaugeValue, b2f(h.Unplaceable), h.Model) } } + +func b2f(b bool) float64 { + if b { + return 1 + } + return 0 +} diff --git a/internal/control/fleetmetrics/fleetmetrics_test.go b/internal/control/fleetmetrics/fleetmetrics_test.go index a468fd7..b86b929 100644 --- a/internal/control/fleetmetrics/fleetmetrics_test.go +++ b/internal/control/fleetmetrics/fleetmetrics_test.go @@ -196,3 +196,35 @@ func TestEveryIBPortIsReportedWithItsCounters(t *testing.T) { } } } + +type healthSource struct { + fakeSource + health []reconciler.ModelHealth +} + +func (h healthSource) ModelHealth() []reconciler.ModelHealth { return h.health } +func (h healthSource) IsLeader() bool { return true } + +func TestAModelThatShouldRunPublishesZeroReplicasReady(t *testing.T) { + // ModelHasNothingServing alerts on llmgw_model_replicas_ready == 0, and + // the series was only ever published while a replica was ready — so the + // alert could not fire for the case it was written for. + src := healthSource{health: []reconciler.ModelHealth{ + {Model: "wanted", Desired: 1, Failures: 5, Parked: true, Reason: "boom"}, + {Model: "stopped", Desired: 0}, + }} + out := gather(t, src) + for _, want := range []string{ + "llmgw_model_replicas_ready model=wanted -> 0", + "llmgw_model_parked model=wanted -> 1", + "llmgw_model_start_failures model=wanted -> 5", + "llmgw_control_plane_leader -> 1", + } { + if !strings.Contains(out, want) { + t.Errorf("missing %s", want) + } + } + if strings.Contains(out, "llmgw_model_replicas_ready model=stopped") { + t.Error("a stopped model publishes a ready count, so the alert would fire for it") + } +} diff --git a/internal/control/reconciler/reconciler.go b/internal/control/reconciler/reconciler.go index 173ba2b..3a922b9 100644 --- a/internal/control/reconciler/reconciler.go +++ b/internal/control/reconciler/reconciler.go @@ -3,11 +3,15 @@ package reconciler import ( "context" crypterand "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" "errors" "fmt" "log/slog" "math/rand/v2" "slices" + "strings" "sync" "time" @@ -66,6 +70,15 @@ type Config struct { // Without a bound, draining a node whose model cannot be placed anywhere // else would never finish. DrainGrace time.Duration + // ParkAfter is how many failures in a row stop a model being retried. + // + // Backoff alone never gives up: it grows to MaxBackoff and keeps trying, + // and a model whose weights its engine cannot load — an architecture the + // installed llama.cpp does not know — was started 155 times over eight + // hours on the live cluster, each time failing the same way, with nothing + // to show for it but the log. A parked model is not placed again until its + // definition changes or an operator runs `llmgw ctl retry`. + ParkAfter int // PortRange is the per-node port range for engine HTTP and dist-init. PortRange [2]int // MgmtInterface is the ethernet interface used for NCCL bootstrap. @@ -126,6 +139,7 @@ func DefaultConfig() Config { EvictionGrace: 2 * time.Minute, MinLifetime: 5 * time.Minute, DrainGrace: 5 * time.Minute, + ParkAfter: 5, PortRange: [2]int{20000, 20999}, MgmtInterface: "eth0", } @@ -165,6 +179,9 @@ func (c *Config) normalize() { if c.DrainGrace <= 0 { c.DrainGrace = d.DrainGrace } + if c.ParkAfter <= 0 { + c.ParkAfter = d.ParkAfter + } if c.PortRange[1] <= c.PortRange[0] { c.PortRange = d.PortRange } @@ -209,6 +226,11 @@ type Reconciler struct { // restarted and not yet begun. One is replaced at a time: replacing them // all at once would need the memory of every replica twice over. restartQueue map[string][]string + // parked holds models that failed ParkAfter times in a row. + parked map[string]parkRecord + // placementLogged remembers the last "no placement" said per model, so it + // is said when it changes rather than on every five-second pass. + placementLogged map[string]placementNote wake chan struct{} stop chan struct{} @@ -230,19 +252,21 @@ func New(cfg Config, cat *catalog.Live, inv *inventory.Store, disp Dispatcher, l } return &Reconciler{ cfg: cfg, cat: cat, inv: inv, disp: disp, log: log, - sched: placement.New(inv), - now: time.Now, - replicas: map[string]*Replica{}, - nextAttempt: map[string]time.Time{}, - attempts: map[string]int{}, - reservedPorts: map[int]struct{}{}, - wanted: map[string]time.Time{}, - restartQueue: map[string][]string{}, - lastExplain: map[string]placement.Result{}, - wake: make(chan struct{}, 1), - stop: make(chan struct{}), - done: make(chan struct{}), - startedAt: time.Now(), + sched: placement.New(inv), + now: time.Now, + replicas: map[string]*Replica{}, + nextAttempt: map[string]time.Time{}, + attempts: map[string]int{}, + reservedPorts: map[int]struct{}{}, + wanted: map[string]time.Time{}, + restartQueue: map[string][]string{}, + parked: map[string]parkRecord{}, + placementLogged: map[string]placementNote{}, + lastExplain: map[string]placement.Result{}, + wake: make(chan struct{}, 1), + stop: make(chan struct{}), + done: make(chan struct{}), + startedAt: time.Now(), } } @@ -464,6 +488,12 @@ func (r *Reconciler) Want(model string) bool { } r.mu.Lock() r.wanted[m.Name] = r.now() + // A placement that failed before this request says nothing about now: + // forgotten, so the pass this triggers decides afresh and Unservable + // does not answer from a stale result. + if res, ok := r.lastExplain[m.Name]; ok && res.Placement == nil { + delete(r.lastExplain, m.Name) + } r.mu.Unlock() // Immediately, not on the next tick. Someone is waiting on this request, // and five seconds of ticker is five seconds they spend staring at nothing. @@ -669,6 +699,17 @@ func (r *Reconciler) reapFailed(ctx context.Context) { // replica is about to disappear, and the backoff has to survive it. r.attempts[rep.Model]++ r.nextAttempt[rep.Model] = r.now().Add(r.backoff(r.attempts[rep.Model])) + if n := r.attempts[rep.Model]; n >= r.cfg.ParkAfter { + if _, already := r.parked[rep.Model]; !already { + m, _ := r.cat.Load().Resolve(rep.Model) + r.parked[rep.Model] = parkRecord{ + Since: r.now(), Reason: rep.Err, Failures: n, fp: modelFingerprint(m), + } + r.log.Error("model parked: it failed every time it started, and is not retried "+ + "until its definition changes or `llmgw ctl retry` is run", + "model", rep.Model, "failures", n, "last_error", rep.Err) + } + } } } r.mu.Unlock() @@ -746,10 +787,22 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) { } } blockedUntil := r.nextAttempt[m.Name] + park, isParked := r.parked[m.Name] + if isParked && park.fp != modelFingerprint(m) { + // Somebody changed the model — a new source, an edited profile — so + // the reason it kept failing may be gone. Worth one more streak. + delete(r.parked, m.Name) + r.attempts[m.Name] = 0 + delete(r.nextAttempt, m.Name) + isParked, blockedUntil = false, time.Time{} + r.log.Info("model unparked: its definition changed", "model", m.Name) + } r.mu.Unlock() desired := r.desiredFor(m) switch { + case len(liveIDs) < desired && isParked: + // Not placed. Its failure is on every status the operator reads. case len(liveIDs) < desired: if r.now().Before(blockedUntil) { // Still in backoff. Rebuilding immediately would hot-loop on a @@ -925,7 +978,7 @@ func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []strin if m.Scaling.OnDemand() && r.makeRoomFor(ctx, m) { return } - r.log.Warn("no placement for model", "model", m.Name, "reasons", res.Explain()) + r.notePlacementFailure(m.Name, res.Explain()) return } if err := r.startReplica(ctx, m, res.Placement); err != nil { @@ -1523,3 +1576,142 @@ func (r *Reconciler) engineKey(replicaID string) string { } return r.cfg.EngineKey(replicaID) } + +// parkRecord is why a model stopped being retried. +type parkRecord struct { + Since time.Time + Reason string + Failures int + // fp is the model's definition when it was parked; a different one + // unparks it. + fp string +} + +type placementNote struct { + text string + at time.Time +} + +// modelFingerprint identifies a model's definition, so a change to it can be +// told from a retry of the same thing. +func modelFingerprint(m catalog.Model) string { + b, _ := json.Marshal(m) + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:8]) +} + +// placementLogEvery is how often an unchanged placement failure is repeated. +const placementLogEvery = 10 * time.Minute + +// notePlacementFailure logs why a model could not be placed, when the reason +// is new or has not been said for a while. Said on every pass it was 250 +// identical lines in two days for one model, burying everything else. +func (r *Reconciler) notePlacementFailure(model, why string) { + r.mu.Lock() + last := r.placementLogged[model] + say := last.text != why || r.now().Sub(last.at) >= placementLogEvery + if say { + r.placementLogged[model] = placementNote{text: why, at: r.now()} + } + r.mu.Unlock() + if say { + r.log.Warn("no placement for model", "model", model, "reasons", why) + } +} + +// Retry clears a parked model and its backoff, so it is placed on the next +// pass. +func (r *Reconciler) Retry(model string) bool { + r.mu.Lock() + _, was := r.parked[model] + delete(r.parked, model) + r.attempts[model] = 0 + delete(r.nextAttempt, model) + r.mu.Unlock() + r.Notify() + return was +} + +// ModelHealth is how one model stands, for status and metrics. +type ModelHealth struct { + Model string + // Desired is how many replicas it should have now; 0 for a stopped model + // or an on-demand one nobody is asking for. + Desired int + Ready int + // Failures counts replicas that failed in a row. + Failures int + // Parked is set when Failures reached ParkAfter; Reason is the last error. + Parked bool + ParkedSince time.Time + Reason string + // Unplaceable is set when it is short of replicas and the last attempt to + // place one found nowhere; Placement says why. + Unplaceable bool + Placement string +} + +// ModelHealth reports every model in the catalog. +func (r *Reconciler) ModelHealth() []ModelHealth { + models := r.cat.Load().Models() + desired := map[string]int{} + for _, m := range models { + desired[m.Name] = r.desiredFor(m) + } + r.mu.Lock() + defer r.mu.Unlock() + ready, live := map[string]int{}, map[string]int{} + for _, rep := range r.replicas { + if rep.State == Ready { + ready[rep.Model]++ + } + if rep.State.Counts() { + live[rep.Model]++ + } + } + out := make([]ModelHealth, 0, len(models)) + for _, m := range models { + h := ModelHealth{Model: m.Name, Desired: desired[m.Name], Ready: ready[m.Name], + Failures: r.attempts[m.Name]} + if p, ok := r.parked[m.Name]; ok { + h.Parked, h.ParkedSince, h.Reason = true, p.Since, p.Reason + } + if res, ok := r.lastExplain[m.Name]; ok && res.Placement == nil && live[m.Name] < h.Desired { + h.Unplaceable, h.Placement = true, res.Explain() + } + out = append(out, h) + } + return out +} + +// Unservable says why a model cannot be served at all, so a request for it is +// told now rather than asked to retry something that will never come up. +func (r *Reconciler) Unservable(model string) (string, bool) { + m, ok := r.cat.Load().Resolve(model) + if !ok { + return "", false + } + r.mu.Lock() + defer r.mu.Unlock() + if p, ok := r.parked[m.Name]; ok { + return fmt.Sprintf("%s failed to start %d times in a row and is not being retried: %s", + m.Name, p.Failures, p.Reason), true + } + for _, rep := range r.replicas { + if rep.Model == m.Name && rep.State.Counts() { + return "", false // something is on its way + } + } + if res, ok := r.lastExplain[m.Name]; ok && res.Placement == nil { + return m.Name + " cannot be placed on any node: " + firstLine(res.Explain()), true + } + return "", false +} + +func firstLine(s string) string { + s = strings.TrimSpace(s) + if i := strings.IndexByte(s, '\n'); i >= 0 { + return strings.TrimSpace(s[:i]) + " (and others; see llmgw ctl explain-placement)" + } + return s +} diff --git a/internal/control/reconciler/reconciler_test.go b/internal/control/reconciler/reconciler_test.go index 237695f..c1394b1 100644 --- a/internal/control/reconciler/reconciler_test.go +++ b/internal/control/reconciler/reconciler_test.go @@ -1,6 +1,7 @@ package reconciler import ( + "bytes" "context" "fmt" "log/slog" @@ -1836,3 +1837,98 @@ func TestReplicasOfAModelAreRestartedOneAtATime(t *testing.T) { t.Error("an unknown replica was accepted") } } + +// failRepeatedly starts and fails the model's replica n times over. +func failRepeatedly(t *testing.T, h *harness, n int, why string) { + t.Helper() + for i := 0; i < n; i++ { + h.disp.reset() + h.r.Reconcile(t.Context()) + started := h.disp.startedShards() + if len(started) == 0 { + return // parked, or still in backoff + } + s := started[0] + h.r.OnShardStatus(s.ReplicaID, s.ShardID, supervisor.Failed, why) + h.r.Reconcile(t.Context()) + h.advance(DefaultConfig().MaxBackoff + time.Second) + } +} + +func TestAModelThatAlwaysFailsIsParkedNotRetriedForever(t *testing.T) { + // kev-0.5b: an architecture the node's llama.cpp did not know, started + // 155 times in eight hours, the same error every time. + h := newHarness(t, testModel(1), gpuNode("1", 8)) + failRepeatedly(t, h, DefaultConfig().ParkAfter, "unknown model architecture: 'ggmlc'") + + h.disp.reset() + h.advance(DefaultConfig().MaxBackoff + time.Second) + h.r.Reconcile(t.Context()) + if n := len(h.disp.startedShards()); n != 0 { + t.Fatalf("a parked model was started again (%d shards)", n) + } + var mh ModelHealth + for _, x := range h.r.ModelHealth() { + if x.Model == "llama" { + mh = x + } + } + if !mh.Parked || !strings.Contains(mh.Reason, "ggmlc") || mh.Failures < DefaultConfig().ParkAfter { + t.Errorf("health %+v", mh) + } + if why, no := h.r.Unservable("llama"); !no || !strings.Contains(why, "ggmlc") { + t.Errorf("a request would not be told why: %q %v", why, no) + } + + // Retried by the operator once fixed. + if !h.r.Retry("llama") { + t.Error("Retry did not report the model as parked") + } + h.r.Reconcile(t.Context()) + if len(h.disp.startedShards()) == 0 { + t.Error("a retried model was not placed") + } +} + +func TestChangingAParkedModelsDefinitionTriesItAgain(t *testing.T) { + h := newHarness(t, testModel(1), gpuNode("1", 8)) + failRepeatedly(t, h, DefaultConfig().ParkAfter, "boom") + + fixed := testModel(1) + fixed.Source = "/models/llama-fixed" + cat, err := catalog.New([]catalog.Model{fixed}) + if err != nil { + t.Fatal(err) + } + h.r.cat.Store(cat) + h.disp.reset() + h.r.Reconcile(t.Context()) + if len(h.disp.startedShards()) == 0 { + t.Error("a model whose definition changed stayed parked") + } +} + +func TestAPlacementFailureIsLoggedWhenItChangesNotEveryPass(t *testing.T) { + var buf bytes.Buffer + h := newHarness(t, testModel(1), gpuNode("1", 1)) // tp4 needs 4 cards + h.r.log = slog.New(slog.NewTextHandler(&buf, nil)) + for i := 0; i < 20; i++ { + h.r.Reconcile(t.Context()) + h.advance(5 * time.Second) + } + if n := strings.Count(buf.String(), "no placement for model"); n != 1 { + t.Errorf("logged %d times over 20 passes", n) + } + h.advance(placementLogEvery) + h.r.Reconcile(t.Context()) + if n := strings.Count(buf.String(), "no placement for model"); n != 2 { + t.Errorf("not repeated after %s: %d", placementLogEvery, n) + } + var mh ModelHealth + for _, x := range h.r.ModelHealth() { + mh = x + } + if !mh.Unplaceable || mh.Desired != 1 { + t.Errorf("health %+v", mh) + } +} diff --git a/internal/router/activate.go b/internal/router/activate.go index 20e5f85..be43317 100644 --- a/internal/router/activate.go +++ b/internal/router/activate.go @@ -17,6 +17,22 @@ type Activator interface { Want(model string) bool } +// unservable is an activator that can also say a model will not come up at +// all — it failed every time it started, or fits on no node — so a request is +// told so at once rather than asked to retry something that never arrives. +type unservable interface { + Unservable(model string) (string, bool) +} + +// whyUnservable is the activator's reason a model cannot be served, if any. +func (r *Router) whyUnservable(model string) (string, bool) { + u, ok := r.activator.(unservable) + if !ok { + return "", false + } + return u.Unservable(model) +} + // SetActivator supplies the control plane that can load models on demand. func (r *Router) SetActivator(a Activator) { r.activator = a } @@ -32,6 +48,8 @@ func (r *Router) SetActivator(a Activator) { r.activator = a } // a slow request. A fifteen-minute wait is a job pretending to be one, and the // client should be told to come back rather than left holding a socket. func (r *Router) activate(ctx context.Context, model string) bool { + // Asked first, always: asking is what makes the control plane try again, + // and a model that fitted nowhere an hour ago may fit now. if r.activator == nil || !r.activator.Want(model) { return false } @@ -56,6 +74,11 @@ func (r *Router) activate(ctx context.Context, model string) bool { if len(r.reg.Snapshots(model)) > 0 { return true } + // Asking set a placement going; if it found nowhere, there is + // nothing to wait for. + if _, no := r.whyUnservable(model); no { + return false + } } } } diff --git a/internal/router/router.go b/internal/router/router.go index 23a782d..4d43f5d 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -629,6 +629,11 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa } } if len(candidates) == 0 { + if why, no := r.whyUnservable(model); no && r.activator != nil { + // Not "loading; retry": it will not load, and a client told to + // retry keeps retrying something that never arrives. + return dispatchResult{err: fmt.Errorf("%w: %s", errNoCapacity, why)} + } if r.activator != nil && r.activator.Want(model) { // Asked for, still loading. Saying "no endpoint serves this" would // be a lie, and a client told to retry will succeed where one told diff --git a/internal/router/router_test.go b/internal/router/router_test.go index b6a9f94..62285dc 100644 --- a/internal/router/router_test.go +++ b/internal/router/router_test.go @@ -619,3 +619,39 @@ func TestTheServedModelHeaderArrivesBeforeTheBody(t *testing.T) { t.Errorf("served-model header = %q, want backup", got) } } + +type unservableActivator struct { + fakeActivator + why map[string]string +} + +func (u *unservableActivator) Unservable(model string) (string, bool) { + w, ok := u.why[model] + return w, ok +} + +func TestAModelThatCannotComeUpIsSaidSoAtOnce(t *testing.T) { + // A model that fits on no node, or failed every time it started, was + // answered "loading; retry shortly" — and a client that retried did so + // forever. + e := newFakeEngine(t, "m") + cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}}) + cfg.Router.ActivationWait = 10 * time.Second + rt := newTestRouter(t, cfg) + rt.SetActivator(&unservableActivator{ + fakeActivator: fakeActivator{onDemand: map[string]bool{"phi-4": true}}, + why: map[string]string{"phi-4": "phi-4 cannot be placed on any node: needs 15 GB"}, + }) + + start := time.Now() + rec := post(t, rt.Handler(), map[string]any{"model": "phi-4", "messages": []any{}}, nil) + if took := time.Since(start); took > 2*time.Second { + t.Errorf("the request waited %s for a model that will not load", took) + } + if rec.Code != http.StatusServiceUnavailable || rec.Header().Get("Retry-After") != "" { + t.Errorf("status %d, Retry-After %q", rec.Code, rec.Header().Get("Retry-After")) + } + if !strings.Contains(rec.Body.String(), "needs 15 GB") { + t.Errorf("the reason was not given: %s", rec.Body.String()) + } +} diff --git a/internal/tlsconf/expiry.go b/internal/tlsconf/expiry.go new file mode 100644 index 0000000..d3d1326 --- /dev/null +++ b/internal/tlsconf/expiry.go @@ -0,0 +1,58 @@ +package tlsconf + +import ( + "crypto/x509" + "encoding/pem" + "os" + + "github.com/prometheus/client_golang/prometheus" +) + +// ExpiryCollector publishes when each certificate in a set of files expires. +// +// Certificates here last a year and nothing renews them; the first sign of one +// running out was going to be every agent refusing to connect on the same day. +// Read at every scrape rather than once, so a certificate replaced on disk is +// seen without a restart. A file that cannot be read publishes nothing, which +// is its own thing to notice. +func ExpiryCollector(files map[string]string) prometheus.Collector { + return &expiry{files: files, desc: prometheus.NewDesc("llmgw_certificate_expiry_timestamp_seconds", + "When a certificate this control plane uses expires, as a Unix time.", + []string{"cert", "subject"}, nil)} +} + +type expiry struct { + files map[string]string + desc *prometheus.Desc +} + +func (e *expiry) Describe(ch chan<- *prometheus.Desc) { ch <- e.desc } + +func (e *expiry) Collect(ch chan<- prometheus.Metric) { + for name, path := range e.files { + if path == "" { + continue + } + b, err := os.ReadFile(path) //nolint:gosec // configured certificate paths + if err != nil { + continue + } + // Every certificate in the file: a CA bundle may hold several. + for { + var blk *pem.Block + blk, b = pem.Decode(b) + if blk == nil { + break + } + if blk.Type != "CERTIFICATE" { + continue + } + c, err := x509.ParseCertificate(blk.Bytes) + if err != nil { + continue + } + ch <- prometheus.MustNewConstMetric(e.desc, prometheus.GaugeValue, + float64(c.NotAfter.Unix()), name, c.Subject.CommonName) + } + } +}