Park a model that fails every time it starts, and say so everywhere
Release / build (push) Successful in 1m3s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m34s

On the live cluster kev-0.5b — an architecture the nodes' llama.cpp did not
know — was started 155 times in eight hours, failing the same way each time,
and phi-4, which fitted on no node, logged "no placement" 250 times in two
days. Neither showed up anywhere an operator looked: the backoff never gave
up, no metric said a model was failing or unplaceable, and the one alert for
a model with nothing serving could not fire, because its series was published
only while a replica was ready.

After five failures in a row a model is parked: not placed again until its
definition changes — a new source or profile — or `llmgw ctl retry` is run.
Status, the model list and the admin API show it with its last error. The
placement warning is logged when its reason changes or every ten minutes.

A request for a parked or unplaceable model is answered at once with why,
rather than "loading; retry shortly", which a client retries forever. Each
request still clears the previous placement result and triggers a fresh
attempt, so a model that fits once room is made is not refused on a stale
answer; the router waits for that attempt, not for its full activation
window.

Metrics now cover model health (desired, ready at 0 for what should run,
consecutive start failures, parked, unplaceable), engine installs per node,
leadership, and certificate expiry read from disk at each scrape, with alerts
for each. `ctl status` counts a model against what it should have now, so an
on-demand model that was asked for and fitted nowhere no longer reads as
satisfied.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 01:21:57 +02:00
1 parent 5e1b6ecea0
commit 37a80a9b44
17 files changed
+740 -24

No files matched your search

+30
View File
@@ -78,6 +78,7 @@ func (a *API) Register(mux *http.ServeMux) {
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
mux.HandleFunc("POST /api/v1/restart", a.restart)
mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel)
}
// --- responses ---
@@ -233,6 +234,18 @@ type ModelView struct {
Max int `json:"max_replicas"`
Ready int `json:"ready_replicas"`
Total int `json:"total_replicas"`
// Desired is how many it should have now: Min, or for an on-demand
// model one while it is being asked for and none otherwise.
Desired int `json:"desired_replicas"`
// Failures counts replicas that failed to start in a row.
Failures int `json:"failures,omitempty"`
// Parked is set once Failures reached the limit; the model is not retried
// until its definition changes or `ctl retry` is run. Reason is the last
// error.
Parked bool `json:"parked,omitempty"`
Reason string `json:"reason,omitempty"`
// Unplaceable is set when it is short and the last attempt found no node.
Unplaceable bool `json:"unplaceable,omitempty"`
Fallbacks []string `json:"fallbacks,omitempty"`
Profiles []string `json:"profiles"`
}
@@ -442,13 +455,20 @@ func (a *API) Models() []ModelView {
}
}
health := map[string]reconciler.ModelHealth{}
for _, h := range a.ctl.Reconciler().ModelHealth() {
health[h.Model] = h
}
out := make([]ModelView, 0)
for _, m := range a.ctl.Catalog().Models() {
h := health[m.Name]
v := ModelView{
Name: m.Name, Description: m.Description, Aliases: m.Aliases, Source: m.Source,
Min: m.Scaling.Min, Max: m.Scaling.Max,
Ready: ready[m.Name], Total: total[m.Name],
Fallbacks: m.Fallbacks,
Desired: h.Desired, Failures: h.Failures,
Parked: h.Parked, Reason: h.Reason, Unplaceable: h.Unplaceable,
}
for _, p := range m.Profiles {
v.Profiles = append(v.Profiles, p.Name)
@@ -1085,3 +1105,13 @@ func (a *API) restart(w http.ResponseWriter, r *http.Request) {
}
writeJSON(w, http.StatusAccepted, RestartResult{Queued: ids})
}
func (a *API) retryModel(w http.ResponseWriter, r *http.Request) {
name := r.PathValue("name")
if _, ok := a.ctl.Catalog().Resolve(name); !ok {
writeError(w, http.StatusNotFound, "no model called "+name)
return
}
was := a.ctl.Reconciler().Retry(name)
writeJSON(w, http.StatusOK, map[string]bool{"was_parked": was})
}
+18 -2
View File
@@ -25,6 +25,9 @@ type Summary struct {
// Unplaceable names models the scheduler currently cannot satisfy. This is
// the line that turns "something is wrong" into a place to look.
Unplaceable []string `json:"unplaceable,omitempty"`
// Parked are models that failed every time they started and are no
// longer retried, with the last error.
Parked []ParkedModel `json:"parked,omitempty"`
// IdleReplicas are serving replicas nothing has asked for in a while, which
// on a fleet with more models than GPUs is the first place to find room.
IdleReplicas []IdleReplica `json:"idle_replicas,omitempty"`
@@ -201,17 +204,30 @@ func (a *API) Summary() Summary {
for _, m := range a.Models() {
s.Models.Total++
if m.Ready >= m.Min {
if m.Parked {
s.Parked = append(s.Parked, ParkedModel{Model: m.Name, Failures: m.Failures, Reason: m.Reason})
}
// Against what it should have now, not its minimum: an on-demand
// model's minimum is 0, so one that was asked for and fitted nowhere
// used to count as satisfied.
if m.Ready >= m.Desired && !m.Parked {
s.Models.Satisfied++
continue
}
s.Models.Short++
// Short of replicas and unable to place more is the actionable case;
// short and still starting is just a cluster doing its job.
if m.Total <= m.Ready {
if m.Unplaceable || (m.Total <= m.Ready && !m.Parked) {
s.Unplaceable = append(s.Unplaceable, m.Name)
}
}
sort.Strings(s.Unplaceable)
return s
}
// ParkedModel is a model no longer retried, and why.
type ParkedModel struct {
Model string `json:"model"`
Failures int `json:"failures"`
Reason string `json:"reason"`
}
+3
View File
@@ -171,6 +171,9 @@ func (c *Controller) Nodes() []inventory.Node { return c.inv.Nodes() }
func (c *Controller) FreeGPUs(nodeID string) []int { return c.inv.FreeGPUs(nodeID) }
func (c *Controller) Replicas() []reconciler.Replica { return c.rec.Replicas() }
// ModelHealth is how each model in the catalog stands.
func (c *Controller) ModelHealth() []reconciler.ModelHealth { return c.rec.ModelHealth() }
// Reconciler exposes the reconciliation loop, for status reports and ctl.
func (c *Controller) Reconciler() *reconciler.Reconciler { return c.rec }
+76 -4
View File
@@ -39,9 +39,25 @@ type Source interface {
// derived from state that already exists, so computing them when asked is both
// simpler and fresher than maintaining a parallel copy that can drift. Nothing
// runs between scrapes.
// HealthSource adds what the reconciler knows about each model and where
// engine installs stand, and whether this control plane is the one placing
// work. Asked for by type assertion, so a source without it — a test's —
// publishes what it did before.
type HealthSource interface {
ModelHealth() []reconciler.ModelHealth
IsLeader() bool
}
type Collector struct {
src Source
modelDesired *prometheus.Desc
modelFailures *prometheus.Desc
modelParked *prometheus.Desc
modelUnplaceable *prometheus.Desc
engineInstall *prometheus.Desc
leader *prometheus.Desc
nodeUp *prometheus.Desc
nodeGPUs *prometheus.Desc
nodeGPUsFree *prometheus.Desc
@@ -137,7 +153,26 @@ func New(src Source) *Collector {
replicaState: prometheus.NewDesc("llmgw_replicas",
"Replicas by model and state.", []string{"model", "state"}, nil),
replicaReady: prometheus.NewDesc("llmgw_model_replicas_ready",
"Replicas of a model that are serving.", []string{"model"}, nil),
"Replicas of a model that are serving. Published, as 0 when none are, for every "+
"model that should be running; a stopped model, or an on-demand one nobody is "+
"asking for, publishes nothing.", []string{"model"}, nil),
modelDesired: prometheus.NewDesc("llmgw_model_replicas_desired",
"Replicas a model should have now: its minimum, or for an on-demand model 1 while "+
"it is being asked for.", []string{"model"}, nil),
modelFailures: prometheus.NewDesc("llmgw_model_start_failures",
"Replicas of a model that failed to start in a row. Resets when one comes up.",
[]string{"model"}, nil),
modelParked: prometheus.NewDesc("llmgw_model_parked",
"1 when a model failed so often in a row that it is no longer retried.",
[]string{"model"}, nil),
modelUnplaceable: prometheus.NewDesc("llmgw_model_unplaceable",
"1 when a model is short of replicas and the last attempt found no node to place one on.",
[]string{"model"}, nil),
engineInstall: prometheus.NewDesc("llmgw_node_engine_install",
"Where installing an engine stands on a node: 1 for the current state.",
[]string{"node", "engine", "version", "state"}, nil),
leader: prometheus.NewDesc("llmgw_control_plane_leader",
"1 when this control plane is the one placing work, 0 when it is a standby.", nil, nil),
replicaIdle: prometheus.NewDesc("llmgw_replica_idle_seconds",
"Seconds since a replica was last sent a request. Absent when nothing has been reported, "+
"which is not the same as idle.", []string{"model", "replica"}, nil),
@@ -201,6 +236,8 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy,
c.gpuNVLink, c.gpuECC,
c.replicaState, c.replicaReady, c.replicaIdle,
c.modelDesired, c.modelFailures, c.modelParked, c.modelUnplaceable,
c.engineInstall, c.leader,
} {
ch <- d
}
@@ -387,8 +424,43 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
ch <- prometheus.MustNewConstMetric(c.replicaState, prometheus.GaugeValue,
float64(v), k[0], k[1])
}
for model, v := range ready {
ch <- prometheus.MustNewConstMetric(c.replicaReady, prometheus.GaugeValue,
float64(v), model)
hs, ok := c.src.(HealthSource)
if !ok {
for model, v := range ready {
ch <- prometheus.MustNewConstMetric(c.replicaReady, prometheus.GaugeValue,
float64(v), model)
}
return
}
for _, n := range c.src.Nodes() {
for _, p := range n.Provision {
ch <- prometheus.MustNewConstMetric(c.engineInstall, prometheus.GaugeValue, 1,
n.ID, p.Engine, p.Version, p.State)
}
}
lead := 0.0
if hs.IsLeader() {
lead = 1
}
ch <- prometheus.MustNewConstMetric(c.leader, prometheus.GaugeValue, lead)
for _, h := range hs.ModelHealth() {
// Ready is published at zero for what should be running — the one
// value ModelHasNothingServing exists to catch, and which, emitted
// only for replicas that were Ready, it never saw.
if h.Desired > 0 || ready[h.Model] > 0 {
ch <- prometheus.MustNewConstMetric(c.replicaReady, prometheus.GaugeValue,
float64(ready[h.Model]), h.Model)
}
ch <- prometheus.MustNewConstMetric(c.modelDesired, prometheus.GaugeValue, float64(h.Desired), h.Model)
ch <- prometheus.MustNewConstMetric(c.modelFailures, prometheus.GaugeValue, float64(h.Failures), h.Model)
ch <- prometheus.MustNewConstMetric(c.modelParked, prometheus.GaugeValue, b2f(h.Parked), h.Model)
ch <- prometheus.MustNewConstMetric(c.modelUnplaceable, prometheus.GaugeValue, b2f(h.Unplaceable), h.Model)
}
}
func b2f(b bool) float64 {
if b {
return 1
}
return 0
}
@@ -196,3 +196,35 @@ func TestEveryIBPortIsReportedWithItsCounters(t *testing.T) {
}
}
}
type healthSource struct {
fakeSource
health []reconciler.ModelHealth
}
func (h healthSource) ModelHealth() []reconciler.ModelHealth { return h.health }
func (h healthSource) IsLeader() bool { return true }
func TestAModelThatShouldRunPublishesZeroReplicasReady(t *testing.T) {
// ModelHasNothingServing alerts on llmgw_model_replicas_ready == 0, and
// the series was only ever published while a replica was ready — so the
// alert could not fire for the case it was written for.
src := healthSource{health: []reconciler.ModelHealth{
{Model: "wanted", Desired: 1, Failures: 5, Parked: true, Reason: "boom"},
{Model: "stopped", Desired: 0},
}}
out := gather(t, src)
for _, want := range []string{
"llmgw_model_replicas_ready model=wanted -> 0",
"llmgw_model_parked model=wanted -> 1",
"llmgw_model_start_failures model=wanted -> 5",
"llmgw_control_plane_leader -> 1",
} {
if !strings.Contains(out, want) {
t.Errorf("missing %s", want)
}
}
if strings.Contains(out, "llmgw_model_replicas_ready model=stopped") {
t.Error("a stopped model publishes a ready count, so the alert would fire for it")
}
}
+206 -14
View File
@@ -3,11 +3,15 @@ package reconciler
import (
"context"
crypterand "crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"log/slog"
"math/rand/v2"
"slices"
"strings"
"sync"
"time"
@@ -66,6 +70,15 @@ type Config struct {
// Without a bound, draining a node whose model cannot be placed anywhere
// else would never finish.
DrainGrace time.Duration
// ParkAfter is how many failures in a row stop a model being retried.
//
// Backoff alone never gives up: it grows to MaxBackoff and keeps trying,
// and a model whose weights its engine cannot load — an architecture the
// installed llama.cpp does not know — was started 155 times over eight
// hours on the live cluster, each time failing the same way, with nothing
// to show for it but the log. A parked model is not placed again until its
// definition changes or an operator runs `llmgw ctl retry`.
ParkAfter int
// PortRange is the per-node port range for engine HTTP and dist-init.
PortRange [2]int
// MgmtInterface is the ethernet interface used for NCCL bootstrap.
@@ -126,6 +139,7 @@ func DefaultConfig() Config {
EvictionGrace: 2 * time.Minute,
MinLifetime: 5 * time.Minute,
DrainGrace: 5 * time.Minute,
ParkAfter: 5,
PortRange: [2]int{20000, 20999},
MgmtInterface: "eth0",
}
@@ -165,6 +179,9 @@ func (c *Config) normalize() {
if c.DrainGrace <= 0 {
c.DrainGrace = d.DrainGrace
}
if c.ParkAfter <= 0 {
c.ParkAfter = d.ParkAfter
}
if c.PortRange[1] <= c.PortRange[0] {
c.PortRange = d.PortRange
}
@@ -209,6 +226,11 @@ type Reconciler struct {
// restarted and not yet begun. One is replaced at a time: replacing them
// all at once would need the memory of every replica twice over.
restartQueue map[string][]string
// parked holds models that failed ParkAfter times in a row.
parked map[string]parkRecord
// placementLogged remembers the last "no placement" said per model, so it
// is said when it changes rather than on every five-second pass.
placementLogged map[string]placementNote
wake chan struct{}
stop chan struct{}
@@ -230,19 +252,21 @@ func New(cfg Config, cat *catalog.Live, inv *inventory.Store, disp Dispatcher, l
}
return &Reconciler{
cfg: cfg, cat: cat, inv: inv, disp: disp, log: log,
sched: placement.New(inv),
now: time.Now,
replicas: map[string]*Replica{},
nextAttempt: map[string]time.Time{},
attempts: map[string]int{},
reservedPorts: map[int]struct{}{},
wanted: map[string]time.Time{},
restartQueue: map[string][]string{},
lastExplain: map[string]placement.Result{},
wake: make(chan struct{}, 1),
stop: make(chan struct{}),
done: make(chan struct{}),
startedAt: time.Now(),
sched: placement.New(inv),
now: time.Now,
replicas: map[string]*Replica{},
nextAttempt: map[string]time.Time{},
attempts: map[string]int{},
reservedPorts: map[int]struct{}{},
wanted: map[string]time.Time{},
restartQueue: map[string][]string{},
parked: map[string]parkRecord{},
placementLogged: map[string]placementNote{},
lastExplain: map[string]placement.Result{},
wake: make(chan struct{}, 1),
stop: make(chan struct{}),
done: make(chan struct{}),
startedAt: time.Now(),
}
}
@@ -464,6 +488,12 @@ func (r *Reconciler) Want(model string) bool {
}
r.mu.Lock()
r.wanted[m.Name] = r.now()
// A placement that failed before this request says nothing about now:
// forgotten, so the pass this triggers decides afresh and Unservable
// does not answer from a stale result.
if res, ok := r.lastExplain[m.Name]; ok && res.Placement == nil {
delete(r.lastExplain, m.Name)
}
r.mu.Unlock()
// Immediately, not on the next tick. Someone is waiting on this request,
// and five seconds of ticker is five seconds they spend staring at nothing.
@@ -669,6 +699,17 @@ func (r *Reconciler) reapFailed(ctx context.Context) {
// replica is about to disappear, and the backoff has to survive it.
r.attempts[rep.Model]++
r.nextAttempt[rep.Model] = r.now().Add(r.backoff(r.attempts[rep.Model]))
if n := r.attempts[rep.Model]; n >= r.cfg.ParkAfter {
if _, already := r.parked[rep.Model]; !already {
m, _ := r.cat.Load().Resolve(rep.Model)
r.parked[rep.Model] = parkRecord{
Since: r.now(), Reason: rep.Err, Failures: n, fp: modelFingerprint(m),
}
r.log.Error("model parked: it failed every time it started, and is not retried "+
"until its definition changes or `llmgw ctl retry` is run",
"model", rep.Model, "failures", n, "last_error", rep.Err)
}
}
}
}
r.mu.Unlock()
@@ -746,10 +787,22 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
}
}
blockedUntil := r.nextAttempt[m.Name]
park, isParked := r.parked[m.Name]
if isParked && park.fp != modelFingerprint(m) {
// Somebody changed the model — a new source, an edited profile — so
// the reason it kept failing may be gone. Worth one more streak.
delete(r.parked, m.Name)
r.attempts[m.Name] = 0
delete(r.nextAttempt, m.Name)
isParked, blockedUntil = false, time.Time{}
r.log.Info("model unparked: its definition changed", "model", m.Name)
}
r.mu.Unlock()
desired := r.desiredFor(m)
switch {
case len(liveIDs) < desired && isParked:
// Not placed. Its failure is on every status the operator reads.
case len(liveIDs) < desired:
if r.now().Before(blockedUntil) {
// Still in backoff. Rebuilding immediately would hot-loop on a
@@ -925,7 +978,7 @@ func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []strin
if m.Scaling.OnDemand() && r.makeRoomFor(ctx, m) {
return
}
r.log.Warn("no placement for model", "model", m.Name, "reasons", res.Explain())
r.notePlacementFailure(m.Name, res.Explain())
return
}
if err := r.startReplica(ctx, m, res.Placement); err != nil {
@@ -1523,3 +1576,142 @@ func (r *Reconciler) engineKey(replicaID string) string {
}
return r.cfg.EngineKey(replicaID)
}
// parkRecord is why a model stopped being retried.
type parkRecord struct {
Since time.Time
Reason string
Failures int
// fp is the model's definition when it was parked; a different one
// unparks it.
fp string
}
type placementNote struct {
text string
at time.Time
}
// modelFingerprint identifies a model's definition, so a change to it can be
// told from a retry of the same thing.
func modelFingerprint(m catalog.Model) string {
b, _ := json.Marshal(m)
sum := sha256.Sum256(b)
return hex.EncodeToString(sum[:8])
}
// placementLogEvery is how often an unchanged placement failure is repeated.
const placementLogEvery = 10 * time.Minute
// notePlacementFailure logs why a model could not be placed, when the reason
// is new or has not been said for a while. Said on every pass it was 250
// identical lines in two days for one model, burying everything else.
func (r *Reconciler) notePlacementFailure(model, why string) {
r.mu.Lock()
last := r.placementLogged[model]
say := last.text != why || r.now().Sub(last.at) >= placementLogEvery
if say {
r.placementLogged[model] = placementNote{text: why, at: r.now()}
}
r.mu.Unlock()
if say {
r.log.Warn("no placement for model", "model", model, "reasons", why)
}
}
// Retry clears a parked model and its backoff, so it is placed on the next
// pass.
func (r *Reconciler) Retry(model string) bool {
r.mu.Lock()
_, was := r.parked[model]
delete(r.parked, model)
r.attempts[model] = 0
delete(r.nextAttempt, model)
r.mu.Unlock()
r.Notify()
return was
}
// ModelHealth is how one model stands, for status and metrics.
type ModelHealth struct {
Model string
// Desired is how many replicas it should have now; 0 for a stopped model
// or an on-demand one nobody is asking for.
Desired int
Ready int
// Failures counts replicas that failed in a row.
Failures int
// Parked is set when Failures reached ParkAfter; Reason is the last error.
Parked bool
ParkedSince time.Time
Reason string
// Unplaceable is set when it is short of replicas and the last attempt to
// place one found nowhere; Placement says why.
Unplaceable bool
Placement string
}
// ModelHealth reports every model in the catalog.
func (r *Reconciler) ModelHealth() []ModelHealth {
models := r.cat.Load().Models()
desired := map[string]int{}
for _, m := range models {
desired[m.Name] = r.desiredFor(m)
}
r.mu.Lock()
defer r.mu.Unlock()
ready, live := map[string]int{}, map[string]int{}
for _, rep := range r.replicas {
if rep.State == Ready {
ready[rep.Model]++
}
if rep.State.Counts() {
live[rep.Model]++
}
}
out := make([]ModelHealth, 0, len(models))
for _, m := range models {
h := ModelHealth{Model: m.Name, Desired: desired[m.Name], Ready: ready[m.Name],
Failures: r.attempts[m.Name]}
if p, ok := r.parked[m.Name]; ok {
h.Parked, h.ParkedSince, h.Reason = true, p.Since, p.Reason
}
if res, ok := r.lastExplain[m.Name]; ok && res.Placement == nil && live[m.Name] < h.Desired {
h.Unplaceable, h.Placement = true, res.Explain()
}
out = append(out, h)
}
return out
}
// Unservable says why a model cannot be served at all, so a request for it is
// told now rather than asked to retry something that will never come up.
func (r *Reconciler) Unservable(model string) (string, bool) {
m, ok := r.cat.Load().Resolve(model)
if !ok {
return "", false
}
r.mu.Lock()
defer r.mu.Unlock()
if p, ok := r.parked[m.Name]; ok {
return fmt.Sprintf("%s failed to start %d times in a row and is not being retried: %s",
m.Name, p.Failures, p.Reason), true
}
for _, rep := range r.replicas {
if rep.Model == m.Name && rep.State.Counts() {
return "", false // something is on its way
}
}
if res, ok := r.lastExplain[m.Name]; ok && res.Placement == nil {
return m.Name + " cannot be placed on any node: " + firstLine(res.Explain()), true
}
return "", false
}
func firstLine(s string) string {
s = strings.TrimSpace(s)
if i := strings.IndexByte(s, '\n'); i >= 0 {
return strings.TrimSpace(s[:i]) + " (and others; see llmgw ctl explain-placement)"
}
return s
}
@@ -1,6 +1,7 @@
package reconciler
import (
"bytes"
"context"
"fmt"
"log/slog"
@@ -1836,3 +1837,98 @@ func TestReplicasOfAModelAreRestartedOneAtATime(t *testing.T) {
t.Error("an unknown replica was accepted")
}
}
// failRepeatedly starts and fails the model's replica n times over.
func failRepeatedly(t *testing.T, h *harness, n int, why string) {
t.Helper()
for i := 0; i < n; i++ {
h.disp.reset()
h.r.Reconcile(t.Context())
started := h.disp.startedShards()
if len(started) == 0 {
return // parked, or still in backoff
}
s := started[0]
h.r.OnShardStatus(s.ReplicaID, s.ShardID, supervisor.Failed, why)
h.r.Reconcile(t.Context())
h.advance(DefaultConfig().MaxBackoff + time.Second)
}
}
func TestAModelThatAlwaysFailsIsParkedNotRetriedForever(t *testing.T) {
// kev-0.5b: an architecture the node's llama.cpp did not know, started
// 155 times in eight hours, the same error every time.
h := newHarness(t, testModel(1), gpuNode("1", 8))
failRepeatedly(t, h, DefaultConfig().ParkAfter, "unknown model architecture: 'ggmlc'")
h.disp.reset()
h.advance(DefaultConfig().MaxBackoff + time.Second)
h.r.Reconcile(t.Context())
if n := len(h.disp.startedShards()); n != 0 {
t.Fatalf("a parked model was started again (%d shards)", n)
}
var mh ModelHealth
for _, x := range h.r.ModelHealth() {
if x.Model == "llama" {
mh = x
}
}
if !mh.Parked || !strings.Contains(mh.Reason, "ggmlc") || mh.Failures < DefaultConfig().ParkAfter {
t.Errorf("health %+v", mh)
}
if why, no := h.r.Unservable("llama"); !no || !strings.Contains(why, "ggmlc") {
t.Errorf("a request would not be told why: %q %v", why, no)
}
// Retried by the operator once fixed.
if !h.r.Retry("llama") {
t.Error("Retry did not report the model as parked")
}
h.r.Reconcile(t.Context())
if len(h.disp.startedShards()) == 0 {
t.Error("a retried model was not placed")
}
}
func TestChangingAParkedModelsDefinitionTriesItAgain(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
failRepeatedly(t, h, DefaultConfig().ParkAfter, "boom")
fixed := testModel(1)
fixed.Source = "/models/llama-fixed"
cat, err := catalog.New([]catalog.Model{fixed})
if err != nil {
t.Fatal(err)
}
h.r.cat.Store(cat)
h.disp.reset()
h.r.Reconcile(t.Context())
if len(h.disp.startedShards()) == 0 {
t.Error("a model whose definition changed stayed parked")
}
}
func TestAPlacementFailureIsLoggedWhenItChangesNotEveryPass(t *testing.T) {
var buf bytes.Buffer
h := newHarness(t, testModel(1), gpuNode("1", 1)) // tp4 needs 4 cards
h.r.log = slog.New(slog.NewTextHandler(&buf, nil))
for i := 0; i < 20; i++ {
h.r.Reconcile(t.Context())
h.advance(5 * time.Second)
}
if n := strings.Count(buf.String(), "no placement for model"); n != 1 {
t.Errorf("logged %d times over 20 passes", n)
}
h.advance(placementLogEvery)
h.r.Reconcile(t.Context())
if n := strings.Count(buf.String(), "no placement for model"); n != 2 {
t.Errorf("not repeated after %s: %d", placementLogEvery, n)
}
var mh ModelHealth
for _, x := range h.r.ModelHealth() {
mh = x
}
if !mh.Unplaceable || mh.Desired != 1 {
t.Errorf("health %+v", mh)
}
}