diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f593e2..9792204 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,38 @@ holds of a node is reserved, not assumed. smallest node the model fits on — beside the memory they already estimate. Running replicas keep their old figures until restarted. +The commands are `llmgw ctl model resources` and `llmgw ctl model class`; in +rc.7 the first was briefly `llmgw ctl resources`. + +### Capacity classes, and preemption + +A model has a class — `critical`, `standard` or `opportunistic` — saying what +its capacity is worth against another's when there is not enough for both. +Unset, an on-demand model is opportunistic and every other model standard, +which is how the catalog behaved before. `llmgw ctl model class MODEL CLASS` +sets it. + +When a model cannot be placed, replicas are stopped to make room for it, by +these rules: + +- A lower class gives way to a higher one. +- Within a class, only replicas above a model's minimum — room it grew into + under load — and only for a model still below its own minimum. +- A critical model's minimum is never taken, nor a replica that has only just + loaded, nor one spanning several nodes. +- Idle on-demand replicas go first, then what lower classes grew into, then + the same within the class, then lower classes' minimums; the longest unused + first within each. +- The set is tried on a copy of the inventory first, at most three replicas, + and nothing is stopped unless it makes the room. + +A replica stopped this way is drained and given 30 seconds to finish what it +is serving. Its model is held back for up to five minutes, or until the model +it gave way to has its replicas, so it does not take the room straight back. +`ctl models` shows each model's class and "gave room to X" while held; the +cluster page shows the class and how many replicas were lost; the metric is +`llmgw_model_preempted_total{model,class}`, and every preemption is logged. + ### The mirror page in tabs /admin/mirror was one long page: what the mirror holds, the Hugging Face diff --git a/deploy/monitoring/llmgw-rules.yml b/deploy/monitoring/llmgw-rules.yml index fc022b5..e28dc53 100644 --- a/deploy/monitoring/llmgw-rules.yml +++ b/deploy/monitoring/llmgw-rules.yml @@ -84,6 +84,17 @@ groups: ` says what each node lacks. For an on-demand model, requests for it are told this rather than asked to retry. + - alert: ModelPreempted + expr: increase(llmgw_model_preempted_total[15m]) > 0 + labels: { severity: info } + annotations: + summary: "{{ $labels.model }} ({{ $labels.class }}) gave up replicas to a higher class" + description: >- + Capacity ran short and a model of a higher class took this one's + room. Expected now and then; if it keeps firing for the same model, + the cluster is too small for what is asked of it at these classes. + `llmgw ctl models` shows who it gave way to. + - alert: EngineInstallFailed expr: llmgw_node_engine_install{state="failed"} == 1 for: 15m diff --git a/docs/reference.md b/docs/reference.md index a1199f3..43b93d6 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -92,6 +92,7 @@ rather than repeated. | `source` | `string` | [Model](#model) | | `profiles` | `[]Profile` | [Model](#model) | | `scaling` | `Scaling` | [Model](#model) | +| `class` | `string` | [Model](#model) | | `fallbacks` | `[]string` | [Model](#model) | | `routing_profile` | `string` | [Model](#model) | | `slo` | `SLO` | [Model](#model) | @@ -736,6 +737,15 @@ Source is the weights path or a HuggingFace id. Takes a [Scaling](#scaling) block. +### `class` + +`string` + +Class is how much this model's capacity is worth against another's, +when there is not enough for both: critical, standard or opportunistic. +Empty is standard, or opportunistic for a model loaded on demand. See +EffectiveClass. + ### `fallbacks` `[]string` diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index aadc972..cec53ce 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -139,7 +139,6 @@ func newCtlCmd() *cobra.Command { newStatusCmd(client, &asJSON), newModelCmd(client), newScaleCmd(client), - newResourcesCmd(client), newRestartCmd(client), newRetryCmd(client), newUpgradeCmd(client), @@ -254,6 +253,8 @@ func modelStatus(m adminapi.ModelView) string { switch { case m.Parked: return "parked" + case m.HeldFor != "": + return "gave room to " + m.HeldFor case m.Unplaceable: return "cannot place" case m.Min == 0 && m.Total == 0: @@ -324,15 +325,15 @@ func newModelsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { } w := tab() - fmt.Fprintln(w, "MODEL\tSERVING\tREPLICAS\tSTATUS\tRESERVES\tALIASES\tPROFILES\tFALLBACKS") + fmt.Fprintln(w, "MODEL\tSERVING\tREPLICAS\tSTATUS\tCLASS\tRESERVES\tALIASES\tPROFILES\tFALLBACKS") var warnings []string for _, m := range models { serving := "no" if m.Ready > 0 { serving = "yes" } - fmt.Fprintf(w, "%s\t%s\t%d/%d\t%s\t%s\t%s\t%s\t%s\n", - m.Name, serving, m.Ready, m.Min, modelStatus(m), reserves(m), + fmt.Fprintf(w, "%s\t%s\t%d/%d\t%s\t%s\t%s\t%s\t%s\t%s\n", + m.Name, serving, m.Ready, m.Min, modelStatus(m), m.Class, reserves(m), join(m.Aliases), join(m.Profiles), join(m.Fallbacks)) for _, x := range m.Warnings { warnings = append(warnings, m.Name+": "+x) diff --git a/internal/cli/model.go b/internal/cli/model.go index fad3dbb..4abcff4 100644 --- a/internal/cli/model.go +++ b/internal/cli/model.go @@ -30,6 +30,7 @@ func newModelCmd(client func() *ctlClient) *cobra.Command { Short: "Add or remove models while the cluster is running", } c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client), + newResourcesCmd(client), newClassCmd(client), newModelServeCmd(client)) return c } @@ -233,3 +234,32 @@ func newResourcesCmd(client func() *ctlClient) *cobra.Command { c.Flags().StringVar(&profile, "profile", "", "one profile; every profile when left out") return c } + +func newClassCmd(client func() *ctlClient) *cobra.Command { + return &cobra.Command{ + Use: "class critical|standard|opportunistic|default", + Short: "Set how much a model's capacity is worth against another's", + Long: "When a model cannot be placed, replicas of a lower class are stopped to\n" + + "make room for it:\n\n" + + " critical never gives up the replicas of its minimum\n" + + " standard gives way to critical\n" + + " opportunistic gives way to both; the default for an on-demand model\n\n" + + "Within a class, a model below its minimum may take what another grew into\n" + + "under load, but not its minimum. \"default\" goes back to what the model's\n" + + "scaling implies: opportunistic when on demand, standard otherwise.", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + class := args[1] + if class == "default" { + class = "" + } + body, _ := json.Marshal(map[string]string{"class": class}) + var res map[string]string + if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/class", body, &res); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "%s is %s\n", res["model"], res["class"]) + return nil + }, + } +} diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index ccc1ab5..4ad938f 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -58,6 +58,25 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("DELETE /api/v1/models/{name}", a.removeModel) mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel) mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources) + mux.HandleFunc("POST /api/v1/models/{name}/class", func(w http.ResponseWriter, r *http.Request) { + if a.models == nil { + writeError(w, http.StatusNotImplemented, "this control plane has no store") + return + } + var req struct { + Class string `json:"class"` + } + if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error()) + return + } + m, err := a.models.SetClass(r.Context(), r.PathValue("name"), req.Class, actorOf(r)) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + writeJSON(w, http.StatusOK, map[string]string{"model": m.Name, "class": m.EffectiveClass()}) + }) mux.HandleFunc("PUT /api/v1/models/{name}/capacity", a.putCapacity) mux.HandleFunc("GET /api/v1/capacity", a.listCapacity) mux.HandleFunc("GET /api/v1/usage", a.usageByScope) @@ -282,6 +301,11 @@ type ModelView struct { // Warnings are what makes its reservation wrong: nothing declared where // placement needs a figure, or less declared than it uses. Warnings []string `json:"warnings,omitempty"` + // Class is its capacity class; Preempted the replicas it lost to a higher + // class; HeldFor the model it is giving room to now, if any. + Class string `json:"class"` + Preempted int `json:"preempted,omitempty"` + HeldFor string `json:"held_for,omitempty"` } // ReplicaView is a replica as `ctl replicas` shows it. @@ -528,6 +552,7 @@ func (a *API) Models() []ModelView { Fallbacks: m.Fallbacks, Desired: h.Desired, Failures: h.Failures, Parked: h.Parked, Reason: h.Reason, Unplaceable: h.Unplaceable, + Class: m.EffectiveClass(), Preempted: h.Preempted, HeldFor: h.HeldFor, } for _, p := range m.Profiles { v.Profiles = append(v.Profiles, p.Name) @@ -608,6 +633,7 @@ type Models interface { Remove(ctx context.Context, name string) error Scale(ctx context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error) Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error) + SetClass(ctx context.Context, name, class, by string) (catalog.Model, error) } // ResourcesRequest sets what a model's profiles reserve of a node. diff --git a/internal/control/adminapi/api_test.go b/internal/control/adminapi/api_test.go index b8dea81..949be67 100644 --- a/internal/control/adminapi/api_test.go +++ b/internal/control/adminapi/api_test.go @@ -258,6 +258,10 @@ type fakeModels struct { err error } +func (f *fakeModels) SetClass(_ context.Context, name, class, _ string) (catalog.Model, error) { + return catalog.Model{Name: name, Class: class}, f.err +} + func (f *fakeModels) Resources(_ context.Context, name, profile string, mem, cores int, _ string) (catalog.Model, error) { if f.err != nil { return catalog.Model{}, f.err diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index 95ae8b0..a6de869 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -149,10 +149,15 @@ type Model struct { // choice by guesswork. Description string `json:"description,omitempty" koanf:"description"` // Source is the weights path or a HuggingFace id. - Source string `json:"source,omitempty" koanf:"source"` - Profiles []Profile `json:"profiles,omitempty" koanf:"profiles"` - Scaling Scaling `json:"scaling,omitempty" koanf:"scaling"` - Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"` + Source string `json:"source,omitempty" koanf:"source"` + Profiles []Profile `json:"profiles,omitempty" koanf:"profiles"` + Scaling Scaling `json:"scaling,omitempty" koanf:"scaling"` + // Class is how much this model's capacity is worth against another's, + // when there is not enough for both: critical, standard or opportunistic. + // Empty is standard, or opportunistic for a model loaded on demand. See + // EffectiveClass. + Class string `json:"class,omitempty" koanf:"class"` + Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"` // RoutingProfile selects how the router scores endpoints for this model. RoutingProfile string `json:"routing_profile,omitempty" koanf:"routing_profile"` SLO SLO `json:"slo,omitempty" koanf:"slo"` @@ -299,6 +304,9 @@ func (m Model) validate() error { if len(m.Profiles) == 0 { return fmt.Errorf("model %q: at least one profile is required", m.Name) } + if !validClass(m.Class) { + return fmt.Errorf("model %q: class %q is not one of critical, standard, opportunistic", m.Name, m.Class) + } if m.Scaling.Max < m.Scaling.Min { return fmt.Errorf("model %q: scaling max %d is below min %d", m.Name, m.Scaling.Max, m.Scaling.Min) } @@ -503,3 +511,40 @@ func SuggestCores(cpuCounts []int) int { } return max(smallest/2, 1) } + +// The capacity classes, lowest first. +const ( + ClassOpportunistic = "opportunistic" + ClassStandard = "standard" + ClassCritical = "critical" +) + +func validClass(c string) bool { + return c == "" || c == ClassOpportunistic || c == ClassStandard || c == ClassCritical +} + +// EffectiveClass is the class a model is treated as: what it says, or, when +// it says nothing, opportunistic for one loaded on demand — it holds capacity +// only while asked for, and has always been the first to give it up — and +// standard for everything else. +func (m Model) EffectiveClass() string { + if m.Class != "" { + return m.Class + } + if m.Scaling.OnDemand() { + return ClassOpportunistic + } + return ClassStandard +} + +// ClassRank orders classes: a higher rank may take capacity from a lower one. +func ClassRank(c string) int { + switch c { + case ClassCritical: + return 3 + case ClassStandard, "": + return 2 + default: + return 1 + } +} diff --git a/internal/control/fleetmetrics/fleetmetrics.go b/internal/control/fleetmetrics/fleetmetrics.go index ecaf5b6..602f9c9 100644 --- a/internal/control/fleetmetrics/fleetmetrics.go +++ b/internal/control/fleetmetrics/fleetmetrics.go @@ -55,6 +55,7 @@ type Collector struct { modelFailures *prometheus.Desc modelParked *prometheus.Desc modelUnplaceable *prometheus.Desc + modelPreempted *prometheus.Desc engineInstall *prometheus.Desc leader *prometheus.Desc @@ -168,6 +169,9 @@ func New(src Source) *Collector { modelUnplaceable: prometheus.NewDesc("llmgw_model_unplaceable", "1 when a model is short of replicas and the last attempt found no node to place one on.", []string{"model"}, nil), + modelPreempted: prometheus.NewDesc("llmgw_model_preempted_total", + "Replicas of a model stopped to make room for a model of a higher capacity class.", + []string{"model", "class"}, nil), engineInstall: prometheus.NewDesc("llmgw_node_engine_install", "Where installing an engine stands on a node: 1 for the current state.", []string{"node", "engine", "version", "state"}, nil), @@ -236,7 +240,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) { c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy, c.gpuNVLink, c.gpuECC, c.replicaState, c.replicaReady, c.replicaIdle, - c.modelDesired, c.modelFailures, c.modelParked, c.modelUnplaceable, + c.modelDesired, c.modelFailures, c.modelParked, c.modelUnplaceable, c.modelPreempted, c.engineInstall, c.leader, } { ch <- d @@ -455,6 +459,7 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric(c.modelFailures, prometheus.GaugeValue, float64(h.Failures), h.Model) ch <- prometheus.MustNewConstMetric(c.modelParked, prometheus.GaugeValue, b2f(h.Parked), h.Model) ch <- prometheus.MustNewConstMetric(c.modelUnplaceable, prometheus.GaugeValue, b2f(h.Unplaceable), h.Model) + ch <- prometheus.MustNewConstMetric(c.modelPreempted, prometheus.CounterValue, float64(h.Preempted), h.Model, h.Class) } } diff --git a/internal/control/modelsvc/modelsvc.go b/internal/control/modelsvc/modelsvc.go index dbdbf61..2778610 100644 --- a/internal/control/modelsvc/modelsvc.go +++ b/internal/control/modelsvc/modelsvc.go @@ -242,6 +242,19 @@ func (s *Service) Resources(ctx context.Context, name, profile string, memoryGB, return final, err } +// SetClass sets a model's capacity class: critical, standard or +// opportunistic, or empty to go back to what its scaling implies. +func (s *Service) SetClass(ctx context.Context, name, class, by string) (catalog.Model, error) { + final, err := s.update(ctx, name, by, "change its class", func(m *catalog.Model) error { + m.Class = class + return nil + }) + if err == nil { + s.log.Info("model class set", "model", name, "class", final.EffectiveClass(), "by", by) + } + return final, err +} + // update changes one model added at runtime and keeps the change. // // Built before anything is written, like Add: catalog.New is what applies the diff --git a/internal/control/reconciler/health.go b/internal/control/reconciler/health.go index 5cb145a..488fb5d 100644 --- a/internal/control/reconciler/health.go +++ b/internal/control/reconciler/health.go @@ -83,6 +83,12 @@ type ModelHealth struct { // place one found nowhere; Placement says why. Unplaceable bool Placement string + // Class is the capacity class it is treated as. Preempted counts the + // replicas it has lost to models of a higher class since the control + // plane started; HeldFor names the model it is giving room to now. + Class string + Preempted int + HeldFor string } // ModelHealth reports every model in the catalog. @@ -106,7 +112,10 @@ func (r *Reconciler) ModelHealth() []ModelHealth { out := make([]ModelHealth, 0, len(models)) for _, m := range models { h := ModelHealth{Model: m.Name, Desired: desired[m.Name], Ready: ready[m.Name], - Failures: r.attempts[m.Name]} + Failures: r.attempts[m.Name], Class: m.EffectiveClass(), Preempted: r.preempted[m.Name]} + if hd, ok := r.held[m.Name]; ok && r.now().Before(hd.Until) { + h.HeldFor = hd.For + } if p, ok := r.parked[m.Name]; ok { h.Parked, h.ParkedSince, h.Reason = true, p.Since, p.Reason } diff --git a/internal/control/reconciler/preempt.go b/internal/control/reconciler/preempt.go new file mode 100644 index 0000000..883855a --- /dev/null +++ b/internal/control/reconciler/preempt.go @@ -0,0 +1,249 @@ +package reconciler + +import ( + "context" + "slices" + "sort" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/placement" +) + +// hold keeps a model that gave up room from taking it back. +type hold struct { + Until time.Time + For string +} + +// maxVictims bounds how many replicas one placement may stop. Past this the +// model is better reported as not fitting than paid for with half a cluster. +const maxVictims = 3 + +// victim is a replica that could be stopped for another model. +type victim struct { + ID string + Model string + Class string + AboveMin bool + Tier int + Idle time.Duration +} + +// preemptFor stops replicas of lower classes when a model cannot otherwise be +// placed, and says whether it did. +// +// What may be taken from whom: +// +// - A lower class gives way to a higher one: opportunistic to standard, +// standard to critical. +// - Within a class, only a replica above its model's minimum, and only for +// a model still below its own. Room a model grew into under load is not a +// promise; its minimum is. +// - A critical model's minimum is never taken. +// +// Victims only ever come from a lower rank, or from above a minimum for a +// model below one, so two models cannot take turns stopping each other. +// +// The set is tried on a copy of the inventory first, and only stopped if it +// would let the model be placed: stopping replicas that do not make room is +// the one outcome worse than not preempting. +func (r *Reconciler) preemptFor(ctx context.Context, m catalog.Model, avoid []string) bool { + cands := r.victims(m) + if len(cands) == 0 { + return false + } + trial := r.inv.Clone() + sched := placement.New(trial) + var chosen []victim + for _, v := range cands { + if len(chosen) == maxVictims { + break + } + trial.ReleaseReplica(v.ID) + chosen = append(chosen, v) + res := sched.Place(m.PlacementProfiles(), avoid) + if res.Placement == nil && len(avoid) > 0 { + res = sched.Place(m.PlacementProfiles(), nil) + } + if res.Placement == nil { + continue + } + r.preempt(ctx, m, chosen) + return true + } + return false +} + +// preempt drains the chosen replicas and holds their models back. +func (r *Reconciler) preempt(ctx context.Context, m catalog.Model, chosen []victim) { + now := r.now() + var ids []string + r.mu.Lock() + for _, v := range chosen { + if rep, ok := r.replicas[v.ID]; ok { + rep.PreemptedFor = m.Name + } + ids = append(ids, v.ID) + r.held[v.Model] = hold{Until: now.Add(r.cfg.PreemptHold), For: m.Name} + r.preempted[v.Model]++ + } + r.preempting[m.Name] = ids + r.mu.Unlock() + + for _, v := range chosen { + r.log.Warn("preempting a replica to make room", + "replica", v.ID, "model", v.Model, "class", v.Class, "above_min", v.AboveMin, + "for", m.Name, "for_class", m.EffectiveClass()) + } + r.scaleDown(ctx, ids) +} + +// awaitingPreemption reports whether replicas stopped for a model are still +// on their way out, so it waits for them rather than stopping more. +func (r *Reconciler) awaitingPreemption(model string) bool { + r.mu.Lock() + defer r.mu.Unlock() + ids := r.preempting[model] + ids = slices.DeleteFunc(slices.Clone(ids), func(id string) bool { + _, ok := r.replicas[id] + return !ok + }) + if len(ids) == 0 { + delete(r.preempting, model) + return false + } + r.preempting[model] = ids + return true +} + +// isHeld reports whether a model gave up room and may not take it back yet. +// The hold ends early once the model it was given to has what it asked for. +// +// Today a preempted replica is only removed by its own model's drain loop, +// which runs after that model has decided not to scale up, so the room it +// frees reaches the model it was freed for first anyway. The hold is what +// keeps that true for any other way a replica can go — a stop reported +// between passes, a reap at the start of one — rather than it resting on the +// order of lines in reconcileModel. +func (r *Reconciler) isHeld(model string) bool { + r.mu.Lock() + h, ok := r.held[model] + r.mu.Unlock() + if !ok { + return false + } + if r.now().After(h.Until) { + r.mu.Lock() + delete(r.held, model) + r.mu.Unlock() + return false + } + if to, ok := r.cat.Load().Resolve(h.For); ok && r.liveCount(h.For) >= r.desiredFor(to) { + r.mu.Lock() + delete(r.held, model) + r.mu.Unlock() + return false + } + return true +} + +// liveCount is how many replicas of a model hold capacity. +func (r *Reconciler) liveCount(model string) int { + r.mu.Lock() + defer r.mu.Unlock() + n := 0 + for _, rep := range r.replicas { + if rep.Model == model && rep.State.Counts() && rep.RetiringAt.IsZero() { + n++ + } + } + return n +} + +// victims lists the replicas that may be stopped for m, in the order they +// should be: idle on-demand ones first, then what lower classes grew into +// under load, then the same within m's class, then lower classes' minimums; +// within each, the longest unused first. +func (r *Reconciler) victims(m catalog.Model) []victim { + cat := r.cat.Load() + now := r.now() + myRank := catalog.ClassRank(m.EffectiveClass()) + belowMin := r.liveCount(m.Name) < m.Scaling.Min + + r.mu.Lock() + defer r.mu.Unlock() + + // Per model, its live replicas newest first: the newest are the ones + // above its minimum. + byModel := map[string][]*Replica{} + for _, rep := range r.replicas { + if rep.Model == m.Name || !rep.State.Counts() || !rep.RetiringAt.IsZero() || rep.PreemptedFor != "" { + continue + } + byModel[rep.Model] = append(byModel[rep.Model], rep) + } + + var out []victim + for model, reps := range byModel { + other, ok := cat.Resolve(model) + if !ok { + continue + } + class := other.EffectiveClass() + rank := catalog.ClassRank(class) + sort.Slice(reps, func(i, j int) bool { return reps[i].CreatedAt.After(reps[j].CreatedAt) }) + above := max(len(reps)-other.Scaling.Min, 0) + for i, rep := range reps { + aboveMin := i < above + tier := -1 + switch { + case rank < myRank && other.Scaling.OnDemand() && idleFor(rep, now) >= r.cfg.EvictionGrace: + tier = 0 + case rank < myRank && aboveMin: + tier = 1 + case rank == myRank && aboveMin && belowMin: + tier = 2 + case rank < myRank && class != catalog.ClassCritical: + tier = 3 + } + if tier < 0 { + continue + } + // Just started: stopping it throws away a model load that has + // only now been paid for. + if !rep.ReadyAt.IsZero() && now.Sub(rep.ReadyAt) < r.cfg.MinLifetime { + continue + } + if len(nodesOf(rep)) > 1 { + continue // see makeRoomFor: a multi-node replica is its own decision + } + out = append(out, victim{ID: rep.ID, Model: model, Class: class, + AboveMin: aboveMin, Tier: tier, Idle: idleFor(rep, now)}) + } + } + sort.SliceStable(out, func(i, j int) bool { + if out[i].Tier != out[j].Tier { + return out[i].Tier < out[j].Tier + } + return out[i].Idle > out[j].Idle + }) + return out +} + +// idleFor is how long a replica has gone unused; zero when nobody has said, +// which is not the same as idle. +func idleFor(rep *Replica, now time.Time) time.Duration { + if rep.LastUsedAt.IsZero() { + return 0 + } + return now.Sub(rep.LastUsedAt) +} + +func nodesOf(rep *Replica) map[string]struct{} { + out := map[string]struct{}{} + for _, sh := range rep.Shards { + out[sh.NodeID] = struct{}{} + } + return out +} diff --git a/internal/control/reconciler/preempt_test.go b/internal/control/reconciler/preempt_test.go new file mode 100644 index 0000000..87cf858 --- /dev/null +++ b/internal/control/reconciler/preempt_test.go @@ -0,0 +1,237 @@ +package reconciler + +import ( + "log/slog" + "strings" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" +) + +// newFleetHarness is newHarness with a catalog of several models. +func newFleetHarness(t *testing.T, models []catalog.Model, nodes ...inventory.Node) *harness { + t.Helper() + cat, err := catalog.New(models) + if err != nil { + t.Fatal(err) + } + inv := inventory.NewStore() + c := newClock() + for _, n := range nodes { + inv.Upsert(n) + } + disp := &fakeDispatcher{} + cfg := DefaultConfig() + cfg.UnreachableAfter = 15 * time.Second + cfg.LostAfter = 60 * time.Second + r := New(cfg, catalog.NewLive(cat), inv, disp, slog.New(slog.DiscardHandler)) + r.SetClock(c.now) + c.advance(cfg.SettleFor) + for _, n := range nodes { + touchAt(inv, n.ID, c.now()) + } + return &harness{r: r, inv: inv, disp: disp, clock: c} +} + +// gpuModel runs on one whole 8-GPU node. +func gpuModel(name, class string, min int) catalog.Model { + return catalog.Model{Name: name, Source: "/models/" + name, Class: class, + Scaling: catalog.Scaling{Min: min, Max: min}, + Profiles: []catalog.Profile{{Name: "tp8", Engine: "vllm", GPUClass: "h100-80g", TP: 8}}} +} + +// settle runs passes until nothing changes, marking what starts ready and +// letting preempted replicas run out their grace. +func (h *harness) settle(t *testing.T, passes int) { + t.Helper() + for range passes { + h.r.Reconcile(t.Context()) + h.markReady() + h.advance(h.r.cfg.PreemptGrace + time.Second) + } +} + +func replicasOf(h *harness, model string) (live int) { + for _, rep := range h.r.Replicas() { + if rep.Model == model && rep.State.Counts() { + live++ + } + } + return live +} + +// fill places the standard model, then ages it past MinLifetime so it may be +// taken from, and only then adds the model that wants its node. +func fillThenWant(t *testing.T, want catalog.Model, have catalog.Model, nodes ...inventory.Node) *harness { + t.Helper() + h := newFleetHarness(t, []catalog.Model{have, withMin(want, 0)}, nodes...) + h.settle(t, 3) + if got := replicasOf(h, have.Name); got != have.Scaling.Min { + t.Fatalf("%s has %d replicas before anything wanted its room", have.Name, got) + } + h.advance(h.r.cfg.MinLifetime + time.Minute) + cat, err := catalog.New([]catalog.Model{have, want}) + if err != nil { + t.Fatal(err) + } + h.r.cat.Store(cat) + return h +} + +func withMin(m catalog.Model, n int) catalog.Model { + m.Scaling = catalog.Scaling{Min: n, Max: n} + return m +} + +func TestACriticalModelTakesAStandardModelsNode(t *testing.T) { + h := fillThenWant(t, gpuModel("vital", catalog.ClassCritical, 1), gpuModel("chat", "", 1), gpuNode("1", 8)) + + h.r.Reconcile(t.Context()) + var draining int + for _, rep := range h.r.Replicas() { + if rep.Model == "chat" && rep.State == Draining && rep.PreemptedFor == "vital" { + draining++ + } + } + if draining != 1 { + t.Fatalf("the standard replica was not drained for the critical one: %+v", h.r.Replicas()) + } + if hs := health(h, "chat"); hs.HeldFor != "vital" || hs.Preempted != 1 { + t.Errorf("the victim's health does not say it gave way: %+v", hs) + } + + h.settle(t, 4) + if replicasOf(h, "vital") != 1 { + t.Fatalf("the critical model was not placed once room was made: %+v", h.r.Replicas()) + } + // Held back until the critical model had its replica, then short again + // because there is no room: it must not have taken the node back — not + // even for a moment, which would load a model only to preempt it again. + if replicasOf(h, "chat") != 0 { + t.Error("the standard model took the node back") + } + started := 0 + for _, cmd := range h.disp.startedShards() { + if strings.HasPrefix(cmd.ReplicaID, "chat-") { + started++ + } + } + if started != 1 { + t.Errorf("the standard model started %d replicas; it should not have tried again", started) + } +} + +func TestAStandardModelDoesNotTakeAnothersMinimum(t *testing.T) { + h := fillThenWant(t, gpuModel("second", "", 1), gpuModel("first", "", 1), gpuNode("1", 8)) + h.settle(t, 3) + if replicasOf(h, "first") != 1 || replicasOf(h, "second") != 0 { + t.Errorf("a model of the same class took another's minimum: %+v", h.r.Replicas()) + } + if hs := health(h, "second"); !hs.Unplaceable { + t.Error("the model left without room should say it cannot be placed") + } +} + +func TestWithinAClassOnlyWhatWasGrownIntoIsTaken(t *testing.T) { + grown := gpuModel("busy", "", 1) + grown.Scaling = catalog.Scaling{Min: 1, Max: 2} + h := newFleetHarness(t, []catalog.Model{grown, withMin(gpuModel("new", "", 1), 0)}, gpuNode("1", 8), gpuNode("2", 8)) + h.settle(t, 2) + // Load grows it to its max: the second replica is above its minimum. + h.r.mu.Lock() + h.r.extra["busy"] = 1 + h.r.mu.Unlock() + h.settle(t, 3) + if replicasOf(h, "busy") != 2 { + t.Fatalf("setup: busy did not grow to 2: %+v", h.r.Replicas()) + } + h.advance(h.r.cfg.MinLifetime + time.Minute) + cat, _ := catalog.New([]catalog.Model{grown, gpuModel("new", "", 1)}) + h.r.cat.Store(cat) + + h.settle(t, 5) + if replicasOf(h, "new") != 1 || replicasOf(h, "busy") != 1 { + t.Errorf("the grown replica should give way to a minimum: busy %d, new %d", + replicasOf(h, "busy"), replicasOf(h, "new")) + } +} + +func TestNothingIsStoppedThatWouldNotMakeRoom(t *testing.T) { + // Two nodes, each half used by an opportunistic model; the critical one + // needs a whole node. Stopping one frees half a node, which is no use — + // and both together are on two nodes, which is no use either. + half := catalog.Model{Name: "half", Source: "/m", Class: catalog.ClassOpportunistic, + Scaling: catalog.Scaling{Min: 2, Max: 2}, + Profiles: []catalog.Profile{{Name: "tp4", Engine: "vllm", GPUClass: "h100-80g", TP: 4}}} + five := func(id string) inventory.Node { return gpuNode(id, 5) } + h := fillThenWant(t, gpuModel("vital", catalog.ClassCritical, 1), half, five("1"), five("2")) + h.r.Reconcile(t.Context()) + for _, rep := range h.r.Replicas() { + if rep.PreemptedFor != "" || rep.State == Draining { + t.Fatalf("a replica was stopped without making room: %+v", rep) + } + } +} + +func TestACriticalMinimumIsNeverTaken(t *testing.T) { + h := fillThenWant(t, gpuModel("also-vital", catalog.ClassCritical, 1), gpuModel("vital", catalog.ClassCritical, 1), gpuNode("1", 8)) + h.settle(t, 3) + if replicasOf(h, "vital") != 1 { + t.Error("a critical model's minimum was taken") + } +} + +func TestAReplicaThatJustStartedIsLeftAlone(t *testing.T) { + h := newFleetHarness(t, []catalog.Model{gpuModel("chat", "", 1), withMin(gpuModel("vital", catalog.ClassCritical, 1), 0)}, gpuNode("1", 8)) + h.settle(t, 2) + cat, _ := catalog.New([]catalog.Model{gpuModel("chat", "", 1), gpuModel("vital", catalog.ClassCritical, 1)}) + h.r.cat.Store(cat) + h.r.Reconcile(t.Context()) + for _, rep := range h.r.Replicas() { + if rep.State == Draining { + t.Fatal("a replica that had only just loaded was preempted") + } + } +} + +func health(h *harness, model string) ModelHealth { + for _, x := range h.r.ModelHealth() { + if x.Model == model { + return x + } + } + return ModelHealth{} +} + +func TestAHoldLastsUntilTheRoomIsUsedOrItExpires(t *testing.T) { + h := fillThenWant(t, gpuModel("vital", catalog.ClassCritical, 1), gpuModel("chat", "", 1), gpuNode("1", 8)) + h.r.Reconcile(t.Context()) + if !h.r.isHeld("chat") { + t.Fatal("the model that gave way is not held") + } + if h.r.isHeld("vital") { + t.Error("the model room was made for is held") + } + // Released as soon as the model it was given to has what it asked for. + h.settle(t, 3) + if replicasOf(h, "vital") != 1 { + t.Fatalf("setup: vital was not placed: %+v", h.r.Replicas()) + } + if h.r.isHeld("chat") { + t.Error("the hold outlasted its purpose") + } + + // And, whatever happens, after PreemptHold. + h.r.mu.Lock() + h.r.held["other"] = hold{Until: h.r.now().Add(time.Minute), For: "never-placed"} + h.r.mu.Unlock() + if !h.r.isHeld("other") { + t.Fatal("a fresh hold is not in force") + } + h.advance(2 * time.Minute) + if h.r.isHeld("other") { + t.Error("a hold outlived its time") + } +} diff --git a/internal/control/reconciler/reconciler.go b/internal/control/reconciler/reconciler.go index 8572f08..3cfb083 100644 --- a/internal/control/reconciler/reconciler.go +++ b/internal/control/reconciler/reconciler.go @@ -65,6 +65,12 @@ type Config struct { // Without a bound, draining a node whose model cannot be placed anywhere // else would never finish. DrainGrace time.Duration + // PreemptGrace is how long a replica stopped to make room for another + // is given to finish what it is serving: a request, not a replacement. + PreemptGrace time.Duration + // PreemptHold is how long a model that gave up room may not take it back, + // so the replica it was given up for is placed first. + PreemptHold time.Duration // ScaleUpWaiting is the average number of requests waiting per ready // replica, held for ScaleUpAfter, that adds a replica — up to the model's // max. ScaleDownAfter is how long with nothing waiting before one of the @@ -141,6 +147,8 @@ func DefaultConfig() Config { EvictionGrace: 2 * time.Minute, MinLifetime: 5 * time.Minute, DrainGrace: 5 * time.Minute, + PreemptGrace: 30 * time.Second, + PreemptHold: 5 * time.Minute, ParkAfter: 5, ScaleUpWaiting: 1, ScaleUpAfter: time.Minute, @@ -158,6 +166,12 @@ func (c *Config) normalize() { if c.BaseBackoff <= 0 { c.BaseBackoff = d.BaseBackoff } + if c.PreemptGrace <= 0 { + c.PreemptGrace = d.PreemptGrace + } + if c.PreemptHold <= 0 { + c.PreemptHold = d.PreemptHold + } if c.MaxBackoff < c.BaseBackoff { c.MaxBackoff = max(c.BaseBackoff, d.MaxBackoff) } @@ -250,6 +264,12 @@ type Reconciler struct { // placementLogged remembers the last "no placement" said per model, so it // is said when it changes rather than on every five-second pass. placementLogged map[string]placementNote + // preempting is, per model, the replicas stopped to make room for it and + // not gone yet; held is, per model that gave them up, until when it may + // not take the room back; preempted counts, per model, replicas it lost. + preempting map[string][]string + held map[string]hold + preempted map[string]int wake chan struct{} stop chan struct{} @@ -285,6 +305,9 @@ func New(cfg Config, cat *catalog.Live, inv *inventory.Store, disp Dispatcher, l quietSince: map[string]time.Time{}, placementLogged: map[string]placementNote{}, lastExplain: map[string]placement.Result{}, + preempting: map[string][]string{}, + held: map[string]hold{}, + preempted: map[string]int{}, wake: make(chan struct{}, 1), stop: make(chan struct{}), done: make(chan struct{}), @@ -704,6 +727,9 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) { switch { case len(liveIDs) < desired && isParked: // Not placed. Its failure is on every status the operator reads. + case len(liveIDs) < desired && r.isHeld(m.Name): + // It gave up room to a model of a higher class, which is placed + // first; taking it back now would undo the preemption. case len(liveIDs) < desired: if r.now().Before(blockedUntil) { // Still in backoff. Rebuilding immediately would hot-loop on a @@ -725,7 +751,11 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) { rep, ok := r.replicas[id] var expired bool if ok { - expired = !rep.DrainingAt.IsZero() && r.now().Sub(rep.DrainingAt) > r.cfg.DrainGrace + grace := r.cfg.DrainGrace + if rep.PreemptedFor != "" { + grace = r.cfg.PreemptGrace + } + expired = !rep.DrainingAt.IsZero() && r.now().Sub(rep.DrainingAt) > grace } r.mu.Unlock() @@ -775,6 +805,9 @@ func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []strin if m.Scaling.OnDemand() && r.makeRoomFor(ctx, m) { return } + if r.awaitingPreemption(m.Name) || r.preemptFor(ctx, m, avoid) { + return + } r.notePlacementFailure(m.Name, res.Explain()) return } diff --git a/internal/control/reconciler/replica.go b/internal/control/reconciler/replica.go index da9fcdc..36f341e 100644 --- a/internal/control/reconciler/replica.go +++ b/internal/control/reconciler/replica.go @@ -156,6 +156,8 @@ type Replica struct { // DrainingAt is when the replica stopped taking new work, so the wait for a // replacement can be bounded. DrainingAt time.Time + // PreemptedFor is the model this replica was stopped to make room for. + PreemptedFor string // RetiringAt is set while the replica is being restarted: it keeps serving // but no longer counts toward the model's replicas, so a replacement is // placed beside it, and it is drained only once that replacement is ready. diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html index 026a9a6..e3183b5 100644 --- a/internal/web/templates/dashboard.html +++ b/internal/web/templates/dashboard.html @@ -80,12 +80,13 @@

Models

{{if .Models}} -{{if .AnyFallbacks}}{{end}} +
ModelAliasesReadyWantedReservesProfilesFallbacksWhat it is for
{{if .AnyFallbacks}}{{end}} {{range .Models}} + {{.Class}}{{if .Preempted}} −{{.Preempted}}{{end}} {{if or .MemoryGB .CPUCores}}{{.MemoryGB}}G{{if .CPUCores}} · {{.CPUCores}}c{{end}}{{else}}—{{end}} {{- if .MeasuredMemoryGB}} {{printf "%.1f" .MeasuredMemoryGB}}G used{{end}}{{if .Warnings}} ⚠{{end}}
ModelAliasesReadyWantedClassReservesProfilesFallbacksWhat it is for
{{.Name}} {{join .Aliases}} {{.Ready}} {{.Min}}–{{.Max}}{{join .Profiles}}