Busy is not unfit: rank files by the hardware the cluster has, and let a model wait for its cards

Ranking what was free first made every file come out on the CPU on a cluster
whose one card was in use. The plan now ranks by hardware alone, free now
only breaking a tie; a model served onto cards that are busy is added and
waits for them, the job ending as waiting rather than failed.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 01:17:32 +02:00
1 parent c2be429554
commit 7cb9d4fa01
7 files changed
+118 -17

No files matched your search

+45 -6
View File
@@ -82,8 +82,14 @@ type Deps struct {
// Pool is the fleet's ready nodes as a plan sees them: what decides
// which file suits the hardware, and how it is spread.
Pool func() []modelplan.PoolNode
// UnplacedGrace is how long a model may go unplaced before the job
// stops waiting for it: a minute unless set.
UnplacedGrace time.Duration
}
// errBusy is a model the cluster has the hardware for, all of it in use now.
var errBusy = errors.New("the hardware it fits is in use")
// Request is what to serve.
type Request struct {
// Repo is the registry repository, owner/name.
@@ -182,6 +188,9 @@ func New(d Deps) *Server {
if d.ReadyTimeout <= 0 {
d.ReadyTimeout = 30 * time.Minute
}
if d.UnplacedGrace <= 0 {
d.UnplacedGrace = time.Minute
}
if d.HTTP == nil {
d.HTTP = &http.Client{Timeout: 5 * time.Minute}
}
@@ -361,7 +370,7 @@ func (s *Server) run(j *Job) {
// vLLM reads the model's shape from its config.json, which came
// with the weights.
s.step(j, "check", Done, fmt.Sprintf("%.1f GB of weights for vLLM", gb(set.Bytes)))
s.addAndStart(ctx, j, repo, set, name, source, profilesFor(layout, req, 0, 0))
s.addAndStart(ctx, j, repo, set, name, source, profilesFor(layout, req, 0, 0), hardwareExists(layout))
return
}
s.step(j, "check", Running, "reading its header from the mirror")
@@ -391,7 +400,7 @@ func (s *Server) run(j *Job) {
cores = catalog.SuggestCores(nc.NodeCPUs(memGB))
}
}
s.addAndStart(ctx, j, repo, set, name, source, profilesFor(layout, req, memGB, cores))
s.addAndStart(ctx, j, repo, set, name, source, profilesFor(layout, req, memGB, cores), hardwareExists(layout))
}
// chooseLayout is where a model runs: what was asked, else the plan's best
@@ -418,6 +427,13 @@ func chooseLayout(req Request, plan Plan, planned bool, set hf.FileSet) ([]model
if o.Nodes > 1 && engine == "llamacpp" {
return nil, errors.New("llama.cpp does not span nodes; serve the original weights with vLLM for that")
}
// The plan's own reckoning of that way, if it has one: what it holds
// and on how many hosts.
for _, p := range opts {
if p.CPU == o.CPU && p.Class == o.Class && p.GPUsPerNode == o.GPUsPerNode && max(p.Nodes, 1) == o.Nodes {
o = p
}
}
out = append(out, o)
case len(opts) > 0:
out = append(out, opts[0])
@@ -477,7 +493,19 @@ func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []cat
// addAndStart puts the model in the catalog with its profile, waits for a
// replica and asks it to answer.
func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSet, name, source string, profs []catalog.Profile) {
// hardwareExists is whether the cluster has what the ways chosen need, busy
// or not: the plan counted hosts for it, or it was asked for by hand.
func hardwareExists(layout []modelplan.Option) bool {
for _, o := range layout {
if o.Hosts > 0 || (o.Class != "" && o.NeedGB == 0) {
return true
}
}
return false
}
func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSet, name, source string,
profs []catalog.Profile, hardware bool) {
req := j.Request
// 4. The catalog.
m := catalog.Model{
@@ -501,7 +529,14 @@ func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.F
s.step(j, "catalog", Done, fmt.Sprintf("%s added, %d replica(s)", name, req.Replicas))
// 5. A replica, ready.
rep, err := s.waitReady(ctx, j, name)
rep, err := s.waitReady(ctx, j, name, hardware)
if errors.Is(err, errBusy) {
// In the catalog, and the hardware for it exists: it starts when
// the cards are free, with nothing more for anyone to do.
s.step(j, "placement", Skipped, err.Error())
s.finish(j)
return
}
if err != nil {
s.fail(j, "placement", err)
return
@@ -664,7 +699,7 @@ func (s *Server) mirror(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSe
// waitReady waits for a replica of the model to serve, reporting what it is
// doing and giving up early when it cannot be placed or keeps failing.
func (s *Server) waitReady(ctx context.Context, j *Job, name string) (reconciler.Replica, error) {
func (s *Server) waitReady(ctx context.Context, j *Job, name string, hardware bool) (reconciler.Replica, error) {
s.step(j, "placement", Running, "waiting for the control plane to place it")
started := time.Now()
deadline := started.Add(s.deps.ReadyTimeout)
@@ -679,7 +714,11 @@ func (s *Server) waitReady(ctx context.Context, j *Job, name string) (reconciler
// A minute's grace from when this step began, not from the job:
// the download before it can take hours, and the first pass may
// not have run yet.
if h.Unplaceable && time.Since(started) > time.Minute {
if h.Unplaceable && time.Since(started) > s.deps.UnplacedGrace {
if hardware {
return reconciler.Replica{}, fmt.Errorf("%w: it is in the catalog and starts when there is room, "+
"with nothing more to do; what holds it now:\n%s", errBusy, h.Placement)
}
return reconciler.Replica{}, fmt.Errorf("no node can host it:\n%s", h.Placement)
}
}
@@ -342,3 +342,36 @@ func TestTheFileThatSuitsTheCardsIsChosenAndRunsOnThem(t *testing.T) {
t.Errorf("%+v", j)
}
}
// busyFleet has the model's cards all in use: nothing placed, and placement
// saying so.
type busyFleet struct{ *fakeFleet }
func (busyFleet) Replicas() []reconciler.Replica { return nil }
func (f busyFleet) ModelHealth() []reconciler.ModelHealth {
return []reconciler.ModelHealth{{Model: "qwen2.5-1.5b-instruct", Unplaceable: true,
Placement: "cuda01: 0 of 1 6000-24g free"}}
}
func TestAModelThatFitsBusyCardsIsAddedToWaitNotRefused(t *testing.T) {
files := []hf.File{{Path: "m-Q8_0.gguf", Bytes: 2 << 30, SHA256: "b"}}
s, m, c, f := setup(t, files, map[string]string{"m-Q8_0.gguf": "b"})
s.deps.Registry = fakeRegistry{repo: hf.Repo{ID: "o/m-GGUF", Revision: "abcdef123456", Files: files}, base: m.srv.URL}
s.deps.Pool = func() []modelplan.PoolNode {
return []modelplan.PoolNode{{ID: "g", Class: "6000-24g", GPUMemGB: 24, GPUs: 1, FreeGPUs: 0, HostMemGB: 8}}
}
s.deps.Fleet = busyFleet{f}
s.deps.UnplacedGrace = 10 * time.Millisecond
job, err := s.Start(Request{Repo: "o/m-GGUF", Name: "qwen2.5-1.5b-instruct", Replicas: 1})
if err != nil {
t.Fatal(err)
}
j := wait(t, s, job.ID)
if j.State != Done || len(c.added) != 1 || c.added[0].Profiles[0].GPUClass != "6000-24g" {
t.Fatalf("%s %+v", j.Error, j.Steps)
}
last := j.Steps[len(j.Steps)-1]
if last.Name != "placement" || last.State != Skipped || !strings.Contains(last.Detail, "starts when there is room") {
t.Errorf("%+v", last)
}
}