Busy is not unfit: rank files by the hardware the cluster has, and let a model wait for its cards
CI / test (push) Successful in 4m43s

Ranking what was free first made every file come out on the CPU on a cluster
whose one card was in use. The plan now ranks by hardware alone, free now
only breaking a tie; a model served onto cards that are busy is added and
waits for them, the job ending as waiting rather than failed.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 01:17:32 +02:00
1 parent c2be429554
commit 7cb9d4fa01
7 files changed
+118 -17

No files matched your search

@@ -342,3 +342,36 @@ func TestTheFileThatSuitsTheCardsIsChosenAndRunsOnThem(t *testing.T) {
t.Errorf("%+v", j)
}
}
// busyFleet has the model's cards all in use: nothing placed, and placement
// saying so.
type busyFleet struct{ *fakeFleet }
func (busyFleet) Replicas() []reconciler.Replica { return nil }
func (f busyFleet) ModelHealth() []reconciler.ModelHealth {
return []reconciler.ModelHealth{{Model: "qwen2.5-1.5b-instruct", Unplaceable: true,
Placement: "cuda01: 0 of 1 6000-24g free"}}
}
func TestAModelThatFitsBusyCardsIsAddedToWaitNotRefused(t *testing.T) {
files := []hf.File{{Path: "m-Q8_0.gguf", Bytes: 2 << 30, SHA256: "b"}}
s, m, c, f := setup(t, files, map[string]string{"m-Q8_0.gguf": "b"})
s.deps.Registry = fakeRegistry{repo: hf.Repo{ID: "o/m-GGUF", Revision: "abcdef123456", Files: files}, base: m.srv.URL}
s.deps.Pool = func() []modelplan.PoolNode {
return []modelplan.PoolNode{{ID: "g", Class: "6000-24g", GPUMemGB: 24, GPUs: 1, FreeGPUs: 0, HostMemGB: 8}}
}
s.deps.Fleet = busyFleet{f}
s.deps.UnplacedGrace = 10 * time.Millisecond
job, err := s.Start(Request{Repo: "o/m-GGUF", Name: "qwen2.5-1.5b-instruct", Replicas: 1})
if err != nil {
t.Fatal(err)
}
j := wait(t, s, job.ID)
if j.State != Done || len(c.added) != 1 || c.added[0].Profiles[0].GPUClass != "6000-24g" {
t.Fatalf("%s %+v", j.Error, j.Steps)
}
last := j.Steps[len(j.Steps)-1]
if last.Name != "placement" || last.State != Skipped || !strings.Contains(last.Detail, "starts when there is room") {
t.Errorf("%+v", last)
}
}