Busy is not unfit: rank files by the hardware the cluster has, and let a model wait for its cards
CI / test (push) Successful in 4m43s
CI / test (push) Successful in 4m43s
Ranking what was free first made every file come out on the CPU on a cluster whose one card was in use. The plan now ranks by hardware alone, free now only breaking a tie; a model served onto cards that are busy is added and waits for them, the job ending as waiting rather than failed. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
c2be429554
commit
7cb9d4fa01
7 files changed
+118
-17
No files matched your search
@@ -342,3 +342,36 @@ func TestTheFileThatSuitsTheCardsIsChosenAndRunsOnThem(t *testing.T) {
|
||||
t.Errorf("%+v", j)
|
||||
}
|
||||
}
|
||||
|
||||
// busyFleet has the model's cards all in use: nothing placed, and placement
|
||||
// saying so.
|
||||
type busyFleet struct{ *fakeFleet }
|
||||
|
||||
func (busyFleet) Replicas() []reconciler.Replica { return nil }
|
||||
func (f busyFleet) ModelHealth() []reconciler.ModelHealth {
|
||||
return []reconciler.ModelHealth{{Model: "qwen2.5-1.5b-instruct", Unplaceable: true,
|
||||
Placement: "cuda01: 0 of 1 6000-24g free"}}
|
||||
}
|
||||
|
||||
func TestAModelThatFitsBusyCardsIsAddedToWaitNotRefused(t *testing.T) {
|
||||
files := []hf.File{{Path: "m-Q8_0.gguf", Bytes: 2 << 30, SHA256: "b"}}
|
||||
s, m, c, f := setup(t, files, map[string]string{"m-Q8_0.gguf": "b"})
|
||||
s.deps.Registry = fakeRegistry{repo: hf.Repo{ID: "o/m-GGUF", Revision: "abcdef123456", Files: files}, base: m.srv.URL}
|
||||
s.deps.Pool = func() []modelplan.PoolNode {
|
||||
return []modelplan.PoolNode{{ID: "g", Class: "6000-24g", GPUMemGB: 24, GPUs: 1, FreeGPUs: 0, HostMemGB: 8}}
|
||||
}
|
||||
s.deps.Fleet = busyFleet{f}
|
||||
s.deps.UnplacedGrace = 10 * time.Millisecond
|
||||
job, err := s.Start(Request{Repo: "o/m-GGUF", Name: "qwen2.5-1.5b-instruct", Replicas: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j := wait(t, s, job.ID)
|
||||
if j.State != Done || len(c.added) != 1 || c.added[0].Profiles[0].GPUClass != "6000-24g" {
|
||||
t.Fatalf("%s %+v", j.Error, j.Steps)
|
||||
}
|
||||
last := j.Steps[len(j.Steps)-1]
|
||||
if last.Name != "placement" || last.State != Skipped || !strings.Contains(last.Detail, "starts when there is room") {
|
||||
t.Errorf("%+v", last)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user