Busy is not unfit: rank files by the hardware the cluster has, and let a model wait for its cards

Ranking what was free first made every file come out on the CPU on a cluster
whose one card was in use. The plan now ranks by hardware alone, free now
only breaking a tie; a model served onto cards that are busy is added and
waits for them, the job ending as waiting rather than failed.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 01:17:32 +02:00
1 parent c2be429554
commit 7cb9d4fa01
7 files changed
+118 -17

No files matched your search

+13 -2
View File
@@ -43,12 +43,23 @@ func TestEachFileIsMatchedToTheCardsThatHoldIt(t *testing.T) {
if !strings.Contains(got, "f16.gguf CPU") {
t.Errorf("f16 not offered the CPU: %s", got)
}
// The card busy: what is free comes first, which is the CPU.
// The card busy: still the card first, said to be busy, and the CPU,
// free, after it — for a fallback.
nodes[0].FreeGPUs = 0
opts = Options(arts, shape14B, nodes, 16384)
if !opts[0].CPU || !opts[0].FreeNow {
if opts[0].CPU || opts[0].FreeNow || opts[0].Artifact != "q8.gguf" {
t.Errorf("with the card busy: %+v", opts[0])
}
cpuAt := -1
for i, o := range opts {
if o.CPU && o.FreeNow {
cpuAt = i
break
}
}
if cpuAt < 0 {
t.Errorf("no free CPU way: %s", strings.Join(labels(opts), "; "))
}
}
func TestALlamaCppModelSpreadsOverANodesCardsByLayer(t *testing.T) {