Busy is not unfit: rank files by the hardware the cluster has, and let a model wait for its cards
CI / test (push) Successful in 4m43s
CI / test (push) Successful in 4m43s
Ranking what was free first made every file come out on the CPU on a cluster whose one card was in use. The plan now ranks by hardware alone, free now only breaking a tie; a model served onto cards that are busy is added and waits for them, the job ending as waiting rather than failed. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
c2be429554
commit
7cb9d4fa01
7 files changed
+118
-17
No files matched your search
@@ -320,15 +320,19 @@ func quality(o Option) float64 {
|
||||
return limit - (o.BitsPerWeight-limit)*0.25
|
||||
}
|
||||
|
||||
// Rank orders options best first: what can start now; on a GPU before the
|
||||
// CPU; the fewest cards, then nodes; then the better precision, up to eight
|
||||
// bits; then the most room left.
|
||||
// Rank orders options best first by what the hardware is, not by what it is
|
||||
// doing: on a GPU before the CPU; the fewest cards, then nodes; then the
|
||||
// better precision, up to eight bits. Whether the cards are free now only
|
||||
// breaks a tie, then the most room left.
|
||||
//
|
||||
// Busy is not unfit. Ranking what was free first made every file of every
|
||||
// repository come out on the CPU on a cluster whose one card was in use —
|
||||
// which turned down the very file the card was there for. A model served onto
|
||||
// cards that are busy waits for one, or with its fallbacks runs elsewhere
|
||||
// until then.
|
||||
func Rank(opts []Option) {
|
||||
sort.SliceStable(opts, func(i, j int) bool {
|
||||
a, b := opts[i], opts[j]
|
||||
if a.FreeNow != b.FreeNow {
|
||||
return a.FreeNow
|
||||
}
|
||||
if a.CPU != b.CPU {
|
||||
return !a.CPU
|
||||
}
|
||||
@@ -341,6 +345,9 @@ func Rank(opts []Option) {
|
||||
if qa, qb := quality(a), quality(b); qa != qb {
|
||||
return qa > qb
|
||||
}
|
||||
if a.FreeNow != b.FreeNow {
|
||||
return a.FreeNow
|
||||
}
|
||||
return a.HaveGB-a.NeedGB > b.HaveGB-b.NeedGB
|
||||
})
|
||||
}
|
||||
|
||||
@@ -43,12 +43,23 @@ func TestEachFileIsMatchedToTheCardsThatHoldIt(t *testing.T) {
|
||||
if !strings.Contains(got, "f16.gguf CPU") {
|
||||
t.Errorf("f16 not offered the CPU: %s", got)
|
||||
}
|
||||
// The card busy: what is free comes first, which is the CPU.
|
||||
// The card busy: still the card first, said to be busy, and the CPU,
|
||||
// free, after it — for a fallback.
|
||||
nodes[0].FreeGPUs = 0
|
||||
opts = Options(arts, shape14B, nodes, 16384)
|
||||
if !opts[0].CPU || !opts[0].FreeNow {
|
||||
if opts[0].CPU || opts[0].FreeNow || opts[0].Artifact != "q8.gguf" {
|
||||
t.Errorf("with the card busy: %+v", opts[0])
|
||||
}
|
||||
cpuAt := -1
|
||||
for i, o := range opts {
|
||||
if o.CPU && o.FreeNow {
|
||||
cpuAt = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if cpuAt < 0 {
|
||||
t.Errorf("no free CPU way: %s", strings.Join(labels(opts), "; "))
|
||||
}
|
||||
}
|
||||
|
||||
func TestALlamaCppModelSpreadsOverANodesCardsByLayer(t *testing.T) {
|
||||
|
||||
Reference in new issue
Block a user