Fill in the memory a model needs, instead of leaving the field blank
CI / test (push) Successful in 1m41s
CI / test (push) Successful in 1m41s
Serving a model from the mirror page left "Needs RAM (GB)" optional and empty. Empty writes no constraint, and a CPU profile has no card to be short of, so every ready node became a candidate — and a 4.68 GB model went onto a 7.9 GB machine and left it 3.5 GB for a KV pool. Placement was not wrong; it was never told what the model needed. The mirror knows the size of the weights, so the field arrives filled in: weights plus half again for the KV pool, plus two for the engine. The half is there because the cache grows with the same things the weights do, layers and heads. It is an estimate and the field is still editable — what it is not is blank. A 1.12 GB model asks for 3 GB, so the small nodes stay usable. Pinning every model to the one large machine would be a different way of getting placement wrong. An unknown size fills in nothing rather than the floor of two, which would exclude no node while reading as though somebody had measured it. And when the estimate is larger than any node in the fleet, the page says so, with the largest node's memory. Better read there than discovered by a node being OOM-killed twenty minutes into a load. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
2acd794022
commit
e0ea4f6be9
3 files changed
+133
-8
No files matched your search
@@ -30,12 +30,15 @@ func (f *fakeCatalog) Declared(name string) bool { return name == f.declared }
|
||||
|
||||
// fleet reports a cluster with a given number of cards and engines.
|
||||
type fakeFleet struct {
|
||||
gpus int
|
||||
engines []string
|
||||
gpus int
|
||||
engines []string
|
||||
memoryGB int
|
||||
}
|
||||
|
||||
func (f fakeFleet) Nodes() []adminapi.NodeView {
|
||||
return []adminapi.NodeView{{ID: "n1", GPUTotal: f.gpus, Engines: f.engines}}
|
||||
return []adminapi.NodeView{
|
||||
{ID: "n1", GPUTotal: f.gpus, Engines: f.engines, MemoryGB: f.memoryGB},
|
||||
}
|
||||
}
|
||||
func (f fakeFleet) Models() []adminapi.ModelView { return nil }
|
||||
func (f fakeFleet) Replicas(string) []adminapi.ReplicaView { return nil }
|
||||
@@ -233,3 +236,62 @@ func TestAFleetOfOlderAgentsIsNotToldEverythingIsUnservable(t *testing.T) {
|
||||
t.Error("silence from older agents must not read as a refusal")
|
||||
}
|
||||
}
|
||||
|
||||
func sized(model, version, file string, gb float64) mirror.Status {
|
||||
n := int64(gb * (1 << 30))
|
||||
return mirror.Status{Model: model, Version: version, File: file,
|
||||
State: mirror.Present, BytesDone: n, BytesTotal: n}
|
||||
}
|
||||
|
||||
func TestTheMemoryFieldIsFilledInFromTheWeights(t *testing.T) {
|
||||
// The real case: 4.68 GB of weights went onto a 7.9 GB node and left it
|
||||
// 3.5 GB for a KV pool, because the field was optional and blank. Blank
|
||||
// writes no constraint, and a CPU profile has no card to be short of.
|
||||
body := mirrorGet(t, servePage(t, []mirror.Status{
|
||||
sized("coder", "v1", "m.gguf", 4.68)}, &fakeCatalog{}, 0, "llamacpp"),
|
||||
"/admin/mirror?serve=coder:v1").Body.String()
|
||||
|
||||
if !strings.Contains(body, `name="min_memory_gb" value="9"`) {
|
||||
t.Errorf("expected a figure derived from 4.68 GB of weights; got %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASmallModelDoesNotDemandALargeNode(t *testing.T) {
|
||||
// The estimate has to leave the small nodes usable, or every model ends up
|
||||
// pinned to the one big machine.
|
||||
body := mirrorGet(t, servePage(t, []mirror.Status{
|
||||
sized("small", "v1", "m.gguf", 1.12)}, &fakeCatalog{}, 0, "llamacpp"),
|
||||
"/admin/mirror?serve=small:v1").Body.String()
|
||||
|
||||
if !strings.Contains(body, `name="min_memory_gb" value="3"`) {
|
||||
t.Errorf("a 1.12 GB model should fit a small node; got %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModelTooBigForEveryNodeSaysSo(t *testing.T) {
|
||||
page := mirrorPageDeps(t, MirrorDeps{
|
||||
Mirror: &fakeMirror{models: []mirror.Status{sized("huge", "v1", "m.gguf", 40)}},
|
||||
Catalog: &fakeCatalog{},
|
||||
Fleet: fakeFleet{gpus: 0, engines: []string{"llamacpp"}, memoryGB: 31},
|
||||
}, true)
|
||||
body := mirrorGet(t, page, "/admin/mirror?serve=huge:v1").Body.String()
|
||||
|
||||
// Better said here than discovered by a node being OOM-killed twenty
|
||||
// minutes into a load.
|
||||
if !strings.Contains(body, "largest node has 31 GB") {
|
||||
t.Errorf("expected the size warning; got %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnEstimateIsNotMadeWhenTheSizeIsUnknown(t *testing.T) {
|
||||
// A version whose size nobody reported would otherwise be given a
|
||||
// requirement of two, which is a constraint that excludes nothing and
|
||||
// reads as though somebody measured it.
|
||||
body := mirrorGet(t, servePage(t, []mirror.Status{
|
||||
{Model: "m", Version: "v", File: "f.gguf", State: mirror.Present}},
|
||||
&fakeCatalog{}, 0, "llamacpp"), "/admin/mirror?serve=m:v").Body.String()
|
||||
|
||||
if !strings.Contains(body, `name="min_memory_gb" value="0"`) {
|
||||
t.Errorf("an unknown size should leave no figure; got %s", body)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user