Fill in the memory a model needs, instead of leaving the field blank

Serving a model from the mirror page left "Needs RAM (GB)" optional and empty.
Empty writes no constraint, and a CPU profile has no card to be short of, so
every ready node became a candidate — and a 4.68 GB model went onto a 7.9 GB
machine and left it 3.5 GB for a KV pool. Placement was not wrong; it was never
told what the model needed.

The mirror knows the size of the weights, so the field arrives filled in:
weights plus half again for the KV pool, plus two for the engine. The half is
there because the cache grows with the same things the weights do, layers and
heads. It is an estimate and the field is still editable — what it is not is
blank.

A 1.12 GB model asks for 3 GB, so the small nodes stay usable. Pinning every
model to the one large machine would be a different way of getting placement
wrong.

An unknown size fills in nothing rather than the floor of two, which would
exclude no node while reading as though somebody had measured it.

And when the estimate is larger than any node in the fleet, the page says so,
with the largest node's memory. Better read there than discovered by a node
being OOM-killed twenty minutes into a load.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-27 16:38:49 +02:00
1 parent 2acd794022
commit e0ea4f6be9
3 files changed
+133 -8

No files matched your search

+65 -3
View File
@@ -30,12 +30,15 @@ func (f *fakeCatalog) Declared(name string) bool { return name == f.declared }
// fleet reports a cluster with a given number of cards and engines.
type fakeFleet struct {
gpus int
engines []string
gpus int
engines []string
memoryGB int
}
func (f fakeFleet) Nodes() []adminapi.NodeView {
return []adminapi.NodeView{{ID: "n1", GPUTotal: f.gpus, Engines: f.engines}}
return []adminapi.NodeView{
{ID: "n1", GPUTotal: f.gpus, Engines: f.engines, MemoryGB: f.memoryGB},
}
}
func (f fakeFleet) Models() []adminapi.ModelView { return nil }
func (f fakeFleet) Replicas(string) []adminapi.ReplicaView { return nil }
@@ -233,3 +236,62 @@ func TestAFleetOfOlderAgentsIsNotToldEverythingIsUnservable(t *testing.T) {
t.Error("silence from older agents must not read as a refusal")
}
}
func sized(model, version, file string, gb float64) mirror.Status {
n := int64(gb * (1 << 30))
return mirror.Status{Model: model, Version: version, File: file,
State: mirror.Present, BytesDone: n, BytesTotal: n}
}
func TestTheMemoryFieldIsFilledInFromTheWeights(t *testing.T) {
// The real case: 4.68 GB of weights went onto a 7.9 GB node and left it
// 3.5 GB for a KV pool, because the field was optional and blank. Blank
// writes no constraint, and a CPU profile has no card to be short of.
body := mirrorGet(t, servePage(t, []mirror.Status{
sized("coder", "v1", "m.gguf", 4.68)}, &fakeCatalog{}, 0, "llamacpp"),
"/admin/mirror?serve=coder:v1").Body.String()
if !strings.Contains(body, `name="min_memory_gb" value="9"`) {
t.Errorf("expected a figure derived from 4.68 GB of weights; got %s", body)
}
}
func TestASmallModelDoesNotDemandALargeNode(t *testing.T) {
// The estimate has to leave the small nodes usable, or every model ends up
// pinned to the one big machine.
body := mirrorGet(t, servePage(t, []mirror.Status{
sized("small", "v1", "m.gguf", 1.12)}, &fakeCatalog{}, 0, "llamacpp"),
"/admin/mirror?serve=small:v1").Body.String()
if !strings.Contains(body, `name="min_memory_gb" value="3"`) {
t.Errorf("a 1.12 GB model should fit a small node; got %s", body)
}
}
func TestAModelTooBigForEveryNodeSaysSo(t *testing.T) {
page := mirrorPageDeps(t, MirrorDeps{
Mirror: &fakeMirror{models: []mirror.Status{sized("huge", "v1", "m.gguf", 40)}},
Catalog: &fakeCatalog{},
Fleet: fakeFleet{gpus: 0, engines: []string{"llamacpp"}, memoryGB: 31},
}, true)
body := mirrorGet(t, page, "/admin/mirror?serve=huge:v1").Body.String()
// Better said here than discovered by a node being OOM-killed twenty
// minutes into a load.
if !strings.Contains(body, "largest node has 31 GB") {
t.Errorf("expected the size warning; got %s", body)
}
}
func TestAnEstimateIsNotMadeWhenTheSizeIsUnknown(t *testing.T) {
// A version whose size nobody reported would otherwise be given a
// requirement of two, which is a constraint that excludes nothing and
// reads as though somebody measured it.
body := mirrorGet(t, servePage(t, []mirror.Status{
{Model: "m", Version: "v", File: "f.gguf", State: mirror.Present}},
&fakeCatalog{}, 0, "llamacpp"), "/admin/mirror?serve=m:v").Body.String()
if !strings.Contains(body, `name="min_memory_gb" value="0"`) {
t.Errorf("an unknown size should leave no figure; got %s", body)
}
}