Fill in the memory a model needs, instead of leaving the field blank
CI / test (push) Successful in 1m41s

Serving a model from the mirror page left "Needs RAM (GB)" optional and empty.
Empty writes no constraint, and a CPU profile has no card to be short of, so
every ready node became a candidate — and a 4.68 GB model went onto a 7.9 GB
machine and left it 3.5 GB for a KV pool. Placement was not wrong; it was never
told what the model needed.

The mirror knows the size of the weights, so the field arrives filled in:
weights plus half again for the KV pool, plus two for the engine. The half is
there because the cache grows with the same things the weights do, layers and
heads. It is an estimate and the field is still editable — what it is not is
blank.

A 1.12 GB model asks for 3 GB, so the small nodes stay usable. Pinning every
model to the one large machine would be a different way of getting placement
wrong.

An unknown size fills in nothing rather than the floor of two, which would
exclude no node while reading as though somebody had measured it.

And when the estimate is larger than any node in the fleet, the page says so,
with the largest node's memory. Better read there than discovered by a node
being OOM-killed twenty minutes into a load.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-27 16:38:49 +02:00
1 parent 2acd794022
commit e0ea4f6be9
3 files changed
+133 -8

No files matched your search

+59
View File
@@ -32,6 +32,14 @@ type serveForm struct {
// be written later. Named for the hardware rather than the model, because
// a profile is about where a model runs.
Profile string
// MinMemoryGB is the host RAM to ask for, estimated from the weights.
//
// Filled in rather than left blank. Blank writes no constraint at all, and
// a CPU profile has no card to be short of, so every ready node becomes a
// candidate: a 4.7 GB model went onto a 7.9 GB machine and left it with
// 3.5 GB for a KV pool. Nothing was wrong with placement — it was never
// told what the model needed.
MinMemoryGB int
// Refusal is set when this version cannot be served as it stands, and says
// why. Offered and then failing at placement is worse than not offered.
@@ -90,6 +98,12 @@ func planServe(st mirror.Status, files []mirror.Status, base string,
"same model runs on the CPU nodes"
}
}
f.MinMemoryGB = memoryFor(st.BytesTotal, st.BytesDone)
if biggest := biggestNode(fleet); biggest > 0 && f.MinMemoryGB > biggest {
f.Warning = strings.TrimSuffix(f.Warning, ".") +
fmt.Sprintf("%sthe largest node has %d GB and this needs about %d",
sep(f.Warning), biggest, f.MinMemoryGB)
}
if cat != nil && cat.Declared(st.Model) {
f.Already = true
f.Refusal = "a model called " + st.Model +
@@ -226,3 +240,48 @@ func positive(v, what string) (int, error) {
}
return n, nil
}
// memoryFor estimates the host RAM a model needs, from the size of its weights.
//
// Weights plus half again, plus two. The half covers the KV pool, which grows
// with the same things the weights do — layers and heads — and the two are the
// engine's own footprint and the page cache it will not get back. It is an
// estimate and the field it fills is editable; what it is not is blank, which
// is what put a 4.7 GB model on a 7.9 GB node.
//
// The honest number comes from running it: `llmgw ctl bench` measures what a
// replica can actually do, and that is a different question from what it needs
// to start.
func memoryFor(total, done int64) int {
size := total
if size <= 0 {
size = done
}
if size <= 0 {
return 0
}
gb := float64(size) / (1 << 30)
return int(gb*1.5) + 2
}
// biggestNode is the most memory any single node reports, or 0 when none says.
func biggestNode(fleet Source) int {
if fleet == nil {
return 0
}
most := 0
for _, n := range fleet.Nodes() {
if n.MemoryGB > most {
most = n.MemoryGB
}
}
return most
}
// sep joins a second clause onto a warning that may be empty.
func sep(existing string) string {
if existing == "" {
return ""
}
return ", and "
}
+65 -3
View File
@@ -30,12 +30,15 @@ func (f *fakeCatalog) Declared(name string) bool { return name == f.declared }
// fleet reports a cluster with a given number of cards and engines.
type fakeFleet struct {
gpus int
engines []string
gpus int
engines []string
memoryGB int
}
func (f fakeFleet) Nodes() []adminapi.NodeView {
return []adminapi.NodeView{{ID: "n1", GPUTotal: f.gpus, Engines: f.engines}}
return []adminapi.NodeView{
{ID: "n1", GPUTotal: f.gpus, Engines: f.engines, MemoryGB: f.memoryGB},
}
}
func (f fakeFleet) Models() []adminapi.ModelView { return nil }
func (f fakeFleet) Replicas(string) []adminapi.ReplicaView { return nil }
@@ -233,3 +236,62 @@ func TestAFleetOfOlderAgentsIsNotToldEverythingIsUnservable(t *testing.T) {
t.Error("silence from older agents must not read as a refusal")
}
}
func sized(model, version, file string, gb float64) mirror.Status {
n := int64(gb * (1 << 30))
return mirror.Status{Model: model, Version: version, File: file,
State: mirror.Present, BytesDone: n, BytesTotal: n}
}
func TestTheMemoryFieldIsFilledInFromTheWeights(t *testing.T) {
// The real case: 4.68 GB of weights went onto a 7.9 GB node and left it
// 3.5 GB for a KV pool, because the field was optional and blank. Blank
// writes no constraint, and a CPU profile has no card to be short of.
body := mirrorGet(t, servePage(t, []mirror.Status{
sized("coder", "v1", "m.gguf", 4.68)}, &fakeCatalog{}, 0, "llamacpp"),
"/admin/mirror?serve=coder:v1").Body.String()
if !strings.Contains(body, `name="min_memory_gb" value="9"`) {
t.Errorf("expected a figure derived from 4.68 GB of weights; got %s", body)
}
}
func TestASmallModelDoesNotDemandALargeNode(t *testing.T) {
// The estimate has to leave the small nodes usable, or every model ends up
// pinned to the one big machine.
body := mirrorGet(t, servePage(t, []mirror.Status{
sized("small", "v1", "m.gguf", 1.12)}, &fakeCatalog{}, 0, "llamacpp"),
"/admin/mirror?serve=small:v1").Body.String()
if !strings.Contains(body, `name="min_memory_gb" value="3"`) {
t.Errorf("a 1.12 GB model should fit a small node; got %s", body)
}
}
func TestAModelTooBigForEveryNodeSaysSo(t *testing.T) {
page := mirrorPageDeps(t, MirrorDeps{
Mirror: &fakeMirror{models: []mirror.Status{sized("huge", "v1", "m.gguf", 40)}},
Catalog: &fakeCatalog{},
Fleet: fakeFleet{gpus: 0, engines: []string{"llamacpp"}, memoryGB: 31},
}, true)
body := mirrorGet(t, page, "/admin/mirror?serve=huge:v1").Body.String()
// Better said here than discovered by a node being OOM-killed twenty
// minutes into a load.
if !strings.Contains(body, "largest node has 31 GB") {
t.Errorf("expected the size warning; got %s", body)
}
}
func TestAnEstimateIsNotMadeWhenTheSizeIsUnknown(t *testing.T) {
// A version whose size nobody reported would otherwise be given a
// requirement of two, which is a constraint that excludes nothing and
// reads as though somebody measured it.
body := mirrorGet(t, servePage(t, []mirror.Status{
{Model: "m", Version: "v", File: "f.gguf", State: mirror.Present}},
&fakeCatalog{}, 0, "llamacpp"), "/admin/mirror?serve=m:v").Body.String()
if !strings.Contains(body, `name="min_memory_gb" value="0"`) {
t.Errorf("an unknown size should leave no figure; got %s", body)
}
}
+9 -5
View File
@@ -110,8 +110,8 @@ reachable from this machine.</p>
</div>
<div class="field-row">
<label>Profile name <input type="text" name="profile" value="{{.Profile}}"></label>
<label>Needs RAM (GB) <span class="opt">optional</span>
<input type="text" name="min_memory_gb" placeholder="8"></label>
<label>Needs RAM (GB) <span class="opt">estimated from the weights</span>
<input type="text" name="min_memory_gb" value="{{.MinMemoryGB}}"></label>
<label>Only nodes labelled <span class="opt">key / value, optional</span>
<input type="text" name="selector_key" placeholder="role" size="8">
<input type="text" name="selector_value" placeholder="cpu-large" size="10"></label>
@@ -120,9 +120,13 @@ reachable from this machine.</p>
</form>
<p class="note">The context pool is what the engine is given; llama.cpp divides
it across its parallel slots, so the number one request may use is the smaller
one above. A node label keeps a large model off the small machines — a CPU
profile has no card to be short of, so without one every ready node looks like
a candidate and the first one picked is the one that runs out of memory.</p>
one above.</p>
<p class="note">The memory figure is the weights plus half again for the KV pool
plus two for the engine — an estimate, and editable. It is filled in rather than
left blank because a CPU profile has no card to be short of: with no figure,
every ready node is a candidate and the first one picked is whichever the
scheduler reached, not whichever can hold the model. A node label does the same
job by name if you would rather pin it.</p>
{{end}}
</td></tr>
{{end}}