From e0ea4f6be93aa11d180d6ea38cd9eeb299ac051f Mon Sep 17 00:00:00 2001
From: Ilker Manap
Date: Sun, 27 Sep 2026 16:38:49 +0200
Subject: [PATCH] Fill in the memory a model needs, instead of leaving the
field blank
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Serving a model from the mirror page left "Needs RAM (GB)" optional and empty.
Empty writes no constraint, and a CPU profile has no card to be short of, so
every ready node became a candidate — and a 4.68 GB model went onto a 7.9 GB
machine and left it 3.5 GB for a KV pool. Placement was not wrong; it was never
told what the model needed.
The mirror knows the size of the weights, so the field arrives filled in:
weights plus half again for the KV pool, plus two for the engine. The half is
there because the cache grows with the same things the weights do, layers and
heads. It is an estimate and the field is still editable — what it is not is
blank.
A 1.12 GB model asks for 3 GB, so the small nodes stay usable. Pinning every
model to the one large machine would be a different way of getting placement
wrong.
An unknown size fills in nothing rather than the floor of two, which would
exclude no node while reading as though somebody had measured it.
And when the estimate is larger than any node in the fleet, the page says so,
with the largest node's memory. Better read there than discovered by a node
being OOM-killed twenty minutes into a load.
Co-Authored-By: Claude Opus 5 (1M context)
---
internal/web/admin_serve.go | 59 ++++++++++++++++++++
internal/web/admin_serve_test.go | 68 ++++++++++++++++++++++--
internal/web/templates/admin_mirror.html | 14 +++--
3 files changed, 133 insertions(+), 8 deletions(-)
diff --git a/internal/web/admin_serve.go b/internal/web/admin_serve.go
index 61e83b4..a583068 100644
--- a/internal/web/admin_serve.go
+++ b/internal/web/admin_serve.go
@@ -32,6 +32,14 @@ type serveForm struct {
// be written later. Named for the hardware rather than the model, because
// a profile is about where a model runs.
Profile string
+ // MinMemoryGB is the host RAM to ask for, estimated from the weights.
+ //
+ // Filled in rather than left blank. Blank writes no constraint at all, and
+ // a CPU profile has no card to be short of, so every ready node becomes a
+ // candidate: a 4.7 GB model went onto a 7.9 GB machine and left it with
+ // 3.5 GB for a KV pool. Nothing was wrong with placement — it was never
+ // told what the model needed.
+ MinMemoryGB int
// Refusal is set when this version cannot be served as it stands, and says
// why. Offered and then failing at placement is worse than not offered.
@@ -90,6 +98,12 @@ func planServe(st mirror.Status, files []mirror.Status, base string,
"same model runs on the CPU nodes"
}
}
+ f.MinMemoryGB = memoryFor(st.BytesTotal, st.BytesDone)
+ if biggest := biggestNode(fleet); biggest > 0 && f.MinMemoryGB > biggest {
+ f.Warning = strings.TrimSuffix(f.Warning, ".") +
+ fmt.Sprintf("%sthe largest node has %d GB and this needs about %d",
+ sep(f.Warning), biggest, f.MinMemoryGB)
+ }
if cat != nil && cat.Declared(st.Model) {
f.Already = true
f.Refusal = "a model called " + st.Model +
@@ -226,3 +240,48 @@ func positive(v, what string) (int, error) {
}
return n, nil
}
+
+// memoryFor estimates the host RAM a model needs, from the size of its weights.
+//
+// Weights plus half again, plus two. The half covers the KV pool, which grows
+// with the same things the weights do — layers and heads — and the two are the
+// engine's own footprint and the page cache it will not get back. It is an
+// estimate and the field it fills is editable; what it is not is blank, which
+// is what put a 4.7 GB model on a 7.9 GB node.
+//
+// The honest number comes from running it: `llmgw ctl bench` measures what a
+// replica can actually do, and that is a different question from what it needs
+// to start.
+func memoryFor(total, done int64) int {
+ size := total
+ if size <= 0 {
+ size = done
+ }
+ if size <= 0 {
+ return 0
+ }
+ gb := float64(size) / (1 << 30)
+ return int(gb*1.5) + 2
+}
+
+// biggestNode is the most memory any single node reports, or 0 when none says.
+func biggestNode(fleet Source) int {
+ if fleet == nil {
+ return 0
+ }
+ most := 0
+ for _, n := range fleet.Nodes() {
+ if n.MemoryGB > most {
+ most = n.MemoryGB
+ }
+ }
+ return most
+}
+
+// sep joins a second clause onto a warning that may be empty.
+func sep(existing string) string {
+ if existing == "" {
+ return ""
+ }
+ return ", and "
+}
diff --git a/internal/web/admin_serve_test.go b/internal/web/admin_serve_test.go
index 147ba00..97f6b8f 100644
--- a/internal/web/admin_serve_test.go
+++ b/internal/web/admin_serve_test.go
@@ -30,12 +30,15 @@ func (f *fakeCatalog) Declared(name string) bool { return name == f.declared }
// fleet reports a cluster with a given number of cards and engines.
type fakeFleet struct {
- gpus int
- engines []string
+ gpus int
+ engines []string
+ memoryGB int
}
func (f fakeFleet) Nodes() []adminapi.NodeView {
- return []adminapi.NodeView{{ID: "n1", GPUTotal: f.gpus, Engines: f.engines}}
+ return []adminapi.NodeView{
+ {ID: "n1", GPUTotal: f.gpus, Engines: f.engines, MemoryGB: f.memoryGB},
+ }
}
func (f fakeFleet) Models() []adminapi.ModelView { return nil }
func (f fakeFleet) Replicas(string) []adminapi.ReplicaView { return nil }
@@ -233,3 +236,62 @@ func TestAFleetOfOlderAgentsIsNotToldEverythingIsUnservable(t *testing.T) {
t.Error("silence from older agents must not read as a refusal")
}
}
+
+func sized(model, version, file string, gb float64) mirror.Status {
+ n := int64(gb * (1 << 30))
+ return mirror.Status{Model: model, Version: version, File: file,
+ State: mirror.Present, BytesDone: n, BytesTotal: n}
+}
+
+func TestTheMemoryFieldIsFilledInFromTheWeights(t *testing.T) {
+ // The real case: 4.68 GB of weights went onto a 7.9 GB node and left it
+ // 3.5 GB for a KV pool, because the field was optional and blank. Blank
+ // writes no constraint, and a CPU profile has no card to be short of.
+ body := mirrorGet(t, servePage(t, []mirror.Status{
+ sized("coder", "v1", "m.gguf", 4.68)}, &fakeCatalog{}, 0, "llamacpp"),
+ "/admin/mirror?serve=coder:v1").Body.String()
+
+ if !strings.Contains(body, `name="min_memory_gb" value="9"`) {
+ t.Errorf("expected a figure derived from 4.68 GB of weights; got %s", body)
+ }
+}
+
+func TestASmallModelDoesNotDemandALargeNode(t *testing.T) {
+ // The estimate has to leave the small nodes usable, or every model ends up
+ // pinned to the one big machine.
+ body := mirrorGet(t, servePage(t, []mirror.Status{
+ sized("small", "v1", "m.gguf", 1.12)}, &fakeCatalog{}, 0, "llamacpp"),
+ "/admin/mirror?serve=small:v1").Body.String()
+
+ if !strings.Contains(body, `name="min_memory_gb" value="3"`) {
+ t.Errorf("a 1.12 GB model should fit a small node; got %s", body)
+ }
+}
+
+func TestAModelTooBigForEveryNodeSaysSo(t *testing.T) {
+ page := mirrorPageDeps(t, MirrorDeps{
+ Mirror: &fakeMirror{models: []mirror.Status{sized("huge", "v1", "m.gguf", 40)}},
+ Catalog: &fakeCatalog{},
+ Fleet: fakeFleet{gpus: 0, engines: []string{"llamacpp"}, memoryGB: 31},
+ }, true)
+ body := mirrorGet(t, page, "/admin/mirror?serve=huge:v1").Body.String()
+
+ // Better said here than discovered by a node being OOM-killed twenty
+ // minutes into a load.
+ if !strings.Contains(body, "largest node has 31 GB") {
+ t.Errorf("expected the size warning; got %s", body)
+ }
+}
+
+func TestAnEstimateIsNotMadeWhenTheSizeIsUnknown(t *testing.T) {
+ // A version whose size nobody reported would otherwise be given a
+ // requirement of two, which is a constraint that excludes nothing and
+ // reads as though somebody measured it.
+ body := mirrorGet(t, servePage(t, []mirror.Status{
+ {Model: "m", Version: "v", File: "f.gguf", State: mirror.Present}},
+ &fakeCatalog{}, 0, "llamacpp"), "/admin/mirror?serve=m:v").Body.String()
+
+ if !strings.Contains(body, `name="min_memory_gb" value="0"`) {
+ t.Errorf("an unknown size should leave no figure; got %s", body)
+ }
+}
diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html
index bb5829d..5bde8ac 100644
--- a/internal/web/templates/admin_mirror.html
+++ b/internal/web/templates/admin_mirror.html
@@ -110,8 +110,8 @@ reachable from this machine.