diff --git a/internal/web/admin_serve.go b/internal/web/admin_serve.go index 61e83b42..a583068d 100644 --- a/internal/web/admin_serve.go +++ b/internal/web/admin_serve.go @@ -32,6 +32,14 @@ type serveForm struct { // be written later. Named for the hardware rather than the model, because // a profile is about where a model runs. Profile string + // MinMemoryGB is the host RAM to ask for, estimated from the weights. + // + // Filled in rather than left blank. Blank writes no constraint at all, and + // a CPU profile has no card to be short of, so every ready node becomes a + // candidate: a 4.7 GB model went onto a 7.9 GB machine and left it with + // 3.5 GB for a KV pool. Nothing was wrong with placement — it was never + // told what the model needed. + MinMemoryGB int // Refusal is set when this version cannot be served as it stands, and says // why. Offered and then failing at placement is worse than not offered. @@ -90,6 +98,12 @@ func planServe(st mirror.Status, files []mirror.Status, base string, "same model runs on the CPU nodes" } } + f.MinMemoryGB = memoryFor(st.BytesTotal, st.BytesDone) + if biggest := biggestNode(fleet); biggest > 0 && f.MinMemoryGB > biggest { + f.Warning = strings.TrimSuffix(f.Warning, ".") + + fmt.Sprintf("%sthe largest node has %d GB and this needs about %d", + sep(f.Warning), biggest, f.MinMemoryGB) + } if cat != nil && cat.Declared(st.Model) { f.Already = true f.Refusal = "a model called " + st.Model + @@ -226,3 +240,48 @@ func positive(v, what string) (int, error) { } return n, nil } + +// memoryFor estimates the host RAM a model needs, from the size of its weights. +// +// Weights plus half again, plus two. The half covers the KV pool, which grows +// with the same things the weights do — layers and heads — and the two are the +// engine's own footprint and the page cache it will not get back. It is an +// estimate and the field it fills is editable; what it is not is blank, which +// is what put a 4.7 GB model on a 7.9 GB node. +// +// The honest number comes from running it: `llmgw ctl bench` measures what a +// replica can actually do, and that is a different question from what it needs +// to start. +func memoryFor(total, done int64) int { + size := total + if size <= 0 { + size = done + } + if size <= 0 { + return 0 + } + gb := float64(size) / (1 << 30) + return int(gb*1.5) + 2 +} + +// biggestNode is the most memory any single node reports, or 0 when none says. +func biggestNode(fleet Source) int { + if fleet == nil { + return 0 + } + most := 0 + for _, n := range fleet.Nodes() { + if n.MemoryGB > most { + most = n.MemoryGB + } + } + return most +} + +// sep joins a second clause onto a warning that may be empty. +func sep(existing string) string { + if existing == "" { + return "" + } + return ", and " +} diff --git a/internal/web/admin_serve_test.go b/internal/web/admin_serve_test.go index 147ba00d..97f6b8fd 100644 --- a/internal/web/admin_serve_test.go +++ b/internal/web/admin_serve_test.go @@ -30,12 +30,15 @@ func (f *fakeCatalog) Declared(name string) bool { return name == f.declared } // fleet reports a cluster with a given number of cards and engines. type fakeFleet struct { - gpus int - engines []string + gpus int + engines []string + memoryGB int } func (f fakeFleet) Nodes() []adminapi.NodeView { - return []adminapi.NodeView{{ID: "n1", GPUTotal: f.gpus, Engines: f.engines}} + return []adminapi.NodeView{ + {ID: "n1", GPUTotal: f.gpus, Engines: f.engines, MemoryGB: f.memoryGB}, + } } func (f fakeFleet) Models() []adminapi.ModelView { return nil } func (f fakeFleet) Replicas(string) []adminapi.ReplicaView { return nil } @@ -233,3 +236,62 @@ func TestAFleetOfOlderAgentsIsNotToldEverythingIsUnservable(t *testing.T) { t.Error("silence from older agents must not read as a refusal") } } + +func sized(model, version, file string, gb float64) mirror.Status { + n := int64(gb * (1 << 30)) + return mirror.Status{Model: model, Version: version, File: file, + State: mirror.Present, BytesDone: n, BytesTotal: n} +} + +func TestTheMemoryFieldIsFilledInFromTheWeights(t *testing.T) { + // The real case: 4.68 GB of weights went onto a 7.9 GB node and left it + // 3.5 GB for a KV pool, because the field was optional and blank. Blank + // writes no constraint, and a CPU profile has no card to be short of. + body := mirrorGet(t, servePage(t, []mirror.Status{ + sized("coder", "v1", "m.gguf", 4.68)}, &fakeCatalog{}, 0, "llamacpp"), + "/admin/mirror?serve=coder:v1").Body.String() + + if !strings.Contains(body, `name="min_memory_gb" value="9"`) { + t.Errorf("expected a figure derived from 4.68 GB of weights; got %s", body) + } +} + +func TestASmallModelDoesNotDemandALargeNode(t *testing.T) { + // The estimate has to leave the small nodes usable, or every model ends up + // pinned to the one big machine. + body := mirrorGet(t, servePage(t, []mirror.Status{ + sized("small", "v1", "m.gguf", 1.12)}, &fakeCatalog{}, 0, "llamacpp"), + "/admin/mirror?serve=small:v1").Body.String() + + if !strings.Contains(body, `name="min_memory_gb" value="3"`) { + t.Errorf("a 1.12 GB model should fit a small node; got %s", body) + } +} + +func TestAModelTooBigForEveryNodeSaysSo(t *testing.T) { + page := mirrorPageDeps(t, MirrorDeps{ + Mirror: &fakeMirror{models: []mirror.Status{sized("huge", "v1", "m.gguf", 40)}}, + Catalog: &fakeCatalog{}, + Fleet: fakeFleet{gpus: 0, engines: []string{"llamacpp"}, memoryGB: 31}, + }, true) + body := mirrorGet(t, page, "/admin/mirror?serve=huge:v1").Body.String() + + // Better said here than discovered by a node being OOM-killed twenty + // minutes into a load. + if !strings.Contains(body, "largest node has 31 GB") { + t.Errorf("expected the size warning; got %s", body) + } +} + +func TestAnEstimateIsNotMadeWhenTheSizeIsUnknown(t *testing.T) { + // A version whose size nobody reported would otherwise be given a + // requirement of two, which is a constraint that excludes nothing and + // reads as though somebody measured it. + body := mirrorGet(t, servePage(t, []mirror.Status{ + {Model: "m", Version: "v", File: "f.gguf", State: mirror.Present}}, + &fakeCatalog{}, 0, "llamacpp"), "/admin/mirror?serve=m:v").Body.String() + + if !strings.Contains(body, `name="min_memory_gb" value="0"`) { + t.Errorf("an unknown size should leave no figure; got %s", body) + } +} diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html index bb5829de..5bde8acc 100644 --- a/internal/web/templates/admin_mirror.html +++ b/internal/web/templates/admin_mirror.html @@ -110,8 +110,8 @@ reachable from this machine.

- + @@ -120,9 +120,13 @@ reachable from this machine.

The context pool is what the engine is given; llama.cpp divides it across its parallel slots, so the number one request may use is the smaller -one above. A node label keeps a large model off the small machines — a CPU -profile has no card to be short of, so without one every ready node looks like -a candidate and the first one picked is the one that runs out of memory.

+one above.

+

The memory figure is the weights plus half again for the KV pool +plus two for the engine — an estimate, and editable. It is filled in rather than +left blank because a CPU profile has no card to be short of: with no figure, +every ready node is a candidate and the first one picked is whichever the +scheduler reached, not whichever can hold the model. A node label does the same +job by name if you would rather pin it.

{{end}} {{end}}