From cba3e329fceed3552849ae48beb337cf3a45bde3 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Thu, 1 Oct 2026 11:59:22 +0200 Subject: [PATCH] Let the mirror's serve form place a model on a GPU The form offered no placement and named every GGUF profile cpu. It now lists the fleet's GPU classes (the freest chosen) and the CPU, a GPU count and parallel requests, names the profile from the placement, shows what the model will run as, and refuses the CPU while GPUs are free unless that is ticked. Refs #2 (item 1). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 22 +++ internal/web/admin_mirror.go | 14 +- internal/web/admin_serve.go | 188 ++++++++++++++++++++++- internal/web/admin_serve_test.go | 96 +++++++++++- internal/web/templates/admin_mirror.html | 55 ++++++- 5 files changed, 364 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ad710d3..25877815 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,28 @@ as nvidia-smi does, by PCI bus, and CUDA's own order is fastest first — on a node with two kinds of GPU, a shard could get a different card from the one it was given. +### Serving from the mirror says where a model will run + +The serve form put every GGUF model on the CPU: it offered no choice, and its +"Profile name" field, pre-filled "cpu", read as though renaming it would move +the model to a GPU. Placement is decided by `gpu_class`, which the form never +set. Now: + +- **Runs on** lists the fleet's GPU classes with how many cards are free, and + for llama.cpp the CPU. On a fleet with GPUs the class with the most free + cards is chosen. +- **GPUs per replica** sets how many cards a replica takes on one node + (`gpus_per_node` for llama.cpp, `tp` for vLLM), and **parallel requests** + how many a llama.cpp replica serves at once. +- The profile is named for where it runs (`t4`, `t4-x4`, `cpu`) rather than + typed. +- A line under the form says what the model will run as — engine, CPU or how + many of which GPU, memory, context pool and how it is divided — and turns + red when the CPU is chosen with GPUs free. The message after adding repeats + it. +- Choosing the CPU while GPUs are free is refused unless **run on the CPU + anyway** is ticked. + ### llama.cpp on several GPUs of one node A llama.cpp profile takes several of a node's GPUs with `gpus_per_node: N`: diff --git a/internal/web/admin_mirror.go b/internal/web/admin_mirror.go index 611615bf..71dd736b 100644 --- a/internal/web/admin_mirror.go +++ b/internal/web/admin_mirror.go @@ -317,6 +317,16 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) { if err != nil { return "", err } + // The CPU while GPUs sit free is a choice to make on purpose: + // made by default, it was a model running at a tenth of the + // speed with nothing saying why. + if model.Profiles[0].CPUOnly() && r.FormValue("cpu_ok") == "" { + if n := freeGPUs(p.fleet); n > 0 { + return "", fmt.Errorf("this fleet has %s free and %s was set to run on the CPU; "+ + "choose a GPU placement, or tick \"run on the CPU anyway\"", + plural(n, "GPU", "GPUs"), model.Name) + } + } // The same check `llmgw ctl model add` makes. This page builds // the source out of a row the mirror reported, so it is right // far more often than something typed — but a file removed @@ -332,8 +342,8 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) { // Placed on the next pass, not now: saying it is serving would // be wrong for however long the weights take to reach a node. return fmt.Sprintf( - "%s added to the catalog, %s; it is placed on the next pass", - model.Name, plural(model.Scaling.Min, "replica", "replicas")), nil + "%s added to the catalog, %s: %s; it is placed on the next pass", + model.Name, plural(model.Scaling.Min, "replica", "replicas"), describeServe(model)), nil })) } diff --git a/internal/web/admin_serve.go b/internal/web/admin_serve.go index a154b8a1..6933240f 100644 --- a/internal/web/admin_serve.go +++ b/internal/web/admin_serve.go @@ -43,8 +43,20 @@ type serveForm struct { Engine string // Profile is the profile name, which doubles as how a node_selector would // be written later. Named for the hardware rather than the model, because - // a profile is about where a model runs. + // a profile is about where a model runs. Derived from the placement, not + // typed: a name is only a name, and a field for it read as though calling + // a profile "gpu" would put it on one. Profile string + // Placements are where it can run: each GPU class the fleet has, and for + // llama.cpp the CPU. One is selected. + Placements []placeChoice + // GPUs is how many cards a replica takes, when placed on a GPU class. + GPUs int + // Parallel is how many requests one replica serves at once. + Parallel int + // FreeGPUs is how many cards no replica holds, across the fleet: what + // choosing the CPU leaves unused. + FreeGPUs int // Files is how many files the version holds, above one for a set. Files int // MinMemoryGB is the host RAM to ask for, estimated from the weights. @@ -126,6 +138,13 @@ func planServe(st mirror.Status, files []mirror.Status, base string, default: f.Profile = "gpu" } + f.GPUs, f.Parallel = 1, defaultParallel + f.Placements, f.FreeGPUs = placementsFor(fleet, f.Engine) + for _, pc := range f.Placements { + if pc.Selected { + f.Profile = pc.profile() + } + } // Whether any node can actually run it. Agents report their installed // engines now, so this is a fact rather than the guess it started as — // which was cards, and would have passed a GPU node with no vLLM on it. @@ -274,12 +293,38 @@ func buildModel(get func(string) string) (catalog.Model, error) { if engine == "" { return m, errors.New("which engine should run it?") } - p := catalog.Profile{ - Name: strings.TrimSpace(get("profile")), Engine: engine, - Weight: 1, Priority: 100, + p := catalog.Profile{Engine: engine, Weight: 1, Priority: 100} + gpus := 1 + if v := strings.TrimSpace(get("gpus")); v != "" { + if gpus, err = positive(v, "the GPUs per replica"); err != nil { + return m, err + } + gpus = max(gpus, 1) } - if p.Name == "" { - p.Name = engine + place := placeChoice{Value: strings.TrimSpace(get("placement"))} + switch class, onGPU := strings.CutPrefix(place.Value, "gpu:"); { + case onGPU && class != "": + // Placement is decided by gpu_class, which is what the selector + // sets; the count is gpus_per_node for llama.cpp, which spreads + // layers over a node's cards, and tp for the others. + p.GPUClass = class + if engine == "llamacpp" { + p.GPUsPerNode = gpus + } else { + p.TP = gpus + } + place.GPUs = gpus + case place.Value == "" || place.Value == "cpu": + if engine != "llamacpp" { + return m, fmt.Errorf("%s runs on GPUs; choose a GPU class to place it on", engine) + } + place.Value = "cpu" + default: + return m, fmt.Errorf("%q is not a placement", place.Value) + } + p.Name = place.profile() + if v := strings.TrimSpace(get("profile")); v != "" { + p.Name = v } // The pool, not the per-request limit: llama.cpp divides it across its // parallel slots, so the number a client may use is smaller and is what @@ -290,6 +335,15 @@ func buildModel(get func(string) string) (catalog.Model, error) { } p.Args = []string{"--ctx-size", v} } + if v := strings.TrimSpace(get("parallel")); v != "" && engine == "llamacpp" { + n, err := positive(v, "the parallel requests") + if err != nil { + return m, err + } + if n > 0 { + p.Args = append(p.Args, "--parallel", strconv.Itoa(n)) + } + } if v := strings.TrimSpace(get("min_memory_gb")); v != "" { if p.MinMemoryGB, err = positive(v, "the memory requirement"); err != nil { return m, err @@ -408,3 +462,125 @@ func modeOf(sc catalog.Scaling) string { // defaultCtxPool is the llama.cpp context pool the serve form starts with. // --parallel divides it among slots; it does not multiply it. const defaultCtxPool = 16384 + +// defaultParallel is the requests a llama.cpp replica serves at once when +// the form is not changed: the agent's own default. +const defaultParallel = 4 + +// placeChoice is one place a model can run: a GPU class, or the CPU. +type placeChoice struct { + // Value is "gpu:" or "cpu", as the form submits it. + Value string + Label string + // GPUs is how many cards a replica takes, for naming the profile. + GPUs int + Selected bool +} + +// profile names the profile for where it runs: the class, with the count when +// it is more than one card ("t4", "t4-x4"), or "cpu". +func (c placeChoice) profile() string { + class, ok := strings.CutPrefix(c.Value, "gpu:") + if !ok || class == "" { + return "cpu" + } + if c.GPUs > 1 { + return fmt.Sprintf("%s-x%d", class, c.GPUs) + } + return class +} + +// placementsFor lists where engine can run on this fleet, with one selected, +// and how many GPUs are free. +// +// A GPU when the fleet has one: a GGUF model was put on the CPU by this form +// on a cluster of GPUs, ran at a tenth of their speed, and nothing on the way +// said why — the form had no choice to offer and named the profile "cpu". The +// class selected is the one with the most cards free. +func placementsFor(fleet Source, engine string) ([]placeChoice, int) { + var out []placeChoice + free, best := 0, -1 + if fleet != nil { + for _, c := range fleet.Summary().GPUs { + n := c.Free() + free += n + out = append(out, placeChoice{ + Value: "gpu:" + c.Class, + Label: fmt.Sprintf("GPU: %s, %d of %d free", c.Class, n, c.Total), + }) + if best < 0 || n > fleet.Summary().GPUs[best].Free() { + best = len(out) - 1 + } + } + } + if best >= 0 { + out[best].Selected = true + } + if engine == "llamacpp" { + label := "CPU only: no GPU, many times slower" + if len(out) == 0 { + label = "CPU: this fleet has no GPU" + } + out = append(out, placeChoice{Value: "cpu", Label: label, Selected: best < 0}) + } + return out, free +} + +// freeGPUs is how many cards no replica holds across the fleet. +func freeGPUs(fleet Source) int { + if fleet == nil { + return 0 + } + n := 0 + for _, c := range fleet.Summary().GPUs { + n += c.Free() + } + return n +} + +// describeServe says, in one line, what a model will run as: the summary +// shown before it is added and in the message after. +func describeServe(m catalog.Model) string { + if len(m.Profiles) == 0 { + return "" + } + p := m.Profiles[0] + engine := map[string]string{"llamacpp": "llama.cpp", "vllm": "vLLM", "sglang": "SGLang"}[p.Engine] + if engine == "" { + engine = p.Engine + } + where := "on the CPU" + if p.GPUClass != "" { + n := max(p.GPUsPerNode, p.TP, 1) + where = fmt.Sprintf("on %s %s", plural(n, "GPU", "GPUs"), p.GPUClass) + if n > 1 { + where = fmt.Sprintf("on %d %s GPUs of one node", n, p.GPUClass) + } + } + parts := []string{engine + " " + where} + if p.MinMemoryGB > 0 { + parts = append(parts, fmt.Sprintf("%d GB of host memory", p.MinMemoryGB)) + } + if p.GPUClass == "" && p.CPUCores > 0 { + parts = append(parts, plural(p.CPUCores, "core", "cores")) + } + pool, parallel := argInt(p.Args, "--ctx-size"), argInt(p.Args, "--parallel") + switch { + case pool > 0 && parallel > 0: + parts = append(parts, fmt.Sprintf("context pool %d over %d parallel requests (%d each)", pool, parallel, pool/parallel)) + case pool > 0: + parts = append(parts, fmt.Sprintf("context pool %d", pool)) + } + return strings.Join(parts, ", ") +} + +// argInt is the number after flag in args, or 0. +func argInt(args []string, flag string) int { + for i := 0; i+1 < len(args); i++ { + if args[i] == flag { + n, _ := strconv.Atoi(args[i+1]) + return n + } + } + return 0 +} diff --git a/internal/web/admin_serve_test.go b/internal/web/admin_serve_test.go index a4274267..91d7049c 100644 --- a/internal/web/admin_serve_test.go +++ b/internal/web/admin_serve_test.go @@ -71,6 +71,8 @@ type fakeFleet struct { gpus int engines []string memoryGB int + // used is how many of the cards replicas hold. + used int } func (f fakeFleet) Nodes() []adminapi.NodeView { @@ -81,7 +83,12 @@ func (f fakeFleet) Nodes() []adminapi.NodeView { func (f fakeFleet) Models() []adminapi.ModelView { return nil } func (f fakeFleet) Replicas(string) []adminapi.ReplicaView { return nil } func (f fakeFleet) Explain(string) adminapi.ExplainView { return adminapi.ExplainView{} } -func (f fakeFleet) Summary() adminapi.Summary { return adminapi.Summary{} } +func (f fakeFleet) Summary() adminapi.Summary { + if f.gpus == 0 { + return adminapi.Summary{} + } + return adminapi.Summary{GPUs: []adminapi.ClassUse{{Class: "t4", Total: f.gpus, Used: f.used}}} +} func held(model, version, file string) mirror.Status { return mirror.Status{Model: model, Version: version, File: file, @@ -452,3 +459,90 @@ func TestServingOnDemandLoadsWhenAskedFor(t *testing.T) { t.Errorf("on demand was added as %+v", sc) } } + +func TestAGGUFOnAFleetWithGPUsIsOfferedTheGPUFirst(t *testing.T) { + gguf := []mirror.Status{held("coder", "v1", "coder-q4_k_m.gguf")} + body := mirrorGet(t, servePage(t, gguf, &fakeCatalog{}, 4, "llamacpp"), + "/admin/mirror?serve=coder:v1").Body.String() + // Found in the field: no choice offered, a profile called "cpu", and a + // model on the CPU of a GPU cluster at a tenth of the speed. + if !strings.Contains(body, ``) { + t.Errorf("the free GPU class should be offered and chosen; got %s", body) + } + if !strings.Contains(body, ``) || + strings.Contains(cpuOnly, `name="cpu_ok"`) { + t.Errorf("a fleet without GPUs has only the CPU, and nothing to confirm; got %s", cpuOnly) + } +} + +func TestTheChosenPlacementDecidesWhereItRuns(t *testing.T) { + form := func(engine, place, gpus string) url.Values { + return url.Values{ + "name": {"m"}, "source": {"http://x/m"}, "engine": {engine}, "replicas": {"1"}, + "context_length": {"4096"}, "ctx_size": {"16384"}, "parallel": {"4"}, + "placement": {place}, "gpus": {gpus}, "min_memory_gb": {"8"}, + } + } + m, err := buildModel(form("llamacpp", "gpu:t4", "4").Get) + if err != nil { + t.Fatal(err) + } + p := m.Profiles[0] + if p.GPUClass != "t4" || p.GPUsPerNode != 4 || p.TP > 1 || p.Name != "t4-x4" { + t.Errorf("llama.cpp on four t4s: %+v", p) + } + if got := describeServe(m); got != "llama.cpp on 4 t4 GPUs of one node, 8 GB of host memory, "+ + "context pool 16384 over 4 parallel requests (4096 each)" { + t.Errorf("summary %q", got) + } + if m, _ := buildModel(form("vllm", "gpu:t4", "2").Get); m.Profiles[0].TP != 2 || m.Profiles[0].GPUsPerNode != 0 { + t.Errorf("vLLM's GPU count is tp: %+v", m.Profiles[0]) + } + if m, _ := buildModel(form("llamacpp", "cpu", "1").Get); m.Profiles[0].GPUClass != "" || m.Profiles[0].Name != "cpu" { + t.Errorf("the CPU: %+v", m.Profiles[0]) + } + if _, err := buildModel(form("vllm", "cpu", "1").Get); err == nil { + t.Error("vLLM on the CPU was accepted") + } +} + +func TestTheCPUWithGPUsFreeIsAChoiceMadeOnPurpose(t *testing.T) { + values := func(extra ...string) url.Values { + v := url.Values{ + "csrf": {"tok"}, "name": {"coder"}, "source": {reachableSource(t)}, "engine": {"llamacpp"}, + "replicas": {"1"}, "context_length": {"4096"}, "placement": {"cpu"}, "min_memory_gb": {"8"}, + } + for i := 0; i+1 < len(extra); i += 2 { + v.Set(extra[i], extra[i+1]) + } + return v + } + cat := &fakeCatalog{} + page := mirrorPageDeps(t, MirrorDeps{Mirror: &fakeMirror{}, Catalog: cat, + Fleet: fakeFleet{gpus: 4, used: 1, engines: []string{"llamacpp"}}}, true) + body := mirrorPost(t, page, "/admin/mirror/serve", values()).Body.String() + if len(cat.added) != 0 || !strings.Contains(body, "3 GPUs free") { + t.Fatalf("the CPU with three GPUs free went through unasked: %s", body) + } + mirrorPost(t, page, "/admin/mirror/serve", values("cpu_ok", "yes")) + if len(cat.added) != 1 { + t.Fatal("ticking the box should add it") + } + // Every card held: nothing to ask. + cat = &fakeCatalog{} + full := mirrorPageDeps(t, MirrorDeps{Mirror: &fakeMirror{}, Catalog: cat, + Fleet: fakeFleet{gpus: 4, used: 4, engines: []string{"llamacpp"}}}, true) + mirrorPost(t, full, "/admin/mirror/serve", values()) + if len(cat.added) != 1 { + t.Error("with no GPU free the CPU is the only choice and should not be questioned") + } +} diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html index 09253572..96e83002 100644 --- a/internal/web/templates/admin_mirror.html +++ b/internal/web/templates/admin_mirror.html @@ -131,7 +131,7 @@ reachable from this machine.

Cannot be served as it stands: {{.Refusal}}.

{{else}} {{if .Warning}}

{{.Warning}}.

{{end}} -
+ @@ -163,8 +163,23 @@ reachable from this machine.

+ {{/* Where it runs is a choice with consequences, so it is a choice: the + GPU classes the fleet has, and the CPU for llama.cpp. A profile name + field stood here once, and reading as though naming a profile "gpu" + would put it on one, it put GGUF models on the CPU of GPU clusters. */}} +
+ + + {{if eq .Engine "llamacpp"}}{{end}} +
+ {{if .FreeGPUs}}{{end}}
-
+
+

The context pool is what the engine is given; llama.cpp divides it across its parallel slots, so the number one request may use is the smaller one above.