Let the mirror's serve form place a model on a GPU

The form offered no placement and named every GGUF profile cpu. It now
lists the fleet's GPU classes (the freest chosen) and the CPU, a GPU
count and parallel requests, names the profile from the placement, shows
what the model will run as, and refuses the CPU while GPUs are free
unless that is ticked.

Refs #2 (item 1).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

CHANGELOG.md

@@ -47,6 +47,28 @@ as nvidia-smi does, by PCI bus, and CUDA's own order is fastest first — on a
 node with two kinds of GPU, a shard could get a different card from the one it
 was given.
 
+### Serving from the mirror says where a model will run
+
+The serve form put every GGUF model on the CPU: it offered no choice, and its
+"Profile name" field, pre-filled "cpu", read as though renaming it would move
+the model to a GPU. Placement is decided by `gpu_class`, which the form never
+set. Now:
+
+- **Runs on** lists the fleet's GPU classes with how many cards are free, and
+  for llama.cpp the CPU. On a fleet with GPUs the class with the most free
+  cards is chosen.
+- **GPUs per replica** sets how many cards a replica takes on one node
+  (`gpus_per_node` for llama.cpp, `tp` for vLLM), and **parallel requests**
+  how many a llama.cpp replica serves at once.
+- The profile is named for where it runs (`t4`, `t4-x4`, `cpu`) rather than
+  typed.
+- A line under the form says what the model will run as — engine, CPU or how
+  many of which GPU, memory, context pool and how it is divided — and turns
+  red when the CPU is chosen with GPUs free. The message after adding repeats
+  it.
+- Choosing the CPU while GPUs are free is refused unless **run on the CPU
+  anyway** is ticked.
+
 ### llama.cpp on several GPUs of one node
 
 A llama.cpp profile takes several of a node's GPUs with `gpus_per_node: N`:

internal/web/admin_mirror.go

@@ -317,6 +317,16 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) {
 				if err != nil {
 					return "", err
 				}
+				// The CPU while GPUs sit free is a choice to make on purpose:
+				// made by default, it was a model running at a tenth of the
+				// speed with nothing saying why.
+				if model.Profiles[0].CPUOnly() && r.FormValue("cpu_ok") == "" {
+					if n := freeGPUs(p.fleet); n > 0 {
+						return "", fmt.Errorf("this fleet has %s free and %s was set to run on the CPU; "+
+							"choose a GPU placement, or tick \"run on the CPU anyway\"",
+							plural(n, "GPU", "GPUs"), model.Name)
+					}
+				}
 				// The same check `llmgw ctl model add` makes. This page builds
 				// the source out of a row the mirror reported, so it is right
 				// far more often than something typed — but a file removed
@@ -332,8 +342,8 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) {
 				// Placed on the next pass, not now: saying it is serving would
 				// be wrong for however long the weights take to reach a node.
 				return fmt.Sprintf(
-					"%s added to the catalog, %s; it is placed on the next pass",
-					model.Name, plural(model.Scaling.Min, "replica", "replicas")), nil
+					"%s added to the catalog, %s: %s; it is placed on the next pass",
+					model.Name, plural(model.Scaling.Min, "replica", "replicas"), describeServe(model)), nil
 			}))
 	}
 

internal/web/admin_serve.go

@@ -43,8 +43,20 @@ type serveForm struct {
 	Engine  string
 	// Profile is the profile name, which doubles as how a node_selector would
 	// be written later. Named for the hardware rather than the model, because
-	// a profile is about where a model runs.
+	// a profile is about where a model runs. Derived from the placement, not
+	// typed: a name is only a name, and a field for it read as though calling
+	// a profile "gpu" would put it on one.
 	Profile string
+	// Placements are where it can run: each GPU class the fleet has, and for
+	// llama.cpp the CPU. One is selected.
+	Placements []placeChoice
+	// GPUs is how many cards a replica takes, when placed on a GPU class.
+	GPUs int
+	// Parallel is how many requests one replica serves at once.
+	Parallel int
+	// FreeGPUs is how many cards no replica holds, across the fleet: what
+	// choosing the CPU leaves unused.
+	FreeGPUs int
 	// Files is how many files the version holds, above one for a set.
 	Files int
 	// MinMemoryGB is the host RAM to ask for, estimated from the weights.
@@ -126,6 +138,13 @@ func planServe(st mirror.Status, files []mirror.Status, base string,
 	default:
 		f.Profile = "gpu"
 	}
+	f.GPUs, f.Parallel = 1, defaultParallel
+	f.Placements, f.FreeGPUs = placementsFor(fleet, f.Engine)
+	for _, pc := range f.Placements {
+		if pc.Selected {
+			f.Profile = pc.profile()
+		}
+	}
 	// Whether any node can actually run it. Agents report their installed
 	// engines now, so this is a fact rather than the guess it started as —
 	// which was cards, and would have passed a GPU node with no vLLM on it.
@@ -274,12 +293,38 @@ func buildModel(get func(string) string) (catalog.Model, error) {
 	if engine == "" {
 		return m, errors.New("which engine should run it?")
 	}
-	p := catalog.Profile{
-		Name: strings.TrimSpace(get("profile")), Engine: engine,
-		Weight: 1, Priority: 100,
+	p := catalog.Profile{Engine: engine, Weight: 1, Priority: 100}
+	gpus := 1
+	if v := strings.TrimSpace(get("gpus")); v != "" {
+		if gpus, err = positive(v, "the GPUs per replica"); err != nil {
+			return m, err
+		}
+		gpus = max(gpus, 1)
 	}
-	if p.Name == "" {
-		p.Name = engine
+	place := placeChoice{Value: strings.TrimSpace(get("placement"))}
+	switch class, onGPU := strings.CutPrefix(place.Value, "gpu:"); {
+	case onGPU && class != "":
+		// Placement is decided by gpu_class, which is what the selector
+		// sets; the count is gpus_per_node for llama.cpp, which spreads
+		// layers over a node's cards, and tp for the others.
+		p.GPUClass = class
+		if engine == "llamacpp" {
+			p.GPUsPerNode = gpus
+		} else {
+			p.TP = gpus
+		}
+		place.GPUs = gpus
+	case place.Value == "" || place.Value == "cpu":
+		if engine != "llamacpp" {
+			return m, fmt.Errorf("%s runs on GPUs; choose a GPU class to place it on", engine)
+		}
+		place.Value = "cpu"
+	default:
+		return m, fmt.Errorf("%q is not a placement", place.Value)
+	}
+	p.Name = place.profile()
+	if v := strings.TrimSpace(get("profile")); v != "" {
+		p.Name = v
 	}
 	// The pool, not the per-request limit: llama.cpp divides it across its
 	// parallel slots, so the number a client may use is smaller and is what
@@ -290,6 +335,15 @@ func buildModel(get func(string) string) (catalog.Model, error) {
 		}
 		p.Args = []string{"--ctx-size", v}
 	}
+	if v := strings.TrimSpace(get("parallel")); v != "" && engine == "llamacpp" {
+		n, err := positive(v, "the parallel requests")
+		if err != nil {
+			return m, err
+		}
+		if n > 0 {
+			p.Args = append(p.Args, "--parallel", strconv.Itoa(n))
+		}
+	}
 	if v := strings.TrimSpace(get("min_memory_gb")); v != "" {
 		if p.MinMemoryGB, err = positive(v, "the memory requirement"); err != nil {
 			return m, err
@@ -408,3 +462,125 @@ func modeOf(sc catalog.Scaling) string {
 // defaultCtxPool is the llama.cpp context pool the serve form starts with.
 // --parallel divides it among slots; it does not multiply it.
 const defaultCtxPool = 16384
+
+// defaultParallel is the requests a llama.cpp replica serves at once when
+// the form is not changed: the agent's own default.
+const defaultParallel = 4
+
+// placeChoice is one place a model can run: a GPU class, or the CPU.
+type placeChoice struct {
+	// Value is "gpu:<class>" or "cpu", as the form submits it.
+	Value string
+	Label string
+	// GPUs is how many cards a replica takes, for naming the profile.
+	GPUs     int
+	Selected bool
+}
+
+// profile names the profile for where it runs: the class, with the count when
+// it is more than one card ("t4", "t4-x4"), or "cpu".
+func (c placeChoice) profile() string {
+	class, ok := strings.CutPrefix(c.Value, "gpu:")
+	if !ok || class == "" {
+		return "cpu"
+	}
+	if c.GPUs > 1 {
+		return fmt.Sprintf("%s-x%d", class, c.GPUs)
+	}
+	return class
+}
+
+// placementsFor lists where engine can run on this fleet, with one selected,
+// and how many GPUs are free.
+//
+// A GPU when the fleet has one: a GGUF model was put on the CPU by this form
+// on a cluster of GPUs, ran at a tenth of their speed, and nothing on the way
+// said why — the form had no choice to offer and named the profile "cpu". The
+// class selected is the one with the most cards free.
+func placementsFor(fleet Source, engine string) ([]placeChoice, int) {
+	var out []placeChoice
+	free, best := 0, -1
+	if fleet != nil {
+		for _, c := range fleet.Summary().GPUs {
+			n := c.Free()
+			free += n
+			out = append(out, placeChoice{
+				Value: "gpu:" + c.Class,
+				Label: fmt.Sprintf("GPU: %s, %d of %d free", c.Class, n, c.Total),
+			})
+			if best < 0 || n > fleet.Summary().GPUs[best].Free() {
+				best = len(out) - 1
+			}
+		}
+	}
+	if best >= 0 {
+		out[best].Selected = true
+	}
+	if engine == "llamacpp" {
+		label := "CPU only: no GPU, many times slower"
+		if len(out) == 0 {
+			label = "CPU: this fleet has no GPU"
+		}
+		out = append(out, placeChoice{Value: "cpu", Label: label, Selected: best < 0})
+	}
+	return out, free
+}
+
+// freeGPUs is how many cards no replica holds across the fleet.
+func freeGPUs(fleet Source) int {
+	if fleet == nil {
+		return 0
+	}
+	n := 0
+	for _, c := range fleet.Summary().GPUs {
+		n += c.Free()
+	}
+	return n
+}
+
+// describeServe says, in one line, what a model will run as: the summary
+// shown before it is added and in the message after.
+func describeServe(m catalog.Model) string {
+	if len(m.Profiles) == 0 {
+		return ""
+	}
+	p := m.Profiles[0]
+	engine := map[string]string{"llamacpp": "llama.cpp", "vllm": "vLLM", "sglang": "SGLang"}[p.Engine]
+	if engine == "" {
+		engine = p.Engine
+	}
+	where := "on the CPU"
+	if p.GPUClass != "" {
+		n := max(p.GPUsPerNode, p.TP, 1)
+		where = fmt.Sprintf("on %s %s", plural(n, "GPU", "GPUs"), p.GPUClass)
+		if n > 1 {
+			where = fmt.Sprintf("on %d %s GPUs of one node", n, p.GPUClass)
+		}
+	}
+	parts := []string{engine + " " + where}
+	if p.MinMemoryGB > 0 {
+		parts = append(parts, fmt.Sprintf("%d GB of host memory", p.MinMemoryGB))
+	}
+	if p.GPUClass == "" && p.CPUCores > 0 {
+		parts = append(parts, plural(p.CPUCores, "core", "cores"))
+	}
+	pool, parallel := argInt(p.Args, "--ctx-size"), argInt(p.Args, "--parallel")
+	switch {
+	case pool > 0 && parallel > 0:
+		parts = append(parts, fmt.Sprintf("context pool %d over %d parallel requests (%d each)", pool, parallel, pool/parallel))
+	case pool > 0:
+		parts = append(parts, fmt.Sprintf("context pool %d", pool))
+	}
+	return strings.Join(parts, ", ")
+}
+
+// argInt is the number after flag in args, or 0.
+func argInt(args []string, flag string) int {
+	for i := 0; i+1 < len(args); i++ {
+		if args[i] == flag {
+			n, _ := strconv.Atoi(args[i+1])
+			return n
+		}
+	}
+	return 0
+}

internal/web/admin_serve_test.go

@@ -71,6 +71,8 @@ type fakeFleet struct {
 	gpus     int
 	engines  []string
 	memoryGB int
+	// used is how many of the cards replicas hold.
+	used int
 }
 
 func (f fakeFleet) Nodes() []adminapi.NodeView {
@@ -81,7 +83,12 @@ func (f fakeFleet) Nodes() []adminapi.NodeView {
 func (f fakeFleet) Models() []adminapi.ModelView           { return nil }
 func (f fakeFleet) Replicas(string) []adminapi.ReplicaView { return nil }
 func (f fakeFleet) Explain(string) adminapi.ExplainView    { return adminapi.ExplainView{} }
-func (f fakeFleet) Summary() adminapi.Summary              { return adminapi.Summary{} }
+func (f fakeFleet) Summary() adminapi.Summary {
+	if f.gpus == 0 {
+		return adminapi.Summary{}
+	}
+	return adminapi.Summary{GPUs: []adminapi.ClassUse{{Class: "t4", Total: f.gpus, Used: f.used}}}
+}
 
 func held(model, version, file string) mirror.Status {
 	return mirror.Status{Model: model, Version: version, File: file,
@@ -452,3 +459,90 @@ func TestServingOnDemandLoadsWhenAskedFor(t *testing.T) {
 		t.Errorf("on demand was added as %+v", sc)
 	}
 }
+
+func TestAGGUFOnAFleetWithGPUsIsOfferedTheGPUFirst(t *testing.T) {
+	gguf := []mirror.Status{held("coder", "v1", "coder-q4_k_m.gguf")}
+	body := mirrorGet(t, servePage(t, gguf, &fakeCatalog{}, 4, "llamacpp"),
+		"/admin/mirror?serve=coder:v1").Body.String()
+	// Found in the field: no choice offered, a profile called "cpu", and a
+	// model on the CPU of a GPU cluster at a tenth of the speed.
+	if !strings.Contains(body, `<option value="gpu:t4" selected>GPU: t4, 4 of 4 free</option>`) {
+		t.Errorf("the free GPU class should be offered and chosen; got %s", body)
+	}
+	if !strings.Contains(body, `<option value="cpu">CPU only`) {
+		t.Error("the CPU should still be on offer, unselected")
+	}
+	if !strings.Contains(body, `name="cpu_ok"`) {
+		t.Error("choosing the CPU with GPUs free needs saying so")
+	}
+
+	cpuOnly := mirrorGet(t, servePage(t, gguf, &fakeCatalog{}, 0, "llamacpp"),
+		"/admin/mirror?serve=coder:v1").Body.String()
+	if !strings.Contains(cpuOnly, `<option value="cpu" selected>CPU: this fleet has no GPU</option>`) ||
+		strings.Contains(cpuOnly, `name="cpu_ok"`) {
+		t.Errorf("a fleet without GPUs has only the CPU, and nothing to confirm; got %s", cpuOnly)
+	}
+}
+
+func TestTheChosenPlacementDecidesWhereItRuns(t *testing.T) {
+	form := func(engine, place, gpus string) url.Values {
+		return url.Values{
+			"name": {"m"}, "source": {"http://x/m"}, "engine": {engine}, "replicas": {"1"},
+			"context_length": {"4096"}, "ctx_size": {"16384"}, "parallel": {"4"},
+			"placement": {place}, "gpus": {gpus}, "min_memory_gb": {"8"},
+		}
+	}
+	m, err := buildModel(form("llamacpp", "gpu:t4", "4").Get)
+	if err != nil {
+		t.Fatal(err)
+	}
+	p := m.Profiles[0]
+	if p.GPUClass != "t4" || p.GPUsPerNode != 4 || p.TP > 1 || p.Name != "t4-x4" {
+		t.Errorf("llama.cpp on four t4s: %+v", p)
+	}
+	if got := describeServe(m); got != "llama.cpp on 4 t4 GPUs of one node, 8 GB of host memory, "+
+		"context pool 16384 over 4 parallel requests (4096 each)" {
+		t.Errorf("summary %q", got)
+	}
+	if m, _ := buildModel(form("vllm", "gpu:t4", "2").Get); m.Profiles[0].TP != 2 || m.Profiles[0].GPUsPerNode != 0 {
+		t.Errorf("vLLM's GPU count is tp: %+v", m.Profiles[0])
+	}
+	if m, _ := buildModel(form("llamacpp", "cpu", "1").Get); m.Profiles[0].GPUClass != "" || m.Profiles[0].Name != "cpu" {
+		t.Errorf("the CPU: %+v", m.Profiles[0])
+	}
+	if _, err := buildModel(form("vllm", "cpu", "1").Get); err == nil {
+		t.Error("vLLM on the CPU was accepted")
+	}
+}
+
+func TestTheCPUWithGPUsFreeIsAChoiceMadeOnPurpose(t *testing.T) {
+	values := func(extra ...string) url.Values {
+		v := url.Values{
+			"csrf": {"tok"}, "name": {"coder"}, "source": {reachableSource(t)}, "engine": {"llamacpp"},
+			"replicas": {"1"}, "context_length": {"4096"}, "placement": {"cpu"}, "min_memory_gb": {"8"},
+		}
+		for i := 0; i+1 < len(extra); i += 2 {
+			v.Set(extra[i], extra[i+1])
+		}
+		return v
+	}
+	cat := &fakeCatalog{}
+	page := mirrorPageDeps(t, MirrorDeps{Mirror: &fakeMirror{}, Catalog: cat,
+		Fleet: fakeFleet{gpus: 4, used: 1, engines: []string{"llamacpp"}}}, true)
+	body := mirrorPost(t, page, "/admin/mirror/serve", values()).Body.String()
+	if len(cat.added) != 0 || !strings.Contains(body, "3 GPUs free") {
+		t.Fatalf("the CPU with three GPUs free went through unasked: %s", body)
+	}
+	mirrorPost(t, page, "/admin/mirror/serve", values("cpu_ok", "yes"))
+	if len(cat.added) != 1 {
+		t.Fatal("ticking the box should add it")
+	}
+	// Every card held: nothing to ask.
+	cat = &fakeCatalog{}
+	full := mirrorPageDeps(t, MirrorDeps{Mirror: &fakeMirror{}, Catalog: cat,
+		Fleet: fakeFleet{gpus: 4, used: 4, engines: []string{"llamacpp"}}}, true)
+	mirrorPost(t, full, "/admin/mirror/serve", values())
+	if len(cat.added) != 1 {
+		t.Error("with no GPU free the CPU is the only choice and should not be questioned")
+	}
+}

internal/web/templates/admin_mirror.html

@@ -131,7 +131,7 @@ reachable from this machine.</p>
 <p class="error">Cannot be served as it stands: {{.Refusal}}.</p>
 {{else}}
 {{if .Warning}}<p class="note error-row">{{.Warning}}.</p>{{end}}
-<form method="post" action="/admin/mirror/serve">
+<form method="post" action="/admin/mirror/serve" class="serve-form" data-engine="{{.Engine}}" data-free-gpus="{{.FreeGPUs}}">
   <input type="hidden" name="csrf" value="{{$.CSRF}}">
   <input type="hidden" name="source" value="{{.Source}}">
   <input type="hidden" name="engine" value="{{.Engine}}">
@@ -163,8 +163,23 @@ reachable from this machine.</p>
     <label>Engine context pool <span class="opt">divided across slots</span>
       <input type="text" name="ctx_size" value="{{.CtxPool}}"></label>
   </div>
+  {{/* Where it runs is a choice with consequences, so it is a choice: the
+       GPU classes the fleet has, and the CPU for llama.cpp. A profile name
+       field stood here once, and reading as though naming a profile "gpu"
+       would put it on one, it put GGUF models on the CPU of GPU clusters. */}}
+  <div class="field-row">
+    <label>Runs on
+      <select name="placement">
+        {{range .Placements}}<option value="{{.Value}}"{{if .Selected}} selected{{end}}>{{.Label}}</option>{{end}}
+      </select></label>
+    <label>GPUs per replica <span class="opt">on one node</span>
+      <input type="text" name="gpus" value="{{.GPUs}}" size="3"></label>
+    {{if eq .Engine "llamacpp"}}<label>Parallel requests <span class="opt">per replica</span>
+      <input type="text" name="parallel" value="{{.Parallel}}" size="3"></label>{{end}}
+  </div>
+  {{if .FreeGPUs}}<label class="cpu-ok"><input type="checkbox" name="cpu_ok" value="yes">
+    Run on the CPU anyway: {{.FreeGPUs}} GPUs are free and would sit unused</label>{{end}}
   <div class="field-row">
-    <label>Profile name <input type="text" name="profile" value="{{.Profile}}"></label>
     <label>Needs RAM (GB) <span class="opt">for the context pool beside it</span>
       <input type="text" name="min_memory_gb" value="{{.MinMemoryGB}}" required></label>
     <label>CPU cores <span class="opt">reserved; the engine's threads</span>
@@ -173,8 +188,44 @@ reachable from this machine.</p>
       <input type="text" name="selector_key" placeholder="role" size="8">
       <input type="text" name="selector_value" placeholder="cpu-large" size="10"></label>
   </div>
+  <p class="sub serve-summary" hidden></p>
   <button type="submit">Add to the catalog</button>
 </form>
+<script>
+// What it will run as, in one line, kept up to date as the form changes:
+// engine, where, how many cards, memory, context and how it is shared.
+(function () {
+  const f = document.currentScript.previousElementSibling;
+  if (!f || !f.classList.contains("serve-form")) return;
+  const out = f.querySelector(".serve-summary"), ok = f.querySelector(".cpu-ok");
+  const v = (n) => (f.elements[n] ? f.elements[n].value.trim() : "");
+  const engine = { llamacpp: "llama.cpp", vllm: "vLLM", sglang: "SGLang" }[f.dataset.engine] || f.dataset.engine;
+  function draw() {
+    const place = v("placement"), gpus = parseInt(v("gpus"), 10) || 1;
+    const onCPU = place === "cpu" || place === "";
+    let where = "on the CPU";
+    if (!onCPU) {
+      const cls = place.replace(/^gpu:/, "");
+      where = gpus > 1 ? "on " + gpus + " " + cls + " GPUs of one node" : "on 1 GPU " + cls;
+    }
+    const parts = [engine + " " + where];
+    if (v("min_memory_gb")) parts.push(v("min_memory_gb") + " GB of host memory");
+    if (onCPU && v("cpu_cores")) parts.push(v("cpu_cores") + " cores");
+    const pool = parseInt(v("ctx_size"), 10), par = parseInt(v("parallel"), 10);
+    if (pool && par) parts.push("context pool " + pool + " over " + par + " parallel requests (" + Math.floor(pool / par) + " each)");
+    else if (pool) parts.push("context pool " + pool);
+    out.textContent = "Will run as: " + parts.join(", ") + ".";
+    const warn = onCPU && +f.dataset.freeGpus > 0;
+    out.classList.toggle("error", warn);
+    if (warn) out.textContent += " " + f.dataset.freeGpus + " GPUs are free; on the CPU this runs many times slower.";
+    if (ok) ok.hidden = !onCPU;
+    out.hidden = false;
+  }
+  f.addEventListener("input", draw);
+  f.addEventListener("change", draw);
+  draw();
+})();
+</script>
 <p class="note">The context pool is what the engine is given; llama.cpp divides
 it across its parallel slots, so the number one request may use is the smaller
 one above.</p>