Let the mirror's serve form place a model on a GPU

The form offered no placement and named every GGUF profile cpu. It now
lists the fleet's GPU classes (the freest chosen) and the CPU, a GPU
count and parallel requests, names the profile from the placement, shows
what the model will run as, and refuses the CPU while GPUs are free
unless that is ticked.

Refs #2 (item 1).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-01 11:59:22 +02:00
1 parent cbd5521f4f
commit cba3e329fc
5 files changed
+364 -11

No files matched your search

+53 -2
View File
@@ -131,7 +131,7 @@ reachable from this machine.</p>
<p class="error">Cannot be served as it stands: {{.Refusal}}.</p>
{{else}}
{{if .Warning}}<p class="note error-row">{{.Warning}}.</p>{{end}}
<form method="post" action="/admin/mirror/serve">
<form method="post" action="/admin/mirror/serve" class="serve-form" data-engine="{{.Engine}}" data-free-gpus="{{.FreeGPUs}}">
<input type="hidden" name="csrf" value="{{$.CSRF}}">
<input type="hidden" name="source" value="{{.Source}}">
<input type="hidden" name="engine" value="{{.Engine}}">
@@ -163,8 +163,23 @@ reachable from this machine.</p>
<label>Engine context pool <span class="opt">divided across slots</span>
<input type="text" name="ctx_size" value="{{.CtxPool}}"></label>
</div>
{{/* Where it runs is a choice with consequences, so it is a choice: the
GPU classes the fleet has, and the CPU for llama.cpp. A profile name
field stood here once, and reading as though naming a profile "gpu"
would put it on one, it put GGUF models on the CPU of GPU clusters. */}}
<div class="field-row">
<label>Runs on
<select name="placement">
{{range .Placements}}<option value="{{.Value}}"{{if .Selected}} selected{{end}}>{{.Label}}</option>{{end}}
</select></label>
<label>GPUs per replica <span class="opt">on one node</span>
<input type="text" name="gpus" value="{{.GPUs}}" size="3"></label>
{{if eq .Engine "llamacpp"}}<label>Parallel requests <span class="opt">per replica</span>
<input type="text" name="parallel" value="{{.Parallel}}" size="3"></label>{{end}}
</div>
{{if .FreeGPUs}}<label class="cpu-ok"><input type="checkbox" name="cpu_ok" value="yes">
Run on the CPU anyway: {{.FreeGPUs}} GPUs are free and would sit unused</label>{{end}}
<div class="field-row">
<label>Profile name <input type="text" name="profile" value="{{.Profile}}"></label>
<label>Needs RAM (GB) <span class="opt">for the context pool beside it</span>
<input type="text" name="min_memory_gb" value="{{.MinMemoryGB}}" required></label>
<label>CPU cores <span class="opt">reserved; the engine's threads</span>
@@ -173,8 +188,44 @@ reachable from this machine.</p>
<input type="text" name="selector_key" placeholder="role" size="8">
<input type="text" name="selector_value" placeholder="cpu-large" size="10"></label>
</div>
<p class="sub serve-summary" hidden></p>
<button type="submit">Add to the catalog</button>
</form>
<script>
// What it will run as, in one line, kept up to date as the form changes:
// engine, where, how many cards, memory, context and how it is shared.
(function () {
const f = document.currentScript.previousElementSibling;
if (!f || !f.classList.contains("serve-form")) return;
const out = f.querySelector(".serve-summary"), ok = f.querySelector(".cpu-ok");
const v = (n) => (f.elements[n] ? f.elements[n].value.trim() : "");
const engine = { llamacpp: "llama.cpp", vllm: "vLLM", sglang: "SGLang" }[f.dataset.engine] || f.dataset.engine;
function draw() {
const place = v("placement"), gpus = parseInt(v("gpus"), 10) || 1;
const onCPU = place === "cpu" || place === "";
let where = "on the CPU";
if (!onCPU) {
const cls = place.replace(/^gpu:/, "");
where = gpus > 1 ? "on " + gpus + " " + cls + " GPUs of one node" : "on 1 GPU " + cls;
}
const parts = [engine + " " + where];
if (v("min_memory_gb")) parts.push(v("min_memory_gb") + " GB of host memory");
if (onCPU && v("cpu_cores")) parts.push(v("cpu_cores") + " cores");
const pool = parseInt(v("ctx_size"), 10), par = parseInt(v("parallel"), 10);
if (pool && par) parts.push("context pool " + pool + " over " + par + " parallel requests (" + Math.floor(pool / par) + " each)");
else if (pool) parts.push("context pool " + pool);
out.textContent = "Will run as: " + parts.join(", ") + ".";
const warn = onCPU && +f.dataset.freeGpus > 0;
out.classList.toggle("error", warn);
if (warn) out.textContent += " " + f.dataset.freeGpus + " GPUs are free; on the CPU this runs many times slower.";
if (ok) ok.hidden = !onCPU;
out.hidden = false;
}
f.addEventListener("input", draw);
f.addEventListener("change", draw);
draw();
})();
</script>
<p class="note">The context pool is what the engine is given; llama.cpp divides
it across its parallel slots, so the number one request may use is the smaller
one above.</p>