Let the mirror's serve form place a model on a GPU

The form offered no placement and named every GGUF profile cpu. It now
lists the fleet's GPU classes (the freest chosen) and the CPU, a GPU
count and parallel requests, names the profile from the placement, shows
what the model will run as, and refuses the CPU while GPUs are free
unless that is ticked.

Refs #2 (item 1).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-01 11:59:22 +02:00
1 parent cbd5521f4f
commit cba3e329fc
5 files changed
+364 -11

No files matched your search

+182 -6
View File
@@ -43,8 +43,20 @@ type serveForm struct {
Engine string
// Profile is the profile name, which doubles as how a node_selector would
// be written later. Named for the hardware rather than the model, because
// a profile is about where a model runs.
// a profile is about where a model runs. Derived from the placement, not
// typed: a name is only a name, and a field for it read as though calling
// a profile "gpu" would put it on one.
Profile string
// Placements are where it can run: each GPU class the fleet has, and for
// llama.cpp the CPU. One is selected.
Placements []placeChoice
// GPUs is how many cards a replica takes, when placed on a GPU class.
GPUs int
// Parallel is how many requests one replica serves at once.
Parallel int
// FreeGPUs is how many cards no replica holds, across the fleet: what
// choosing the CPU leaves unused.
FreeGPUs int
// Files is how many files the version holds, above one for a set.
Files int
// MinMemoryGB is the host RAM to ask for, estimated from the weights.
@@ -126,6 +138,13 @@ func planServe(st mirror.Status, files []mirror.Status, base string,
default:
f.Profile = "gpu"
}
f.GPUs, f.Parallel = 1, defaultParallel
f.Placements, f.FreeGPUs = placementsFor(fleet, f.Engine)
for _, pc := range f.Placements {
if pc.Selected {
f.Profile = pc.profile()
}
}
// Whether any node can actually run it. Agents report their installed
// engines now, so this is a fact rather than the guess it started as —
// which was cards, and would have passed a GPU node with no vLLM on it.
@@ -274,12 +293,38 @@ func buildModel(get func(string) string) (catalog.Model, error) {
if engine == "" {
return m, errors.New("which engine should run it?")
}
p := catalog.Profile{
Name: strings.TrimSpace(get("profile")), Engine: engine,
Weight: 1, Priority: 100,
p := catalog.Profile{Engine: engine, Weight: 1, Priority: 100}
gpus := 1
if v := strings.TrimSpace(get("gpus")); v != "" {
if gpus, err = positive(v, "the GPUs per replica"); err != nil {
return m, err
}
gpus = max(gpus, 1)
}
if p.Name == "" {
p.Name = engine
place := placeChoice{Value: strings.TrimSpace(get("placement"))}
switch class, onGPU := strings.CutPrefix(place.Value, "gpu:"); {
case onGPU && class != "":
// Placement is decided by gpu_class, which is what the selector
// sets; the count is gpus_per_node for llama.cpp, which spreads
// layers over a node's cards, and tp for the others.
p.GPUClass = class
if engine == "llamacpp" {
p.GPUsPerNode = gpus
} else {
p.TP = gpus
}
place.GPUs = gpus
case place.Value == "" || place.Value == "cpu":
if engine != "llamacpp" {
return m, fmt.Errorf("%s runs on GPUs; choose a GPU class to place it on", engine)
}
place.Value = "cpu"
default:
return m, fmt.Errorf("%q is not a placement", place.Value)
}
p.Name = place.profile()
if v := strings.TrimSpace(get("profile")); v != "" {
p.Name = v
}
// The pool, not the per-request limit: llama.cpp divides it across its
// parallel slots, so the number a client may use is smaller and is what
@@ -290,6 +335,15 @@ func buildModel(get func(string) string) (catalog.Model, error) {
}
p.Args = []string{"--ctx-size", v}
}
if v := strings.TrimSpace(get("parallel")); v != "" && engine == "llamacpp" {
n, err := positive(v, "the parallel requests")
if err != nil {
return m, err
}
if n > 0 {
p.Args = append(p.Args, "--parallel", strconv.Itoa(n))
}
}
if v := strings.TrimSpace(get("min_memory_gb")); v != "" {
if p.MinMemoryGB, err = positive(v, "the memory requirement"); err != nil {
return m, err
@@ -408,3 +462,125 @@ func modeOf(sc catalog.Scaling) string {
// defaultCtxPool is the llama.cpp context pool the serve form starts with.
// --parallel divides it among slots; it does not multiply it.
const defaultCtxPool = 16384
// defaultParallel is the requests a llama.cpp replica serves at once when
// the form is not changed: the agent's own default.
const defaultParallel = 4
// placeChoice is one place a model can run: a GPU class, or the CPU.
type placeChoice struct {
// Value is "gpu:<class>" or "cpu", as the form submits it.
Value string
Label string
// GPUs is how many cards a replica takes, for naming the profile.
GPUs int
Selected bool
}
// profile names the profile for where it runs: the class, with the count when
// it is more than one card ("t4", "t4-x4"), or "cpu".
func (c placeChoice) profile() string {
class, ok := strings.CutPrefix(c.Value, "gpu:")
if !ok || class == "" {
return "cpu"
}
if c.GPUs > 1 {
return fmt.Sprintf("%s-x%d", class, c.GPUs)
}
return class
}
// placementsFor lists where engine can run on this fleet, with one selected,
// and how many GPUs are free.
//
// A GPU when the fleet has one: a GGUF model was put on the CPU by this form
// on a cluster of GPUs, ran at a tenth of their speed, and nothing on the way
// said why — the form had no choice to offer and named the profile "cpu". The
// class selected is the one with the most cards free.
func placementsFor(fleet Source, engine string) ([]placeChoice, int) {
var out []placeChoice
free, best := 0, -1
if fleet != nil {
for _, c := range fleet.Summary().GPUs {
n := c.Free()
free += n
out = append(out, placeChoice{
Value: "gpu:" + c.Class,
Label: fmt.Sprintf("GPU: %s, %d of %d free", c.Class, n, c.Total),
})
if best < 0 || n > fleet.Summary().GPUs[best].Free() {
best = len(out) - 1
}
}
}
if best >= 0 {
out[best].Selected = true
}
if engine == "llamacpp" {
label := "CPU only: no GPU, many times slower"
if len(out) == 0 {
label = "CPU: this fleet has no GPU"
}
out = append(out, placeChoice{Value: "cpu", Label: label, Selected: best < 0})
}
return out, free
}
// freeGPUs is how many cards no replica holds across the fleet.
func freeGPUs(fleet Source) int {
if fleet == nil {
return 0
}
n := 0
for _, c := range fleet.Summary().GPUs {
n += c.Free()
}
return n
}
// describeServe says, in one line, what a model will run as: the summary
// shown before it is added and in the message after.
func describeServe(m catalog.Model) string {
if len(m.Profiles) == 0 {
return ""
}
p := m.Profiles[0]
engine := map[string]string{"llamacpp": "llama.cpp", "vllm": "vLLM", "sglang": "SGLang"}[p.Engine]
if engine == "" {
engine = p.Engine
}
where := "on the CPU"
if p.GPUClass != "" {
n := max(p.GPUsPerNode, p.TP, 1)
where = fmt.Sprintf("on %s %s", plural(n, "GPU", "GPUs"), p.GPUClass)
if n > 1 {
where = fmt.Sprintf("on %d %s GPUs of one node", n, p.GPUClass)
}
}
parts := []string{engine + " " + where}
if p.MinMemoryGB > 0 {
parts = append(parts, fmt.Sprintf("%d GB of host memory", p.MinMemoryGB))
}
if p.GPUClass == "" && p.CPUCores > 0 {
parts = append(parts, plural(p.CPUCores, "core", "cores"))
}
pool, parallel := argInt(p.Args, "--ctx-size"), argInt(p.Args, "--parallel")
switch {
case pool > 0 && parallel > 0:
parts = append(parts, fmt.Sprintf("context pool %d over %d parallel requests (%d each)", pool, parallel, pool/parallel))
case pool > 0:
parts = append(parts, fmt.Sprintf("context pool %d", pool))
}
return strings.Join(parts, ", ")
}
// argInt is the number after flag in args, or 0.
func argInt(args []string, flag string) int {
for i := 0; i+1 < len(args); i++ {
if args[i] == flag {
n, _ := strconv.Atoi(args[i+1])
return n
}
}
return 0
}