diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ad710d3..25877815 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,28 @@ as nvidia-smi does, by PCI bus, and CUDA's own order is fastest first — on a node with two kinds of GPU, a shard could get a different card from the one it was given. +### Serving from the mirror says where a model will run + +The serve form put every GGUF model on the CPU: it offered no choice, and its +"Profile name" field, pre-filled "cpu", read as though renaming it would move +the model to a GPU. Placement is decided by `gpu_class`, which the form never +set. Now: + +- **Runs on** lists the fleet's GPU classes with how many cards are free, and + for llama.cpp the CPU. On a fleet with GPUs the class with the most free + cards is chosen. +- **GPUs per replica** sets how many cards a replica takes on one node + (`gpus_per_node` for llama.cpp, `tp` for vLLM), and **parallel requests** + how many a llama.cpp replica serves at once. +- The profile is named for where it runs (`t4`, `t4-x4`, `cpu`) rather than + typed. +- A line under the form says what the model will run as — engine, CPU or how + many of which GPU, memory, context pool and how it is divided — and turns + red when the CPU is chosen with GPUs free. The message after adding repeats + it. +- Choosing the CPU while GPUs are free is refused unless **run on the CPU + anyway** is ticked. + ### llama.cpp on several GPUs of one node A llama.cpp profile takes several of a node's GPUs with `gpus_per_node: N`: diff --git a/internal/web/admin_mirror.go b/internal/web/admin_mirror.go index 611615bf..71dd736b 100644 --- a/internal/web/admin_mirror.go +++ b/internal/web/admin_mirror.go @@ -317,6 +317,16 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) { if err != nil { return "", err } + // The CPU while GPUs sit free is a choice to make on purpose: + // made by default, it was a model running at a tenth of the + // speed with nothing saying why. + if model.Profiles[0].CPUOnly() && r.FormValue("cpu_ok") == "" { + if n := freeGPUs(p.fleet); n > 0 { + return "", fmt.Errorf("this fleet has %s free and %s was set to run on the CPU; "+ + "choose a GPU placement, or tick \"run on the CPU anyway\"", + plural(n, "GPU", "GPUs"), model.Name) + } + } // The same check `llmgw ctl model add` makes. This page builds // the source out of a row the mirror reported, so it is right // far more often than something typed — but a file removed @@ -332,8 +342,8 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) { // Placed on the next pass, not now: saying it is serving would // be wrong for however long the weights take to reach a node. return fmt.Sprintf( - "%s added to the catalog, %s; it is placed on the next pass", - model.Name, plural(model.Scaling.Min, "replica", "replicas")), nil + "%s added to the catalog, %s: %s; it is placed on the next pass", + model.Name, plural(model.Scaling.Min, "replica", "replicas"), describeServe(model)), nil })) } diff --git a/internal/web/admin_serve.go b/internal/web/admin_serve.go index a154b8a1..6933240f 100644 --- a/internal/web/admin_serve.go +++ b/internal/web/admin_serve.go @@ -43,8 +43,20 @@ type serveForm struct { Engine string // Profile is the profile name, which doubles as how a node_selector would // be written later. Named for the hardware rather than the model, because - // a profile is about where a model runs. + // a profile is about where a model runs. Derived from the placement, not + // typed: a name is only a name, and a field for it read as though calling + // a profile "gpu" would put it on one. Profile string + // Placements are where it can run: each GPU class the fleet has, and for + // llama.cpp the CPU. One is selected. + Placements []placeChoice + // GPUs is how many cards a replica takes, when placed on a GPU class. + GPUs int + // Parallel is how many requests one replica serves at once. + Parallel int + // FreeGPUs is how many cards no replica holds, across the fleet: what + // choosing the CPU leaves unused. + FreeGPUs int // Files is how many files the version holds, above one for a set. Files int // MinMemoryGB is the host RAM to ask for, estimated from the weights. @@ -126,6 +138,13 @@ func planServe(st mirror.Status, files []mirror.Status, base string, default: f.Profile = "gpu" } + f.GPUs, f.Parallel = 1, defaultParallel + f.Placements, f.FreeGPUs = placementsFor(fleet, f.Engine) + for _, pc := range f.Placements { + if pc.Selected { + f.Profile = pc.profile() + } + } // Whether any node can actually run it. Agents report their installed // engines now, so this is a fact rather than the guess it started as — // which was cards, and would have passed a GPU node with no vLLM on it. @@ -274,12 +293,38 @@ func buildModel(get func(string) string) (catalog.Model, error) { if engine == "" { return m, errors.New("which engine should run it?") } - p := catalog.Profile{ - Name: strings.TrimSpace(get("profile")), Engine: engine, - Weight: 1, Priority: 100, + p := catalog.Profile{Engine: engine, Weight: 1, Priority: 100} + gpus := 1 + if v := strings.TrimSpace(get("gpus")); v != "" { + if gpus, err = positive(v, "the GPUs per replica"); err != nil { + return m, err + } + gpus = max(gpus, 1) } - if p.Name == "" { - p.Name = engine + place := placeChoice{Value: strings.TrimSpace(get("placement"))} + switch class, onGPU := strings.CutPrefix(place.Value, "gpu:"); { + case onGPU && class != "": + // Placement is decided by gpu_class, which is what the selector + // sets; the count is gpus_per_node for llama.cpp, which spreads + // layers over a node's cards, and tp for the others. + p.GPUClass = class + if engine == "llamacpp" { + p.GPUsPerNode = gpus + } else { + p.TP = gpus + } + place.GPUs = gpus + case place.Value == "" || place.Value == "cpu": + if engine != "llamacpp" { + return m, fmt.Errorf("%s runs on GPUs; choose a GPU class to place it on", engine) + } + place.Value = "cpu" + default: + return m, fmt.Errorf("%q is not a placement", place.Value) + } + p.Name = place.profile() + if v := strings.TrimSpace(get("profile")); v != "" { + p.Name = v } // The pool, not the per-request limit: llama.cpp divides it across its // parallel slots, so the number a client may use is smaller and is what @@ -290,6 +335,15 @@ func buildModel(get func(string) string) (catalog.Model, error) { } p.Args = []string{"--ctx-size", v} } + if v := strings.TrimSpace(get("parallel")); v != "" && engine == "llamacpp" { + n, err := positive(v, "the parallel requests") + if err != nil { + return m, err + } + if n > 0 { + p.Args = append(p.Args, "--parallel", strconv.Itoa(n)) + } + } if v := strings.TrimSpace(get("min_memory_gb")); v != "" { if p.MinMemoryGB, err = positive(v, "the memory requirement"); err != nil { return m, err @@ -408,3 +462,125 @@ func modeOf(sc catalog.Scaling) string { // defaultCtxPool is the llama.cpp context pool the serve form starts with. // --parallel divides it among slots; it does not multiply it. const defaultCtxPool = 16384 + +// defaultParallel is the requests a llama.cpp replica serves at once when +// the form is not changed: the agent's own default. +const defaultParallel = 4 + +// placeChoice is one place a model can run: a GPU class, or the CPU. +type placeChoice struct { + // Value is "gpu:" or "cpu", as the form submits it. + Value string + Label string + // GPUs is how many cards a replica takes, for naming the profile. + GPUs int + Selected bool +} + +// profile names the profile for where it runs: the class, with the count when +// it is more than one card ("t4", "t4-x4"), or "cpu". +func (c placeChoice) profile() string { + class, ok := strings.CutPrefix(c.Value, "gpu:") + if !ok || class == "" { + return "cpu" + } + if c.GPUs > 1 { + return fmt.Sprintf("%s-x%d", class, c.GPUs) + } + return class +} + +// placementsFor lists where engine can run on this fleet, with one selected, +// and how many GPUs are free. +// +// A GPU when the fleet has one: a GGUF model was put on the CPU by this form +// on a cluster of GPUs, ran at a tenth of their speed, and nothing on the way +// said why — the form had no choice to offer and named the profile "cpu". The +// class selected is the one with the most cards free. +func placementsFor(fleet Source, engine string) ([]placeChoice, int) { + var out []placeChoice + free, best := 0, -1 + if fleet != nil { + for _, c := range fleet.Summary().GPUs { + n := c.Free() + free += n + out = append(out, placeChoice{ + Value: "gpu:" + c.Class, + Label: fmt.Sprintf("GPU: %s, %d of %d free", c.Class, n, c.Total), + }) + if best < 0 || n > fleet.Summary().GPUs[best].Free() { + best = len(out) - 1 + } + } + } + if best >= 0 { + out[best].Selected = true + } + if engine == "llamacpp" { + label := "CPU only: no GPU, many times slower" + if len(out) == 0 { + label = "CPU: this fleet has no GPU" + } + out = append(out, placeChoice{Value: "cpu", Label: label, Selected: best < 0}) + } + return out, free +} + +// freeGPUs is how many cards no replica holds across the fleet. +func freeGPUs(fleet Source) int { + if fleet == nil { + return 0 + } + n := 0 + for _, c := range fleet.Summary().GPUs { + n += c.Free() + } + return n +} + +// describeServe says, in one line, what a model will run as: the summary +// shown before it is added and in the message after. +func describeServe(m catalog.Model) string { + if len(m.Profiles) == 0 { + return "" + } + p := m.Profiles[0] + engine := map[string]string{"llamacpp": "llama.cpp", "vllm": "vLLM", "sglang": "SGLang"}[p.Engine] + if engine == "" { + engine = p.Engine + } + where := "on the CPU" + if p.GPUClass != "" { + n := max(p.GPUsPerNode, p.TP, 1) + where = fmt.Sprintf("on %s %s", plural(n, "GPU", "GPUs"), p.GPUClass) + if n > 1 { + where = fmt.Sprintf("on %d %s GPUs of one node", n, p.GPUClass) + } + } + parts := []string{engine + " " + where} + if p.MinMemoryGB > 0 { + parts = append(parts, fmt.Sprintf("%d GB of host memory", p.MinMemoryGB)) + } + if p.GPUClass == "" && p.CPUCores > 0 { + parts = append(parts, plural(p.CPUCores, "core", "cores")) + } + pool, parallel := argInt(p.Args, "--ctx-size"), argInt(p.Args, "--parallel") + switch { + case pool > 0 && parallel > 0: + parts = append(parts, fmt.Sprintf("context pool %d over %d parallel requests (%d each)", pool, parallel, pool/parallel)) + case pool > 0: + parts = append(parts, fmt.Sprintf("context pool %d", pool)) + } + return strings.Join(parts, ", ") +} + +// argInt is the number after flag in args, or 0. +func argInt(args []string, flag string) int { + for i := 0; i+1 < len(args); i++ { + if args[i] == flag { + n, _ := strconv.Atoi(args[i+1]) + return n + } + } + return 0 +} diff --git a/internal/web/admin_serve_test.go b/internal/web/admin_serve_test.go index a4274267..91d7049c 100644 --- a/internal/web/admin_serve_test.go +++ b/internal/web/admin_serve_test.go @@ -71,6 +71,8 @@ type fakeFleet struct { gpus int engines []string memoryGB int + // used is how many of the cards replicas hold. + used int } func (f fakeFleet) Nodes() []adminapi.NodeView { @@ -81,7 +83,12 @@ func (f fakeFleet) Nodes() []adminapi.NodeView { func (f fakeFleet) Models() []adminapi.ModelView { return nil } func (f fakeFleet) Replicas(string) []adminapi.ReplicaView { return nil } func (f fakeFleet) Explain(string) adminapi.ExplainView { return adminapi.ExplainView{} } -func (f fakeFleet) Summary() adminapi.Summary { return adminapi.Summary{} } +func (f fakeFleet) Summary() adminapi.Summary { + if f.gpus == 0 { + return adminapi.Summary{} + } + return adminapi.Summary{GPUs: []adminapi.ClassUse{{Class: "t4", Total: f.gpus, Used: f.used}}} +} func held(model, version, file string) mirror.Status { return mirror.Status{Model: model, Version: version, File: file, @@ -452,3 +459,90 @@ func TestServingOnDemandLoadsWhenAskedFor(t *testing.T) { t.Errorf("on demand was added as %+v", sc) } } + +func TestAGGUFOnAFleetWithGPUsIsOfferedTheGPUFirst(t *testing.T) { + gguf := []mirror.Status{held("coder", "v1", "coder-q4_k_m.gguf")} + body := mirrorGet(t, servePage(t, gguf, &fakeCatalog{}, 4, "llamacpp"), + "/admin/mirror?serve=coder:v1").Body.String() + // Found in the field: no choice offered, a profile called "cpu", and a + // model on the CPU of a GPU cluster at a tenth of the speed. + if !strings.Contains(body, ``) { + t.Errorf("the free GPU class should be offered and chosen; got %s", body) + } + if !strings.Contains(body, ``) || + strings.Contains(cpuOnly, `name="cpu_ok"`) { + t.Errorf("a fleet without GPUs has only the CPU, and nothing to confirm; got %s", cpuOnly) + } +} + +func TestTheChosenPlacementDecidesWhereItRuns(t *testing.T) { + form := func(engine, place, gpus string) url.Values { + return url.Values{ + "name": {"m"}, "source": {"http://x/m"}, "engine": {engine}, "replicas": {"1"}, + "context_length": {"4096"}, "ctx_size": {"16384"}, "parallel": {"4"}, + "placement": {place}, "gpus": {gpus}, "min_memory_gb": {"8"}, + } + } + m, err := buildModel(form("llamacpp", "gpu:t4", "4").Get) + if err != nil { + t.Fatal(err) + } + p := m.Profiles[0] + if p.GPUClass != "t4" || p.GPUsPerNode != 4 || p.TP > 1 || p.Name != "t4-x4" { + t.Errorf("llama.cpp on four t4s: %+v", p) + } + if got := describeServe(m); got != "llama.cpp on 4 t4 GPUs of one node, 8 GB of host memory, "+ + "context pool 16384 over 4 parallel requests (4096 each)" { + t.Errorf("summary %q", got) + } + if m, _ := buildModel(form("vllm", "gpu:t4", "2").Get); m.Profiles[0].TP != 2 || m.Profiles[0].GPUsPerNode != 0 { + t.Errorf("vLLM's GPU count is tp: %+v", m.Profiles[0]) + } + if m, _ := buildModel(form("llamacpp", "cpu", "1").Get); m.Profiles[0].GPUClass != "" || m.Profiles[0].Name != "cpu" { + t.Errorf("the CPU: %+v", m.Profiles[0]) + } + if _, err := buildModel(form("vllm", "cpu", "1").Get); err == nil { + t.Error("vLLM on the CPU was accepted") + } +} + +func TestTheCPUWithGPUsFreeIsAChoiceMadeOnPurpose(t *testing.T) { + values := func(extra ...string) url.Values { + v := url.Values{ + "csrf": {"tok"}, "name": {"coder"}, "source": {reachableSource(t)}, "engine": {"llamacpp"}, + "replicas": {"1"}, "context_length": {"4096"}, "placement": {"cpu"}, "min_memory_gb": {"8"}, + } + for i := 0; i+1 < len(extra); i += 2 { + v.Set(extra[i], extra[i+1]) + } + return v + } + cat := &fakeCatalog{} + page := mirrorPageDeps(t, MirrorDeps{Mirror: &fakeMirror{}, Catalog: cat, + Fleet: fakeFleet{gpus: 4, used: 1, engines: []string{"llamacpp"}}}, true) + body := mirrorPost(t, page, "/admin/mirror/serve", values()).Body.String() + if len(cat.added) != 0 || !strings.Contains(body, "3 GPUs free") { + t.Fatalf("the CPU with three GPUs free went through unasked: %s", body) + } + mirrorPost(t, page, "/admin/mirror/serve", values("cpu_ok", "yes")) + if len(cat.added) != 1 { + t.Fatal("ticking the box should add it") + } + // Every card held: nothing to ask. + cat = &fakeCatalog{} + full := mirrorPageDeps(t, MirrorDeps{Mirror: &fakeMirror{}, Catalog: cat, + Fleet: fakeFleet{gpus: 4, used: 4, engines: []string{"llamacpp"}}}, true) + mirrorPost(t, full, "/admin/mirror/serve", values()) + if len(cat.added) != 1 { + t.Error("with no GPU free the CPU is the only choice and should not be questioned") + } +} diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html index 09253572..96e83002 100644 --- a/internal/web/templates/admin_mirror.html +++ b/internal/web/templates/admin_mirror.html @@ -131,7 +131,7 @@ reachable from this machine.

Cannot be served as it stands: {{.Refusal}}.

{{else}} {{if .Warning}}

{{.Warning}}.

{{end}} -
+ @@ -163,8 +163,23 @@ reachable from this machine.

+ {{/* Where it runs is a choice with consequences, so it is a choice: the + GPU classes the fleet has, and the CPU for llama.cpp. A profile name + field stood here once, and reading as though naming a profile "gpu" + would put it on one, it put GGUF models on the CPU of GPU clusters. */}} +
+ + + {{if eq .Engine "llamacpp"}}{{end}} +
+ {{if .FreeGPUs}}{{end}}
-
+
+

The context pool is what the engine is given; llama.cpp divides it across its parallel slots, so the number one request may use is the smaller one above.