Let the mirror's serve form place a model on a GPU
The form offered no placement and named every GGUF profile cpu. It now lists the fleet's GPU classes (the freest chosen) and the CPU, a GPU count and parallel requests, names the profile from the placement, shows what the model will run as, and refuses the CPU while GPUs are free unless that is ticked. Refs #2 (item 1). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
cbd5521f4f
commit
cba3e329fc
5 files changed
+364
-11
No files matched your search
@@ -71,6 +71,8 @@ type fakeFleet struct {
|
||||
gpus int
|
||||
engines []string
|
||||
memoryGB int
|
||||
// used is how many of the cards replicas hold.
|
||||
used int
|
||||
}
|
||||
|
||||
func (f fakeFleet) Nodes() []adminapi.NodeView {
|
||||
@@ -81,7 +83,12 @@ func (f fakeFleet) Nodes() []adminapi.NodeView {
|
||||
func (f fakeFleet) Models() []adminapi.ModelView { return nil }
|
||||
func (f fakeFleet) Replicas(string) []adminapi.ReplicaView { return nil }
|
||||
func (f fakeFleet) Explain(string) adminapi.ExplainView { return adminapi.ExplainView{} }
|
||||
func (f fakeFleet) Summary() adminapi.Summary { return adminapi.Summary{} }
|
||||
func (f fakeFleet) Summary() adminapi.Summary {
|
||||
if f.gpus == 0 {
|
||||
return adminapi.Summary{}
|
||||
}
|
||||
return adminapi.Summary{GPUs: []adminapi.ClassUse{{Class: "t4", Total: f.gpus, Used: f.used}}}
|
||||
}
|
||||
|
||||
func held(model, version, file string) mirror.Status {
|
||||
return mirror.Status{Model: model, Version: version, File: file,
|
||||
@@ -452,3 +459,90 @@ func TestServingOnDemandLoadsWhenAskedFor(t *testing.T) {
|
||||
t.Errorf("on demand was added as %+v", sc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGGUFOnAFleetWithGPUsIsOfferedTheGPUFirst(t *testing.T) {
|
||||
gguf := []mirror.Status{held("coder", "v1", "coder-q4_k_m.gguf")}
|
||||
body := mirrorGet(t, servePage(t, gguf, &fakeCatalog{}, 4, "llamacpp"),
|
||||
"/admin/mirror?serve=coder:v1").Body.String()
|
||||
// Found in the field: no choice offered, a profile called "cpu", and a
|
||||
// model on the CPU of a GPU cluster at a tenth of the speed.
|
||||
if !strings.Contains(body, `<option value="gpu:t4" selected>GPU: t4, 4 of 4 free</option>`) {
|
||||
t.Errorf("the free GPU class should be offered and chosen; got %s", body)
|
||||
}
|
||||
if !strings.Contains(body, `<option value="cpu">CPU only`) {
|
||||
t.Error("the CPU should still be on offer, unselected")
|
||||
}
|
||||
if !strings.Contains(body, `name="cpu_ok"`) {
|
||||
t.Error("choosing the CPU with GPUs free needs saying so")
|
||||
}
|
||||
|
||||
cpuOnly := mirrorGet(t, servePage(t, gguf, &fakeCatalog{}, 0, "llamacpp"),
|
||||
"/admin/mirror?serve=coder:v1").Body.String()
|
||||
if !strings.Contains(cpuOnly, `<option value="cpu" selected>CPU: this fleet has no GPU</option>`) ||
|
||||
strings.Contains(cpuOnly, `name="cpu_ok"`) {
|
||||
t.Errorf("a fleet without GPUs has only the CPU, and nothing to confirm; got %s", cpuOnly)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheChosenPlacementDecidesWhereItRuns(t *testing.T) {
|
||||
form := func(engine, place, gpus string) url.Values {
|
||||
return url.Values{
|
||||
"name": {"m"}, "source": {"http://x/m"}, "engine": {engine}, "replicas": {"1"},
|
||||
"context_length": {"4096"}, "ctx_size": {"16384"}, "parallel": {"4"},
|
||||
"placement": {place}, "gpus": {gpus}, "min_memory_gb": {"8"},
|
||||
}
|
||||
}
|
||||
m, err := buildModel(form("llamacpp", "gpu:t4", "4").Get)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := m.Profiles[0]
|
||||
if p.GPUClass != "t4" || p.GPUsPerNode != 4 || p.TP > 1 || p.Name != "t4-x4" {
|
||||
t.Errorf("llama.cpp on four t4s: %+v", p)
|
||||
}
|
||||
if got := describeServe(m); got != "llama.cpp on 4 t4 GPUs of one node, 8 GB of host memory, "+
|
||||
"context pool 16384 over 4 parallel requests (4096 each)" {
|
||||
t.Errorf("summary %q", got)
|
||||
}
|
||||
if m, _ := buildModel(form("vllm", "gpu:t4", "2").Get); m.Profiles[0].TP != 2 || m.Profiles[0].GPUsPerNode != 0 {
|
||||
t.Errorf("vLLM's GPU count is tp: %+v", m.Profiles[0])
|
||||
}
|
||||
if m, _ := buildModel(form("llamacpp", "cpu", "1").Get); m.Profiles[0].GPUClass != "" || m.Profiles[0].Name != "cpu" {
|
||||
t.Errorf("the CPU: %+v", m.Profiles[0])
|
||||
}
|
||||
if _, err := buildModel(form("vllm", "cpu", "1").Get); err == nil {
|
||||
t.Error("vLLM on the CPU was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheCPUWithGPUsFreeIsAChoiceMadeOnPurpose(t *testing.T) {
|
||||
values := func(extra ...string) url.Values {
|
||||
v := url.Values{
|
||||
"csrf": {"tok"}, "name": {"coder"}, "source": {reachableSource(t)}, "engine": {"llamacpp"},
|
||||
"replicas": {"1"}, "context_length": {"4096"}, "placement": {"cpu"}, "min_memory_gb": {"8"},
|
||||
}
|
||||
for i := 0; i+1 < len(extra); i += 2 {
|
||||
v.Set(extra[i], extra[i+1])
|
||||
}
|
||||
return v
|
||||
}
|
||||
cat := &fakeCatalog{}
|
||||
page := mirrorPageDeps(t, MirrorDeps{Mirror: &fakeMirror{}, Catalog: cat,
|
||||
Fleet: fakeFleet{gpus: 4, used: 1, engines: []string{"llamacpp"}}}, true)
|
||||
body := mirrorPost(t, page, "/admin/mirror/serve", values()).Body.String()
|
||||
if len(cat.added) != 0 || !strings.Contains(body, "3 GPUs free") {
|
||||
t.Fatalf("the CPU with three GPUs free went through unasked: %s", body)
|
||||
}
|
||||
mirrorPost(t, page, "/admin/mirror/serve", values("cpu_ok", "yes"))
|
||||
if len(cat.added) != 1 {
|
||||
t.Fatal("ticking the box should add it")
|
||||
}
|
||||
// Every card held: nothing to ask.
|
||||
cat = &fakeCatalog{}
|
||||
full := mirrorPageDeps(t, MirrorDeps{Mirror: &fakeMirror{}, Catalog: cat,
|
||||
Fleet: fakeFleet{gpus: 4, used: 4, engines: []string{"llamacpp"}}}, true)
|
||||
mirrorPost(t, full, "/admin/mirror/serve", values())
|
||||
if len(cat.added) != 1 {
|
||||
t.Error("with no GPU free the CPU is the only choice and should not be questioned")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user