Files
llmgw/internal/web/admin_serve_test.go
T
Ilker ManapandClaude Opus 5 e0ea4f6be9
CI / test (push) Successful in 1m41s
Fill in the memory a model needs, instead of leaving the field blank
Serving a model from the mirror page left "Needs RAM (GB)" optional and empty.
Empty writes no constraint, and a CPU profile has no card to be short of, so
every ready node became a candidate — and a 4.68 GB model went onto a 7.9 GB
machine and left it 3.5 GB for a KV pool. Placement was not wrong; it was never
told what the model needed.

The mirror knows the size of the weights, so the field arrives filled in:
weights plus half again for the KV pool, plus two for the engine. The half is
there because the cache grows with the same things the weights do, layers and
heads. It is an estimate and the field is still editable — what it is not is
blank.

A 1.12 GB model asks for 3 GB, so the small nodes stay usable. Pinning every
model to the one large machine would be a different way of getting placement
wrong.

An unknown size fills in nothing rather than the floor of two, which would
exclude no node while reading as though somebody had measured it.

And when the estimate is larger than any node in the fleet, the page says so,
with the largest node's memory. Better read there than discovered by a node
being OOM-killed twenty minutes into a load.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-27 16:38:49 +02:00

298 lines
11 KiB
Go

package web
import (
"context"
"errors"
"net/http"
"net/url"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
)
type fakeCatalog struct {
added []catalog.Model
declared string
err error
}
func (f *fakeCatalog) Add(_ context.Context, m catalog.Model, _ string) error {
if f.err != nil {
return f.err
}
f.added = append(f.added, m)
return nil
}
func (f *fakeCatalog) Declared(name string) bool { return name == f.declared }
// fleet reports a cluster with a given number of cards and engines.
type fakeFleet struct {
gpus int
engines []string
memoryGB int
}
func (f fakeFleet) Nodes() []adminapi.NodeView {
return []adminapi.NodeView{
{ID: "n1", GPUTotal: f.gpus, Engines: f.engines, MemoryGB: f.memoryGB},
}
}
func (f fakeFleet) Models() []adminapi.ModelView { return nil }
func (f fakeFleet) Replicas(string) []adminapi.ReplicaView { return nil }
func (f fakeFleet) Explain(string) adminapi.ExplainView { return adminapi.ExplainView{} }
func (f fakeFleet) Summary() adminapi.Summary { return adminapi.Summary{} }
func held(model, version, file string) mirror.Status {
return mirror.Status{Model: model, Version: version, File: file,
State: mirror.Present, BytesDone: 10, BytesTotal: 10}
}
func servePage(t *testing.T, models []mirror.Status, cat CatalogAdmin, gpus int,
engines ...string,
) *http.ServeMux {
t.Helper()
return mirrorPageDeps(t, MirrorDeps{
Mirror: &fakeMirror{models: models}, Catalog: cat,
Fleet: fakeFleet{gpus: gpus, engines: engines},
}, true)
}
func TestAGGUFCanBeServedStraightFromTheList(t *testing.T) {
cat := &fakeCatalog{}
body := mirrorGet(t, servePage(t, []mirror.Status{
held("qwen2.5-coder-7b", "c03e6d35", "qwen-q4_k_m.gguf")}, cat, 0),
"/admin/mirror?serve=qwen2.5-coder-7b:c03e6d35").Body.String()
// The mirror already knows the name, the URL and what reads that kind of
// file; leaving an operator to write YAML for it was the gap.
if !strings.Contains(body, `name="engine" value="llamacpp"`) {
t.Errorf("the engine should be inferred from the extension; got %s", body)
}
if !strings.Contains(body, "/qwen2.5-coder-7b/c03e6d35/qwen-q4_k_m.gguf") {
t.Error("the source should be the mirror's own URL")
}
}
func TestAShardedVersionIsRefusedWithTheReason(t *testing.T) {
models := []mirror.Status{
held("llama", "v1", "model-00001-of-00002.safetensors"),
held("llama", "v1", "model-00002-of-00002.safetensors"),
}
body := mirrorGet(t, servePage(t, models, &fakeCatalog{}, 8),
"/admin/mirror?serve=llama:v1").Body.String()
// A catalogue entry names one source, so an entry for a sharded version is
// an entry for one shard — which loads as a broken model rather than
// failing to load.
if !strings.Contains(body, "a catalogue entry names one source") {
t.Errorf("the refusal should say why; got %s", body)
}
if strings.Contains(body, `action="/admin/mirror/serve"`) {
t.Error("no form should be offered for something that cannot be served")
}
}
func TestSafetensorsOnACPUFleetWarnsBeforeAnybodyTries(t *testing.T) {
body := mirrorGet(t, servePage(t, []mirror.Status{
held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 0, "llamacpp"),
"/admin/mirror?serve=qwen:v1").Body.String()
// Fifteen gigabytes arrived before anybody noticed the fleet had no card
// and no vLLM to run them with. The page can say so.
if !strings.Contains(body, "no node reports having vllm installed") {
t.Errorf("expected the warning; got %s", body)
}
// And the way out, which on a fleet with no cards is a different build of
// the same model rather than installing vLLM.
if !strings.Contains(body, "GGUF build of the same model") {
t.Error("a fleet with no GPU should be pointed at a GGUF")
}
// A warning, not a refusal: a card could be added tomorrow.
if !strings.Contains(body, `action="/admin/mirror/serve"`) {
t.Error("it should still be possible to add it")
}
}
func TestSafetensorsOnAGPUFleetIsNotWarnedAbout(t *testing.T) {
body := mirrorGet(t, servePage(t, []mirror.Status{
held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 8, "vllm"),
"/admin/mirror?serve=qwen:v1").Body.String()
if strings.Contains(body, "no node reports having") {
t.Error("a fleet that runs vllm should not be warned about")
}
if !strings.Contains(body, `value="vllm"`) {
t.Error("safetensors should propose vllm")
}
}
func TestAModelDeclaredInConfigurationIsNotOfferedTwice(t *testing.T) {
cat := &fakeCatalog{declared: "qwen"}
body := mirrorGet(t, servePage(t, []mirror.Status{
held("qwen", "v1", "m.gguf")}, cat, 0),
"/admin/mirror?serve=qwen:v1").Body.String()
if !strings.Contains(body, "declared in configuration") {
t.Errorf("expected the refusal; got %s", body)
}
}
func TestServingBuildsTheEntryFromTheForm(t *testing.T) {
cat := &fakeCatalog{}
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
"csrf": {"tok"}, "name": {"coder"}, "aliases": {"c7b, coder-7b"},
"source": {"http://mirror:8081/coder/v1/m.gguf"}, "engine": {"llamacpp"},
"replicas": {"3"}, "context_length": {"4096"}, "max_output": {"1024"},
"ctx_size": {"16384"}, "profile": {"cpu-large"},
"min_memory_gb": {"8"}, "selector_key": {"role"}, "selector_value": {"cpu-large"},
})
if len(cat.added) != 1 {
t.Fatalf("expected one model added, got %d", len(cat.added))
}
m := cat.added[0]
if m.Name != "coder" || len(m.Aliases) != 2 {
t.Errorf("name and aliases did not arrive: %+v", m)
}
if m.Scaling.Min != 3 || m.Scaling.Max != 3 {
t.Errorf("expected three replicas, got %+v", m.Scaling)
}
if len(m.Profiles) != 1 {
t.Fatalf("expected one profile, got %+v", m.Profiles)
}
p := m.Profiles[0]
if p.Engine != "llamacpp" || p.Name != "cpu-large" || p.MinMemoryGB != 8 {
t.Errorf("the profile is wrong: %+v", p)
}
// The label is what keeps a large model off the small machines.
if p.NodeSelector["role"] != "cpu-large" {
t.Errorf("the node selector was lost: %+v", p.NodeSelector)
}
if strings.Join(p.Args, " ") != "--ctx-size 16384" {
t.Errorf("the engine's context pool was lost: %v", p.Args)
}
}
func TestRubbishInTheFormIsRefusedBeforeTheCatalog(t *testing.T) {
cat := &fakeCatalog{}
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
"csrf": {"tok"}, "name": {"coder"}, "source": {"http://x/m.gguf"},
"engine": {"llamacpp"}, "replicas": {"lots"}, "context_length": {"4096"},
})
if len(cat.added) != 0 {
t.Error("nothing should have reached the catalog")
}
}
func TestTheCatalogsOwnRefusalReachesTheOperator(t *testing.T) {
cat := &fakeCatalog{err: errors.New("that name collides with an alias")}
body := mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
"csrf": {"tok"}, "name": {"coder"}, "source": {"http://x/m.gguf"},
"engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"},
}).Body.String()
if !strings.Contains(body, "collides with an alias") {
t.Errorf("the catalog's reason should be shown; got %s", body)
}
}
func TestWithNoCatalogThereIsNoServeAction(t *testing.T) {
body := mirrorGet(t, mirrorPageDeps(t, MirrorDeps{
Mirror: &fakeMirror{models: []mirror.Status{held("qwen", "v1", "m.gguf")}},
}, true), "/admin/mirror").Body.String()
// Without a store a model added at runtime would not survive a restart, so
// the action is absent rather than offered and then refused.
if strings.Contains(body, "?serve=") {
t.Error("serving should not be offered without somewhere to persist it")
}
}
func TestAGPUNodeWithoutVLLMIsStillWarnedAbout(t *testing.T) {
// The first version of this checked cards, which would have waved through
// a fleet of eight-GPU nodes that had never had vLLM installed. Agents
// report their engines now, so it is a fact.
body := mirrorGet(t, servePage(t, []mirror.Status{
held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 8, "llamacpp"),
"/admin/mirror?serve=qwen:v1").Body.String()
if !strings.Contains(body, "no node reports having vllm installed") {
t.Errorf("cards are not the question; the engine is. Got %s", body)
}
}
func TestAFleetOfOlderAgentsIsNotToldEverythingIsUnservable(t *testing.T) {
// An agent older than the engine probe reports nothing, which is not the
// same as having no engines. Placement makes the same allowance, and the
// two disagreeing would be worse than either.
body := mirrorGet(t, servePage(t, []mirror.Status{
held("qwen", "v1", "m.gguf")}, &fakeCatalog{}, 0),
"/admin/mirror?serve=qwen:v1").Body.String()
if strings.Contains(body, "no node reports having") {
t.Error("silence from older agents must not read as a refusal")
}
}
func sized(model, version, file string, gb float64) mirror.Status {
n := int64(gb * (1 << 30))
return mirror.Status{Model: model, Version: version, File: file,
State: mirror.Present, BytesDone: n, BytesTotal: n}
}
func TestTheMemoryFieldIsFilledInFromTheWeights(t *testing.T) {
// The real case: 4.68 GB of weights went onto a 7.9 GB node and left it
// 3.5 GB for a KV pool, because the field was optional and blank. Blank
// writes no constraint, and a CPU profile has no card to be short of.
body := mirrorGet(t, servePage(t, []mirror.Status{
sized("coder", "v1", "m.gguf", 4.68)}, &fakeCatalog{}, 0, "llamacpp"),
"/admin/mirror?serve=coder:v1").Body.String()
if !strings.Contains(body, `name="min_memory_gb" value="9"`) {
t.Errorf("expected a figure derived from 4.68 GB of weights; got %s", body)
}
}
func TestASmallModelDoesNotDemandALargeNode(t *testing.T) {
// The estimate has to leave the small nodes usable, or every model ends up
// pinned to the one big machine.
body := mirrorGet(t, servePage(t, []mirror.Status{
sized("small", "v1", "m.gguf", 1.12)}, &fakeCatalog{}, 0, "llamacpp"),
"/admin/mirror?serve=small:v1").Body.String()
if !strings.Contains(body, `name="min_memory_gb" value="3"`) {
t.Errorf("a 1.12 GB model should fit a small node; got %s", body)
}
}
func TestAModelTooBigForEveryNodeSaysSo(t *testing.T) {
page := mirrorPageDeps(t, MirrorDeps{
Mirror: &fakeMirror{models: []mirror.Status{sized("huge", "v1", "m.gguf", 40)}},
Catalog: &fakeCatalog{},
Fleet: fakeFleet{gpus: 0, engines: []string{"llamacpp"}, memoryGB: 31},
}, true)
body := mirrorGet(t, page, "/admin/mirror?serve=huge:v1").Body.String()
// Better said here than discovered by a node being OOM-killed twenty
// minutes into a load.
if !strings.Contains(body, "largest node has 31 GB") {
t.Errorf("expected the size warning; got %s", body)
}
}
func TestAnEstimateIsNotMadeWhenTheSizeIsUnknown(t *testing.T) {
// A version whose size nobody reported would otherwise be given a
// requirement of two, which is a constraint that excludes nothing and
// reads as though somebody measured it.
body := mirrorGet(t, servePage(t, []mirror.Status{
{Model: "m", Version: "v", File: "f.gguf", State: mirror.Present}},
&fakeCatalog{}, 0, "llamacpp"), "/admin/mirror?serve=m:v").Body.String()
if !strings.Contains(body, `name="min_memory_gb" value="0"`) {
t.Errorf("an unknown size should leave no figure; got %s", body)
}
}