Files
llmgw/internal/web/admin_serve_test.go
T
Ilker ManapandClaude Opus 5 dfbb7557dd
CI / test (push) Successful in 1m42s
Serve a model made of several files
The mirror would happily hold a four-shard model that nothing could serve. A
catalog entry names one source, the agent fetched one URL, and so a version of
several files had no way to be described — which is every safetensors model,
and every GGUF too large to ship whole.

A source ending in a slash names a version rather than a file. The agent fetches
every file in it into one directory and hands the engine the directory: vLLM
reads the shards and the config out of the folder, and llama.cpp is given the
first part and finds the rest beside it.

The trailing slash is the whole signal. A flag beside the source would be a
second place for the two to disagree, and a URL already distinguishes a
directory from a file everywhere else.

The mirror answers a version with a manifest instead of a file listing. A plain
file server gives HTML there, and something that has to know what a version
consists of would be left parsing it. It is on the byte listener, not the
management API: fetching weights is the one thing every node does, and making
it depend on the administrative port would undo the split that keeps that port
closeable.

Built from the directory rather than a file written beside it. A manifest on
disk is a second record of what is there, and the two disagree the first time
somebody copies a file in by hand — which the mirror explicitly allows.

The whole set or nothing. A directory with three of four shards loads as a
broken model rather than failing to load, so a completion marker is written
last and an interrupted fetch starts the missing files again rather than being
found and served. Files already there at the right size are not fetched twice.

Progress is reported under the version's address, not each file's: the shards
waiting on it asked for the version, and progress under a file's own URL
reaches nobody.

A filename from the manifest is refused if it contains a separator. The mirror
is a place an operator copies files into by hand, so what it reports is not
automatically safe to join onto a path.

An HTML listing where a manifest was expected says so. "invalid character '<'"
sends somebody looking at the wrong thing.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-27 21:07:26 +02:00

345 lines
13 KiB
Go

package web
import (
"context"
"errors"
"net/http"
"net/url"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
)
type fakeCatalog struct {
added []catalog.Model
declared string
err error
}
func (f *fakeCatalog) Add(_ context.Context, m catalog.Model, _ string) error {
if f.err != nil {
return f.err
}
f.added = append(f.added, m)
return nil
}
func (f *fakeCatalog) Declared(name string) bool { return name == f.declared }
// fleet reports a cluster with a given number of cards and engines.
type fakeFleet struct {
gpus int
engines []string
memoryGB int
}
func (f fakeFleet) Nodes() []adminapi.NodeView {
return []adminapi.NodeView{
{ID: "n1", GPUTotal: f.gpus, Engines: f.engines, MemoryGB: f.memoryGB},
}
}
func (f fakeFleet) Models() []adminapi.ModelView { return nil }
func (f fakeFleet) Replicas(string) []adminapi.ReplicaView { return nil }
func (f fakeFleet) Explain(string) adminapi.ExplainView { return adminapi.ExplainView{} }
func (f fakeFleet) Summary() adminapi.Summary { return adminapi.Summary{} }
func held(model, version, file string) mirror.Status {
return mirror.Status{Model: model, Version: version, File: file,
State: mirror.Present, BytesDone: 10, BytesTotal: 10}
}
func servePage(t *testing.T, models []mirror.Status, cat CatalogAdmin, gpus int,
engines ...string,
) *http.ServeMux {
t.Helper()
return mirrorPageDeps(t, MirrorDeps{
Mirror: &fakeMirror{models: models}, Catalog: cat,
Fleet: fakeFleet{gpus: gpus, engines: engines},
}, true)
}
func TestAGGUFCanBeServedStraightFromTheList(t *testing.T) {
cat := &fakeCatalog{}
body := mirrorGet(t, servePage(t, []mirror.Status{
held("qwen2.5-coder-7b", "c03e6d35", "qwen-q4_k_m.gguf")}, cat, 0),
"/admin/mirror?serve=qwen2.5-coder-7b:c03e6d35").Body.String()
// The mirror already knows the name, the URL and what reads that kind of
// file; leaving an operator to write YAML for it was the gap.
if !strings.Contains(body, `name="engine" value="llamacpp"`) {
t.Errorf("the engine should be inferred from the extension; got %s", body)
}
if !strings.Contains(body, "/qwen2.5-coder-7b/c03e6d35/qwen-q4_k_m.gguf") {
t.Error("the source should be the mirror's own URL")
}
}
func TestAShardedVersionIsServedAsTheWholeVersion(t *testing.T) {
models := []mirror.Status{
held("llama", "v1", "model-00001-of-00002.safetensors"),
held("llama", "v1", "model-00002-of-00002.safetensors"),
}
body := mirrorGet(t, servePage(t, models, &fakeCatalog{}, 8, "vllm"),
"/admin/mirror?serve=llama:v1").Body.String()
// A catalogue entry names one source, and for a version of several files
// that source is the version. Refusing it meant a sharded model could be
// mirrored and never served — which is most of what is not a GGUF.
if !strings.Contains(body, `value="http://mirror:8081/llama/v1/"`) {
t.Errorf("the source should be the version directory; got %s", body)
}
if !strings.Contains(body, "The whole version, 2 files") {
t.Error("the form should say it is fetching a set")
}
if !strings.Contains(body, `action="/admin/mirror/serve"`) {
t.Error("it should be servable")
}
}
func TestTheEngineForASetComesFromItsWeights(t *testing.T) {
// A version is a mixture only when somebody has put a file there by hand.
// config.json decides nothing; the safetensors do.
models := []mirror.Status{
held("llama", "v1", "config.json"),
held("llama", "v1", "model-00001-of-00002.safetensors"),
}
body := mirrorGet(t, servePage(t, models, &fakeCatalog{}, 8, "vllm"),
"/admin/mirror?serve=llama:v1").Body.String()
if !strings.Contains(body, `name="engine" value="vllm"`) {
t.Errorf("the weights should pick the engine; got %s", body)
}
}
func TestSafetensorsOnACPUFleetWarnsBeforeAnybodyTries(t *testing.T) {
body := mirrorGet(t, servePage(t, []mirror.Status{
held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 0, "llamacpp"),
"/admin/mirror?serve=qwen:v1").Body.String()
// Fifteen gigabytes arrived before anybody noticed the fleet had no card
// and no vLLM to run them with. The page can say so.
if !strings.Contains(body, "no node reports having vllm installed") {
t.Errorf("expected the warning; got %s", body)
}
// And the way out, which on a fleet with no cards is a different build of
// the same model rather than installing vLLM.
if !strings.Contains(body, "GGUF build of the same model") {
t.Error("a fleet with no GPU should be pointed at a GGUF")
}
// A warning, not a refusal: a card could be added tomorrow.
if !strings.Contains(body, `action="/admin/mirror/serve"`) {
t.Error("it should still be possible to add it")
}
}
func TestSafetensorsOnAGPUFleetIsNotWarnedAbout(t *testing.T) {
body := mirrorGet(t, servePage(t, []mirror.Status{
held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 8, "vllm"),
"/admin/mirror?serve=qwen:v1").Body.String()
if strings.Contains(body, "no node reports having") {
t.Error("a fleet that runs vllm should not be warned about")
}
if !strings.Contains(body, `value="vllm"`) {
t.Error("safetensors should propose vllm")
}
}
func TestAModelDeclaredInConfigurationIsNotOfferedTwice(t *testing.T) {
cat := &fakeCatalog{declared: "qwen"}
body := mirrorGet(t, servePage(t, []mirror.Status{
held("qwen", "v1", "m.gguf")}, cat, 0),
"/admin/mirror?serve=qwen:v1").Body.String()
if !strings.Contains(body, "declared in configuration") {
t.Errorf("expected the refusal; got %s", body)
}
}
func TestServingBuildsTheEntryFromTheForm(t *testing.T) {
cat := &fakeCatalog{}
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
"csrf": {"tok"}, "name": {"coder"}, "aliases": {"c7b, coder-7b"},
"source": {"http://mirror:8081/coder/v1/m.gguf"}, "engine": {"llamacpp"},
"replicas": {"3"}, "context_length": {"4096"}, "max_output": {"1024"},
"ctx_size": {"16384"}, "profile": {"cpu-large"},
"min_memory_gb": {"8"}, "selector_key": {"role"}, "selector_value": {"cpu-large"},
})
if len(cat.added) != 1 {
t.Fatalf("expected one model added, got %d", len(cat.added))
}
m := cat.added[0]
if m.Name != "coder" || len(m.Aliases) != 2 {
t.Errorf("name and aliases did not arrive: %+v", m)
}
if m.Scaling.Min != 3 || m.Scaling.Max != 3 {
t.Errorf("expected three replicas, got %+v", m.Scaling)
}
if len(m.Profiles) != 1 {
t.Fatalf("expected one profile, got %+v", m.Profiles)
}
p := m.Profiles[0]
if p.Engine != "llamacpp" || p.Name != "cpu-large" || p.MinMemoryGB != 8 {
t.Errorf("the profile is wrong: %+v", p)
}
// The label is what keeps a large model off the small machines.
if p.NodeSelector["role"] != "cpu-large" {
t.Errorf("the node selector was lost: %+v", p.NodeSelector)
}
if strings.Join(p.Args, " ") != "--ctx-size 16384" {
t.Errorf("the engine's context pool was lost: %v", p.Args)
}
}
func TestRubbishInTheFormIsRefusedBeforeTheCatalog(t *testing.T) {
cat := &fakeCatalog{}
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
"csrf": {"tok"}, "name": {"coder"}, "source": {"http://x/m.gguf"},
"engine": {"llamacpp"}, "replicas": {"lots"}, "context_length": {"4096"},
})
if len(cat.added) != 0 {
t.Error("nothing should have reached the catalog")
}
}
func TestTheCatalogsOwnRefusalReachesTheOperator(t *testing.T) {
cat := &fakeCatalog{err: errors.New("that name collides with an alias")}
body := mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
"csrf": {"tok"}, "name": {"coder"}, "source": {"http://x/m.gguf"},
"engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"},
}).Body.String()
if !strings.Contains(body, "collides with an alias") {
t.Errorf("the catalog's reason should be shown; got %s", body)
}
}
func TestWithNoCatalogThereIsNoServeAction(t *testing.T) {
body := mirrorGet(t, mirrorPageDeps(t, MirrorDeps{
Mirror: &fakeMirror{models: []mirror.Status{held("qwen", "v1", "m.gguf")}},
}, true), "/admin/mirror").Body.String()
// Without a store a model added at runtime would not survive a restart, so
// the action is absent rather than offered and then refused.
if strings.Contains(body, "?serve=") {
t.Error("serving should not be offered without somewhere to persist it")
}
}
func TestAGPUNodeWithoutVLLMIsStillWarnedAbout(t *testing.T) {
// The first version of this checked cards, which would have waved through
// a fleet of eight-GPU nodes that had never had vLLM installed. Agents
// report their engines now, so it is a fact.
body := mirrorGet(t, servePage(t, []mirror.Status{
held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 8, "llamacpp"),
"/admin/mirror?serve=qwen:v1").Body.String()
if !strings.Contains(body, "no node reports having vllm installed") {
t.Errorf("cards are not the question; the engine is. Got %s", body)
}
}
func TestAFleetOfOlderAgentsIsNotToldEverythingIsUnservable(t *testing.T) {
// An agent older than the engine probe reports nothing, which is not the
// same as having no engines. Placement makes the same allowance, and the
// two disagreeing would be worse than either.
body := mirrorGet(t, servePage(t, []mirror.Status{
held("qwen", "v1", "m.gguf")}, &fakeCatalog{}, 0),
"/admin/mirror?serve=qwen:v1").Body.String()
if strings.Contains(body, "no node reports having") {
t.Error("silence from older agents must not read as a refusal")
}
}
func sized(model, version, file string, gb float64) mirror.Status {
n := int64(gb * (1 << 30))
return mirror.Status{Model: model, Version: version, File: file,
State: mirror.Present, BytesDone: n, BytesTotal: n}
}
func TestTheMemoryFieldIsFilledInFromTheWeights(t *testing.T) {
// The real case: 4.68 GB of weights went onto a 7.9 GB node and left it
// 3.5 GB for a KV pool, because the field was optional and blank. Blank
// writes no constraint, and a CPU profile has no card to be short of.
body := mirrorGet(t, servePage(t, []mirror.Status{
sized("coder", "v1", "m.gguf", 4.68)}, &fakeCatalog{}, 0, "llamacpp"),
"/admin/mirror?serve=coder:v1").Body.String()
if !strings.Contains(body, `name="min_memory_gb" value="9"`) {
t.Errorf("expected a figure derived from 4.68 GB of weights; got %s", body)
}
}
func TestASmallModelDoesNotDemandALargeNode(t *testing.T) {
// The estimate has to leave the small nodes usable, or every model ends up
// pinned to the one big machine.
body := mirrorGet(t, servePage(t, []mirror.Status{
sized("small", "v1", "m.gguf", 1.12)}, &fakeCatalog{}, 0, "llamacpp"),
"/admin/mirror?serve=small:v1").Body.String()
if !strings.Contains(body, `name="min_memory_gb" value="3"`) {
t.Errorf("a 1.12 GB model should fit a small node; got %s", body)
}
}
func TestAModelTooBigForEveryNodeSaysSo(t *testing.T) {
page := mirrorPageDeps(t, MirrorDeps{
Mirror: &fakeMirror{models: []mirror.Status{sized("huge", "v1", "m.gguf", 40)}},
Catalog: &fakeCatalog{},
Fleet: fakeFleet{gpus: 0, engines: []string{"llamacpp"}, memoryGB: 31},
}, true)
body := mirrorGet(t, page, "/admin/mirror?serve=huge:v1").Body.String()
// Better said here than discovered by a node being OOM-killed twenty
// minutes into a load.
if !strings.Contains(body, "largest node has 31 GB") {
t.Errorf("expected the size warning; got %s", body)
}
}
func TestAnEstimateIsNotMadeWhenTheSizeIsUnknown(t *testing.T) {
// A version whose size nobody reported would otherwise be given a
// requirement of two, which is a constraint that excludes nothing and
// reads as though somebody measured it.
body := mirrorGet(t, servePage(t, []mirror.Status{
{Model: "m", Version: "v", File: "f.gguf", State: mirror.Present}},
&fakeCatalog{}, 0, "llamacpp"), "/admin/mirror?serve=m:v").Body.String()
if !strings.Contains(body, `name="min_memory_gb" value="0"`) {
t.Errorf("an unknown size should leave no figure; got %s", body)
}
}
func TestADescriptionReachesTheCatalog(t *testing.T) {
cat := &fakeCatalog{}
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
"csrf": {"tok"}, "name": {"coder"}, "source": {"http://x/m.gguf"},
"engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"},
"description": {"code completion and review; 7B, fits the large node"},
})
if len(cat.added) != 1 {
t.Fatalf("expected one model, got %d", len(cat.added))
}
// The line a client sees in /v1/models, which for most of them is the only
// view of what this gateway serves.
if cat.added[0].Description != "code completion and review; 7B, fits the large node" {
t.Errorf("the description was lost: %q", cat.added[0].Description)
}
}
func TestAModelWithNoDescriptionIsStillAccepted(t *testing.T) {
cat := &fakeCatalog{}
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
"csrf": {"tok"}, "name": {"coder"}, "source": {"http://x/m.gguf"},
"engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"},
})
if len(cat.added) != 1 || cat.added[0].Description != "" {
t.Errorf("a description is optional: %+v", cat.added)
}
}