CI / test (push) Successful in 1m35s
Fetching a model put it on the mirror and left the operator to hand-write a catalogue entry and run `ctl model add` on the control plane — for a source the page was already displaying, with an engine decided by the file extension it had just shown them. A present row now offers to serve it. The form opens on that row with the name, the mirror's own URL and the engine filled in, and asks for the two things nobody can infer: the context pool and how many replicas. What it refuses is the point. A version of several files is refused with the reason. A catalogue entry names one source, so an entry for a sharded model is an entry for one shard, which loads as a broken model rather than failing to load. Nothing said so before — the mirror happily holds a four-shard model that no catalogue entry can describe. Safetensors on a fleet with no GPU is warned about before anything is added. This is the failure we hit today: fifteen gigabytes of shards arrived, and only then did looking at a node show no card, no vLLM and no SGLang. The control plane is not told which engines a node has installed, so the check is on cards, which it does know — a warning rather than a refusal, because a card can be added tomorrow. A name already declared in configuration is refused here too, matching what the model service would say anyway, but before the form is filled in rather than after. Without a store there is no serve action at all: a model added at runtime would not survive a restart, and offering it then refusing is worse than not offering. The mirror page's dependencies became a struct. Three are optional interfaces and two are the same shape, which at a call site is a swap nobody notices until a page shows the wrong thing. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
199 lines
7.1 KiB
Go
199 lines
7.1 KiB
Go
package web
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
|
)
|
|
|
|
type fakeCatalog struct {
|
|
added []catalog.Model
|
|
declared string
|
|
err error
|
|
}
|
|
|
|
func (f *fakeCatalog) Add(_ context.Context, m catalog.Model, _ string) error {
|
|
if f.err != nil {
|
|
return f.err
|
|
}
|
|
f.added = append(f.added, m)
|
|
return nil
|
|
}
|
|
func (f *fakeCatalog) Declared(name string) bool { return name == f.declared }
|
|
|
|
// fleet reports a cluster with a given number of cards.
|
|
type fakeFleet struct{ gpus int }
|
|
|
|
func (f fakeFleet) Nodes() []adminapi.NodeView {
|
|
return []adminapi.NodeView{{ID: "n1", GPUTotal: f.gpus}}
|
|
}
|
|
func (f fakeFleet) Models() []adminapi.ModelView { return nil }
|
|
func (f fakeFleet) Replicas(string) []adminapi.ReplicaView { return nil }
|
|
func (f fakeFleet) Explain(string) adminapi.ExplainView { return adminapi.ExplainView{} }
|
|
func (f fakeFleet) Summary() adminapi.Summary { return adminapi.Summary{} }
|
|
|
|
func held(model, version, file string) mirror.Status {
|
|
return mirror.Status{Model: model, Version: version, File: file,
|
|
State: mirror.Present, BytesDone: 10, BytesTotal: 10}
|
|
}
|
|
|
|
func servePage(t *testing.T, models []mirror.Status, cat CatalogAdmin, gpus int) *http.ServeMux {
|
|
t.Helper()
|
|
return mirrorPageDeps(t, MirrorDeps{
|
|
Mirror: &fakeMirror{models: models}, Catalog: cat, Fleet: fakeFleet{gpus: gpus},
|
|
}, true)
|
|
}
|
|
|
|
func TestAGGUFCanBeServedStraightFromTheList(t *testing.T) {
|
|
cat := &fakeCatalog{}
|
|
body := mirrorGet(t, servePage(t, []mirror.Status{
|
|
held("qwen2.5-coder-7b", "c03e6d35", "qwen-q4_k_m.gguf")}, cat, 0),
|
|
"/admin/mirror?serve=qwen2.5-coder-7b:c03e6d35").Body.String()
|
|
|
|
// The mirror already knows the name, the URL and what reads that kind of
|
|
// file; leaving an operator to write YAML for it was the gap.
|
|
if !strings.Contains(body, `name="engine" value="llamacpp"`) {
|
|
t.Errorf("the engine should be inferred from the extension; got %s", body)
|
|
}
|
|
if !strings.Contains(body, "/qwen2.5-coder-7b/c03e6d35/qwen-q4_k_m.gguf") {
|
|
t.Error("the source should be the mirror's own URL")
|
|
}
|
|
}
|
|
|
|
func TestAShardedVersionIsRefusedWithTheReason(t *testing.T) {
|
|
models := []mirror.Status{
|
|
held("llama", "v1", "model-00001-of-00002.safetensors"),
|
|
held("llama", "v1", "model-00002-of-00002.safetensors"),
|
|
}
|
|
body := mirrorGet(t, servePage(t, models, &fakeCatalog{}, 8),
|
|
"/admin/mirror?serve=llama:v1").Body.String()
|
|
|
|
// A catalogue entry names one source, so an entry for a sharded version is
|
|
// an entry for one shard — which loads as a broken model rather than
|
|
// failing to load.
|
|
if !strings.Contains(body, "a catalogue entry names one source") {
|
|
t.Errorf("the refusal should say why; got %s", body)
|
|
}
|
|
if strings.Contains(body, `action="/admin/mirror/serve"`) {
|
|
t.Error("no form should be offered for something that cannot be served")
|
|
}
|
|
}
|
|
|
|
func TestSafetensorsOnACPUFleetWarnsBeforeAnybodyTries(t *testing.T) {
|
|
body := mirrorGet(t, servePage(t, []mirror.Status{
|
|
held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 0),
|
|
"/admin/mirror?serve=qwen:v1").Body.String()
|
|
|
|
// Fifteen gigabytes arrived before anybody noticed the fleet had no card
|
|
// to run them on. The page can say so.
|
|
if !strings.Contains(body, "no node in this fleet reports a GPU") {
|
|
t.Errorf("expected the warning; got %s", body)
|
|
}
|
|
// A warning, not a refusal: a card could be added tomorrow.
|
|
if !strings.Contains(body, `action="/admin/mirror/serve"`) {
|
|
t.Error("it should still be possible to add it")
|
|
}
|
|
}
|
|
|
|
func TestSafetensorsOnAGPUFleetIsNotWarnedAbout(t *testing.T) {
|
|
body := mirrorGet(t, servePage(t, []mirror.Status{
|
|
held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 8),
|
|
"/admin/mirror?serve=qwen:v1").Body.String()
|
|
|
|
if strings.Contains(body, "reports a GPU") {
|
|
t.Error("a fleet with cards should not be warned about")
|
|
}
|
|
if !strings.Contains(body, `value="vllm"`) {
|
|
t.Error("safetensors should propose vllm")
|
|
}
|
|
}
|
|
|
|
func TestAModelDeclaredInConfigurationIsNotOfferedTwice(t *testing.T) {
|
|
cat := &fakeCatalog{declared: "qwen"}
|
|
body := mirrorGet(t, servePage(t, []mirror.Status{
|
|
held("qwen", "v1", "m.gguf")}, cat, 0),
|
|
"/admin/mirror?serve=qwen:v1").Body.String()
|
|
|
|
if !strings.Contains(body, "declared in configuration") {
|
|
t.Errorf("expected the refusal; got %s", body)
|
|
}
|
|
}
|
|
|
|
func TestServingBuildsTheEntryFromTheForm(t *testing.T) {
|
|
cat := &fakeCatalog{}
|
|
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
|
|
"csrf": {"tok"}, "name": {"coder"}, "aliases": {"c7b, coder-7b"},
|
|
"source": {"http://mirror:8081/coder/v1/m.gguf"}, "engine": {"llamacpp"},
|
|
"replicas": {"3"}, "context_length": {"4096"}, "max_output": {"1024"},
|
|
"ctx_size": {"16384"}, "profile": {"cpu-large"},
|
|
"min_memory_gb": {"8"}, "selector_key": {"role"}, "selector_value": {"cpu-large"},
|
|
})
|
|
|
|
if len(cat.added) != 1 {
|
|
t.Fatalf("expected one model added, got %d", len(cat.added))
|
|
}
|
|
m := cat.added[0]
|
|
if m.Name != "coder" || len(m.Aliases) != 2 {
|
|
t.Errorf("name and aliases did not arrive: %+v", m)
|
|
}
|
|
if m.Scaling.Min != 3 || m.Scaling.Max != 3 {
|
|
t.Errorf("expected three replicas, got %+v", m.Scaling)
|
|
}
|
|
if len(m.Profiles) != 1 {
|
|
t.Fatalf("expected one profile, got %+v", m.Profiles)
|
|
}
|
|
p := m.Profiles[0]
|
|
if p.Engine != "llamacpp" || p.Name != "cpu-large" || p.MinMemoryGB != 8 {
|
|
t.Errorf("the profile is wrong: %+v", p)
|
|
}
|
|
// The label is what keeps a large model off the small machines.
|
|
if p.NodeSelector["role"] != "cpu-large" {
|
|
t.Errorf("the node selector was lost: %+v", p.NodeSelector)
|
|
}
|
|
if strings.Join(p.Args, " ") != "--ctx-size 16384" {
|
|
t.Errorf("the engine's context pool was lost: %v", p.Args)
|
|
}
|
|
}
|
|
|
|
func TestRubbishInTheFormIsRefusedBeforeTheCatalog(t *testing.T) {
|
|
cat := &fakeCatalog{}
|
|
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
|
|
"csrf": {"tok"}, "name": {"coder"}, "source": {"http://x/m.gguf"},
|
|
"engine": {"llamacpp"}, "replicas": {"lots"}, "context_length": {"4096"},
|
|
})
|
|
if len(cat.added) != 0 {
|
|
t.Error("nothing should have reached the catalog")
|
|
}
|
|
}
|
|
|
|
func TestTheCatalogsOwnRefusalReachesTheOperator(t *testing.T) {
|
|
cat := &fakeCatalog{err: errors.New("that name collides with an alias")}
|
|
body := mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
|
|
"csrf": {"tok"}, "name": {"coder"}, "source": {"http://x/m.gguf"},
|
|
"engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"},
|
|
}).Body.String()
|
|
|
|
if !strings.Contains(body, "collides with an alias") {
|
|
t.Errorf("the catalog's reason should be shown; got %s", body)
|
|
}
|
|
}
|
|
|
|
func TestWithNoCatalogThereIsNoServeAction(t *testing.T) {
|
|
body := mirrorGet(t, mirrorPageDeps(t, MirrorDeps{
|
|
Mirror: &fakeMirror{models: []mirror.Status{held("qwen", "v1", "m.gguf")}},
|
|
}, true), "/admin/mirror").Body.String()
|
|
|
|
// Without a store a model added at runtime would not survive a restart, so
|
|
// the action is absent rather than offered and then refused.
|
|
if strings.Contains(body, "?serve=") {
|
|
t.Error("serving should not be offered without somewhere to persist it")
|
|
}
|
|
}
|