Ilker ManapandClaude Opus 5
d998ea7ef5
A gateway serving nine models offered nine names and nothing to choose between them. /v1/models is the only view most clients ever have, and it listed ids. Models carry a description now: written when the model is added, shown on the dashboard and sent to clients in /v1/models. Written rather than derived. A registry offers a pipeline tag and a wall of README, and neither answers the question somebody picking from a list is asking — which of these should I send this request to. "General chat, strong Swedish; slower than the 8B" is not in any metadata field. It is optional, and absent means absent rather than an empty string in the response: a model nobody described should read as undescribed, not as described with nothing. On the dashboard it sits under its row rather than in a column. A sentence in a table cell either wraps every other column into a stripe or gets cut off. Co-Authored-By: Claude Opus 5 (1M context) <moreply@anthropic.com>
327 lines
12 KiB
Go
327 lines
12 KiB
Go
package web
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
|
)
|
|
|
|
type fakeCatalog struct {
|
|
added []catalog.Model
|
|
declared string
|
|
err error
|
|
}
|
|
|
|
func (f *fakeCatalog) Add(_ context.Context, m catalog.Model, _ string) error {
|
|
if f.err != nil {
|
|
return f.err
|
|
}
|
|
f.added = append(f.added, m)
|
|
return nil
|
|
}
|
|
func (f *fakeCatalog) Declared(name string) bool { return name == f.declared }
|
|
|
|
// fleet reports a cluster with a given number of cards and engines.
|
|
type fakeFleet struct {
|
|
gpus int
|
|
engines []string
|
|
memoryGB int
|
|
}
|
|
|
|
func (f fakeFleet) Nodes() []adminapi.NodeView {
|
|
return []adminapi.NodeView{
|
|
{ID: "n1", GPUTotal: f.gpus, Engines: f.engines, MemoryGB: f.memoryGB},
|
|
}
|
|
}
|
|
func (f fakeFleet) Models() []adminapi.ModelView { return nil }
|
|
func (f fakeFleet) Replicas(string) []adminapi.ReplicaView { return nil }
|
|
func (f fakeFleet) Explain(string) adminapi.ExplainView { return adminapi.ExplainView{} }
|
|
func (f fakeFleet) Summary() adminapi.Summary { return adminapi.Summary{} }
|
|
|
|
func held(model, version, file string) mirror.Status {
|
|
return mirror.Status{Model: model, Version: version, File: file,
|
|
State: mirror.Present, BytesDone: 10, BytesTotal: 10}
|
|
}
|
|
|
|
func servePage(t *testing.T, models []mirror.Status, cat CatalogAdmin, gpus int,
|
|
engines ...string,
|
|
) *http.ServeMux {
|
|
t.Helper()
|
|
return mirrorPageDeps(t, MirrorDeps{
|
|
Mirror: &fakeMirror{models: models}, Catalog: cat,
|
|
Fleet: fakeFleet{gpus: gpus, engines: engines},
|
|
}, true)
|
|
}
|
|
|
|
func TestAGGUFCanBeServedStraightFromTheList(t *testing.T) {
|
|
cat := &fakeCatalog{}
|
|
body := mirrorGet(t, servePage(t, []mirror.Status{
|
|
held("qwen2.5-coder-7b", "c03e6d35", "qwen-q4_k_m.gguf")}, cat, 0),
|
|
"/admin/mirror?serve=qwen2.5-coder-7b:c03e6d35").Body.String()
|
|
|
|
// The mirror already knows the name, the URL and what reads that kind of
|
|
// file; leaving an operator to write YAML for it was the gap.
|
|
if !strings.Contains(body, `name="engine" value="llamacpp"`) {
|
|
t.Errorf("the engine should be inferred from the extension; got %s", body)
|
|
}
|
|
if !strings.Contains(body, "/qwen2.5-coder-7b/c03e6d35/qwen-q4_k_m.gguf") {
|
|
t.Error("the source should be the mirror's own URL")
|
|
}
|
|
}
|
|
|
|
func TestAShardedVersionIsRefusedWithTheReason(t *testing.T) {
|
|
models := []mirror.Status{
|
|
held("llama", "v1", "model-00001-of-00002.safetensors"),
|
|
held("llama", "v1", "model-00002-of-00002.safetensors"),
|
|
}
|
|
body := mirrorGet(t, servePage(t, models, &fakeCatalog{}, 8),
|
|
"/admin/mirror?serve=llama:v1").Body.String()
|
|
|
|
// A catalogue entry names one source, so an entry for a sharded version is
|
|
// an entry for one shard — which loads as a broken model rather than
|
|
// failing to load.
|
|
if !strings.Contains(body, "a catalogue entry names one source") {
|
|
t.Errorf("the refusal should say why; got %s", body)
|
|
}
|
|
if strings.Contains(body, `action="/admin/mirror/serve"`) {
|
|
t.Error("no form should be offered for something that cannot be served")
|
|
}
|
|
}
|
|
|
|
func TestSafetensorsOnACPUFleetWarnsBeforeAnybodyTries(t *testing.T) {
|
|
body := mirrorGet(t, servePage(t, []mirror.Status{
|
|
held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 0, "llamacpp"),
|
|
"/admin/mirror?serve=qwen:v1").Body.String()
|
|
|
|
// Fifteen gigabytes arrived before anybody noticed the fleet had no card
|
|
// and no vLLM to run them with. The page can say so.
|
|
if !strings.Contains(body, "no node reports having vllm installed") {
|
|
t.Errorf("expected the warning; got %s", body)
|
|
}
|
|
// And the way out, which on a fleet with no cards is a different build of
|
|
// the same model rather than installing vLLM.
|
|
if !strings.Contains(body, "GGUF build of the same model") {
|
|
t.Error("a fleet with no GPU should be pointed at a GGUF")
|
|
}
|
|
// A warning, not a refusal: a card could be added tomorrow.
|
|
if !strings.Contains(body, `action="/admin/mirror/serve"`) {
|
|
t.Error("it should still be possible to add it")
|
|
}
|
|
}
|
|
|
|
func TestSafetensorsOnAGPUFleetIsNotWarnedAbout(t *testing.T) {
|
|
body := mirrorGet(t, servePage(t, []mirror.Status{
|
|
held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 8, "vllm"),
|
|
"/admin/mirror?serve=qwen:v1").Body.String()
|
|
|
|
if strings.Contains(body, "no node reports having") {
|
|
t.Error("a fleet that runs vllm should not be warned about")
|
|
}
|
|
if !strings.Contains(body, `value="vllm"`) {
|
|
t.Error("safetensors should propose vllm")
|
|
}
|
|
}
|
|
|
|
func TestAModelDeclaredInConfigurationIsNotOfferedTwice(t *testing.T) {
|
|
cat := &fakeCatalog{declared: "qwen"}
|
|
body := mirrorGet(t, servePage(t, []mirror.Status{
|
|
held("qwen", "v1", "m.gguf")}, cat, 0),
|
|
"/admin/mirror?serve=qwen:v1").Body.String()
|
|
|
|
if !strings.Contains(body, "declared in configuration") {
|
|
t.Errorf("expected the refusal; got %s", body)
|
|
}
|
|
}
|
|
|
|
func TestServingBuildsTheEntryFromTheForm(t *testing.T) {
|
|
cat := &fakeCatalog{}
|
|
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
|
|
"csrf": {"tok"}, "name": {"coder"}, "aliases": {"c7b, coder-7b"},
|
|
"source": {"http://mirror:8081/coder/v1/m.gguf"}, "engine": {"llamacpp"},
|
|
"replicas": {"3"}, "context_length": {"4096"}, "max_output": {"1024"},
|
|
"ctx_size": {"16384"}, "profile": {"cpu-large"},
|
|
"min_memory_gb": {"8"}, "selector_key": {"role"}, "selector_value": {"cpu-large"},
|
|
})
|
|
|
|
if len(cat.added) != 1 {
|
|
t.Fatalf("expected one model added, got %d", len(cat.added))
|
|
}
|
|
m := cat.added[0]
|
|
if m.Name != "coder" || len(m.Aliases) != 2 {
|
|
t.Errorf("name and aliases did not arrive: %+v", m)
|
|
}
|
|
if m.Scaling.Min != 3 || m.Scaling.Max != 3 {
|
|
t.Errorf("expected three replicas, got %+v", m.Scaling)
|
|
}
|
|
if len(m.Profiles) != 1 {
|
|
t.Fatalf("expected one profile, got %+v", m.Profiles)
|
|
}
|
|
p := m.Profiles[0]
|
|
if p.Engine != "llamacpp" || p.Name != "cpu-large" || p.MinMemoryGB != 8 {
|
|
t.Errorf("the profile is wrong: %+v", p)
|
|
}
|
|
// The label is what keeps a large model off the small machines.
|
|
if p.NodeSelector["role"] != "cpu-large" {
|
|
t.Errorf("the node selector was lost: %+v", p.NodeSelector)
|
|
}
|
|
if strings.Join(p.Args, " ") != "--ctx-size 16384" {
|
|
t.Errorf("the engine's context pool was lost: %v", p.Args)
|
|
}
|
|
}
|
|
|
|
func TestRubbishInTheFormIsRefusedBeforeTheCatalog(t *testing.T) {
|
|
cat := &fakeCatalog{}
|
|
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
|
|
"csrf": {"tok"}, "name": {"coder"}, "source": {"http://x/m.gguf"},
|
|
"engine": {"llamacpp"}, "replicas": {"lots"}, "context_length": {"4096"},
|
|
})
|
|
if len(cat.added) != 0 {
|
|
t.Error("nothing should have reached the catalog")
|
|
}
|
|
}
|
|
|
|
func TestTheCatalogsOwnRefusalReachesTheOperator(t *testing.T) {
|
|
cat := &fakeCatalog{err: errors.New("that name collides with an alias")}
|
|
body := mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
|
|
"csrf": {"tok"}, "name": {"coder"}, "source": {"http://x/m.gguf"},
|
|
"engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"},
|
|
}).Body.String()
|
|
|
|
if !strings.Contains(body, "collides with an alias") {
|
|
t.Errorf("the catalog's reason should be shown; got %s", body)
|
|
}
|
|
}
|
|
|
|
func TestWithNoCatalogThereIsNoServeAction(t *testing.T) {
|
|
body := mirrorGet(t, mirrorPageDeps(t, MirrorDeps{
|
|
Mirror: &fakeMirror{models: []mirror.Status{held("qwen", "v1", "m.gguf")}},
|
|
}, true), "/admin/mirror").Body.String()
|
|
|
|
// Without a store a model added at runtime would not survive a restart, so
|
|
// the action is absent rather than offered and then refused.
|
|
if strings.Contains(body, "?serve=") {
|
|
t.Error("serving should not be offered without somewhere to persist it")
|
|
}
|
|
}
|
|
|
|
func TestAGPUNodeWithoutVLLMIsStillWarnedAbout(t *testing.T) {
|
|
// The first version of this checked cards, which would have waved through
|
|
// a fleet of eight-GPU nodes that had never had vLLM installed. Agents
|
|
// report their engines now, so it is a fact.
|
|
body := mirrorGet(t, servePage(t, []mirror.Status{
|
|
held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 8, "llamacpp"),
|
|
"/admin/mirror?serve=qwen:v1").Body.String()
|
|
|
|
if !strings.Contains(body, "no node reports having vllm installed") {
|
|
t.Errorf("cards are not the question; the engine is. Got %s", body)
|
|
}
|
|
}
|
|
|
|
func TestAFleetOfOlderAgentsIsNotToldEverythingIsUnservable(t *testing.T) {
|
|
// An agent older than the engine probe reports nothing, which is not the
|
|
// same as having no engines. Placement makes the same allowance, and the
|
|
// two disagreeing would be worse than either.
|
|
body := mirrorGet(t, servePage(t, []mirror.Status{
|
|
held("qwen", "v1", "m.gguf")}, &fakeCatalog{}, 0),
|
|
"/admin/mirror?serve=qwen:v1").Body.String()
|
|
|
|
if strings.Contains(body, "no node reports having") {
|
|
t.Error("silence from older agents must not read as a refusal")
|
|
}
|
|
}
|
|
|
|
func sized(model, version, file string, gb float64) mirror.Status {
|
|
n := int64(gb * (1 << 30))
|
|
return mirror.Status{Model: model, Version: version, File: file,
|
|
State: mirror.Present, BytesDone: n, BytesTotal: n}
|
|
}
|
|
|
|
func TestTheMemoryFieldIsFilledInFromTheWeights(t *testing.T) {
|
|
// The real case: 4.68 GB of weights went onto a 7.9 GB node and left it
|
|
// 3.5 GB for a KV pool, because the field was optional and blank. Blank
|
|
// writes no constraint, and a CPU profile has no card to be short of.
|
|
body := mirrorGet(t, servePage(t, []mirror.Status{
|
|
sized("coder", "v1", "m.gguf", 4.68)}, &fakeCatalog{}, 0, "llamacpp"),
|
|
"/admin/mirror?serve=coder:v1").Body.String()
|
|
|
|
if !strings.Contains(body, `name="min_memory_gb" value="9"`) {
|
|
t.Errorf("expected a figure derived from 4.68 GB of weights; got %s", body)
|
|
}
|
|
}
|
|
|
|
func TestASmallModelDoesNotDemandALargeNode(t *testing.T) {
|
|
// The estimate has to leave the small nodes usable, or every model ends up
|
|
// pinned to the one big machine.
|
|
body := mirrorGet(t, servePage(t, []mirror.Status{
|
|
sized("small", "v1", "m.gguf", 1.12)}, &fakeCatalog{}, 0, "llamacpp"),
|
|
"/admin/mirror?serve=small:v1").Body.String()
|
|
|
|
if !strings.Contains(body, `name="min_memory_gb" value="3"`) {
|
|
t.Errorf("a 1.12 GB model should fit a small node; got %s", body)
|
|
}
|
|
}
|
|
|
|
func TestAModelTooBigForEveryNodeSaysSo(t *testing.T) {
|
|
page := mirrorPageDeps(t, MirrorDeps{
|
|
Mirror: &fakeMirror{models: []mirror.Status{sized("huge", "v1", "m.gguf", 40)}},
|
|
Catalog: &fakeCatalog{},
|
|
Fleet: fakeFleet{gpus: 0, engines: []string{"llamacpp"}, memoryGB: 31},
|
|
}, true)
|
|
body := mirrorGet(t, page, "/admin/mirror?serve=huge:v1").Body.String()
|
|
|
|
// Better said here than discovered by a node being OOM-killed twenty
|
|
// minutes into a load.
|
|
if !strings.Contains(body, "largest node has 31 GB") {
|
|
t.Errorf("expected the size warning; got %s", body)
|
|
}
|
|
}
|
|
|
|
func TestAnEstimateIsNotMadeWhenTheSizeIsUnknown(t *testing.T) {
|
|
// A version whose size nobody reported would otherwise be given a
|
|
// requirement of two, which is a constraint that excludes nothing and
|
|
// reads as though somebody measured it.
|
|
body := mirrorGet(t, servePage(t, []mirror.Status{
|
|
{Model: "m", Version: "v", File: "f.gguf", State: mirror.Present}},
|
|
&fakeCatalog{}, 0, "llamacpp"), "/admin/mirror?serve=m:v").Body.String()
|
|
|
|
if !strings.Contains(body, `name="min_memory_gb" value="0"`) {
|
|
t.Errorf("an unknown size should leave no figure; got %s", body)
|
|
}
|
|
}
|
|
|
|
func TestADescriptionReachesTheCatalog(t *testing.T) {
|
|
cat := &fakeCatalog{}
|
|
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
|
|
"csrf": {"tok"}, "name": {"coder"}, "source": {"http://x/m.gguf"},
|
|
"engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"},
|
|
"description": {"code completion and review; 7B, fits the large node"},
|
|
})
|
|
|
|
if len(cat.added) != 1 {
|
|
t.Fatalf("expected one model, got %d", len(cat.added))
|
|
}
|
|
// The line a client sees in /v1/models, which for most of them is the only
|
|
// view of what this gateway serves.
|
|
if cat.added[0].Description != "code completion and review; 7B, fits the large node" {
|
|
t.Errorf("the description was lost: %q", cat.added[0].Description)
|
|
}
|
|
}
|
|
|
|
func TestAModelWithNoDescriptionIsStillAccepted(t *testing.T) {
|
|
cat := &fakeCatalog{}
|
|
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
|
|
"csrf": {"tok"}, "name": {"coder"}, "source": {"http://x/m.gguf"},
|
|
"engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"},
|
|
})
|
|
if len(cat.added) != 1 || cat.added[0].Description != "" {
|
|
t.Errorf("a description is optional: %+v", cat.added)
|
|
}
|
|
}
|