package web
import (
"context"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
)
type fakeCatalog struct {
added []catalog.Model
declared string
sources map[string][]catalog.Model
scaled []string
lastScale catalog.Scaling
err error
classes []string
owners []string
resources []string
adapterSets [][]catalog.Adapter
// stops, starts and splits are the models stopped, started and counted
// per profile, splitCPU the profile on CPUs last asked for.
stops, starts []string
splits []map[string]int
splitCPU *modelsvc.CPUProfile
}
func (f *fakeCatalog) SetProfileReplicas(_ context.Context, name string, counts map[string]int, cpu *modelsvc.CPUProfile,
_ string,
) (catalog.Model, error) {
f.splits, f.splitCPU = append(f.splits, counts), cpu
m := catalog.Model{Name: name}
for p, n := range counts {
m.Profiles = append(m.Profiles, catalog.Profile{Name: p, Replicas: n})
}
return m, f.err
}
func (f *fakeCatalog) Add(_ context.Context, m catalog.Model, _ string) error {
if f.err != nil {
return f.err
}
f.added = append(f.added, m)
return nil
}
func (f *fakeCatalog) Declared(name string) bool { return name == f.declared }
func (f *fakeCatalog) Sources(context.Context) (map[string][]catalog.Model, error) {
return f.sources, nil
}
func (f *fakeCatalog) SetAdapters(_ context.Context, name string, as []catalog.Adapter, _ string) (catalog.Model, error) {
f.adapterSets = append(f.adapterSets, as)
return catalog.Model{Name: name, Adapters: as}, f.err
}
func (f *fakeCatalog) SetClass(_ context.Context, name, class, _ string) (catalog.Model, error) {
f.classes = append(f.classes, name+"="+class)
return catalog.Model{Name: name, Class: class}, f.err
}
func (f *fakeCatalog) SetOwner(_ context.Context, name, owner, _ string) (catalog.Model, error) {
f.owners = append(f.owners, name+"="+owner)
return catalog.Model{Name: name, Owner: owner}, f.err
}
func (f *fakeCatalog) Resources(_ context.Context, name, _ string, mem, cores int, _ string) (catalog.Model, error) {
f.resources = append(f.resources, fmt.Sprintf("%s=%d/%d", name, mem, cores))
return catalog.Model{Name: name}, f.err
}
func (f *fakeCatalog) Scale(_ context.Context, name string, sc catalog.Scaling, _ string) (catalog.Model, error) {
f.scaled = append(f.scaled, name)
f.lastScale = sc
return catalog.Model{Name: name, Scaling: sc}, f.err
}
// fleet reports a cluster with a given number of cards and engines.
type fakeFleet struct {
gpus int
engines []string
memoryGB int
// used is how many of the cards replicas hold.
used int
}
func (f fakeFleet) Nodes() []adminapi.NodeView {
return []adminapi.NodeView{
{ID: "n1", GPUTotal: f.gpus, Engines: f.engines, MemoryGB: f.memoryGB},
}
}
func (f fakeFleet) Models() []adminapi.ModelView { return nil }
func (f fakeFleet) Replicas(string) []adminapi.ReplicaView { return nil }
func (f fakeFleet) Explain(string) adminapi.ExplainView { return adminapi.ExplainView{} }
func (f fakeFleet) Summary() adminapi.Summary {
if f.gpus == 0 {
return adminapi.Summary{}
}
return adminapi.Summary{GPUs: []adminapi.ClassUse{{Class: "t4", Total: f.gpus, Used: f.used}}}
}
func held(model, version, file string) mirror.Status {
return mirror.Status{Model: model, Version: version, File: file,
State: mirror.Present, BytesDone: 10, BytesTotal: 10}
}
func servePage(t *testing.T, models []mirror.Status, cat CatalogAdmin, gpus int,
engines ...string,
) *http.ServeMux {
t.Helper()
return mirrorPageDeps(t, MirrorDeps{
Mirror: &fakeMirror{models: models}, Catalog: cat,
Fleet: fakeFleet{gpus: gpus, engines: engines},
}, true)
}
// reachableSource stands in for a mirror holding the file. The serve handler
// checks the source before the catalog sees it, the same as the CLI does, so a
// test that means to exercise the form has to offer something that answers.
func reachableSource(t *testing.T) string {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
t.Cleanup(srv.Close)
return srv.URL + "/coder/v1/m.gguf"
}
func TestAGGUFCanBeServedStraightFromTheList(t *testing.T) {
cat := &fakeCatalog{}
body := mirrorGet(t, servePage(t, []mirror.Status{
held("qwen2.5-coder-7b", "c03e6d35", "qwen-q4_k_m.gguf")}, cat, 0),
"/admin/mirror?serve=qwen2.5-coder-7b:c03e6d35").Body.String()
// The mirror already knows the name, the URL and what reads that kind of
// file; leaving an operator to write YAML for it was the gap.
if !strings.Contains(body, `name="engine" value="llamacpp"`) {
t.Errorf("the engine should be inferred from the extension; got %s", body)
}
if !strings.Contains(body, "/qwen2.5-coder-7b/c03e6d35/qwen-q4_k_m.gguf") {
t.Error("the source should be the mirror's own URL")
}
}
func TestAShardedVersionIsServedAsTheWholeVersion(t *testing.T) {
models := []mirror.Status{
held("llama", "v1", "model-00001-of-00002.safetensors"),
held("llama", "v1", "model-00002-of-00002.safetensors"),
}
body := mirrorGet(t, servePage(t, models, &fakeCatalog{}, 8, "vllm"),
"/admin/mirror?serve=llama:v1").Body.String()
// A catalogue entry names one source, and for a version of several files
// that source is the version. Refusing it meant a sharded model could be
// mirrored and never served — which is most of what is not a GGUF.
if !strings.Contains(body, `value="http://mirror:8081/llama/v1/"`) {
t.Errorf("the source should be the version directory; got %s", body)
}
if !strings.Contains(body, "The whole version, 2 files") {
t.Error("the form should say it is fetching a set")
}
if !strings.Contains(body, `action="/admin/mirror/serve"`) {
t.Error("it should be servable")
}
}
func TestTheEngineForASetComesFromItsWeights(t *testing.T) {
// A version is a mixture only when somebody has put a file there by hand.
// config.json decides nothing; the safetensors do.
models := []mirror.Status{
held("llama", "v1", "config.json"),
held("llama", "v1", "model-00001-of-00002.safetensors"),
}
body := mirrorGet(t, servePage(t, models, &fakeCatalog{}, 8, "vllm"),
"/admin/mirror?serve=llama:v1").Body.String()
if !strings.Contains(body, `name="engine" value="vllm"`) {
t.Errorf("the weights should pick the engine; got %s", body)
}
}
func TestSafetensorsOnACPUFleetWarnsBeforeAnybodyTries(t *testing.T) {
body := mirrorGet(t, servePage(t, []mirror.Status{
held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 0, "llamacpp"),
"/admin/mirror?serve=qwen:v1").Body.String()
// Fifteen gigabytes arrived before anybody noticed the fleet had no card
// and no vLLM to run them with. The page can say so.
if !strings.Contains(body, "no node reports having vllm installed") {
t.Errorf("expected the warning; got %s", body)
}
// And the way out, which on a fleet with no cards is a different build of
// the same model rather than installing vLLM.
if !strings.Contains(body, "GGUF build of the same model") {
t.Error("a fleet with no GPU should be pointed at a GGUF")
}
// A warning, not a refusal: a card could be added tomorrow.
if !strings.Contains(body, `action="/admin/mirror/serve"`) {
t.Error("it should still be possible to add it")
}
}
func TestSafetensorsOnAGPUFleetIsNotWarnedAbout(t *testing.T) {
body := mirrorGet(t, servePage(t, []mirror.Status{
held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 8, "vllm"),
"/admin/mirror?serve=qwen:v1").Body.String()
if strings.Contains(body, "no node reports having") {
t.Error("a fleet that runs vllm should not be warned about")
}
if !strings.Contains(body, `value="vllm"`) {
t.Error("safetensors should propose vllm")
}
}
func TestAModelDeclaredInConfigurationIsNotOfferedTwice(t *testing.T) {
cat := &fakeCatalog{declared: "qwen"}
body := mirrorGet(t, servePage(t, []mirror.Status{
held("qwen", "v1", "m.gguf")}, cat, 0),
"/admin/mirror?serve=qwen:v1").Body.String()
if !strings.Contains(body, "declared in configuration") {
t.Errorf("expected the refusal; got %s", body)
}
}
func TestServingBuildsTheEntryFromTheForm(t *testing.T) {
cat := &fakeCatalog{}
source := reachableSource(t)
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
"csrf": {"tok"}, "name": {"coder"}, "aliases": {"c7b, coder-7b"},
"source": {source}, "engine": {"llamacpp"},
"replicas": {"3"}, "context_length": {"4096"}, "max_output": {"1024"},
"ctx_size": {"16384"}, "profile": {"cpu-large"},
"min_memory_gb": {"8"}, "selector_key": {"role"}, "selector_value": {"cpu-large"},
})
if len(cat.added) != 1 {
t.Fatalf("expected one model added, got %d", len(cat.added))
}
m := cat.added[0]
if m.Name != "coder" || len(m.Aliases) != 2 {
t.Errorf("name and aliases did not arrive: %+v", m)
}
if m.Scaling.Min != 3 || m.Scaling.Max != 3 {
t.Errorf("expected three replicas, got %+v", m.Scaling)
}
if len(m.Profiles) != 1 {
t.Fatalf("expected one profile, got %+v", m.Profiles)
}
p := m.Profiles[0]
if p.Engine != "llamacpp" || p.Name != "cpu-large" || p.MinMemoryGB != 8 {
t.Errorf("the profile is wrong: %+v", p)
}
// The label is what keeps a large model off the small machines.
if p.NodeSelector["role"] != "cpu-large" {
t.Errorf("the node selector was lost: %+v", p.NodeSelector)
}
if strings.Join(p.Args, " ") != "--ctx-size 16384" {
t.Errorf("the engine's context pool was lost: %v", p.Args)
}
}
func TestRubbishInTheFormIsRefusedBeforeTheCatalog(t *testing.T) {
cat := &fakeCatalog{}
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
"csrf": {"tok"}, "name": {"coder"}, "source": {"http://x/m.gguf"},
"engine": {"llamacpp"}, "replicas": {"lots"}, "context_length": {"4096"},
})
if len(cat.added) != 0 {
t.Error("nothing should have reached the catalog")
}
}
func TestTheCatalogsOwnRefusalReachesTheOperator(t *testing.T) {
cat := &fakeCatalog{err: errors.New("that name collides with an alias")}
source := reachableSource(t)
body := mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
"csrf": {"tok"}, "name": {"coder"}, "source": {source},
"engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"},
}).Body.String()
if !strings.Contains(body, "collides with an alias") {
t.Errorf("the catalog's reason should be shown; got %s", body)
}
}
func TestWithNoCatalogThereIsNoServeAction(t *testing.T) {
body := mirrorGet(t, mirrorPageDeps(t, MirrorDeps{
Mirror: &fakeMirror{models: []mirror.Status{held("qwen", "v1", "m.gguf")}},
}, true), "/admin/mirror").Body.String()
// Without a store a model added at runtime would not survive a restart, so
// the action is absent rather than offered and then refused.
if strings.Contains(body, "?serve=") {
t.Error("serving should not be offered without somewhere to persist it")
}
}
func TestAGPUNodeWithoutVLLMIsStillWarnedAbout(t *testing.T) {
// The first version of this checked cards, which would have waved through
// a fleet of eight-GPU nodes that had never had vLLM installed. Agents
// report their engines now, so it is a fact.
body := mirrorGet(t, servePage(t, []mirror.Status{
held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 8, "llamacpp"),
"/admin/mirror?serve=qwen:v1").Body.String()
if !strings.Contains(body, "no node reports having vllm installed") {
t.Errorf("cards are not the question; the engine is. Got %s", body)
}
}
func TestAFleetOfOlderAgentsIsNotToldEverythingIsUnservable(t *testing.T) {
// An agent older than the engine probe reports nothing, which is not the
// same as having no engines. Placement makes the same allowance, and the
// two disagreeing would be worse than either.
body := mirrorGet(t, servePage(t, []mirror.Status{
held("qwen", "v1", "m.gguf")}, &fakeCatalog{}, 0),
"/admin/mirror?serve=qwen:v1").Body.String()
if strings.Contains(body, "no node reports having") {
t.Error("silence from older agents must not read as a refusal")
}
}
func sized(model, version, file string, gb float64) mirror.Status {
n := int64(gb * (1 << 30))
return mirror.Status{Model: model, Version: version, File: file,
State: mirror.Present, BytesDone: n, BytesTotal: n}
}
func TestTheMemoryFieldIsFilledInFromTheWeights(t *testing.T) {
// The real case: 4.68 GB of weights went onto a 7.9 GB node and left it
// 3.5 GB for a KV pool, because the field was optional and blank. Blank
// writes no constraint, and a CPU profile has no card to be short of.
body := mirrorGet(t, servePage(t, []mirror.Status{
sized("coder", "v1", "m.gguf", 4.68)}, &fakeCatalog{}, 0, "llamacpp"),
"/admin/mirror?serve=coder:v1").Body.String()
if !strings.Contains(body, `name="min_memory_gb" value="9"`) {
t.Errorf("expected a figure derived from 4.68 GB of weights; got %s", body)
}
}
func TestASmallModelDoesNotDemandALargeNode(t *testing.T) {
// The estimate has to leave the small nodes usable, or every model ends up
// pinned to the one big machine.
body := mirrorGet(t, servePage(t, []mirror.Status{
sized("small", "v1", "m.gguf", 1.12)}, &fakeCatalog{}, 0, "llamacpp"),
"/admin/mirror?serve=small:v1").Body.String()
if !strings.Contains(body, `name="min_memory_gb" value="3"`) {
t.Errorf("a 1.12 GB model should fit a small node; got %s", body)
}
}
func TestAModelTooBigForEveryNodeSaysSo(t *testing.T) {
page := mirrorPageDeps(t, MirrorDeps{
Mirror: &fakeMirror{models: []mirror.Status{sized("huge", "v1", "m.gguf", 40)}},
Catalog: &fakeCatalog{},
Fleet: fakeFleet{gpus: 0, engines: []string{"llamacpp"}, memoryGB: 31},
}, true)
body := mirrorGet(t, page, "/admin/mirror?serve=huge:v1").Body.String()
// Better said here than discovered by a node being OOM-killed twenty
// minutes into a load.
if !strings.Contains(body, "largest node has 31 GB") {
t.Errorf("expected the size warning; got %s", body)
}
}
func TestAnEstimateIsNotMadeWhenTheSizeIsUnknown(t *testing.T) {
// A version whose size nobody reported would otherwise be given a
// requirement of two, which is a constraint that excludes nothing and
// reads as though somebody measured it.
body := mirrorGet(t, servePage(t, []mirror.Status{
{Model: "m", Version: "v", File: "f.gguf", State: mirror.Present}},
&fakeCatalog{}, 0, "llamacpp"), "/admin/mirror?serve=m:v").Body.String()
if !strings.Contains(body, `name="min_memory_gb" value="0"`) {
t.Errorf("an unknown size should leave no figure; got %s", body)
}
}
func TestADescriptionReachesTheCatalog(t *testing.T) {
cat := &fakeCatalog{}
source := reachableSource(t)
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
"csrf": {"tok"}, "name": {"coder"}, "source": {source},
"engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"},
"description": {"code completion and review; 7B, fits the large node"},
})
if len(cat.added) != 1 {
t.Fatalf("expected one model, got %d", len(cat.added))
}
// The line a client sees in /v1/models, which for most of them is the only
// view of what this gateway serves.
if cat.added[0].Description != "code completion and review; 7B, fits the large node" {
t.Errorf("the description was lost: %q", cat.added[0].Description)
}
}
func TestAModelWithNoDescriptionIsStillAccepted(t *testing.T) {
cat := &fakeCatalog{}
source := reachableSource(t)
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
"csrf": {"tok"}, "name": {"coder"}, "source": {source},
"engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"},
})
if len(cat.added) != 1 || cat.added[0].Description != "" {
t.Errorf("a description is optional: %+v", cat.added)
}
}
func TestAnUnreachableSourceIsRefusedBeforeTheCatalog(t *testing.T) {
// A port nothing is listening on: refused at once rather than after a
// timeout, and the same shape of failure as a mirror that is down.
srv := httptest.NewServer(http.HandlerFunc(
func(http.ResponseWriter, *http.Request) {}))
dead := srv.URL
srv.Close()
cat := &fakeCatalog{}
body := mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
"csrf": {"tok"}, "name": {"coder"}, "source": {dead + "/m.gguf"},
"engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"},
}).Body.String()
if len(cat.added) != 0 {
t.Error("a model whose weights are not there should not reach the catalog")
}
// The address, so the operator can see which one it could not reach; the
// page used to accept this and leave the failure to a node, minutes later.
if !strings.Contains(body, "cannot reach") {
t.Errorf("the reason should name the source; got %s", body)
}
}
func TestASourceTheMirrorNoLongerHoldsIsRefused(t *testing.T) {
gone := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "gone", http.StatusNotFound) }))
t.Cleanup(gone.Close)
cat := &fakeCatalog{}
body := mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
"csrf": {"tok"}, "name": {"coder"}, "source": {gone.URL + "/m.gguf"},
"engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"},
}).Body.String()
if len(cat.added) != 0 {
t.Error("a file removed between the listing and the click should be caught")
}
if !strings.Contains(body, "404") {
t.Errorf("the server's answer should be shown; got %s", body)
}
}
func TestServingOnDemandLoadsWhenAskedFor(t *testing.T) {
cat := &fakeCatalog{}
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
"csrf": {"tok"}, "name": {"big"}, "source": {reachableSource(t)}, "engine": {"llamacpp"},
"replicas": {"1"}, "mode": {"ondemand"}, "context_length": {"4096"}, "profile": {"cpu"},
})
if len(cat.added) != 1 {
t.Fatalf("expected one model added, got %d", len(cat.added))
}
if sc := cat.added[0].Scaling; sc.Min != 0 || sc.Max != 1 || sc.IdleTimeout == 0 {
t.Errorf("on demand was added as %+v", sc)
}
}
func TestAGGUFOnAFleetWithGPUsIsOfferedTheGPUFirst(t *testing.T) {
gguf := []mirror.Status{held("coder", "v1", "coder-q4_k_m.gguf")}
body := mirrorGet(t, servePage(t, gguf, &fakeCatalog{}, 4, "llamacpp"),
"/admin/mirror?serve=coder:v1").Body.String()
// Found in the field: no choice offered, a profile called "cpu", and a
// model on the CPU of a GPU cluster at a tenth of the speed.
if !strings.Contains(body, ``) {
t.Errorf("the free GPU class should be offered and chosen; got %s", body)
}
if !strings.Contains(body, ``) ||
strings.Contains(cpuOnly, `name="cpu_ok"`) {
t.Errorf("a fleet without GPUs has only the CPU, and nothing to confirm; got %s", cpuOnly)
}
}
func TestTheChosenPlacementDecidesWhereItRuns(t *testing.T) {
form := func(engine, place, gpus string) url.Values {
return url.Values{
"name": {"m"}, "source": {"http://x/m"}, "engine": {engine}, "replicas": {"1"},
"context_length": {"4096"}, "ctx_size": {"16384"}, "parallel": {"4"},
"placement": {place}, "gpus": {gpus}, "min_memory_gb": {"8"},
}
}
m, err := buildModel(english, form("llamacpp", "gpu:t4", "4").Get)
if err != nil {
t.Fatal(err)
}
p := m.Profiles[0]
if p.GPUClass != "t4" || p.GPUsPerNode != 4 || p.TP > 1 || p.Name != "t4-x4" {
t.Errorf("llama.cpp on four t4s: %+v", p)
}
if got := describeServe(english, m); got != "llama.cpp on 4 t4 GPUs of one node, 8 GB of host memory, "+
"context pool 16384 over 4 parallel requests (4096 each)" {
t.Errorf("summary %q", got)
}
if m, _ := buildModel(english, form("vllm", "gpu:t4", "2").Get); m.Profiles[0].TP != 2 || m.Profiles[0].GPUsPerNode != 0 {
t.Errorf("vLLM's GPU count is tp: %+v", m.Profiles[0])
}
if m, _ := buildModel(english, form("llamacpp", "cpu", "1").Get); m.Profiles[0].GPUClass != "" || m.Profiles[0].Name != "cpu" {
t.Errorf("the CPU: %+v", m.Profiles[0])
}
if _, err := buildModel(english, form("vllm", "cpu", "1").Get); err == nil {
t.Error("vLLM on the CPU was accepted")
}
}
func TestTheCPUWithGPUsFreeIsAChoiceMadeOnPurpose(t *testing.T) {
values := func(extra ...string) url.Values {
v := url.Values{
"csrf": {"tok"}, "name": {"coder"}, "source": {reachableSource(t)}, "engine": {"llamacpp"},
"replicas": {"1"}, "context_length": {"4096"}, "placement": {"cpu"}, "min_memory_gb": {"8"},
}
for i := 0; i+1 < len(extra); i += 2 {
v.Set(extra[i], extra[i+1])
}
return v
}
cat := &fakeCatalog{}
page := mirrorPageDeps(t, MirrorDeps{Mirror: &fakeMirror{}, Catalog: cat,
Fleet: fakeFleet{gpus: 4, used: 1, engines: []string{"llamacpp"}}}, true)
body := mirrorPost(t, page, "/admin/mirror/serve", values()).Body.String()
if len(cat.added) != 0 || !strings.Contains(body, "3 GPUs free") {
t.Fatalf("the CPU with three GPUs free went through unasked: %s", body)
}
mirrorPost(t, page, "/admin/mirror/serve", values("cpu_ok", "yes"))
if len(cat.added) != 1 {
t.Fatal("ticking the box should add it")
}
// Every card held: nothing to ask.
cat = &fakeCatalog{}
full := mirrorPageDeps(t, MirrorDeps{Mirror: &fakeMirror{}, Catalog: cat,
Fleet: fakeFleet{gpus: 4, used: 4, engines: []string{"llamacpp"}}}, true)
mirrorPost(t, full, "/admin/mirror/serve", values())
if len(cat.added) != 1 {
t.Error("with no GPU free the CPU is the only choice and should not be questioned")
}
}
func (f *fakeCatalog) Stop(_ context.Context, name, _ string) (catalog.Model, error) {
f.stops = append(f.stops, name)
return catalog.Model{Name: name, Stopped: &catalog.Scaling{Min: 1, Max: 1}}, f.err
}
func (f *fakeCatalog) Start(_ context.Context, name, _ string) (catalog.Model, error) {
f.starts = append(f.starts, name)
return catalog.Model{Name: name, Scaling: catalog.Scaling{Min: 1, Max: 1}}, f.err
}
// A version of a family Strata runs, on a fleet with Strata, is served on it:
// one card, its family and size, the experts' memory, from the directory.
func TestAStrataFamilysVersionIsServedOnStrata(t *testing.T) {
st := held("q38", "abc123", "Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00001-of-00002.gguf")
st.Source = "https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF/resolve/abc/IQ2_XS/" + st.File
files := []mirror.Status{st, held("q38", "abc123", "Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00002-of-00002.gguf")}
f := planServe(english, st, files, "http://mirror:8081", fakeFleet{gpus: 1, engines: []string{"llamacpp", "strata"}, memoryGB: 251},
nil, func(string) gguf.Verdict { return gguf.Verdict{Checked: true, Reason: "unknown architecture"} })
if f.Engine != "strata" || f.StrataFamily != "qwen" || f.StrataSize != "IQ2_XS" || f.Refusal != "" ||
f.MinMemoryGB != 52 || f.Context != 32768 || !strings.HasSuffix(f.Source, "/q38/abc123/") {
t.Fatalf("%+v", f)
}
// Without Strata on any node, llama.cpp as before, and its refusal.
if f := planServe(english, st, files, "http://mirror:8081", fakeFleet{gpus: 1, engines: []string{"llamacpp"}},
nil, func(string) gguf.Verdict { return gguf.Verdict{Checked: true, Reason: "unknown architecture"} }); f.Engine != "llamacpp" || f.Refusal == "" {
t.Errorf("%+v", f)
}
form := url.Values{"name": {"q38"}, "source": {f.Source}, "engine": {"strata"}, "replicas": {"1"},
"context_length": {"32768"}, "placement": {"gpu:6000-24g"}, "min_memory_gb": {"52"},
"strata_family": {"qwen"}, "strata_size": {"IQ2_XS"}}
m, err := buildModel(english, form.Get)
if err != nil {
t.Fatal(err)
}
p := m.Profiles[0]
if p.Engine != "strata" || p.GPUClass != "6000-24g" || p.GPUsPerNode != 1 || p.MinMemoryGB != 52 ||
strings.Join(p.Args, " ") != "--family qwen --size IQ2_XS --context 32768 --vision no" {
t.Errorf("%+v", p)
}
form.Set("strata_size", "IQ9")
if _, err := buildModel(english, form.Get); err == nil {
t.Error("an unknown size was taken")
}
}
// speedFleet is two nodes with GPUs whose memory rates were measured: a
// large server and a small one, as on a fleet where every node has cards.
type speedFleet struct{ fakeFleet }
func (speedFleet) Nodes() []adminapi.NodeView {
return []adminapi.NodeView{
{ID: "big", State: "ready", MemoryGB: 512, CPUCount: 64, MemBandwidthGBs: 300, MemBandwidthCoreGBs: 10,
Engines: []string{"llamacpp"}},
{ID: "small", State: "ready", MemoryGB: 64, CPUCount: 8, MemBandwidthGBs: 40, MemBandwidthCoreGBs: 10,
Engines: []string{"llamacpp"}},
}
}
// The speed on the CPU follows the threads a replica is given: four of a
// large server read at four cores' rate, not the server's.
func TestTheCPUsSpeedFollowsTheCoresGiven(t *testing.T) {
hosts := cpuHosts(speedFleet{}, 32)
if len(hosts) != 2 {
t.Fatalf("hosts %+v", hosts)
}
few, _ := bestCPUSpeed(hosts, 18e9, 4)
half, node := bestCPUSpeed(hosts, 18e9, 0)
if few > 2.5 || half < 8 || node != "big" {
t.Errorf("four cores %.1f tok/s, half the big node's %.1f on %s", few, half, node)
}
if hosts := cpuHosts(speedFleet{}, 100); len(hosts) != 1 || hosts[0].ID != "big" {
t.Errorf("a model of 100 GB fits the big node only: %+v", hosts)
}
}
// Slow enough that a reply is minutes, the CPU is a choice made on purpose,
// GPUs free or not.
func TestAVerySlowModelOnTheCPUIsAChoiceMadeOnPurpose(t *testing.T) {
big := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Length", "18000000000")
w.WriteHeader(http.StatusOK)
}))
t.Cleanup(big.Close)
values := func(extra ...string) url.Values {
v := url.Values{
"csrf": {"tok"}, "name": {"big-dense"}, "source": {big.URL + "/m/v1/m.gguf"}, "engine": {"llamacpp"},
"replicas": {"1"}, "context_length": {"4096"}, "placement": {"cpu"}, "min_memory_gb": {"24"},
"cpu_cores": {"4"},
}
for i := 0; i+1 < len(extra); i += 2 {
v.Set(extra[i], extra[i+1])
}
return v
}
cat := &fakeCatalog{}
page := mirrorPageDeps(t, MirrorDeps{Mirror: &fakeMirror{}, Catalog: cat, Fleet: speedFleet{}}, true)
body := mirrorPost(t, page, "/admin/mirror/serve", values()).Body.String()
if len(cat.added) != 0 || !strings.Contains(body, "tokens a second on the CPU") {
t.Fatalf("an 18 GB model on four cores went through unasked: %s", body)
}
mirrorPost(t, page, "/admin/mirror/serve", values("cpu_ok", "yes"))
if len(cat.added) != 1 {
t.Fatal("ticking the box should add it")
}
}
func TestTheServeFormStartsAtTheLongestContextThatFits(t *testing.T) {
// An 8B's shape: 36 layers, 8 KV heads of 128, so 144 KiB a token.
h := gguf.Header{BlockCount: 36, HeadCount: 32, HeadCountKV: 8, KeyLength: 128, ValueLength: 128,
ContextLength: 40960}
card := []placeChoice{{Value: "gpu:6000-24g", Selected: true}, {Value: "cpu"}}
cpu := []placeChoice{{Value: "gpu:6000-24g"}, {Value: "cpu", Selected: true}}
for _, c := range []struct {
name string
weights int64
places []placeChoice
header gguf.Header
tokens, slots int
}{
// All it was trained for, and two slots of it in the room left.
{"on a card with room", 5 << 30, card, h, 40960, 2},
// The weights take most of the card: the most that fits, one slot.
{"on a card nearly full", 18 << 30, card, h, 18432, 1},
{"on the CPU of a 64 GB node", 5 << 30, cpu, h, 40960, 4},
{"weights larger than the card", 30 << 30, card, h, 0, 0},
{"a header that does not say", 5 << 30, card, gguf.Header{BlockCount: 36, HeadCountKV: 8, HeadCount: 32, KeyLength: 128}, 0, 0},
} {
tokens, slots := mostContext(c.header, c.weights, c.places, 1, fakeFleet{memoryGB: 64})
if tokens != c.tokens || slots != c.slots {
t.Errorf("%s: %d tokens × %d slots, want %d × %d", c.name, tokens, slots, c.tokens, c.slots)
}
}
}