Ilker ManapandClaude Opus 5.5
1a3f385be4
CI / test (push) Successful in 10m51s
The row of a served model leads to its card, which now changes its context and longest answer, its cards, its replicas on each profile with a profile on CPUs to add and its speed beside, stops, starts and restarts it, and removes it with its files or without. Stopped on the row stops the model rather than setting its scaling to nothing. The console gains Replicas per profile, and ctl models shows the counts. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
673 lines
26 KiB
Go
673 lines
26 KiB
Go
package web
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/url"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
|
)
|
|
|
|
type fakeCatalog struct {
|
|
added []catalog.Model
|
|
declared string
|
|
sources map[string][]catalog.Model
|
|
scaled []string
|
|
lastScale catalog.Scaling
|
|
err error
|
|
classes []string
|
|
owners []string
|
|
resources []string
|
|
adapterSets [][]catalog.Adapter
|
|
// stops, starts and splits are the models stopped, started and counted
|
|
// per profile, splitCPU the profile on CPUs last asked for.
|
|
stops, starts []string
|
|
splits []map[string]int
|
|
splitCPU *modelsvc.CPUProfile
|
|
}
|
|
|
|
func (f *fakeCatalog) SetProfileReplicas(_ context.Context, name string, counts map[string]int, cpu *modelsvc.CPUProfile,
|
|
_ string,
|
|
) (catalog.Model, error) {
|
|
f.splits, f.splitCPU = append(f.splits, counts), cpu
|
|
m := catalog.Model{Name: name}
|
|
for p, n := range counts {
|
|
m.Profiles = append(m.Profiles, catalog.Profile{Name: p, Replicas: n})
|
|
}
|
|
return m, f.err
|
|
}
|
|
|
|
func (f *fakeCatalog) Add(_ context.Context, m catalog.Model, _ string) error {
|
|
if f.err != nil {
|
|
return f.err
|
|
}
|
|
f.added = append(f.added, m)
|
|
return nil
|
|
}
|
|
func (f *fakeCatalog) Declared(name string) bool { return name == f.declared }
|
|
func (f *fakeCatalog) Sources(context.Context) (map[string][]catalog.Model, error) {
|
|
return f.sources, nil
|
|
}
|
|
|
|
func (f *fakeCatalog) SetAdapters(_ context.Context, name string, as []catalog.Adapter, _ string) (catalog.Model, error) {
|
|
f.adapterSets = append(f.adapterSets, as)
|
|
return catalog.Model{Name: name, Adapters: as}, f.err
|
|
}
|
|
|
|
func (f *fakeCatalog) SetClass(_ context.Context, name, class, _ string) (catalog.Model, error) {
|
|
f.classes = append(f.classes, name+"="+class)
|
|
return catalog.Model{Name: name, Class: class}, f.err
|
|
}
|
|
|
|
func (f *fakeCatalog) SetOwner(_ context.Context, name, owner, _ string) (catalog.Model, error) {
|
|
f.owners = append(f.owners, name+"="+owner)
|
|
return catalog.Model{Name: name, Owner: owner}, f.err
|
|
}
|
|
|
|
func (f *fakeCatalog) Resources(_ context.Context, name, _ string, mem, cores int, _ string) (catalog.Model, error) {
|
|
f.resources = append(f.resources, fmt.Sprintf("%s=%d/%d", name, mem, cores))
|
|
return catalog.Model{Name: name}, f.err
|
|
}
|
|
|
|
func (f *fakeCatalog) Scale(_ context.Context, name string, sc catalog.Scaling, _ string) (catalog.Model, error) {
|
|
f.scaled = append(f.scaled, name)
|
|
f.lastScale = sc
|
|
return catalog.Model{Name: name, Scaling: sc}, f.err
|
|
}
|
|
|
|
// fleet reports a cluster with a given number of cards and engines.
|
|
type fakeFleet struct {
|
|
gpus int
|
|
engines []string
|
|
memoryGB int
|
|
// used is how many of the cards replicas hold.
|
|
used int
|
|
}
|
|
|
|
func (f fakeFleet) Nodes() []adminapi.NodeView {
|
|
return []adminapi.NodeView{
|
|
{ID: "n1", GPUTotal: f.gpus, Engines: f.engines, MemoryGB: f.memoryGB},
|
|
}
|
|
}
|
|
func (f fakeFleet) Models() []adminapi.ModelView { return nil }
|
|
func (f fakeFleet) Replicas(string) []adminapi.ReplicaView { return nil }
|
|
func (f fakeFleet) Explain(string) adminapi.ExplainView { return adminapi.ExplainView{} }
|
|
func (f fakeFleet) Summary() adminapi.Summary {
|
|
if f.gpus == 0 {
|
|
return adminapi.Summary{}
|
|
}
|
|
return adminapi.Summary{GPUs: []adminapi.ClassUse{{Class: "t4", Total: f.gpus, Used: f.used}}}
|
|
}
|
|
|
|
func held(model, version, file string) mirror.Status {
|
|
return mirror.Status{Model: model, Version: version, File: file,
|
|
State: mirror.Present, BytesDone: 10, BytesTotal: 10}
|
|
}
|
|
|
|
func servePage(t *testing.T, models []mirror.Status, cat CatalogAdmin, gpus int,
|
|
engines ...string,
|
|
) *http.ServeMux {
|
|
t.Helper()
|
|
return mirrorPageDeps(t, MirrorDeps{
|
|
Mirror: &fakeMirror{models: models}, Catalog: cat,
|
|
Fleet: fakeFleet{gpus: gpus, engines: engines},
|
|
}, true)
|
|
}
|
|
|
|
// reachableSource stands in for a mirror holding the file. The serve handler
|
|
// checks the source before the catalog sees it, the same as the CLI does, so a
|
|
// test that means to exercise the form has to offer something that answers.
|
|
func reachableSource(t *testing.T) string {
|
|
t.Helper()
|
|
srv := httptest.NewServer(http.HandlerFunc(
|
|
func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
|
|
t.Cleanup(srv.Close)
|
|
return srv.URL + "/coder/v1/m.gguf"
|
|
}
|
|
|
|
func TestAGGUFCanBeServedStraightFromTheList(t *testing.T) {
|
|
cat := &fakeCatalog{}
|
|
body := mirrorGet(t, servePage(t, []mirror.Status{
|
|
held("qwen2.5-coder-7b", "c03e6d35", "qwen-q4_k_m.gguf")}, cat, 0),
|
|
"/admin/mirror?serve=qwen2.5-coder-7b:c03e6d35").Body.String()
|
|
|
|
// The mirror already knows the name, the URL and what reads that kind of
|
|
// file; leaving an operator to write YAML for it was the gap.
|
|
if !strings.Contains(body, `name="engine" value="llamacpp"`) {
|
|
t.Errorf("the engine should be inferred from the extension; got %s", body)
|
|
}
|
|
if !strings.Contains(body, "/qwen2.5-coder-7b/c03e6d35/qwen-q4_k_m.gguf") {
|
|
t.Error("the source should be the mirror's own URL")
|
|
}
|
|
}
|
|
|
|
func TestAShardedVersionIsServedAsTheWholeVersion(t *testing.T) {
|
|
models := []mirror.Status{
|
|
held("llama", "v1", "model-00001-of-00002.safetensors"),
|
|
held("llama", "v1", "model-00002-of-00002.safetensors"),
|
|
}
|
|
body := mirrorGet(t, servePage(t, models, &fakeCatalog{}, 8, "vllm"),
|
|
"/admin/mirror?serve=llama:v1").Body.String()
|
|
|
|
// A catalogue entry names one source, and for a version of several files
|
|
// that source is the version. Refusing it meant a sharded model could be
|
|
// mirrored and never served — which is most of what is not a GGUF.
|
|
if !strings.Contains(body, `value="http://mirror:8081/llama/v1/"`) {
|
|
t.Errorf("the source should be the version directory; got %s", body)
|
|
}
|
|
if !strings.Contains(body, "The whole version, 2 files") {
|
|
t.Error("the form should say it is fetching a set")
|
|
}
|
|
if !strings.Contains(body, `action="/admin/mirror/serve"`) {
|
|
t.Error("it should be servable")
|
|
}
|
|
}
|
|
|
|
func TestTheEngineForASetComesFromItsWeights(t *testing.T) {
|
|
// A version is a mixture only when somebody has put a file there by hand.
|
|
// config.json decides nothing; the safetensors do.
|
|
models := []mirror.Status{
|
|
held("llama", "v1", "config.json"),
|
|
held("llama", "v1", "model-00001-of-00002.safetensors"),
|
|
}
|
|
body := mirrorGet(t, servePage(t, models, &fakeCatalog{}, 8, "vllm"),
|
|
"/admin/mirror?serve=llama:v1").Body.String()
|
|
|
|
if !strings.Contains(body, `name="engine" value="vllm"`) {
|
|
t.Errorf("the weights should pick the engine; got %s", body)
|
|
}
|
|
}
|
|
|
|
func TestSafetensorsOnACPUFleetWarnsBeforeAnybodyTries(t *testing.T) {
|
|
body := mirrorGet(t, servePage(t, []mirror.Status{
|
|
held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 0, "llamacpp"),
|
|
"/admin/mirror?serve=qwen:v1").Body.String()
|
|
|
|
// Fifteen gigabytes arrived before anybody noticed the fleet had no card
|
|
// and no vLLM to run them with. The page can say so.
|
|
if !strings.Contains(body, "no node reports having vllm installed") {
|
|
t.Errorf("expected the warning; got %s", body)
|
|
}
|
|
// And the way out, which on a fleet with no cards is a different build of
|
|
// the same model rather than installing vLLM.
|
|
if !strings.Contains(body, "GGUF build of the same model") {
|
|
t.Error("a fleet with no GPU should be pointed at a GGUF")
|
|
}
|
|
// A warning, not a refusal: a card could be added tomorrow.
|
|
if !strings.Contains(body, `action="/admin/mirror/serve"`) {
|
|
t.Error("it should still be possible to add it")
|
|
}
|
|
}
|
|
|
|
func TestSafetensorsOnAGPUFleetIsNotWarnedAbout(t *testing.T) {
|
|
body := mirrorGet(t, servePage(t, []mirror.Status{
|
|
held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 8, "vllm"),
|
|
"/admin/mirror?serve=qwen:v1").Body.String()
|
|
|
|
if strings.Contains(body, "no node reports having") {
|
|
t.Error("a fleet that runs vllm should not be warned about")
|
|
}
|
|
if !strings.Contains(body, `value="vllm"`) {
|
|
t.Error("safetensors should propose vllm")
|
|
}
|
|
}
|
|
|
|
func TestAModelDeclaredInConfigurationIsNotOfferedTwice(t *testing.T) {
|
|
cat := &fakeCatalog{declared: "qwen"}
|
|
body := mirrorGet(t, servePage(t, []mirror.Status{
|
|
held("qwen", "v1", "m.gguf")}, cat, 0),
|
|
"/admin/mirror?serve=qwen:v1").Body.String()
|
|
|
|
if !strings.Contains(body, "declared in configuration") {
|
|
t.Errorf("expected the refusal; got %s", body)
|
|
}
|
|
}
|
|
|
|
func TestServingBuildsTheEntryFromTheForm(t *testing.T) {
|
|
cat := &fakeCatalog{}
|
|
source := reachableSource(t)
|
|
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
|
|
"csrf": {"tok"}, "name": {"coder"}, "aliases": {"c7b, coder-7b"},
|
|
"source": {source}, "engine": {"llamacpp"},
|
|
"replicas": {"3"}, "context_length": {"4096"}, "max_output": {"1024"},
|
|
"ctx_size": {"16384"}, "profile": {"cpu-large"},
|
|
"min_memory_gb": {"8"}, "selector_key": {"role"}, "selector_value": {"cpu-large"},
|
|
})
|
|
|
|
if len(cat.added) != 1 {
|
|
t.Fatalf("expected one model added, got %d", len(cat.added))
|
|
}
|
|
m := cat.added[0]
|
|
if m.Name != "coder" || len(m.Aliases) != 2 {
|
|
t.Errorf("name and aliases did not arrive: %+v", m)
|
|
}
|
|
if m.Scaling.Min != 3 || m.Scaling.Max != 3 {
|
|
t.Errorf("expected three replicas, got %+v", m.Scaling)
|
|
}
|
|
if len(m.Profiles) != 1 {
|
|
t.Fatalf("expected one profile, got %+v", m.Profiles)
|
|
}
|
|
p := m.Profiles[0]
|
|
if p.Engine != "llamacpp" || p.Name != "cpu-large" || p.MinMemoryGB != 8 {
|
|
t.Errorf("the profile is wrong: %+v", p)
|
|
}
|
|
// The label is what keeps a large model off the small machines.
|
|
if p.NodeSelector["role"] != "cpu-large" {
|
|
t.Errorf("the node selector was lost: %+v", p.NodeSelector)
|
|
}
|
|
if strings.Join(p.Args, " ") != "--ctx-size 16384" {
|
|
t.Errorf("the engine's context pool was lost: %v", p.Args)
|
|
}
|
|
}
|
|
|
|
func TestRubbishInTheFormIsRefusedBeforeTheCatalog(t *testing.T) {
|
|
cat := &fakeCatalog{}
|
|
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
|
|
"csrf": {"tok"}, "name": {"coder"}, "source": {"http://x/m.gguf"},
|
|
"engine": {"llamacpp"}, "replicas": {"lots"}, "context_length": {"4096"},
|
|
})
|
|
if len(cat.added) != 0 {
|
|
t.Error("nothing should have reached the catalog")
|
|
}
|
|
}
|
|
|
|
func TestTheCatalogsOwnRefusalReachesTheOperator(t *testing.T) {
|
|
cat := &fakeCatalog{err: errors.New("that name collides with an alias")}
|
|
source := reachableSource(t)
|
|
body := mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
|
|
"csrf": {"tok"}, "name": {"coder"}, "source": {source},
|
|
"engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"},
|
|
}).Body.String()
|
|
|
|
if !strings.Contains(body, "collides with an alias") {
|
|
t.Errorf("the catalog's reason should be shown; got %s", body)
|
|
}
|
|
}
|
|
|
|
func TestWithNoCatalogThereIsNoServeAction(t *testing.T) {
|
|
body := mirrorGet(t, mirrorPageDeps(t, MirrorDeps{
|
|
Mirror: &fakeMirror{models: []mirror.Status{held("qwen", "v1", "m.gguf")}},
|
|
}, true), "/admin/mirror").Body.String()
|
|
|
|
// Without a store a model added at runtime would not survive a restart, so
|
|
// the action is absent rather than offered and then refused.
|
|
if strings.Contains(body, "?serve=") {
|
|
t.Error("serving should not be offered without somewhere to persist it")
|
|
}
|
|
}
|
|
|
|
func TestAGPUNodeWithoutVLLMIsStillWarnedAbout(t *testing.T) {
|
|
// The first version of this checked cards, which would have waved through
|
|
// a fleet of eight-GPU nodes that had never had vLLM installed. Agents
|
|
// report their engines now, so it is a fact.
|
|
body := mirrorGet(t, servePage(t, []mirror.Status{
|
|
held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 8, "llamacpp"),
|
|
"/admin/mirror?serve=qwen:v1").Body.String()
|
|
|
|
if !strings.Contains(body, "no node reports having vllm installed") {
|
|
t.Errorf("cards are not the question; the engine is. Got %s", body)
|
|
}
|
|
}
|
|
|
|
func TestAFleetOfOlderAgentsIsNotToldEverythingIsUnservable(t *testing.T) {
|
|
// An agent older than the engine probe reports nothing, which is not the
|
|
// same as having no engines. Placement makes the same allowance, and the
|
|
// two disagreeing would be worse than either.
|
|
body := mirrorGet(t, servePage(t, []mirror.Status{
|
|
held("qwen", "v1", "m.gguf")}, &fakeCatalog{}, 0),
|
|
"/admin/mirror?serve=qwen:v1").Body.String()
|
|
|
|
if strings.Contains(body, "no node reports having") {
|
|
t.Error("silence from older agents must not read as a refusal")
|
|
}
|
|
}
|
|
|
|
func sized(model, version, file string, gb float64) mirror.Status {
|
|
n := int64(gb * (1 << 30))
|
|
return mirror.Status{Model: model, Version: version, File: file,
|
|
State: mirror.Present, BytesDone: n, BytesTotal: n}
|
|
}
|
|
|
|
func TestTheMemoryFieldIsFilledInFromTheWeights(t *testing.T) {
|
|
// The real case: 4.68 GB of weights went onto a 7.9 GB node and left it
|
|
// 3.5 GB for a KV pool, because the field was optional and blank. Blank
|
|
// writes no constraint, and a CPU profile has no card to be short of.
|
|
body := mirrorGet(t, servePage(t, []mirror.Status{
|
|
sized("coder", "v1", "m.gguf", 4.68)}, &fakeCatalog{}, 0, "llamacpp"),
|
|
"/admin/mirror?serve=coder:v1").Body.String()
|
|
|
|
if !strings.Contains(body, `name="min_memory_gb" value="9"`) {
|
|
t.Errorf("expected a figure derived from 4.68 GB of weights; got %s", body)
|
|
}
|
|
}
|
|
|
|
func TestASmallModelDoesNotDemandALargeNode(t *testing.T) {
|
|
// The estimate has to leave the small nodes usable, or every model ends up
|
|
// pinned to the one big machine.
|
|
body := mirrorGet(t, servePage(t, []mirror.Status{
|
|
sized("small", "v1", "m.gguf", 1.12)}, &fakeCatalog{}, 0, "llamacpp"),
|
|
"/admin/mirror?serve=small:v1").Body.String()
|
|
|
|
if !strings.Contains(body, `name="min_memory_gb" value="3"`) {
|
|
t.Errorf("a 1.12 GB model should fit a small node; got %s", body)
|
|
}
|
|
}
|
|
|
|
func TestAModelTooBigForEveryNodeSaysSo(t *testing.T) {
|
|
page := mirrorPageDeps(t, MirrorDeps{
|
|
Mirror: &fakeMirror{models: []mirror.Status{sized("huge", "v1", "m.gguf", 40)}},
|
|
Catalog: &fakeCatalog{},
|
|
Fleet: fakeFleet{gpus: 0, engines: []string{"llamacpp"}, memoryGB: 31},
|
|
}, true)
|
|
body := mirrorGet(t, page, "/admin/mirror?serve=huge:v1").Body.String()
|
|
|
|
// Better said here than discovered by a node being OOM-killed twenty
|
|
// minutes into a load.
|
|
if !strings.Contains(body, "largest node has 31 GB") {
|
|
t.Errorf("expected the size warning; got %s", body)
|
|
}
|
|
}
|
|
|
|
func TestAnEstimateIsNotMadeWhenTheSizeIsUnknown(t *testing.T) {
|
|
// A version whose size nobody reported would otherwise be given a
|
|
// requirement of two, which is a constraint that excludes nothing and
|
|
// reads as though somebody measured it.
|
|
body := mirrorGet(t, servePage(t, []mirror.Status{
|
|
{Model: "m", Version: "v", File: "f.gguf", State: mirror.Present}},
|
|
&fakeCatalog{}, 0, "llamacpp"), "/admin/mirror?serve=m:v").Body.String()
|
|
|
|
if !strings.Contains(body, `name="min_memory_gb" value="0"`) {
|
|
t.Errorf("an unknown size should leave no figure; got %s", body)
|
|
}
|
|
}
|
|
|
|
func TestADescriptionReachesTheCatalog(t *testing.T) {
|
|
cat := &fakeCatalog{}
|
|
source := reachableSource(t)
|
|
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
|
|
"csrf": {"tok"}, "name": {"coder"}, "source": {source},
|
|
"engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"},
|
|
"description": {"code completion and review; 7B, fits the large node"},
|
|
})
|
|
|
|
if len(cat.added) != 1 {
|
|
t.Fatalf("expected one model, got %d", len(cat.added))
|
|
}
|
|
// The line a client sees in /v1/models, which for most of them is the only
|
|
// view of what this gateway serves.
|
|
if cat.added[0].Description != "code completion and review; 7B, fits the large node" {
|
|
t.Errorf("the description was lost: %q", cat.added[0].Description)
|
|
}
|
|
}
|
|
|
|
func TestAModelWithNoDescriptionIsStillAccepted(t *testing.T) {
|
|
cat := &fakeCatalog{}
|
|
source := reachableSource(t)
|
|
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
|
|
"csrf": {"tok"}, "name": {"coder"}, "source": {source},
|
|
"engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"},
|
|
})
|
|
if len(cat.added) != 1 || cat.added[0].Description != "" {
|
|
t.Errorf("a description is optional: %+v", cat.added)
|
|
}
|
|
}
|
|
|
|
func TestAnUnreachableSourceIsRefusedBeforeTheCatalog(t *testing.T) {
|
|
// A port nothing is listening on: refused at once rather than after a
|
|
// timeout, and the same shape of failure as a mirror that is down.
|
|
srv := httptest.NewServer(http.HandlerFunc(
|
|
func(http.ResponseWriter, *http.Request) {}))
|
|
dead := srv.URL
|
|
srv.Close()
|
|
|
|
cat := &fakeCatalog{}
|
|
body := mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
|
|
"csrf": {"tok"}, "name": {"coder"}, "source": {dead + "/m.gguf"},
|
|
"engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"},
|
|
}).Body.String()
|
|
|
|
if len(cat.added) != 0 {
|
|
t.Error("a model whose weights are not there should not reach the catalog")
|
|
}
|
|
// The address, so the operator can see which one it could not reach; the
|
|
// page used to accept this and leave the failure to a node, minutes later.
|
|
if !strings.Contains(body, "cannot reach") {
|
|
t.Errorf("the reason should name the source; got %s", body)
|
|
}
|
|
}
|
|
|
|
func TestASourceTheMirrorNoLongerHoldsIsRefused(t *testing.T) {
|
|
gone := httptest.NewServer(http.HandlerFunc(
|
|
func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "gone", http.StatusNotFound) }))
|
|
t.Cleanup(gone.Close)
|
|
|
|
cat := &fakeCatalog{}
|
|
body := mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
|
|
"csrf": {"tok"}, "name": {"coder"}, "source": {gone.URL + "/m.gguf"},
|
|
"engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"},
|
|
}).Body.String()
|
|
|
|
if len(cat.added) != 0 {
|
|
t.Error("a file removed between the listing and the click should be caught")
|
|
}
|
|
if !strings.Contains(body, "404") {
|
|
t.Errorf("the server's answer should be shown; got %s", body)
|
|
}
|
|
}
|
|
|
|
func TestServingOnDemandLoadsWhenAskedFor(t *testing.T) {
|
|
cat := &fakeCatalog{}
|
|
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
|
|
"csrf": {"tok"}, "name": {"big"}, "source": {reachableSource(t)}, "engine": {"llamacpp"},
|
|
"replicas": {"1"}, "mode": {"ondemand"}, "context_length": {"4096"}, "profile": {"cpu"},
|
|
})
|
|
if len(cat.added) != 1 {
|
|
t.Fatalf("expected one model added, got %d", len(cat.added))
|
|
}
|
|
if sc := cat.added[0].Scaling; sc.Min != 0 || sc.Max != 1 || sc.IdleTimeout == 0 {
|
|
t.Errorf("on demand was added as %+v", sc)
|
|
}
|
|
}
|
|
|
|
func TestAGGUFOnAFleetWithGPUsIsOfferedTheGPUFirst(t *testing.T) {
|
|
gguf := []mirror.Status{held("coder", "v1", "coder-q4_k_m.gguf")}
|
|
body := mirrorGet(t, servePage(t, gguf, &fakeCatalog{}, 4, "llamacpp"),
|
|
"/admin/mirror?serve=coder:v1").Body.String()
|
|
// Found in the field: no choice offered, a profile called "cpu", and a
|
|
// model on the CPU of a GPU cluster at a tenth of the speed.
|
|
if !strings.Contains(body, `<option value="gpu:t4" selected>GPU: t4, 4 of 4 free</option>`) {
|
|
t.Errorf("the free GPU class should be offered and chosen; got %s", body)
|
|
}
|
|
if !strings.Contains(body, `<option value="cpu">CPU only`) {
|
|
t.Error("the CPU should still be on offer, unselected")
|
|
}
|
|
if !strings.Contains(body, `name="cpu_ok"`) {
|
|
t.Error("choosing the CPU with GPUs free needs saying so")
|
|
}
|
|
|
|
cpuOnly := mirrorGet(t, servePage(t, gguf, &fakeCatalog{}, 0, "llamacpp"),
|
|
"/admin/mirror?serve=coder:v1").Body.String()
|
|
if !strings.Contains(cpuOnly, `<option value="cpu" selected>CPU: this fleet has no GPU</option>`) ||
|
|
strings.Contains(cpuOnly, `name="cpu_ok"`) {
|
|
t.Errorf("a fleet without GPUs has only the CPU, and nothing to confirm; got %s", cpuOnly)
|
|
}
|
|
}
|
|
|
|
func TestTheChosenPlacementDecidesWhereItRuns(t *testing.T) {
|
|
form := func(engine, place, gpus string) url.Values {
|
|
return url.Values{
|
|
"name": {"m"}, "source": {"http://x/m"}, "engine": {engine}, "replicas": {"1"},
|
|
"context_length": {"4096"}, "ctx_size": {"16384"}, "parallel": {"4"},
|
|
"placement": {place}, "gpus": {gpus}, "min_memory_gb": {"8"},
|
|
}
|
|
}
|
|
m, err := buildModel(english, form("llamacpp", "gpu:t4", "4").Get)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
p := m.Profiles[0]
|
|
if p.GPUClass != "t4" || p.GPUsPerNode != 4 || p.TP > 1 || p.Name != "t4-x4" {
|
|
t.Errorf("llama.cpp on four t4s: %+v", p)
|
|
}
|
|
if got := describeServe(english, m); got != "llama.cpp on 4 t4 GPUs of one node, 8 GB of host memory, "+
|
|
"context pool 16384 over 4 parallel requests (4096 each)" {
|
|
t.Errorf("summary %q", got)
|
|
}
|
|
if m, _ := buildModel(english, form("vllm", "gpu:t4", "2").Get); m.Profiles[0].TP != 2 || m.Profiles[0].GPUsPerNode != 0 {
|
|
t.Errorf("vLLM's GPU count is tp: %+v", m.Profiles[0])
|
|
}
|
|
if m, _ := buildModel(english, form("llamacpp", "cpu", "1").Get); m.Profiles[0].GPUClass != "" || m.Profiles[0].Name != "cpu" {
|
|
t.Errorf("the CPU: %+v", m.Profiles[0])
|
|
}
|
|
if _, err := buildModel(english, form("vllm", "cpu", "1").Get); err == nil {
|
|
t.Error("vLLM on the CPU was accepted")
|
|
}
|
|
}
|
|
|
|
func TestTheCPUWithGPUsFreeIsAChoiceMadeOnPurpose(t *testing.T) {
|
|
values := func(extra ...string) url.Values {
|
|
v := url.Values{
|
|
"csrf": {"tok"}, "name": {"coder"}, "source": {reachableSource(t)}, "engine": {"llamacpp"},
|
|
"replicas": {"1"}, "context_length": {"4096"}, "placement": {"cpu"}, "min_memory_gb": {"8"},
|
|
}
|
|
for i := 0; i+1 < len(extra); i += 2 {
|
|
v.Set(extra[i], extra[i+1])
|
|
}
|
|
return v
|
|
}
|
|
cat := &fakeCatalog{}
|
|
page := mirrorPageDeps(t, MirrorDeps{Mirror: &fakeMirror{}, Catalog: cat,
|
|
Fleet: fakeFleet{gpus: 4, used: 1, engines: []string{"llamacpp"}}}, true)
|
|
body := mirrorPost(t, page, "/admin/mirror/serve", values()).Body.String()
|
|
if len(cat.added) != 0 || !strings.Contains(body, "3 GPUs free") {
|
|
t.Fatalf("the CPU with three GPUs free went through unasked: %s", body)
|
|
}
|
|
mirrorPost(t, page, "/admin/mirror/serve", values("cpu_ok", "yes"))
|
|
if len(cat.added) != 1 {
|
|
t.Fatal("ticking the box should add it")
|
|
}
|
|
// Every card held: nothing to ask.
|
|
cat = &fakeCatalog{}
|
|
full := mirrorPageDeps(t, MirrorDeps{Mirror: &fakeMirror{}, Catalog: cat,
|
|
Fleet: fakeFleet{gpus: 4, used: 4, engines: []string{"llamacpp"}}}, true)
|
|
mirrorPost(t, full, "/admin/mirror/serve", values())
|
|
if len(cat.added) != 1 {
|
|
t.Error("with no GPU free the CPU is the only choice and should not be questioned")
|
|
}
|
|
}
|
|
|
|
func (f *fakeCatalog) Stop(_ context.Context, name, _ string) (catalog.Model, error) {
|
|
f.stops = append(f.stops, name)
|
|
return catalog.Model{Name: name, Stopped: &catalog.Scaling{Min: 1, Max: 1}}, f.err
|
|
}
|
|
|
|
func (f *fakeCatalog) Start(_ context.Context, name, _ string) (catalog.Model, error) {
|
|
f.starts = append(f.starts, name)
|
|
return catalog.Model{Name: name, Scaling: catalog.Scaling{Min: 1, Max: 1}}, f.err
|
|
}
|
|
|
|
// A version of a family Strata runs, on a fleet with Strata, is served on it:
|
|
// one card, its family and size, the experts' memory, from the directory.
|
|
func TestAStrataFamilysVersionIsServedOnStrata(t *testing.T) {
|
|
st := held("q38", "abc123", "Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00001-of-00002.gguf")
|
|
st.Source = "https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF/resolve/abc/IQ2_XS/" + st.File
|
|
files := []mirror.Status{st, held("q38", "abc123", "Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00002-of-00002.gguf")}
|
|
f := planServe(english, st, files, "http://mirror:8081", fakeFleet{gpus: 1, engines: []string{"llamacpp", "strata"}, memoryGB: 251},
|
|
nil, func(string) gguf.Verdict { return gguf.Verdict{Checked: true, Reason: "unknown architecture"} })
|
|
if f.Engine != "strata" || f.StrataFamily != "qwen" || f.StrataSize != "IQ2_XS" || f.Refusal != "" ||
|
|
f.MinMemoryGB != 52 || f.Context != 32768 || !strings.HasSuffix(f.Source, "/q38/abc123/") {
|
|
t.Fatalf("%+v", f)
|
|
}
|
|
// Without Strata on any node, llama.cpp as before, and its refusal.
|
|
if f := planServe(english, st, files, "http://mirror:8081", fakeFleet{gpus: 1, engines: []string{"llamacpp"}},
|
|
nil, func(string) gguf.Verdict { return gguf.Verdict{Checked: true, Reason: "unknown architecture"} }); f.Engine != "llamacpp" || f.Refusal == "" {
|
|
t.Errorf("%+v", f)
|
|
}
|
|
form := url.Values{"name": {"q38"}, "source": {f.Source}, "engine": {"strata"}, "replicas": {"1"},
|
|
"context_length": {"32768"}, "placement": {"gpu:6000-24g"}, "min_memory_gb": {"52"},
|
|
"strata_family": {"qwen"}, "strata_size": {"IQ2_XS"}}
|
|
m, err := buildModel(english, form.Get)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
p := m.Profiles[0]
|
|
if p.Engine != "strata" || p.GPUClass != "6000-24g" || p.GPUsPerNode != 1 || p.MinMemoryGB != 52 ||
|
|
strings.Join(p.Args, " ") != "--family qwen --size IQ2_XS --context 32768 --vision no" {
|
|
t.Errorf("%+v", p)
|
|
}
|
|
form.Set("strata_size", "IQ9")
|
|
if _, err := buildModel(english, form.Get); err == nil {
|
|
t.Error("an unknown size was taken")
|
|
}
|
|
}
|
|
|
|
// speedFleet is two nodes with GPUs whose memory rates were measured: a
|
|
// large server and a small one, as on a fleet where every node has cards.
|
|
type speedFleet struct{ fakeFleet }
|
|
|
|
func (speedFleet) Nodes() []adminapi.NodeView {
|
|
return []adminapi.NodeView{
|
|
{ID: "big", State: "ready", MemoryGB: 512, CPUCount: 64, MemBandwidthGBs: 300, MemBandwidthCoreGBs: 10,
|
|
Engines: []string{"llamacpp"}},
|
|
{ID: "small", State: "ready", MemoryGB: 64, CPUCount: 8, MemBandwidthGBs: 40, MemBandwidthCoreGBs: 10,
|
|
Engines: []string{"llamacpp"}},
|
|
}
|
|
}
|
|
|
|
// The speed on the CPU follows the threads a replica is given: four of a
|
|
// large server read at four cores' rate, not the server's.
|
|
func TestTheCPUsSpeedFollowsTheCoresGiven(t *testing.T) {
|
|
hosts := cpuHosts(speedFleet{}, 32)
|
|
if len(hosts) != 2 {
|
|
t.Fatalf("hosts %+v", hosts)
|
|
}
|
|
few, _ := bestCPUSpeed(hosts, 18e9, 4)
|
|
half, node := bestCPUSpeed(hosts, 18e9, 0)
|
|
if few > 2.5 || half < 8 || node != "big" {
|
|
t.Errorf("four cores %.1f tok/s, half the big node's %.1f on %s", few, half, node)
|
|
}
|
|
if hosts := cpuHosts(speedFleet{}, 100); len(hosts) != 1 || hosts[0].ID != "big" {
|
|
t.Errorf("a model of 100 GB fits the big node only: %+v", hosts)
|
|
}
|
|
}
|
|
|
|
// Slow enough that a reply is minutes, the CPU is a choice made on purpose,
|
|
// GPUs free or not.
|
|
func TestAVerySlowModelOnTheCPUIsAChoiceMadeOnPurpose(t *testing.T) {
|
|
big := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.Header().Set("Content-Length", "18000000000")
|
|
w.WriteHeader(http.StatusOK)
|
|
}))
|
|
t.Cleanup(big.Close)
|
|
values := func(extra ...string) url.Values {
|
|
v := url.Values{
|
|
"csrf": {"tok"}, "name": {"big-dense"}, "source": {big.URL + "/m/v1/m.gguf"}, "engine": {"llamacpp"},
|
|
"replicas": {"1"}, "context_length": {"4096"}, "placement": {"cpu"}, "min_memory_gb": {"24"},
|
|
"cpu_cores": {"4"},
|
|
}
|
|
for i := 0; i+1 < len(extra); i += 2 {
|
|
v.Set(extra[i], extra[i+1])
|
|
}
|
|
return v
|
|
}
|
|
cat := &fakeCatalog{}
|
|
page := mirrorPageDeps(t, MirrorDeps{Mirror: &fakeMirror{}, Catalog: cat, Fleet: speedFleet{}}, true)
|
|
body := mirrorPost(t, page, "/admin/mirror/serve", values()).Body.String()
|
|
if len(cat.added) != 0 || !strings.Contains(body, "tokens a second on the CPU") {
|
|
t.Fatalf("an 18 GB model on four cores went through unasked: %s", body)
|
|
}
|
|
mirrorPost(t, page, "/admin/mirror/serve", values("cpu_ok", "yes"))
|
|
if len(cat.added) != 1 {
|
|
t.Fatal("ticking the box should add it")
|
|
}
|
|
}
|