Ilker ManapandClaude Opus 5.5
663770226e
An adapters view in the explorer, filtered by the model a held model was made from; GGUF files told apart as adapter or model by their first bytes; held adapter files marked; and a model's card attaches and detaches held adapters of its architecture, refusing models and mismatched architectures. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
336 lines
14 KiB
Go
336 lines
14 KiB
Go
package web
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// roomyFleet runs one replica of "coder" on n2 and has room for `more`.
|
|
type roomyFleet struct {
|
|
fakeFleet
|
|
more int
|
|
blocked []string
|
|
asked int
|
|
preempt int
|
|
preempts []string
|
|
}
|
|
|
|
func (f *roomyFleet) Nodes() []adminapi.NodeView {
|
|
return []adminapi.NodeView{{ID: "n1", MemoryGB: 16}, {ID: "n2", MemoryGB: 32}}
|
|
}
|
|
func (f *roomyFleet) Models() []adminapi.ModelView {
|
|
return []adminapi.ModelView{{Name: "coder", Min: 1, Max: 1, Ready: 1, Total: 1, Desired: 1}}
|
|
}
|
|
func (f *roomyFleet) Replicas(string) []adminapi.ReplicaView {
|
|
return []adminapi.ReplicaView{{ID: "coder-abc", Model: "coder", State: "ready", AgeSec: 3700,
|
|
EngineKey: "secret-engine-key",
|
|
Shards: []adminapi.ShardView{{Node: "n2", CPUCores: 3.5, MemoryBytes: 6 << 30}}}}
|
|
}
|
|
func (f *roomyFleet) Headroom(model string, limit int) (adminapi.HeadroomView, bool) {
|
|
f.asked = limit
|
|
more := min(f.more, limit)
|
|
h := adminapi.HeadroomView{Model: model, Current: 1, More: more}
|
|
for range more {
|
|
h.Nodes = append(h.Nodes, []string{"n1"})
|
|
}
|
|
if more < limit {
|
|
h.Blocked = f.blocked
|
|
} else {
|
|
h.Limited = true
|
|
}
|
|
h.ByPreempting, h.Preempts = f.preempt, f.preempts
|
|
return h, true
|
|
}
|
|
|
|
func detailPage(t *testing.T, fleet *roomyFleet, cat *fakeCatalog) func(string) string {
|
|
t.Helper()
|
|
m := &fakeMirror{models: []mirror.Status{
|
|
{Model: "coder", Version: "v1", File: "c.gguf", State: mirror.Present, BytesDone: 5 << 30},
|
|
}}
|
|
mux := mirrorPageDeps(t, MirrorDeps{Mirror: m, Catalog: cat, Fleet: fleet}, true)
|
|
return func(path string) string { return mirrorGet(t, mux, path).Body.String() }
|
|
}
|
|
|
|
func coderCatalog() *fakeCatalog {
|
|
return &fakeCatalog{sources: map[string][]catalog.Model{"http://mirror:8081/coder/v1/c.gguf": {
|
|
{Name: "coder", Scaling: catalog.Scaling{Min: 1, Max: 1}}}}}
|
|
}
|
|
|
|
func TestAHeldModelOpensToWhereItRunsAndHowMuchRoomIsLeft(t *testing.T) {
|
|
fleet := &roomyFleet{more: 2, blocked: []string{"n2: 2 GB free of 32, profile needs 6"}}
|
|
get := detailPage(t, fleet, coderCatalog())
|
|
|
|
closed := get("/admin/mirror")
|
|
if strings.Contains(closed, "held-card") {
|
|
t.Error("a card is open before anything was clicked")
|
|
}
|
|
if !strings.Contains(closed, `href="/admin/mirror?tab=held&open=coder%3av1"`) {
|
|
t.Error("the model's name does not open its card")
|
|
}
|
|
|
|
body := get("/admin/mirror?tab=held&open=coder:v1")
|
|
for _, want := range []string{
|
|
"held-card", `<a href="/admin/nodes/n2">n2</a>`, "32 GB", "coder-abc", "1h01m", "3.5 cores", "6.0 GiB",
|
|
"Room for 2 more now, on n1, n1.", "n2: 2 GB free of 32, profile needs 6",
|
|
`name="min" value="1" min="0" max="3"`, `action="/admin/mirror/replicas"`,
|
|
`name="open" value="coder:v1"`,
|
|
} {
|
|
if !strings.Contains(body, want) {
|
|
t.Errorf("the card is missing %q", want)
|
|
}
|
|
}
|
|
if strings.Contains(body, "secret-engine-key") {
|
|
t.Error("the engine's key reached the page")
|
|
}
|
|
if strings.Contains(body, `http-equiv="refresh"`) {
|
|
t.Error("a page with an open form reloads under the operator")
|
|
}
|
|
}
|
|
|
|
func TestAReplicaCountTheClusterHasNoRoomForIsRefused(t *testing.T) {
|
|
fleet := &roomyFleet{more: 1, blocked: []string{"n1: 1 GB free of 16, profile needs 6"}}
|
|
cat := coderCatalog()
|
|
mux := mirrorPageDeps(t, MirrorDeps{Mirror: &fakeMirror{models: []mirror.Status{
|
|
{Model: "coder", Version: "v1", File: "c.gguf", State: mirror.Present, BytesDone: 1},
|
|
}}, Catalog: cat, Fleet: fleet}, true)
|
|
post := func(lo, hi string) string {
|
|
return mirrorPost(t, mux, "/admin/mirror/replicas", url.Values{"csrf": {"tok"},
|
|
"model": {"coder"}, "min": {lo}, "max": {hi}, "open": {"coder:v1"}}).Body.String()
|
|
}
|
|
|
|
body := post("3", "3")
|
|
if len(cat.scaled) != 0 {
|
|
t.Fatalf("three always-on replicas were set with room for two: %+v", cat.lastScale)
|
|
}
|
|
if !strings.Contains(body, "can always run at most 2 now") || !strings.Contains(body, "1 GB free of 16") {
|
|
t.Errorf("the refusal should say how many fit and why: %s", body)
|
|
}
|
|
if !strings.Contains(body, "held-card") {
|
|
t.Error("the card closed on the refusal")
|
|
}
|
|
|
|
body = post("2", "4")
|
|
if len(cat.scaled) != 1 || cat.lastScale.Min != 2 || cat.lastScale.Max != 4 {
|
|
t.Fatalf("what fits was not set: %+v", cat.lastScale)
|
|
}
|
|
if !strings.Contains(body, "coder runs 2, and up to 4 under load; there is room for 2 now") {
|
|
t.Errorf("a ceiling above the room should be taken and said: %s", body)
|
|
}
|
|
|
|
post("0", "2")
|
|
if cat.lastScale.Min != 0 || cat.lastScale.Max != 2 || cat.lastScale.IdleTimeout == 0 {
|
|
t.Errorf("on demand was set as %+v", cat.lastScale)
|
|
}
|
|
for _, bad := range [][2]string{{"3", "2"}, {"-1", "1"}, {"x", "1"}, {"1", "100"}} {
|
|
n := len(cat.scaled)
|
|
post(bad[0], bad[1])
|
|
if len(cat.scaled) != n {
|
|
t.Errorf("min %s max %s was accepted", bad[0], bad[1])
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAModelSetInConfigurationIsShownButNotChanged(t *testing.T) {
|
|
cat := coderCatalog()
|
|
cat.declared = "coder"
|
|
body := detailPage(t, &roomyFleet{more: 20}, cat)("/admin/mirror?open=coder:v1")
|
|
if strings.Contains(body, `action="/admin/mirror/replicas"`) || !strings.Contains(body, "change its replicas there") {
|
|
t.Error("a declared model offered a change that would revert")
|
|
}
|
|
if !strings.Contains(body, "Room for at least 16 more") {
|
|
t.Error("room that reached the limit asked should read as at least")
|
|
}
|
|
}
|
|
|
|
func TestAHeldRowSaysHowItsModelRunsWithoutOpening(t *testing.T) {
|
|
body := detailPage(t, &roomyFleet{more: 1}, coderCatalog())("/admin/mirror")
|
|
if strings.Contains(body, "held-card") {
|
|
t.Fatal("the card opened by itself")
|
|
}
|
|
if !strings.Contains(body, `title="replicas ready / wanted">1/1 · <span class="node" title="n2">n2</span></span>`) {
|
|
t.Errorf("the row does not say where its replicas run: %s", body)
|
|
}
|
|
}
|
|
|
|
func TestRunningOnNamesEachNodeOnceAndCountsWhatIsStarting(t *testing.T) {
|
|
nodes, starting := runningOn([]adminapi.ReplicaView{
|
|
{State: "ready", Shards: []adminapi.ShardView{{Node: "n2"}}},
|
|
{State: "starting", Shards: []adminapi.ShardView{{Node: "n1"}}},
|
|
{State: "ready", Shards: []adminapi.ShardView{{Node: "n2"}}},
|
|
})
|
|
if strings.Join(nodes, ",") != "n1,n2" || starting != 1 {
|
|
t.Errorf("nodes %v, starting %d", nodes, starting)
|
|
}
|
|
}
|
|
|
|
func TestNodeNamesLoseOnlyWhatEveryNodeShares(t *testing.T) {
|
|
for _, c := range []struct {
|
|
ids []string
|
|
want string
|
|
}{
|
|
{[]string{"llmgw-node01", "llmgw-node02", "llmgw-node06"}, "llmgw-"},
|
|
{[]string{"gpu-a.dc1", "gpu-b.dc1"}, "gpu-"},
|
|
{[]string{"n1", "n2"}, ""},
|
|
{[]string{"llmgw-node01"}, ""},
|
|
{[]string{"alpha", "beta"}, ""},
|
|
} {
|
|
if got := sharedPrefix(c.ids); got != c.want {
|
|
t.Errorf("%v: %q, expected %q", c.ids, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func settingsPage(t *testing.T, cat *fakeCatalog, withQuotas bool) *http.ServeMux {
|
|
t.Helper()
|
|
a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: true}}
|
|
p := NewAccountPages(a, nil, nil, "v0", slog.New(slog.DiscardHandler))
|
|
if withQuotas {
|
|
p.SetCapacity(&fakeCapacity{})
|
|
}
|
|
mux := http.NewServeMux()
|
|
p.RegisterAdminMirror(mux, MirrorDeps{Mirror: &fakeMirror{models: []mirror.Status{
|
|
{Model: "coder", Version: "v1", File: "c.gguf", State: mirror.Present, BytesDone: 1},
|
|
}}, Catalog: cat, Fleet: &roomyFleet{more: 3}})
|
|
return mux
|
|
}
|
|
|
|
func TestACardSetsClassOwnerAndReservationAndOnlyWhatChanged(t *testing.T) {
|
|
cat := coderCatalog()
|
|
cat.sources["http://mirror:8081/coder/v1/c.gguf"][0].Profiles = []catalog.Profile{{Name: "cpu", MinMemoryGB: 12, CPUCores: 4}}
|
|
mux := settingsPage(t, cat, true)
|
|
|
|
card := mirrorGet(t, mux, "/admin/mirror?open=coder:v1").Body.String()
|
|
for _, want := range []string{`name="class"`, `<option value="" selected>default (standard)</option>`,
|
|
`name="owner"`, `<option value="team:t1">Acme / ml</option>`, `name="memory" value="12"`, `name="cores" value="4"`} {
|
|
if !strings.Contains(card, want) {
|
|
t.Errorf("the card lacks %q", want)
|
|
}
|
|
}
|
|
post := func(v url.Values) string {
|
|
v.Set("csrf", "tok")
|
|
v.Set("model", "coder")
|
|
v.Set("open", "coder:v1")
|
|
return mirrorPost(t, mux, "/admin/mirror/replicas", v).Body.String()
|
|
}
|
|
same := url.Values{"min": {"1"}, "max": {"1"}, "class": {""}, "owner": {""}, "memory": {"12"}, "cores": {"4"}}
|
|
if body := post(same); !strings.Contains(body, "coder: nothing changed") ||
|
|
len(cat.classes)+len(cat.owners)+len(cat.resources)+len(cat.scaled) != 0 {
|
|
t.Errorf("an unchanged form changed something: %v %v %v %v", cat.classes, cat.owners, cat.resources, cat.scaled)
|
|
}
|
|
changed := url.Values{"min": {"1"}, "max": {"1"}, "class": {"critical"}, "owner": {"team:t1"}, "memory": {"14"}, "cores": {"4"}}
|
|
body := post(changed)
|
|
if strings.Join(cat.resources, ",") != "coder=14/4" || strings.Join(cat.classes, ",") != "coder=critical" ||
|
|
strings.Join(cat.owners, ",") != "coder=team:t1" || len(cat.scaled) != 0 {
|
|
t.Errorf("applied %v %v %v %v", cat.resources, cat.classes, cat.owners, cat.scaled)
|
|
}
|
|
if !strings.Contains(body, "a replica reserves 14 GB and 4 cores") || !strings.Contains(body, "counts against team:t1") {
|
|
t.Errorf("the change was not said: %s", body)
|
|
}
|
|
|
|
// Without capacity quotas there is no owner to choose, and none is sent.
|
|
plain := mirrorGet(t, settingsPage(t, coderCatalog(), false), "/admin/mirror?open=coder:v1").Body.String()
|
|
if strings.Contains(plain, `name="owner"`) {
|
|
t.Error("an owner choice with no quotas to count against")
|
|
}
|
|
}
|
|
|
|
func TestRoomAHigherClassWouldMakeIsOfferedAndSaid(t *testing.T) {
|
|
fleet := &roomyFleet{more: 0, preempt: 1, preempts: []string{"chat"}, blocked: []string{"n1: full"}}
|
|
cat := coderCatalog()
|
|
mux := mirrorPageDeps(t, MirrorDeps{Mirror: &fakeMirror{models: []mirror.Status{
|
|
{Model: "coder", Version: "v1", File: "c.gguf", State: mirror.Present, BytesDone: 1},
|
|
}}, Catalog: cat, Fleet: fleet}, true)
|
|
|
|
card := mirrorGet(t, mux, "/admin/mirror?open=coder:v1").Body.String()
|
|
if !strings.Contains(card, "No free room for another replica now.") ||
|
|
!strings.Contains(card, "1 by stopping replicas its class outranks: chat.") ||
|
|
!strings.Contains(card, `name="min" value="1" min="0" max="2"`) {
|
|
t.Errorf("the card should offer the room preemption makes, and say whose")
|
|
}
|
|
body := mirrorPost(t, mux, "/admin/mirror/replicas", url.Values{"csrf": {"tok"},
|
|
"model": {"coder"}, "min": {"2"}, "max": {"2"}, "open": {"coder:v1"}}).Body.String()
|
|
if len(cat.scaled) != 1 || cat.lastScale.Min != 2 {
|
|
t.Fatalf("two were refused though preemption makes room: %s", body)
|
|
}
|
|
if !strings.Contains(body, "replicas of chat will be stopped") {
|
|
t.Errorf("stopping somebody else's replica should be said: %s", body)
|
|
}
|
|
body = mirrorPost(t, mux, "/admin/mirror/replicas", url.Values{"csrf": {"tok"},
|
|
"model": {"coder"}, "min": {"3"}, "max": {"3"}, "open": {"coder:v1"}}).Body.String()
|
|
if len(cat.scaled) != 1 || !strings.Contains(body, "at most 2") {
|
|
t.Errorf("three is past even what preemption makes: %s", body)
|
|
}
|
|
}
|
|
|
|
func TestAnAdapterHeldInTheMirrorIsAttachedFromItsModelsCard(t *testing.T) {
|
|
prev := kindOf
|
|
t.Cleanup(func() { kindOf = prev })
|
|
kindOf = func(_ context.Context, u string) (gguf.Header, error) {
|
|
switch {
|
|
case strings.Contains(u, "/sql/"):
|
|
return gguf.Header{Architecture: "qwen2", Type: "adapter"}, nil
|
|
case strings.Contains(u, "/llama-lora/"):
|
|
return gguf.Header{Architecture: "llama", Type: "adapter"}, nil
|
|
}
|
|
return gguf.Header{Architecture: "qwen2"}, nil
|
|
}
|
|
cat := coderCatalog()
|
|
coder := &cat.sources["http://mirror:8081/coder/v1/c.gguf"][0]
|
|
coder.Source = "http://mirror:8081/coder/v1/c.gguf"
|
|
coder.Profiles = []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 12}}
|
|
m := &fakeMirror{models: []mirror.Status{
|
|
{Model: "coder", Version: "v1", File: "c.gguf", State: mirror.Present, BytesDone: 1,
|
|
Source: "https://huggingface.co/bartowski/Qwen2.5-Coder-7B-Instruct-GGUF/resolve/abc/c.gguf"},
|
|
{Model: "sql", Version: "v1", File: "sql.gguf", State: mirror.Present, BytesDone: 1},
|
|
{Model: "llama-lora", Version: "v1", File: "l.gguf", State: mirror.Present, BytesDone: 1},
|
|
}}
|
|
mux := mirrorPageDeps(t, MirrorDeps{Mirror: m, Catalog: cat, Fleet: &roomyFleet{more: 1}}, true)
|
|
|
|
list := mirrorGet(t, mux, "/admin/mirror").Body.String()
|
|
if strings.Count(list, ">LoRA adapter</span>") != 2 {
|
|
t.Error("the two adapter files are not marked as adapters in the list")
|
|
}
|
|
card := mirrorGet(t, mux, "/admin/mirror?open=coder:v1").Body.String()
|
|
if !strings.Contains(card, `<option value="http://mirror:8081/sql/v1/sql.gguf" data-suggest="coder-sql">`) {
|
|
t.Error("the qwen2 adapter is not offered for the qwen2 model")
|
|
}
|
|
if strings.Contains(card, "llama-lora") && strings.Contains(card, `value="http://mirror:8081/llama-lora/v1/l.gguf"`) {
|
|
t.Error("an adapter for another architecture is offered")
|
|
}
|
|
if !strings.Contains(card, "view=adapters&from=bartowski%2FQwen2.5-Coder-7B-Instruct-GGUF") {
|
|
t.Error("finding adapters should start from the model's own repository")
|
|
}
|
|
|
|
post := func(v url.Values) string {
|
|
v.Set("csrf", "tok")
|
|
v.Set("model", "coder")
|
|
v.Set("open", "coder:v1")
|
|
return mirrorPost(t, mux, "/admin/mirror/adapters", v).Body.String()
|
|
}
|
|
body := post(url.Values{"op": {"add"}, "name": {"coder-sql"}, "source": {"http://mirror:8081/sql/v1/sql.gguf"}, "scale": {"0.8"}})
|
|
if len(cat.adapterSets) != 1 || len(cat.adapterSets[0]) != 1 || cat.adapterSets[0][0].Scale != 0.8 {
|
|
t.Fatalf("attached %+v: %s", cat.adapterSets, body)
|
|
}
|
|
if !strings.Contains(body, "adapter coder-sql added") {
|
|
t.Errorf("the attach was not said: %s", body)
|
|
}
|
|
for name, src := range map[string]string{"another architecture": "http://mirror:8081/llama-lora/v1/l.gguf",
|
|
"a model": "http://mirror:8081/coder/v1/c.gguf"} {
|
|
n := len(cat.adapterSets)
|
|
out := post(url.Values{"op": {"add"}, "name": {"x"}, "source": {src}})
|
|
if len(cat.adapterSets) != n || !strings.Contains(out, `class="error"`) {
|
|
t.Errorf("%s was attached", name)
|
|
}
|
|
}
|
|
}
|