Files
llmgw/internal/web/admin_mirror_test.go
T
Ilker ManapandClaude Opus 5.5 6289b01ed6 The cluster's settings live in the store, and every change takes effect
Everything in server.yaml that is not one machine's own is now a versioned
document in the store (cluster_settings), the same on every control plane.
The first start against a store imports the file's cluster keys, with the
baseline guard policy when there is none; after that those keys in a file
are ignored, with a warning.

Each control plane polls the document's version and its own server.yaml's
hash. A change is validated and applied: guard live (Gate.Replace), anything
else by a graceful restart that systemd brings back, standbys first and the
leader 30s later. A version that does not load on a machine is not applied;
at start the newest one that loads is used, so a bad edit cannot keep every
control plane down.

llmgw ctl config show/get/set/unset/edit/history/rollback, the admin API
under /api/v1/settings, and the console's Settings screen, which layers the
store's document over the machine's file, write it. Commands beside the
control plane that read cluster keys (init, node add, server add, backup,
upgrade) read the layered configuration; init --bundle writes offline to
the store once it holds the settings.

A standby that becomes the leader re-reads models, engine choices and
facility holds from the store.

Retires appendGuardIfMissing: the import does it.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-05 15:25:23 +02:00

748 lines
29 KiB
Go

package web
import (
"context"
"errors"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/account"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
)
// fakeAccounts resolves one session, for whoever the test says is looking.
type fakeAccounts struct{ user store.User }
func (f fakeAccounts) SignUp(context.Context, string, string, string) error { return nil }
func (f fakeAccounts) SignIn(context.Context, string, string) (string, string, error) {
return "", "", nil
}
func (f fakeAccounts) Resolve(_ context.Context, token string) (store.Session, store.User, error) {
if token != "good" {
return store.Session{}, store.User{}, errors.New("no such session")
}
return store.Session{UserID: f.user.ID}, f.user, nil
}
func (f fakeAccounts) SignOut(context.Context, string) error { return nil }
func (f fakeAccounts) CheckCSRF(_ store.Session, sub string) bool { return sub == "tok" }
func (f fakeAccounts) SignupOpen() bool { return false }
func (f fakeAccounts) OpenSession(context.Context, string) (string, string, error) {
return "", "", nil
}
func (f fakeAccounts) ChangePassword(context.Context, account.PasswordChanger,
store.User, string, string,
) error {
return nil
}
// fakeMirror stands in for the mirror's API, which lives on another machine.
type fakeMirror struct {
models []mirror.Status
listErr error
fetched mirror.FetchRequest
all []mirror.FetchRequest
fetchErr error
removed [2]string
removeErr error
}
func (f *fakeMirror) Addr() string { return "http://mirror:8082" }
func (f *fakeMirror) List(context.Context) (mirror.ModelsResponse, error) {
if f.listErr != nil {
return mirror.ModelsResponse{}, f.listErr
}
return mirror.ModelsResponse{Base: "http://mirror:8081", Models: f.models}, nil
}
func (f *fakeMirror) Fetch(_ context.Context, r mirror.FetchRequest) (mirror.FetchResponse, error) {
f.fetched = r
f.all = append(f.all, r)
if f.fetchErr != nil {
return mirror.FetchResponse{}, f.fetchErr
}
return mirror.FetchResponse{
Status: mirror.Status{Model: r.Model, Version: r.Version, State: mirror.Fetching},
}, nil
}
func (f *fakeMirror) Remove(_ context.Context, model, version string) ([]string, error) {
f.removed = [2]string{model, version}
if f.removeErr != nil {
return nil, f.removeErr
}
return []string{"model.gguf"}, nil
}
// mirrorPage mounts the page for an admin, or for somebody who is not one.
func mirrorPage(t *testing.T, m MirrorAdmin, admin bool) *http.ServeMux {
return mirrorPageWith(t, m, nil, admin)
}
// mirrorPageWith mounts it with a registry behind the explorer.
func mirrorPageWith(t *testing.T, m MirrorAdmin, reg Registry, admin bool) *http.ServeMux {
return mirrorPageDeps(t, MirrorDeps{Mirror: m, Registry: reg}, admin)
}
// mirrorPageDeps mounts it with everything chosen.
func mirrorPageDeps(t *testing.T, d MirrorDeps, admin bool) *http.ServeMux {
t.Helper()
a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: admin}}
p := NewAccountPages(a, nil, nil, "v0",
slog.New(slog.NewTextHandler(io.Discard, nil)))
mux := http.NewServeMux()
p.RegisterAdminMirror(mux, d)
return mux
}
func mirrorGet(t *testing.T, mux *http.ServeMux, path string) *httptest.ResponseRecorder {
t.Helper()
r := httptest.NewRequest("GET", path, nil)
r.AddCookie(&http.Cookie{Name: SessionCookie, Value: "good"})
r.AddCookie(&http.Cookie{Name: csrfCookie, Value: "tok"})
w := httptest.NewRecorder()
mux.ServeHTTP(w, r)
return w
}
func mirrorPost(t *testing.T, mux *http.ServeMux, path string, form url.Values) *httptest.ResponseRecorder {
t.Helper()
r := httptest.NewRequest("POST", path, strings.NewReader(form.Encode()))
r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
r.AddCookie(&http.Cookie{Name: SessionCookie, Value: "good"})
r.AddCookie(&http.Cookie{Name: csrfCookie, Value: "tok"})
w := httptest.NewRecorder()
mux.ServeHTTP(w, r)
return w
}
func TestTheMirrorPageShowsTheAddressACatalogShouldUse(t *testing.T) {
m := &fakeMirror{models: []mirror.Status{{
Model: "qwen", Version: "2026-09", File: "model.gguf",
State: mirror.Present, BytesDone: 491400032, BytesTotal: 491400032,
SHA256: "74a4da8c9fdbcd15bd1f6d01d621410d31c6fc00986f5eb687824e7b93d7a9db",
}}}
body := mirrorGet(t, mirrorPage(t, m, true), "/admin/mirror").Body.String()
// The one string on the page somebody will copy. Rendered from the base the
// mirror reported, not assembled here — two spellings of one address is
// what makes a file that is in use look unreferenced.
if !strings.Contains(body, "http://mirror:8081/qwen/2026-09/model.gguf") {
t.Error("the page should render the URL a catalog entry needs")
}
if !strings.Contains(body, "468.6 MiB") {
t.Errorf("the size should be readable; body was %s", body)
}
}
func TestAnUnreachableMirrorIsNotAnEmptyOne(t *testing.T) {
m := &fakeMirror{listErr: errors.New("reaching the mirror at http://mirror:8082: refused")}
w := mirrorGet(t, mirrorPage(t, m, true), "/admin/mirror")
// The two render almost identically and mean opposite things: one is a
// mirror waiting to be filled, the other is a machine to go and look at.
body := w.Body.String()
if !strings.Contains(body, "not currently reachable") {
t.Errorf("the page should say the mirror did not answer; got %s", body)
}
if strings.Contains(body, "holds nothing yet") {
t.Error("an unreachable mirror must not be reported as an empty one")
}
}
func TestSomebodyWhoIsNotAnAdminIsNotToldThePageExists(t *testing.T) {
w := mirrorGet(t, mirrorPage(t, &fakeMirror{}, false), "/admin/mirror")
// 404 rather than 403: a 403 confirms the page is there, which is free
// reconnaissance for anybody probing.
if w.Code != http.StatusNotFound {
t.Errorf("expected 404 for a non-admin, got %d", w.Code)
}
}
func TestFetchingWithoutTheCSRFTokenIsRefused(t *testing.T) {
m := &fakeMirror{}
w := mirrorPost(t, mirrorPage(t, m, true), "/admin/mirror/fetch", url.Values{
"source": {"http://x/model.gguf"}, "model": {"m"}, "version": {"v"},
})
if w.Code != http.StatusBadRequest {
t.Errorf("expected 400 without a CSRF token, got %d", w.Code)
}
if m.fetched.Source != "" {
t.Error("nothing should have reached the mirror")
}
}
func TestFetchSaysStartedRatherThanAdded(t *testing.T) {
m := &fakeMirror{}
body := mirrorPost(t, mirrorPage(t, m, true), "/admin/mirror/fetch", url.Values{
"csrf": {"tok"},
"source": {" http://x/model.gguf "}, "model": {"qwen"}, "version": {"2026-09"},
}).Body.String()
// Trimmed, because a URL pasted from a terminal carries whitespace and the
// mirror would take it as part of the address.
if m.fetched.Source != "http://x/model.gguf" {
t.Errorf("the source should be trimmed; got %q", m.fetched.Source)
}
// "started", not "added": the mirror answered 202 and the bytes are still
// moving. A message saying it was added invites pointing the catalog at a
// partial file.
if !strings.Contains(body, "started downloading qwen/2026-09") {
t.Errorf("the message should say the download began; got %s", body)
}
}
func TestTheMirrorsRefusalReachesTheOperator(t *testing.T) {
m := &fakeMirror{removeErr: &mirror.APIError{
Status: http.StatusConflict, Msg: "qwen/2026-09 is still downloading; nothing was removed",
}}
body := mirrorPost(t, mirrorPage(t, m, true), "/admin/mirror/qwen/2026-09/remove",
url.Values{"csrf": {"tok"}}).Body.String()
// The mirror's own sentence, not "the mirror said 409". The reason is in
// the body of its answer precisely so it can be shown here.
if !strings.Contains(body, "still downloading") {
t.Errorf("the mirror's reason should be on the page; got %s", body)
}
}
func TestRemovePassesTheVersionThrough(t *testing.T) {
m := &fakeMirror{}
mirrorPost(t, mirrorPage(t, m, true), "/admin/mirror/qwen/2026-09/remove",
url.Values{"csrf": {"tok"}})
if m.removed != [2]string{"qwen", "2026-09"} {
t.Errorf("the version to remove did not arrive intact: %v", m.removed)
}
}
func TestThePageReloadsOnlyWhileSomethingIsDownloading(t *testing.T) {
idle := mirrorGet(t, mirrorPage(t, &fakeMirror{models: []mirror.Status{
{Model: "m", Version: "v", File: "f", State: mirror.Present},
}}, true), "/admin/mirror").Body.String()
// A reload while nothing is happening would take with it whatever the
// operator was typing into the fetch form.
if strings.Contains(idle, "http-equiv=\"refresh\"") {
t.Error("an idle page should not reload itself")
}
busy := mirrorGet(t, mirrorPage(t, &fakeMirror{models: []mirror.Status{
{Model: "m", Version: "v", File: "f", State: mirror.Fetching},
}}, true), "/admin/mirror").Body.String()
if !strings.Contains(busy, "http-equiv=\"refresh\"") {
t.Error("a page with a download running should follow it")
}
}
func TestNoPercentageIsInventedWhenTheTotalIsUnknown(t *testing.T) {
body := mirrorGet(t, mirrorPage(t, &fakeMirror{models: []mirror.Status{{
Model: "m", Version: "v", File: "f", State: mirror.Fetching,
BytesDone: 1048576, BytesTotal: 0,
}}}, true), "/admin/mirror").Body.String()
// A bar that guesses its own end and then stops moving is worse than a
// number that only goes up.
if strings.Contains(body, "%)") {
t.Errorf("no percentage should appear without a total; got %s", body)
}
if !strings.Contains(body, "1.0 MiB so far") {
t.Errorf("the byte count should still be shown; got %s", body)
}
}
func TestDownloadsAndFailuresComeFirst(t *testing.T) {
body := mirrorGet(t, mirrorPage(t, &fakeMirror{models: []mirror.Status{
{Model: "aaa", Version: "v", File: "f", State: mirror.Present},
{Model: "zzz", Version: "v", File: "f", State: mirror.Fetching},
{Model: "mmm", Version: "v", File: "f", State: mirror.Failed, Err: "upstream 404"},
}}, true), "/admin/mirror").Body.String()
// The two states somebody opened this page for are the two at the top,
// whatever they are called alphabetically.
fetching := strings.Index(body, "zzz")
failed := strings.Index(body, "mmm")
present := strings.Index(body, "aaa")
if !(fetching < failed && failed < present) {
t.Errorf("expected fetching, then failed, then present; got %d %d %d",
fetching, failed, present)
}
if !strings.Contains(body, "upstream 404") {
t.Error("a failed fetch should show the reason it failed")
}
}
func TestThePageRendersThroughTheRealClient(t *testing.T) {
// The three pieces together: the page, the HTTP client, and the mirror's
// own API behind a real listener. Each is tested on its own elsewhere, and
// each can be right on its own while the three disagree about a field name
// or a status code — which is a defect that only shows up in a browser.
dir := t.TempDir()
if err := os.MkdirAll(filepath.Join(dir, "qwen", "2026-09"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(
filepath.Join(dir, "qwen", "2026-09", "model.gguf"), []byte("weights"), 0o600); err != nil {
t.Fatal(err)
}
log := slog.New(slog.NewTextHandler(io.Discard, nil))
srv := httptest.NewServer(
mirror.NewAPI(privateOK(mirror.NewManager(dir, log)), "http://mirror:8081").Handler())
defer srv.Close()
c, err := mirror.NewClient(srv.URL, tlsconf.Config{}, 0)
if err != nil {
t.Fatal(err)
}
body := mirrorGet(t, mirrorPage(t, c, true), "/admin/mirror").Body.String()
if !strings.Contains(body, "http://mirror:8081/qwen/2026-09/model.gguf") {
t.Errorf("the catalog URL should survive all three hops; got %s", body)
}
if !strings.Contains(body, srv.URL) {
t.Error("the page should name the mirror it is reading")
}
}
func TestAFailedRowOffersRetryRatherThanRemove(t *testing.T) {
body := mirrorGet(t, mirrorPage(t, &fakeMirror{models: []mirror.Status{{
Model: "m", Version: "v", File: "model.gguf", State: mirror.Failed,
Source: "http://upstream/model.gguf", Err: "upstream answered 404",
}}}, true), "/admin/mirror").Body.String()
// A fetch that failed left nothing on disk, so a remove would come back
// "the mirror holds nothing for m/v" — a button whose only outcome is a
// confusing error.
if strings.Contains(body, "/admin/mirror/m/v/remove") {
t.Error("a failed row should not offer a remove that can only 404")
}
if !strings.Contains(body, ">retry<") {
t.Error("a failed row should offer to start the download again")
}
if !strings.Contains(body, `value="http://upstream/model.gguf"`) {
t.Error("retry has to carry the source it should fetch again")
}
}
type fakeRegistry struct {
results []hf.Model
repo hf.Repo
err error
asked string
query hf.Query
explored int
}
func (f *fakeRegistry) Explore(_ context.Context, q hf.Query) ([]hf.Model, error) {
f.asked = q.Search
f.query = q
f.explored++
return f.results, f.err
}
func (f *fakeRegistry) Repo(_ context.Context, id string) (hf.Repo, error) {
f.asked = id
return f.repo, f.err
}
func TestSearchingTheRegistryListsRepositories(t *testing.T) {
reg := &fakeRegistry{results: []hf.Model{
{ID: "Qwen/Qwen2.5-Coder-7B-Instruct-GGUF", Downloads: 295447, PipelineTag: "text-generation"},
}}
body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{}, reg, true),
"/admin/mirror?q=qwen+coder").Body.String()
if reg.asked != "qwen coder" {
t.Errorf("the query did not arrive: %q", reg.asked)
}
if !strings.Contains(body, "Qwen2.5-Coder-7B-Instruct-GGUF") {
t.Errorf("the result is missing; got %s", body)
}
}
func TestOnlyWeightFilesAreOffered(t *testing.T) {
reg := &fakeRegistry{repo: hf.Repo{
ID: "Qwen/X", Revision: "abc123def456",
Files: []hf.File{
{Path: "README.md", Bytes: 900},
{Path: "model.gguf", Bytes: 1117320736, SHA256: "6a1a2eb6d15622bf3c96857206351ba97e1af16c30d7a74ee38970e434e9407e"},
},
}}
body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{}, reg, true),
"/admin/mirror?repo=Qwen/X").Body.String()
// A repository is mostly a README and a tokenizer; listing those buries
// the four files somebody came for.
if strings.Contains(body, "README.md") {
t.Error("documentation should not be offered for mirroring")
}
if !strings.Contains(body, "model.gguf") || !strings.Contains(body, "1.0 GiB") {
t.Errorf("the weights and their size should be listed; got %s", body)
}
}
func TestTheFetchButtonPinsTheRevisionAndSuggestsANames(t *testing.T) {
reg := &fakeRegistry{repo: hf.Repo{
ID: "Qwen/Qwen2.5-1.5B-Instruct-GGUF", Revision: "abc123def456789",
Files: []hf.File{{Path: "model.gguf", Bytes: 10}},
}}
body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{}, reg, true),
"/admin/mirror?repo=Qwen/Qwen2.5-1.5B-Instruct-GGUF").Body.String()
// Pinned, so what arrives is what was listed even if the branch moves.
if !strings.Contains(body, "/resolve/abc123def456789/model.gguf") {
t.Errorf("the source should pin the revision; got %s", body)
}
// The owner is not part of what anybody calls the model, and -GGUF
// describes the file rather than the model.
if !strings.Contains(body, `value="qwen2.5-1.5b-instruct"`) {
t.Error("the suggested name should drop the owner and the format marker")
}
if !strings.Contains(body, `value="abc123de"`) {
t.Error("the version should default to the revision that was listed")
}
}
func TestARegistryThatIsDownDoesNotBreakTheMirrorPage(t *testing.T) {
reg := &fakeRegistry{err: errors.New("reaching the model registry: timeout")}
body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{models: []mirror.Status{{
Model: "held", Version: "v", File: "f", State: mirror.Present,
}}}, reg, true), "/admin/mirror?q=qwen").Body.String()
// The registry is somebody else's server. It being down says nothing about
// the mirror, which is what this page is for.
if !strings.Contains(body, "The registry:") {
t.Error("the registry failure should be reported on its own line")
}
if !strings.Contains(body, "held") {
t.Error("what the mirror holds must still be listed")
}
}
func TestWithNoRegistryThereIsNoSearchBox(t *testing.T) {
body := mirrorGet(t, mirrorPage(t, &fakeMirror{}, true), "/admin/mirror").Body.String()
// A deployment with no way out to the internet gets no search box, rather
// than one that can only time out.
if strings.Contains(body, "Search Hugging Face") {
t.Error("the explorer should be absent without a registry")
}
if strings.Contains(body, "Explore Hugging Face") {
t.Error("a tab for an explorer that is not there")
}
fetch := mirrorGet(t, mirrorPage(t, &fakeMirror{}, true), "/admin/mirror?tab=fetch").Body.String()
if !strings.Contains(fetch, `name="source" required`) {
t.Error("fetching by hand must still be there")
}
}
func shardedRepo() *fakeRegistry {
return &fakeRegistry{repo: hf.Repo{
ID: "Qwen/Qwen2.5-7B-Instruct", Revision: "abc123def456",
Files: []hf.File{
{Path: "model-00001-of-00004.safetensors", Bytes: 4_000_000_000},
{Path: "model-00002-of-00004.safetensors", Bytes: 4_000_000_000},
{Path: "model-00003-of-00004.safetensors", Bytes: 4_000_000_000},
{Path: "model-00004-of-00004.safetensors", Bytes: 3_000_000_000},
{Path: "README.md", Bytes: 900},
},
}}
}
func TestAShardedModelIsOneRowAndOneButton(t *testing.T) {
body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{}, shardedRepo(), true),
"/admin/mirror?repo=Qwen/Qwen2.5-7B-Instruct").Body.String()
// Fifteen shards is one decision, not fifteen. Four here, and one form.
if n := strings.Count(body, `action="/admin/mirror/fetch"`); n != 1 {
t.Errorf("expected the four shards as a single form, found %d forms", n)
}
if !strings.Contains(body, "4 shards, fetched together") {
t.Errorf("the row should say it is a set; got %s", body)
}
// Summed, so the operator sees what the disk is in for.
if !strings.Contains(body, "14.0 GiB") {
t.Error("the set should total its shards")
}
}
func TestFetchingASetStartsEveryShard(t *testing.T) {
m := &fakeMirror{}
form := url.Values{"csrf": {"tok"}, "model": {"qwen"}, "version": {"abc123de"}}
for i, p := range []string{"model-00001-of-00004.safetensors", "model-00002-of-00004.safetensors"} {
form.Add("source", "https://hf/resolve/abc123def456/"+p)
form.Add("file", p)
_ = i
}
body := mirrorPost(t, mirrorPageWith(t, m, shardedRepo(), true),
"/admin/mirror/fetch", form).Body.String()
if len(m.all) != 2 {
t.Fatalf("expected both shards to be started, got %d", len(m.all))
}
// Each shard keeps its own filename under one model and version, which is
// what makes them one version on disk.
if m.all[0].File == m.all[1].File {
t.Errorf("the shards collided on %q", m.all[0].File)
}
if m.all[0].Model != "qwen" || m.all[1].Version != "abc123de" {
t.Errorf("the set should share one model and version: %+v", m.all)
}
if !strings.Contains(body, "2 files") {
t.Errorf("the message should say how many started; got %s", body)
}
}
func TestASetThatFailsPartWaySaysHowFarItGot(t *testing.T) {
m := &fakeMirror{fetchErr: errors.New("the mirror is full")}
form := url.Values{"csrf": {"tok"}, "model": {"qwen"}, "version": {"v"}}
form.Add("source", "https://hf/a.safetensors")
form.Add("file", "a.safetensors")
body := mirrorPost(t, mirrorPageWith(t, m, shardedRepo(), true),
"/admin/mirror/fetch", form).Body.String()
// "0 of 1 started, then …" rather than a bare error: a set that fails on
// its fourth shard has three arriving, and reading it as nothing started
// would send somebody to fetch them all again.
if !strings.Contains(body, "0 of 1 started") {
t.Errorf("the message should say how far it got; got %s", body)
}
}
func TestAlternativeQuantisationsStaySeparateRows(t *testing.T) {
reg := &fakeRegistry{repo: hf.Repo{
ID: "Qwen/X", Revision: "abc",
Files: []hf.File{
{Path: "m-q4_k_m.gguf", Bytes: 1},
{Path: "m-q8_0.gguf", Bytes: 2},
},
}}
body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{}, reg, true),
"/admin/mirror?repo=Qwen/X").Body.String()
// Two GGUFs are alternatives, not halves. Offering "fetch all" here would
// spend gigabytes on a copy nobody asked for.
if n := strings.Count(body, `action="/admin/mirror/fetch"`); n != 2 {
t.Errorf("expected a form each, found %d", n)
}
}
func TestAHeldVersionIsOneLineAndSaysWhoUsesIt(t *testing.T) {
m := &fakeMirror{models: []mirror.Status{
{Model: "big", Version: "v1", File: "a-00001-of-00002.gguf", State: mirror.Present, BytesDone: 1 << 30},
{Model: "big", Version: "v1", File: "a-00002-of-00002.gguf", State: mirror.Present, BytesDone: 1 << 30},
{Model: "small", Version: "v1", File: "s.gguf", State: mirror.Present, BytesDone: 1 << 20},
}}
cat := &fakeCatalog{sources: map[string][]catalog.Model{"http://mirror:8081/small/v1/s.gguf": {
{Name: "small-chat", Scaling: catalog.Scaling{Min: 1, Max: 1}}}}}
body := mirrorGet(t, mirrorPageDeps(t, MirrorDeps{Mirror: m, Catalog: cat}, true),
"/admin/mirror").Body.String()
if n := strings.Count(body, "/admin/mirror/big/v1/remove"); n != 1 {
t.Errorf("a two-file version should be one row, found %d", n)
}
if !strings.Contains(body, "2 files") || !strings.Contains(body, "2.0 GiB") {
t.Error("the row should count its files and add up their size")
}
if !strings.Contains(body, `data-url="http://mirror:8081/big/v1/"`) {
t.Error("a set's address is its directory, trailing slash included")
}
if !strings.Contains(body, ">as small-chat<") {
t.Error("the model serving a file should be named on its row")
}
// Already served: offering to serve it again invites a second catalog
// entry for the same weights.
if strings.Contains(body, "serve=small:v1") {
t.Error("a file in use should not be offered for serving")
}
if !strings.Contains(body, "serve=big:v1") {
t.Error("a file nobody uses should be offered for serving")
}
}
func TestHowAModelRunsIsChangedWhereItsFileIsShown(t *testing.T) {
m := &fakeMirror{models: []mirror.Status{
{Model: "small", Version: "v1", File: "s.gguf", State: mirror.Present, BytesDone: 1 << 20},
}}
cat := &fakeCatalog{declared: "fixed", sources: map[string][]catalog.Model{"http://mirror:8081/small/v1/s.gguf": {
{Name: "small-chat", Scaling: catalog.Scaling{Min: 1, Max: 1}},
{Name: "fixed", Scaling: catalog.Scaling{Min: 0, Max: 1}},
}}}
mux := mirrorPageDeps(t, MirrorDeps{Mirror: m, Catalog: cat}, true)
body := mirrorGet(t, mux, "/admin/mirror").Body.String()
if !strings.Contains(body, `<option value="always" selected>always on</option>`) {
t.Error("the running model's mode is not shown as chosen")
}
// Declared in the cluster's settings: shown, not offered, since it is
// changed there.
if !strings.Contains(body, `title="set in the cluster&#39;s settings (llmgw ctl config)">on demand · yaml</span>`) {
t.Error("a declared model should say where its mode is set")
}
if strings.Count(body, `name="mode" class="mode"`) != 1 {
t.Error("only the runtime model should have a mode selector")
}
mirrorPost(t, mux, "/admin/mirror/mode", url.Values{"csrf": {"tok"},
"model": {"small-chat"}, "mode": {"ondemand"}, "replicas": {"1"}})
if len(cat.scaled) != 1 || cat.lastScale.Min != 0 || cat.lastScale.Max != 1 || cat.lastScale.IdleTimeout == 0 {
t.Errorf("on demand was sent as %+v", cat.lastScale)
}
}
func TestTheMirrorsTableIsGroupedByFamily(t *testing.T) {
// Four qwens under four different names were four rows with nothing in
// common; grouped, they sit under one heading. A family of one gets no
// heading, which would only repeat its row.
m := &fakeMirror{models: []mirror.Status{
{Model: "qwen2.5-coder-7b-instruct", Version: "v1", File: "a.gguf", State: mirror.Present, BytesDone: 100},
{Model: "llama-3.1-8b-instruct", Version: "v1", File: "l.gguf", State: mirror.Present, BytesDone: 5},
{Model: "qwen2.5-14b-instruct", Version: "v1", File: "b.gguf", State: mirror.Present, BytesDone: 40},
{Model: "qwen3-8b", Version: "v2", File: "c.gguf", State: mirror.Fetching, BytesDone: 10},
}}
body := mirrorGet(t, mirrorPage(t, m, true), "/admin/mirror").Body.String()
if n := strings.Count(body, `class="model-group"`); n != 1 {
t.Fatalf("%d family headings, want 1 (qwen)", n)
}
if !strings.Contains(body, "<strong>qwen</strong>") || !strings.Contains(body, "3 models · 150 B") ||
!strings.Contains(body, `<span class="tag new">downloading</span>`) {
t.Error("the qwen heading should count its models, total them and say one is downloading")
}
// The downloading family first, its models together, then llama.
var at []int
for _, name := range []string{"qwen3-8b", "qwen2.5-14b-instruct", "qwen2.5-coder-7b-instruct", "llama-3.1-8b-instruct"} {
at = append(at, strings.Index(body, "<strong>"+name+"</strong>"))
}
for i := 1; i < len(at); i++ {
if at[i-1] < 0 || at[i] < at[i-1] {
t.Errorf("rows out of order: %v", at)
}
}
}
func TestAModelsFamilyIsHowItsNameStarts(t *testing.T) {
for name, want := range map[string]string{
"qwen2.5-coder-14b-instruct": "qwen",
"qwen3-8b": "qwen",
"Qwen2.5-14B-Instruct": "qwen",
"llama-3.1-8b-instruct": "llama",
"deepseek-r1-distill-qwen-14b": "deepseek",
"phi-4": "phi",
"bge-m3": "bge",
"4bit-thing": "4bit",
} {
if got := modelFamily(name); got != want {
t.Errorf("%s is %q, want %q", name, got, want)
}
}
}
// privateOK lets a test's manager fetch from the httptest upstreams on
// loopback, which a real one refuses.
func privateOK(m *mirror.Manager) *mirror.Manager {
m.AllowPrivateSources()
return m
}
func TestEachTabDoesOnlyItsOwnWork(t *testing.T) {
reg := &fakeRegistry{}
mux := mirrorPageWith(t, &fakeMirror{models: []mirror.Status{
{Model: "m", Version: "v", File: "m.gguf", State: mirror.Present, BytesDone: 1},
{Model: "n", Version: "v", File: "n.gguf", State: mirror.Fetching, BytesDone: 1},
}}, reg, true)
held := mirrorGet(t, mux, "/admin/mirror").Body.String()
if reg.explored != 0 {
t.Error("the list asked the registry for repositories it does not show")
}
if !strings.Contains(held, `class="on" aria-current="page">Held <span class="count">1 · 1 downloading</span>`) {
t.Errorf("the Held tab should be chosen and count what it holds: %s", held)
}
if strings.Contains(held, `name="source" required`) || strings.Contains(held, "Search Hugging Face") {
t.Error("other tabs' forms are on the list")
}
if !strings.Contains(held, `http-equiv="refresh"`) {
t.Error("the list stopped following a download")
}
explore := mirrorGet(t, mux, "/admin/mirror?tab=explore").Body.String()
if reg.explored != 1 || !strings.Contains(explore, "Search Hugging Face") {
t.Error("the explorer tab did not explore")
}
if strings.Contains(explore, `id="held"`) || strings.Contains(explore, `http-equiv="refresh"`) {
t.Error("the explorer shows the list, or reloads away a search")
}
// A link from before there were tabs still lands on the explorer.
if old := mirrorGet(t, mux, "/admin/mirror?q=qwen").Body.String(); !strings.Contains(old, "Search Hugging Face") {
t.Error("a search address no longer opens the explorer")
}
}
func TestARefusedFetchReturnsToItsFormAndAStartedOneToTheList(t *testing.T) {
m := &fakeMirror{fetchErr: errors.New("upstream said 404")}
mux := mirrorPage(t, m, true)
body := mirrorPost(t, mux, "/admin/mirror/fetch", url.Values{"csrf": {"tok"}, "source": {"https://h.example/m.gguf"}, "model": {"m"},
"version": {"v"}, "force": {"1"}, "tab": {"fetch"}}).Body.String()
if !strings.Contains(body, "upstream said 404") || !strings.Contains(body, `name="source" required`) {
t.Errorf("a refusal should come back on the form it came from: %s", body)
}
m.fetchErr = nil
body = mirrorPost(t, mux, "/admin/mirror/fetch", url.Values{"csrf": {"tok"}, "source": {"https://h.example/m.gguf"}, "model": {"m"},
"version": {"v"}, "force": {"1"}, "tab": {"fetch"}}).Body.String()
if !strings.Contains(body, `id="held"`) {
t.Errorf("a started download should be shown where it is followed: %s", body)
}
}
func TestASafetensorsSetIsFetchedWithWhatTheEngineReadsBesideIt(t *testing.T) {
// The field: a four-shard AWQ model mirrored as its shards alone, and
// vLLM failing at start for want of config.json.
reg := &fakeRegistry{repo: hf.Repo{
ID: "Qwen/Qwen2.5-32B-Instruct-AWQ", Revision: "abc123def456",
Files: []hf.File{
{Path: "model-00001-of-00002.safetensors", Bytes: 4_000_000_000},
{Path: "model-00002-of-00002.safetensors", Bytes: 3_000_000_000},
{Path: "model.safetensors.index.json", Bytes: 90_000},
{Path: "config.json", Bytes: 900},
{Path: "tokenizer.json", Bytes: 7_000_000},
{Path: "tokenizer_config.json", Bytes: 7_000},
{Path: "README.md", Bytes: 9_000},
{Path: "modeling_custom.py", Bytes: 40_000},
{Path: "onnx/config.json", Bytes: 900},
},
}}
body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{}, reg, true),
"/admin/mirror?repo=Qwen/Qwen2.5-32B-Instruct-AWQ").Body.String()
for _, want := range []string{"config.json", "tokenizer.json", "tokenizer_config.json", "model.safetensors.index.json"} {
if !strings.Contains(body, `name="file" value="`+want+`"`) {
t.Errorf("%s is not fetched with the set", want)
}
}
for _, not := range []string{"README.md", "modeling_custom.py", "onnx/config.json"} {
if strings.Contains(body, `name="file" value="`+not+`"`) {
t.Errorf("%s was swept in with the set", not)
}
}
if !strings.Contains(body, "with 4 files the engine reads beside them") {
t.Errorf("the companions are not said; got %s", body)
}
}