Files
llmgw/internal/web/admin_mirror_test.go
T
Ilker ManapandClaude Opus 5 ed530133b6 Find a model in the registry instead of pasting a URL into a box
Mirroring anything meant knowing a repository existed, knowing which of its
thirty files were the weights, and assembling a resolve URL by hand. The mirror
page had a text box for that URL and no way to find one.

It now searches Hugging Face and lists a repository's weight files with their
sizes and published checksums, each with a button that fills the fetch form.

Only weight files. A repository is mostly a README and a tokenizer, and listing
those buries the four entries somebody came for.

The size shown is the LFS size, not the plain one. For an LFS-stored file —
every weights file — the plain field is the size of the pointer, so the other
one reports a gigabyte model as 133 bytes, and that is the number somebody
would then plan disk against.

The source URL is pinned to the commit the listing came from rather than to a
branch, so what the mirror fetches is what was on screen. A repository
re-uploaded under the same branch otherwise leaves two different sets of
weights with identical provenance and no way to tell them apart. The version
defaults to that revision for the same reason.

The registry's own failures are reported on their own line. It is somebody
else's server on the other side of the internet, and it being down says nothing
about the mirror — which is what the page is for and has to keep working
regardless. With no registry configured there is no search box at all, rather
than one that can only time out.

Gating decodes from a bool or a string, because the field is documented as the
first and is the second in practice: strict decoding fails the whole response,
so one gated model would hide the other nineteen in a page of results.

And a bug this made obvious: downloads were keyed by model and version, so the
second shard of a sharded model was refused as "already fetching" until the
first had finished. A fifteen-shard model had to be pulled one file at a time,
each waiting on the last. Keyed by file now — which is what the single-flight
was ever protecting, since the race is two writers on one .part. A version made
of several files reports as the worst of them: it is not present until every
shard is, and one shard failing makes the version unusable however many others
arrived.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-27 15:18:57 +02:00

438 lines
16 KiB
Go

package web
import (
"context"
"errors"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/account"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
)
// fakeAccounts resolves one session, for whoever the test says is looking.
type fakeAccounts struct{ user store.User }
func (f fakeAccounts) SignUp(context.Context, string, string, string) error { return nil }
func (f fakeAccounts) SignIn(context.Context, string, string) (string, string, error) {
return "", "", nil
}
func (f fakeAccounts) Resolve(_ context.Context, token string) (store.Session, store.User, error) {
if token != "good" {
return store.Session{}, store.User{}, errors.New("no such session")
}
return store.Session{UserID: f.user.ID}, f.user, nil
}
func (f fakeAccounts) SignOut(context.Context, string) error { return nil }
func (f fakeAccounts) CheckCSRF(_ store.Session, sub string) bool { return sub == "tok" }
func (f fakeAccounts) SignupOpen() bool { return false }
func (f fakeAccounts) OpenSession(context.Context, string) (string, string, error) {
return "", "", nil
}
func (f fakeAccounts) ChangePassword(context.Context, account.PasswordChanger,
store.User, string, string,
) error {
return nil
}
// fakeMirror stands in for the mirror's API, which lives on another machine.
type fakeMirror struct {
models []mirror.Status
listErr error
fetched mirror.FetchRequest
fetchErr error
removed [2]string
removeErr error
}
func (f *fakeMirror) Addr() string { return "http://mirror:8082" }
func (f *fakeMirror) List(context.Context) (mirror.ModelsResponse, error) {
if f.listErr != nil {
return mirror.ModelsResponse{}, f.listErr
}
return mirror.ModelsResponse{Base: "http://mirror:8081", Models: f.models}, nil
}
func (f *fakeMirror) Fetch(_ context.Context, r mirror.FetchRequest) (mirror.FetchResponse, error) {
f.fetched = r
if f.fetchErr != nil {
return mirror.FetchResponse{}, f.fetchErr
}
return mirror.FetchResponse{
Status: mirror.Status{Model: r.Model, Version: r.Version, State: mirror.Fetching},
}, nil
}
func (f *fakeMirror) Remove(_ context.Context, model, version string) ([]string, error) {
f.removed = [2]string{model, version}
if f.removeErr != nil {
return nil, f.removeErr
}
return []string{"model.gguf"}, nil
}
// mirrorPage mounts the page for an admin, or for somebody who is not one.
func mirrorPage(t *testing.T, m MirrorAdmin, admin bool) *http.ServeMux {
return mirrorPageWith(t, m, nil, admin)
}
// mirrorPageWith mounts it with a registry behind the explorer.
func mirrorPageWith(t *testing.T, m MirrorAdmin, reg Registry, admin bool) *http.ServeMux {
t.Helper()
a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: admin}}
p := NewAccountPages(a, nil, nil, "v0",
slog.New(slog.NewTextHandler(io.Discard, nil)))
mux := http.NewServeMux()
p.RegisterAdminMirror(mux, m, reg)
return mux
}
func mirrorGet(t *testing.T, mux *http.ServeMux, path string) *httptest.ResponseRecorder {
t.Helper()
r := httptest.NewRequest("GET", path, nil)
r.AddCookie(&http.Cookie{Name: SessionCookie, Value: "good"})
r.AddCookie(&http.Cookie{Name: csrfCookie, Value: "tok"})
w := httptest.NewRecorder()
mux.ServeHTTP(w, r)
return w
}
func mirrorPost(t *testing.T, mux *http.ServeMux, path string, form url.Values) *httptest.ResponseRecorder {
t.Helper()
r := httptest.NewRequest("POST", path, strings.NewReader(form.Encode()))
r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
r.AddCookie(&http.Cookie{Name: SessionCookie, Value: "good"})
r.AddCookie(&http.Cookie{Name: csrfCookie, Value: "tok"})
w := httptest.NewRecorder()
mux.ServeHTTP(w, r)
return w
}
func TestTheMirrorPageShowsTheAddressACatalogShouldUse(t *testing.T) {
m := &fakeMirror{models: []mirror.Status{{
Model: "qwen", Version: "2026-09", File: "model.gguf",
State: mirror.Present, BytesDone: 491400032, BytesTotal: 491400032,
SHA256: "74a4da8c9fdbcd15bd1f6d01d621410d31c6fc00986f5eb687824e7b93d7a9db",
}}}
body := mirrorGet(t, mirrorPage(t, m, true), "/admin/mirror").Body.String()
// The one string on the page somebody will copy. Rendered from the base the
// mirror reported, not assembled here — two spellings of one address is
// what makes a file that is in use look unreferenced.
if !strings.Contains(body, "http://mirror:8081/qwen/2026-09/model.gguf") {
t.Error("the page should render the URL a catalog entry needs")
}
if !strings.Contains(body, "468.6 MiB") {
t.Errorf("the size should be readable; body was %s", body)
}
}
func TestAnUnreachableMirrorIsNotAnEmptyOne(t *testing.T) {
m := &fakeMirror{listErr: errors.New("reaching the mirror at http://mirror:8082: refused")}
w := mirrorGet(t, mirrorPage(t, m, true), "/admin/mirror")
// The two render almost identically and mean opposite things: one is a
// mirror waiting to be filled, the other is a machine to go and look at.
body := w.Body.String()
if !strings.Contains(body, "not currently reachable") {
t.Errorf("the page should say the mirror did not answer; got %s", body)
}
if strings.Contains(body, "holds nothing yet") {
t.Error("an unreachable mirror must not be reported as an empty one")
}
}
func TestSomebodyWhoIsNotAnAdminIsNotToldThePageExists(t *testing.T) {
w := mirrorGet(t, mirrorPage(t, &fakeMirror{}, false), "/admin/mirror")
// 404 rather than 403: a 403 confirms the page is there, which is free
// reconnaissance for anybody probing.
if w.Code != http.StatusNotFound {
t.Errorf("expected 404 for a non-admin, got %d", w.Code)
}
}
func TestFetchingWithoutTheCSRFTokenIsRefused(t *testing.T) {
m := &fakeMirror{}
w := mirrorPost(t, mirrorPage(t, m, true), "/admin/mirror/fetch", url.Values{
"source": {"http://x/model.gguf"}, "model": {"m"}, "version": {"v"},
})
if w.Code != http.StatusBadRequest {
t.Errorf("expected 400 without a CSRF token, got %d", w.Code)
}
if m.fetched.Source != "" {
t.Error("nothing should have reached the mirror")
}
}
func TestFetchSaysStartedRatherThanAdded(t *testing.T) {
m := &fakeMirror{}
body := mirrorPost(t, mirrorPage(t, m, true), "/admin/mirror/fetch", url.Values{
"csrf": {"tok"},
"source": {" http://x/model.gguf "}, "model": {"qwen"}, "version": {"2026-09"},
}).Body.String()
// Trimmed, because a URL pasted from a terminal carries whitespace and the
// mirror would take it as part of the address.
if m.fetched.Source != "http://x/model.gguf" {
t.Errorf("the source should be trimmed; got %q", m.fetched.Source)
}
// "started", not "added": the mirror answered 202 and the bytes are still
// moving. A message saying it was added invites pointing the catalog at a
// partial file.
if !strings.Contains(body, "started downloading qwen/2026-09") {
t.Errorf("the message should say the download began; got %s", body)
}
}
func TestTheMirrorsRefusalReachesTheOperator(t *testing.T) {
m := &fakeMirror{removeErr: &mirror.APIError{
Status: http.StatusConflict, Msg: "qwen/2026-09 is still downloading; nothing was removed",
}}
body := mirrorPost(t, mirrorPage(t, m, true), "/admin/mirror/qwen/2026-09/remove",
url.Values{"csrf": {"tok"}}).Body.String()
// The mirror's own sentence, not "the mirror said 409". The reason is in
// the body of its answer precisely so it can be shown here.
if !strings.Contains(body, "still downloading") {
t.Errorf("the mirror's reason should be on the page; got %s", body)
}
}
func TestRemovePassesTheVersionThrough(t *testing.T) {
m := &fakeMirror{}
mirrorPost(t, mirrorPage(t, m, true), "/admin/mirror/qwen/2026-09/remove",
url.Values{"csrf": {"tok"}})
if m.removed != [2]string{"qwen", "2026-09"} {
t.Errorf("the version to remove did not arrive intact: %v", m.removed)
}
}
func TestThePageReloadsOnlyWhileSomethingIsDownloading(t *testing.T) {
idle := mirrorGet(t, mirrorPage(t, &fakeMirror{models: []mirror.Status{
{Model: "m", Version: "v", File: "f", State: mirror.Present},
}}, true), "/admin/mirror").Body.String()
// A reload while nothing is happening would take with it whatever the
// operator was typing into the fetch form.
if strings.Contains(idle, "http-equiv=\"refresh\"") {
t.Error("an idle page should not reload itself")
}
busy := mirrorGet(t, mirrorPage(t, &fakeMirror{models: []mirror.Status{
{Model: "m", Version: "v", File: "f", State: mirror.Fetching},
}}, true), "/admin/mirror").Body.String()
if !strings.Contains(busy, "http-equiv=\"refresh\"") {
t.Error("a page with a download running should follow it")
}
}
func TestNoPercentageIsInventedWhenTheTotalIsUnknown(t *testing.T) {
body := mirrorGet(t, mirrorPage(t, &fakeMirror{models: []mirror.Status{{
Model: "m", Version: "v", File: "f", State: mirror.Fetching,
BytesDone: 1048576, BytesTotal: 0,
}}}, true), "/admin/mirror").Body.String()
// A bar that guesses its own end and then stops moving is worse than a
// number that only goes up.
if strings.Contains(body, "%)") {
t.Errorf("no percentage should appear without a total; got %s", body)
}
if !strings.Contains(body, "1.0 MiB so far") {
t.Errorf("the byte count should still be shown; got %s", body)
}
}
func TestDownloadsAndFailuresComeFirst(t *testing.T) {
body := mirrorGet(t, mirrorPage(t, &fakeMirror{models: []mirror.Status{
{Model: "aaa", Version: "v", File: "f", State: mirror.Present},
{Model: "zzz", Version: "v", File: "f", State: mirror.Fetching},
{Model: "mmm", Version: "v", File: "f", State: mirror.Failed, Err: "upstream 404"},
}}, true), "/admin/mirror").Body.String()
// The two states somebody opened this page for are the two at the top,
// whatever they are called alphabetically.
fetching := strings.Index(body, "zzz")
failed := strings.Index(body, "mmm")
present := strings.Index(body, "aaa")
if !(fetching < failed && failed < present) {
t.Errorf("expected fetching, then failed, then present; got %d %d %d",
fetching, failed, present)
}
if !strings.Contains(body, "upstream 404") {
t.Error("a failed fetch should show the reason it failed")
}
}
func TestThePageRendersThroughTheRealClient(t *testing.T) {
// The three pieces together: the page, the HTTP client, and the mirror's
// own API behind a real listener. Each is tested on its own elsewhere, and
// each can be right on its own while the three disagree about a field name
// or a status code — which is a defect that only shows up in a browser.
dir := t.TempDir()
if err := os.MkdirAll(filepath.Join(dir, "qwen", "2026-09"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(
filepath.Join(dir, "qwen", "2026-09", "model.gguf"), []byte("weights"), 0o600); err != nil {
t.Fatal(err)
}
log := slog.New(slog.NewTextHandler(io.Discard, nil))
srv := httptest.NewServer(
mirror.NewAPI(mirror.NewManager(dir, log), "http://mirror:8081").Handler())
defer srv.Close()
c, err := mirror.NewClient(srv.URL, tlsconf.Config{}, 0)
if err != nil {
t.Fatal(err)
}
body := mirrorGet(t, mirrorPage(t, c, true), "/admin/mirror").Body.String()
if !strings.Contains(body, "http://mirror:8081/qwen/2026-09/model.gguf") {
t.Errorf("the catalog URL should survive all three hops; got %s", body)
}
if !strings.Contains(body, srv.URL) {
t.Error("the page should name the mirror it is reading")
}
}
func TestAFailedRowOffersRetryRatherThanRemove(t *testing.T) {
body := mirrorGet(t, mirrorPage(t, &fakeMirror{models: []mirror.Status{{
Model: "m", Version: "v", File: "model.gguf", State: mirror.Failed,
Source: "http://upstream/model.gguf", Err: "upstream answered 404",
}}}, true), "/admin/mirror").Body.String()
// A fetch that failed left nothing on disk, so a remove would come back
// "the mirror holds nothing for m/v" — a button whose only outcome is a
// confusing error.
if strings.Contains(body, "/admin/mirror/m/v/remove") {
t.Error("a failed row should not offer a remove that can only 404")
}
if !strings.Contains(body, "Retry") {
t.Error("a failed row should offer to start the download again")
}
if !strings.Contains(body, `value="http://upstream/model.gguf"`) {
t.Error("retry has to carry the source it should fetch again")
}
}
type fakeRegistry struct {
results []hf.Model
repo hf.Repo
err error
asked string
}
func (f *fakeRegistry) Search(_ context.Context, q string, _ int) ([]hf.Model, error) {
f.asked = q
return f.results, f.err
}
func (f *fakeRegistry) Repo(_ context.Context, id string) (hf.Repo, error) {
f.asked = id
return f.repo, f.err
}
func TestSearchingTheRegistryListsRepositories(t *testing.T) {
reg := &fakeRegistry{results: []hf.Model{
{ID: "Qwen/Qwen2.5-Coder-7B-Instruct-GGUF", Downloads: 295447, PipelineTag: "text-generation"},
}}
body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{}, reg, true),
"/admin/mirror?q=qwen+coder").Body.String()
if reg.asked != "qwen coder" {
t.Errorf("the query did not arrive: %q", reg.asked)
}
if !strings.Contains(body, "Qwen2.5-Coder-7B-Instruct-GGUF") {
t.Errorf("the result is missing; got %s", body)
}
}
func TestOnlyWeightFilesAreOffered(t *testing.T) {
reg := &fakeRegistry{repo: hf.Repo{
ID: "Qwen/X", Revision: "abc123def456",
Files: []hf.File{
{Path: "README.md", Bytes: 900},
{Path: "model.gguf", Bytes: 1117320736, SHA256: "6a1a2eb6d15622bf3c96857206351ba97e1af16c30d7a74ee38970e434e9407e"},
},
}}
body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{}, reg, true),
"/admin/mirror?repo=Qwen/X").Body.String()
// A repository is mostly a README and a tokenizer; listing those buries
// the four files somebody came for.
if strings.Contains(body, "README.md") {
t.Error("documentation should not be offered for mirroring")
}
if !strings.Contains(body, "model.gguf") || !strings.Contains(body, "1.0 GiB") {
t.Errorf("the weights and their size should be listed; got %s", body)
}
}
func TestTheFetchButtonPinsTheRevisionAndSuggestsANames(t *testing.T) {
reg := &fakeRegistry{repo: hf.Repo{
ID: "Qwen/Qwen2.5-1.5B-Instruct-GGUF", Revision: "abc123def456789",
Files: []hf.File{{Path: "model.gguf", Bytes: 10}},
}}
body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{}, reg, true),
"/admin/mirror?repo=Qwen/Qwen2.5-1.5B-Instruct-GGUF").Body.String()
// Pinned, so what arrives is what was listed even if the branch moves.
if !strings.Contains(body, "/resolve/abc123def456789/model.gguf") {
t.Errorf("the source should pin the revision; got %s", body)
}
// The owner is not part of what anybody calls the model, and -GGUF
// describes the file rather than the model.
if !strings.Contains(body, `value="qwen2.5-1.5b-instruct"`) {
t.Error("the suggested name should drop the owner and the format marker")
}
if !strings.Contains(body, `value="abc123de"`) {
t.Error("the version should default to the revision that was listed")
}
}
func TestARegistryThatIsDownDoesNotBreakTheMirrorPage(t *testing.T) {
reg := &fakeRegistry{err: errors.New("reaching the model registry: timeout")}
body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{models: []mirror.Status{{
Model: "held", Version: "v", File: "f", State: mirror.Present,
}}}, reg, true), "/admin/mirror?q=qwen").Body.String()
// The registry is somebody else's server. It being down says nothing about
// the mirror, which is what this page is for.
if !strings.Contains(body, "The registry:") {
t.Error("the registry failure should be reported on its own line")
}
if !strings.Contains(body, "held") {
t.Error("what the mirror holds must still be listed")
}
}
func TestWithNoRegistryThereIsNoSearchBox(t *testing.T) {
body := mirrorGet(t, mirrorPage(t, &fakeMirror{}, true), "/admin/mirror").Body.String()
// A deployment with no way out to the internet gets no search box, rather
// than one that can only time out.
if strings.Contains(body, "Search Hugging Face") {
t.Error("the explorer should be absent without a registry")
}
if !strings.Contains(body, "Fetch a model") {
t.Error("fetching by hand must still be there")
}
}