Give the mirror a page, so managing it is not a curl command
CI / test (push) Successful in 1m49s

/admin/mirror, behind the same admin session as the other admin pages. What is
held, what is arriving with its byte count, what failed and why; start a fetch,
retry a failed one, remove a version.

Not mounted at all without a mirror configured. The right page for a deployment
with no mirror is no page, rather than an empty one pointing at nothing.

It reloads itself only while something is downloading. A reload at any other
moment takes with it whatever is half-typed into the fetch form, and there is
nothing to watch when nothing is moving.

A failed row offers Retry, not Remove. A fetch that failed left nothing on disk,
so the remove could only answer "the mirror holds nothing for this" — a button
whose single outcome is a confusing error. Starting the same download again is
what somebody looking at that row wants, and the mirror allows it: only a
download still running is refused.

An unreachable mirror is said outright instead of rendering as an empty table.
The two look nearly identical and mean opposite things: one is a mirror waiting
to be filled, the other is a machine to go and look at.

Dashboard takes NavExtras rather than a second bool. Two adjacent booleans at a
call site get swapped silently and are discovered by somebody clicking the
wrong link.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-27 10:17:12 +02:00
1 parent 744a283cf1
commit cd7eedf09e
17 files changed
+889 -24

No files matched your search

+16 -1
View File
@@ -27,6 +27,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fleetmetrics"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/router"
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
@@ -170,7 +171,7 @@ func newServerCmd() *cobra.Command {
usage = st
}
adminMux.HandleFunc("GET /", web.Dashboard(
api, usage, version.Version, 5*time.Second, true, log))
api, usage, version.Version, 5*time.Second, true, web.NavExtras{}, log))
adminMux.Handle("GET /_static/", web.Static())
// The same page on the client-facing port at /admin, behind a
@@ -179,6 +180,20 @@ func newServerCmd() *cobra.Command {
rt.SetClusterView(api, usage)
}
// The model mirror, when one is configured. Built here and not at
// first use so a mistyped address stops the control plane starting,
// rather than turning into a broken admin page hours later when
// somebody finally opens it.
if cfg.Mirror.API != "" {
mc, err := mirror.NewClient(cfg.Mirror.API, cfg.Mirror.TLS, cfg.Mirror.Timeout)
if err != nil {
return fmt.Errorf("the model mirror: %w", err)
}
rt.SetMirror(mc)
log.Info("model mirror configured",
"api", cfg.Mirror.API, "mtls", cfg.Mirror.TLS.Enabled())
}
apiSrv := &http.Server{
Handler: rt.Handler(),
ReadHeaderTimeout: 10 * time.Second,
+30
View File
@@ -65,6 +65,10 @@ type Server struct {
// trusted certificate, while this one is mutual and private.
AgentTLS tlsconf.Config `koanf:"agent_tls"`
// Mirror points the control plane at a model mirror it can administer.
// Empty means there is none, and /admin/mirror stays unmounted.
Mirror Mirror `koanf:"mirror"`
Router Router `koanf:"router"`
Control Control `koanf:"control"`
Models []catalog.Model `koanf:"models"`
@@ -75,6 +79,32 @@ type Server struct {
Endpoints []Endpoint `koanf:"endpoints"`
}
// Mirror is a model mirror this control plane administers over its API.
//
// The mirror is a separate machine with a large disk, not part of the gateway,
// so what is configured here is an address and how to authenticate to it —
// never a directory. A control plane that could write to the mirror's disk
// directly would be a way to replace a model with another one that every node
// would then fetch without complaint.
type Mirror struct {
// API is the mirror's management address, scheme included, for example
// https://models.internal:8082. Empty means no mirror is configured.
//
// The management port, not the one nodes fetch bytes from. Those are two
// listeners on the mirror precisely so this one can be locked down without
// re-certifying every agent's fetcher.
API string `koanf:"api"`
// TLS authenticates to the mirror's management API, with the same CA the
// agents use. Empty leaves the connection in plain HTTP, which is only
// acceptable on a network where nothing untrusted can reach that port —
// the mirror says as much in its own log when it starts that way.
TLS tlsconf.Config `koanf:"tls"`
// Timeout bounds a management call. It does not bound a download: the
// mirror answers a fetch as soon as it has started one, and the bytes keep
// arriving long after this deadline has passed.
Timeout time.Duration `koanf:"timeout"`
}
// Landing configures the public page at "/".
//
// It exists for one thing the gateway cannot know: who to ask for a key. A page
+9 -1
View File
@@ -153,9 +153,17 @@ func (r *Router) Handler() http.Handler {
// the admin listener — which also carries pprof, and pprof can dump
// the heap that holds every API key in plaintext.
mux.HandleFunc("GET /admin", pages.AdminPage(web.Dashboard(
r.clusterView, r.clusterUsage, version.Version, 5*time.Second, false, r.log)))
r.clusterView, r.clusterUsage, version.Version, 5*time.Second, false,
web.NavExtras{Mirror: r.mirror != nil}, r.log)))
pages.RegisterAdminUsers(mux, r.store)
}
// Independent of the cluster view: the mirror is a separate machine
// with its own disk, and it is worth administering whether or not this
// process also has a cluster to show.
if r.mirror != nil {
pages.RegisterAdminMirror(mux, r.mirror)
}
}
if r.auth == nil {
+11
View File
@@ -97,6 +97,11 @@ type Router struct {
// in a router-only deployment.
clusterView web.Source
clusterUsage web.UsageSource
// mirror is nil unless a model mirror is configured, which leaves
// /admin/mirror unmounted — the right page for a deployment with no mirror
// is no page, not an empty one pointing at nothing.
mirror web.MirrorAdmin
}
// SetClusterView gives the router the data the /admin page renders.
@@ -108,6 +113,12 @@ func (r *Router) SetClusterView(src web.Source, usage web.UsageSource) {
r.clusterView, r.clusterUsage = src, usage
}
// SetMirror gives the router a model mirror to administer.
//
// Set by the server command when one is configured, for the same reason as the
// cluster view: the router is built before the things its admin pages show.
func (r *Router) SetMirror(m web.MirrorAdmin) { r.mirror = m }
// Auth exposes the authenticator so ctl and the batch writer can reach it.
// Nil when authentication is off.
func (r *Router) Auth() *auth.Authenticator { return r.auth }
+3
View File
@@ -52,6 +52,9 @@ type AccountPages struct {
// field in that snippet is real, so the one placeholder left in it was
// copied verbatim and sent as the literal string MODEL.
served func() []string
// hasMirror is set when /admin/mirror is registered, so the other admin
// pages can link to it and only then.
hasMirror bool
}
// ServedModels supplies the names the profile page's example should use.
+228
View File
@@ -0,0 +1,228 @@
package web
import (
"context"
"errors"
"net/http"
"sort"
"strconv"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// MirrorAdmin is the mirror's management API as this page needs it.
//
// An interface rather than the client itself so the page can be tested without
// a mirror, and so a deployment without one leaves it nil and the page
// unmounted — the same shape the cluster dashboard uses.
type MirrorAdmin interface {
List(ctx context.Context) (mirror.ModelsResponse, error)
Fetch(ctx context.Context, req mirror.FetchRequest) (mirror.FetchResponse, error)
Remove(ctx context.Context, model, version string) ([]string, error)
// Addr is the mirror this points at, shown on the page because an operator
// looking at an empty table needs to know which machine is empty.
Addr() string
}
// mirrorRow is one model version as the page shows it.
type mirrorRow struct {
mirror.Status
// URL is what a catalog entry should say. The one string on this page
// somebody will copy, so it is rendered rather than left to be assembled.
URL string
Percent int
// HasPercent is false when the upstream never sent a length. The bar is
// then not drawn at all: a progress bar that has to invent its own total is
// worse than a byte count that only goes up.
HasPercent bool
}
type adminMirrorData struct {
Addr string
Base string
Rows []mirrorRow
Fetching int
Present int
Failed int
// Unreachable separates "the mirror holds nothing" from "the mirror did not
// answer". They render almost identically and mean opposite things: one is
// a mirror waiting to be filled, the other is a machine to go and look at.
Unreachable bool
Message string
Error string
CSRF string
Version string
Now string
// RefreshSec is 0 when nothing is downloading. The page reloads only while
// there is something to watch, because a reload while an operator is typing
// a source URL into the form takes the URL with it.
RefreshSec int
}
// RegisterAdminMirror mounts the model mirror under /admin/mirror.
//
// Fetching and removing, and nothing else. What the mirror holds is operator
// state rather than account state, but the reasoning is the same as
// /admin/users: these two are the work somebody does repeatedly and should not
// need a shell for, while anything that would let a stolen session reach
// further — pointing the control plane at a different mirror, say — stays in
// configuration where changing it means access to the host.
func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, m MirrorAdmin) {
p.hasMirror = true
mux.HandleFunc("GET /admin/mirror", p.adminOnly(
func(w http.ResponseWriter, r *http.Request, _ store.User) {
p.renderMirror(w, r, m, "", "")
}))
mux.HandleFunc("POST /admin/mirror/fetch", p.mirrorAction(m,
func(ctx context.Context, admin store.User, r *http.Request) (string, error) {
req := mirror.FetchRequest{
Source: strings.TrimSpace(r.FormValue("source")),
Model: strings.TrimSpace(r.FormValue("model")),
Version: strings.TrimSpace(r.FormValue("version")),
File: strings.TrimSpace(r.FormValue("file")),
}
if req.Source == "" {
return "", errors.New("a source URL is required")
}
got, err := m.Fetch(ctx, req)
if err != nil {
return "", err
}
// Named as started, not as done. The mirror answered 202 and the
// bytes are still moving; a message saying "added" would invite
// somebody to point the catalog at a partial file.
return "started downloading " + got.Status.Model + "/" + got.Status.Version, nil
}))
mux.HandleFunc("POST /admin/mirror/{model}/{version}/remove", p.mirrorAction(m,
func(ctx context.Context, _ store.User, r *http.Request) (string, error) {
model, ver := r.PathValue("model"), r.PathValue("version")
removed, err := m.Remove(ctx, model, ver)
if err != nil {
return "", err
}
return "removed " + model + "/" + ver +
" (" + plural(len(removed), "file", "files") + ")", nil
}))
}
// mirrorAction wraps a change: admin session, CSRF, audit line, re-render.
//
// The audit line is not optional. Removing a version deletes weights every node
// may be about to fetch, and the only record that it happened on purpose is
// this log entry naming who clicked it.
func (p *AccountPages) mirrorAction(m MirrorAdmin,
do func(ctx context.Context, admin store.User, r *http.Request) (string, error),
) http.HandlerFunc {
return p.adminOnly(func(w http.ResponseWriter, r *http.Request, admin store.User) {
sess, _, _ := p.session(r)
if !p.accounts.CheckCSRF(sess, r.FormValue("csrf")) {
http.Error(w, "bad request", http.StatusBadRequest)
return
}
msg, err := do(r.Context(), admin, r)
if err != nil {
p.log.Warn("a mirror action was refused",
"admin", admin.Email, "mirror", m.Addr(), "err", err)
p.renderMirror(w, r, m, "", err.Error())
return
}
p.log.Info("the mirror was changed",
"admin", admin.Email, "mirror", m.Addr(), "what", msg)
p.renderMirror(w, r, m, msg, "")
})
}
func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request,
m MirrorAdmin, msg, errMsg string,
) {
d := adminMirrorData{
Addr: m.Addr(),
Message: msg,
Error: errMsg,
CSRF: csrfFor(r),
Version: p.version,
Now: time.Now().UTC().Format(time.RFC3339),
}
got, err := m.List(r.Context())
if err != nil {
p.log.Error("could not read the mirror", "mirror", m.Addr(), "err", err)
d.Unreachable = true
if d.Error == "" {
d.Error = err.Error()
}
render(w, "admin_mirror", d)
return
}
d.Base = got.Base
for _, st := range got.Models {
row := mirrorRow{
Status: st,
URL: mirror.Entry{Model: st.Model, Version: st.Version, File: st.File}.
URL(got.Base),
}
if pc, ok := st.Percent(); ok {
row.Percent, row.HasPercent = pc, true
}
switch st.State {
case mirror.Fetching:
d.Fetching++
case mirror.Failed:
d.Failed++
default:
d.Present++
}
d.Rows = append(d.Rows, row)
}
// Downloads first, then failures, then what is simply here. The two states
// somebody opened this page for are at the top; the rest is a list they
// scroll. Within a state the order is fixed so rows do not shuffle between
// reloads while a byte count ticks.
sort.SliceStable(d.Rows, func(i, j int) bool {
a, b := d.Rows[i], d.Rows[j]
if ra, rb := mirrorRank(a.State), mirrorRank(b.State); ra != rb {
return ra < rb
}
if a.Model != b.Model {
return a.Model < b.Model
}
if a.Version != b.Version {
return a.Version < b.Version
}
return a.File < b.File
})
if d.Fetching > 0 {
d.RefreshSec = 5
}
render(w, "admin_mirror", d)
}
func mirrorRank(s mirror.State) int {
switch s {
case mirror.Fetching:
return 0
case mirror.Failed:
return 1
default:
return 2
}
}
func plural(n int, one, many string) string {
if n == 1 {
return "1 " + one
}
return strconv.Itoa(n) + " " + many
}
+327
View File
@@ -0,0 +1,327 @@
package web
import (
"context"
"errors"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/account"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
)
// fakeAccounts resolves one session, for whoever the test says is looking.
type fakeAccounts struct{ user store.User }
func (f fakeAccounts) SignUp(context.Context, string, string, string) error { return nil }
func (f fakeAccounts) SignIn(context.Context, string, string) (string, string, error) {
return "", "", nil
}
func (f fakeAccounts) Resolve(_ context.Context, token string) (store.Session, store.User, error) {
if token != "good" {
return store.Session{}, store.User{}, errors.New("no such session")
}
return store.Session{UserID: f.user.ID}, f.user, nil
}
func (f fakeAccounts) SignOut(context.Context, string) error { return nil }
func (f fakeAccounts) CheckCSRF(_ store.Session, sub string) bool { return sub == "tok" }
func (f fakeAccounts) SignupOpen() bool { return false }
func (f fakeAccounts) OpenSession(context.Context, string) (string, string, error) {
return "", "", nil
}
func (f fakeAccounts) ChangePassword(context.Context, account.PasswordChanger,
store.User, string, string,
) error {
return nil
}
// fakeMirror stands in for the mirror's API, which lives on another machine.
type fakeMirror struct {
models []mirror.Status
listErr error
fetched mirror.FetchRequest
fetchErr error
removed [2]string
removeErr error
}
func (f *fakeMirror) Addr() string { return "http://mirror:8082" }
func (f *fakeMirror) List(context.Context) (mirror.ModelsResponse, error) {
if f.listErr != nil {
return mirror.ModelsResponse{}, f.listErr
}
return mirror.ModelsResponse{Base: "http://mirror:8081", Models: f.models}, nil
}
func (f *fakeMirror) Fetch(_ context.Context, r mirror.FetchRequest) (mirror.FetchResponse, error) {
f.fetched = r
if f.fetchErr != nil {
return mirror.FetchResponse{}, f.fetchErr
}
return mirror.FetchResponse{
Status: mirror.Status{Model: r.Model, Version: r.Version, State: mirror.Fetching},
}, nil
}
func (f *fakeMirror) Remove(_ context.Context, model, version string) ([]string, error) {
f.removed = [2]string{model, version}
if f.removeErr != nil {
return nil, f.removeErr
}
return []string{"model.gguf"}, nil
}
// mirrorPage mounts the page for an admin, or for somebody who is not one.
func mirrorPage(t *testing.T, m MirrorAdmin, admin bool) *http.ServeMux {
t.Helper()
a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: admin}}
p := NewAccountPages(a, nil, nil, "v0",
slog.New(slog.NewTextHandler(io.Discard, nil)))
mux := http.NewServeMux()
p.RegisterAdminMirror(mux, m)
return mux
}
func mirrorGet(t *testing.T, mux *http.ServeMux, path string) *httptest.ResponseRecorder {
t.Helper()
r := httptest.NewRequest("GET", path, nil)
r.AddCookie(&http.Cookie{Name: SessionCookie, Value: "good"})
r.AddCookie(&http.Cookie{Name: csrfCookie, Value: "tok"})
w := httptest.NewRecorder()
mux.ServeHTTP(w, r)
return w
}
func mirrorPost(t *testing.T, mux *http.ServeMux, path string, form url.Values) *httptest.ResponseRecorder {
t.Helper()
r := httptest.NewRequest("POST", path, strings.NewReader(form.Encode()))
r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
r.AddCookie(&http.Cookie{Name: SessionCookie, Value: "good"})
r.AddCookie(&http.Cookie{Name: csrfCookie, Value: "tok"})
w := httptest.NewRecorder()
mux.ServeHTTP(w, r)
return w
}
func TestTheMirrorPageShowsTheAddressACatalogShouldUse(t *testing.T) {
m := &fakeMirror{models: []mirror.Status{{
Model: "qwen", Version: "2026-09", File: "model.gguf",
State: mirror.Present, BytesDone: 491400032, BytesTotal: 491400032,
SHA256: "74a4da8c9fdbcd15bd1f6d01d621410d31c6fc00986f5eb687824e7b93d7a9db",
}}}
body := mirrorGet(t, mirrorPage(t, m, true), "/admin/mirror").Body.String()
// The one string on the page somebody will copy. Rendered from the base the
// mirror reported, not assembled here — two spellings of one address is
// what makes a file that is in use look unreferenced.
if !strings.Contains(body, "http://mirror:8081/qwen/2026-09/model.gguf") {
t.Error("the page should render the URL a catalog entry needs")
}
if !strings.Contains(body, "468.6 MiB") {
t.Errorf("the size should be readable; body was %s", body)
}
}
func TestAnUnreachableMirrorIsNotAnEmptyOne(t *testing.T) {
m := &fakeMirror{listErr: errors.New("reaching the mirror at http://mirror:8082: refused")}
w := mirrorGet(t, mirrorPage(t, m, true), "/admin/mirror")
// The two render almost identically and mean opposite things: one is a
// mirror waiting to be filled, the other is a machine to go and look at.
body := w.Body.String()
if !strings.Contains(body, "not currently reachable") {
t.Errorf("the page should say the mirror did not answer; got %s", body)
}
if strings.Contains(body, "holds nothing yet") {
t.Error("an unreachable mirror must not be reported as an empty one")
}
}
func TestSomebodyWhoIsNotAnAdminIsNotToldThePageExists(t *testing.T) {
w := mirrorGet(t, mirrorPage(t, &fakeMirror{}, false), "/admin/mirror")
// 404 rather than 403: a 403 confirms the page is there, which is free
// reconnaissance for anybody probing.
if w.Code != http.StatusNotFound {
t.Errorf("expected 404 for a non-admin, got %d", w.Code)
}
}
func TestFetchingWithoutTheCSRFTokenIsRefused(t *testing.T) {
m := &fakeMirror{}
w := mirrorPost(t, mirrorPage(t, m, true), "/admin/mirror/fetch", url.Values{
"source": {"http://x/model.gguf"}, "model": {"m"}, "version": {"v"},
})
if w.Code != http.StatusBadRequest {
t.Errorf("expected 400 without a CSRF token, got %d", w.Code)
}
if m.fetched.Source != "" {
t.Error("nothing should have reached the mirror")
}
}
func TestFetchSaysStartedRatherThanAdded(t *testing.T) {
m := &fakeMirror{}
body := mirrorPost(t, mirrorPage(t, m, true), "/admin/mirror/fetch", url.Values{
"csrf": {"tok"},
"source": {" http://x/model.gguf "}, "model": {"qwen"}, "version": {"2026-09"},
}).Body.String()
// Trimmed, because a URL pasted from a terminal carries whitespace and the
// mirror would take it as part of the address.
if m.fetched.Source != "http://x/model.gguf" {
t.Errorf("the source should be trimmed; got %q", m.fetched.Source)
}
// "started", not "added": the mirror answered 202 and the bytes are still
// moving. A message saying it was added invites pointing the catalog at a
// partial file.
if !strings.Contains(body, "started downloading qwen/2026-09") {
t.Errorf("the message should say the download began; got %s", body)
}
}
func TestTheMirrorsRefusalReachesTheOperator(t *testing.T) {
m := &fakeMirror{removeErr: &mirror.APIError{
Status: http.StatusConflict, Msg: "qwen/2026-09 is still downloading; nothing was removed",
}}
body := mirrorPost(t, mirrorPage(t, m, true), "/admin/mirror/qwen/2026-09/remove",
url.Values{"csrf": {"tok"}}).Body.String()
// The mirror's own sentence, not "the mirror said 409". The reason is in
// the body of its answer precisely so it can be shown here.
if !strings.Contains(body, "still downloading") {
t.Errorf("the mirror's reason should be on the page; got %s", body)
}
}
func TestRemovePassesTheVersionThrough(t *testing.T) {
m := &fakeMirror{}
mirrorPost(t, mirrorPage(t, m, true), "/admin/mirror/qwen/2026-09/remove",
url.Values{"csrf": {"tok"}})
if m.removed != [2]string{"qwen", "2026-09"} {
t.Errorf("the version to remove did not arrive intact: %v", m.removed)
}
}
func TestThePageReloadsOnlyWhileSomethingIsDownloading(t *testing.T) {
idle := mirrorGet(t, mirrorPage(t, &fakeMirror{models: []mirror.Status{
{Model: "m", Version: "v", File: "f", State: mirror.Present},
}}, true), "/admin/mirror").Body.String()
// A reload while nothing is happening would take with it whatever the
// operator was typing into the fetch form.
if strings.Contains(idle, "http-equiv=\"refresh\"") {
t.Error("an idle page should not reload itself")
}
busy := mirrorGet(t, mirrorPage(t, &fakeMirror{models: []mirror.Status{
{Model: "m", Version: "v", File: "f", State: mirror.Fetching},
}}, true), "/admin/mirror").Body.String()
if !strings.Contains(busy, "http-equiv=\"refresh\"") {
t.Error("a page with a download running should follow it")
}
}
func TestNoPercentageIsInventedWhenTheTotalIsUnknown(t *testing.T) {
body := mirrorGet(t, mirrorPage(t, &fakeMirror{models: []mirror.Status{{
Model: "m", Version: "v", File: "f", State: mirror.Fetching,
BytesDone: 1048576, BytesTotal: 0,
}}}, true), "/admin/mirror").Body.String()
// A bar that guesses its own end and then stops moving is worse than a
// number that only goes up.
if strings.Contains(body, "%)") {
t.Errorf("no percentage should appear without a total; got %s", body)
}
if !strings.Contains(body, "1.0 MiB so far") {
t.Errorf("the byte count should still be shown; got %s", body)
}
}
func TestDownloadsAndFailuresComeFirst(t *testing.T) {
body := mirrorGet(t, mirrorPage(t, &fakeMirror{models: []mirror.Status{
{Model: "aaa", Version: "v", File: "f", State: mirror.Present},
{Model: "zzz", Version: "v", File: "f", State: mirror.Fetching},
{Model: "mmm", Version: "v", File: "f", State: mirror.Failed, Err: "upstream 404"},
}}, true), "/admin/mirror").Body.String()
// The two states somebody opened this page for are the two at the top,
// whatever they are called alphabetically.
fetching := strings.Index(body, "zzz")
failed := strings.Index(body, "mmm")
present := strings.Index(body, "aaa")
if !(fetching < failed && failed < present) {
t.Errorf("expected fetching, then failed, then present; got %d %d %d",
fetching, failed, present)
}
if !strings.Contains(body, "upstream 404") {
t.Error("a failed fetch should show the reason it failed")
}
}
func TestThePageRendersThroughTheRealClient(t *testing.T) {
// The three pieces together: the page, the HTTP client, and the mirror's
// own API behind a real listener. Each is tested on its own elsewhere, and
// each can be right on its own while the three disagree about a field name
// or a status code — which is a defect that only shows up in a browser.
dir := t.TempDir()
if err := os.MkdirAll(filepath.Join(dir, "qwen", "2026-09"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(
filepath.Join(dir, "qwen", "2026-09", "model.gguf"), []byte("weights"), 0o600); err != nil {
t.Fatal(err)
}
log := slog.New(slog.NewTextHandler(io.Discard, nil))
srv := httptest.NewServer(
mirror.NewAPI(mirror.NewManager(dir, log), "http://mirror:8081").Handler())
defer srv.Close()
c, err := mirror.NewClient(srv.URL, tlsconf.Config{}, 0)
if err != nil {
t.Fatal(err)
}
body := mirrorGet(t, mirrorPage(t, c, true), "/admin/mirror").Body.String()
if !strings.Contains(body, "http://mirror:8081/qwen/2026-09/model.gguf") {
t.Errorf("the catalog URL should survive all three hops; got %s", body)
}
if !strings.Contains(body, srv.URL) {
t.Error("the page should name the mirror it is reading")
}
}
func TestAFailedRowOffersRetryRatherThanRemove(t *testing.T) {
body := mirrorGet(t, mirrorPage(t, &fakeMirror{models: []mirror.Status{{
Model: "m", Version: "v", File: "model.gguf", State: mirror.Failed,
Source: "http://upstream/model.gguf", Err: "upstream answered 404",
}}}, true), "/admin/mirror").Body.String()
// A fetch that failed left nothing on disk, so a remove would come back
// "the mirror holds nothing for m/v" — a button whose only outcome is a
// confusing error.
if strings.Contains(body, "/admin/mirror/m/v/remove") {
t.Error("a failed row should not offer a remove that can only 404")
}
if !strings.Contains(body, "Retry") {
t.Error("a failed row should offer to start the download again")
}
if !strings.Contains(body, `value="http://upstream/model.gguf"`) {
t.Error("retry has to carry the source it should fetch again")
}
}
+6 -3
View File
@@ -28,6 +28,8 @@ type adminUsersData struct {
CSRF string
Version string
Now string
// HasMirror keeps the nav from offering a page that was never mounted.
HasMirror bool
}
// RegisterAdminUsers mounts account administration under /admin/users.
@@ -139,9 +141,10 @@ func (p *AccountPages) renderUsers(w http.ResponseWriter, r *http.Request,
) {
d := adminUsersData{
Message: msg, Error: errMsg,
CSRF: csrfFor(r),
Version: p.version,
Now: time.Now().UTC().Format(time.RFC3339),
CSRF: csrfFor(r),
Version: p.version,
HasMirror: p.hasMirror,
Now: time.Now().UTC().Format(time.RFC3339),
}
all, err := s.Users(r.Context(), "")
+21 -4
View File
@@ -58,9 +58,15 @@ type DashboardData struct {
// the other into a 404 — or, on the public port, into "no api key was
// provided", which is a far more confusing way to say the same thing.
OnAdminListener bool
RefreshSec int
Version string
Now string
// Nav says which optional admin pages are mounted. Asked rather than
// assumed: a link to a page that was never registered lands on whatever the
// catch-all route serves, which on the client-facing port is the landing
// page — an admin clicking "mirror" and getting the marketing copy has no
// way to tell that from a bug.
Nav NavExtras
RefreshSec int
Version string
Now string
}
// Source is what the dashboard reads. An interface so the page can be rendered
@@ -104,8 +110,18 @@ type UsageSource interface {
// Read-only on purpose. Anything that mints a credential stays in `llmgw keys`,
// where it needs a shell on the machine rather than a browser session this
// project has no way to authenticate.
// NavExtras says which optional admin pages exist.
//
// A struct rather than another bool parameter: Dashboard already takes one, and
// two adjacent booleans at a call site are swapped silently and discovered by
// somebody clicking the wrong link.
type NavExtras struct {
// Mirror is true when /admin/mirror is mounted.
Mirror bool
}
func Dashboard(src Source, usage UsageSource, version string, refresh time.Duration,
onAdminListener bool, log *slog.Logger,
onAdminListener bool, nav NavExtras, log *slog.Logger,
) http.HandlerFunc {
if log == nil {
log = slog.Default()
@@ -116,6 +132,7 @@ func Dashboard(src Source, usage UsageSource, version string, refresh time.Durat
return func(w http.ResponseWriter, r *http.Request) {
d := DashboardData{
OnAdminListener: onAdminListener,
Nav: nav,
RefreshSec: int(refresh.Seconds()),
Version: version,
Now: time.Now().UTC().Format(time.RFC3339),
+15
View File
@@ -262,3 +262,18 @@ pre .copy:hover { color: var(--fg); filter: none; }
.endpoints > li span { color: var(--muted); font-size: .85rem; }
/* A button revealed by hover is a button a phone never has. */
@media (hover: none) { pre .copy { opacity: 1; } }
/* The mirror page. A model version spills onto a second line under its row:
the catalog URL is long, and a column wide enough for it would squeeze every
other column on the table into uselessness. */
.url-row code, .error-row { font-size: .78rem; padding-top: 0; }
.url-row { color: var(--muted); }
.url-row code { color: var(--muted); }
.error-row { color: var(--bad); }
tr:has(+ .url-row) td, tr:has(+ .error-row) td { border-bottom: none; }
/* The fetch form's three short fields sit on one line on a wide screen and
stack on a narrow one, because two of them are a word long and a full-width
input for "2026-09" reads as though more is wanted. */
.field-row { display: flex; gap: 1rem; flex-wrap: wrap; }
.field-row label { flex: 1 1 12rem; }
.opt { font-weight: 400; color: var(--muted); font-size: .8rem; }
+111
View File
@@ -0,0 +1,111 @@
{{define "admin_mirror"}}<!doctype html>
<html lang="en"><head>
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
{{/* Only while something is downloading. A reload at any other time would
take with it whatever the operator was typing into the form below. */}}
{{if .RefreshSec}}<meta http-equiv="refresh" content="{{.RefreshSec}}">{{end}}
<title>Mirror · llmgw</title><link rel="stylesheet" href="/_static/base.css">
</head><body><div class="wrap">
<header>
<h1>Model mirror</h1>
<p class="sub">
{{if .Unreachable}}{{.Addr}} did not answer{{else}}{{.Present}} held ·
{{.Fetching}} downloading · {{.Failed}} failed · {{.Addr}}{{end}}
</p>
<nav><a href="/admin">cluster</a> <a href="/admin/users">accounts</a>
<a href="/profile">your keys</a></nav>
</header>
{{if .Error}}<p class="error">{{.Error}}</p>{{end}}
{{if .Message}}<p class="ok">{{.Message}}</p>{{end}}
{{if .Unreachable}}
{{/* Said outright rather than shown as an empty table. A mirror holding
nothing and a mirror that is not there look the same in a list of zero
rows, and they call for opposite responses. */}}
<p class="note">Nothing was read from the mirror, so this page is not a list of
what it holds — it is not currently reachable at <code>{{.Addr}}</code>. Check
that <code>llmgw-mirror</code> is running there and that its management port is
reachable from this machine.</p>
{{else}}
<h2>What the mirror holds</h2>
{{if .Rows}}
<table>
<tr><th>Model</th><th>Version</th><th>File</th><th>State</th><th>Size</th>
<th>Checksum</th><th></th></tr>
{{range .Rows}}<tr>
<td>{{.Model}}</td>
<td>{{.Version}}</td>
<td><code>{{.File}}</code></td>
<td class="state state-{{.State}}">{{.State}}</td>
<td class="num">
{{if eq (printf "%s" .State) "fetching"}}
{{if .HasPercent}}{{bytes .BytesDone}} / {{bytes .BytesTotal}} ({{.Percent}}%)
{{else}}{{/* No total came from upstream, so no percentage is invented —
a bar that guesses its own end and then stops moving is worse
than a number that only goes up. */}}
{{bytes .BytesDone}} so far{{end}}
{{else if eq (printf "%s" .State) "failed"}}—
{{else}}{{bytes .BytesDone}}{{end}}
</td>
<td>{{if .SHA256}}<code title="{{.SHA256}}">{{slice .SHA256 0 12}}</code>{{else}}—{{end}}</td>
<td>
{{if eq (printf "%s" .State) "fetching"}}
<span class="note">downloading</span>
{{else if eq (printf "%s" .State) "failed"}}
{{/* Retry, not remove. A fetch that failed left nothing on disk, so the
remove would come back "the mirror holds nothing for this" — a button
whose only outcome is a confusing error. Starting the same download
again is what somebody looking at a failed row wants, and the mirror
allows it: only a download still running is refused. */}}
<form method="post" action="/admin/mirror/fetch" class="inline">
<input type="hidden" name="csrf" value="{{$.CSRF}}">
<input type="hidden" name="source" value="{{.Source}}">
<input type="hidden" name="model" value="{{.Model}}">
<input type="hidden" name="version" value="{{.Version}}">
<input type="hidden" name="file" value="{{.File}}">
<button type="submit" class="link">Retry</button>
</form>
{{else}}
<form method="post" action="/admin/mirror/{{.Model}}/{{.Version}}/remove"
class="inline" onsubmit="return confirm('Remove {{.Model}}/{{.Version}} and every file in it?')">
<input type="hidden" name="csrf" value="{{$.CSRF}}">
<button type="submit" class="link danger">Remove</button>
</form>
{{end}}
</td>
</tr>
{{if .Err}}<tr><td colspan="7" class="error-row">{{.Err}}</td></tr>{{end}}
{{if not .Err}}<tr><td colspan="7" class="url-row"><code>{{.URL}}</code></td></tr>{{end}}
{{end}}
</table>
<p class="note">The address under each row is what a catalog entry should say.
It is rendered here rather than assembled by hand because the mirror and the
catalog have to spell the same file the same way — two spellings is what makes a
file that is in use look unreferenced.</p>
{{else}}<p class="empty">The mirror is reachable and holds nothing yet.</p>{{end}}
<h2>Fetch a model</h2>
<form method="post" action="/admin/mirror/fetch">
<input type="hidden" name="csrf" value="{{.CSRF}}">
<label>Source URL
<input type="text" name="source" required
placeholder="https://huggingface.co/…/resolve/main/model.gguf"></label>
<div class="field-row">
<label>Model <input type="text" name="model" required placeholder="qwen2.5-coder-7b"></label>
<label>Version <input type="text" name="version" required placeholder="2026-09"></label>
<label>File <span class="opt">optional</span>
<input type="text" name="file" placeholder="taken from the URL"></label>
</div>
<button type="submit">Start download</button>
</form>
<p class="note">The mirror answers as soon as the download is under way, not when
it finishes — a model is tens of gigabytes and nothing here holds a connection
open for that. The row above will show it arriving. A version is never
overwritten: fetching into one that exists is refused, so a new build is a new
version and the old weights stay where they are.</p>
{{end}}
<footer><span>llmgw {{.Version}}</span><span>{{.Now}}</span></footer>
</div></body></html>{{end}}
+3 -1
View File
@@ -6,7 +6,9 @@
<header>
<h1>Accounts</h1>
<p class="sub">{{len .Pending}} waiting for approval · {{len .Active}} active</p>
<nav><a href="/admin">cluster</a> <a href="/profile">your keys</a></nav>
<nav><a href="/admin">cluster</a>
{{if .HasMirror}} <a href="/admin/mirror">mirror</a>
{{end}} <a href="/profile">your keys</a></nav>
</header>
{{if .Error}}<p class="error">{{.Error}}</p>{{end}}
+2
View File
@@ -14,6 +14,8 @@
<a href="/api/v1/nodes">api</a>
{{else}}
<a href="/admin/users">accounts</a>
{{if .Nav.Mirror}} <a href="/admin/mirror">mirror</a>
{{end}}
<a href="/profile">your keys</a>
<a href="/chat">chat</a>
{{end}}
+16
View File
@@ -78,6 +78,22 @@ var funcs = template.FuncMap{
"units": func(micros int64) string {
return fmt.Sprintf("%.2f", float64(micros)/1e6)
},
// Model weights are read in gigabytes, so that is what the mirror page
// shows. Binary units because that is what `ls -lh` and every disk-full
// message on the host will say, and a page that disagrees with the shell
// about the size of the same file starts an argument about neither.
"bytes": func(n int64) string {
const unit = 1024
if n < unit {
return fmt.Sprintf("%d B", n)
}
div, exp := int64(unit), 0
for v := n / unit; v >= unit; v /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
},
"date": func(t time.Time) string {
if t.IsZero() {
return "—"
+14 -14
View File
@@ -187,7 +187,7 @@ func (f fakeUsage) UsageSince(context.Context, time.Time) ([]store.KeyUsage, err
}
func TestDashboardShowsTheCluster(t *testing.T) {
h := Dashboard(fakeSource{}, fakeUsage{}, "v1", time.Second, true, nil)
h := Dashboard(fakeSource{}, fakeUsage{}, "v1", time.Second, true, NavExtras{}, nil)
code, body := get(t, h, "http://127.0.0.1:9090/", nil)
if code != http.StatusOK {
t.Fatalf("status %d", code)
@@ -205,7 +205,7 @@ func TestDashboardShowsTheCluster(t *testing.T) {
func TestDashboardExplainsOnlyWhatIsUnplaced(t *testing.T) {
// Asking why a satisfied model was not placed gives an empty answer, and a
// page full of empty answers hides the one that matters.
h := Dashboard(fakeSource{}, nil, "v1", time.Second, true, nil)
h := Dashboard(fakeSource{}, nil, "v1", time.Second, true, NavExtras{}, nil)
_, body := get(t, h, "http://127.0.0.1:9090/", nil)
if !strings.Contains(body, "needs 8") {
@@ -219,7 +219,7 @@ func TestDashboardExplainsOnlyWhatIsUnplaced(t *testing.T) {
func TestDashboardSurvivesAFailingUsageQuery(t *testing.T) {
// A reporting query failing must not take the page with it: the cluster
// state is what someone is most likely here for.
h := Dashboard(fakeSource{}, fakeUsage{err: context.DeadlineExceeded}, "v1", time.Second, true, nil)
h := Dashboard(fakeSource{}, fakeUsage{err: context.DeadlineExceeded}, "v1", time.Second, true, NavExtras{}, nil)
code, body := get(t, h, "http://127.0.0.1:9090/", nil)
if code != http.StatusOK {
t.Fatalf("status %d", code)
@@ -230,7 +230,7 @@ func TestDashboardSurvivesAFailingUsageQuery(t *testing.T) {
}
func TestDashboardSaysWhenThereIsNothingToAccount(t *testing.T) {
h := Dashboard(fakeSource{}, nil, "v1", time.Second, true, nil)
h := Dashboard(fakeSource{}, nil, "v1", time.Second, true, NavExtras{}, nil)
_, body := get(t, h, "http://127.0.0.1:9090/", nil)
if !strings.Contains(body, "Authentication is off") {
t.Error("the page does not explain the empty usage section")
@@ -240,7 +240,7 @@ func TestDashboardSaysWhenThereIsNothingToAccount(t *testing.T) {
func TestDashboardRendersAnEmptyCluster(t *testing.T) {
// The state a fresh install is in, and the one most likely to be looked at
// with a template that has never seen it.
h := Dashboard(emptySource{}, nil, "v1", time.Second, true, nil)
h := Dashboard(emptySource{}, nil, "v1", time.Second, true, NavExtras{}, nil)
code, body := get(t, h, "http://127.0.0.1:9090/", nil)
if code != http.StatusOK {
t.Fatalf("status %d", code)
@@ -381,7 +381,7 @@ func TestTheDashboardDrawsTrafficWhenThereIsSome(t *testing.T) {
{Start: now.Add(-time.Hour)},
{Start: now, Requests: 95, Prompt: 9000, Completion: 2100, LatencyMs: 310},
}}
h := Dashboard(fakeSource{}, usage, "v1", time.Second, true, nil)
h := Dashboard(fakeSource{}, usage, "v1", time.Second, true, NavExtras{}, nil)
_, body := get(t, h, "http://127.0.0.1:9090/", nil)
if !strings.Contains(body, "<svg") {
@@ -398,7 +398,7 @@ func TestTheDashboardDrawsTrafficWhenThereIsSome(t *testing.T) {
}
func TestTheDashboardSaysWhenThereIsNoTrafficYet(t *testing.T) {
h := Dashboard(fakeSource{}, fakeUsage{}, "v1", time.Second, true, nil)
h := Dashboard(fakeSource{}, fakeUsage{}, "v1", time.Second, true, NavExtras{}, nil)
_, body := get(t, h, "http://127.0.0.1:9090/", nil)
if !strings.Contains(body, "No traffic recorded yet") {
t.Error("an empty history renders nothing at all")
@@ -414,7 +414,7 @@ func TestTheDashboardOffersOnlyLinksThatExistWhereItIsServed(t *testing.T) {
// through to the API-key middleware, so a visitor clicking "metrics" was
// told "no api key was provided", which is a far more confusing way to say
// the link does not belong there.
_, admin := get(t, Dashboard(fakeSource{}, nil, "v1", time.Second, true, nil),
_, admin := get(t, Dashboard(fakeSource{}, nil, "v1", time.Second, true, NavExtras{}, nil),
"http://127.0.0.1:9090/", nil)
for _, want := range []string{`href="/metrics"`, `href="/debug/pprof/"`, `href="/api/v1/nodes"`} {
if !strings.Contains(admin, want) {
@@ -425,7 +425,7 @@ func TestTheDashboardOffersOnlyLinksThatExistWhereItIsServed(t *testing.T) {
t.Error("the admin listener's page links to a page only the public port serves")
}
_, public := get(t, Dashboard(fakeSource{}, nil, "v1", time.Second, false, nil),
_, public := get(t, Dashboard(fakeSource{}, nil, "v1", time.Second, false, NavExtras{}, nil),
"https://llmgw.example/admin", nil)
for _, unwanted := range []string{`href="/metrics"`, `href="/debug/pprof/"`, `href="/api/v1/nodes"`} {
if strings.Contains(public, unwanted) {
@@ -447,7 +447,7 @@ func TestTheDashboardShowsCapacityWithoutPrometheus(t *testing.T) {
// is should not require standing up a monitoring stack beside it. The
// tables say what exists; this says how much of it is in use, which is the
// question asked first.
h := Dashboard(fakeSource{}, fakeUsage{}, "v1", time.Second, true, nil)
h := Dashboard(fakeSource{}, fakeUsage{}, "v1", time.Second, true, NavExtras{}, nil)
code, page := get(t, h, "http://127.0.0.1:9090/", nil)
if code != http.StatusOK {
t.Fatalf("status %d", code)
@@ -466,7 +466,7 @@ func TestTheDashboardShowsCapacityWithoutPrometheus(t *testing.T) {
func TestAFaultyCardIsShownAsNeitherUsedNorFree(t *testing.T) {
// Counting a faulty card as free is how a cluster looks like it has room it
// does not, and the page must not disagree with the scheduler about that.
h := Dashboard(fakeSource{}, fakeUsage{}, "v1", time.Second, true, nil)
h := Dashboard(fakeSource{}, fakeUsage{}, "v1", time.Second, true, NavExtras{}, nil)
code, page := get(t, h, "http://127.0.0.1:9090/", nil)
if code != http.StatusOK {
t.Fatalf("status %d", code)
@@ -499,7 +499,7 @@ func TestTheDashboardShowsWhoIsUsingTheCluster(t *testing.T) {
// Per key is the right unit for revoking something and the wrong one for
// answering who is using this cluster: one person holds a laptop key, a CI
// key and a notebook key, and adding three rows up by eye gets skipped.
h := Dashboard(fakeSource{}, fakeUsage{}, "v1", time.Second, true, nil)
h := Dashboard(fakeSource{}, fakeUsage{}, "v1", time.Second, true, NavExtras{}, nil)
code, page := get(t, h, "http://127.0.0.1:9090/", nil)
if code != http.StatusOK {
t.Fatalf("status %d", code)
@@ -517,7 +517,7 @@ func TestTheDashboardShowsWhoIsUsingTheCluster(t *testing.T) {
}
func TestTheDashboardShowsWhatIsActuallyRunning(t *testing.T) {
h := Dashboard(fakeSource{}, fakeUsage{}, "v1", time.Second, true, nil)
h := Dashboard(fakeSource{}, fakeUsage{}, "v1", time.Second, true, NavExtras{}, nil)
_, page := get(t, h, "http://127.0.0.1:9090/", nil)
for _, want := range []string{"What is being run", "qwen-coder", "240 ms"} {
if !strings.Contains(page, want) {
@@ -529,7 +529,7 @@ func TestTheDashboardShowsWhatIsActuallyRunning(t *testing.T) {
func TestAReportingFailureDoesNotTakeThePageWithIt(t *testing.T) {
// The cluster state is what someone is most likely here for, and a usage
// query is the part most likely to be slow or broken.
h := Dashboard(fakeSource{}, brokenUsage{}, "v1", time.Second, true, nil)
h := Dashboard(fakeSource{}, brokenUsage{}, "v1", time.Second, true, NavExtras{}, nil)
code, page := get(t, h, "http://127.0.0.1:9090/", nil)
if code != http.StatusOK {
t.Fatalf("status %d; a reporting failure took the page down", code)