llmgw/internal/web/admin_mirror.go

package web

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"html/template"
	"net/http"
	"slices"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
	"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
	"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
	"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
	"gitea.mynodes.xyz/ilker/llmgw/internal/safedial"
	"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)

// MirrorAdmin is the mirror's management API as this page needs it.
//
// An interface rather than the client itself so the page can be tested without
// a mirror, and so a deployment without one leaves it nil and the page
// unmounted — the same shape the cluster dashboard uses.
type MirrorAdmin interface {
	List(ctx context.Context) (mirror.ModelsResponse, error)
	Fetch(ctx context.Context, req mirror.FetchRequest) (mirror.FetchResponse, error)
	Remove(ctx context.Context, model, version string) ([]string, error)
	// Addr is the mirror this points at, shown on the page because an operator
	// looking at an empty table needs to know which machine is empty.
	Addr() string
}

// Registry is the model registry the explorer browses. Nil leaves the explorer
// off, which is right for a deployment with no way out to the internet: a
// search box that can only time out is worse than no search box.
type Registry interface {
	Explore(ctx context.Context, q hf.Query) ([]hf.Model, error)
	Repo(ctx context.Context, id string) (hf.Repo, error)
}

// repoSet is one thing worth fetching: a file, or every shard of one.
type repoSet struct {
	// Kind is what the file says it is, read from its first bytes: "adapter"
	// for a LoRA adapter, "model", or empty when it was not read. Arch is the
	// architecture it names, which an adapter must share with its model.
	Kind, Arch string
	// Verdict is whether the nodes' llama.cpp can load it, from the
	// architecture the registry reports; a set that cannot be served is
	// shown with the reason and no fetch button.
	Verdict gguf.Verdict
	hf.FileSet
	// Sources are every file in the set, in the order the registry listed them.
	// A shard set goes as one action — a fifteen-shard model is not fifteen
	// decisions, it is one.
	Sources []repoSource
	// Companions are the files fetched with a safetensors set that are not
	// weights: config.json, the tokenizer, the shard index.
	Companions []string
	// Model and Version are what the fetch form starts with. Editable, because
	// the name a catalog entry uses is the operator's to choose and the
	// registry's idea of it is only a good guess.
	Model   string
	Version string
}

type repoSource struct{ URL, Path string }

// modelUse is a catalog model that runs a mirrored version, and how.
type modelUse struct {
	Name     string
	Mode     string // always, ondemand or stopped
	Replicas int
	// Min and Max are its scaling as the catalog holds it.
	Min, Max int
	// Class is what it set, empty for its default; EffClass what that comes
	// to. Owner is org:ID or empty for shared. MemoryGB and CPUCores are what
	// a replica of its first profile reserves.
	Class, EffClass, Owner string
	MemoryGB, CPUCores     int
	// Adapters are the LoRA adapters it serves.
	Adapters []catalog.Adapter
	// Ready, Wanted and Nodes are how it runs now, for the row itself:
	// replicas ready of those it should have, and the nodes they are on.
	// Fleet is false where nothing could be asked, so the row says nothing
	// rather than "0 of 0".
	Fleet         bool
	Ready, Wanted int
	Starting      int
	Nodes         []nodeName
	Parked        bool
	Unplaceable   bool
	// Declared models are read from server.yaml on every start, so a mode
	// changed here would revert; the page says where to change it instead.
	Declared bool
}

// mirrorRow is one model version as the page shows it.
type mirrorRow struct {
	mirror.Status
	// URL is what a catalog entry should say. The one string on this page
	// somebody will copy, so it is rendered rather than left to be assembled.
	URL     string
	Percent int
	// Serve is the offer to put this version into service, open on the row the
	// operator clicked. Nil on a row that is still arriving: a catalog entry
	// pointing at a partial file is the thing versioned paths exist to prevent.
	Serve *serveForm
	// Detail is the card opened beneath the row: what runs it, on which
	// nodes, and how many replicas it runs.
	Detail *heldDetail
	// Kind is what a single GGUF file is — "adapter" or "model" — and Arch
	// its architecture; empty when not read.
	Kind, Arch string
	// Open is the row's ?open= value.
	Open string

	// HasPercent is false when the upstream never sent a length. The bar is
	// then not drawn at all: a progress bar that has to invent its own total is
	// worse than a byte count that only goes up.
	HasPercent bool

	// Files is how many files a held version has. A version of several is one
	// row — a fifteen-shard model is one thing to serve or remove, and fifteen
	// rows of it pushed everything else off the page.
	Files int
	// UsedBy is the catalog models whose source is this row's address, and how
	// each runs, so a file in use does not look like one waiting to be served
	// and how it runs can be changed where it is seen.
	UsedBy []modelUse
}

type adminMirrorData struct {
	Nav Nav
	// Tab is the part of the page shown: held, explore or fetch. Tabs are the
	// links to each.
	Tab  string
	Tabs []mirrorTab
	Addr string
	Base string
	Rows []mirrorRow
	// Groups are the same rows under their model.
	Groups []mirrorGroup

	Fetching int
	Present  int
	Failed   int

	// Unreachable separates "the mirror holds nothing" from "the mirror did not
	// answer". They render almost identically and mean opposite things: one is
	// a mirror waiting to be filled, the other is a machine to go and look at.
	Unreachable bool

	// HasCatalog is false where the catalogue cannot be changed at runtime, and
	// the serve action is then not offered rather than offered and refused.
	HasCatalog bool

	// The explorer.
	HasRegistry bool
	Query       string
	Explore     *explorer
	// Back is the listing an open repository was reached from.
	Back template.URL
	// Inline is a repository rendered beneath its row in the listing, without
	// the page around it.
	Inline bool
	// Card is the open repository's model card, rendered.
	Card template.HTML
	// RepoURL is the open repository on the registry's own site.
	RepoURL     string
	Repo        *hf.Repo
	RepoSets    []repoSet
	RegistryErr string

	Message string
	Error   string
	CSRF    string
	Version string
	Now     string
	// RefreshSec is 0 when nothing is downloading. The page reloads only while
	// there is something to watch, because a reload while an operator is typing
	// a source URL into the form takes the URL with it.
	RefreshSec int
}

// RegisterAdminMirror mounts the model mirror under /admin/mirror.
//
// Fetching and removing, and nothing else. What the mirror holds is operator
// state rather than account state, but the reasoning is the same as
// /admin/users: these two are the work somebody does repeatedly and should not
// need a shell for, while anything that would let a stolen session reach
// further — pointing the control plane at a different mirror, say — stays in
// configuration where changing it means access to the host.
// MirrorDeps is what the mirror page can reach.
//
// A struct rather than five parameters: three of them are optional interfaces
// and two are the same shape, which at a call site is a swap nobody notices
// until a page shows the wrong thing.
type MirrorDeps struct {
	Mirror MirrorAdmin
	// Registry browses a model registry. Nil turns the explorer off.
	Registry Registry
	// Catalog puts a mirrored model into service. Nil leaves the serve action
	// off, which is right where the catalogue cannot be changed at runtime.
	Catalog CatalogAdmin
	// Fleet is the cluster, for saying whether anything could run a model
	// before somebody spends an hour downloading it.
	Fleet Source
}

func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) {
	m := d.Mirror
	p.hasMirror = true
	p.registry = d.Registry
	p.catalog = d.Catalog
	p.fleet = d.Fleet

	mux.HandleFunc("GET /admin/mirror", p.adminOnly(
		func(w http.ResponseWriter, r *http.Request, _ store.User) {
			p.renderMirror(w, r, m, "", "")
		}))

	// One repository as a fragment, for the row beneath it in the listing.
	mux.HandleFunc("GET /admin/mirror/repo", p.adminOnly(
		func(w http.ResponseWriter, r *http.Request, _ store.User) {
			repo := strings.TrimSpace(r.URL.Query().Get("repo"))
			if p.registry == nil || repo == "" {
				http.NotFound(w, r)
				return
			}
			d := adminMirrorData{CSRF: csrfFor(r), Inline: true, HasRegistry: true}
			p.openRepo(r, &d, repo)
			if d.RegistryErr != "" {
				// Read by the page's own script, which shows it beneath the
				// row, so it is in the page's language.
				http.Error(w, langOf(r).T("explore.registry_err", d.RegistryErr), http.StatusBadGateway)
				return
			}
			render(w, r, "mirror_repo", d)
		}))

	mux.HandleFunc("POST /admin/mirror/fetch", p.mirrorAction(m,
		func(ctx context.Context, admin store.User, r *http.Request) (string, error) {
			// One form posts one source or fifteen. A sharded model is one
			// decision, not one per shard, and the mirror fetches each file
			// independently anyway.
			l := langOf(r)
			sources, files := r.Form["source"], r.Form["file"]
			if len(sources) == 0 || strings.TrimSpace(sources[0]) == "" {
				return "", errors.New(l.T("mirror.err.source_required"))
			}
			model := strings.TrimSpace(r.FormValue("model"))
			version := strings.TrimSpace(r.FormValue("version"))

			// Read before anything is downloaded: the header says whether the
			// nodes can load it, and a file that cannot be served is an hour
			// of download and a replica failing on every node.
			if r.FormValue("force") == "" {
				if v := p.checkSource(ctx, strings.TrimSpace(sources[0])); !v.Servable {
					return "", errors.New(l.T("mirror.err.not_fetched", v.Reason))
				}
			}

			var started int
			for i, src := range sources {
				src = strings.TrimSpace(src)
				if src == "" {
					continue
				}
				req := mirror.FetchRequest{Source: src, Model: model, Version: version}
				if i < len(files) {
					req.File = strings.TrimSpace(files[i])
				}
				if _, err := m.Fetch(ctx, req); err != nil {
					// Reported with what already went, so a set that fails on
					// its fourth shard does not read as though nothing started.
					return "", errors.New(l.T("mirror.err.partly_started",
						started, len(sources), mirror.LastPathElement(src), l.Err(err)))
				}
				started++
			}
			// Named as started, not as done. The mirror answered 202 and the
			// bytes are still moving; a message saying "added" would invite
			// somebody to point the catalog at a partial file.
			return l.T("mirror.msg.started", model, version, l.N("mirror.files", int64(started))), nil
		}))

	if d.Catalog != nil {
		// How a model runs, changed where the file it runs is seen.
		mux.HandleFunc("POST /admin/mirror/mode", p.mirrorAction(m,
			func(ctx context.Context, admin store.User, r *http.Request) (string, error) {
				l := langOf(r)
				name, mode := r.FormValue("model"), r.FormValue("mode")
				if name == "" || (mode != modeAlways && mode != modeOnDemand && mode != modeStopped) {
					return "", errors.New(l.T("mirror.err.choose_mode"))
				}
				replicas, _ := strconv.Atoi(r.FormValue("replicas"))
				sc := scalingFor(mode, replicas, 0)
				if _, err := d.Catalog.Scale(ctx, name, sc, admin.Email); err != nil {
					return "", err
				}
				switch mode {
				case modeOnDemand:
					return l.T("mirror.msg.ondemand", name), nil
				case modeStopped:
					return l.T("mirror.msg.stopped", name), nil
				}
				return l.T("mirror.msg.always", name), nil
			}))

		mux.HandleFunc("POST /admin/mirror/replicas", p.mirrorAction(m, p.setModel))
		mux.HandleFunc("POST /admin/mirror/adapters", p.mirrorAction(m, p.setAdapters))

		mux.HandleFunc("POST /admin/mirror/serve", p.mirrorAction(m,
			func(ctx context.Context, admin store.User, r *http.Request) (string, error) {
				l := langOf(r)
				model, err := buildModel(l, r.FormValue)
				if err != nil {
					return "", err
				}
				// A judge's weights are served as a judge, under the name the
				// policies ask: served as a model clients use, the policies
				// never find it and refuse every request they judge.
				judge, renamed := p.asJudge(&model)
				// The CPU while GPUs sit free is a choice to make on purpose:
				// made by default, it was a model running at a tenth of the
				// speed with nothing saying why. A judge is meant for the CPU.
				if !judge && model.Profiles[0].CPUOnly() && r.FormValue("cpu_ok") == "" {
					if n := freeGPUs(p.fleet); n > 0 {
						return "", errors.New(l.N("serve.err.cpu_with_free_gpus", int64(n), model.Name))
					}
				}
				// The same check `llmgw ctl model add` makes. This page builds
				// the source out of a row the mirror reported, so it is right
				// far more often than something typed — but a file removed
				// between the listing and the click, or a mirror that has gone
				// down since, both end as a replica failing on a node minutes
				// later with an error naming the node. Asking costs one HEAD.
				size, err := sourcecheck.Check(ctx, model.Source)
				if err != nil {
					return "", err
				}
				// The weights and the context's cache have to fit the card:
				// an 8B asked for a 131072-token pool on 24 GB loaded its
				// weights and failed to make its context, again and again.
				if err := p.fitsCard(ctx, l, model, size); err != nil {
					return "", err
				}
				if err := d.Catalog.Add(ctx, model, admin.Email); err != nil {
					return "", err
				}
				// Placed on the next pass, not now: saying it is serving would
				// be wrong for however long the weights take to reach a node.
				msg := l.T("serve.msg.added", model.Name,
					l.N("serve.replicas_n", int64(model.Scaling.Min)), describeServe(l, model))
				switch {
				case renamed != "":
					msg += " " + l.T("serve.msg.judge_renamed", renamed, model.Name)
				case judge:
					msg += " " + l.T("serve.msg.judge", model.Name)
				}
				return msg, nil
			}))
	}

	mux.HandleFunc("POST /admin/mirror/{model}/{version}/remove", p.mirrorAction(m,
		func(ctx context.Context, _ store.User, r *http.Request) (string, error) {
			model, ver := r.PathValue("model"), r.PathValue("version")
			// Weights a catalog model still loads are not taken from under
			// it: removed from the mirror alone, the model stayed in the
			// catalog and its replicas went on loading the copy on the node.
			if users := p.mirrorUsers(ctx, model, ver); len(users) > 0 {
				return "", fmt.Errorf("%s/%s is the weights of %s in the catalog: remove %s first "+
					"(the models page, or llmgw ctl model remove %s), then the weights",
					model, ver, strings.Join(users, ", "), strings.Join(users, ", "), users[0])
			}
			removed, err := m.Remove(ctx, model, ver)
			if err != nil {
				return "", err
			}
			l := langOf(r)
			return l.T("mirror.msg.removed", model, ver, l.N("mirror.files", int64(len(removed)))), nil
		}))
}

// mirrorAction wraps a change: admin session, CSRF, audit line, re-render.
//
// The audit line is not optional. Removing a version deletes weights every node
// may be about to fetch, and the only record that it happened on purpose is
// this log entry naming who clicked it.
func (p *AccountPages) mirrorAction(m MirrorAdmin,
	do func(ctx context.Context, admin store.User, r *http.Request) (string, error),
) http.HandlerFunc {
	return p.adminOnly(func(w http.ResponseWriter, r *http.Request, admin store.User) {
		sess, _, _ := p.session(r)
		if !p.accounts.CheckCSRF(sess, r.FormValue("csrf")) {
			badForm(w, r)
			return
		}
		msg, err := do(r.Context(), admin, r)
		// Asked for from the page's own script — a fetch started from a
		// repository opened in the list — the answer is the outcome alone,
		// so the list, and the repository open in it, stay where they are.
		wantJSON := strings.Contains(r.Header.Get("Accept"), "application/json")
		if err != nil {
			p.log.Warn("a mirror action was refused",
				"admin", admin.Email, "mirror", m.Addr(), "err", err)
			if wantJSON {
				writeJSONStatus(w, http.StatusUnprocessableEntity, map[string]string{"error": langOf(r).Err(err)})
				return
			}
			p.renderMirror(w, r, m, "", langOf(r).Err(err))
			return
		}
		p.log.Info("the mirror was changed",
			"admin", admin.Email, "mirror", m.Addr(), "what", msg)
		if wantJSON {
			writeJSONStatus(w, http.StatusOK, map[string]string{"message": msg})
			return
		}
		p.renderMirror(w, r, m, msg, "")
	})
}

func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request,
	m MirrorAdmin, msg, errMsg string,
) {
	l := langOf(r)
	d := adminMirrorData{
		Nav:     navFrom(r, "mirror"),
		Addr:    m.Addr(),
		Message: msg,
		Error:   errMsg,
		CSRF:    csrfFor(r),
		Version: p.version,
		Now:     time.Now().UTC().Format(time.RFC3339),
	}

	d.HasCatalog = p.catalog != nil
	d.HasRegistry = p.registry != nil
	d.Tab = mirrorTabFor(r, errMsg)
	// Only the tab shown does its work: the registry is a server across the
	// internet, and asking it for sixty repositories to show somebody who came
	// to look at a download's progress made every visit wait on it.
	if d.Tab == tabExplore {
		p.explore(r, &d)
	}

	got, err := m.List(r.Context())
	if err != nil {
		p.log.Error("could not read the mirror", "mirror", m.Addr(), "err", err)
		d.Unreachable = true
		if d.Error == "" {
			d.Error = l.Err(err)
		}
		d.Tabs = mirrorTabs(l, d)
		render(w, r, "admin_mirror", d)
		return
	}

	d.Base = got.Base
	var sources map[string][]catalog.Model
	if p.catalog != nil {
		if sources, err = p.catalog.Sources(r.Context()); err != nil {
			p.log.Warn("could not read which models use the mirror", "err", err)
		}
	}
	held := map[string]int{} // model/version -> index of its row
	for _, st := range got.Models {
		switch st.State {
		case mirror.Fetching:
			d.Fetching++
		case mirror.Failed:
			d.Failed++
		default:
			key := st.Model + "/" + st.Version
			if i, ok := held[key]; ok {
				// Another file of a version already shown: counted into it.
				d.Rows[i].Files++
				d.Rows[i].BytesDone += st.BytesDone
				d.Rows[i].URL = strings.TrimSuffix(
					mirror.Entry{Model: st.Model, Version: st.Version}.URL(got.Base), "/") + "/"
				continue
			}
			held[key] = len(d.Rows)
			d.Present++ // versions, not files: what the rows below count
		}
		row := mirrorRow{
			Status: st,
			Files:  1,
			URL: mirror.Entry{Model: st.Model, Version: st.Version, File: st.File}.
				URL(got.Base),
		}
		if pc, ok := st.Percent(); ok {
			row.Percent, row.HasPercent = pc, true
		}
		d.Rows = append(d.Rows, row)
	}
	var views map[string]adminapi.ModelView
	prefix := ""
	if p.fleet != nil && len(sources) > 0 {
		views = map[string]adminapi.ModelView{}
		for _, v := range p.fleet.Models() {
			views[v.Name] = v
		}
		var ids []string
		for _, n := range p.fleet.Nodes() {
			ids = append(ids, n.ID)
		}
		prefix = sharedPrefix(ids)
	}
	for i := range d.Rows {
		for _, m := range sources[d.Rows[i].URL] {
			u := modelUse{
				Name: m.Name, Mode: modeOf(m.Scaling), Replicas: max(m.Scaling.Max, 1),
				Min: m.Scaling.Min, Max: m.Scaling.Max,
				Class: m.Class, EffClass: m.EffectiveClass(), Owner: m.Owner,
				Adapters: m.Adapters,
				Declared: p.catalog.Declared(m.Name),
			}
			if len(m.Profiles) > 0 {
				u.MemoryGB, u.CPUCores = m.Profiles[0].MinMemoryGB, m.Profiles[0].CPUCores
			}
			if v, ok := views[m.Name]; ok {
				u.Fleet, u.Ready, u.Wanted = true, v.Ready, v.Desired
				u.Parked, u.Unplaceable = v.Parked, v.Unplaceable
				var nodes []string
				nodes, u.Starting = runningOn(p.fleet.Replicas(m.Name))
				for _, n := range nodes {
					u.Nodes = append(u.Nodes, nodeName{Full: n, Short: strings.TrimPrefix(n, prefix)})
				}
			}
			d.Rows[i].UsedBy = append(d.Rows[i].UsedBy, u)
		}
	}

	p.heldKinds(r.Context(), d.Rows)

	// The one row whose card is open. From the form too, so a change made in
	// the card comes back with the card still open to show it.
	open := r.FormValue("open")
	for i := range d.Rows {
		d.Rows[i].Open = openKey(d.Rows[i].Model, d.Rows[i].Version)
		if open != "" && d.Rows[i].Open == open && d.Rows[i].State == mirror.Present {
			d.Rows[i].Detail = p.heldDetailFor(r.Context(), l, d.Rows[i])
			d.Rows[i].Detail.addAdapterChoices(d.Rows[i], d.Rows)
		}
	}

	// The one row whose serve form is open, and only when the catalog can be
	// changed at all.
	if p.catalog != nil {
		if want := r.URL.Query().Get("serve"); want != "" {
			byVersion := map[string][]mirror.Status{}
			for _, st := range got.Models {
				byVersion[st.Model+"/"+st.Version] = append(byVersion[st.Model+"/"+st.Version], st)
			}
			for i := range d.Rows {
				key := d.Rows[i].Model + ":" + d.Rows[i].Version
				if key != want || d.Rows[i].State != mirror.Present {
					continue
				}
				form := planServe(l, d.Rows[i].Status,
					byVersion[d.Rows[i].Model+"/"+d.Rows[i].Version], got.Base, p.fleet, p.catalog, func(src string) gguf.Verdict { return p.checkSource(r.Context(), src) })
				d.Rows[i].Serve = &form
				break
			}
		}
	}

	// Downloads first, then failures, then what is simply here. The two states
	// somebody opened this page for are at the top; the rest is a list they
	// scroll. Within a state the order is fixed so rows do not shuffle between
	// reloads while a byte count ticks.
	sort.SliceStable(d.Rows, func(i, j int) bool {
		a, b := d.Rows[i], d.Rows[j]
		if ra, rb := mirrorRank(a.State), mirrorRank(b.State); ra != rb {
			return ra < rb
		}
		if a.Model != b.Model {
			return a.Model < b.Model
		}
		if a.Version != b.Version {
			return a.Version < b.Version
		}
		return a.File < b.File
	})

	d.Groups = groupByModel(d.Rows)

	// Only on the list: a reload anywhere else takes with it a search, an
	// open repository or a URL half typed.
	if d.Fetching > 0 && d.Tab == tabHeld && open == "" {
		d.RefreshSec = 5
	}
	d.Tabs = mirrorTabs(l, d)
	render(w, r, "admin_mirror", d)
}

const (
	tabHeld    = "held"
	tabExplore = "explore"
	tabFetch   = "fetch"
)

// mirrorTab is one of the page's tabs.
type mirrorTab struct {
	Label  string
	Count  string // beside the label, empty for none
	Href   string
	Active bool
}

// mirrorTabFor is the tab a request is for.
//
// Named in the address when it is chosen, and otherwise read from what the
// address carries, so links made before the page had tabs — a search, an
// open repository — still land where they meant. After a change the list is
// shown, where its effect is, unless it was refused: then the form it came
// from, to try again.
func mirrorTabFor(r *http.Request, errMsg string) string {
	valid := func(t string) bool { return t == tabHeld || t == tabExplore || t == tabFetch }
	if r.Method == http.MethodPost {
		if t := r.PostFormValue("tab"); errMsg != "" && valid(t) {
			return t
		}
		return tabHeld
	}
	q := r.URL.Query()
	if t := q.Get("tab"); valid(t) {
		return t
	}
	for _, k := range []string{"q", "repo", "view", "task", "size", "x"} {
		if q.Has(k) {
			return tabExplore
		}
	}
	return tabHeld
}

func mirrorTabs(l *Lang, d adminMirrorData) []mirrorTab {
	held := mirrorTab{Label: l.T("mirror.tab.held"), Href: "/admin/mirror?tab=held", Active: d.Tab == tabHeld}
	switch {
	case d.Unreachable:
		held.Count = l.T("mirror.tab.unreachable")
	case d.Fetching > 0:
		held.Count = l.T("mirror.tab.downloading", l.Int(int64(d.Present)), l.Int(int64(d.Fetching)))
	default:
		held.Count = l.Int(int64(d.Present))
	}
	if d.Failed > 0 {
		held.Count += " · " + l.T("mirror.tab.failed", l.Int(int64(d.Failed)))
	}
	tabs := []mirrorTab{held}
	if d.HasRegistry {
		tabs = append(tabs, mirrorTab{Label: l.T("mirror.tab.explore"),
			Href: "/admin/mirror?tab=explore", Active: d.Tab == tabExplore})
	}
	return append(tabs, mirrorTab{Label: l.T("mirror.tab.fetch"),
		Href: "/admin/mirror?tab=fetch", Active: d.Tab == tabFetch})
}

// mirrorGroup is a family of models the mirror holds — every qwen, every
// llama — and their versions.
type mirrorGroup struct {
	Family string
	Rows   []mirrorRow
	// Models is how many different models are in it; a family of one is
	// shown without a heading, which would only repeat its row.
	Models int
	// Bytes is what the versions take on disk, downloads so far included.
	Bytes            int64
	Fetching, Failed int
	InUse            bool
}

// modelFamily is the family a model's name puts it in: the letters its name
// starts with. "qwen2.5-coder-7b-instruct" and "qwen3-8b" are qwen,
// "llama-3.1-8b-instruct" is llama, "deepseek-r1-distill-qwen-14b" is
// deepseek — which is how people talk about them, and all a name says.
func modelFamily(model string) string {
	name := strings.ToLower(model)
	end := strings.IndexFunc(name, func(r rune) bool { return r < 'a' || r > 'z' })
	if end <= 0 {
		// Starts with a digit or a symbol: the first part whole.
		first, _, _ := strings.Cut(name, "-")
		return first
	}
	return name[:end]
}

// groupByModel gathers rows by family, in the order the rows are already in:
// a family with a download or a failure comes first because its first row
// does, and its models and versions stay in that order beneath it.
//
// Grouped because a family is what somebody looks for. Sorted by state and
// name alone, "which qwens do we have" meant reading the whole table.
func groupByModel(rows []mirrorRow) []mirrorGroup {
	var out []mirrorGroup
	at := map[string]int{}
	models := map[string]map[string]bool{}
	for _, r := range rows {
		fam := modelFamily(r.Model)
		i, ok := at[fam]
		if !ok {
			i = len(out)
			at[fam] = i
			out = append(out, mirrorGroup{Family: fam})
			models[fam] = map[string]bool{}
		}
		g := &out[i]
		g.Rows = append(g.Rows, r)
		g.Bytes += r.BytesDone
		models[fam][r.Model] = true
		switch r.State {
		case mirror.Fetching:
			g.Fetching++
		case mirror.Failed:
			g.Failed++
		}
		if len(r.UsedBy) > 0 {
			g.InUse = true
		}
	}
	for i := range out {
		out[i].Models = len(models[out[i].Family])
	}
	return out
}

func mirrorRank(s mirror.State) int {
	switch s {
	case mirror.Fetching:
		return 0
	case mirror.Failed:
		return 1
	default:
		return 2
	}
}

func plural(n int, one, many string) string {
	if n == 1 {
		return "1 " + one
	}
	return strconv.Itoa(n) + " " + many
}

// explore fills in the registry half of the page: a listing, or one repository.
//
// A failure here is reported in its own line rather than as the page's error.
// The registry is somebody else's server on the other side of the internet, and
// it being down says nothing about the mirror — which is the thing this page is
// actually for, and which must keep working when Hugging Face does not.
func (p *AccountPages) explore(r *http.Request, d *adminMirrorData) {
	if p.registry == nil {
		return
	}
	d.HasRegistry = true
	d.Query = strings.TrimSpace(r.URL.Query().Get("q"))

	if repo := strings.TrimSpace(r.URL.Query().Get("repo")); repo != "" {
		d.Back = exploreState(r.URL.Query()).href(d.Query, "", "")
		p.openRepo(r, d, repo)
		return
	}

	p.exploreList(r, d)
}

// openRepo reads one repository: its card, and its weights grouped into what
// can be fetched.
func (p *AccountPages) openRepo(r *http.Request, d *adminMirrorData, repo string) {
	got, err := p.registry.Repo(r.Context(), repo)
	if err != nil {
		d.RegistryErr = langOf(r).Err(err)
		return
	}
	d.Repo = &got
	d.RepoURL = strings.TrimSuffix(got.DownloadURL("", ""), "/resolve/"+revOrMain(got.Revision)+"/")
	d.Card = renderCard(got.Card, d.RepoURL+"/blob/"+revOrMain(got.Revision)+"/")
	var weights []hf.File
	for _, f := range got.Files {
		if !f.IsWeights() {
			// Thirty files of which four are worth gigabytes. The rest are
			// a README and a tokenizer, and listing them buries the ones
			// somebody came for.
			continue
		}
		weights = append(weights, f)
	}
	for _, set := range hf.Group(weights) {
		rs := repoSet{
			FileSet: set,
			Model:   defaultModelName(got.ID),
			// The revision, not the date: two fetches of the same commit
			// land in the same place, and a repository re-uploaded under
			// the same branch lands somewhere else.
			Version: got.ShortRevision(),
		}
		rs.Verdict = p.verdictFor(langOf(r), got.Architecture, set.Name)
		for _, f := range set.Files {
			rs.Sources = append(rs.Sources,
				repoSource{URL: got.DownloadURL("", f.Path), Path: f.Path})
		}
		// Safetensors are not a model on their own: vLLM loads a directory,
		// and reads its config and tokenizer before any weight. Fetched as
		// part of the set, or the set arrives as shards nothing can load.
		if strings.HasSuffix(strings.ToLower(set.Name), ".safetensors") {
			for _, f := range hf.Companions(got.Files) {
				rs.Sources = append(rs.Sources,
					repoSource{URL: got.DownloadURL("", f.Path), Path: f.Path})
				rs.Companions = append(rs.Companions, f.Path)
				rs.Bytes += f.Bytes
			}
		}
		d.RepoSets = append(d.RepoSets, rs)
	}
	p.probeKinds(r.Context(), d.RepoSets)
}

// kindOf reads what a GGUF file is; a variable so tests need no registry.
var kindOf = func(ctx context.Context, url string) (gguf.Header, error) {
	return gguf.KindFromURL(ctx, safedial.Client(safedial.ControlPlane, 30*time.Second), url)
}

// probeKinds reads the start of each single-file GGUF set, to tell LoRA
// adapters from models: a repository tagged "lora" holds a model with the
// adapter merged in as often as an adapter, and only the file says which.
// A few at once and a few in all, since each is a request to the registry.
func (p *AccountPages) probeKinds(ctx context.Context, sets []repoSet) {
	const most, together = 12, 6
	sem := make(chan struct{}, together)
	var wg sync.WaitGroup
	n := 0
	for i := range sets {
		s := &sets[i]
		if len(s.Sources) != 1 || !gguf.IsGGUF(s.Sources[0].Path) || n == most {
			continue
		}
		n++
		wg.Add(1)
		go func() {
			defer wg.Done()
			sem <- struct{}{}
			defer func() { <-sem }()
			cctx, cancel := context.WithTimeout(ctx, 10*time.Second)
			defer cancel()
			if h, err := kindOf(cctx, s.Sources[0].URL); err == nil {
				s.Arch = h.Architecture
				s.Kind = "model"
				if h.IsAdapter() {
					s.Kind = "adapter"
				}
			}
		}()
	}
	wg.Wait()
}

// defaultModelName is the catalog name a repository suggests.
//
// The last element, lowercased, with the quantisation marker dropped: the owner
// is not part of what anybody calls the model, and "-GGUF" describes the file
// format rather than the model, which is already clear from the file being
// fetched.
func defaultModelName(repoID string) string { return hf.ModelName(repoID) }

// verdictFor judges a registry file set by the repository's architecture. A
// vision projector is its own architecture whatever the repository says.
func (p *AccountPages) verdictFor(l *Lang, arch, name string) gguf.Verdict {
	// Not refused here — the mirror keeps what it is given — but a node
	// refuses to fetch it for a model without allow_pickle.
	if weights.Pickled(name) {
		return gguf.Verdict{Checked: true, Reason: l.T("card.pickle")}
	}
	if strings.Contains(strings.ToLower(name), "mmproj") {
		return gguf.LlamaCpp(p.llamaVersion, "clip")
	}
	// Published beside the quantisations they were made with, and a GGUF
	// file like them, so the repository's architecture would pass it.
	if strings.Contains(strings.ToLower(name), "imatrix") {
		return gguf.Verdict{Checked: true,
			Reason: l.T("card.imatrix")}
	}
	if arch == "" || !gguf.IsGGUF(name) {
		return gguf.Verdict{Servable: true}
	}
	return gguf.LlamaCpp(p.llamaVersion, arch)
}

// checkSource reads a GGUF source's header and judges it; anything else, or a
// header that cannot be read, passes.
func (p *AccountPages) checkSource(ctx context.Context, source string) gguf.Verdict {
	url, ok := sourcecheck.HTTPSource(source)
	if !ok || !gguf.IsGGUF(url) || p.llamaVersion == "" {
		return gguf.Verdict{Servable: true}
	}
	return gguf.Check(ctx, url, p.llamaVersion)
}

func revOrMain(rev string) string {
	if rev == "" {
		return "main"
	}
	return rev
}

func writeJSONStatus(w http.ResponseWriter, code int, v any) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(code)
	_ = json.NewEncoder(w).Encode(v)
}

// runningOn is the nodes a model's replicas are on, each once and in order,
// and how many of its replicas are not ready yet.
func runningOn(reps []adminapi.ReplicaView) (nodes []string, starting int) {
	for _, r := range reps {
		if r.State != "ready" {
			starting++
		}
		for _, sh := range r.Shards {
			if !slices.Contains(nodes, sh.Node) {
				nodes = append(nodes, sh.Node)
			}
		}
	}
	slices.Sort(nodes)
	return nodes, starting
}

// nodeName is a node as a row names it: short, with the full name beside.
type nodeName struct{ Short, Full string }

// sharedPrefix is what every node's name starts with, up to and including a
// separator — "llmgw-" of llmgw-node01 and llmgw-node02 — so a row can say
// node01 and fit on a line. Nothing when the names share no such part, or
// when there is one node and dropping it would leave only a number.
func sharedPrefix(ids []string) string {
	if len(ids) < 2 {
		return ""
	}
	p := ids[0]
	for _, id := range ids[1:] {
		for !strings.HasPrefix(id, p) {
			p = p[:len(p)-1]
		}
	}
	if i := strings.LastIndexAny(p, "-._"); i >= 0 {
		return p[:i+1]
	}
	return ""
}

// mirrorUsers are the catalog models whose weights are the mirror's
// model/version.
func (p *AccountPages) mirrorUsers(ctx context.Context, model, ver string) []string {
	if p.catalog == nil {
		return nil
	}
	srcs, err := p.catalog.Sources(ctx)
	if err != nil {
		return nil
	}
	part := "/" + model + "/" + ver + "/"
	var out []string
	for src, ms := range srcs {
		if !strings.Contains(src, part) {
			continue
		}
		for _, m := range ms {
			if !slices.Contains(out, m.Name) {
				out = append(out, m.Name)
			}
		}
	}
	slices.Sort(out)
	return out
}

// fitsCard refuses a model whose weights and context pool do not fit the
// cards its profile names, saying the largest pool that would.
func (p *AccountPages) fitsCard(ctx context.Context, l *Lang, m catalog.Model, size int64) error {
	if len(m.Profiles) == 0 || m.Profiles[0].GPUClass == "" || size <= 0 {
		return nil
	}
	pr := m.Profiles[0]
	v := p.checkSource(ctx, m.Source)
	cards := max(pr.GPUsPerNode, pr.TP, 1)
	most := v.Header.PoolThatFits(size, gguf.ClassMemGB(pr.GPUClass), cards)
	pool := argInt(pr.Args, "--ctx-size")
	if pool <= 0 {
		pool = defaultCtxPool
	}
	switch {
	case most < 0:
		return errors.New(l.T("serve.err.weights_too_big", l.Num(float64(size)/(1<<30), 1), pr.GPUClass))
	case most > 0 && pool > most:
		return errors.New(l.T("serve.err.pool_too_big", l.Int(int64(pool)), pr.GPUClass, l.Int(int64(most/1024*1024))))
	}
	return nil
}