An adapters view in the explorer, filtered by the model a held model was made from; GGUF files told apart as adapter or model by their first bytes; held adapter files marked; and a model's card attaches and detaches held adapters of its architecture, refusing models and mismatched architectures. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
434 lines
14 KiB
Go
434 lines
14 KiB
Go
package web
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"html/template"
|
|
"net/http"
|
|
"net/url"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// HeadroomSource is a fleet that can say how many more replicas of a model
|
|
// fit now. Optional: without it the card shows what runs and offers no change
|
|
// it cannot check.
|
|
type HeadroomSource interface {
|
|
Headroom(model string, limit int) (adminapi.HeadroomView, bool)
|
|
}
|
|
|
|
// maxReplicas bounds what the form accepts. Not a cluster limit, a typing one:
|
|
// 300 where 3 was meant would otherwise be taken as a ceiling.
|
|
const maxReplicas = 64
|
|
|
|
// heldDetail is the card opened beneath a held version: what runs it, where,
|
|
// and room to change it.
|
|
type heldDetail struct {
|
|
Models []heldModel
|
|
// Scopes are the parts of the tenancy tree a model can belong to; empty
|
|
// when capacity quotas are not set up, and the owner choice is left off.
|
|
Scopes []adminapi.ScopeName
|
|
}
|
|
|
|
// heldModel is one catalog model running a held version.
|
|
type heldModel struct {
|
|
modelUse
|
|
// Adapters are the model's own; AdapterChoices the held adapter files
|
|
// of its architecture not attached yet; FindAdapters the explorer's
|
|
// list of adapters made for it.
|
|
Adapters []catalog.Adapter
|
|
AdapterChoices []adapterChoice
|
|
FindAdapters template.URL
|
|
View adminapi.ModelView
|
|
Replicas []heldReplica
|
|
Headroom *adminapi.HeadroomView
|
|
// MostNow is how many replicas could always run, Current plus the room
|
|
// found; zero when the room was not measured or had no end in sight.
|
|
MostNow int
|
|
}
|
|
|
|
// heldReplica is one replica, by the nodes it runs on.
|
|
type heldReplica struct {
|
|
ID string
|
|
State string
|
|
Restart string
|
|
Age int64
|
|
Endpoint string
|
|
Nodes []heldShard
|
|
}
|
|
|
|
type heldShard struct {
|
|
Node string
|
|
MemoryGB int // the node's RAM, 0 when unknown
|
|
CPU float64
|
|
MemBytes int64
|
|
}
|
|
|
|
// openKey names a row the way ?open= does.
|
|
func openKey(model, version string) string { return model + ":" + version }
|
|
|
|
// heldDetailFor reads, for each model using a row, how it runs and how much
|
|
// more room it has.
|
|
func (p *AccountPages) heldDetailFor(ctx context.Context, row mirrorRow) *heldDetail {
|
|
d := &heldDetail{}
|
|
if p.capacity != nil {
|
|
d.Scopes, _ = p.capacity.Scopes(ctx)
|
|
}
|
|
if p.fleet == nil {
|
|
for _, u := range row.UsedBy {
|
|
d.Models = append(d.Models, heldModel{modelUse: u, Adapters: u.Adapters})
|
|
}
|
|
return d
|
|
}
|
|
views := map[string]adminapi.ModelView{}
|
|
for _, v := range p.fleet.Models() {
|
|
views[v.Name] = v
|
|
}
|
|
memory := map[string]int{}
|
|
for _, n := range p.fleet.Nodes() {
|
|
memory[n.ID] = n.MemoryGB
|
|
}
|
|
hs, _ := p.fleet.(HeadroomSource)
|
|
for _, u := range row.UsedBy {
|
|
hm := heldModel{modelUse: u, View: views[u.Name], Adapters: u.Adapters}
|
|
for _, rep := range p.fleet.Replicas(u.Name) {
|
|
// Built field by field: a ReplicaView carries the engine's key,
|
|
// which is for the admin listener and never for a page.
|
|
hr := heldReplica{ID: rep.ID, State: rep.State, Restart: rep.Restart,
|
|
Age: rep.AgeSec, Endpoint: rep.Endpoint}
|
|
for _, sh := range rep.Shards {
|
|
hr.Nodes = append(hr.Nodes, heldShard{Node: sh.Node, MemoryGB: memory[sh.Node],
|
|
CPU: sh.CPUCores, MemBytes: sh.MemoryBytes})
|
|
}
|
|
hm.Replicas = append(hm.Replicas, hr)
|
|
}
|
|
if hs != nil {
|
|
if h, ok := hs.Headroom(u.Name, 16); ok {
|
|
hm.Headroom = &h
|
|
if !h.Limited {
|
|
hm.MostNow = h.Current + h.More + h.ByPreempting
|
|
}
|
|
}
|
|
}
|
|
d.Models = append(d.Models, hm)
|
|
}
|
|
return d
|
|
}
|
|
|
|
// setReplicas changes how many replicas a model always runs and how far it
|
|
// may grow, refusing an always-on count the cluster has no room for.
|
|
//
|
|
// Refused rather than taken and left short: a minimum placement cannot meet
|
|
// is a model reported as unplaceable on every pass from then on, which is the
|
|
// same answer given later and less plainly. A maximum above the room is
|
|
// accepted and said: it is reached only under load, by which time something
|
|
// may have made room, and refusing it would make the ceiling depend on what
|
|
// happened to be running when it was set.
|
|
func (p *AccountPages) setReplicas(ctx context.Context, admin store.User, r *http.Request) (string, error) {
|
|
name := r.FormValue("model")
|
|
lo, err1 := strconv.Atoi(strings.TrimSpace(r.FormValue("min")))
|
|
hi, err2 := strconv.Atoi(strings.TrimSpace(r.FormValue("max")))
|
|
switch {
|
|
case name == "":
|
|
return "", errors.New("which model?")
|
|
case err1 != nil || err2 != nil:
|
|
return "", errors.New("replicas are whole numbers")
|
|
case lo < 0 || hi < lo:
|
|
return "", fmt.Errorf("always running (%d) cannot be more than the most under load (%d)", lo, hi)
|
|
case hi > maxReplicas:
|
|
return "", fmt.Errorf("at most %d replicas", maxReplicas)
|
|
}
|
|
current, err := p.currentModel(ctx, name)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if p.catalog.Declared(name) {
|
|
return "", fmt.Errorf("%s is set in server.yaml; change it there", name)
|
|
}
|
|
|
|
note := ""
|
|
if hs, ok := p.fleet.(HeadroomSource); ok && hi > 0 {
|
|
h, ok := hs.Headroom(name, hi)
|
|
if !ok {
|
|
return "", fmt.Errorf("no model %q", name)
|
|
}
|
|
fits := h.Current + h.More + h.ByPreempting
|
|
if lo > fits && !h.Limited {
|
|
why := "nothing more fits"
|
|
if len(h.Blocked) > 0 {
|
|
why = strings.Join(h.Blocked, "; ")
|
|
}
|
|
return "", fmt.Errorf("%s can always run at most %d now (%d running, room for %d more): %s",
|
|
name, fits, h.Current, h.More, why)
|
|
}
|
|
if hi > fits && !h.Limited {
|
|
note = fmt.Sprintf("; there is room for %d now, so it grows past that only if room is made", fits)
|
|
}
|
|
// Room its class would make by stopping others: said, since it is
|
|
// somebody else's replica that goes.
|
|
if lo > h.Current+h.More && len(h.Preempts) > 0 {
|
|
note += "; to make room, replicas of " + strings.Join(h.Preempts, ", ") + " will be stopped"
|
|
}
|
|
}
|
|
|
|
sc := catalog.Scaling{Min: lo, Max: hi, IdleTimeout: current.Scaling.IdleTimeout}
|
|
if sc.OnDemand() && sc.IdleTimeout <= 0 {
|
|
sc.IdleTimeout = defaultIdle
|
|
}
|
|
if _, err := p.catalog.Scale(ctx, name, sc, admin.Email); err != nil {
|
|
return "", err
|
|
}
|
|
switch {
|
|
case hi == 0:
|
|
return name + " is stopped; its replicas are being taken down", nil
|
|
case lo == 0:
|
|
return fmt.Sprintf("%s loads when asked for, up to %d under load%s", name, hi, note), nil
|
|
case lo == hi:
|
|
return fmt.Sprintf("%s runs %s%s", name, plural(lo, "replica", "replicas"), note), nil
|
|
}
|
|
return fmt.Sprintf("%s runs %d, and up to %d under load%s", name, lo, hi, note), nil
|
|
}
|
|
|
|
// currentModel is a model as the catalog holds it now.
|
|
func (p *AccountPages) currentModel(ctx context.Context, name string) (catalog.Model, error) {
|
|
sources, err := p.catalog.Sources(ctx)
|
|
if err != nil {
|
|
return catalog.Model{}, err
|
|
}
|
|
for _, ms := range sources {
|
|
for _, m := range ms {
|
|
if m.Name == name {
|
|
return m, nil
|
|
}
|
|
}
|
|
}
|
|
return catalog.Model{}, fmt.Errorf("no model %q runs from the mirror", name)
|
|
}
|
|
|
|
// age renders seconds the way the replicas table does.
|
|
func age(sec int64) string {
|
|
d := time.Duration(sec) * time.Second
|
|
switch {
|
|
case d < time.Minute:
|
|
return fmt.Sprintf("%ds", sec)
|
|
case d < time.Hour:
|
|
return fmt.Sprintf("%dm", int(d.Minutes()))
|
|
case d < 48*time.Hour:
|
|
return fmt.Sprintf("%dh%02dm", int(d.Hours()), int(d.Minutes())%60)
|
|
}
|
|
return fmt.Sprintf("%dd", int(d.Hours()/24))
|
|
}
|
|
|
|
// setModel applies the card's settings form: what a replica reserves, the
|
|
// model's class and owner, then how many replicas it runs — in that order, so
|
|
// the room checked for the replicas is the room its new class and reservation
|
|
// would get. Only what changed is applied, and each change is said.
|
|
func (p *AccountPages) setModel(ctx context.Context, admin store.User, r *http.Request) (string, error) {
|
|
name := r.FormValue("model")
|
|
current, err := p.currentModel(ctx, name)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if p.catalog.Declared(name) {
|
|
return "", fmt.Errorf("%s is set in server.yaml; change it there", name)
|
|
}
|
|
var done []string
|
|
if r.Form.Has("memory") && len(current.Profiles) > 0 {
|
|
mem, err1 := strconv.Atoi(strings.TrimSpace(r.FormValue("memory")))
|
|
cores, err2 := strconv.Atoi(strings.TrimSpace(r.FormValue("cores")))
|
|
if err1 != nil || err2 != nil || mem < 0 || cores < 0 {
|
|
return "", errors.New("memory and cores are whole numbers")
|
|
}
|
|
if cur := current.Profiles[0]; mem != cur.MinMemoryGB || cores != cur.CPUCores {
|
|
if _, err := p.catalog.Resources(ctx, name, "", mem, cores, admin.Email); err != nil {
|
|
return "", err
|
|
}
|
|
done = append(done, fmt.Sprintf("a replica reserves %d GB and %d cores (running ones keep theirs until restarted)", mem, cores))
|
|
}
|
|
}
|
|
if r.Form.Has("class") {
|
|
if c := r.FormValue("class"); c != current.Class {
|
|
m, err := p.catalog.SetClass(ctx, name, c, admin.Email)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
done = append(done, "is "+m.EffectiveClass())
|
|
}
|
|
}
|
|
if r.Form.Has("owner") && p.capacity != nil {
|
|
if o := r.FormValue("owner"); o != current.Owner {
|
|
if _, err := p.catalog.SetOwner(ctx, name, o, admin.Email); err != nil {
|
|
return "", err
|
|
}
|
|
if o == "" {
|
|
done = append(done, "is shared")
|
|
} else {
|
|
done = append(done, "counts against "+o)
|
|
}
|
|
}
|
|
}
|
|
lo, _ := strconv.Atoi(strings.TrimSpace(r.FormValue("min")))
|
|
hi, _ := strconv.Atoi(strings.TrimSpace(r.FormValue("max")))
|
|
if r.Form.Has("min") && (lo != current.Scaling.Min || hi != current.Scaling.Max) {
|
|
msg, err := p.setReplicas(ctx, admin, r)
|
|
if err != nil {
|
|
if len(done) > 0 {
|
|
return "", fmt.Errorf("%s %s, but not the replicas: %w", name, strings.Join(done, ", "), err)
|
|
}
|
|
return "", err
|
|
}
|
|
done = append(done, strings.TrimPrefix(msg, name+" "))
|
|
}
|
|
if len(done) == 0 {
|
|
return name + ": nothing changed", nil
|
|
}
|
|
return name + " " + strings.Join(done, "; "), nil
|
|
}
|
|
|
|
// heldKinds reads what each held single-file GGUF is — an adapter or a model —
|
|
// once per file, remembered after.
|
|
func (p *AccountPages) heldKinds(ctx context.Context, rows []mirrorRow) {
|
|
var wg sync.WaitGroup
|
|
sem := make(chan struct{}, 6)
|
|
for i := range rows {
|
|
row := &rows[i]
|
|
if row.State != mirror.Present || row.Files != 1 || !gguf.IsGGUF(row.File) {
|
|
continue
|
|
}
|
|
if h, ok := p.kinds.Load(row.URL); ok {
|
|
row.Kind, row.Arch = kindName(h.(gguf.Header)), h.(gguf.Header).Architecture
|
|
continue
|
|
}
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
sem <- struct{}{}
|
|
defer func() { <-sem }()
|
|
cctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
defer cancel()
|
|
if h, err := kindOf(cctx, row.URL); err == nil {
|
|
p.kinds.Store(row.URL, h)
|
|
row.Kind, row.Arch = kindName(h), h.Architecture
|
|
}
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
}
|
|
|
|
func kindName(h gguf.Header) string {
|
|
if h.IsAdapter() {
|
|
return "adapter"
|
|
}
|
|
return "model"
|
|
}
|
|
|
|
// adapterChoice is a held adapter file that could be attached to a model.
|
|
type adapterChoice struct {
|
|
URL, Label, Suggest string
|
|
}
|
|
|
|
// addAdapterChoices offers, on each model of a held model's card, the held
|
|
// adapter files of the same architecture, and a way to find more.
|
|
func (d *heldDetail) addAdapterChoices(row mirrorRow, rows []mirrorRow) {
|
|
for i := range d.Models {
|
|
hm := &d.Models[i]
|
|
hm.FindAdapters = findAdaptersHref(row.Source)
|
|
if row.Kind == "adapter" {
|
|
continue
|
|
}
|
|
for _, r := range rows {
|
|
if r.Kind != "adapter" || (row.Arch != "" && r.Arch != row.Arch) {
|
|
continue
|
|
}
|
|
attached := false
|
|
for _, a := range hm.Adapters {
|
|
attached = attached || a.Source == r.URL
|
|
}
|
|
if !attached {
|
|
hm.AdapterChoices = append(hm.AdapterChoices, adapterChoice{URL: r.URL,
|
|
Label: r.Model + " " + r.Version + " · " + r.File, Suggest: hm.Name + "-" + r.Model})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// findAdaptersHref is the explorer's list of adapters for a held model: those
|
|
// made for the model its repository was made from, when it came from the
|
|
// registry, and every adapter otherwise.
|
|
func findAdaptersHref(source string) template.URL {
|
|
const base = "/admin/mirror?tab=explore&view=adapters"
|
|
u, err := url.Parse(source)
|
|
if err != nil || u.Host != "huggingface.co" {
|
|
return base
|
|
}
|
|
parts := strings.Split(strings.Trim(u.Path, "/"), "/")
|
|
if len(parts) < 2 {
|
|
return base
|
|
}
|
|
return template.URL(base + "&from=" + url.QueryEscape(parts[0]+"/"+parts[1]))
|
|
}
|
|
|
|
// setAdapters attaches or detaches one adapter on a model, from its card.
|
|
// An adapter is checked before it is attached: a file that is not an adapter,
|
|
// or one made for another architecture, is a replica that fails to start.
|
|
func (p *AccountPages) setAdapters(ctx context.Context, admin store.User, r *http.Request) (string, error) {
|
|
name := r.FormValue("model")
|
|
current, err := p.currentModel(ctx, name)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
adapters := slices.Clone(current.Adapters)
|
|
switch r.FormValue("op") {
|
|
case "remove":
|
|
gone := r.FormValue("name")
|
|
adapters = slices.DeleteFunc(adapters, func(a catalog.Adapter) bool { return a.Name == gone })
|
|
if len(adapters) == len(current.Adapters) {
|
|
return "", fmt.Errorf("%s has no adapter %q", name, gone)
|
|
}
|
|
if _, err := p.catalog.SetAdapters(ctx, name, adapters, admin.Email); err != nil {
|
|
return "", err
|
|
}
|
|
return fmt.Sprintf("%s: adapter %s removed; replicas drop it when restarted (llmgw ctl restart --model %s)",
|
|
name, gone, name), nil
|
|
case "add":
|
|
an := strings.TrimSpace(r.FormValue("name"))
|
|
src := strings.TrimSpace(r.FormValue("source"))
|
|
scale := 1.0
|
|
if v := strings.TrimSpace(r.FormValue("scale")); v != "" {
|
|
if scale, err = strconv.ParseFloat(v, 64); err != nil || scale <= 0 {
|
|
return "", errors.New("the scale is a positive number")
|
|
}
|
|
}
|
|
if an == "" || src == "" {
|
|
return "", errors.New("an adapter needs a name and a file")
|
|
}
|
|
h, err := kindOf(ctx, src)
|
|
if err != nil {
|
|
return "", fmt.Errorf("the adapter file could not be read: %w", err)
|
|
}
|
|
if !h.IsAdapter() {
|
|
return "", errors.New("that file is a model, not a LoRA adapter")
|
|
}
|
|
if base, err := kindOf(ctx, current.Source); err == nil && base.Architecture != h.Architecture {
|
|
return "", fmt.Errorf("that adapter is for %s models and %s is %s", h.Architecture, name, base.Architecture)
|
|
}
|
|
adapters = append(adapters, catalog.Adapter{Name: an, Source: src, Scale: scale})
|
|
if _, err := p.catalog.SetAdapters(ctx, name, adapters, admin.Email); err != nil {
|
|
return "", err
|
|
}
|
|
return fmt.Sprintf("%s: adapter %s added; ask for it by that name once the replicas are restarted "+
|
|
"(llmgw ctl restart --model %s)", name, an, name), nil
|
|
}
|
|
return "", errors.New("add or remove an adapter")
|
|
}
|