package web import ( "context" "errors" "fmt" "html/template" "net/http" "net/url" "slices" "strconv" "strings" "sync" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // HeadroomSource is a fleet that can say how many more replicas of a model // fit now. Optional: without it the card shows what runs and offers no change // it cannot check. type HeadroomSource interface { Headroom(model string, limit int) (adminapi.HeadroomView, bool) } // maxReplicas bounds what the form accepts. Not a cluster limit, a typing one: // 300 where 3 was meant would otherwise be taken as a ceiling. const maxReplicas = 64 // heldDetail is the card opened beneath a held version: what runs it, where, // and room to change it. type heldDetail struct { Models []heldModel // Scopes are the parts of the tenancy tree a model can belong to; empty // when capacity quotas are not set up, and the owner choice is left off. Scopes []adminapi.ScopeName } // heldModel is one catalog model running a held version. type heldModel struct { modelUse // Adapters are the model's own; AdapterChoices the held adapter files // of its architecture not attached yet; FindAdapters the explorer's // list of adapters made for it. Adapters []catalog.Adapter AdapterChoices []adapterChoice FindAdapters template.URL View adminapi.ModelView Replicas []heldReplica Headroom *adminapi.HeadroomView // MostNow is how many replicas could always run, Current plus the room // found; zero when the room was not measured or had no end in sight. MostNow int } // heldReplica is one replica, by the nodes it runs on. type heldReplica struct { ID string State string Restart string Age int64 Endpoint string Nodes []heldShard } type heldShard struct { Node string MemoryGB int // the node's RAM, 0 when unknown CPU float64 MemBytes int64 } // openKey names a row the way ?open= does. func openKey(model, version string) string { return model + ":" + version } // heldDetailFor reads, for each model using a row, how it runs and how much // more room it has. func (p *AccountPages) heldDetailFor(ctx context.Context, row mirrorRow) *heldDetail { d := &heldDetail{} if p.capacity != nil { d.Scopes, _ = p.capacity.Scopes(ctx) } if p.fleet == nil { for _, u := range row.UsedBy { d.Models = append(d.Models, heldModel{modelUse: u, Adapters: u.Adapters}) } return d } views := map[string]adminapi.ModelView{} for _, v := range p.fleet.Models() { views[v.Name] = v } memory := map[string]int{} for _, n := range p.fleet.Nodes() { memory[n.ID] = n.MemoryGB } hs, _ := p.fleet.(HeadroomSource) for _, u := range row.UsedBy { hm := heldModel{modelUse: u, View: views[u.Name], Adapters: u.Adapters} for _, rep := range p.fleet.Replicas(u.Name) { // Built field by field: a ReplicaView carries the engine's key, // which is for the admin listener and never for a page. hr := heldReplica{ID: rep.ID, State: rep.State, Restart: rep.Restart, Age: rep.AgeSec, Endpoint: rep.Endpoint} for _, sh := range rep.Shards { hr.Nodes = append(hr.Nodes, heldShard{Node: sh.Node, MemoryGB: memory[sh.Node], CPU: sh.CPUCores, MemBytes: sh.MemoryBytes}) } hm.Replicas = append(hm.Replicas, hr) } if hs != nil { if h, ok := hs.Headroom(u.Name, 16); ok { hm.Headroom = &h if !h.Limited { hm.MostNow = h.Current + h.More + h.ByPreempting } } } d.Models = append(d.Models, hm) } return d } // setReplicas changes how many replicas a model always runs and how far it // may grow, refusing an always-on count the cluster has no room for. // // Refused rather than taken and left short: a minimum placement cannot meet // is a model reported as unplaceable on every pass from then on, which is the // same answer given later and less plainly. A maximum above the room is // accepted and said: it is reached only under load, by which time something // may have made room, and refusing it would make the ceiling depend on what // happened to be running when it was set. func (p *AccountPages) setReplicas(ctx context.Context, admin store.User, r *http.Request) (string, error) { name := r.FormValue("model") lo, err1 := strconv.Atoi(strings.TrimSpace(r.FormValue("min"))) hi, err2 := strconv.Atoi(strings.TrimSpace(r.FormValue("max"))) switch { case name == "": return "", errors.New("which model?") case err1 != nil || err2 != nil: return "", errors.New("replicas are whole numbers") case lo < 0 || hi < lo: return "", fmt.Errorf("always running (%d) cannot be more than the most under load (%d)", lo, hi) case hi > maxReplicas: return "", fmt.Errorf("at most %d replicas", maxReplicas) } current, err := p.currentModel(ctx, name) if err != nil { return "", err } if p.catalog.Declared(name) { return "", fmt.Errorf("%s is set in server.yaml; change it there", name) } note := "" if hs, ok := p.fleet.(HeadroomSource); ok && hi > 0 { h, ok := hs.Headroom(name, hi) if !ok { return "", fmt.Errorf("no model %q", name) } fits := h.Current + h.More + h.ByPreempting if lo > fits && !h.Limited { why := "nothing more fits" if len(h.Blocked) > 0 { why = strings.Join(h.Blocked, "; ") } return "", fmt.Errorf("%s can always run at most %d now (%d running, room for %d more): %s", name, fits, h.Current, h.More, why) } if hi > fits && !h.Limited { note = fmt.Sprintf("; there is room for %d now, so it grows past that only if room is made", fits) } // Room its class would make by stopping others: said, since it is // somebody else's replica that goes. if lo > h.Current+h.More && len(h.Preempts) > 0 { note += "; to make room, replicas of " + strings.Join(h.Preempts, ", ") + " will be stopped" } } sc := catalog.Scaling{Min: lo, Max: hi, IdleTimeout: current.Scaling.IdleTimeout} if sc.OnDemand() && sc.IdleTimeout <= 0 { sc.IdleTimeout = defaultIdle } if _, err := p.catalog.Scale(ctx, name, sc, admin.Email); err != nil { return "", err } switch { case hi == 0: return name + " is stopped; its replicas are being taken down", nil case lo == 0: return fmt.Sprintf("%s loads when asked for, up to %d under load%s", name, hi, note), nil case lo == hi: return fmt.Sprintf("%s runs %s%s", name, plural(lo, "replica", "replicas"), note), nil } return fmt.Sprintf("%s runs %d, and up to %d under load%s", name, lo, hi, note), nil } // currentModel is a model as the catalog holds it now. func (p *AccountPages) currentModel(ctx context.Context, name string) (catalog.Model, error) { sources, err := p.catalog.Sources(ctx) if err != nil { return catalog.Model{}, err } for _, ms := range sources { for _, m := range ms { if m.Name == name { return m, nil } } } return catalog.Model{}, fmt.Errorf("no model %q runs from the mirror", name) } // age renders seconds the way the replicas table does. func age(sec int64) string { d := time.Duration(sec) * time.Second switch { case d < time.Minute: return fmt.Sprintf("%ds", sec) case d < time.Hour: return fmt.Sprintf("%dm", int(d.Minutes())) case d < 48*time.Hour: return fmt.Sprintf("%dh%02dm", int(d.Hours()), int(d.Minutes())%60) } return fmt.Sprintf("%dd", int(d.Hours()/24)) } // setModel applies the card's settings form: what a replica reserves, the // model's class and owner, then how many replicas it runs โ€” in that order, so // the room checked for the replicas is the room its new class and reservation // would get. Only what changed is applied, and each change is said. func (p *AccountPages) setModel(ctx context.Context, admin store.User, r *http.Request) (string, error) { name := r.FormValue("model") current, err := p.currentModel(ctx, name) if err != nil { return "", err } if p.catalog.Declared(name) { return "", fmt.Errorf("%s is set in server.yaml; change it there", name) } var done []string if r.Form.Has("memory") && len(current.Profiles) > 0 { mem, err1 := strconv.Atoi(strings.TrimSpace(r.FormValue("memory"))) cores, err2 := strconv.Atoi(strings.TrimSpace(r.FormValue("cores"))) if err1 != nil || err2 != nil || mem < 0 || cores < 0 { return "", errors.New("memory and cores are whole numbers") } if cur := current.Profiles[0]; mem != cur.MinMemoryGB || cores != cur.CPUCores { if _, err := p.catalog.Resources(ctx, name, "", mem, cores, admin.Email); err != nil { return "", err } done = append(done, fmt.Sprintf("a replica reserves %d GB and %d cores (running ones keep theirs until restarted)", mem, cores)) } } if r.Form.Has("class") { if c := r.FormValue("class"); c != current.Class { m, err := p.catalog.SetClass(ctx, name, c, admin.Email) if err != nil { return "", err } done = append(done, "is "+m.EffectiveClass()) } } if r.Form.Has("owner") && p.capacity != nil { if o := r.FormValue("owner"); o != current.Owner { if _, err := p.catalog.SetOwner(ctx, name, o, admin.Email); err != nil { return "", err } if o == "" { done = append(done, "is shared") } else { done = append(done, "counts against "+o) } } } lo, _ := strconv.Atoi(strings.TrimSpace(r.FormValue("min"))) hi, _ := strconv.Atoi(strings.TrimSpace(r.FormValue("max"))) if r.Form.Has("min") && (lo != current.Scaling.Min || hi != current.Scaling.Max) { msg, err := p.setReplicas(ctx, admin, r) if err != nil { if len(done) > 0 { return "", fmt.Errorf("%s %s, but not the replicas: %w", name, strings.Join(done, ", "), err) } return "", err } done = append(done, strings.TrimPrefix(msg, name+" ")) } if len(done) == 0 { return name + ": nothing changed", nil } return name + " " + strings.Join(done, "; "), nil } // heldKinds reads what each held single-file GGUF is โ€” an adapter or a model โ€” // once per file, remembered after. func (p *AccountPages) heldKinds(ctx context.Context, rows []mirrorRow) { var wg sync.WaitGroup sem := make(chan struct{}, 6) for i := range rows { row := &rows[i] if row.State != mirror.Present || row.Files != 1 || !gguf.IsGGUF(row.File) { continue } if h, ok := p.kinds.Load(row.URL); ok { row.Kind, row.Arch = kindName(h.(gguf.Header)), h.(gguf.Header).Architecture continue } wg.Add(1) go func() { defer wg.Done() sem <- struct{}{} defer func() { <-sem }() cctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() if h, err := kindOf(cctx, row.URL); err == nil { p.kinds.Store(row.URL, h) row.Kind, row.Arch = kindName(h), h.Architecture } }() } wg.Wait() } func kindName(h gguf.Header) string { if h.IsAdapter() { return "adapter" } return "model" } // adapterChoice is a held adapter file that could be attached to a model. type adapterChoice struct { URL, Label, Suggest string } // addAdapterChoices offers, on each model of a held model's card, the held // adapter files of the same architecture, and a way to find more. func (d *heldDetail) addAdapterChoices(row mirrorRow, rows []mirrorRow) { for i := range d.Models { hm := &d.Models[i] hm.FindAdapters = findAdaptersHref(row.Source) if row.Kind == "adapter" { continue } for _, r := range rows { if r.Kind != "adapter" || (row.Arch != "" && r.Arch != row.Arch) { continue } attached := false for _, a := range hm.Adapters { attached = attached || a.Source == r.URL } if !attached { hm.AdapterChoices = append(hm.AdapterChoices, adapterChoice{URL: r.URL, Label: r.Model + " " + r.Version + " ยท " + r.File, Suggest: hm.Name + "-" + r.Model}) } } } } // findAdaptersHref is the explorer's list of adapters for a held model: those // made for the model its repository was made from, when it came from the // registry, and every adapter otherwise. func findAdaptersHref(source string) template.URL { const base = "/admin/mirror?tab=explore&view=adapters" u, err := url.Parse(source) if err != nil || u.Host != "huggingface.co" { return base } parts := strings.Split(strings.Trim(u.Path, "/"), "/") if len(parts) < 2 { return base } return template.URL(base + "&from=" + url.QueryEscape(parts[0]+"/"+parts[1])) } // setAdapters attaches or detaches one adapter on a model, from its card. // An adapter is checked before it is attached: a file that is not an adapter, // or one made for another architecture, is a replica that fails to start. func (p *AccountPages) setAdapters(ctx context.Context, admin store.User, r *http.Request) (string, error) { name := r.FormValue("model") current, err := p.currentModel(ctx, name) if err != nil { return "", err } adapters := slices.Clone(current.Adapters) switch r.FormValue("op") { case "remove": gone := r.FormValue("name") adapters = slices.DeleteFunc(adapters, func(a catalog.Adapter) bool { return a.Name == gone }) if len(adapters) == len(current.Adapters) { return "", fmt.Errorf("%s has no adapter %q", name, gone) } if _, err := p.catalog.SetAdapters(ctx, name, adapters, admin.Email); err != nil { return "", err } return fmt.Sprintf("%s: adapter %s removed; replicas drop it when restarted (llmgw ctl restart --model %s)", name, gone, name), nil case "add": an := strings.TrimSpace(r.FormValue("name")) src := strings.TrimSpace(r.FormValue("source")) scale := 1.0 if v := strings.TrimSpace(r.FormValue("scale")); v != "" { if scale, err = strconv.ParseFloat(v, 64); err != nil || scale <= 0 { return "", errors.New("the scale is a positive number") } } if an == "" || src == "" { return "", errors.New("an adapter needs a name and a file") } h, err := kindOf(ctx, src) if err != nil { return "", fmt.Errorf("the adapter file could not be read: %w", err) } if !h.IsAdapter() { return "", errors.New("that file is a model, not a LoRA adapter") } if base, err := kindOf(ctx, current.Source); err == nil && base.Architecture != h.Architecture { return "", fmt.Errorf("that adapter is for %s models and %s is %s", h.Architecture, name, base.Architecture) } adapters = append(adapters, catalog.Adapter{Name: an, Source: src, Scale: scale}) if _, err := p.catalog.SetAdapters(ctx, name, adapters, admin.Email); err != nil { return "", err } return fmt.Sprintf("%s: adapter %s added; ask for it by that name once the replicas are restarted "+ "(llmgw ctl restart --model %s)", name, an, name), nil } return "", errors.New("add or remove an adapter") }