package web import ( "context" "errors" "html/template" "net/http" "net/url" "slices" "strconv" "strings" "sync" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // HeadroomSource is a fleet that can say how many more replicas of a model // fit now. Optional: without it the card shows what runs and offers no change // it cannot check. type HeadroomSource interface { Headroom(model string, limit int) (adminapi.HeadroomView, bool) } // maxReplicas bounds what the form accepts. Not a cluster limit, a typing one: // 300 where 3 was meant would otherwise be taken as a ceiling. const maxReplicas = 64 // heldDetail is the card opened beneath a held version: what runs it, where, // and room to change it. type heldDetail struct { Models []heldModel // Scopes are the parts of the tenancy tree a model can belong to; empty // when capacity quotas are not set up, and the owner choice is left off. Scopes []adminapi.ScopeName } // heldModel is one catalog model running a held version. type heldModel struct { modelUse // Adapters are the model's own; AdapterChoices the held adapter files // of its architecture not attached yet; FindAdapters the explorer's // list of adapters made for it. Adapters []catalog.Adapter AdapterChoices []adapterChoice FindAdapters template.URL View adminapi.ModelView Replicas []heldReplica Headroom *adminapi.HeadroomView // MostNow is how many replicas could always run, Current plus the room // found; zero when the room was not measured or had no end in sight. MostNow int // RoomText and PreemptText say the headroom in a sentence each, built // here because the nodes it names are a list inside the sentence. RoomText, PreemptText string } // heldReplica is one replica, by the nodes it runs on. type heldReplica struct { ID string State string // StateText is State as the page says it; Restart is already worded. StateText string Restart string Age int64 Endpoint string Nodes []heldShard } type heldShard struct { Node string MemoryGB int // the node's RAM, 0 when unknown CPU float64 MemBytes int64 } // openKey names a row the way ?open= does. func openKey(model, version string) string { return model + ":" + version } // heldDetailFor reads, for each model using a row, how it runs and how much // more room it has. func (p *AccountPages) heldDetailFor(ctx context.Context, l *Lang, row mirrorRow) *heldDetail { d := &heldDetail{} if p.capacity != nil { d.Scopes, _ = p.capacity.Scopes(ctx) } if p.fleet == nil { for _, u := range row.UsedBy { d.Models = append(d.Models, heldModel{modelUse: u, Adapters: u.Adapters}) } return d } views := map[string]adminapi.ModelView{} for _, v := range p.fleet.Models() { views[v.Name] = v } memory := map[string]int{} for _, n := range p.fleet.Nodes() { memory[n.ID] = n.MemoryGB } hs, _ := p.fleet.(HeadroomSource) for _, u := range row.UsedBy { hm := heldModel{modelUse: u, View: views[u.Name], Adapters: u.Adapters} for _, rep := range p.fleet.Replicas(u.Name) { // Built field by field: a ReplicaView carries the engine's key, // which is for the admin listener and never for a page. hr := heldReplica{ID: rep.ID, State: rep.State, StateText: replicaState(l, rep.State), Restart: restartState(l, rep.Restart), Age: rep.AgeSec, Endpoint: rep.Endpoint} for _, sh := range rep.Shards { hr.Nodes = append(hr.Nodes, heldShard{Node: sh.Node, MemoryGB: memory[sh.Node], CPU: sh.CPUCores, MemBytes: sh.MemoryBytes}) } hm.Replicas = append(hm.Replicas, hr) } if hs != nil { if h, ok := hs.Headroom(u.Name, 16); ok { hm.Headroom = &h if !h.Limited { hm.MostNow = h.Current + h.More + h.ByPreempting } hm.RoomText, hm.PreemptText = roomText(l, h) } } d.Models = append(d.Models, hm) } return d } // replicaStates are the states a replica reports, each with its word. var replicaStates = []string{"placing", "starting", "ready", "draining", "stopped", "failed", "unknown"} func init() { for _, s := range replicaStates { registerKeys("mirror.replica." + s) } registerKeys("mirror.mode.always", "mirror.mode.ondemand", "mirror.mode.stopped", "mirror.class.critical", "mirror.class.standard", "mirror.class.opportunistic") } // replicaState is a replica's state in words; one this page does not know // is shown as the control plane said it. func replicaState(l *Lang, s string) string { if slices.Contains(replicaStates, s) { return l.T("mirror.replica." + s) } return s } // restartState is where an operator's restart stands. A refusal carries // the control plane's own reason, which is shown as it came. func restartState(l *Lang, s string) string { switch s { case "queued": return l.T("mirror.restart.queued") case "replacing": return l.T("mirror.restart.replacing") case "in place": return l.T("mirror.restart.in_place") } return s } // roomText says how much more of a model fits, and how much more it could // have by stopping what its class outranks. func roomText(l *Lang, h adminapi.HeadroomView) (room, preempt string) { switch { case h.Limited: room = l.T("mirror.room.at_least", h.More) case h.More > 0 && len(h.Nodes) > 0: var on []string for _, n := range h.Nodes { if len(n) == 0 { on = append(on, "—") continue } on = append(on, strings.Join(n, ", ")) } room = l.T("mirror.room.more_on", h.More, strings.Join(on, ", ")) case h.More > 0: room = l.T("mirror.room.more", h.More) default: room = l.T("mirror.room.none") } if h.ByPreempting > 0 { who := "—" if len(h.Preempts) > 0 { who = strings.Join(h.Preempts, ", ") } if h.More > 0 { preempt = l.T("mirror.room.and_preempting", h.ByPreempting, who) } else { preempt = l.T("mirror.room.preempting", h.ByPreempting, who) } } return room, preempt } // setReplicas changes how many replicas a model always runs and how far it // may grow, refusing an always-on count the cluster has no room for. // // Refused rather than taken and left short: a minimum placement cannot meet // is a model reported as unplaceable on every pass from then on, which is the // same answer given later and less plainly. A maximum above the room is // accepted and said: it is reached only under load, by which time something // may have made room, and refusing it would make the ceiling depend on what // happened to be running when it was set. func (p *AccountPages) setReplicas(ctx context.Context, admin store.User, r *http.Request) (string, error) { l := langOf(r) name := r.FormValue("model") lo, err1 := strconv.Atoi(strings.TrimSpace(r.FormValue("min"))) hi, err2 := strconv.Atoi(strings.TrimSpace(r.FormValue("max"))) switch { case name == "": return "", errors.New(l.T("mirror.err.which_model")) case err1 != nil || err2 != nil: return "", errors.New(l.T("mirror.err.replicas_whole")) case lo < 0 || hi < lo: return "", errors.New(l.T("mirror.err.min_over_max", lo, hi)) case hi > maxReplicas: return "", errors.New(l.T("mirror.err.at_most", maxReplicas)) } current, err := p.currentModel(ctx, l, name) if err != nil { return "", err } if p.catalog.Declared(name) { return "", errors.New(l.T("mirror.err.declared", name)) } note := "" if hs, ok := p.fleet.(HeadroomSource); ok && hi > 0 { h, ok := hs.Headroom(name, hi) if !ok { return "", errors.New(l.T("mirror.err.no_model", name)) } fits := h.Current + h.More + h.ByPreempting if lo > fits && !h.Limited { why := l.T("mirror.err.nothing_fits") if len(h.Blocked) > 0 { why = strings.Join(h.Blocked, "; ") } return "", errors.New(l.T("mirror.err.no_room", name, fits, h.Current, h.More, why)) } if hi > fits && !h.Limited { note = "; " + l.T("mirror.msg.room_for", fits) } // Room its class would make by stopping others: said, since it is // somebody else's replica that goes. if lo > h.Current+h.More && len(h.Preempts) > 0 { note += "; " + l.T("mirror.msg.will_stop", strings.Join(h.Preempts, ", ")) } } sc := catalog.Scaling{Min: lo, Max: hi, IdleTimeout: current.Scaling.IdleTimeout} if sc.OnDemand() && sc.IdleTimeout <= 0 { sc.IdleTimeout = defaultIdle } if _, err := p.catalog.Scale(ctx, name, sc, admin.Email); err != nil { return "", err } // Each says the model's name first and the rest after it, which setModel // relies on to join this onto what else it changed. switch { case hi == 0: return l.T("mirror.msg.stopped", name), nil case lo == 0: return l.T("mirror.msg.loads_up_to", name, hi) + note, nil case lo == hi: return l.T("mirror.msg.runs_n", name, l.N("serve.replicas_n", int64(lo))) + note, nil } return l.T("mirror.msg.runs_up_to", name, lo, hi) + note, nil } // currentModel is a model as the catalog holds it now. func (p *AccountPages) currentModel(ctx context.Context, l *Lang, name string) (catalog.Model, error) { sources, err := p.catalog.Sources(ctx) if err != nil { return catalog.Model{}, err } for _, ms := range sources { for _, m := range ms { if m.Name == name { return m, nil } } } return catalog.Model{}, errors.New(l.T("mirror.err.not_from_mirror", name)) } // age renders seconds the way the replicas table does. func age(sec int64) string { return english.Age(sec) } // setModel applies the card's settings form: what a replica reserves, the // model's class and owner, then how many replicas it runs — in that order, so // the room checked for the replicas is the room its new class and reservation // would get. Only what changed is applied, and each change is said. func (p *AccountPages) setModel(ctx context.Context, admin store.User, r *http.Request) (string, error) { l := langOf(r) name := r.FormValue("model") current, err := p.currentModel(ctx, l, name) if err != nil { return "", err } if p.catalog.Declared(name) { return "", errors.New(l.T("mirror.err.declared", name)) } var done []string if r.Form.Has("memory") && len(current.Profiles) > 0 { mem, err1 := strconv.Atoi(strings.TrimSpace(r.FormValue("memory"))) cores, err2 := strconv.Atoi(strings.TrimSpace(r.FormValue("cores"))) if err1 != nil || err2 != nil || mem < 0 || cores < 0 { return "", errors.New(l.T("mirror.err.memory_whole")) } if cur := current.Profiles[0]; mem != cur.MinMemoryGB || cores != cur.CPUCores { if _, err := p.catalog.Resources(ctx, name, "", mem, cores, admin.Email); err != nil { return "", err } done = append(done, l.T("mirror.msg.reserves", mem, cores)) } } if r.Form.Has("class") { if c := r.FormValue("class"); c != current.Class { m, err := p.catalog.SetClass(ctx, name, c, admin.Email) if err != nil { return "", err } done = append(done, l.T("mirror.msg.is_class", l.T("mirror.class."+m.EffectiveClass()))) } } if r.Form.Has("owner") && p.capacity != nil { if o := r.FormValue("owner"); o != current.Owner { if _, err := p.catalog.SetOwner(ctx, name, o, admin.Email); err != nil { return "", err } if o == "" { done = append(done, l.T("mirror.msg.is_shared")) } else { done = append(done, l.T("mirror.msg.counts_against", o)) } } } lo, _ := strconv.Atoi(strings.TrimSpace(r.FormValue("min"))) hi, _ := strconv.Atoi(strings.TrimSpace(r.FormValue("max"))) if r.Form.Has("min") && (lo != current.Scaling.Min || hi != current.Scaling.Max) { msg, err := p.setReplicas(ctx, admin, r) if err != nil { if len(done) > 0 { return "", errors.New(l.T("mirror.err.partly_set", name, strings.Join(done, ", "), l.Err(err))) } return "", err } done = append(done, strings.TrimPrefix(msg, name+" ")) } if len(done) == 0 { return l.T("mirror.msg.nothing_changed", name), nil } return l.T("mirror.msg.changed", name, strings.Join(done, "; ")), nil } // heldKinds reads what each held single-file GGUF is — an adapter or a model — // once per file, remembered after. func (p *AccountPages) heldKinds(ctx context.Context, rows []mirrorRow) { var wg sync.WaitGroup sem := make(chan struct{}, 6) for i := range rows { row := &rows[i] if row.State != mirror.Present || row.Files != 1 || !gguf.IsGGUF(row.File) { continue } if h, ok := p.kinds.Load(row.URL); ok { row.Kind, row.Arch = kindName(h.(gguf.Header)), h.(gguf.Header).Architecture continue } wg.Add(1) go func() { defer wg.Done() sem <- struct{}{} defer func() { <-sem }() cctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() if h, err := kindOf(cctx, row.URL); err == nil { p.kinds.Store(row.URL, h) row.Kind, row.Arch = kindName(h), h.Architecture } }() } wg.Wait() } func kindName(h gguf.Header) string { if h.IsAdapter() { return "adapter" } return "model" } // adapterChoice is a held adapter file that could be attached to a model. type adapterChoice struct { URL, Label, Suggest string } // addAdapterChoices offers, on each model of a held model's card, the held // adapter files of the same architecture, and a way to find more. func (d *heldDetail) addAdapterChoices(row mirrorRow, rows []mirrorRow) { for i := range d.Models { hm := &d.Models[i] hm.FindAdapters = findAdaptersHref(row.Source) if row.Kind == "adapter" { continue } for _, r := range rows { if r.Kind != "adapter" || (row.Arch != "" && r.Arch != row.Arch) { continue } attached := false for _, a := range hm.Adapters { attached = attached || a.Source == r.URL } if !attached { hm.AdapterChoices = append(hm.AdapterChoices, adapterChoice{URL: r.URL, Label: r.Model + " " + r.Version + " · " + r.File, Suggest: hm.Name + "-" + r.Model}) } } } } // findAdaptersHref is the explorer's list of adapters for a held model: those // made for the model its repository was made from, when it came from the // registry, and every adapter otherwise. func findAdaptersHref(source string) template.URL { const base = "/admin/mirror?tab=explore&view=adapters" u, err := url.Parse(source) if err != nil || u.Host != "huggingface.co" { return base } parts := strings.Split(strings.Trim(u.Path, "/"), "/") if len(parts) < 2 { return base } return template.URL(base + "&from=" + url.QueryEscape(parts[0]+"/"+parts[1])) } // setAdapters attaches or detaches one adapter on a model, from its card. // An adapter is checked before it is attached: a file that is not an adapter, // or one made for another architecture, is a replica that fails to start. func (p *AccountPages) setAdapters(ctx context.Context, admin store.User, r *http.Request) (string, error) { l := langOf(r) name := r.FormValue("model") current, err := p.currentModel(ctx, l, name) if err != nil { return "", err } adapters := slices.Clone(current.Adapters) switch r.FormValue("op") { case "remove": gone := r.FormValue("name") adapters = slices.DeleteFunc(adapters, func(a catalog.Adapter) bool { return a.Name == gone }) if len(adapters) == len(current.Adapters) { return "", errors.New(l.T("mirror.err.no_adapter", name, gone)) } if _, err := p.catalog.SetAdapters(ctx, name, adapters, admin.Email); err != nil { return "", err } return l.T("mirror.msg.adapter_removed", name, gone), nil case "add": an := strings.TrimSpace(r.FormValue("name")) src := strings.TrimSpace(r.FormValue("source")) scale := 1.0 if v := strings.TrimSpace(r.FormValue("scale")); v != "" { if scale, err = strconv.ParseFloat(v, 64); err != nil || scale <= 0 { return "", errors.New(l.T("mirror.err.scale")) } } if an == "" || src == "" { return "", errors.New(l.T("mirror.err.adapter_name_file")) } h, err := kindOf(ctx, src) if err != nil { return "", errors.New(l.T("mirror.err.adapter_unreadable", l.Err(err))) } if !h.IsAdapter() { return "", errors.New(l.T("mirror.err.not_adapter")) } if base, err := kindOf(ctx, current.Source); err == nil && base.Architecture != h.Architecture { return "", errors.New(l.T("mirror.err.wrong_arch", h.Architecture, name, base.Architecture)) } adapters = append(adapters, catalog.Adapter{Name: an, Source: src, Scale: scale}) if _, err := p.catalog.SetAdapters(ctx, name, adapters, admin.Email); err != nil { return "", err } return l.T("mirror.msg.adapter_added", name, an), nil } return "", errors.New(l.T("mirror.err.adapter_op")) }