package web import ( "context" "errors" "fmt" "net/http" "strconv" "strings" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // HeadroomSource is a fleet that can say how many more replicas of a model // fit now. Optional: without it the card shows what runs and offers no change // it cannot check. type HeadroomSource interface { Headroom(model string, limit int) (adminapi.HeadroomView, bool) } // maxReplicas bounds what the form accepts. Not a cluster limit, a typing one: // 300 where 3 was meant would otherwise be taken as a ceiling. const maxReplicas = 64 // heldDetail is the card opened beneath a held version: what runs it, where, // and room to change it. type heldDetail struct { Models []heldModel } // heldModel is one catalog model running a held version. type heldModel struct { modelUse View adminapi.ModelView Replicas []heldReplica Headroom *adminapi.HeadroomView // MostNow is how many replicas could always run, Current plus the room // found; zero when the room was not measured or had no end in sight. MostNow int } // heldReplica is one replica, by the nodes it runs on. type heldReplica struct { ID string State string Restart string Age int64 Endpoint string Nodes []heldShard } type heldShard struct { Node string MemoryGB int // the node's RAM, 0 when unknown CPU float64 MemBytes int64 } // openKey names a row the way ?open= does. func openKey(model, version string) string { return model + ":" + version } // heldDetailFor reads, for each model using a row, how it runs and how much // more room it has. func (p *AccountPages) heldDetailFor(row mirrorRow) *heldDetail { d := &heldDetail{} if p.fleet == nil { for _, u := range row.UsedBy { d.Models = append(d.Models, heldModel{modelUse: u}) } return d } views := map[string]adminapi.ModelView{} for _, v := range p.fleet.Models() { views[v.Name] = v } memory := map[string]int{} for _, n := range p.fleet.Nodes() { memory[n.ID] = n.MemoryGB } hs, _ := p.fleet.(HeadroomSource) for _, u := range row.UsedBy { hm := heldModel{modelUse: u, View: views[u.Name]} for _, rep := range p.fleet.Replicas(u.Name) { // Built field by field: a ReplicaView carries the engine's key, // which is for the admin listener and never for a page. hr := heldReplica{ID: rep.ID, State: rep.State, Restart: rep.Restart, Age: rep.AgeSec, Endpoint: rep.Endpoint} for _, sh := range rep.Shards { hr.Nodes = append(hr.Nodes, heldShard{Node: sh.Node, MemoryGB: memory[sh.Node], CPU: sh.CPUCores, MemBytes: sh.MemoryBytes}) } hm.Replicas = append(hm.Replicas, hr) } if hs != nil { if h, ok := hs.Headroom(u.Name, 16); ok { hm.Headroom = &h if !h.Limited { hm.MostNow = h.Current + h.More } } } d.Models = append(d.Models, hm) } return d } // setReplicas changes how many replicas a model always runs and how far it // may grow, refusing an always-on count the cluster has no room for. // // Refused rather than taken and left short: a minimum placement cannot meet // is a model reported as unplaceable on every pass from then on, which is the // same answer given later and less plainly. A maximum above the room is // accepted and said: it is reached only under load, by which time something // may have made room, and refusing it would make the ceiling depend on what // happened to be running when it was set. func (p *AccountPages) setReplicas(ctx context.Context, admin store.User, r *http.Request) (string, error) { name := r.FormValue("model") lo, err1 := strconv.Atoi(strings.TrimSpace(r.FormValue("min"))) hi, err2 := strconv.Atoi(strings.TrimSpace(r.FormValue("max"))) switch { case name == "": return "", errors.New("which model?") case err1 != nil || err2 != nil: return "", errors.New("replicas are whole numbers") case lo < 0 || hi < lo: return "", fmt.Errorf("always running (%d) cannot be more than the most under load (%d)", lo, hi) case hi > maxReplicas: return "", fmt.Errorf("at most %d replicas", maxReplicas) } current, err := p.currentModel(ctx, name) if err != nil { return "", err } if p.catalog.Declared(name) { return "", fmt.Errorf("%s is set in server.yaml; change it there", name) } note := "" if hs, ok := p.fleet.(HeadroomSource); ok && hi > 0 { h, ok := hs.Headroom(name, hi) if !ok { return "", fmt.Errorf("no model %q", name) } fits := h.Current + h.More if lo > fits && !h.Limited { why := "nothing more fits" if len(h.Blocked) > 0 { why = strings.Join(h.Blocked, "; ") } return "", fmt.Errorf("%s can always run at most %d now (%d running, room for %d more): %s", name, fits, h.Current, h.More, why) } if hi > fits && !h.Limited { note = fmt.Sprintf("; there is room for %d now, so it grows past that only if room is made", fits) } } sc := catalog.Scaling{Min: lo, Max: hi, IdleTimeout: current.Scaling.IdleTimeout} if sc.OnDemand() && sc.IdleTimeout <= 0 { sc.IdleTimeout = defaultIdle } if _, err := p.catalog.Scale(ctx, name, sc, admin.Email); err != nil { return "", err } switch { case hi == 0: return name + " is stopped; its replicas are being taken down", nil case lo == 0: return fmt.Sprintf("%s loads when asked for, up to %d under load%s", name, hi, note), nil case lo == hi: return fmt.Sprintf("%s runs %s", name, plural(lo, "replica", "replicas")), nil } return fmt.Sprintf("%s runs %d, and up to %d under load%s", name, lo, hi, note), nil } // currentModel is a model as the catalog holds it now. func (p *AccountPages) currentModel(ctx context.Context, name string) (catalog.Model, error) { sources, err := p.catalog.Sources(ctx) if err != nil { return catalog.Model{}, err } for _, ms := range sources { for _, m := range ms { if m.Name == name { return m, nil } } } return catalog.Model{}, fmt.Errorf("no model %q runs from the mirror", name) } // age renders seconds the way the replicas table does. func age(sec int64) string { d := time.Duration(sec) * time.Second switch { case d < time.Minute: return fmt.Sprintf("%ds", sec) case d < time.Hour: return fmt.Sprintf("%dm", int(d.Minutes())) case d < 48*time.Hour: return fmt.Sprintf("%dh%02dm", int(d.Hours()), int(d.Minutes())%60) } return fmt.Sprintf("%dd", int(d.Hours()/24)) }