The Held tab's model name opens a card: each replica with its node, state, age, CPU and memory, the room for more found by dry-running the scheduler on a copy of the inventory, and a form for the always-running and under-load replica counts. An always-running count placement cannot meet is refused with the reason. Headroom is also on the admin API. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
208 lines
6.3 KiB
Go
208 lines
6.3 KiB
Go
package web
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// HeadroomSource is a fleet that can say how many more replicas of a model
|
|
// fit now. Optional: without it the card shows what runs and offers no change
|
|
// it cannot check.
|
|
type HeadroomSource interface {
|
|
Headroom(model string, limit int) (adminapi.HeadroomView, bool)
|
|
}
|
|
|
|
// maxReplicas bounds what the form accepts. Not a cluster limit, a typing one:
|
|
// 300 where 3 was meant would otherwise be taken as a ceiling.
|
|
const maxReplicas = 64
|
|
|
|
// heldDetail is the card opened beneath a held version: what runs it, where,
|
|
// and room to change it.
|
|
type heldDetail struct {
|
|
Models []heldModel
|
|
}
|
|
|
|
// heldModel is one catalog model running a held version.
|
|
type heldModel struct {
|
|
modelUse
|
|
View adminapi.ModelView
|
|
Replicas []heldReplica
|
|
Headroom *adminapi.HeadroomView
|
|
// MostNow is how many replicas could always run, Current plus the room
|
|
// found; zero when the room was not measured or had no end in sight.
|
|
MostNow int
|
|
}
|
|
|
|
// heldReplica is one replica, by the nodes it runs on.
|
|
type heldReplica struct {
|
|
ID string
|
|
State string
|
|
Restart string
|
|
Age int64
|
|
Endpoint string
|
|
Nodes []heldShard
|
|
}
|
|
|
|
type heldShard struct {
|
|
Node string
|
|
MemoryGB int // the node's RAM, 0 when unknown
|
|
CPU float64
|
|
MemBytes int64
|
|
}
|
|
|
|
// openKey names a row the way ?open= does.
|
|
func openKey(model, version string) string { return model + ":" + version }
|
|
|
|
// heldDetailFor reads, for each model using a row, how it runs and how much
|
|
// more room it has.
|
|
func (p *AccountPages) heldDetailFor(row mirrorRow) *heldDetail {
|
|
d := &heldDetail{}
|
|
if p.fleet == nil {
|
|
for _, u := range row.UsedBy {
|
|
d.Models = append(d.Models, heldModel{modelUse: u})
|
|
}
|
|
return d
|
|
}
|
|
views := map[string]adminapi.ModelView{}
|
|
for _, v := range p.fleet.Models() {
|
|
views[v.Name] = v
|
|
}
|
|
memory := map[string]int{}
|
|
for _, n := range p.fleet.Nodes() {
|
|
memory[n.ID] = n.MemoryGB
|
|
}
|
|
hs, _ := p.fleet.(HeadroomSource)
|
|
for _, u := range row.UsedBy {
|
|
hm := heldModel{modelUse: u, View: views[u.Name]}
|
|
for _, rep := range p.fleet.Replicas(u.Name) {
|
|
// Built field by field: a ReplicaView carries the engine's key,
|
|
// which is for the admin listener and never for a page.
|
|
hr := heldReplica{ID: rep.ID, State: rep.State, Restart: rep.Restart,
|
|
Age: rep.AgeSec, Endpoint: rep.Endpoint}
|
|
for _, sh := range rep.Shards {
|
|
hr.Nodes = append(hr.Nodes, heldShard{Node: sh.Node, MemoryGB: memory[sh.Node],
|
|
CPU: sh.CPUCores, MemBytes: sh.MemoryBytes})
|
|
}
|
|
hm.Replicas = append(hm.Replicas, hr)
|
|
}
|
|
if hs != nil {
|
|
if h, ok := hs.Headroom(u.Name, 16); ok {
|
|
hm.Headroom = &h
|
|
if !h.Limited {
|
|
hm.MostNow = h.Current + h.More
|
|
}
|
|
}
|
|
}
|
|
d.Models = append(d.Models, hm)
|
|
}
|
|
return d
|
|
}
|
|
|
|
// setReplicas changes how many replicas a model always runs and how far it
|
|
// may grow, refusing an always-on count the cluster has no room for.
|
|
//
|
|
// Refused rather than taken and left short: a minimum placement cannot meet
|
|
// is a model reported as unplaceable on every pass from then on, which is the
|
|
// same answer given later and less plainly. A maximum above the room is
|
|
// accepted and said: it is reached only under load, by which time something
|
|
// may have made room, and refusing it would make the ceiling depend on what
|
|
// happened to be running when it was set.
|
|
func (p *AccountPages) setReplicas(ctx context.Context, admin store.User, r *http.Request) (string, error) {
|
|
name := r.FormValue("model")
|
|
lo, err1 := strconv.Atoi(strings.TrimSpace(r.FormValue("min")))
|
|
hi, err2 := strconv.Atoi(strings.TrimSpace(r.FormValue("max")))
|
|
switch {
|
|
case name == "":
|
|
return "", errors.New("which model?")
|
|
case err1 != nil || err2 != nil:
|
|
return "", errors.New("replicas are whole numbers")
|
|
case lo < 0 || hi < lo:
|
|
return "", fmt.Errorf("always running (%d) cannot be more than the most under load (%d)", lo, hi)
|
|
case hi > maxReplicas:
|
|
return "", fmt.Errorf("at most %d replicas", maxReplicas)
|
|
}
|
|
current, err := p.currentModel(ctx, name)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if p.catalog.Declared(name) {
|
|
return "", fmt.Errorf("%s is set in server.yaml; change it there", name)
|
|
}
|
|
|
|
note := ""
|
|
if hs, ok := p.fleet.(HeadroomSource); ok && hi > 0 {
|
|
h, ok := hs.Headroom(name, hi)
|
|
if !ok {
|
|
return "", fmt.Errorf("no model %q", name)
|
|
}
|
|
fits := h.Current + h.More
|
|
if lo > fits && !h.Limited {
|
|
why := "nothing more fits"
|
|
if len(h.Blocked) > 0 {
|
|
why = strings.Join(h.Blocked, "; ")
|
|
}
|
|
return "", fmt.Errorf("%s can always run at most %d now (%d running, room for %d more): %s",
|
|
name, fits, h.Current, h.More, why)
|
|
}
|
|
if hi > fits && !h.Limited {
|
|
note = fmt.Sprintf("; there is room for %d now, so it grows past that only if room is made", fits)
|
|
}
|
|
}
|
|
|
|
sc := catalog.Scaling{Min: lo, Max: hi, IdleTimeout: current.Scaling.IdleTimeout}
|
|
if sc.OnDemand() && sc.IdleTimeout <= 0 {
|
|
sc.IdleTimeout = defaultIdle
|
|
}
|
|
if _, err := p.catalog.Scale(ctx, name, sc, admin.Email); err != nil {
|
|
return "", err
|
|
}
|
|
switch {
|
|
case hi == 0:
|
|
return name + " is stopped; its replicas are being taken down", nil
|
|
case lo == 0:
|
|
return fmt.Sprintf("%s loads when asked for, up to %d under load%s", name, hi, note), nil
|
|
case lo == hi:
|
|
return fmt.Sprintf("%s runs %s", name, plural(lo, "replica", "replicas")), nil
|
|
}
|
|
return fmt.Sprintf("%s runs %d, and up to %d under load%s", name, lo, hi, note), nil
|
|
}
|
|
|
|
// currentModel is a model as the catalog holds it now.
|
|
func (p *AccountPages) currentModel(ctx context.Context, name string) (catalog.Model, error) {
|
|
sources, err := p.catalog.Sources(ctx)
|
|
if err != nil {
|
|
return catalog.Model{}, err
|
|
}
|
|
for _, ms := range sources {
|
|
for _, m := range ms {
|
|
if m.Name == name {
|
|
return m, nil
|
|
}
|
|
}
|
|
}
|
|
return catalog.Model{}, fmt.Errorf("no model %q runs from the mirror", name)
|
|
}
|
|
|
|
// age renders seconds the way the replicas table does.
|
|
func age(sec int64) string {
|
|
d := time.Duration(sec) * time.Second
|
|
switch {
|
|
case d < time.Minute:
|
|
return fmt.Sprintf("%ds", sec)
|
|
case d < time.Hour:
|
|
return fmt.Sprintf("%dm", int(d.Minutes()))
|
|
case d < 48*time.Hour:
|
|
return fmt.Sprintf("%dh%02dm", int(d.Hours()), int(d.Minutes())%60)
|
|
}
|
|
return fmt.Sprintf("%dd", int(d.Hours()/24))
|
|
}
|