Files
llmgw/internal/web/mirror_detail.go
T
Ilker ManapandClaude Opus 5.5 d7f1c1d545
Release / build (push) Successful in 1m6s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m48s
Open a held model to its replicas, nodes and room for more
The Held tab's model name opens a card: each replica with its node, state,
age, CPU and memory, the room for more found by dry-running the scheduler
on a copy of the inventory, and a form for the always-running and
under-load replica counts. An always-running count placement cannot meet
is refused with the reason. Headroom is also on the admin API.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 10:19:31 +02:00

208 lines
6.3 KiB
Go

package web
import (
"context"
"errors"
"fmt"
"net/http"
"strconv"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// HeadroomSource is a fleet that can say how many more replicas of a model
// fit now. Optional: without it the card shows what runs and offers no change
// it cannot check.
type HeadroomSource interface {
Headroom(model string, limit int) (adminapi.HeadroomView, bool)
}
// maxReplicas bounds what the form accepts. Not a cluster limit, a typing one:
// 300 where 3 was meant would otherwise be taken as a ceiling.
const maxReplicas = 64
// heldDetail is the card opened beneath a held version: what runs it, where,
// and room to change it.
type heldDetail struct {
Models []heldModel
}
// heldModel is one catalog model running a held version.
type heldModel struct {
modelUse
View adminapi.ModelView
Replicas []heldReplica
Headroom *adminapi.HeadroomView
// MostNow is how many replicas could always run, Current plus the room
// found; zero when the room was not measured or had no end in sight.
MostNow int
}
// heldReplica is one replica, by the nodes it runs on.
type heldReplica struct {
ID string
State string
Restart string
Age int64
Endpoint string
Nodes []heldShard
}
type heldShard struct {
Node string
MemoryGB int // the node's RAM, 0 when unknown
CPU float64
MemBytes int64
}
// openKey names a row the way ?open= does.
func openKey(model, version string) string { return model + ":" + version }
// heldDetailFor reads, for each model using a row, how it runs and how much
// more room it has.
func (p *AccountPages) heldDetailFor(row mirrorRow) *heldDetail {
d := &heldDetail{}
if p.fleet == nil {
for _, u := range row.UsedBy {
d.Models = append(d.Models, heldModel{modelUse: u})
}
return d
}
views := map[string]adminapi.ModelView{}
for _, v := range p.fleet.Models() {
views[v.Name] = v
}
memory := map[string]int{}
for _, n := range p.fleet.Nodes() {
memory[n.ID] = n.MemoryGB
}
hs, _ := p.fleet.(HeadroomSource)
for _, u := range row.UsedBy {
hm := heldModel{modelUse: u, View: views[u.Name]}
for _, rep := range p.fleet.Replicas(u.Name) {
// Built field by field: a ReplicaView carries the engine's key,
// which is for the admin listener and never for a page.
hr := heldReplica{ID: rep.ID, State: rep.State, Restart: rep.Restart,
Age: rep.AgeSec, Endpoint: rep.Endpoint}
for _, sh := range rep.Shards {
hr.Nodes = append(hr.Nodes, heldShard{Node: sh.Node, MemoryGB: memory[sh.Node],
CPU: sh.CPUCores, MemBytes: sh.MemoryBytes})
}
hm.Replicas = append(hm.Replicas, hr)
}
if hs != nil {
if h, ok := hs.Headroom(u.Name, 16); ok {
hm.Headroom = &h
if !h.Limited {
hm.MostNow = h.Current + h.More
}
}
}
d.Models = append(d.Models, hm)
}
return d
}
// setReplicas changes how many replicas a model always runs and how far it
// may grow, refusing an always-on count the cluster has no room for.
//
// Refused rather than taken and left short: a minimum placement cannot meet
// is a model reported as unplaceable on every pass from then on, which is the
// same answer given later and less plainly. A maximum above the room is
// accepted and said: it is reached only under load, by which time something
// may have made room, and refusing it would make the ceiling depend on what
// happened to be running when it was set.
func (p *AccountPages) setReplicas(ctx context.Context, admin store.User, r *http.Request) (string, error) {
name := r.FormValue("model")
lo, err1 := strconv.Atoi(strings.TrimSpace(r.FormValue("min")))
hi, err2 := strconv.Atoi(strings.TrimSpace(r.FormValue("max")))
switch {
case name == "":
return "", errors.New("which model?")
case err1 != nil || err2 != nil:
return "", errors.New("replicas are whole numbers")
case lo < 0 || hi < lo:
return "", fmt.Errorf("always running (%d) cannot be more than the most under load (%d)", lo, hi)
case hi > maxReplicas:
return "", fmt.Errorf("at most %d replicas", maxReplicas)
}
current, err := p.currentModel(ctx, name)
if err != nil {
return "", err
}
if p.catalog.Declared(name) {
return "", fmt.Errorf("%s is set in server.yaml; change it there", name)
}
note := ""
if hs, ok := p.fleet.(HeadroomSource); ok && hi > 0 {
h, ok := hs.Headroom(name, hi)
if !ok {
return "", fmt.Errorf("no model %q", name)
}
fits := h.Current + h.More
if lo > fits && !h.Limited {
why := "nothing more fits"
if len(h.Blocked) > 0 {
why = strings.Join(h.Blocked, "; ")
}
return "", fmt.Errorf("%s can always run at most %d now (%d running, room for %d more): %s",
name, fits, h.Current, h.More, why)
}
if hi > fits && !h.Limited {
note = fmt.Sprintf("; there is room for %d now, so it grows past that only if room is made", fits)
}
}
sc := catalog.Scaling{Min: lo, Max: hi, IdleTimeout: current.Scaling.IdleTimeout}
if sc.OnDemand() && sc.IdleTimeout <= 0 {
sc.IdleTimeout = defaultIdle
}
if _, err := p.catalog.Scale(ctx, name, sc, admin.Email); err != nil {
return "", err
}
switch {
case hi == 0:
return name + " is stopped; its replicas are being taken down", nil
case lo == 0:
return fmt.Sprintf("%s loads when asked for, up to %d under load%s", name, hi, note), nil
case lo == hi:
return fmt.Sprintf("%s runs %s", name, plural(lo, "replica", "replicas")), nil
}
return fmt.Sprintf("%s runs %d, and up to %d under load%s", name, lo, hi, note), nil
}
// currentModel is a model as the catalog holds it now.
func (p *AccountPages) currentModel(ctx context.Context, name string) (catalog.Model, error) {
sources, err := p.catalog.Sources(ctx)
if err != nil {
return catalog.Model{}, err
}
for _, ms := range sources {
for _, m := range ms {
if m.Name == name {
return m, nil
}
}
}
return catalog.Model{}, fmt.Errorf("no model %q runs from the mirror", name)
}
// age renders seconds the way the replicas table does.
func age(sec int64) string {
d := time.Duration(sec) * time.Second
switch {
case d < time.Minute:
return fmt.Sprintf("%ds", sec)
case d < time.Hour:
return fmt.Sprintf("%dm", int(d.Minutes()))
case d < 48*time.Hour:
return fmt.Sprintf("%dh%02dm", int(d.Hours()), int(d.Minutes())%60)
}
return fmt.Sprintf("%dd", int(d.Hours()/24))
}