Ilker ManapandClaude Opus 5.5
0d3eb82b9b
Nodes say where they are with llmgw.io/site, room, row, rack and pdu labels, from agent.yaml or `llmgw ctl label` (kept in the store, over the agent's). Placement spreads a model's replicas across racks and PDUs, keeps a multi-node replica within one domain when its profile says keep_within, places nothing in a domain held with `llmgw ctl facility hold` (optionally draining it), and refuses a shard that would take a domain past its power cap from facility.budgets, counting declared, derived and measured draw. Explain-placement names the domain, and keeps the rejections of a placed replica; `ctl nodes` gains LOCATION and --domain; `ctl facility` lists the domains; the console's Nodes view shows and sets location and holds. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
899 lines
32 KiB
Go
899 lines
32 KiB
Go
package cli
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"net/url"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
tea "github.com/charmbracelet/bubbletea"
|
||
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||
)
|
||
|
||
// What can be done from the console's tables: Enter on a row opens a menu of
|
||
// what applies to it — its details first — and each item is a form, a
|
||
// question, or done at once, through the same admin API `llmgw ctl` uses.
|
||
|
||
type menuItem struct {
|
||
label string
|
||
act func(m *consoleModel) tea.Cmd
|
||
}
|
||
|
||
type menuDialog struct {
|
||
title string
|
||
items []menuItem
|
||
sel int
|
||
}
|
||
|
||
type formField struct {
|
||
label, hint string
|
||
text textField
|
||
options []string
|
||
opt int
|
||
}
|
||
|
||
func (f formField) value() string {
|
||
if f.options != nil {
|
||
return f.options[f.opt]
|
||
}
|
||
return strings.TrimSpace(f.text.Value())
|
||
}
|
||
|
||
type formDialog struct {
|
||
title, note string
|
||
fields []formField
|
||
focus int
|
||
err string
|
||
submit func(m *consoleModel, vals []string) (tea.Cmd, error)
|
||
}
|
||
|
||
type confirmDialog struct {
|
||
title, text string
|
||
yes func(m *consoleModel) tea.Cmd
|
||
}
|
||
|
||
// actionDoneMsg is an action's outcome, shown as a dialog.
|
||
type actionDoneMsg struct {
|
||
title, text string
|
||
err error
|
||
}
|
||
|
||
func textF(label, value, hint string) formField {
|
||
t := newTextField()
|
||
t.Width = 30
|
||
t.SetValue(value)
|
||
return formField{label: label, hint: hint, text: t}
|
||
}
|
||
|
||
func choiceF(label string, options []string, current string) formField {
|
||
f := formField{label: label, options: options}
|
||
for i, o := range options {
|
||
if o == current {
|
||
f.opt = i
|
||
}
|
||
}
|
||
return f
|
||
}
|
||
|
||
func (m *consoleModel) openForm(f *formDialog) {
|
||
if len(f.fields) > 0 {
|
||
f.fields[0].text.Focus()
|
||
}
|
||
m.formD = f
|
||
}
|
||
|
||
// call runs one admin API request in the background and reports it.
|
||
func (m *consoleModel) call(title, method, path string, body any, say func(raw json.RawMessage) string) tea.Cmd {
|
||
m.busy = title
|
||
c := m.client
|
||
return func() tea.Msg {
|
||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||
defer cancel()
|
||
var raw json.RawMessage
|
||
// A change may answer with no body at all.
|
||
if err := c.do(ctx, method, path, body, &raw); err != nil && !errors.Is(err, io.EOF) {
|
||
return actionDoneMsg{title: title, err: err}
|
||
}
|
||
text := "Done."
|
||
if say != nil {
|
||
text = say(raw)
|
||
}
|
||
return actionDoneMsg{title: title, text: text}
|
||
}
|
||
}
|
||
|
||
func (m consoleModel) actionKey(k tea.KeyMsg) (tea.Model, tea.Cmd, bool) {
|
||
switch {
|
||
case m.formD != nil:
|
||
f := m.formD
|
||
switch k.Type {
|
||
case tea.KeyEsc:
|
||
m.formD = nil
|
||
case tea.KeyTab, tea.KeyDown:
|
||
// From Apply, which is past the last field, round to the first.
|
||
if f.focus < len(f.fields) {
|
||
f.fields[f.focus].text.Blur()
|
||
}
|
||
f.focus = (f.focus + 1) % (len(f.fields) + 1)
|
||
if f.focus < len(f.fields) {
|
||
f.fields[f.focus].text.Focus()
|
||
}
|
||
case tea.KeyShiftTab, tea.KeyUp:
|
||
if f.focus < len(f.fields) {
|
||
f.fields[f.focus].text.Blur()
|
||
}
|
||
f.focus = (f.focus + len(f.fields)) % (len(f.fields) + 1)
|
||
if f.focus < len(f.fields) {
|
||
f.fields[f.focus].text.Focus()
|
||
}
|
||
case tea.KeyLeft, tea.KeyRight:
|
||
if f.focus < len(f.fields) && f.fields[f.focus].options != nil {
|
||
fl := &f.fields[f.focus]
|
||
d := 1
|
||
if k.Type == tea.KeyLeft {
|
||
d = len(fl.options) - 1
|
||
}
|
||
fl.opt = (fl.opt + d) % len(fl.options)
|
||
}
|
||
case tea.KeyEnter:
|
||
// A form of one field is applied by Enter on it.
|
||
if f.focus < len(f.fields) && len(f.fields) > 1 {
|
||
f.fields[f.focus].text.Blur()
|
||
f.focus++
|
||
if f.focus < len(f.fields) {
|
||
f.fields[f.focus].text.Focus()
|
||
}
|
||
return m, nil, true
|
||
}
|
||
vals := make([]string, len(f.fields))
|
||
for i, fl := range f.fields {
|
||
vals[i] = fl.value()
|
||
}
|
||
cmd, err := f.submit(&m, vals)
|
||
if err != nil {
|
||
f.err = err.Error()
|
||
return m, nil, true
|
||
}
|
||
m.formD = nil
|
||
return m, cmd, true
|
||
default:
|
||
if f.focus < len(f.fields) && f.fields[f.focus].options == nil {
|
||
var cmd tea.Cmd
|
||
f.fields[f.focus].text, cmd = f.fields[f.focus].text.Update(k)
|
||
return m, cmd, true
|
||
}
|
||
}
|
||
return m, nil, true
|
||
case m.confirmD != nil:
|
||
c := m.confirmD
|
||
m.confirmD = nil
|
||
if strings.EqualFold(k.String(), "y") {
|
||
return m, c.yes(&m), true
|
||
}
|
||
return m, nil, true
|
||
case m.menuD != nil:
|
||
d := m.menuD
|
||
switch k.String() {
|
||
case "up", "k":
|
||
d.sel = (d.sel - 1 + len(d.items)) % len(d.items)
|
||
case "down", "j", "tab":
|
||
d.sel = (d.sel + 1) % len(d.items)
|
||
case "esc", "q":
|
||
m.menuD = nil
|
||
case "enter":
|
||
m.menuD = nil
|
||
return m, d.items[d.sel].act(&m), true
|
||
}
|
||
return m, nil, true
|
||
}
|
||
return m, nil, false
|
||
}
|
||
|
||
// actionDialog draws whichever of the three is open, or "".
|
||
func (m consoleModel) actionDialog() (title, body string) {
|
||
switch {
|
||
case m.formD != nil:
|
||
f := m.formD
|
||
var b strings.Builder
|
||
if f.note != "" {
|
||
b.WriteString(f.note + "\n\n")
|
||
}
|
||
w := 0
|
||
for _, fl := range f.fields {
|
||
w = max(w, len(fl.label))
|
||
}
|
||
for i, fl := range f.fields {
|
||
label := fmt.Sprintf("%-*s", w, fl.label)
|
||
if i == f.focus {
|
||
label = fSelected.Render(label)
|
||
}
|
||
v := fl.text.View()
|
||
if fl.options != nil && len(fl.options) > 3 {
|
||
v = fmt.Sprintf("◀ %s ▶ %s", sOK.Render(fl.options[fl.opt]), sDim.Render(fmt.Sprintf("%d of %d", fl.opt+1, len(fl.options))))
|
||
} else if fl.options != nil {
|
||
var opts []string
|
||
for j, o := range fl.options {
|
||
if j == fl.opt {
|
||
opts = append(opts, "("+sOK.Render("•")+") "+o)
|
||
} else {
|
||
opts = append(opts, "( ) "+sDim.Render(o))
|
||
}
|
||
}
|
||
v = strings.Join(opts, " ")
|
||
}
|
||
fmt.Fprintf(&b, "%s %s\n", label, v)
|
||
if fl.hint != "" {
|
||
fmt.Fprintf(&b, "%s %s\n", strings.Repeat(" ", w), sDim.Render(fl.hint))
|
||
}
|
||
}
|
||
b.WriteString("\n" + button(f.focus == len(f.fields), "Apply") + "\n")
|
||
if f.err != "" {
|
||
b.WriteString("\n" + sBad.Render(f.err) + "\n")
|
||
}
|
||
b.WriteString("\n" + sDim.Render("Tab next · ←→ choose · Enter on Apply · Esc cancel"))
|
||
return f.title, b.String()
|
||
case m.confirmD != nil:
|
||
return m.confirmD.title, m.confirmD.text + "\n\n" + sDim.Render("Y yes · any other key no")
|
||
case m.menuD != nil:
|
||
var b strings.Builder
|
||
// A long menu shows the part around the choice.
|
||
items, first := m.menuD.items, 0
|
||
if room := max(5, m.height-12); len(items) > room {
|
||
first = min(max(0, m.menuD.sel-room/2), len(items)-room)
|
||
items = items[first : first+room]
|
||
if first > 0 {
|
||
b.WriteString(sDim.Render(" ↑ more") + "\n")
|
||
}
|
||
}
|
||
for j, it := range items {
|
||
i := first + j
|
||
if i == m.menuD.sel {
|
||
b.WriteString(fSelected.Render("› "+it.label) + "\n")
|
||
} else {
|
||
b.WriteString(" " + it.label + "\n")
|
||
}
|
||
}
|
||
if first+len(items) < len(m.menuD.items) {
|
||
b.WriteString(sDim.Render(" ↓ more") + "\n")
|
||
}
|
||
b.WriteString("\n" + sDim.Render("↑↓ choose · Enter · Esc close"))
|
||
return m.menuD.title, b.String()
|
||
}
|
||
return "", ""
|
||
}
|
||
|
||
func details(m *consoleModel) tea.Cmd {
|
||
m.detail, m.scroll = true, 0
|
||
return m.fetchDetail()
|
||
}
|
||
|
||
// --- models ---
|
||
|
||
func (m consoleModel) modelMenu(md adminapi.ModelView) *menuDialog {
|
||
path := "/api/v1/models/" + url.PathEscape(md.Name)
|
||
items := []menuItem{
|
||
{"Details", details},
|
||
{"Replicas (scale)…", func(m *consoleModel) tea.Cmd {
|
||
m.openForm(&formDialog{title: "Scale " + md.Name,
|
||
note: "How many replicas it keeps. Max 0 stops it; min 0 with max above loads it on demand.",
|
||
fields: []formField{
|
||
textF("min", strconv.Itoa(md.Min), "replicas kept running"),
|
||
textF("max", strconv.Itoa(md.Max), "the most it scales to"),
|
||
textF("idle timeout", "", "on demand only: unload after this long unused, like 30m"),
|
||
},
|
||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||
minR, err1 := strconv.Atoi(v[0])
|
||
maxR, err2 := strconv.Atoi(v[1])
|
||
if err1 != nil || err2 != nil || minR < 0 || maxR < minR {
|
||
return nil, fmt.Errorf("min and max are whole numbers, max at least min")
|
||
}
|
||
req := adminapi.ScaleRequest{Min: minR, Max: maxR}
|
||
if v[2] != "" {
|
||
d, err := time.ParseDuration(v[2])
|
||
if err != nil {
|
||
return nil, fmt.Errorf("the idle timeout is a duration, like 30m")
|
||
}
|
||
req.IdleTimeout = d
|
||
}
|
||
return m.call("Scale "+md.Name, http.MethodPost, path+"/scale", req,
|
||
func(json.RawMessage) string { return fmt.Sprintf("%s keeps %d to %d replicas.", md.Name, minR, maxR) }), nil
|
||
}})
|
||
return nil
|
||
}},
|
||
{"Class…", func(m *consoleModel) tea.Cmd {
|
||
m.openForm(&formDialog{title: "Class of " + md.Name,
|
||
note: "How much its capacity is worth against another's when there is not enough for both.",
|
||
fields: []formField{choiceF("class", []string{"critical", "standard", "opportunistic"}, md.Class)},
|
||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||
return m.call("Class of "+md.Name, http.MethodPost, path+"/class", map[string]string{"class": v[0]},
|
||
func(json.RawMessage) string { return md.Name + " is " + v[0] + "." }), nil
|
||
}})
|
||
return nil
|
||
}},
|
||
{"Owner…", func(m *consoleModel) tea.Cmd {
|
||
m.openForm(&formDialog{title: "Owner of " + md.Name,
|
||
note: "Whose capacity quota it counts against. Empty: the cluster's, nobody's.",
|
||
fields: []formField{textF("owner", md.Owner, "org:ID, team:ID or project:ID")},
|
||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||
return m.call("Owner of "+md.Name, http.MethodPost, path+"/owner", map[string]string{"owner": v[0]}, nil), nil
|
||
}})
|
||
return nil
|
||
}},
|
||
{"Resources…", func(m *consoleModel) tea.Cmd {
|
||
profiles := md.Profiles
|
||
if len(profiles) == 0 {
|
||
profiles = []string{""}
|
||
}
|
||
m.openForm(&formDialog{title: "Resources of " + md.Name,
|
||
note: "What one replica reserves of its node.",
|
||
fields: []formField{
|
||
choiceF("profile", profiles, profiles[0]),
|
||
textF("memory GB", strconv.Itoa(md.MemoryGB), fmt.Sprintf("measured: %.1f GB", md.MeasuredMemoryGB)),
|
||
textF("CPU cores", strconv.Itoa(md.CPUCores), "for a profile that computes on them"),
|
||
},
|
||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||
mem, err1 := strconv.Atoi(v[1])
|
||
cores, err2 := strconv.Atoi(v[2])
|
||
if err1 != nil || err2 != nil || mem < 0 || cores < 0 {
|
||
return nil, fmt.Errorf("memory and cores are whole numbers")
|
||
}
|
||
return m.call("Resources of "+md.Name, http.MethodPost, path+"/resources",
|
||
adminapi.ResourcesRequest{Profile: v[0], MemoryGB: mem, CPUCores: cores}, nil), nil
|
||
}})
|
||
return nil
|
||
}},
|
||
{"LoRA adapters…", func(m *consoleModel) tea.Cmd {
|
||
var cur []string
|
||
for _, a := range md.Adapters {
|
||
s := a.Name + "=" + a.Source
|
||
if a.Scale != 0 {
|
||
s += "@" + strconv.FormatFloat(a.Scale, 'f', -1, 64)
|
||
}
|
||
cur = append(cur, s)
|
||
}
|
||
m.openForm(&formDialog{title: "Adapters of " + md.Name,
|
||
note: "Every replica loads every adapter; a request names one as its model.",
|
||
fields: []formField{textF("adapters", strings.Join(cur, ", "), "name=source[@scale], separated by commas; empty removes all")},
|
||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||
var next []catalog.Adapter
|
||
for _, part := range strings.Split(v[0], ",") {
|
||
if part = strings.TrimSpace(part); part == "" {
|
||
continue
|
||
}
|
||
name, rest, ok := strings.Cut(part, "=")
|
||
if !ok {
|
||
return nil, fmt.Errorf("%q is not name=source", part)
|
||
}
|
||
a := catalog.Adapter{Name: strings.TrimSpace(name), Source: strings.TrimSpace(rest)}
|
||
if src, sc, ok := strings.Cut(a.Source, "@"); ok {
|
||
f, err := strconv.ParseFloat(sc, 64)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("%q: the scale is a number", part)
|
||
}
|
||
a.Source, a.Scale = src, f
|
||
}
|
||
next = append(next, a)
|
||
}
|
||
return m.call("Adapters of "+md.Name, http.MethodPost, path+"/adapters",
|
||
adminapi.AdaptersRequest{Adapters: next}, nil), nil
|
||
}})
|
||
return nil
|
||
}},
|
||
{"Restart its replicas…", func(m *consoleModel) tea.Cmd {
|
||
m.openForm(&formDialog{title: "Restart " + md.Name,
|
||
note: "Each replica is replaced: a new one started before the old is stopped, unless in place.",
|
||
fields: []formField{choiceF("how", []string{"one by one", "in place"}, "one by one")},
|
||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||
return m.call("Restart "+md.Name, http.MethodPost, "/api/v1/restart",
|
||
adminapi.RestartRequest{Models: []string{md.Name}, InPlace: v[0] == "in place"}, nil), nil
|
||
}})
|
||
return nil
|
||
}},
|
||
{"Why is it placed where it is", func(m *consoleModel) tea.Cmd {
|
||
return m.call("Placement of "+md.Name, http.MethodGet, "/api/v1/explain-placement?model="+url.QueryEscape(md.Name), nil,
|
||
func(raw json.RawMessage) string {
|
||
var e adminapi.ExplainView
|
||
if json.Unmarshal(raw, &e) != nil {
|
||
return string(raw)
|
||
}
|
||
var b strings.Builder
|
||
if e.Placed {
|
||
b.WriteString("Placed.\n")
|
||
}
|
||
for _, r := range e.Rejections {
|
||
fmt.Fprintf(&b, "%s on %s %s: %s\n", r.Profile, r.Pool, r.Node, r.Reason)
|
||
}
|
||
if b.Len() == 0 {
|
||
return "Nothing to explain: it has what it asked for."
|
||
}
|
||
return b.String()
|
||
})
|
||
}},
|
||
}
|
||
items = append(items, m.updateItems(md)...)
|
||
items = append(items, m.evalItems(md)...)
|
||
if md.Parked {
|
||
items = append(items, menuItem{"Try again (it is parked)", func(m *consoleModel) tea.Cmd {
|
||
return m.call("Retry "+md.Name, http.MethodPost, path+"/retry", map[string]string{}, nil)
|
||
}})
|
||
}
|
||
items = append(items, menuItem{"Remove…", func(m *consoleModel) tea.Cmd {
|
||
m.confirmD = &confirmDialog{title: "Remove " + md.Name,
|
||
text: "Its replicas stop and it leaves the catalog. A model from server.yaml cannot be removed here.",
|
||
yes: func(m *consoleModel) tea.Cmd {
|
||
return m.call("Remove "+md.Name, http.MethodDelete, path, nil, nil)
|
||
}}
|
||
return nil
|
||
}})
|
||
return &menuDialog{title: md.Name, items: items}
|
||
}
|
||
|
||
// --- nodes ---
|
||
|
||
func (m consoleModel) nodeMenu(n adminapi.NodeView) *menuDialog {
|
||
path := "/api/v1/nodes/" + url.PathEscape(n.ID)
|
||
state := func(s, label, title string) menuItem {
|
||
return menuItem{label, func(m *consoleModel) tea.Cmd {
|
||
return m.call(title+" "+n.ID, http.MethodPost, path+"/state", map[string]string{"state": s},
|
||
func(json.RawMessage) string { return n.ID + " is " + s + "." })
|
||
}}
|
||
}
|
||
items := []menuItem{{"Details", details}}
|
||
if n.State == "cordoned" || n.State == "draining" {
|
||
items = append(items, state("ready", "Uncordon", "Uncordon"))
|
||
} else {
|
||
items = append(items, state("cordoned", "Cordon: place nothing new here", "Cordon"))
|
||
items = append(items, menuItem{"Drain…", func(m *consoleModel) tea.Cmd {
|
||
m.confirmD = &confirmDialog{title: "Drain " + n.ID,
|
||
text: "Its replicas move off one at a time, each replacement up before the original stops.",
|
||
yes: func(m *consoleModel) tea.Cmd {
|
||
return m.call("Drain "+n.ID, http.MethodPost, path+"/state", map[string]string{"state": "draining"}, nil)
|
||
}}
|
||
return nil
|
||
}})
|
||
}
|
||
items = append(items, m.locationItems(n)...)
|
||
if n.GPUTotal > 0 {
|
||
for _, engine := range []string{"vllm", "sglang"} {
|
||
items = append(items, menuItem{engine + " build…", func(m *consoleModel) tea.Cmd {
|
||
m.openForm(&formDialog{title: engine + " on " + n.ID,
|
||
note: "Which build of " + engine + " this node installs: auto follows its hardware, none installs none.",
|
||
fields: []formField{choiceF("build", control.EngineBuilds(engine), n.EngineBuilds[engine])},
|
||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||
return m.call(engine+" on "+n.ID, http.MethodPost, path+"/engines",
|
||
adminapi.NodeEngineRequest{Engine: engine, Build: v[0]}, nil), nil
|
||
}})
|
||
return nil
|
||
}})
|
||
}
|
||
}
|
||
items = append(items,
|
||
menuItem{"Upgrade its agent…", func(m *consoleModel) tea.Cmd {
|
||
m.openForm(&formDialog{title: "Upgrade " + n.ID,
|
||
note: "It fetches the release for its platform, checks it, swaps it in and restarts; engines keep serving.",
|
||
fields: []formField{textF("version", version.Version, "a release, like v0.11.1")},
|
||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||
return m.call("Upgrade "+n.ID, http.MethodPost, path+"/upgrade", adminapi.UpgradeRequest{Version: v[0]}, nil), nil
|
||
}})
|
||
return nil
|
||
}},
|
||
menuItem{"Delete model files nothing uses", func(m *consoleModel) tea.Cmd {
|
||
return m.call("Unused files on "+n.ID, http.MethodPost, "/api/v1/weights/collect", map[string]string{"node": n.ID},
|
||
func(raw json.RawMessage) string {
|
||
var rs []struct {
|
||
Files int `json:"files"`
|
||
Bytes int64 `json:"bytes"`
|
||
}
|
||
_ = json.Unmarshal(raw, &rs)
|
||
var files int
|
||
var b int64
|
||
for _, r := range rs {
|
||
files, b = files+r.Files, b+r.Bytes
|
||
}
|
||
return fmt.Sprintf("%d files, %.1f GB freed.", files, float64(b)/(1<<30))
|
||
})
|
||
}},
|
||
menuItem{"Remove from the cluster…", func(m *consoleModel) tea.Cmd {
|
||
m.confirmD = &confirmDialog{title: "Remove " + n.ID,
|
||
text: "The control plane forgets it. Its replicas are placed elsewhere; the machine is not touched.",
|
||
yes: func(m *consoleModel) tea.Cmd {
|
||
return m.call("Remove "+n.ID, http.MethodDelete, path, nil, nil)
|
||
}}
|
||
return nil
|
||
}},
|
||
)
|
||
return &menuDialog{title: n.ID, items: items}
|
||
}
|
||
|
||
// --- replicas ---
|
||
|
||
func (m consoleModel) replicaMenu(r adminapi.ReplicaView) *menuDialog {
|
||
return &menuDialog{title: r.ID, items: []menuItem{
|
||
{"Restart it…", func(m *consoleModel) tea.Cmd {
|
||
m.openForm(&formDialog{title: "Restart " + r.ID,
|
||
fields: []formField{choiceF("how", []string{"replace", "in place"}, "replace")},
|
||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||
return m.call("Restart "+r.ID, http.MethodPost, "/api/v1/restart",
|
||
adminapi.RestartRequest{Replicas: []string{r.ID}, InPlace: v[0] == "in place"}, nil), nil
|
||
}})
|
||
return nil
|
||
}},
|
||
}}
|
||
}
|
||
|
||
// --- capacity quotas ---
|
||
|
||
func intPtr(s string) (*int, error) {
|
||
s = strings.TrimSpace(s)
|
||
if s == "" {
|
||
return nil, nil
|
||
}
|
||
n, err := strconv.Atoi(s)
|
||
if err != nil || n < 0 {
|
||
return nil, fmt.Errorf("%q is not a whole number; leave it empty for none", s)
|
||
}
|
||
return &n, nil
|
||
}
|
||
|
||
func ptrText(p *int) string {
|
||
if p == nil {
|
||
return ""
|
||
}
|
||
return strconv.Itoa(*p)
|
||
}
|
||
|
||
func (m *consoleModel) quotaForm(q adminapi.QuotaView) {
|
||
scope := textF("scope", q.Scope, "org:ID, team:ID or project:ID")
|
||
m.openForm(&formDialog{title: "Capacity quota",
|
||
note: "Guaranteed is never taken for someone else's model; at most is the ceiling. Empty is not governed.",
|
||
fields: []formField{scope,
|
||
textF("guaranteed GPUs", ptrText(q.Guarantee.GPUs), ""),
|
||
textF("guaranteed cores", ptrText(q.Guarantee.CPUCores), ""),
|
||
textF("guaranteed memory GB", ptrText(q.Guarantee.MemoryGB), ""),
|
||
textF("at most GPUs", ptrText(q.Limit.GPUs), ""),
|
||
textF("at most cores", ptrText(q.Limit.CPUCores), ""),
|
||
textF("at most memory GB", ptrText(q.Limit.MemoryGB), ""),
|
||
},
|
||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||
req := adminapi.QuotaRequest{Scope: v[0]}
|
||
dst := []**int{&req.Guarantee.GPUs, &req.Guarantee.CPUCores, &req.Guarantee.MemoryGB,
|
||
&req.Limit.GPUs, &req.Limit.CPUCores, &req.Limit.MemoryGB}
|
||
for i, d := range dst {
|
||
p, err := intPtr(v[i+1])
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
*d = p
|
||
}
|
||
return m.call("Quota for "+v[0], http.MethodPut, "/api/v1/quotas", req, nil), nil
|
||
}})
|
||
}
|
||
|
||
func (m consoleModel) quotaMenu(q adminapi.QuotaView) *menuDialog {
|
||
return &menuDialog{title: q.Scope, items: []menuItem{
|
||
{"Change…", func(m *consoleModel) tea.Cmd { m.quotaForm(q); return nil }},
|
||
{"Remove…", func(m *consoleModel) tea.Cmd {
|
||
m.confirmD = &confirmDialog{title: "Remove the quota on " + q.Scope,
|
||
text: "Its models then share the cluster by their class alone.",
|
||
yes: func(m *consoleModel) tea.Cmd {
|
||
return m.call("Quota on "+q.Scope, http.MethodDelete, "/api/v1/quotas/"+url.PathEscape(q.Scope), nil, nil)
|
||
}}
|
||
return nil
|
||
}},
|
||
}}
|
||
}
|
||
|
||
type consoleQuotasMsg struct {
|
||
rows []adminapi.QuotaView
|
||
err error
|
||
}
|
||
|
||
func (m consoleModel) fetchQuotas() tea.Cmd {
|
||
c := m.client
|
||
return func() tea.Msg {
|
||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||
defer cancel()
|
||
var rows []adminapi.QuotaView
|
||
err := c.get(ctx, "/api/v1/quotas", &rows)
|
||
return consoleQuotasMsg{rows, err}
|
||
}
|
||
}
|
||
|
||
func (m consoleModel) quotasPanel(pw int) (string, string, string) {
|
||
keys := "→ or Enter open · ↑↓ choose · Q leave"
|
||
if m.inPanel {
|
||
keys = "↑↓ row · Enter actions · A add · ← back · Q leave"
|
||
}
|
||
head := "\n " + sDim.Render("Capacity each org, team or project is guaranteed and may at most hold. A to add one.") + "\n\n"
|
||
if m.quotasErr != nil {
|
||
return "Quotas", head + " " + sBad.Render(m.quotasErr.Error()), keys
|
||
}
|
||
cell := func(p *int) string {
|
||
if p == nil {
|
||
return "—"
|
||
}
|
||
return strconv.Itoa(*p)
|
||
}
|
||
var rows [][]string
|
||
for _, q := range m.quotas {
|
||
name := q.Name
|
||
if name == "" {
|
||
name = q.Scope
|
||
}
|
||
rows = append(rows, []string{name,
|
||
cell(q.Guarantee.GPUs) + " / " + cell(q.Limit.GPUs),
|
||
cell(q.Guarantee.CPUCores) + " / " + cell(q.Limit.CPUCores),
|
||
cell(q.Guarantee.MemoryGB) + " / " + cell(q.Limit.MemoryGB),
|
||
fmt.Sprintf("%d GPU, %d cores, %d GB", q.Holds.GPUs, q.Holds.CPUCores, q.Holds.MemoryGB)})
|
||
}
|
||
if len(rows) == 0 {
|
||
return "Quotas", head + " " + sDim.Render("None: models share the cluster by their class alone."), keys
|
||
}
|
||
return "Quotas", head + table([]string{"WHO", "GPUS (GUAR / MAX)", "CORES", "MEMORY GB", "HOLDS"}, rows, m.row, pw, m.inPanel), keys
|
||
}
|
||
|
||
// --- the cluster ---
|
||
|
||
// runAfter closes the screen and runs llmgw with these arguments, showing its
|
||
// output; the console comes back after.
|
||
func (m *consoleModel) runAfter(args ...string) tea.Cmd {
|
||
m.action = consoleAction{kind: "run", targets: append([]string{}, args...)}
|
||
return tea.Quit
|
||
}
|
||
|
||
func (m *consoleModel) doEntry() tea.Cmd {
|
||
server := "--server=" + m.client.base
|
||
switch m.menu {
|
||
case cUpgrade:
|
||
m.openForm(&formDialog{title: "Upgrade the cluster",
|
||
note: "This machine runs " + version.Version + ".",
|
||
fields: []formField{textF("version", "", "like v0.11.2"), choiceF("this machine too", []string{"yes", "no, nodes only"}, "yes")},
|
||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||
if !release.Tag.MatchString(v[0]) {
|
||
return nil, fmt.Errorf("%q is not a release version (v1.2.3)", v[0])
|
||
}
|
||
args := []string{"ctl", server, "upgrade", v[0]}
|
||
if v[1] != "yes" {
|
||
args = append(args, "--skip-control-plane")
|
||
}
|
||
return m.runAfter(args...), nil
|
||
}})
|
||
case cRestartAll:
|
||
m.confirmD = &confirmDialog{title: "Restart every replica",
|
||
text: "Each is replaced in turn, the new one up before the old one stops.",
|
||
yes: func(m *consoleModel) tea.Cmd {
|
||
return m.call("Restart every replica", http.MethodPost, "/api/v1/restart", adminapi.RestartRequest{All: true}, nil)
|
||
}}
|
||
case cBackup:
|
||
return m.runAfter("server", "backup", "--config="+m.file)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// canaryItems try a model's newer weights on a share of its requests: start
|
||
// one, change its share, promote it, or take it out.
|
||
func (m consoleModel) canaryItems(md adminapi.ModelView) []menuItem {
|
||
path := "/api/v1/models/" + url.PathEscape(md.Name)
|
||
result := func(raw json.RawMessage) string {
|
||
var out map[string]string
|
||
_ = json.Unmarshal(raw, &out)
|
||
return out["result"] + "."
|
||
}
|
||
if cn := md.Canary; cn != nil {
|
||
return []menuItem{
|
||
{fmt.Sprintf("Canary's share (%d%%)…", cn.Percent), func(m *consoleModel) tea.Cmd {
|
||
m.openForm(&formDialog{title: "Share of " + md.Name + " for " + cn.Model,
|
||
fields: []formField{textF("percent", strconv.Itoa(cn.Percent), "0 to 100; 0 keeps it running and gives it none")},
|
||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||
n, err := strconv.Atoi(v[0])
|
||
if err != nil || n < 0 || n > 100 {
|
||
return nil, fmt.Errorf("a whole number from 0 to 100")
|
||
}
|
||
return m.call("Canary of "+md.Name, http.MethodPut, path+"/canary", map[string]int{"percent": n}, result), nil
|
||
}})
|
||
return nil
|
||
}},
|
||
{"Promote the canary…", func(m *consoleModel) tea.Cmd {
|
||
m.confirmD = &confirmDialog{title: "Promote " + cn.Model,
|
||
text: md.Name + " moves to the canary's weights: its replicas are replaced one at a time,\n" +
|
||
"and the canary is removed. Going back is \"Go back to\" on this menu." + gateNote(md),
|
||
yes: func(m *consoleModel) tea.Cmd {
|
||
return m.call("Promote "+cn.Model, http.MethodPost, path+"/promote", map[string]string{}, startedSays)
|
||
}}
|
||
return nil
|
||
}},
|
||
{"Take the canary out…", func(m *consoleModel) tea.Cmd {
|
||
m.confirmD = &confirmDialog{title: "Take " + cn.Model + " out",
|
||
text: md.Name + " serves all of its requests again, and the canary is removed.",
|
||
yes: func(m *consoleModel) tea.Cmd {
|
||
return m.call("Canary of "+md.Name, http.MethodDelete, path+"/canary", nil, result)
|
||
}}
|
||
return nil
|
||
}},
|
||
}
|
||
}
|
||
if u := md.Update; u == nil || !u.Available || md.CanaryOf != "" {
|
||
return nil
|
||
}
|
||
return []menuItem{{"Try the new weights as a canary…", func(m *consoleModel) tea.Cmd {
|
||
m.openForm(&formDialog{title: "Canary of " + md.Name,
|
||
note: "The new weights run beside it, one replica, given this share of its requests.\n" +
|
||
"Taken out by itself if it errs or answers more slowly than " + md.Name + ".",
|
||
fields: []formField{textF("percent", "10", "share of its requests, 0 to 100")},
|
||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||
n, err := strconv.Atoi(v[0])
|
||
if err != nil || n < 0 || n > 100 {
|
||
return nil, fmt.Errorf("a whole number from 0 to 100")
|
||
}
|
||
return m.call("Canary of "+md.Name, http.MethodPost, path+"/canary", catalog.Canary{Percent: n}, startedSays), nil
|
||
}})
|
||
return nil
|
||
}}}
|
||
}
|
||
|
||
// updateItems are what can be done about a model's repository publishing new
|
||
// weights: move to them, ask now, or go back.
|
||
func (m consoleModel) updateItems(md adminapi.ModelView) []menuItem {
|
||
origin := menuItem{"Where its weights came from…", func(m *consoleModel) tea.Cmd {
|
||
repo := ""
|
||
if md.Origin != nil {
|
||
repo = md.Origin.Repo
|
||
}
|
||
m.openForm(&formDialog{title: "Origin of " + md.Name,
|
||
note: "The registry repository its files were fetched from. Checked: it must have the\n" +
|
||
"commit the mirror's directory is named after, with the same files at the same sizes.",
|
||
fields: []formField{textF("repository", repo, "owner/name, like bartowski/phi-4-GGUF")},
|
||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||
if !strings.Contains(v[0], "/") {
|
||
return nil, fmt.Errorf("owner/name")
|
||
}
|
||
return m.call("Origin of "+md.Name, http.MethodPost, "/api/v1/models/"+url.PathEscape(md.Name)+"/origin",
|
||
map[string]string{"repo": v[0]}, func(raw json.RawMessage) string {
|
||
var o adminapi.OriginView
|
||
_ = json.Unmarshal(raw, &o)
|
||
return fmt.Sprintf("%s came from %s at %s; its updates are watched.", md.Name, o.Repo,
|
||
catalog.ShortRevision(o.Revision))
|
||
}), nil
|
||
}})
|
||
return nil
|
||
}}
|
||
if md.Origin == nil {
|
||
if strings.Contains(md.Source, "://") {
|
||
return []menuItem{origin}
|
||
}
|
||
return nil
|
||
}
|
||
var items []menuItem
|
||
if u := md.Update; u != nil && u.Available {
|
||
items = append(items, menuItem{fmt.Sprintf("Update to %s (%s)…", catalog.ShortRevision(u.Latest), bytesShort(u.Bytes)),
|
||
func(m *consoleModel) tea.Cmd {
|
||
m.confirmD = &confirmDialog{title: "Update " + md.Name,
|
||
text: fmt.Sprintf("%s has new weights at %s: %s.\n\n"+
|
||
"They are fetched into the mirror beside the ones in use and checked, then the\n"+
|
||
"replicas are replaced one at a time. The old files stay, to go back to.%s",
|
||
u.Repo, catalog.ShortRevision(u.Latest), strings.Join(u.Changed, ", "), gateNote(md)),
|
||
yes: func(m *consoleModel) tea.Cmd {
|
||
return m.call("Update "+md.Name, http.MethodPost, "/api/v1/models/"+url.PathEscape(md.Name)+"/update",
|
||
map[string]string{}, startedSays)
|
||
}}
|
||
return nil
|
||
}})
|
||
}
|
||
items = append(items, m.canaryItems(md)...)
|
||
items = append(items, menuItem{"Ask the registry for new weights now", func(m *consoleModel) tea.Cmd {
|
||
return m.call("New weights for "+md.Name, http.MethodPost, "/api/v1/updates/check", map[string]string{"model": md.Name},
|
||
func(raw json.RawMessage) string {
|
||
var us []modelserve.UpdateInfo
|
||
if json.Unmarshal(raw, &us) != nil || len(us) == 0 {
|
||
return string(raw)
|
||
}
|
||
u := us[0]
|
||
if u.Available {
|
||
return fmt.Sprintf("%s has new weights at %s: %s, %s.", u.Repo, catalog.ShortRevision(u.Latest),
|
||
strings.Join(u.Changed, ", "), bytesShort(u.Bytes))
|
||
}
|
||
return fmt.Sprintf("%s at %s: %s", u.Repo, catalog.ShortRevision(u.Latest), updateCell(u))
|
||
})
|
||
}})
|
||
items = append(items, origin)
|
||
if prev := md.Origin.Previous; prev != "" {
|
||
items = append(items, menuItem{"Go back to " + catalog.ShortRevision(prev) + "…", func(m *consoleModel) tea.Cmd {
|
||
m.confirmD = &confirmDialog{title: "Go back on " + md.Name,
|
||
text: "It runs the weights its last update replaced, " + catalog.ShortRevision(prev) +
|
||
", if they are still on the mirror;\nits replicas are replaced one at a time.",
|
||
yes: func(m *consoleModel) tea.Cmd {
|
||
return m.call("Go back on "+md.Name, http.MethodPost, "/api/v1/models/"+url.PathEscape(md.Name)+"/rollback",
|
||
map[string]string{}, func(raw json.RawMessage) string {
|
||
var out map[string]string
|
||
_ = json.Unmarshal(raw, &out)
|
||
return out["result"] + "."
|
||
})
|
||
}}
|
||
return nil
|
||
}})
|
||
}
|
||
return items
|
||
}
|
||
|
||
// locationItems are the node menu's facility entries: where the node is,
|
||
// and holding or releasing its rack.
|
||
func (m consoleModel) locationItems(n adminapi.NodeView) []menuItem {
|
||
path := "/api/v1/nodes/" + url.PathEscape(n.ID) + "/labels"
|
||
var loc facility.Location
|
||
if n.Location != nil {
|
||
loc = *n.Location
|
||
}
|
||
items := []menuItem{{"Location…", func(m *consoleModel) tea.Cmd {
|
||
m.openForm(&formDialog{title: "Where " + n.ID + " is",
|
||
note: "Placement spreads a model's replicas across racks and PDUs, and keeps out of held ones. " +
|
||
"Empty leaves a level unknown; a PDU list is comma separated, for a dual-fed rack.",
|
||
fields: []formField{
|
||
textF("site", loc.Site, "sto1"), textF("room", loc.Room, "hall-b"),
|
||
textF("row", loc.Row, "12"), textF("rack", loc.Rack, "r12-07"),
|
||
textF("pdu", strings.Join(loc.PDUs, ","), "r12-07-a,r12-07-b"),
|
||
},
|
||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||
req := locationRequest(n, v)
|
||
if len(req.Set) == 0 && len(req.Remove) == 0 {
|
||
return nil, fmt.Errorf("nothing changed")
|
||
}
|
||
return m.call("Location of "+n.ID, http.MethodPost, path, req,
|
||
func(json.RawMessage) string { return n.ID + "'s location is set." }), nil
|
||
}})
|
||
return nil
|
||
}}}
|
||
switch {
|
||
case n.HeldDomain != "":
|
||
d := n.HeldDomain
|
||
items = append(items, menuItem{"Release " + d, func(m *consoleModel) tea.Cmd {
|
||
return m.call("Release "+d, http.MethodDelete, "/api/v1/facility/holds/"+url.PathEscape(d), nil,
|
||
func(json.RawMessage) string { return d + " is released." })
|
||
}})
|
||
case loc.Rack != "":
|
||
d := string(facility.Of(facility.Rack, loc.Rack))
|
||
items = append(items, menuItem{"Hold rack " + loc.Rack + "…", func(m *consoleModel) tea.Cmd {
|
||
m.openForm(&formDialog{title: "Hold rack " + loc.Rack,
|
||
note: "Nothing new is placed in the rack. Drain also moves its replicas off, one at a time.",
|
||
fields: []formField{textF("reason", "", "PDU A replacement"),
|
||
choiceF("drain", []string{"no", "yes"}, "no")},
|
||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||
return m.call("Hold "+d, http.MethodPost, "/api/v1/facility/holds",
|
||
adminapi.HoldRequest{Domain: d, Reason: v[0], Drain: v[1] == "yes"},
|
||
func(json.RawMessage) string { return d + " is held." }), nil
|
||
}})
|
||
return nil
|
||
}})
|
||
}
|
||
return items
|
||
}
|
||
|
||
// locationRequest turns the location form's values, site to pdu, into the
|
||
// labels to set and the ones to remove: a level emptied that was set through
|
||
// the control plane is removed, which leaves the agent's own value if any.
|
||
func locationRequest(n adminapi.NodeView, vals []string) adminapi.NodeLabelsRequest {
|
||
req := adminapi.NodeLabelsRequest{Set: map[string]string{}}
|
||
for i, lv := range facility.Levels {
|
||
k, v := lv.Label(), strings.TrimSpace(vals[i])
|
||
switch {
|
||
case v != "" && n.Labels[k] != v:
|
||
req.Set[k] = v
|
||
case v == "" && n.OperatorLabels[k] != "":
|
||
req.Remove = append(req.Remove, k)
|
||
}
|
||
}
|
||
return req
|
||
}
|