Act on the console's rows: scale, class, owner, resources, adapters, restart, remove; cordon, drain, engines, upgrade; quotas; cluster tasks

Enter on a row of Models, Nodes or Replicas opens what can be done to it,
as forms and questions over the admin API. A Quotas view sets and clears
capacity quotas. Do gains serving a model, upgrading the cluster,
restarting every replica and a backup.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-03 23:23:12 +02:00
1 parent e0384fb148
commit c7d5ee16b5
3 files changed
+934 -12

No files matched your search

+686
View File
@@ -0,0 +1,686 @@
package cli
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"time"
tea "github.com/charmbracelet/bubbletea"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
)
// What can be done from the console's tables: Enter on a row opens a menu of
// what applies to it — its details first — and each item is a form, a
// question, or done at once, through the same admin API `llmgw ctl` uses.
type menuItem struct {
label string
act func(m *consoleModel) tea.Cmd
}
type menuDialog struct {
title string
items []menuItem
sel int
}
type formField struct {
label, hint string
text textField
options []string
opt int
}
func (f formField) value() string {
if f.options != nil {
return f.options[f.opt]
}
return strings.TrimSpace(f.text.Value())
}
type formDialog struct {
title, note string
fields []formField
focus int
err string
submit func(m *consoleModel, vals []string) (tea.Cmd, error)
}
type confirmDialog struct {
title, text string
yes func(m *consoleModel) tea.Cmd
}
// actionDoneMsg is an action's outcome, shown as a dialog.
type actionDoneMsg struct {
title, text string
err error
}
func textF(label, value, hint string) formField {
t := newTextField()
t.Width = 30
t.SetValue(value)
return formField{label: label, hint: hint, text: t}
}
func choiceF(label string, options []string, current string) formField {
f := formField{label: label, options: options}
for i, o := range options {
if o == current {
f.opt = i
}
}
return f
}
func (m *consoleModel) openForm(f *formDialog) {
if len(f.fields) > 0 {
f.fields[0].text.Focus()
}
m.formD = f
}
// call runs one admin API request in the background and reports it.
func (m *consoleModel) call(title, method, path string, body any, say func(raw json.RawMessage) string) tea.Cmd {
m.busy = title
c := m.client
return func() tea.Msg {
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
var raw json.RawMessage
// A change may answer with no body at all.
if err := c.do(ctx, method, path, body, &raw); err != nil && !errors.Is(err, io.EOF) {
return actionDoneMsg{title: title, err: err}
}
text := "Done."
if say != nil {
text = say(raw)
}
return actionDoneMsg{title: title, text: text}
}
}
func (m consoleModel) actionKey(k tea.KeyMsg) (tea.Model, tea.Cmd, bool) {
switch {
case m.formD != nil:
f := m.formD
switch k.Type {
case tea.KeyEsc:
m.formD = nil
case tea.KeyTab, tea.KeyDown:
f.fields[f.focus].text.Blur()
f.focus = (f.focus + 1) % (len(f.fields) + 1)
if f.focus < len(f.fields) {
f.fields[f.focus].text.Focus()
}
case tea.KeyShiftTab, tea.KeyUp:
if f.focus < len(f.fields) {
f.fields[f.focus].text.Blur()
}
f.focus = (f.focus + len(f.fields)) % (len(f.fields) + 1)
if f.focus < len(f.fields) {
f.fields[f.focus].text.Focus()
}
case tea.KeyLeft, tea.KeyRight:
if f.focus < len(f.fields) && f.fields[f.focus].options != nil {
fl := &f.fields[f.focus]
d := 1
if k.Type == tea.KeyLeft {
d = len(fl.options) - 1
}
fl.opt = (fl.opt + d) % len(fl.options)
}
case tea.KeyEnter:
if f.focus < len(f.fields) {
f.fields[f.focus].text.Blur()
f.focus++
if f.focus < len(f.fields) {
f.fields[f.focus].text.Focus()
}
return m, nil, true
}
vals := make([]string, len(f.fields))
for i, fl := range f.fields {
vals[i] = fl.value()
}
cmd, err := f.submit(&m, vals)
if err != nil {
f.err = err.Error()
return m, nil, true
}
m.formD = nil
return m, cmd, true
default:
if f.focus < len(f.fields) && f.fields[f.focus].options == nil {
var cmd tea.Cmd
f.fields[f.focus].text, cmd = f.fields[f.focus].text.Update(k)
return m, cmd, true
}
}
return m, nil, true
case m.confirmD != nil:
c := m.confirmD
m.confirmD = nil
if strings.EqualFold(k.String(), "y") {
return m, c.yes(&m), true
}
return m, nil, true
case m.menuD != nil:
d := m.menuD
switch k.String() {
case "up", "k":
d.sel = (d.sel - 1 + len(d.items)) % len(d.items)
case "down", "j", "tab":
d.sel = (d.sel + 1) % len(d.items)
case "esc", "q":
m.menuD = nil
case "enter":
m.menuD = nil
return m, d.items[d.sel].act(&m), true
}
return m, nil, true
}
return m, nil, false
}
// actionDialog draws whichever of the three is open, or "".
func (m consoleModel) actionDialog() (title, body string) {
switch {
case m.formD != nil:
f := m.formD
var b strings.Builder
if f.note != "" {
b.WriteString(f.note + "\n\n")
}
w := 0
for _, fl := range f.fields {
w = max(w, len(fl.label))
}
for i, fl := range f.fields {
label := fmt.Sprintf("%-*s", w, fl.label)
if i == f.focus {
label = fSelected.Render(label)
}
v := fl.text.View()
if fl.options != nil && len(fl.options) > 3 {
v = fmt.Sprintf("◀ %s ▶ %s", sOK.Render(fl.options[fl.opt]), sDim.Render(fmt.Sprintf("%d of %d", fl.opt+1, len(fl.options))))
} else if fl.options != nil {
var opts []string
for j, o := range fl.options {
if j == fl.opt {
opts = append(opts, "("+sOK.Render("•")+") "+o)
} else {
opts = append(opts, "( ) "+sDim.Render(o))
}
}
v = strings.Join(opts, " ")
}
fmt.Fprintf(&b, "%s %s\n", label, v)
if fl.hint != "" {
fmt.Fprintf(&b, "%s %s\n", strings.Repeat(" ", w), sDim.Render(fl.hint))
}
}
b.WriteString("\n" + button(f.focus == len(f.fields), "Apply") + "\n")
if f.err != "" {
b.WriteString("\n" + sBad.Render(f.err) + "\n")
}
b.WriteString("\n" + sDim.Render("Tab next · ←→ choose · Enter on Apply · Esc cancel"))
return f.title, b.String()
case m.confirmD != nil:
return m.confirmD.title, m.confirmD.text + "\n\n" + sDim.Render("Y yes · any other key no")
case m.menuD != nil:
var b strings.Builder
for i, it := range m.menuD.items {
if i == m.menuD.sel {
b.WriteString(fSelected.Render("› "+it.label) + "\n")
} else {
b.WriteString(" " + it.label + "\n")
}
}
b.WriteString("\n" + sDim.Render("↑↓ choose · Enter · Esc close"))
return m.menuD.title, b.String()
}
return "", ""
}
func details(m *consoleModel) tea.Cmd {
m.detail, m.scroll = true, 0
return nil
}
// --- models ---
func (m consoleModel) modelMenu(md adminapi.ModelView) *menuDialog {
path := "/api/v1/models/" + url.PathEscape(md.Name)
items := []menuItem{
{"Details", details},
{"Replicas (scale)…", func(m *consoleModel) tea.Cmd {
m.openForm(&formDialog{title: "Scale " + md.Name,
note: "How many replicas it keeps. Max 0 stops it; min 0 with max above loads it on demand.",
fields: []formField{
textF("min", strconv.Itoa(md.Min), "replicas kept running"),
textF("max", strconv.Itoa(md.Max), "the most it scales to"),
textF("idle timeout", "", "on demand only: unload after this long unused, like 30m"),
},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
minR, err1 := strconv.Atoi(v[0])
maxR, err2 := strconv.Atoi(v[1])
if err1 != nil || err2 != nil || minR < 0 || maxR < minR {
return nil, fmt.Errorf("min and max are whole numbers, max at least min")
}
req := adminapi.ScaleRequest{Min: minR, Max: maxR}
if v[2] != "" {
d, err := time.ParseDuration(v[2])
if err != nil {
return nil, fmt.Errorf("the idle timeout is a duration, like 30m")
}
req.IdleTimeout = d
}
return m.call("Scale "+md.Name, http.MethodPost, path+"/scale", req,
func(json.RawMessage) string { return fmt.Sprintf("%s keeps %d to %d replicas.", md.Name, minR, maxR) }), nil
}})
return nil
}},
{"Class…", func(m *consoleModel) tea.Cmd {
m.openForm(&formDialog{title: "Class of " + md.Name,
note: "How much its capacity is worth against another's when there is not enough for both.",
fields: []formField{choiceF("class", []string{"critical", "standard", "opportunistic"}, md.Class)},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
return m.call("Class of "+md.Name, http.MethodPost, path+"/class", map[string]string{"class": v[0]},
func(json.RawMessage) string { return md.Name + " is " + v[0] + "." }), nil
}})
return nil
}},
{"Owner…", func(m *consoleModel) tea.Cmd {
m.openForm(&formDialog{title: "Owner of " + md.Name,
note: "Whose capacity quota it counts against. Empty: the cluster's, nobody's.",
fields: []formField{textF("owner", md.Owner, "org:ID, team:ID or project:ID")},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
return m.call("Owner of "+md.Name, http.MethodPost, path+"/owner", map[string]string{"owner": v[0]}, nil), nil
}})
return nil
}},
{"Resources…", func(m *consoleModel) tea.Cmd {
profiles := md.Profiles
if len(profiles) == 0 {
profiles = []string{""}
}
m.openForm(&formDialog{title: "Resources of " + md.Name,
note: "What one replica reserves of its node.",
fields: []formField{
choiceF("profile", profiles, profiles[0]),
textF("memory GB", strconv.Itoa(md.MemoryGB), fmt.Sprintf("measured: %.1f GB", md.MeasuredMemoryGB)),
textF("CPU cores", strconv.Itoa(md.CPUCores), "for a profile that computes on them"),
},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
mem, err1 := strconv.Atoi(v[1])
cores, err2 := strconv.Atoi(v[2])
if err1 != nil || err2 != nil || mem < 0 || cores < 0 {
return nil, fmt.Errorf("memory and cores are whole numbers")
}
return m.call("Resources of "+md.Name, http.MethodPost, path+"/resources",
adminapi.ResourcesRequest{Profile: v[0], MemoryGB: mem, CPUCores: cores}, nil), nil
}})
return nil
}},
{"LoRA adapters…", func(m *consoleModel) tea.Cmd {
var cur []string
for _, a := range md.Adapters {
s := a.Name + "=" + a.Source
if a.Scale != 0 {
s += "@" + strconv.FormatFloat(a.Scale, 'f', -1, 64)
}
cur = append(cur, s)
}
m.openForm(&formDialog{title: "Adapters of " + md.Name,
note: "Every replica loads every adapter; a request names one as its model.",
fields: []formField{textF("adapters", strings.Join(cur, ", "), "name=source[@scale], separated by commas; empty removes all")},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
var next []catalog.Adapter
for _, part := range strings.Split(v[0], ",") {
if part = strings.TrimSpace(part); part == "" {
continue
}
name, rest, ok := strings.Cut(part, "=")
if !ok {
return nil, fmt.Errorf("%q is not name=source", part)
}
a := catalog.Adapter{Name: strings.TrimSpace(name), Source: strings.TrimSpace(rest)}
if src, sc, ok := strings.Cut(a.Source, "@"); ok {
f, err := strconv.ParseFloat(sc, 64)
if err != nil {
return nil, fmt.Errorf("%q: the scale is a number", part)
}
a.Source, a.Scale = src, f
}
next = append(next, a)
}
return m.call("Adapters of "+md.Name, http.MethodPost, path+"/adapters",
adminapi.AdaptersRequest{Adapters: next}, nil), nil
}})
return nil
}},
{"Restart its replicas…", func(m *consoleModel) tea.Cmd {
m.openForm(&formDialog{title: "Restart " + md.Name,
note: "Each replica is replaced: a new one started before the old is stopped, unless in place.",
fields: []formField{choiceF("how", []string{"one by one", "in place"}, "one by one")},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
return m.call("Restart "+md.Name, http.MethodPost, "/api/v1/restart",
adminapi.RestartRequest{Models: []string{md.Name}, InPlace: v[0] == "in place"}, nil), nil
}})
return nil
}},
{"Why is it placed where it is", func(m *consoleModel) tea.Cmd {
return m.call("Placement of "+md.Name, http.MethodGet, "/api/v1/explain-placement?model="+url.QueryEscape(md.Name), nil,
func(raw json.RawMessage) string {
var e adminapi.ExplainView
if json.Unmarshal(raw, &e) != nil {
return string(raw)
}
var b strings.Builder
if e.Placed {
b.WriteString("Placed.\n")
}
for _, r := range e.Rejections {
fmt.Fprintf(&b, "%s on %s %s: %s\n", r.Profile, r.Pool, r.Node, r.Reason)
}
if b.Len() == 0 {
return "Nothing to explain: it has what it asked for."
}
return b.String()
})
}},
}
if md.Parked {
items = append(items, menuItem{"Try again (it is parked)", func(m *consoleModel) tea.Cmd {
return m.call("Retry "+md.Name, http.MethodPost, path+"/retry", map[string]string{}, nil)
}})
}
items = append(items, menuItem{"Remove…", func(m *consoleModel) tea.Cmd {
m.confirmD = &confirmDialog{title: "Remove " + md.Name,
text: "Its replicas stop and it leaves the catalog. A model from server.yaml cannot be removed here.",
yes: func(m *consoleModel) tea.Cmd {
return m.call("Remove "+md.Name, http.MethodDelete, path, nil, nil)
}}
return nil
}})
return &menuDialog{title: md.Name, items: items}
}
// --- nodes ---
func (m consoleModel) nodeMenu(n adminapi.NodeView) *menuDialog {
path := "/api/v1/nodes/" + url.PathEscape(n.ID)
state := func(s, label, title string) menuItem {
return menuItem{label, func(m *consoleModel) tea.Cmd {
return m.call(title+" "+n.ID, http.MethodPost, path+"/state", map[string]string{"state": s},
func(json.RawMessage) string { return n.ID + " is " + s + "." })
}}
}
items := []menuItem{{"Details", details}}
if n.State == "cordoned" || n.State == "draining" {
items = append(items, state("ready", "Uncordon", "Uncordon"))
} else {
items = append(items, state("cordoned", "Cordon: place nothing new here", "Cordon"))
items = append(items, menuItem{"Drain…", func(m *consoleModel) tea.Cmd {
m.confirmD = &confirmDialog{title: "Drain " + n.ID,
text: "Its replicas move off one at a time, each replacement up before the original stops.",
yes: func(m *consoleModel) tea.Cmd {
return m.call("Drain "+n.ID, http.MethodPost, path+"/state", map[string]string{"state": "draining"}, nil)
}}
return nil
}})
}
if n.GPUTotal > 0 {
for _, engine := range []string{"vllm", "sglang"} {
items = append(items, menuItem{engine + " build…", func(m *consoleModel) tea.Cmd {
m.openForm(&formDialog{title: engine + " on " + n.ID,
note: "Which build of " + engine + " this node installs: auto follows its hardware, none installs none.",
fields: []formField{choiceF("build", control.EngineBuilds(engine), n.EngineBuilds[engine])},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
return m.call(engine+" on "+n.ID, http.MethodPost, path+"/engines",
adminapi.NodeEngineRequest{Engine: engine, Build: v[0]}, nil), nil
}})
return nil
}})
}
}
items = append(items,
menuItem{"Upgrade its agent…", func(m *consoleModel) tea.Cmd {
m.openForm(&formDialog{title: "Upgrade " + n.ID,
note: "It fetches the release for its platform, checks it, swaps it in and restarts; engines keep serving.",
fields: []formField{textF("version", version.Version, "a release, like v0.11.1")},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
return m.call("Upgrade "+n.ID, http.MethodPost, path+"/upgrade", adminapi.UpgradeRequest{Version: v[0]}, nil), nil
}})
return nil
}},
menuItem{"Delete model files nothing uses", func(m *consoleModel) tea.Cmd {
return m.call("Unused files on "+n.ID, http.MethodPost, "/api/v1/weights/collect", map[string]string{"node": n.ID},
func(raw json.RawMessage) string {
var rs []struct {
Files int `json:"files"`
Bytes int64 `json:"bytes"`
}
_ = json.Unmarshal(raw, &rs)
var files int
var b int64
for _, r := range rs {
files, b = files+r.Files, b+r.Bytes
}
return fmt.Sprintf("%d files, %.1f GB freed.", files, float64(b)/(1<<30))
})
}},
menuItem{"Remove from the cluster…", func(m *consoleModel) tea.Cmd {
m.confirmD = &confirmDialog{title: "Remove " + n.ID,
text: "The control plane forgets it. Its replicas are placed elsewhere; the machine is not touched.",
yes: func(m *consoleModel) tea.Cmd {
return m.call("Remove "+n.ID, http.MethodDelete, path, nil, nil)
}}
return nil
}},
)
return &menuDialog{title: n.ID, items: items}
}
// --- replicas ---
func (m consoleModel) replicaMenu(r adminapi.ReplicaView) *menuDialog {
return &menuDialog{title: r.ID, items: []menuItem{
{"Restart it…", func(m *consoleModel) tea.Cmd {
m.openForm(&formDialog{title: "Restart " + r.ID,
fields: []formField{choiceF("how", []string{"replace", "in place"}, "replace")},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
return m.call("Restart "+r.ID, http.MethodPost, "/api/v1/restart",
adminapi.RestartRequest{Replicas: []string{r.ID}, InPlace: v[0] == "in place"}, nil), nil
}})
return nil
}},
}}
}
// --- capacity quotas ---
func intPtr(s string) (*int, error) {
s = strings.TrimSpace(s)
if s == "" {
return nil, nil
}
n, err := strconv.Atoi(s)
if err != nil || n < 0 {
return nil, fmt.Errorf("%q is not a whole number; leave it empty for none", s)
}
return &n, nil
}
func ptrText(p *int) string {
if p == nil {
return ""
}
return strconv.Itoa(*p)
}
func (m *consoleModel) quotaForm(q adminapi.QuotaView) {
scope := textF("scope", q.Scope, "org:ID, team:ID or project:ID")
m.openForm(&formDialog{title: "Capacity quota",
note: "Guaranteed is never taken for someone else's model; at most is the ceiling. Empty is not governed.",
fields: []formField{scope,
textF("guaranteed GPUs", ptrText(q.Guarantee.GPUs), ""),
textF("guaranteed cores", ptrText(q.Guarantee.CPUCores), ""),
textF("guaranteed memory GB", ptrText(q.Guarantee.MemoryGB), ""),
textF("at most GPUs", ptrText(q.Limit.GPUs), ""),
textF("at most cores", ptrText(q.Limit.CPUCores), ""),
textF("at most memory GB", ptrText(q.Limit.MemoryGB), ""),
},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
req := adminapi.QuotaRequest{Scope: v[0]}
dst := []**int{&req.Guarantee.GPUs, &req.Guarantee.CPUCores, &req.Guarantee.MemoryGB,
&req.Limit.GPUs, &req.Limit.CPUCores, &req.Limit.MemoryGB}
for i, d := range dst {
p, err := intPtr(v[i+1])
if err != nil {
return nil, err
}
*d = p
}
return m.call("Quota for "+v[0], http.MethodPut, "/api/v1/quotas", req, nil), nil
}})
}
func (m consoleModel) quotaMenu(q adminapi.QuotaView) *menuDialog {
return &menuDialog{title: q.Scope, items: []menuItem{
{"Change…", func(m *consoleModel) tea.Cmd { m.quotaForm(q); return nil }},
{"Remove…", func(m *consoleModel) tea.Cmd {
m.confirmD = &confirmDialog{title: "Remove the quota on " + q.Scope,
text: "Its models then share the cluster by their class alone.",
yes: func(m *consoleModel) tea.Cmd {
return m.call("Quota on "+q.Scope, http.MethodDelete, "/api/v1/quotas/"+url.PathEscape(q.Scope), nil, nil)
}}
return nil
}},
}}
}
type consoleQuotasMsg struct {
rows []adminapi.QuotaView
err error
}
func (m consoleModel) fetchQuotas() tea.Cmd {
c := m.client
return func() tea.Msg {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
var rows []adminapi.QuotaView
err := c.get(ctx, "/api/v1/quotas", &rows)
return consoleQuotasMsg{rows, err}
}
}
func (m consoleModel) quotasPanel(pw int) (string, string, string) {
keys := "→ or Enter open · ↑↓ choose · Q leave"
if m.inPanel {
keys = "↑↓ row · Enter actions · A add · ← back · Q leave"
}
head := "\n " + sDim.Render("Capacity each org, team or project is guaranteed and may at most hold. A to add one.") + "\n\n"
if m.quotasErr != nil {
return "Quotas", head + " " + sBad.Render(m.quotasErr.Error()), keys
}
cell := func(p *int) string {
if p == nil {
return "—"
}
return strconv.Itoa(*p)
}
var rows [][]string
for _, q := range m.quotas {
name := q.Name
if name == "" {
name = q.Scope
}
rows = append(rows, []string{name,
cell(q.Guarantee.GPUs) + " / " + cell(q.Limit.GPUs),
cell(q.Guarantee.CPUCores) + " / " + cell(q.Limit.CPUCores),
cell(q.Guarantee.MemoryGB) + " / " + cell(q.Limit.MemoryGB),
fmt.Sprintf("%d GPU, %d cores, %d GB", q.Holds.GPUs, q.Holds.CPUCores, q.Holds.MemoryGB)})
}
if len(rows) == 0 {
return "Quotas", head + " " + sDim.Render("None: models share the cluster by their class alone."), keys
}
return "Quotas", head + table([]string{"WHO", "GPUS (GUAR / MAX)", "CORES", "MEMORY GB", "HOLDS"}, rows, m.row, pw, m.inPanel), keys
}
// --- the cluster ---
// runAfter closes the screen and runs llmgw with these arguments, showing its
// output; the console comes back after.
func (m *consoleModel) runAfter(args ...string) tea.Cmd {
m.action = consoleAction{kind: "run", targets: append([]string{}, args...)}
return tea.Quit
}
func (m *consoleModel) doEntry() tea.Cmd {
server := "--server=" + m.client.base
switch m.menu {
case cServe:
m.openForm(&formDialog{title: "Serve a model",
note: "A GGUF repository on Hugging Face, like bartowski/Qwen2.5-3B-Instruct-GGUF.",
fields: []formField{
textF("repository", "", "OWNER/REPO"),
textF("quantisation", "Q4_K_M", ""),
textF("name", "", "in the catalog; empty takes it from the repository"),
textF("replicas", "1", "0 runs none until it is first asked for"),
textF("context length", "4096", "what one request may use"),
},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
if !strings.Contains(v[0], "/") {
return nil, fmt.Errorf("the repository is OWNER/REPO")
}
args := []string{"ctl", server, "model", "serve", v[0], "--quant=" + v[1], "--replicas=" + v[3],
"--context-length=" + v[4]}
if v[3] == "0" {
args = append(args, "--on-demand")
}
if v[2] != "" {
args = append(args, "--name="+v[2])
}
return m.runAfter(args...), nil
}})
case cUpgrade:
m.openForm(&formDialog{title: "Upgrade the cluster",
note: "This machine runs " + version.Version + ".",
fields: []formField{textF("version", "", "like v0.11.2"), choiceF("this machine too", []string{"yes", "no, nodes only"}, "yes")},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
if !release.Tag.MatchString(v[0]) {
return nil, fmt.Errorf("%q is not a release version (v1.2.3)", v[0])
}
args := []string{"ctl", server, "upgrade", v[0]}
if v[1] != "yes" {
args = append(args, "--skip-control-plane")
}
return m.runAfter(args...), nil
}})
case cRestartAll:
m.confirmD = &confirmDialog{title: "Restart every replica",
text: "Each is replaced in turn, the new one up before the old one stops.",
yes: func(m *consoleModel) tea.Cmd {
return m.call("Restart every replica", http.MethodPost, "/api/v1/restart", adminapi.RestartRequest{All: true}, nil)
}}
case cBackup:
return m.runAfter("server", "backup", "--config="+m.file)
}
return nil
}
+153
View File
@@ -0,0 +1,153 @@
package cli
import (
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
tea "github.com/charmbracelet/bubbletea"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
)
type seen struct {
mu sync.Mutex
reqs []string
}
func (s *seen) last() string {
s.mu.Lock()
defer s.mu.Unlock()
if len(s.reqs) == 0 {
return ""
}
return s.reqs[len(s.reqs)-1]
}
// adminStub answers every change with 204 and records it as "METHOD path body".
func adminStub(t *testing.T) (*seen, string) {
s := &seen{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodGet {
_, _ = w.Write([]byte("[]"))
return
}
b, _ := io.ReadAll(r.Body)
s.mu.Lock()
s.reqs = append(s.reqs, r.Method+" "+r.URL.Path+" "+strings.TrimSpace(string(b)))
s.mu.Unlock()
w.WriteHeader(http.StatusNoContent)
}))
t.Cleanup(srv.Close)
return s, srv.URL
}
// pressRun presses keys and runs, one level deep, whatever the last one
// returns that is not a tick, the way the program would.
func pressRun(m tea.Model, keys ...string) tea.Model {
m = press(m, keys[:len(keys)-1]...)
var msg tea.Msg = tea.KeyMsg{Type: tea.KeyEnter}
if k := keys[len(keys)-1]; k != "enter" {
msg = tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(k)}
}
m, cmd := m.Update(msg)
if cmd != nil {
if done, ok := cmd().(actionDoneMsg); ok {
m, _ = m.Update(done)
}
}
return m
}
func consoleWith(t *testing.T, menu int) (consoleModel, *seen) {
s, url := adminStub(t)
m := newConsoleModel(url, "/nonexistent/server.yaml")
m.loaded, m.menu, m.inPanel = true, menu, true
m.data.models = []adminapi.ModelView{{Name: "qwen", Min: 1, Max: 1, Class: "standard", Profiles: []string{"cpu"}}}
m.data.nodes = []adminapi.NodeView{{ID: "n1", State: "ready"}}
m.data.replicas = []adminapi.ReplicaView{{ID: "qwen-1", Model: "qwen"}}
return m, s
}
func TestAModelsReplicasAreSetFromItsRow(t *testing.T) {
m, s := consoleWith(t, cModels)
var tm tea.Model = m
tm = press(tm, "enter")
if !strings.Contains(tm.View(), "Replicas (scale)") {
t.Fatalf("no action menu:\n%s", tm.View())
}
tm = press(tm, "down", "enter")
f := tm.(consoleModel).formD
if f == nil {
t.Fatal("no form")
}
f.fields[1].text.SetValue("3")
// A max below the min is refused in the form, and nothing is sent.
f.fields[0].text.SetValue("5")
tm = press(tm, "tab", "tab", "tab", "enter")
if f.err == "" || s.last() != "" {
t.Fatalf("a max below the min went through: %q %q", f.err, s.last())
}
f.fields[0].text.SetValue("2")
tm = pressRun(tm, "enter")
var body adminapi.ScaleRequest
got := s.last()
if !strings.HasPrefix(got, "POST /api/v1/models/qwen/scale ") ||
json.Unmarshal([]byte(strings.SplitN(got, " ", 3)[2]), &body) != nil || body.Min != 2 || body.Max != 3 {
t.Fatalf("sent %q", got)
}
if !strings.Contains(tm.View(), "keeps 2 to 3") {
t.Errorf("the outcome was not shown:\n%s", tm.View())
}
}
func TestANodeIsCordonedAndAModelRemovedOnlyWhenConfirmed(t *testing.T) {
m, s := consoleWith(t, cNodes)
var tm tea.Model = m
tm = pressRun(tm, "enter", "down", "enter")
if got := s.last(); got != `POST /api/v1/nodes/n1/state {"state":"cordoned"}` {
t.Fatalf("sent %q", got)
}
m, s = consoleWith(t, cModels)
tm = m
tm = press(tm, "enter")
n := len(tm.(consoleModel).menuD.items)
keys := []string{}
for range n - 1 {
keys = append(keys, "down")
}
tm = press(tm, append(keys, "enter")...)
if tm.(consoleModel).confirmD == nil {
t.Fatal("removing did not ask first")
}
tm = press(tm, "n")
if s.last() != "" {
t.Fatalf("removed though the answer was no: %q", s.last())
}
tm = press(tm, "enter")
tm = press(tm, append(keys, "enter")...)
_ = pressRun(tm, "y")
if got := s.last(); got != "DELETE /api/v1/models/qwen " {
t.Fatalf("sent %q", got)
}
}
func TestAQuotaIsAddedWithOnlyWhatIsGiven(t *testing.T) {
m, s := consoleWith(t, cQuotas)
var tm tea.Model = press(m, "a")
f := tm.(consoleModel).formD
f.fields[0].text.SetValue("team:acme/ml")
f.fields[4].text.SetValue("2")
for range len(f.fields) {
tm = press(tm, "tab")
}
_ = pressRun(tm, "enter")
if got := s.last(); got != `PUT /api/v1/quotas {"scope":"team:acme/ml","guarantee":{},"limit":{"gpus":2}}` {
t.Fatalf("sent %q", got)
}
}
+95 -12
View File
@@ -35,21 +35,27 @@ const (
cModels
cReplicas
cUsage
cQuotas
cLogs
cSettings
cAddNode
cReplicate
cServe
cUpgrade
cRestartAll
cBackup
cLeave
cCount
)
var consoleLabels = [cCount]string{"Overview", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Logs",
"server.yaml", "Add a node", "Replicate the store", "Leave"}
var consoleLabels = [cCount]string{"Overview", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Logs",
"server.yaml", "Add a node", "Replicate the store", "Serve a model", "Upgrade the cluster",
"Restart all replicas", "Back up now", "Leave"}
// consoleAction is what is done once the screen closes: each runs its own
// screen, and the console comes back after.
type consoleAction struct {
kind string // "", "node", "patroni", "restart"
kind string // "", "node", "patroni", "restart", "run" (llmgw with targets as its arguments)
targets []string
}
@@ -127,6 +133,16 @@ type consoleModel struct {
editKind settings.Kind
action consoleAction
// What can be done to a row: a menu, a form, a question, and the request
// under way.
menuD *menuDialog
formD *formDialog
confirmD *confirmDialog
busy string
quotas []adminapi.QuotaView
quotasErr error
}
var usageGroupings = []string{"team", "project", "org"}
@@ -235,6 +251,9 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
if m.menu == cUsage {
cmds = append(cmds, m.fetchUsage())
}
if m.menu == cQuotas {
cmds = append(cmds, m.fetchQuotas())
}
if m.menu == cLogs && m.detail && m.logs.node != "" {
cmds = append(cmds, m.fetchLogs(m.logs.node))
}
@@ -250,6 +269,18 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
m.usage, m.usageErr = msg.rows, msg.err
case consoleLogsMsg:
m.logs = msg
case consoleQuotasMsg:
m.quotas, m.quotasErr = msg.rows, msg.err
case actionDoneMsg:
m.busy = ""
m.dialog, m.dialogTitle = "error", msg.title
if msg.err != nil {
m.dialogText = sBad.Render("✗ ") + msg.err.Error()
} else {
m.dialogText = sOK.Render("✓ ") + msg.text
}
m.dialogText += "\n\n" + sDim.Render("any key")
return m, tea.Batch(m.fetch(), m.onEnter())
case tea.KeyMsg:
if msg.Type == tea.KeyCtrlC {
return m, tea.Quit
@@ -269,9 +300,15 @@ func (m consoleModel) entryOK(i int) bool {
func (m consoleModel) storeReplicated() bool { return len(m.data.sum.Store) > 0 }
func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
if m.busy != "" {
return m, nil
}
if m.dialog != "" {
return m.dialogKey(k)
}
if mm, cmd, ok := m.actionKey(k); ok {
return mm, cmd
}
switch k.Type {
case tea.KeyPgDown:
m.scroll += max(1, m.height-8)
@@ -310,6 +347,8 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
return m.ask("Replicate the store", "user@address of database machine", 2)
case cLeave:
return m.leave()
case cServe, cUpgrade, cRestartAll, cBackup:
return m, m.doEntry()
case cOverview:
return m, nil
}
@@ -331,13 +370,27 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
m.row++
}
case "enter":
if !m.detail && n > 0 && m.menu != cGPUs && m.menu != cReplicas && m.menu != cUsage {
if m.detail || n == 0 || m.row >= n {
break
}
switch m.menu {
case cLogs:
m.detail, m.scroll = true, 0
if m.menu == cLogs {
node := m.data.nodes[m.row].ID
m.logs = consoleLogsMsg{node: node}
return m, m.fetchLogs(node)
}
node := m.data.nodes[m.row].ID
m.logs = consoleLogsMsg{node: node}
return m, m.fetchLogs(node)
case cNodes:
m.menuD = m.nodeMenu(m.data.nodes[m.row])
case cModels:
m.menuD = m.modelMenu(m.data.models[m.row])
case cReplicas:
m.menuD = m.replicaMenu(m.data.replicas[m.row])
case cQuotas:
m.menuD = m.quotaMenu(m.quotas[m.row])
}
case "a":
if m.menu == cQuotas {
m.quotaForm(adminapi.QuotaView{})
}
case "right", "l":
if m.menu == cUsage {
@@ -358,8 +411,11 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
// onEnter loads what a view needs when it is chosen.
func (m consoleModel) onEnter() tea.Cmd {
if m.menu == cUsage {
switch m.menu {
case cUsage:
return m.fetchUsage()
case cQuotas:
return m.fetchQuotas()
}
return nil
}
@@ -659,6 +715,8 @@ func (m consoleModel) rowCount() int {
return len(m.data.replicas)
case cUsage:
return len(m.usage)
case cQuotas:
return len(m.quotas)
}
return 0
}
@@ -718,6 +776,10 @@ func (m consoleModel) View() string {
}
f.Dialog = q + "\n\n" + m.prompt.View() + "\n\n" + sDim.Render("Enter · Esc cancel")
}
} else if m.busy != "" {
f.DialogTitle, f.Dialog = m.busy, m.spin.View()+" asking the control plane…"
} else if t, d := m.actionDialog(); d != "" {
f.DialogTitle, f.Dialog = t, d
}
return f.render()
}
@@ -725,7 +787,7 @@ func (m consoleModel) View() string {
// panel is the right-hand side for the entry chosen.
func (m consoleModel) panel() (title, body, keys string) {
pw := max(40, m.width-navWidth-8)
rowKeys := "↑↓ row · Enter detail · ← back · PgUp/PgDn · Q leave"
rowKeys := "↑↓ row · Enter actions · ← back · PgUp/PgDn · Q leave"
if !m.inPanel {
keys = "↑↓ choose · → or Enter open · Q leave · refreshed every three seconds"
}
@@ -823,7 +885,7 @@ func (m consoleModel) panel() (title, body, keys string) {
(time.Duration(r.AgeSec) * time.Second).Round(time.Second).String()})
}
if m.inPanel {
keys = "↑↓ row · ← back · PgUp/PgDn · Q leave"
keys = rowKeys
}
return "Replicas", "\n" + table([]string{"REPLICA", "MODEL", "STATE", "POOL", "ON", "AGE"}, rows, m.row, pw, m.inPanel), keys
case cUsage:
@@ -841,6 +903,8 @@ func (m consoleModel) panel() (title, body, keys string) {
keys = "↑↓ row · → grouping · ← back · Q leave"
}
return "Usage", head + table([]string{"WHO", "REQUESTS", "TOKENS", "ERRORS", "SPENT"}, rows, m.row, pw, m.inPanel), keys
case cQuotas:
return m.quotasPanel(pw)
case cSettings:
return m.settingsPanel(pw)
}
@@ -853,6 +917,18 @@ func (m consoleModel) panel() (title, body, keys string) {
item = "\n Moves the store onto two machines under Patroni, replicated,\n so losing one loses nothing.\n\n " +
sDim.Render("On the command line:") +
"\n sudo --preserve-env=SSH_AUTH_SOCK llmgw server init \\\n --patroni <user>@<a> --patroni <user>@<b> --no-mirror --no-node --no-model"
case cServe:
item = "\n Takes a GGUF model from Hugging Face to a replica that answers:\n the mirror fetches it, its checksum and memory are checked, it joins\n the catalog, and a replica is started and asked to say hello.\n\n " +
sDim.Render("On the command line:") + "\n llmgw ctl model serve OWNER/REPO"
case cUpgrade:
item = "\n Moves the cluster to a release: this machine first, then each node\n in turn. Engines keep serving; a node that does not come back stops it.\n\n " +
sDim.Render("On the command line:") + "\n sudo llmgw ctl upgrade VERSION"
case cRestartAll:
item = "\n Replaces every replica, one at a time, each new one up before the\n old one stops.\n\n " +
sDim.Render("On the command line:") + "\n llmgw ctl restart --all"
case cBackup:
item = "\n Writes the store and /etc/llmgw to one archive in " + DefaultBackupDir + ",\n and keeps the last few. Copy it off this machine.\n\n " +
sDim.Render("On the command line:") + "\n sudo llmgw server backup"
case cLeave:
item = "\n Closes the console. The cluster keeps running."
}
@@ -1114,6 +1190,13 @@ func runConsole(ctx context.Context, out io.Writer, file string) error {
// Already set up: only the store changes.
in.noMirror, in.noNode, in.noModel = true, true, true
err = in.runInitTUI(ctx, out)
case "run":
c := Root()
c.SetArgs(act.targets)
c.SetOut(out)
c.SetErr(out)
fmt.Fprintln(out, sDim.Render("llmgw "+strings.Join(act.targets, " ")))
err = c.ExecuteContext(ctx)
case "restart":
fmt.Fprintf(out, "restarting %s…\n", fm.service())
if b, e := exec.CommandContext(ctx, "systemctl", "restart", fm.service()).CombinedOutput(); e != nil {