Files
llmgw/internal/cli/tui_console.go
T
2026-10-04 16:30:56 +02:00

1449 lines
41 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package cli
import (
"bufio"
"bytes"
"context"
"fmt"
"io"
"net/url"
"os"
"os/exec"
"strconv"
"strings"
"time"
tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/lipgloss"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/docgen"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// The console: what a bare `sudo llmgw` opens on a control plane. Down the
// left, three groups — what is running (watched live), the settings (every
// key of server.yaml, from the generated reference), and what can be done.
// Sidebar entries, in order. Headings are not entries.
const (
cOverview = iota
cMonitor
cNodes
cGPUs
cModels
cReplicas
cUsage
cQuotas
cMirror
cJobs
cLogs
cAccounts
cKeys
cOrgs
cAudit
cManual
cSettings
cAddNode
cReplicate
cServe
cUpgrade
cRestartAll
cBackup
cLeave
cCount
)
var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Audit", "Manual",
"server.yaml", "Add a node", "Replicate the store", "Serve a model", "Upgrade the cluster",
"Restart all replicas", "Back up now", "Leave"}
// consoleAction is what is done once the screen closes: each runs its own
// screen, and the console comes back after.
type consoleAction struct {
kind string // "", "node", "patroni", "restart", "run" (llmgw with targets as its arguments)
targets []string
}
type consoleData struct {
sum adminapi.Summary
nodes []adminapi.NodeView
models []adminapi.ModelView
replicas []adminapi.ReplicaView
jobs []modelserve.Job
audit []store.AuditEvent
err error
}
type (
consoleDataMsg consoleData
consoleUsageMsg struct {
rows []adminapi.ScopeUsageView
err error
}
consoleLogsMsg struct {
node string
lines []string
err error
}
consoleTickMsg struct{}
)
// settingsLevel is where the settings view is: a block, or a list or map of
// blocks, at a path of keys in the file.
type settingsLevel struct {
title string
block *docgen.Block
path []string
kind settings.Kind // Block, BlockList or BlockMap
}
type consoleModel struct {
client *ctlClient
width, height int
spin spinnerModel
data consoleData
loaded bool
usage []adminapi.ScopeUsageView
// usageBy is the grouping on the usage view.
usageBy int
usageErr error
logs consoleLogsMsg
// menu is the sidebar entry; inPanel says the keys go to the panel.
menu int
inPanel bool
row int
// detail is the panel showing one row's detail rather than the table.
detail bool
scroll int
// node is the console of a node: its agent.yaml and nothing to watch,
// since the cluster is watched from the control plane.
node bool
// The settings.
file string
doc *settings.Doc
cfg any // the file as loaded, for the values that apply
levels []settingsLevel
docErr error
saveNote string
// A question open as a dialog.
dialog string // "", "target", "edit", "addname", "saved", "confirm-leave", "error"
dialogTitle string
dialogText string
prompt textField
asked []string
editPath []string
editKind settings.Kind
action consoleAction
// What can be done to a row: a menu, a form, a question, and the request
// under way.
menuD *menuDialog
formD *formDialog
confirmD *confirmDialog
busy string
quotas []adminapi.QuotaView
quotasErr error
st *storeHolder
people peopleData
mirror consoleMirrorMsg
hf hfState
docs docViewer
mon monitorState
det monitorState
}
var usageGroupings = []string{"team", "project", "org"}
func newConsoleModel(addr, file string) consoleModel {
m := consoleModel{client: newCtlClient(addr), spin: newSpinner(), width: 100, height: 30, file: file,
st: &storeHolder{file: file}, docs: newDocViewer()}
m.node = strings.HasSuffix(file, "agent.yaml")
if m.node {
m.menu = cSettings
}
m.loadSettings()
return m
}
// load reads the file as the service would, defaults and checks included.
func (m consoleModel) load(path string) (any, error) {
if m.node {
return config.LoadAgent(path)
}
return config.Load(path)
}
func (m consoleModel) service() string {
if m.node {
return "llmgw-agent"
}
return "llmgw-server"
}
func (m *consoleModel) loadSettings() {
m.doc, m.docErr = settings.Open(m.file)
if m.docErr != nil {
return
}
if cfg, err := m.load(m.file); err == nil {
m.cfg = cfg
} else if !m.node {
m.cfg = config.Default()
}
if root, err := settings.Root(m.file); err == nil && len(m.levels) == 0 {
m.levels = []settingsLevel{{title: filepathBase(m.file), block: root, kind: settings.Block}}
}
}
func (m consoleModel) Init() tea.Cmd {
if m.node {
return m.spin.Tick
}
return tea.Batch(m.fetch(), m.spin.Tick, tea.Tick(3*time.Second, func(time.Time) tea.Msg { return consoleTickMsg{} }))
}
// fetch reads everything the watched views show, in one go.
func (m consoleModel) fetch() tea.Cmd {
c := m.client
return func() tea.Msg {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
var d consoleData
if d.err = c.get(ctx, "/api/v1/summary", &d.sum); d.err != nil {
return consoleDataMsg(d)
}
_ = c.get(ctx, "/api/v1/nodes", &d.nodes)
_ = c.get(ctx, "/api/v1/models", &d.models)
_ = c.get(ctx, "/api/v1/replicas", &d.replicas)
_ = c.get(ctx, "/api/v1/jobs", &d.jobs)
_ = c.get(ctx, "/api/v1/audit?since=7d&limit=300", &d.audit)
return consoleDataMsg(d)
}
}
func (m consoleModel) fetchUsage() tea.Cmd {
c, by := m.client, usageGroupings[m.usageBy]
return func() tea.Msg {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
var rows []adminapi.ScopeUsageView
err := c.get(ctx, "/api/v1/usage?by="+by+"&since=24h", &rows)
return consoleUsageMsg{rows, err}
}
}
func (m consoleModel) fetchLogs(node string) tea.Cmd {
c := m.client
return func() tea.Msg {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
var out struct {
Lines []string `json:"lines"`
}
err := c.get(ctx, "/api/v1/nodes/"+url.PathEscape(node)+"/logs?lines=200", &out)
return consoleLogsMsg{node: node, lines: out.Lines, err: err}
}
}
func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
switch msg := msg.(type) {
case tea.WindowSizeMsg:
m.width, m.height = msg.Width, msg.Height
case spinTickMsg:
var cmd tea.Cmd
m.spin, cmd = m.spin.Update(msg)
return m, cmd
case consoleTickMsg:
if m.node {
return m, nil
}
cmds := []tea.Cmd{m.fetch(), tea.Tick(3*time.Second, func(time.Time) tea.Msg { return consoleTickMsg{} })}
if m.menu == cUsage {
cmds = append(cmds, m.fetchUsage())
}
if m.menu == cQuotas {
cmds = append(cmds, m.fetchQuotas())
}
if m.menu == cMirror {
cmds = append(cmds, m.fetchMirror())
}
if m.menu == cMonitor && m.monitorDue() {
cmds = append(cmds, m.fetchMonitor())
}
if m.detailDue() {
cmds = append(cmds, m.fetchDetail())
}
if m.peopleView() {
cmds = append(cmds, m.fetchPeople())
}
if m.menu == cLogs && m.detail && m.logs.node != "" {
cmds = append(cmds, m.fetchLogs(m.logs.node))
}
return m, tea.Batch(cmds...)
case consoleDataMsg:
m.loaded = true
if msg.err != nil {
m.data.err = msg.err
} else {
m.data = consoleData(msg)
m.mon.sample(m.data)
}
case detailMsg:
if msg.key == m.detailKey() {
m.det.res, m.det.err, m.det.key = msg.res, msg.err, msg.key
}
case monitorMsg:
// Kept if it is still what the screen wants: the range and width.
if msg.key == m.monitorKey() {
m.mon.res, m.mon.err, m.mon.key = msg.res, msg.err, msg.key
}
case consoleUsageMsg:
m.usage, m.usageErr = msg.rows, msg.err
case consoleLogsMsg:
m.logs = msg
case consoleMirrorMsg:
m.mirror = msg
case consoleHFMsg:
if msg.query == m.hf.query && msg.sort == m.hf.sort && msg.st == m.hf.safetensors {
m.hf.models, m.hf.err = msg.models, msg.err
if m.hf.models == nil && msg.err == nil {
m.hf.models = []hf.Model{}
}
}
case consoleRepoMsg:
m.busy = ""
if msg.err != nil {
m.dialog, m.dialogTitle = "error", "Hugging Face"
m.dialogText = sBad.Render("✗ ") + msg.err.Error() + "\n\n" + sDim.Render("any key")
} else {
m.menuD = m.repoMenu(msg.plan)
}
case consolePeopleMsg:
m.people = peopleData(msg)
case consoleQuotasMsg:
m.quotas, m.quotasErr = msg.rows, msg.err
case actionDoneMsg:
m.busy = ""
m.dialog, m.dialogTitle = "error", msg.title
if msg.err != nil {
m.dialogText = sBad.Render("✗ ") + msg.err.Error()
} else {
m.dialogText = sOK.Render("✓ ") + msg.text
}
m.dialogText += "\n\n" + sDim.Render("any key")
return m, tea.Batch(m.fetch(), m.onEnter())
case tea.KeyMsg:
if msg.Type == tea.KeyCtrlC {
return m, tea.Quit
}
return m.key(msg)
}
return m, nil
}
func (m consoleModel) entryOK(i int) bool {
if m.node {
return i == cSettings || i == cManual || i == cLeave
}
return i != cReplicate || !m.storeReplicated()
}
func (m consoleModel) storeReplicated() bool { return len(m.data.sum.Store) > 0 }
func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
if m.busy != "" {
return m, nil
}
if m.dialog != "" {
return m.dialogKey(k)
}
if mm, cmd, ok := m.actionKey(k); ok {
return mm, cmd
}
// The manual, from anywhere: its page about this screen.
if k.String() == "?" && !(m.menu == cManual && m.detail) {
m.menu, m.inPanel, m.detail = cManual, true, true
m.docs.show(max(0, findTopic(m.docs.topics, "console")))
return m, nil
}
if m.menu == cManual && m.inPanel && m.detail {
pw := max(40, m.width-navWidth-8)
if m.docs.key(k, pw-4, m.height) {
return m, nil
}
switch k.String() {
case "esc", "left", "h", "backspace":
m.detail = false
m.row = m.docs.open
case "q":
return m.leave()
}
return m, nil
}
switch k.Type {
case tea.KeyPgDown:
m.scroll += max(1, m.height-8)
return m, nil
case tea.KeyPgUp:
m.scroll = max(0, m.scroll-max(1, m.height-8))
return m, nil
}
if !m.inPanel {
switch k.String() {
case "up", "k":
for {
m.menu = (m.menu - 1 + cCount) % cCount
if m.entryOK(m.menu) {
break
}
}
m.row, m.detail, m.scroll = 0, false, 0
return m, m.onEnter()
case "down", "j":
for {
m.menu = (m.menu + 1) % cCount
if m.entryOK(m.menu) {
break
}
}
m.row, m.detail, m.scroll = 0, false, 0
return m, m.onEnter()
case "q":
return m.leave()
case "right", "l", "tab", "enter":
switch m.menu {
case cAddNode:
return m.ask("Add a node", "user@address of the machine", 1)
case cReplicate:
return m.ask("Replicate the store", "user@address of database machine", 2)
case cLeave:
return m.leave()
case cUpgrade, cRestartAll, cBackup:
return m, m.doEntry()
case cOverview:
return m, nil
}
m.inPanel = true
}
return m, nil
}
if m.menu == cSettings {
return m.settingsKey(k)
}
n := m.rowCount()
switch k.String() {
case "up", "k":
if !m.detail && m.row > 0 {
m.row--
}
case "down", "j":
if !m.detail && m.row < n-1 {
m.row++
}
case "enter":
if m.detail || n == 0 || m.row >= n {
break
}
switch m.menu {
case cLogs:
m.detail, m.scroll = true, 0
node := m.data.nodes[m.row].ID
m.logs = consoleLogsMsg{node: node}
return m, m.fetchLogs(node)
case cNodes:
m.menuD = m.nodeMenu(m.data.nodes[m.row])
case cModels:
m.menuD = m.modelMenu(m.data.models[m.row])
case cReplicas:
m.menuD = m.replicaMenu(m.data.replicas[m.row])
case cQuotas:
m.menuD = m.quotaMenu(m.quotas[m.row])
case cMirror:
m.menuD = m.mirrorMenu(m.mirrorGroups()[m.row])
case cJobs:
m.dialog = "error"
m.dialogTitle, m.dialogText = jobDialog(m.data.jobs[m.row])
case cAudit:
m.dialog = "error"
m.dialogTitle, m.dialogText = auditDialog(m.data.audit[m.row])
case cServe:
return m, m.fetchRepo(m.hf.models[m.row].ID)
case cAccounts:
m.menuD = m.userMenu(m.people.users[m.row])
case cKeys:
m.menuD = m.keyMenu(m.people.keys[m.row])
case cOrgs:
m.menuD = m.orgMenu(m.people.orgs[m.row])
case cManual:
m.docs.show(m.row)
m.detail = true
}
case "1", "2", "3", "4", "r":
if kind, _ := m.detailTarget(); m.menu == cMonitor || kind != "" {
if k.String() == "r" {
m.mon.rng = (m.mon.rng + 1) % len(monRanges)
} else {
m.mon.rng = int(k.String()[0] - '1')
}
if kind != "" {
return m, m.fetchDetail()
}
return m, m.fetchMonitor()
}
case "/":
if m.menu == cServe {
m.hfSearchDialog()
}
case "f":
if m.menu == cServe {
m.hf.safetensors = !m.hf.safetensors
m.hf.models, m.hf.err, m.row = nil, nil, 0
return m, m.fetchHF()
}
case "s":
if m.menu == cServe {
m.hf.sort = (m.hf.sort + 1) % len(hfSorts)
m.hf.models, m.hf.err, m.row = nil, nil, 0
return m, m.fetchHF()
}
case "a":
switch m.menu {
case cQuotas:
m.quotaForm(adminapi.QuotaView{})
case cModels:
m.addModelForm()
case cMirror:
m.mirrorFetchForm()
case cAccounts:
m.createUserForm()
case cKeys:
m.createKeyForm()
case cOrgs:
m.createOrgForm()
}
case "right", "l":
if m.menu == cUsage {
m.usageBy = (m.usageBy + 1) % len(usageGroupings)
return m, m.fetchUsage()
}
case "esc", "left", "h", "backspace":
if m.detail {
m.detail, m.scroll = false, 0
} else {
m.inPanel = false
}
case "q":
return m.leave()
}
return m, nil
}
// onEnter loads what a view needs when it is chosen.
func (m consoleModel) onEnter() tea.Cmd {
switch m.menu {
case cUsage:
return m.fetchUsage()
case cQuotas:
return m.fetchQuotas()
case cMirror:
return m.fetchMirror()
case cMonitor:
return m.fetchMonitor()
case cServe:
if m.hf.models == nil {
return m.fetchHF()
}
case cAccounts, cKeys, cOrgs:
// The quotas too, for an organization's quota form.
return tea.Batch(m.fetchPeople(), m.fetchQuotas())
}
return nil
}
func (m consoleModel) peopleView() bool {
return m.menu == cAccounts || m.menu == cKeys || m.menu == cOrgs
}
func (m consoleModel) leave() (tea.Model, tea.Cmd) {
if m.doc != nil && m.doc.Changed > 0 {
m.dialog, m.dialogTitle = "confirm-leave", "Unsaved changes"
m.dialogText = fmt.Sprintf("%d change(s) to %s are not saved.\nLeave without saving them?\n\n%s",
m.doc.Changed, m.file, sDim.Render("Y leave · N stay"))
return m, nil
}
return m, tea.Quit
}
func (m consoleModel) ask(title, question string, n int) (tea.Model, tea.Cmd) {
m.dialog, m.dialogTitle, m.dialogText, m.asked = "target", title, question, nil
m.prompt = newTextField()
m.prompt.Placeholder, m.prompt.Width = "ubuntu@10.0.0.20", 40
m.prompt.Focus()
m.editKind = settings.Kind(n) // how many to ask for
return m, nil
}
func (m consoleModel) dialogKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
switch m.dialog {
case "error":
m.dialog = ""
return m, nil
case "confirm-leave":
if strings.EqualFold(k.String(), "y") {
return m, tea.Quit
}
m.dialog = ""
return m, nil
case "saved":
if strings.EqualFold(k.String(), "y") {
m.action = consoleAction{kind: "restart"}
return m, tea.Quit
}
m.dialog = ""
return m, nil
}
switch k.Type {
case tea.KeyEsc:
m.dialog = ""
return m, nil
case tea.KeyEnter:
v := strings.TrimSpace(m.prompt.Value())
switch m.dialog {
case "target":
if v == "" {
return m, nil
}
m.asked = append(m.asked, v)
m.prompt.SetValue("")
if len(m.asked) == int(m.editKind) {
kind := "node"
if m.menu == cReplicate {
kind = "patroni"
}
m.action = consoleAction{kind: kind, targets: m.asked}
return m, tea.Quit
}
case "edit":
if err := m.doc.Set(m.editPath, m.editKind, m.prompt.Value()); err != nil {
m.dialogText = m.editQuestion() + "\n\n" + sBad.Render(err.Error())
return m, nil
}
m.dialog = ""
case "addname":
if v == "" {
return m, nil
}
if err := m.doc.AddEntry(m.editPath, v); err != nil {
m.dialogText = "Name of the new entry\n\n" + sBad.Render(err.Error())
return m, nil
}
m.dialog = ""
}
return m, nil
}
var cmd tea.Cmd
m.prompt, cmd = m.prompt.Update(k)
return m, cmd
}
// --- settings ---
// settingsRow is one line of the settings view.
type settingsRow struct {
label, value string
key *settings.Key
// index is a list entry's position, or a map entry's name.
entry string
add bool
}
func (m consoleModel) level() settingsLevel { return m.levels[len(m.levels)-1] }
func (m consoleModel) settingsRows() []settingsRow {
lv := m.level()
var rows []settingsRow
switch lv.kind {
case settings.Block:
for _, k := range settings.Keys(lv.block) {
k := k
path := append(append([]string{}, lv.path...), k.Key)
rows = append(rows, settingsRow{label: k.Key, value: m.valueOf(k, path), key: &k})
}
case settings.BlockList:
for i := 0; i < m.doc.Len(lv.path); i++ {
rows = append(rows, settingsRow{label: m.entryLabel(append(append([]string{}, lv.path...), strconv.Itoa(i)), i),
value: "›", entry: strconv.Itoa(i)})
}
rows = append(rows, settingsRow{label: "+ add an entry", add: true})
case settings.BlockMap:
for _, name := range m.doc.MapKeys(lv.path) {
rows = append(rows, settingsRow{label: name, value: "›", entry: name})
}
rows = append(rows, settingsRow{label: "+ add an entry", add: true})
}
return rows
}
// entryLabel names a list entry by its name or id, else its position.
func (m consoleModel) entryLabel(path []string, i int) string {
for _, k := range []string{"name", "id", "key", "url"} {
if v, ok := m.doc.Text(append(path, k)); ok && v != "" {
return v
}
}
return fmt.Sprintf("entry %d", i+1)
}
func (m consoleModel) valueOf(k settings.Key, path []string) string {
switch k.Kind {
case settings.Block:
return "›"
case settings.BlockList, settings.BlockMap:
return fmt.Sprintf("%d ›", m.doc.Len(path))
}
if v, ok := m.doc.Text(path); ok {
return strings.ReplaceAll(v, "\n", ", ")
}
if v, ok := settings.Effective(m.cfg, path); ok && v != "" {
return sDim.Render(v + " (default)")
}
return sDim.Render("—")
}
func (m consoleModel) editQuestion() string {
hint := map[settings.Kind]string{
settings.Duration: "a duration: 30s, 5m, 2h",
settings.Int: "a whole number",
settings.Float: "a number",
settings.TextList: "values, separated by commas",
settings.TextMap: "key = value, separated by commas",
}[m.editKind]
q := strings.Join(m.editPath, ".")
if hint != "" {
q += "\n" + sDim.Render(hint)
}
return q
}
func (m consoleModel) settingsKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
if m.doc == nil {
if k.String() == "esc" || k.String() == "left" {
m.inPanel = false
}
return m, nil
}
rows := m.settingsRows()
lv := m.level()
switch k.String() {
case "up", "k":
if m.row > 0 {
m.row--
}
case "down", "j":
if m.row < len(rows)-1 {
m.row++
}
case "esc", "left", "h", "backspace":
if len(m.levels) > 1 {
m.levels = m.levels[:len(m.levels)-1]
m.row, m.scroll = 0, 0
} else {
m.inPanel = false
}
case "s":
return m.save()
case "u", "delete":
if m.row >= len(rows) {
return m, nil
}
r := rows[m.row]
switch {
case r.key != nil:
m.doc.Unset(append(append([]string{}, lv.path...), r.key.Key))
case r.entry != "":
m.doc.Unset(append(append([]string{}, lv.path...), r.entry))
m.row = max(0, m.row-1)
}
case "enter", "right", "l":
if m.row >= len(rows) {
return m, nil
}
r := rows[m.row]
switch {
case r.add && lv.kind == settings.BlockList:
i := m.doc.Append(lv.path)
m.levels = append(m.levels, settingsLevel{title: fmt.Sprintf("entry %d", i+1), block: lv.block,
path: append(append([]string{}, lv.path...), strconv.Itoa(i)), kind: settings.Block})
m.row = 0
case r.add && lv.kind == settings.BlockMap:
m.dialog, m.dialogTitle, m.dialogText = "addname", "Add an entry", "Name of the new entry"
m.editPath = lv.path
m.prompt = newTextField()
m.prompt.Width = 40
m.prompt.Focus()
case r.entry != "":
m.levels = append(m.levels, settingsLevel{title: r.label, block: lv.block,
path: append(append([]string{}, lv.path...), r.entry), kind: settings.Block})
m.row = 0
case r.key != nil:
path := append(append([]string{}, lv.path...), r.key.Key)
switch r.key.Kind {
case settings.Block:
m.levels = append(m.levels, settingsLevel{title: r.key.Key, block: r.key.Block, path: path, kind: settings.Block})
m.row = 0
case settings.BlockList, settings.BlockMap:
m.levels = append(m.levels, settingsLevel{title: r.key.Key, block: r.key.Block, path: path, kind: r.key.Kind})
m.row = 0
case settings.Bool:
cur, ok := m.doc.Text(path)
if !ok {
cur, _ = settings.Effective(m.cfg, path)
}
b, _ := strconv.ParseBool(cur)
_ = m.doc.Set(path, settings.Bool, strconv.FormatBool(!b))
default:
m.editPath, m.editKind = path, r.key.Kind
cur, ok := m.doc.Text(path)
if !ok {
cur, _ = settings.Effective(m.cfg, path)
}
m.prompt = newTextField()
m.prompt.Width = 50
m.prompt.SetValue(strings.ReplaceAll(cur, "\n", ", "))
m.prompt.Focus()
m.dialog, m.dialogTitle, m.dialogText = "edit", "Change "+r.key.Key, m.editQuestion()
}
}
m.scroll = 0
}
return m, nil
}
func (m consoleModel) save() (tea.Model, tea.Cmd) {
if m.doc.Changed == 0 {
m.dialog, m.dialogTitle, m.dialogText = "error", "Nothing to save", "No change has been made."
return m, nil
}
backup, err := m.doc.Save(func(p string) error { _, err := m.load(p); return err })
if err != nil {
m.dialog, m.dialogTitle = "error", "Not saved"
m.dialogText = "The configuration would not load:\n\n" + sBad.Render(err.Error()) + "\n\n" + sDim.Render("Nothing was written. Any key to go back.")
return m, nil
}
m.loadSettings()
note := ""
if backup != "" {
note = "The old file is " + backup + ".\n"
}
m.dialog, m.dialogTitle = "saved", "Saved"
m.dialogText = "Saved " + m.file + ".\n" + note +
fmt.Sprintf("\n%s reads it when it starts. Restart it now?\n", m.service())
if !m.node {
m.dialogText += sDim.Render("Another control plane keeps its own copy: make the change there too.") + "\n"
}
m.dialogText += "\n" + sDim.Render("Y restart · N later")
return m, nil
}
// --- views ---
func (m consoleModel) rowCount() int {
switch m.menu {
case cNodes, cLogs:
return len(m.data.nodes)
case cGPUs:
return len(m.gpuRows())
case cModels:
return len(m.data.models)
case cReplicas:
return len(m.data.replicas)
case cUsage:
return len(m.usage)
case cQuotas:
return len(m.quotas)
case cMirror:
return len(m.mirrorGroups())
case cJobs:
return len(m.data.jobs)
case cAudit:
return len(m.data.audit)
case cServe:
return len(m.hf.models)
case cAccounts:
return len(m.people.users)
case cKeys:
return len(m.people.keys)
case cOrgs:
return len(m.people.orgs)
case cManual:
return len(m.docs.topics)
}
return 0
}
func (m consoleModel) View() string {
host, _ := os.Hostname()
nav := []navItem{{Label: "Watch", Heading: true}}
if m.node {
nav = nil
}
for i := 0; i < cCount; i++ {
if m.node && !m.entryOK(i) {
continue
}
switch i {
case cAccounts:
nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "People", Heading: true})
case cManual:
nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "Help", Heading: true})
case cSettings:
nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "Settings", Heading: true})
case cAddNode, cLeave:
if (i == cAddNode) != m.node {
nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "Do", Heading: true})
}
}
label := consoleLabels[i]
if i == cSettings {
label = filepathBase(m.file)
}
if n := runningJobs(m.data.jobs); i == cJobs && n > 0 {
label += fmt.Sprintf(" (%d)", n)
}
if i == cSettings && m.doc != nil && m.doc.Changed > 0 {
label += fmt.Sprintf(" (%d)", m.doc.Changed)
}
nav = append(nav, navItem{Label: label, Current: i == m.menu, Dim: !m.entryOK(i)})
}
title, body, keys := m.panel()
title0 := "llmgw console"
if m.node {
title0 = "llmgw · node"
}
f := frame{
Width: m.width, Height: m.height, Title: title0, Where: host,
NavTitle: "", Nav: nav, PanelTitle: title, Body: body, Scroll: m.scroll, Keys: keys,
}
if m.menu == cManual && m.detail {
f.Scroll = m.docs.scroll
}
// The selected row kept on screen.
if m.inPanel && !m.detail {
inner := m.height - 4
top := m.row + 3
if top-m.scroll >= inner-1 {
f.Scroll = top - inner + 2
}
}
if m.dialog != "" {
f.DialogTitle = m.dialogTitle
f.Dialog = m.dialogText
switch m.dialog {
case "target", "edit", "addname":
q := m.dialogText
if m.dialog == "target" && m.editKind > 1 {
q = fmt.Sprintf("%s %d of %d", q, len(m.asked)+1, int(m.editKind))
}
f.Dialog = q + "\n\n" + m.prompt.View() + "\n\n" + sDim.Render("Enter · Esc cancel")
}
} else if t, d := m.docs.dialog(); d != "" && m.menu == cManual {
f.DialogTitle, f.Dialog = t, d
} else if m.busy != "" {
f.DialogTitle, f.Dialog = m.busy, m.spin.View()+" asking the control plane…"
} else if t, d := m.actionDialog(); d != "" {
f.DialogTitle, f.Dialog = t, d
}
return f.render()
}
// panel is the right-hand side for the entry chosen.
func (m consoleModel) panel() (title, body, keys string) {
pw := max(40, m.width-navWidth-8)
rowKeys := "↑↓ row · Enter actions · ← back · PgUp/PgDn · Q leave"
if !m.inPanel {
keys = "↑↓ choose · → or Enter open · Q leave · refreshed every three seconds"
}
waiting := func() string {
if m.data.err != nil {
return "\n " + sBad.Render("The control plane does not answer: ") + m.data.err.Error() +
"\n\n " + sDim.Render("systemctl status llmgw-server says why.")
}
return "\n " + m.spin.View() + " asking the control plane…"
}
if (!m.loaded || m.data.err != nil) && !m.node {
if m.menu < cSettings {
return consoleLabels[m.menu], waiting(), keys
}
}
switch m.menu {
case cOverview:
var b bytes.Buffer
c := &cobra.Command{}
c.SetOut(&b)
printSummary(c, m.data.sum)
return "Overview", "\n" + indent(b.String(), " "), keys
case cNodes, cLogs:
if m.detail && m.row < len(m.data.nodes) {
n := m.data.nodes[m.row]
if m.menu == cLogs {
return "Agent log · " + n.ID, m.logView(), "← back · PgUp/PgDn · refreshed every three seconds"
}
return "Node · " + n.ID, m.nodeDetail(n) + m.detailCharts(pw), "← back · 1–4 or R range · PgUp/PgDn · Q leave"
}
var rows [][]string
for _, n := range m.data.nodes {
cpu, mem := "—", "—"
if n.Load != nil {
cpu, mem = fmt.Sprintf("%.0f%%", n.Load.CPUPercent), fmt.Sprintf("%.0f%%", n.Load.MemPercent)
}
state := n.State
if state != "ready" {
state = sWarn.Render(state)
}
rows = append(rows, []string{n.ID, state, cpu, mem, fmt.Sprintf("%d/%d", n.GPUFree, n.GPUTotal),
strconv.Itoa(n.Replicas), fmt.Sprintf("%d/%d GB", n.DiskFreeGB, n.DiskTotalGB), n.AgentVersion, n.LastSeenAgo})
}
t := "Nodes"
if m.menu == cLogs {
t = "Logs · choose a node"
}
if m.inPanel {
keys = rowKeys
}
return t, "\n" + table([]string{"NODE", "STATE", "CPU", "MEM", "GPU FREE", "REPLICAS", "DISK FREE", "AGENT", "SEEN"},
rows, m.row, pw, m.inPanel), keys
case cGPUs:
if m.inPanel {
keys = "↑↓ row · ← back · PgUp/PgDn · Q leave"
}
return "GPUs", "\n" + table([]string{"NODE", "GPU", "CARD", "BUSY", "MEMORY", "TEMP", "POWER", "HELD BY"},
m.gpuRows(), m.row, pw, m.inPanel), keys
case cModels:
if m.detail && m.row < len(m.data.models) {
return "Model · " + m.data.models[m.row].Name, m.modelDetail(m.data.models[m.row]) + m.detailCharts(pw),
"← back · 1–4 or R range · PgUp/PgDn · Q leave"
}
var rows [][]string
for _, md := range m.data.models {
status := "stopped"
switch {
case md.Parked:
status = sBad.Render("parked")
case md.Unplaceable:
status = sWarn.Render("not placed")
case md.Ready > 0 && md.Ready >= md.Desired:
status = sOK.Render("ready")
case md.Desired > 0:
status = sWarn.Render("starting")
}
warn := ""
if len(md.Warnings) > 0 {
warn = sWarn.Render("!")
}
upd := ""
if md.Update != nil && md.Update.Available {
upd = sWarn.Render("↑ " + catalog.ShortRevision(md.Update.Latest))
}
rows = append(rows, []string{md.Name, fmt.Sprintf("%d/%d", md.Ready, md.Desired), status, md.Class,
strings.Join(md.Profiles, ","), upd, warn})
}
if m.inPanel {
keys = "↑↓ row · Enter actions · A add from a file · ← back · Q leave"
}
return "Models", "\n" + table([]string{"MODEL", "READY", "STATUS", "CLASS", "PROFILES", "UPDATE", ""}, rows, m.row, pw, m.inPanel), keys
case cReplicas:
var rows [][]string
for _, r := range m.data.replicas {
var nodes []string
for _, s := range r.Shards {
nodes = append(nodes, fmt.Sprintf("%s%v", s.Node, s.GPUs))
}
rows = append(rows, []string{r.ID, r.Model, r.State, r.Pool, strings.Join(nodes, " "),
(time.Duration(r.AgeSec) * time.Second).Round(time.Second).String()})
}
if m.inPanel {
keys = rowKeys
}
return "Replicas", "\n" + table([]string{"REPLICA", "MODEL", "STATE", "POOL", "ON", "AGE"}, rows, m.row, pw, m.inPanel), keys
case cUsage:
head := "\n Last 24 hours, by " + lipgloss.NewStyle().Bold(true).Render(usageGroupings[m.usageBy]) +
sDim.Render(" (→ to group by "+usageGroupings[(m.usageBy+1)%len(usageGroupings)]+")") + "\n\n"
if m.usageErr != nil {
return "Usage", head + " " + sBad.Render(m.usageErr.Error()), keys
}
var rows [][]string
for _, u := range m.usage {
rows = append(rows, []string{u.Name, compactNum(u.Requests), compactNum(u.Prompt + u.Completion),
compactNum(u.Errors), fmt.Sprintf("%.2f", float64(u.CostMicros)/1e6)})
}
if m.inPanel {
keys = "↑↓ row · → grouping · ← back · Q leave"
}
return "Usage", head + table([]string{"WHO", "REQUESTS", "TOKENS", "ERRORS", "SPENT"}, rows, m.row, pw, m.inPanel), keys
case cQuotas:
return m.quotasPanel(pw)
case cAccounts, cKeys, cOrgs:
return m.peoplePanel(pw)
case cManual:
if m.detail && m.docs.open >= 0 {
k := docKeys
if m.docs.note != "" {
k = m.docs.note
}
return "Manual · " + m.docs.topics[m.docs.open].title, "\n" + m.docs.body(pw-4), k
}
d := m.docs
d.sel = m.row
if m.inPanel {
keys = "↑↓ topic · Enter read · ← back · ? the console's page · Q leave"
}
return "Manual", d.list(pw, m.inPanel), keys
case cMirror:
return m.mirrorPanel(pw)
case cJobs:
return m.jobsPanel(pw)
case cAudit:
return m.auditPanel(pw)
case cMonitor:
return m.monitorPanel(pw)
case cServe:
return m.hfPanel(pw)
case cSettings:
return m.settingsPanel(pw)
}
var item string
switch m.menu {
case cAddNode:
item = "\n Adds a machine as a node: installs llmgw on it over ssh, checks it,\n gives it a certificate and starts its agent.\n\n " +
sDim.Render("On the command line:") + "\n sudo llmgw node add <user>@<address>"
case cReplicate:
item = "\n Moves the store onto two machines under Patroni, replicated,\n so losing one loses nothing.\n\n " +
sDim.Render("On the command line:") +
"\n sudo --preserve-env=SSH_AUTH_SOCK llmgw server init \\\n --patroni <user>@<a> --patroni <user>@<b> --no-mirror --no-node --no-model"
case cUpgrade:
item = "\n Moves the cluster to a release: this machine first, then each node\n in turn. Engines keep serving; a node that does not come back stops it.\n\n " +
sDim.Render("On the command line:") + "\n sudo llmgw ctl upgrade VERSION"
case cRestartAll:
item = "\n Replaces every replica, one at a time, each new one up before the\n old one stops.\n\n " +
sDim.Render("On the command line:") + "\n llmgw ctl restart --all"
case cBackup:
item = "\n Writes the store and /etc/llmgw to one archive in " + DefaultBackupDir + ",\n and keeps the last few. Copy it off this machine.\n\n " +
sDim.Render("On the command line:") + "\n sudo llmgw server backup"
case cLeave:
item = "\n Closes the console. The cluster keeps running."
}
return consoleLabels[m.menu], item, "Enter do it · ↑↓ choose · Q leave"
}
func (m consoleModel) settingsPanel(pw int) (string, string, string) {
if m.docErr != nil {
return "server.yaml", "\n " + sBad.Render(m.docErr.Error()), "← back"
}
var crumbs []string
for _, l := range m.levels {
crumbs = append(crumbs, l.title)
}
rows := m.settingsRows()
var tr [][]string
for _, r := range rows {
typ := ""
if r.key != nil {
typ = sDim.Render(r.key.Type)
}
tr = append(tr, []string{r.label, r.value, typ})
}
body := "\n" + table([]string{"KEY", "VALUE", "TYPE"}, tr, m.row, pw, m.inPanel)
// What the chosen key is for, from its comment in the code.
if m.inPanel && m.row < len(rows) {
doc := ""
if r := rows[m.row]; r.key != nil {
doc = r.key.Doc
if doc == "" && r.key.Block != nil {
doc = r.key.Block.Doc
}
} else if l := m.level(); l.block != nil {
doc = l.block.Doc
}
if doc != "" {
body += "\n " + sDim.Render(strings.Repeat("─", min(pw-4, 60))) + "\n" +
indent(wrap(firstParagraphs(doc, 2), pw-4, ""), " ")
}
}
keys := "→ or Enter open · ↑↓ row · Q leave"
if m.inPanel {
keys = "Enter change/open · U back to default · S save · Esc up · PgUp/PgDn"
if l := m.level(); l.kind != settings.Block {
keys = "Enter open · D remove entry · S save · Esc up"
}
}
return strings.Join(crumbs, " › "), body, keys
}
func firstParagraphs(s string, n int) string {
p := strings.Split(s, "\n\n")
if len(p) > n {
p = p[:n]
}
return strings.Join(p, "\n\n")
}
func (m consoleModel) gpuRows() [][]string {
held := map[string]string{}
for _, r := range m.data.replicas {
for _, s := range r.Shards {
for _, g := range s.GPUs {
held[s.Node+"/"+strconv.Itoa(g)] = r.Model
}
}
}
var rows [][]string
for _, n := range m.data.nodes {
for _, g := range n.GPUs {
busy, mem, temp, power := "—", "—", "—", "—"
if n.Load != nil {
for _, s := range n.Load.GPUs {
if s.Index != g.Index {
continue
}
busy = fmt.Sprintf("%.0f%%", s.UtilPercent)
if s.MemoryTotalMB > 0 {
mem = fmt.Sprintf("%.1f/%.0f GB", s.MemoryUsedMB/1024, s.MemoryTotalMB/1024)
}
if s.TempC > 0 {
temp = fmt.Sprintf("%.0f°C", s.TempC)
}
if s.PowerW > 0 {
power = fmt.Sprintf("%.0f/%.0f W", s.PowerW, s.PowerLimitW)
}
}
}
holder := held[n.ID+"/"+strconv.Itoa(g.Index)]
if holder == "" {
holder = sDim.Render("free")
}
rows = append(rows, []string{n.ID, strconv.Itoa(g.Index), g.Name, busy, mem, temp, power, holder})
}
}
return rows
}
func (m consoleModel) nodeDetail(n adminapi.NodeView) string {
var b strings.Builder
kv := func(k, v string) { fmt.Fprintf(&b, " %-14s %s\n", sDim.Render(k), v) }
b.WriteString("\n")
kv("state", n.State)
kv("address", n.MgmtAddr)
kv("system", n.OS)
kv("agent", n.AgentVersion+", seen "+n.LastSeenAgo+" ago")
if n.Load != nil {
l := n.Load
kv("cpu", fmt.Sprintf("%.0f%% of %d · load %.2f %.2f %.2f", l.CPUPercent, l.CPUs, l.Load1, l.Load5, l.Load15))
kv("memory", fmt.Sprintf("%.0f%% · %.1f of %.1f GB", l.MemPercent, float64(l.MemUsed)/(1<<30), float64(l.MemTotal)/(1<<30)))
}
kv("disk", fmt.Sprintf("%d GB free of %d", n.DiskFreeGB, n.DiskTotalGB))
kv("replicas", strconv.Itoa(n.Replicas))
if len(n.Provision) > 0 {
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("Engines") + "\n")
for _, p := range n.Provision {
fmt.Fprintf(&b, " %-9s %-10s %-8s %s\n", p.Engine, p.Version, p.State, sDim.Render(truncate(p.Detail, 60)))
}
}
if len(n.GPUs) > 0 {
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("GPUs") + "\n")
for _, r := range m.gpuRows() {
if r[0] == n.ID {
fmt.Fprintf(&b, " %s %-18s busy %-5s mem %-14s %-6s %s %s\n", r[1], r[2], r[3], r[4], r[5], r[6], r[7])
}
}
}
for _, w := range append(n.Warnings, n.Unhealthy...) {
b.WriteString("\n " + sWarn.Render("! ") + w)
}
return b.String()
}
func (m consoleModel) modelDetail(md adminapi.ModelView) string {
var b strings.Builder
kv := func(k, v string) {
if v != "" {
fmt.Fprintf(&b, " %-14s %s\n", sDim.Render(k), v)
}
}
b.WriteString("\n")
if md.Description != "" {
b.WriteString(" " + md.Description + "\n\n")
}
kv("replicas", fmt.Sprintf("%d ready of %d wanted (min %d, max %d)", md.Ready, md.Desired, md.Min, md.Max))
kv("class", md.Class)
kv("owner", md.Owner)
kv("profiles", strings.Join(md.Profiles, ", "))
kv("aliases", strings.Join(md.Aliases, ", "))
kv("source", md.Source)
if o := md.Origin; o != nil {
from := o.Repo + " at " + catalog.ShortRevision(o.Revision)
if o.Previous != "" {
from += ", updated from " + catalog.ShortRevision(o.Previous)
}
kv("from", from)
}
if u := md.Update; u != nil {
kv("registry", updateCell(*u)+sDim.Render(", asked "+ago(u.CheckedAt, time.Now())))
}
if md.MemoryGB > 0 {
kv("reserves", fmt.Sprintf("%d GB, %d cores (uses %.1f GB)", md.MemoryGB, md.CPUCores, md.MeasuredMemoryGB))
}
kv("fallbacks", strings.Join(md.Fallbacks, ", "))
if md.Reason != "" {
b.WriteString("\n " + sWarn.Render("! ") + md.Reason + "\n")
}
for _, w := range md.Warnings {
b.WriteString(" " + sWarn.Render("! ") + w + "\n")
}
var reps [][]string
for _, r := range m.data.replicas {
if r.Model == md.Name {
reps = append(reps, []string{r.ID, r.State, r.Pool, r.Endpoint})
}
}
if len(reps) > 0 {
b.WriteString("\n" + table([]string{"REPLICA", "STATE", "POOL", "ENDPOINT"}, reps, -1, max(40, m.width-navWidth-8), false))
}
return b.String()
}
func (m consoleModel) logView() string {
if m.logs.err != nil {
return "\n " + sBad.Render(m.logs.err.Error())
}
if len(m.logs.lines) == 0 {
return "\n " + m.spin.View() + " reading the log…"
}
// The newest at the bottom, where a log is read; the panel starts there.
lines := m.logs.lines
if k := m.height - 6; len(lines) > k && m.scroll == 0 {
lines = lines[len(lines)-k:]
}
var b strings.Builder
b.WriteString("\n")
for _, l := range lines {
switch {
case strings.Contains(l, "level=ERROR"):
l = sBad.Render(l)
case strings.Contains(l, "level=WARN"):
l = sWarn.Render(l)
}
b.WriteString(" " + l + "\n")
}
return b.String()
}
func compactNum(n int64) string {
switch {
case n >= 1_000_000:
return fmt.Sprintf("%.1fM", float64(n)/1e6)
case n >= 1_000:
return fmt.Sprintf("%.1fk", float64(n)/1e3)
}
return strconv.FormatInt(n, 10)
}
func indent(s, pre string) string {
lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
for i := range lines {
lines[i] = pre + lines[i]
}
return strings.Join(lines, "\n")
}
func filepathBase(p string) string {
if i := strings.LastIndexByte(p, '/'); i >= 0 {
return p[i+1:]
}
return p
}
// runConsole shows the console, runs what is chosen on it, and comes back.
// file is server.yaml on a control plane, agent.yaml on a node.
func runConsole(ctx context.Context, out io.Writer, file string) error {
addr := "127.0.0.1:9090"
if cfg, err := config.Load(file); err == nil {
if cfg.AdminListen != "" {
addr = cfg.AdminListen
if strings.HasPrefix(addr, ":") {
addr = "127.0.0.1" + addr
}
}
// The admin token, for this screen and every command it runs in
// this process — `llmgw ctl` included — from this configuration's
// file rather than only the usual place.
if t := readAdminToken(adminTokenPath(file, cfg)); t != "" {
_ = os.Setenv("LLMGW_ADMIN_TOKEN", t)
}
}
menu := cOverview
if strings.HasSuffix(file, "agent.yaml") {
menu = cSettings
}
for {
m := newConsoleModel(addr, file)
m.menu = menu
final, err := tea.NewProgram(m, tea.WithOutput(out), tea.WithAltScreen()).Run()
if err != nil {
return err
}
fm := final.(consoleModel)
fm.st.close()
menu = fm.menu
act := fm.action
switch act.kind {
case "":
return nil
case "node":
c := newNodeAddCmd()
c.SetArgs(act.targets)
c.SetOut(out)
err = c.ExecuteContext(ctx)
case "patroni":
in := newDefaultServerInit()
in.patroni = act.targets
// Already set up: only the store changes.
in.noMirror, in.noNode, in.noModel = true, true, true
err = in.runInitTUI(ctx, out)
case "run":
c := Root()
c.SetArgs(act.targets)
c.SetOut(out)
c.SetErr(out)
fmt.Fprintln(out, sDim.Render("llmgw "+strings.Join(act.targets, " ")))
err = c.ExecuteContext(ctx)
case "restart":
fmt.Fprintf(out, "restarting %s…\n", fm.service())
if b, e := exec.CommandContext(ctx, "systemctl", "restart", fm.service()).CombinedOutput(); e != nil {
err = fmt.Errorf("%v: %s", e, strings.TrimSpace(string(b)))
} else {
fmt.Fprintln(out, sOK.Render("✓ ")+"restarted; it picks up the configuration as it starts")
}
}
if err != nil {
fmt.Fprintln(out, sBad.Render("✗ ")+err.Error())
}
fmt.Fprint(out, "\n"+sDim.Render("enter to go back to the console")+" ")
_, _ = bufio.NewReader(os.Stdin).ReadString('\n')
}
}