package cli import ( "bufio" "bytes" "context" "fmt" "io" "net/url" "os" "os/exec" "strconv" "strings" "time" tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" "github.com/spf13/cobra" "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" "gitea.mynodes.xyz/ilker/llmgw/internal/docgen" "gitea.mynodes.xyz/ilker/llmgw/internal/hf" "gitea.mynodes.xyz/ilker/llmgw/internal/settings" ) // The console: what a bare `sudo llmgw` opens on a control plane. Down the // left, three groups — what is running (watched live), the settings (every // key of server.yaml, from the generated reference), and what can be done. // Sidebar entries, in order. Headings are not entries. const ( cOverview = iota cMonitor cNodes cGPUs cModels cReplicas cUsage cQuotas cMirror cJobs cLogs cAccounts cKeys cOrgs cManual cSettings cAddNode cReplicate cServe cUpgrade cRestartAll cBackup cLeave cCount ) var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Manual", "server.yaml", "Add a node", "Replicate the store", "Serve a model", "Upgrade the cluster", "Restart all replicas", "Back up now", "Leave"} // consoleAction is what is done once the screen closes: each runs its own // screen, and the console comes back after. type consoleAction struct { kind string // "", "node", "patroni", "restart", "run" (llmgw with targets as its arguments) targets []string } type consoleData struct { sum adminapi.Summary nodes []adminapi.NodeView models []adminapi.ModelView replicas []adminapi.ReplicaView jobs []modelserve.Job err error } type ( consoleDataMsg consoleData consoleUsageMsg struct { rows []adminapi.ScopeUsageView err error } consoleLogsMsg struct { node string lines []string err error } consoleTickMsg struct{} ) // settingsLevel is where the settings view is: a block, or a list or map of // blocks, at a path of keys in the file. type settingsLevel struct { title string block *docgen.Block path []string kind settings.Kind // Block, BlockList or BlockMap } type consoleModel struct { client *ctlClient width, height int spin spinnerModel data consoleData loaded bool usage []adminapi.ScopeUsageView // usageBy is the grouping on the usage view. usageBy int usageErr error logs consoleLogsMsg // menu is the sidebar entry; inPanel says the keys go to the panel. menu int inPanel bool row int // detail is the panel showing one row's detail rather than the table. detail bool scroll int // node is the console of a node: its agent.yaml and nothing to watch, // since the cluster is watched from the control plane. node bool // The settings. file string doc *settings.Doc cfg any // the file as loaded, for the values that apply levels []settingsLevel docErr error saveNote string // A question open as a dialog. dialog string // "", "target", "edit", "addname", "saved", "confirm-leave", "error" dialogTitle string dialogText string prompt textField asked []string editPath []string editKind settings.Kind action consoleAction // What can be done to a row: a menu, a form, a question, and the request // under way. menuD *menuDialog formD *formDialog confirmD *confirmDialog busy string quotas []adminapi.QuotaView quotasErr error st *storeHolder people peopleData mirror consoleMirrorMsg hf hfState docs docViewer mon monitorState det monitorState } var usageGroupings = []string{"team", "project", "org"} func newConsoleModel(addr, file string) consoleModel { m := consoleModel{client: newCtlClient(addr), spin: newSpinner(), width: 100, height: 30, file: file, st: &storeHolder{file: file}, docs: newDocViewer()} m.node = strings.HasSuffix(file, "agent.yaml") if m.node { m.menu = cSettings } m.loadSettings() return m } // load reads the file as the service would, defaults and checks included. func (m consoleModel) load(path string) (any, error) { if m.node { return config.LoadAgent(path) } return config.Load(path) } func (m consoleModel) service() string { if m.node { return "llmgw-agent" } return "llmgw-server" } func (m *consoleModel) loadSettings() { m.doc, m.docErr = settings.Open(m.file) if m.docErr != nil { return } if cfg, err := m.load(m.file); err == nil { m.cfg = cfg } else if !m.node { m.cfg = config.Default() } if root, err := settings.Root(m.file); err == nil && len(m.levels) == 0 { m.levels = []settingsLevel{{title: filepathBase(m.file), block: root, kind: settings.Block}} } } func (m consoleModel) Init() tea.Cmd { if m.node { return m.spin.Tick } return tea.Batch(m.fetch(), m.spin.Tick, tea.Tick(3*time.Second, func(time.Time) tea.Msg { return consoleTickMsg{} })) } // fetch reads everything the watched views show, in one go. func (m consoleModel) fetch() tea.Cmd { c := m.client return func() tea.Msg { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() var d consoleData if d.err = c.get(ctx, "/api/v1/summary", &d.sum); d.err != nil { return consoleDataMsg(d) } _ = c.get(ctx, "/api/v1/nodes", &d.nodes) _ = c.get(ctx, "/api/v1/models", &d.models) _ = c.get(ctx, "/api/v1/replicas", &d.replicas) _ = c.get(ctx, "/api/v1/jobs", &d.jobs) return consoleDataMsg(d) } } func (m consoleModel) fetchUsage() tea.Cmd { c, by := m.client, usageGroupings[m.usageBy] return func() tea.Msg { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() var rows []adminapi.ScopeUsageView err := c.get(ctx, "/api/v1/usage?by="+by+"&since=24h", &rows) return consoleUsageMsg{rows, err} } } func (m consoleModel) fetchLogs(node string) tea.Cmd { c := m.client return func() tea.Msg { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() var out struct { Lines []string `json:"lines"` } err := c.get(ctx, "/api/v1/nodes/"+url.PathEscape(node)+"/logs?lines=200", &out) return consoleLogsMsg{node: node, lines: out.Lines, err: err} } } func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { switch msg := msg.(type) { case tea.WindowSizeMsg: m.width, m.height = msg.Width, msg.Height case spinTickMsg: var cmd tea.Cmd m.spin, cmd = m.spin.Update(msg) return m, cmd case consoleTickMsg: if m.node { return m, nil } cmds := []tea.Cmd{m.fetch(), tea.Tick(3*time.Second, func(time.Time) tea.Msg { return consoleTickMsg{} })} if m.menu == cUsage { cmds = append(cmds, m.fetchUsage()) } if m.menu == cQuotas { cmds = append(cmds, m.fetchQuotas()) } if m.menu == cMirror { cmds = append(cmds, m.fetchMirror()) } if m.menu == cMonitor && m.monitorDue() { cmds = append(cmds, m.fetchMonitor()) } if m.detailDue() { cmds = append(cmds, m.fetchDetail()) } if m.peopleView() { cmds = append(cmds, m.fetchPeople()) } if m.menu == cLogs && m.detail && m.logs.node != "" { cmds = append(cmds, m.fetchLogs(m.logs.node)) } return m, tea.Batch(cmds...) case consoleDataMsg: m.loaded = true if msg.err != nil { m.data.err = msg.err } else { m.data = consoleData(msg) m.mon.sample(m.data) } case detailMsg: if msg.key == m.detailKey() { m.det.res, m.det.err, m.det.key = msg.res, msg.err, msg.key } case monitorMsg: // Kept if it is still what the screen wants: the range and width. if msg.key == m.monitorKey() { m.mon.res, m.mon.err, m.mon.key = msg.res, msg.err, msg.key } case consoleUsageMsg: m.usage, m.usageErr = msg.rows, msg.err case consoleLogsMsg: m.logs = msg case consoleMirrorMsg: m.mirror = msg case consoleHFMsg: if msg.query == m.hf.query && msg.sort == m.hf.sort && msg.st == m.hf.safetensors { m.hf.models, m.hf.err = msg.models, msg.err if m.hf.models == nil && msg.err == nil { m.hf.models = []hf.Model{} } } case consoleRepoMsg: m.busy = "" if msg.err != nil { m.dialog, m.dialogTitle = "error", "Hugging Face" m.dialogText = sBad.Render("✗ ") + msg.err.Error() + "\n\n" + sDim.Render("any key") } else { m.menuD = m.repoMenu(msg.plan) } case consolePeopleMsg: m.people = peopleData(msg) case consoleQuotasMsg: m.quotas, m.quotasErr = msg.rows, msg.err case actionDoneMsg: m.busy = "" m.dialog, m.dialogTitle = "error", msg.title if msg.err != nil { m.dialogText = sBad.Render("✗ ") + msg.err.Error() } else { m.dialogText = sOK.Render("✓ ") + msg.text } m.dialogText += "\n\n" + sDim.Render("any key") return m, tea.Batch(m.fetch(), m.onEnter()) case tea.KeyMsg: if msg.Type == tea.KeyCtrlC { return m, tea.Quit } return m.key(msg) } return m, nil } func (m consoleModel) entryOK(i int) bool { if m.node { return i == cSettings || i == cManual || i == cLeave } return i != cReplicate || !m.storeReplicated() } func (m consoleModel) storeReplicated() bool { return len(m.data.sum.Store) > 0 } func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) { if m.busy != "" { return m, nil } if m.dialog != "" { return m.dialogKey(k) } if mm, cmd, ok := m.actionKey(k); ok { return mm, cmd } // The manual, from anywhere: its page about this screen. if k.String() == "?" && !(m.menu == cManual && m.detail) { m.menu, m.inPanel, m.detail = cManual, true, true m.docs.show(max(0, findTopic(m.docs.topics, "console"))) return m, nil } if m.menu == cManual && m.inPanel && m.detail { pw := max(40, m.width-navWidth-8) if m.docs.key(k, pw-4, m.height) { return m, nil } switch k.String() { case "esc", "left", "h", "backspace": m.detail = false m.row = m.docs.open case "q": return m.leave() } return m, nil } switch k.Type { case tea.KeyPgDown: m.scroll += max(1, m.height-8) return m, nil case tea.KeyPgUp: m.scroll = max(0, m.scroll-max(1, m.height-8)) return m, nil } if !m.inPanel { switch k.String() { case "up", "k": for { m.menu = (m.menu - 1 + cCount) % cCount if m.entryOK(m.menu) { break } } m.row, m.detail, m.scroll = 0, false, 0 return m, m.onEnter() case "down", "j": for { m.menu = (m.menu + 1) % cCount if m.entryOK(m.menu) { break } } m.row, m.detail, m.scroll = 0, false, 0 return m, m.onEnter() case "q": return m.leave() case "right", "l", "tab", "enter": switch m.menu { case cAddNode: return m.ask("Add a node", "user@address of the machine", 1) case cReplicate: return m.ask("Replicate the store", "user@address of database machine", 2) case cLeave: return m.leave() case cUpgrade, cRestartAll, cBackup: return m, m.doEntry() case cOverview: return m, nil } m.inPanel = true } return m, nil } if m.menu == cSettings { return m.settingsKey(k) } n := m.rowCount() switch k.String() { case "up", "k": if !m.detail && m.row > 0 { m.row-- } case "down", "j": if !m.detail && m.row < n-1 { m.row++ } case "enter": if m.detail || n == 0 || m.row >= n { break } switch m.menu { case cLogs: m.detail, m.scroll = true, 0 node := m.data.nodes[m.row].ID m.logs = consoleLogsMsg{node: node} return m, m.fetchLogs(node) case cNodes: m.menuD = m.nodeMenu(m.data.nodes[m.row]) case cModels: m.menuD = m.modelMenu(m.data.models[m.row]) case cReplicas: m.menuD = m.replicaMenu(m.data.replicas[m.row]) case cQuotas: m.menuD = m.quotaMenu(m.quotas[m.row]) case cMirror: m.menuD = m.mirrorMenu(m.mirrorGroups()[m.row]) case cJobs: m.dialog = "error" m.dialogTitle, m.dialogText = jobDialog(m.data.jobs[m.row]) case cServe: return m, m.fetchRepo(m.hf.models[m.row].ID) case cAccounts: m.menuD = m.userMenu(m.people.users[m.row]) case cKeys: m.menuD = m.keyMenu(m.people.keys[m.row]) case cOrgs: m.menuD = m.orgMenu(m.people.orgs[m.row]) case cManual: m.docs.show(m.row) m.detail = true } case "1", "2", "3", "4", "r": if kind, _ := m.detailTarget(); m.menu == cMonitor || kind != "" { if k.String() == "r" { m.mon.rng = (m.mon.rng + 1) % len(monRanges) } else { m.mon.rng = int(k.String()[0] - '1') } if kind != "" { return m, m.fetchDetail() } return m, m.fetchMonitor() } case "/": if m.menu == cServe { m.hfSearchDialog() } case "f": if m.menu == cServe { m.hf.safetensors = !m.hf.safetensors m.hf.models, m.hf.err, m.row = nil, nil, 0 return m, m.fetchHF() } case "s": if m.menu == cServe { m.hf.sort = (m.hf.sort + 1) % len(hfSorts) m.hf.models, m.hf.err, m.row = nil, nil, 0 return m, m.fetchHF() } case "a": switch m.menu { case cQuotas: m.quotaForm(adminapi.QuotaView{}) case cModels: m.addModelForm() case cMirror: m.mirrorFetchForm() case cAccounts: m.createUserForm() case cKeys: m.createKeyForm() case cOrgs: m.createOrgForm() } case "right", "l": if m.menu == cUsage { m.usageBy = (m.usageBy + 1) % len(usageGroupings) return m, m.fetchUsage() } case "esc", "left", "h", "backspace": if m.detail { m.detail, m.scroll = false, 0 } else { m.inPanel = false } case "q": return m.leave() } return m, nil } // onEnter loads what a view needs when it is chosen. func (m consoleModel) onEnter() tea.Cmd { switch m.menu { case cUsage: return m.fetchUsage() case cQuotas: return m.fetchQuotas() case cMirror: return m.fetchMirror() case cMonitor: return m.fetchMonitor() case cServe: if m.hf.models == nil { return m.fetchHF() } case cAccounts, cKeys, cOrgs: // The quotas too, for an organization's quota form. return tea.Batch(m.fetchPeople(), m.fetchQuotas()) } return nil } func (m consoleModel) peopleView() bool { return m.menu == cAccounts || m.menu == cKeys || m.menu == cOrgs } func (m consoleModel) leave() (tea.Model, tea.Cmd) { if m.doc != nil && m.doc.Changed > 0 { m.dialog, m.dialogTitle = "confirm-leave", "Unsaved changes" m.dialogText = fmt.Sprintf("%d change(s) to %s are not saved.\nLeave without saving them?\n\n%s", m.doc.Changed, m.file, sDim.Render("Y leave · N stay")) return m, nil } return m, tea.Quit } func (m consoleModel) ask(title, question string, n int) (tea.Model, tea.Cmd) { m.dialog, m.dialogTitle, m.dialogText, m.asked = "target", title, question, nil m.prompt = newTextField() m.prompt.Placeholder, m.prompt.Width = "ubuntu@10.0.0.20", 40 m.prompt.Focus() m.editKind = settings.Kind(n) // how many to ask for return m, nil } func (m consoleModel) dialogKey(k tea.KeyMsg) (tea.Model, tea.Cmd) { switch m.dialog { case "error": m.dialog = "" return m, nil case "confirm-leave": if strings.EqualFold(k.String(), "y") { return m, tea.Quit } m.dialog = "" return m, nil case "saved": if strings.EqualFold(k.String(), "y") { m.action = consoleAction{kind: "restart"} return m, tea.Quit } m.dialog = "" return m, nil } switch k.Type { case tea.KeyEsc: m.dialog = "" return m, nil case tea.KeyEnter: v := strings.TrimSpace(m.prompt.Value()) switch m.dialog { case "target": if v == "" { return m, nil } m.asked = append(m.asked, v) m.prompt.SetValue("") if len(m.asked) == int(m.editKind) { kind := "node" if m.menu == cReplicate { kind = "patroni" } m.action = consoleAction{kind: kind, targets: m.asked} return m, tea.Quit } case "edit": if err := m.doc.Set(m.editPath, m.editKind, m.prompt.Value()); err != nil { m.dialogText = m.editQuestion() + "\n\n" + sBad.Render(err.Error()) return m, nil } m.dialog = "" case "addname": if v == "" { return m, nil } if err := m.doc.AddEntry(m.editPath, v); err != nil { m.dialogText = "Name of the new entry\n\n" + sBad.Render(err.Error()) return m, nil } m.dialog = "" } return m, nil } var cmd tea.Cmd m.prompt, cmd = m.prompt.Update(k) return m, cmd } // --- settings --- // settingsRow is one line of the settings view. type settingsRow struct { label, value string key *settings.Key // index is a list entry's position, or a map entry's name. entry string add bool } func (m consoleModel) level() settingsLevel { return m.levels[len(m.levels)-1] } func (m consoleModel) settingsRows() []settingsRow { lv := m.level() var rows []settingsRow switch lv.kind { case settings.Block: for _, k := range settings.Keys(lv.block) { k := k path := append(append([]string{}, lv.path...), k.Key) rows = append(rows, settingsRow{label: k.Key, value: m.valueOf(k, path), key: &k}) } case settings.BlockList: for i := 0; i < m.doc.Len(lv.path); i++ { rows = append(rows, settingsRow{label: m.entryLabel(append(append([]string{}, lv.path...), strconv.Itoa(i)), i), value: "›", entry: strconv.Itoa(i)}) } rows = append(rows, settingsRow{label: "+ add an entry", add: true}) case settings.BlockMap: for _, name := range m.doc.MapKeys(lv.path) { rows = append(rows, settingsRow{label: name, value: "›", entry: name}) } rows = append(rows, settingsRow{label: "+ add an entry", add: true}) } return rows } // entryLabel names a list entry by its name or id, else its position. func (m consoleModel) entryLabel(path []string, i int) string { for _, k := range []string{"name", "id", "key", "url"} { if v, ok := m.doc.Text(append(path, k)); ok && v != "" { return v } } return fmt.Sprintf("entry %d", i+1) } func (m consoleModel) valueOf(k settings.Key, path []string) string { switch k.Kind { case settings.Block: return "›" case settings.BlockList, settings.BlockMap: return fmt.Sprintf("%d ›", m.doc.Len(path)) } if v, ok := m.doc.Text(path); ok { return strings.ReplaceAll(v, "\n", ", ") } if v, ok := settings.Effective(m.cfg, path); ok && v != "" { return sDim.Render(v + " (default)") } return sDim.Render("—") } func (m consoleModel) editQuestion() string { hint := map[settings.Kind]string{ settings.Duration: "a duration: 30s, 5m, 2h", settings.Int: "a whole number", settings.Float: "a number", settings.TextList: "values, separated by commas", settings.TextMap: "key = value, separated by commas", }[m.editKind] q := strings.Join(m.editPath, ".") if hint != "" { q += "\n" + sDim.Render(hint) } return q } func (m consoleModel) settingsKey(k tea.KeyMsg) (tea.Model, tea.Cmd) { if m.doc == nil { if k.String() == "esc" || k.String() == "left" { m.inPanel = false } return m, nil } rows := m.settingsRows() lv := m.level() switch k.String() { case "up", "k": if m.row > 0 { m.row-- } case "down", "j": if m.row < len(rows)-1 { m.row++ } case "esc", "left", "h", "backspace": if len(m.levels) > 1 { m.levels = m.levels[:len(m.levels)-1] m.row, m.scroll = 0, 0 } else { m.inPanel = false } case "s": return m.save() case "u", "delete": if m.row >= len(rows) { return m, nil } r := rows[m.row] switch { case r.key != nil: m.doc.Unset(append(append([]string{}, lv.path...), r.key.Key)) case r.entry != "": m.doc.Unset(append(append([]string{}, lv.path...), r.entry)) m.row = max(0, m.row-1) } case "enter", "right", "l": if m.row >= len(rows) { return m, nil } r := rows[m.row] switch { case r.add && lv.kind == settings.BlockList: i := m.doc.Append(lv.path) m.levels = append(m.levels, settingsLevel{title: fmt.Sprintf("entry %d", i+1), block: lv.block, path: append(append([]string{}, lv.path...), strconv.Itoa(i)), kind: settings.Block}) m.row = 0 case r.add && lv.kind == settings.BlockMap: m.dialog, m.dialogTitle, m.dialogText = "addname", "Add an entry", "Name of the new entry" m.editPath = lv.path m.prompt = newTextField() m.prompt.Width = 40 m.prompt.Focus() case r.entry != "": m.levels = append(m.levels, settingsLevel{title: r.label, block: lv.block, path: append(append([]string{}, lv.path...), r.entry), kind: settings.Block}) m.row = 0 case r.key != nil: path := append(append([]string{}, lv.path...), r.key.Key) switch r.key.Kind { case settings.Block: m.levels = append(m.levels, settingsLevel{title: r.key.Key, block: r.key.Block, path: path, kind: settings.Block}) m.row = 0 case settings.BlockList, settings.BlockMap: m.levels = append(m.levels, settingsLevel{title: r.key.Key, block: r.key.Block, path: path, kind: r.key.Kind}) m.row = 0 case settings.Bool: cur, ok := m.doc.Text(path) if !ok { cur, _ = settings.Effective(m.cfg, path) } b, _ := strconv.ParseBool(cur) _ = m.doc.Set(path, settings.Bool, strconv.FormatBool(!b)) default: m.editPath, m.editKind = path, r.key.Kind cur, ok := m.doc.Text(path) if !ok { cur, _ = settings.Effective(m.cfg, path) } m.prompt = newTextField() m.prompt.Width = 50 m.prompt.SetValue(strings.ReplaceAll(cur, "\n", ", ")) m.prompt.Focus() m.dialog, m.dialogTitle, m.dialogText = "edit", "Change "+r.key.Key, m.editQuestion() } } m.scroll = 0 } return m, nil } func (m consoleModel) save() (tea.Model, tea.Cmd) { if m.doc.Changed == 0 { m.dialog, m.dialogTitle, m.dialogText = "error", "Nothing to save", "No change has been made." return m, nil } backup, err := m.doc.Save(func(p string) error { _, err := m.load(p); return err }) if err != nil { m.dialog, m.dialogTitle = "error", "Not saved" m.dialogText = "The configuration would not load:\n\n" + sBad.Render(err.Error()) + "\n\n" + sDim.Render("Nothing was written. Any key to go back.") return m, nil } m.loadSettings() note := "" if backup != "" { note = "The old file is " + backup + ".\n" } m.dialog, m.dialogTitle = "saved", "Saved" m.dialogText = "Saved " + m.file + ".\n" + note + fmt.Sprintf("\n%s reads it when it starts. Restart it now?\n", m.service()) if !m.node { m.dialogText += sDim.Render("Another control plane keeps its own copy: make the change there too.") + "\n" } m.dialogText += "\n" + sDim.Render("Y restart · N later") return m, nil } // --- views --- func (m consoleModel) rowCount() int { switch m.menu { case cNodes, cLogs: return len(m.data.nodes) case cGPUs: return len(m.gpuRows()) case cModels: return len(m.data.models) case cReplicas: return len(m.data.replicas) case cUsage: return len(m.usage) case cQuotas: return len(m.quotas) case cMirror: return len(m.mirrorGroups()) case cJobs: return len(m.data.jobs) case cServe: return len(m.hf.models) case cAccounts: return len(m.people.users) case cKeys: return len(m.people.keys) case cOrgs: return len(m.people.orgs) case cManual: return len(m.docs.topics) } return 0 } func (m consoleModel) View() string { host, _ := os.Hostname() nav := []navItem{{Label: "Watch", Heading: true}} if m.node { nav = nil } for i := 0; i < cCount; i++ { if m.node && !m.entryOK(i) { continue } switch i { case cAccounts: nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "People", Heading: true}) case cManual: nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "Help", Heading: true}) case cSettings: nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "Settings", Heading: true}) case cAddNode, cLeave: if (i == cAddNode) != m.node { nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "Do", Heading: true}) } } label := consoleLabels[i] if i == cSettings { label = filepathBase(m.file) } if n := runningJobs(m.data.jobs); i == cJobs && n > 0 { label += fmt.Sprintf(" (%d)", n) } if i == cSettings && m.doc != nil && m.doc.Changed > 0 { label += fmt.Sprintf(" (%d)", m.doc.Changed) } nav = append(nav, navItem{Label: label, Current: i == m.menu, Dim: !m.entryOK(i)}) } title, body, keys := m.panel() title0 := "llmgw console" if m.node { title0 = "llmgw · node" } f := frame{ Width: m.width, Height: m.height, Title: title0, Where: host, NavTitle: "", Nav: nav, PanelTitle: title, Body: body, Scroll: m.scroll, Keys: keys, } if m.menu == cManual && m.detail { f.Scroll = m.docs.scroll } // The selected row kept on screen. if m.inPanel && !m.detail { inner := m.height - 4 top := m.row + 3 if top-m.scroll >= inner-1 { f.Scroll = top - inner + 2 } } if m.dialog != "" { f.DialogTitle = m.dialogTitle f.Dialog = m.dialogText switch m.dialog { case "target", "edit", "addname": q := m.dialogText if m.dialog == "target" && m.editKind > 1 { q = fmt.Sprintf("%s %d of %d", q, len(m.asked)+1, int(m.editKind)) } f.Dialog = q + "\n\n" + m.prompt.View() + "\n\n" + sDim.Render("Enter · Esc cancel") } } else if t, d := m.docs.dialog(); d != "" && m.menu == cManual { f.DialogTitle, f.Dialog = t, d } else if m.busy != "" { f.DialogTitle, f.Dialog = m.busy, m.spin.View()+" asking the control plane…" } else if t, d := m.actionDialog(); d != "" { f.DialogTitle, f.Dialog = t, d } return f.render() } // panel is the right-hand side for the entry chosen. func (m consoleModel) panel() (title, body, keys string) { pw := max(40, m.width-navWidth-8) rowKeys := "↑↓ row · Enter actions · ← back · PgUp/PgDn · Q leave" if !m.inPanel { keys = "↑↓ choose · → or Enter open · Q leave · refreshed every three seconds" } waiting := func() string { if m.data.err != nil { return "\n " + sBad.Render("The control plane does not answer: ") + m.data.err.Error() + "\n\n " + sDim.Render("systemctl status llmgw-server says why.") } return "\n " + m.spin.View() + " asking the control plane…" } if (!m.loaded || m.data.err != nil) && !m.node { if m.menu < cSettings { return consoleLabels[m.menu], waiting(), keys } } switch m.menu { case cOverview: var b bytes.Buffer c := &cobra.Command{} c.SetOut(&b) printSummary(c, m.data.sum) return "Overview", "\n" + indent(b.String(), " "), keys case cNodes, cLogs: if m.detail && m.row < len(m.data.nodes) { n := m.data.nodes[m.row] if m.menu == cLogs { return "Agent log · " + n.ID, m.logView(), "← back · PgUp/PgDn · refreshed every three seconds" } return "Node · " + n.ID, m.nodeDetail(n) + m.detailCharts(pw), "← back · 1–4 or R range · PgUp/PgDn · Q leave" } var rows [][]string for _, n := range m.data.nodes { cpu, mem := "—", "—" if n.Load != nil { cpu, mem = fmt.Sprintf("%.0f%%", n.Load.CPUPercent), fmt.Sprintf("%.0f%%", n.Load.MemPercent) } state := n.State if state != "ready" { state = sWarn.Render(state) } rows = append(rows, []string{n.ID, state, cpu, mem, fmt.Sprintf("%d/%d", n.GPUFree, n.GPUTotal), strconv.Itoa(n.Replicas), fmt.Sprintf("%d/%d GB", n.DiskFreeGB, n.DiskTotalGB), n.AgentVersion, n.LastSeenAgo}) } t := "Nodes" if m.menu == cLogs { t = "Logs · choose a node" } if m.inPanel { keys = rowKeys } return t, "\n" + table([]string{"NODE", "STATE", "CPU", "MEM", "GPU FREE", "REPLICAS", "DISK FREE", "AGENT", "SEEN"}, rows, m.row, pw, m.inPanel), keys case cGPUs: if m.inPanel { keys = "↑↓ row · ← back · PgUp/PgDn · Q leave" } return "GPUs", "\n" + table([]string{"NODE", "GPU", "CARD", "BUSY", "MEMORY", "TEMP", "POWER", "HELD BY"}, m.gpuRows(), m.row, pw, m.inPanel), keys case cModels: if m.detail && m.row < len(m.data.models) { return "Model · " + m.data.models[m.row].Name, m.modelDetail(m.data.models[m.row]) + m.detailCharts(pw), "← back · 1–4 or R range · PgUp/PgDn · Q leave" } var rows [][]string for _, md := range m.data.models { status := "stopped" switch { case md.Parked: status = sBad.Render("parked") case md.Unplaceable: status = sWarn.Render("not placed") case md.Ready > 0 && md.Ready >= md.Desired: status = sOK.Render("ready") case md.Desired > 0: status = sWarn.Render("starting") } warn := "" if len(md.Warnings) > 0 { warn = sWarn.Render("!") } upd := "" if md.Update != nil && md.Update.Available { upd = sWarn.Render("↑ " + catalog.ShortRevision(md.Update.Latest)) } rows = append(rows, []string{md.Name, fmt.Sprintf("%d/%d", md.Ready, md.Desired), status, md.Class, strings.Join(md.Profiles, ","), upd, warn}) } if m.inPanel { keys = "↑↓ row · Enter actions · A add from a file · ← back · Q leave" } return "Models", "\n" + table([]string{"MODEL", "READY", "STATUS", "CLASS", "PROFILES", "UPDATE", ""}, rows, m.row, pw, m.inPanel), keys case cReplicas: var rows [][]string for _, r := range m.data.replicas { var nodes []string for _, s := range r.Shards { nodes = append(nodes, fmt.Sprintf("%s%v", s.Node, s.GPUs)) } rows = append(rows, []string{r.ID, r.Model, r.State, r.Pool, strings.Join(nodes, " "), (time.Duration(r.AgeSec) * time.Second).Round(time.Second).String()}) } if m.inPanel { keys = rowKeys } return "Replicas", "\n" + table([]string{"REPLICA", "MODEL", "STATE", "POOL", "ON", "AGE"}, rows, m.row, pw, m.inPanel), keys case cUsage: head := "\n Last 24 hours, by " + lipgloss.NewStyle().Bold(true).Render(usageGroupings[m.usageBy]) + sDim.Render(" (→ to group by "+usageGroupings[(m.usageBy+1)%len(usageGroupings)]+")") + "\n\n" if m.usageErr != nil { return "Usage", head + " " + sBad.Render(m.usageErr.Error()), keys } var rows [][]string for _, u := range m.usage { rows = append(rows, []string{u.Name, compactNum(u.Requests), compactNum(u.Prompt + u.Completion), compactNum(u.Errors), fmt.Sprintf("%.2f", float64(u.CostMicros)/1e6)}) } if m.inPanel { keys = "↑↓ row · → grouping · ← back · Q leave" } return "Usage", head + table([]string{"WHO", "REQUESTS", "TOKENS", "ERRORS", "SPENT"}, rows, m.row, pw, m.inPanel), keys case cQuotas: return m.quotasPanel(pw) case cAccounts, cKeys, cOrgs: return m.peoplePanel(pw) case cManual: if m.detail && m.docs.open >= 0 { k := docKeys if m.docs.note != "" { k = m.docs.note } return "Manual · " + m.docs.topics[m.docs.open].title, "\n" + m.docs.body(pw-4), k } d := m.docs d.sel = m.row if m.inPanel { keys = "↑↓ topic · Enter read · ← back · ? the console's page · Q leave" } return "Manual", d.list(pw, m.inPanel), keys case cMirror: return m.mirrorPanel(pw) case cJobs: return m.jobsPanel(pw) case cMonitor: return m.monitorPanel(pw) case cServe: return m.hfPanel(pw) case cSettings: return m.settingsPanel(pw) } var item string switch m.menu { case cAddNode: item = "\n Adds a machine as a node: installs llmgw on it over ssh, checks it,\n gives it a certificate and starts its agent.\n\n " + sDim.Render("On the command line:") + "\n sudo llmgw node add @
" case cReplicate: item = "\n Moves the store onto two machines under Patroni, replicated,\n so losing one loses nothing.\n\n " + sDim.Render("On the command line:") + "\n sudo --preserve-env=SSH_AUTH_SOCK llmgw server init \\\n --patroni @ --patroni @ --no-mirror --no-node --no-model" case cUpgrade: item = "\n Moves the cluster to a release: this machine first, then each node\n in turn. Engines keep serving; a node that does not come back stops it.\n\n " + sDim.Render("On the command line:") + "\n sudo llmgw ctl upgrade VERSION" case cRestartAll: item = "\n Replaces every replica, one at a time, each new one up before the\n old one stops.\n\n " + sDim.Render("On the command line:") + "\n llmgw ctl restart --all" case cBackup: item = "\n Writes the store and /etc/llmgw to one archive in " + DefaultBackupDir + ",\n and keeps the last few. Copy it off this machine.\n\n " + sDim.Render("On the command line:") + "\n sudo llmgw server backup" case cLeave: item = "\n Closes the console. The cluster keeps running." } return consoleLabels[m.menu], item, "Enter do it · ↑↓ choose · Q leave" } func (m consoleModel) settingsPanel(pw int) (string, string, string) { if m.docErr != nil { return "server.yaml", "\n " + sBad.Render(m.docErr.Error()), "← back" } var crumbs []string for _, l := range m.levels { crumbs = append(crumbs, l.title) } rows := m.settingsRows() var tr [][]string for _, r := range rows { typ := "" if r.key != nil { typ = sDim.Render(r.key.Type) } tr = append(tr, []string{r.label, r.value, typ}) } body := "\n" + table([]string{"KEY", "VALUE", "TYPE"}, tr, m.row, pw, m.inPanel) // What the chosen key is for, from its comment in the code. if m.inPanel && m.row < len(rows) { doc := "" if r := rows[m.row]; r.key != nil { doc = r.key.Doc if doc == "" && r.key.Block != nil { doc = r.key.Block.Doc } } else if l := m.level(); l.block != nil { doc = l.block.Doc } if doc != "" { body += "\n " + sDim.Render(strings.Repeat("─", min(pw-4, 60))) + "\n" + indent(wrap(firstParagraphs(doc, 2), pw-4, ""), " ") } } keys := "→ or Enter open · ↑↓ row · Q leave" if m.inPanel { keys = "Enter change/open · U back to default · S save · Esc up · PgUp/PgDn" if l := m.level(); l.kind != settings.Block { keys = "Enter open · D remove entry · S save · Esc up" } } return strings.Join(crumbs, " › "), body, keys } func firstParagraphs(s string, n int) string { p := strings.Split(s, "\n\n") if len(p) > n { p = p[:n] } return strings.Join(p, "\n\n") } func (m consoleModel) gpuRows() [][]string { held := map[string]string{} for _, r := range m.data.replicas { for _, s := range r.Shards { for _, g := range s.GPUs { held[s.Node+"/"+strconv.Itoa(g)] = r.Model } } } var rows [][]string for _, n := range m.data.nodes { for _, g := range n.GPUs { busy, mem, temp, power := "—", "—", "—", "—" if n.Load != nil { for _, s := range n.Load.GPUs { if s.Index != g.Index { continue } busy = fmt.Sprintf("%.0f%%", s.UtilPercent) if s.MemoryTotalMB > 0 { mem = fmt.Sprintf("%.1f/%.0f GB", s.MemoryUsedMB/1024, s.MemoryTotalMB/1024) } if s.TempC > 0 { temp = fmt.Sprintf("%.0f°C", s.TempC) } if s.PowerW > 0 { power = fmt.Sprintf("%.0f/%.0f W", s.PowerW, s.PowerLimitW) } } } holder := held[n.ID+"/"+strconv.Itoa(g.Index)] if holder == "" { holder = sDim.Render("free") } rows = append(rows, []string{n.ID, strconv.Itoa(g.Index), g.Name, busy, mem, temp, power, holder}) } } return rows } func (m consoleModel) nodeDetail(n adminapi.NodeView) string { var b strings.Builder kv := func(k, v string) { fmt.Fprintf(&b, " %-14s %s\n", sDim.Render(k), v) } b.WriteString("\n") kv("state", n.State) kv("address", n.MgmtAddr) kv("system", n.OS) kv("agent", n.AgentVersion+", seen "+n.LastSeenAgo+" ago") if n.Load != nil { l := n.Load kv("cpu", fmt.Sprintf("%.0f%% of %d · load %.2f %.2f %.2f", l.CPUPercent, l.CPUs, l.Load1, l.Load5, l.Load15)) kv("memory", fmt.Sprintf("%.0f%% · %.1f of %.1f GB", l.MemPercent, float64(l.MemUsed)/(1<<30), float64(l.MemTotal)/(1<<30))) } kv("disk", fmt.Sprintf("%d GB free of %d", n.DiskFreeGB, n.DiskTotalGB)) kv("replicas", strconv.Itoa(n.Replicas)) if len(n.Provision) > 0 { b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("Engines") + "\n") for _, p := range n.Provision { fmt.Fprintf(&b, " %-9s %-10s %-8s %s\n", p.Engine, p.Version, p.State, sDim.Render(truncate(p.Detail, 60))) } } if len(n.GPUs) > 0 { b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("GPUs") + "\n") for _, r := range m.gpuRows() { if r[0] == n.ID { fmt.Fprintf(&b, " %s %-18s busy %-5s mem %-14s %-6s %s %s\n", r[1], r[2], r[3], r[4], r[5], r[6], r[7]) } } } for _, w := range append(n.Warnings, n.Unhealthy...) { b.WriteString("\n " + sWarn.Render("! ") + w) } return b.String() } func (m consoleModel) modelDetail(md adminapi.ModelView) string { var b strings.Builder kv := func(k, v string) { if v != "" { fmt.Fprintf(&b, " %-14s %s\n", sDim.Render(k), v) } } b.WriteString("\n") if md.Description != "" { b.WriteString(" " + md.Description + "\n\n") } kv("replicas", fmt.Sprintf("%d ready of %d wanted (min %d, max %d)", md.Ready, md.Desired, md.Min, md.Max)) kv("class", md.Class) kv("owner", md.Owner) kv("profiles", strings.Join(md.Profiles, ", ")) kv("aliases", strings.Join(md.Aliases, ", ")) kv("source", md.Source) if o := md.Origin; o != nil { from := o.Repo + " at " + catalog.ShortRevision(o.Revision) if o.Previous != "" { from += ", updated from " + catalog.ShortRevision(o.Previous) } kv("from", from) } if u := md.Update; u != nil { kv("registry", updateCell(*u)+sDim.Render(", asked "+ago(u.CheckedAt, time.Now()))) } if md.MemoryGB > 0 { kv("reserves", fmt.Sprintf("%d GB, %d cores (uses %.1f GB)", md.MemoryGB, md.CPUCores, md.MeasuredMemoryGB)) } kv("fallbacks", strings.Join(md.Fallbacks, ", ")) if md.Reason != "" { b.WriteString("\n " + sWarn.Render("! ") + md.Reason + "\n") } for _, w := range md.Warnings { b.WriteString(" " + sWarn.Render("! ") + w + "\n") } var reps [][]string for _, r := range m.data.replicas { if r.Model == md.Name { reps = append(reps, []string{r.ID, r.State, r.Pool, r.Endpoint}) } } if len(reps) > 0 { b.WriteString("\n" + table([]string{"REPLICA", "STATE", "POOL", "ENDPOINT"}, reps, -1, max(40, m.width-navWidth-8), false)) } return b.String() } func (m consoleModel) logView() string { if m.logs.err != nil { return "\n " + sBad.Render(m.logs.err.Error()) } if len(m.logs.lines) == 0 { return "\n " + m.spin.View() + " reading the log…" } // The newest at the bottom, where a log is read; the panel starts there. lines := m.logs.lines if k := m.height - 6; len(lines) > k && m.scroll == 0 { lines = lines[len(lines)-k:] } var b strings.Builder b.WriteString("\n") for _, l := range lines { switch { case strings.Contains(l, "level=ERROR"): l = sBad.Render(l) case strings.Contains(l, "level=WARN"): l = sWarn.Render(l) } b.WriteString(" " + l + "\n") } return b.String() } func compactNum(n int64) string { switch { case n >= 1_000_000: return fmt.Sprintf("%.1fM", float64(n)/1e6) case n >= 1_000: return fmt.Sprintf("%.1fk", float64(n)/1e3) } return strconv.FormatInt(n, 10) } func indent(s, pre string) string { lines := strings.Split(strings.TrimRight(s, "\n"), "\n") for i := range lines { lines[i] = pre + lines[i] } return strings.Join(lines, "\n") } func filepathBase(p string) string { if i := strings.LastIndexByte(p, '/'); i >= 0 { return p[i+1:] } return p } // runConsole shows the console, runs what is chosen on it, and comes back. // file is server.yaml on a control plane, agent.yaml on a node. func runConsole(ctx context.Context, out io.Writer, file string) error { addr := "127.0.0.1:9090" if cfg, err := config.Load(file); err == nil && cfg.AdminListen != "" { addr = cfg.AdminListen if strings.HasPrefix(addr, ":") { addr = "127.0.0.1" + addr } } menu := cOverview if strings.HasSuffix(file, "agent.yaml") { menu = cSettings } for { m := newConsoleModel(addr, file) m.menu = menu final, err := tea.NewProgram(m, tea.WithOutput(out), tea.WithAltScreen()).Run() if err != nil { return err } fm := final.(consoleModel) fm.st.close() menu = fm.menu act := fm.action switch act.kind { case "": return nil case "node": c := newNodeAddCmd() c.SetArgs(act.targets) c.SetOut(out) err = c.ExecuteContext(ctx) case "patroni": in := newDefaultServerInit() in.patroni = act.targets // Already set up: only the store changes. in.noMirror, in.noNode, in.noModel = true, true, true err = in.runInitTUI(ctx, out) case "run": c := Root() c.SetArgs(act.targets) c.SetOut(out) c.SetErr(out) fmt.Fprintln(out, sDim.Render("llmgw "+strings.Join(act.targets, " "))) err = c.ExecuteContext(ctx) case "restart": fmt.Fprintf(out, "restarting %s…\n", fm.service()) if b, e := exec.CommandContext(ctx, "systemctl", "restart", fm.service()).CombinedOutput(); e != nil { err = fmt.Errorf("%v: %s", e, strings.TrimSpace(string(b))) } else { fmt.Fprintln(out, sOK.Render("✓ ")+"restarted; it picks up the configuration as it starts") } } if err != nil { fmt.Fprintln(out, sBad.Render("✗ ")+err.Error()) } fmt.Fprint(out, "\n"+sDim.Render("enter to go back to the console")+" ") _, _ = bufio.NewReader(os.Stdin).ReadString('\n') } }