Ilker ManapandClaude Opus 5.5
9ab93a0aa6
The console gains live views of nodes, GPUs, models, replicas, usage and agent logs, and a settings tree over every key of server.yaml (agent.yaml on a node), generated from the configuration structs: values with their defaults and doc comments, edited in dialogs, saved only once the service's own loader accepts them, comments kept. The generator now tells blocks apart by package, which brings the TLS, engine and supervisor settings back into the reference. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1132 lines
32 KiB
Go
1132 lines
32 KiB
Go
package cli
|
||
|
||
import (
|
||
"bufio"
|
||
"bytes"
|
||
"context"
|
||
"fmt"
|
||
"io"
|
||
"net/url"
|
||
"os"
|
||
"os/exec"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
tea "github.com/charmbracelet/bubbletea"
|
||
"github.com/charmbracelet/lipgloss"
|
||
"github.com/spf13/cobra"
|
||
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/docgen"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
|
||
)
|
||
|
||
// The console: what a bare `sudo llmgw` opens on a control plane. Down the
|
||
// left, three groups — what is running (watched live), the settings (every
|
||
// key of server.yaml, from the generated reference), and what can be done.
|
||
|
||
// Sidebar entries, in order. Headings are not entries.
|
||
const (
|
||
cOverview = iota
|
||
cNodes
|
||
cGPUs
|
||
cModels
|
||
cReplicas
|
||
cUsage
|
||
cLogs
|
||
cSettings
|
||
cAddNode
|
||
cReplicate
|
||
cLeave
|
||
cCount
|
||
)
|
||
|
||
var consoleLabels = [cCount]string{"Overview", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Logs",
|
||
"server.yaml", "Add a node", "Replicate the store", "Leave"}
|
||
|
||
// consoleAction is what is done once the screen closes: each runs its own
|
||
// screen, and the console comes back after.
|
||
type consoleAction struct {
|
||
kind string // "", "node", "patroni", "restart"
|
||
targets []string
|
||
}
|
||
|
||
type consoleData struct {
|
||
sum adminapi.Summary
|
||
nodes []adminapi.NodeView
|
||
models []adminapi.ModelView
|
||
replicas []adminapi.ReplicaView
|
||
err error
|
||
}
|
||
|
||
type (
|
||
consoleDataMsg consoleData
|
||
consoleUsageMsg struct {
|
||
rows []adminapi.ScopeUsageView
|
||
err error
|
||
}
|
||
consoleLogsMsg struct {
|
||
node string
|
||
lines []string
|
||
err error
|
||
}
|
||
consoleTickMsg struct{}
|
||
)
|
||
|
||
// settingsLevel is where the settings view is: a block, or a list or map of
|
||
// blocks, at a path of keys in the file.
|
||
type settingsLevel struct {
|
||
title string
|
||
block *docgen.Block
|
||
path []string
|
||
kind settings.Kind // Block, BlockList or BlockMap
|
||
}
|
||
|
||
type consoleModel struct {
|
||
client *ctlClient
|
||
width, height int
|
||
spin spinnerModel
|
||
|
||
data consoleData
|
||
loaded bool
|
||
usage []adminapi.ScopeUsageView
|
||
// usageBy is the grouping on the usage view.
|
||
usageBy int
|
||
usageErr error
|
||
logs consoleLogsMsg
|
||
|
||
// menu is the sidebar entry; inPanel says the keys go to the panel.
|
||
menu int
|
||
inPanel bool
|
||
row int
|
||
// detail is the panel showing one row's detail rather than the table.
|
||
detail bool
|
||
scroll int
|
||
|
||
// node is the console of a node: its agent.yaml and nothing to watch,
|
||
// since the cluster is watched from the control plane.
|
||
node bool
|
||
|
||
// The settings.
|
||
file string
|
||
doc *settings.Doc
|
||
cfg any // the file as loaded, for the values that apply
|
||
levels []settingsLevel
|
||
docErr error
|
||
saveNote string
|
||
|
||
// A question open as a dialog.
|
||
dialog string // "", "target", "edit", "addname", "saved", "confirm-leave", "error"
|
||
dialogTitle string
|
||
dialogText string
|
||
prompt textField
|
||
asked []string
|
||
editPath []string
|
||
editKind settings.Kind
|
||
|
||
action consoleAction
|
||
}
|
||
|
||
var usageGroupings = []string{"team", "project", "org"}
|
||
|
||
func newConsoleModel(addr, file string) consoleModel {
|
||
m := consoleModel{client: newCtlClient(addr), spin: newSpinner(), width: 100, height: 30, file: file}
|
||
m.node = strings.HasSuffix(file, "agent.yaml")
|
||
if m.node {
|
||
m.menu = cSettings
|
||
}
|
||
m.loadSettings()
|
||
return m
|
||
}
|
||
|
||
// load reads the file as the service would, defaults and checks included.
|
||
func (m consoleModel) load(path string) (any, error) {
|
||
if m.node {
|
||
return config.LoadAgent(path)
|
||
}
|
||
return config.Load(path)
|
||
}
|
||
|
||
func (m consoleModel) service() string {
|
||
if m.node {
|
||
return "llmgw-agent"
|
||
}
|
||
return "llmgw-server"
|
||
}
|
||
|
||
func (m *consoleModel) loadSettings() {
|
||
m.doc, m.docErr = settings.Open(m.file)
|
||
if m.docErr != nil {
|
||
return
|
||
}
|
||
if cfg, err := m.load(m.file); err == nil {
|
||
m.cfg = cfg
|
||
} else if !m.node {
|
||
m.cfg = config.Default()
|
||
}
|
||
if root, err := settings.Root(m.file); err == nil && len(m.levels) == 0 {
|
||
m.levels = []settingsLevel{{title: filepathBase(m.file), block: root, kind: settings.Block}}
|
||
}
|
||
}
|
||
|
||
func (m consoleModel) Init() tea.Cmd {
|
||
if m.node {
|
||
return m.spin.Tick
|
||
}
|
||
return tea.Batch(m.fetch(), m.spin.Tick, tea.Tick(3*time.Second, func(time.Time) tea.Msg { return consoleTickMsg{} }))
|
||
}
|
||
|
||
// fetch reads everything the watched views show, in one go.
|
||
func (m consoleModel) fetch() tea.Cmd {
|
||
c := m.client
|
||
return func() tea.Msg {
|
||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||
defer cancel()
|
||
var d consoleData
|
||
if d.err = c.get(ctx, "/api/v1/summary", &d.sum); d.err != nil {
|
||
return consoleDataMsg(d)
|
||
}
|
||
_ = c.get(ctx, "/api/v1/nodes", &d.nodes)
|
||
_ = c.get(ctx, "/api/v1/models", &d.models)
|
||
_ = c.get(ctx, "/api/v1/replicas", &d.replicas)
|
||
return consoleDataMsg(d)
|
||
}
|
||
}
|
||
|
||
func (m consoleModel) fetchUsage() tea.Cmd {
|
||
c, by := m.client, usageGroupings[m.usageBy]
|
||
return func() tea.Msg {
|
||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||
defer cancel()
|
||
var rows []adminapi.ScopeUsageView
|
||
err := c.get(ctx, "/api/v1/usage?by="+by+"&since=24h", &rows)
|
||
return consoleUsageMsg{rows, err}
|
||
}
|
||
}
|
||
|
||
func (m consoleModel) fetchLogs(node string) tea.Cmd {
|
||
c := m.client
|
||
return func() tea.Msg {
|
||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||
defer cancel()
|
||
var out struct {
|
||
Lines []string `json:"lines"`
|
||
}
|
||
err := c.get(ctx, "/api/v1/nodes/"+url.PathEscape(node)+"/logs?lines=200", &out)
|
||
return consoleLogsMsg{node: node, lines: out.Lines, err: err}
|
||
}
|
||
}
|
||
|
||
func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||
switch msg := msg.(type) {
|
||
case tea.WindowSizeMsg:
|
||
m.width, m.height = msg.Width, msg.Height
|
||
case spinTickMsg:
|
||
var cmd tea.Cmd
|
||
m.spin, cmd = m.spin.Update(msg)
|
||
return m, cmd
|
||
case consoleTickMsg:
|
||
if m.node {
|
||
return m, nil
|
||
}
|
||
cmds := []tea.Cmd{m.fetch(), tea.Tick(3*time.Second, func(time.Time) tea.Msg { return consoleTickMsg{} })}
|
||
if m.menu == cUsage {
|
||
cmds = append(cmds, m.fetchUsage())
|
||
}
|
||
if m.menu == cLogs && m.detail && m.logs.node != "" {
|
||
cmds = append(cmds, m.fetchLogs(m.logs.node))
|
||
}
|
||
return m, tea.Batch(cmds...)
|
||
case consoleDataMsg:
|
||
m.loaded = true
|
||
if msg.err != nil {
|
||
m.data.err = msg.err
|
||
} else {
|
||
m.data = consoleData(msg)
|
||
}
|
||
case consoleUsageMsg:
|
||
m.usage, m.usageErr = msg.rows, msg.err
|
||
case consoleLogsMsg:
|
||
m.logs = msg
|
||
case tea.KeyMsg:
|
||
if msg.Type == tea.KeyCtrlC {
|
||
return m, tea.Quit
|
||
}
|
||
return m.key(msg)
|
||
}
|
||
return m, nil
|
||
}
|
||
|
||
func (m consoleModel) entryOK(i int) bool {
|
||
if m.node {
|
||
return i == cSettings || i == cLeave
|
||
}
|
||
return i != cReplicate || !m.storeReplicated()
|
||
}
|
||
|
||
func (m consoleModel) storeReplicated() bool { return len(m.data.sum.Store) > 0 }
|
||
|
||
func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||
if m.dialog != "" {
|
||
return m.dialogKey(k)
|
||
}
|
||
switch k.Type {
|
||
case tea.KeyPgDown:
|
||
m.scroll += max(1, m.height-8)
|
||
return m, nil
|
||
case tea.KeyPgUp:
|
||
m.scroll = max(0, m.scroll-max(1, m.height-8))
|
||
return m, nil
|
||
}
|
||
if !m.inPanel {
|
||
switch k.String() {
|
||
case "up", "k":
|
||
for {
|
||
m.menu = (m.menu - 1 + cCount) % cCount
|
||
if m.entryOK(m.menu) {
|
||
break
|
||
}
|
||
}
|
||
m.row, m.detail, m.scroll = 0, false, 0
|
||
return m, m.onEnter()
|
||
case "down", "j":
|
||
for {
|
||
m.menu = (m.menu + 1) % cCount
|
||
if m.entryOK(m.menu) {
|
||
break
|
||
}
|
||
}
|
||
m.row, m.detail, m.scroll = 0, false, 0
|
||
return m, m.onEnter()
|
||
case "q":
|
||
return m.leave()
|
||
case "right", "l", "tab", "enter":
|
||
switch m.menu {
|
||
case cAddNode:
|
||
return m.ask("Add a node", "user@address of the machine", 1)
|
||
case cReplicate:
|
||
return m.ask("Replicate the store", "user@address of database machine", 2)
|
||
case cLeave:
|
||
return m.leave()
|
||
case cOverview:
|
||
return m, nil
|
||
}
|
||
m.inPanel = true
|
||
}
|
||
return m, nil
|
||
}
|
||
if m.menu == cSettings {
|
||
return m.settingsKey(k)
|
||
}
|
||
n := m.rowCount()
|
||
switch k.String() {
|
||
case "up", "k":
|
||
if !m.detail && m.row > 0 {
|
||
m.row--
|
||
}
|
||
case "down", "j":
|
||
if !m.detail && m.row < n-1 {
|
||
m.row++
|
||
}
|
||
case "enter":
|
||
if !m.detail && n > 0 && m.menu != cGPUs && m.menu != cReplicas && m.menu != cUsage {
|
||
m.detail, m.scroll = true, 0
|
||
if m.menu == cLogs {
|
||
node := m.data.nodes[m.row].ID
|
||
m.logs = consoleLogsMsg{node: node}
|
||
return m, m.fetchLogs(node)
|
||
}
|
||
}
|
||
case "right", "l":
|
||
if m.menu == cUsage {
|
||
m.usageBy = (m.usageBy + 1) % len(usageGroupings)
|
||
return m, m.fetchUsage()
|
||
}
|
||
case "esc", "left", "h", "backspace":
|
||
if m.detail {
|
||
m.detail, m.scroll = false, 0
|
||
} else {
|
||
m.inPanel = false
|
||
}
|
||
case "q":
|
||
return m.leave()
|
||
}
|
||
return m, nil
|
||
}
|
||
|
||
// onEnter loads what a view needs when it is chosen.
|
||
func (m consoleModel) onEnter() tea.Cmd {
|
||
if m.menu == cUsage {
|
||
return m.fetchUsage()
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (m consoleModel) leave() (tea.Model, tea.Cmd) {
|
||
if m.doc != nil && m.doc.Changed > 0 {
|
||
m.dialog, m.dialogTitle = "confirm-leave", "Unsaved changes"
|
||
m.dialogText = fmt.Sprintf("%d change(s) to %s are not saved.\nLeave without saving them?\n\n%s",
|
||
m.doc.Changed, m.file, sDim.Render("Y leave · N stay"))
|
||
return m, nil
|
||
}
|
||
return m, tea.Quit
|
||
}
|
||
|
||
func (m consoleModel) ask(title, question string, n int) (tea.Model, tea.Cmd) {
|
||
m.dialog, m.dialogTitle, m.dialogText, m.asked = "target", title, question, nil
|
||
m.prompt = newTextField()
|
||
m.prompt.Placeholder, m.prompt.Width = "ubuntu@10.0.0.20", 40
|
||
m.prompt.Focus()
|
||
m.editKind = settings.Kind(n) // how many to ask for
|
||
return m, nil
|
||
}
|
||
|
||
func (m consoleModel) dialogKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||
switch m.dialog {
|
||
case "error":
|
||
m.dialog = ""
|
||
return m, nil
|
||
case "confirm-leave":
|
||
if strings.EqualFold(k.String(), "y") {
|
||
return m, tea.Quit
|
||
}
|
||
m.dialog = ""
|
||
return m, nil
|
||
case "saved":
|
||
if strings.EqualFold(k.String(), "y") {
|
||
m.action = consoleAction{kind: "restart"}
|
||
return m, tea.Quit
|
||
}
|
||
m.dialog = ""
|
||
return m, nil
|
||
}
|
||
switch k.Type {
|
||
case tea.KeyEsc:
|
||
m.dialog = ""
|
||
return m, nil
|
||
case tea.KeyEnter:
|
||
v := strings.TrimSpace(m.prompt.Value())
|
||
switch m.dialog {
|
||
case "target":
|
||
if v == "" {
|
||
return m, nil
|
||
}
|
||
m.asked = append(m.asked, v)
|
||
m.prompt.SetValue("")
|
||
if len(m.asked) == int(m.editKind) {
|
||
kind := "node"
|
||
if m.menu == cReplicate {
|
||
kind = "patroni"
|
||
}
|
||
m.action = consoleAction{kind: kind, targets: m.asked}
|
||
return m, tea.Quit
|
||
}
|
||
case "edit":
|
||
if err := m.doc.Set(m.editPath, m.editKind, m.prompt.Value()); err != nil {
|
||
m.dialogText = m.editQuestion() + "\n\n" + sBad.Render(err.Error())
|
||
return m, nil
|
||
}
|
||
m.dialog = ""
|
||
case "addname":
|
||
if v == "" {
|
||
return m, nil
|
||
}
|
||
if err := m.doc.AddEntry(m.editPath, v); err != nil {
|
||
m.dialogText = "Name of the new entry\n\n" + sBad.Render(err.Error())
|
||
return m, nil
|
||
}
|
||
m.dialog = ""
|
||
}
|
||
return m, nil
|
||
}
|
||
var cmd tea.Cmd
|
||
m.prompt, cmd = m.prompt.Update(k)
|
||
return m, cmd
|
||
}
|
||
|
||
// --- settings ---
|
||
|
||
// settingsRow is one line of the settings view.
|
||
type settingsRow struct {
|
||
label, value string
|
||
key *settings.Key
|
||
// index is a list entry's position, or a map entry's name.
|
||
entry string
|
||
add bool
|
||
}
|
||
|
||
func (m consoleModel) level() settingsLevel { return m.levels[len(m.levels)-1] }
|
||
|
||
func (m consoleModel) settingsRows() []settingsRow {
|
||
lv := m.level()
|
||
var rows []settingsRow
|
||
switch lv.kind {
|
||
case settings.Block:
|
||
for _, k := range settings.Keys(lv.block) {
|
||
k := k
|
||
path := append(append([]string{}, lv.path...), k.Key)
|
||
rows = append(rows, settingsRow{label: k.Key, value: m.valueOf(k, path), key: &k})
|
||
}
|
||
case settings.BlockList:
|
||
for i := 0; i < m.doc.Len(lv.path); i++ {
|
||
rows = append(rows, settingsRow{label: m.entryLabel(append(append([]string{}, lv.path...), strconv.Itoa(i)), i),
|
||
value: "›", entry: strconv.Itoa(i)})
|
||
}
|
||
rows = append(rows, settingsRow{label: "+ add an entry", add: true})
|
||
case settings.BlockMap:
|
||
for _, name := range m.doc.MapKeys(lv.path) {
|
||
rows = append(rows, settingsRow{label: name, value: "›", entry: name})
|
||
}
|
||
rows = append(rows, settingsRow{label: "+ add an entry", add: true})
|
||
}
|
||
return rows
|
||
}
|
||
|
||
// entryLabel names a list entry by its name or id, else its position.
|
||
func (m consoleModel) entryLabel(path []string, i int) string {
|
||
for _, k := range []string{"name", "id", "key", "url"} {
|
||
if v, ok := m.doc.Text(append(path, k)); ok && v != "" {
|
||
return v
|
||
}
|
||
}
|
||
return fmt.Sprintf("entry %d", i+1)
|
||
}
|
||
|
||
func (m consoleModel) valueOf(k settings.Key, path []string) string {
|
||
switch k.Kind {
|
||
case settings.Block:
|
||
return "›"
|
||
case settings.BlockList, settings.BlockMap:
|
||
return fmt.Sprintf("%d ›", m.doc.Len(path))
|
||
}
|
||
if v, ok := m.doc.Text(path); ok {
|
||
return strings.ReplaceAll(v, "\n", ", ")
|
||
}
|
||
if v, ok := settings.Effective(m.cfg, path); ok && v != "" {
|
||
return sDim.Render(v + " (default)")
|
||
}
|
||
return sDim.Render("—")
|
||
}
|
||
|
||
func (m consoleModel) editQuestion() string {
|
||
hint := map[settings.Kind]string{
|
||
settings.Duration: "a duration: 30s, 5m, 2h",
|
||
settings.Int: "a whole number",
|
||
settings.Float: "a number",
|
||
settings.TextList: "values, separated by commas",
|
||
settings.TextMap: "key = value, separated by commas",
|
||
}[m.editKind]
|
||
q := strings.Join(m.editPath, ".")
|
||
if hint != "" {
|
||
q += "\n" + sDim.Render(hint)
|
||
}
|
||
return q
|
||
}
|
||
|
||
func (m consoleModel) settingsKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||
if m.doc == nil {
|
||
if k.String() == "esc" || k.String() == "left" {
|
||
m.inPanel = false
|
||
}
|
||
return m, nil
|
||
}
|
||
rows := m.settingsRows()
|
||
lv := m.level()
|
||
switch k.String() {
|
||
case "up", "k":
|
||
if m.row > 0 {
|
||
m.row--
|
||
}
|
||
case "down", "j":
|
||
if m.row < len(rows)-1 {
|
||
m.row++
|
||
}
|
||
case "esc", "left", "h", "backspace":
|
||
if len(m.levels) > 1 {
|
||
m.levels = m.levels[:len(m.levels)-1]
|
||
m.row, m.scroll = 0, 0
|
||
} else {
|
||
m.inPanel = false
|
||
}
|
||
case "s":
|
||
return m.save()
|
||
case "u", "delete":
|
||
if m.row >= len(rows) {
|
||
return m, nil
|
||
}
|
||
r := rows[m.row]
|
||
switch {
|
||
case r.key != nil:
|
||
m.doc.Unset(append(append([]string{}, lv.path...), r.key.Key))
|
||
case r.entry != "":
|
||
m.doc.Unset(append(append([]string{}, lv.path...), r.entry))
|
||
m.row = max(0, m.row-1)
|
||
}
|
||
case "enter", "right", "l":
|
||
if m.row >= len(rows) {
|
||
return m, nil
|
||
}
|
||
r := rows[m.row]
|
||
switch {
|
||
case r.add && lv.kind == settings.BlockList:
|
||
i := m.doc.Append(lv.path)
|
||
m.levels = append(m.levels, settingsLevel{title: fmt.Sprintf("entry %d", i+1), block: lv.block,
|
||
path: append(append([]string{}, lv.path...), strconv.Itoa(i)), kind: settings.Block})
|
||
m.row = 0
|
||
case r.add && lv.kind == settings.BlockMap:
|
||
m.dialog, m.dialogTitle, m.dialogText = "addname", "Add an entry", "Name of the new entry"
|
||
m.editPath = lv.path
|
||
m.prompt = newTextField()
|
||
m.prompt.Width = 40
|
||
m.prompt.Focus()
|
||
case r.entry != "":
|
||
m.levels = append(m.levels, settingsLevel{title: r.label, block: lv.block,
|
||
path: append(append([]string{}, lv.path...), r.entry), kind: settings.Block})
|
||
m.row = 0
|
||
case r.key != nil:
|
||
path := append(append([]string{}, lv.path...), r.key.Key)
|
||
switch r.key.Kind {
|
||
case settings.Block:
|
||
m.levels = append(m.levels, settingsLevel{title: r.key.Key, block: r.key.Block, path: path, kind: settings.Block})
|
||
m.row = 0
|
||
case settings.BlockList, settings.BlockMap:
|
||
m.levels = append(m.levels, settingsLevel{title: r.key.Key, block: r.key.Block, path: path, kind: r.key.Kind})
|
||
m.row = 0
|
||
case settings.Bool:
|
||
cur, ok := m.doc.Text(path)
|
||
if !ok {
|
||
cur, _ = settings.Effective(m.cfg, path)
|
||
}
|
||
b, _ := strconv.ParseBool(cur)
|
||
_ = m.doc.Set(path, settings.Bool, strconv.FormatBool(!b))
|
||
default:
|
||
m.editPath, m.editKind = path, r.key.Kind
|
||
cur, ok := m.doc.Text(path)
|
||
if !ok {
|
||
cur, _ = settings.Effective(m.cfg, path)
|
||
}
|
||
m.prompt = newTextField()
|
||
m.prompt.Width = 50
|
||
m.prompt.SetValue(strings.ReplaceAll(cur, "\n", ", "))
|
||
m.prompt.Focus()
|
||
m.dialog, m.dialogTitle, m.dialogText = "edit", "Change "+r.key.Key, m.editQuestion()
|
||
}
|
||
}
|
||
m.scroll = 0
|
||
}
|
||
return m, nil
|
||
}
|
||
|
||
func (m consoleModel) save() (tea.Model, tea.Cmd) {
|
||
if m.doc.Changed == 0 {
|
||
m.dialog, m.dialogTitle, m.dialogText = "error", "Nothing to save", "No change has been made."
|
||
return m, nil
|
||
}
|
||
backup, err := m.doc.Save(func(p string) error { _, err := m.load(p); return err })
|
||
if err != nil {
|
||
m.dialog, m.dialogTitle = "error", "Not saved"
|
||
m.dialogText = "The configuration would not load:\n\n" + sBad.Render(err.Error()) + "\n\n" + sDim.Render("Nothing was written. Any key to go back.")
|
||
return m, nil
|
||
}
|
||
m.loadSettings()
|
||
note := ""
|
||
if backup != "" {
|
||
note = "The old file is " + backup + ".\n"
|
||
}
|
||
m.dialog, m.dialogTitle = "saved", "Saved"
|
||
m.dialogText = "Saved " + m.file + ".\n" + note +
|
||
fmt.Sprintf("\n%s reads it when it starts. Restart it now?\n", m.service())
|
||
if !m.node {
|
||
m.dialogText += sDim.Render("Another control plane keeps its own copy: make the change there too.") + "\n"
|
||
}
|
||
m.dialogText += "\n" + sDim.Render("Y restart · N later")
|
||
return m, nil
|
||
}
|
||
|
||
// --- views ---
|
||
|
||
func (m consoleModel) rowCount() int {
|
||
switch m.menu {
|
||
case cNodes, cLogs:
|
||
return len(m.data.nodes)
|
||
case cGPUs:
|
||
return len(m.gpuRows())
|
||
case cModels:
|
||
return len(m.data.models)
|
||
case cReplicas:
|
||
return len(m.data.replicas)
|
||
case cUsage:
|
||
return len(m.usage)
|
||
}
|
||
return 0
|
||
}
|
||
|
||
func (m consoleModel) View() string {
|
||
host, _ := os.Hostname()
|
||
nav := []navItem{{Label: "Watch", Heading: true}}
|
||
if m.node {
|
||
nav = nil
|
||
}
|
||
for i := 0; i < cCount; i++ {
|
||
if m.node && !m.entryOK(i) {
|
||
continue
|
||
}
|
||
switch i {
|
||
case cSettings:
|
||
nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "Settings", Heading: true})
|
||
case cAddNode, cLeave:
|
||
if (i == cAddNode) != m.node {
|
||
nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "Do", Heading: true})
|
||
}
|
||
}
|
||
label := consoleLabels[i]
|
||
if i == cSettings {
|
||
label = filepathBase(m.file)
|
||
}
|
||
if i == cSettings && m.doc != nil && m.doc.Changed > 0 {
|
||
label += fmt.Sprintf(" (%d)", m.doc.Changed)
|
||
}
|
||
nav = append(nav, navItem{Label: label, Current: i == m.menu, Dim: !m.entryOK(i)})
|
||
}
|
||
title, body, keys := m.panel()
|
||
title0 := "llmgw console"
|
||
if m.node {
|
||
title0 = "llmgw · node"
|
||
}
|
||
f := frame{
|
||
Width: m.width, Height: m.height, Title: title0, Where: host,
|
||
NavTitle: "", Nav: nav, PanelTitle: title, Body: body, Scroll: m.scroll, Keys: keys,
|
||
}
|
||
// The selected row kept on screen.
|
||
if m.inPanel && !m.detail {
|
||
inner := m.height - 4
|
||
top := m.row + 3
|
||
if top-m.scroll >= inner-1 {
|
||
f.Scroll = top - inner + 2
|
||
}
|
||
}
|
||
if m.dialog != "" {
|
||
f.DialogTitle = m.dialogTitle
|
||
f.Dialog = m.dialogText
|
||
switch m.dialog {
|
||
case "target", "edit", "addname":
|
||
q := m.dialogText
|
||
if m.dialog == "target" && m.editKind > 1 {
|
||
q = fmt.Sprintf("%s %d of %d", q, len(m.asked)+1, int(m.editKind))
|
||
}
|
||
f.Dialog = q + "\n\n" + m.prompt.View() + "\n\n" + sDim.Render("Enter · Esc cancel")
|
||
}
|
||
}
|
||
return f.render()
|
||
}
|
||
|
||
// panel is the right-hand side for the entry chosen.
|
||
func (m consoleModel) panel() (title, body, keys string) {
|
||
pw := max(40, m.width-navWidth-8)
|
||
rowKeys := "↑↓ row · Enter detail · ← back · PgUp/PgDn · Q leave"
|
||
if !m.inPanel {
|
||
keys = "↑↓ choose · → or Enter open · Q leave · refreshed every three seconds"
|
||
}
|
||
waiting := func() string {
|
||
if m.data.err != nil {
|
||
return "\n " + sBad.Render("The control plane does not answer: ") + m.data.err.Error() +
|
||
"\n\n " + sDim.Render("systemctl status llmgw-server says why.")
|
||
}
|
||
return "\n " + m.spin.View() + " asking the control plane…"
|
||
}
|
||
if (!m.loaded || m.data.err != nil) && !m.node {
|
||
if m.menu < cSettings {
|
||
return consoleLabels[m.menu], waiting(), keys
|
||
}
|
||
}
|
||
switch m.menu {
|
||
case cOverview:
|
||
var b bytes.Buffer
|
||
c := &cobra.Command{}
|
||
c.SetOut(&b)
|
||
printSummary(c, m.data.sum)
|
||
return "Overview", "\n" + indent(b.String(), " "), keys
|
||
case cNodes, cLogs:
|
||
if m.detail && m.row < len(m.data.nodes) {
|
||
n := m.data.nodes[m.row]
|
||
if m.menu == cLogs {
|
||
return "Agent log · " + n.ID, m.logView(), "← back · PgUp/PgDn · refreshed every three seconds"
|
||
}
|
||
return "Node · " + n.ID, m.nodeDetail(n), "← back · PgUp/PgDn · Q leave"
|
||
}
|
||
var rows [][]string
|
||
for _, n := range m.data.nodes {
|
||
cpu, mem := "—", "—"
|
||
if n.Load != nil {
|
||
cpu, mem = fmt.Sprintf("%.0f%%", n.Load.CPUPercent), fmt.Sprintf("%.0f%%", n.Load.MemPercent)
|
||
}
|
||
state := n.State
|
||
if state != "ready" {
|
||
state = sWarn.Render(state)
|
||
}
|
||
rows = append(rows, []string{n.ID, state, cpu, mem, fmt.Sprintf("%d/%d", n.GPUFree, n.GPUTotal),
|
||
strconv.Itoa(n.Replicas), fmt.Sprintf("%d/%d GB", n.DiskFreeGB, n.DiskTotalGB), n.AgentVersion, n.LastSeenAgo})
|
||
}
|
||
t := "Nodes"
|
||
if m.menu == cLogs {
|
||
t = "Logs · choose a node"
|
||
}
|
||
if m.inPanel {
|
||
keys = rowKeys
|
||
}
|
||
return t, "\n" + table([]string{"NODE", "STATE", "CPU", "MEM", "GPU FREE", "REPLICAS", "DISK FREE", "AGENT", "SEEN"},
|
||
rows, m.row, pw, m.inPanel), keys
|
||
case cGPUs:
|
||
if m.inPanel {
|
||
keys = "↑↓ row · ← back · PgUp/PgDn · Q leave"
|
||
}
|
||
return "GPUs", "\n" + table([]string{"NODE", "GPU", "CARD", "BUSY", "MEMORY", "TEMP", "POWER", "HELD BY"},
|
||
m.gpuRows(), m.row, pw, m.inPanel), keys
|
||
case cModels:
|
||
if m.detail && m.row < len(m.data.models) {
|
||
return "Model · " + m.data.models[m.row].Name, m.modelDetail(m.data.models[m.row]), "← back · PgUp/PgDn · Q leave"
|
||
}
|
||
var rows [][]string
|
||
for _, md := range m.data.models {
|
||
status := "stopped"
|
||
switch {
|
||
case md.Parked:
|
||
status = sBad.Render("parked")
|
||
case md.Unplaceable:
|
||
status = sWarn.Render("not placed")
|
||
case md.Ready > 0 && md.Ready >= md.Desired:
|
||
status = sOK.Render("ready")
|
||
case md.Desired > 0:
|
||
status = sWarn.Render("starting")
|
||
}
|
||
warn := ""
|
||
if len(md.Warnings) > 0 {
|
||
warn = sWarn.Render("!")
|
||
}
|
||
rows = append(rows, []string{md.Name, fmt.Sprintf("%d/%d", md.Ready, md.Desired), status, md.Class,
|
||
strings.Join(md.Profiles, ","), warn})
|
||
}
|
||
if m.inPanel {
|
||
keys = rowKeys
|
||
}
|
||
return "Models", "\n" + table([]string{"MODEL", "READY", "STATUS", "CLASS", "PROFILES", ""}, rows, m.row, pw, m.inPanel), keys
|
||
case cReplicas:
|
||
var rows [][]string
|
||
for _, r := range m.data.replicas {
|
||
var nodes []string
|
||
for _, s := range r.Shards {
|
||
nodes = append(nodes, fmt.Sprintf("%s%v", s.Node, s.GPUs))
|
||
}
|
||
rows = append(rows, []string{r.ID, r.Model, r.State, r.Pool, strings.Join(nodes, " "),
|
||
(time.Duration(r.AgeSec) * time.Second).Round(time.Second).String()})
|
||
}
|
||
if m.inPanel {
|
||
keys = "↑↓ row · ← back · PgUp/PgDn · Q leave"
|
||
}
|
||
return "Replicas", "\n" + table([]string{"REPLICA", "MODEL", "STATE", "POOL", "ON", "AGE"}, rows, m.row, pw, m.inPanel), keys
|
||
case cUsage:
|
||
head := "\n Last 24 hours, by " + lipgloss.NewStyle().Bold(true).Render(usageGroupings[m.usageBy]) +
|
||
sDim.Render(" (→ to group by "+usageGroupings[(m.usageBy+1)%len(usageGroupings)]+")") + "\n\n"
|
||
if m.usageErr != nil {
|
||
return "Usage", head + " " + sBad.Render(m.usageErr.Error()), keys
|
||
}
|
||
var rows [][]string
|
||
for _, u := range m.usage {
|
||
rows = append(rows, []string{u.Name, compactNum(u.Requests), compactNum(u.Prompt + u.Completion),
|
||
compactNum(u.Errors), fmt.Sprintf("%.2f", float64(u.CostMicros)/1e6)})
|
||
}
|
||
if m.inPanel {
|
||
keys = "↑↓ row · → grouping · ← back · Q leave"
|
||
}
|
||
return "Usage", head + table([]string{"WHO", "REQUESTS", "TOKENS", "ERRORS", "SPENT"}, rows, m.row, pw, m.inPanel), keys
|
||
case cSettings:
|
||
return m.settingsPanel(pw)
|
||
}
|
||
var item string
|
||
switch m.menu {
|
||
case cAddNode:
|
||
item = "\n Adds a machine as a node: installs llmgw on it over ssh, checks it,\n gives it a certificate and starts its agent.\n\n " +
|
||
sDim.Render("On the command line:") + "\n sudo llmgw node add <user>@<address>"
|
||
case cReplicate:
|
||
item = "\n Moves the store onto two machines under Patroni, replicated,\n so losing one loses nothing.\n\n " +
|
||
sDim.Render("On the command line:") +
|
||
"\n sudo --preserve-env=SSH_AUTH_SOCK llmgw server init \\\n --patroni <user>@<a> --patroni <user>@<b> --no-mirror --no-node --no-model"
|
||
case cLeave:
|
||
item = "\n Closes the console. The cluster keeps running."
|
||
}
|
||
return consoleLabels[m.menu], item, "Enter do it · ↑↓ choose · Q leave"
|
||
}
|
||
|
||
func (m consoleModel) settingsPanel(pw int) (string, string, string) {
|
||
if m.docErr != nil {
|
||
return "server.yaml", "\n " + sBad.Render(m.docErr.Error()), "← back"
|
||
}
|
||
var crumbs []string
|
||
for _, l := range m.levels {
|
||
crumbs = append(crumbs, l.title)
|
||
}
|
||
rows := m.settingsRows()
|
||
var tr [][]string
|
||
for _, r := range rows {
|
||
typ := ""
|
||
if r.key != nil {
|
||
typ = sDim.Render(r.key.Type)
|
||
}
|
||
tr = append(tr, []string{r.label, r.value, typ})
|
||
}
|
||
body := "\n" + table([]string{"KEY", "VALUE", "TYPE"}, tr, m.row, pw, m.inPanel)
|
||
// What the chosen key is for, from its comment in the code.
|
||
if m.inPanel && m.row < len(rows) {
|
||
doc := ""
|
||
if r := rows[m.row]; r.key != nil {
|
||
doc = r.key.Doc
|
||
if doc == "" && r.key.Block != nil {
|
||
doc = r.key.Block.Doc
|
||
}
|
||
} else if l := m.level(); l.block != nil {
|
||
doc = l.block.Doc
|
||
}
|
||
if doc != "" {
|
||
body += "\n " + sDim.Render(strings.Repeat("─", min(pw-4, 60))) + "\n" +
|
||
indent(wrap(firstParagraphs(doc, 2), pw-4, ""), " ")
|
||
}
|
||
}
|
||
keys := "→ or Enter open · ↑↓ row · Q leave"
|
||
if m.inPanel {
|
||
keys = "Enter change/open · U back to default · S save · Esc up · PgUp/PgDn"
|
||
if l := m.level(); l.kind != settings.Block {
|
||
keys = "Enter open · D remove entry · S save · Esc up"
|
||
}
|
||
}
|
||
return strings.Join(crumbs, " › "), body, keys
|
||
}
|
||
|
||
func firstParagraphs(s string, n int) string {
|
||
p := strings.Split(s, "\n\n")
|
||
if len(p) > n {
|
||
p = p[:n]
|
||
}
|
||
return strings.Join(p, "\n\n")
|
||
}
|
||
|
||
func (m consoleModel) gpuRows() [][]string {
|
||
held := map[string]string{}
|
||
for _, r := range m.data.replicas {
|
||
for _, s := range r.Shards {
|
||
for _, g := range s.GPUs {
|
||
held[s.Node+"/"+strconv.Itoa(g)] = r.Model
|
||
}
|
||
}
|
||
}
|
||
var rows [][]string
|
||
for _, n := range m.data.nodes {
|
||
for _, g := range n.GPUs {
|
||
busy, mem, temp, power := "—", "—", "—", "—"
|
||
if n.Load != nil {
|
||
for _, s := range n.Load.GPUs {
|
||
if s.Index != g.Index {
|
||
continue
|
||
}
|
||
busy = fmt.Sprintf("%.0f%%", s.UtilPercent)
|
||
if s.MemoryTotalMB > 0 {
|
||
mem = fmt.Sprintf("%.1f/%.0f GB", s.MemoryUsedMB/1024, s.MemoryTotalMB/1024)
|
||
}
|
||
if s.TempC > 0 {
|
||
temp = fmt.Sprintf("%.0f°C", s.TempC)
|
||
}
|
||
if s.PowerW > 0 {
|
||
power = fmt.Sprintf("%.0f/%.0f W", s.PowerW, s.PowerLimitW)
|
||
}
|
||
}
|
||
}
|
||
holder := held[n.ID+"/"+strconv.Itoa(g.Index)]
|
||
if holder == "" {
|
||
holder = sDim.Render("free")
|
||
}
|
||
rows = append(rows, []string{n.ID, strconv.Itoa(g.Index), g.Name, busy, mem, temp, power, holder})
|
||
}
|
||
}
|
||
return rows
|
||
}
|
||
|
||
func (m consoleModel) nodeDetail(n adminapi.NodeView) string {
|
||
var b strings.Builder
|
||
kv := func(k, v string) { fmt.Fprintf(&b, " %-14s %s\n", sDim.Render(k), v) }
|
||
b.WriteString("\n")
|
||
kv("state", n.State)
|
||
kv("address", n.MgmtAddr)
|
||
kv("system", n.OS)
|
||
kv("agent", n.AgentVersion+", seen "+n.LastSeenAgo+" ago")
|
||
if n.Load != nil {
|
||
l := n.Load
|
||
kv("cpu", fmt.Sprintf("%.0f%% of %d · load %.2f %.2f %.2f", l.CPUPercent, l.CPUs, l.Load1, l.Load5, l.Load15))
|
||
kv("memory", fmt.Sprintf("%.0f%% · %.1f of %.1f GB", l.MemPercent, float64(l.MemUsed)/(1<<30), float64(l.MemTotal)/(1<<30)))
|
||
}
|
||
kv("disk", fmt.Sprintf("%d GB free of %d", n.DiskFreeGB, n.DiskTotalGB))
|
||
kv("replicas", strconv.Itoa(n.Replicas))
|
||
if len(n.Provision) > 0 {
|
||
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("Engines") + "\n")
|
||
for _, p := range n.Provision {
|
||
fmt.Fprintf(&b, " %-9s %-10s %-8s %s\n", p.Engine, p.Version, p.State, sDim.Render(truncate(p.Detail, 60)))
|
||
}
|
||
}
|
||
if len(n.GPUs) > 0 {
|
||
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("GPUs") + "\n")
|
||
for _, r := range m.gpuRows() {
|
||
if r[0] == n.ID {
|
||
fmt.Fprintf(&b, " %s %-18s busy %-5s mem %-14s %-6s %s %s\n", r[1], r[2], r[3], r[4], r[5], r[6], r[7])
|
||
}
|
||
}
|
||
}
|
||
for _, w := range append(n.Warnings, n.Unhealthy...) {
|
||
b.WriteString("\n " + sWarn.Render("! ") + w)
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
func (m consoleModel) modelDetail(md adminapi.ModelView) string {
|
||
var b strings.Builder
|
||
kv := func(k, v string) {
|
||
if v != "" {
|
||
fmt.Fprintf(&b, " %-14s %s\n", sDim.Render(k), v)
|
||
}
|
||
}
|
||
b.WriteString("\n")
|
||
if md.Description != "" {
|
||
b.WriteString(" " + md.Description + "\n\n")
|
||
}
|
||
kv("replicas", fmt.Sprintf("%d ready of %d wanted (min %d, max %d)", md.Ready, md.Desired, md.Min, md.Max))
|
||
kv("class", md.Class)
|
||
kv("owner", md.Owner)
|
||
kv("profiles", strings.Join(md.Profiles, ", "))
|
||
kv("aliases", strings.Join(md.Aliases, ", "))
|
||
kv("source", md.Source)
|
||
if md.MemoryGB > 0 {
|
||
kv("reserves", fmt.Sprintf("%d GB, %d cores (uses %.1f GB)", md.MemoryGB, md.CPUCores, md.MeasuredMemoryGB))
|
||
}
|
||
kv("fallbacks", strings.Join(md.Fallbacks, ", "))
|
||
if md.Reason != "" {
|
||
b.WriteString("\n " + sWarn.Render("! ") + md.Reason + "\n")
|
||
}
|
||
for _, w := range md.Warnings {
|
||
b.WriteString(" " + sWarn.Render("! ") + w + "\n")
|
||
}
|
||
var reps [][]string
|
||
for _, r := range m.data.replicas {
|
||
if r.Model == md.Name {
|
||
reps = append(reps, []string{r.ID, r.State, r.Pool, r.Endpoint})
|
||
}
|
||
}
|
||
if len(reps) > 0 {
|
||
b.WriteString("\n" + table([]string{"REPLICA", "STATE", "POOL", "ENDPOINT"}, reps, -1, max(40, m.width-navWidth-8), false))
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
func (m consoleModel) logView() string {
|
||
if m.logs.err != nil {
|
||
return "\n " + sBad.Render(m.logs.err.Error())
|
||
}
|
||
if len(m.logs.lines) == 0 {
|
||
return "\n " + m.spin.View() + " reading the log…"
|
||
}
|
||
// The newest at the bottom, where a log is read; the panel starts there.
|
||
lines := m.logs.lines
|
||
if k := m.height - 6; len(lines) > k && m.scroll == 0 {
|
||
lines = lines[len(lines)-k:]
|
||
}
|
||
var b strings.Builder
|
||
b.WriteString("\n")
|
||
for _, l := range lines {
|
||
switch {
|
||
case strings.Contains(l, "level=ERROR"):
|
||
l = sBad.Render(l)
|
||
case strings.Contains(l, "level=WARN"):
|
||
l = sWarn.Render(l)
|
||
}
|
||
b.WriteString(" " + l + "\n")
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
func compactNum(n int64) string {
|
||
switch {
|
||
case n >= 1_000_000:
|
||
return fmt.Sprintf("%.1fM", float64(n)/1e6)
|
||
case n >= 1_000:
|
||
return fmt.Sprintf("%.1fk", float64(n)/1e3)
|
||
}
|
||
return strconv.FormatInt(n, 10)
|
||
}
|
||
|
||
func indent(s, pre string) string {
|
||
lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
|
||
for i := range lines {
|
||
lines[i] = pre + lines[i]
|
||
}
|
||
return strings.Join(lines, "\n")
|
||
}
|
||
|
||
func filepathBase(p string) string {
|
||
if i := strings.LastIndexByte(p, '/'); i >= 0 {
|
||
return p[i+1:]
|
||
}
|
||
return p
|
||
}
|
||
|
||
// runConsole shows the console, runs what is chosen on it, and comes back.
|
||
// file is server.yaml on a control plane, agent.yaml on a node.
|
||
func runConsole(ctx context.Context, out io.Writer, file string) error {
|
||
addr := "127.0.0.1:9090"
|
||
if cfg, err := config.Load(file); err == nil && cfg.AdminListen != "" {
|
||
addr = cfg.AdminListen
|
||
if strings.HasPrefix(addr, ":") {
|
||
addr = "127.0.0.1" + addr
|
||
}
|
||
}
|
||
menu := cOverview
|
||
if strings.HasSuffix(file, "agent.yaml") {
|
||
menu = cSettings
|
||
}
|
||
for {
|
||
m := newConsoleModel(addr, file)
|
||
m.menu = menu
|
||
final, err := tea.NewProgram(m, tea.WithOutput(out), tea.WithAltScreen()).Run()
|
||
if err != nil {
|
||
return err
|
||
}
|
||
fm := final.(consoleModel)
|
||
menu = fm.menu
|
||
act := fm.action
|
||
switch act.kind {
|
||
case "":
|
||
return nil
|
||
case "node":
|
||
c := newNodeAddCmd()
|
||
c.SetArgs(act.targets)
|
||
c.SetOut(out)
|
||
err = c.ExecuteContext(ctx)
|
||
case "patroni":
|
||
in := newDefaultServerInit()
|
||
in.patroni = act.targets
|
||
// Already set up: only the store changes.
|
||
in.noMirror, in.noNode, in.noModel = true, true, true
|
||
err = in.runInitTUI(ctx, out)
|
||
case "restart":
|
||
fmt.Fprintf(out, "restarting %s…\n", fm.service())
|
||
if b, e := exec.CommandContext(ctx, "systemctl", "restart", fm.service()).CombinedOutput(); e != nil {
|
||
err = fmt.Errorf("%v: %s", e, strings.TrimSpace(string(b)))
|
||
} else {
|
||
fmt.Fprintln(out, sOK.Render("✓ ")+"restarted; it picks up the configuration as it starts")
|
||
}
|
||
}
|
||
if err != nil {
|
||
fmt.Fprintln(out, sBad.Render("✗ ")+err.Error())
|
||
}
|
||
fmt.Fprint(out, "\n"+sDim.Render("enter to go back to the console")+" ")
|
||
_, _ = bufio.NewReader(os.Stdin).ReadString('\n')
|
||
}
|
||
}
|