Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1449 lines
41 KiB
Go
1449 lines
41 KiB
Go
package cli
|
||
|
||
import (
|
||
"bufio"
|
||
"bytes"
|
||
"context"
|
||
"fmt"
|
||
"io"
|
||
"net/url"
|
||
"os"
|
||
"os/exec"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
tea "github.com/charmbracelet/bubbletea"
|
||
"github.com/charmbracelet/lipgloss"
|
||
"github.com/spf13/cobra"
|
||
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/docgen"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||
)
|
||
|
||
// The console: what a bare `sudo llmgw` opens on a control plane. Down the
|
||
// left, three groups — what is running (watched live), the settings (every
|
||
// key of server.yaml, from the generated reference), and what can be done.
|
||
|
||
// Sidebar entries, in order. Headings are not entries.
|
||
const (
|
||
cOverview = iota
|
||
cMonitor
|
||
cNodes
|
||
cGPUs
|
||
cModels
|
||
cReplicas
|
||
cUsage
|
||
cQuotas
|
||
cMirror
|
||
cJobs
|
||
cLogs
|
||
cAccounts
|
||
cKeys
|
||
cOrgs
|
||
cAudit
|
||
cManual
|
||
cSettings
|
||
cAddNode
|
||
cReplicate
|
||
cServe
|
||
cUpgrade
|
||
cRestartAll
|
||
cBackup
|
||
cLeave
|
||
cCount
|
||
)
|
||
|
||
var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Audit", "Manual",
|
||
"server.yaml", "Add a node", "Replicate the store", "Serve a model", "Upgrade the cluster",
|
||
"Restart all replicas", "Back up now", "Leave"}
|
||
|
||
// consoleAction is what is done once the screen closes: each runs its own
|
||
// screen, and the console comes back after.
|
||
type consoleAction struct {
|
||
kind string // "", "node", "patroni", "restart", "run" (llmgw with targets as its arguments)
|
||
targets []string
|
||
}
|
||
|
||
type consoleData struct {
|
||
sum adminapi.Summary
|
||
nodes []adminapi.NodeView
|
||
models []adminapi.ModelView
|
||
replicas []adminapi.ReplicaView
|
||
jobs []modelserve.Job
|
||
audit []store.AuditEvent
|
||
err error
|
||
}
|
||
|
||
type (
|
||
consoleDataMsg consoleData
|
||
consoleUsageMsg struct {
|
||
rows []adminapi.ScopeUsageView
|
||
err error
|
||
}
|
||
consoleLogsMsg struct {
|
||
node string
|
||
lines []string
|
||
err error
|
||
}
|
||
consoleTickMsg struct{}
|
||
)
|
||
|
||
// settingsLevel is where the settings view is: a block, or a list or map of
|
||
// blocks, at a path of keys in the file.
|
||
type settingsLevel struct {
|
||
title string
|
||
block *docgen.Block
|
||
path []string
|
||
kind settings.Kind // Block, BlockList or BlockMap
|
||
}
|
||
|
||
type consoleModel struct {
|
||
client *ctlClient
|
||
width, height int
|
||
spin spinnerModel
|
||
|
||
data consoleData
|
||
loaded bool
|
||
usage []adminapi.ScopeUsageView
|
||
// usageBy is the grouping on the usage view.
|
||
usageBy int
|
||
usageErr error
|
||
logs consoleLogsMsg
|
||
|
||
// menu is the sidebar entry; inPanel says the keys go to the panel.
|
||
menu int
|
||
inPanel bool
|
||
row int
|
||
// detail is the panel showing one row's detail rather than the table.
|
||
detail bool
|
||
scroll int
|
||
|
||
// node is the console of a node: its agent.yaml and nothing to watch,
|
||
// since the cluster is watched from the control plane.
|
||
node bool
|
||
|
||
// The settings.
|
||
file string
|
||
doc *settings.Doc
|
||
cfg any // the file as loaded, for the values that apply
|
||
levels []settingsLevel
|
||
docErr error
|
||
saveNote string
|
||
|
||
// A question open as a dialog.
|
||
dialog string // "", "target", "edit", "addname", "saved", "confirm-leave", "error"
|
||
dialogTitle string
|
||
dialogText string
|
||
prompt textField
|
||
asked []string
|
||
editPath []string
|
||
editKind settings.Kind
|
||
|
||
action consoleAction
|
||
|
||
// What can be done to a row: a menu, a form, a question, and the request
|
||
// under way.
|
||
menuD *menuDialog
|
||
formD *formDialog
|
||
confirmD *confirmDialog
|
||
busy string
|
||
|
||
quotas []adminapi.QuotaView
|
||
quotasErr error
|
||
|
||
st *storeHolder
|
||
people peopleData
|
||
mirror consoleMirrorMsg
|
||
hf hfState
|
||
docs docViewer
|
||
mon monitorState
|
||
det monitorState
|
||
}
|
||
|
||
var usageGroupings = []string{"team", "project", "org"}
|
||
|
||
func newConsoleModel(addr, file string) consoleModel {
|
||
m := consoleModel{client: newCtlClient(addr), spin: newSpinner(), width: 100, height: 30, file: file,
|
||
st: &storeHolder{file: file}, docs: newDocViewer()}
|
||
m.node = strings.HasSuffix(file, "agent.yaml")
|
||
if m.node {
|
||
m.menu = cSettings
|
||
}
|
||
m.loadSettings()
|
||
return m
|
||
}
|
||
|
||
// load reads the file as the service would, defaults and checks included.
|
||
func (m consoleModel) load(path string) (any, error) {
|
||
if m.node {
|
||
return config.LoadAgent(path)
|
||
}
|
||
return config.Load(path)
|
||
}
|
||
|
||
func (m consoleModel) service() string {
|
||
if m.node {
|
||
return "llmgw-agent"
|
||
}
|
||
return "llmgw-server"
|
||
}
|
||
|
||
func (m *consoleModel) loadSettings() {
|
||
m.doc, m.docErr = settings.Open(m.file)
|
||
if m.docErr != nil {
|
||
return
|
||
}
|
||
if cfg, err := m.load(m.file); err == nil {
|
||
m.cfg = cfg
|
||
} else if !m.node {
|
||
m.cfg = config.Default()
|
||
}
|
||
if root, err := settings.Root(m.file); err == nil && len(m.levels) == 0 {
|
||
m.levels = []settingsLevel{{title: filepathBase(m.file), block: root, kind: settings.Block}}
|
||
}
|
||
}
|
||
|
||
func (m consoleModel) Init() tea.Cmd {
|
||
if m.node {
|
||
return m.spin.Tick
|
||
}
|
||
return tea.Batch(m.fetch(), m.spin.Tick, tea.Tick(3*time.Second, func(time.Time) tea.Msg { return consoleTickMsg{} }))
|
||
}
|
||
|
||
// fetch reads everything the watched views show, in one go.
|
||
func (m consoleModel) fetch() tea.Cmd {
|
||
c := m.client
|
||
return func() tea.Msg {
|
||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||
defer cancel()
|
||
var d consoleData
|
||
if d.err = c.get(ctx, "/api/v1/summary", &d.sum); d.err != nil {
|
||
return consoleDataMsg(d)
|
||
}
|
||
_ = c.get(ctx, "/api/v1/nodes", &d.nodes)
|
||
_ = c.get(ctx, "/api/v1/models", &d.models)
|
||
_ = c.get(ctx, "/api/v1/replicas", &d.replicas)
|
||
_ = c.get(ctx, "/api/v1/jobs", &d.jobs)
|
||
_ = c.get(ctx, "/api/v1/audit?since=7d&limit=300", &d.audit)
|
||
return consoleDataMsg(d)
|
||
}
|
||
}
|
||
|
||
func (m consoleModel) fetchUsage() tea.Cmd {
|
||
c, by := m.client, usageGroupings[m.usageBy]
|
||
return func() tea.Msg {
|
||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||
defer cancel()
|
||
var rows []adminapi.ScopeUsageView
|
||
err := c.get(ctx, "/api/v1/usage?by="+by+"&since=24h", &rows)
|
||
return consoleUsageMsg{rows, err}
|
||
}
|
||
}
|
||
|
||
func (m consoleModel) fetchLogs(node string) tea.Cmd {
|
||
c := m.client
|
||
return func() tea.Msg {
|
||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||
defer cancel()
|
||
var out struct {
|
||
Lines []string `json:"lines"`
|
||
}
|
||
err := c.get(ctx, "/api/v1/nodes/"+url.PathEscape(node)+"/logs?lines=200", &out)
|
||
return consoleLogsMsg{node: node, lines: out.Lines, err: err}
|
||
}
|
||
}
|
||
|
||
func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||
switch msg := msg.(type) {
|
||
case tea.WindowSizeMsg:
|
||
m.width, m.height = msg.Width, msg.Height
|
||
case spinTickMsg:
|
||
var cmd tea.Cmd
|
||
m.spin, cmd = m.spin.Update(msg)
|
||
return m, cmd
|
||
case consoleTickMsg:
|
||
if m.node {
|
||
return m, nil
|
||
}
|
||
cmds := []tea.Cmd{m.fetch(), tea.Tick(3*time.Second, func(time.Time) tea.Msg { return consoleTickMsg{} })}
|
||
if m.menu == cUsage {
|
||
cmds = append(cmds, m.fetchUsage())
|
||
}
|
||
if m.menu == cQuotas {
|
||
cmds = append(cmds, m.fetchQuotas())
|
||
}
|
||
if m.menu == cMirror {
|
||
cmds = append(cmds, m.fetchMirror())
|
||
}
|
||
if m.menu == cMonitor && m.monitorDue() {
|
||
cmds = append(cmds, m.fetchMonitor())
|
||
}
|
||
if m.detailDue() {
|
||
cmds = append(cmds, m.fetchDetail())
|
||
}
|
||
if m.peopleView() {
|
||
cmds = append(cmds, m.fetchPeople())
|
||
}
|
||
if m.menu == cLogs && m.detail && m.logs.node != "" {
|
||
cmds = append(cmds, m.fetchLogs(m.logs.node))
|
||
}
|
||
return m, tea.Batch(cmds...)
|
||
case consoleDataMsg:
|
||
m.loaded = true
|
||
if msg.err != nil {
|
||
m.data.err = msg.err
|
||
} else {
|
||
m.data = consoleData(msg)
|
||
m.mon.sample(m.data)
|
||
}
|
||
case detailMsg:
|
||
if msg.key == m.detailKey() {
|
||
m.det.res, m.det.err, m.det.key = msg.res, msg.err, msg.key
|
||
}
|
||
case monitorMsg:
|
||
// Kept if it is still what the screen wants: the range and width.
|
||
if msg.key == m.monitorKey() {
|
||
m.mon.res, m.mon.err, m.mon.key = msg.res, msg.err, msg.key
|
||
}
|
||
case consoleUsageMsg:
|
||
m.usage, m.usageErr = msg.rows, msg.err
|
||
case consoleLogsMsg:
|
||
m.logs = msg
|
||
case consoleMirrorMsg:
|
||
m.mirror = msg
|
||
case consoleHFMsg:
|
||
if msg.query == m.hf.query && msg.sort == m.hf.sort && msg.st == m.hf.safetensors {
|
||
m.hf.models, m.hf.err = msg.models, msg.err
|
||
if m.hf.models == nil && msg.err == nil {
|
||
m.hf.models = []hf.Model{}
|
||
}
|
||
}
|
||
case consoleRepoMsg:
|
||
m.busy = ""
|
||
if msg.err != nil {
|
||
m.dialog, m.dialogTitle = "error", "Hugging Face"
|
||
m.dialogText = sBad.Render("✗ ") + msg.err.Error() + "\n\n" + sDim.Render("any key")
|
||
} else {
|
||
m.menuD = m.repoMenu(msg.plan)
|
||
}
|
||
case consolePeopleMsg:
|
||
m.people = peopleData(msg)
|
||
case consoleQuotasMsg:
|
||
m.quotas, m.quotasErr = msg.rows, msg.err
|
||
case actionDoneMsg:
|
||
m.busy = ""
|
||
m.dialog, m.dialogTitle = "error", msg.title
|
||
if msg.err != nil {
|
||
m.dialogText = sBad.Render("✗ ") + msg.err.Error()
|
||
} else {
|
||
m.dialogText = sOK.Render("✓ ") + msg.text
|
||
}
|
||
m.dialogText += "\n\n" + sDim.Render("any key")
|
||
return m, tea.Batch(m.fetch(), m.onEnter())
|
||
case tea.KeyMsg:
|
||
if msg.Type == tea.KeyCtrlC {
|
||
return m, tea.Quit
|
||
}
|
||
return m.key(msg)
|
||
}
|
||
return m, nil
|
||
}
|
||
|
||
func (m consoleModel) entryOK(i int) bool {
|
||
if m.node {
|
||
return i == cSettings || i == cManual || i == cLeave
|
||
}
|
||
return i != cReplicate || !m.storeReplicated()
|
||
}
|
||
|
||
func (m consoleModel) storeReplicated() bool { return len(m.data.sum.Store) > 0 }
|
||
|
||
func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||
if m.busy != "" {
|
||
return m, nil
|
||
}
|
||
if m.dialog != "" {
|
||
return m.dialogKey(k)
|
||
}
|
||
if mm, cmd, ok := m.actionKey(k); ok {
|
||
return mm, cmd
|
||
}
|
||
// The manual, from anywhere: its page about this screen.
|
||
if k.String() == "?" && !(m.menu == cManual && m.detail) {
|
||
m.menu, m.inPanel, m.detail = cManual, true, true
|
||
m.docs.show(max(0, findTopic(m.docs.topics, "console")))
|
||
return m, nil
|
||
}
|
||
if m.menu == cManual && m.inPanel && m.detail {
|
||
pw := max(40, m.width-navWidth-8)
|
||
if m.docs.key(k, pw-4, m.height) {
|
||
return m, nil
|
||
}
|
||
switch k.String() {
|
||
case "esc", "left", "h", "backspace":
|
||
m.detail = false
|
||
m.row = m.docs.open
|
||
case "q":
|
||
return m.leave()
|
||
}
|
||
return m, nil
|
||
}
|
||
switch k.Type {
|
||
case tea.KeyPgDown:
|
||
m.scroll += max(1, m.height-8)
|
||
return m, nil
|
||
case tea.KeyPgUp:
|
||
m.scroll = max(0, m.scroll-max(1, m.height-8))
|
||
return m, nil
|
||
}
|
||
if !m.inPanel {
|
||
switch k.String() {
|
||
case "up", "k":
|
||
for {
|
||
m.menu = (m.menu - 1 + cCount) % cCount
|
||
if m.entryOK(m.menu) {
|
||
break
|
||
}
|
||
}
|
||
m.row, m.detail, m.scroll = 0, false, 0
|
||
return m, m.onEnter()
|
||
case "down", "j":
|
||
for {
|
||
m.menu = (m.menu + 1) % cCount
|
||
if m.entryOK(m.menu) {
|
||
break
|
||
}
|
||
}
|
||
m.row, m.detail, m.scroll = 0, false, 0
|
||
return m, m.onEnter()
|
||
case "q":
|
||
return m.leave()
|
||
case "right", "l", "tab", "enter":
|
||
switch m.menu {
|
||
case cAddNode:
|
||
return m.ask("Add a node", "user@address of the machine", 1)
|
||
case cReplicate:
|
||
return m.ask("Replicate the store", "user@address of database machine", 2)
|
||
case cLeave:
|
||
return m.leave()
|
||
case cUpgrade, cRestartAll, cBackup:
|
||
return m, m.doEntry()
|
||
case cOverview:
|
||
return m, nil
|
||
}
|
||
m.inPanel = true
|
||
}
|
||
return m, nil
|
||
}
|
||
if m.menu == cSettings {
|
||
return m.settingsKey(k)
|
||
}
|
||
n := m.rowCount()
|
||
switch k.String() {
|
||
case "up", "k":
|
||
if !m.detail && m.row > 0 {
|
||
m.row--
|
||
}
|
||
case "down", "j":
|
||
if !m.detail && m.row < n-1 {
|
||
m.row++
|
||
}
|
||
case "enter":
|
||
if m.detail || n == 0 || m.row >= n {
|
||
break
|
||
}
|
||
switch m.menu {
|
||
case cLogs:
|
||
m.detail, m.scroll = true, 0
|
||
node := m.data.nodes[m.row].ID
|
||
m.logs = consoleLogsMsg{node: node}
|
||
return m, m.fetchLogs(node)
|
||
case cNodes:
|
||
m.menuD = m.nodeMenu(m.data.nodes[m.row])
|
||
case cModels:
|
||
m.menuD = m.modelMenu(m.data.models[m.row])
|
||
case cReplicas:
|
||
m.menuD = m.replicaMenu(m.data.replicas[m.row])
|
||
case cQuotas:
|
||
m.menuD = m.quotaMenu(m.quotas[m.row])
|
||
case cMirror:
|
||
m.menuD = m.mirrorMenu(m.mirrorGroups()[m.row])
|
||
case cJobs:
|
||
m.dialog = "error"
|
||
m.dialogTitle, m.dialogText = jobDialog(m.data.jobs[m.row])
|
||
case cAudit:
|
||
m.dialog = "error"
|
||
m.dialogTitle, m.dialogText = auditDialog(m.data.audit[m.row])
|
||
case cServe:
|
||
return m, m.fetchRepo(m.hf.models[m.row].ID)
|
||
case cAccounts:
|
||
m.menuD = m.userMenu(m.people.users[m.row])
|
||
case cKeys:
|
||
m.menuD = m.keyMenu(m.people.keys[m.row])
|
||
case cOrgs:
|
||
m.menuD = m.orgMenu(m.people.orgs[m.row])
|
||
case cManual:
|
||
m.docs.show(m.row)
|
||
m.detail = true
|
||
}
|
||
case "1", "2", "3", "4", "r":
|
||
if kind, _ := m.detailTarget(); m.menu == cMonitor || kind != "" {
|
||
if k.String() == "r" {
|
||
m.mon.rng = (m.mon.rng + 1) % len(monRanges)
|
||
} else {
|
||
m.mon.rng = int(k.String()[0] - '1')
|
||
}
|
||
if kind != "" {
|
||
return m, m.fetchDetail()
|
||
}
|
||
return m, m.fetchMonitor()
|
||
}
|
||
case "/":
|
||
if m.menu == cServe {
|
||
m.hfSearchDialog()
|
||
}
|
||
case "f":
|
||
if m.menu == cServe {
|
||
m.hf.safetensors = !m.hf.safetensors
|
||
m.hf.models, m.hf.err, m.row = nil, nil, 0
|
||
return m, m.fetchHF()
|
||
}
|
||
case "s":
|
||
if m.menu == cServe {
|
||
m.hf.sort = (m.hf.sort + 1) % len(hfSorts)
|
||
m.hf.models, m.hf.err, m.row = nil, nil, 0
|
||
return m, m.fetchHF()
|
||
}
|
||
case "a":
|
||
switch m.menu {
|
||
case cQuotas:
|
||
m.quotaForm(adminapi.QuotaView{})
|
||
case cModels:
|
||
m.addModelForm()
|
||
case cMirror:
|
||
m.mirrorFetchForm()
|
||
case cAccounts:
|
||
m.createUserForm()
|
||
case cKeys:
|
||
m.createKeyForm()
|
||
case cOrgs:
|
||
m.createOrgForm()
|
||
}
|
||
case "right", "l":
|
||
if m.menu == cUsage {
|
||
m.usageBy = (m.usageBy + 1) % len(usageGroupings)
|
||
return m, m.fetchUsage()
|
||
}
|
||
case "esc", "left", "h", "backspace":
|
||
if m.detail {
|
||
m.detail, m.scroll = false, 0
|
||
} else {
|
||
m.inPanel = false
|
||
}
|
||
case "q":
|
||
return m.leave()
|
||
}
|
||
return m, nil
|
||
}
|
||
|
||
// onEnter loads what a view needs when it is chosen.
|
||
func (m consoleModel) onEnter() tea.Cmd {
|
||
switch m.menu {
|
||
case cUsage:
|
||
return m.fetchUsage()
|
||
case cQuotas:
|
||
return m.fetchQuotas()
|
||
case cMirror:
|
||
return m.fetchMirror()
|
||
case cMonitor:
|
||
return m.fetchMonitor()
|
||
case cServe:
|
||
if m.hf.models == nil {
|
||
return m.fetchHF()
|
||
}
|
||
case cAccounts, cKeys, cOrgs:
|
||
// The quotas too, for an organization's quota form.
|
||
return tea.Batch(m.fetchPeople(), m.fetchQuotas())
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (m consoleModel) peopleView() bool {
|
||
return m.menu == cAccounts || m.menu == cKeys || m.menu == cOrgs
|
||
}
|
||
|
||
func (m consoleModel) leave() (tea.Model, tea.Cmd) {
|
||
if m.doc != nil && m.doc.Changed > 0 {
|
||
m.dialog, m.dialogTitle = "confirm-leave", "Unsaved changes"
|
||
m.dialogText = fmt.Sprintf("%d change(s) to %s are not saved.\nLeave without saving them?\n\n%s",
|
||
m.doc.Changed, m.file, sDim.Render("Y leave · N stay"))
|
||
return m, nil
|
||
}
|
||
return m, tea.Quit
|
||
}
|
||
|
||
func (m consoleModel) ask(title, question string, n int) (tea.Model, tea.Cmd) {
|
||
m.dialog, m.dialogTitle, m.dialogText, m.asked = "target", title, question, nil
|
||
m.prompt = newTextField()
|
||
m.prompt.Placeholder, m.prompt.Width = "ubuntu@10.0.0.20", 40
|
||
m.prompt.Focus()
|
||
m.editKind = settings.Kind(n) // how many to ask for
|
||
return m, nil
|
||
}
|
||
|
||
func (m consoleModel) dialogKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||
switch m.dialog {
|
||
case "error":
|
||
m.dialog = ""
|
||
return m, nil
|
||
case "confirm-leave":
|
||
if strings.EqualFold(k.String(), "y") {
|
||
return m, tea.Quit
|
||
}
|
||
m.dialog = ""
|
||
return m, nil
|
||
case "saved":
|
||
if strings.EqualFold(k.String(), "y") {
|
||
m.action = consoleAction{kind: "restart"}
|
||
return m, tea.Quit
|
||
}
|
||
m.dialog = ""
|
||
return m, nil
|
||
}
|
||
switch k.Type {
|
||
case tea.KeyEsc:
|
||
m.dialog = ""
|
||
return m, nil
|
||
case tea.KeyEnter:
|
||
v := strings.TrimSpace(m.prompt.Value())
|
||
switch m.dialog {
|
||
case "target":
|
||
if v == "" {
|
||
return m, nil
|
||
}
|
||
m.asked = append(m.asked, v)
|
||
m.prompt.SetValue("")
|
||
if len(m.asked) == int(m.editKind) {
|
||
kind := "node"
|
||
if m.menu == cReplicate {
|
||
kind = "patroni"
|
||
}
|
||
m.action = consoleAction{kind: kind, targets: m.asked}
|
||
return m, tea.Quit
|
||
}
|
||
case "edit":
|
||
if err := m.doc.Set(m.editPath, m.editKind, m.prompt.Value()); err != nil {
|
||
m.dialogText = m.editQuestion() + "\n\n" + sBad.Render(err.Error())
|
||
return m, nil
|
||
}
|
||
m.dialog = ""
|
||
case "addname":
|
||
if v == "" {
|
||
return m, nil
|
||
}
|
||
if err := m.doc.AddEntry(m.editPath, v); err != nil {
|
||
m.dialogText = "Name of the new entry\n\n" + sBad.Render(err.Error())
|
||
return m, nil
|
||
}
|
||
m.dialog = ""
|
||
}
|
||
return m, nil
|
||
}
|
||
var cmd tea.Cmd
|
||
m.prompt, cmd = m.prompt.Update(k)
|
||
return m, cmd
|
||
}
|
||
|
||
// --- settings ---
|
||
|
||
// settingsRow is one line of the settings view.
|
||
type settingsRow struct {
|
||
label, value string
|
||
key *settings.Key
|
||
// index is a list entry's position, or a map entry's name.
|
||
entry string
|
||
add bool
|
||
}
|
||
|
||
func (m consoleModel) level() settingsLevel { return m.levels[len(m.levels)-1] }
|
||
|
||
func (m consoleModel) settingsRows() []settingsRow {
|
||
lv := m.level()
|
||
var rows []settingsRow
|
||
switch lv.kind {
|
||
case settings.Block:
|
||
for _, k := range settings.Keys(lv.block) {
|
||
k := k
|
||
path := append(append([]string{}, lv.path...), k.Key)
|
||
rows = append(rows, settingsRow{label: k.Key, value: m.valueOf(k, path), key: &k})
|
||
}
|
||
case settings.BlockList:
|
||
for i := 0; i < m.doc.Len(lv.path); i++ {
|
||
rows = append(rows, settingsRow{label: m.entryLabel(append(append([]string{}, lv.path...), strconv.Itoa(i)), i),
|
||
value: "›", entry: strconv.Itoa(i)})
|
||
}
|
||
rows = append(rows, settingsRow{label: "+ add an entry", add: true})
|
||
case settings.BlockMap:
|
||
for _, name := range m.doc.MapKeys(lv.path) {
|
||
rows = append(rows, settingsRow{label: name, value: "›", entry: name})
|
||
}
|
||
rows = append(rows, settingsRow{label: "+ add an entry", add: true})
|
||
}
|
||
return rows
|
||
}
|
||
|
||
// entryLabel names a list entry by its name or id, else its position.
|
||
func (m consoleModel) entryLabel(path []string, i int) string {
|
||
for _, k := range []string{"name", "id", "key", "url"} {
|
||
if v, ok := m.doc.Text(append(path, k)); ok && v != "" {
|
||
return v
|
||
}
|
||
}
|
||
return fmt.Sprintf("entry %d", i+1)
|
||
}
|
||
|
||
func (m consoleModel) valueOf(k settings.Key, path []string) string {
|
||
switch k.Kind {
|
||
case settings.Block:
|
||
return "›"
|
||
case settings.BlockList, settings.BlockMap:
|
||
return fmt.Sprintf("%d ›", m.doc.Len(path))
|
||
}
|
||
if v, ok := m.doc.Text(path); ok {
|
||
return strings.ReplaceAll(v, "\n", ", ")
|
||
}
|
||
if v, ok := settings.Effective(m.cfg, path); ok && v != "" {
|
||
return sDim.Render(v + " (default)")
|
||
}
|
||
return sDim.Render("—")
|
||
}
|
||
|
||
func (m consoleModel) editQuestion() string {
|
||
hint := map[settings.Kind]string{
|
||
settings.Duration: "a duration: 30s, 5m, 2h",
|
||
settings.Int: "a whole number",
|
||
settings.Float: "a number",
|
||
settings.TextList: "values, separated by commas",
|
||
settings.TextMap: "key = value, separated by commas",
|
||
}[m.editKind]
|
||
q := strings.Join(m.editPath, ".")
|
||
if hint != "" {
|
||
q += "\n" + sDim.Render(hint)
|
||
}
|
||
return q
|
||
}
|
||
|
||
func (m consoleModel) settingsKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||
if m.doc == nil {
|
||
if k.String() == "esc" || k.String() == "left" {
|
||
m.inPanel = false
|
||
}
|
||
return m, nil
|
||
}
|
||
rows := m.settingsRows()
|
||
lv := m.level()
|
||
switch k.String() {
|
||
case "up", "k":
|
||
if m.row > 0 {
|
||
m.row--
|
||
}
|
||
case "down", "j":
|
||
if m.row < len(rows)-1 {
|
||
m.row++
|
||
}
|
||
case "esc", "left", "h", "backspace":
|
||
if len(m.levels) > 1 {
|
||
m.levels = m.levels[:len(m.levels)-1]
|
||
m.row, m.scroll = 0, 0
|
||
} else {
|
||
m.inPanel = false
|
||
}
|
||
case "s":
|
||
return m.save()
|
||
case "u", "delete":
|
||
if m.row >= len(rows) {
|
||
return m, nil
|
||
}
|
||
r := rows[m.row]
|
||
switch {
|
||
case r.key != nil:
|
||
m.doc.Unset(append(append([]string{}, lv.path...), r.key.Key))
|
||
case r.entry != "":
|
||
m.doc.Unset(append(append([]string{}, lv.path...), r.entry))
|
||
m.row = max(0, m.row-1)
|
||
}
|
||
case "enter", "right", "l":
|
||
if m.row >= len(rows) {
|
||
return m, nil
|
||
}
|
||
r := rows[m.row]
|
||
switch {
|
||
case r.add && lv.kind == settings.BlockList:
|
||
i := m.doc.Append(lv.path)
|
||
m.levels = append(m.levels, settingsLevel{title: fmt.Sprintf("entry %d", i+1), block: lv.block,
|
||
path: append(append([]string{}, lv.path...), strconv.Itoa(i)), kind: settings.Block})
|
||
m.row = 0
|
||
case r.add && lv.kind == settings.BlockMap:
|
||
m.dialog, m.dialogTitle, m.dialogText = "addname", "Add an entry", "Name of the new entry"
|
||
m.editPath = lv.path
|
||
m.prompt = newTextField()
|
||
m.prompt.Width = 40
|
||
m.prompt.Focus()
|
||
case r.entry != "":
|
||
m.levels = append(m.levels, settingsLevel{title: r.label, block: lv.block,
|
||
path: append(append([]string{}, lv.path...), r.entry), kind: settings.Block})
|
||
m.row = 0
|
||
case r.key != nil:
|
||
path := append(append([]string{}, lv.path...), r.key.Key)
|
||
switch r.key.Kind {
|
||
case settings.Block:
|
||
m.levels = append(m.levels, settingsLevel{title: r.key.Key, block: r.key.Block, path: path, kind: settings.Block})
|
||
m.row = 0
|
||
case settings.BlockList, settings.BlockMap:
|
||
m.levels = append(m.levels, settingsLevel{title: r.key.Key, block: r.key.Block, path: path, kind: r.key.Kind})
|
||
m.row = 0
|
||
case settings.Bool:
|
||
cur, ok := m.doc.Text(path)
|
||
if !ok {
|
||
cur, _ = settings.Effective(m.cfg, path)
|
||
}
|
||
b, _ := strconv.ParseBool(cur)
|
||
_ = m.doc.Set(path, settings.Bool, strconv.FormatBool(!b))
|
||
default:
|
||
m.editPath, m.editKind = path, r.key.Kind
|
||
cur, ok := m.doc.Text(path)
|
||
if !ok {
|
||
cur, _ = settings.Effective(m.cfg, path)
|
||
}
|
||
m.prompt = newTextField()
|
||
m.prompt.Width = 50
|
||
m.prompt.SetValue(strings.ReplaceAll(cur, "\n", ", "))
|
||
m.prompt.Focus()
|
||
m.dialog, m.dialogTitle, m.dialogText = "edit", "Change "+r.key.Key, m.editQuestion()
|
||
}
|
||
}
|
||
m.scroll = 0
|
||
}
|
||
return m, nil
|
||
}
|
||
|
||
func (m consoleModel) save() (tea.Model, tea.Cmd) {
|
||
if m.doc.Changed == 0 {
|
||
m.dialog, m.dialogTitle, m.dialogText = "error", "Nothing to save", "No change has been made."
|
||
return m, nil
|
||
}
|
||
backup, err := m.doc.Save(func(p string) error { _, err := m.load(p); return err })
|
||
if err != nil {
|
||
m.dialog, m.dialogTitle = "error", "Not saved"
|
||
m.dialogText = "The configuration would not load:\n\n" + sBad.Render(err.Error()) + "\n\n" + sDim.Render("Nothing was written. Any key to go back.")
|
||
return m, nil
|
||
}
|
||
m.loadSettings()
|
||
note := ""
|
||
if backup != "" {
|
||
note = "The old file is " + backup + ".\n"
|
||
}
|
||
m.dialog, m.dialogTitle = "saved", "Saved"
|
||
m.dialogText = "Saved " + m.file + ".\n" + note +
|
||
fmt.Sprintf("\n%s reads it when it starts. Restart it now?\n", m.service())
|
||
if !m.node {
|
||
m.dialogText += sDim.Render("Another control plane keeps its own copy: make the change there too.") + "\n"
|
||
}
|
||
m.dialogText += "\n" + sDim.Render("Y restart · N later")
|
||
return m, nil
|
||
}
|
||
|
||
// --- views ---
|
||
|
||
func (m consoleModel) rowCount() int {
|
||
switch m.menu {
|
||
case cNodes, cLogs:
|
||
return len(m.data.nodes)
|
||
case cGPUs:
|
||
return len(m.gpuRows())
|
||
case cModels:
|
||
return len(m.data.models)
|
||
case cReplicas:
|
||
return len(m.data.replicas)
|
||
case cUsage:
|
||
return len(m.usage)
|
||
case cQuotas:
|
||
return len(m.quotas)
|
||
case cMirror:
|
||
return len(m.mirrorGroups())
|
||
case cJobs:
|
||
return len(m.data.jobs)
|
||
case cAudit:
|
||
return len(m.data.audit)
|
||
case cServe:
|
||
return len(m.hf.models)
|
||
case cAccounts:
|
||
return len(m.people.users)
|
||
case cKeys:
|
||
return len(m.people.keys)
|
||
case cOrgs:
|
||
return len(m.people.orgs)
|
||
case cManual:
|
||
return len(m.docs.topics)
|
||
}
|
||
return 0
|
||
}
|
||
|
||
func (m consoleModel) View() string {
|
||
host, _ := os.Hostname()
|
||
nav := []navItem{{Label: "Watch", Heading: true}}
|
||
if m.node {
|
||
nav = nil
|
||
}
|
||
for i := 0; i < cCount; i++ {
|
||
if m.node && !m.entryOK(i) {
|
||
continue
|
||
}
|
||
switch i {
|
||
case cAccounts:
|
||
nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "People", Heading: true})
|
||
case cManual:
|
||
nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "Help", Heading: true})
|
||
case cSettings:
|
||
nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "Settings", Heading: true})
|
||
case cAddNode, cLeave:
|
||
if (i == cAddNode) != m.node {
|
||
nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "Do", Heading: true})
|
||
}
|
||
}
|
||
label := consoleLabels[i]
|
||
if i == cSettings {
|
||
label = filepathBase(m.file)
|
||
}
|
||
if n := runningJobs(m.data.jobs); i == cJobs && n > 0 {
|
||
label += fmt.Sprintf(" (%d)", n)
|
||
}
|
||
if i == cSettings && m.doc != nil && m.doc.Changed > 0 {
|
||
label += fmt.Sprintf(" (%d)", m.doc.Changed)
|
||
}
|
||
nav = append(nav, navItem{Label: label, Current: i == m.menu, Dim: !m.entryOK(i)})
|
||
}
|
||
title, body, keys := m.panel()
|
||
title0 := "llmgw console"
|
||
if m.node {
|
||
title0 = "llmgw · node"
|
||
}
|
||
f := frame{
|
||
Width: m.width, Height: m.height, Title: title0, Where: host,
|
||
NavTitle: "", Nav: nav, PanelTitle: title, Body: body, Scroll: m.scroll, Keys: keys,
|
||
}
|
||
if m.menu == cManual && m.detail {
|
||
f.Scroll = m.docs.scroll
|
||
}
|
||
// The selected row kept on screen.
|
||
if m.inPanel && !m.detail {
|
||
inner := m.height - 4
|
||
top := m.row + 3
|
||
if top-m.scroll >= inner-1 {
|
||
f.Scroll = top - inner + 2
|
||
}
|
||
}
|
||
if m.dialog != "" {
|
||
f.DialogTitle = m.dialogTitle
|
||
f.Dialog = m.dialogText
|
||
switch m.dialog {
|
||
case "target", "edit", "addname":
|
||
q := m.dialogText
|
||
if m.dialog == "target" && m.editKind > 1 {
|
||
q = fmt.Sprintf("%s %d of %d", q, len(m.asked)+1, int(m.editKind))
|
||
}
|
||
f.Dialog = q + "\n\n" + m.prompt.View() + "\n\n" + sDim.Render("Enter · Esc cancel")
|
||
}
|
||
} else if t, d := m.docs.dialog(); d != "" && m.menu == cManual {
|
||
f.DialogTitle, f.Dialog = t, d
|
||
} else if m.busy != "" {
|
||
f.DialogTitle, f.Dialog = m.busy, m.spin.View()+" asking the control plane…"
|
||
} else if t, d := m.actionDialog(); d != "" {
|
||
f.DialogTitle, f.Dialog = t, d
|
||
}
|
||
return f.render()
|
||
}
|
||
|
||
// panel is the right-hand side for the entry chosen.
|
||
func (m consoleModel) panel() (title, body, keys string) {
|
||
pw := max(40, m.width-navWidth-8)
|
||
rowKeys := "↑↓ row · Enter actions · ← back · PgUp/PgDn · Q leave"
|
||
if !m.inPanel {
|
||
keys = "↑↓ choose · → or Enter open · Q leave · refreshed every three seconds"
|
||
}
|
||
waiting := func() string {
|
||
if m.data.err != nil {
|
||
return "\n " + sBad.Render("The control plane does not answer: ") + m.data.err.Error() +
|
||
"\n\n " + sDim.Render("systemctl status llmgw-server says why.")
|
||
}
|
||
return "\n " + m.spin.View() + " asking the control plane…"
|
||
}
|
||
if (!m.loaded || m.data.err != nil) && !m.node {
|
||
if m.menu < cSettings {
|
||
return consoleLabels[m.menu], waiting(), keys
|
||
}
|
||
}
|
||
switch m.menu {
|
||
case cOverview:
|
||
var b bytes.Buffer
|
||
c := &cobra.Command{}
|
||
c.SetOut(&b)
|
||
printSummary(c, m.data.sum)
|
||
return "Overview", "\n" + indent(b.String(), " "), keys
|
||
case cNodes, cLogs:
|
||
if m.detail && m.row < len(m.data.nodes) {
|
||
n := m.data.nodes[m.row]
|
||
if m.menu == cLogs {
|
||
return "Agent log · " + n.ID, m.logView(), "← back · PgUp/PgDn · refreshed every three seconds"
|
||
}
|
||
return "Node · " + n.ID, m.nodeDetail(n) + m.detailCharts(pw), "← back · 1–4 or R range · PgUp/PgDn · Q leave"
|
||
}
|
||
var rows [][]string
|
||
for _, n := range m.data.nodes {
|
||
cpu, mem := "—", "—"
|
||
if n.Load != nil {
|
||
cpu, mem = fmt.Sprintf("%.0f%%", n.Load.CPUPercent), fmt.Sprintf("%.0f%%", n.Load.MemPercent)
|
||
}
|
||
state := n.State
|
||
if state != "ready" {
|
||
state = sWarn.Render(state)
|
||
}
|
||
rows = append(rows, []string{n.ID, state, cpu, mem, fmt.Sprintf("%d/%d", n.GPUFree, n.GPUTotal),
|
||
strconv.Itoa(n.Replicas), fmt.Sprintf("%d/%d GB", n.DiskFreeGB, n.DiskTotalGB), n.AgentVersion, n.LastSeenAgo})
|
||
}
|
||
t := "Nodes"
|
||
if m.menu == cLogs {
|
||
t = "Logs · choose a node"
|
||
}
|
||
if m.inPanel {
|
||
keys = rowKeys
|
||
}
|
||
return t, "\n" + table([]string{"NODE", "STATE", "CPU", "MEM", "GPU FREE", "REPLICAS", "DISK FREE", "AGENT", "SEEN"},
|
||
rows, m.row, pw, m.inPanel), keys
|
||
case cGPUs:
|
||
if m.inPanel {
|
||
keys = "↑↓ row · ← back · PgUp/PgDn · Q leave"
|
||
}
|
||
return "GPUs", "\n" + table([]string{"NODE", "GPU", "CARD", "BUSY", "MEMORY", "TEMP", "POWER", "HELD BY"},
|
||
m.gpuRows(), m.row, pw, m.inPanel), keys
|
||
case cModels:
|
||
if m.detail && m.row < len(m.data.models) {
|
||
return "Model · " + m.data.models[m.row].Name, m.modelDetail(m.data.models[m.row]) + m.detailCharts(pw),
|
||
"← back · 1–4 or R range · PgUp/PgDn · Q leave"
|
||
}
|
||
var rows [][]string
|
||
for _, md := range m.data.models {
|
||
status := "stopped"
|
||
switch {
|
||
case md.Parked:
|
||
status = sBad.Render("parked")
|
||
case md.Unplaceable:
|
||
status = sWarn.Render("not placed")
|
||
case md.Ready > 0 && md.Ready >= md.Desired:
|
||
status = sOK.Render("ready")
|
||
case md.Desired > 0:
|
||
status = sWarn.Render("starting")
|
||
}
|
||
warn := ""
|
||
if len(md.Warnings) > 0 {
|
||
warn = sWarn.Render("!")
|
||
}
|
||
upd := ""
|
||
if md.Update != nil && md.Update.Available {
|
||
upd = sWarn.Render("↑ " + catalog.ShortRevision(md.Update.Latest))
|
||
}
|
||
rows = append(rows, []string{md.Name, fmt.Sprintf("%d/%d", md.Ready, md.Desired), status, md.Class,
|
||
strings.Join(md.Profiles, ","), upd, warn})
|
||
}
|
||
if m.inPanel {
|
||
keys = "↑↓ row · Enter actions · A add from a file · ← back · Q leave"
|
||
}
|
||
return "Models", "\n" + table([]string{"MODEL", "READY", "STATUS", "CLASS", "PROFILES", "UPDATE", ""}, rows, m.row, pw, m.inPanel), keys
|
||
case cReplicas:
|
||
var rows [][]string
|
||
for _, r := range m.data.replicas {
|
||
var nodes []string
|
||
for _, s := range r.Shards {
|
||
nodes = append(nodes, fmt.Sprintf("%s%v", s.Node, s.GPUs))
|
||
}
|
||
rows = append(rows, []string{r.ID, r.Model, r.State, r.Pool, strings.Join(nodes, " "),
|
||
(time.Duration(r.AgeSec) * time.Second).Round(time.Second).String()})
|
||
}
|
||
if m.inPanel {
|
||
keys = rowKeys
|
||
}
|
||
return "Replicas", "\n" + table([]string{"REPLICA", "MODEL", "STATE", "POOL", "ON", "AGE"}, rows, m.row, pw, m.inPanel), keys
|
||
case cUsage:
|
||
head := "\n Last 24 hours, by " + lipgloss.NewStyle().Bold(true).Render(usageGroupings[m.usageBy]) +
|
||
sDim.Render(" (→ to group by "+usageGroupings[(m.usageBy+1)%len(usageGroupings)]+")") + "\n\n"
|
||
if m.usageErr != nil {
|
||
return "Usage", head + " " + sBad.Render(m.usageErr.Error()), keys
|
||
}
|
||
var rows [][]string
|
||
for _, u := range m.usage {
|
||
rows = append(rows, []string{u.Name, compactNum(u.Requests), compactNum(u.Prompt + u.Completion),
|
||
compactNum(u.Errors), fmt.Sprintf("%.2f", float64(u.CostMicros)/1e6)})
|
||
}
|
||
if m.inPanel {
|
||
keys = "↑↓ row · → grouping · ← back · Q leave"
|
||
}
|
||
return "Usage", head + table([]string{"WHO", "REQUESTS", "TOKENS", "ERRORS", "SPENT"}, rows, m.row, pw, m.inPanel), keys
|
||
case cQuotas:
|
||
return m.quotasPanel(pw)
|
||
case cAccounts, cKeys, cOrgs:
|
||
return m.peoplePanel(pw)
|
||
case cManual:
|
||
if m.detail && m.docs.open >= 0 {
|
||
k := docKeys
|
||
if m.docs.note != "" {
|
||
k = m.docs.note
|
||
}
|
||
return "Manual · " + m.docs.topics[m.docs.open].title, "\n" + m.docs.body(pw-4), k
|
||
}
|
||
d := m.docs
|
||
d.sel = m.row
|
||
if m.inPanel {
|
||
keys = "↑↓ topic · Enter read · ← back · ? the console's page · Q leave"
|
||
}
|
||
return "Manual", d.list(pw, m.inPanel), keys
|
||
case cMirror:
|
||
return m.mirrorPanel(pw)
|
||
case cJobs:
|
||
return m.jobsPanel(pw)
|
||
case cAudit:
|
||
return m.auditPanel(pw)
|
||
case cMonitor:
|
||
return m.monitorPanel(pw)
|
||
case cServe:
|
||
return m.hfPanel(pw)
|
||
case cSettings:
|
||
return m.settingsPanel(pw)
|
||
}
|
||
var item string
|
||
switch m.menu {
|
||
case cAddNode:
|
||
item = "\n Adds a machine as a node: installs llmgw on it over ssh, checks it,\n gives it a certificate and starts its agent.\n\n " +
|
||
sDim.Render("On the command line:") + "\n sudo llmgw node add <user>@<address>"
|
||
case cReplicate:
|
||
item = "\n Moves the store onto two machines under Patroni, replicated,\n so losing one loses nothing.\n\n " +
|
||
sDim.Render("On the command line:") +
|
||
"\n sudo --preserve-env=SSH_AUTH_SOCK llmgw server init \\\n --patroni <user>@<a> --patroni <user>@<b> --no-mirror --no-node --no-model"
|
||
case cUpgrade:
|
||
item = "\n Moves the cluster to a release: this machine first, then each node\n in turn. Engines keep serving; a node that does not come back stops it.\n\n " +
|
||
sDim.Render("On the command line:") + "\n sudo llmgw ctl upgrade VERSION"
|
||
case cRestartAll:
|
||
item = "\n Replaces every replica, one at a time, each new one up before the\n old one stops.\n\n " +
|
||
sDim.Render("On the command line:") + "\n llmgw ctl restart --all"
|
||
case cBackup:
|
||
item = "\n Writes the store and /etc/llmgw to one archive in " + DefaultBackupDir + ",\n and keeps the last few. Copy it off this machine.\n\n " +
|
||
sDim.Render("On the command line:") + "\n sudo llmgw server backup"
|
||
case cLeave:
|
||
item = "\n Closes the console. The cluster keeps running."
|
||
}
|
||
return consoleLabels[m.menu], item, "Enter do it · ↑↓ choose · Q leave"
|
||
}
|
||
|
||
func (m consoleModel) settingsPanel(pw int) (string, string, string) {
|
||
if m.docErr != nil {
|
||
return "server.yaml", "\n " + sBad.Render(m.docErr.Error()), "← back"
|
||
}
|
||
var crumbs []string
|
||
for _, l := range m.levels {
|
||
crumbs = append(crumbs, l.title)
|
||
}
|
||
rows := m.settingsRows()
|
||
var tr [][]string
|
||
for _, r := range rows {
|
||
typ := ""
|
||
if r.key != nil {
|
||
typ = sDim.Render(r.key.Type)
|
||
}
|
||
tr = append(tr, []string{r.label, r.value, typ})
|
||
}
|
||
body := "\n" + table([]string{"KEY", "VALUE", "TYPE"}, tr, m.row, pw, m.inPanel)
|
||
// What the chosen key is for, from its comment in the code.
|
||
if m.inPanel && m.row < len(rows) {
|
||
doc := ""
|
||
if r := rows[m.row]; r.key != nil {
|
||
doc = r.key.Doc
|
||
if doc == "" && r.key.Block != nil {
|
||
doc = r.key.Block.Doc
|
||
}
|
||
} else if l := m.level(); l.block != nil {
|
||
doc = l.block.Doc
|
||
}
|
||
if doc != "" {
|
||
body += "\n " + sDim.Render(strings.Repeat("─", min(pw-4, 60))) + "\n" +
|
||
indent(wrap(firstParagraphs(doc, 2), pw-4, ""), " ")
|
||
}
|
||
}
|
||
keys := "→ or Enter open · ↑↓ row · Q leave"
|
||
if m.inPanel {
|
||
keys = "Enter change/open · U back to default · S save · Esc up · PgUp/PgDn"
|
||
if l := m.level(); l.kind != settings.Block {
|
||
keys = "Enter open · D remove entry · S save · Esc up"
|
||
}
|
||
}
|
||
return strings.Join(crumbs, " › "), body, keys
|
||
}
|
||
|
||
func firstParagraphs(s string, n int) string {
|
||
p := strings.Split(s, "\n\n")
|
||
if len(p) > n {
|
||
p = p[:n]
|
||
}
|
||
return strings.Join(p, "\n\n")
|
||
}
|
||
|
||
func (m consoleModel) gpuRows() [][]string {
|
||
held := map[string]string{}
|
||
for _, r := range m.data.replicas {
|
||
for _, s := range r.Shards {
|
||
for _, g := range s.GPUs {
|
||
held[s.Node+"/"+strconv.Itoa(g)] = r.Model
|
||
}
|
||
}
|
||
}
|
||
var rows [][]string
|
||
for _, n := range m.data.nodes {
|
||
for _, g := range n.GPUs {
|
||
busy, mem, temp, power := "—", "—", "—", "—"
|
||
if n.Load != nil {
|
||
for _, s := range n.Load.GPUs {
|
||
if s.Index != g.Index {
|
||
continue
|
||
}
|
||
busy = fmt.Sprintf("%.0f%%", s.UtilPercent)
|
||
if s.MemoryTotalMB > 0 {
|
||
mem = fmt.Sprintf("%.1f/%.0f GB", s.MemoryUsedMB/1024, s.MemoryTotalMB/1024)
|
||
}
|
||
if s.TempC > 0 {
|
||
temp = fmt.Sprintf("%.0f°C", s.TempC)
|
||
}
|
||
if s.PowerW > 0 {
|
||
power = fmt.Sprintf("%.0f/%.0f W", s.PowerW, s.PowerLimitW)
|
||
}
|
||
}
|
||
}
|
||
holder := held[n.ID+"/"+strconv.Itoa(g.Index)]
|
||
if holder == "" {
|
||
holder = sDim.Render("free")
|
||
}
|
||
rows = append(rows, []string{n.ID, strconv.Itoa(g.Index), g.Name, busy, mem, temp, power, holder})
|
||
}
|
||
}
|
||
return rows
|
||
}
|
||
|
||
func (m consoleModel) nodeDetail(n adminapi.NodeView) string {
|
||
var b strings.Builder
|
||
kv := func(k, v string) { fmt.Fprintf(&b, " %-14s %s\n", sDim.Render(k), v) }
|
||
b.WriteString("\n")
|
||
kv("state", n.State)
|
||
kv("address", n.MgmtAddr)
|
||
kv("system", n.OS)
|
||
kv("agent", n.AgentVersion+", seen "+n.LastSeenAgo+" ago")
|
||
if n.Load != nil {
|
||
l := n.Load
|
||
kv("cpu", fmt.Sprintf("%.0f%% of %d · load %.2f %.2f %.2f", l.CPUPercent, l.CPUs, l.Load1, l.Load5, l.Load15))
|
||
kv("memory", fmt.Sprintf("%.0f%% · %.1f of %.1f GB", l.MemPercent, float64(l.MemUsed)/(1<<30), float64(l.MemTotal)/(1<<30)))
|
||
}
|
||
kv("disk", fmt.Sprintf("%d GB free of %d", n.DiskFreeGB, n.DiskTotalGB))
|
||
kv("replicas", strconv.Itoa(n.Replicas))
|
||
if len(n.Provision) > 0 {
|
||
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("Engines") + "\n")
|
||
for _, p := range n.Provision {
|
||
fmt.Fprintf(&b, " %-9s %-10s %-8s %s\n", p.Engine, p.Version, p.State, sDim.Render(truncate(p.Detail, 60)))
|
||
}
|
||
}
|
||
if len(n.GPUs) > 0 {
|
||
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("GPUs") + "\n")
|
||
for _, r := range m.gpuRows() {
|
||
if r[0] == n.ID {
|
||
fmt.Fprintf(&b, " %s %-18s busy %-5s mem %-14s %-6s %s %s\n", r[1], r[2], r[3], r[4], r[5], r[6], r[7])
|
||
}
|
||
}
|
||
}
|
||
for _, w := range append(n.Warnings, n.Unhealthy...) {
|
||
b.WriteString("\n " + sWarn.Render("! ") + w)
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
func (m consoleModel) modelDetail(md adminapi.ModelView) string {
|
||
var b strings.Builder
|
||
kv := func(k, v string) {
|
||
if v != "" {
|
||
fmt.Fprintf(&b, " %-14s %s\n", sDim.Render(k), v)
|
||
}
|
||
}
|
||
b.WriteString("\n")
|
||
if md.Description != "" {
|
||
b.WriteString(" " + md.Description + "\n\n")
|
||
}
|
||
kv("replicas", fmt.Sprintf("%d ready of %d wanted (min %d, max %d)", md.Ready, md.Desired, md.Min, md.Max))
|
||
kv("class", md.Class)
|
||
kv("owner", md.Owner)
|
||
kv("profiles", strings.Join(md.Profiles, ", "))
|
||
kv("aliases", strings.Join(md.Aliases, ", "))
|
||
kv("source", md.Source)
|
||
if o := md.Origin; o != nil {
|
||
from := o.Repo + " at " + catalog.ShortRevision(o.Revision)
|
||
if o.Previous != "" {
|
||
from += ", updated from " + catalog.ShortRevision(o.Previous)
|
||
}
|
||
kv("from", from)
|
||
}
|
||
if u := md.Update; u != nil {
|
||
kv("registry", updateCell(*u)+sDim.Render(", asked "+ago(u.CheckedAt, time.Now())))
|
||
}
|
||
if md.MemoryGB > 0 {
|
||
kv("reserves", fmt.Sprintf("%d GB, %d cores (uses %.1f GB)", md.MemoryGB, md.CPUCores, md.MeasuredMemoryGB))
|
||
}
|
||
kv("fallbacks", strings.Join(md.Fallbacks, ", "))
|
||
if md.Reason != "" {
|
||
b.WriteString("\n " + sWarn.Render("! ") + md.Reason + "\n")
|
||
}
|
||
for _, w := range md.Warnings {
|
||
b.WriteString(" " + sWarn.Render("! ") + w + "\n")
|
||
}
|
||
var reps [][]string
|
||
for _, r := range m.data.replicas {
|
||
if r.Model == md.Name {
|
||
reps = append(reps, []string{r.ID, r.State, r.Pool, r.Endpoint})
|
||
}
|
||
}
|
||
if len(reps) > 0 {
|
||
b.WriteString("\n" + table([]string{"REPLICA", "STATE", "POOL", "ENDPOINT"}, reps, -1, max(40, m.width-navWidth-8), false))
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
func (m consoleModel) logView() string {
|
||
if m.logs.err != nil {
|
||
return "\n " + sBad.Render(m.logs.err.Error())
|
||
}
|
||
if len(m.logs.lines) == 0 {
|
||
return "\n " + m.spin.View() + " reading the log…"
|
||
}
|
||
// The newest at the bottom, where a log is read; the panel starts there.
|
||
lines := m.logs.lines
|
||
if k := m.height - 6; len(lines) > k && m.scroll == 0 {
|
||
lines = lines[len(lines)-k:]
|
||
}
|
||
var b strings.Builder
|
||
b.WriteString("\n")
|
||
for _, l := range lines {
|
||
switch {
|
||
case strings.Contains(l, "level=ERROR"):
|
||
l = sBad.Render(l)
|
||
case strings.Contains(l, "level=WARN"):
|
||
l = sWarn.Render(l)
|
||
}
|
||
b.WriteString(" " + l + "\n")
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
func compactNum(n int64) string {
|
||
switch {
|
||
case n >= 1_000_000:
|
||
return fmt.Sprintf("%.1fM", float64(n)/1e6)
|
||
case n >= 1_000:
|
||
return fmt.Sprintf("%.1fk", float64(n)/1e3)
|
||
}
|
||
return strconv.FormatInt(n, 10)
|
||
}
|
||
|
||
func indent(s, pre string) string {
|
||
lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
|
||
for i := range lines {
|
||
lines[i] = pre + lines[i]
|
||
}
|
||
return strings.Join(lines, "\n")
|
||
}
|
||
|
||
func filepathBase(p string) string {
|
||
if i := strings.LastIndexByte(p, '/'); i >= 0 {
|
||
return p[i+1:]
|
||
}
|
||
return p
|
||
}
|
||
|
||
// runConsole shows the console, runs what is chosen on it, and comes back.
|
||
// file is server.yaml on a control plane, agent.yaml on a node.
|
||
func runConsole(ctx context.Context, out io.Writer, file string) error {
|
||
addr := "127.0.0.1:9090"
|
||
if cfg, err := config.Load(file); err == nil {
|
||
if cfg.AdminListen != "" {
|
||
addr = cfg.AdminListen
|
||
if strings.HasPrefix(addr, ":") {
|
||
addr = "127.0.0.1" + addr
|
||
}
|
||
}
|
||
// The admin token, for this screen and every command it runs in
|
||
// this process — `llmgw ctl` included — from this configuration's
|
||
// file rather than only the usual place.
|
||
if t := readAdminToken(adminTokenPath(file, cfg)); t != "" {
|
||
_ = os.Setenv("LLMGW_ADMIN_TOKEN", t)
|
||
}
|
||
}
|
||
menu := cOverview
|
||
if strings.HasSuffix(file, "agent.yaml") {
|
||
menu = cSettings
|
||
}
|
||
for {
|
||
m := newConsoleModel(addr, file)
|
||
m.menu = menu
|
||
final, err := tea.NewProgram(m, tea.WithOutput(out), tea.WithAltScreen()).Run()
|
||
if err != nil {
|
||
return err
|
||
}
|
||
fm := final.(consoleModel)
|
||
fm.st.close()
|
||
menu = fm.menu
|
||
act := fm.action
|
||
switch act.kind {
|
||
case "":
|
||
return nil
|
||
case "node":
|
||
c := newNodeAddCmd()
|
||
c.SetArgs(act.targets)
|
||
c.SetOut(out)
|
||
err = c.ExecuteContext(ctx)
|
||
case "patroni":
|
||
in := newDefaultServerInit()
|
||
in.patroni = act.targets
|
||
// Already set up: only the store changes.
|
||
in.noMirror, in.noNode, in.noModel = true, true, true
|
||
err = in.runInitTUI(ctx, out)
|
||
case "run":
|
||
c := Root()
|
||
c.SetArgs(act.targets)
|
||
c.SetOut(out)
|
||
c.SetErr(out)
|
||
fmt.Fprintln(out, sDim.Render("llmgw "+strings.Join(act.targets, " ")))
|
||
err = c.ExecuteContext(ctx)
|
||
case "restart":
|
||
fmt.Fprintf(out, "restarting %s…\n", fm.service())
|
||
if b, e := exec.CommandContext(ctx, "systemctl", "restart", fm.service()).CombinedOutput(); e != nil {
|
||
err = fmt.Errorf("%v: %s", e, strings.TrimSpace(string(b)))
|
||
} else {
|
||
fmt.Fprintln(out, sOK.Render("✓ ")+"restarted; it picks up the configuration as it starts")
|
||
}
|
||
}
|
||
if err != nil {
|
||
fmt.Fprintln(out, sBad.Render("✗ ")+err.Error())
|
||
}
|
||
fmt.Fprint(out, "\n"+sDim.Render("enter to go back to the console")+" ")
|
||
_, _ = bufio.NewReader(os.Stdin).ReadString('\n')
|
||
}
|
||
}
|