A control plane being stopped held every agent's stream for the thirty seconds of a graceful shutdown, so the standby led within four seconds and had no nodes for thirty. The agent server is now closed, not drained. A control plane that loses its lease while it lives on lets its agents go, rather than keeping nodes the new leader cannot reach. ctl status and the console's Overview say which control plane leads, which stand by, and whether the floating address is on the leader. The console adds a control plane (Do → Add a control plane), asking for the machine and the floating address. server add says when the mirror would refuse the new control plane. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1572 lines
45 KiB
Go
1572 lines
45 KiB
Go
package cli
|
||
|
||
import (
|
||
"bufio"
|
||
"bytes"
|
||
"context"
|
||
"fmt"
|
||
"io"
|
||
"net"
|
||
"net/url"
|
||
"os"
|
||
"os/exec"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
tea "github.com/charmbracelet/bubbletea"
|
||
"github.com/charmbracelet/lipgloss"
|
||
"github.com/spf13/cobra"
|
||
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/docgen"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||
)
|
||
|
||
// The console: what a bare `sudo llmgw` opens on a control plane. Down the
|
||
// left, three groups — what is running (watched live), the settings (every
|
||
// key of server.yaml, from the generated reference), and what can be done.
|
||
|
||
// Sidebar entries, in order. Headings are not entries.
|
||
const (
|
||
cOverview = iota
|
||
cMonitor
|
||
cNodes
|
||
cGPUs
|
||
cModels
|
||
cReplicas
|
||
cUsage
|
||
cQuotas
|
||
cFederation
|
||
cMirror
|
||
cJobs
|
||
cLogs
|
||
cAccounts
|
||
cKeys
|
||
cOrgs
|
||
cAudit
|
||
cManual
|
||
cSettings
|
||
cAddNode
|
||
cAddControl
|
||
cReplicate
|
||
cServe
|
||
cUpgrade
|
||
cRestartAll
|
||
cBackup
|
||
cLeave
|
||
cCount
|
||
)
|
||
|
||
var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Federation", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Audit", "Manual",
|
||
"server.yaml", "Add a node", "Add a control plane", "Replicate the store", "Serve a model", "Upgrade the cluster",
|
||
"Restart all replicas", "Back up now", "Leave"}
|
||
|
||
// consoleAction is what is done once the screen closes: each runs its own
|
||
// screen, and the console comes back after.
|
||
type consoleAction struct {
|
||
kind string // "", "node", "patroni", "restart", "run" (llmgw with targets as its arguments)
|
||
targets []string
|
||
}
|
||
|
||
type consoleData struct {
|
||
sum adminapi.Summary
|
||
nodes []adminapi.NodeView
|
||
models []adminapi.ModelView
|
||
replicas []adminapi.ReplicaView
|
||
jobs []modelserve.Job
|
||
audit []store.AuditEvent
|
||
// federation is the other llmgw clusters whose models are offered
|
||
// here; empty on a control plane without a store.
|
||
federation adminapi.FederationView
|
||
err error
|
||
}
|
||
|
||
type (
|
||
consoleDataMsg consoleData
|
||
consoleUsageMsg struct {
|
||
rows []adminapi.ScopeUsageView
|
||
err error
|
||
}
|
||
consoleLogsMsg struct {
|
||
node string
|
||
lines []string
|
||
err error
|
||
}
|
||
consoleTickMsg struct{}
|
||
)
|
||
|
||
// settingsLevel is where the settings view is: a block, or a list or map of
|
||
// blocks, at a path of keys in the file.
|
||
type settingsLevel struct {
|
||
title string
|
||
block *docgen.Block
|
||
path []string
|
||
kind settings.Kind // Block, BlockList or BlockMap
|
||
}
|
||
|
||
type consoleModel struct {
|
||
client *ctlClient
|
||
width, height int
|
||
spin spinnerModel
|
||
|
||
data consoleData
|
||
loaded bool
|
||
usage []adminapi.ScopeUsageView
|
||
// usageBy is the grouping on the usage view.
|
||
usageBy int
|
||
usageErr error
|
||
logs consoleLogsMsg
|
||
|
||
// menu is the sidebar entry; inPanel says the keys go to the panel.
|
||
menu int
|
||
inPanel bool
|
||
row int
|
||
// detail is the panel showing one row's detail rather than the table.
|
||
detail bool
|
||
scroll int
|
||
|
||
// node is the console of a node: its agent.yaml and nothing to watch,
|
||
// since the cluster is watched from the control plane.
|
||
node bool
|
||
|
||
// The settings.
|
||
file string
|
||
doc *settings.Doc
|
||
cfg any // the file as loaded, for the values that apply
|
||
levels []settingsLevel
|
||
docErr error
|
||
saveNote string
|
||
|
||
// A question open as a dialog.
|
||
dialog string // "", "target", "edit", "addname", "saved", "confirm-leave", "error"
|
||
dialogTitle string
|
||
dialogText string
|
||
prompt textField
|
||
asked []string
|
||
editPath []string
|
||
editKind settings.Kind
|
||
|
||
action consoleAction
|
||
|
||
// What can be done to a row: a menu, a form, a question, and the request
|
||
// under way.
|
||
menuD *menuDialog
|
||
formD *formDialog
|
||
confirmD *confirmDialog
|
||
busy string
|
||
// busyNote says how far the request under way has got, when it can;
|
||
// evalWatch is the quality run being followed, which Esc stops
|
||
// following while it carries on.
|
||
busyNote string
|
||
evalWatch string
|
||
|
||
quotas []adminapi.QuotaView
|
||
quotasErr error
|
||
|
||
st *storeHolder
|
||
people peopleData
|
||
mirror consoleMirrorMsg
|
||
hf hfState
|
||
docs docViewer
|
||
mon monitorState
|
||
det monitorState
|
||
}
|
||
|
||
var usageGroupings = []string{"team", "project", "org"}
|
||
|
||
func newConsoleModel(addr, file string) consoleModel {
|
||
m := consoleModel{client: newCtlClient(addr), spin: newSpinner(), width: 100, height: 30, file: file,
|
||
st: &storeHolder{file: file}, docs: newDocViewer()}
|
||
m.node = strings.HasSuffix(file, "agent.yaml")
|
||
if m.node {
|
||
m.menu = cSettings
|
||
}
|
||
m.loadSettings()
|
||
return m
|
||
}
|
||
|
||
// load reads the file as the service would, defaults and checks included.
|
||
func (m consoleModel) load(path string) (any, error) {
|
||
if m.node {
|
||
return config.LoadAgent(path)
|
||
}
|
||
return config.Load(path)
|
||
}
|
||
|
||
func (m consoleModel) service() string {
|
||
if m.node {
|
||
return "llmgw-agent"
|
||
}
|
||
return "llmgw-server"
|
||
}
|
||
|
||
func (m *consoleModel) loadSettings() {
|
||
m.doc, m.docErr = settings.Open(m.file)
|
||
if m.docErr != nil {
|
||
return
|
||
}
|
||
if cfg, err := m.load(m.file); err == nil {
|
||
m.cfg = cfg
|
||
} else if !m.node {
|
||
m.cfg = config.Default()
|
||
}
|
||
if root, err := settings.Root(m.file); err == nil && len(m.levels) == 0 {
|
||
m.levels = []settingsLevel{{title: filepathBase(m.file), block: root, kind: settings.Block}}
|
||
}
|
||
}
|
||
|
||
func (m consoleModel) Init() tea.Cmd {
|
||
if m.node {
|
||
return m.spin.Tick
|
||
}
|
||
return tea.Batch(m.fetch(), m.spin.Tick, tea.Tick(3*time.Second, func(time.Time) tea.Msg { return consoleTickMsg{} }))
|
||
}
|
||
|
||
// fetch reads everything the watched views show, in one go.
|
||
func (m consoleModel) fetch() tea.Cmd {
|
||
c := m.client
|
||
return func() tea.Msg {
|
||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||
defer cancel()
|
||
var d consoleData
|
||
if d.err = c.get(ctx, "/api/v1/summary", &d.sum); d.err != nil {
|
||
return consoleDataMsg(d)
|
||
}
|
||
_ = c.get(ctx, "/api/v1/nodes", &d.nodes)
|
||
_ = c.get(ctx, "/api/v1/models", &d.models)
|
||
_ = c.get(ctx, "/api/v1/replicas", &d.replicas)
|
||
_ = c.get(ctx, "/api/v1/jobs", &d.jobs)
|
||
_ = c.get(ctx, "/api/v1/audit?since=7d&limit=300", &d.audit)
|
||
_ = c.get(ctx, "/api/v1/federation", &d.federation)
|
||
return consoleDataMsg(d)
|
||
}
|
||
}
|
||
|
||
func (m consoleModel) fetchUsage() tea.Cmd {
|
||
c, by := m.client, usageGroupings[m.usageBy]
|
||
return func() tea.Msg {
|
||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||
defer cancel()
|
||
var rows []adminapi.ScopeUsageView
|
||
err := c.get(ctx, "/api/v1/usage?by="+by+"&since=24h", &rows)
|
||
return consoleUsageMsg{rows, err}
|
||
}
|
||
}
|
||
|
||
func (m consoleModel) fetchLogs(node string) tea.Cmd {
|
||
c := m.client
|
||
return func() tea.Msg {
|
||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||
defer cancel()
|
||
var out struct {
|
||
Lines []string `json:"lines"`
|
||
}
|
||
err := c.get(ctx, "/api/v1/nodes/"+url.PathEscape(node)+"/logs?lines=200", &out)
|
||
return consoleLogsMsg{node: node, lines: out.Lines, err: err}
|
||
}
|
||
}
|
||
|
||
func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||
switch msg := msg.(type) {
|
||
case tea.WindowSizeMsg:
|
||
m.width, m.height = msg.Width, msg.Height
|
||
case spinTickMsg:
|
||
var cmd tea.Cmd
|
||
m.spin, cmd = m.spin.Update(msg)
|
||
return m, cmd
|
||
case consoleTickMsg:
|
||
if m.node {
|
||
return m, nil
|
||
}
|
||
cmds := []tea.Cmd{m.fetch(), tea.Tick(3*time.Second, func(time.Time) tea.Msg { return consoleTickMsg{} })}
|
||
if m.menu == cUsage {
|
||
cmds = append(cmds, m.fetchUsage())
|
||
}
|
||
if m.menu == cQuotas {
|
||
cmds = append(cmds, m.fetchQuotas())
|
||
}
|
||
if m.menu == cMirror {
|
||
cmds = append(cmds, m.fetchMirror())
|
||
}
|
||
if m.menu == cMonitor && m.monitorDue() {
|
||
cmds = append(cmds, m.fetchMonitor())
|
||
}
|
||
if m.detailDue() {
|
||
cmds = append(cmds, m.fetchDetail())
|
||
}
|
||
if m.peopleView() {
|
||
cmds = append(cmds, m.fetchPeople())
|
||
}
|
||
if m.menu == cLogs && m.detail && m.logs.node != "" {
|
||
cmds = append(cmds, m.fetchLogs(m.logs.node))
|
||
}
|
||
return m, tea.Batch(cmds...)
|
||
case consoleDataMsg:
|
||
m.loaded = true
|
||
if msg.err != nil {
|
||
m.data.err = msg.err
|
||
} else {
|
||
m.data = consoleData(msg)
|
||
m.mon.sample(m.data)
|
||
}
|
||
case detailMsg:
|
||
if msg.key == m.detailKey() {
|
||
m.det.res, m.det.err, m.det.key = msg.res, msg.err, msg.key
|
||
}
|
||
case monitorMsg:
|
||
// Kept if it is still what the screen wants: the range and width.
|
||
if msg.key == m.monitorKey() {
|
||
m.mon.res, m.mon.err, m.mon.key = msg.res, msg.err, msg.key
|
||
}
|
||
case consoleUsageMsg:
|
||
m.usage, m.usageErr = msg.rows, msg.err
|
||
case consoleLogsMsg:
|
||
m.logs = msg
|
||
case consoleMirrorMsg:
|
||
m.mirror = msg
|
||
case consoleHFMsg:
|
||
if msg.query == m.hf.query && msg.sort == m.hf.sort && msg.st == m.hf.safetensors {
|
||
m.hf.models, m.hf.err = msg.models, msg.err
|
||
if m.hf.models == nil && msg.err == nil {
|
||
m.hf.models = []hf.Model{}
|
||
}
|
||
}
|
||
case consoleRepoMsg:
|
||
m.busy = ""
|
||
if msg.err != nil {
|
||
m.dialog, m.dialogTitle = "error", "Hugging Face"
|
||
m.dialogText = sBad.Render("✗ ") + msg.err.Error() + "\n\n" + sDim.Render("any key")
|
||
} else {
|
||
m.menuD = m.repoMenu(msg.plan)
|
||
}
|
||
case consolePeopleMsg:
|
||
m.people = peopleData(msg)
|
||
case consoleQuotasMsg:
|
||
m.quotas, m.quotasErr = msg.rows, msg.err
|
||
case evalSuitesMsg:
|
||
return m.onEvalSuites(msg)
|
||
case evalPollMsg:
|
||
return m.onEvalPoll(msg)
|
||
case actionDoneMsg:
|
||
m.busy, m.busyNote, m.evalWatch = "", "", ""
|
||
m.dialog, m.dialogTitle = "error", msg.title
|
||
if msg.err != nil {
|
||
m.dialogText = sBad.Render("✗ ") + msg.err.Error()
|
||
} else {
|
||
m.dialogText = sOK.Render("✓ ") + msg.text
|
||
}
|
||
m.dialogText += "\n\n" + sDim.Render("any key")
|
||
return m, tea.Batch(m.fetch(), m.onEnter())
|
||
case tea.KeyMsg:
|
||
if msg.Type == tea.KeyCtrlC {
|
||
return m, tea.Quit
|
||
}
|
||
return m.key(msg)
|
||
}
|
||
return m, nil
|
||
}
|
||
|
||
func (m consoleModel) entryOK(i int) bool {
|
||
if m.node {
|
||
return i == cSettings || i == cManual || i == cLeave
|
||
}
|
||
return i != cReplicate || !m.storeReplicated()
|
||
}
|
||
|
||
func (m consoleModel) storeReplicated() bool { return len(m.data.sum.Store) > 0 }
|
||
|
||
func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||
if m.busy != "" {
|
||
if m.evalWatch != "" && k.Type == tea.KeyEsc {
|
||
m.busy, m.busyNote, m.evalWatch = "", "", ""
|
||
}
|
||
return m, nil
|
||
}
|
||
if m.dialog != "" {
|
||
return m.dialogKey(k)
|
||
}
|
||
if mm, cmd, ok := m.actionKey(k); ok {
|
||
return mm, cmd
|
||
}
|
||
// The manual, from anywhere: its page about this screen.
|
||
if k.String() == "?" && !(m.menu == cManual && m.detail) {
|
||
m.menu, m.inPanel, m.detail = cManual, true, true
|
||
m.docs.show(max(0, findTopic(m.docs.topics, "console")))
|
||
return m, nil
|
||
}
|
||
if m.menu == cManual && m.inPanel && m.detail {
|
||
pw := max(40, m.width-navWidth-8)
|
||
if m.docs.key(k, pw-4, m.height) {
|
||
return m, nil
|
||
}
|
||
switch k.String() {
|
||
case "esc", "left", "h", "backspace":
|
||
m.detail = false
|
||
m.row = m.docs.open
|
||
case "q":
|
||
return m.leave()
|
||
}
|
||
return m, nil
|
||
}
|
||
switch k.Type {
|
||
case tea.KeyPgDown:
|
||
m.scroll += max(1, m.height-8)
|
||
return m, nil
|
||
case tea.KeyPgUp:
|
||
m.scroll = max(0, m.scroll-max(1, m.height-8))
|
||
return m, nil
|
||
}
|
||
if !m.inPanel {
|
||
switch k.String() {
|
||
case "up", "k":
|
||
for {
|
||
m.menu = (m.menu - 1 + cCount) % cCount
|
||
if m.entryOK(m.menu) {
|
||
break
|
||
}
|
||
}
|
||
m.row, m.detail, m.scroll = 0, false, 0
|
||
return m, m.onEnter()
|
||
case "down", "j":
|
||
for {
|
||
m.menu = (m.menu + 1) % cCount
|
||
if m.entryOK(m.menu) {
|
||
break
|
||
}
|
||
}
|
||
m.row, m.detail, m.scroll = 0, false, 0
|
||
return m, m.onEnter()
|
||
case "q":
|
||
return m.leave()
|
||
case "right", "l", "tab", "enter":
|
||
switch m.menu {
|
||
case cAddNode:
|
||
return m.ask("Add a node", "user@address of the machine", 1)
|
||
case cAddControl:
|
||
if !m.storeReplicated() {
|
||
m.dialog, m.dialogTitle = "error", "Add a control plane"
|
||
m.dialogText = "A second control plane needs a store both can reach, and this one's\n" +
|
||
"is on one machine. Replicate the store first (Do → Replicate the store)."
|
||
return m, nil
|
||
}
|
||
return m.ask("Add a control plane", "user@address of the machine", 2)
|
||
case cReplicate:
|
||
return m.ask("Replicate the store", "user@address of database machine", 2)
|
||
case cLeave:
|
||
return m.leave()
|
||
case cUpgrade, cRestartAll, cBackup:
|
||
return m, m.doEntry()
|
||
case cOverview:
|
||
return m, nil
|
||
}
|
||
m.inPanel = true
|
||
}
|
||
return m, nil
|
||
}
|
||
if m.menu == cSettings {
|
||
return m.settingsKey(k)
|
||
}
|
||
n := m.rowCount()
|
||
switch k.String() {
|
||
case "up", "k":
|
||
if !m.detail && m.row > 0 {
|
||
m.row--
|
||
}
|
||
case "down", "j":
|
||
if !m.detail && m.row < n-1 {
|
||
m.row++
|
||
}
|
||
case "enter":
|
||
if m.detail || n == 0 || m.row >= n {
|
||
break
|
||
}
|
||
switch m.menu {
|
||
case cLogs:
|
||
m.detail, m.scroll = true, 0
|
||
node := m.data.nodes[m.row].ID
|
||
m.logs = consoleLogsMsg{node: node}
|
||
return m, m.fetchLogs(node)
|
||
case cNodes:
|
||
m.menuD = m.nodeMenu(m.data.nodes[m.row])
|
||
case cModels:
|
||
m.menuD = m.modelMenu(m.data.models[m.row])
|
||
case cReplicas:
|
||
m.menuD = m.replicaMenu(m.data.replicas[m.row])
|
||
case cQuotas:
|
||
m.menuD = m.quotaMenu(m.quotas[m.row])
|
||
case cMirror:
|
||
m.menuD = m.mirrorMenu(m.mirrorGroups()[m.row])
|
||
case cJobs:
|
||
m.dialog = "error"
|
||
m.dialogTitle, m.dialogText = jobDialog(m.data.jobs[m.row])
|
||
case cAudit:
|
||
m.dialog = "error"
|
||
m.dialogTitle, m.dialogText = auditDialog(m.data.audit[m.row])
|
||
case cFederation:
|
||
m.dialog = "error"
|
||
m.dialogTitle, m.dialogText = peerDialog(m.data.federation.Peers[m.row])
|
||
case cServe:
|
||
return m, m.fetchRepo(m.hf.models[m.row].ID)
|
||
case cAccounts:
|
||
m.menuD = m.userMenu(m.people.users[m.row])
|
||
case cKeys:
|
||
m.menuD = m.keyMenu(m.people.keys[m.row])
|
||
case cOrgs:
|
||
m.menuD = m.orgMenu(m.people.orgs[m.row])
|
||
case cManual:
|
||
m.docs.show(m.row)
|
||
m.detail = true
|
||
}
|
||
case "1", "2", "3", "4", "r":
|
||
if kind, _ := m.detailTarget(); m.menu == cMonitor || kind != "" {
|
||
if k.String() == "r" {
|
||
m.mon.rng = (m.mon.rng + 1) % len(monRanges)
|
||
} else {
|
||
m.mon.rng = int(k.String()[0] - '1')
|
||
}
|
||
if kind != "" {
|
||
return m, m.fetchDetail()
|
||
}
|
||
return m, m.fetchMonitor()
|
||
}
|
||
case "/":
|
||
if m.menu == cServe {
|
||
m.hfSearchDialog()
|
||
}
|
||
case "f":
|
||
if m.menu == cServe {
|
||
m.hf.safetensors = !m.hf.safetensors
|
||
m.hf.models, m.hf.err, m.row = nil, nil, 0
|
||
return m, m.fetchHF()
|
||
}
|
||
case "s":
|
||
if m.menu == cServe {
|
||
m.hf.sort = (m.hf.sort + 1) % len(hfSorts)
|
||
m.hf.models, m.hf.err, m.row = nil, nil, 0
|
||
return m, m.fetchHF()
|
||
}
|
||
case "a":
|
||
switch m.menu {
|
||
case cQuotas:
|
||
m.quotaForm(adminapi.QuotaView{})
|
||
case cModels:
|
||
m.addModelForm()
|
||
case cMirror:
|
||
m.mirrorFetchForm()
|
||
case cAccounts:
|
||
m.createUserForm()
|
||
case cKeys:
|
||
m.createKeyForm()
|
||
case cOrgs:
|
||
m.createOrgForm()
|
||
}
|
||
case "i":
|
||
if m.menu == cMirror {
|
||
m.mirrorImportOllamaForm()
|
||
}
|
||
case "n":
|
||
if m.menu == cMonitor {
|
||
return m, m.runCLI("Test the webhooks", "ctl", "--server="+m.client.base, "notify", "test")
|
||
}
|
||
case "e":
|
||
if m.menu == cUsage {
|
||
m.usageExportForm()
|
||
return m, nil
|
||
}
|
||
case "right", "l":
|
||
if m.menu == cUsage {
|
||
m.usageBy = (m.usageBy + 1) % len(usageGroupings)
|
||
return m, m.fetchUsage()
|
||
}
|
||
case "esc", "left", "h", "backspace":
|
||
if m.detail {
|
||
m.detail, m.scroll = false, 0
|
||
} else {
|
||
m.inPanel = false
|
||
}
|
||
case "q":
|
||
return m.leave()
|
||
}
|
||
return m, nil
|
||
}
|
||
|
||
// onEnter loads what a view needs when it is chosen.
|
||
func (m consoleModel) onEnter() tea.Cmd {
|
||
switch m.menu {
|
||
case cUsage:
|
||
return m.fetchUsage()
|
||
case cQuotas:
|
||
return m.fetchQuotas()
|
||
case cMirror:
|
||
return m.fetchMirror()
|
||
case cMonitor:
|
||
return m.fetchMonitor()
|
||
case cServe:
|
||
if m.hf.models == nil {
|
||
return m.fetchHF()
|
||
}
|
||
case cAccounts, cKeys, cOrgs:
|
||
// The quotas too, for an organization's quota form.
|
||
return tea.Batch(m.fetchPeople(), m.fetchQuotas())
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (m consoleModel) peopleView() bool {
|
||
return m.menu == cAccounts || m.menu == cKeys || m.menu == cOrgs
|
||
}
|
||
|
||
func (m consoleModel) leave() (tea.Model, tea.Cmd) {
|
||
if m.doc != nil && m.doc.Changed > 0 {
|
||
m.dialog, m.dialogTitle = "confirm-leave", "Unsaved changes"
|
||
m.dialogText = fmt.Sprintf("%d change(s) to %s are not saved.\nLeave without saving them?\n\n%s",
|
||
m.doc.Changed, m.file, sDim.Render("Y leave · N stay"))
|
||
return m, nil
|
||
}
|
||
return m, tea.Quit
|
||
}
|
||
|
||
func (m consoleModel) ask(title, question string, n int) (tea.Model, tea.Cmd) {
|
||
m.dialog, m.dialogTitle, m.dialogText, m.asked = "target", title, question, nil
|
||
m.prompt = newTextField()
|
||
m.prompt.Placeholder, m.prompt.Width = "ubuntu@10.0.0.20", 40
|
||
m.prompt.Focus()
|
||
m.editKind = settings.Kind(n) // how many to ask for
|
||
return m, nil
|
||
}
|
||
|
||
func (m consoleModel) dialogKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||
switch m.dialog {
|
||
case "error":
|
||
m.dialog = ""
|
||
return m, nil
|
||
case "confirm-leave":
|
||
if strings.EqualFold(k.String(), "y") {
|
||
return m, tea.Quit
|
||
}
|
||
m.dialog = ""
|
||
return m, nil
|
||
case "saved":
|
||
if strings.EqualFold(k.String(), "y") {
|
||
m.action = consoleAction{kind: "restart"}
|
||
return m, tea.Quit
|
||
}
|
||
m.dialog = ""
|
||
return m, nil
|
||
}
|
||
switch k.Type {
|
||
case tea.KeyEsc:
|
||
m.dialog = ""
|
||
return m, nil
|
||
case tea.KeyEnter:
|
||
v := strings.TrimSpace(m.prompt.Value())
|
||
switch m.dialog {
|
||
case "target":
|
||
// The floating address is the one answer that may be left empty.
|
||
floating := m.menu == cAddControl && len(m.asked) == 1
|
||
if v == "" && !floating {
|
||
return m, nil
|
||
}
|
||
if floating && v != "" {
|
||
if _, _, err := net.ParseCIDR(v); err != nil {
|
||
m.dialogText = floatingQuestion + "\n\n" + sBad.Render("not an address with its prefix, like 10.0.0.161/24")
|
||
return m, nil
|
||
}
|
||
}
|
||
m.asked = append(m.asked, v)
|
||
m.prompt.SetValue("")
|
||
if len(m.asked) == int(m.editKind) {
|
||
switch m.menu {
|
||
case cAddControl:
|
||
args := []string{"server", "add", m.asked[0]}
|
||
if m.asked[1] != "" {
|
||
args = append(args, "--vip", m.asked[1])
|
||
}
|
||
m.action = consoleAction{kind: "run", targets: args}
|
||
case cReplicate:
|
||
m.action = consoleAction{kind: "patroni", targets: m.asked}
|
||
default:
|
||
m.action = consoleAction{kind: "node", targets: m.asked}
|
||
}
|
||
return m, tea.Quit
|
||
}
|
||
if m.menu == cAddControl {
|
||
m.dialogText = floatingQuestion
|
||
m.prompt.Placeholder = "10.0.0.161/24"
|
||
}
|
||
case "edit":
|
||
if err := m.doc.Set(m.editPath, m.editKind, m.prompt.Value()); err != nil {
|
||
m.dialogText = m.editQuestion() + "\n\n" + sBad.Render(err.Error())
|
||
return m, nil
|
||
}
|
||
m.dialog = ""
|
||
case "addname":
|
||
if v == "" {
|
||
return m, nil
|
||
}
|
||
if err := m.doc.AddEntry(m.editPath, v); err != nil {
|
||
m.dialogText = "Name of the new entry\n\n" + sBad.Render(err.Error())
|
||
return m, nil
|
||
}
|
||
m.dialog = ""
|
||
}
|
||
return m, nil
|
||
}
|
||
var cmd tea.Cmd
|
||
m.prompt, cmd = m.prompt.Update(k)
|
||
return m, cmd
|
||
}
|
||
|
||
// --- settings ---
|
||
|
||
// settingsRow is one line of the settings view.
|
||
type settingsRow struct {
|
||
label, value string
|
||
key *settings.Key
|
||
// index is a list entry's position, or a map entry's name.
|
||
entry string
|
||
add bool
|
||
}
|
||
|
||
func (m consoleModel) level() settingsLevel { return m.levels[len(m.levels)-1] }
|
||
|
||
func (m consoleModel) settingsRows() []settingsRow {
|
||
lv := m.level()
|
||
var rows []settingsRow
|
||
switch lv.kind {
|
||
case settings.Block:
|
||
for _, k := range settings.Keys(lv.block) {
|
||
k := k
|
||
path := append(append([]string{}, lv.path...), k.Key)
|
||
rows = append(rows, settingsRow{label: k.Key, value: m.valueOf(k, path), key: &k})
|
||
}
|
||
case settings.BlockList:
|
||
for i := 0; i < m.doc.Len(lv.path); i++ {
|
||
rows = append(rows, settingsRow{label: m.entryLabel(append(append([]string{}, lv.path...), strconv.Itoa(i)), i),
|
||
value: "›", entry: strconv.Itoa(i)})
|
||
}
|
||
rows = append(rows, settingsRow{label: "+ add an entry", add: true})
|
||
case settings.BlockMap:
|
||
for _, name := range m.doc.MapKeys(lv.path) {
|
||
rows = append(rows, settingsRow{label: name, value: "›", entry: name})
|
||
}
|
||
rows = append(rows, settingsRow{label: "+ add an entry", add: true})
|
||
}
|
||
return rows
|
||
}
|
||
|
||
// entryLabel names a list entry by its name or id, else its position.
|
||
func (m consoleModel) entryLabel(path []string, i int) string {
|
||
for _, k := range []string{"name", "id", "key", "url"} {
|
||
if v, ok := m.doc.Text(append(path, k)); ok && v != "" {
|
||
return v
|
||
}
|
||
}
|
||
return fmt.Sprintf("entry %d", i+1)
|
||
}
|
||
|
||
func (m consoleModel) valueOf(k settings.Key, path []string) string {
|
||
switch k.Kind {
|
||
case settings.Block:
|
||
return "›"
|
||
case settings.BlockList, settings.BlockMap:
|
||
return fmt.Sprintf("%d ›", m.doc.Len(path))
|
||
}
|
||
if v, ok := m.doc.Text(path); ok {
|
||
return strings.ReplaceAll(v, "\n", ", ")
|
||
}
|
||
if v, ok := settings.Effective(m.cfg, path); ok && v != "" {
|
||
return sDim.Render(v + " (default)")
|
||
}
|
||
return sDim.Render("—")
|
||
}
|
||
|
||
func (m consoleModel) editQuestion() string {
|
||
hint := map[settings.Kind]string{
|
||
settings.Duration: "a duration: 30s, 5m, 2h",
|
||
settings.Int: "a whole number",
|
||
settings.Float: "a number",
|
||
settings.TextList: "values, separated by commas",
|
||
settings.TextMap: "key = value, separated by commas",
|
||
}[m.editKind]
|
||
q := strings.Join(m.editPath, ".")
|
||
if hint != "" {
|
||
q += "\n" + sDim.Render(hint)
|
||
}
|
||
return q
|
||
}
|
||
|
||
func (m consoleModel) settingsKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||
if m.doc == nil {
|
||
if k.String() == "esc" || k.String() == "left" {
|
||
m.inPanel = false
|
||
}
|
||
return m, nil
|
||
}
|
||
rows := m.settingsRows()
|
||
lv := m.level()
|
||
switch k.String() {
|
||
case "up", "k":
|
||
if m.row > 0 {
|
||
m.row--
|
||
}
|
||
case "down", "j":
|
||
if m.row < len(rows)-1 {
|
||
m.row++
|
||
}
|
||
case "esc", "left", "h", "backspace":
|
||
if len(m.levels) > 1 {
|
||
m.levels = m.levels[:len(m.levels)-1]
|
||
m.row, m.scroll = 0, 0
|
||
} else {
|
||
m.inPanel = false
|
||
}
|
||
case "s":
|
||
return m.save()
|
||
case "u", "delete":
|
||
if m.row >= len(rows) {
|
||
return m, nil
|
||
}
|
||
r := rows[m.row]
|
||
switch {
|
||
case r.key != nil:
|
||
m.doc.Unset(append(append([]string{}, lv.path...), r.key.Key))
|
||
case r.entry != "":
|
||
m.doc.Unset(append(append([]string{}, lv.path...), r.entry))
|
||
m.row = max(0, m.row-1)
|
||
}
|
||
case "enter", "right", "l":
|
||
if m.row >= len(rows) {
|
||
return m, nil
|
||
}
|
||
r := rows[m.row]
|
||
switch {
|
||
case r.add && lv.kind == settings.BlockList:
|
||
i := m.doc.Append(lv.path)
|
||
m.levels = append(m.levels, settingsLevel{title: fmt.Sprintf("entry %d", i+1), block: lv.block,
|
||
path: append(append([]string{}, lv.path...), strconv.Itoa(i)), kind: settings.Block})
|
||
m.row = 0
|
||
case r.add && lv.kind == settings.BlockMap:
|
||
m.dialog, m.dialogTitle, m.dialogText = "addname", "Add an entry", "Name of the new entry"
|
||
m.editPath = lv.path
|
||
m.prompt = newTextField()
|
||
m.prompt.Width = 40
|
||
m.prompt.Focus()
|
||
case r.entry != "":
|
||
m.levels = append(m.levels, settingsLevel{title: r.label, block: lv.block,
|
||
path: append(append([]string{}, lv.path...), r.entry), kind: settings.Block})
|
||
m.row = 0
|
||
case r.key != nil:
|
||
path := append(append([]string{}, lv.path...), r.key.Key)
|
||
switch r.key.Kind {
|
||
case settings.Block:
|
||
m.levels = append(m.levels, settingsLevel{title: r.key.Key, block: r.key.Block, path: path, kind: settings.Block})
|
||
m.row = 0
|
||
case settings.BlockList, settings.BlockMap:
|
||
m.levels = append(m.levels, settingsLevel{title: r.key.Key, block: r.key.Block, path: path, kind: r.key.Kind})
|
||
m.row = 0
|
||
case settings.Bool:
|
||
cur, ok := m.doc.Text(path)
|
||
if !ok {
|
||
cur, _ = settings.Effective(m.cfg, path)
|
||
}
|
||
b, _ := strconv.ParseBool(cur)
|
||
_ = m.doc.Set(path, settings.Bool, strconv.FormatBool(!b))
|
||
default:
|
||
m.editPath, m.editKind = path, r.key.Kind
|
||
cur, ok := m.doc.Text(path)
|
||
if !ok {
|
||
cur, _ = settings.Effective(m.cfg, path)
|
||
}
|
||
m.prompt = newTextField()
|
||
m.prompt.Width = 50
|
||
m.prompt.SetValue(strings.ReplaceAll(cur, "\n", ", "))
|
||
m.prompt.Focus()
|
||
m.dialog, m.dialogTitle, m.dialogText = "edit", "Change "+r.key.Key, m.editQuestion()
|
||
}
|
||
}
|
||
m.scroll = 0
|
||
}
|
||
return m, nil
|
||
}
|
||
|
||
func (m consoleModel) save() (tea.Model, tea.Cmd) {
|
||
if m.doc.Changed == 0 {
|
||
m.dialog, m.dialogTitle, m.dialogText = "error", "Nothing to save", "No change has been made."
|
||
return m, nil
|
||
}
|
||
backup, err := m.doc.Save(func(p string) error { _, err := m.load(p); return err })
|
||
if err != nil {
|
||
m.dialog, m.dialogTitle = "error", "Not saved"
|
||
m.dialogText = "The configuration would not load:\n\n" + sBad.Render(err.Error()) + "\n\n" + sDim.Render("Nothing was written. Any key to go back.")
|
||
return m, nil
|
||
}
|
||
m.loadSettings()
|
||
note := ""
|
||
if backup != "" {
|
||
note = "The old file is " + backup + ".\n"
|
||
}
|
||
m.dialog, m.dialogTitle = "saved", "Saved"
|
||
m.dialogText = "Saved " + m.file + ".\n" + note +
|
||
fmt.Sprintf("\n%s reads it when it starts. Restart it now?\n", m.service())
|
||
if !m.node {
|
||
m.dialogText += sDim.Render("Another control plane keeps its own copy: make the change there too.") + "\n"
|
||
}
|
||
m.dialogText += "\n" + sDim.Render("Y restart · N later")
|
||
return m, nil
|
||
}
|
||
|
||
// --- views ---
|
||
|
||
func (m consoleModel) rowCount() int {
|
||
switch m.menu {
|
||
case cNodes, cLogs:
|
||
return len(m.data.nodes)
|
||
case cGPUs:
|
||
return len(m.gpuRows())
|
||
case cModels:
|
||
return len(m.data.models)
|
||
case cReplicas:
|
||
return len(m.data.replicas)
|
||
case cUsage:
|
||
return len(m.usage)
|
||
case cQuotas:
|
||
return len(m.quotas)
|
||
case cMirror:
|
||
return len(m.mirrorGroups())
|
||
case cJobs:
|
||
return len(m.data.jobs)
|
||
case cAudit:
|
||
return len(m.data.audit)
|
||
case cFederation:
|
||
return len(m.data.federation.Peers)
|
||
case cServe:
|
||
return len(m.hf.models)
|
||
case cAccounts:
|
||
return len(m.people.users)
|
||
case cKeys:
|
||
return len(m.people.keys)
|
||
case cOrgs:
|
||
return len(m.people.orgs)
|
||
case cManual:
|
||
return len(m.docs.topics)
|
||
}
|
||
return 0
|
||
}
|
||
|
||
func (m consoleModel) View() string {
|
||
host, _ := os.Hostname()
|
||
nav := []navItem{{Label: "Watch", Heading: true}}
|
||
if m.node {
|
||
nav = nil
|
||
}
|
||
for i := 0; i < cCount; i++ {
|
||
if m.node && !m.entryOK(i) {
|
||
continue
|
||
}
|
||
switch i {
|
||
case cAccounts:
|
||
nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "People", Heading: true})
|
||
case cManual:
|
||
nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "Help", Heading: true})
|
||
case cSettings:
|
||
nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "Settings", Heading: true})
|
||
case cAddNode, cLeave:
|
||
if (i == cAddNode) != m.node {
|
||
nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "Do", Heading: true})
|
||
}
|
||
}
|
||
label := consoleLabels[i]
|
||
if i == cSettings {
|
||
label = filepathBase(m.file)
|
||
}
|
||
if n := runningJobs(m.data.jobs); i == cJobs && n > 0 {
|
||
label += fmt.Sprintf(" (%d)", n)
|
||
}
|
||
if i == cSettings && m.doc != nil && m.doc.Changed > 0 {
|
||
label += fmt.Sprintf(" (%d)", m.doc.Changed)
|
||
}
|
||
nav = append(nav, navItem{Label: label, Current: i == m.menu, Dim: !m.entryOK(i)})
|
||
}
|
||
title, body, keys := m.panel()
|
||
title0 := "llmgw console"
|
||
if m.node {
|
||
title0 = "llmgw · node"
|
||
}
|
||
f := frame{
|
||
Width: m.width, Height: m.height, Title: title0, Where: host,
|
||
NavTitle: "", Nav: nav, PanelTitle: title, Body: body, Scroll: m.scroll, Keys: keys,
|
||
}
|
||
if m.menu == cManual && m.detail {
|
||
f.Scroll = m.docs.scroll
|
||
}
|
||
// The selected row kept on screen.
|
||
if m.inPanel && !m.detail {
|
||
inner := m.height - 4
|
||
top := m.row + 3
|
||
if top-m.scroll >= inner-1 {
|
||
f.Scroll = top - inner + 2
|
||
}
|
||
}
|
||
if m.dialog != "" {
|
||
f.DialogTitle = m.dialogTitle
|
||
f.Dialog = m.dialogText
|
||
switch m.dialog {
|
||
case "target", "edit", "addname":
|
||
q := m.dialogText
|
||
if m.dialog == "target" && m.editKind > 1 {
|
||
q = fmt.Sprintf("%s %d of %d", q, len(m.asked)+1, int(m.editKind))
|
||
}
|
||
f.Dialog = q + "\n\n" + m.prompt.View() + "\n\n" + sDim.Render("Enter · Esc cancel")
|
||
}
|
||
} else if t, d := m.docs.dialog(); d != "" && m.menu == cManual {
|
||
f.DialogTitle, f.Dialog = t, d
|
||
} else if m.busy != "" {
|
||
note := "asking the control plane…"
|
||
if m.busyNote != "" {
|
||
note = m.busyNote
|
||
}
|
||
f.DialogTitle, f.Dialog = m.busy, m.spin.View()+" "+note
|
||
} else if t, d := m.actionDialog(); d != "" {
|
||
f.DialogTitle, f.Dialog = t, d
|
||
}
|
||
return f.render()
|
||
}
|
||
|
||
// panel is the right-hand side for the entry chosen.
|
||
func (m consoleModel) panel() (title, body, keys string) {
|
||
pw := max(40, m.width-navWidth-8)
|
||
rowKeys := "↑↓ row · Enter actions · ← back · PgUp/PgDn · Q leave"
|
||
if !m.inPanel {
|
||
keys = "↑↓ choose · → or Enter open · Q leave · refreshed every three seconds"
|
||
}
|
||
waiting := func() string {
|
||
if m.data.err != nil {
|
||
return "\n " + sBad.Render("The control plane does not answer: ") + m.data.err.Error() +
|
||
"\n\n " + sDim.Render("systemctl status llmgw-server says why.")
|
||
}
|
||
return "\n " + m.spin.View() + " asking the control plane…"
|
||
}
|
||
if (!m.loaded || m.data.err != nil) && !m.node {
|
||
if m.menu < cSettings {
|
||
return consoleLabels[m.menu], waiting(), keys
|
||
}
|
||
}
|
||
switch m.menu {
|
||
case cOverview:
|
||
var b bytes.Buffer
|
||
c := &cobra.Command{}
|
||
c.SetOut(&b)
|
||
printSummary(c, m.data.sum)
|
||
return "Overview", "\n" + indent(b.String(), " "), keys
|
||
case cNodes, cLogs:
|
||
if m.detail && m.row < len(m.data.nodes) {
|
||
n := m.data.nodes[m.row]
|
||
if m.menu == cLogs {
|
||
return "Agent log · " + n.ID, m.logView(), "← back · PgUp/PgDn · refreshed every three seconds"
|
||
}
|
||
return "Node · " + n.ID, m.nodeDetail(n) + m.detailCharts(pw), "← back · 1–4 or R range · PgUp/PgDn · Q leave"
|
||
}
|
||
var rows [][]string
|
||
for _, n := range m.data.nodes {
|
||
cpu, mem := "—", "—"
|
||
if n.Load != nil {
|
||
cpu, mem = fmt.Sprintf("%.0f%%", n.Load.CPUPercent), fmt.Sprintf("%.0f%%", n.Load.MemPercent)
|
||
}
|
||
state := stateColumn(n)
|
||
if state != "ready" {
|
||
state = sWarn.Render(state)
|
||
}
|
||
rows = append(rows, []string{n.ID, state, locationColumn(n), cpu, mem, fmt.Sprintf("%d/%d", n.GPUFree, n.GPUTotal),
|
||
strconv.Itoa(n.Replicas), fmt.Sprintf("%d/%d GB", n.DiskFreeGB, n.DiskTotalGB), n.AgentVersion, n.LastSeenAgo})
|
||
}
|
||
t := "Nodes"
|
||
if m.menu == cLogs {
|
||
t = "Logs · choose a node"
|
||
}
|
||
if m.inPanel {
|
||
keys = rowKeys
|
||
}
|
||
return t, "\n" + table([]string{"NODE", "STATE", "WHERE", "CPU", "MEM", "GPU FREE", "REPLICAS", "DISK FREE", "AGENT", "SEEN"},
|
||
rows, m.row, pw, m.inPanel), keys
|
||
case cGPUs:
|
||
if m.inPanel {
|
||
keys = "↑↓ row · ← back · PgUp/PgDn · Q leave"
|
||
}
|
||
return "GPUs", "\n" + table([]string{"NODE", "GPU", "CARD", "BUSY", "MEMORY", "TEMP", "POWER", "HELD BY"},
|
||
m.gpuRows(), m.row, pw, m.inPanel), keys
|
||
case cModels:
|
||
if m.detail && m.row < len(m.data.models) {
|
||
return "Model · " + m.data.models[m.row].Name, m.modelDetail(m.data.models[m.row]) + m.detailCharts(pw),
|
||
"← back · 1–4 or R range · PgUp/PgDn · Q leave"
|
||
}
|
||
var rows [][]string
|
||
for _, md := range m.data.models {
|
||
status := "stopped"
|
||
switch {
|
||
case md.Parked:
|
||
status = sBad.Render("parked")
|
||
case md.Unplaceable:
|
||
status = sWarn.Render("not placed")
|
||
case md.Ready > 0 && md.Ready >= md.Desired:
|
||
status = sOK.Render("ready")
|
||
case md.Desired > 0:
|
||
status = sWarn.Render("starting")
|
||
}
|
||
warn := ""
|
||
if len(md.Warnings) > 0 {
|
||
warn = sWarn.Render("!")
|
||
}
|
||
upd := ""
|
||
if md.Update != nil && md.Update.Available {
|
||
upd = sWarn.Render("↑ " + catalog.ShortRevision(md.Update.Latest))
|
||
}
|
||
rows = append(rows, []string{md.Name, fmt.Sprintf("%d/%d", md.Ready, md.Desired), status, md.Class,
|
||
strings.Join(md.Profiles, ","), upd, warn})
|
||
}
|
||
if m.inPanel {
|
||
keys = "↑↓ row · Enter actions · A add from a file · ← back · Q leave"
|
||
}
|
||
return "Models", "\n" + table([]string{"MODEL", "READY", "STATUS", "CLASS", "PROFILES", "UPDATE", ""}, rows, m.row, pw, m.inPanel), keys
|
||
case cReplicas:
|
||
var rows [][]string
|
||
for _, r := range m.data.replicas {
|
||
var nodes []string
|
||
for _, s := range r.Shards {
|
||
nodes = append(nodes, fmt.Sprintf("%s%v", s.Node, discovery.DeviceLabels(s.GPUs)))
|
||
}
|
||
rows = append(rows, []string{r.ID, r.Model, r.State, r.Pool, strings.Join(nodes, " "),
|
||
(time.Duration(r.AgeSec) * time.Second).Round(time.Second).String()})
|
||
}
|
||
if m.inPanel {
|
||
keys = rowKeys
|
||
}
|
||
return "Replicas", "\n" + table([]string{"REPLICA", "MODEL", "STATE", "POOL", "ON", "AGE"}, rows, m.row, pw, m.inPanel), keys
|
||
case cUsage:
|
||
head := "\n Last 24 hours, by " + lipgloss.NewStyle().Bold(true).Render(usageGroupings[m.usageBy]) +
|
||
sDim.Render(" (→ to group by "+usageGroupings[(m.usageBy+1)%len(usageGroupings)]+")") + "\n\n"
|
||
if m.usageErr != nil {
|
||
return "Usage", head + " " + sBad.Render(m.usageErr.Error()), keys
|
||
}
|
||
var rows [][]string
|
||
for _, u := range m.usage {
|
||
rows = append(rows, []string{u.Name, compactNum(u.Requests), compactNum(u.Prompt + u.Completion),
|
||
compactNum(u.Errors), fmt.Sprintf("%.2f", float64(u.CostMicros)/1e6)})
|
||
}
|
||
if m.inPanel {
|
||
keys = "↑↓ row · → grouping · E export as CSV · ← back · Q leave"
|
||
}
|
||
return "Usage", head + table([]string{"WHO", "REQUESTS", "TOKENS", "ERRORS", "SPENT"}, rows, m.row, pw, m.inPanel), keys
|
||
case cQuotas:
|
||
return m.quotasPanel(pw)
|
||
case cAccounts, cKeys, cOrgs:
|
||
return m.peoplePanel(pw)
|
||
case cManual:
|
||
if m.detail && m.docs.open >= 0 {
|
||
k := docKeys
|
||
if m.docs.note != "" {
|
||
k = m.docs.note
|
||
}
|
||
return "Manual · " + m.docs.topics[m.docs.open].title, "\n" + m.docs.body(pw-4), k
|
||
}
|
||
d := m.docs
|
||
d.sel = m.row
|
||
if m.inPanel {
|
||
keys = "↑↓ topic · Enter read · ← back · ? the console's page · Q leave"
|
||
}
|
||
return "Manual", d.list(pw, m.inPanel), keys
|
||
case cMirror:
|
||
return m.mirrorPanel(pw)
|
||
case cJobs:
|
||
return m.jobsPanel(pw)
|
||
case cAudit:
|
||
return m.auditPanel(pw)
|
||
case cFederation:
|
||
return m.federationPanel(pw)
|
||
case cMonitor:
|
||
return m.monitorPanel(pw)
|
||
case cServe:
|
||
return m.hfPanel(pw)
|
||
case cSettings:
|
||
return m.settingsPanel(pw)
|
||
}
|
||
var item string
|
||
switch m.menu {
|
||
case cAddNode:
|
||
item = "\n Adds a machine as a node: installs llmgw on it over ssh, checks it,\n gives it a certificate and starts its agent.\n\n " +
|
||
sDim.Render("On the command line:") + "\n sudo llmgw node add <user>@<address>"
|
||
case cAddControl:
|
||
item = "\n Makes another machine a control plane of this cluster: a standby\n that takes over within seconds when this one is lost. Every node is\n told of it; none is touched. With a floating address, keepalived on\n both keeps it on whichever leads — the address to give clients.\n\n " +
|
||
sDim.Render("Needs the store replicated first, so both reach it.") + "\n\n " +
|
||
sDim.Render("On the command line:") + "\n sudo --preserve-env=SSH_AUTH_SOCK llmgw server add <user>@<address> --vip <address>/<prefix>"
|
||
case cReplicate:
|
||
item = "\n Moves the store onto two machines under Patroni, replicated,\n so losing one loses nothing.\n\n " +
|
||
sDim.Render("On the command line:") +
|
||
"\n sudo --preserve-env=SSH_AUTH_SOCK llmgw server init \\\n --patroni <user>@<a> --patroni <user>@<b> --no-mirror --no-node --no-model"
|
||
case cUpgrade:
|
||
item = "\n Moves the cluster to a release: this machine first, then each node\n in turn. Engines keep serving; a node that does not come back stops it.\n\n " +
|
||
sDim.Render("On the command line:") + "\n sudo llmgw ctl upgrade VERSION"
|
||
case cRestartAll:
|
||
item = "\n Replaces every replica, one at a time, each new one up before the\n old one stops.\n\n " +
|
||
sDim.Render("On the command line:") + "\n llmgw ctl restart --all"
|
||
case cBackup:
|
||
item = "\n Writes the store and /etc/llmgw to one archive in " + DefaultBackupDir + ",\n and keeps the last few. Copy it off this machine.\n\n " +
|
||
sDim.Render("On the command line:") + "\n sudo llmgw server backup"
|
||
case cLeave:
|
||
item = "\n Closes the console. The cluster keeps running."
|
||
}
|
||
return consoleLabels[m.menu], item, "Enter do it · ↑↓ choose · Q leave"
|
||
}
|
||
|
||
func (m consoleModel) settingsPanel(pw int) (string, string, string) {
|
||
if m.docErr != nil {
|
||
return "server.yaml", "\n " + sBad.Render(m.docErr.Error()), "← back"
|
||
}
|
||
var crumbs []string
|
||
for _, l := range m.levels {
|
||
crumbs = append(crumbs, l.title)
|
||
}
|
||
rows := m.settingsRows()
|
||
var tr [][]string
|
||
for _, r := range rows {
|
||
typ := ""
|
||
if r.key != nil {
|
||
typ = sDim.Render(r.key.Type)
|
||
}
|
||
tr = append(tr, []string{r.label, r.value, typ})
|
||
}
|
||
body := "\n" + table([]string{"KEY", "VALUE", "TYPE"}, tr, m.row, pw, m.inPanel)
|
||
// What the chosen key is for, from its comment in the code.
|
||
if m.inPanel && m.row < len(rows) {
|
||
doc := ""
|
||
if r := rows[m.row]; r.key != nil {
|
||
doc = r.key.Doc
|
||
if doc == "" && r.key.Block != nil {
|
||
doc = r.key.Block.Doc
|
||
}
|
||
} else if l := m.level(); l.block != nil {
|
||
doc = l.block.Doc
|
||
}
|
||
if doc != "" {
|
||
body += "\n " + sDim.Render(strings.Repeat("─", min(pw-4, 60))) + "\n" +
|
||
indent(wrap(firstParagraphs(doc, 2), pw-4, ""), " ")
|
||
}
|
||
}
|
||
keys := "→ or Enter open · ↑↓ row · Q leave"
|
||
if m.inPanel {
|
||
keys = "Enter change/open · U back to default · S save · Esc up · PgUp/PgDn"
|
||
if l := m.level(); l.kind != settings.Block {
|
||
keys = "Enter open · D remove entry · S save · Esc up"
|
||
}
|
||
}
|
||
return strings.Join(crumbs, " › "), body, keys
|
||
}
|
||
|
||
func firstParagraphs(s string, n int) string {
|
||
p := strings.Split(s, "\n\n")
|
||
if len(p) > n {
|
||
p = p[:n]
|
||
}
|
||
return strings.Join(p, "\n\n")
|
||
}
|
||
|
||
func (m consoleModel) gpuRows() [][]string {
|
||
held := map[string]string{}
|
||
for _, r := range m.data.replicas {
|
||
for _, s := range r.Shards {
|
||
for _, g := range s.GPUs {
|
||
held[s.Node+"/"+strconv.Itoa(g)] = r.Model
|
||
}
|
||
}
|
||
}
|
||
var rows [][]string
|
||
for _, n := range m.data.nodes {
|
||
for _, g := range n.GPUs {
|
||
busy, mem, temp, power := "—", "—", "—", "—"
|
||
// A MIG instance under its card: its own memory, the card's heat
|
||
// and power, and no utilisation, which no driver measures per
|
||
// instance — said, rather than shown as the card's or as zero.
|
||
mig := g.MIGProfile != ""
|
||
if mig {
|
||
busy = sDim.Render("n/a on MIG")
|
||
if g.MemoryTotalMB > 0 {
|
||
mem = fmt.Sprintf("%.1f/%.0f GB", float64(g.MemoryUsedMB)/1024, float64(g.MemoryTotalMB)/1024)
|
||
}
|
||
}
|
||
card := g.Card
|
||
if g.Label == "" {
|
||
// A control plane older than MIG, which sends no card.
|
||
card = g.Index
|
||
}
|
||
if n.Load != nil {
|
||
for _, s := range n.Load.GPUs {
|
||
if s.Index != card {
|
||
continue
|
||
}
|
||
if !mig {
|
||
busy = fmt.Sprintf("%.0f%%", s.UtilPercent)
|
||
if s.MemoryTotalMB > 0 {
|
||
mem = fmt.Sprintf("%.1f/%.0f GB", s.MemoryUsedMB/1024, s.MemoryTotalMB/1024)
|
||
}
|
||
}
|
||
if s.TempC > 0 {
|
||
temp = fmt.Sprintf("%.0f°C", s.TempC)
|
||
}
|
||
if s.PowerW > 0 {
|
||
power = fmt.Sprintf("%.0f/%.0f W", s.PowerW, s.PowerLimitW)
|
||
}
|
||
}
|
||
}
|
||
holder := held[n.ID+"/"+strconv.Itoa(g.Index)]
|
||
if holder == "" {
|
||
holder = sDim.Render("free")
|
||
}
|
||
label := g.Label
|
||
if label == "" {
|
||
// A control plane older than labels.
|
||
label = strconv.Itoa(g.Index)
|
||
}
|
||
rows = append(rows, []string{n.ID, label, g.Name, busy, mem, temp, power, holder})
|
||
}
|
||
}
|
||
return rows
|
||
}
|
||
|
||
func (m consoleModel) nodeDetail(n adminapi.NodeView) string {
|
||
var b strings.Builder
|
||
kv := func(k, v string) { fmt.Fprintf(&b, " %-14s %s\n", sDim.Render(k), v) }
|
||
b.WriteString("\n")
|
||
kv("state", n.State)
|
||
if n.Location != nil {
|
||
where := []string{}
|
||
for _, d := range n.Location.Domains() {
|
||
where = append(where, d.Human())
|
||
}
|
||
kv("location", strings.Join(where, " · "))
|
||
} else {
|
||
kv("location", sDim.Render("not labelled: Enter, then Location…"))
|
||
}
|
||
if n.Held != "" {
|
||
kv("held", sWarn.Render(n.Held))
|
||
}
|
||
kv("address", n.MgmtAddr)
|
||
kv("system", n.OS)
|
||
kv("agent", n.AgentVersion+", seen "+n.LastSeenAgo+" ago")
|
||
if n.Load != nil {
|
||
l := n.Load
|
||
kv("cpu", fmt.Sprintf("%.0f%% of %d · load %.2f %.2f %.2f", l.CPUPercent, l.CPUs, l.Load1, l.Load5, l.Load15))
|
||
kv("memory", fmt.Sprintf("%.0f%% · %.1f of %.1f GB", l.MemPercent, float64(l.MemUsed)/(1<<30), float64(l.MemTotal)/(1<<30)))
|
||
}
|
||
kv("disk", fmt.Sprintf("%d GB free of %d", n.DiskFreeGB, n.DiskTotalGB))
|
||
kv("replicas", strconv.Itoa(n.Replicas))
|
||
// By class, which on a node with a card in MIG mode is the instances'
|
||
// profiles beside any whole card: what a profile's gpu_class can name.
|
||
for _, c := range n.ClassGPUs {
|
||
what := fmt.Sprintf("%d, %d free", c.Total, c.Free)
|
||
if c.MIG {
|
||
what += " · MIG instances, one to a replica"
|
||
}
|
||
kv(c.Class, what)
|
||
}
|
||
if len(n.Provision) > 0 {
|
||
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("Engines") + "\n")
|
||
for _, p := range n.Provision {
|
||
fmt.Fprintf(&b, " %-9s %-10s %-8s %s\n", p.Engine, p.Version, p.State, sDim.Render(truncate(p.Detail, 60)))
|
||
}
|
||
}
|
||
if len(n.GPUs) > 0 {
|
||
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("GPUs") + "\n")
|
||
for _, r := range m.gpuRows() {
|
||
if r[0] == n.ID {
|
||
fmt.Fprintf(&b, " %s %-18s busy %-5s mem %-14s %-6s %s %s\n", r[1], r[2], r[3], r[4], r[5], r[6], r[7])
|
||
}
|
||
}
|
||
}
|
||
for _, w := range append(n.Warnings, n.Unhealthy...) {
|
||
b.WriteString("\n " + sWarn.Render("! ") + w)
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
func (m consoleModel) modelDetail(md adminapi.ModelView) string {
|
||
var b strings.Builder
|
||
kv := func(k, v string) {
|
||
if v != "" {
|
||
fmt.Fprintf(&b, " %-14s %s\n", sDim.Render(k), v)
|
||
}
|
||
}
|
||
b.WriteString("\n")
|
||
if md.Description != "" {
|
||
b.WriteString(" " + md.Description + "\n\n")
|
||
}
|
||
kv("replicas", fmt.Sprintf("%d ready of %d wanted (min %d, max %d)", md.Ready, md.Desired, md.Min, md.Max))
|
||
kv("class", md.Class)
|
||
kv("owner", md.Owner)
|
||
kv("profiles", strings.Join(md.Profiles, ", "))
|
||
kv("aliases", strings.Join(md.Aliases, ", "))
|
||
kv("source", md.Source)
|
||
if o := md.Origin; o != nil {
|
||
from := o.Repo + " at " + catalog.ShortRevision(o.Revision)
|
||
if o.Previous != "" {
|
||
from += ", updated from " + catalog.ShortRevision(o.Previous)
|
||
}
|
||
kv("from", from)
|
||
}
|
||
if u := md.Update; u != nil {
|
||
kv("registry", updateCell(*u)+sDim.Render(", asked "+ago(u.CheckedAt, time.Now())))
|
||
}
|
||
if md.MemoryGB > 0 {
|
||
kv("reserves", fmt.Sprintf("%d GB, %d cores (uses %.1f GB)", md.MemoryGB, md.CPUCores, md.MeasuredMemoryGB))
|
||
}
|
||
kv("fallbacks", strings.Join(md.Fallbacks, ", "))
|
||
if md.Reason != "" {
|
||
b.WriteString("\n " + sWarn.Render("! ") + md.Reason + "\n")
|
||
}
|
||
for _, w := range md.Warnings {
|
||
b.WriteString(" " + sWarn.Render("! ") + w + "\n")
|
||
}
|
||
var reps [][]string
|
||
for _, r := range m.data.replicas {
|
||
if r.Model == md.Name {
|
||
reps = append(reps, []string{r.ID, r.State, r.Pool, r.Endpoint})
|
||
}
|
||
}
|
||
if len(reps) > 0 {
|
||
b.WriteString("\n" + table([]string{"REPLICA", "STATE", "POOL", "ENDPOINT"}, reps, -1, max(40, m.width-navWidth-8), false))
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
func (m consoleModel) logView() string {
|
||
if m.logs.err != nil {
|
||
return "\n " + sBad.Render(m.logs.err.Error())
|
||
}
|
||
if len(m.logs.lines) == 0 {
|
||
return "\n " + m.spin.View() + " reading the log…"
|
||
}
|
||
// The newest at the bottom, where a log is read; the panel starts there.
|
||
lines := m.logs.lines
|
||
if k := m.height - 6; len(lines) > k && m.scroll == 0 {
|
||
lines = lines[len(lines)-k:]
|
||
}
|
||
var b strings.Builder
|
||
b.WriteString("\n")
|
||
for _, l := range lines {
|
||
switch {
|
||
case strings.Contains(l, "level=ERROR"):
|
||
l = sBad.Render(l)
|
||
case strings.Contains(l, "level=WARN"):
|
||
l = sWarn.Render(l)
|
||
}
|
||
b.WriteString(" " + l + "\n")
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
func compactNum(n int64) string {
|
||
switch {
|
||
case n >= 1_000_000:
|
||
return fmt.Sprintf("%.1fM", float64(n)/1e6)
|
||
case n >= 1_000:
|
||
return fmt.Sprintf("%.1fk", float64(n)/1e3)
|
||
}
|
||
return strconv.FormatInt(n, 10)
|
||
}
|
||
|
||
func indent(s, pre string) string {
|
||
lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
|
||
for i := range lines {
|
||
lines[i] = pre + lines[i]
|
||
}
|
||
return strings.Join(lines, "\n")
|
||
}
|
||
|
||
func filepathBase(p string) string {
|
||
if i := strings.LastIndexByte(p, '/'); i >= 0 {
|
||
return p[i+1:]
|
||
}
|
||
return p
|
||
}
|
||
|
||
// runConsole shows the console, runs what is chosen on it, and comes back.
|
||
// file is server.yaml on a control plane, agent.yaml on a node.
|
||
func runConsole(ctx context.Context, out io.Writer, file string) error {
|
||
addr := "127.0.0.1:9090"
|
||
if cfg, err := config.Load(file); err == nil {
|
||
if cfg.AdminListen != "" {
|
||
addr = cfg.AdminListen
|
||
if strings.HasPrefix(addr, ":") {
|
||
addr = "127.0.0.1" + addr
|
||
}
|
||
}
|
||
// The admin token, for this screen and every command it runs in
|
||
// this process — `llmgw ctl` included — from this configuration's
|
||
// file rather than only the usual place.
|
||
if t := readAdminToken(adminTokenPath(file, cfg)); t != "" {
|
||
_ = os.Setenv("LLMGW_ADMIN_TOKEN", t)
|
||
}
|
||
}
|
||
menu := cOverview
|
||
if strings.HasSuffix(file, "agent.yaml") {
|
||
menu = cSettings
|
||
}
|
||
for {
|
||
m := newConsoleModel(addr, file)
|
||
m.menu = menu
|
||
final, err := tea.NewProgram(m, tea.WithOutput(out), tea.WithAltScreen()).Run()
|
||
if err != nil {
|
||
return err
|
||
}
|
||
fm := final.(consoleModel)
|
||
fm.st.close()
|
||
menu = fm.menu
|
||
act := fm.action
|
||
switch act.kind {
|
||
case "":
|
||
return nil
|
||
case "node":
|
||
c := newNodeAddCmd()
|
||
c.SetArgs(act.targets)
|
||
c.SetOut(out)
|
||
err = c.ExecuteContext(ctx)
|
||
case "patroni":
|
||
in := newDefaultServerInit()
|
||
in.patroni = act.targets
|
||
// Already set up: only the store changes.
|
||
in.noMirror, in.noNode, in.noModel = true, true, true
|
||
err = in.runInitTUI(ctx, out)
|
||
case "run":
|
||
c := Root()
|
||
c.SetArgs(act.targets)
|
||
c.SetOut(out)
|
||
c.SetErr(out)
|
||
fmt.Fprintln(out, sDim.Render("llmgw "+strings.Join(act.targets, " ")))
|
||
err = c.ExecuteContext(ctx)
|
||
case "restart":
|
||
fmt.Fprintf(out, "restarting %s…\n", fm.service())
|
||
if b, e := exec.CommandContext(ctx, "systemctl", "restart", fm.service()).CombinedOutput(); e != nil {
|
||
err = fmt.Errorf("%v: %s", e, strings.TrimSpace(string(b)))
|
||
} else {
|
||
fmt.Fprintln(out, sOK.Render("✓ ")+"restarted; it picks up the configuration as it starts")
|
||
}
|
||
}
|
||
if err != nil {
|
||
fmt.Fprintln(out, sBad.Render("✗ ")+err.Error())
|
||
}
|
||
fmt.Fprint(out, "\n"+sDim.Render("enter to go back to the console")+" ")
|
||
_, _ = bufio.NewReader(os.Stdin).ReadString('\n')
|
||
}
|
||
}
|
||
|
||
// floatingQuestion is the second question when adding a control plane.
|
||
const floatingQuestion = "A floating address for clients, kept on whichever control plane leads,\n" +
|
||
"as 10.0.0.161/24 — a free one on this network. Empty for none, when a load\n" +
|
||
"balancer in front checks /ready instead."
|