Watch the cluster and edit every setting from the console
The console gains live views of nodes, GPUs, models, replicas, usage and agent logs, and a settings tree over every key of server.yaml (agent.yaml on a node), generated from the configuration structs: values with their defaults and doc comments, edited in dialogs, saved only once the service's own loader accepts them, comments kept. The generator now tells blocks apart by package, which brings the TLS, engine and supervisor settings back into the reference. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
aa3efcebbe
commit
9ab93a0aa6
13 files changed
+2702
-472
No files matched your search
File diff suppressed because it is too large.
Load diff
@@ -24,6 +24,8 @@ type navItem struct {
|
||||
Current bool
|
||||
// Dim greys an entry that cannot be chosen now.
|
||||
Dim bool
|
||||
// Heading is a title between groups of entries, not an entry.
|
||||
Heading bool
|
||||
}
|
||||
|
||||
type frame struct {
|
||||
@@ -70,6 +72,10 @@ func (f frame) render() string {
|
||||
for _, it := range f.Nav {
|
||||
mark := " "
|
||||
label := it.Label
|
||||
if it.Heading {
|
||||
nav.WriteString(sDim.Render(label) + "\n")
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case it.State == "done":
|
||||
mark = sOK.Render("✓ ")
|
||||
@@ -182,3 +188,50 @@ func truncateWidth(s string, w int) string {
|
||||
}
|
||||
return ansi.Truncate(s, max(w-1, 0), "…")
|
||||
}
|
||||
|
||||
// table draws rows under a header in columns as wide as they need, cut to
|
||||
// width; the selected row is shown reversed when the panel has the focus.
|
||||
func table(headers []string, rows [][]string, selected int, width int, focused bool) string {
|
||||
widths := make([]int, len(headers))
|
||||
for i, h := range headers {
|
||||
widths[i] = ansi.StringWidth(h)
|
||||
}
|
||||
for _, r := range rows {
|
||||
for i := range headers {
|
||||
if i < len(r) {
|
||||
widths[i] = min(max(widths[i], ansi.StringWidth(r[i])), 44)
|
||||
}
|
||||
}
|
||||
}
|
||||
line := func(cells []string) string {
|
||||
var b strings.Builder
|
||||
for i := range headers {
|
||||
c := ""
|
||||
if i < len(cells) {
|
||||
c = truncateWidth(cells[i], widths[i])
|
||||
}
|
||||
b.WriteString(c)
|
||||
if i < len(headers)-1 {
|
||||
b.WriteString(strings.Repeat(" ", widths[i]-ansi.StringWidth(c)+2))
|
||||
}
|
||||
}
|
||||
return truncateWidth(b.String(), width)
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString(" " + lipgloss.NewStyle().Bold(true).Foreground(tuiDim).Render(line(headers)) + "\n")
|
||||
for i, r := range rows {
|
||||
l := line(r)
|
||||
switch {
|
||||
case i == selected && focused:
|
||||
pad := max(0, width-ansi.StringWidth(ansi.Strip(l)))
|
||||
l = fBar.Render(ansi.Strip(l) + strings.Repeat(" ", pad))
|
||||
case i == selected:
|
||||
l = lipgloss.NewStyle().Bold(true).Render(l)
|
||||
}
|
||||
b.WriteString(" " + l + "\n")
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
b.WriteString(" " + sDim.Render("nothing yet") + "\n")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
+4
-224
@@ -1,23 +1,13 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/mattn/go-isatty"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
// A bare `llmgw` in a terminal: the setup screens on a machine with nothing
|
||||
@@ -43,11 +33,11 @@ func runHome(cmd *cobra.Command) error {
|
||||
// that stopped halfway has written server.yaml and nothing that runs, and
|
||||
// the setup screens are what finish it.
|
||||
case exists("/etc/llmgw/server.yaml") && exists("/etc/systemd/system/llmgw-server.service"):
|
||||
return runConsole(cmd.Context(), out)
|
||||
return runConsole(cmd.Context(), out, "/etc/llmgw/server.yaml")
|
||||
case exists("/etc/llmgw/agent.yaml"):
|
||||
fmt.Fprintln(out, "This machine is a node. Its cluster is managed from the control plane:")
|
||||
fmt.Fprintln(out, "run llmgw there. Here, `llmgw doctor` checks the machine.")
|
||||
return nil
|
||||
// A node: its own settings. The cluster is watched from the
|
||||
// control plane.
|
||||
return runConsole(cmd.Context(), out, "/etc/llmgw/agent.yaml")
|
||||
}
|
||||
return runSetupTUI(cmd.Context(), out)
|
||||
}
|
||||
@@ -56,213 +46,3 @@ func exists(p string) bool {
|
||||
_, err := os.Stat(p)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// What the console asks to be done once its screen closes: each runs its own
|
||||
// screen, and the console comes back after.
|
||||
type consoleAction struct {
|
||||
kind string // "", "node", "patroni"
|
||||
targets []string
|
||||
}
|
||||
|
||||
type summaryMsg struct {
|
||||
text string
|
||||
store bool
|
||||
err error
|
||||
}
|
||||
|
||||
type consoleItem struct {
|
||||
label, command, ask string
|
||||
kind string
|
||||
n int // how many targets to ask for
|
||||
}
|
||||
|
||||
type consoleModel struct {
|
||||
client *ctlClient
|
||||
summary string
|
||||
store bool
|
||||
err error
|
||||
focus int
|
||||
asking bool
|
||||
asked []string
|
||||
prompt textField
|
||||
action consoleAction
|
||||
width int
|
||||
height int
|
||||
spin spinnerModel
|
||||
loaded bool
|
||||
}
|
||||
|
||||
func (m consoleModel) items() []consoleItem {
|
||||
its := []consoleItem{{label: "Add a node", kind: "node", n: 1,
|
||||
ask: "user@address of the machine", command: "sudo llmgw node add <user>@<address>"}}
|
||||
if !m.store {
|
||||
its = append(its, consoleItem{label: "Replicate the store across two machines (Patroni)", kind: "patroni", n: 2,
|
||||
ask: "user@address of database machine",
|
||||
command: "sudo --preserve-env=SSH_AUTH_SOCK llmgw server init --patroni <user>@<a> --patroni <user>@<b> --no-mirror --no-node --no-model"})
|
||||
}
|
||||
return append(its, consoleItem{label: "Leave", kind: "quit"})
|
||||
}
|
||||
|
||||
func (m consoleModel) fetch() tea.Msg {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
var s adminapi.Summary
|
||||
if err := m.client.get(ctx, "/api/v1/summary", &s); err != nil {
|
||||
return summaryMsg{err: err}
|
||||
}
|
||||
var b bytes.Buffer
|
||||
c := &cobra.Command{}
|
||||
c.SetOut(&b)
|
||||
printSummary(c, s)
|
||||
return summaryMsg{text: b.String(), store: len(s.Store) > 0}
|
||||
}
|
||||
|
||||
func (m consoleModel) Init() tea.Cmd { return tea.Batch(m.fetch, m.spin.Tick) }
|
||||
|
||||
func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
switch msg := msg.(type) {
|
||||
case tea.WindowSizeMsg:
|
||||
m.width, m.height = msg.Width, msg.Height
|
||||
case spinTickMsg:
|
||||
var cmd tea.Cmd
|
||||
m.spin, cmd = m.spin.Update(msg)
|
||||
return m, cmd
|
||||
case summaryMsg:
|
||||
m.loaded = true
|
||||
m.err = msg.err
|
||||
if msg.err == nil {
|
||||
m.summary, m.store = msg.text, msg.store
|
||||
}
|
||||
return m, tea.Tick(5*time.Second, func(time.Time) tea.Msg { return m.fetch() })
|
||||
case tea.KeyMsg:
|
||||
if msg.Type == tea.KeyCtrlC {
|
||||
return m, tea.Quit
|
||||
}
|
||||
its := m.items()
|
||||
if m.asking {
|
||||
switch msg.Type {
|
||||
case tea.KeyEsc:
|
||||
m.asking, m.asked = false, nil
|
||||
case tea.KeyEnter:
|
||||
v := strings.TrimSpace(m.prompt.Value())
|
||||
if v == "" {
|
||||
return m, nil
|
||||
}
|
||||
m.asked = append(m.asked, v)
|
||||
m.prompt.SetValue("")
|
||||
if it := its[m.focus]; len(m.asked) == it.n {
|
||||
m.action = consoleAction{kind: it.kind, targets: m.asked}
|
||||
return m, tea.Quit
|
||||
}
|
||||
default:
|
||||
var cmd tea.Cmd
|
||||
m.prompt, cmd = m.prompt.Update(msg)
|
||||
return m, cmd
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
switch msg.String() {
|
||||
case "up", "k", "shift+tab":
|
||||
m.focus = (m.focus - 1 + len(its)) % len(its)
|
||||
case "down", "j", "tab":
|
||||
m.focus = (m.focus + 1) % len(its)
|
||||
case "q":
|
||||
return m, tea.Quit
|
||||
case "enter":
|
||||
it := its[m.focus]
|
||||
if it.kind == "quit" {
|
||||
return m, tea.Quit
|
||||
}
|
||||
m.asking, m.asked = true, nil
|
||||
m.prompt = newTextField()
|
||||
m.prompt.Placeholder, m.prompt.Width = "ubuntu@10.0.0.20", 40
|
||||
m.prompt.Focus()
|
||||
}
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m consoleModel) View() string {
|
||||
host, _ := os.Hostname()
|
||||
var nav []navItem
|
||||
for i, it := range m.items() {
|
||||
nav = append(nav, navItem{Label: it.label, Current: i == m.focus})
|
||||
}
|
||||
var body string
|
||||
switch {
|
||||
case !m.loaded:
|
||||
body = "\n " + m.spin.View() + " asking the control plane…"
|
||||
case m.err != nil:
|
||||
body = "\n " + sBad.Render("The control plane does not answer:") + "\n " + m.err.Error() +
|
||||
"\n\n " + sDim.Render("systemctl status llmgw-server says why.")
|
||||
default:
|
||||
body = "\n" + indent(m.summary, " ")
|
||||
}
|
||||
if it := m.items()[m.focus]; it.command != "" {
|
||||
body += "\n\n " + sDim.Render("On the command line:") + "\n " + it.command
|
||||
}
|
||||
f := frame{
|
||||
Width: m.width, Height: m.height,
|
||||
Title: "llmgw console", Where: host,
|
||||
NavTitle: "Do", Nav: nav,
|
||||
PanelTitle: "Cluster", Body: body,
|
||||
Keys: "↑↓ choose · Enter do it · Q leave · refreshed every five seconds",
|
||||
}
|
||||
if m.asking {
|
||||
it := m.items()[m.focus]
|
||||
ask := it.ask
|
||||
if it.n > 1 {
|
||||
ask = fmt.Sprintf("%s %d of %d", ask, len(m.asked)+1, it.n)
|
||||
}
|
||||
f.DialogTitle = it.label
|
||||
f.Dialog = ask + "\n\n" + m.prompt.View() + "\n\n" + sDim.Render("Enter · Esc cancel")
|
||||
}
|
||||
return f.render()
|
||||
}
|
||||
|
||||
func indent(s, pre string) string {
|
||||
lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
|
||||
for i := range lines {
|
||||
lines[i] = pre + lines[i]
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
// runConsole shows the console, runs what is chosen on it, and comes back.
|
||||
func runConsole(ctx context.Context, out io.Writer) error {
|
||||
addr := "127.0.0.1:9090"
|
||||
if cfg, err := config.Load("/etc/llmgw/server.yaml"); err == nil && cfg.AdminListen != "" {
|
||||
addr = cfg.AdminListen
|
||||
if strings.HasPrefix(addr, ":") {
|
||||
addr = "127.0.0.1" + addr
|
||||
}
|
||||
}
|
||||
for {
|
||||
m := consoleModel{client: newCtlClient(addr), spin: newSpinner(), width: 100, height: 30}
|
||||
final, err := tea.NewProgram(m, tea.WithOutput(out), tea.WithAltScreen()).Run()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
act := final.(consoleModel).action
|
||||
switch act.kind {
|
||||
case "":
|
||||
return nil
|
||||
case "node":
|
||||
c := newNodeAddCmd()
|
||||
c.SetArgs(act.targets)
|
||||
c.SetOut(out)
|
||||
err = c.ExecuteContext(ctx)
|
||||
case "patroni":
|
||||
in := newDefaultServerInit()
|
||||
in.patroni = act.targets
|
||||
// Already set up: only the store changes.
|
||||
in.noMirror, in.noNode, in.noModel = true, true, true
|
||||
err = in.runInitTUI(ctx, out)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintln(out, sBad.Render("✗ ")+err.Error())
|
||||
}
|
||||
fmt.Fprint(out, "\n"+sDim.Render("enter to go back to the console")+" ")
|
||||
_, _ = bufio.NewReader(os.Stdin).ReadString('\n')
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -104,35 +106,68 @@ func TestTheChecksHoldTheSetupBackOnlyForAFailure(t *testing.T) {
|
||||
|
||||
func TestTheConsoleOffersToReplicateOnlyAStoreThatIsNot(t *testing.T) {
|
||||
m := consoleModel{}
|
||||
if !strings.Contains(m.items()[1].label, "Replicate") {
|
||||
if !m.entryOK(cReplicate) {
|
||||
t.Error("a single store is not offered replication")
|
||||
}
|
||||
m.store = true
|
||||
for _, it := range m.items() {
|
||||
if strings.Contains(it.label, "Replicate") {
|
||||
t.Error("a replicated store is offered replication")
|
||||
}
|
||||
m.data.sum.Store = []adminapi.StoreMember{{Name: "a", Role: "leader"}}
|
||||
if m.entryOK(cReplicate) {
|
||||
t.Error("a replicated store is offered replication")
|
||||
}
|
||||
// A node is asked for, then the console closes with it to do.
|
||||
var mm tea.Model = consoleModel{summary: "x", loaded: true}
|
||||
// A node is asked for in a dialog, then the console closes with it to do.
|
||||
var mm tea.Model = consoleModel{menu: cAddNode, loaded: true, width: 100, height: 30}
|
||||
mm = press(mm, "enter", "ubuntu@10.0.0.9", "enter")
|
||||
if a := mm.(consoleModel).action; a.kind != "node" || a.targets[0] != "ubuntu@10.0.0.9" {
|
||||
t.Errorf("%+v", a)
|
||||
}
|
||||
_ = adminapi.Summary{}
|
||||
}
|
||||
|
||||
func TestTheSetupIsDrawnAsWindows(t *testing.T) {
|
||||
m := newSetupModel("ctl", []string{"10.0.0.5"})
|
||||
m.width, m.height = 100, 30
|
||||
v := m.View()
|
||||
lines := strings.Split(v, "\n")
|
||||
if len(lines) != 30 {
|
||||
t.Errorf("%d lines for a 30-line terminal", len(lines))
|
||||
func TestTheConsoleWatchesTheClusterAndEditsItsSettings(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
file := filepath.Join(dir, "server.yaml")
|
||||
if err := os.WriteFile(file, []byte("# mine\nlisten: \":8080\"\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{"llmgw setup", "Steps", "Start", "╭", "One machine", "Ctrl+C"} {
|
||||
if !strings.Contains(v, want) {
|
||||
t.Errorf("the screen lacks %q:\n%s", want, v)
|
||||
m := newConsoleModel("127.0.0.1:1", file)
|
||||
m.width, m.height, m.loaded = 120, 40, true
|
||||
m.data.nodes = []adminapi.NodeView{{ID: "cuda01", State: "ready", GPUTotal: 1, GPUs: []adminapi.GPUView{{Index: 0, Name: "Quadro RTX 6000"}}}}
|
||||
m.data.replicas = []adminapi.ReplicaView{{ID: "r1", Model: "coder-14b", State: "ready",
|
||||
Shards: []adminapi.ShardView{{Node: "cuda01", GPUs: []int{0}}}}}
|
||||
|
||||
// Watching: the GPUs view says which model holds each card.
|
||||
m.menu = cGPUs
|
||||
if v := m.View(); !strings.Contains(v, "Quadro RTX 6000") || !strings.Contains(v, "coder-14b") {
|
||||
t.Errorf("the GPUs view:\n%s", v)
|
||||
}
|
||||
|
||||
// Settings: into router, change retries, save.
|
||||
var mm tea.Model = m
|
||||
mm.(consoleModel).View()
|
||||
cm := mm.(consoleModel)
|
||||
cm.menu, cm.inPanel = cSettings, true
|
||||
for i, r := range cm.settingsRows() {
|
||||
if r.label == "router" {
|
||||
cm.row = i
|
||||
}
|
||||
}
|
||||
mm = press(cm, "enter") // into router
|
||||
cm = mm.(consoleModel)
|
||||
for i, r := range cm.settingsRows() {
|
||||
if r.label == "retries" {
|
||||
cm.row = i
|
||||
}
|
||||
}
|
||||
if v := cm.View(); !strings.Contains(v, "(default)") {
|
||||
t.Errorf("an unset key does not show its default:\n%s", v)
|
||||
}
|
||||
mm = press(cm, "enter") // the edit dialog
|
||||
cm = mm.(consoleModel)
|
||||
cm.prompt.SetValue("5")
|
||||
mm = press(cm, "enter", "s")
|
||||
if d := mm.(consoleModel).dialog; d != "saved" {
|
||||
t.Fatalf("after saving the dialog is %q: %s", d, mm.(consoleModel).dialogText)
|
||||
}
|
||||
b, _ := os.ReadFile(file)
|
||||
if !strings.Contains(string(b), "retries: 5") || !strings.Contains(string(b), "# mine") {
|
||||
t.Errorf("server.yaml:\n%s", b)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user