Ilker ManapandClaude Opus 5.5
51ca09287f
CI / test (push) Successful in 5m10s
The setup screen asks for up to two judge machines after the store's two, and server init takes --judge-node for each. Each is added as a node labelled llmgw.role: judge, and the judge's profile selects that label, with cores by init's rule over those machines. A judge beside a busy control plane refused 38 of 60 requests under load; alone, 2. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
997 lines
29 KiB
Go
997 lines
29 KiB
Go
package cli
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"io"
|
||
"net"
|
||
"os"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
tea "github.com/charmbracelet/bubbletea"
|
||
"github.com/charmbracelet/lipgloss"
|
||
|
||
joinpkg "gitea.mynodes.xyz/ilker/llmgw/internal/join"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/preflight"
|
||
)
|
||
|
||
// The setup screens: what a bare `llmgw` shows on a machine with nothing
|
||
// installed. One machine, or three for high availability — this one as the
|
||
// control plane and etcd's third vote, two more for the store under Patroni —
|
||
// with every machine checked before anything is installed on any of them.
|
||
//
|
||
// The other machines are reached with Go's own ssh rather than the system's,
|
||
// so a password can be typed here when there is no key; it is held in memory
|
||
// for the run and written nowhere. A sudo password, likewise, is used once to
|
||
// let the user sudo for the length of the install, and that is taken back
|
||
// when it finishes.
|
||
|
||
type setupPage int
|
||
|
||
const (
|
||
pWelcome setupPage = iota
|
||
// pChoices is what is installed, part by part, each with its settings.
|
||
pChoices
|
||
pMachines
|
||
// pWait shows the codes for machines joining rather than logged into,
|
||
// until they have all dialled in.
|
||
pWait
|
||
pCheck
|
||
pPlan
|
||
pRun
|
||
)
|
||
|
||
// How a database machine is logged into.
|
||
const (
|
||
authAgent = iota
|
||
authKey
|
||
authPassword
|
||
// authJoin is no login at all: the machine runs `llmgw join` with a code
|
||
// shown here and dials out to this one.
|
||
authJoin
|
||
authCount
|
||
)
|
||
|
||
var authNames = [authCount]string{"ssh agent", "key file", "password", "join code"}
|
||
|
||
// Fields of one database machine, in order.
|
||
const (
|
||
mfAddr = iota
|
||
mfUser
|
||
mfAuth
|
||
mfSecret
|
||
mfSudo
|
||
mfCount
|
||
)
|
||
|
||
type dbMachine struct {
|
||
addr, user, secret, sudo textField
|
||
auth int
|
||
// For a machine joining: its code, and it once it has dialled in.
|
||
code string
|
||
joined *joinpkg.Remote
|
||
arrive <-chan *joinpkg.Remote
|
||
}
|
||
|
||
// visible reports whether a field is shown, and so can be moved to: a joining
|
||
// machine is asked nothing but how it joins.
|
||
func (d dbMachine) visible(f int) bool {
|
||
switch {
|
||
case d.auth == authJoin:
|
||
return f == mfAuth
|
||
case f == mfSecret:
|
||
return d.auth != authAgent
|
||
}
|
||
return true
|
||
}
|
||
|
||
// dbConn is a database machine as it is reached, whichever way, or a judge
|
||
// machine when judge is set.
|
||
type dbConn struct {
|
||
remote onboard.Remote
|
||
addr string
|
||
ssh *onboard.GoSSH
|
||
joined *joinpkg.Remote
|
||
judge bool
|
||
target string
|
||
}
|
||
|
||
// The machines the page asks for: the store's two, then up to maxJudges for
|
||
// the guardrails' judges alone.
|
||
const maxJudges = 2
|
||
|
||
func newDBMachine(n int) dbMachine {
|
||
f := func(ph string, mask bool) textField {
|
||
t := newTextField()
|
||
t.Placeholder, t.Mask, t.CharLimit, t.Width = ph, mask, 200, 34
|
||
return t
|
||
}
|
||
m := dbMachine{
|
||
addr: f(fmt.Sprintf("10.0.0.%d or db%d.internal", 10+n, n), false),
|
||
user: f("ubuntu", false),
|
||
secret: f("", true),
|
||
sudo: f("only if sudo asks for one", true),
|
||
}
|
||
if os.Getenv("SSH_AUTH_SOCK") == "" {
|
||
m.auth = authKey
|
||
}
|
||
// As choosing the way to log in sets them: a key file's path is not a
|
||
// secret to hide.
|
||
m.secret.Mask = m.auth == authPassword
|
||
m.secret.Placeholder = map[int]string{authKey: "/root/.ssh/id_ed25519", authPassword: "the user's ssh password"}[m.auth]
|
||
return m
|
||
}
|
||
|
||
// newJudgeMachine is the fields of a machine for the judges.
|
||
func newJudgeMachine(n int) dbMachine {
|
||
d := newDBMachine(n + 2)
|
||
d.addr.Placeholder = fmt.Sprintf("10.0.0.%d or judge%d.internal", 20+n, n)
|
||
return d
|
||
}
|
||
|
||
// shown is how many machines the page asks for.
|
||
func (m setupModel) shown() int { return 2 + m.judges }
|
||
|
||
// machineTitle names machine i: the store's, or the judges'.
|
||
func machineTitle(i int) string {
|
||
if i < 2 {
|
||
return fmt.Sprintf("Database machine %d", i+1)
|
||
}
|
||
return fmt.Sprintf("Judge machine %d", i-1)
|
||
}
|
||
|
||
// field is the text field at a machine's position, nil for the selector.
|
||
func (d *dbMachine) field(i int) *textField {
|
||
switch i {
|
||
case mfAddr:
|
||
return &d.addr
|
||
case mfUser:
|
||
return &d.user
|
||
case mfSecret:
|
||
return &d.secret
|
||
case mfSudo:
|
||
return &d.sudo
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// Messages from the work behind the screens.
|
||
type (
|
||
checkDoneMsg struct{ rep preflight.Report }
|
||
joinedMsg struct {
|
||
i int
|
||
r *joinpkg.Remote
|
||
}
|
||
)
|
||
|
||
// progRef lets work started from Update send to the program it runs in.
|
||
type progRef struct{ p *tea.Program }
|
||
|
||
type setupModel struct {
|
||
ref *progRef
|
||
|
||
page setupPage
|
||
host string
|
||
ips []string
|
||
choice int // 0 one machine, 1 high availability
|
||
|
||
choices *installForm
|
||
|
||
db [2 + maxJudges]dbMachine
|
||
// judges is how many judge machines are asked for, none to maxJudges.
|
||
judges int
|
||
mfocus int // over the machines' fields, then the judges' choice, then the check button
|
||
|
||
conns []dbConn
|
||
listener *joinpkg.Listener
|
||
report preflight.Report
|
||
checking bool
|
||
|
||
in *serverInit
|
||
run initModel
|
||
finished bool
|
||
log *[]string
|
||
logMu *sync.Mutex
|
||
|
||
spin spinnerModel
|
||
formErr string
|
||
width int
|
||
height int
|
||
// scroll is how far the panel is scrolled, reset with each page.
|
||
scroll int
|
||
quit bool
|
||
// single is the choice of one machine: the screen closes and init's own
|
||
// form takes over.
|
||
single bool
|
||
}
|
||
|
||
func newSetupModel(host string, ips []string) setupModel {
|
||
in := newDefaultServerInit()
|
||
// The control plane of a replicated setup is not where models run;
|
||
// nodes are added once it is up. Either can still be chosen.
|
||
in.noNode, in.noModel = true, true
|
||
in.standbys = true
|
||
// Two judges, on two of the three machines' nodes.
|
||
in.judge.Replicas = 2
|
||
return setupModel{
|
||
ref: &progRef{}, host: host, ips: ips, in: &in, choices: newInstallForm(&in, host, ips, true),
|
||
db: [2 + maxJudges]dbMachine{newDBMachine(1), newDBMachine(2), newJudgeMachine(1), newJudgeMachine(2)},
|
||
spin: newSpinner(), width: 100, height: 30,
|
||
log: &[]string{}, logMu: &sync.Mutex{},
|
||
}
|
||
}
|
||
|
||
func (m setupModel) Init() tea.Cmd { return m.spin.Tick }
|
||
|
||
func (m setupModel) thisAddr() string {
|
||
if len(m.ips) > 0 {
|
||
return m.ips[0]
|
||
}
|
||
return m.host
|
||
}
|
||
|
||
func (m setupModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||
switch msg := msg.(type) {
|
||
case tea.WindowSizeMsg:
|
||
m.width, m.height = msg.Width, msg.Height
|
||
m.choices.height = msg.Height
|
||
m.run.width = msg.Width - navWidth - 6
|
||
case spinTickMsg:
|
||
var cmd tea.Cmd
|
||
m.spin, cmd = m.spin.Update(msg)
|
||
m.run.spin = m.spin
|
||
return m, cmd
|
||
case checkDoneMsg:
|
||
m.checking, m.report = false, msg.rep
|
||
case joinedMsg:
|
||
m.db[msg.i].joined = msg.r
|
||
if m.page == pWait && m.allJoined() {
|
||
m.page = pCheck
|
||
cmd := m.startCheck()
|
||
return m, cmd
|
||
}
|
||
case initLineMsg:
|
||
m.run.addLine(string(msg))
|
||
case initDoneMsg:
|
||
m.run.done, m.run.err, m.finished = true, msg.err, true
|
||
case tea.KeyMsg:
|
||
if msg.Type == tea.KeyCtrlC {
|
||
m.quit = true
|
||
return m, tea.Quit
|
||
}
|
||
before := m.page
|
||
next, cmd := m.key(msg)
|
||
if nm := next.(setupModel); nm.page != before {
|
||
// A new page starts at its top.
|
||
nm.scroll = 0
|
||
return nm, cmd
|
||
}
|
||
return next, cmd
|
||
}
|
||
return m, nil
|
||
}
|
||
|
||
func (m setupModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||
// The choices' own dialogs take their keys, Esc among them.
|
||
if m.page == pChoices && (m.choices.dialog != nil || m.choices.err != "") {
|
||
m.choices.key(k, m.in)
|
||
return m, nil
|
||
}
|
||
// A dialog open takes the next key, whatever it is, to close it.
|
||
if m.formErr != "" {
|
||
m.formErr = ""
|
||
return m, nil
|
||
}
|
||
switch k.Type {
|
||
case tea.KeyPgDown:
|
||
m.scroll += max(1, m.height-8)
|
||
return m, nil
|
||
case tea.KeyPgUp:
|
||
m.scroll = max(0, m.scroll-max(1, m.height-8))
|
||
return m, nil
|
||
}
|
||
if k.Type == tea.KeyEsc && m.page > pWelcome && m.page < pRun && !m.checking {
|
||
m.page--
|
||
if m.page == pWait {
|
||
m.page = pMachines
|
||
}
|
||
m.formErr = ""
|
||
return m, nil
|
||
}
|
||
switch m.page {
|
||
case pWelcome:
|
||
switch k.String() {
|
||
case "up", "down", "tab", "k", "j":
|
||
m.choice = 1 - m.choice
|
||
case "enter":
|
||
if m.choice == 0 {
|
||
m.single = true
|
||
return m, tea.Quit
|
||
}
|
||
m.page = pChoices
|
||
}
|
||
case pChoices:
|
||
if m.choices.key(k, m.in) {
|
||
m.page, m.mfocus = pMachines, 0
|
||
m.focusMachine()
|
||
}
|
||
case pMachines:
|
||
return m.machinesKey(k)
|
||
case pCheck:
|
||
switch k.String() {
|
||
case "r":
|
||
if !m.checking {
|
||
cmd := m.startCheck()
|
||
return m, cmd
|
||
}
|
||
case "enter":
|
||
if !m.checking && m.report.Worst() < preflight.Fail {
|
||
m.page = pPlan
|
||
}
|
||
}
|
||
case pPlan:
|
||
if k.String() == "enter" {
|
||
m.page = pRun
|
||
cmd := m.startRun()
|
||
return m, cmd
|
||
}
|
||
case pRun:
|
||
if m.finished && (k.String() == "q" || k.String() == "enter") {
|
||
return m, tea.Quit
|
||
}
|
||
}
|
||
return m, nil
|
||
}
|
||
|
||
func (m *setupModel) focusMachine() {
|
||
for i := range m.db {
|
||
for f := 0; f < mfCount; f++ {
|
||
if t := m.db[i].field(f); t != nil {
|
||
t.Blur()
|
||
}
|
||
}
|
||
}
|
||
if m.mfocus < m.shown()*mfCount {
|
||
if t := m.db[m.mfocus/mfCount].field(m.mfocus % mfCount); t != nil {
|
||
t.Focus()
|
||
}
|
||
}
|
||
}
|
||
|
||
func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||
sel := m.shown() * mfCount
|
||
n := sel + 2
|
||
switch k.Type {
|
||
case tea.KeyTab, tea.KeyDown:
|
||
m.step(1, n)
|
||
return m, nil
|
||
case tea.KeyShiftTab, tea.KeyUp:
|
||
m.step(-1, n)
|
||
return m, nil
|
||
case tea.KeyEnter:
|
||
if m.mfocus <= sel {
|
||
m.step(1, n)
|
||
return m, nil
|
||
}
|
||
if err := m.validateMachines(); err != "" {
|
||
m.formErr = err
|
||
return m, nil
|
||
}
|
||
m.formErr = ""
|
||
if cmd, waiting := m.startJoins(); waiting {
|
||
m.page = pWait
|
||
return m, cmd
|
||
}
|
||
m.page = pCheck
|
||
cmd := m.startCheck()
|
||
return m, cmd
|
||
}
|
||
if m.mfocus == sel {
|
||
switch k.String() {
|
||
case "left", "h":
|
||
m.judges = (m.judges + maxJudges) % (maxJudges + 1)
|
||
case "right", "l", " ":
|
||
m.judges = (m.judges + 1) % (maxJudges + 1)
|
||
}
|
||
m.mfocus = m.shown() * mfCount
|
||
return m, nil
|
||
}
|
||
if m.mfocus > sel {
|
||
return m, nil
|
||
}
|
||
d := &m.db[m.mfocus/mfCount]
|
||
if m.mfocus%mfCount == mfAuth {
|
||
switch k.String() {
|
||
case "left", "h":
|
||
d.auth = (d.auth + authCount - 1) % authCount
|
||
case "right", "l", " ":
|
||
d.auth = (d.auth + 1) % authCount
|
||
}
|
||
if d.auth != authJoin && d.joined != nil {
|
||
_ = d.joined.Close()
|
||
d.joined, d.code, d.arrive = nil, "", nil
|
||
}
|
||
d.secret.SetValue("")
|
||
d.secret.Mask = d.auth == authPassword
|
||
d.secret.Placeholder = map[int]string{authKey: "/root/.ssh/id_ed25519", authPassword: "the user's ssh password"}[d.auth]
|
||
return m, nil
|
||
}
|
||
var cmd tea.Cmd
|
||
t := d.field(m.mfocus % mfCount)
|
||
*t, cmd = t.Update(k)
|
||
return m, cmd
|
||
}
|
||
|
||
// step moves the focus over the fields that are shown.
|
||
func (m *setupModel) step(d, n int) {
|
||
for range n {
|
||
m.mfocus = (m.mfocus + d + n) % n
|
||
if m.mfocus >= m.shown()*mfCount || m.db[m.mfocus/mfCount].visible(m.mfocus%mfCount) {
|
||
break
|
||
}
|
||
}
|
||
m.focusMachine()
|
||
}
|
||
|
||
func (m setupModel) allJoined() bool {
|
||
for _, d := range m.db[:m.shown()] {
|
||
if d.auth == authJoin && d.joined == nil {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// startJoins listens for the machines that join with a code, makes their
|
||
// codes, and waits for them; it says whether there is anything to wait for.
|
||
func (m *setupModel) startJoins() (tea.Cmd, bool) {
|
||
var cmds []tea.Cmd
|
||
for i := range m.db[:m.shown()] {
|
||
d := &m.db[i]
|
||
if d.auth != authJoin || d.joined != nil {
|
||
continue
|
||
}
|
||
if m.listener == nil {
|
||
// On the address the codes name, not every interface: a code is
|
||
// a credential, and the port it is spent on need not face more
|
||
// than the network the machines join from.
|
||
addr := net.JoinHostPort(m.thisAddr(), fmt.Sprint(joinpkg.DefaultPort))
|
||
l, err := joinpkg.Listen(addr, addr, time.Hour)
|
||
if err != nil {
|
||
m.formErr = err.Error()
|
||
return nil, false
|
||
}
|
||
m.listener = l
|
||
}
|
||
if d.arrive == nil {
|
||
role := "database"
|
||
if i >= 2 {
|
||
role = "judge"
|
||
}
|
||
c, ch := m.listener.Code(role)
|
||
d.code, d.arrive = c.String(), ch
|
||
}
|
||
ch, i := d.arrive, i
|
||
cmds = append(cmds, func() tea.Msg { return joinedMsg{i, <-ch} })
|
||
}
|
||
return tea.Batch(cmds...), len(cmds) > 0
|
||
}
|
||
|
||
func (m setupModel) validateMachines() string {
|
||
seen := map[string]bool{m.thisAddr(): true, m.host: true}
|
||
for i, d := range m.db[:m.shown()] {
|
||
if d.auth == authJoin {
|
||
continue
|
||
}
|
||
addr, title := strings.TrimSpace(d.addr.Value()), strings.ToLower(machineTitle(i)[:1])+machineTitle(i)[1:]
|
||
switch {
|
||
case addr == "":
|
||
return title + " has no address"
|
||
case seen[addr]:
|
||
return fmt.Sprintf("%s is given twice, or is this machine: each must be a different machine", addr)
|
||
case strings.TrimSpace(d.user.Value()) == "":
|
||
return title + " has no user to log in as"
|
||
case d.auth != authAgent && d.secret.Value() == "":
|
||
return fmt.Sprintf("%s needs its %s", title, authNames[d.auth])
|
||
}
|
||
seen[addr] = true
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// startCheck connects to the database machines and runs the checks on all
|
||
// three, this one included.
|
||
func (m *setupModel) startCheck() tea.Cmd {
|
||
for _, c := range m.conns {
|
||
if c.ssh != nil {
|
||
_ = c.ssh.Close()
|
||
}
|
||
}
|
||
m.conns = nil
|
||
machines := []preflight.Machine{{
|
||
Name: m.host + " (this machine)", Addr: m.thisAddr(), Local: true, Remote: pgha.Local{},
|
||
Roles: []preflight.Role{preflight.RoleControl, preflight.RoleWitness},
|
||
}}
|
||
for i, d := range m.db[:m.shown()] {
|
||
roles, judge := m.dbRoles(), i >= 2
|
||
if judge {
|
||
roles = []preflight.Role{preflight.RoleNode}
|
||
}
|
||
if d.auth == authJoin {
|
||
m.conns = append(m.conns, dbConn{remote: d.joined, addr: d.joined.Addr, joined: d.joined,
|
||
judge: judge, target: "root@" + d.joined.Addr})
|
||
machines = append(machines, preflight.Machine{Name: d.joined.Hostname + " (" + d.joined.Addr + ", joined)",
|
||
Addr: d.joined.Addr, Roles: roles, Remote: d.joined})
|
||
continue
|
||
}
|
||
g := &onboard.GoSSH{
|
||
Addr: strings.TrimSpace(d.addr.Value()), User: strings.TrimSpace(d.user.Value()),
|
||
UseAgent: d.auth == authAgent, SudoPassword: d.sudo.Value(),
|
||
}
|
||
switch d.auth {
|
||
case authKey:
|
||
g.KeyFile = strings.TrimSpace(d.secret.Value())
|
||
case authPassword:
|
||
g.Password = d.secret.Value()
|
||
}
|
||
m.conns = append(m.conns, dbConn{remote: g, addr: g.Addr, ssh: g, judge: judge, target: g.User + "@" + g.Addr})
|
||
machines = append(machines, preflight.Machine{Name: g.Addr, Addr: g.Addr, Roles: roles, Remote: g})
|
||
}
|
||
m.checking = true
|
||
m.report = preflight.Report{}
|
||
conns := m.conns
|
||
return func() tea.Msg {
|
||
rep := preflight.Run(context.Background(), machines)
|
||
// A sudo that asks for a password is fine with one given: the
|
||
// install grants itself sudo with it.
|
||
for i := range rep.Findings {
|
||
f := &rep.Findings[i]
|
||
if f.Check == "sudo" && f.Level == preflight.Warn {
|
||
for _, c := range conns {
|
||
g := c.ssh
|
||
switch {
|
||
case g == nil || g.Addr != f.Machine:
|
||
case g.SudoPassword == "":
|
||
f.Level = preflight.Fail
|
||
f.Fix = "give this user's sudo password on the previous screen"
|
||
default:
|
||
f.Level = preflight.OK
|
||
f.Detail = "with the password given"
|
||
f.Fix = ""
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return checkDoneMsg{rep}
|
||
}
|
||
}
|
||
|
||
// dbRoles are what the other two machines are to be: the store, and a
|
||
// control plane too when they stand by.
|
||
func (m setupModel) dbRoles() []preflight.Role {
|
||
if m.in.standbys {
|
||
return []preflight.Role{preflight.RoleDatabase, preflight.RoleControl}
|
||
}
|
||
return []preflight.Role{preflight.RoleDatabase}
|
||
}
|
||
|
||
// targets are the database machines as user@address, for the command line.
|
||
func (m setupModel) targets() []string {
|
||
var out []string
|
||
for _, d := range m.db[:2] {
|
||
if d.joined != nil {
|
||
out = append(out, "root@"+d.joined.Addr)
|
||
continue
|
||
}
|
||
out = append(out, strings.TrimSpace(d.user.Value())+"@"+strings.TrimSpace(d.addr.Value()))
|
||
}
|
||
return out
|
||
}
|
||
|
||
// judgeTargets are the judge machines as user@address.
|
||
func (m setupModel) judgeTargets() []judgeMachine {
|
||
var out []judgeMachine
|
||
for _, d := range m.db[2:m.shown()] {
|
||
if d.joined != nil {
|
||
out = append(out, judgeMachine{target: "root@" + d.joined.Addr})
|
||
continue
|
||
}
|
||
out = append(out, judgeMachine{target: strings.TrimSpace(d.user.Value()) + "@" + strings.TrimSpace(d.addr.Value())})
|
||
}
|
||
return out
|
||
}
|
||
|
||
// command is what this setup is on the command line, for doing it again
|
||
// from a script — with keys in place, since a password cannot be given there.
|
||
func (m setupModel) command() string {
|
||
in := *m.in
|
||
in.patroni = m.targets()
|
||
in.judgeMachines = m.judgeTargets()
|
||
return strings.Join(append([]string{"sudo --preserve-env=SSH_AUTH_SOCK llmgw server init"}, in.initFlags()...), " \\\n ")
|
||
}
|
||
|
||
// startRun installs: sudo granted where it asks for a password, the control
|
||
// plane here, the store on the other two, then sudo taken back.
|
||
func (m *setupModel) startRun() tea.Cmd {
|
||
in := m.in
|
||
in.patroni = m.targets()
|
||
in.judgeMachines = nil
|
||
m.run = newInitModel(in, nil, false)
|
||
m.run.running, m.run.width, m.run.spin = true, m.width, m.spin
|
||
ref, conns, logp, mu, listener := m.ref, m.conns, m.log, m.logMu, m.listener
|
||
return func() tea.Msg {
|
||
send := func(l string) {
|
||
mu.Lock()
|
||
*logp = append(*logp, l)
|
||
mu.Unlock()
|
||
ref.p.Send(initLineMsg(l))
|
||
}
|
||
in.out = &lineWriter{send: send}
|
||
err := func() error {
|
||
ctx := context.Background()
|
||
for _, c := range conns {
|
||
if c.ssh != nil {
|
||
fmt.Fprintf(in.out, " .. sudo on %s\n", c.addr)
|
||
if err := c.ssh.PrepareSudo(ctx); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
if c.judge {
|
||
// Added as a node once the control plane is up.
|
||
in.judgeMachines = append(in.judgeMachines, judgeMachine{target: c.target, remote: c.remote})
|
||
continue
|
||
}
|
||
out, err := c.remote.Run(ctx, "hostname", nil)
|
||
if err != nil {
|
||
return fmt.Errorf("%s: %v", c.addr, err)
|
||
}
|
||
ip, err := ipv4Of(c.addr)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
in.patroniNodes = append(in.patroniNodes, pgha.Node{
|
||
Member: pgha.Member{Name: strings.TrimSpace(string(out)), IP: ip}, Remote: c.remote})
|
||
}
|
||
return in.run(ctx)
|
||
}()
|
||
// Whatever happened: a sudo grant taken back, a joined machine's
|
||
// `llmgw join` told it is done, the port for joining closed.
|
||
for _, c := range conns {
|
||
if c.ssh != nil {
|
||
_ = c.ssh.Close()
|
||
}
|
||
if c.joined != nil {
|
||
_ = c.joined.Close()
|
||
}
|
||
}
|
||
if listener != nil {
|
||
_ = listener.Close()
|
||
}
|
||
return initDoneMsg{err}
|
||
}
|
||
}
|
||
|
||
// setupSteps are the sidebar's steps, by page.
|
||
var setupSteps = []struct {
|
||
label string
|
||
page setupPage
|
||
}{
|
||
{"Start", pWelcome}, {"Choices", pChoices}, {"Machines", pMachines},
|
||
{"Checks", pCheck}, {"Plan", pPlan}, {"Install", pRun},
|
||
}
|
||
|
||
func (m setupModel) View() string {
|
||
at := m.page
|
||
if at == pWait {
|
||
at = pMachines
|
||
}
|
||
var nav []navItem
|
||
for _, st := range setupSteps {
|
||
if m.choice == 0 && st.page > pWelcome {
|
||
break
|
||
}
|
||
state := "todo"
|
||
switch {
|
||
case st.page < at || (st.page == pRun && m.finished && m.run.err == nil):
|
||
state = "done"
|
||
case st.page == at:
|
||
state = "current"
|
||
}
|
||
nav = append(nav, navItem{Label: st.label, State: state})
|
||
}
|
||
title, body, keys := m.pageView()
|
||
f := frame{
|
||
Width: m.width, Height: m.height,
|
||
Title: "llmgw setup", Where: m.host + " · " + m.thisAddr(),
|
||
NavTitle: "Steps", Nav: nav,
|
||
PanelTitle: title, Body: body, Scroll: m.scroll, Keys: keys,
|
||
}
|
||
if m.formErr != "" {
|
||
f.DialogTitle, f.Dialog = "Not yet", m.formErr+"\n\n"+sDim.Render("any key to go back")
|
||
}
|
||
if m.page == pChoices {
|
||
f.DialogTitle, f.Dialog = m.choices.overlay()
|
||
}
|
||
return f.render()
|
||
}
|
||
|
||
// pageView is the panel for the page: its title, what is in it, and its keys.
|
||
func (m setupModel) pageView() (string, string, string) {
|
||
switch m.page {
|
||
case pWelcome:
|
||
return "Start", m.welcomeView(), "↑↓ choose · Enter go on · Ctrl+C leave"
|
||
case pChoices:
|
||
keys := m.choices.keys()
|
||
if m.choices.dialog == nil && m.choices.err == "" {
|
||
keys = strings.Replace(keys, " · Ctrl+C", " · Esc back · Ctrl+C", 1)
|
||
}
|
||
return "What to install", m.choices.view(m.width - navWidth - 6), keys
|
||
case pMachines:
|
||
return "Database machines", m.machinesView(), "Tab/↑↓ move · ←→ how to log in · Enter next · Esc back · PgUp/PgDn scroll"
|
||
case pWait:
|
||
return "Waiting for machines to join", m.waitView(), "Esc back · PgUp/PgDn scroll · Ctrl+C leave"
|
||
case pCheck:
|
||
keys := "R check again · Esc back · PgUp/PgDn scroll"
|
||
if !m.checking && m.report.Worst() < preflight.Fail {
|
||
keys = "Enter see the plan · " + keys
|
||
}
|
||
return "Checks", m.checkView(), keys
|
||
case pPlan:
|
||
return "Plan", m.planView(), "Enter install · Esc back · PgUp/PgDn scroll · Ctrl+C leave"
|
||
}
|
||
v := m.run.View()
|
||
if i := strings.Index(v, "\n\n"); i >= 0 {
|
||
v = v[i+2:] // init's own title; the frame has one
|
||
}
|
||
keys := "Ctrl+C stop"
|
||
if m.finished {
|
||
keys = "Enter or Q leave · PgUp/PgDn scroll"
|
||
if m.run.err == nil {
|
||
v += "\n" + sDim.Render("The same, from a script, with ssh keys in place:") + "\n" + m.command() + "\n"
|
||
}
|
||
}
|
||
return "Install", v, keys
|
||
}
|
||
|
||
func (m setupModel) welcomeView() string {
|
||
var b strings.Builder
|
||
fmt.Fprintf(&b, "\n Nothing of llmgw is installed on %s.\n\n", lipgloss.NewStyle().Bold(true).Render(m.host))
|
||
opt := func(i int, title, detail string) {
|
||
mark := "( )"
|
||
if m.choice == i {
|
||
mark = "(" + sOK.Render("•") + ")"
|
||
title = fSelected.Render(title)
|
||
}
|
||
fmt.Fprintf(&b, " %s %s\n %s\n\n", mark, title, sDim.Render(detail))
|
||
}
|
||
opt(0, "One machine", "This machine runs everything: control plane, model mirror,\n and models if it has room.")
|
||
opt(1, "High availability — three machines",
|
||
"Each of the three can run the gateway; two keep the store, replicated,\n"+
|
||
" and etcd votes on all three, so losing any one machine loses nothing.")
|
||
return b.String()
|
||
}
|
||
|
||
// button is a [ Label ] that shows when it has the focus.
|
||
func button(focused bool, label string) string {
|
||
if focused {
|
||
return " " + fBar.Render(" "+label+" ")
|
||
}
|
||
return " " + lipgloss.NewStyle().Bold(true).Render("[ "+label+" ]")
|
||
}
|
||
|
||
func (m setupModel) machinesView() string {
|
||
var b strings.Builder
|
||
b.WriteString("\n The two machines that keep the store: Ubuntu 22.04/24.04 or Debian 12.\n" +
|
||
" PostgreSQL, Patroni and etcd are installed on them.\n\n")
|
||
for i, d := range m.db[:m.shown()] {
|
||
base := i * mfCount
|
||
if i == 2 {
|
||
b.WriteString(" " + sDim.Render("Machines for the guardrails' judges alone: each becomes a node that runs only\n"+
|
||
" the judges, so a busy control plane cannot leave a request waiting for a judgment.") + "\n\n")
|
||
}
|
||
// Padded before it is coloured: the colour's escape codes count
|
||
// toward a width and threw the focused row out of line.
|
||
label := func(f int, s string) string {
|
||
s = fmt.Sprintf("%-15s", s)
|
||
if m.mfocus == base+f {
|
||
return fSelected.Render("› " + s)
|
||
}
|
||
return " " + s
|
||
}
|
||
fmt.Fprintf(&b, " %s\n", lipgloss.NewStyle().Bold(true).Render(machineTitle(i)))
|
||
if d.visible(mfAddr) {
|
||
fmt.Fprintf(&b, " %s %s\n", label(mfAddr, "Address"), d.addr.View())
|
||
fmt.Fprintf(&b, " %s %s\n", label(mfUser, "User"), d.user.View())
|
||
}
|
||
var auth []string
|
||
for a := 0; a < authCount; a++ {
|
||
if a == d.auth {
|
||
auth = append(auth, "("+sOK.Render("•")+") "+authNames[a])
|
||
} else {
|
||
auth = append(auth, "( ) "+sDim.Render(authNames[a]))
|
||
}
|
||
}
|
||
fmt.Fprintf(&b, " %s %s\n", label(mfAuth, "Log in with"), strings.Join(auth, " "))
|
||
if d.visible(mfSecret) {
|
||
fmt.Fprintf(&b, " %s %s\n", label(mfSecret, strings.ToUpper(authNames[d.auth][:1])+authNames[d.auth][1:]), d.secret.View())
|
||
}
|
||
if d.visible(mfSudo) {
|
||
fmt.Fprintf(&b, " %s %s\n", label(mfSudo, "Sudo password"), d.sudo.View())
|
||
}
|
||
if d.auth == authJoin {
|
||
b.WriteString(" " + sDim.Render("Nothing logs into it: it runs llmgw join with a code shown next.") + "\n")
|
||
}
|
||
b.WriteString("\n")
|
||
}
|
||
sel := m.shown() * mfCount
|
||
var opts []string
|
||
for n, name := range []string{"none", "one", "two"} {
|
||
if n == m.judges {
|
||
opts = append(opts, "("+sOK.Render("•")+") "+name)
|
||
} else {
|
||
opts = append(opts, "( ) "+sDim.Render(name))
|
||
}
|
||
}
|
||
judgeLabel := fmt.Sprintf(" %-15s", "Judge machines")
|
||
if m.mfocus == sel {
|
||
judgeLabel = fSelected.Render(fmt.Sprintf("› %-15s", "Judge machines"))
|
||
}
|
||
fmt.Fprintf(&b, " %s %s\n", judgeLabel, strings.Join(opts, " "))
|
||
b.WriteString(" " + sDim.Render("Without them the judges run on whichever node has room, added later.") + "\n\n")
|
||
b.WriteString(button(m.mfocus == sel+1, fmt.Sprintf("Check the %d machines", m.shown()+1)) + "\n\n")
|
||
b.WriteString(" " + sDim.Render("Passwords are kept in memory only, for this setup.") + "\n")
|
||
return b.String()
|
||
}
|
||
|
||
func (m setupModel) waitView() string {
|
||
var b strings.Builder
|
||
fmt.Fprintf(&b, "\n On each machine below, as root. It dials this machine on port %d;\n nothing dials in. Each code is for one machine and good for an hour.\n\n", joinpkg.DefaultPort)
|
||
for i, d := range m.db[:m.shown()] {
|
||
if d.auth != authJoin {
|
||
continue
|
||
}
|
||
title := lipgloss.NewStyle().Bold(true).Render(machineTitle(i))
|
||
if d.joined != nil {
|
||
fmt.Fprintf(&b, " %s %s joined as %s (%s)\n\n", title, sOK.Render("✓"), d.joined.Hostname, d.joined.Addr)
|
||
continue
|
||
}
|
||
fmt.Fprintf(&b, " %s %s waiting\n", title, m.spin.View())
|
||
b.WriteString(" " + sDim.Render("without llmgw yet:") + "\n")
|
||
for _, l := range joinInstall() {
|
||
b.WriteString(" " + strings.ReplaceAll(l, "\n", "\n ") + "\n")
|
||
}
|
||
b.WriteString(" " + sDim.Render("then:") + "\n sudo llmgw join " + d.code + "\n\n")
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
func (m setupModel) checkView() string {
|
||
var b strings.Builder
|
||
b.WriteString("\n")
|
||
if m.checking {
|
||
b.WriteString(" " + m.spin.View() + " checking the machines…\n")
|
||
return b.String()
|
||
}
|
||
switch m.report.Worst() {
|
||
case preflight.Fail:
|
||
b.WriteString(" " + sBad.Render("✗ Fix what is marked ✗, then check again (R).") + "\n")
|
||
case preflight.Warn:
|
||
b.WriteString(" " + sWarn.Render("! It can go ahead; what is marked ! is worth a look.") + "\n")
|
||
default:
|
||
b.WriteString(" " + sOK.Render("✓ All are ready.") + "\n")
|
||
}
|
||
machine := ""
|
||
for _, f := range m.report.Findings {
|
||
if f.Machine != machine {
|
||
machine = f.Machine
|
||
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render(machine) + "\n")
|
||
}
|
||
mark := sOK.Render("✓")
|
||
switch f.Level {
|
||
case preflight.Warn:
|
||
mark = sWarn.Render("!")
|
||
case preflight.Fail:
|
||
mark = sBad.Render("✗")
|
||
}
|
||
fmt.Fprintf(&b, " %s %-13s %s\n", mark, f.Check, f.Detail)
|
||
if f.Level != preflight.OK && f.Fix != "" {
|
||
b.WriteString(" " + sDim.Render(f.Fix) + "\n")
|
||
}
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
func (m setupModel) planView() string {
|
||
var b strings.Builder
|
||
b.WriteString(m.choices.planView(m.width - navWidth - 6))
|
||
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("The machines") + "\n")
|
||
here := "control plane, leading at first; model mirror; etcd"
|
||
there := "PostgreSQL under Patroni, etcd"
|
||
if m.in.standbys {
|
||
there = "control plane, standing by; " + there
|
||
}
|
||
if m.in.vip != "" {
|
||
here += "; keepalived"
|
||
there += "; keepalived"
|
||
}
|
||
fmt.Fprintf(&b, " %s %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(m.host), sDim.Render(m.thisAddr()), here)
|
||
granted := false
|
||
for i, d := range m.db[:m.shown()] {
|
||
addr := strings.TrimSpace(d.addr.Value())
|
||
if d.joined != nil {
|
||
addr = d.joined.Hostname + " " + sDim.Render(d.joined.Addr+", joined")
|
||
}
|
||
role := there
|
||
if i >= 2 {
|
||
role = "a node for the guardrails' judges alone"
|
||
}
|
||
fmt.Fprintf(&b, " %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(addr), role)
|
||
granted = granted || d.sudo.Value() != ""
|
||
}
|
||
b.WriteString("\n The store is replicated from the first to the second; if the primary is lost\n" +
|
||
" the standby takes over in about thirty seconds, and the control plane follows.\n")
|
||
if granted {
|
||
b.WriteString(" " + sDim.Render("Where a sudo password was given, the user may sudo without it until the\n install finishes, and not after.") + "\n")
|
||
}
|
||
b.WriteString("\n " + sDim.Render("The same from a script, with ssh keys in place:") + "\n")
|
||
for _, l := range strings.Split(m.command(), "\n") {
|
||
b.WriteString(" " + l + "\n")
|
||
}
|
||
for _, d := range m.db[:m.shown()] {
|
||
if d.joined != nil {
|
||
b.WriteString(" " + sDim.Render("(a machine that joined with a code needs ssh for that)") + "\n")
|
||
break
|
||
}
|
||
}
|
||
b.WriteString("\n" + button(true, "Enter to install") + "\n")
|
||
return b.String()
|
||
}
|
||
|
||
// runSetupTUI is the setup screens, until they finish or hand over to init's
|
||
// own form for one machine.
|
||
func runSetupTUI(ctx context.Context, out io.Writer) error {
|
||
host, _ := os.Hostname()
|
||
var ips []string
|
||
for _, ip := range localAddrs() {
|
||
if ip.To4() != nil {
|
||
ips = append(ips, ip.String())
|
||
}
|
||
}
|
||
m := newSetupModel(host, ips)
|
||
p := tea.NewProgram(m, tea.WithOutput(out), tea.WithAltScreen())
|
||
m.ref.p = p
|
||
final, err := p.Run()
|
||
if err != nil {
|
||
return err
|
||
}
|
||
fm := final.(setupModel)
|
||
if fm.single {
|
||
in := newDefaultServerInit()
|
||
in.out = out
|
||
return in.runInitTUI(ctx, out)
|
||
}
|
||
if fm.page < pRun {
|
||
return nil
|
||
}
|
||
fmt.Fprintln(out)
|
||
if fm.run.err != nil {
|
||
fm.logMu.Lock()
|
||
fmt.Fprintln(out, strings.Join(*fm.log, "\n"))
|
||
fm.logMu.Unlock()
|
||
return fm.run.err
|
||
}
|
||
if !fm.finished {
|
||
return fmt.Errorf("left while installing; running llmgw again finishes what was started")
|
||
}
|
||
if fm.run.password != "" {
|
||
fmt.Fprintf(out, "Administrator %s, password %s (shown once)\n\n", fm.in.admin, fm.run.password)
|
||
}
|
||
fmt.Fprintln(out, "Next: add nodes from the console — run llmgw again — or with")
|
||
fmt.Fprintln(out, " sudo llmgw node add <user>@<address>")
|
||
return nil
|
||
}
|