CI / test (push) Successful in 8m42s
A control plane that is also a node, serving the starter model and two judges, needs about 10.5 GB: 3.5 for itself, 2 for the starter, 2 for each judge, and one more because the scheduler counts whole gigabytes. On 8 GB the install did every step and then had nowhere to run the starter. The setup's checks now fail this machine's memory and name what it would serve, and server init refuses before it installs anything, naming the flags that leave those models off it. quick-install.md said 8 GB; it says 12. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
999 lines
29 KiB
Go
999 lines
29 KiB
Go
package cli
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"io"
|
||
"net"
|
||
"os"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
tea "github.com/charmbracelet/bubbletea"
|
||
"github.com/charmbracelet/lipgloss"
|
||
|
||
joinpkg "gitea.mynodes.xyz/ilker/llmgw/internal/join"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/preflight"
|
||
)
|
||
|
||
// The setup screens: what a bare `llmgw` shows on a machine with nothing
|
||
// installed. One machine, or three for high availability — this one as the
|
||
// control plane and etcd's third vote, two more for the store under Patroni —
|
||
// with every machine checked before anything is installed on any of them.
|
||
//
|
||
// The other machines are reached with Go's own ssh rather than the system's,
|
||
// so a password can be typed here when there is no key; it is held in memory
|
||
// for the run and written nowhere. A sudo password, likewise, is used once to
|
||
// let the user sudo for the length of the install, and that is taken back
|
||
// when it finishes.
|
||
|
||
type setupPage int
|
||
|
||
const (
|
||
pWelcome setupPage = iota
|
||
// pChoices is what is installed, part by part, each with its settings.
|
||
pChoices
|
||
pMachines
|
||
// pWait shows the codes for machines joining rather than logged into,
|
||
// until they have all dialled in.
|
||
pWait
|
||
pCheck
|
||
pPlan
|
||
pRun
|
||
)
|
||
|
||
// How a database machine is logged into.
|
||
const (
|
||
authAgent = iota
|
||
authKey
|
||
authPassword
|
||
// authJoin is no login at all: the machine runs `llmgw join` with a code
|
||
// shown here and dials out to this one.
|
||
authJoin
|
||
authCount
|
||
)
|
||
|
||
var authNames = [authCount]string{"ssh agent", "key file", "password", "join code"}
|
||
|
||
// Fields of one database machine, in order.
|
||
const (
|
||
mfAddr = iota
|
||
mfUser
|
||
mfAuth
|
||
mfSecret
|
||
mfSudo
|
||
mfCount
|
||
)
|
||
|
||
type dbMachine struct {
|
||
addr, user, secret, sudo textField
|
||
auth int
|
||
// For a machine joining: its code, and it once it has dialled in.
|
||
code string
|
||
joined *joinpkg.Remote
|
||
arrive <-chan *joinpkg.Remote
|
||
}
|
||
|
||
// visible reports whether a field is shown, and so can be moved to: a joining
|
||
// machine is asked nothing but how it joins.
|
||
func (d dbMachine) visible(f int) bool {
|
||
switch {
|
||
case d.auth == authJoin:
|
||
return f == mfAuth
|
||
case f == mfSecret:
|
||
return d.auth != authAgent
|
||
}
|
||
return true
|
||
}
|
||
|
||
// dbConn is a database machine as it is reached, whichever way, or a judge
|
||
// machine when judge is set.
|
||
type dbConn struct {
|
||
remote onboard.Remote
|
||
addr string
|
||
ssh *onboard.GoSSH
|
||
joined *joinpkg.Remote
|
||
judge bool
|
||
target string
|
||
}
|
||
|
||
// The machines the page asks for: the store's two, then up to maxJudges for
|
||
// the guardrails' judges alone.
|
||
const maxJudges = 2
|
||
|
||
func newDBMachine(n int) dbMachine {
|
||
f := func(ph string, mask bool) textField {
|
||
t := newTextField()
|
||
t.Placeholder, t.Mask, t.CharLimit, t.Width = ph, mask, 200, 34
|
||
return t
|
||
}
|
||
m := dbMachine{
|
||
addr: f(fmt.Sprintf("10.0.0.%d or db%d.internal", 10+n, n), false),
|
||
user: f("ubuntu", false),
|
||
secret: f("", true),
|
||
sudo: f("only if sudo asks for one", true),
|
||
}
|
||
if os.Getenv("SSH_AUTH_SOCK") == "" {
|
||
m.auth = authKey
|
||
}
|
||
// As choosing the way to log in sets them: a key file's path is not a
|
||
// secret to hide.
|
||
m.secret.Mask = m.auth == authPassword
|
||
m.secret.Placeholder = map[int]string{authKey: "/root/.ssh/id_ed25519", authPassword: "the user's ssh password"}[m.auth]
|
||
return m
|
||
}
|
||
|
||
// newJudgeMachine is the fields of a machine for the judges.
|
||
func newJudgeMachine(n int) dbMachine {
|
||
d := newDBMachine(n + 2)
|
||
d.addr.Placeholder = fmt.Sprintf("10.0.0.%d or judge%d.internal", 20+n, n)
|
||
return d
|
||
}
|
||
|
||
// shown is how many machines the page asks for.
|
||
func (m setupModel) shown() int { return 2 + m.judges }
|
||
|
||
// machineTitle names machine i: the store's, or the judges'.
|
||
func machineTitle(i int) string {
|
||
if i < 2 {
|
||
return fmt.Sprintf("Database machine %d", i+1)
|
||
}
|
||
return fmt.Sprintf("Judge machine %d", i-1)
|
||
}
|
||
|
||
// field is the text field at a machine's position, nil for the selector.
|
||
func (d *dbMachine) field(i int) *textField {
|
||
switch i {
|
||
case mfAddr:
|
||
return &d.addr
|
||
case mfUser:
|
||
return &d.user
|
||
case mfSecret:
|
||
return &d.secret
|
||
case mfSudo:
|
||
return &d.sudo
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// Messages from the work behind the screens.
|
||
type (
|
||
checkDoneMsg struct{ rep preflight.Report }
|
||
joinedMsg struct {
|
||
i int
|
||
r *joinpkg.Remote
|
||
}
|
||
)
|
||
|
||
// progRef lets work started from Update send to the program it runs in.
|
||
type progRef struct{ p *tea.Program }
|
||
|
||
type setupModel struct {
|
||
ref *progRef
|
||
|
||
page setupPage
|
||
host string
|
||
ips []string
|
||
choice int // 0 one machine, 1 high availability
|
||
|
||
choices *installForm
|
||
|
||
db [2 + maxJudges]dbMachine
|
||
// judges is how many judge machines are asked for, none to maxJudges.
|
||
judges int
|
||
mfocus int // over the machines' fields, then the judges' choice, then the check button
|
||
|
||
conns []dbConn
|
||
listener *joinpkg.Listener
|
||
report preflight.Report
|
||
checking bool
|
||
|
||
in *serverInit
|
||
run initModel
|
||
finished bool
|
||
log *[]string
|
||
logMu *sync.Mutex
|
||
|
||
spin spinnerModel
|
||
formErr string
|
||
width int
|
||
height int
|
||
// scroll is how far the panel is scrolled, reset with each page.
|
||
scroll int
|
||
quit bool
|
||
// single is the choice of one machine: the screen closes and init's own
|
||
// form takes over.
|
||
single bool
|
||
}
|
||
|
||
func newSetupModel(host string, ips []string) setupModel {
|
||
in := newDefaultServerInit()
|
||
// The control plane of a replicated setup is not where models run;
|
||
// nodes are added once it is up. Either can still be chosen.
|
||
in.noNode, in.noModel = true, true
|
||
in.standbys = true
|
||
// Two judges, on two of the three machines' nodes.
|
||
in.judge.Replicas = 2
|
||
return setupModel{
|
||
ref: &progRef{}, host: host, ips: ips, in: &in, choices: newInstallForm(&in, host, ips, true),
|
||
db: [2 + maxJudges]dbMachine{newDBMachine(1), newDBMachine(2), newJudgeMachine(1), newJudgeMachine(2)},
|
||
spin: newSpinner(), width: 100, height: 30,
|
||
log: &[]string{}, logMu: &sync.Mutex{},
|
||
}
|
||
}
|
||
|
||
func (m setupModel) Init() tea.Cmd { return m.spin.Tick }
|
||
|
||
func (m setupModel) thisAddr() string {
|
||
if len(m.ips) > 0 {
|
||
return m.ips[0]
|
||
}
|
||
return m.host
|
||
}
|
||
|
||
func (m setupModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||
switch msg := msg.(type) {
|
||
case tea.WindowSizeMsg:
|
||
m.width, m.height = msg.Width, msg.Height
|
||
m.choices.height = msg.Height
|
||
m.run.width = msg.Width - navWidth - 6
|
||
case spinTickMsg:
|
||
var cmd tea.Cmd
|
||
m.spin, cmd = m.spin.Update(msg)
|
||
m.run.spin = m.spin
|
||
return m, cmd
|
||
case checkDoneMsg:
|
||
m.checking, m.report = false, msg.rep
|
||
case joinedMsg:
|
||
m.db[msg.i].joined = msg.r
|
||
if m.page == pWait && m.allJoined() {
|
||
m.page = pCheck
|
||
cmd := m.startCheck()
|
||
return m, cmd
|
||
}
|
||
case initLineMsg:
|
||
m.run.addLine(string(msg))
|
||
case initDoneMsg:
|
||
m.run.done, m.run.err, m.finished = true, msg.err, true
|
||
case tea.KeyMsg:
|
||
if msg.Type == tea.KeyCtrlC {
|
||
m.quit = true
|
||
return m, tea.Quit
|
||
}
|
||
before := m.page
|
||
next, cmd := m.key(msg)
|
||
if nm := next.(setupModel); nm.page != before {
|
||
// A new page starts at its top.
|
||
nm.scroll = 0
|
||
return nm, cmd
|
||
}
|
||
return next, cmd
|
||
}
|
||
return m, nil
|
||
}
|
||
|
||
func (m setupModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||
// The choices' own dialogs take their keys, Esc among them.
|
||
if m.page == pChoices && (m.choices.dialog != nil || m.choices.err != "") {
|
||
m.choices.key(k, m.in)
|
||
return m, nil
|
||
}
|
||
// A dialog open takes the next key, whatever it is, to close it.
|
||
if m.formErr != "" {
|
||
m.formErr = ""
|
||
return m, nil
|
||
}
|
||
switch k.Type {
|
||
case tea.KeyPgDown:
|
||
m.scroll += max(1, m.height-8)
|
||
return m, nil
|
||
case tea.KeyPgUp:
|
||
m.scroll = max(0, m.scroll-max(1, m.height-8))
|
||
return m, nil
|
||
}
|
||
if k.Type == tea.KeyEsc && m.page > pWelcome && m.page < pRun && !m.checking {
|
||
m.page--
|
||
if m.page == pWait {
|
||
m.page = pMachines
|
||
}
|
||
m.formErr = ""
|
||
return m, nil
|
||
}
|
||
switch m.page {
|
||
case pWelcome:
|
||
switch k.String() {
|
||
case "up", "down", "tab", "k", "j":
|
||
m.choice = 1 - m.choice
|
||
case "enter":
|
||
if m.choice == 0 {
|
||
m.single = true
|
||
return m, tea.Quit
|
||
}
|
||
m.page = pChoices
|
||
}
|
||
case pChoices:
|
||
if m.choices.key(k, m.in) {
|
||
m.page, m.mfocus = pMachines, 0
|
||
m.focusMachine()
|
||
}
|
||
case pMachines:
|
||
return m.machinesKey(k)
|
||
case pCheck:
|
||
switch k.String() {
|
||
case "r":
|
||
if !m.checking {
|
||
cmd := m.startCheck()
|
||
return m, cmd
|
||
}
|
||
case "enter":
|
||
if !m.checking && m.report.Worst() < preflight.Fail {
|
||
m.page = pPlan
|
||
}
|
||
}
|
||
case pPlan:
|
||
if k.String() == "enter" {
|
||
m.page = pRun
|
||
cmd := m.startRun()
|
||
return m, cmd
|
||
}
|
||
case pRun:
|
||
if m.finished && (k.String() == "q" || k.String() == "enter") {
|
||
return m, tea.Quit
|
||
}
|
||
}
|
||
return m, nil
|
||
}
|
||
|
||
func (m *setupModel) focusMachine() {
|
||
for i := range m.db {
|
||
for f := 0; f < mfCount; f++ {
|
||
if t := m.db[i].field(f); t != nil {
|
||
t.Blur()
|
||
}
|
||
}
|
||
}
|
||
if m.mfocus < m.shown()*mfCount {
|
||
if t := m.db[m.mfocus/mfCount].field(m.mfocus % mfCount); t != nil {
|
||
t.Focus()
|
||
}
|
||
}
|
||
}
|
||
|
||
func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||
sel := m.shown() * mfCount
|
||
n := sel + 2
|
||
switch k.Type {
|
||
case tea.KeyTab, tea.KeyDown:
|
||
m.step(1, n)
|
||
return m, nil
|
||
case tea.KeyShiftTab, tea.KeyUp:
|
||
m.step(-1, n)
|
||
return m, nil
|
||
case tea.KeyEnter:
|
||
if m.mfocus <= sel {
|
||
m.step(1, n)
|
||
return m, nil
|
||
}
|
||
if err := m.validateMachines(); err != "" {
|
||
m.formErr = err
|
||
return m, nil
|
||
}
|
||
m.formErr = ""
|
||
if cmd, waiting := m.startJoins(); waiting {
|
||
m.page = pWait
|
||
return m, cmd
|
||
}
|
||
m.page = pCheck
|
||
cmd := m.startCheck()
|
||
return m, cmd
|
||
}
|
||
if m.mfocus == sel {
|
||
switch k.String() {
|
||
case "left", "h":
|
||
m.judges = (m.judges + maxJudges) % (maxJudges + 1)
|
||
case "right", "l", " ":
|
||
m.judges = (m.judges + 1) % (maxJudges + 1)
|
||
}
|
||
m.mfocus = m.shown() * mfCount
|
||
return m, nil
|
||
}
|
||
if m.mfocus > sel {
|
||
return m, nil
|
||
}
|
||
d := &m.db[m.mfocus/mfCount]
|
||
if m.mfocus%mfCount == mfAuth {
|
||
switch k.String() {
|
||
case "left", "h":
|
||
d.auth = (d.auth + authCount - 1) % authCount
|
||
case "right", "l", " ":
|
||
d.auth = (d.auth + 1) % authCount
|
||
}
|
||
if d.auth != authJoin && d.joined != nil {
|
||
_ = d.joined.Close()
|
||
d.joined, d.code, d.arrive = nil, "", nil
|
||
}
|
||
d.secret.SetValue("")
|
||
d.secret.Mask = d.auth == authPassword
|
||
d.secret.Placeholder = map[int]string{authKey: "/root/.ssh/id_ed25519", authPassword: "the user's ssh password"}[d.auth]
|
||
return m, nil
|
||
}
|
||
var cmd tea.Cmd
|
||
t := d.field(m.mfocus % mfCount)
|
||
*t, cmd = t.Update(k)
|
||
return m, cmd
|
||
}
|
||
|
||
// step moves the focus over the fields that are shown.
|
||
func (m *setupModel) step(d, n int) {
|
||
for range n {
|
||
m.mfocus = (m.mfocus + d + n) % n
|
||
if m.mfocus >= m.shown()*mfCount || m.db[m.mfocus/mfCount].visible(m.mfocus%mfCount) {
|
||
break
|
||
}
|
||
}
|
||
m.focusMachine()
|
||
}
|
||
|
||
func (m setupModel) allJoined() bool {
|
||
for _, d := range m.db[:m.shown()] {
|
||
if d.auth == authJoin && d.joined == nil {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// startJoins listens for the machines that join with a code, makes their
|
||
// codes, and waits for them; it says whether there is anything to wait for.
|
||
func (m *setupModel) startJoins() (tea.Cmd, bool) {
|
||
var cmds []tea.Cmd
|
||
for i := range m.db[:m.shown()] {
|
||
d := &m.db[i]
|
||
if d.auth != authJoin || d.joined != nil {
|
||
continue
|
||
}
|
||
if m.listener == nil {
|
||
// On the address the codes name, not every interface: a code is
|
||
// a credential, and the port it is spent on need not face more
|
||
// than the network the machines join from.
|
||
addr := net.JoinHostPort(m.thisAddr(), fmt.Sprint(joinpkg.DefaultPort))
|
||
l, err := joinpkg.Listen(addr, addr, time.Hour)
|
||
if err != nil {
|
||
m.formErr = err.Error()
|
||
return nil, false
|
||
}
|
||
m.listener = l
|
||
}
|
||
if d.arrive == nil {
|
||
role := "database"
|
||
if i >= 2 {
|
||
role = "judge"
|
||
}
|
||
c, ch := m.listener.Code(role)
|
||
d.code, d.arrive = c.String(), ch
|
||
}
|
||
ch, i := d.arrive, i
|
||
cmds = append(cmds, func() tea.Msg { return joinedMsg{i, <-ch} })
|
||
}
|
||
return tea.Batch(cmds...), len(cmds) > 0
|
||
}
|
||
|
||
func (m setupModel) validateMachines() string {
|
||
seen := map[string]bool{m.thisAddr(): true, m.host: true}
|
||
for i, d := range m.db[:m.shown()] {
|
||
if d.auth == authJoin {
|
||
continue
|
||
}
|
||
addr, title := strings.TrimSpace(d.addr.Value()), strings.ToLower(machineTitle(i)[:1])+machineTitle(i)[1:]
|
||
switch {
|
||
case addr == "":
|
||
return title + " has no address"
|
||
case seen[addr]:
|
||
return fmt.Sprintf("%s is given twice, or is this machine: each must be a different machine", addr)
|
||
case strings.TrimSpace(d.user.Value()) == "":
|
||
return title + " has no user to log in as"
|
||
case d.auth != authAgent && d.secret.Value() == "":
|
||
return fmt.Sprintf("%s needs its %s", title, authNames[d.auth])
|
||
}
|
||
seen[addr] = true
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// startCheck connects to the database machines and runs the checks on all
|
||
// three, this one included.
|
||
func (m *setupModel) startCheck() tea.Cmd {
|
||
for _, c := range m.conns {
|
||
if c.ssh != nil {
|
||
_ = c.ssh.Close()
|
||
}
|
||
}
|
||
m.conns = nil
|
||
serves, what := m.in.servingMemoryMB(m.judges > 0)
|
||
machines := []preflight.Machine{{
|
||
Name: m.host + " (this machine)", Addr: m.thisAddr(), Local: true, Remote: pgha.Local{},
|
||
Roles: []preflight.Role{preflight.RoleControl, preflight.RoleWitness},
|
||
ServesMB: serves, Serves: what,
|
||
}}
|
||
for i, d := range m.db[:m.shown()] {
|
||
roles, judge := m.dbRoles(), i >= 2
|
||
if judge {
|
||
roles = []preflight.Role{preflight.RoleNode}
|
||
}
|
||
if d.auth == authJoin {
|
||
m.conns = append(m.conns, dbConn{remote: d.joined, addr: d.joined.Addr, joined: d.joined,
|
||
judge: judge, target: "root@" + d.joined.Addr})
|
||
machines = append(machines, preflight.Machine{Name: d.joined.Hostname + " (" + d.joined.Addr + ", joined)",
|
||
Addr: d.joined.Addr, Roles: roles, Remote: d.joined})
|
||
continue
|
||
}
|
||
g := &onboard.GoSSH{
|
||
Addr: strings.TrimSpace(d.addr.Value()), User: strings.TrimSpace(d.user.Value()),
|
||
UseAgent: d.auth == authAgent, SudoPassword: d.sudo.Value(),
|
||
}
|
||
switch d.auth {
|
||
case authKey:
|
||
g.KeyFile = strings.TrimSpace(d.secret.Value())
|
||
case authPassword:
|
||
g.Password = d.secret.Value()
|
||
}
|
||
m.conns = append(m.conns, dbConn{remote: g, addr: g.Addr, ssh: g, judge: judge, target: g.User + "@" + g.Addr})
|
||
machines = append(machines, preflight.Machine{Name: g.Addr, Addr: g.Addr, Roles: roles, Remote: g})
|
||
}
|
||
m.checking = true
|
||
m.report = preflight.Report{}
|
||
conns := m.conns
|
||
return func() tea.Msg {
|
||
rep := preflight.Run(context.Background(), machines)
|
||
// A sudo that asks for a password is fine with one given: the
|
||
// install grants itself sudo with it.
|
||
for i := range rep.Findings {
|
||
f := &rep.Findings[i]
|
||
if f.Check == "sudo" && f.Level == preflight.Warn {
|
||
for _, c := range conns {
|
||
g := c.ssh
|
||
switch {
|
||
case g == nil || g.Addr != f.Machine:
|
||
case g.SudoPassword == "":
|
||
f.Level = preflight.Fail
|
||
f.Fix = "give this user's sudo password on the previous screen"
|
||
default:
|
||
f.Level = preflight.OK
|
||
f.Detail = "with the password given"
|
||
f.Fix = ""
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return checkDoneMsg{rep}
|
||
}
|
||
}
|
||
|
||
// dbRoles are what the other two machines are to be: the store, and a
|
||
// control plane too when they stand by.
|
||
func (m setupModel) dbRoles() []preflight.Role {
|
||
if m.in.standbys {
|
||
return []preflight.Role{preflight.RoleDatabase, preflight.RoleControl}
|
||
}
|
||
return []preflight.Role{preflight.RoleDatabase}
|
||
}
|
||
|
||
// targets are the database machines as user@address, for the command line.
|
||
func (m setupModel) targets() []string {
|
||
var out []string
|
||
for _, d := range m.db[:2] {
|
||
if d.joined != nil {
|
||
out = append(out, "root@"+d.joined.Addr)
|
||
continue
|
||
}
|
||
out = append(out, strings.TrimSpace(d.user.Value())+"@"+strings.TrimSpace(d.addr.Value()))
|
||
}
|
||
return out
|
||
}
|
||
|
||
// judgeTargets are the judge machines as user@address.
|
||
func (m setupModel) judgeTargets() []judgeMachine {
|
||
var out []judgeMachine
|
||
for _, d := range m.db[2:m.shown()] {
|
||
if d.joined != nil {
|
||
out = append(out, judgeMachine{target: "root@" + d.joined.Addr})
|
||
continue
|
||
}
|
||
out = append(out, judgeMachine{target: strings.TrimSpace(d.user.Value()) + "@" + strings.TrimSpace(d.addr.Value())})
|
||
}
|
||
return out
|
||
}
|
||
|
||
// command is what this setup is on the command line, for doing it again
|
||
// from a script — with keys in place, since a password cannot be given there.
|
||
func (m setupModel) command() string {
|
||
in := *m.in
|
||
in.patroni = m.targets()
|
||
in.judgeMachines = m.judgeTargets()
|
||
return strings.Join(append([]string{"sudo --preserve-env=SSH_AUTH_SOCK llmgw server init"}, in.initFlags()...), " \\\n ")
|
||
}
|
||
|
||
// startRun installs: sudo granted where it asks for a password, the control
|
||
// plane here, the store on the other two, then sudo taken back.
|
||
func (m *setupModel) startRun() tea.Cmd {
|
||
in := m.in
|
||
in.patroni = m.targets()
|
||
in.judgeMachines = nil
|
||
m.run = newInitModel(in, nil, false)
|
||
m.run.running, m.run.width, m.run.spin = true, m.width, m.spin
|
||
ref, conns, logp, mu, listener := m.ref, m.conns, m.log, m.logMu, m.listener
|
||
return func() tea.Msg {
|
||
send := func(l string) {
|
||
mu.Lock()
|
||
*logp = append(*logp, l)
|
||
mu.Unlock()
|
||
ref.p.Send(initLineMsg(l))
|
||
}
|
||
in.out = &lineWriter{send: send}
|
||
err := func() error {
|
||
ctx := context.Background()
|
||
for _, c := range conns {
|
||
if c.ssh != nil {
|
||
fmt.Fprintf(in.out, " .. sudo on %s\n", c.addr)
|
||
if err := c.ssh.PrepareSudo(ctx); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
if c.judge {
|
||
// Added as a node once the control plane is up.
|
||
in.judgeMachines = append(in.judgeMachines, judgeMachine{target: c.target, remote: c.remote})
|
||
continue
|
||
}
|
||
out, err := c.remote.Run(ctx, "hostname", nil)
|
||
if err != nil {
|
||
return fmt.Errorf("%s: %v", c.addr, err)
|
||
}
|
||
ip, err := ipv4Of(c.addr)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
in.patroniNodes = append(in.patroniNodes, pgha.Node{
|
||
Member: pgha.Member{Name: strings.TrimSpace(string(out)), IP: ip}, Remote: c.remote})
|
||
}
|
||
return in.run(ctx)
|
||
}()
|
||
// Whatever happened: a sudo grant taken back, a joined machine's
|
||
// `llmgw join` told it is done, the port for joining closed.
|
||
for _, c := range conns {
|
||
if c.ssh != nil {
|
||
_ = c.ssh.Close()
|
||
}
|
||
if c.joined != nil {
|
||
_ = c.joined.Close()
|
||
}
|
||
}
|
||
if listener != nil {
|
||
_ = listener.Close()
|
||
}
|
||
return initDoneMsg{err}
|
||
}
|
||
}
|
||
|
||
// setupSteps are the sidebar's steps, by page.
|
||
var setupSteps = []struct {
|
||
label string
|
||
page setupPage
|
||
}{
|
||
{"Start", pWelcome}, {"Choices", pChoices}, {"Machines", pMachines},
|
||
{"Checks", pCheck}, {"Plan", pPlan}, {"Install", pRun},
|
||
}
|
||
|
||
func (m setupModel) View() string {
|
||
at := m.page
|
||
if at == pWait {
|
||
at = pMachines
|
||
}
|
||
var nav []navItem
|
||
for _, st := range setupSteps {
|
||
if m.choice == 0 && st.page > pWelcome {
|
||
break
|
||
}
|
||
state := "todo"
|
||
switch {
|
||
case st.page < at || (st.page == pRun && m.finished && m.run.err == nil):
|
||
state = "done"
|
||
case st.page == at:
|
||
state = "current"
|
||
}
|
||
nav = append(nav, navItem{Label: st.label, State: state})
|
||
}
|
||
title, body, keys := m.pageView()
|
||
f := frame{
|
||
Width: m.width, Height: m.height,
|
||
Title: "llmgw setup", Where: m.host + " · " + m.thisAddr(),
|
||
NavTitle: "Steps", Nav: nav,
|
||
PanelTitle: title, Body: body, Scroll: m.scroll, Keys: keys,
|
||
}
|
||
if m.formErr != "" {
|
||
f.DialogTitle, f.Dialog = "Not yet", m.formErr+"\n\n"+sDim.Render("any key to go back")
|
||
}
|
||
if m.page == pChoices {
|
||
f.DialogTitle, f.Dialog = m.choices.overlay()
|
||
}
|
||
return f.render()
|
||
}
|
||
|
||
// pageView is the panel for the page: its title, what is in it, and its keys.
|
||
func (m setupModel) pageView() (string, string, string) {
|
||
switch m.page {
|
||
case pWelcome:
|
||
return "Start", m.welcomeView(), "↑↓ choose · Enter go on · Ctrl+C leave"
|
||
case pChoices:
|
||
keys := m.choices.keys()
|
||
if m.choices.dialog == nil && m.choices.err == "" {
|
||
keys = strings.Replace(keys, " · Ctrl+C", " · Esc back · Ctrl+C", 1)
|
||
}
|
||
return "What to install", m.choices.view(m.width - navWidth - 6), keys
|
||
case pMachines:
|
||
return "Database machines", m.machinesView(), "Tab/↑↓ move · ←→ how to log in · Enter next · Esc back · PgUp/PgDn scroll"
|
||
case pWait:
|
||
return "Waiting for machines to join", m.waitView(), "Esc back · PgUp/PgDn scroll · Ctrl+C leave"
|
||
case pCheck:
|
||
keys := "R check again · Esc back · PgUp/PgDn scroll"
|
||
if !m.checking && m.report.Worst() < preflight.Fail {
|
||
keys = "Enter see the plan · " + keys
|
||
}
|
||
return "Checks", m.checkView(), keys
|
||
case pPlan:
|
||
return "Plan", m.planView(), "Enter install · Esc back · PgUp/PgDn scroll · Ctrl+C leave"
|
||
}
|
||
v := m.run.View()
|
||
if i := strings.Index(v, "\n\n"); i >= 0 {
|
||
v = v[i+2:] // init's own title; the frame has one
|
||
}
|
||
keys := "Ctrl+C stop"
|
||
if m.finished {
|
||
keys = "Enter or Q leave · PgUp/PgDn scroll"
|
||
if m.run.err == nil {
|
||
v += "\n" + sDim.Render("The same, from a script, with ssh keys in place:") + "\n" + m.command() + "\n"
|
||
}
|
||
}
|
||
return "Install", v, keys
|
||
}
|
||
|
||
func (m setupModel) welcomeView() string {
|
||
var b strings.Builder
|
||
fmt.Fprintf(&b, "\n Nothing of llmgw is installed on %s.\n\n", lipgloss.NewStyle().Bold(true).Render(m.host))
|
||
opt := func(i int, title, detail string) {
|
||
mark := "( )"
|
||
if m.choice == i {
|
||
mark = "(" + sOK.Render("•") + ")"
|
||
title = fSelected.Render(title)
|
||
}
|
||
fmt.Fprintf(&b, " %s %s\n %s\n\n", mark, title, sDim.Render(detail))
|
||
}
|
||
opt(0, "One machine", "This machine runs everything: control plane, model mirror,\n and models if it has room.")
|
||
opt(1, "High availability — three machines",
|
||
"Each of the three can run the gateway; two keep the store, replicated,\n"+
|
||
" and etcd votes on all three, so losing any one machine loses nothing.")
|
||
return b.String()
|
||
}
|
||
|
||
// button is a [ Label ] that shows when it has the focus.
|
||
func button(focused bool, label string) string {
|
||
if focused {
|
||
return " " + fBar.Render(" "+label+" ")
|
||
}
|
||
return " " + lipgloss.NewStyle().Bold(true).Render("[ "+label+" ]")
|
||
}
|
||
|
||
func (m setupModel) machinesView() string {
|
||
var b strings.Builder
|
||
b.WriteString("\n The two machines that keep the store: Ubuntu 22.04/24.04 or Debian 12.\n" +
|
||
" PostgreSQL, Patroni and etcd are installed on them.\n\n")
|
||
for i, d := range m.db[:m.shown()] {
|
||
base := i * mfCount
|
||
if i == 2 {
|
||
b.WriteString(" " + sDim.Render("Machines for the guardrails' judges alone: each becomes a node that runs only\n"+
|
||
" the judges, so a busy control plane cannot leave a request waiting for a judgment.") + "\n\n")
|
||
}
|
||
// Padded before it is coloured: the colour's escape codes count
|
||
// toward a width and threw the focused row out of line.
|
||
label := func(f int, s string) string {
|
||
s = fmt.Sprintf("%-15s", s)
|
||
if m.mfocus == base+f {
|
||
return fSelected.Render("› " + s)
|
||
}
|
||
return " " + s
|
||
}
|
||
fmt.Fprintf(&b, " %s\n", lipgloss.NewStyle().Bold(true).Render(machineTitle(i)))
|
||
if d.visible(mfAddr) {
|
||
fmt.Fprintf(&b, " %s %s\n", label(mfAddr, "Address"), d.addr.View())
|
||
fmt.Fprintf(&b, " %s %s\n", label(mfUser, "User"), d.user.View())
|
||
}
|
||
var auth []string
|
||
for a := 0; a < authCount; a++ {
|
||
if a == d.auth {
|
||
auth = append(auth, "("+sOK.Render("•")+") "+authNames[a])
|
||
} else {
|
||
auth = append(auth, "( ) "+sDim.Render(authNames[a]))
|
||
}
|
||
}
|
||
fmt.Fprintf(&b, " %s %s\n", label(mfAuth, "Log in with"), strings.Join(auth, " "))
|
||
if d.visible(mfSecret) {
|
||
fmt.Fprintf(&b, " %s %s\n", label(mfSecret, strings.ToUpper(authNames[d.auth][:1])+authNames[d.auth][1:]), d.secret.View())
|
||
}
|
||
if d.visible(mfSudo) {
|
||
fmt.Fprintf(&b, " %s %s\n", label(mfSudo, "Sudo password"), d.sudo.View())
|
||
}
|
||
if d.auth == authJoin {
|
||
b.WriteString(" " + sDim.Render("Nothing logs into it: it runs llmgw join with a code shown next.") + "\n")
|
||
}
|
||
b.WriteString("\n")
|
||
}
|
||
sel := m.shown() * mfCount
|
||
var opts []string
|
||
for n, name := range []string{"none", "one", "two"} {
|
||
if n == m.judges {
|
||
opts = append(opts, "("+sOK.Render("•")+") "+name)
|
||
} else {
|
||
opts = append(opts, "( ) "+sDim.Render(name))
|
||
}
|
||
}
|
||
judgeLabel := fmt.Sprintf(" %-15s", "Judge machines")
|
||
if m.mfocus == sel {
|
||
judgeLabel = fSelected.Render(fmt.Sprintf("› %-15s", "Judge machines"))
|
||
}
|
||
fmt.Fprintf(&b, " %s %s\n", judgeLabel, strings.Join(opts, " "))
|
||
b.WriteString(" " + sDim.Render("Without them the judges run on whichever node has room, added later.") + "\n\n")
|
||
b.WriteString(button(m.mfocus == sel+1, fmt.Sprintf("Check the %d machines", m.shown()+1)) + "\n\n")
|
||
b.WriteString(" " + sDim.Render("Passwords are kept in memory only, for this setup.") + "\n")
|
||
return b.String()
|
||
}
|
||
|
||
func (m setupModel) waitView() string {
|
||
var b strings.Builder
|
||
fmt.Fprintf(&b, "\n On each machine below, as root. It dials this machine on port %d;\n nothing dials in. Each code is for one machine and good for an hour.\n\n", joinpkg.DefaultPort)
|
||
for i, d := range m.db[:m.shown()] {
|
||
if d.auth != authJoin {
|
||
continue
|
||
}
|
||
title := lipgloss.NewStyle().Bold(true).Render(machineTitle(i))
|
||
if d.joined != nil {
|
||
fmt.Fprintf(&b, " %s %s joined as %s (%s)\n\n", title, sOK.Render("✓"), d.joined.Hostname, d.joined.Addr)
|
||
continue
|
||
}
|
||
fmt.Fprintf(&b, " %s %s waiting\n", title, m.spin.View())
|
||
b.WriteString(" " + sDim.Render("without llmgw yet:") + "\n")
|
||
for _, l := range joinInstall() {
|
||
b.WriteString(" " + strings.ReplaceAll(l, "\n", "\n ") + "\n")
|
||
}
|
||
b.WriteString(" " + sDim.Render("then:") + "\n sudo llmgw join " + d.code + "\n\n")
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
func (m setupModel) checkView() string {
|
||
var b strings.Builder
|
||
b.WriteString("\n")
|
||
if m.checking {
|
||
b.WriteString(" " + m.spin.View() + " checking the machines…\n")
|
||
return b.String()
|
||
}
|
||
switch m.report.Worst() {
|
||
case preflight.Fail:
|
||
b.WriteString(" " + sBad.Render("✗ Fix what is marked ✗, then check again (R).") + "\n")
|
||
case preflight.Warn:
|
||
b.WriteString(" " + sWarn.Render("! It can go ahead; what is marked ! is worth a look.") + "\n")
|
||
default:
|
||
b.WriteString(" " + sOK.Render("✓ All are ready.") + "\n")
|
||
}
|
||
machine := ""
|
||
for _, f := range m.report.Findings {
|
||
if f.Machine != machine {
|
||
machine = f.Machine
|
||
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render(machine) + "\n")
|
||
}
|
||
mark := sOK.Render("✓")
|
||
switch f.Level {
|
||
case preflight.Warn:
|
||
mark = sWarn.Render("!")
|
||
case preflight.Fail:
|
||
mark = sBad.Render("✗")
|
||
}
|
||
fmt.Fprintf(&b, " %s %-13s %s\n", mark, f.Check, f.Detail)
|
||
if f.Level != preflight.OK && f.Fix != "" {
|
||
b.WriteString(" " + sDim.Render(f.Fix) + "\n")
|
||
}
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
func (m setupModel) planView() string {
|
||
var b strings.Builder
|
||
b.WriteString(m.choices.planView(m.width - navWidth - 6))
|
||
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("The machines") + "\n")
|
||
here := "control plane, leading at first; model mirror; etcd"
|
||
there := "PostgreSQL under Patroni, etcd"
|
||
if m.in.standbys {
|
||
there = "control plane, standing by; " + there
|
||
}
|
||
if m.in.vip != "" {
|
||
here += "; keepalived"
|
||
there += "; keepalived"
|
||
}
|
||
fmt.Fprintf(&b, " %s %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(m.host), sDim.Render(m.thisAddr()), here)
|
||
granted := false
|
||
for i, d := range m.db[:m.shown()] {
|
||
addr := strings.TrimSpace(d.addr.Value())
|
||
if d.joined != nil {
|
||
addr = d.joined.Hostname + " " + sDim.Render(d.joined.Addr+", joined")
|
||
}
|
||
role := there
|
||
if i >= 2 {
|
||
role = "a node for the guardrails' judges alone"
|
||
}
|
||
fmt.Fprintf(&b, " %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(addr), role)
|
||
granted = granted || d.sudo.Value() != ""
|
||
}
|
||
b.WriteString("\n The store is replicated from the first to the second; if the primary is lost\n" +
|
||
" the standby takes over in about thirty seconds, and the control plane follows.\n")
|
||
if granted {
|
||
b.WriteString(" " + sDim.Render("Where a sudo password was given, the user may sudo without it until the\n install finishes, and not after.") + "\n")
|
||
}
|
||
b.WriteString("\n " + sDim.Render("The same from a script, with ssh keys in place:") + "\n")
|
||
for _, l := range strings.Split(m.command(), "\n") {
|
||
b.WriteString(" " + l + "\n")
|
||
}
|
||
for _, d := range m.db[:m.shown()] {
|
||
if d.joined != nil {
|
||
b.WriteString(" " + sDim.Render("(a machine that joined with a code needs ssh for that)") + "\n")
|
||
break
|
||
}
|
||
}
|
||
b.WriteString("\n" + button(true, "Enter to install") + "\n")
|
||
return b.String()
|
||
}
|
||
|
||
// runSetupTUI is the setup screens, until they finish or hand over to init's
|
||
// own form for one machine.
|
||
func runSetupTUI(ctx context.Context, out io.Writer) error {
|
||
host, _ := os.Hostname()
|
||
var ips []string
|
||
for _, ip := range localAddrs() {
|
||
if ip.To4() != nil {
|
||
ips = append(ips, ip.String())
|
||
}
|
||
}
|
||
m := newSetupModel(host, ips)
|
||
p := tea.NewProgram(m, tea.WithOutput(out), tea.WithAltScreen())
|
||
m.ref.p = p
|
||
final, err := p.Run()
|
||
if err != nil {
|
||
return err
|
||
}
|
||
fm := final.(setupModel)
|
||
if fm.single {
|
||
in := newDefaultServerInit()
|
||
in.out = out
|
||
return in.runInitTUI(ctx, out)
|
||
}
|
||
if fm.page < pRun {
|
||
return nil
|
||
}
|
||
fmt.Fprintln(out)
|
||
if fm.run.err != nil {
|
||
fm.logMu.Lock()
|
||
fmt.Fprintln(out, strings.Join(*fm.log, "\n"))
|
||
fm.logMu.Unlock()
|
||
return fm.run.err
|
||
}
|
||
if !fm.finished {
|
||
return fmt.Errorf("left while installing; running llmgw again finishes what was started")
|
||
}
|
||
if fm.run.password != "" {
|
||
fmt.Fprintf(out, "Administrator %s, password %s (shown once)\n\n", fm.in.admin, fm.run.password)
|
||
}
|
||
fmt.Fprintln(out, "Next: add nodes from the console — run llmgw again — or with")
|
||
fmt.Fprintln(out, " sudo llmgw node add <user>@<address>")
|
||
return nil
|
||
}
|