notify.webhooks existed and setup never asked for one, so a new site learnt of a parked model or a lost node from its users. `llmgw server init --webhook URL` (repeatable) and a Notification webhooks field on both setup screens take them now. Slack's address, Mattermost's and Rocket.Chat's /hooks/ and Discord's …/slack are sent Slack's text, any other the event as JSON; slack: or json: in front says which. Once the control plane runs, they are added to the cluster's settings beside the webhooks already there, none twice, and setup waits for the control planes to start again on the change before sending each a test, saying which answered. A webhook's path is its secret, so it is shown by host only. A failure is said, not fatal: the cluster serves without its alerts, and ctl notify test tries again. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
992 lines
28 KiB
Go
992 lines
28 KiB
Go
package cli
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"io"
|
||
"net"
|
||
"net/mail"
|
||
"os"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
tea "github.com/charmbracelet/bubbletea"
|
||
"github.com/charmbracelet/lipgloss"
|
||
|
||
joinpkg "gitea.mynodes.xyz/ilker/llmgw/internal/join"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/preflight"
|
||
)
|
||
|
||
// The setup screens: what a bare `llmgw` shows on a machine with nothing
|
||
// installed. One machine, or three for high availability — this one as the
|
||
// control plane and etcd's third vote, two more for the store under Patroni —
|
||
// with every machine checked before anything is installed on any of them.
|
||
//
|
||
// The other machines are reached with Go's own ssh rather than the system's,
|
||
// so a password can be typed here when there is no key; it is held in memory
|
||
// for the run and written nowhere. A sudo password, likewise, is used once to
|
||
// let the user sudo for the length of the install, and that is taken back
|
||
// when it finishes.
|
||
|
||
type setupPage int
|
||
|
||
const (
|
||
pWelcome setupPage = iota
|
||
pAdmin
|
||
pMachines
|
||
// pWait shows the codes for machines joining rather than logged into,
|
||
// until they have all dialled in.
|
||
pWait
|
||
pCheck
|
||
pPlan
|
||
pRun
|
||
)
|
||
|
||
// How a database machine is logged into.
|
||
const (
|
||
authAgent = iota
|
||
authKey
|
||
authPassword
|
||
// authJoin is no login at all: the machine runs `llmgw join` with a code
|
||
// shown here and dials out to this one.
|
||
authJoin
|
||
authCount
|
||
)
|
||
|
||
var authNames = [authCount]string{"ssh agent", "key file", "password", "join code"}
|
||
|
||
// Fields of one database machine, in order.
|
||
const (
|
||
mfAddr = iota
|
||
mfUser
|
||
mfAuth
|
||
mfSecret
|
||
mfSudo
|
||
mfCount
|
||
)
|
||
|
||
type dbMachine struct {
|
||
addr, user, secret, sudo textField
|
||
auth int
|
||
// For a machine joining: its code, and it once it has dialled in.
|
||
code string
|
||
joined *joinpkg.Remote
|
||
arrive <-chan *joinpkg.Remote
|
||
}
|
||
|
||
// visible reports whether a field is shown, and so can be moved to: a joining
|
||
// machine is asked nothing but how it joins.
|
||
func (d dbMachine) visible(f int) bool {
|
||
switch {
|
||
case d.auth == authJoin:
|
||
return f == mfAuth
|
||
case f == mfSecret:
|
||
return d.auth != authAgent
|
||
}
|
||
return true
|
||
}
|
||
|
||
// dbConn is a database machine as it is reached, whichever way.
|
||
type dbConn struct {
|
||
remote onboard.Remote
|
||
addr string
|
||
ssh *onboard.GoSSH
|
||
joined *joinpkg.Remote
|
||
}
|
||
|
||
func newDBMachine(n int) dbMachine {
|
||
f := func(ph string, mask bool) textField {
|
||
t := newTextField()
|
||
t.Placeholder, t.Mask, t.CharLimit, t.Width = ph, mask, 200, 34
|
||
return t
|
||
}
|
||
m := dbMachine{
|
||
addr: f(fmt.Sprintf("10.0.0.%d or db%d.internal", 10+n, n), false),
|
||
user: f("ubuntu", false),
|
||
secret: f("", true),
|
||
sudo: f("only if sudo asks for one", true),
|
||
}
|
||
if os.Getenv("SSH_AUTH_SOCK") == "" {
|
||
m.auth = authKey
|
||
}
|
||
return m
|
||
}
|
||
|
||
// field is the text field at a machine's position, nil for the selector.
|
||
func (d *dbMachine) field(i int) *textField {
|
||
switch i {
|
||
case mfAddr:
|
||
return &d.addr
|
||
case mfUser:
|
||
return &d.user
|
||
case mfSecret:
|
||
return &d.secret
|
||
case mfSudo:
|
||
return &d.sudo
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// Messages from the work behind the screens.
|
||
type (
|
||
checkDoneMsg struct{ rep preflight.Report }
|
||
joinedMsg struct {
|
||
i int
|
||
r *joinpkg.Remote
|
||
}
|
||
)
|
||
|
||
// progRef lets work started from Update send to the program it runs in.
|
||
type progRef struct{ p *tea.Program }
|
||
|
||
type setupModel struct {
|
||
ref *progRef
|
||
|
||
page setupPage
|
||
host string
|
||
ips []string
|
||
choice int // 0 one machine, 1 high availability
|
||
|
||
admin, names, webhooks textField
|
||
afocus int // admin, names, webhooks, next
|
||
|
||
db [2]dbMachine
|
||
mfocus int // over both machines' fields, then the check button
|
||
|
||
conns []dbConn
|
||
listener *joinpkg.Listener
|
||
report preflight.Report
|
||
checking bool
|
||
|
||
in *serverInit
|
||
run initModel
|
||
finished bool
|
||
log *[]string
|
||
logMu *sync.Mutex
|
||
|
||
spin spinnerModel
|
||
formErr string
|
||
width int
|
||
height int
|
||
// scroll is how far the panel is scrolled, reset with each page.
|
||
scroll int
|
||
quit bool
|
||
// single is the choice of one machine: the screen closes and init's own
|
||
// form takes over.
|
||
single bool
|
||
}
|
||
|
||
func newSetupModel(host string, ips []string) setupModel {
|
||
admin := newTextField()
|
||
admin.Placeholder, admin.CharLimit, admin.Width = "you@example.com", 200, 40
|
||
names := newTextField()
|
||
names.Placeholder, names.CharLimit, names.Width = "llm.example.com, 203.0.113.10", 400, 40
|
||
webhooks := newTextField()
|
||
webhooks.Placeholder, webhooks.CharLimit, webhooks.Width = "https://hooks.slack.com/services/…", 800, 40
|
||
return setupModel{
|
||
ref: &progRef{}, host: host, ips: ips, admin: admin, names: names, webhooks: webhooks,
|
||
db: [2]dbMachine{newDBMachine(1), newDBMachine(2)}, spin: newSpinner(), width: 100, height: 30,
|
||
log: &[]string{}, logMu: &sync.Mutex{},
|
||
}
|
||
}
|
||
|
||
func (m setupModel) Init() tea.Cmd { return m.spin.Tick }
|
||
|
||
func (m setupModel) thisAddr() string {
|
||
if len(m.ips) > 0 {
|
||
return m.ips[0]
|
||
}
|
||
return m.host
|
||
}
|
||
|
||
func (m setupModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||
switch msg := msg.(type) {
|
||
case tea.WindowSizeMsg:
|
||
m.width, m.height = msg.Width, msg.Height
|
||
m.run.width = msg.Width - navWidth - 6
|
||
case spinTickMsg:
|
||
var cmd tea.Cmd
|
||
m.spin, cmd = m.spin.Update(msg)
|
||
m.run.spin = m.spin
|
||
return m, cmd
|
||
case checkDoneMsg:
|
||
m.checking, m.report = false, msg.rep
|
||
case joinedMsg:
|
||
m.db[msg.i].joined = msg.r
|
||
if m.page == pWait && m.allJoined() {
|
||
m.page = pCheck
|
||
cmd := m.startCheck()
|
||
return m, cmd
|
||
}
|
||
case initLineMsg:
|
||
m.run.addLine(string(msg))
|
||
case initDoneMsg:
|
||
m.run.done, m.run.err, m.finished = true, msg.err, true
|
||
case tea.KeyMsg:
|
||
if msg.Type == tea.KeyCtrlC {
|
||
m.quit = true
|
||
return m, tea.Quit
|
||
}
|
||
before := m.page
|
||
next, cmd := m.key(msg)
|
||
if nm := next.(setupModel); nm.page != before {
|
||
// A new page starts at its top.
|
||
nm.scroll = 0
|
||
return nm, cmd
|
||
}
|
||
return next, cmd
|
||
}
|
||
return m, nil
|
||
}
|
||
|
||
func (m setupModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||
// A dialog open takes the next key, whatever it is, to close it.
|
||
if m.formErr != "" {
|
||
m.formErr = ""
|
||
return m, nil
|
||
}
|
||
switch k.Type {
|
||
case tea.KeyPgDown:
|
||
m.scroll += max(1, m.height-8)
|
||
return m, nil
|
||
case tea.KeyPgUp:
|
||
m.scroll = max(0, m.scroll-max(1, m.height-8))
|
||
return m, nil
|
||
}
|
||
if k.Type == tea.KeyEsc && m.page > pWelcome && m.page < pRun && !m.checking {
|
||
m.page--
|
||
if m.page == pWait {
|
||
m.page = pMachines
|
||
}
|
||
m.formErr = ""
|
||
return m, nil
|
||
}
|
||
switch m.page {
|
||
case pWelcome:
|
||
switch k.String() {
|
||
case "up", "down", "tab", "k", "j":
|
||
m.choice = 1 - m.choice
|
||
case "enter":
|
||
if m.choice == 0 {
|
||
m.single = true
|
||
return m, tea.Quit
|
||
}
|
||
m.page = pAdmin
|
||
m.afocus = 0
|
||
m.admin.Focus()
|
||
}
|
||
case pAdmin:
|
||
return m.adminKey(k)
|
||
case pMachines:
|
||
return m.machinesKey(k)
|
||
case pCheck:
|
||
switch k.String() {
|
||
case "r":
|
||
if !m.checking {
|
||
cmd := m.startCheck()
|
||
return m, cmd
|
||
}
|
||
case "enter":
|
||
if !m.checking && m.report.Worst() < preflight.Fail {
|
||
m.page = pPlan
|
||
}
|
||
}
|
||
case pPlan:
|
||
if k.String() == "enter" {
|
||
m.page = pRun
|
||
cmd := m.startRun()
|
||
return m, cmd
|
||
}
|
||
case pRun:
|
||
if m.finished && (k.String() == "q" || k.String() == "enter") {
|
||
return m, tea.Quit
|
||
}
|
||
}
|
||
return m, nil
|
||
}
|
||
|
||
func (m setupModel) adminKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||
move := func(d int) {
|
||
m.afocus = (m.afocus + d + 4) % 4
|
||
m.admin.Blur()
|
||
m.names.Blur()
|
||
m.webhooks.Blur()
|
||
switch m.afocus {
|
||
case 0:
|
||
m.admin.Focus()
|
||
case 1:
|
||
m.names.Focus()
|
||
case 2:
|
||
m.webhooks.Focus()
|
||
}
|
||
}
|
||
switch k.Type {
|
||
case tea.KeyTab, tea.KeyDown:
|
||
move(1)
|
||
return m, nil
|
||
case tea.KeyShiftTab, tea.KeyUp:
|
||
move(-1)
|
||
return m, nil
|
||
case tea.KeyEnter:
|
||
if m.afocus < 3 {
|
||
move(1)
|
||
return m, nil
|
||
}
|
||
if _, err := mail.ParseAddress(strings.TrimSpace(m.admin.Value())); err != nil {
|
||
m.formErr = "the administrator's email is needed, and that is not one"
|
||
m.afocus = 0
|
||
move(0)
|
||
return m, nil
|
||
}
|
||
if _, err := parseWebhooks(m.webhookList()); err != nil {
|
||
m.formErr = err.Error()
|
||
m.afocus = 2
|
||
move(0)
|
||
return m, nil
|
||
}
|
||
m.formErr = ""
|
||
m.page, m.mfocus = pMachines, 0
|
||
m.db[0].addr.Focus()
|
||
return m, nil
|
||
}
|
||
var cmd tea.Cmd
|
||
switch m.afocus {
|
||
case 0:
|
||
m.admin, cmd = m.admin.Update(k)
|
||
case 1:
|
||
m.names, cmd = m.names.Update(k)
|
||
case 2:
|
||
m.webhooks, cmd = m.webhooks.Update(k)
|
||
}
|
||
return m, cmd
|
||
}
|
||
|
||
// webhookList is the webhooks asked for, one an item.
|
||
func (m setupModel) webhookList() []string {
|
||
var out []string
|
||
for _, s := range strings.Split(m.webhooks.Value(), ",") {
|
||
if s = strings.TrimSpace(s); s != "" {
|
||
out = append(out, s)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
func (m *setupModel) focusMachine() {
|
||
for i := range m.db {
|
||
for f := 0; f < mfCount; f++ {
|
||
if t := m.db[i].field(f); t != nil {
|
||
t.Blur()
|
||
}
|
||
}
|
||
}
|
||
if m.mfocus < 2*mfCount {
|
||
if t := m.db[m.mfocus/mfCount].field(m.mfocus % mfCount); t != nil {
|
||
t.Focus()
|
||
}
|
||
}
|
||
}
|
||
|
||
func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||
n := 2*mfCount + 1
|
||
switch k.Type {
|
||
case tea.KeyTab, tea.KeyDown:
|
||
m.step(1, n)
|
||
return m, nil
|
||
case tea.KeyShiftTab, tea.KeyUp:
|
||
m.step(-1, n)
|
||
return m, nil
|
||
case tea.KeyEnter:
|
||
if m.mfocus < 2*mfCount {
|
||
m.step(1, n)
|
||
return m, nil
|
||
}
|
||
if err := m.validateMachines(); err != "" {
|
||
m.formErr = err
|
||
return m, nil
|
||
}
|
||
m.formErr = ""
|
||
if cmd, waiting := m.startJoins(); waiting {
|
||
m.page = pWait
|
||
return m, cmd
|
||
}
|
||
m.page = pCheck
|
||
cmd := m.startCheck()
|
||
return m, cmd
|
||
}
|
||
if m.mfocus == 2*mfCount {
|
||
return m, nil
|
||
}
|
||
d := &m.db[m.mfocus/mfCount]
|
||
if m.mfocus%mfCount == mfAuth {
|
||
switch k.String() {
|
||
case "left", "h":
|
||
d.auth = (d.auth + authCount - 1) % authCount
|
||
case "right", "l", " ":
|
||
d.auth = (d.auth + 1) % authCount
|
||
}
|
||
if d.auth != authJoin && d.joined != nil {
|
||
_ = d.joined.Close()
|
||
d.joined, d.code, d.arrive = nil, "", nil
|
||
}
|
||
d.secret.SetValue("")
|
||
d.secret.Mask = d.auth == authPassword
|
||
d.secret.Placeholder = map[int]string{authKey: "/root/.ssh/id_ed25519", authPassword: "the user's ssh password"}[d.auth]
|
||
return m, nil
|
||
}
|
||
var cmd tea.Cmd
|
||
t := d.field(m.mfocus % mfCount)
|
||
*t, cmd = t.Update(k)
|
||
return m, cmd
|
||
}
|
||
|
||
// step moves the focus over the fields that are shown.
|
||
func (m *setupModel) step(d, n int) {
|
||
for range n {
|
||
m.mfocus = (m.mfocus + d + n) % n
|
||
if m.mfocus == 2*mfCount || m.db[m.mfocus/mfCount].visible(m.mfocus%mfCount) {
|
||
break
|
||
}
|
||
}
|
||
m.focusMachine()
|
||
}
|
||
|
||
func (m setupModel) allJoined() bool {
|
||
for _, d := range m.db {
|
||
if d.auth == authJoin && d.joined == nil {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// startJoins listens for the machines that join with a code, makes their
|
||
// codes, and waits for them; it says whether there is anything to wait for.
|
||
func (m *setupModel) startJoins() (tea.Cmd, bool) {
|
||
var cmds []tea.Cmd
|
||
for i := range m.db {
|
||
d := &m.db[i]
|
||
if d.auth != authJoin || d.joined != nil {
|
||
continue
|
||
}
|
||
if m.listener == nil {
|
||
// On the address the codes name, not every interface: a code is
|
||
// a credential, and the port it is spent on need not face more
|
||
// than the network the machines join from.
|
||
addr := net.JoinHostPort(m.thisAddr(), fmt.Sprint(joinpkg.DefaultPort))
|
||
l, err := joinpkg.Listen(addr, addr, time.Hour)
|
||
if err != nil {
|
||
m.formErr = err.Error()
|
||
return nil, false
|
||
}
|
||
m.listener = l
|
||
}
|
||
if d.arrive == nil {
|
||
c, ch := m.listener.Code("database")
|
||
d.code, d.arrive = c.String(), ch
|
||
}
|
||
ch, i := d.arrive, i
|
||
cmds = append(cmds, func() tea.Msg { return joinedMsg{i, <-ch} })
|
||
}
|
||
return tea.Batch(cmds...), len(cmds) > 0
|
||
}
|
||
|
||
func (m setupModel) validateMachines() string {
|
||
seen := map[string]bool{m.thisAddr(): true, m.host: true}
|
||
for i, d := range m.db {
|
||
if d.auth == authJoin {
|
||
continue
|
||
}
|
||
addr := strings.TrimSpace(d.addr.Value())
|
||
switch {
|
||
case addr == "":
|
||
return fmt.Sprintf("database machine %d has no address", i+1)
|
||
case seen[addr]:
|
||
return fmt.Sprintf("%s is given twice, or is this machine: the three must be different machines", addr)
|
||
case strings.TrimSpace(d.user.Value()) == "":
|
||
return fmt.Sprintf("database machine %d has no user to log in as", i+1)
|
||
case d.auth != authAgent && d.secret.Value() == "":
|
||
return fmt.Sprintf("database machine %d needs its %s", i+1, authNames[d.auth])
|
||
}
|
||
seen[addr] = true
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// startCheck connects to the database machines and runs the checks on all
|
||
// three, this one included.
|
||
func (m *setupModel) startCheck() tea.Cmd {
|
||
for _, c := range m.conns {
|
||
if c.ssh != nil {
|
||
_ = c.ssh.Close()
|
||
}
|
||
}
|
||
m.conns = nil
|
||
machines := []preflight.Machine{{
|
||
Name: m.host + " (this machine)", Addr: m.thisAddr(), Local: true, Remote: pgha.Local{},
|
||
Roles: []preflight.Role{preflight.RoleControl, preflight.RoleWitness},
|
||
}}
|
||
for _, d := range m.db {
|
||
if d.auth == authJoin {
|
||
m.conns = append(m.conns, dbConn{remote: d.joined, addr: d.joined.Addr, joined: d.joined})
|
||
machines = append(machines, preflight.Machine{Name: d.joined.Hostname + " (" + d.joined.Addr + ", joined)",
|
||
Addr: d.joined.Addr, Roles: []preflight.Role{preflight.RoleDatabase}, Remote: d.joined})
|
||
continue
|
||
}
|
||
g := &onboard.GoSSH{
|
||
Addr: strings.TrimSpace(d.addr.Value()), User: strings.TrimSpace(d.user.Value()),
|
||
UseAgent: d.auth == authAgent, SudoPassword: d.sudo.Value(),
|
||
}
|
||
switch d.auth {
|
||
case authKey:
|
||
g.KeyFile = strings.TrimSpace(d.secret.Value())
|
||
case authPassword:
|
||
g.Password = d.secret.Value()
|
||
}
|
||
m.conns = append(m.conns, dbConn{remote: g, addr: g.Addr, ssh: g})
|
||
machines = append(machines, preflight.Machine{Name: g.Addr, Addr: g.Addr,
|
||
Roles: []preflight.Role{preflight.RoleDatabase}, Remote: g})
|
||
}
|
||
m.checking = true
|
||
m.report = preflight.Report{}
|
||
conns := m.conns
|
||
return func() tea.Msg {
|
||
rep := preflight.Run(context.Background(), machines)
|
||
// A sudo that asks for a password is fine with one given: the
|
||
// install grants itself sudo with it.
|
||
for i := range rep.Findings {
|
||
f := &rep.Findings[i]
|
||
if f.Check == "sudo" && f.Level == preflight.Warn {
|
||
for _, c := range conns {
|
||
g := c.ssh
|
||
switch {
|
||
case g == nil || g.Addr != f.Machine:
|
||
case g.SudoPassword == "":
|
||
f.Level = preflight.Fail
|
||
f.Fix = "give this user's sudo password on the previous screen"
|
||
default:
|
||
f.Level = preflight.OK
|
||
f.Detail = "with the password given"
|
||
f.Fix = ""
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return checkDoneMsg{rep}
|
||
}
|
||
}
|
||
|
||
// targets are the database machines as user@address, for the command line.
|
||
func (m setupModel) targets() []string {
|
||
var out []string
|
||
for _, d := range m.db {
|
||
if d.joined != nil {
|
||
out = append(out, "root@"+d.joined.Addr)
|
||
continue
|
||
}
|
||
out = append(out, strings.TrimSpace(d.user.Value())+"@"+strings.TrimSpace(d.addr.Value()))
|
||
}
|
||
return out
|
||
}
|
||
|
||
func (m setupModel) namesList() []string {
|
||
var out []string
|
||
for _, h := range strings.Split(m.names.Value(), ",") {
|
||
if h = strings.TrimSpace(h); h != "" {
|
||
out = append(out, h)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// command is what this setup is on the command line, for doing it again
|
||
// from a script — with keys in place, since a password cannot be given there.
|
||
func (m setupModel) command() string {
|
||
parts := []string{"sudo --preserve-env=SSH_AUTH_SOCK llmgw server init",
|
||
"--admin " + strings.TrimSpace(m.admin.Value())}
|
||
for _, h := range m.namesList() {
|
||
parts = append(parts, "--host "+h)
|
||
}
|
||
for _, t := range m.targets() {
|
||
parts = append(parts, "--patroni "+t)
|
||
}
|
||
for _, w := range m.webhookList() {
|
||
parts = append(parts, "--webhook "+w)
|
||
}
|
||
parts = append(parts, "--no-node", "--no-model")
|
||
return strings.Join(parts, " \\\n ")
|
||
}
|
||
|
||
// startRun installs: sudo granted where it asks for a password, the control
|
||
// plane here, the store on the other two, then sudo taken back.
|
||
func (m *setupModel) startRun() tea.Cmd {
|
||
in := newDefaultServerInit()
|
||
in.admin = strings.TrimSpace(m.admin.Value())
|
||
in.hosts = m.namesList()
|
||
in.patroni = m.targets()
|
||
in.webhooks = m.webhookList()
|
||
// The control plane of a replicated setup is not where models run;
|
||
// nodes are added once it is up.
|
||
in.noNode, in.noModel = true, true
|
||
m.in = &in
|
||
m.run = newInitModel(&in, nil, false)
|
||
m.run.running, m.run.width, m.run.spin = true, m.width, m.spin
|
||
ref, conns, logp, mu, listener := m.ref, m.conns, m.log, m.logMu, m.listener
|
||
return func() tea.Msg {
|
||
send := func(l string) {
|
||
mu.Lock()
|
||
*logp = append(*logp, l)
|
||
mu.Unlock()
|
||
ref.p.Send(initLineMsg(l))
|
||
}
|
||
in.out = &lineWriter{send: send}
|
||
err := func() error {
|
||
ctx := context.Background()
|
||
for _, c := range conns {
|
||
if c.ssh != nil {
|
||
fmt.Fprintf(in.out, " .. sudo on %s\n", c.addr)
|
||
if err := c.ssh.PrepareSudo(ctx); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
out, err := c.remote.Run(ctx, "hostname", nil)
|
||
if err != nil {
|
||
return fmt.Errorf("%s: %v", c.addr, err)
|
||
}
|
||
ip, err := ipv4Of(c.addr)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
in.patroniNodes = append(in.patroniNodes, pgha.Node{
|
||
Member: pgha.Member{Name: strings.TrimSpace(string(out)), IP: ip}, Remote: c.remote})
|
||
}
|
||
return in.run(ctx)
|
||
}()
|
||
// Whatever happened: a sudo grant taken back, a joined machine's
|
||
// `llmgw join` told it is done, the port for joining closed.
|
||
for _, c := range conns {
|
||
if c.ssh != nil {
|
||
_ = c.ssh.Close()
|
||
}
|
||
if c.joined != nil {
|
||
_ = c.joined.Close()
|
||
}
|
||
}
|
||
if listener != nil {
|
||
_ = listener.Close()
|
||
}
|
||
return initDoneMsg{err}
|
||
}
|
||
}
|
||
|
||
// setupSteps are the sidebar's steps, by page.
|
||
var setupSteps = []struct {
|
||
label string
|
||
page setupPage
|
||
}{
|
||
{"Start", pWelcome}, {"Administrator", pAdmin}, {"Machines", pMachines},
|
||
{"Checks", pCheck}, {"Plan", pPlan}, {"Install", pRun},
|
||
}
|
||
|
||
func (m setupModel) View() string {
|
||
at := m.page
|
||
if at == pWait {
|
||
at = pMachines
|
||
}
|
||
var nav []navItem
|
||
for _, st := range setupSteps {
|
||
if m.choice == 0 && st.page > pWelcome {
|
||
break
|
||
}
|
||
state := "todo"
|
||
switch {
|
||
case st.page < at || (st.page == pRun && m.finished && m.run.err == nil):
|
||
state = "done"
|
||
case st.page == at:
|
||
state = "current"
|
||
}
|
||
nav = append(nav, navItem{Label: st.label, State: state})
|
||
}
|
||
title, body, keys := m.pageView()
|
||
f := frame{
|
||
Width: m.width, Height: m.height,
|
||
Title: "llmgw setup", Where: m.host + " · " + m.thisAddr(),
|
||
NavTitle: "Steps", Nav: nav,
|
||
PanelTitle: title, Body: body, Scroll: m.scroll, Keys: keys,
|
||
}
|
||
if m.formErr != "" {
|
||
f.DialogTitle, f.Dialog = "Not yet", m.formErr+"\n\n"+sDim.Render("any key to go back")
|
||
}
|
||
return f.render()
|
||
}
|
||
|
||
// pageView is the panel for the page: its title, what is in it, and its keys.
|
||
func (m setupModel) pageView() (string, string, string) {
|
||
switch m.page {
|
||
case pWelcome:
|
||
return "Start", m.welcomeView(), "↑↓ choose · Enter go on · Ctrl+C leave"
|
||
case pAdmin:
|
||
return "Administrator", m.adminView(), "Tab/↑↓ move · Enter next · Esc back · Ctrl+C leave"
|
||
case pMachines:
|
||
return "Database machines", m.machinesView(), "Tab/↑↓ move · ←→ how to log in · Enter next · Esc back · PgUp/PgDn scroll"
|
||
case pWait:
|
||
return "Waiting for machines to join", m.waitView(), "Esc back · PgUp/PgDn scroll · Ctrl+C leave"
|
||
case pCheck:
|
||
keys := "R check again · Esc back · PgUp/PgDn scroll"
|
||
if !m.checking && m.report.Worst() < preflight.Fail {
|
||
keys = "Enter see the plan · " + keys
|
||
}
|
||
return "Checks", m.checkView(), keys
|
||
case pPlan:
|
||
return "Plan", m.planView(), "Enter install · Esc back · PgUp/PgDn scroll · Ctrl+C leave"
|
||
}
|
||
v := m.run.View()
|
||
if i := strings.Index(v, "\n\n"); i >= 0 {
|
||
v = v[i+2:] // init's own title; the frame has one
|
||
}
|
||
keys := "Ctrl+C stop"
|
||
if m.finished {
|
||
keys = "Enter or Q leave · PgUp/PgDn scroll"
|
||
if m.run.err == nil {
|
||
v += "\n" + sDim.Render("The same, from a script, with ssh keys in place:") + "\n" + m.command() + "\n"
|
||
}
|
||
}
|
||
return "Install", v, keys
|
||
}
|
||
|
||
func (m setupModel) welcomeView() string {
|
||
var b strings.Builder
|
||
fmt.Fprintf(&b, "\n Nothing of llmgw is installed on %s.\n\n", lipgloss.NewStyle().Bold(true).Render(m.host))
|
||
opt := func(i int, title, detail string) {
|
||
mark := "( )"
|
||
if m.choice == i {
|
||
mark = "(" + sOK.Render("•") + ")"
|
||
title = fSelected.Render(title)
|
||
}
|
||
fmt.Fprintf(&b, " %s %s\n %s\n\n", mark, title, sDim.Render(detail))
|
||
}
|
||
opt(0, "One machine", "This machine runs everything: control plane, model mirror,\n and models if it has room.")
|
||
opt(1, "High availability — three machines",
|
||
"This one is the control plane; two more keep the store, replicated,\n so losing any one machine loses nothing.")
|
||
return b.String()
|
||
}
|
||
|
||
func (m setupModel) adminView() string {
|
||
var b strings.Builder
|
||
label := func(i int, s string) string {
|
||
if m.afocus == i {
|
||
return fSelected.Render("› " + s)
|
||
}
|
||
return " " + s
|
||
}
|
||
b.WriteString("\n")
|
||
fmt.Fprintf(&b, "%s\n %s\n %s\n\n", label(0, "Administrator's email"), m.admin.View(),
|
||
sDim.Render("A password is generated and shown once, at the end."))
|
||
fmt.Fprintf(&b, "%s\n %s\n %s\n\n", label(1, "Other names nodes will reach this machine by"), m.names.View(),
|
||
sDim.Render("Known: "+strings.Join(append([]string{m.host}, m.ips...), ", ")+". Add a DNS name or a floating address."))
|
||
fmt.Fprintf(&b, "%s\n %s\n %s\n\n", label(2, "Notification webhooks"), m.webhooks.View(),
|
||
sDim.Render("Where a parked model, a lost node or a spent budget is said: Slack, Mattermost, or any URL taking JSON. Optional; a test is sent."))
|
||
b.WriteString(button(m.afocus == 3, "Next") + "\n")
|
||
return b.String()
|
||
}
|
||
|
||
// button is a [ Label ] that shows when it has the focus.
|
||
func button(focused bool, label string) string {
|
||
if focused {
|
||
return " " + fBar.Render(" "+label+" ")
|
||
}
|
||
return " " + lipgloss.NewStyle().Bold(true).Render("[ "+label+" ]")
|
||
}
|
||
|
||
func (m setupModel) machinesView() string {
|
||
var b strings.Builder
|
||
b.WriteString("\n The two machines that keep the store: Ubuntu 22.04/24.04 or Debian 12.\n" +
|
||
" PostgreSQL, Patroni and etcd are installed on them.\n\n")
|
||
for i, d := range m.db {
|
||
base := i * mfCount
|
||
// Padded before it is coloured: the colour's escape codes count
|
||
// toward a width and threw the focused row out of line.
|
||
label := func(f int, s string) string {
|
||
s = fmt.Sprintf("%-15s", s)
|
||
if m.mfocus == base+f {
|
||
return fSelected.Render("› " + s)
|
||
}
|
||
return " " + s
|
||
}
|
||
fmt.Fprintf(&b, " %s\n", lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("Database machine %d", i+1)))
|
||
if d.visible(mfAddr) {
|
||
fmt.Fprintf(&b, " %s %s\n", label(mfAddr, "Address"), d.addr.View())
|
||
fmt.Fprintf(&b, " %s %s\n", label(mfUser, "User"), d.user.View())
|
||
}
|
||
var auth []string
|
||
for a := 0; a < authCount; a++ {
|
||
if a == d.auth {
|
||
auth = append(auth, "("+sOK.Render("•")+") "+authNames[a])
|
||
} else {
|
||
auth = append(auth, "( ) "+sDim.Render(authNames[a]))
|
||
}
|
||
}
|
||
fmt.Fprintf(&b, " %s %s\n", label(mfAuth, "Log in with"), strings.Join(auth, " "))
|
||
if d.visible(mfSecret) {
|
||
fmt.Fprintf(&b, " %s %s\n", label(mfSecret, strings.ToUpper(authNames[d.auth][:1])+authNames[d.auth][1:]), d.secret.View())
|
||
}
|
||
if d.visible(mfSudo) {
|
||
fmt.Fprintf(&b, " %s %s\n", label(mfSudo, "Sudo password"), d.sudo.View())
|
||
}
|
||
if d.auth == authJoin {
|
||
b.WriteString(" " + sDim.Render("Nothing logs into it: it runs llmgw join with a code shown next.") + "\n")
|
||
}
|
||
b.WriteString("\n")
|
||
}
|
||
b.WriteString(button(m.mfocus == 2*mfCount, "Check the three machines") + "\n\n")
|
||
b.WriteString(" " + sDim.Render("Passwords are kept in memory only, for this setup.") + "\n")
|
||
return b.String()
|
||
}
|
||
|
||
func (m setupModel) waitView() string {
|
||
var b strings.Builder
|
||
fmt.Fprintf(&b, "\n On each machine below, as root. It dials this machine on port %d;\n nothing dials in. Each code is for one machine and good for an hour.\n\n", joinpkg.DefaultPort)
|
||
for i, d := range m.db {
|
||
if d.auth != authJoin {
|
||
continue
|
||
}
|
||
title := lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("Database machine %d", i+1))
|
||
if d.joined != nil {
|
||
fmt.Fprintf(&b, " %s %s joined as %s (%s)\n\n", title, sOK.Render("✓"), d.joined.Hostname, d.joined.Addr)
|
||
continue
|
||
}
|
||
fmt.Fprintf(&b, " %s %s waiting\n", title, m.spin.View())
|
||
b.WriteString(" " + sDim.Render("without llmgw yet:") + "\n")
|
||
for _, l := range joinInstall() {
|
||
b.WriteString(" " + strings.ReplaceAll(l, "\n", "\n ") + "\n")
|
||
}
|
||
b.WriteString(" " + sDim.Render("then:") + "\n sudo llmgw join " + d.code + "\n\n")
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
func (m setupModel) checkView() string {
|
||
var b strings.Builder
|
||
b.WriteString("\n")
|
||
if m.checking {
|
||
b.WriteString(" " + m.spin.View() + " checking the three machines…\n")
|
||
return b.String()
|
||
}
|
||
switch m.report.Worst() {
|
||
case preflight.Fail:
|
||
b.WriteString(" " + sBad.Render("✗ Fix what is marked ✗, then check again (R).") + "\n")
|
||
case preflight.Warn:
|
||
b.WriteString(" " + sWarn.Render("! It can go ahead; what is marked ! is worth a look.") + "\n")
|
||
default:
|
||
b.WriteString(" " + sOK.Render("✓ All three are ready.") + "\n")
|
||
}
|
||
machine := ""
|
||
for _, f := range m.report.Findings {
|
||
if f.Machine != machine {
|
||
machine = f.Machine
|
||
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render(machine) + "\n")
|
||
}
|
||
mark := sOK.Render("✓")
|
||
switch f.Level {
|
||
case preflight.Warn:
|
||
mark = sWarn.Render("!")
|
||
case preflight.Fail:
|
||
mark = sBad.Render("✗")
|
||
}
|
||
fmt.Fprintf(&b, " %s %-13s %s\n", mark, f.Check, f.Detail)
|
||
if f.Level != preflight.OK && f.Fix != "" {
|
||
b.WriteString(" " + sDim.Render(f.Fix) + "\n")
|
||
}
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
func (m setupModel) planView() string {
|
||
var b strings.Builder
|
||
b.WriteString("\n What will be installed:\n\n")
|
||
fmt.Fprintf(&b, " %s %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(m.host), sDim.Render(m.thisAddr()),
|
||
"control plane and gateway, model mirror, etcd (the third vote)")
|
||
fmt.Fprintf(&b, " administrator %s\n", strings.TrimSpace(m.admin.Value()))
|
||
if hooks, _ := parseWebhooks(m.webhookList()); len(hooks) > 0 {
|
||
var where []string
|
||
for _, h := range hooks {
|
||
where = append(where, notify.Redact(h.URL)+" ("+h.Format+")")
|
||
}
|
||
fmt.Fprintf(&b, " alerts to %s\n", strings.Join(where, ", "))
|
||
}
|
||
granted := false
|
||
for _, d := range m.db {
|
||
addr := strings.TrimSpace(d.addr.Value())
|
||
if d.joined != nil {
|
||
addr = d.joined.Hostname + " " + sDim.Render(d.joined.Addr+", joined")
|
||
}
|
||
fmt.Fprintf(&b, " %s\n PostgreSQL under Patroni, etcd\n", lipgloss.NewStyle().Bold(true).Render(addr))
|
||
granted = granted || d.sudo.Value() != ""
|
||
}
|
||
b.WriteString("\n The store is replicated from the first to the second; if the primary is lost\n" +
|
||
" the standby takes over in about thirty seconds, and the control plane follows.\n")
|
||
if granted {
|
||
b.WriteString(" " + sDim.Render("Where a sudo password was given, the user may sudo without it until the\n install finishes, and not after.") + "\n")
|
||
}
|
||
b.WriteString("\n " + sDim.Render("The same from a script, with ssh keys in place:") + "\n")
|
||
for _, l := range strings.Split(m.command(), "\n") {
|
||
b.WriteString(" " + l + "\n")
|
||
}
|
||
for _, d := range m.db {
|
||
if d.joined != nil {
|
||
b.WriteString(" " + sDim.Render("(a machine that joined with a code needs ssh for that)") + "\n")
|
||
break
|
||
}
|
||
}
|
||
b.WriteString("\n" + button(true, "Enter to install") + "\n")
|
||
return b.String()
|
||
}
|
||
|
||
// runSetupTUI is the setup screens, until they finish or hand over to init's
|
||
// own form for one machine.
|
||
func runSetupTUI(ctx context.Context, out io.Writer) error {
|
||
host, _ := os.Hostname()
|
||
var ips []string
|
||
for _, ip := range localAddrs() {
|
||
if ip.To4() != nil {
|
||
ips = append(ips, ip.String())
|
||
}
|
||
}
|
||
m := newSetupModel(host, ips)
|
||
p := tea.NewProgram(m, tea.WithOutput(out), tea.WithAltScreen())
|
||
m.ref.p = p
|
||
final, err := p.Run()
|
||
if err != nil {
|
||
return err
|
||
}
|
||
fm := final.(setupModel)
|
||
if fm.single {
|
||
in := newDefaultServerInit()
|
||
in.out = out
|
||
return in.runInitTUI(ctx, out)
|
||
}
|
||
if fm.page < pRun {
|
||
return nil
|
||
}
|
||
fmt.Fprintln(out)
|
||
if fm.run.err != nil {
|
||
fm.logMu.Lock()
|
||
fmt.Fprintln(out, strings.Join(*fm.log, "\n"))
|
||
fm.logMu.Unlock()
|
||
return fm.run.err
|
||
}
|
||
if !fm.finished {
|
||
return fmt.Errorf("left while installing; running llmgw again finishes what was started")
|
||
}
|
||
if fm.run.password != "" {
|
||
fmt.Fprintf(out, "Administrator %s, password %s (shown once)\n\n", fm.in.admin, fm.run.password)
|
||
}
|
||
fmt.Fprintln(out, "Next: add nodes from the console — run llmgw again — or with")
|
||
fmt.Fprintln(out, " sudo llmgw node add <user>@<address>")
|
||
return nil
|
||
}
|