Setup lets every part be chosen, each with its settings; three machines each run the gateway
CI / test (push) Failing after 30m4s

The setup screens list the parts of an install: control plane, store, model
mirror, this machine as a node, starter model, pictures and 3D, the judge,
firewall, notifications, limits, on-demand turns and an offline bundle. Each
says what it is or what is missing without it. Space turns a part on or off;
Enter opens its settings in a dialog that explains it. A plan page lists what
is installed and what is left out, and the same install as a command. Every
setting is a server init flag too, written into server.yaml when init writes
it.

With three machines, the two database machines become control planes too,
standing by (server add's steps), with an optional floating address. The
mirror's management API answers all three control planes, so a standby that
comes to lead can still fetch models.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-15 11:22:36 +02:00
1 parent 733e20924c
commit 94401e21e6
16 files changed
+2255 -483

No files matched your search

+61 -152
View File
@@ -5,7 +5,6 @@ import (
"fmt"
"io"
"net"
"net/mail"
"os"
"strings"
"sync"
@@ -15,7 +14,6 @@ import (
"github.com/charmbracelet/lipgloss"
joinpkg "gitea.mynodes.xyz/ilker/llmgw/internal/join"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
"gitea.mynodes.xyz/ilker/llmgw/internal/preflight"
@@ -36,7 +34,8 @@ type setupPage int
const (
pWelcome setupPage = iota
pAdmin
// pChoices is what is installed, part by part, each with its settings.
pChoices
pMachines
// pWait shows the codes for machines joining rather than logged into,
// until they have all dialled in.
@@ -151,8 +150,7 @@ type setupModel struct {
ips []string
choice int // 0 one machine, 1 high availability
admin, names, webhooks textField
afocus int // admin, names, webhooks, next
choices *installForm
db [2]dbMachine
mfocus int // over both machines' fields, then the check button
@@ -181,14 +179,13 @@ type setupModel struct {
}
func newSetupModel(host string, ips []string) setupModel {
admin := newTextField()
admin.Placeholder, admin.CharLimit, admin.Width = "you@example.com", 200, 40
names := newTextField()
names.Placeholder, names.CharLimit, names.Width = "llm.example.com, 203.0.113.10", 400, 40
webhooks := newTextField()
webhooks.Placeholder, webhooks.CharLimit, webhooks.Width = "https://hooks.slack.com/services/…", 800, 40
in := newDefaultServerInit()
// The control plane of a replicated setup is not where models run;
// nodes are added once it is up. Either can still be chosen.
in.noNode, in.noModel = true, true
in.standbys = true
return setupModel{
ref: &progRef{}, host: host, ips: ips, admin: admin, names: names, webhooks: webhooks,
ref: &progRef{}, host: host, ips: ips, in: &in, choices: newInstallForm(&in, host, ips, true),
db: [2]dbMachine{newDBMachine(1), newDBMachine(2)}, spin: newSpinner(), width: 100, height: 30,
log: &[]string{}, logMu: &sync.Mutex{},
}
@@ -207,6 +204,7 @@ func (m setupModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
switch msg := msg.(type) {
case tea.WindowSizeMsg:
m.width, m.height = msg.Width, msg.Height
m.choices.height = msg.Height
m.run.width = msg.Width - navWidth - 6
case spinTickMsg:
var cmd tea.Cmd
@@ -244,6 +242,11 @@ func (m setupModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
}
func (m setupModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
// The choices' own dialogs take their keys, Esc among them.
if m.page == pChoices && (m.choices.dialog != nil || m.choices.err != "") {
m.choices.key(k, m.in)
return m, nil
}
// A dialog open takes the next key, whatever it is, to close it.
if m.formErr != "" {
m.formErr = ""
@@ -275,12 +278,13 @@ func (m setupModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
m.single = true
return m, tea.Quit
}
m.page = pAdmin
m.afocus = 0
m.admin.Focus()
m.page = pChoices
}
case pChoices:
if m.choices.key(k, m.in) {
m.page, m.mfocus = pMachines, 0
m.focusMachine()
}
case pAdmin:
return m.adminKey(k)
case pMachines:
return m.machinesKey(k)
case pCheck:
@@ -309,73 +313,6 @@ func (m setupModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
return m, nil
}
func (m setupModel) adminKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
move := func(d int) {
m.afocus = (m.afocus + d + 4) % 4
m.admin.Blur()
m.names.Blur()
m.webhooks.Blur()
switch m.afocus {
case 0:
m.admin.Focus()
case 1:
m.names.Focus()
case 2:
m.webhooks.Focus()
}
}
switch k.Type {
case tea.KeyTab, tea.KeyDown:
move(1)
return m, nil
case tea.KeyShiftTab, tea.KeyUp:
move(-1)
return m, nil
case tea.KeyEnter:
if m.afocus < 3 {
move(1)
return m, nil
}
if _, err := mail.ParseAddress(strings.TrimSpace(m.admin.Value())); err != nil {
m.formErr = "the administrator's email is needed, and that is not one"
m.afocus = 0
move(0)
return m, nil
}
if _, err := parseWebhooks(m.webhookList()); err != nil {
m.formErr = err.Error()
m.afocus = 2
move(0)
return m, nil
}
m.formErr = ""
m.page, m.mfocus = pMachines, 0
m.db[0].addr.Focus()
return m, nil
}
var cmd tea.Cmd
switch m.afocus {
case 0:
m.admin, cmd = m.admin.Update(k)
case 1:
m.names, cmd = m.names.Update(k)
case 2:
m.webhooks, cmd = m.webhooks.Update(k)
}
return m, cmd
}
// webhookList is the webhooks asked for, one an item.
func (m setupModel) webhookList() []string {
var out []string
for _, s := range strings.Split(m.webhooks.Value(), ",") {
if s = strings.TrimSpace(s); s != "" {
out = append(out, s)
}
}
return out
}
func (m *setupModel) focusMachine() {
for i := range m.db {
for f := 0; f < mfCount; f++ {
@@ -534,7 +471,7 @@ func (m *setupModel) startCheck() tea.Cmd {
if d.auth == authJoin {
m.conns = append(m.conns, dbConn{remote: d.joined, addr: d.joined.Addr, joined: d.joined})
machines = append(machines, preflight.Machine{Name: d.joined.Hostname + " (" + d.joined.Addr + ", joined)",
Addr: d.joined.Addr, Roles: []preflight.Role{preflight.RoleDatabase}, Remote: d.joined})
Addr: d.joined.Addr, Roles: m.dbRoles(), Remote: d.joined})
continue
}
g := &onboard.GoSSH{
@@ -549,7 +486,7 @@ func (m *setupModel) startCheck() tea.Cmd {
}
m.conns = append(m.conns, dbConn{remote: g, addr: g.Addr, ssh: g})
machines = append(machines, preflight.Machine{Name: g.Addr, Addr: g.Addr,
Roles: []preflight.Role{preflight.RoleDatabase}, Remote: g})
Roles: m.dbRoles(), Remote: g})
}
m.checking = true
m.report = preflight.Report{}
@@ -580,6 +517,15 @@ func (m *setupModel) startCheck() tea.Cmd {
}
}
// dbRoles are what the other two machines are to be: the store, and a
// control plane too when they stand by.
func (m setupModel) dbRoles() []preflight.Role {
if m.in.standbys {
return []preflight.Role{preflight.RoleDatabase, preflight.RoleControl}
}
return []preflight.Role{preflight.RoleDatabase}
}
// targets are the database machines as user@address, for the command line.
func (m setupModel) targets() []string {
var out []string
@@ -593,47 +539,20 @@ func (m setupModel) targets() []string {
return out
}
func (m setupModel) namesList() []string {
var out []string
for _, h := range strings.Split(m.names.Value(), ",") {
if h = strings.TrimSpace(h); h != "" {
out = append(out, h)
}
}
return out
}
// command is what this setup is on the command line, for doing it again
// from a script — with keys in place, since a password cannot be given there.
func (m setupModel) command() string {
parts := []string{"sudo --preserve-env=SSH_AUTH_SOCK llmgw server init",
"--admin " + strings.TrimSpace(m.admin.Value())}
for _, h := range m.namesList() {
parts = append(parts, "--host "+h)
}
for _, t := range m.targets() {
parts = append(parts, "--patroni "+t)
}
for _, w := range m.webhookList() {
parts = append(parts, "--webhook "+w)
}
parts = append(parts, "--no-node", "--no-model")
return strings.Join(parts, " \\\n ")
in := *m.in
in.patroni = m.targets()
return strings.Join(append([]string{"sudo --preserve-env=SSH_AUTH_SOCK llmgw server init"}, in.initFlags()...), " \\\n ")
}
// startRun installs: sudo granted where it asks for a password, the control
// plane here, the store on the other two, then sudo taken back.
func (m *setupModel) startRun() tea.Cmd {
in := newDefaultServerInit()
in.admin = strings.TrimSpace(m.admin.Value())
in.hosts = m.namesList()
in := m.in
in.patroni = m.targets()
in.webhooks = m.webhookList()
// The control plane of a replicated setup is not where models run;
// nodes are added once it is up.
in.noNode, in.noModel = true, true
m.in = &in
m.run = newInitModel(&in, nil, false)
m.run = newInitModel(in, nil, false)
m.run.running, m.run.width, m.run.spin = true, m.width, m.spin
ref, conns, logp, mu, listener := m.ref, m.conns, m.log, m.logMu, m.listener
return func() tea.Msg {
@@ -688,7 +607,7 @@ var setupSteps = []struct {
label string
page setupPage
}{
{"Start", pWelcome}, {"Administrator", pAdmin}, {"Machines", pMachines},
{"Start", pWelcome}, {"Choices", pChoices}, {"Machines", pMachines},
{"Checks", pCheck}, {"Plan", pPlan}, {"Install", pRun},
}
@@ -721,6 +640,9 @@ func (m setupModel) View() string {
if m.formErr != "" {
f.DialogTitle, f.Dialog = "Not yet", m.formErr+"\n\n"+sDim.Render("any key to go back")
}
if m.page == pChoices {
f.DialogTitle, f.Dialog = m.choices.overlay()
}
return f.render()
}
@@ -729,8 +651,12 @@ func (m setupModel) pageView() (string, string, string) {
switch m.page {
case pWelcome:
return "Start", m.welcomeView(), "↑↓ choose · Enter go on · Ctrl+C leave"
case pAdmin:
return "Administrator", m.adminView(), "Tab/↑↓ move · Enter next · Esc back · Ctrl+C leave"
case pChoices:
keys := m.choices.keys()
if m.choices.dialog == nil && m.choices.err == "" {
keys = strings.Replace(keys, " · Ctrl+C", " · Esc back · Ctrl+C", 1)
}
return "What to install", m.choices.view(m.width - navWidth - 6), keys
case pMachines:
return "Database machines", m.machinesView(), "Tab/↑↓ move · ←→ how to log in · Enter next · Esc back · PgUp/PgDn scroll"
case pWait:
@@ -771,26 +697,8 @@ func (m setupModel) welcomeView() string {
}
opt(0, "One machine", "This machine runs everything: control plane, model mirror,\n and models if it has room.")
opt(1, "High availability — three machines",
"This one is the control plane; two more keep the store, replicated,\n so losing any one machine loses nothing.")
return b.String()
}
func (m setupModel) adminView() string {
var b strings.Builder
label := func(i int, s string) string {
if m.afocus == i {
return fSelected.Render("› " + s)
}
return " " + s
}
b.WriteString("\n")
fmt.Fprintf(&b, "%s\n %s\n %s\n\n", label(0, "Administrator's email"), m.admin.View(),
sDim.Render("A password is generated and shown once, at the end."))
fmt.Fprintf(&b, "%s\n %s\n %s\n\n", label(1, "Other names nodes will reach this machine by"), m.names.View(),
sDim.Render("Known: "+strings.Join(append([]string{m.host}, m.ips...), ", ")+". Add a DNS name or a floating address."))
fmt.Fprintf(&b, "%s\n %s\n %s\n\n", label(2, "Notification webhooks"), m.webhooks.View(),
sDim.Render("Where a parked model, a lost node or a spent budget is said: Slack, Mattermost, or any URL taking JSON. Optional; a test is sent."))
b.WriteString(button(m.afocus == 3, "Next") + "\n")
"Each of the three can run the gateway; two keep the store, replicated,\n"+
" and etcd votes on all three, so losing any one machine loses nothing.")
return b.String()
}
@@ -907,24 +815,25 @@ func (m setupModel) checkView() string {
func (m setupModel) planView() string {
var b strings.Builder
b.WriteString("\n What will be installed:\n\n")
fmt.Fprintf(&b, " %s %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(m.host), sDim.Render(m.thisAddr()),
"control plane and gateway, model mirror, etcd (the third vote)")
fmt.Fprintf(&b, " administrator %s\n", strings.TrimSpace(m.admin.Value()))
if hooks, _ := parseWebhooks(m.webhookList()); len(hooks) > 0 {
var where []string
for _, h := range hooks {
where = append(where, notify.Redact(h.URL)+" ("+h.Format+")")
}
fmt.Fprintf(&b, " alerts to %s\n", strings.Join(where, ", "))
b.WriteString(m.choices.planView(m.width - navWidth - 6))
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("The three machines") + "\n")
here := "control plane, leading at first; model mirror; etcd"
there := "PostgreSQL under Patroni, etcd"
if m.in.standbys {
there = "control plane, standing by; " + there
}
if m.in.vip != "" {
here += "; keepalived"
there += "; keepalived"
}
fmt.Fprintf(&b, " %s %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(m.host), sDim.Render(m.thisAddr()), here)
granted := false
for _, d := range m.db {
addr := strings.TrimSpace(d.addr.Value())
if d.joined != nil {
addr = d.joined.Hostname + " " + sDim.Render(d.joined.Addr+", joined")
}
fmt.Fprintf(&b, " %s\n PostgreSQL under Patroni, etcd\n", lipgloss.NewStyle().Bold(true).Render(addr))
fmt.Fprintf(&b, " %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(addr), there)
granted = granted || d.sudo.Value() != ""
}
b.WriteString("\n The store is replicated from the first to the second; if the primary is lost\n" +