From 38f2671ed957abcac8d92de6e5934b75c4ac7851 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sat, 3 Oct 2026 21:33:59 +0200 Subject: [PATCH] Open setup screens or a console when llmgw runs bare in a terminal With nothing installed, sudo llmgw asks for one machine or three, takes how to reach the other two (ssh agent, key or password, and a sudo password used only for the install), checks every machine first, shows the plan and its command line, and installs. On a control plane it opens a console with the cluster's state and the next steps. Machines are reached with Go's ssh so a password works; the store's etcd and access rules use this machine's routed address rather than a client-facing name. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 28 + docs/install.md | 17 +- internal/cli/root.go | 8 + internal/cli/serverinit.go | 14 +- internal/cli/serverinit_patroni.go | 19 +- internal/cli/serverinit_postgres.go | 2 +- internal/cli/tui_home.go | 259 ++++++++++ internal/cli/tui_setup.go | 741 +++++++++++++++++++++++++++ internal/cli/tui_setup_test.go | 119 +++++ internal/cli/tui_widgets.go | 5 + internal/onboard/gossh.go | 278 ++++++++++ internal/preflight/preflight.go | 360 +++++++++++++ internal/preflight/preflight_test.go | 109 ++++ 13 files changed, 1954 insertions(+), 5 deletions(-) create mode 100644 internal/cli/tui_home.go create mode 100644 internal/cli/tui_setup.go create mode 100644 internal/cli/tui_setup_test.go create mode 100644 internal/onboard/gossh.go create mode 100644 internal/preflight/preflight.go create mode 100644 internal/preflight/preflight_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 2201aa28..2ba3a36d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,6 +75,34 @@ and a figure the driver does not give is left out rather than shown as zero. each card and whose quota it counts against, for joining the GPU series to models and tenants. +### Setting up from nothing in screens: `sudo llmgw` + +Run with nothing after it in a terminal, as root, `llmgw` opens full-screen +setup on a machine with nothing installed, and a console on a control plane. +In a pipe or a script it prints its help, as before. + +- **Setup** asks one machine or three. For three — this one as the control + plane and etcd's third vote, two more for the store under Patroni — it asks + how to reach the other two: the ssh agent, a key file, or a password, and a + sudo password where sudo asks for one. Passwords stay in memory for the run; + a sudo password grants the user passwordless sudo until the install + finishes, and the grant is removed after, failed or not. +- **Every machine is checked before anything is installed**: login, system + (Ubuntu 22.04/24.04, Debian 12), architecture, CPUs, memory and disk for its + role, clock against this one, NTP, the ports its role needs, internet, and + whether it reaches the other machines. Failures stop it, with what to do; + warnings let it go on. +- **Then the plan, and the command line that does the same** from a script, + then the install with its progress, and the administrator's password. +- **The console** shows `ctl status`, refreshed every five seconds, and offers + to add a node or, for a store on one machine, to replicate it. +- Machines are reached with Go's ssh client rather than the system's, so a + password can be used; a host key is recorded on first sight and refused if it + changes. + +Tried end to end on three new machines whose user logs in and sudoes only with +a password. + ### A store that survives losing its machine `deploy/ha/` now covers the store as well as the control planes: PostgreSQL diff --git a/docs/install.md b/docs/install.md index 91216d45..5b1a63e4 100644 --- a/docs/install.md +++ b/docs/install.md @@ -87,7 +87,22 @@ CGO_ENABLED=0 go build -o llmgw ./cmd/llmgw ## 2. The control plane -On the machine that will be the control plane, as root: +**The short way is the screens.** On the machine that will be the control +plane, run `sudo llmgw` with nothing after it. With nothing installed it asks +whether this is one machine or three for high availability; for three, it asks +for the other two — address, user, and an ssh key, the ssh agent, or a +password, and the sudo password if sudo asks for one — checks all three before +installing anything (system, room, clock, ports, internet, whether they reach +one another), shows the plan, and installs. Passwords are held in memory for +the run only; a sudo password lets the user sudo without one until the install +finishes, and is then taken back. At the end it shows the administrator's +password and the command that does the same from a script. + +Run again later, `sudo llmgw` is the console: the cluster's state, refreshed, +and what to do next — add a node, replicate the store. A setup that stopped +halfway opens the setup screens again, which finish it. + +The rest of this section is the same by command, as root: ``` sudo llmgw server init --admin you@example.com diff --git a/internal/cli/root.go b/internal/cli/root.go index 1245bc8a..a1d46133 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -30,6 +30,14 @@ func Root() *cobra.Command { PersistentPreRun: func(_ *cobra.Command, _ []string) { obs.Setup(obs.Options{Level: global.logLevel, Format: global.logFormat}) }, + // Bare, in a terminal: the setup screens or the console. Otherwise + // the help, as before. + RunE: func(cmd *cobra.Command, _ []string) error { + if interactive(cmd) { + return runHome(cmd) + } + return cmd.Help() + }, } pf := root.PersistentFlags() diff --git a/internal/cli/serverinit.go b/internal/cli/serverinit.go index 23d0111d..408a183e 100644 --- a/internal/cli/serverinit.go +++ b/internal/cli/serverinit.go @@ -23,6 +23,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/account" "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/pgha" "gitea.mynodes.xyz/ilker/llmgw/internal/store" "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" ) @@ -72,6 +73,10 @@ type serverInit struct { // patroni are user@host of the machines to keep the store on under // Patroni, replicated; this machine is the third etcd vote for two. patroni []string + // patroniNodes are the same machines already connected to, which the + // setup screen does with the password it was given; when set, patroni + // only names them. + patroniNodes []pgha.Node // noMirror, noNode and noModel turn off the parts of a first install that // make it serve something: a mirror, this machine as a node, and a small // model on it. @@ -89,14 +94,19 @@ type serverInit struct { changed bool } -func newServerInitCmd() *cobra.Command { - in := serverInit{ +// newDefaultServerInit is init as it sets up a real machine. +func newDefaultServerInit() serverInit { + return serverInit{ etcDir: "/etc/llmgw", stateDir: "/var/lib/llmgw", pkiDir: defaultPKIDir, unitPath: "/etc/systemd/system/llmgw-server.service", user: "llmgw", } +} + +func newServerInitCmd() *cobra.Command { + in := newDefaultServerInit() var plain bool cmd := &cobra.Command{ Use: "init", diff --git a/internal/cli/serverinit_patroni.go b/internal/cli/serverinit_patroni.go index 3342dea2..075434c1 100644 --- a/internal/cli/serverinit_patroni.go +++ b/internal/cli/serverinit_patroni.go @@ -26,7 +26,11 @@ func (in *serverInit) ensurePatroni(ctx context.Context) (err error) { in.kept("%s: %s", step, detail) } }} + setup.DB = in.patroniNodes for _, t := range in.patroni { + if len(in.patroniNodes) > 0 { + break + } user, host, ok := strings.Cut(t, "@") if !ok { user, host = "", t @@ -46,7 +50,7 @@ func (in *serverInit) ensurePatroni(ctx context.Context) (err error) { Remote: remote, }) } - self, err := ipv4Of(in.reachableAddr()) + self, err := sourceIPToward(setup.DB[0].IP) if err != nil { return err } @@ -146,6 +150,19 @@ func (in *serverInit) moveInto(ctx context.Context, from, to string) error { return nil } +// sourceIPToward is this machine's address on the route to another: the one +// the other machine sees it connect from, which is what etcd advertises and +// what the database lets in. Not the first --host, which names the gateway +// for its clients and may not resolve yet, or resolve to a floating address. +func sourceIPToward(ip string) (string, error) { + c, err := net.Dial("udp", net.JoinHostPort(ip, "5432")) + if err != nil { + return "", fmt.Errorf("no route from this machine to %s: %w", ip, err) + } + defer c.Close() + return c.LocalAddr().(*net.UDPAddr).IP.String(), nil +} + // ipv4Of is the address a host name or address stands for, as the other // machines will be told to reach it. func ipv4Of(host string) (string, error) { diff --git a/internal/cli/serverinit_postgres.go b/internal/cli/serverinit_postgres.go index 9f37f7a8..4351f240 100644 --- a/internal/cli/serverinit_postgres.go +++ b/internal/cli/serverinit_postgres.go @@ -44,7 +44,7 @@ func (in *serverInit) storeAddress() string { // means reissuing every key and being unable to say which was which; and the // administrator init created a minute ago would be gone too. func (in *serverInit) ensureDatabase(ctx context.Context) (err error) { - if len(in.patroni) > 0 { + if len(in.patroni) > 0 || len(in.patroniNodes) > 0 { if in.postgres { return fmt.Errorf("--postgres keeps the store on this machine and --patroni on others: give one") } diff --git a/internal/cli/tui_home.go b/internal/cli/tui_home.go new file mode 100644 index 00000000..ff329a14 --- /dev/null +++ b/internal/cli/tui_home.go @@ -0,0 +1,259 @@ +package cli + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "runtime" + "strings" + "time" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/mattn/go-isatty" + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" +) + +// A bare `llmgw` in a terminal: the setup screens on a machine with nothing +// installed, the console on a control plane. Anywhere else — a pipe, a +// script, another operating system — it prints its help, as it always did. + +func interactive(cmd *cobra.Command) bool { + return isTerminal(cmd.OutOrStdout()) && isatty.IsTerminal(os.Stdin.Fd()) +} + +func runHome(cmd *cobra.Command) error { + out := cmd.OutOrStdout() + if runtime.GOOS != "linux" { + fmt.Fprintln(out, "The setup and console screens run on the Linux machine they manage.") + fmt.Fprintln(out) + return cmd.Help() + } + if os.Geteuid() != 0 { + return errors.New("run it as root — sudo llmgw — since it installs software and holds the certificate authority") + } + switch { + // Installed is the service in place, not only its configuration: a setup + // that stopped halfway has written server.yaml and nothing that runs, and + // the setup screens are what finish it. + case exists("/etc/llmgw/server.yaml") && exists("/etc/systemd/system/llmgw-server.service"): + return runConsole(cmd.Context(), out) + case exists("/etc/llmgw/agent.yaml"): + fmt.Fprintln(out, "This machine is a node. Its cluster is managed from the control plane:") + fmt.Fprintln(out, "run llmgw there. Here, `llmgw doctor` checks the machine.") + return nil + } + return runSetupTUI(cmd.Context(), out) +} + +func exists(p string) bool { + _, err := os.Stat(p) + return err == nil +} + +// What the console asks to be done once its screen closes: each runs its own +// screen, and the console comes back after. +type consoleAction struct { + kind string // "", "node", "patroni" + targets []string +} + +type summaryMsg struct { + text string + store bool + err error +} + +type consoleItem struct { + label, command, ask string + kind string + n int // how many targets to ask for +} + +type consoleModel struct { + client *ctlClient + summary string + store bool + err error + focus int + asking bool + asked []string + prompt textField + action consoleAction + width int + spin spinnerModel + loaded bool +} + +func (m consoleModel) items() []consoleItem { + its := []consoleItem{{label: "Add a node", kind: "node", n: 1, + ask: "user@address of the machine", command: "sudo llmgw node add @
"}} + if !m.store { + its = append(its, consoleItem{label: "Replicate the store across two machines (Patroni)", kind: "patroni", n: 2, + ask: "user@address of database machine", + command: "sudo --preserve-env=SSH_AUTH_SOCK llmgw server init --patroni @ --patroni @ --no-mirror --no-node --no-model"}) + } + return append(its, consoleItem{label: "Leave", kind: "quit"}) +} + +func (m consoleModel) fetch() tea.Msg { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + var s adminapi.Summary + if err := m.client.get(ctx, "/api/v1/summary", &s); err != nil { + return summaryMsg{err: err} + } + var b bytes.Buffer + c := &cobra.Command{} + c.SetOut(&b) + printSummary(c, s) + return summaryMsg{text: b.String(), store: len(s.Store) > 0} +} + +func (m consoleModel) Init() tea.Cmd { return tea.Batch(m.fetch, m.spin.Tick) } + +func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case tea.WindowSizeMsg: + m.width = msg.Width + case spinTickMsg: + var cmd tea.Cmd + m.spin, cmd = m.spin.Update(msg) + return m, cmd + case summaryMsg: + m.loaded = true + m.err = msg.err + if msg.err == nil { + m.summary, m.store = msg.text, msg.store + } + return m, tea.Tick(5*time.Second, func(time.Time) tea.Msg { return m.fetch() }) + case tea.KeyMsg: + if msg.Type == tea.KeyCtrlC { + return m, tea.Quit + } + its := m.items() + if m.asking { + switch msg.Type { + case tea.KeyEsc: + m.asking, m.asked = false, nil + case tea.KeyEnter: + v := strings.TrimSpace(m.prompt.Value()) + if v == "" { + return m, nil + } + m.asked = append(m.asked, v) + m.prompt.SetValue("") + if it := its[m.focus]; len(m.asked) == it.n { + m.action = consoleAction{kind: it.kind, targets: m.asked} + return m, tea.Quit + } + default: + var cmd tea.Cmd + m.prompt, cmd = m.prompt.Update(msg) + return m, cmd + } + return m, nil + } + switch msg.String() { + case "up", "k", "shift+tab": + m.focus = (m.focus - 1 + len(its)) % len(its) + case "down", "j", "tab": + m.focus = (m.focus + 1) % len(its) + case "q": + return m, tea.Quit + case "enter": + it := its[m.focus] + if it.kind == "quit" { + return m, tea.Quit + } + m.asking, m.asked = true, nil + m.prompt = newTextField() + m.prompt.Placeholder, m.prompt.Width = "ubuntu@10.0.0.20", 40 + m.prompt.Focus() + } + } + return m, nil +} + +func (m consoleModel) View() string { + var b strings.Builder + host, _ := os.Hostname() + b.WriteString(sTitle.Render("llmgw · console") + " " + sDim.Render(host) + "\n\n") + switch { + case !m.loaded: + b.WriteString(" " + m.spin.View() + " asking the control plane…\n") + case m.err != nil: + b.WriteString(sBad.Render("The control plane does not answer: ") + m.err.Error() + "\n" + + sDim.Render("systemctl status llmgw-server says why.") + "\n") + default: + b.WriteString(m.summary) + } + b.WriteString("\n" + lipgloss.NewStyle().Bold(true).Render("Do") + "\n") + for i, it := range m.items() { + line := " " + it.label + if i == m.focus { + line = sTitle.Render("› " + it.label) + } + b.WriteString(line + "\n") + if i == m.focus && it.command != "" { + b.WriteString(" " + sDim.Render(it.command) + "\n") + } + } + if m.asking { + it := m.items()[m.focus] + ask := it.ask + if it.n > 1 { + ask = fmt.Sprintf("%s %d of %d", ask, len(m.asked)+1, it.n) + } + b.WriteString("\n" + ask + "\n " + m.prompt.View() + "\n" + sDim.Render("enter · esc to cancel") + "\n") + } else { + b.WriteString("\n" + sDim.Render("↑↓ to choose · enter · q to leave · refreshed every five seconds") + "\n") + } + return b.String() +} + +// runConsole shows the console, runs what is chosen on it, and comes back. +func runConsole(ctx context.Context, out io.Writer) error { + addr := "127.0.0.1:9090" + if cfg, err := config.Load("/etc/llmgw/server.yaml"); err == nil && cfg.AdminListen != "" { + addr = cfg.AdminListen + if strings.HasPrefix(addr, ":") { + addr = "127.0.0.1" + addr + } + } + for { + m := consoleModel{client: newCtlClient(addr), spin: newSpinner(), width: 100} + final, err := tea.NewProgram(m, tea.WithOutput(out)).Run() + if err != nil { + return err + } + act := final.(consoleModel).action + switch act.kind { + case "": + return nil + case "node": + c := newNodeAddCmd() + c.SetArgs(act.targets) + c.SetOut(out) + err = c.ExecuteContext(ctx) + case "patroni": + in := newDefaultServerInit() + in.patroni = act.targets + // Already set up: only the store changes. + in.noMirror, in.noNode, in.noModel = true, true, true + err = in.runInitTUI(ctx, out) + } + if err != nil { + fmt.Fprintln(out, sBad.Render("✗ ")+err.Error()) + } + fmt.Fprint(out, "\n"+sDim.Render("enter to go back to the console")+" ") + _, _ = bufio.NewReader(os.Stdin).ReadString('\n') + } +} diff --git a/internal/cli/tui_setup.go b/internal/cli/tui_setup.go new file mode 100644 index 00000000..43fb9a16 --- /dev/null +++ b/internal/cli/tui_setup.go @@ -0,0 +1,741 @@ +package cli + +import ( + "context" + "fmt" + "io" + "net/mail" + "os" + "strings" + "sync" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + + "gitea.mynodes.xyz/ilker/llmgw/internal/onboard" + "gitea.mynodes.xyz/ilker/llmgw/internal/pgha" + "gitea.mynodes.xyz/ilker/llmgw/internal/preflight" +) + +// The setup screens: what a bare `llmgw` shows on a machine with nothing +// installed. One machine, or three for high availability — this one as the +// control plane and etcd's third vote, two more for the store under Patroni — +// with every machine checked before anything is installed on any of them. +// +// The other machines are reached with Go's own ssh rather than the system's, +// so a password can be typed here when there is no key; it is held in memory +// for the run and written nowhere. A sudo password, likewise, is used once to +// let the user sudo for the length of the install, and that is taken back +// when it finishes. + +type setupPage int + +const ( + pWelcome setupPage = iota + pAdmin + pMachines + pCheck + pPlan + pRun +) + +// How a database machine is logged into. +const ( + authAgent = iota + authKey + authPassword + authCount +) + +var authNames = [authCount]string{"ssh agent", "key file", "password"} + +// Fields of one database machine, in order. +const ( + mfAddr = iota + mfUser + mfAuth + mfSecret + mfSudo + mfCount +) + +type dbMachine struct { + addr, user, secret, sudo textField + auth int +} + +func newDBMachine(n int) dbMachine { + f := func(ph string, mask bool) textField { + t := newTextField() + t.Placeholder, t.Mask, t.CharLimit, t.Width = ph, mask, 200, 34 + return t + } + m := dbMachine{ + addr: f(fmt.Sprintf("10.0.0.%d or db%d.internal", 10+n, n), false), + user: f("ubuntu", false), + secret: f("", true), + sudo: f("only if sudo asks for one", true), + } + if os.Getenv("SSH_AUTH_SOCK") == "" { + m.auth = authKey + } + return m +} + +// field is the text field at a machine's position, nil for the selector. +func (d *dbMachine) field(i int) *textField { + switch i { + case mfAddr: + return &d.addr + case mfUser: + return &d.user + case mfSecret: + return &d.secret + case mfSudo: + return &d.sudo + } + return nil +} + +// Messages from the work behind the screens. +type ( + checkDoneMsg struct{ rep preflight.Report } + setupRunMsg struct{} +) + +// progRef lets work started from Update send to the program it runs in. +type progRef struct{ p *tea.Program } + +type setupModel struct { + ref *progRef + + page setupPage + host string + ips []string + choice int // 0 one machine, 1 high availability + + admin, names textField + afocus int // admin, names, next + + db [2]dbMachine + mfocus int // over both machines' fields, then the check button + + remotes []*onboard.GoSSH + report preflight.Report + checking bool + + in *serverInit + run initModel + finished bool + log *[]string + logMu *sync.Mutex + + spin spinnerModel + formErr string + width int + quit bool + // single is the choice of one machine: the screen closes and init's own + // form takes over. + single bool +} + +func newSetupModel(host string, ips []string) setupModel { + admin := newTextField() + admin.Placeholder, admin.CharLimit, admin.Width = "you@example.com", 200, 40 + names := newTextField() + names.Placeholder, names.CharLimit, names.Width = "llm.example.com, 203.0.113.10", 400, 40 + return setupModel{ + ref: &progRef{}, host: host, ips: ips, admin: admin, names: names, + db: [2]dbMachine{newDBMachine(1), newDBMachine(2)}, spin: newSpinner(), width: 100, + log: &[]string{}, logMu: &sync.Mutex{}, + } +} + +func (m setupModel) Init() tea.Cmd { return m.spin.Tick } + +func (m setupModel) thisAddr() string { + if len(m.ips) > 0 { + return m.ips[0] + } + return m.host +} + +func (m setupModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case tea.WindowSizeMsg: + m.width = msg.Width + m.run.width = msg.Width + case spinTickMsg: + var cmd tea.Cmd + m.spin, cmd = m.spin.Update(msg) + m.run.spin = m.spin + return m, cmd + case checkDoneMsg: + m.checking, m.report = false, msg.rep + case initLineMsg: + m.run.addLine(string(msg)) + case initDoneMsg: + m.run.done, m.run.err, m.finished = true, msg.err, true + case tea.KeyMsg: + if msg.Type == tea.KeyCtrlC { + m.quit = true + return m, tea.Quit + } + return m.key(msg) + } + return m, nil +} + +func (m setupModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) { + if k.Type == tea.KeyEsc && m.page > pWelcome && m.page < pRun && !m.checking { + m.page-- + if m.page == pMachines && m.choice == 0 { + m.page = pAdmin + } + m.formErr = "" + return m, nil + } + switch m.page { + case pWelcome: + switch k.String() { + case "up", "down", "tab", "k", "j": + m.choice = 1 - m.choice + case "enter": + if m.choice == 0 { + m.single = true + return m, tea.Quit + } + m.page = pAdmin + m.afocus = 0 + m.admin.Focus() + } + case pAdmin: + return m.adminKey(k) + case pMachines: + return m.machinesKey(k) + case pCheck: + switch k.String() { + case "r": + if !m.checking { + cmd := m.startCheck() + return m, cmd + } + case "enter": + if !m.checking && m.report.Worst() < preflight.Fail { + m.page = pPlan + } + } + case pPlan: + if k.String() == "enter" { + m.page = pRun + cmd := m.startRun() + return m, cmd + } + case pRun: + if m.finished && (k.String() == "q" || k.String() == "enter") { + return m, tea.Quit + } + } + return m, nil +} + +func (m setupModel) adminKey(k tea.KeyMsg) (tea.Model, tea.Cmd) { + move := func(d int) { + m.afocus = (m.afocus + d + 3) % 3 + m.admin.Blur() + m.names.Blur() + switch m.afocus { + case 0: + m.admin.Focus() + case 1: + m.names.Focus() + } + } + switch k.Type { + case tea.KeyTab, tea.KeyDown: + move(1) + return m, nil + case tea.KeyShiftTab, tea.KeyUp: + move(-1) + return m, nil + case tea.KeyEnter: + if m.afocus < 2 { + move(1) + return m, nil + } + if _, err := mail.ParseAddress(strings.TrimSpace(m.admin.Value())); err != nil { + m.formErr = "the administrator's email is needed, and that is not one" + m.afocus = 0 + move(0) + return m, nil + } + m.formErr = "" + m.page, m.mfocus = pMachines, 0 + m.db[0].addr.Focus() + return m, nil + } + var cmd tea.Cmd + switch m.afocus { + case 0: + m.admin, cmd = m.admin.Update(k) + case 1: + m.names, cmd = m.names.Update(k) + } + return m, cmd +} + +func (m *setupModel) focusMachine() { + for i := range m.db { + for f := 0; f < mfCount; f++ { + if t := m.db[i].field(f); t != nil { + t.Blur() + } + } + } + if m.mfocus < 2*mfCount { + if t := m.db[m.mfocus/mfCount].field(m.mfocus % mfCount); t != nil { + t.Focus() + } + } +} + +func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) { + n := 2*mfCount + 1 + switch k.Type { + case tea.KeyTab, tea.KeyDown: + m.mfocus = (m.mfocus + 1) % n + if m.mfocus < 2*mfCount && m.mfocus%mfCount == mfSecret && m.db[m.mfocus/mfCount].auth == authAgent { + m.mfocus++ + } + m.focusMachine() + return m, nil + case tea.KeyShiftTab, tea.KeyUp: + m.mfocus = (m.mfocus - 1 + n) % n + if m.mfocus < 2*mfCount && m.mfocus%mfCount == mfSecret && m.db[m.mfocus/mfCount].auth == authAgent { + m.mfocus-- + } + m.focusMachine() + return m, nil + case tea.KeyEnter: + if m.mfocus < 2*mfCount { + m.mfocus++ + m.focusMachine() + return m, nil + } + if err := m.validateMachines(); err != "" { + m.formErr = err + return m, nil + } + m.formErr = "" + m.page = pCheck + cmd := m.startCheck() + return m, cmd + } + if m.mfocus == 2*mfCount { + return m, nil + } + d := &m.db[m.mfocus/mfCount] + if m.mfocus%mfCount == mfAuth { + switch k.String() { + case "left", "h": + d.auth = (d.auth + authCount - 1) % authCount + case "right", "l", " ": + d.auth = (d.auth + 1) % authCount + } + d.secret.SetValue("") + d.secret.Mask = d.auth == authPassword + d.secret.Placeholder = map[int]string{authKey: "/root/.ssh/id_ed25519", authPassword: "the user's ssh password"}[d.auth] + return m, nil + } + var cmd tea.Cmd + t := d.field(m.mfocus % mfCount) + *t, cmd = t.Update(k) + return m, cmd +} + +func (m setupModel) validateMachines() string { + seen := map[string]bool{m.thisAddr(): true, m.host: true} + for i, d := range m.db { + addr := strings.TrimSpace(d.addr.Value()) + switch { + case addr == "": + return fmt.Sprintf("database machine %d has no address", i+1) + case seen[addr]: + return fmt.Sprintf("%s is given twice, or is this machine: the three must be different machines", addr) + case strings.TrimSpace(d.user.Value()) == "": + return fmt.Sprintf("database machine %d has no user to log in as", i+1) + case d.auth != authAgent && d.secret.Value() == "": + return fmt.Sprintf("database machine %d needs its %s", i+1, authNames[d.auth]) + } + seen[addr] = true + } + return "" +} + +// startCheck connects to the database machines and runs the checks on all +// three, this one included. +func (m *setupModel) startCheck() tea.Cmd { + for _, r := range m.remotes { + _ = r.Close() + } + m.remotes = nil + machines := []preflight.Machine{{ + Name: m.host + " (this machine)", Addr: m.thisAddr(), Local: true, Remote: pgha.Local{}, + Roles: []preflight.Role{preflight.RoleControl, preflight.RoleWitness}, + }} + for _, d := range m.db { + g := &onboard.GoSSH{ + Addr: strings.TrimSpace(d.addr.Value()), User: strings.TrimSpace(d.user.Value()), + UseAgent: d.auth == authAgent, SudoPassword: d.sudo.Value(), + } + switch d.auth { + case authKey: + g.KeyFile = strings.TrimSpace(d.secret.Value()) + case authPassword: + g.Password = d.secret.Value() + } + m.remotes = append(m.remotes, g) + machines = append(machines, preflight.Machine{Name: g.Addr, Addr: g.Addr, + Roles: []preflight.Role{preflight.RoleDatabase}, Remote: g}) + } + m.checking = true + m.report = preflight.Report{} + remotes := m.remotes + return func() tea.Msg { + rep := preflight.Run(context.Background(), machines) + // A sudo that asks for a password is fine with one given: the + // install grants itself sudo with it. + for i := range rep.Findings { + f := &rep.Findings[i] + if f.Check == "sudo" && f.Level == preflight.Warn { + for _, g := range remotes { + switch { + case g.Addr != f.Machine: + case g.SudoPassword == "": + f.Level = preflight.Fail + f.Fix = "give this user's sudo password on the previous screen" + default: + f.Level = preflight.OK + f.Detail = "with the password given" + f.Fix = "" + } + } + } + } + return checkDoneMsg{rep} + } +} + +// targets are the database machines as user@address, for the command line. +func (m setupModel) targets() []string { + var out []string + for _, d := range m.db { + out = append(out, strings.TrimSpace(d.user.Value())+"@"+strings.TrimSpace(d.addr.Value())) + } + return out +} + +func (m setupModel) namesList() []string { + var out []string + for _, h := range strings.Split(m.names.Value(), ",") { + if h = strings.TrimSpace(h); h != "" { + out = append(out, h) + } + } + return out +} + +// command is what this setup is on the command line, for doing it again +// from a script — with keys in place, since a password cannot be given there. +func (m setupModel) command() string { + parts := []string{"sudo --preserve-env=SSH_AUTH_SOCK llmgw server init", + "--admin " + strings.TrimSpace(m.admin.Value())} + for _, h := range m.namesList() { + parts = append(parts, "--host "+h) + } + for _, t := range m.targets() { + parts = append(parts, "--patroni "+t) + } + parts = append(parts, "--no-node", "--no-model") + return strings.Join(parts, " \\\n ") +} + +// startRun installs: sudo granted where it asks for a password, the control +// plane here, the store on the other two, then sudo taken back. +func (m *setupModel) startRun() tea.Cmd { + in := newDefaultServerInit() + in.admin = strings.TrimSpace(m.admin.Value()) + in.hosts = m.namesList() + in.patroni = m.targets() + // The control plane of a replicated setup is not where models run; + // nodes are added once it is up. + in.noNode, in.noModel = true, true + m.in = &in + m.run = newInitModel(&in, nil, false) + m.run.running, m.run.width, m.run.spin = true, m.width, m.spin + ref, remotes, logp, mu := m.ref, m.remotes, m.log, m.logMu + return func() tea.Msg { + send := func(l string) { + mu.Lock() + *logp = append(*logp, l) + mu.Unlock() + ref.p.Send(initLineMsg(l)) + } + in.out = &lineWriter{send: send} + err := func() error { + ctx := context.Background() + for _, g := range remotes { + fmt.Fprintf(in.out, " .. sudo on %s\n", g.Addr) + if err := g.PrepareSudo(ctx); err != nil { + return err + } + out, err := g.Run(ctx, "hostname", nil) + if err != nil { + return fmt.Errorf("%s: %v", g.Addr, err) + } + ip, err := ipv4Of(g.Addr) + if err != nil { + return err + } + in.patroniNodes = append(in.patroniNodes, pgha.Node{ + Member: pgha.Member{Name: strings.TrimSpace(string(out)), IP: ip}, Remote: g}) + } + return in.run(ctx) + }() + for _, g := range remotes { + // The grant is taken back whatever happened. + _ = g.Close() + } + return initDoneMsg{err} + } +} + +func (m setupModel) View() string { + var b strings.Builder + b.WriteString(sTitle.Render("llmgw · setup") + " " + sDim.Render(m.crumbs()) + "\n\n") + switch m.page { + case pWelcome: + b.WriteString(m.welcomeView()) + case pAdmin: + b.WriteString(m.adminView()) + case pMachines: + b.WriteString(m.machinesView()) + case pCheck: + b.WriteString(m.checkView()) + case pPlan: + b.WriteString(m.planView()) + case pRun: + v := m.run.View() + if i := strings.Index(v, "\n\n"); i >= 0 { + v = v[i+2:] // its own title; this screen has one + } + b.WriteString(v) + if m.finished { + if m.run.err == nil { + b.WriteString("\n" + sDim.Render("The same, from a script, with ssh keys in place:") + "\n" + m.command() + "\n") + } + b.WriteString("\n" + sDim.Render("enter or q to leave") + "\n") + } + } + if m.formErr != "" { + b.WriteString("\n" + sBad.Render(m.formErr) + "\n") + } + return b.String() +} + +func (m setupModel) crumbs() string { + steps := []string{"start", "administrator", "machines", "checks", "plan", "install"} + if m.choice == 0 { + return steps[0] + } + var out []string + for i, s := range steps { + if setupPage(i) == m.page { + s = lipgloss.NewStyle().Bold(true).Render(s) + } + out = append(out, s) + } + return strings.Join(out, " › ") +} + +func (m setupModel) welcomeView() string { + var b strings.Builder + fmt.Fprintf(&b, "Nothing of llmgw is installed on %s (%s).\n\n", m.host, strings.Join(m.ips, ", ")) + opt := func(i int, title, detail string) { + mark := "( )" + if m.choice == i { + mark = "(" + sOK.Render("•") + ")" + title = sTitle.Render(title) + } + fmt.Fprintf(&b, " %s %s\n %s\n\n", mark, title, sDim.Render(detail)) + } + opt(0, "One machine", "this machine runs everything: the control plane, a model mirror, and models if it has room.") + opt(1, "High availability — three machines", + "this one is the control plane; two more keep the store, replicated, so losing any one loses nothing.") + b.WriteString(sDim.Render("↑↓ to choose · enter to go on · ctrl+c to leave") + "\n") + return b.String() +} + +func (m setupModel) adminView() string { + var b strings.Builder + label := func(i int, s string) string { + if m.afocus == i { + return sTitle.Render("› " + s) + } + return " " + s + } + fmt.Fprintf(&b, "%s\n %s\n %s\n\n", label(0, "Administrator's email"), m.admin.View(), + sDim.Render("A password is generated and shown once, at the end.")) + fmt.Fprintf(&b, "%s\n %s\n %s\n\n", label(1, "Other names nodes will reach this machine by"), m.names.View(), + sDim.Render("Already known: "+strings.Join(append([]string{m.host}, m.ips...), ", ")+". Add a DNS name or a floating address.")) + b.WriteString(label(2, lipgloss.NewStyle().Bold(true).Render("Next")) + "\n\n") + b.WriteString(sDim.Render("tab or ↑↓ to move · enter on Next · esc back") + "\n") + return b.String() +} + +func (m setupModel) machinesView() string { + var b strings.Builder + b.WriteString("The two machines that keep the store. They need ssh, a user that can sudo, and\n" + + "Ubuntu 22.04/24.04 or Debian 12; PostgreSQL, Patroni and etcd are installed on them.\n\n") + for i, d := range m.db { + base := i * mfCount + // Padded before it is coloured: the colour's escape codes count + // toward a width and threw the focused row out of line. + label := func(f int, s string) string { + s = fmt.Sprintf("%-16s", s) + if m.mfocus == base+f { + return sTitle.Render("› " + s) + } + return " " + s + } + fmt.Fprintf(&b, "%s\n", lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("Database machine %d", i+1))) + fmt.Fprintf(&b, " %s %s\n", label(mfAddr, "address"), d.addr.View()) + fmt.Fprintf(&b, " %s %s\n", label(mfUser, "user"), d.user.View()) + var auth []string + for a := 0; a < authCount; a++ { + if a == d.auth { + auth = append(auth, sOK.Render("["+authNames[a]+"]")) + } else { + auth = append(auth, sDim.Render(" "+authNames[a]+" ")) + } + } + fmt.Fprintf(&b, " %s %s\n", label(mfAuth, "log in with"), strings.Join(auth, " ")) + if d.auth != authAgent { + fmt.Fprintf(&b, " %s %s\n", label(mfSecret, authNames[d.auth]), d.secret.View()) + } + fmt.Fprintf(&b, " %s %s\n\n", label(mfSudo, "sudo password"), d.sudo.View()) + } + btn := lipgloss.NewStyle().Bold(true).Render("Check the three machines") + if m.mfocus == 2*mfCount { + btn = sTitle.Render("› " + btn) + } else { + btn = " " + btn + } + b.WriteString(btn + "\n\n") + b.WriteString(sDim.Render("tab or ↑↓ to move · ←→ to choose how to log in · passwords are kept in memory only · esc back") + "\n") + return b.String() +} + +func (m setupModel) checkView() string { + var b strings.Builder + if m.checking { + b.WriteString(" " + m.spin.View() + " checking the three machines…\n") + return b.String() + } + machine := "" + for _, f := range m.report.Findings { + if f.Machine != machine { + machine = f.Machine + b.WriteString("\n" + lipgloss.NewStyle().Bold(true).Render(machine) + "\n") + } + mark := sOK.Render("✓") + switch f.Level { + case preflight.Warn: + mark = sWarn.Render("!") + case preflight.Fail: + mark = sBad.Render("✗") + } + line := fmt.Sprintf(" %s %-13s %s", mark, f.Check, truncate(f.Detail, max(20, m.width-22))) + if f.Level != preflight.OK && f.Fix != "" { + line += "\n " + sDim.Render(truncate(f.Fix, max(20, m.width-20))) + } + b.WriteString(line + "\n") + } + b.WriteString("\n") + switch m.report.Worst() { + case preflight.Fail: + b.WriteString(sBad.Render("Fix what is marked ✗, then check again.") + "\n" + + sDim.Render("r to check again · esc to change the machines") + "\n") + case preflight.Warn: + b.WriteString(sWarn.Render("It can go ahead; what is marked ! is worth a look.") + "\n" + + sDim.Render("enter to see the plan · r to check again · esc back") + "\n") + default: + b.WriteString(sOK.Render("All three are ready.") + "\n" + sDim.Render("enter to see the plan · esc back") + "\n") + } + return b.String() +} + +func (m setupModel) planView() string { + var b strings.Builder + b.WriteString("What will be installed:\n\n") + fmt.Fprintf(&b, " %s %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(m.host), sDim.Render(m.thisAddr()), + "control plane and gateway, model mirror, etcd (the third vote), administrator "+strings.TrimSpace(m.admin.Value())) + granted := false + for _, d := range m.db { + fmt.Fprintf(&b, " %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(strings.TrimSpace(d.addr.Value())), + "PostgreSQL under Patroni, etcd") + granted = granted || d.sudo.Value() != "" + } + b.WriteString("\n The store is replicated from the first to the second; if the primary is lost the\n" + + " standby takes over in about thirty seconds, and the control plane follows it.\n") + if granted { + b.WriteString(" " + sDim.Render("Where a sudo password was given, the user may sudo without it until the install\n finishes, and not after.") + "\n") + } + b.WriteString("\n" + sDim.Render("The same from a script, with ssh keys in place:") + "\n" + m.command() + "\n\n") + b.WriteString(lipgloss.NewStyle().Bold(true).Render("enter to install") + sDim.Render(" · esc back · ctrl+c to leave") + "\n") + return b.String() +} + +// runSetupTUI is the setup screens, until they finish or hand over to init's +// own form for one machine. +func runSetupTUI(ctx context.Context, out io.Writer) error { + host, _ := os.Hostname() + var ips []string + for _, ip := range localAddrs() { + if ip.To4() != nil { + ips = append(ips, ip.String()) + } + } + m := newSetupModel(host, ips) + p := tea.NewProgram(m, tea.WithOutput(out)) + m.ref.p = p + final, err := p.Run() + if err != nil { + return err + } + fm := final.(setupModel) + if fm.single { + in := newDefaultServerInit() + in.out = out + return in.runInitTUI(ctx, out) + } + if fm.page < pRun { + return nil + } + fmt.Fprintln(out) + if fm.run.err != nil { + fm.logMu.Lock() + fmt.Fprintln(out, strings.Join(*fm.log, "\n")) + fm.logMu.Unlock() + return fm.run.err + } + if !fm.finished { + return fmt.Errorf("left while installing; running llmgw again finishes what was started") + } + if fm.run.password != "" { + fmt.Fprintf(out, "Administrator %s, password %s (shown once)\n\n", fm.in.admin, fm.run.password) + } + fmt.Fprintln(out, "Next: add nodes from the console — run llmgw again — or with") + fmt.Fprintln(out, " sudo llmgw node add @
") + return nil +} diff --git a/internal/cli/tui_setup_test.go b/internal/cli/tui_setup_test.go new file mode 100644 index 00000000..5f778206 --- /dev/null +++ b/internal/cli/tui_setup_test.go @@ -0,0 +1,119 @@ +package cli + +import ( + "strings" + "testing" + + tea "github.com/charmbracelet/bubbletea" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/preflight" +) + +func press(m tea.Model, keys ...string) tea.Model { + for _, k := range keys { + var msg tea.KeyMsg + switch k { + case "enter": + msg = tea.KeyMsg{Type: tea.KeyEnter} + case "tab": + msg = tea.KeyMsg{Type: tea.KeyTab} + case "down": + msg = tea.KeyMsg{Type: tea.KeyDown} + case "esc": + msg = tea.KeyMsg{Type: tea.KeyEsc} + case "right": + msg = tea.KeyMsg{Type: tea.KeyRight} + default: + msg = tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(k)} + } + m, _ = m.Update(msg) + } + return m +} + +func TestTheSetupAsksForWhatHighAvailabilityNeedsAndRefusesWhatWillNotDo(t *testing.T) { + m := tea.Model(newSetupModel("ctl", []string{"10.0.0.5"})) + m = press(m, "down", "enter") + if s := m.(setupModel); s.page != pAdmin { + t.Fatalf("high availability did not lead to the administrator: page %d", s.page) + } + // No email: refused, with a reason. + m = press(m, "tab", "tab", "enter") + if s := m.(setupModel); s.page != pAdmin || !strings.Contains(s.formErr, "email") { + t.Fatalf("went on without an administrator: %+v", s.formErr) + } + // The refusal puts the cursor back on the email. + m = press(m, "a@example.com", "tab", "tab", "enter") + s := m.(setupModel) + if s.page != pMachines { + t.Fatalf("page %d: %s", s.page, s.formErr) + } + // This machine's own address is not a database machine. + s.db[0].addr.SetValue("10.0.0.5") + s.db[0].user.SetValue("ubuntu") + s.db[0].auth = authAgent + s.db[1].addr.SetValue("10.0.0.7") + s.db[1].user.SetValue("ubuntu") + s.db[1].auth = authPassword + if err := s.validateMachines(); !strings.Contains(err, "this machine") { + t.Errorf("this machine as a database machine: %q", err) + } + s.db[0].addr.SetValue("10.0.0.6") + if err := s.validateMachines(); !strings.Contains(err, "password") { + t.Errorf("a password login with no password: %q", err) + } + s.db[1].secret.SetValue("pw") + if err := s.validateMachines(); err != "" { + t.Errorf("a valid pair refused: %s", err) + } + cmd := s.command() + for _, want := range []string{"--admin a@example.com", "--patroni ubuntu@10.0.0.6", "--patroni ubuntu@10.0.0.7", "--no-node"} { + if !strings.Contains(cmd, want) { + t.Errorf("command lacks %q:\n%s", want, cmd) + } + } + if strings.Contains(cmd, "pw") { + t.Error("the password is in the command line") + } +} + +func TestChoosingOneMachineHandsOverToInit(t *testing.T) { + m := press(tea.Model(newSetupModel("ctl", nil)), "enter").(setupModel) + if !m.single { + t.Error("one machine did not hand over") + } +} + +func TestTheChecksHoldTheSetupBackOnlyForAFailure(t *testing.T) { + m := newSetupModel("ctl", nil) + m.choice, m.page = 1, pCheck + m.report = preflight.Report{Findings: []preflight.Finding{{Machine: "a", Check: "disk", Level: preflight.Fail, Fix: "more"}}} + if s := press(m, "enter").(setupModel); s.page != pCheck { + t.Error("went on past a failed check") + } + m.report.Findings[0].Level = preflight.Warn + if s := press(m, "enter").(setupModel); s.page != pPlan { + t.Error("a warning held it back") + } +} + +func TestTheConsoleOffersToReplicateOnlyAStoreThatIsNot(t *testing.T) { + m := consoleModel{} + if !strings.Contains(m.items()[1].label, "Replicate") { + t.Error("a single store is not offered replication") + } + m.store = true + for _, it := range m.items() { + if strings.Contains(it.label, "Replicate") { + t.Error("a replicated store is offered replication") + } + } + // A node is asked for, then the console closes with it to do. + var mm tea.Model = consoleModel{summary: "x", loaded: true} + mm = press(mm, "enter", "ubuntu@10.0.0.9", "enter") + if a := mm.(consoleModel).action; a.kind != "node" || a.targets[0] != "ubuntu@10.0.0.9" { + t.Errorf("%+v", a) + } + _ = adminapi.Summary{} +} diff --git a/internal/cli/tui_widgets.go b/internal/cli/tui_widgets.go index 1058cdc6..ef423f37 100644 --- a/internal/cli/tui_widgets.go +++ b/internal/cli/tui_widgets.go @@ -47,6 +47,8 @@ type textField struct { Placeholder string CharLimit int Width int + // Mask shows a dot for every character: a password. + Mask bool } func newTextField() textField { return textField{Prompt: "> "} } @@ -84,6 +86,9 @@ func (f textField) Update(msg tea.Msg) (textField, tea.Cmd) { func (f textField) View() string { text := string(f.value) + if f.Mask { + text = strings.Repeat("•", len(f.value)) + } shown := text if text == "" && f.Placeholder != "" { shown = sDim.Render(f.Placeholder) diff --git a/internal/onboard/gossh.go b/internal/onboard/gossh.go new file mode 100644 index 00000000..cc95a405 --- /dev/null +++ b/internal/onboard/gossh.go @@ -0,0 +1,278 @@ +package onboard + +import ( + "bytes" + "context" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/agent" + "golang.org/x/crypto/ssh/knownhosts" +) + +// GoSSH is Remote through Go's own ssh client rather than the system's: what +// the setup screens use, because they ask for a password when there is no key, +// which the system ssh in batch mode cannot take. +// +// Authentication is whichever is given, tried in order: the running ssh +// agent, a key file, a password. The password is held in memory for the +// session and never written anywhere. +type GoSSH struct { + Addr string + Port int + User string + + UseAgent bool + KeyFile string + // KeyPassphrase opens an encrypted KeyFile. + KeyPassphrase string + Password string + + // SudoPassword is asked for when the user cannot sudo without one. With + // it, PrepareSudo lets the user sudo without a password until + // RevokeSudo, so every script can use plain sudo as on any other node. + SudoPassword string + + // KnownHosts is where host keys are kept; "" is ~/.ssh/known_hosts. + KnownHosts string + + mu sync.Mutex + client *ssh.Client + sudoGrant bool +} + +// Run runs script with sh on the machine. +func (g *GoSSH) Run(ctx context.Context, script string, stdin []byte) ([]byte, error) { + c, err := g.dial(ctx) + if err != nil { + return nil, err + } + sess, err := c.NewSession() + if err != nil { + // A connection that died since: dial again once. + g.drop() + if c, err = g.dial(ctx); err != nil { + return nil, err + } + if sess, err = c.NewSession(); err != nil { + return nil, err + } + } + defer sess.Close() + if stdin != nil { + sess.Stdin = bytes.NewReader(stdin) + } + var stdout, stderr bytes.Buffer + sess.Stdout, sess.Stderr = &stdout, &stderr + done := make(chan error, 1) + go func() { done <- sess.Run("sh -c " + shellQuote(script)) }() + select { + case <-ctx.Done(): + _ = sess.Signal(ssh.SIGKILL) + return stdout.Bytes(), ctx.Err() + case err := <-done: + if err != nil { + return append(stdout.Bytes(), stderr.Bytes()...), err + } + return stdout.Bytes(), nil + } +} + +// Close ends the connection, taking back a sudo grant first. +func (g *GoSSH) Close() error { + _ = g.RevokeSudo(context.Background()) + g.drop() + return nil +} + +func (g *GoSSH) drop() { + g.mu.Lock() + defer g.mu.Unlock() + if g.client != nil { + _ = g.client.Close() + g.client = nil + } +} + +func (g *GoSSH) target() string { + port := g.Port + if port == 0 { + port = 22 + } + return net.JoinHostPort(g.Addr, strconv.Itoa(port)) +} + +func (g *GoSSH) dial(ctx context.Context) (*ssh.Client, error) { + g.mu.Lock() + defer g.mu.Unlock() + if g.client != nil { + return g.client, nil + } + var methods []ssh.AuthMethod + if g.UseAgent { + if sock := os.Getenv("SSH_AUTH_SOCK"); sock != "" { + if conn, err := net.Dial("unix", sock); err == nil { + methods = append(methods, ssh.PublicKeysCallback(agent.NewClient(conn).Signers)) + } + } + } + if g.KeyFile != "" { + signer, err := loadKey(g.KeyFile, g.KeyPassphrase) + if err != nil { + return nil, err + } + methods = append(methods, ssh.PublicKeys(signer)) + } + if g.Password != "" { + pw := g.Password + methods = append(methods, ssh.Password(pw), + ssh.KeyboardInteractive(func(_, _ string, qs []string, _ []bool) ([]string, error) { + out := make([]string, len(qs)) + for i := range qs { + out[i] = pw + } + return out, nil + })) + } + if len(methods) == 0 { + return nil, fmt.Errorf("no way to log in to %s: no ssh agent, key or password", g.Addr) + } + hostKey, err := g.hostKeyCallback() + if err != nil { + return nil, err + } + cfg := &ssh.ClientConfig{User: g.User, Auth: methods, HostKeyCallback: hostKey, Timeout: 10 * time.Second} + d := net.Dialer{Timeout: 10 * time.Second} + conn, err := d.DialContext(ctx, "tcp", g.target()) + if err != nil { + return nil, fmt.Errorf("cannot reach %s: %w", g.target(), err) + } + cc, chans, reqs, err := ssh.NewClientConn(conn, g.target(), cfg) + if err != nil { + _ = conn.Close() + return nil, fmt.Errorf("cannot log in to %s as %s: %w", g.Addr, g.User, err) + } + g.client = ssh.NewClient(cc, chans, reqs) + return g.client, nil +} + +func loadKey(path, passphrase string) (ssh.Signer, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading the key %s: %w", path, err) + } + if passphrase != "" { + return ssh.ParsePrivateKeyWithPassphrase(b, []byte(passphrase)) + } + s, err := ssh.ParsePrivateKey(b) + var missing *ssh.PassphraseMissingError + if errors.As(err, &missing) { + return nil, fmt.Errorf("the key %s needs its passphrase", path) + } + return s, err +} + +// hostKeyCallback trusts a machine on first sight and records its key, and +// refuses one whose key has changed since — accept-new, as onboarding's +// system ssh does: a machine provisioned a minute ago has a key nobody has +// seen, and the risk guarded against is a machine substituted for one that +// was known. +func (g *GoSSH) hostKeyCallback() (ssh.HostKeyCallback, error) { + path := g.KnownHosts + if path == "" { + home, err := os.UserHomeDir() + if err != nil { + return nil, err + } + path = filepath.Join(home, ".ssh", "known_hosts") + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return nil, err + } + if f, err := os.OpenFile(path, os.O_CREATE|os.O_RDONLY, 0o600); err == nil { + _ = f.Close() + } + known, err := knownhosts.New(path) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", path, err) + } + return func(host string, remote net.Addr, key ssh.PublicKey) error { + err := known(host, remote, key) + var ke *knownhosts.KeyError + if errors.As(err, &ke) && len(ke.Want) == 0 { + f, ferr := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o600) + if ferr != nil { + return ferr + } + defer f.Close() + _, ferr = fmt.Fprintln(f, knownhosts.Line([]string{knownhosts.Normalize(host)}, key)) + return ferr + } + if errors.As(err, &ke) { + return fmt.Errorf("%s's host key is not the one recorded in %s: "+ + "if the machine was rebuilt, remove its line there; otherwise something is in the way", host, path) + } + return err + }, nil +} + +// sudoersDrop is the grant PrepareSudo writes and RevokeSudo removes. +const sudoersDrop = "/etc/sudoers.d/llmgw-setup" + +// CanSudo reports whether the user can sudo without a password now. +func (g *GoSSH) CanSudo(ctx context.Context) bool { + _, err := g.Run(ctx, "sudo -n true", nil) + return err == nil +} + +// PrepareSudo makes plain sudo work for the rest of the session: nothing to do +// when it already does, otherwise a sudoers grant written with SudoPassword +// and taken back by RevokeSudo. A grant rather than the password piped into +// every sudo, because the scripts feed their own standard input to the +// commands they run under it. +func (g *GoSSH) PrepareSudo(ctx context.Context) error { + if g.CanSudo(ctx) { + return nil + } + if g.SudoPassword == "" { + return fmt.Errorf("%s cannot use sudo without a password, and none was given", g.User) + } + line := fmt.Sprintf("%s ALL=(ALL) NOPASSWD:ALL", g.User) + // Checked by visudo before it is put in place: a broken file in + // sudoers.d breaks sudo for everyone on the machine. + inner := fmt.Sprintf("umask 337; printf '%%s\\n' %s %s > %[3]s.new && visudo -cf %[3]s.new >/dev/null && mv -f %[3]s.new %[3]s", + shellQuote("# llmgw setup: removed when it finishes"), shellQuote(line), sudoersDrop) + script := "sudo -S -p '' sh -c " + shellQuote(inner) + if out, err := g.Run(ctx, script, []byte(g.SudoPassword+"\n")); err != nil { + return fmt.Errorf("sudo with the password given failed on %s: %v: %s", g.Addr, err, strings.TrimSpace(string(out))) + } + g.mu.Lock() + g.sudoGrant = true + g.mu.Unlock() + return nil +} + +// RevokeSudo removes the grant PrepareSudo wrote, if it wrote one. +func (g *GoSSH) RevokeSudo(ctx context.Context) error { + g.mu.Lock() + granted := g.sudoGrant + g.mu.Unlock() + if !granted { + return nil + } + if _, err := g.Run(ctx, "sudo rm -f "+sudoersDrop, nil); err != nil { + return err + } + g.mu.Lock() + g.sudoGrant = false + g.mu.Unlock() + return nil +} diff --git a/internal/preflight/preflight.go b/internal/preflight/preflight.go new file mode 100644 index 00000000..76464915 --- /dev/null +++ b/internal/preflight/preflight.go @@ -0,0 +1,360 @@ +// Package preflight checks machines before anything is installed on them: +// that they can be logged into and administered, run a supported system, have +// room for their role, keep time, have the ports free, and can reach the +// internet and each other. Everything wrong is found at once, with what to +// do about it, rather than one failure at a time halfway through an install. +package preflight + +import ( + "context" + "fmt" + "sort" + "strconv" + "strings" + "sync" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/onboard" +) + +// Role is what a machine is going to be. +type Role string + +// The roles a setup gives machines. +const ( + RoleControl Role = "control plane" + RoleDatabase Role = "database" + RoleWitness Role = "etcd witness" + RoleNode Role = "node" +) + +// Need is the least a role asks of a machine. +type Need struct { + CPUs int + MemoryMB int + DiskGB int + Ports []int +} + +// Needs are what each role asks for; a machine with several roles needs the +// largest of each and the ports of all. +var Needs = map[Role]Need{ + RoleControl: {CPUs: 2, MemoryMB: 3500, DiskGB: 15, Ports: []int{7443, 8080}}, + RoleDatabase: {CPUs: 2, MemoryMB: 3500, DiskGB: 30, Ports: []int{5432, 8008, 2379, 2380}}, + RoleWitness: {CPUs: 1, MemoryMB: 900, DiskGB: 5, Ports: []int{2379, 2380}}, + RoleNode: {CPUs: 2, MemoryMB: 3500, DiskGB: 20}, +} + +// Machine is one machine to check. +type Machine struct { + Name string // how the screen calls it + Addr string // the address the others will use + Roles []Role + Remote onboard.Remote + // Local is this machine, where sudo is not needed (setup runs as root). + Local bool +} + +// Level is how bad a finding is. +type Level int + +// Levels, mildest first. +const ( + OK Level = iota + Warn + Fail +) + +func (l Level) String() string { return [...]string{"ok", "warn", "fail"}[l] } + +// Finding is one thing checked on one machine. +type Finding struct { + Machine string + Check string + Level Level + Detail string + // Fix says what to do, for anything that is not OK. + Fix string +} + +// Report is everything found. +type Report struct { + Findings []Finding +} + +// Worst is the most serious level found. +func (r Report) Worst() Level { + w := OK + for _, f := range r.Findings { + if f.Level > w { + w = f.Level + } + } + return w +} + +// For is one machine's findings, in check order. +func (r Report) For(machine string) []Finding { + var out []Finding + for _, f := range r.Findings { + if f.Machine == machine { + out = append(out, f) + } + } + return out +} + +// probeScript gathers everything in one round trip, as key=value lines, so +// the checks are decided here and can be tested without a machine. +const probeScript = ` +. /etc/os-release 2>/dev/null; echo "os=$ID"; echo "version=$VERSION_ID"; echo "pretty=$PRETTY_NAME" +echo "arch=$(uname -m)" +echo "cpus=$(nproc)" +echo "mem_mb=$(awk '/MemTotal/{print int($2/1024)}' /proc/meminfo)" +echo "disk_gb=$(df -BG --output=avail / | tail -1 | tr -dc 0-9)" +echo "now=$(date +%s.%N)" +echo "ntp=$(timedatectl show -p NTPSynchronized --value 2>/dev/null)" +echo "listening=$(ss -Hltn 2>/dev/null | awk '{n=split($4,a,":"); print a[n]}' | sort -un | tr '\n' ' ')" +if sudo -n true 2>/dev/null; then echo "sudo=yes"; else echo "sudo=no"; fi +echo "apt=$(command -v apt-get >/dev/null && echo yes || echo no)" +echo "internet=$(timeout 6 bash -c 'exec 3<>/dev/tcp/deb.debian.org/80' 2>/dev/null && echo yes || (command -v curl >/dev/null && curl -s -m 6 -o /dev/null http://deb.debian.org && echo yes) || echo no)" +` + +// reachScript tries one TCP port on each address: whether the machines can +// reach one another at all. Port 22 because it is open everywhere before +// anything is installed; the ports llmgw will use have nothing listening yet. +func reachScript(addrs []string) string { + var b strings.Builder + for _, a := range addrs { + fmt.Fprintf(&b, "if timeout 4 bash -c 'exec 3<>/dev/tcp/%s/22' 2>/dev/null; then echo \"reach %s yes\"; else echo \"reach %s no\"; fi\n", a, a, a) + } + return b.String() +} + +// Run checks every machine, concurrently. +func Run(ctx context.Context, machines []Machine) Report { + var ( + mu sync.Mutex + rep Report + wg sync.WaitGroup + ) + add := func(fs ...Finding) { + mu.Lock() + rep.Findings = append(rep.Findings, fs...) + mu.Unlock() + } + var addrs []string + for _, m := range machines { + addrs = append(addrs, m.Addr) + } + order := map[string]int{} + for i, m := range machines { + order[m.Name] = i + wg.Add(1) + go func(m Machine) { + defer wg.Done() + add(check(ctx, m, addrs)...) + }(m) + } + wg.Wait() + // Machines in the order given, checks in the order made. + sort.SliceStable(rep.Findings, func(i, j int) bool { + return order[rep.Findings[i].Machine] < order[rep.Findings[j].Machine] + }) + return rep +} + +func check(ctx context.Context, m Machine, all []string) []Finding { + sent := time.Now() + out, err := m.Remote.Run(ctx, probeScript, nil) + took := time.Since(sent) + if err != nil { + return []Finding{{Machine: m.Name, Check: "login", Level: Fail, + Detail: strings.TrimSpace(firstLine(err.Error() + " " + string(out))), + Fix: "check the address, the user and the key or password given for it"}} + } + facts := parse(out) + fs := Evaluate(m, facts, sent.Add(took/2)) + var others []string + for _, a := range all { + if a != m.Addr { + others = append(others, a) + } + } + if len(others) > 0 { + rout, _ := m.Remote.Run(ctx, reachScript(others), nil) + fs = append(fs, evaluateReach(m, string(rout))...) + } + return fs +} + +func firstLine(s string) string { + s = strings.TrimSpace(s) + if i := strings.IndexByte(s, '\n'); i >= 0 { + return s[:i] + } + return s +} + +func parse(out []byte) map[string]string { + facts := map[string]string{} + for _, line := range strings.Split(string(out), "\n") { + if k, v, ok := strings.Cut(strings.TrimSpace(line), "="); ok { + facts[k] = strings.TrimSpace(v) + } + } + return facts +} + +// supported are the systems the setup installs packages on with apt. +var supported = map[string][]string{ + "ubuntu": {"22.04", "24.04"}, + "debian": {"12"}, +} + +// Evaluate turns one machine's facts into findings. at is when the machine's +// clock was read, by this machine's clock. +func Evaluate(m Machine, facts map[string]string, at time.Time) []Finding { + var fs []Finding + f := func(check string, l Level, detail, fix string) { + fs = append(fs, Finding{Machine: m.Name, Check: check, Level: l, Detail: detail, Fix: fix}) + } + f("login", OK, "logged in", "") + + // The system. + os, ver := facts["os"], facts["version"] + switch vers, ok := supported[os]; { + case !ok: + f("system", Fail, facts["pretty"], "use Ubuntu 22.04 or 24.04, or Debian 12") + case !contains(vers, ver): + f("system", Warn, facts["pretty"]+" (tried on "+strings.Join(vers, ", ")+")", "it may work; Ubuntu 24.04 is what this is tested on") + default: + f("system", OK, facts["pretty"], "") + } + switch facts["arch"] { + case "x86_64", "aarch64": + f("architecture", OK, facts["arch"], "") + default: + f("architecture", Fail, facts["arch"], "llmgw is built for amd64 and arm64") + } + + // Administering it. + if !m.Local { + if facts["sudo"] == "yes" { + f("sudo", OK, "without a password", "") + } else { + // Not a failure: the setup asks for the password and grants + // itself sudo for the length of the install. + f("sudo", Warn, "asks for a password", "give the sudo password on the access screen") + } + } + if facts["apt"] != "yes" { + f("packages", Fail, "no apt-get", "the setup installs PostgreSQL, Patroni and etcd with apt") + } + + // Room for its roles. + need := needOf(m.Roles) + cpus, _ := strconv.Atoi(facts["cpus"]) + mem, _ := strconv.Atoi(facts["mem_mb"]) + disk, _ := strconv.Atoi(facts["disk_gb"]) + room := func(check string, have, want int, unit string) { + d := fmt.Sprintf("%d %s (needs %d)", have, unit, want) + switch { + case have >= want: + f(check, OK, d, "") + case have*2 >= want: + f(check, Warn, d, "it will run, short of what the role is sized for") + default: + f(check, Fail, d, "give the machine more") + } + } + room("cpus", cpus, need.CPUs, "CPUs") + room("memory", mem, need.MemoryMB, "MB") + room("disk", disk, need.DiskGB, "GB free") + + // Time. Leases and etcd's elections go wrong on clocks that disagree. + if now, err := strconv.ParseFloat(facts["now"], 64); err == nil { + skew := time.Duration((now - float64(at.UnixNano())/1e9) * float64(time.Second)) + if skew < 0 { + skew = -skew + } + switch { + case skew > 5*time.Second: + f("clock", Fail, fmt.Sprintf("%s off from this machine", skew.Round(time.Second)), "turn on NTP (timedatectl set-ntp true) on both") + case skew > time.Second: + f("clock", Warn, fmt.Sprintf("%s off from this machine", skew.Round(100*time.Millisecond)), "turn on NTP on both") + default: + f("clock", OK, "within a second of this machine", "") + } + } + if facts["ntp"] != "yes" { + f("time sync", Warn, "NTP is not synchronised", "timedatectl set-ntp true") + } + + // Ports its roles will listen on. + listening := map[string]bool{} + for _, p := range strings.Fields(facts["listening"]) { + listening[p] = true + } + var busy []string + for _, p := range need.Ports { + if listening[strconv.Itoa(p)] { + busy = append(busy, strconv.Itoa(p)) + } + } + if len(busy) > 0 { + // A warning: a re-run finds its own services listening. + f("ports", Warn, "already listening: "+strings.Join(busy, ", "), + "fine if it is llmgw's own from an earlier run; otherwise stop what holds them") + } else if len(need.Ports) > 0 { + f("ports", OK, "free", "") + } + + if facts["internet"] == "yes" { + f("internet", OK, "reaches the package mirrors", "") + } else { + f("internet", Fail, "cannot reach deb.debian.org", "packages and releases are downloaded; allow outbound HTTP/HTTPS or set a proxy") + } + return fs +} + +func evaluateReach(m Machine, out string) []Finding { + var no []string + for _, line := range strings.Split(out, "\n") { + parts := strings.Fields(line) + if len(parts) == 3 && parts[0] == "reach" && parts[2] == "no" { + no = append(no, parts[1]) + } + } + if len(no) > 0 { + return []Finding{{Machine: m.Name, Check: "network", Level: Fail, + Detail: "cannot reach " + strings.Join(no, ", "), + Fix: "the machines must reach one another; check routes and firewalls"}} + } + return []Finding{{Machine: m.Name, Check: "network", Level: OK, Detail: "reaches every other machine"}} +} + +func needOf(roles []Role) Need { + var n Need + seen := map[int]bool{} + for _, r := range roles { + x := Needs[r] + n.CPUs, n.MemoryMB, n.DiskGB = max(n.CPUs, x.CPUs), max(n.MemoryMB, x.MemoryMB), max(n.DiskGB, x.DiskGB) + for _, p := range x.Ports { + if !seen[p] { + seen[p] = true + n.Ports = append(n.Ports, p) + } + } + } + return n +} + +func contains(xs []string, x string) bool { + for _, y := range xs { + if y == x { + return true + } + } + return false +} diff --git a/internal/preflight/preflight_test.go b/internal/preflight/preflight_test.go new file mode 100644 index 00000000..e95bc22d --- /dev/null +++ b/internal/preflight/preflight_test.go @@ -0,0 +1,109 @@ +package preflight + +import ( + "context" + "os" + "strconv" + "strings" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/onboard" +) + +func facts(over map[string]string) map[string]string { + f := map[string]string{ + "os": "ubuntu", "version": "24.04", "pretty": "Ubuntu 24.04.3 LTS", "arch": "x86_64", + "cpus": "2", "mem_mb": "3900", "disk_gb": "35", "ntp": "yes", "listening": "22 53", + "sudo": "yes", "apt": "yes", "internet": "yes", + "now": strconv.FormatFloat(float64(time.Now().UnixNano())/1e9, 'f', 3, 64), + } + for k, v := range over { + f[k] = v + } + return f +} + +func levelOf(fs []Finding, check string) (Level, string) { + for _, f := range fs { + if f.Check == check { + return f.Level, f.Detail + } + } + return -1, "" +} + +func TestAMachineFitForItsRoleHasNothingWrong(t *testing.T) { + fs := Evaluate(Machine{Name: "pg01", Roles: []Role{RoleDatabase}}, facts(nil), time.Now()) + if r := (Report{Findings: fs}); r.Worst() != OK { + t.Errorf("%+v", fs) + } +} + +func TestEachProblemIsFoundAndSaysWhatToDo(t *testing.T) { + m := Machine{Name: "x", Roles: []Role{RoleDatabase, RoleControl}} + cases := []struct { + over map[string]string + check string + want Level + }{ + {map[string]string{"os": "centos", "version": "9"}, "system", Fail}, + {map[string]string{"version": "20.04"}, "system", Warn}, + {map[string]string{"arch": "riscv64"}, "architecture", Fail}, + {map[string]string{"sudo": "no"}, "sudo", Warn}, + {map[string]string{"mem_mb": "1000"}, "memory", Fail}, + {map[string]string{"disk_gb": "20"}, "disk", Warn}, + {map[string]string{"ntp": "no"}, "time sync", Warn}, + {map[string]string{"listening": "22 5432"}, "ports", Warn}, + {map[string]string{"internet": "no"}, "internet", Fail}, + {map[string]string{"now": strconv.FormatFloat(float64(time.Now().Add(-30*time.Second).UnixNano())/1e9, 'f', 3, 64)}, "clock", Fail}, + } + for _, c := range cases { + fs := Evaluate(m, facts(c.over), time.Now()) + got, detail := levelOf(fs, c.check) + if got != c.want { + t.Errorf("%v: %s is %v (%s), want %v", c.over, c.check, got, detail, c.want) + } + for _, f := range fs { + if f.Level != OK && f.Fix == "" { + t.Errorf("%s: %s says nothing about what to do", c.check, f.Check) + } + } + } + // This machine is run as root; sudo is not its question. + if l, _ := levelOf(Evaluate(Machine{Name: "here", Local: true}, facts(map[string]string{"sudo": "no"}), time.Now()), "sudo"); l != -1 { + t.Error("the local machine was asked about sudo") + } +} + +func TestAMachineThatCannotReachAnotherIsCalledOut(t *testing.T) { + fs := evaluateReach(Machine{Name: "a"}, "reach 10.0.0.2 yes\nreach 10.0.0.3 no\n") + if fs[0].Level != Fail || !strings.Contains(fs[0].Detail, "10.0.0.3") { + t.Errorf("%+v", fs) + } +} + +// TestAgainstRealMachines runs the probe over ssh when LLMGW_TEST_PREFLIGHT +// names user@host,user@host: the scripts are shell, and only a real machine +// says whether they parse. +func TestAgainstRealMachines(t *testing.T) { + targets := os.Getenv("LLMGW_TEST_PREFLIGHT") + if targets == "" { + t.Skip("set LLMGW_TEST_PREFLIGHT=user@host,user@host to run against machines") + } + var ms []Machine + for _, tgt := range strings.Split(targets, ",") { + user, host, _ := strings.Cut(tgt, "@") + ms = append(ms, Machine{Name: host, Addr: host, Roles: []Role{RoleDatabase}, + Remote: &onboard.GoSSH{Addr: host, User: user, UseAgent: true}}) + } + rep := Run(context.Background(), ms) + for _, f := range rep.Findings { + t.Logf("%-12s %-12s %-5s %s", f.Machine, f.Check, f.Level, f.Detail) + } + for _, f := range rep.Findings { + if f.Check == "login" && f.Level != OK { + t.Fatalf("could not log in: %s", f.Detail) + } + } +}