diff --git a/CHANGELOG.md b/CHANGELOG.md index 0acf0e0e..63ca6e67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,24 @@ one too old for the test is refused at once instead of waited for. Written against perftest's real output and tested with it, but not yet run over real InfiniBand or RoCE: this cluster has neither. +### `server init` asks, and `node add` takes several nodes + +- **`sudo llmgw server init`** in a terminal, without `--admin`, asks for the + administrator's email, the other names nodes will reach the machine by + (showing the addresses it already found), SQLite or PostgreSQL, and whether + to run the mirror, a node and the starter model here. Then its steps fill + in as a list, downloads updating in place, and the administrator's password + stays in a box until the end — and is printed again after, with the whole + run, so it is in the terminal's history. The last screen says how to add a + node and to back up the authority's key. `--plain`, or no terminal, prints + lines as before. +- **`llmgw node add a b c`** adds each in turn and ends with which were added. + A fabric named for one node's adapters is recognised on the next by what the + adapters report, and not asked again. + +Both tried on empty VMs: init from the form to a starter model answering, and +two nodes added to it in one command in 40 seconds. + ## v0.10.0 — 2026-09-29 Independent InfiniBand fabrics and NVLink domains, told apart, drawn and diff --git a/docs/quick-install.md b/docs/quick-install.md index e16f2fba..36ead553 100644 --- a/docs/quick-install.md +++ b/docs/quick-install.md @@ -35,10 +35,12 @@ install it. ## 2. Set up the control plane ``` -sudo llmgw server init --admin you@example.com +sudo llmgw server init ``` -This one command sets up everything on this machine, and takes a few minutes: +It asks for your email and a few choices, then sets up everything on this +machine, in a few minutes. In a script, give them as flags instead: +`sudo llmgw server init --admin you@example.com`. - the service, its certificates and its configuration; - your administrator account. **The password is printed once. Copy it now.** @@ -60,12 +62,14 @@ the account page. ## 3. Add the GPU nodes -On the control plane, for each node: +On the control plane: ``` sudo llmgw node add ubuntu@10.0.0.21 ``` +Several at once: `sudo llmgw node add ubuntu@10.0.0.21 ubuntu@10.0.0.22 ubuntu@10.0.0.23`. + Replace `ubuntu` with the user you log in to the node as, and `10.0.0.21` with its address. That user needs `sudo` without a password on the node. diff --git a/internal/cli/node.go b/internal/cli/node.go index 36be281e..712beaa9 100644 --- a/internal/cli/node.go +++ b/internal/cli/node.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "slices" "strings" "time" @@ -64,12 +65,15 @@ type nodeAdd struct { noWait bool plain bool serverCfg string + // named are the fabric names given in this run, by what the adapters + // report the fabric to be. + named map[string]string } func newNodeAddCmd() *cobra.Command { var na nodeAdd cmd := &cobra.Command{ - Use: "add [user@]address", + Use: "add [user@]address...", Short: "Make a machine a node: check it, give it an identity, start its agent", Long: `Run on the machine holding the certificate authority — the control plane, after 'llmgw server init' — as root. It logs in to the machine with ssh, @@ -87,14 +91,18 @@ Nothing else needs giving, except --fabric for InfiniBand or RoCE: which fabric an adapter is on has a name only you can choose, and it must be the same on every node attached to it. Doctor suggests one when it sees adapters. +Several addresses add several nodes, one after another; a fabric named for +one is recognised on the next and not asked again. + The user needs sudo on the machine without a password. Run it again at any time: the node gets a fresh certificate and configuration, and its running engines are kept.`, Example: ` sudo llmgw node add 10.0.0.21 + sudo llmgw node add ubuntu@10.0.0.21 ubuntu@10.0.0.22 ubuntu@10.0.0.23 sudo llmgw node add ubuntu@gpu01 --fabric ib-a:mlx5_0,mlx5_1,mlx5_2,mlx5_3`, - Args: cobra.ExactArgs(1), + Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return na.run(cmd.Context(), args[0], cmd.OutOrStdout()) + return na.run(cmd.Context(), args, cmd.OutOrStdout()) }, } f := cmd.Flags() @@ -120,18 +128,9 @@ engines are kept.`, return cmd } -func (na *nodeAdd) prepare(addr string) error { +// prepare works out what every node added in this run shares. +func (na *nodeAdd) prepare() error { o := &na.opts - o.Addr = addr - if user, host, ok := strings.Cut(addr, "@"); ok { - o.User, o.Addr = user, host - } - if o.User == "" { - // Under sudo, the person who ran it — their account is usually the - // one that exists on the nodes too. root's own keys still do the - // logging in, since that is who is running ssh. - o.User = os.Getenv("SUDO_USER") - } for _, s := range na.fabrics { fab, err := onboard.ParseFabric(s) if err != nil { @@ -158,32 +157,124 @@ func (na *nodeAdd) prepare(addr string) error { o.Binary = exe } onboard.AgentUnit = agentUnit + na.named = map[string]string{} return nil } -func (na *nodeAdd) run(ctx context.Context, addr string, out io.Writer) error { - if err := na.prepare(addr); err != nil { +// target is the options for one address, [user@]host. +func (na *nodeAdd) target(addr string) onboard.Options { + o := na.opts + o.Fabrics = slices.Clone(na.opts.Fabrics) + o.Addr = addr + if user, host, ok := strings.Cut(addr, "@"); ok { + o.User, o.Addr = user, host + } + if o.User == "" { + // Under sudo, the person who ran it — their account is usually the + // one that exists on the nodes too. root's own keys still do the + // logging in, since that is who is running ssh. + o.User = os.Getenv("SUDO_USER") + } + return o +} + +// rememberFabrics wraps the question of which fabric a node's adapters are +// on, so a name given for one node is used for the next whose adapters +// report the same fabric, without asking again. ask nil is a script, which +// is asked nothing and gets only what an earlier answer said. +func (na *nodeAdd) rememberFabrics(ask func([]onboard.Fabric) []onboard.Fabric) func([]onboard.Fabric) []onboard.Fabric { + return func(s []onboard.Fabric) []onboard.Fabric { + var known []onboard.Fabric + all := true + for i, f := range s { + if name, ok := na.named[f.Key]; ok && f.Key != "" && f.Key != "unknown" { + f.Name = name + s[i].Name = name + known = append(known, f) + } else { + all = false + } + } + if all || ask == nil { + return known + } + got := ask(s) + for _, f := range got { + if f.Key != "" && f.Key != "unknown" { + na.named[f.Key] = f.Name + } + } + return got + } +} + +func (na *nodeAdd) run(ctx context.Context, addrs []string, out io.Writer) error { + if len(addrs) > 1 && na.opts.Node != "" { + return errors.New("--node names one node; give one address with it") + } + if err := na.prepare(); err != nil { return err } - if !na.plain && isTerminal(out) { - return na.runTUI(ctx) + tui := !na.plain && isTerminal(out) + type outcome struct { + addr, node string + err error } + var results []outcome + for i, addr := range addrs { + o := na.target(addr) + title := "llmgw · adding " + o.Addr + if len(addrs) > 1 { + title += fmt.Sprintf(" (%d of %d)", i+1, len(addrs)) + } + var node string + var err error + if tui { + node, err = na.runTUI(ctx, o, title) + } else { + node, err = na.runPlain(ctx, o, title, out) + } + results = append(results, outcome{addr, node, err}) + if ctx.Err() != nil { + break + } + } + if len(addrs) == 1 { + return results[0].err + } + fmt.Fprintln(out) + failed := 0 + for _, r := range results { + if r.err != nil { + failed++ + fmt.Fprintf(out, " FAIL %-20s %s\n", r.addr, r.err) + } else { + fmt.Fprintf(out, " ok %-20s %s\n", r.addr, r.node) + } + } + if failed > 0 { + return fmt.Errorf("%d of %d not added; running the same command again finishes them and leaves the rest as they are", failed, len(addrs)) + } + return nil +} + +func (na *nodeAdd) runPlain(ctx context.Context, o onboard.Options, title string, out io.Writer) (string, error) { p := &plainProgress{out: out} - fmt.Fprintf(out, "adding %s\n", na.opts.Addr) - r := onboard.New(na.opts, p.event) - res, err := r.Run(ctx) + fmt.Fprintf(out, "%s\n", title) + o.AskFabrics = na.rememberFabrics(nil) + res, err := onboard.New(o, p.event).Run(ctx) if err != nil { - return err + return "", err } if na.noWait { - fmt.Fprintf(out, "\n%s is added. 'llmgw ctl nodes' shows it once it joins.\n", res.Node) - return nil + fmt.Fprintf(out, "\n%s is added. 'llmgw ctl nodes' shows it once it joins.\n\n", res.Node) + return res.Node, nil } if err := na.watch(ctx, res.Node, p.event); err != nil { - return err + return res.Node, err } - fmt.Fprintf(out, "\n%s is in the cluster.\n", res.Node) - return nil + fmt.Fprintf(out, "\n%s is in the cluster.\n\n", res.Node) + return res.Node, nil } // adminAPI is the control plane's admin API, from its configuration when diff --git a/internal/cli/serverinit.go b/internal/cli/serverinit.go index 36c8706e..e1aa577d 100644 --- a/internal/cli/serverinit.go +++ b/internal/cli/serverinit.go @@ -92,6 +92,7 @@ func newServerInitCmd() *cobra.Command { unitPath: "/etc/systemd/system/llmgw-server.service", user: "llmgw", } + var plain bool cmd := &cobra.Command{ Use: "init", Short: "Set this machine up as a control plane and start it", @@ -122,10 +123,14 @@ func newServerInitCmd() *cobra.Command { return errors.New("init creates a user, writes /etc and starts a service: run it as root") } in.out = cmd.OutOrStdout() + if !plain && isTerminal(in.out) { + return in.runInitTUI(cmd.Context(), in.out) + } return in.run(cmd.Context()) }, } f := cmd.Flags() + f.BoolVar(&plain, "plain", false, "plain output, even in a terminal") f.StringArrayVar(&in.hosts, "host", nil, "an address agents dial besides the ones found on this machine; repeatable") f.StringVar(&in.admin, "admin", "", "email of the first administrator; a password is generated and printed once") diff --git a/internal/cli/tui_init.go b/internal/cli/tui_init.go new file mode 100644 index 00000000..0edc9d59 --- /dev/null +++ b/internal/cli/tui_init.go @@ -0,0 +1,395 @@ +package cli + +import ( + "context" + "fmt" + "io" + "net/mail" + "strings" + "sync" + + "github.com/charmbracelet/bubbles/spinner" + "github.com/charmbracelet/bubbles/textinput" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" +) + +// The full-screen way through 'llmgw server init': a short form for what +// init would otherwise take as flags, then init's own lines as a list that +// fills in, with the administrator's password kept in view — and printed +// again once the screen closes, so it is in the terminal's history. + +// initLineMsg is one line init wrote. +type initLineMsg string + +// initDoneMsg is init finishing. +type initDoneMsg struct{ err error } + +// lineWriter turns init's output into one message per line. +type lineWriter struct { + mu sync.Mutex + buf []byte + send func(string) +} + +func (w *lineWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + w.buf = append(w.buf, p...) + for { + i := strings.IndexByte(string(w.buf), '\n') + if i < 0 { + return len(p), nil + } + w.send(string(w.buf[:i])) + w.buf = w.buf[i+1:] + } +} + +// Form fields, in order. +const ( + fAdmin = iota + fHosts + fPostgres + fMirror + fNode + fModel + fStart + fCount +) + +type initModel struct { + in *serverInit + found []string // this machine's addresses + + // The form, until it is submitted. + form bool + focus int + admin textinput.Model + hosts textinput.Model + toggles [fCount]bool + formError string + + // Then the run. + running bool + lines []string // init's lines, as it wrote them + password string + next []string // the closing "Next:" text + spin spinner.Model + width int + done bool + err error + quit bool + start func() +} + +func newInitModel(in *serverInit, found []string, askForm bool) initModel { + admin := textinput.New() + admin.Placeholder = "you@example.com" + admin.SetValue(in.admin) + admin.CharLimit = 200 + admin.Width = 40 + hosts := textinput.New() + hosts.Placeholder = "llm.example.com, 203.0.113.10" + hosts.SetValue(strings.Join(in.hosts, ", ")) + hosts.CharLimit = 400 + hosts.Width = 40 + sp := spinner.New() + sp.Spinner = spinner.MiniDot + sp.Style = lipgloss.NewStyle().Foreground(tuiAccent) + m := initModel{in: in, found: found, form: askForm, admin: admin, hosts: hosts, spin: sp, width: 100} + m.toggles[fPostgres] = in.postgres + m.toggles[fMirror] = !in.noMirror + m.toggles[fNode] = !in.noNode + m.toggles[fModel] = !in.noModel + if askForm { + m.admin.Focus() + } + return m +} + +func (m initModel) Init() tea.Cmd { + if m.form { + return textinput.Blink + } + return tea.Batch(m.spin.Tick, func() tea.Msg { return startInitMsg{} }) +} + +type startInitMsg struct{} + +func (m initModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case tea.WindowSizeMsg: + m.width = msg.Width + case spinner.TickMsg: + var cmd tea.Cmd + m.spin, cmd = m.spin.Update(msg) + return m, cmd + case startInitMsg: + m.running = true + if m.start != nil { + m.start() + } + case initLineMsg: + m.addLine(string(msg)) + case initDoneMsg: + m.done, m.err = true, msg.err + return m, tea.Quit + case tea.KeyMsg: + if msg.Type == tea.KeyCtrlC { + m.quit = true + return m, tea.Quit + } + if m.form { + return m.formKey(msg) + } + } + return m, nil +} + +// addLine files one of init's lines: the password aside, the closing text +// aside, a progress line replacing the one before it. +func (m *initModel) addLine(l string) { + t := strings.TrimSpace(l) + switch { + case t == "": + return + case strings.HasPrefix(t, "password: "): + m.password = strings.TrimPrefix(t, "password: ") + return + case strings.HasPrefix(t, "shown once"): + return + case !strings.HasPrefix(l, " ") || len(m.next) > 0: + // Unindented lines are the closing text, and everything after its + // first line belongs to it. + m.next = append(m.next, l) + return + } + if strings.HasPrefix(t, "..") && len(m.lines) > 0 && strings.HasPrefix(strings.TrimSpace(m.lines[len(m.lines)-1]), "..") { + m.lines[len(m.lines)-1] = l + return + } + m.lines = append(m.lines, l) +} + +func (m initModel) formKey(k tea.KeyMsg) (tea.Model, tea.Cmd) { + switch k.Type { + case tea.KeyTab, tea.KeyDown: + m.moveFocus(1) + return m, nil + case tea.KeyShiftTab, tea.KeyUp: + m.moveFocus(-1) + return m, nil + case tea.KeyEnter: + if m.focus != fStart { + m.moveFocus(1) + return m, nil + } + admin := strings.TrimSpace(m.admin.Value()) + if admin != "" { + if _, err := mail.ParseAddress(admin); err != nil { + m.formError = "the administrator's email is not an address" + m.focus = fAdmin + m.admin.Focus() + return m, nil + } + } + m.in.admin = admin + m.in.hosts = nil + for _, h := range strings.Split(m.hosts.Value(), ",") { + if h = strings.TrimSpace(h); h != "" { + m.in.hosts = append(m.in.hosts, h) + } + } + m.in.postgres = m.toggles[fPostgres] + m.in.noMirror = !m.toggles[fMirror] + m.in.noNode = !m.toggles[fNode] + m.in.noModel = !m.toggles[fModel] + m.form = false + return m, tea.Batch(m.spin.Tick, func() tea.Msg { return startInitMsg{} }) + } + switch m.focus { + case fAdmin: + var cmd tea.Cmd + m.admin, cmd = m.admin.Update(k) + return m, cmd + case fHosts: + var cmd tea.Cmd + m.hosts, cmd = m.hosts.Update(k) + return m, cmd + case fPostgres, fMirror, fNode, fModel: + if k.String() == " " || k.String() == "x" { + m.toggles[m.focus] = !m.toggles[m.focus] + } + } + return m, nil +} + +func (m *initModel) moveFocus(d int) { + m.focus = (m.focus + d + fCount) % fCount + m.admin.Blur() + m.hosts.Blur() + switch m.focus { + case fAdmin: + m.admin.Focus() + case fHosts: + m.hosts.Focus() + } +} + +func (m initModel) View() string { + var b strings.Builder + b.WriteString(sTitle.Render("llmgw · setting up this machine as a control plane") + "\n\n") + if m.form { + return b.String() + m.formView() + } + for _, l := range m.visibleLines() { + b.WriteString(m.lineView(l) + "\n") + } + if m.password != "" { + box := sBox.BorderForeground(tuiWarn).Render( + fmt.Sprintf("Administrator %s\npassword %s\n%s", m.in.admin, + lipgloss.NewStyle().Bold(true).Render(m.password), + sDim.Render("Shown once. Copy it now; change it on the account page after signing in."))) + b.WriteString("\n" + box + "\n") + } + switch { + case m.done && m.err != nil: + b.WriteString("\n" + sBad.Render("✗ ") + wrap(m.err.Error(), m.width-4, " ") + "\n" + + sDim.Render(" It is safe to run init again once that is fixed: it finishes what is missing.") + "\n") + case m.done: + b.WriteString("\n" + sOK.Render("✓ ") + "The control plane is up. Add a node: " + + lipgloss.NewStyle().Bold(true).Render("sudo llmgw node add @
") + "\n") + b.WriteString(" " + sDim.Render("Back up "+m.in.pkiDir+": its ca-key.pem signs every node's identity, and nothing recovers it.") + "\n") + case m.quit: + b.WriteString("\n" + sDim.Render("stopped; run init again to finish") + "\n") + case m.running: + b.WriteString("\n" + sDim.Render("ctrl+c to stop") + "\n") + } + return b.String() +} + +// visibleLines are the last lines, enough to follow without the screen +// outgrowing the terminal: the whole log is printed when it closes. +func (m initModel) visibleLines() []string { + const keep = 16 + if len(m.lines) > keep { + return m.lines[len(m.lines)-keep:] + } + return m.lines +} + +func (m initModel) lineView(l string) string { + t := strings.TrimSpace(l) + word, rest, _ := strings.Cut(t, " ") + rest = truncate(strings.TrimSpace(rest), max(20, m.width-6)) + switch word { + case "ok": + return " " + sOK.Render("✓") + " " + rest + case "kept": + return " " + sDim.Render("· "+rest) + case "skip": + return " " + sDim.Render("- "+rest) + case "!!": + return " " + sWarn.Render("! ") + rest + case "..": + if m.done { + return " " + sDim.Render("· "+rest) + } + return " " + m.spin.View() + " " + rest + } + return " " + t +} + +func (m initModel) formView() string { + var b strings.Builder + label := func(i int, s string) string { + if m.focus == i { + return sTitle.Render("› " + s) + } + return " " + s + } + check := func(i int, s, hint string) string { + box := "[ ]" + if m.toggles[i] { + box = "[" + sOK.Render("x") + "]" + } + return fmt.Sprintf("%s %s %s\n", label(i, box), s, sDim.Render(hint)) + } + fmt.Fprintf(&b, "%s\n %s\n", label(fAdmin, "Administrator's email"), m.admin.View()) + b.WriteString(" " + sDim.Render("A password is generated and shown once. Leave empty to create accounts later.") + "\n\n") + fmt.Fprintf(&b, "%s\n %s\n", label(fHosts, "Other names nodes will reach this machine by"), m.hosts.View()) + found := "none found" + if len(m.found) > 0 { + found = strings.Join(m.found, ", ") + } + b.WriteString(" " + sDim.Render("Already in the certificate: "+found+". Add a DNS name or NAT address here.") + "\n\n") + b.WriteString(check(fPostgres, "PostgreSQL", "rather than SQLite; installed here, for several control planes or a larger store")) + b.WriteString(check(fMirror, "Model mirror", "downloads model files once for every node")) + b.WriteString(check(fNode, "This machine as a node", "serves models here too, on its GPUs or CPU")) + b.WriteString(check(fModel, "Starter model", "a small Qwen, so /chat answers at once")) + b.WriteString("\n" + label(fStart, lipgloss.NewStyle().Bold(true).Render("Set up")) + "\n") + if m.formError != "" { + b.WriteString("\n" + sBad.Render(m.formError) + "\n") + } + b.WriteString("\n" + sDim.Render("tab or ↑↓ to move · space to tick · enter on Set up to start · ctrl+c to leave") + "\n") + return b.String() +} + +// runInitTUI runs init with the screen above, asking the form first when +// the flags left the administrator unset. +func (in *serverInit) runInitTUI(ctx context.Context, out io.Writer) error { + var found []string + for _, ip := range localAddrs() { + found = append(found, ip.String()) + } + m := newInitModel(in, found, in.admin == "") + var p *tea.Program + + var ( + logMu sync.Mutex + log []string + ) + w := &lineWriter{send: func(l string) { + logMu.Lock() + log = append(log, l) + logMu.Unlock() + p.Send(initLineMsg(l)) + }} + ctx, cancel := context.WithCancel(ctx) + defer cancel() + started := make(chan struct{}) + m.start = func() { + close(started) + go func() { + in.out = w + err := in.run(ctx) + p.Send(initDoneMsg{err}) + }() + } + p = tea.NewProgram(m, tea.WithOutput(out)) + final, err := p.Run() + if err != nil { + return err + } + fm := final.(initModel) + select { + case <-started: + default: + return fmt.Errorf("left before anything was done") + } + if fm.quit && !fm.done { + cancel() + return fmt.Errorf("stopped; run init again to finish what was started") + } + + // The screen showed the last lines; the whole run goes to the terminal's + // history, the password with it. + logMu.Lock() + defer logMu.Unlock() + fmt.Fprintln(out) + for _, l := range log { + fmt.Fprintln(out, l) + } + return fm.err +} diff --git a/internal/cli/tui_init_test.go b/internal/cli/tui_init_test.go new file mode 100644 index 00000000..d1c1de97 --- /dev/null +++ b/internal/cli/tui_init_test.go @@ -0,0 +1,89 @@ +package cli + +import ( + "strings" + "testing" + + tea "github.com/charmbracelet/bubbletea" +) + +func typeIn(m tea.Model, s string) tea.Model { + return send(m, tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(s)}) +} + +func TestTheInitFormBecomesInitsSettings(t *testing.T) { + in := &serverInit{pkiDir: "/etc/llmgw/pki"} + var m tea.Model = newInitModel(in, []string{"10.0.0.140"}, true) + if v := m.View(); !strings.Contains(v, "Already in the certificate: 10.0.0.140") { + t.Fatalf("the addresses found are not shown:\n%s", v) + } + m = typeIn(m, "ops@example.com") + m = send(m, tea.KeyMsg{Type: tea.KeyTab}) + m = typeIn(m, "llm.example.com, 203.0.113.10") + m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(" ")}) // PostgreSQL on + m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyTab}, + tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(" ")}) // starter model off + m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyEnter}) + + if in.admin != "ops@example.com" || strings.Join(in.hosts, "|") != "llm.example.com|203.0.113.10" { + t.Errorf("admin %q hosts %q", in.admin, in.hosts) + } + if !in.postgres || in.noMirror || in.noNode || !in.noModel { + t.Errorf("toggles: postgres %v mirror %v node %v model %v", in.postgres, !in.noMirror, !in.noNode, !in.noModel) + } + if m.(initModel).form { + t.Error("the form stayed up after Set up") + } +} + +func TestAnEmailThatIsNotOneIsSaidSo(t *testing.T) { + in := &serverInit{} + var m tea.Model = newInitModel(in, nil, true) + m = typeIn(m, "not an address") + for range fStart { + m = send(m, tea.KeyMsg{Type: tea.KeyTab}) + } + m = send(m, tea.KeyMsg{Type: tea.KeyEnter}) + if !m.(initModel).form || !strings.Contains(m.View(), "not an address") { + t.Errorf("a bad email was accepted:\n%s", m.View()) + } +} + +func TestInitsLinesAreFiledAndThePasswordKeptInView(t *testing.T) { + in := &serverInit{admin: "ops@example.com", pkiDir: "/etc/llmgw/pki"} + var m tea.Model = newInitModel(in, nil, false) + for _, l := range []string{ + " ok system user llmgw", + " kept certificate authority in /etc/llmgw/pki", + " ok administrator ops@example.com (u1)", + "", + " password: Tr0ub4dor-3", + " shown once — sign in and change it from /profile", + "", + " .. downloading qwen2.5-0.5b-instruct into the mirror", + " .. 40%", + " .. 80%", + " !! an NVIDIA card is on the PCI bus but nvidia-smi does not answer", + "The control plane is up. Next:", + " llmgw ctl status", + } { + m = send(m, initLineMsg(l)) + } + m = send(m, initDoneMsg{}) + im := m.(initModel) + if im.password != "Tr0ub4dor-3" { + t.Errorf("password %q", im.password) + } + v := m.View() + for _, want := range []string{"✓ system user llmgw", "Tr0ub4dor-3", "Shown once", "80%", "! an NVIDIA card", "Back up /etc/llmgw/pki", "node add"} { + if !strings.Contains(v, want) { + t.Errorf("screen lacks %q:\n%s", want, v) + } + } + if strings.Contains(v, "40%") { + t.Error("progress lines piled up instead of replacing each other") + } + if strings.Contains(v, "llmgw ctl status") { + t.Error("the closing text was listed as a step") + } +} diff --git a/internal/cli/tui_node.go b/internal/cli/tui_node.go index 29bf69e8..0662720d 100644 --- a/internal/cli/tui_node.go +++ b/internal/cli/tui_node.go @@ -176,7 +176,7 @@ func (m nodeModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) { var out []onboard.Fabric for i, in := range m.inputs { if name := strings.TrimSpace(in.Value()); name != "" { - out = append(out, onboard.Fabric{Name: name, HCAs: m.fabrics.suggested[i].HCAs}) + out = append(out, onboard.Fabric{Name: name, HCAs: m.fabrics.suggested[i].HCAs, Key: m.fabrics.suggested[i].Key}) } } m.fabrics.reply <- out @@ -297,15 +297,15 @@ func wrap(s string, width int, indent string) string { return strings.Join(lines, "\n"+indent) } -// runTUI adds the node with the screen above. -func (na *nodeAdd) runTUI(ctx context.Context) error { +// runTUI adds one node with the screen above. +func (na *nodeAdd) runTUI(ctx context.Context, o onboard.Options, title string) (string, error) { ctx, cancel := context.WithCancel(ctx) defer cancel() - m := newNodeModel("llmgw · adding "+na.opts.Addr, cancel) + m := newNodeModel(title, cancel) p := tea.NewProgram(m) emit := func(e onboard.Event) { p.Send(evMsg(e)) } - na.opts.AskFabrics = func(s []onboard.Fabric) []onboard.Fabric { + o.AskFabrics = na.rememberFabrics(func(s []onboard.Fabric) []onboard.Fabric { reply := make(chan []onboard.Fabric) p.Send(askFabricsMsg{s, reply}) select { @@ -314,8 +314,8 @@ func (na *nodeAdd) runTUI(ctx context.Context) error { case <-ctx.Done(): return nil } - } - na.opts.AskContinue = func(fails int) bool { + }) + o.AskContinue = func(fails int) bool { reply := make(chan bool) p.Send(askContinueMsg{fails, reply}) select { @@ -327,7 +327,7 @@ func (na *nodeAdd) runTUI(ctx context.Context) error { } go func() { - res, err := onboard.New(na.opts, emit).Run(ctx) + res, err := onboard.New(o, emit).Run(ctx) node := "" if res != nil { node = res.Node @@ -344,11 +344,12 @@ func (na *nodeAdd) runTUI(ctx context.Context) error { final, err := p.Run() if err != nil { - return err + return "", err } fm := final.(nodeModel) if fm.quitAsked && !fm.done { - return fmt.Errorf("stopped; the node may be half set up, and running this again finishes it") + cancel() + return fm.node, fmt.Errorf("stopped; the node may be half set up, and running this again finishes it") } - return fm.err + return fm.node, fm.err } diff --git a/internal/cli/tui_node_test.go b/internal/cli/tui_node_test.go index 79de13e2..06a53ffe 100644 --- a/internal/cli/tui_node_test.go +++ b/internal/cli/tui_node_test.go @@ -94,3 +94,24 @@ func TestANodeCountsAsJoinedOnlyWhenHeardSinceItsAgentStarted(t *testing.T) { } } } + +func TestAFabricNamedForOneNodeIsNotAskedAgainForTheNext(t *testing.T) { + na := &nodeAdd{named: map[string]string{}} + asked := 0 + ask := func(s []onboard.Fabric) []onboard.Fabric { + asked++ + return []onboard.Fabric{{Name: "ib-compute", HCAs: s[0].HCAs, Key: s[0].Key}} + } + first := na.rememberFabrics(ask)([]onboard.Fabric{{Name: "fabric", HCAs: []string{"mlx5_0"}, Key: "sm:0xa"}}) + second := na.rememberFabrics(ask)([]onboard.Fabric{{Name: "fabric", HCAs: []string{"mlx5_0", "mlx5_1"}, Key: "sm:0xa"}}) + if asked != 1 { + t.Errorf("asked %d times", asked) + } + if len(second) != 1 || second[0].Name != "ib-compute" || len(second[0].HCAs) != 2 || first[0].Name != "ib-compute" { + t.Errorf("first %+v second %+v", first, second) + } + // A fabric not seen before is asked about; in a script nothing is asked. + if got := na.rememberFabrics(nil)([]onboard.Fabric{{Name: "fabric", Key: "sm:0xb"}}); len(got) != 0 { + t.Errorf("a script was given a fabric nobody named: %+v", got) + } +} diff --git a/internal/onboard/onboard.go b/internal/onboard/onboard.go index c69c40a4..67e8170a 100644 --- a/internal/onboard/onboard.go +++ b/internal/onboard/onboard.go @@ -35,6 +35,10 @@ import ( type Fabric struct { Name string HCAs []string + // Key is what the adapters report the fabric to be — its subnet manager, + // or a RoCE network — on a suggestion, so a name given for one node can + // be given again to the next on the same fabric. + Key string } // ParseFabric reads "name:hca1,hca2", or "name" alone. @@ -462,7 +466,7 @@ func suggestFabrics(hcas []discovery.HCA) []Fabric { if len(keys) > 1 { name = fmt.Sprintf("fabric%d", i+1) } - out = append(out, Fabric{Name: name, HCAs: byKey[k]}) + out = append(out, Fabric{Name: name, HCAs: byKey[k], Key: k}) } return out }