Let server init ask for what it needs, and node add take several nodes
In a terminal and without --admin, server init shows a short form: the administrator's email, the other names nodes will use (with the addresses it found), SQLite or PostgreSQL, and the mirror, node and starter model. Its own lines then fill in as a list — read from the writer init already uses, so nothing in init changed — with downloads updating in place and the password in a box until the end. The whole run, password included, is printed again when the screen closes, so it is in the terminal's history. The last screen says how to add a node and to back up the authority's key. node add takes several addresses and adds them in turn, with a summary. Suggested fabrics carry what the adapters report them to be, so a name given for one node is used for the next on the same fabric without asking. Tried on empty VMs: init from the form through to the starter model serving, and two nodes added to that cluster in one command. The first attempt at the form lost its keystrokes to the terminal colour query in a pseudo-terminal that never answers it; a real terminal does. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
7071c67487
commit
a1b5daceb8
9 files changed
+670
-42
No files matched your search
+118
-27
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -64,12 +65,15 @@ type nodeAdd struct {
|
||||
noWait bool
|
||||
plain bool
|
||||
serverCfg string
|
||||
// named are the fabric names given in this run, by what the adapters
|
||||
// report the fabric to be.
|
||||
named map[string]string
|
||||
}
|
||||
|
||||
func newNodeAddCmd() *cobra.Command {
|
||||
var na nodeAdd
|
||||
cmd := &cobra.Command{
|
||||
Use: "add [user@]address",
|
||||
Use: "add [user@]address...",
|
||||
Short: "Make a machine a node: check it, give it an identity, start its agent",
|
||||
Long: `Run on the machine holding the certificate authority — the control plane,
|
||||
after 'llmgw server init' — as root. It logs in to the machine with ssh,
|
||||
@@ -87,14 +91,18 @@ Nothing else needs giving, except --fabric for InfiniBand or RoCE: which
|
||||
fabric an adapter is on has a name only you can choose, and it must be the
|
||||
same on every node attached to it. Doctor suggests one when it sees adapters.
|
||||
|
||||
Several addresses add several nodes, one after another; a fabric named for
|
||||
one is recognised on the next and not asked again.
|
||||
|
||||
The user needs sudo on the machine without a password. Run it again at any
|
||||
time: the node gets a fresh certificate and configuration, and its running
|
||||
engines are kept.`,
|
||||
Example: ` sudo llmgw node add 10.0.0.21
|
||||
sudo llmgw node add ubuntu@10.0.0.21 ubuntu@10.0.0.22 ubuntu@10.0.0.23
|
||||
sudo llmgw node add ubuntu@gpu01 --fabric ib-a:mlx5_0,mlx5_1,mlx5_2,mlx5_3`,
|
||||
Args: cobra.ExactArgs(1),
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return na.run(cmd.Context(), args[0], cmd.OutOrStdout())
|
||||
return na.run(cmd.Context(), args, cmd.OutOrStdout())
|
||||
},
|
||||
}
|
||||
f := cmd.Flags()
|
||||
@@ -120,18 +128,9 @@ engines are kept.`,
|
||||
return cmd
|
||||
}
|
||||
|
||||
func (na *nodeAdd) prepare(addr string) error {
|
||||
// prepare works out what every node added in this run shares.
|
||||
func (na *nodeAdd) prepare() error {
|
||||
o := &na.opts
|
||||
o.Addr = addr
|
||||
if user, host, ok := strings.Cut(addr, "@"); ok {
|
||||
o.User, o.Addr = user, host
|
||||
}
|
||||
if o.User == "" {
|
||||
// Under sudo, the person who ran it — their account is usually the
|
||||
// one that exists on the nodes too. root's own keys still do the
|
||||
// logging in, since that is who is running ssh.
|
||||
o.User = os.Getenv("SUDO_USER")
|
||||
}
|
||||
for _, s := range na.fabrics {
|
||||
fab, err := onboard.ParseFabric(s)
|
||||
if err != nil {
|
||||
@@ -158,32 +157,124 @@ func (na *nodeAdd) prepare(addr string) error {
|
||||
o.Binary = exe
|
||||
}
|
||||
onboard.AgentUnit = agentUnit
|
||||
na.named = map[string]string{}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (na *nodeAdd) run(ctx context.Context, addr string, out io.Writer) error {
|
||||
if err := na.prepare(addr); err != nil {
|
||||
// target is the options for one address, [user@]host.
|
||||
func (na *nodeAdd) target(addr string) onboard.Options {
|
||||
o := na.opts
|
||||
o.Fabrics = slices.Clone(na.opts.Fabrics)
|
||||
o.Addr = addr
|
||||
if user, host, ok := strings.Cut(addr, "@"); ok {
|
||||
o.User, o.Addr = user, host
|
||||
}
|
||||
if o.User == "" {
|
||||
// Under sudo, the person who ran it — their account is usually the
|
||||
// one that exists on the nodes too. root's own keys still do the
|
||||
// logging in, since that is who is running ssh.
|
||||
o.User = os.Getenv("SUDO_USER")
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
// rememberFabrics wraps the question of which fabric a node's adapters are
|
||||
// on, so a name given for one node is used for the next whose adapters
|
||||
// report the same fabric, without asking again. ask nil is a script, which
|
||||
// is asked nothing and gets only what an earlier answer said.
|
||||
func (na *nodeAdd) rememberFabrics(ask func([]onboard.Fabric) []onboard.Fabric) func([]onboard.Fabric) []onboard.Fabric {
|
||||
return func(s []onboard.Fabric) []onboard.Fabric {
|
||||
var known []onboard.Fabric
|
||||
all := true
|
||||
for i, f := range s {
|
||||
if name, ok := na.named[f.Key]; ok && f.Key != "" && f.Key != "unknown" {
|
||||
f.Name = name
|
||||
s[i].Name = name
|
||||
known = append(known, f)
|
||||
} else {
|
||||
all = false
|
||||
}
|
||||
}
|
||||
if all || ask == nil {
|
||||
return known
|
||||
}
|
||||
got := ask(s)
|
||||
for _, f := range got {
|
||||
if f.Key != "" && f.Key != "unknown" {
|
||||
na.named[f.Key] = f.Name
|
||||
}
|
||||
}
|
||||
return got
|
||||
}
|
||||
}
|
||||
|
||||
func (na *nodeAdd) run(ctx context.Context, addrs []string, out io.Writer) error {
|
||||
if len(addrs) > 1 && na.opts.Node != "" {
|
||||
return errors.New("--node names one node; give one address with it")
|
||||
}
|
||||
if err := na.prepare(); err != nil {
|
||||
return err
|
||||
}
|
||||
if !na.plain && isTerminal(out) {
|
||||
return na.runTUI(ctx)
|
||||
tui := !na.plain && isTerminal(out)
|
||||
type outcome struct {
|
||||
addr, node string
|
||||
err error
|
||||
}
|
||||
var results []outcome
|
||||
for i, addr := range addrs {
|
||||
o := na.target(addr)
|
||||
title := "llmgw · adding " + o.Addr
|
||||
if len(addrs) > 1 {
|
||||
title += fmt.Sprintf(" (%d of %d)", i+1, len(addrs))
|
||||
}
|
||||
var node string
|
||||
var err error
|
||||
if tui {
|
||||
node, err = na.runTUI(ctx, o, title)
|
||||
} else {
|
||||
node, err = na.runPlain(ctx, o, title, out)
|
||||
}
|
||||
results = append(results, outcome{addr, node, err})
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(addrs) == 1 {
|
||||
return results[0].err
|
||||
}
|
||||
fmt.Fprintln(out)
|
||||
failed := 0
|
||||
for _, r := range results {
|
||||
if r.err != nil {
|
||||
failed++
|
||||
fmt.Fprintf(out, " FAIL %-20s %s\n", r.addr, r.err)
|
||||
} else {
|
||||
fmt.Fprintf(out, " ok %-20s %s\n", r.addr, r.node)
|
||||
}
|
||||
}
|
||||
if failed > 0 {
|
||||
return fmt.Errorf("%d of %d not added; running the same command again finishes them and leaves the rest as they are", failed, len(addrs))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (na *nodeAdd) runPlain(ctx context.Context, o onboard.Options, title string, out io.Writer) (string, error) {
|
||||
p := &plainProgress{out: out}
|
||||
fmt.Fprintf(out, "adding %s\n", na.opts.Addr)
|
||||
r := onboard.New(na.opts, p.event)
|
||||
res, err := r.Run(ctx)
|
||||
fmt.Fprintf(out, "%s\n", title)
|
||||
o.AskFabrics = na.rememberFabrics(nil)
|
||||
res, err := onboard.New(o, p.event).Run(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
if na.noWait {
|
||||
fmt.Fprintf(out, "\n%s is added. 'llmgw ctl nodes' shows it once it joins.\n", res.Node)
|
||||
return nil
|
||||
fmt.Fprintf(out, "\n%s is added. 'llmgw ctl nodes' shows it once it joins.\n\n", res.Node)
|
||||
return res.Node, nil
|
||||
}
|
||||
if err := na.watch(ctx, res.Node, p.event); err != nil {
|
||||
return err
|
||||
return res.Node, err
|
||||
}
|
||||
fmt.Fprintf(out, "\n%s is in the cluster.\n", res.Node)
|
||||
return nil
|
||||
fmt.Fprintf(out, "\n%s is in the cluster.\n\n", res.Node)
|
||||
return res.Node, nil
|
||||
}
|
||||
|
||||
// adminAPI is the control plane's admin API, from its configuration when
|
||||
|
||||
Reference in new issue
Block a user