Adding a node was deploy/bootstrap/onboard-node.sh, which is not installed
anywhere and wanted the node's name, the control plane's address and a
dozen flags. It is now in the binary, run on the control plane:
sudo llmgw node add ubuntu@10.0.0.21
The node id is the machine's hostname, and the address its agent dials is
this machine's on the route to it, checked against the server certificate.
It installs llmgw there (the release for its architecture, fetched and
checked by the node; uname rather than dpkg, so RHEL works), runs doctor
there and stops on a problem before changing anything, issues and installs
the certificate, writes agent.yaml and the unit, starts the agent, and waits
for it to join and install llama.cpp. The authority is read first, so a
missing one is said before the node is touched.
The steps are events. In a terminal they draw a checklist (bubbletea) that
asks what a script cannot: which fabric each group of active adapters is on,
grouped by the subnet manager or RoCE network they report, and whether to go
on past a problem. Elsewhere, one line per step.
Found by running it on an empty VM: ssh's "Permanently added" warning on
stderr was read as the machine's hostname, so standard error is now kept
apart; and a node added again while cordoned never counted as joined, so
joining now means a heartbeat since the restart.
llmgw node remove forgets a lost node, which otherwise stays listed until
the control plane restarts. ctl nodes no longer lists vLLM as skipped on
every CPU node. The quick install, install and onboarding docs use it.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
283 lines
8.8 KiB
Go
283 lines
8.8 KiB
Go
package onboard
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"regexp"
|
|
"runtime"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/doctor"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
|
)
|
|
|
|
// fakeMachine answers what onboarding asks a machine, and keeps what it is
|
|
// given.
|
|
type fakeMachine struct {
|
|
mu sync.Mutex
|
|
hostname string
|
|
arch string
|
|
noSudo bool
|
|
report doctorReport
|
|
files map[string][]byte
|
|
scripts []string
|
|
download error
|
|
}
|
|
|
|
type doctorReport struct {
|
|
Findings []doctor.Finding `json:"findings"`
|
|
Facts doctor.Facts `json:"facts"`
|
|
}
|
|
|
|
var putPath = regexp.MustCompile(`cat > (\S+)\.new`)
|
|
|
|
func (f *fakeMachine) Run(_ context.Context, script string, stdin []byte) ([]byte, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.scripts = append(f.scripts, script)
|
|
switch {
|
|
case script == "hostname && uname -m":
|
|
return []byte(f.hostname + "\n" + f.arch + "\n"), nil
|
|
case script == "sudo -n true":
|
|
if f.noSudo {
|
|
return []byte("sudo: a password is required"), errors.New("exit 1")
|
|
}
|
|
return nil, nil
|
|
case strings.Contains(script, "checksums.txt"):
|
|
if f.download != nil {
|
|
return []byte("curl: (6) Could not resolve host"), f.download
|
|
}
|
|
return nil, nil
|
|
case strings.Contains(script, "install -m 0755") && stdin != nil:
|
|
f.files["/usr/local/bin/llmgw"] = stdin
|
|
return nil, nil
|
|
case strings.HasPrefix(script, "sudo llmgw doctor --json"):
|
|
b, _ := json.Marshal(f.report)
|
|
return b, nil
|
|
case putPath.MatchString(script):
|
|
f.files[putPath.FindStringSubmatch(script)[1]] = stdin
|
|
return nil, nil
|
|
case strings.Contains(script, "systemctl restart llmgw-agent"):
|
|
return []byte("active\n"), nil
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeMachine) ran(s string) bool {
|
|
for _, x := range f.scripts {
|
|
if strings.Contains(x, s) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func newFake() *fakeMachine {
|
|
return &fakeMachine{hostname: "gpu01", arch: map[string]string{"amd64": "x86_64", "arm64": "aarch64"}[runtime.GOARCH],
|
|
files: map[string][]byte{}}
|
|
}
|
|
|
|
func runner(t *testing.T, m *fakeMachine, o Options) (*Runner, *[]Event) {
|
|
t.Helper()
|
|
pki := t.TempDir()
|
|
ca, err := tlsconf.NewCA("test")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := ca.Write(pki); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
o.Addr = "10.0.0.21"
|
|
o.PKI, o.EtcDir = pki, t.TempDir()
|
|
if o.Servers == nil {
|
|
o.Servers = []string{"https://10.0.0.10:7443"}
|
|
}
|
|
o.Binary = "onboard_test.go" // anything readable
|
|
AgentUnit = "[Unit]\n"
|
|
var events []Event
|
|
var mu sync.Mutex
|
|
return &Runner{Opts: o, Remote: m, Emit: func(e Event) { mu.Lock(); events = append(events, e); mu.Unlock() }}, &events
|
|
}
|
|
|
|
func TestANodeIsAddedFromItsAddressAlone(t *testing.T) {
|
|
m := newFake()
|
|
r, _ := runner(t, m, Options{Version: "v1.2.3-4-gabc"})
|
|
res, err := r.Run(t.Context())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if res.Node != "gpu01" {
|
|
t.Errorf("node id %q, want the machine's hostname", res.Node)
|
|
}
|
|
for _, f := range []string{"/etc/llmgw/ca.pem", "/etc/llmgw/agent.pem", "/etc/llmgw/agent-key.pem",
|
|
"/etc/llmgw/agent.yaml", "/etc/systemd/system/llmgw-agent.service", "/usr/local/bin/llmgw"} {
|
|
if len(m.files[f]) == 0 {
|
|
t.Errorf("%s was not written", f)
|
|
}
|
|
}
|
|
yaml := string(m.files["/etc/llmgw/agent.yaml"])
|
|
if !strings.Contains(yaml, `server: "https://10.0.0.10:7443"`) || strings.Contains(yaml, "node_id") {
|
|
t.Errorf("agent.yaml:\n%s", yaml)
|
|
}
|
|
// The key is never readable by others, not even for a moment on the way.
|
|
if !m.ran("umask 077; cat > /etc/llmgw/agent-key.pem.new") || !m.ran("chmod 0600 /etc/llmgw/agent-key.pem.new") {
|
|
t.Error("the key was not written private")
|
|
}
|
|
if !m.ran("systemctl restart llmgw-agent") {
|
|
t.Error("the agent was not started")
|
|
}
|
|
}
|
|
|
|
func TestAReleaseIsFetchedByTheNodeAndCheckedThere(t *testing.T) {
|
|
m := newFake()
|
|
r, _ := runner(t, m, Options{Version: "v1.2.3", Releases: "https://example.test/releases/download"})
|
|
if _, err := r.Run(t.Context()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !m.ran("sha256sum -c") || !m.ran("https://example.test/releases/download/v1.2.3") {
|
|
t.Error("the release was not fetched and verified on the node")
|
|
}
|
|
if _, copied := m.files["/usr/local/bin/llmgw"]; copied {
|
|
t.Error("this machine's binary was copied although the node fetched the release")
|
|
}
|
|
}
|
|
|
|
func TestANodeThatCannotDownloadGetsThisMachinesCopy(t *testing.T) {
|
|
m := newFake()
|
|
m.download = errors.New("exit 6")
|
|
r, events := runner(t, m, Options{Version: "v1.2.3", Releases: "https://example.test"})
|
|
if _, err := r.Run(t.Context()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(m.files["/usr/local/bin/llmgw"]) == 0 {
|
|
t.Error("the binary was not copied")
|
|
}
|
|
if !said(*events, Warn, "could not download") {
|
|
t.Error("the failed download was not mentioned")
|
|
}
|
|
}
|
|
|
|
func TestAnotherArchitectureIsNotGivenThisMachinesBinary(t *testing.T) {
|
|
m := newFake()
|
|
m.arch = map[string]string{"amd64": "aarch64", "arm64": "x86_64"}[runtime.GOARCH]
|
|
r, _ := runner(t, m, Options{Version: "dev"})
|
|
_, err := r.Run(t.Context())
|
|
if err == nil || !strings.Contains(err.Error(), "cannot be copied") {
|
|
t.Errorf("got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestNoSudoIsSaidPlainly(t *testing.T) {
|
|
m := newFake()
|
|
m.noSudo = true
|
|
r, _ := runner(t, m, Options{})
|
|
_, err := r.Run(t.Context())
|
|
if err == nil || !strings.Contains(err.Error(), "sudo without a password") {
|
|
t.Errorf("got %v", err)
|
|
}
|
|
if len(m.files) > 0 {
|
|
t.Error("the machine was changed before finding it could not be finished")
|
|
}
|
|
}
|
|
|
|
func TestADoctorProblemStopsItUnlessAnswered(t *testing.T) {
|
|
fail := doctor.Finding{Level: doctor.Fail, Area: "nvlink", Message: "fabric manager inactive", Fix: "systemctl enable --now nvidia-fabricmanager"}
|
|
|
|
m := newFake()
|
|
m.report.Findings = []doctor.Finding{fail}
|
|
r, events := runner(t, m, Options{Version: "dev"})
|
|
if _, err := r.Run(t.Context()); !errors.Is(err, ErrChecks) {
|
|
t.Fatalf("got %v, want ErrChecks", err)
|
|
}
|
|
if !said(*events, Fail, "fabric manager inactive") {
|
|
t.Error("the problem itself was not reported")
|
|
}
|
|
if _, ok := m.files["/etc/llmgw/agent.pem"]; ok {
|
|
t.Error("a certificate was installed on a machine that failed its checks")
|
|
}
|
|
|
|
m = newFake()
|
|
m.report.Findings = []doctor.Finding{fail}
|
|
asked := 0
|
|
r, _ = runner(t, m, Options{Version: "dev", AskContinue: func(n int) bool { asked = n; return true }})
|
|
if _, err := r.Run(t.Context()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if asked != 1 {
|
|
t.Errorf("asked about %d problems, want 1", asked)
|
|
}
|
|
}
|
|
|
|
func ibPort(sm string) discovery.Port {
|
|
return discovery.Port{Num: 1, State: "ACTIVE", LinkLayer: "InfiniBand", SM: &discovery.SMInfo{GUID: sm}}
|
|
}
|
|
|
|
func TestAdaptersAreGroupedByTheFabricTheyReport(t *testing.T) {
|
|
hcas := []discovery.HCA{
|
|
{Name: "mlx5_0", Ports: []discovery.Port{ibPort("0xa")}},
|
|
{Name: "mlx5_1", Ports: []discovery.Port{ibPort("0xa")}},
|
|
{Name: "mlx5_4", Ports: []discovery.Port{ibPort("0xb")}},
|
|
{Name: "mlx5_5", Ports: []discovery.Port{{Num: 1, State: "DOWN"}}},
|
|
}
|
|
got := suggestFabrics(hcas)
|
|
if len(got) != 2 || strings.Join(got[0].HCAs, ",") != "mlx5_0,mlx5_1" || strings.Join(got[1].HCAs, ",") != "mlx5_4" {
|
|
t.Errorf("got %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestTheFabricsChosenAreWrittenToTheAgent(t *testing.T) {
|
|
m := newFake()
|
|
m.report.Facts.HCAs = []discovery.HCA{{Name: "mlx5_0", Ports: []discovery.Port{ibPort("0xa")}}}
|
|
r, _ := runner(t, m, Options{Version: "dev", AskFabrics: func(s []Fabric) []Fabric {
|
|
return []Fabric{{Name: "ib-a", HCAs: s[0].HCAs}}
|
|
}})
|
|
if _, err := r.Run(t.Context()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if yaml := string(m.files["/etc/llmgw/agent.yaml"]); !strings.Contains(yaml, `- name: "ib-a"`) || !strings.Contains(yaml, "hcas: [mlx5_0]") {
|
|
t.Errorf("agent.yaml:\n%s", yaml)
|
|
}
|
|
|
|
// Asked nothing, a script is told what to add instead.
|
|
m = newFake()
|
|
m.report.Facts.HCAs = []discovery.HCA{{Name: "mlx5_0", Ports: []discovery.Port{ibPort("0xa")}}}
|
|
r, events := runner(t, m, Options{Version: "dev"})
|
|
if _, err := r.Run(t.Context()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !said(*events, Warn, "no fabric was given") {
|
|
t.Error("adapters up with no fabric given went unmentioned")
|
|
}
|
|
}
|
|
|
|
func TestANodeNamedOtherThanItsHostnameSaysSo(t *testing.T) {
|
|
res := &Result{Node: "gpu-a", Hostname: "ip-10-0-0-21", Servers: []string{"https://a:7443", "https://b:7443"}}
|
|
y := AgentYAML(Options{}, res)
|
|
if !strings.Contains(y, `node_id: "gpu-a"`) || !strings.Contains(y, "servers:\n - \"https://a:7443\"") {
|
|
t.Errorf("agent.yaml:\n%s", y)
|
|
}
|
|
}
|
|
|
|
func TestParseFabric(t *testing.T) {
|
|
f, err := ParseFabric("ib-a:mlx5_0, mlx5_1")
|
|
if err != nil || f.Name != "ib-a" || strings.Join(f.HCAs, ",") != "mlx5_0,mlx5_1" {
|
|
t.Errorf("got %+v %v", f, err)
|
|
}
|
|
if _, err := ParseFabric(":mlx5_0"); err == nil {
|
|
t.Error("a fabric with no name was accepted")
|
|
}
|
|
}
|
|
|
|
func said(events []Event, level, s string) bool {
|
|
for _, e := range events {
|
|
if e.Level == level && strings.Contains(e.Detail, s) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|