Ilker ManapandClaude Opus 5.5
2a16c26575
Adding a node was deploy/bootstrap/onboard-node.sh, which is not installed
anywhere and wanted the node's name, the control plane's address and a
dozen flags. It is now in the binary, run on the control plane:
sudo llmgw node add ubuntu@10.0.0.21
The node id is the machine's hostname, and the address its agent dials is
this machine's on the route to it, checked against the server certificate.
It installs llmgw there (the release for its architecture, fetched and
checked by the node; uname rather than dpkg, so RHEL works), runs doctor
there and stops on a problem before changing anything, issues and installs
the certificate, writes agent.yaml and the unit, starts the agent, and waits
for it to join and install llama.cpp. The authority is read first, so a
missing one is said before the node is touched.
The steps are events. In a terminal they draw a checklist (bubbletea) that
asks what a script cannot: which fabric each group of active adapters is on,
grouped by the subnet manager or RoCE network they report, and whether to go
on past a problem. Elsewhere, one line per step.
Found by running it on an empty VM: ssh's "Permanently added" warning on
stderr was read as the machine's hostname, so standard error is now kept
apart; and a node added again while cordoned never counted as joined, so
joining now means a heartbeat since the restart.
llmgw node remove forgets a lost node, which otherwise stays listed until
the control plane restarts. ctl nodes no longer lists vLLM as skipped on
every CPU node. The quick install, install and onboarding docs use it.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
97 lines
3.1 KiB
Go
97 lines
3.1 KiB
Go
package cli
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
|
|
"time"
|
|
|
|
tea "github.com/charmbracelet/bubbletea"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
|
|
)
|
|
|
|
func send(m tea.Model, msgs ...tea.Msg) tea.Model {
|
|
for _, msg := range msgs {
|
|
m, _ = m.Update(msg)
|
|
}
|
|
return m
|
|
}
|
|
|
|
func TestTheNodeScreenFillsInAsStepsFinish(t *testing.T) {
|
|
var m tea.Model = newNodeModel("adding gpu01", func() {})
|
|
m = send(m,
|
|
evMsg{Step: onboard.StepConnect, Level: onboard.Done, Detail: "gpu01 (amd64)"},
|
|
evMsg{Step: onboard.StepCheck, Level: onboard.Warn, Detail: "fabric: nvidia_peermem is not loaded", Fix: "modprobe nvidia_peermem"},
|
|
evMsg{Step: onboard.StepCheck, Level: onboard.Running, Detail: "running llmgw doctor"},
|
|
)
|
|
v := m.View()
|
|
for _, want := range []string{"✓", "gpu01 (amd64)", "nvidia_peermem is not loaded", "→ modprobe nvidia_peermem", "running llmgw doctor", "engines"} {
|
|
if !strings.Contains(v, want) {
|
|
t.Errorf("screen lacks %q:\n%s", want, v)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTheFabricQuestionAnswersWithWhatWasTyped(t *testing.T) {
|
|
var m tea.Model = newNodeModel("adding gpu01", func() {})
|
|
reply := make(chan []onboard.Fabric, 1)
|
|
m = send(m, askFabricsMsg{suggested: []onboard.Fabric{
|
|
{Name: "fabric1", HCAs: []string{"mlx5_0", "mlx5_1"}},
|
|
{Name: "fabric2", HCAs: []string{"mlx5_4"}},
|
|
}, reply: reply})
|
|
if v := m.View(); !strings.Contains(v, "Which fabric") || !strings.Contains(v, "mlx5_0, mlx5_1") {
|
|
t.Fatalf("no question on screen:\n%s", m.View())
|
|
}
|
|
// Rename the first to ib-a, empty the second to leave it out.
|
|
for range len("fabric1") {
|
|
m = send(m, tea.KeyMsg{Type: tea.KeyBackspace})
|
|
}
|
|
m = send(m, tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("ib-a")}, tea.KeyMsg{Type: tea.KeyTab})
|
|
for range len("fabric2") {
|
|
m = send(m, tea.KeyMsg{Type: tea.KeyBackspace})
|
|
}
|
|
m = send(m, tea.KeyMsg{Type: tea.KeyEnter})
|
|
got := <-reply
|
|
if len(got) != 1 || got[0].Name != "ib-a" || strings.Join(got[0].HCAs, ",") != "mlx5_0,mlx5_1" {
|
|
t.Errorf("got %+v", got)
|
|
}
|
|
if strings.Contains(m.View(), "Which fabric") {
|
|
t.Error("the question stayed on screen after it was answered")
|
|
}
|
|
}
|
|
|
|
func TestContinuingPastAProblemIsNoUnlessSaidYes(t *testing.T) {
|
|
var m tea.Model = newNodeModel("adding gpu01", func() {})
|
|
reply := make(chan bool, 1)
|
|
m = send(m, askContinueMsg{fails: 2, reply: reply}, tea.KeyMsg{Type: tea.KeyEnter})
|
|
if <-reply {
|
|
t.Error("enter went on past the problems")
|
|
}
|
|
m = send(m, askContinueMsg{fails: 2, reply: reply}, tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("y")})
|
|
if !<-reply {
|
|
t.Error("y did not go on")
|
|
}
|
|
_ = m
|
|
}
|
|
|
|
func TestANodeCountsAsJoinedOnlyWhenHeardSinceItsAgentStarted(t *testing.T) {
|
|
start := time.Now().Add(-10 * time.Second)
|
|
cases := []struct {
|
|
state, ago string
|
|
want bool
|
|
}{
|
|
{"ready", "3s", true},
|
|
{"cordoned", "3s", true}, // added again while cordoned
|
|
{"ready", "30s", false}, // the agent before the restart
|
|
{"unreachable", "3s", false},
|
|
{"ready", "", false},
|
|
}
|
|
for _, c := range cases {
|
|
if got := heardSince(adminapi.NodeView{State: c.state, LastSeenAgo: c.ago}, start); got != c.want {
|
|
t.Errorf("%s seen %q ago: got %v", c.state, c.ago, got)
|
|
}
|
|
}
|
|
}
|