Ilker ManapandClaude Opus 5.5
7bb360fc78
doctor checks a node alone, and every check can pass while two nodes still cannot reach each other. ctl fabric-test A B asks B's agent for an ib_write_bw server and A's for the client, adapter by adapter as NCCL pairs them, one rail at a time, and grades each against its link's rate. It goes through the agents' streams, so there is no SSH between nodes; the agent installs perftest the first time, and names the RoCE v2 GID itself. The result is read from perftest's table by its heading, not a column number. Agents now list the requests they answer beyond the usual (Register.features), and the control plane refuses a test on one too old to answer instead of waiting minutes for it. The forwarding dispatcher carries the new capability, and its test says so: a missing forward would have made the command quietly unavailable in production. Not yet run over real InfiniBand or RoCE; this cluster has neither. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
101 lines
3.4 KiB
Go
101 lines
3.4 KiB
Go
package fabrictest
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
)
|
|
|
|
func node(id string, hcas ...string) inventory.Node {
|
|
n := inventory.Node{ID: id, MgmtAddr: "10.0.0." + id[len(id)-1:], Features: []string{"fabric-test"},
|
|
Fabrics: []inventory.FabricAttachment{{Fabric: "ib-a", HCAs: hcas}}}
|
|
for _, h := range hcas {
|
|
n.IBPorts = append(n.IBPorts, inventory.IBPort{HCA: h, Port: 1, State: "ACTIVE", RateGbps: 400})
|
|
}
|
|
return n
|
|
}
|
|
|
|
// fakeFabric answers as two agents would: the server waits for its client.
|
|
type fakeFabric struct {
|
|
mu sync.Mutex
|
|
gbps map[string]float64 // by client adapter
|
|
fails map[string]string
|
|
asked []string
|
|
}
|
|
|
|
func (f *fakeFabric) FabricTest(_ context.Context, n string, req *llmgwv1.FabricTestRequest) (*llmgwv1.FabricTestReply, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.asked = append(f.asked, n+" "+req.GetRole()+" "+req.GetHca())
|
|
if req.GetRole() == "server" {
|
|
return &llmgwv1.FabricTestReply{}, nil
|
|
}
|
|
if e := f.fails[req.GetHca()]; e != "" {
|
|
return &llmgwv1.FabricTestReply{Error: e}, nil
|
|
}
|
|
return &llmgwv1.FabricTestReply{Gbps: f.gbps[req.GetHca()]}, nil
|
|
}
|
|
|
|
func TestRailsArePairedByPositionOnTheSharedFabric(t *testing.T) {
|
|
a, b := node("n1", "mlx5_0", "mlx5_1"), node("n2", "mlx5_0", "mlx5_1", "mlx5_2")
|
|
fabric, rails, notes, err := Plan(a, b, "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if fabric != "ib-a" || len(rails) != 2 || rails[1].A != "mlx5_1" || rails[1].B != "mlx5_1" || rails[0].RateGbps != 400 {
|
|
t.Errorf("got %s %+v", fabric, rails)
|
|
}
|
|
if len(notes) != 1 || !strings.Contains(notes[0], "only the first 2") {
|
|
t.Errorf("an unequal count went unmentioned: %v", notes)
|
|
}
|
|
}
|
|
|
|
func TestNodesWithNoFabricInCommonAreSaidSo(t *testing.T) {
|
|
b := node("n2", "mlx5_0")
|
|
b.Fabrics[0].Fabric = "ib-b"
|
|
if _, _, _, err := Plan(node("n1", "mlx5_0"), b, ""); err == nil || !strings.Contains(err.Error(), "no fabric in common") {
|
|
t.Errorf("got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestEachRailIsGradedAgainstItsLink(t *testing.T) {
|
|
a, b := node("n1", "mlx5_0", "mlx5_1", "mlx5_2"), node("n2", "mlx5_0", "mlx5_1", "mlx5_2")
|
|
_, rails, _, _ := Plan(a, b, "ib-a")
|
|
f := &fakeFabric{
|
|
gbps: map[string]float64{"mlx5_0": 388, "mlx5_1": 240},
|
|
fails: map[string]string{"mlx5_2": "ib_write_bw: Couldn't connect to 10.0.0.2:18517"},
|
|
}
|
|
rep := Run(context.Background(), f, a, b, "ib-a", rails, Options{Seconds: 1, Settle: time.Millisecond})
|
|
want := []string{"ok", "slow", "fail"}
|
|
for i, r := range rep.Results {
|
|
if r.Verdict != want[i] {
|
|
t.Errorf("rail %d: %s (%s), want %s", i, r.Verdict, r.Detail, want[i])
|
|
}
|
|
}
|
|
if rep.OK() {
|
|
t.Error("a report with a failed rail is OK")
|
|
}
|
|
if !strings.Contains(rep.Results[2].Detail, "Couldn't connect") {
|
|
t.Errorf("the reason was lost: %q", rep.Results[2].Detail)
|
|
}
|
|
// Servers on the second node, clients on the first, one rail at a time.
|
|
if f.asked[0] != "n2 server mlx5_0" || f.asked[1] != "n1 client mlx5_0" {
|
|
t.Errorf("asked %v", f.asked)
|
|
}
|
|
}
|
|
|
|
func TestAnAgentTooOldToAnswerIsRefusedAtOnce(t *testing.T) {
|
|
// It would ignore the request, and the control plane would wait minutes.
|
|
old := node("n2", "mlx5_0")
|
|
old.Features, old.AgentVersion = nil, "v0.10.0"
|
|
_, _, _, err := Plan(node("n1", "mlx5_0"), old, "")
|
|
if err == nil || !strings.Contains(err.Error(), "n2's agent (v0.10.0) is too old") {
|
|
t.Errorf("got %v", err)
|
|
}
|
|
}
|