Measure two nodes' fabric rail by rail: llmgw ctl fabric-test

doctor checks a node alone, and every check can pass while two nodes still
cannot reach each other. ctl fabric-test A B asks B's agent for an
ib_write_bw server and A's for the client, adapter by adapter as NCCL pairs
them, one rail at a time, and grades each against its link's rate. It goes
through the agents' streams, so there is no SSH between nodes; the agent
installs perftest the first time, and names the RoCE v2 GID itself.

The result is read from perftest's table by its heading, not a column
number. Agents now list the requests they answer beyond the usual
(Register.features), and the control plane refuses a test on one too old to
answer instead of waiting minutes for it. The forwarding dispatcher carries
the new capability, and its test says so: a missing forward would have made
the command quietly unavailable in production.

Not yet run over real InfiniBand or RoCE; this cluster has neither.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-29 21:29:23 +02:00
1 parent af6f8e023e
commit 7bb360fc78
21 files changed
+1265 -62

No files matched your search

+75
View File
@@ -844,6 +844,7 @@ func (a *Agent) session(ctx context.Context, server string) error {
reg.EngineVersions = a.engines(parent)
reg.EnginesProbed = true
reg.InstallableEngines = provision.Installable()
reg.Features = []string{FeatureFabricTest}
reg.RunningShards = a.runningShards()
a.mu.Lock()
h := a.prov.Host
@@ -1145,6 +1146,16 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha
case <-ctx.Done():
}
}(m.FabricScan)
case *llmgwv1.ControlMessage_FabricTest:
// Seconds of traffic, and a package install the first time: off the
// receive loop too.
go func(req *llmgwv1.FabricTestRequest) {
reply := a.fabricTest(ctx, req)
select {
case out <- &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_FabricTest{FabricTest: reply}}:
case <-ctx.Done():
}
}(m.FabricTest)
case *llmgwv1.ControlMessage_Ack:
if !m.Ack.GetOk() {
a.log.Warn("control plane rejected a message",
@@ -1758,6 +1769,70 @@ func (a *Agent) ensureDiags(ctx context.Context) {
}
}
// FeatureFabricTest is the agent answering FabricTestRequest.
const FeatureFabricTest = "fabric-test"
// runWriteBW is discovery.RunWriteBW, and haveWriteBW whether it is
// installed; both replaced in tests.
var (
runWriteBW = discovery.RunWriteBW
haveWriteBW = func() bool { _, err := exec.LookPath("ib_write_bw"); return err == nil }
)
// fabricTest runs this node's side of a bandwidth test with another.
func (a *Agent) fabricTest(ctx context.Context, req *llmgwv1.FabricTestRequest) *llmgwv1.FabricTestReply {
start := time.Now()
reply := &llmgwv1.FabricTestReply{RequestId: req.GetRequestId()}
defer func() { reply.TookMs = time.Since(start).Milliseconds() }()
w := discovery.WriteBW{HCA: req.GetHca(), Port: int(req.GetPort()), GIDIndex: -1,
TCPPort: int(req.GetTcpPort()), Seconds: int(req.GetSeconds())}
if req.GetRole() == "client" {
w.Peer = req.GetPeer()
}
a.mu.Lock()
found := false
for _, h := range a.inv.HCAs {
for _, p := range h.Ports {
if h.Name == w.HCA && p.Num == w.Port {
found = true
// RoCE needs its v2 GID named, as NCCL does; InfiniBand uses the LID.
if p.LinkLayer == discovery.LinkEthernet && p.RoCEv2GID >= 0 {
w.GIDIndex = p.RoCEv2GID
}
}
}
}
a.mu.Unlock()
if !found {
reply.Error = fmt.Sprintf("this node has no port %s/%d", w.HCA, w.Port)
return reply
}
if !haveWriteBW() && a.prov != nil {
a.log.Info("installing perftest, for a bandwidth test with another node")
pkgs := map[string][]string{"deb": {"perftest"}, "rpm": {"perftest"}}
if err := a.prov.InstallPackages(ctx, pkgs); err != nil {
reply.Error = "installing perftest: " + err.Error()
return reply
}
}
gbps, out, err := runWriteBW(ctx, w)
reply.Gbps = gbps
reply.Output = tailLines(out, 12)
if err != nil {
reply.Error = err.Error()
}
return reply
}
func tailLines(s string, n int) string {
lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
if len(lines) > n {
lines = lines[len(lines)-n:]
}
return strings.Join(lines, "\n")
}
// discoverGPUFabric and discoverIMEX are replaced in tests.
var (
discoverGPUFabric = discovery.DiscoverGPUFabric
+32
View File
@@ -23,6 +23,7 @@ import (
"golang.org/x/net/http2"
"golang.org/x/net/http2/h2c"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
@@ -881,3 +882,34 @@ func TestAFabricScanIsAskedForAndAnswered(t *testing.T) {
t.Errorf("asked %q, got %+v", asked, r)
}
}
func TestABandwidthTestIsRunOnTheAdapterAskedFor(t *testing.T) {
oldRun, oldHave := runWriteBW, haveWriteBW
t.Cleanup(func() { runWriteBW, haveWriteBW = oldRun, oldHave })
haveWriteBW = func() bool { return true }
var got discovery.WriteBW
runWriteBW = func(_ context.Context, w discovery.WriteBW) (float64, string, error) {
got = w
return 388.2, " #bytes #iterations BW peak[Gb/sec] BW average[Gb/sec]\n", nil
}
h := start(t, func(c *Config) { c.Discovery.IBRoot = ibTreeWithActivePorts(t, "mlx5_0") })
r, err := h.svc.FabricTest(t.Context(), "n1", &llmgwv1.FabricTestRequest{
Role: "client", Hca: "mlx5_0", Port: 1, Peer: "10.0.0.2", TcpPort: 18515, Seconds: 3})
if err != nil {
t.Fatal(err)
}
if r.GetGbps() != 388.2 || r.GetError() != "" {
t.Errorf("reply %+v", r)
}
if got.HCA != "mlx5_0" || got.Peer != "10.0.0.2" || got.TCPPort != 18515 || got.GIDIndex != -1 {
t.Errorf("ran %+v", got)
}
// An adapter the node does not have is said so, without running anything.
got = discovery.WriteBW{}
r, _ = h.svc.FabricTest(t.Context(), "n1", &llmgwv1.FabricTestRequest{Role: "server", Hca: "mlx5_9", Port: 1, Seconds: 1})
if !strings.Contains(r.GetError(), "no port mlx5_9/1") || got.HCA != "" {
t.Errorf("reply %+v, ran %+v", r, got)
}
}
+133
View File
@@ -0,0 +1,133 @@
package discovery
import (
"context"
"errors"
"fmt"
"os/exec"
"regexp"
"strconv"
"strings"
"time"
)
// WriteBW is one side of an ib_write_bw run, from perftest: an RDMA write
// bandwidth test between two adapters, which is what a collective spanning
// nodes rides on.
type WriteBW struct {
HCA string
Port int
// GIDIndex is the RoCE v2 GID to use; negative on InfiniBand, where the
// port's LID is used.
GIDIndex int
// Peer is the server's address, for the client; empty makes the server.
Peer string
TCPPort int
Seconds int
}
// Args are the command line: a duration rather than a count, so the test
// takes as long on a slow link as a fast one, and results in Gb/s.
func (w WriteBW) Args() []string {
args := []string{"-d", w.HCA, "-i", strconv.Itoa(max(w.Port, 1)),
"-p", strconv.Itoa(w.TCPPort), "-D", strconv.Itoa(max(w.Seconds, 1)),
"-F", "--report_gbits"}
if w.GIDIndex >= 0 {
args = append(args, "-x", strconv.Itoa(w.GIDIndex))
}
if w.Peer != "" {
args = append(args, w.Peer)
}
return args
}
// RunWriteBW runs one side and returns the average bandwidth the output
// reports, in Gb/s, and the output. The client retries its connection for a
// while: the server is started first, but a few seconds may pass before it
// listens.
func RunWriteBW(ctx context.Context, w WriteBW) (float64, string, error) {
bin, err := exec.LookPath("ib_write_bw")
if err != nil {
return 0, "", errors.New("ib_write_bw is not installed (the perftest package)")
}
limit := time.Duration(w.Seconds+60) * time.Second
var out []byte
for attempt := 0; ; attempt++ {
cctx, cancel := context.WithTimeout(ctx, limit)
out, err = exec.CommandContext(cctx, bin, w.Args()...).CombinedOutput() //nolint:gosec // fixed tool
cancel()
if err == nil || w.Peer == "" || !refused(string(out)) || attempt >= 10 {
break
}
select {
case <-ctx.Done():
return 0, string(out), ctx.Err()
case <-time.After(time.Second):
}
}
text := string(out)
if err != nil {
return 0, text, fmt.Errorf("ib_write_bw: %v: %s", err, lastNonEmpty(text))
}
gbps, perr := ParseWriteBW(text)
if perr != nil && w.Peer != "" {
// Only the client is sure to print the result; a server that does not
// is not a failure.
return 0, text, perr
}
return gbps, text, nil
}
func refused(out string) bool {
for _, s := range []string{"Couldn't connect", "Unable to init the socket connection", "Connection refused"} {
if strings.Contains(out, s) {
return true
}
}
return false
}
var columnSplit = regexp.MustCompile(`\s{2,}`)
// ParseWriteBW finds the average bandwidth in perftest's result table:
//
// #bytes #iterations BW peak[Gb/sec] BW average[Gb/sec] MsgRate[Mpps]
// 65536 887036 0.00 96.93 0.184877
//
// by the heading's position rather than a fixed column, since versions add
// columns and the peak reads 0.00 when the test runs for a duration.
func ParseWriteBW(out string) (float64, error) {
lines := strings.Split(out, "\n")
for i, l := range lines {
if !strings.Contains(l, "BW average") {
continue
}
heads := columnSplit.Split(strings.TrimSpace(l), -1)
col := -1
for j, h := range heads {
if strings.HasPrefix(h, "BW average") {
col = j
}
}
for _, d := range lines[i+1:] {
f := strings.Fields(d)
if len(f) == 0 || strings.HasPrefix(f[0], "-") {
continue
}
if len(f) != len(heads) || col < 0 {
return 0, fmt.Errorf("perftest's result table has a shape this does not know: %q / %q", l, d)
}
v, err := strconv.ParseFloat(f[col], 64)
if err != nil {
return 0, fmt.Errorf("reading the average bandwidth %q: %w", f[col], err)
}
return v, nil
}
}
return 0, errors.New("perftest printed no result")
}
func lastNonEmpty(s string) string {
lines := strings.Split(strings.TrimSpace(s), "\n")
return strings.TrimSpace(lines[len(lines)-1])
}
+49
View File
@@ -0,0 +1,49 @@
package discovery
import (
"slices"
"testing"
)
// The client's output of perftest 24.x, with a duration.
const writeBWClient = `---------------------------------------------------------------------------------------
RDMA_Write BW Test
Dual-port : OFF Device : mlx5_0
Number of qps : 1 Transport type : IB
Connection type : RC Using SRQ : OFF
TX depth : 128
CQ Moderation : 100
Mtu : 4096[B]
Link type : IB
Max inline data : 0[B]
rdma_cm QPs : OFF
Data ex. method : Ethernet
---------------------------------------------------------------------------------------
local address: LID 0x05 QPN 0x0112 PSN 0x7b1f02 RKey 0x1fffbd VAddr 0x007f1d2a5f0000
remote address: LID 0x09 QPN 0x0113 PSN 0x2c1e44 RKey 0x1ffeaf VAddr 0x007fa4e1a60000
---------------------------------------------------------------------------------------
#bytes #iterations BW peak[Gb/sec] BW average[Gb/sec] MsgRate[Mpps]
65536 1851214 0.00 388.21 0.740422
---------------------------------------------------------------------------------------
`
func TestTheAverageBandwidthIsReadFromPerftest(t *testing.T) {
got, err := ParseWriteBW(writeBWClient)
if err != nil || got != 388.21 {
t.Errorf("got %v, %v", got, err)
}
if _, err := ParseWriteBW(" Couldn't connect to 10.0.0.2:18515\n"); err == nil {
t.Error("output with no result table gave a number")
}
}
func TestWriteBWArguments(t *testing.T) {
ib := WriteBW{HCA: "mlx5_0", Port: 1, GIDIndex: -1, TCPPort: 18515, Seconds: 3}.Args()
if slices.Contains(ib, "-x") {
t.Errorf("InfiniBand was given a GID index: %v", ib)
}
roce := WriteBW{HCA: "mlx5_0", Port: 1, GIDIndex: 3, Peer: "10.0.0.2", TCPPort: 18515, Seconds: 3}.Args()
if !slices.Contains(roce, "-x") || roce[len(roce)-1] != "10.0.0.2" {
t.Errorf("RoCE client: %v", roce)
}
}