Ilker ManapandClaude Opus 5.5
7bb360fc78
doctor checks a node alone, and every check can pass while two nodes still cannot reach each other. ctl fabric-test A B asks B's agent for an ib_write_bw server and A's for the client, adapter by adapter as NCCL pairs them, one rail at a time, and grades each against its link's rate. It goes through the agents' streams, so there is no SSH between nodes; the agent installs perftest the first time, and names the RoCE v2 GID itself. The result is read from perftest's table by its heading, not a column number. Agents now list the requests they answer beyond the usual (Register.features), and the control plane refuses a test on one too old to answer instead of waiting minutes for it. The forwarding dispatcher carries the new capability, and its test says so: a missing forward would have made the command quietly unavailable in production. Not yet run over real InfiniBand or RoCE; this cluster has neither. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
50 lines
1.9 KiB
Go
50 lines
1.9 KiB
Go
package discovery
|
|
|
|
import (
|
|
"slices"
|
|
"testing"
|
|
)
|
|
|
|
// The client's output of perftest 24.x, with a duration.
|
|
const writeBWClient = `---------------------------------------------------------------------------------------
|
|
RDMA_Write BW Test
|
|
Dual-port : OFF Device : mlx5_0
|
|
Number of qps : 1 Transport type : IB
|
|
Connection type : RC Using SRQ : OFF
|
|
TX depth : 128
|
|
CQ Moderation : 100
|
|
Mtu : 4096[B]
|
|
Link type : IB
|
|
Max inline data : 0[B]
|
|
rdma_cm QPs : OFF
|
|
Data ex. method : Ethernet
|
|
---------------------------------------------------------------------------------------
|
|
local address: LID 0x05 QPN 0x0112 PSN 0x7b1f02 RKey 0x1fffbd VAddr 0x007f1d2a5f0000
|
|
remote address: LID 0x09 QPN 0x0113 PSN 0x2c1e44 RKey 0x1ffeaf VAddr 0x007fa4e1a60000
|
|
---------------------------------------------------------------------------------------
|
|
#bytes #iterations BW peak[Gb/sec] BW average[Gb/sec] MsgRate[Mpps]
|
|
65536 1851214 0.00 388.21 0.740422
|
|
---------------------------------------------------------------------------------------
|
|
`
|
|
|
|
func TestTheAverageBandwidthIsReadFromPerftest(t *testing.T) {
|
|
got, err := ParseWriteBW(writeBWClient)
|
|
if err != nil || got != 388.21 {
|
|
t.Errorf("got %v, %v", got, err)
|
|
}
|
|
if _, err := ParseWriteBW(" Couldn't connect to 10.0.0.2:18515\n"); err == nil {
|
|
t.Error("output with no result table gave a number")
|
|
}
|
|
}
|
|
|
|
func TestWriteBWArguments(t *testing.T) {
|
|
ib := WriteBW{HCA: "mlx5_0", Port: 1, GIDIndex: -1, TCPPort: 18515, Seconds: 3}.Args()
|
|
if slices.Contains(ib, "-x") {
|
|
t.Errorf("InfiniBand was given a GID index: %v", ib)
|
|
}
|
|
roce := WriteBW{HCA: "mlx5_0", Port: 1, GIDIndex: 3, Peer: "10.0.0.2", TCPPort: 18515, Seconds: 3}.Args()
|
|
if !slices.Contains(roce, "-x") || roce[len(roce)-1] != "10.0.0.2" {
|
|
t.Errorf("RoCE client: %v", roce)
|
|
}
|
|
}
|