A fabric was what agent.yaml called it. With several independent InfiniBand fabrics at a site, two given one name would have had a replica spread across both, starting and then hanging on its first collective. Each active InfiniBand port now reports the master subnet manager it sees (sminfo, asked once a minute), which every port on one fabric shares and ports on separate fabrics cannot; a RoCE port reports the IP network of its RoCE v2 address. Ports also carry their node and port GUIDs. Placement divides a declared fabric by that physical identity and forms a replica on one part only; a node whose adapters for one name are on two fabrics is refused, and a node that cannot say is kept apart from those that can. /api/v1/fabrics and the cluster page list each physical fabric with its subnet manager, and say when one name covers two fabrics, two names one fabric, an adapter is up but undeclared, or a node cannot be checked. doctor reports each port's subnet manager, and an agent on a node with adapters installs infiniband-diags when sminfo is missing. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
326 lines
9.6 KiB
Go
326 lines
9.6 KiB
Go
package discovery
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
)
|
|
|
|
// fakeSysfs builds an InfiniBand tree shaped like the real one.
|
|
func fakeSysfs(t *testing.T) string {
|
|
t.Helper()
|
|
root := t.TempDir()
|
|
|
|
write := func(path, content string) {
|
|
t.Helper()
|
|
full := filepath.Join(root, path)
|
|
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(full, []byte(content+"\n"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// mlx5_0: one active IB port.
|
|
write("ib/mlx5_0/fw_ver", "20.35.1012")
|
|
write("ib/mlx5_0/ports/1/state", "4: ACTIVE")
|
|
write("ib/mlx5_0/ports/1/phys_state", "5: LinkUp")
|
|
write("ib/mlx5_0/ports/1/rate", "200 Gb/sec (4X HDR)")
|
|
write("ib/mlx5_0/ports/1/link_layer", "InfiniBand")
|
|
write("ib/mlx5_0/ports/1/gids/0", "FEC0:0000:0000:000A:0002:C903:00A0:1234")
|
|
write("ib/mlx5_0/ports/1/lid", "0x1a")
|
|
write("ib/mlx5_0/ports/1/sm_lid", "1")
|
|
write("ib/mlx5_0/ports/1/counters/symbol_error", "3")
|
|
write("ib/mlx5_0/ports/1/counters/link_downed", "1")
|
|
write("ib/mlx5_0/ports/1/counters/port_xmit_data", "1000")
|
|
|
|
// mlx5_1: two ports, the second one down.
|
|
write("ib/mlx5_1/ports/1/state", "4: ACTIVE")
|
|
write("ib/mlx5_1/ports/1/rate", "200 Gb/sec (4X HDR)")
|
|
write("ib/mlx5_1/ports/1/link_layer", "InfiniBand")
|
|
write("ib/mlx5_1/ports/1/gids/0", "fec0:0000:0000:000a:0002:c903:00a0:5678")
|
|
write("ib/mlx5_1/ports/2/state", "1: DOWN")
|
|
write("ib/mlx5_1/ports/2/phys_state", "3: Disabled")
|
|
write("ib/mlx5_1/ports/2/rate", "0")
|
|
write("ib/mlx5_1/ports/2/link_layer", "InfiniBand")
|
|
|
|
// mlx5_2: RoCE rather than native IB.
|
|
write("ib/mlx5_2/ports/1/state", "4: ACTIVE")
|
|
write("ib/mlx5_2/ports/1/rate", "100 Gb/sec (4X EDR)")
|
|
write("ib/mlx5_2/ports/1/link_layer", "Ethernet")
|
|
write("ib/mlx5_2/ports/1/gids/0", "fe80:0000:0000:0000:0000:0000:0000:0001")
|
|
write("ib/mlx5_2/ports/1/gid_attrs/types/0", "IB/RoCE v1")
|
|
write("ib/mlx5_2/ports/1/gids/1", "fe80:0000:0000:0000:0000:0000:0000:0001")
|
|
write("ib/mlx5_2/ports/1/gid_attrs/types/1", "RoCE v2")
|
|
write("ib/mlx5_2/ports/1/gids/3", "0000:0000:0000:0000:0000:ffff:0a00:0001")
|
|
write("ib/mlx5_2/ports/1/gid_attrs/types/3", "RoCE v2")
|
|
write("ib/mlx5_2/ports/1/gid_attrs/ndevs/3", "ens1f0")
|
|
|
|
// A PCI device with a NUMA node, linked the way the driver does it.
|
|
pci := filepath.Join(root, "pci", "0000:0b:00.0")
|
|
if err := os.MkdirAll(pci, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(pci, "numa_node"), []byte("1\n"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.Symlink(pci, filepath.Join(root, "ib", "mlx5_0", "device")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
return filepath.Join(root, "ib")
|
|
}
|
|
|
|
func TestDiscoverIB(t *testing.T) {
|
|
hcas, err := DiscoverIB(fakeSysfs(t))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(hcas) != 3 {
|
|
t.Fatalf("expected 3 HCAs, got %d", len(hcas))
|
|
}
|
|
// Sorted by name, so the order is predictable for the rest of the test.
|
|
if hcas[0].Name != "mlx5_0" || hcas[2].Name != "mlx5_2" {
|
|
t.Fatalf("HCAs are not sorted by name: %v %v", hcas[0].Name, hcas[2].Name)
|
|
}
|
|
|
|
h := hcas[0]
|
|
if h.FirmwareVersion != "20.35.1012" {
|
|
t.Errorf("fw_ver=%q", h.FirmwareVersion)
|
|
}
|
|
if h.PCIAddr != "0000:0b:00.0" {
|
|
t.Errorf("PCIAddr=%q", h.PCIAddr)
|
|
}
|
|
if h.NUMANode != 1 {
|
|
t.Errorf("NUMANode=%d; GPU-to-HCA affinity depends on this", h.NUMANode)
|
|
}
|
|
|
|
p := h.Ports[0]
|
|
if p.State != "ACTIVE" || !p.Active() {
|
|
t.Errorf("state=%q", p.State)
|
|
}
|
|
if p.PhysState != "LinkUp" {
|
|
t.Errorf("phys_state=%q", p.PhysState)
|
|
}
|
|
if p.RateGbps != 200 {
|
|
t.Errorf("rate=%d", p.RateGbps)
|
|
}
|
|
if p.LinkLayer != LinkInfiniBand {
|
|
t.Errorf("link_layer=%q", p.LinkLayer)
|
|
}
|
|
if p.GIDPrefix != "fec0:0000:0000:000a" {
|
|
t.Errorf("GIDPrefix=%q; it must be lowercased and cut to four groups", p.GIDPrefix)
|
|
}
|
|
if p.LID != 26 {
|
|
t.Errorf("LID=%d, expected 0x1a to parse as 26", p.LID)
|
|
}
|
|
if p.SMLID != 1 {
|
|
t.Errorf("SMLID=%d, expected a decimal value to parse too", p.SMLID)
|
|
}
|
|
}
|
|
|
|
func TestActivePortsExcludesDown(t *testing.T) {
|
|
hcas, err := DiscoverIB(fakeSysfs(t))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h := hcas[1] // mlx5_1
|
|
if len(h.Ports) != 2 {
|
|
t.Fatalf("expected 2 ports, got %d", len(h.Ports))
|
|
}
|
|
active := h.ActivePorts()
|
|
if len(active) != 1 || active[0].Num != 1 {
|
|
t.Errorf("a DOWN port was reported as active: %+v", active)
|
|
}
|
|
}
|
|
|
|
func TestRoCEIsDistinguished(t *testing.T) {
|
|
// RoCE is usable but behaves differently from native IB; the placement
|
|
// logic needs to be able to tell them apart.
|
|
hcas, err := DiscoverIB(fakeSysfs(t))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := hcas[2].Ports[0].LinkLayer; got != LinkEthernet {
|
|
t.Errorf("mlx5_2 link_layer=%q, expected Ethernet (RoCE)", got)
|
|
}
|
|
}
|
|
|
|
func TestMissingRootIsNotAnError(t *testing.T) {
|
|
// A node without InfiniBand is perfectly normal; it just forms its own
|
|
// single-node island.
|
|
hcas, err := DiscoverIB(filepath.Join(t.TempDir(), "nonexistent"))
|
|
if err != nil {
|
|
t.Fatalf("a node without IB should not be an error: %v", err)
|
|
}
|
|
if len(hcas) != 0 {
|
|
t.Errorf("expected no HCAs, got %d", len(hcas))
|
|
}
|
|
}
|
|
|
|
func TestUnreadableAdapterDoesNotHideOthers(t *testing.T) {
|
|
root := fakeSysfs(t)
|
|
// An adapter with no ports directory at all.
|
|
if err := os.MkdirAll(filepath.Join(root, "mlx5_broken"), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
hcas, err := DiscoverIB(root)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(hcas) != 3 {
|
|
t.Errorf("a broken adapter changed the result: got %d HCAs", len(hcas))
|
|
}
|
|
}
|
|
|
|
func TestParseHelpers(t *testing.T) {
|
|
t.Run("rate", func(t *testing.T) {
|
|
for in, want := range map[string]int{
|
|
"200 Gb/sec (4X HDR)": 200,
|
|
"100 Gb/sec (4X EDR)": 100,
|
|
"2.5 Gb/sec (1X SDR)": 2,
|
|
"": 0,
|
|
"garbage": 0,
|
|
} {
|
|
if got := parseRate(in); got != want {
|
|
t.Errorf("parseRate(%q)=%d, want %d", in, got, want)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("state", func(t *testing.T) {
|
|
for in, want := range map[string]string{
|
|
"4: ACTIVE": "ACTIVE",
|
|
"1: DOWN": "DOWN",
|
|
"5: LinkUp": "LinkUp",
|
|
"ACTIVE": "ACTIVE",
|
|
"": "",
|
|
} {
|
|
if got := parseEnumeratedState(in); got != want {
|
|
t.Errorf("parseEnumeratedState(%q)=%q, want %q", in, got, want)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("hex", func(t *testing.T) {
|
|
for in, want := range map[string]int{
|
|
"0x1a": 26, "0X1A": 26, "26": 26, "": 0, "zzz": 0,
|
|
} {
|
|
if got := parseMaybeHex(in); got != want {
|
|
t.Errorf("parseMaybeHex(%q)=%d, want %d", in, got, want)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("gid", func(t *testing.T) {
|
|
if got := parseGIDPrefix("FE80:0000:0000:0000:0002:C903:00A0:1234"); got != "fe80:0000:0000:0000" {
|
|
t.Errorf("parseGIDPrefix=%q", got)
|
|
}
|
|
if got := parseGIDPrefix("short:gid"); got != "" {
|
|
t.Errorf("a malformed GID should yield an empty prefix, got %q", got)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestAPortsCountersAreRead(t *testing.T) {
|
|
hcas, err := DiscoverIB(fakeSysfs(t))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
c := hcas[0].Ports[0].Counters
|
|
// Data in bytes: sysfs counts four-byte words.
|
|
if c["symbol_error"] != 3 || c["link_downed"] != 1 || c["port_xmit_data"] != 4000 {
|
|
t.Errorf("counters = %v", c)
|
|
}
|
|
if _, ok := c["port_rcv_errors"]; ok {
|
|
t.Error("a counter the driver does not have should be absent, not zero")
|
|
}
|
|
}
|
|
|
|
func TestRoCEFindsItsRoutableGID(t *testing.T) {
|
|
// NCCL's default GID index is a v1 link-local entry on most hosts, which
|
|
// does not route: the job hangs. The v2 entry holding an IPv4 address is
|
|
// the one to use, over an earlier v2 link-local one.
|
|
hcas, err := DiscoverIB(fakeSysfs(t))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, h := range hcas {
|
|
p := h.Ports[0]
|
|
switch h.Name {
|
|
case "mlx5_2":
|
|
if p.RoCEv2GID != 3 {
|
|
t.Errorf("RoCE GID index = %d, want 3", p.RoCEv2GID)
|
|
}
|
|
default:
|
|
if p.RoCEv2GID != -1 {
|
|
t.Errorf("%s is InfiniBand and has a GID index %d", h.Name, p.RoCEv2GID)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTheSubnetManagerAPortSeesIsRead(t *testing.T) {
|
|
sm, err := ParseSMInfo("sminfo: sm lid 1 sm guid 0x98039B03009fcfd6, activity count 5452 priority 15 state 3 SMINFO_MASTER\n")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if sm.GUID != "0x98039b03009fcfd6" || sm.LID != 1 || sm.Priority != 15 || sm.State != "SMINFO_MASTER" {
|
|
t.Errorf("sm = %+v", sm)
|
|
}
|
|
if _, err := ParseSMInfo("ibwarn: [1234] mad_rpc_open_port: can't open UMAD port"); err == nil {
|
|
t.Error("an error from sminfo should not parse")
|
|
}
|
|
}
|
|
|
|
func TestAPortsFabricIsItsSubnetManagerOrItsNetwork(t *testing.T) {
|
|
// Two ports on one InfiniBand fabric see one master SM, and on separate
|
|
// fabrics cannot; RoCE has no SM, and is its IP network.
|
|
hcas, err := DiscoverIB(fakeSysfs(t))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if hcas[0].NodeGUID != "" {
|
|
t.Errorf("no node_guid file, yet %q", hcas[0].NodeGUID)
|
|
}
|
|
err = AttachSM(hcas, func(hca string, port int) (*SMInfo, error) {
|
|
return &SMInfo{GUID: "0x00000000000000aa", LID: 1, State: "SMINFO_MASTER"}, nil
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := hcas[0].Ports[0].FabricKey(); got != "sm:0x00000000000000aa" {
|
|
t.Errorf("IB key = %q", got)
|
|
}
|
|
if got := hcas[1].Ports[1].FabricKey(); got != "" {
|
|
t.Errorf("a port that is down is on no fabric, got %q", got)
|
|
}
|
|
if hcas[2].Ports[0].SM != nil {
|
|
t.Error("RoCE was asked for a subnet manager")
|
|
}
|
|
if got := hcas[0].Ports[0].PortGUID; got != "0x0002c90300a01234" {
|
|
t.Errorf("port GUID = %q", got)
|
|
}
|
|
}
|
|
|
|
func TestARoCEPortsNetworkIsItsFabric(t *testing.T) {
|
|
old := RoCENetwork
|
|
t.Cleanup(func() { RoCENetwork = old })
|
|
RoCENetwork = func(ndev, gid string) string {
|
|
if ip := gidIPv4(gid); ip != nil && ip.String() == "10.0.0.1" {
|
|
return "10.0.0.0/24"
|
|
}
|
|
return ""
|
|
}
|
|
hcas, err := DiscoverIB(fakeSysfs(t))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
roce := hcas[2].Ports[0]
|
|
if roce.Network != "10.0.0.0/24" || roce.FabricKey() != "roce:10.0.0.0/24" {
|
|
t.Errorf("RoCE network %q key %q", roce.Network, roce.FabricKey())
|
|
}
|
|
}
|