A fabric was what agent.yaml called it. With several independent InfiniBand fabrics at a site, two given one name would have had a replica spread across both, starting and then hanging on its first collective. Each active InfiniBand port now reports the master subnet manager it sees (sminfo, asked once a minute), which every port on one fabric shares and ports on separate fabrics cannot; a RoCE port reports the IP network of its RoCE v2 address. Ports also carry their node and port GUIDs. Placement divides a declared fabric by that physical identity and forms a replica on one part only; a node whose adapters for one name are on two fabrics is refused, and a node that cannot say is kept apart from those that can. /api/v1/fabrics and the cluster page list each physical fabric with its subnet manager, and say when one name covers two fabrics, two names one fabric, an adapter is up but undeclared, or a node cannot be checked. doctor reports each port's subnet manager, and an agent on a node with adapters installs infiniband-diags when sminfo is missing. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
92 lines
3.1 KiB
Go
92 lines
3.1 KiB
Go
package fabrics
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
)
|
|
|
|
func ibNode(id, declared, sm string, hcas ...string) inventory.Node {
|
|
n := inventory.Node{ID: id}
|
|
if declared != "" {
|
|
n.Fabrics = []inventory.FabricAttachment{{Fabric: declared, HCAs: hcas}}
|
|
}
|
|
for _, h := range hcas {
|
|
n.IBPorts = append(n.IBPorts, inventory.IBPort{HCA: h, Port: 1, State: "ACTIVE", RateGbps: 400,
|
|
LinkLayer: "InfiniBand", SMGUID: sm, SMLID: 1, GIDPrefix: "fe80:0000:0000:0000"})
|
|
}
|
|
return n
|
|
}
|
|
|
|
func problem(t *testing.T, r Report, l Level, fragment string) {
|
|
t.Helper()
|
|
for _, p := range r.Problems {
|
|
if p.Level == l && strings.Contains(p.Message, fragment) {
|
|
return
|
|
}
|
|
}
|
|
t.Errorf("no %s problem containing %q in %+v", l, fragment, r.Problems)
|
|
}
|
|
|
|
func TestIndependentFabricsWithTheirOwnNamesAreFine(t *testing.T) {
|
|
r := Analyze([]inventory.Node{
|
|
ibNode("a1", "ib-a", "0x01", "mlx5_0"), ibNode("a2", "ib-a", "0x01", "mlx5_0"),
|
|
ibNode("b1", "ib-b", "0x02", "mlx5_0"),
|
|
})
|
|
if len(r.Physical) != 2 {
|
|
t.Fatalf("physical = %+v", r.Physical)
|
|
}
|
|
if r.Physical[0].Declared[0] != "ib-a" || len(r.Physical[0].Members) != 2 || r.Physical[0].SM.GUID != "0x01" {
|
|
t.Errorf("ib-a = %+v", r.Physical[0])
|
|
}
|
|
for _, p := range r.Problems {
|
|
if p.Level != Info {
|
|
t.Errorf("unexpected problem: %+v", p)
|
|
}
|
|
}
|
|
// Both on the default prefix: said, quietly.
|
|
problem(t, r, Info, "share the subnet prefix")
|
|
}
|
|
|
|
func TestOneNameOnTwoFabricsIsAFailure(t *testing.T) {
|
|
r := Analyze([]inventory.Node{
|
|
ibNode("a1", "ib-a", "0x01", "mlx5_0"), ibNode("b1", "ib-a", "0x02", "mlx5_0"),
|
|
})
|
|
problem(t, r, Fail, "ib-a is 2 separate fabrics")
|
|
}
|
|
|
|
func TestTwoNamesOnOneFabricAreSaid(t *testing.T) {
|
|
r := Analyze([]inventory.Node{
|
|
ibNode("a1", "ib-a", "0x01", "mlx5_0"), ibNode("b1", "ib-b", "0x01", "mlx5_0"),
|
|
})
|
|
problem(t, r, Warn, "ib-a and ib-b are one fabric")
|
|
}
|
|
|
|
func TestAnAdapterUpButUndeclaredIsSaid(t *testing.T) {
|
|
n := ibNode("a1", "ib-a", "0x01", "mlx5_0")
|
|
n.IBPorts = append(n.IBPorts, inventory.IBPort{HCA: "mlx5_3", Port: 1, State: "ACTIVE", LinkLayer: "InfiniBand", SMGUID: "0x09"})
|
|
problem(t, Analyze([]inventory.Node{n}), Warn, "mlx5_3 is up on subnet manager 0x09")
|
|
}
|
|
|
|
func TestOneNodesAdaptersOnTwoFabricsAreAFailure(t *testing.T) {
|
|
n := ibNode("x", "ib-a", "0x01", "mlx5_0", "mlx5_1")
|
|
n.IBPorts[1].SMGUID = "0x02"
|
|
problem(t, Analyze([]inventory.Node{n}), Fail, "cabled to 2 different fabrics")
|
|
}
|
|
|
|
func TestANodeThatCannotSayIsReported(t *testing.T) {
|
|
n := ibNode("x", "ib-a", "", "mlx5_0")
|
|
n.IBPorts[0].SMError = "sminfo not found: install infiniband-diags"
|
|
problem(t, Analyze([]inventory.Node{n}), Info, "install infiniband-diags")
|
|
}
|
|
|
|
func TestRoCEIsItsNetwork(t *testing.T) {
|
|
n := inventory.Node{ID: "r1", Fabrics: []inventory.FabricAttachment{{Fabric: "roce-a", HCAs: []string{"mlx5_0"}}},
|
|
IBPorts: []inventory.IBPort{{HCA: "mlx5_0", Port: 1, State: "ACTIVE", LinkLayer: "Ethernet", Network: "10.8.0.0/24"}}}
|
|
r := Analyze([]inventory.Node{n})
|
|
if len(r.Physical) != 1 || r.Physical[0].Kind != "roce" || r.Physical[0].Label != "RoCE 10.8.0.0/24" {
|
|
t.Errorf("physical = %+v", r.Physical)
|
|
}
|
|
}
|