package fabrics import ( "strings" "testing" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" ) func ibNode(id, declared, sm string, hcas ...string) inventory.Node { n := inventory.Node{ID: id} if declared != "" { n.Fabrics = []inventory.FabricAttachment{{Fabric: declared, HCAs: hcas}} } for _, h := range hcas { n.IBPorts = append(n.IBPorts, inventory.IBPort{HCA: h, Port: 1, State: "ACTIVE", RateGbps: 400, LinkLayer: "InfiniBand", SMGUID: sm, SMLID: 1, GIDPrefix: "fe80:0000:0000:0000"}) } return n } func problem(t *testing.T, r Report, l Level, fragment string) { t.Helper() for _, p := range r.Problems { if p.Level == l && strings.Contains(p.Message, fragment) { return } } t.Errorf("no %s problem containing %q in %+v", l, fragment, r.Problems) } func TestIndependentFabricsWithTheirOwnNamesAreFine(t *testing.T) { r := analyze([]inventory.Node{ ibNode("a1", "ib-a", "0x01", "mlx5_0"), ibNode("a2", "ib-a", "0x01", "mlx5_0"), ibNode("b1", "ib-b", "0x02", "mlx5_0"), }) if len(r.Physical) != 2 { t.Fatalf("physical = %+v", r.Physical) } if r.Physical[0].Declared[0] != "ib-a" || len(r.Physical[0].Members) != 2 || r.Physical[0].SM.GUID != "0x01" { t.Errorf("ib-a = %+v", r.Physical[0]) } for _, p := range r.Problems { if p.Level != Info { t.Errorf("unexpected problem: %+v", p) } } // Both on the default prefix: said, quietly. problem(t, r, Info, "share the subnet prefix") } func TestOneNameOnTwoFabricsIsAFailure(t *testing.T) { r := analyze([]inventory.Node{ ibNode("a1", "ib-a", "0x01", "mlx5_0"), ibNode("b1", "ib-a", "0x02", "mlx5_0"), }) problem(t, r, Fail, "ib-a is 2 separate fabrics") } func TestTwoNamesOnOneFabricAreSaid(t *testing.T) { r := analyze([]inventory.Node{ ibNode("a1", "ib-a", "0x01", "mlx5_0"), ibNode("b1", "ib-b", "0x01", "mlx5_0"), }) problem(t, r, Warn, "ib-a and ib-b are one fabric") } func TestAnAdapterUpButUndeclaredIsSaid(t *testing.T) { n := ibNode("a1", "ib-a", "0x01", "mlx5_0") n.IBPorts = append(n.IBPorts, inventory.IBPort{HCA: "mlx5_3", Port: 1, State: "ACTIVE", LinkLayer: "InfiniBand", SMGUID: "0x09"}) problem(t, analyze([]inventory.Node{n}), Warn, "mlx5_3 is up on fabric of subnet manager 0x09") } func TestOneNodesAdaptersOnTwoFabricsAreAFailure(t *testing.T) { n := ibNode("x", "ib-a", "0x01", "mlx5_0", "mlx5_1") n.IBPorts[1].SMGUID = "0x02" problem(t, analyze([]inventory.Node{n}), Fail, "cabled to 2 different fabrics") } func TestANodeThatCannotSayIsReported(t *testing.T) { n := ibNode("x", "ib-a", "", "mlx5_0") n.IBPorts[0].SMError = "sminfo not found: install infiniband-diags" problem(t, analyze([]inventory.Node{n}), Info, "install infiniband-diags") } func TestRoCEIsItsNetwork(t *testing.T) { n := inventory.Node{ID: "r1", Fabrics: []inventory.FabricAttachment{{Fabric: "roce-a", HCAs: []string{"mlx5_0"}}}, IBPorts: []inventory.IBPort{{HCA: "mlx5_0", Port: 1, State: "ACTIVE", LinkLayer: "Ethernet", Network: "10.8.0.0/24"}}} r := analyze([]inventory.Node{n}) if len(r.Physical) != 1 || r.Physical[0].Kind != "roce" || r.Physical[0].Label != "RoCE 10.8.0.0/24" { t.Errorf("physical = %+v", r.Physical) } } func analyze(nodes []inventory.Node) Report { return Analyze(nodes, nil, time.Now()) } func withSMs(n inventory.Node, sms ...inventory.SubnetManager) inventory.Node { n.IBPorts[0].SMs = sms return n } func TestAFailoverKeepsAFabricOneFabric(t *testing.T) { // Mid-failover some ports see the old master, some the new. Keyed by the // master alone, one fabric read as two for a minute — and a declared name // covering it as "two separate fabrics". The list of managers is the same // before and after, which is what keeps it one. sms := []inventory.SubnetManager{{GUID: "0x01", State: "master", Priority: 15}, {GUID: "0x02", State: "standby", Priority: 14}} r := analyze([]inventory.Node{ withSMs(ibNode("a1", "ib-a", "0x01", "mlx5_0"), sms...), ibNode("a2", "ib-a", "0x02", "mlx5_0"), }) if len(r.Physical) != 1 { t.Fatalf("one fabric read as %d: %+v", len(r.Physical), r.Physical) } for _, p := range r.Problems { if p.Level == Fail { t.Errorf("a failover was reported as a failure: %s", p.Message) } } problem(t, r, Warn, "do not agree on the master subnet manager") if r.Physical[0].Standbys != 1 || len(r.Physical[0].SMs) != 2 || r.Physical[0].SMs[0].State != "master" { t.Errorf("managers = %+v", r.Physical[0].SMs) } } func TestAFabricWithNoStandbyIsSaid(t *testing.T) { r := analyze([]inventory.Node{withSMs(ibNode("a1", "ib-a", "0x01", "mlx5_0"), inventory.SubnetManager{GUID: "0x01", State: "master"})}) problem(t, r, Warn, "one subnet manager and no standby") } func TestPortsUpWithoutASubnetManagerAreAFailure(t *testing.T) { n := ibNode("a1", "ib-a", "", "mlx5_0") n.IBPorts[0].State, n.IBPorts[0].PhysState = "INIT", "LinkUp" problem(t, analyze([]inventory.Node{n}), Fail, "no subnet manager has configured them") } func TestARecentFailoverIsShown(t *testing.T) { now := time.Now() nodes := []inventory.Node{withSMs(ibNode("a1", "ib-a", "0x02", "mlx5_0"), inventory.SubnetManager{GUID: "0x01", State: "standby"}, inventory.SubnetManager{GUID: "0x02", State: "master"})} changes := []inventory.SMChange{ {At: now.Add(-20 * time.Minute), Node: "a1", Port: "mlx5_0/1", From: "0x01", To: "0x02"}, {At: now.Add(-48 * time.Hour), Node: "a1", Port: "mlx5_0/1", From: "0x02", To: "0x01"}, } r := Analyze(nodes, changes, now) if got := r.Physical[0].Changes; len(got) != 1 || got[0].To != "0x02" { t.Errorf("changes in the last day = %+v", got) } problem(t, r, Warn, "the master subnet manager changed 20m0s ago") }