llmgw/internal/control/fabrics/fabrics_test.go

package fabrics

import (
	"strings"
	"testing"
	"time"

	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)

func ibNode(id, declared, sm string, hcas ...string) inventory.Node {
	n := inventory.Node{ID: id}
	if declared != "" {
		n.Fabrics = []inventory.FabricAttachment{{Fabric: declared, HCAs: hcas}}
	}
	for _, h := range hcas {
		n.IBPorts = append(n.IBPorts, inventory.IBPort{HCA: h, Port: 1, State: "ACTIVE", RateGbps: 400,
			LinkLayer: "InfiniBand", SMGUID: sm, SMLID: 1, GIDPrefix: "fe80:0000:0000:0000"})
	}
	return n
}

func problem(t *testing.T, r Report, l Level, fragment string) {
	t.Helper()
	for _, p := range r.Problems {
		if p.Level == l && strings.Contains(p.Message, fragment) {
			return
		}
	}
	t.Errorf("no %s problem containing %q in %+v", l, fragment, r.Problems)
}

func TestIndependentFabricsWithTheirOwnNamesAreFine(t *testing.T) {
	r := analyze([]inventory.Node{
		ibNode("a1", "ib-a", "0x01", "mlx5_0"), ibNode("a2", "ib-a", "0x01", "mlx5_0"),
		ibNode("b1", "ib-b", "0x02", "mlx5_0"),
	})
	if len(r.Physical) != 2 {
		t.Fatalf("physical = %+v", r.Physical)
	}
	if r.Physical[0].Declared[0] != "ib-a" || len(r.Physical[0].Members) != 2 || r.Physical[0].SM.GUID != "0x01" {
		t.Errorf("ib-a = %+v", r.Physical[0])
	}
	for _, p := range r.Problems {
		if p.Level != Info {
			t.Errorf("unexpected problem: %+v", p)
		}
	}
	// Both on the default prefix: said, quietly.
	problem(t, r, Info, "share the subnet prefix")
}

func TestOneNameOnTwoFabricsIsAFailure(t *testing.T) {
	r := analyze([]inventory.Node{
		ibNode("a1", "ib-a", "0x01", "mlx5_0"), ibNode("b1", "ib-a", "0x02", "mlx5_0"),
	})
	problem(t, r, Fail, "ib-a is 2 separate fabrics")
}

func TestTwoNamesOnOneFabricAreSaid(t *testing.T) {
	r := analyze([]inventory.Node{
		ibNode("a1", "ib-a", "0x01", "mlx5_0"), ibNode("b1", "ib-b", "0x01", "mlx5_0"),
	})
	problem(t, r, Warn, "ib-a and ib-b are one fabric")
}

func TestAnAdapterUpButUndeclaredIsSaid(t *testing.T) {
	n := ibNode("a1", "ib-a", "0x01", "mlx5_0")
	n.IBPorts = append(n.IBPorts, inventory.IBPort{HCA: "mlx5_3", Port: 1, State: "ACTIVE", LinkLayer: "InfiniBand", SMGUID: "0x09"})
	problem(t, analyze([]inventory.Node{n}), Warn, "mlx5_3 is up on fabric of subnet manager 0x09")
}

func TestOneNodesAdaptersOnTwoFabricsAreAFailure(t *testing.T) {
	n := ibNode("x", "ib-a", "0x01", "mlx5_0", "mlx5_1")
	n.IBPorts[1].SMGUID = "0x02"
	problem(t, analyze([]inventory.Node{n}), Fail, "cabled to 2 different fabrics")
}

func TestANodeThatCannotSayIsReported(t *testing.T) {
	n := ibNode("x", "ib-a", "", "mlx5_0")
	n.IBPorts[0].SMError = "sminfo not found: install infiniband-diags"
	problem(t, analyze([]inventory.Node{n}), Info, "install infiniband-diags")
}

func TestRoCEIsItsNetwork(t *testing.T) {
	n := inventory.Node{ID: "r1", Fabrics: []inventory.FabricAttachment{{Fabric: "roce-a", HCAs: []string{"mlx5_0"}}},
		IBPorts: []inventory.IBPort{{HCA: "mlx5_0", Port: 1, State: "ACTIVE", LinkLayer: "Ethernet", Network: "10.8.0.0/24"}}}
	r := analyze([]inventory.Node{n})
	if len(r.Physical) != 1 || r.Physical[0].Kind != "roce" || r.Physical[0].Label != "RoCE 10.8.0.0/24" {
		t.Errorf("physical = %+v", r.Physical)
	}
}

func analyze(nodes []inventory.Node) Report { return Analyze(nodes, nil, time.Now()) }

func withSMs(n inventory.Node, sms ...inventory.SubnetManager) inventory.Node {
	n.IBPorts[0].SMs = sms
	return n
}

func TestAFailoverKeepsAFabricOneFabric(t *testing.T) {
	// Mid-failover some ports see the old master, some the new. Keyed by the
	// master alone, one fabric read as two for a minute — and a declared name
	// covering it as "two separate fabrics". The list of managers is the same
	// before and after, which is what keeps it one.
	sms := []inventory.SubnetManager{{GUID: "0x01", State: "master", Priority: 15}, {GUID: "0x02", State: "standby", Priority: 14}}
	r := analyze([]inventory.Node{
		withSMs(ibNode("a1", "ib-a", "0x01", "mlx5_0"), sms...),
		ibNode("a2", "ib-a", "0x02", "mlx5_0"),
	})
	if len(r.Physical) != 1 {
		t.Fatalf("one fabric read as %d: %+v", len(r.Physical), r.Physical)
	}
	for _, p := range r.Problems {
		if p.Level == Fail {
			t.Errorf("a failover was reported as a failure: %s", p.Message)
		}
	}
	problem(t, r, Warn, "do not agree on the master subnet manager")
	if r.Physical[0].Standbys != 1 || len(r.Physical[0].SMs) != 2 || r.Physical[0].SMs[0].State != "master" {
		t.Errorf("managers = %+v", r.Physical[0].SMs)
	}
}

func TestAFabricWithNoStandbyIsSaid(t *testing.T) {
	r := analyze([]inventory.Node{withSMs(ibNode("a1", "ib-a", "0x01", "mlx5_0"),
		inventory.SubnetManager{GUID: "0x01", State: "master"})})
	problem(t, r, Warn, "one subnet manager and no standby")
}

func TestPortsUpWithoutASubnetManagerAreAFailure(t *testing.T) {
	n := ibNode("a1", "ib-a", "", "mlx5_0")
	n.IBPorts[0].State, n.IBPorts[0].PhysState = "INIT", "LinkUp"
	problem(t, analyze([]inventory.Node{n}), Fail, "no subnet manager has configured them")
}

func TestARecentFailoverIsShown(t *testing.T) {
	now := time.Now()
	nodes := []inventory.Node{withSMs(ibNode("a1", "ib-a", "0x02", "mlx5_0"),
		inventory.SubnetManager{GUID: "0x01", State: "standby"}, inventory.SubnetManager{GUID: "0x02", State: "master"})}
	changes := []inventory.SMChange{
		{At: now.Add(-20 * time.Minute), Node: "a1", Port: "mlx5_0/1", From: "0x01", To: "0x02"},
		{At: now.Add(-48 * time.Hour), Node: "a1", Port: "mlx5_0/1", From: "0x02", To: "0x01"},
	}
	r := Analyze(nodes, changes, now)
	if got := r.Physical[0].Changes; len(got) != 1 || got[0].To != "0x02" {
		t.Errorf("changes in the last day = %+v", got)
	}
	problem(t, r, Warn, "the master subnet manager changed 20m0s ago")
}