package fabrictest import ( "context" "strings" "sync" "testing" "time" llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" ) func node(id string, hcas ...string) inventory.Node { n := inventory.Node{ID: id, MgmtAddr: "10.0.0." + id[len(id)-1:], Features: []string{"fabric-test"}, Fabrics: []inventory.FabricAttachment{{Fabric: "ib-a", HCAs: hcas}}} for _, h := range hcas { n.IBPorts = append(n.IBPorts, inventory.IBPort{HCA: h, Port: 1, State: "ACTIVE", RateGbps: 400}) } return n } // fakeFabric answers as two agents would: the server waits for its client. type fakeFabric struct { mu sync.Mutex gbps map[string]float64 // by client adapter fails map[string]string asked []string } func (f *fakeFabric) FabricTest(_ context.Context, n string, req *llmgwv1.FabricTestRequest) (*llmgwv1.FabricTestReply, error) { f.mu.Lock() defer f.mu.Unlock() f.asked = append(f.asked, n+" "+req.GetRole()+" "+req.GetHca()) if req.GetRole() == "server" { return &llmgwv1.FabricTestReply{}, nil } if e := f.fails[req.GetHca()]; e != "" { return &llmgwv1.FabricTestReply{Error: e}, nil } return &llmgwv1.FabricTestReply{Gbps: f.gbps[req.GetHca()]}, nil } func TestRailsArePairedByPositionOnTheSharedFabric(t *testing.T) { a, b := node("n1", "mlx5_0", "mlx5_1"), node("n2", "mlx5_0", "mlx5_1", "mlx5_2") fabric, rails, notes, err := Plan(a, b, "") if err != nil { t.Fatal(err) } if fabric != "ib-a" || len(rails) != 2 || rails[1].A != "mlx5_1" || rails[1].B != "mlx5_1" || rails[0].RateGbps != 400 { t.Errorf("got %s %+v", fabric, rails) } if len(notes) != 1 || !strings.Contains(notes[0], "only the first 2") { t.Errorf("an unequal count went unmentioned: %v", notes) } } func TestNodesWithNoFabricInCommonAreSaidSo(t *testing.T) { b := node("n2", "mlx5_0") b.Fabrics[0].Fabric = "ib-b" if _, _, _, err := Plan(node("n1", "mlx5_0"), b, ""); err == nil || !strings.Contains(err.Error(), "no fabric in common") { t.Errorf("got %v", err) } } func TestEachRailIsGradedAgainstItsLink(t *testing.T) { a, b := node("n1", "mlx5_0", "mlx5_1", "mlx5_2"), node("n2", "mlx5_0", "mlx5_1", "mlx5_2") _, rails, _, _ := Plan(a, b, "ib-a") f := &fakeFabric{ gbps: map[string]float64{"mlx5_0": 388, "mlx5_1": 240}, fails: map[string]string{"mlx5_2": "ib_write_bw: Couldn't connect to 10.0.0.2:18517"}, } rep := Run(context.Background(), f, a, b, "ib-a", rails, Options{Seconds: 1, Settle: time.Millisecond}) want := []string{"ok", "slow", "fail"} for i, r := range rep.Results { if r.Verdict != want[i] { t.Errorf("rail %d: %s (%s), want %s", i, r.Verdict, r.Detail, want[i]) } } if rep.OK() { t.Error("a report with a failed rail is OK") } if !strings.Contains(rep.Results[2].Detail, "Couldn't connect") { t.Errorf("the reason was lost: %q", rep.Results[2].Detail) } // Servers on the second node, clients on the first, one rail at a time. if f.asked[0] != "n2 server mlx5_0" || f.asked[1] != "n1 client mlx5_0" { t.Errorf("asked %v", f.asked) } } func TestAnAgentTooOldToAnswerIsRefusedAtOnce(t *testing.T) { // It would ignore the request, and the control plane would wait minutes. old := node("n2", "mlx5_0") old.Features, old.AgentVersion = nil, "v0.10.0" _, _, _, err := Plan(node("n1", "mlx5_0"), old, "") if err == nil || !strings.Contains(err.Error(), "n2's agent (v0.10.0) is too old") { t.Errorf("got %v", err) } }