package discovery import ( "strings" "testing" ) func hca(name string, ports ...Port) HCA { return HCA{Name: name, Ports: ports} } func active(rate int) Port { return Port{Num: 1, State: "ACTIVE", PhysState: "LinkUp", RateGbps: rate, LinkLayer: LinkInfiniBand} } func TestADeclaredAdapterThatIsNotThereIsReported(t *testing.T) { // A typo, or a card that enumerated differently after a reboot. Taken on // trust it becomes NCCL_IB_HCA naming an adapter that does not exist, and // the replica hangs on its first all-reduce. got := CheckFabrics( map[string][]string{"ib-a": {"mlx5_0", "mlx5_9"}}, []HCA{hca("mlx5_0", active(200))}, ) if len(got) != 1 { t.Fatalf("got %d checks", len(got)) } c := got[0] if len(c.Usable) != 1 || c.Usable[0] != "mlx5_0" { t.Errorf("usable = %v, want only the adapter that exists", c.Usable) } if len(c.Problems) != 1 || !strings.Contains(c.Problems[0], "mlx5_9") { t.Errorf("problems = %v", c.Problems) } // The message names what is actually present, because the operator is // looking at a typo and cannot see the node. if !strings.Contains(c.Problems[0], "mlx5_0") { t.Errorf("the problem does not say what was found: %q", c.Problems[0]) } } func TestAnAdapterWithNoActivePortCannotCarryTraffic(t *testing.T) { // A port that is physically linked but has no subnet manager sits in INIT. // It looks healthy to anything counting adapters and passes no data, which // is the case worth catching: it survives a reboot and looks like software. got := CheckFabrics( map[string][]string{"ib-a": {"mlx5_0"}}, []HCA{hca("mlx5_0", Port{Num: 1, State: "INIT", PhysState: "LinkUp", RateGbps: 200})}, ) c := got[0] if c.OK() { t.Error("a fabric with no active port was reported usable") } if len(c.Problems) != 1 || !strings.Contains(c.Problems[0], "INIT") { t.Errorf("the problem does not say what state the port is in: %v", c.Problems) } } func TestTheRateIsTheSlowestActiveLink(t *testing.T) { // A collective runs at the speed of its slowest link, so reporting the // fastest would have the scheduler believe a fabric is quicker than it is. got := CheckFabrics( map[string][]string{"ib-a": {"fast", "slow"}}, []HCA{hca("fast", active(400)), hca("slow", active(100))}, ) if got[0].RateGbps != 100 { t.Errorf("rate = %d, want 100", got[0].RateGbps) } } func TestAnAdapterWithSeveralPortsUsesItsBestActiveOne(t *testing.T) { // A dual-port card with one cable in is a perfectly good adapter, and // judging it by the dead port would take a working fabric out of service. got := CheckFabrics( map[string][]string{"ib-a": {"mlx5_0"}}, []HCA{hca("mlx5_0", Port{Num: 1, State: "DOWN", PhysState: "Disabled"}, active(200), )}, ) if !got[0].OK() { t.Fatalf("a card with one live port was rejected: %+v", got[0]) } if got[0].RateGbps != 200 { t.Errorf("rate = %d, want the active port's 200", got[0].RateGbps) } } func TestAFabricWithNothingUsableIsNotOK(t *testing.T) { got := CheckFabrics( map[string][]string{"ib-dead": {"mlx5_0"}}, []HCA{hca("mlx5_0", Port{Num: 1, State: "DOWN", PhysState: "Polling"})}, ) if got[0].OK() { t.Error("a fabric whose only adapter is down was reported usable") } } func TestANodeWithNoFabricsDeclaredProducesNoChecks(t *testing.T) { // Every node without InfiniBand, which must stay entirely unaffected. if got := CheckFabrics(nil, []HCA{hca("mlx5_0", active(200))}); len(got) != 0 { t.Errorf("got %d checks for a node that declared none", len(got)) } } func TestChecksComeBackInAStableOrder(t *testing.T) { // Map iteration would otherwise reorder log lines between restarts, which // makes a diff of two registrations unreadable. declared := map[string][]string{"ib-c": {"m"}, "ib-a": {"m"}, "ib-b": {"m"}} got := CheckFabrics(declared, nil) if len(got) != 3 { t.Fatalf("got %d", len(got)) } if got[0].Fabric != "ib-a" || got[1].Fabric != "ib-b" || got[2].Fabric != "ib-c" { t.Errorf("order = %s, %s, %s", got[0].Fabric, got[1].Fabric, got[2].Fabric) } }