llmgw/internal/agent/discovery/fabrics_test.go

package discovery

import (
	"strings"
	"testing"
)

func hca(name string, ports ...Port) HCA { return HCA{Name: name, Ports: ports} }

func active(rate int) Port {
	return Port{Num: 1, State: "ACTIVE", PhysState: "LinkUp", RateGbps: rate, LinkLayer: LinkInfiniBand}
}

func TestADeclaredAdapterThatIsNotThereIsReported(t *testing.T) {
	// A typo, or a card that enumerated differently after a reboot. Taken on
	// trust it becomes NCCL_IB_HCA naming an adapter that does not exist, and
	// the replica hangs on its first all-reduce.
	got := CheckFabrics(
		map[string][]string{"ib-a": {"mlx5_0", "mlx5_9"}},
		[]HCA{hca("mlx5_0", active(200))},
	)
	if len(got) != 1 {
		t.Fatalf("got %d checks", len(got))
	}
	c := got[0]
	if len(c.Usable) != 1 || c.Usable[0] != "mlx5_0" {
		t.Errorf("usable = %v, want only the adapter that exists", c.Usable)
	}
	if len(c.Problems) != 1 || !strings.Contains(c.Problems[0], "mlx5_9") {
		t.Errorf("problems = %v", c.Problems)
	}
	// The message names what is actually present, because the operator is
	// looking at a typo and cannot see the node.
	if !strings.Contains(c.Problems[0], "mlx5_0") {
		t.Errorf("the problem does not say what was found: %q", c.Problems[0])
	}
}

func TestAnAdapterWithNoActivePortCannotCarryTraffic(t *testing.T) {
	// A port that is physically linked but has no subnet manager sits in INIT.
	// It looks healthy to anything counting adapters and passes no data, which
	// is the case worth catching: it survives a reboot and looks like software.
	got := CheckFabrics(
		map[string][]string{"ib-a": {"mlx5_0"}},
		[]HCA{hca("mlx5_0", Port{Num: 1, State: "INIT", PhysState: "LinkUp", RateGbps: 200})},
	)
	c := got[0]
	if c.OK() {
		t.Error("a fabric with no active port was reported usable")
	}
	if len(c.Problems) != 1 || !strings.Contains(c.Problems[0], "INIT") {
		t.Errorf("the problem does not say what state the port is in: %v", c.Problems)
	}
}

func TestTheRateIsTheSlowestActiveLink(t *testing.T) {
	// A collective runs at the speed of its slowest link, so reporting the
	// fastest would have the scheduler believe a fabric is quicker than it is.
	got := CheckFabrics(
		map[string][]string{"ib-a": {"fast", "slow"}},
		[]HCA{hca("fast", active(400)), hca("slow", active(100))},
	)
	if got[0].RateGbps != 100 {
		t.Errorf("rate = %d, want 100", got[0].RateGbps)
	}
}

func TestAnAdapterWithSeveralPortsUsesItsBestActiveOne(t *testing.T) {
	// A dual-port card with one cable in is a perfectly good adapter, and
	// judging it by the dead port would take a working fabric out of service.
	got := CheckFabrics(
		map[string][]string{"ib-a": {"mlx5_0"}},
		[]HCA{hca("mlx5_0",
			Port{Num: 1, State: "DOWN", PhysState: "Disabled"},
			active(200),
		)},
	)
	if !got[0].OK() {
		t.Fatalf("a card with one live port was rejected: %+v", got[0])
	}
	if got[0].RateGbps != 200 {
		t.Errorf("rate = %d, want the active port's 200", got[0].RateGbps)
	}
}

func TestAFabricWithNothingUsableIsNotOK(t *testing.T) {
	got := CheckFabrics(
		map[string][]string{"ib-dead": {"mlx5_0"}},
		[]HCA{hca("mlx5_0", Port{Num: 1, State: "DOWN", PhysState: "Polling"})},
	)
	if got[0].OK() {
		t.Error("a fabric whose only adapter is down was reported usable")
	}
}

func TestANodeWithNoFabricsDeclaredProducesNoChecks(t *testing.T) {
	// Every node without InfiniBand, which must stay entirely unaffected.
	if got := CheckFabrics(nil, []HCA{hca("mlx5_0", active(200))}); len(got) != 0 {
		t.Errorf("got %d checks for a node that declared none", len(got))
	}
}

func TestChecksComeBackInAStableOrder(t *testing.T) {
	// Map iteration would otherwise reorder log lines between restarts, which
	// makes a diff of two registrations unreadable.
	declared := map[string][]string{"ib-c": {"m"}, "ib-a": {"m"}, "ib-b": {"m"}}
	got := CheckFabrics(declared, nil)
	if len(got) != 3 {
		t.Fatalf("got %d", len(got))
	}
	if got[0].Fabric != "ib-a" || got[1].Fabric != "ib-b" || got[2].Fabric != "ib-c" {
		t.Errorf("order = %s, %s, %s", got[0].Fabric, got[1].Fabric, got[2].Fabric)
	}
}