Files
llmgw/internal/agent/discovery/ib_test.go
T
Ilker ManapandClaude Opus 5.5 9ee21dc814 Fix what a multi-node launch would have got wrong on real hardware
vLLM's mp launch gave workers no rank and the leader's address as their
own VLLM_HOST_IP; every node now runs serve with --nnodes, --node-rank,
--master-addr and --master-port, and the workers --headless, as vLLM
documents it.

NCCL_SOCKET_IFNAME and GLOO_SOCKET_IFNAME were the control plane's one
interface name for every node; the agent already worked out its own -
the one holding its management address - and now uses it.

On RoCE, NCCL's default GID index is a v1 link-local entry on most hosts,
which does not route. The agent finds the RoCE v2 entry holding an IPv4
address and passes it as NCCL_IB_GID_INDEX; InfiniBand gets none.

A worker whose only process needs the leader - a Ray join - started
without waiting for it, since the wait was honoured for sidecars alone;
it found no head, exited and failed the replica. It now waits.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-29 00:17:31 +02:00

263 lines
7.6 KiB
Go

package discovery
import (
"os"
"path/filepath"
"testing"
)
// fakeSysfs builds an InfiniBand tree shaped like the real one.
func fakeSysfs(t *testing.T) string {
t.Helper()
root := t.TempDir()
write := func(path, content string) {
t.Helper()
full := filepath.Join(root, path)
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(full, []byte(content+"\n"), 0o644); err != nil {
t.Fatal(err)
}
}
// mlx5_0: one active IB port.
write("ib/mlx5_0/fw_ver", "20.35.1012")
write("ib/mlx5_0/ports/1/state", "4: ACTIVE")
write("ib/mlx5_0/ports/1/phys_state", "5: LinkUp")
write("ib/mlx5_0/ports/1/rate", "200 Gb/sec (4X HDR)")
write("ib/mlx5_0/ports/1/link_layer", "InfiniBand")
write("ib/mlx5_0/ports/1/gids/0", "FEC0:0000:0000:000A:0002:C903:00A0:1234")
write("ib/mlx5_0/ports/1/lid", "0x1a")
write("ib/mlx5_0/ports/1/sm_lid", "1")
write("ib/mlx5_0/ports/1/counters/symbol_error", "3")
write("ib/mlx5_0/ports/1/counters/link_downed", "1")
write("ib/mlx5_0/ports/1/counters/port_xmit_data", "1000")
// mlx5_1: two ports, the second one down.
write("ib/mlx5_1/ports/1/state", "4: ACTIVE")
write("ib/mlx5_1/ports/1/rate", "200 Gb/sec (4X HDR)")
write("ib/mlx5_1/ports/1/link_layer", "InfiniBand")
write("ib/mlx5_1/ports/1/gids/0", "fec0:0000:0000:000a:0002:c903:00a0:5678")
write("ib/mlx5_1/ports/2/state", "1: DOWN")
write("ib/mlx5_1/ports/2/phys_state", "3: Disabled")
write("ib/mlx5_1/ports/2/rate", "0")
write("ib/mlx5_1/ports/2/link_layer", "InfiniBand")
// mlx5_2: RoCE rather than native IB.
write("ib/mlx5_2/ports/1/state", "4: ACTIVE")
write("ib/mlx5_2/ports/1/rate", "100 Gb/sec (4X EDR)")
write("ib/mlx5_2/ports/1/link_layer", "Ethernet")
write("ib/mlx5_2/ports/1/gids/0", "fe80:0000:0000:0000:0000:0000:0000:0001")
write("ib/mlx5_2/ports/1/gid_attrs/types/0", "IB/RoCE v1")
write("ib/mlx5_2/ports/1/gids/1", "fe80:0000:0000:0000:0000:0000:0000:0001")
write("ib/mlx5_2/ports/1/gid_attrs/types/1", "RoCE v2")
write("ib/mlx5_2/ports/1/gids/3", "0000:0000:0000:0000:0000:ffff:0a00:0001")
write("ib/mlx5_2/ports/1/gid_attrs/types/3", "RoCE v2")
// A PCI device with a NUMA node, linked the way the driver does it.
pci := filepath.Join(root, "pci", "0000:0b:00.0")
if err := os.MkdirAll(pci, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(pci, "numa_node"), []byte("1\n"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.Symlink(pci, filepath.Join(root, "ib", "mlx5_0", "device")); err != nil {
t.Fatal(err)
}
return filepath.Join(root, "ib")
}
func TestDiscoverIB(t *testing.T) {
hcas, err := DiscoverIB(fakeSysfs(t))
if err != nil {
t.Fatal(err)
}
if len(hcas) != 3 {
t.Fatalf("expected 3 HCAs, got %d", len(hcas))
}
// Sorted by name, so the order is predictable for the rest of the test.
if hcas[0].Name != "mlx5_0" || hcas[2].Name != "mlx5_2" {
t.Fatalf("HCAs are not sorted by name: %v %v", hcas[0].Name, hcas[2].Name)
}
h := hcas[0]
if h.FirmwareVersion != "20.35.1012" {
t.Errorf("fw_ver=%q", h.FirmwareVersion)
}
if h.PCIAddr != "0000:0b:00.0" {
t.Errorf("PCIAddr=%q", h.PCIAddr)
}
if h.NUMANode != 1 {
t.Errorf("NUMANode=%d; GPU-to-HCA affinity depends on this", h.NUMANode)
}
p := h.Ports[0]
if p.State != "ACTIVE" || !p.Active() {
t.Errorf("state=%q", p.State)
}
if p.PhysState != "LinkUp" {
t.Errorf("phys_state=%q", p.PhysState)
}
if p.RateGbps != 200 {
t.Errorf("rate=%d", p.RateGbps)
}
if p.LinkLayer != LinkInfiniBand {
t.Errorf("link_layer=%q", p.LinkLayer)
}
if p.GIDPrefix != "fec0:0000:0000:000a" {
t.Errorf("GIDPrefix=%q; it must be lowercased and cut to four groups", p.GIDPrefix)
}
if p.LID != 26 {
t.Errorf("LID=%d, expected 0x1a to parse as 26", p.LID)
}
if p.SMLID != 1 {
t.Errorf("SMLID=%d, expected a decimal value to parse too", p.SMLID)
}
}
func TestActivePortsExcludesDown(t *testing.T) {
hcas, err := DiscoverIB(fakeSysfs(t))
if err != nil {
t.Fatal(err)
}
h := hcas[1] // mlx5_1
if len(h.Ports) != 2 {
t.Fatalf("expected 2 ports, got %d", len(h.Ports))
}
active := h.ActivePorts()
if len(active) != 1 || active[0].Num != 1 {
t.Errorf("a DOWN port was reported as active: %+v", active)
}
}
func TestRoCEIsDistinguished(t *testing.T) {
// RoCE is usable but behaves differently from native IB; the placement
// logic needs to be able to tell them apart.
hcas, err := DiscoverIB(fakeSysfs(t))
if err != nil {
t.Fatal(err)
}
if got := hcas[2].Ports[0].LinkLayer; got != LinkEthernet {
t.Errorf("mlx5_2 link_layer=%q, expected Ethernet (RoCE)", got)
}
}
func TestMissingRootIsNotAnError(t *testing.T) {
// A node without InfiniBand is perfectly normal; it just forms its own
// single-node island.
hcas, err := DiscoverIB(filepath.Join(t.TempDir(), "nonexistent"))
if err != nil {
t.Fatalf("a node without IB should not be an error: %v", err)
}
if len(hcas) != 0 {
t.Errorf("expected no HCAs, got %d", len(hcas))
}
}
func TestUnreadableAdapterDoesNotHideOthers(t *testing.T) {
root := fakeSysfs(t)
// An adapter with no ports directory at all.
if err := os.MkdirAll(filepath.Join(root, "mlx5_broken"), 0o755); err != nil {
t.Fatal(err)
}
hcas, err := DiscoverIB(root)
if err != nil {
t.Fatal(err)
}
if len(hcas) != 3 {
t.Errorf("a broken adapter changed the result: got %d HCAs", len(hcas))
}
}
func TestParseHelpers(t *testing.T) {
t.Run("rate", func(t *testing.T) {
for in, want := range map[string]int{
"200 Gb/sec (4X HDR)": 200,
"100 Gb/sec (4X EDR)": 100,
"2.5 Gb/sec (1X SDR)": 2,
"": 0,
"garbage": 0,
} {
if got := parseRate(in); got != want {
t.Errorf("parseRate(%q)=%d, want %d", in, got, want)
}
}
})
t.Run("state", func(t *testing.T) {
for in, want := range map[string]string{
"4: ACTIVE": "ACTIVE",
"1: DOWN": "DOWN",
"5: LinkUp": "LinkUp",
"ACTIVE": "ACTIVE",
"": "",
} {
if got := parseEnumeratedState(in); got != want {
t.Errorf("parseEnumeratedState(%q)=%q, want %q", in, got, want)
}
}
})
t.Run("hex", func(t *testing.T) {
for in, want := range map[string]int{
"0x1a": 26, "0X1A": 26, "26": 26, "": 0, "zzz": 0,
} {
if got := parseMaybeHex(in); got != want {
t.Errorf("parseMaybeHex(%q)=%d, want %d", in, got, want)
}
}
})
t.Run("gid", func(t *testing.T) {
if got := parseGIDPrefix("FE80:0000:0000:0000:0002:C903:00A0:1234"); got != "fe80:0000:0000:0000" {
t.Errorf("parseGIDPrefix=%q", got)
}
if got := parseGIDPrefix("short:gid"); got != "" {
t.Errorf("a malformed GID should yield an empty prefix, got %q", got)
}
})
}
func TestAPortsCountersAreRead(t *testing.T) {
hcas, err := DiscoverIB(fakeSysfs(t))
if err != nil {
t.Fatal(err)
}
c := hcas[0].Ports[0].Counters
// Data in bytes: sysfs counts four-byte words.
if c["symbol_error"] != 3 || c["link_downed"] != 1 || c["port_xmit_data"] != 4000 {
t.Errorf("counters = %v", c)
}
if _, ok := c["port_rcv_errors"]; ok {
t.Error("a counter the driver does not have should be absent, not zero")
}
}
func TestRoCEFindsItsRoutableGID(t *testing.T) {
// NCCL's default GID index is a v1 link-local entry on most hosts, which
// does not route: the job hangs. The v2 entry holding an IPv4 address is
// the one to use, over an earlier v2 link-local one.
hcas, err := DiscoverIB(fakeSysfs(t))
if err != nil {
t.Fatal(err)
}
for _, h := range hcas {
p := h.Ports[0]
switch h.Name {
case "mlx5_2":
if p.RoCEv2GID != 3 {
t.Errorf("RoCE GID index = %d, want 3", p.RoCEv2GID)
}
default:
if p.RoCEv2GID != -1 {
t.Errorf("%s is InfiniBand and has a GID index %d", h.Name, p.RoCEv2GID)
}
}
}
}