vLLM's mp launch gave workers no rank and the leader's address as their own VLLM_HOST_IP; every node now runs serve with --nnodes, --node-rank, --master-addr and --master-port, and the workers --headless, as vLLM documents it. NCCL_SOCKET_IFNAME and GLOO_SOCKET_IFNAME were the control plane's one interface name for every node; the agent already worked out its own - the one holding its management address - and now uses it. On RoCE, NCCL's default GID index is a v1 link-local entry on most hosts, which does not route. The agent finds the RoCE v2 entry holding an IPv4 address and passes it as NCCL_IB_GID_INDEX; InfiniBand gets none. A worker whose only process needs the leader - a Ray join - started without waiting for it, since the wait was honoured for sidecars alone; it found no head, exited and failed the replica. It now waits. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
263 lines
7.6 KiB
Go
263 lines
7.6 KiB
Go
package discovery
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
)
|
|
|
|
// fakeSysfs builds an InfiniBand tree shaped like the real one.
|
|
func fakeSysfs(t *testing.T) string {
|
|
t.Helper()
|
|
root := t.TempDir()
|
|
|
|
write := func(path, content string) {
|
|
t.Helper()
|
|
full := filepath.Join(root, path)
|
|
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(full, []byte(content+"\n"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// mlx5_0: one active IB port.
|
|
write("ib/mlx5_0/fw_ver", "20.35.1012")
|
|
write("ib/mlx5_0/ports/1/state", "4: ACTIVE")
|
|
write("ib/mlx5_0/ports/1/phys_state", "5: LinkUp")
|
|
write("ib/mlx5_0/ports/1/rate", "200 Gb/sec (4X HDR)")
|
|
write("ib/mlx5_0/ports/1/link_layer", "InfiniBand")
|
|
write("ib/mlx5_0/ports/1/gids/0", "FEC0:0000:0000:000A:0002:C903:00A0:1234")
|
|
write("ib/mlx5_0/ports/1/lid", "0x1a")
|
|
write("ib/mlx5_0/ports/1/sm_lid", "1")
|
|
write("ib/mlx5_0/ports/1/counters/symbol_error", "3")
|
|
write("ib/mlx5_0/ports/1/counters/link_downed", "1")
|
|
write("ib/mlx5_0/ports/1/counters/port_xmit_data", "1000")
|
|
|
|
// mlx5_1: two ports, the second one down.
|
|
write("ib/mlx5_1/ports/1/state", "4: ACTIVE")
|
|
write("ib/mlx5_1/ports/1/rate", "200 Gb/sec (4X HDR)")
|
|
write("ib/mlx5_1/ports/1/link_layer", "InfiniBand")
|
|
write("ib/mlx5_1/ports/1/gids/0", "fec0:0000:0000:000a:0002:c903:00a0:5678")
|
|
write("ib/mlx5_1/ports/2/state", "1: DOWN")
|
|
write("ib/mlx5_1/ports/2/phys_state", "3: Disabled")
|
|
write("ib/mlx5_1/ports/2/rate", "0")
|
|
write("ib/mlx5_1/ports/2/link_layer", "InfiniBand")
|
|
|
|
// mlx5_2: RoCE rather than native IB.
|
|
write("ib/mlx5_2/ports/1/state", "4: ACTIVE")
|
|
write("ib/mlx5_2/ports/1/rate", "100 Gb/sec (4X EDR)")
|
|
write("ib/mlx5_2/ports/1/link_layer", "Ethernet")
|
|
write("ib/mlx5_2/ports/1/gids/0", "fe80:0000:0000:0000:0000:0000:0000:0001")
|
|
write("ib/mlx5_2/ports/1/gid_attrs/types/0", "IB/RoCE v1")
|
|
write("ib/mlx5_2/ports/1/gids/1", "fe80:0000:0000:0000:0000:0000:0000:0001")
|
|
write("ib/mlx5_2/ports/1/gid_attrs/types/1", "RoCE v2")
|
|
write("ib/mlx5_2/ports/1/gids/3", "0000:0000:0000:0000:0000:ffff:0a00:0001")
|
|
write("ib/mlx5_2/ports/1/gid_attrs/types/3", "RoCE v2")
|
|
|
|
// A PCI device with a NUMA node, linked the way the driver does it.
|
|
pci := filepath.Join(root, "pci", "0000:0b:00.0")
|
|
if err := os.MkdirAll(pci, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(pci, "numa_node"), []byte("1\n"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.Symlink(pci, filepath.Join(root, "ib", "mlx5_0", "device")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
return filepath.Join(root, "ib")
|
|
}
|
|
|
|
func TestDiscoverIB(t *testing.T) {
|
|
hcas, err := DiscoverIB(fakeSysfs(t))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(hcas) != 3 {
|
|
t.Fatalf("expected 3 HCAs, got %d", len(hcas))
|
|
}
|
|
// Sorted by name, so the order is predictable for the rest of the test.
|
|
if hcas[0].Name != "mlx5_0" || hcas[2].Name != "mlx5_2" {
|
|
t.Fatalf("HCAs are not sorted by name: %v %v", hcas[0].Name, hcas[2].Name)
|
|
}
|
|
|
|
h := hcas[0]
|
|
if h.FirmwareVersion != "20.35.1012" {
|
|
t.Errorf("fw_ver=%q", h.FirmwareVersion)
|
|
}
|
|
if h.PCIAddr != "0000:0b:00.0" {
|
|
t.Errorf("PCIAddr=%q", h.PCIAddr)
|
|
}
|
|
if h.NUMANode != 1 {
|
|
t.Errorf("NUMANode=%d; GPU-to-HCA affinity depends on this", h.NUMANode)
|
|
}
|
|
|
|
p := h.Ports[0]
|
|
if p.State != "ACTIVE" || !p.Active() {
|
|
t.Errorf("state=%q", p.State)
|
|
}
|
|
if p.PhysState != "LinkUp" {
|
|
t.Errorf("phys_state=%q", p.PhysState)
|
|
}
|
|
if p.RateGbps != 200 {
|
|
t.Errorf("rate=%d", p.RateGbps)
|
|
}
|
|
if p.LinkLayer != LinkInfiniBand {
|
|
t.Errorf("link_layer=%q", p.LinkLayer)
|
|
}
|
|
if p.GIDPrefix != "fec0:0000:0000:000a" {
|
|
t.Errorf("GIDPrefix=%q; it must be lowercased and cut to four groups", p.GIDPrefix)
|
|
}
|
|
if p.LID != 26 {
|
|
t.Errorf("LID=%d, expected 0x1a to parse as 26", p.LID)
|
|
}
|
|
if p.SMLID != 1 {
|
|
t.Errorf("SMLID=%d, expected a decimal value to parse too", p.SMLID)
|
|
}
|
|
}
|
|
|
|
func TestActivePortsExcludesDown(t *testing.T) {
|
|
hcas, err := DiscoverIB(fakeSysfs(t))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h := hcas[1] // mlx5_1
|
|
if len(h.Ports) != 2 {
|
|
t.Fatalf("expected 2 ports, got %d", len(h.Ports))
|
|
}
|
|
active := h.ActivePorts()
|
|
if len(active) != 1 || active[0].Num != 1 {
|
|
t.Errorf("a DOWN port was reported as active: %+v", active)
|
|
}
|
|
}
|
|
|
|
func TestRoCEIsDistinguished(t *testing.T) {
|
|
// RoCE is usable but behaves differently from native IB; the placement
|
|
// logic needs to be able to tell them apart.
|
|
hcas, err := DiscoverIB(fakeSysfs(t))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := hcas[2].Ports[0].LinkLayer; got != LinkEthernet {
|
|
t.Errorf("mlx5_2 link_layer=%q, expected Ethernet (RoCE)", got)
|
|
}
|
|
}
|
|
|
|
func TestMissingRootIsNotAnError(t *testing.T) {
|
|
// A node without InfiniBand is perfectly normal; it just forms its own
|
|
// single-node island.
|
|
hcas, err := DiscoverIB(filepath.Join(t.TempDir(), "nonexistent"))
|
|
if err != nil {
|
|
t.Fatalf("a node without IB should not be an error: %v", err)
|
|
}
|
|
if len(hcas) != 0 {
|
|
t.Errorf("expected no HCAs, got %d", len(hcas))
|
|
}
|
|
}
|
|
|
|
func TestUnreadableAdapterDoesNotHideOthers(t *testing.T) {
|
|
root := fakeSysfs(t)
|
|
// An adapter with no ports directory at all.
|
|
if err := os.MkdirAll(filepath.Join(root, "mlx5_broken"), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
hcas, err := DiscoverIB(root)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(hcas) != 3 {
|
|
t.Errorf("a broken adapter changed the result: got %d HCAs", len(hcas))
|
|
}
|
|
}
|
|
|
|
func TestParseHelpers(t *testing.T) {
|
|
t.Run("rate", func(t *testing.T) {
|
|
for in, want := range map[string]int{
|
|
"200 Gb/sec (4X HDR)": 200,
|
|
"100 Gb/sec (4X EDR)": 100,
|
|
"2.5 Gb/sec (1X SDR)": 2,
|
|
"": 0,
|
|
"garbage": 0,
|
|
} {
|
|
if got := parseRate(in); got != want {
|
|
t.Errorf("parseRate(%q)=%d, want %d", in, got, want)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("state", func(t *testing.T) {
|
|
for in, want := range map[string]string{
|
|
"4: ACTIVE": "ACTIVE",
|
|
"1: DOWN": "DOWN",
|
|
"5: LinkUp": "LinkUp",
|
|
"ACTIVE": "ACTIVE",
|
|
"": "",
|
|
} {
|
|
if got := parseEnumeratedState(in); got != want {
|
|
t.Errorf("parseEnumeratedState(%q)=%q, want %q", in, got, want)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("hex", func(t *testing.T) {
|
|
for in, want := range map[string]int{
|
|
"0x1a": 26, "0X1A": 26, "26": 26, "": 0, "zzz": 0,
|
|
} {
|
|
if got := parseMaybeHex(in); got != want {
|
|
t.Errorf("parseMaybeHex(%q)=%d, want %d", in, got, want)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("gid", func(t *testing.T) {
|
|
if got := parseGIDPrefix("FE80:0000:0000:0000:0002:C903:00A0:1234"); got != "fe80:0000:0000:0000" {
|
|
t.Errorf("parseGIDPrefix=%q", got)
|
|
}
|
|
if got := parseGIDPrefix("short:gid"); got != "" {
|
|
t.Errorf("a malformed GID should yield an empty prefix, got %q", got)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestAPortsCountersAreRead(t *testing.T) {
|
|
hcas, err := DiscoverIB(fakeSysfs(t))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
c := hcas[0].Ports[0].Counters
|
|
// Data in bytes: sysfs counts four-byte words.
|
|
if c["symbol_error"] != 3 || c["link_downed"] != 1 || c["port_xmit_data"] != 4000 {
|
|
t.Errorf("counters = %v", c)
|
|
}
|
|
if _, ok := c["port_rcv_errors"]; ok {
|
|
t.Error("a counter the driver does not have should be absent, not zero")
|
|
}
|
|
}
|
|
|
|
func TestRoCEFindsItsRoutableGID(t *testing.T) {
|
|
// NCCL's default GID index is a v1 link-local entry on most hosts, which
|
|
// does not route: the job hangs. The v2 entry holding an IPv4 address is
|
|
// the one to use, over an earlier v2 link-local one.
|
|
hcas, err := DiscoverIB(fakeSysfs(t))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, h := range hcas {
|
|
p := h.Ports[0]
|
|
switch h.Name {
|
|
case "mlx5_2":
|
|
if p.RoCEv2GID != 3 {
|
|
t.Errorf("RoCE GID index = %d, want 3", p.RoCEv2GID)
|
|
}
|
|
default:
|
|
if p.RoCEv2GID != -1 {
|
|
t.Errorf("%s is InfiniBand and has a GID index %d", h.Name, p.RoCEv2GID)
|
|
}
|
|
}
|
|
}
|
|
}
|