Everything here works without GPUs and is covered by tests. The control plane connection is not built yet, so llmgw agent itself is still a stub and the phase exit criterion is not met. - discovery: IB from /sys/class/infiniband, parsing the formats sysfs actually uses and telling native InfiniBand from RoCE; GPUs via nvidia-smi, the cgo-free path, with health flags for uncorrected ECC and memory already held by another process. A node without IB is not an error, it is a single-node island. - engines: vLLM and SGLang command and environment construction, kept free of process handling so it stays testable. The NCCL environment is the part the plan marks critical: NCCL_IB_HCA lists only the chosen fabric's adapters, because leaving it open lets NCCL pick an adapter on a fabric where the peer is unreachable, and the job then hangs instead of failing. - validation before launch: no mixed GPU types in a tensor-parallel group, the node/GPU/TP arithmetic has to add up, and fp8 is refused below compute capability 8.9 rather than silently falling back to a slower path. - supervisor: process groups, because engines fork workers that hold GPU memory and outlive a signal sent only to the leader; a bounded log tail, since engine output is the only diagnosis when loading fails; Starting separate from Loading, because a 70B model legitimately takes minutes; and orphan adoption through pid files fingerprinted with the command line, since pids get reused and adopting an unrelated process is worse than leaking the original. - readiness is two questions: /health says the process is up, /v1/models says the weights loaded. Routing on /health alone sends traffic into a loading model. - llmgw agent inventory reports what the scheduler would see on a node. One plan correction: GPU class sizes come from the driver in GiB, which does not always match the marketing name. An L40S is sold as 48GB and reports 44.99 GiB, so its class is l40s-45g, not l40s-48g. Consistency matters more than matching the box, because that number decides whether a model fits. Plan §2 and §5.1 updated. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
217 lines
6.0 KiB
Go
217 lines
6.0 KiB
Go
package discovery
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
)
|
|
|
|
// fakeSysfs builds an InfiniBand tree shaped like the real one.
|
|
func fakeSysfs(t *testing.T) string {
|
|
t.Helper()
|
|
root := t.TempDir()
|
|
|
|
write := func(path, content string) {
|
|
t.Helper()
|
|
full := filepath.Join(root, path)
|
|
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(full, []byte(content+"\n"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// mlx5_0: one active IB port.
|
|
write("ib/mlx5_0/fw_ver", "20.35.1012")
|
|
write("ib/mlx5_0/ports/1/state", "4: ACTIVE")
|
|
write("ib/mlx5_0/ports/1/phys_state", "5: LinkUp")
|
|
write("ib/mlx5_0/ports/1/rate", "200 Gb/sec (4X HDR)")
|
|
write("ib/mlx5_0/ports/1/link_layer", "InfiniBand")
|
|
write("ib/mlx5_0/ports/1/gids/0", "FEC0:0000:0000:000A:0002:C903:00A0:1234")
|
|
write("ib/mlx5_0/ports/1/lid", "0x1a")
|
|
write("ib/mlx5_0/ports/1/sm_lid", "1")
|
|
|
|
// mlx5_1: two ports, the second one down.
|
|
write("ib/mlx5_1/ports/1/state", "4: ACTIVE")
|
|
write("ib/mlx5_1/ports/1/rate", "200 Gb/sec (4X HDR)")
|
|
write("ib/mlx5_1/ports/1/link_layer", "InfiniBand")
|
|
write("ib/mlx5_1/ports/1/gids/0", "fec0:0000:0000:000a:0002:c903:00a0:5678")
|
|
write("ib/mlx5_1/ports/2/state", "1: DOWN")
|
|
write("ib/mlx5_1/ports/2/phys_state", "3: Disabled")
|
|
write("ib/mlx5_1/ports/2/rate", "0")
|
|
write("ib/mlx5_1/ports/2/link_layer", "InfiniBand")
|
|
|
|
// mlx5_2: RoCE rather than native IB.
|
|
write("ib/mlx5_2/ports/1/state", "4: ACTIVE")
|
|
write("ib/mlx5_2/ports/1/rate", "100 Gb/sec (4X EDR)")
|
|
write("ib/mlx5_2/ports/1/link_layer", "Ethernet")
|
|
write("ib/mlx5_2/ports/1/gids/0", "0000:0000:0000:0000:0000:ffff:0a00:0001")
|
|
|
|
// A PCI device with a NUMA node, linked the way the driver does it.
|
|
pci := filepath.Join(root, "pci", "0000:0b:00.0")
|
|
if err := os.MkdirAll(pci, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(pci, "numa_node"), []byte("1\n"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.Symlink(pci, filepath.Join(root, "ib", "mlx5_0", "device")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
return filepath.Join(root, "ib")
|
|
}
|
|
|
|
func TestDiscoverIB(t *testing.T) {
|
|
hcas, err := DiscoverIB(fakeSysfs(t))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(hcas) != 3 {
|
|
t.Fatalf("expected 3 HCAs, got %d", len(hcas))
|
|
}
|
|
// Sorted by name, so the order is predictable for the rest of the test.
|
|
if hcas[0].Name != "mlx5_0" || hcas[2].Name != "mlx5_2" {
|
|
t.Fatalf("HCAs are not sorted by name: %v %v", hcas[0].Name, hcas[2].Name)
|
|
}
|
|
|
|
h := hcas[0]
|
|
if h.FirmwareVersion != "20.35.1012" {
|
|
t.Errorf("fw_ver=%q", h.FirmwareVersion)
|
|
}
|
|
if h.PCIAddr != "0000:0b:00.0" {
|
|
t.Errorf("PCIAddr=%q", h.PCIAddr)
|
|
}
|
|
if h.NUMANode != 1 {
|
|
t.Errorf("NUMANode=%d; GPU-to-HCA affinity depends on this", h.NUMANode)
|
|
}
|
|
|
|
p := h.Ports[0]
|
|
if p.State != "ACTIVE" || !p.Active() {
|
|
t.Errorf("state=%q", p.State)
|
|
}
|
|
if p.PhysState != "LinkUp" {
|
|
t.Errorf("phys_state=%q", p.PhysState)
|
|
}
|
|
if p.RateGbps != 200 {
|
|
t.Errorf("rate=%d", p.RateGbps)
|
|
}
|
|
if p.LinkLayer != LinkInfiniBand {
|
|
t.Errorf("link_layer=%q", p.LinkLayer)
|
|
}
|
|
if p.GIDPrefix != "fec0:0000:0000:000a" {
|
|
t.Errorf("GIDPrefix=%q; it must be lowercased and cut to four groups", p.GIDPrefix)
|
|
}
|
|
if p.LID != 26 {
|
|
t.Errorf("LID=%d, expected 0x1a to parse as 26", p.LID)
|
|
}
|
|
if p.SMLID != 1 {
|
|
t.Errorf("SMLID=%d, expected a decimal value to parse too", p.SMLID)
|
|
}
|
|
}
|
|
|
|
func TestActivePortsExcludesDown(t *testing.T) {
|
|
hcas, err := DiscoverIB(fakeSysfs(t))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h := hcas[1] // mlx5_1
|
|
if len(h.Ports) != 2 {
|
|
t.Fatalf("expected 2 ports, got %d", len(h.Ports))
|
|
}
|
|
active := h.ActivePorts()
|
|
if len(active) != 1 || active[0].Num != 1 {
|
|
t.Errorf("a DOWN port was reported as active: %+v", active)
|
|
}
|
|
}
|
|
|
|
func TestRoCEIsDistinguished(t *testing.T) {
|
|
// RoCE is usable but behaves differently from native IB; the placement
|
|
// logic needs to be able to tell them apart.
|
|
hcas, err := DiscoverIB(fakeSysfs(t))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := hcas[2].Ports[0].LinkLayer; got != LinkEthernet {
|
|
t.Errorf("mlx5_2 link_layer=%q, expected Ethernet (RoCE)", got)
|
|
}
|
|
}
|
|
|
|
func TestMissingRootIsNotAnError(t *testing.T) {
|
|
// A node without InfiniBand is perfectly normal; it just forms its own
|
|
// single-node island.
|
|
hcas, err := DiscoverIB(filepath.Join(t.TempDir(), "nonexistent"))
|
|
if err != nil {
|
|
t.Fatalf("a node without IB should not be an error: %v", err)
|
|
}
|
|
if len(hcas) != 0 {
|
|
t.Errorf("expected no HCAs, got %d", len(hcas))
|
|
}
|
|
}
|
|
|
|
func TestUnreadableAdapterDoesNotHideOthers(t *testing.T) {
|
|
root := fakeSysfs(t)
|
|
// An adapter with no ports directory at all.
|
|
if err := os.MkdirAll(filepath.Join(root, "mlx5_broken"), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
hcas, err := DiscoverIB(root)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(hcas) != 3 {
|
|
t.Errorf("a broken adapter changed the result: got %d HCAs", len(hcas))
|
|
}
|
|
}
|
|
|
|
func TestParseHelpers(t *testing.T) {
|
|
t.Run("rate", func(t *testing.T) {
|
|
for in, want := range map[string]int{
|
|
"200 Gb/sec (4X HDR)": 200,
|
|
"100 Gb/sec (4X EDR)": 100,
|
|
"2.5 Gb/sec (1X SDR)": 2,
|
|
"": 0,
|
|
"garbage": 0,
|
|
} {
|
|
if got := parseRate(in); got != want {
|
|
t.Errorf("parseRate(%q)=%d, want %d", in, got, want)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("state", func(t *testing.T) {
|
|
for in, want := range map[string]string{
|
|
"4: ACTIVE": "ACTIVE",
|
|
"1: DOWN": "DOWN",
|
|
"5: LinkUp": "LinkUp",
|
|
"ACTIVE": "ACTIVE",
|
|
"": "",
|
|
} {
|
|
if got := parseEnumeratedState(in); got != want {
|
|
t.Errorf("parseEnumeratedState(%q)=%q, want %q", in, got, want)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("hex", func(t *testing.T) {
|
|
for in, want := range map[string]int{
|
|
"0x1a": 26, "0X1A": 26, "26": 26, "": 0, "zzz": 0,
|
|
} {
|
|
if got := parseMaybeHex(in); got != want {
|
|
t.Errorf("parseMaybeHex(%q)=%d, want %d", in, got, want)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("gid", func(t *testing.T) {
|
|
if got := parseGIDPrefix("FE80:0000:0000:0000:0002:C903:00A0:1234"); got != "fe80:0000:0000:0000" {
|
|
t.Errorf("parseGIDPrefix=%q", got)
|
|
}
|
|
if got := parseGIDPrefix("short:gid"); got != "" {
|
|
t.Errorf("a malformed GID should yield an empty prefix, got %q", got)
|
|
}
|
|
})
|
|
}
|