Ilker ManapandClaude Opus 5.5
20479a2d5f
A GB200 NVL72 rack or an NVLink Switch system is a scale-up island: its GPUs reach each other over NVLink whatever node they are in. Each GPU now reports its fabric from nvidia-smi -q - state, status, ClusterUUID, CliqueId, bandwidth - at registration and once a minute after, since a fabric manager restarting drops every card out at once; and each node reports nvidia-imex and its channels, without which the domain's GPUs cannot share memory across nodes. A single machine's NVSwitch, which reports a cluster of zeros, is no domain. A replica spanning nodes is placed inside one domain first, on nodes whose every card is joined and whose IMEX is up, and its shards are told NCCL_MNNVL_ENABLE=1; InfiniBand is the fallback. /admin/nvlink draws each domain as a rack - the NVLink Switch band on top, light running down a backplane, trays of GPU chips that glow when joined, pulse red when not, turn amber on degraded bandwidth - and lists what keeps a tray out. A node's page shows its GPU-by-GPU NVLink matrix, each card's NUMA node and nearest adapter, and its domain and IMEX. doctor checks joining, bandwidth, IMEX and its channel; metrics llmgw_gpu_nvlink_joined and llmgw_imex_up; /api/v1/nvlink. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
56 lines
1.9 KiB
Go
56 lines
1.9 KiB
Go
package nvlink
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
)
|
|
|
|
func tray(id string, gpus int, state string) inventory.Node {
|
|
n := inventory.Node{ID: id, IMEX: discovery.IMEX{State: "active", Channels: 1}}
|
|
for i := range gpus {
|
|
n.GPUs = append(n.GPUs, discovery.GPU{Index: i, Name: "NVIDIA GB200", Fabric: &discovery.GPUFabric{
|
|
State: state, Status: "Success", ClusterUUID: "76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa", CliqueID: "1", Bandwidth: "Full"}})
|
|
}
|
|
return n
|
|
}
|
|
|
|
func has(t *testing.T, r Report, l Level, s string) {
|
|
t.Helper()
|
|
for _, p := range r.Problems {
|
|
if p.Level == l && strings.Contains(p.Message, s) {
|
|
return
|
|
}
|
|
}
|
|
t.Errorf("no %s problem with %q in %+v", l, s, r.Problems)
|
|
}
|
|
|
|
func TestARackOfJoinedTraysIsOneUsableDomain(t *testing.T) {
|
|
r := Analyze([]inventory.Node{tray("t1", 4, "Completed"), tray("t2", 4, "Completed"), {ID: "cpu"}})
|
|
if len(r.Domains) != 1 || r.Domains[0].Usable != 2 || r.Domains[0].Joined != 8 || len(r.Problems) != 0 {
|
|
t.Errorf("report = %+v", r)
|
|
}
|
|
if Short(r.Domains[0].Key) != "76b1ad76·1" {
|
|
t.Errorf("short = %q", Short(r.Domains[0].Key))
|
|
}
|
|
}
|
|
|
|
func TestWhatKeepsATrayOutIsSaid(t *testing.T) {
|
|
half := tray("t2", 4, "Completed")
|
|
half.GPUs[3].Fabric = &discovery.GPUFabric{State: "In Progress"}
|
|
noIMEX := tray("t3", 4, "Completed")
|
|
noIMEX.IMEX = discovery.IMEX{State: "inactive"}
|
|
slow := tray("t4", 4, "Completed")
|
|
slow.GPUs[0].Fabric.Bandwidth = "Degraded"
|
|
r := Analyze([]inventory.Node{tray("t1", 4, "Completed"), half, noIMEX, slow})
|
|
has(t, r, Fail, "t2 GPU 3 is not joined to its domain (In Progress)")
|
|
has(t, r, Fail, "t2 is half joined")
|
|
has(t, r, Fail, "t3: nvidia-imex is inactive")
|
|
has(t, r, Warn, "t4 GPU 0 has degraded NVLink bandwidth")
|
|
if r.Domains[0].Usable != 2 {
|
|
t.Errorf("usable = %d, want t1 and t4", r.Domains[0].Usable)
|
|
}
|
|
}
|