Find multi-node NVLink domains, place across them, and show them as racks
A GB200 NVL72 rack or an NVLink Switch system is a scale-up island: its GPUs reach each other over NVLink whatever node they are in. Each GPU now reports its fabric from nvidia-smi -q - state, status, ClusterUUID, CliqueId, bandwidth - at registration and once a minute after, since a fabric manager restarting drops every card out at once; and each node reports nvidia-imex and its channels, without which the domain's GPUs cannot share memory across nodes. A single machine's NVSwitch, which reports a cluster of zeros, is no domain. A replica spanning nodes is placed inside one domain first, on nodes whose every card is joined and whose IMEX is up, and its shards are told NCCL_MNNVL_ENABLE=1; InfiniBand is the fallback. /admin/nvlink draws each domain as a rack - the NVLink Switch band on top, light running down a backplane, trays of GPU chips that glow when joined, pulse red when not, turn amber on degraded bandwidth - and lists what keeps a tray out. A node's page shows its GPU-by-GPU NVLink matrix, each card's NUMA node and nearest adapter, and its domain and IMEX. doctor checks joining, bandwidth, IMEX and its channel; metrics llmgw_gpu_nvlink_joined and llmgw_imex_up; /api/v1/nvlink. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
5bbc619a31
commit
20479a2d5f
31 files changed
+1535
-131
No files matched your search
@@ -0,0 +1,172 @@
|
||||
// Package nvlink works out the cluster's multi-node NVLink domains — an NVL72
|
||||
// rack, an NVLink Switch system — from what each GPU reports, and what keeps
|
||||
// one from being usable.
|
||||
//
|
||||
// A domain is a scale-up island: its GPUs reach each other over NVLink
|
||||
// whatever node they are in, so a tensor-parallel group can span its nodes
|
||||
// at NVLink speed. What it needs to be one is every card joined by the
|
||||
// fabric manager and IMEX running on every node; either missing, and a
|
||||
// replica placed across it fails or crawls.
|
||||
package nvlink
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
// Level is how much a problem matters.
|
||||
type Level string
|
||||
|
||||
// The levels.
|
||||
const (
|
||||
Fail Level = "fail"
|
||||
Warn Level = "warn"
|
||||
)
|
||||
|
||||
// Problem is one thing wrong with a domain.
|
||||
type Problem struct {
|
||||
Level Level `json:"level"`
|
||||
Domain string `json:"domain"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// GPU is one card as its domain sees it.
|
||||
type GPU struct {
|
||||
Index int `json:"index"`
|
||||
Name string `json:"name"`
|
||||
State string `json:"state"`
|
||||
Status string `json:"status,omitempty"`
|
||||
Bandwidth string `json:"bandwidth,omitempty"`
|
||||
Joined bool `json:"joined"`
|
||||
}
|
||||
|
||||
// Member is one node of a domain.
|
||||
type Member struct {
|
||||
Node string `json:"node"`
|
||||
GPUs []GPU `json:"gpus"`
|
||||
// Usable is every card joined and IMEX up: what placement requires.
|
||||
Usable bool `json:"usable"`
|
||||
IMEX string `json:"imex"`
|
||||
IMEXChannels int `json:"imex_channels"`
|
||||
}
|
||||
|
||||
// Domain is one multi-node NVLink domain.
|
||||
type Domain struct {
|
||||
Key string `json:"key"`
|
||||
ClusterUUID string `json:"cluster_uuid"`
|
||||
CliqueID string `json:"clique_id"`
|
||||
Members []Member `json:"members"`
|
||||
GPUs int `json:"gpus"`
|
||||
Joined int `json:"joined"`
|
||||
Usable int `json:"usable_nodes"`
|
||||
}
|
||||
|
||||
// Report is every domain and what is wrong with them.
|
||||
type Report struct {
|
||||
Domains []Domain `json:"domains"`
|
||||
Problems []Problem `json:"problems"`
|
||||
}
|
||||
|
||||
// Analyze reads every node's GPUs.
|
||||
func Analyze(nodes []inventory.Node) Report {
|
||||
var r Report
|
||||
by := map[string]*Domain{}
|
||||
for _, n := range nodes {
|
||||
// The domain a node is in is the one any of its cards joined; a card
|
||||
// still joining has no domain of its own yet, and is counted against
|
||||
// the node's.
|
||||
key := ""
|
||||
for _, g := range n.GPUs {
|
||||
if d := g.Fabric.Domain(); d != "" {
|
||||
key = d
|
||||
break
|
||||
}
|
||||
}
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
d := by[key]
|
||||
if d == nil {
|
||||
uuid, clique, _ := strings.Cut(key, "/")
|
||||
d = &Domain{Key: key, ClusterUUID: uuid, CliqueID: clique}
|
||||
by[key] = d
|
||||
}
|
||||
m := Member{Node: n.ID, IMEX: n.IMEX.State, IMEXChannels: n.IMEX.Channels}
|
||||
joined := 0
|
||||
for _, g := range n.GPUs {
|
||||
v := GPU{Index: g.Index, Name: g.Name}
|
||||
if f := g.Fabric; f != nil {
|
||||
v.State, v.Status, v.Bandwidth = f.State, f.Status, f.Bandwidth
|
||||
v.Joined = f.Domain() == key
|
||||
}
|
||||
if v.Joined {
|
||||
joined++
|
||||
} else {
|
||||
why := v.State
|
||||
if why == "" {
|
||||
why = "no fabric"
|
||||
}
|
||||
if v.Status != "" && !strings.EqualFold(v.Status, "Success") {
|
||||
why += ": " + v.Status
|
||||
}
|
||||
r.Problems = append(r.Problems, Problem{Level: Fail, Domain: key, Message: fmt.Sprintf(
|
||||
"%s GPU %d is not joined to its domain (%s)", n.ID, g.Index, why)})
|
||||
}
|
||||
if v.Bandwidth != "" && !strings.EqualFold(v.Bandwidth, "Full") {
|
||||
r.Problems = append(r.Problems, Problem{Level: Warn, Domain: key, Message: fmt.Sprintf(
|
||||
"%s GPU %d has %s NVLink bandwidth", n.ID, g.Index, strings.ToLower(v.Bandwidth))})
|
||||
}
|
||||
m.GPUs = append(m.GPUs, v)
|
||||
}
|
||||
imexOK := n.IMEX.State == "active" && n.IMEX.Channels > 0
|
||||
switch {
|
||||
case n.IMEX.State != "active":
|
||||
state := n.IMEX.State
|
||||
if state == "" {
|
||||
state = "not installed"
|
||||
}
|
||||
r.Problems = append(r.Problems, Problem{Level: Fail, Domain: key, Message: fmt.Sprintf(
|
||||
"%s: nvidia-imex is %s — without it the domain's GPUs cannot share memory across nodes",
|
||||
n.ID, state)})
|
||||
case n.IMEX.Channels == 0:
|
||||
r.Problems = append(r.Problems, Problem{Level: Fail, Domain: key, Message: fmt.Sprintf(
|
||||
"%s: nvidia-imex runs but there is no IMEX channel for an engine to open (/dev/nvidia-caps-imex-channels)", n.ID)})
|
||||
}
|
||||
m.Usable = joined == len(n.GPUs) && joined > 0 && imexOK
|
||||
if joined > 0 && joined < len(n.GPUs) {
|
||||
r.Problems = append(r.Problems, Problem{Level: Fail, Domain: key, Message: fmt.Sprintf(
|
||||
"%s is half joined (%d of %d GPUs), so no replica spans the domain through it", n.ID, joined, len(n.GPUs))})
|
||||
}
|
||||
d.GPUs += len(n.GPUs)
|
||||
d.Joined += joined
|
||||
if m.Usable {
|
||||
d.Usable++
|
||||
}
|
||||
d.Members = append(d.Members, m)
|
||||
}
|
||||
for _, d := range by {
|
||||
sort.Slice(d.Members, func(i, j int) bool { return d.Members[i].Node < d.Members[j].Node })
|
||||
r.Domains = append(r.Domains, *d)
|
||||
}
|
||||
sort.Slice(r.Domains, func(i, j int) bool { return r.Domains[i].Key < r.Domains[j].Key })
|
||||
sort.SliceStable(r.Problems, func(i, j int) bool {
|
||||
if r.Problems[i].Level != r.Problems[j].Level {
|
||||
return r.Problems[i].Level == Fail
|
||||
}
|
||||
return r.Problems[i].Message < r.Problems[j].Message
|
||||
})
|
||||
return r
|
||||
}
|
||||
|
||||
// Short is a domain's key as a person reads it: the cluster's first group
|
||||
// and the clique.
|
||||
func Short(key string) string {
|
||||
uuid, clique, _ := strings.Cut(key, "/")
|
||||
if i := strings.IndexByte(uuid, '-'); i > 0 {
|
||||
uuid = uuid[:i]
|
||||
}
|
||||
return uuid + "·" + clique
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package nvlink
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
func tray(id string, gpus int, state string) inventory.Node {
|
||||
n := inventory.Node{ID: id, IMEX: discovery.IMEX{State: "active", Channels: 1}}
|
||||
for i := range gpus {
|
||||
n.GPUs = append(n.GPUs, discovery.GPU{Index: i, Name: "NVIDIA GB200", Fabric: &discovery.GPUFabric{
|
||||
State: state, Status: "Success", ClusterUUID: "76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa", CliqueID: "1", Bandwidth: "Full"}})
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func has(t *testing.T, r Report, l Level, s string) {
|
||||
t.Helper()
|
||||
for _, p := range r.Problems {
|
||||
if p.Level == l && strings.Contains(p.Message, s) {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Errorf("no %s problem with %q in %+v", l, s, r.Problems)
|
||||
}
|
||||
|
||||
func TestARackOfJoinedTraysIsOneUsableDomain(t *testing.T) {
|
||||
r := Analyze([]inventory.Node{tray("t1", 4, "Completed"), tray("t2", 4, "Completed"), {ID: "cpu"}})
|
||||
if len(r.Domains) != 1 || r.Domains[0].Usable != 2 || r.Domains[0].Joined != 8 || len(r.Problems) != 0 {
|
||||
t.Errorf("report = %+v", r)
|
||||
}
|
||||
if Short(r.Domains[0].Key) != "76b1ad76·1" {
|
||||
t.Errorf("short = %q", Short(r.Domains[0].Key))
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatKeepsATrayOutIsSaid(t *testing.T) {
|
||||
half := tray("t2", 4, "Completed")
|
||||
half.GPUs[3].Fabric = &discovery.GPUFabric{State: "In Progress"}
|
||||
noIMEX := tray("t3", 4, "Completed")
|
||||
noIMEX.IMEX = discovery.IMEX{State: "inactive"}
|
||||
slow := tray("t4", 4, "Completed")
|
||||
slow.GPUs[0].Fabric.Bandwidth = "Degraded"
|
||||
r := Analyze([]inventory.Node{tray("t1", 4, "Completed"), half, noIMEX, slow})
|
||||
has(t, r, Fail, "t2 GPU 3 is not joined to its domain (In Progress)")
|
||||
has(t, r, Fail, "t2 is half joined")
|
||||
has(t, r, Fail, "t3: nvidia-imex is inactive")
|
||||
has(t, r, Warn, "t4 GPU 0 has degraded NVLink bandwidth")
|
||||
if r.Domains[0].Usable != 2 {
|
||||
t.Errorf("usable = %d, want t1 and t4", r.Domains[0].Usable)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user