Find multi-node NVLink domains, place across them, and show them as racks
A GB200 NVL72 rack or an NVLink Switch system is a scale-up island: its GPUs reach each other over NVLink whatever node they are in. Each GPU now reports its fabric from nvidia-smi -q - state, status, ClusterUUID, CliqueId, bandwidth - at registration and once a minute after, since a fabric manager restarting drops every card out at once; and each node reports nvidia-imex and its channels, without which the domain's GPUs cannot share memory across nodes. A single machine's NVSwitch, which reports a cluster of zeros, is no domain. A replica spanning nodes is placed inside one domain first, on nodes whose every card is joined and whose IMEX is up, and its shards are told NCCL_MNNVL_ENABLE=1; InfiniBand is the fallback. /admin/nvlink draws each domain as a rack - the NVLink Switch band on top, light running down a backplane, trays of GPU chips that glow when joined, pulse red when not, turn amber on degraded bandwidth - and lists what keeps a tray out. A node's page shows its GPU-by-GPU NVLink matrix, each card's NUMA node and nearest adapter, and its domain and IMEX. doctor checks joining, bandwidth, IMEX and its channel; metrics llmgw_gpu_nvlink_joined and llmgw_imex_up; /api/v1/nvlink. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
5bbc619a31
commit
20479a2d5f
31 files changed
+1535
-131
No files matched your search
@@ -16,11 +16,13 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bench"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/topology"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
@@ -53,6 +55,9 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("GET /api/v1/fabrics", func(w http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(w, http.StatusOK, a.Fabrics())
|
||||
})
|
||||
mux.HandleFunc("GET /api/v1/nvlink", func(w http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(w, http.StatusOK, a.NVLink())
|
||||
})
|
||||
mux.HandleFunc("GET /api/v1/fabrics/{id}/topology", func(w http.ResponseWriter, r *http.Request) {
|
||||
t, ok := a.Topology(r.PathValue("id"))
|
||||
if !ok {
|
||||
@@ -134,6 +139,27 @@ type NodeView struct {
|
||||
Load *LoadView `json:"load,omitempty"`
|
||||
// IBPorts are the node's InfiniBand and RoCE ports at the last heartbeat.
|
||||
IBPorts []IBPortView `json:"ib_ports,omitempty"`
|
||||
// GPUs are the cards, with how they are wired: NVLink peers, the
|
||||
// adapter nearest each, and the NVLink domain each has joined.
|
||||
GPUs []GPUView `json:"gpus,omitempty"`
|
||||
// NVLinkDomain is the multi-node domain every card has joined, if any.
|
||||
NVLinkDomain string `json:"nvlink_domain,omitempty"`
|
||||
IMEX string `json:"imex,omitempty"`
|
||||
IMEXChannels int `json:"imex_channels,omitempty"`
|
||||
}
|
||||
|
||||
// GPUView is one card as a node's page shows it.
|
||||
type GPUView struct {
|
||||
Index int `json:"index"`
|
||||
Name string `json:"name"`
|
||||
NUMA int `json:"numa"`
|
||||
NVLinkPeers []int `json:"nvlink_peers,omitempty"`
|
||||
// NearestNIC is the adapter closest to the card, and how close.
|
||||
NearestNIC string `json:"nearest_nic,omitempty"`
|
||||
NICPath string `json:"nic_path,omitempty"`
|
||||
FabricState string `json:"fabric_state,omitempty"`
|
||||
Domain string `json:"domain,omitempty"`
|
||||
Bandwidth string `json:"bandwidth,omitempty"`
|
||||
}
|
||||
|
||||
// IBPortView is one port as a page shows it.
|
||||
@@ -344,6 +370,20 @@ func (a *API) Nodes() []NodeView {
|
||||
v.LastSeenAgo = time.Since(n.LastSeen).Round(time.Second).String()
|
||||
}
|
||||
v.Load = loadView(n.Load, time.Now())
|
||||
v.NVLinkDomain, v.IMEX, v.IMEXChannels = n.NVLinkDomain(), n.IMEX.State, n.IMEX.Channels
|
||||
for _, g := range n.GPUs {
|
||||
gv := GPUView{Index: g.Index, Name: g.Name, NUMA: g.NUMANode, NVLinkPeers: g.NVLinkPeers}
|
||||
for hca, rel := range g.NICPaths {
|
||||
if gv.NearestNIC == "" || discovery.PathRank(rel) < discovery.PathRank(gv.NICPath) ||
|
||||
(discovery.PathRank(rel) == discovery.PathRank(gv.NICPath) && hca < gv.NearestNIC) {
|
||||
gv.NearestNIC, gv.NICPath = hca, rel
|
||||
}
|
||||
}
|
||||
if f := g.Fabric; f != nil {
|
||||
gv.FabricState, gv.Domain, gv.Bandwidth = f.State, f.Domain(), f.Bandwidth
|
||||
}
|
||||
v.GPUs = append(v.GPUs, gv)
|
||||
}
|
||||
fabricOf := map[string]string{}
|
||||
for _, f := range n.Fabrics {
|
||||
for _, h := range f.HCAs {
|
||||
@@ -959,3 +999,9 @@ func (a *API) Topology(id string) (TopologyView, bool) {
|
||||
}
|
||||
return t, true
|
||||
}
|
||||
|
||||
// NVLink is the cluster's multi-node NVLink domains, and what keeps any of
|
||||
// their nodes out.
|
||||
func (a *API) NVLink() nvlink.Report {
|
||||
return nvlink.Analyze(a.ctl.Inventory().Nodes())
|
||||
}
|
||||
@@ -61,6 +61,8 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
|
||||
CUDAVersion: g.GetCudaVersion(),
|
||||
NVLinkPeers: intSlice(g.GetNvlinkPeers()),
|
||||
NICPaths: maps.Clone(g.GetNicPaths()),
|
||||
Fabric: fabricFrom(g.GetFabricState(), g.GetFabricStatus(), g.GetClusterUuid(),
|
||||
g.GetCliqueId(), g.GetFabricBandwidth()),
|
||||
Health: discovery.GPUHealth{
|
||||
ECCErrors: int(g.GetEccErrors()),
|
||||
TemperatureC: int(g.GetTemperatureC()),
|
||||
@@ -68,6 +70,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
|
||||
},
|
||||
})
|
||||
}
|
||||
n.IMEX = discovery.IMEX{State: r.GetImex().GetState(), Channels: int(r.GetImex().GetChannels())}
|
||||
for _, f := range r.GetFabrics() {
|
||||
n.Fabrics = append(n.Fabrics, inventory.FabricAttachment{
|
||||
Fabric: f.GetFabric(),
|
||||
@@ -119,7 +122,13 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string,
|
||||
TemperatureC: int32(g.Health.TemperatureC),
|
||||
EccErrors: int32(g.Health.ECCErrors),
|
||||
})
|
||||
if f := g.Fabric; f != nil {
|
||||
last := r.Gpus[len(r.Gpus)-1]
|
||||
last.FabricState, last.FabricStatus, last.ClusterUuid = f.State, f.Status, f.ClusterUUID
|
||||
last.CliqueId, last.FabricBandwidth = f.CliqueID, f.Bandwidth
|
||||
}
|
||||
}
|
||||
r.Imex = &llmgwv1.IMEX{State: inv.IMEX.State, Channels: int32(inv.IMEX.Channels)}
|
||||
for _, f := range fabrics {
|
||||
r.Fabrics = append(r.Fabrics, &llmgwv1.FabricAttachment{
|
||||
Fabric: f.Fabric, Hcas: f.HCAs, RateGbps: int32(f.RateGbps),
|
||||
@@ -266,3 +275,12 @@ func int32Slice(v []int) []int32 {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// fabricFrom is a GPU's NVLink fabric as the wire carries it, nil when the
|
||||
// driver reported none.
|
||||
func fabricFrom(state, status, cluster, clique, bandwidth string) *discovery.GPUFabric {
|
||||
if state == "" {
|
||||
return nil
|
||||
}
|
||||
return &discovery.GPUFabric{State: state, Status: status, ClusterUUID: cluster, CliqueID: clique, Bandwidth: bandwidth}
|
||||
}
|
||||
@@ -242,3 +242,25 @@ func TestAPortGoingDownTakesItsFabricAway(t *testing.T) {
|
||||
t.Error("a heartbeat without a fabric change emptied the fabrics")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGPUFallingOutOfItsDomainTakesTheNodeOut(t *testing.T) {
|
||||
// A fabric manager restarting drops cards out of the domain; the node has
|
||||
// to stop being offered work that spans it until they are back.
|
||||
s := newTestService()
|
||||
fab := &discovery.GPUFabric{State: "Completed", ClusterUUID: "76b1ad76-7c24", CliqueID: "1"}
|
||||
nodeWith(s, inventory.Node{ID: "t1", State: inventory.Ready, GPUs: []discovery.GPU{
|
||||
{Index: 0, Fabric: fab}, {Index: 1, Fabric: fab}}})
|
||||
if n, _ := s.inv.Get("t1"); n.NVLinkDomain() == "" {
|
||||
t.Fatal("the node should start in its domain")
|
||||
}
|
||||
s.applyHeartbeat("t1", &llmgwv1.Heartbeat{NodeId: "t1",
|
||||
GpuFabric: []*llmgwv1.GPUFabricState{{Index: 1, State: "In Progress"}},
|
||||
Imex: &llmgwv1.IMEX{State: "active", Channels: 1}})
|
||||
n, _ := s.inv.Get("t1")
|
||||
if n.NVLinkDomain() != "" {
|
||||
t.Errorf("a node with a card out of the domain is still in it")
|
||||
}
|
||||
if n.IMEX.State != "active" || n.IMEX.Channels != 1 {
|
||||
t.Errorf("imex = %+v", n.IMEX)
|
||||
}
|
||||
}
|
||||
@@ -322,7 +322,7 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
||||
if len(used) == 0 && len(problems) == 0 && len(storage) == 0 &&
|
||||
!hb.GetWeightsChanged() && memMB == 0 && !hb.GetEnginesChanged() &&
|
||||
!hb.GetProvisionChanged() && hb.GetLoad() == nil && len(hb.GetIbPorts()) == 0 &&
|
||||
!hb.GetFabricsChanged() {
|
||||
!hb.GetFabricsChanged() && len(hb.GetGpuFabric()) == 0 && hb.GetImex() == nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -457,6 +457,24 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
||||
if len(hb.GetIbPorts()) > 0 {
|
||||
n.IBAt = time.Now()
|
||||
}
|
||||
// NVLink domain membership, which a fabric manager restarting takes
|
||||
// away from every card at once.
|
||||
if fab := hb.GetGpuFabric(); len(fab) > 0 {
|
||||
before := n.NVLinkDomain()
|
||||
for _, f := range fab {
|
||||
for i := range n.GPUs {
|
||||
if n.GPUs[i].Index == int(f.GetIndex()) {
|
||||
n.GPUs[i].Fabric = fabricFrom(f.GetState(), f.GetStatus(), f.GetClusterUuid(), f.GetCliqueId(), f.GetBandwidth())
|
||||
}
|
||||
}
|
||||
}
|
||||
if after := n.NVLinkDomain(); after != before {
|
||||
s.log.Warn("a node's NVLink domain changed", "node", nodeID, "from", before, "to", after)
|
||||
}
|
||||
}
|
||||
if m := hb.GetImex(); m != nil {
|
||||
n.IMEX = discovery.IMEX{State: m.GetState(), Channels: int(m.GetChannels())}
|
||||
}
|
||||
// A port went down, or came back. Placement reads the fabrics, so this is
|
||||
// what stops a node being offered multi-node work it can no longer carry.
|
||||
if hb.GetFabricsChanged() {
|
||||
|
||||
@@ -74,6 +74,9 @@ type Collector struct {
|
||||
ibRate *prometheus.Desc
|
||||
ibCounter *prometheus.Desc
|
||||
|
||||
gpuNVLJoined *prometheus.Desc
|
||||
imexUp *prometheus.Desc
|
||||
|
||||
fabricSMs *prometheus.Desc
|
||||
fabricMaster *prometheus.Desc
|
||||
fabricProblems *prometheus.Desc
|
||||
@@ -166,6 +169,11 @@ func New(src Source) *Collector {
|
||||
ibRate: prometheus.NewDesc("llmgw_ib_port_rate_gbps",
|
||||
"A port's negotiated rate. A link that trained below its usual speed is a "+
|
||||
"cable or transceiver to look at.", []string{"node", "hca", "port"}, nil),
|
||||
gpuNVLJoined: prometheus.NewDesc("llmgw_gpu_nvlink_joined",
|
||||
"1 when a GPU is joined to its multi-node NVLink domain, 0 when it reports a fabric it has not "+
|
||||
"joined. Absent on cards with no multi-node fabric.", []string{"node", "gpu", "domain", "state"}, nil),
|
||||
imexUp: prometheus.NewDesc("llmgw_imex_up",
|
||||
"1 when nvidia-imex is active with a channel, on nodes whose GPUs report a fabric.", []string{"node"}, nil),
|
||||
fabricSMs: prometheus.NewDesc("llmgw_fabric_subnet_managers",
|
||||
"Subnet managers a physical fabric lists, by state. No standby means the fabric "+
|
||||
"cannot recover from a change if its master stops.", []string{"fabric", "physical", "state"}, nil),
|
||||
@@ -189,6 +197,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
|
||||
c.nodeDiskFree, c.nodeDiskSize,
|
||||
c.nodeCPU, c.nodeCPUs, c.nodeLoad, c.nodeMemUsed, c.nodeMemAvail, c.shardCPU, c.shardMemory,
|
||||
c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems,
|
||||
c.gpuNVLJoined, c.imexUp,
|
||||
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy,
|
||||
c.gpuNVLink, c.gpuECC,
|
||||
c.replicaState, c.replicaReady, c.replicaIdle,
|
||||
@@ -263,6 +272,28 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
|
||||
}
|
||||
}
|
||||
|
||||
fabricGPUs := 0
|
||||
for _, g := range n.GPUs {
|
||||
fb := g.Fabric
|
||||
if fb == nil || strings.EqualFold(fb.State, "Not Supported") {
|
||||
continue
|
||||
}
|
||||
fabricGPUs++
|
||||
joined := 0.0
|
||||
if fb.Domain() != "" {
|
||||
joined = 1
|
||||
}
|
||||
ch <- prometheus.MustNewConstMetric(c.gpuNVLJoined, prometheus.GaugeValue, joined,
|
||||
n.ID, strconv.Itoa(g.Index), fb.Domain(), fb.State)
|
||||
}
|
||||
if fabricGPUs > 0 {
|
||||
up := 0.0
|
||||
if n.IMEX.State == "active" && n.IMEX.Channels > 0 {
|
||||
up = 1
|
||||
}
|
||||
ch <- prometheus.MustNewConstMetric(c.imexUp, prometheus.GaugeValue, up, n.ID)
|
||||
}
|
||||
|
||||
byClass := map[string]int{}
|
||||
for _, g := range n.GPUs {
|
||||
class := g.Class()
|
||||
|
||||
@@ -120,6 +120,24 @@ type Node struct {
|
||||
IBPorts []IBPort `json:"-"`
|
||||
// IBAt is when IBPorts was read, for the rates.
|
||||
IBAt time.Time `json:"-"`
|
||||
// IMEX is the service a multi-node NVLink domain needs on this node.
|
||||
IMEX discovery.IMEX `json:"imex"`
|
||||
}
|
||||
|
||||
// NVLinkDomain is the multi-node NVLink domain this node's GPUs are joined
|
||||
// to, when every one of them is joined to the same. A node half joined — a
|
||||
// card still coming up, or one that fell out — is in none: a replica that
|
||||
// counts on its NVLink would find a card that has none.
|
||||
func (n Node) NVLinkDomain() string {
|
||||
d := ""
|
||||
for _, g := range n.GPUs {
|
||||
gd := g.Fabric.Domain()
|
||||
if gd == "" || (d != "" && gd != d) {
|
||||
return ""
|
||||
}
|
||||
d = gd
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// IBPort is one port's state, identity and cumulative counters.
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
// Package nvlink works out the cluster's multi-node NVLink domains — an NVL72
|
||||
// rack, an NVLink Switch system — from what each GPU reports, and what keeps
|
||||
// one from being usable.
|
||||
//
|
||||
// A domain is a scale-up island: its GPUs reach each other over NVLink
|
||||
// whatever node they are in, so a tensor-parallel group can span its nodes
|
||||
// at NVLink speed. What it needs to be one is every card joined by the
|
||||
// fabric manager and IMEX running on every node; either missing, and a
|
||||
// replica placed across it fails or crawls.
|
||||
package nvlink
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
// Level is how much a problem matters.
|
||||
type Level string
|
||||
|
||||
// The levels.
|
||||
const (
|
||||
Fail Level = "fail"
|
||||
Warn Level = "warn"
|
||||
)
|
||||
|
||||
// Problem is one thing wrong with a domain.
|
||||
type Problem struct {
|
||||
Level Level `json:"level"`
|
||||
Domain string `json:"domain"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// GPU is one card as its domain sees it.
|
||||
type GPU struct {
|
||||
Index int `json:"index"`
|
||||
Name string `json:"name"`
|
||||
State string `json:"state"`
|
||||
Status string `json:"status,omitempty"`
|
||||
Bandwidth string `json:"bandwidth,omitempty"`
|
||||
Joined bool `json:"joined"`
|
||||
}
|
||||
|
||||
// Member is one node of a domain.
|
||||
type Member struct {
|
||||
Node string `json:"node"`
|
||||
GPUs []GPU `json:"gpus"`
|
||||
// Usable is every card joined and IMEX up: what placement requires.
|
||||
Usable bool `json:"usable"`
|
||||
IMEX string `json:"imex"`
|
||||
IMEXChannels int `json:"imex_channels"`
|
||||
}
|
||||
|
||||
// Domain is one multi-node NVLink domain.
|
||||
type Domain struct {
|
||||
Key string `json:"key"`
|
||||
ClusterUUID string `json:"cluster_uuid"`
|
||||
CliqueID string `json:"clique_id"`
|
||||
Members []Member `json:"members"`
|
||||
GPUs int `json:"gpus"`
|
||||
Joined int `json:"joined"`
|
||||
Usable int `json:"usable_nodes"`
|
||||
}
|
||||
|
||||
// Report is every domain and what is wrong with them.
|
||||
type Report struct {
|
||||
Domains []Domain `json:"domains"`
|
||||
Problems []Problem `json:"problems"`
|
||||
}
|
||||
|
||||
// Analyze reads every node's GPUs.
|
||||
func Analyze(nodes []inventory.Node) Report {
|
||||
var r Report
|
||||
by := map[string]*Domain{}
|
||||
for _, n := range nodes {
|
||||
// The domain a node is in is the one any of its cards joined; a card
|
||||
// still joining has no domain of its own yet, and is counted against
|
||||
// the node's.
|
||||
key := ""
|
||||
for _, g := range n.GPUs {
|
||||
if d := g.Fabric.Domain(); d != "" {
|
||||
key = d
|
||||
break
|
||||
}
|
||||
}
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
d := by[key]
|
||||
if d == nil {
|
||||
uuid, clique, _ := strings.Cut(key, "/")
|
||||
d = &Domain{Key: key, ClusterUUID: uuid, CliqueID: clique}
|
||||
by[key] = d
|
||||
}
|
||||
m := Member{Node: n.ID, IMEX: n.IMEX.State, IMEXChannels: n.IMEX.Channels}
|
||||
joined := 0
|
||||
for _, g := range n.GPUs {
|
||||
v := GPU{Index: g.Index, Name: g.Name}
|
||||
if f := g.Fabric; f != nil {
|
||||
v.State, v.Status, v.Bandwidth = f.State, f.Status, f.Bandwidth
|
||||
v.Joined = f.Domain() == key
|
||||
}
|
||||
if v.Joined {
|
||||
joined++
|
||||
} else {
|
||||
why := v.State
|
||||
if why == "" {
|
||||
why = "no fabric"
|
||||
}
|
||||
if v.Status != "" && !strings.EqualFold(v.Status, "Success") {
|
||||
why += ": " + v.Status
|
||||
}
|
||||
r.Problems = append(r.Problems, Problem{Level: Fail, Domain: key, Message: fmt.Sprintf(
|
||||
"%s GPU %d is not joined to its domain (%s)", n.ID, g.Index, why)})
|
||||
}
|
||||
if v.Bandwidth != "" && !strings.EqualFold(v.Bandwidth, "Full") {
|
||||
r.Problems = append(r.Problems, Problem{Level: Warn, Domain: key, Message: fmt.Sprintf(
|
||||
"%s GPU %d has %s NVLink bandwidth", n.ID, g.Index, strings.ToLower(v.Bandwidth))})
|
||||
}
|
||||
m.GPUs = append(m.GPUs, v)
|
||||
}
|
||||
imexOK := n.IMEX.State == "active" && n.IMEX.Channels > 0
|
||||
switch {
|
||||
case n.IMEX.State != "active":
|
||||
state := n.IMEX.State
|
||||
if state == "" {
|
||||
state = "not installed"
|
||||
}
|
||||
r.Problems = append(r.Problems, Problem{Level: Fail, Domain: key, Message: fmt.Sprintf(
|
||||
"%s: nvidia-imex is %s — without it the domain's GPUs cannot share memory across nodes",
|
||||
n.ID, state)})
|
||||
case n.IMEX.Channels == 0:
|
||||
r.Problems = append(r.Problems, Problem{Level: Fail, Domain: key, Message: fmt.Sprintf(
|
||||
"%s: nvidia-imex runs but there is no IMEX channel for an engine to open (/dev/nvidia-caps-imex-channels)", n.ID)})
|
||||
}
|
||||
m.Usable = joined == len(n.GPUs) && joined > 0 && imexOK
|
||||
if joined > 0 && joined < len(n.GPUs) {
|
||||
r.Problems = append(r.Problems, Problem{Level: Fail, Domain: key, Message: fmt.Sprintf(
|
||||
"%s is half joined (%d of %d GPUs), so no replica spans the domain through it", n.ID, joined, len(n.GPUs))})
|
||||
}
|
||||
d.GPUs += len(n.GPUs)
|
||||
d.Joined += joined
|
||||
if m.Usable {
|
||||
d.Usable++
|
||||
}
|
||||
d.Members = append(d.Members, m)
|
||||
}
|
||||
for _, d := range by {
|
||||
sort.Slice(d.Members, func(i, j int) bool { return d.Members[i].Node < d.Members[j].Node })
|
||||
r.Domains = append(r.Domains, *d)
|
||||
}
|
||||
sort.Slice(r.Domains, func(i, j int) bool { return r.Domains[i].Key < r.Domains[j].Key })
|
||||
sort.SliceStable(r.Problems, func(i, j int) bool {
|
||||
if r.Problems[i].Level != r.Problems[j].Level {
|
||||
return r.Problems[i].Level == Fail
|
||||
}
|
||||
return r.Problems[i].Message < r.Problems[j].Message
|
||||
})
|
||||
return r
|
||||
}
|
||||
|
||||
// Short is a domain's key as a person reads it: the cluster's first group
|
||||
// and the clique.
|
||||
func Short(key string) string {
|
||||
uuid, clique, _ := strings.Cut(key, "/")
|
||||
if i := strings.IndexByte(uuid, '-'); i > 0 {
|
||||
uuid = uuid[:i]
|
||||
}
|
||||
return uuid + "·" + clique
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package nvlink
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
func tray(id string, gpus int, state string) inventory.Node {
|
||||
n := inventory.Node{ID: id, IMEX: discovery.IMEX{State: "active", Channels: 1}}
|
||||
for i := range gpus {
|
||||
n.GPUs = append(n.GPUs, discovery.GPU{Index: i, Name: "NVIDIA GB200", Fabric: &discovery.GPUFabric{
|
||||
State: state, Status: "Success", ClusterUUID: "76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa", CliqueID: "1", Bandwidth: "Full"}})
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func has(t *testing.T, r Report, l Level, s string) {
|
||||
t.Helper()
|
||||
for _, p := range r.Problems {
|
||||
if p.Level == l && strings.Contains(p.Message, s) {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Errorf("no %s problem with %q in %+v", l, s, r.Problems)
|
||||
}
|
||||
|
||||
func TestARackOfJoinedTraysIsOneUsableDomain(t *testing.T) {
|
||||
r := Analyze([]inventory.Node{tray("t1", 4, "Completed"), tray("t2", 4, "Completed"), {ID: "cpu"}})
|
||||
if len(r.Domains) != 1 || r.Domains[0].Usable != 2 || r.Domains[0].Joined != 8 || len(r.Problems) != 0 {
|
||||
t.Errorf("report = %+v", r)
|
||||
}
|
||||
if Short(r.Domains[0].Key) != "76b1ad76·1" {
|
||||
t.Errorf("short = %q", Short(r.Domains[0].Key))
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatKeepsATrayOutIsSaid(t *testing.T) {
|
||||
half := tray("t2", 4, "Completed")
|
||||
half.GPUs[3].Fabric = &discovery.GPUFabric{State: "In Progress"}
|
||||
noIMEX := tray("t3", 4, "Completed")
|
||||
noIMEX.IMEX = discovery.IMEX{State: "inactive"}
|
||||
slow := tray("t4", 4, "Completed")
|
||||
slow.GPUs[0].Fabric.Bandwidth = "Degraded"
|
||||
r := Analyze([]inventory.Node{tray("t1", 4, "Completed"), half, noIMEX, slow})
|
||||
has(t, r, Fail, "t2 GPU 3 is not joined to its domain (In Progress)")
|
||||
has(t, r, Fail, "t2 is half joined")
|
||||
has(t, r, Fail, "t3: nvidia-imex is inactive")
|
||||
has(t, r, Warn, "t4 GPU 0 has degraded NVLink bandwidth")
|
||||
if r.Domains[0].Usable != 2 {
|
||||
t.Errorf("usable = %d, want t1 and t4", r.Domains[0].Usable)
|
||||
}
|
||||
}
|
||||
@@ -239,13 +239,20 @@ func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rej
|
||||
var rejections []Rejection
|
||||
|
||||
fabrics := s.inv.Fabrics()
|
||||
if len(fabrics) == 0 {
|
||||
if len(fabrics) == 0 && !s.anyNVLinkDomain() {
|
||||
return nil, []Rejection{{
|
||||
Profile: p.Name,
|
||||
Reason: fmt.Sprintf("needs %d nodes on one fabric, but no node reports a fabric", p.Nodes),
|
||||
}}
|
||||
}
|
||||
|
||||
// A multi-node NVLink domain first: a replica inside one runs tensor
|
||||
// parallelism across its nodes at NVLink speed, which no InfiniBand
|
||||
// fabric matches.
|
||||
if pl := s.placeOnNVLink(p, avoid, &rejections); pl != nil {
|
||||
return pl, rejections
|
||||
}
|
||||
|
||||
canon := inventory.CanonicalSMs(s.inv.Nodes())
|
||||
for _, fabric := range fabrics {
|
||||
all := s.eligibleNodes(p, fabric, avoid, &rejections)
|
||||
@@ -262,6 +269,66 @@ func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rej
|
||||
return nil, rejections
|
||||
}
|
||||
|
||||
// NVLinkFabric is how a placement names a multi-node NVLink domain in its
|
||||
// Fabric, so the shards on it are told to use it.
|
||||
const NVLinkFabric = "nvl:"
|
||||
|
||||
func (s *Scheduler) anyNVLinkDomain() bool {
|
||||
for _, n := range s.inv.Nodes() {
|
||||
if n.NVLinkDomain() != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// placeOnNVLink forms a replica inside one multi-node NVLink domain, from
|
||||
// nodes whose every card is joined to it and whose IMEX is up: a node
|
||||
// missing either would leave the replica a card, or its memory, short.
|
||||
func (s *Scheduler) placeOnNVLink(p Profile, avoid []string, rejections *[]Rejection) *Placement {
|
||||
var none []Rejection
|
||||
cands := s.eligibleNodes(p, "", avoid, &none)
|
||||
by := map[string][]candidate{}
|
||||
for _, c := range cands {
|
||||
d := c.node.NVLinkDomain()
|
||||
if d == "" {
|
||||
continue
|
||||
}
|
||||
if c.node.IMEX.State != "active" || c.node.IMEX.Channels == 0 {
|
||||
*rejections = append(*rejections, Rejection{Profile: p.Name, Pool: NVLinkFabric + shortDomain(d), Node: c.node.ID,
|
||||
Reason: "in an NVLink domain, but nvidia-imex is not up"})
|
||||
continue
|
||||
}
|
||||
by[d] = append(by[d], c)
|
||||
}
|
||||
keys := make([]string, 0, len(by))
|
||||
for k := range by {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if len(by[keys[i]]) != len(by[keys[j]]) {
|
||||
return len(by[keys[i]]) > len(by[keys[j]])
|
||||
}
|
||||
return keys[i] < keys[j]
|
||||
})
|
||||
for _, d := range keys {
|
||||
pool := NVLinkFabric + shortDomain(d) + "/" + p.classLabel()
|
||||
if pl := s.placeOnFabric(p, NVLinkFabric+d, pool, by[d], rejections); pl != nil {
|
||||
return pl
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// shortDomain is an NVLink domain as a pool name carries it.
|
||||
func shortDomain(d string) string {
|
||||
uuid, clique, _ := strings.Cut(d, "/")
|
||||
if i := strings.IndexByte(uuid, '-'); i > 0 {
|
||||
uuid = uuid[:i]
|
||||
}
|
||||
return uuid + "." + clique
|
||||
}
|
||||
|
||||
// fabricGroup is the candidates on one physical fabric.
|
||||
type fabricGroup struct {
|
||||
key string
|
||||
|
||||
@@ -1018,3 +1018,56 @@ func TestAReplicaStaysUnderOneLeafAndOffDegradedCables(t *testing.T) {
|
||||
t.Errorf("placed on %v, want the two good nodes under leaf-b", got)
|
||||
}
|
||||
}
|
||||
|
||||
// inDomain joins every card of a node to one multi-node NVLink domain, with
|
||||
// IMEX up.
|
||||
func inDomain(n inventory.Node, clique string) inventory.Node {
|
||||
for i := range n.GPUs {
|
||||
n.GPUs[i].Fabric = &discovery.GPUFabric{State: "Completed", ClusterUUID: "76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa", CliqueID: clique}
|
||||
}
|
||||
n.IMEX = discovery.IMEX{State: "active", Channels: 1}
|
||||
return n
|
||||
}
|
||||
|
||||
func TestAReplicaSpansNodesOverNVLinkBeforeInfiniBand(t *testing.T) {
|
||||
// Four trays on one InfiniBand fabric, two of them in an NVLink domain:
|
||||
// the pair is placed across the domain, where tensor parallelism between
|
||||
// nodes runs at NVLink speed, and given no adapters to use.
|
||||
st := store(
|
||||
onSM(node("ib1", "h100-80g", 4, "ib-a"), "0x01"), onSM(node("ib2", "h100-80g", 4, "ib-a"), "0x01"),
|
||||
inDomain(onSM(node("nv1", "h100-80g", 4, "ib-a"), "0x01"), "7"),
|
||||
inDomain(onSM(node("nv2", "h100-80g", 4, "ib-a"), "0x01"), "7"),
|
||||
)
|
||||
res := New(st).Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 2}}, nil)
|
||||
if res.Placement == nil {
|
||||
t.Fatalf("no placement:\n%s", res.Explain())
|
||||
}
|
||||
if !strings.HasPrefix(res.Placement.Fabric, NVLinkFabric) || !strings.HasPrefix(res.Placement.Pool, "nvl:76b1ad76.7/") {
|
||||
t.Errorf("fabric %q pool %q", res.Placement.Fabric, res.Placement.Pool)
|
||||
}
|
||||
for _, a := range res.Placement.Nodes {
|
||||
if !strings.HasPrefix(a.NodeID, "nv") || len(a.HCAs) != 0 {
|
||||
t.Errorf("assignment %+v", a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeWithoutIMEXIsNotCountedInItsDomain(t *testing.T) {
|
||||
a := inDomain(onSM(node("nv1", "h100-80g", 4, "ib-a"), "0x01"), "7")
|
||||
b := inDomain(onSM(node("nv2", "h100-80g", 4, "ib-a"), "0x01"), "7")
|
||||
b.IMEX.State = "inactive"
|
||||
res := New(store(a, b)).Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 2}}, nil)
|
||||
// Not across the domain; InfiniBand still carries it.
|
||||
if res.Placement == nil || strings.HasPrefix(res.Placement.Fabric, NVLinkFabric) {
|
||||
t.Fatalf("placement %+v\n%s", res.Placement, res.Explain())
|
||||
}
|
||||
|
||||
// With no InfiniBand to fall back on, the refusal says why.
|
||||
a2 := inDomain(node("nv1", "h100-80g", 4), "7")
|
||||
b2 := inDomain(node("nv2", "h100-80g", 4), "7")
|
||||
b2.IMEX.State = "inactive"
|
||||
res = New(store(a2, b2)).Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 2}}, nil)
|
||||
if res.Placement != nil || !strings.Contains(res.Explain(), "nvidia-imex is not up") {
|
||||
t.Errorf("the explanation should say why the domain was passed over:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user