Find multi-node NVLink domains, place across them, and show them as racks

A GB200 NVL72 rack or an NVLink Switch system is a scale-up island: its
GPUs reach each other over NVLink whatever node they are in. Each GPU
now reports its fabric from nvidia-smi -q - state, status, ClusterUUID,
CliqueId, bandwidth - at registration and once a minute after, since a
fabric manager restarting drops every card out at once; and each node
reports nvidia-imex and its channels, without which the domain's GPUs
cannot share memory across nodes. A single machine's NVSwitch, which
reports a cluster of zeros, is no domain.

A replica spanning nodes is placed inside one domain first, on nodes
whose every card is joined and whose IMEX is up, and its shards are
told NCCL_MNNVL_ENABLE=1; InfiniBand is the fallback.

/admin/nvlink draws each domain as a rack - the NVLink Switch band on
top, light running down a backplane, trays of GPU chips that glow when
joined, pulse red when not, turn amber on degraded bandwidth - and
lists what keeps a tray out. A node's page shows its GPU-by-GPU NVLink
matrix, each card's NUMA node and nearest adapter, and its domain and
IMEX. doctor checks joining, bandwidth, IMEX and its channel; metrics
llmgw_gpu_nvlink_joined and llmgw_imex_up; /api/v1/nvlink.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-29 09:07:59 +02:00
1 parent 5bbc619a31
commit 20479a2d5f
31 files changed
+1535 -131

No files matched your search

+46
View File
@@ -16,11 +16,13 @@ import (
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/topology"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
@@ -53,6 +55,9 @@ func (a *API) Register(mux *http.ServeMux) {
mux.HandleFunc("GET /api/v1/fabrics", func(w http.ResponseWriter, _ *http.Request) {
writeJSON(w, http.StatusOK, a.Fabrics())
})
mux.HandleFunc("GET /api/v1/nvlink", func(w http.ResponseWriter, _ *http.Request) {
writeJSON(w, http.StatusOK, a.NVLink())
})
mux.HandleFunc("GET /api/v1/fabrics/{id}/topology", func(w http.ResponseWriter, r *http.Request) {
t, ok := a.Topology(r.PathValue("id"))
if !ok {
@@ -134,6 +139,27 @@ type NodeView struct {
Load *LoadView `json:"load,omitempty"`
// IBPorts are the node's InfiniBand and RoCE ports at the last heartbeat.
IBPorts []IBPortView `json:"ib_ports,omitempty"`
// GPUs are the cards, with how they are wired: NVLink peers, the
// adapter nearest each, and the NVLink domain each has joined.
GPUs []GPUView `json:"gpus,omitempty"`
// NVLinkDomain is the multi-node domain every card has joined, if any.
NVLinkDomain string `json:"nvlink_domain,omitempty"`
IMEX string `json:"imex,omitempty"`
IMEXChannels int `json:"imex_channels,omitempty"`
}
// GPUView is one card as a node's page shows it.
type GPUView struct {
Index int `json:"index"`
Name string `json:"name"`
NUMA int `json:"numa"`
NVLinkPeers []int `json:"nvlink_peers,omitempty"`
// NearestNIC is the adapter closest to the card, and how close.
NearestNIC string `json:"nearest_nic,omitempty"`
NICPath string `json:"nic_path,omitempty"`
FabricState string `json:"fabric_state,omitempty"`
Domain string `json:"domain,omitempty"`
Bandwidth string `json:"bandwidth,omitempty"`
}
// IBPortView is one port as a page shows it.
@@ -344,6 +370,20 @@ func (a *API) Nodes() []NodeView {
v.LastSeenAgo = time.Since(n.LastSeen).Round(time.Second).String()
}
v.Load = loadView(n.Load, time.Now())
v.NVLinkDomain, v.IMEX, v.IMEXChannels = n.NVLinkDomain(), n.IMEX.State, n.IMEX.Channels
for _, g := range n.GPUs {
gv := GPUView{Index: g.Index, Name: g.Name, NUMA: g.NUMANode, NVLinkPeers: g.NVLinkPeers}
for hca, rel := range g.NICPaths {
if gv.NearestNIC == "" || discovery.PathRank(rel) < discovery.PathRank(gv.NICPath) ||
(discovery.PathRank(rel) == discovery.PathRank(gv.NICPath) && hca < gv.NearestNIC) {
gv.NearestNIC, gv.NICPath = hca, rel
}
}
if f := g.Fabric; f != nil {
gv.FabricState, gv.Domain, gv.Bandwidth = f.State, f.Domain(), f.Bandwidth
}
v.GPUs = append(v.GPUs, gv)
}
fabricOf := map[string]string{}
for _, f := range n.Fabrics {
for _, h := range f.HCAs {
@@ -959,3 +999,9 @@ func (a *API) Topology(id string) (TopologyView, bool) {
}
return t, true
}
// NVLink is the cluster's multi-node NVLink domains, and what keeps any of
// their nodes out.
func (a *API) NVLink() nvlink.Report {
return nvlink.Analyze(a.ctl.Inventory().Nodes())
}