Ilker ManapandClaude Opus 5.5
20479a2d5f
A GB200 NVL72 rack or an NVLink Switch system is a scale-up island: its GPUs reach each other over NVLink whatever node they are in. Each GPU now reports its fabric from nvidia-smi -q - state, status, ClusterUUID, CliqueId, bandwidth - at registration and once a minute after, since a fabric manager restarting drops every card out at once; and each node reports nvidia-imex and its channels, without which the domain's GPUs cannot share memory across nodes. A single machine's NVSwitch, which reports a cluster of zeros, is no domain. A replica spanning nodes is placed inside one domain first, on nodes whose every card is joined and whose IMEX is up, and its shards are told NCCL_MNNVL_ENABLE=1; InfiniBand is the fallback. /admin/nvlink draws each domain as a rack - the NVLink Switch band on top, light running down a backplane, trays of GPU chips that glow when joined, pulse red when not, turn amber on degraded bandwidth - and lists what keeps a tray out. A node's page shows its GPU-by-GPU NVLink matrix, each card's NUMA node and nearest adapter, and its domain and IMEX. doctor checks joining, bandwidth, IMEX and its channel; metrics llmgw_gpu_nvlink_joined and llmgw_imex_up; /api/v1/nvlink. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
57 lines
1.3 KiB
Go
57 lines
1.3 KiB
Go
package web
|
|
|
|
import (
|
|
"net/http"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
|
|
)
|
|
|
|
// NVLinkSource is a Source that can report multi-node NVLink domains.
|
|
type NVLinkSource interface {
|
|
NVLink() nvlink.Report
|
|
}
|
|
|
|
type nvlinkPageData struct {
|
|
Nav Nav
|
|
OnAdminListener bool
|
|
Nodes string
|
|
Domains []nvlinkDomainView
|
|
Problems []nvlink.Problem
|
|
Version, Now string
|
|
}
|
|
|
|
type nvlinkDomainView struct {
|
|
nvlink.Domain
|
|
Short string
|
|
Problems []nvlink.Problem
|
|
}
|
|
|
|
// NVLinkPage shows every multi-node NVLink domain as a rack of trays.
|
|
func NVLinkPage(src Source, onAdminListener bool, version string) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
ns, ok := src.(NVLinkSource)
|
|
if !ok {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
rep := ns.NVLink()
|
|
d := nvlinkPageData{Nav: navFrom(r, "cluster"), OnAdminListener: onAdminListener, Nodes: "/admin/nodes",
|
|
Problems: rep.Problems, Version: version, Now: time.Now().UTC().Format(time.RFC3339)}
|
|
if onAdminListener {
|
|
d.Nodes = "/nodes"
|
|
}
|
|
for _, dom := range rep.Domains {
|
|
v := nvlinkDomainView{Domain: dom, Short: nvlink.Short(dom.Key)}
|
|
for _, p := range rep.Problems {
|
|
if p.Domain == dom.Key {
|
|
v.Problems = append(v.Problems, p)
|
|
}
|
|
}
|
|
d.Domains = append(d.Domains, v)
|
|
}
|
|
noStore(w)
|
|
render(w, "nvlink", d)
|
|
}
|
|
}
|