Files
llmgw/internal/doctor/doctor.go
T
Ilker ManapandClaude Opus 5.5 45e3f52da3 Launch multi-node vLLM its own way by default, not through Ray
An empty launch_mode meant ray: a Ray head and joins on every node, a
dependency to install, and a path never checked against a real cluster.
It now means mp - every node runs vllm serve with its rank and the
leader's address, the workers headless - which is how vLLM documents a
multi-node launch today and needs nothing but vLLM. ray is still there
for a vLLM too old for mp's flags. The example agent configuration,
doctor and the reference say so.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-29 09:13:59 +02:00

581 lines
18 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Package doctor checks a node for what llmgw needs of its GPUs and fabric,
// and says what is wrong in words an operator can act on.
//
// Written for a machine nobody here can log into. The report is meant to be
// run on the node, read there, and sent back when it is not enough: every
// fact it concluded from is in it, so the conclusion can be checked.
package doctor
import (
"context"
"fmt"
"net"
"os"
"os/exec"
"path/filepath"
"slices"
"sort"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
)
// Level is how much a finding matters.
type Level string
// The levels, most serious first.
const (
Fail Level = "FAIL"
Warn Level = "WARN"
OK Level = "OK"
Info Level = "INFO"
)
// Finding is one conclusion, and what to do about it.
type Finding struct {
Level Level `json:"level"`
Area string `json:"area"`
Message string `json:"message"`
Fix string `json:"fix,omitempty"`
}
// Facts are what was read from the node. Every finding is drawn from these
// and nothing else, so a report sent back can be re-read here.
type Facts struct {
At time.Time `json:"at"`
Hostname string `json:"hostname"`
Host discovery.Host `json:"host"`
SMIFound bool `json:"nvidia_smi_found"`
GPUs []discovery.GPU `json:"gpus"`
GPUError string `json:"gpu_error,omitempty"`
TopoRaw string `json:"topo_raw,omitempty"`
TopoErr string `json:"topo_error,omitempty"`
// NVSwitch is set when the machine has NVSwitch devices, which need the
// fabric manager running before CUDA will start.
NVSwitch bool `json:"nvswitch"`
// IMEX is the service multi-node NVLink needs.
IMEX discovery.IMEX `json:"imex"`
FabricManager string `json:"fabric_manager,omitempty"`
HCAs []discovery.HCA `json:"hcas"`
// SMErrors say, per port, why the subnet manager could not be asked.
SMErrors map[string]string `json:"sm_errors,omitempty"`
// SMLists are every subnet manager on each fabric, by master GUID;
// SMListErrors why a fabric's could not be listed.
SMLists map[string][]discovery.SMRecord `json:"sm_lists,omitempty"`
SMListErrors map[string]string `json:"sm_list_errors,omitempty"`
Modules map[string]bool `json:"modules"`
// ConfigPath is the agent configuration read, empty when there was none.
ConfigPath string `json:"config_path,omitempty"`
ConfigErr string `json:"config_error,omitempty"`
Fabrics []discovery.FabricCheck `json:"fabrics,omitempty"`
MgmtAddr string `json:"mgmt_addr,omitempty"`
MgmtIface string `json:"mgmt_interface,omitempty"`
IfaceExists bool `json:"mgmt_interface_exists"`
LaunchMode string `json:"vllm_launch_mode,omitempty"`
NCCL map[string]map[string]string `json:"nccl_env,omitempty"`
UnitMemlock string `json:"agent_unit_memlock,omitempty"`
Engines map[string]string `json:"engines"`
}
// Modules checked, and why each matters.
var watchedModules = []string{"ib_core", "ib_uverbs", "mlx5_ib", "rdma_ucm", "nvidia_peermem", "nv_peer_mem"}
// Options says where to look.
type Options struct {
ConfigPath string
IBRoot string
NvidiaSMI string
}
// Collect reads the node.
func Collect(ctx context.Context, o Options) Facts {
f := Facts{At: time.Now().UTC(), Modules: map[string]bool{}, Engines: map[string]string{}}
f.Hostname, _ = os.Hostname()
f.Host = discovery.DiscoverHost()
smi := o.NvidiaSMI
if smi == "" {
if p, err := exec.LookPath("nvidia-smi"); err == nil {
smi = p
}
}
if smi != "" {
f.SMIFound = true
gpus, err := discovery.DiscoverGPUsSMI(ctx, smi)
if err != nil {
f.GPUError = err.Error()
}
out, err := exec.CommandContext(ctx, smi, "topo", "-m").Output()
if err != nil {
f.TopoErr = err.Error()
} else {
f.TopoRaw = string(out)
if t, err := discovery.ParseTopology(f.TopoRaw); err == nil {
t.Apply(gpus)
} else {
f.TopoErr = err.Error()
}
}
if fab, err := discovery.DiscoverGPUFabric(ctx, smi); err == nil {
discovery.ApplyFabric(gpus, fab)
}
f.GPUs = gpus
f.IMEX = discovery.DiscoverIMEX(ctx)
}
if m, _ := filepath.Glob("/dev/nvidia-nvswitch*"); len(m) > 0 {
f.NVSwitch = true
f.FabricManager = systemctl(ctx, "is-active", "nvidia-fabricmanager")
}
root := o.IBRoot
if root == "" {
root = discovery.SysfsRoot
}
f.HCAs, _ = discovery.DiscoverIB(root)
f.SMErrors = map[string]string{}
_ = discovery.AttachSM(f.HCAs, func(hca string, port int) (*discovery.SMInfo, error) {
sm, err := discovery.QuerySM(ctx, hca, port)
if err != nil {
f.SMErrors[fmt.Sprintf("%s/%d", hca, port)] = err.Error()
}
return sm, err
})
f.SMLists, f.SMListErrors = map[string][]discovery.SMRecord{}, map[string]string{}
for _, h := range f.HCAs {
for _, p := range h.Ports {
if p.SM == nil {
continue
}
if _, done := f.SMLists[p.SM.GUID]; done {
continue
}
if _, done := f.SMListErrors[p.SM.GUID]; done {
continue
}
if recs, err := discovery.QuerySMs(ctx, h.Name, p.Num); err != nil {
f.SMListErrors[p.SM.GUID] = err.Error()
} else {
f.SMLists[p.SM.GUID] = recs
}
}
}
for _, m := range watchedModules {
_, err := os.Stat(filepath.Join("/sys/module", m))
f.Modules[m] = err == nil
}
f.readConfig(o)
f.UnitMemlock = strings.TrimPrefix(systemctl(ctx, "show", "llmgw-agent", "-p", "LimitMEMLOCK"), "LimitMEMLOCK=")
for name, bin := range map[string]string{"vllm": "vllm", "ray": "ray", "sglang": "python3"} {
if name == "sglang" {
continue // no binary of its own to ask
}
if p, err := exec.LookPath(bin); err == nil {
out, _ := exec.CommandContext(ctx, p, "--version").CombinedOutput()
f.Engines[name] = firstLine(string(out))
}
}
return f
}
func (f *Facts) readConfig(o Options) {
path := o.ConfigPath
if path == "" {
if _, err := os.Stat("/etc/llmgw/agent.yaml"); err != nil {
return
}
path = "/etc/llmgw/agent.yaml"
}
f.ConfigPath = path
cfg, err := loadAgent(path)
if err != nil {
f.ConfigErr = err.Error()
return
}
declared := map[string][]string{}
for _, fa := range cfg.Fabrics {
declared[fa.Fabric] = fa.HCAs
}
f.Fabrics = discovery.CheckFabrics(declared, f.HCAs)
f.MgmtAddr, f.MgmtIface = agent.ResolveManagement(cfg)
if f.MgmtIface == "" {
f.MgmtIface = "eth0"
}
_, err = net.InterfaceByName(f.MgmtIface)
f.IfaceExists = err == nil
f.LaunchMode = cfg.Engines.VLLM.LaunchMode
if f.LaunchMode == "" {
f.LaunchMode = "mp"
}
// What a two-node shard on each fabric would be given, the part of an
// engine's environment that decides whether NCCL finds its peers.
f.NCCL = map[string]map[string]string{}
for _, c := range f.Fabrics {
if !c.OK() {
continue
}
spec := engines.ShardSpec{NNodes: 2, HCAs: c.Usable, MgmtInterface: f.MgmtIface, GIDIndex: rocev2(f.HCAs, c.Usable)}
env := engines.NCCLEnv(spec)
delete(env, "CUDA_VISIBLE_DEVICES")
f.NCCL[c.Fabric] = env
}
}
// loadAgent is replaced in tests; the real one lives in config, which this
// package cannot import without a cycle through the CLI.
var loadAgent = func(string) (agent.Config, error) {
return agent.Config{}, fmt.Errorf("no agent configuration loader")
}
// SetConfigLoader gives Collect the configuration loader.
func SetConfigLoader(fn func(string) (agent.Config, error)) { loadAgent = fn }
func rocev2(hcas []discovery.HCA, usable []string) *int {
for _, h := range hcas {
if !slices.Contains(usable, h.Name) {
continue
}
for _, p := range h.Ports {
if p.Active() && p.RoCEv2GID >= 0 {
v := p.RoCEv2GID
return &v
}
}
}
return nil
}
func systemctl(ctx context.Context, args ...string) string {
out, err := exec.CommandContext(ctx, "systemctl", args...).Output()
if err != nil && len(out) == 0 {
return "unknown"
}
return strings.TrimSpace(string(out))
}
func firstLine(s string) string {
s = strings.TrimSpace(s)
if i := strings.IndexByte(s, '\n'); i >= 0 {
s = s[:i]
}
return s
}
// Evaluate draws the findings from the facts, most serious first.
func Evaluate(f Facts) []Finding {
var out []Finding
add := func(l Level, area, msg, fix string) {
out = append(out, Finding{Level: l, Area: area, Message: msg, Fix: fix})
}
// --- GPUs ---
switch {
case !f.SMIFound:
add(Info, "gpu", "no nvidia-smi: a CPU node, or the driver is not installed", "")
case f.GPUError != "":
add(Fail, "gpu", "nvidia-smi fails: "+f.GPUError, "check the driver: run nvidia-smi by hand")
case len(f.GPUs) == 0:
add(Warn, "gpu", "nvidia-smi is installed but reports no GPUs", "")
default:
add(OK, "gpu", fmt.Sprintf("%d GPUs: %s", len(f.GPUs), gpuNames(f.GPUs)), "")
for _, g := range f.GPUs {
if !g.Health.OK() {
add(Warn, "gpu", fmt.Sprintf("GPU %d is kept out of placement: %s", g.Index, strings.Join(g.Health.Problems, "; ")), "")
}
}
}
if f.NVSwitch && f.FabricManager != "active" {
add(Fail, "nvlink", "this machine has NVSwitch but nvidia-fabricmanager is "+f.FabricManager+
": CUDA will not initialise on its GPUs",
"systemctl enable --now nvidia-fabricmanager (its version must match the driver's)")
}
if len(f.GPUs) > 1 {
pairs, groups := nvlinkGroups(f.GPUs)
switch {
case f.TopoErr != "":
add(Warn, "nvlink", "could not read nvidia-smi topo -m: "+f.TopoErr+
"; tensor-parallel groups will be chosen without NVLink", "")
case pairs == 0 && f.NVSwitch:
add(Warn, "nvlink", "NVSwitch is present but no GPU reports an NVLink peer", "check nvidia-smi nvlink --status")
case pairs == 0:
add(Info, "nvlink", "no NVLink: GPUs talk over PCIe, and tensor parallelism across them is slow", "")
default:
add(OK, "nvlink", "NVLink groups: "+groups, "")
}
}
// --- multi-node NVLink ---
domain := ""
for _, g := range f.GPUs {
fb := g.Fabric
if fb == nil || strings.EqualFold(fb.State, "Not Supported") {
continue
}
if d := fb.Domain(); d != "" {
domain = d
if fb.Bandwidth != "" && !strings.EqualFold(fb.Bandwidth, "Full") {
add(Warn, "nvlink", fmt.Sprintf("GPU %d's NVLink bandwidth is %s", g.Index, strings.ToLower(fb.Bandwidth)),
"check its NVLink cables and trays: nvidia-smi -q -d FABRIC")
}
continue
}
if !strings.EqualFold(fb.State, "Completed") {
why := fb.State
if fb.Status != "" && !strings.EqualFold(fb.Status, "Success") {
why += ": " + fb.Status
}
add(Fail, "nvlink", fmt.Sprintf("GPU %d is not joined to its NVLink domain (%s)", g.Index, why),
"the fabric manager joins it: check nvidia-fabricmanager and the NVLink switch trays")
}
}
if domain != "" {
add(OK, "nvlink", "the GPUs are in multi-node NVLink domain "+domain, "")
switch {
case f.IMEX.State != "active":
state := f.IMEX.State
if state == "" {
state = "not installed"
}
add(Fail, "nvlink", "nvidia-imex is "+state+": without it the domain's GPUs cannot share memory across nodes",
"systemctl enable --now nvidia-imex, with every node of the domain in /etc/nvidia-imex/nodes_config.cfg")
case f.IMEX.Channels == 0:
add(Fail, "nvlink", "nvidia-imex runs but no IMEX channel exists for an engine to open",
"load the driver with options nvidia NVreg_CreateImexChannel0=1")
default:
add(OK, "nvlink", fmt.Sprintf("nvidia-imex is active with %d channel(s)", f.IMEX.Channels), "")
}
}
// --- InfiniBand and RoCE ---
if len(f.HCAs) == 0 {
add(Info, "fabric", "no InfiniBand or RoCE adapters: this node can only run single-node replicas", "")
}
declared := map[string]string{} // hca -> fabric
for _, c := range f.Fabrics {
for _, h := range c.Declared {
declared[h] = c.Fabric
}
}
for _, h := range f.HCAs {
for _, p := range h.Ports {
where := fmt.Sprintf("%s/%d", h.Name, p.Num)
level := Warn
if declared[h.Name] != "" {
level = Fail
}
switch {
case p.Active() && p.LinkLayer == discovery.LinkEthernet && p.RoCEv2GID < 0:
add(Fail, "fabric", where+" is RoCE with no RoCE v2 GID entry: NCCL would use a v1 entry and hang",
"give the interface an IPv4 address, or check the adapter's RoCE mode")
case p.Active():
on := ""
switch {
case p.SM != nil:
on = fmt.Sprintf(", subnet manager %s (LID %d, %s)", p.SM.GUID, p.SM.LID, p.SM.State)
case p.Network != "":
on = ", network " + p.Network
}
add(OK, "fabric", fmt.Sprintf("%s %s at %d Gb/s (%s)%s", where, p.State, p.RateGbps, p.LinkLayer, on), "")
if p.LinkLayer == discovery.LinkInfiniBand && p.SM == nil {
why := f.SMErrors[where]
fix := ""
if strings.Contains(why, "infiniband-diags") {
fix = "install infiniband-diags"
}
add(Warn, "fabric", where+": which fabric it is on cannot be told — "+orUnknown(why)+
"; without it, separate fabrics given one name cannot be caught", fix)
}
if p.LinkLayer == discovery.LinkInfiniBand && p.SMLID == 0 {
add(Warn, "fabric", where+" is active but reports no subnet manager LID", "")
}
case strings.EqualFold(p.PhysState, "LinkUp"):
add(level, "fabric", where+" has a link but is "+p.State+": no subnet manager has configured it",
"run a subnet manager on the fabric (opensm, or the switch's own)")
default:
add(level, "fabric", fmt.Sprintf("%s is %s (%s): no link", where, p.State, p.PhysState),
"check the cable and the switch port")
}
for _, name := range []string{"symbol_error", "link_downed", "link_error_recovery", "port_rcv_errors", "local_link_integrity_errors"} {
if v := p.Counters[name]; v > 0 {
add(Warn, "fabric", fmt.Sprintf("%s: %s = %d since the adapter came up", where, name, v),
"a count that keeps growing is a cable or transceiver going bad")
}
}
}
}
for _, master := range sortedKeys(f.SMLists) {
recs := f.SMLists[master]
var parts []string
standby := 0
for _, r := range recs {
parts = append(parts, fmt.Sprintf("%s (%s, priority %d)", r.GUID, r.State, r.Priority))
if r.State == "standby" {
standby++
}
}
if standby == 0 {
add(Warn, "fabric", "the fabric of subnet manager "+master+" has no standby: "+strings.Join(parts, ", "),
"run a second subnet manager, on a switch or a host, at a lower priority")
} else {
add(OK, "fabric", "subnet managers of "+master+"'s fabric: "+strings.Join(parts, ", "), "")
}
}
for _, master := range sortedKeys(f.SMListErrors) {
add(Info, "fabric", "the subnet managers of "+master+"'s fabric could not be listed: "+f.SMListErrors[master], "")
}
if len(f.HCAs) > 0 {
if !f.Modules["ib_uverbs"] {
add(Fail, "fabric", "ib_uverbs is not loaded: nothing in user space can use RDMA", "modprobe ib_uverbs")
}
if len(f.GPUs) > 0 && !f.Modules["nvidia_peermem"] && !f.Modules["nv_peer_mem"] {
add(Warn, "fabric", "nvidia_peermem is not loaded: GPUDirect RDMA is off, and NCCL copies every transfer through host memory",
"modprobe nvidia_peermem")
}
}
// --- the agent's view ---
switch {
case f.ConfigErr != "":
add(Fail, "agent", "the agent configuration "+f.ConfigPath+" does not load: "+f.ConfigErr, "")
case f.ConfigPath == "":
add(Info, "agent", "no agent configuration found; declared fabrics were not checked", "pass --config")
default:
for _, c := range f.Fabrics {
if c.OK() {
add(OK, "agent", fmt.Sprintf("fabric %s: %s usable at %d Gb/s", c.Fabric, strings.Join(c.Usable, ","), c.RateGbps), "")
}
for _, p := range c.Problems {
lvl := Warn
if !c.OK() {
lvl = Fail
}
add(lvl, "agent", "fabric "+c.Fabric+": "+p, "")
}
if !c.OK() {
add(Fail, "agent", "fabric "+c.Fabric+" will not be advertised: no multi-node replica will be placed here", "")
}
}
active := activeHCAs(f.HCAs)
if len(f.Fabrics) == 0 && len(active) > 0 {
add(Warn, "agent", "adapters are up ("+strings.Join(active, ", ")+") but agent.yaml declares no fabric: this node will never get multi-node work",
"add fabrics: [{name: <fabric>, hcas: ["+strings.Join(active, ", ")+"]}]")
}
if !f.IfaceExists {
add(Fail, "agent", "the management interface "+f.MgmtIface+" does not exist: NCCL and Gloo would bootstrap over nothing",
"set mgmt_interface in agent.yaml")
} else {
add(OK, "agent", "NCCL bootstraps over "+f.MgmtIface+" ("+f.MgmtAddr+")", "")
}
if len(f.Fabrics) > 0 && f.LaunchMode == "ray" && f.Engines["ray"] == "" && f.Engines["vllm"] != "" {
add(Fail, "agent", "vLLM's launch_mode is ray but ray is not installed", "install ray, or set engines.vllm.launch_mode: mp")
}
}
if f.UnitMemlock != "" && f.UnitMemlock != "unknown" && f.UnitMemlock != "infinity" && len(f.HCAs) > 0 {
add(Warn, "agent", "the llmgw-agent unit limits locked memory to "+f.UnitMemlock+"; RDMA pins what it registers",
"LimitMEMLOCK=infinity in the unit")
}
sort.SliceStable(out, func(i, j int) bool { return rank(out[i].Level) < rank(out[j].Level) })
return out
}
func rank(l Level) int {
switch l {
case Fail:
return 0
case Warn:
return 1
case OK:
return 2
}
return 3
}
func gpuNames(gpus []discovery.GPU) string {
count := map[string]int{}
var order []string
for _, g := range gpus {
if count[g.Name] == 0 {
order = append(order, g.Name)
}
count[g.Name]++
}
parts := make([]string, len(order))
for i, n := range order {
parts[i] = fmt.Sprintf("%d× %s", count[n], n)
}
return strings.Join(parts, ", ")
}
// nvlinkGroups counts NVLink pairs and names the connected groups.
func nvlinkGroups(gpus []discovery.GPU) (int, string) {
parent := map[int]int{}
var find func(int) int
find = func(x int) int {
if p, ok := parent[x]; ok && p != x {
parent[x] = find(p)
return parent[x]
}
parent[x] = x
return x
}
pairs := 0
for _, g := range gpus {
find(g.Index)
for _, p := range g.NVLinkPeers {
if p > g.Index {
pairs++
}
parent[find(g.Index)] = find(p)
}
}
groups := map[int][]int{}
for _, g := range gpus {
r := find(g.Index)
groups[r] = append(groups[r], g.Index)
}
var names []string
for _, members := range groups {
sort.Ints(members)
if len(members) > 1 {
names = append(names, fmt.Sprint(members))
}
}
sort.Strings(names)
return pairs, strings.Join(names, " ")
}
func activeHCAs(hcas []discovery.HCA) []string {
var out []string
for _, h := range hcas {
if len(h.ActivePorts()) > 0 {
out = append(out, h.Name)
}
}
return out
}
func orUnknown(s string) string {
if s == "" {
return "no answer from the subnet manager"
}
return s
}
func sortedKeys[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}