Files
llmgw/internal/doctor/doctor.go
T
Ilker ManapandClaude Opus 5.5 f2b2e24e1f A node's own readiness and its multi-node readiness are kept apart
doctor sums findings up as two readinesses - the node's, which decides
whether it joins, and multi-node, its fabrics for replicas spanning nodes -
prints fabric findings in a section of their own, and gives both in --json.
node add says the multi-node one. NodeView.multi_node says whether a node's
declared fabrics are up, naming adapters that are down, and ctl nodes shows
it beside the fabric.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-07 13:11:16 +02:00

864 lines
29 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Package doctor checks a node for what llmgw needs of its GPUs and fabric,
// and says what is wrong in words an operator can act on.
//
// Written for a machine nobody here can log into. The report is meant to be
// run on the node, read there, and sent back when it is not enough: every
// fact it concluded from is in it, so the conclusion can be checked.
package doctor
import (
"context"
"errors"
"fmt"
"maps"
"net"
"os"
"os/exec"
"path/filepath"
"slices"
"sort"
"strconv"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
)
// Level is how much a finding matters.
type Level string
// The levels, most serious first.
const (
Fail Level = "FAIL"
Warn Level = "WARN"
OK Level = "OK"
Info Level = "INFO"
)
// Finding is one conclusion, and what to do about it.
type Finding struct {
Level Level `json:"level"`
Area string `json:"area"`
Message string `json:"message"`
Fix string `json:"fix,omitempty"`
}
// Facts are what was read from the node. Every finding is drawn from these
// and nothing else, so a report sent back can be re-read here.
type Facts struct {
At time.Time `json:"at"`
Hostname string `json:"hostname"`
Host discovery.Host `json:"host"`
SMIFound bool `json:"nvidia_smi_found"`
GPUs []discovery.GPU `json:"gpus"`
GPUError string `json:"gpu_error,omitempty"`
TopoRaw string `json:"topo_raw,omitempty"`
TopoErr string `json:"topo_error,omitempty"`
// MIG is each MIG-capable card's mode and instances; MIGErr why it could
// not be read.
MIG []discovery.MIGCard `json:"mig,omitempty"`
MIGErr string `json:"mig_error,omitempty"`
// NVSwitch is set when the machine has NVSwitch devices, which need the
// fabric manager running before CUDA will start.
NVSwitch bool `json:"nvswitch"`
// IMEX is the service multi-node NVLink needs.
IMEX discovery.IMEX `json:"imex"`
FabricManager string `json:"fabric_manager,omitempty"`
HCAs []discovery.HCA `json:"hcas"`
// SMErrors say, per port, why the subnet manager could not be asked.
SMErrors map[string]string `json:"sm_errors,omitempty"`
// SMLists are every subnet manager on each fabric, by master GUID;
// SMListErrors why a fabric's could not be listed.
SMLists map[string][]discovery.SMRecord `json:"sm_lists,omitempty"`
SMListErrors map[string]string `json:"sm_list_errors,omitempty"`
Modules map[string]bool `json:"modules"`
// ConfigPath is the agent configuration read, empty when there was none.
ConfigPath string `json:"config_path,omitempty"`
ConfigErr string `json:"config_error,omitempty"`
Fabrics []discovery.FabricCheck `json:"fabrics,omitempty"`
MgmtAddr string `json:"mgmt_addr,omitempty"`
MgmtIface string `json:"mgmt_interface,omitempty"`
IfaceExists bool `json:"mgmt_interface_exists"`
LaunchMode string `json:"vllm_launch_mode,omitempty"`
NCCL map[string]map[string]string `json:"nccl_env,omitempty"`
UnitMemlock string `json:"agent_unit_memlock,omitempty"`
Engines map[string]string `json:"engines"`
// What installing vLLM here would meet: the version this llmgw installs,
// where, the space there, and whether it already has.
VLLMVersion string `json:"vllm_version,omitempty"`
// DisabledEngines are those agent.yaml says this node never runs.
DisabledEngines []string `json:"disabled_engines,omitempty"`
EnginesDir string `json:"engines_dir,omitempty"`
EnginesFreeGB float64 `json:"engines_free_gb,omitempty"`
VLLMInstalled bool `json:"vllm_installed"`
// Firewall is whether llmgw's nftables table is loaded: "present",
// "absent", or "" when it could not be read (not root).
Firewall string `json:"firewall,omitempty"`
}
// Modules checked, and why each matters.
var watchedModules = []string{"ib_core", "ib_uverbs", "mlx5_ib", "rdma_ucm", "nvidia_peermem", "nv_peer_mem"}
// Options says where to look.
type Options struct {
ConfigPath string
IBRoot string
NvidiaSMI string
// VLLMVersion is the vLLM this llmgw installs; its prerequisites are
// checked when set.
VLLMVersion string
// Fabrics are fabrics about to be declared, by name, with their
// adapters: what `node add --fabric` will write into agent.yaml, which
// does not exist yet when doctor runs before it.
Fabrics map[string][]string
}
// Collect reads the node.
func Collect(ctx context.Context, o Options) Facts {
f := Facts{At: time.Now().UTC(), Modules: map[string]bool{}, Engines: map[string]string{}}
f.Hostname, _ = os.Hostname()
f.Host = discovery.DiscoverHost()
smi := o.NvidiaSMI
if smi == "" {
if p, err := exec.LookPath("nvidia-smi"); err == nil {
smi = p
}
}
if smi != "" {
f.SMIFound = true
gpus, err := discovery.DiscoverGPUsSMI(ctx, smi)
if err != nil {
f.GPUError = err.Error()
}
out, err := exec.CommandContext(ctx, smi, "topo", "-m").Output()
if err != nil {
f.TopoErr = err.Error()
} else {
f.TopoRaw = string(out)
if t, err := discovery.ParseTopology(f.TopoRaw); err == nil {
t.Apply(gpus)
} else {
f.TopoErr = err.Error()
}
}
if fab, err := discovery.DiscoverGPUFabric(ctx, smi); err == nil {
discovery.ApplyFabric(gpus, fab)
}
f.GPUs = gpus
f.IMEX = discovery.DiscoverIMEX(ctx)
if slices.ContainsFunc(gpus, func(g discovery.GPU) bool {
cc, _ := strconv.ParseFloat(g.ComputeCapability, 64)
return g.IsMIG() || cc >= 8
}) {
if mig, err := discovery.DiscoverMIG(ctx, smi); err != nil {
f.MIGErr = err.Error()
} else {
f.MIG = mig
}
}
}
if m, _ := filepath.Glob("/dev/nvidia-nvswitch*"); len(m) > 0 {
f.NVSwitch = true
f.FabricManager = systemctl(ctx, "is-active", "nvidia-fabricmanager")
}
root := o.IBRoot
if root == "" {
root = discovery.SysfsRoot
}
f.HCAs, _ = discovery.DiscoverIB(root)
f.SMErrors = map[string]string{}
_ = discovery.AttachSM(f.HCAs, func(hca string, port int) (*discovery.SMInfo, error) {
sm, err := discovery.QuerySM(ctx, hca, port)
if err != nil {
f.SMErrors[fmt.Sprintf("%s/%d", hca, port)] = err.Error()
}
return sm, err
})
f.SMLists, f.SMListErrors = map[string][]discovery.SMRecord{}, map[string]string{}
for _, h := range f.HCAs {
for _, p := range h.Ports {
if p.SM == nil {
continue
}
if _, done := f.SMLists[p.SM.GUID]; done {
continue
}
if _, done := f.SMListErrors[p.SM.GUID]; done {
continue
}
if recs, err := discovery.QuerySMs(ctx, h.Name, p.Num); err != nil {
f.SMListErrors[p.SM.GUID] = err.Error()
} else {
f.SMLists[p.SM.GUID] = recs
}
}
}
for _, m := range watchedModules {
_, err := os.Stat(filepath.Join("/sys/module", m))
f.Modules[m] = err == nil
}
f.readConfig(o)
if len(o.Fabrics) > 0 {
declared := map[string][]string{}
for _, c := range f.Fabrics {
declared[c.Fabric] = c.Declared
}
maps.Copy(declared, o.Fabrics)
f.Fabrics = discovery.CheckFabrics(declared, f.HCAs)
}
f.Host.MinComputeCapability = discovery.MinComputeCapability(f.GPUs)
if f.VLLMVersion = o.VLLMVersion; f.VLLMVersion != "" {
if f.EnginesDir == "" {
f.EnginesDir = provision.DefaultRoot
}
// The directory may not exist yet: the space is the filesystem's.
for dir := f.EnginesDir; dir != "/" && dir != "."; dir = filepath.Dir(dir) {
if gb, err := provision.FreeGB(dir); err == nil {
f.EnginesFreeGB = gb
break
}
}
matches, _ := filepath.Glob(filepath.Join(f.EnginesDir, "vllm", f.VLLMVersion+"-*", ".llmgw-installed"))
f.VLLMInstalled = len(matches) > 0
}
f.UnitMemlock = strings.TrimPrefix(systemctl(ctx, "show", "llmgw-agent", "-p", "LimitMEMLOCK"), "LimitMEMLOCK=")
for name, bin := range map[string]string{"vllm": "vllm", "ray": "ray", "sglang": "python3"} {
if name == "sglang" {
continue // no binary of its own to ask
}
if p, err := exec.LookPath(bin); err == nil {
out, _ := exec.CommandContext(ctx, p, "--version").CombinedOutput()
f.Engines[name] = firstLine(string(out))
}
}
f.Firewall = firewallState(ctx)
return f
}
// firewallState reads whether llmgw's table is loaded.
func firewallState(ctx context.Context) string {
out, err := exec.CommandContext(ctx, "nft", "list", "table", "inet", "llmgw").CombinedOutput()
switch {
case err == nil:
return "present"
case strings.Contains(string(out), "No such file or directory"):
return "absent"
}
return "" // no nft, or not root: unknown
}
func (f *Facts) readConfig(o Options) {
path := o.ConfigPath
if path == "" {
if _, err := os.Stat("/etc/llmgw/agent.yaml"); err != nil {
return
}
path = "/etc/llmgw/agent.yaml"
}
f.ConfigPath = path
cfg, err := loadAgent(path)
if err != nil {
f.ConfigErr = err.Error()
return
}
declared := map[string][]string{}
for _, fa := range cfg.Fabrics {
declared[fa.Fabric] = fa.HCAs
}
f.Fabrics = discovery.CheckFabrics(declared, f.HCAs)
f.MgmtAddr, f.MgmtIface = agent.ResolveManagement(cfg)
if f.MgmtIface == "" {
f.MgmtIface = "eth0"
}
_, err = net.InterfaceByName(f.MgmtIface)
f.IfaceExists = err == nil
f.EnginesDir = cfg.EnginesDir
f.DisabledEngines = cfg.Engines.Disabled
f.LaunchMode = cfg.Engines.VLLM.LaunchMode
if f.LaunchMode == "" {
f.LaunchMode = "mp"
}
// What a two-node shard on each fabric would be given, the part of an
// engine's environment that decides whether NCCL finds its peers.
f.NCCL = map[string]map[string]string{}
for _, c := range f.Fabrics {
if !c.OK() {
continue
}
spec := engines.ShardSpec{NNodes: 2, HCAs: c.Usable, MgmtInterface: f.MgmtIface, GIDIndex: rocev2(f.HCAs, c.Usable)}
env := engines.NCCLEnv(spec)
delete(env, "CUDA_VISIBLE_DEVICES")
f.NCCL[c.Fabric] = env
}
}
// loadAgent is replaced in tests; the real one lives in config, which this
// package cannot import without a cycle through the CLI.
var loadAgent = func(string) (agent.Config, error) {
return agent.Config{}, fmt.Errorf("no agent configuration loader")
}
// SetConfigLoader gives Collect the configuration loader.
func SetConfigLoader(fn func(string) (agent.Config, error)) { loadAgent = fn }
func rocev2(hcas []discovery.HCA, usable []string) *int {
for _, h := range hcas {
if !slices.Contains(usable, h.Name) {
continue
}
for _, p := range h.Ports {
if p.Active() && p.RoCEv2GID >= 0 {
v := p.RoCEv2GID
return &v
}
}
}
return nil
}
func systemctl(ctx context.Context, args ...string) string {
out, err := exec.CommandContext(ctx, "systemctl", args...).Output()
if err != nil && len(out) == 0 {
return "unknown"
}
return strings.TrimSpace(string(out))
}
func firstLine(s string) string {
s = strings.TrimSpace(s)
if i := strings.IndexByte(s, '\n'); i >= 0 {
s = s[:i]
}
return s
}
// Evaluate draws the findings from the facts, most serious first.
func Evaluate(f Facts) []Finding {
var out []Finding
add := func(l Level, area, msg, fix string) {
out = append(out, Finding{Level: l, Area: area, Message: msg, Fix: fix})
}
// --- CPU ---
// Read from /proc/cpuinfo, so only on a node whose OS was read too;
// elsewhere an empty list means unknown, not missing.
if f.Host.Arch == "amd64" && f.Host.OSID != "" {
if f.Host.HasCPUFlag("avx2") {
add(OK, "cpu", "vector extensions: "+strings.Join(f.Host.CPUFlags, ", "), "")
} else {
has := "none of AVX, AVX2 or FMA"
if len(f.Host.CPUFlags) > 0 {
has = "only " + strings.Join(f.Host.CPUFlags, ", ")
}
// Nothing else looks wrong on such a node: it registers, answers
// correctly, and is several times slower than the hardware.
add(Warn, "cpu", "this CPU offers no AVX2 ("+has+"): llama.cpp runs several times slower on it",
"on a VM, give it the host's CPU type (Proxmox: qm set <vmid> --cpu host, then restart the VM)")
}
}
// --- GPUs ---
switch {
case !f.SMIFound:
add(Info, "gpu", "no nvidia-smi: a CPU node, or the driver is not installed", "")
case f.GPUError != "":
add(Fail, "gpu", "nvidia-smi fails: "+f.GPUError, "check the driver: run nvidia-smi by hand")
case len(f.GPUs) == 0:
add(Warn, "gpu", "nvidia-smi is installed but reports no GPUs", "")
default:
add(OK, "gpu", fmt.Sprintf("%d GPUs: %s", len(f.GPUs), gpuNames(f.GPUs)), "")
for _, g := range f.GPUs {
if !g.Health.OK() {
add(Warn, "gpu", fmt.Sprintf("GPU %s is kept out of placement: %s",
discovery.DeviceLabel(g.Index), strings.Join(g.Health.Problems, "; ")), "")
}
}
}
evaluateMIG(f, add)
if f.NVSwitch && f.FabricManager != "active" {
add(Fail, "nvlink", "this machine has NVSwitch but nvidia-fabricmanager is "+f.FabricManager+
": CUDA will not initialise on its GPUs",
"systemctl enable --now nvidia-fabricmanager (its version must match the driver's)")
}
// NVLink is between whole cards: a MIG instance has none to report, and
// counting instances would call a node of MIG cards one without NVLink.
cards := slices.DeleteFunc(slices.Clone(f.GPUs), discovery.GPU.IsMIG)
if len(cards) > 1 {
pairs, groups := nvlinkGroups(cards)
switch {
case f.TopoErr != "":
add(Warn, "nvlink", "could not read nvidia-smi topo -m: "+f.TopoErr+
"; tensor-parallel groups will be chosen without NVLink", "")
case pairs == 0 && f.NVSwitch:
add(Warn, "nvlink", "NVSwitch is present but no GPU reports an NVLink peer", "check nvidia-smi nvlink --status")
case pairs == 0:
add(Info, "nvlink", "no NVLink: GPUs talk over PCIe, and tensor parallelism across them is slow", "")
default:
add(OK, "nvlink", "NVLink groups: "+groups, "")
}
}
// --- multi-node NVLink ---
domain := ""
for _, g := range cards {
fb := g.Fabric
if fb == nil || strings.EqualFold(fb.State, "Not Supported") {
continue
}
if d := fb.Domain(); d != "" {
domain = d
if fb.Bandwidth != "" && !strings.EqualFold(fb.Bandwidth, "Full") {
add(Warn, "nvlink", fmt.Sprintf("GPU %d's NVLink bandwidth is %s", g.Index, strings.ToLower(fb.Bandwidth)),
"check its NVLink cables and trays: nvidia-smi -q -d FABRIC")
}
continue
}
if !strings.EqualFold(fb.State, "Completed") {
why := fb.State
if fb.Status != "" && !strings.EqualFold(fb.Status, "Success") {
why += ": " + fb.Status
}
add(Fail, "nvlink", fmt.Sprintf("GPU %d is not joined to its NVLink domain (%s)", g.Index, why),
"the fabric manager joins it: check nvidia-fabricmanager and the NVLink switch trays")
}
}
if domain != "" {
add(OK, "nvlink", "the GPUs are in multi-node NVLink domain "+domain, "")
switch {
case f.IMEX.State != "active":
state := f.IMEX.State
if state == "" {
state = "not installed"
}
add(Fail, "nvlink", "nvidia-imex is "+state+": without it the domain's GPUs cannot share memory across nodes",
"systemctl enable --now nvidia-imex, with every node of the domain in /etc/nvidia-imex/nodes_config.cfg")
case f.IMEX.Channels == 0:
add(Fail, "nvlink", "nvidia-imex runs but no IMEX channel exists for an engine to open",
"load the driver with options nvidia NVreg_CreateImexChannel0=1")
default:
add(OK, "nvlink", fmt.Sprintf("nvidia-imex is active with %d channel(s)", f.IMEX.Channels), "")
}
}
// --- InfiniBand and RoCE ---
if len(f.HCAs) == 0 {
add(Info, "fabric", "no InfiniBand or RoCE adapters: this node can only run single-node replicas", "")
}
declared := map[string]string{} // hca -> fabric
for _, c := range f.Fabrics {
for _, h := range c.Declared {
declared[h] = c.Fabric
}
}
for _, h := range f.HCAs {
for _, p := range h.Ports {
where := fmt.Sprintf("%s/%d", h.Name, p.Num)
level := Warn
if declared[h.Name] != "" {
level = Fail
}
switch {
case p.Active() && p.LinkLayer == discovery.LinkEthernet && p.RoCEv2GID < 0:
// Fatal only for a port a declared fabric uses. An adapter
// nothing is configured to use — a RoCE NIC on a node added
// for single-node inference — kept nodes out for a fabric
// nobody asked for.
msg := where + " is RoCE with no RoCE v2 GID entry: NCCL would use a v1 entry and hang"
if declared[h.Name] == "" {
msg += "; no declared fabric uses it, so nothing will until one does"
}
add(level, "fabric", msg, "give the interface an IPv4 address, or check the adapter's RoCE mode")
case p.Active():
on := ""
switch {
case p.SM != nil:
on = fmt.Sprintf(", subnet manager %s (LID %d, %s)", p.SM.GUID, p.SM.LID, p.SM.State)
case p.Network != "":
on = ", network " + p.Network
}
add(OK, "fabric", fmt.Sprintf("%s %s at %d Gb/s (%s)%s", where, p.State, p.RateGbps, p.LinkLayer, on), "")
// Questions about which fabric a port is on matter to the
// fabrics that use it; for a port none does they are noted,
// not warned of.
quiet := Warn
if declared[h.Name] == "" {
quiet = Info
}
if p.LinkLayer == discovery.LinkInfiniBand && p.SM == nil {
why := f.SMErrors[where]
fix := ""
if strings.Contains(why, "infiniband-diags") {
fix = "install infiniband-diags"
}
add(quiet, "fabric", where+": which fabric it is on cannot be told — "+orUnknown(why)+
"; without it, separate fabrics given one name cannot be caught", fix)
}
if p.LinkLayer == discovery.LinkInfiniBand && p.SMLID == 0 {
add(quiet, "fabric", where+" is active but reports no subnet manager LID", "")
}
case strings.EqualFold(p.PhysState, "LinkUp"):
add(level, "fabric", where+" has a link but is "+p.State+": no subnet manager has configured it",
"run a subnet manager on the fabric (opensm, or the switch's own)")
default:
add(level, "fabric", fmt.Sprintf("%s is %s (%s): no link", where, p.State, p.PhysState),
"check the cable and the switch port")
}
for _, name := range []string{"symbol_error", "link_downed", "link_error_recovery", "port_rcv_errors", "local_link_integrity_errors"} {
if v := p.Counters[name]; v > 0 {
add(Warn, "fabric", fmt.Sprintf("%s: %s = %d since the adapter came up", where, name, v),
"a count that keeps growing is a cable or transceiver going bad")
}
}
}
}
for _, master := range sortedKeys(f.SMLists) {
recs := f.SMLists[master]
var parts []string
standby := 0
for _, r := range recs {
parts = append(parts, fmt.Sprintf("%s (%s, priority %d)", r.GUID, r.State, r.Priority))
if r.State == "standby" {
standby++
}
}
if standby == 0 {
add(Warn, "fabric", "the fabric of subnet manager "+master+" has no standby: "+strings.Join(parts, ", "),
"run a second subnet manager, on a switch or a host, at a lower priority")
} else {
add(OK, "fabric", "subnet managers of "+master+"'s fabric: "+strings.Join(parts, ", "), "")
}
}
for _, master := range sortedKeys(f.SMListErrors) {
add(Info, "fabric", "the subnet managers of "+master+"'s fabric could not be listed: "+f.SMListErrors[master], "")
}
if len(f.HCAs) > 0 {
if !f.Modules["ib_uverbs"] {
// RDMA is for replicas spanning nodes; a node that declares no
// fabric runs none, and is a node without it.
if len(declared) > 0 {
add(Fail, "fabric", "ib_uverbs is not loaded: nothing in user space can use RDMA", "modprobe ib_uverbs")
} else {
add(Warn, "fabric", "ib_uverbs is not loaded: nothing in user space can use RDMA, which a fabric declared "+
"later would need; this node runs single-node replicas without it", "modprobe ib_uverbs")
}
}
if len(f.GPUs) > 0 && !f.Modules["nvidia_peermem"] && !f.Modules["nv_peer_mem"] {
add(Warn, "fabric", "nvidia_peermem is not loaded: GPUDirect RDMA is off, and NCCL copies every transfer through host memory",
"modprobe nvidia_peermem")
}
}
// --- the agent's view ---
switch {
case f.ConfigErr != "":
add(Fail, "agent", "the agent configuration "+f.ConfigPath+" does not load: "+f.ConfigErr, "")
case f.ConfigPath == "":
add(Info, "agent", "no agent configuration found; declared fabrics were not checked", "pass --config")
default:
for _, c := range f.Fabrics {
if c.OK() {
add(OK, "agent", fmt.Sprintf("fabric %s: %s usable at %d Gb/s", c.Fabric, strings.Join(c.Usable, ","), c.RateGbps), "")
}
for _, p := range c.Problems {
lvl := Warn
if !c.OK() {
lvl = Fail
}
add(lvl, "agent", "fabric "+c.Fabric+": "+p, "")
}
if !c.OK() {
add(Fail, "agent", "fabric "+c.Fabric+" will not be advertised: no multi-node replica will be placed here", "")
}
}
active := activeHCAs(f.HCAs)
if len(f.Fabrics) == 0 && len(active) > 0 {
add(Warn, "agent", "adapters are up ("+strings.Join(active, ", ")+") but agent.yaml declares no fabric: this node will never get multi-node work",
"add fabrics: [{name: <fabric>, hcas: ["+strings.Join(active, ", ")+"]}]")
}
if !f.IfaceExists {
add(Fail, "agent", "the management interface "+f.MgmtIface+" does not exist: NCCL and Gloo would bootstrap over nothing",
"set mgmt_interface in agent.yaml")
} else {
add(OK, "agent", "NCCL bootstraps over "+f.MgmtIface+" ("+f.MgmtAddr+")", "")
}
if len(f.Fabrics) > 0 && f.LaunchMode == "ray" && f.Engines["ray"] == "" && f.Engines["vllm"] != "" {
add(Fail, "agent", "vLLM's launch_mode is ray but ray is not installed", "install ray, or set engines.vllm.launch_mode: mp")
}
}
// --- engines ---
// An engine disabled on this node: whether it could be installed is not
// a question anybody is asking.
if f.VLLMVersion != "" && !slices.Contains(f.DisabledEngines, "vllm") {
vllmFindings(f, add)
}
if f.UnitMemlock != "" && f.UnitMemlock != "unknown" && f.UnitMemlock != "infinity" && len(f.HCAs) > 0 {
add(Warn, "agent", "the llmgw-agent unit limits locked memory to "+f.UnitMemlock+"; RDMA pins what it registers",
"LimitMEMLOCK=infinity in the unit")
}
// --- firewall ---
// Only on a node already set up: before that, onboarding writes it.
if f.ConfigPath != "" {
switch f.Firewall {
case "present":
add(OK, "firewall", "llmgw's ports answer the cluster's networks only", "")
case "absent":
add(Warn, "firewall", "no llmgw firewall: the engines' ports answer every network this machine is on",
"run node add for it again, or list extra networks in firewall_networks first")
}
}
sort.SliceStable(out, func(i, j int) bool { return rank(out[i].Level) < rank(out[j].Level) })
return out
}
// vllmFindings says whether the agent can install vLLM here, and if not why:
// the same decision the agent makes, so the report and the node agree.
func vllmFindings(f Facts, add func(Level, string, string, string)) {
v := f.VLLMVersion
builds, err := provision.Recipes["vllm"].Candidates(v, "", f.Host)
var b provision.Build
if err == nil {
// The first the agent would try; it moves to the next by itself if
// that one does not work here.
b = builds[0]
if b.Version != "" {
v = b.Version
}
}
var nh provision.NotHere
switch {
case errors.As(err, &nh):
add(Info, "engines", "vLLM will not be installed: "+nh.Reason, "")
return
case err != nil:
// A warning, not a failure: the node is still a node, and serves
// llama.cpp. Refusing it here kept older GPU generations out of a
// cluster for an engine nobody meant to run on them.
add(Warn, "engines", "vLLM "+v+" will not run here, and the node joins without it: "+err.Error(),
"add it with --disable-engine vllm if it is meant for llama.cpp alone")
return
case f.VLLMInstalled:
add(OK, "engines", "vLLM "+v+" is installed under "+f.EnginesDir, "")
return
}
if f.EnginesFreeGB > 0 && f.EnginesFreeGB < float64(b.FreeGB) {
add(Warn, "engines", fmt.Sprintf("vLLM %s needs about %d GB free under %s and there are %.1f GB; the node joins without it until there is",
v, b.FreeGB, f.EnginesDir, f.EnginesFreeGB),
"free space there, or point engines_dir in agent.yaml at a larger disk")
return
}
note := ""
if b.Note != "" {
note = "; " + b.Note
}
add(OK, "engines", fmt.Sprintf("vLLM %s (%s) can be installed here: driver %s, %.1f GB free under %s%s",
v, b.Variant, f.Host.NVIDIADriver, f.EnginesFreeGB, f.EnginesDir, note), "")
}
func rank(l Level) int {
switch l {
case Fail:
return 0
case Warn:
return 1
case OK:
return 2
}
return 3
}
// evaluateMIG reports each MIG-capable card's mode and instances. Quiet for a
// card with MIG off, which is every card on most machines; what it warns of
// are the states in which a card is in MIG mode and nothing can run on it.
func evaluateMIG(f Facts, add func(Level, string, string, string)) {
if f.MIGErr != "" {
add(Warn, "mig", "could not read the cards' MIG state: "+f.MIGErr, "run nvidia-smi -q by hand")
return
}
for _, c := range f.MIG {
card := fmt.Sprintf("GPU %d (%s)", c.Index, c.Name)
if c.Enabled != c.PendingEnabled {
want := "off"
if c.PendingEnabled {
want = "on"
}
add(Warn, "mig", fmt.Sprintf("%s has MIG turning %s at its next reset, and until then runs as before", card, want),
fmt.Sprintf("reset the card when nothing runs on it: nvidia-smi -i %d -r (or reboot)", c.Index))
}
if !c.Enabled {
continue
}
// A GPU instance with no compute instance is carved but not usable:
// it is not a device until one is made inside it.
var bare []string
for _, gi := range c.GPUInstances {
has := slices.ContainsFunc(c.ComputeInstances, func(ci discovery.MIGInstance) bool {
return ci.GPUInstance == gi.GPUInstance
})
if !has {
bare = append(bare, fmt.Sprintf("%d (%s)", gi.GPUInstance, gi.Profile))
}
}
switch {
case len(c.Instances) == 0 && len(c.GPUInstances) == 0:
add(Warn, "mig", card+" is in MIG mode with no instances created: nothing can run on it",
fmt.Sprintf("create instances, for example nvidia-smi mig -i %d -cgi 3g.40gb,3g.40gb -C "+
"(nvidia-smi mig -i %d -lgip lists the profiles), or turn MIG off: nvidia-smi -i %d -mig 0",
c.Index, c.Index, c.Index))
case len(c.Instances) == 0:
add(Warn, "mig", fmt.Sprintf("%s is in MIG mode with GPU instances %s but no compute instances: nothing can run on it",
card, strings.Join(bare, ", ")),
fmt.Sprintf("create them: nvidia-smi mig -i %d -cci", c.Index))
default:
var names []string
for _, in := range c.Instances {
names = append(names, fmt.Sprintf("%d: %s %s", in.Device, in.Profile, in.UUID))
}
add(OK, "mig", fmt.Sprintf("%s is in MIG mode with %d instance(s), each a device of its own: %s",
card, len(c.Instances), strings.Join(names, "; ")), "")
if len(bare) > 0 {
add(Warn, "mig", fmt.Sprintf("%s has GPU instances %s with no compute instance, which are not devices yet",
card, strings.Join(bare, ", ")),
fmt.Sprintf("create them: nvidia-smi mig -i %d -cci, or destroy the GPU instances", c.Index))
}
}
}
}
func gpuNames(gpus []discovery.GPU) string {
count := map[string]int{}
var order []string
for _, g := range gpus {
if count[g.Name] == 0 {
order = append(order, g.Name)
}
count[g.Name]++
}
parts := make([]string, len(order))
for i, n := range order {
parts[i] = fmt.Sprintf("%d× %s", count[n], n)
}
return strings.Join(parts, ", ")
}
// nvlinkGroups counts NVLink pairs and names the connected groups.
func nvlinkGroups(gpus []discovery.GPU) (int, string) {
parent := map[int]int{}
var find func(int) int
find = func(x int) int {
if p, ok := parent[x]; ok && p != x {
parent[x] = find(p)
return parent[x]
}
parent[x] = x
return x
}
pairs := 0
for _, g := range gpus {
find(g.Index)
for _, p := range g.NVLinkPeers {
if p > g.Index {
pairs++
}
parent[find(g.Index)] = find(p)
}
}
groups := map[int][]int{}
for _, g := range gpus {
r := find(g.Index)
groups[r] = append(groups[r], g.Index)
}
var names []string
for _, members := range groups {
sort.Ints(members)
if len(members) > 1 {
names = append(names, fmt.Sprint(members))
}
}
sort.Strings(names)
return pairs, strings.Join(names, " ")
}
func activeHCAs(hcas []discovery.HCA) []string {
var out []string
for _, h := range hcas {
if len(h.ActivePorts()) > 0 {
out = append(out, h.Name)
}
}
return out
}
func orUnknown(s string) string {
if s == "" {
return "no answer from the subnet manager"
}
return s
}
func sortedKeys[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
// MultiNode is whether a finding is about the fabrics replicas spanning
// nodes use, rather than about the node itself.
func (f Finding) MultiNode() bool { return f.Area == "fabric" }
// Readiness is a node's two readinesses, kept apart: whether it can join and
// serve replicas of its own, and whether it can also take part in replicas
// spanning nodes over a fabric. The second never keeps a node out unless a
// fabric was declared for it, whose problems are then the node's.
type Readiness struct {
// Node is "ready", or how many problems keep it out.
Node string `json:"node"`
// MultiNode is "ready", "not used" (no fabric adapter), or "not ready:"
// and the first reason.
MultiNode string `json:"multi_node"`
}
// ReadinessOf sums findings up as the two readinesses.
func ReadinessOf(fs []Finding) Readiness {
r := Readiness{Node: "ready", MultiNode: "not used"}
fails := 0
for _, f := range fs {
if f.Level == Fail {
fails++
}
}
if fails > 0 {
r.Node = fmt.Sprintf("%d problem(s) keep it out", fails)
}
for _, f := range fs {
if !f.MultiNode() {
continue
}
if r.MultiNode == "not used" {
r.MultiNode = "ready"
}
if (f.Level == Fail || f.Level == Warn) && !strings.HasPrefix(r.MultiNode, "not ready") {
r.MultiNode = "not ready: " + f.Message
}
}
return r
}