Files
llmgw/internal/doctor/doctor.go
T
Ilker ManapandClaude Opus 5.5 dc483d8c68 Choose each engine's build from the node's hardware, for every GPU generation
Each engine has a list of builds, best first, from what its project
publishes and how each was compiled: llama.cpp CUDA 13.3, CUDA 12.8
(driver 570 for the GPUs it ships as PTX), then Vulkan, never the CPU
build on an NVIDIA node; vLLM CUDA 13, then CUDA 12.9, and for GPUs
older than every build of the version asked for, the newest built for
them (V100: 0.19.1). The node takes the first its hardware meets and
that installs and computes on the GPU, moving on when one does not, and
reports the version it runs. GPU processes are read from nvidia-smi's
full report, so Vulkan engines count as on their GPU.

Refs #2.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-01 14:35:58 +02:00

715 lines
24 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Package doctor checks a node for what llmgw needs of its GPUs and fabric,
// and says what is wrong in words an operator can act on.
//
// Written for a machine nobody here can log into. The report is meant to be
// run on the node, read there, and sent back when it is not enough: every
// fact it concluded from is in it, so the conclusion can be checked.
package doctor
import (
"context"
"errors"
"fmt"
"maps"
"net"
"os"
"os/exec"
"path/filepath"
"slices"
"sort"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
)
// Level is how much a finding matters.
type Level string
// The levels, most serious first.
const (
Fail Level = "FAIL"
Warn Level = "WARN"
OK Level = "OK"
Info Level = "INFO"
)
// Finding is one conclusion, and what to do about it.
type Finding struct {
Level Level `json:"level"`
Area string `json:"area"`
Message string `json:"message"`
Fix string `json:"fix,omitempty"`
}
// Facts are what was read from the node. Every finding is drawn from these
// and nothing else, so a report sent back can be re-read here.
type Facts struct {
At time.Time `json:"at"`
Hostname string `json:"hostname"`
Host discovery.Host `json:"host"`
SMIFound bool `json:"nvidia_smi_found"`
GPUs []discovery.GPU `json:"gpus"`
GPUError string `json:"gpu_error,omitempty"`
TopoRaw string `json:"topo_raw,omitempty"`
TopoErr string `json:"topo_error,omitempty"`
// NVSwitch is set when the machine has NVSwitch devices, which need the
// fabric manager running before CUDA will start.
NVSwitch bool `json:"nvswitch"`
// IMEX is the service multi-node NVLink needs.
IMEX discovery.IMEX `json:"imex"`
FabricManager string `json:"fabric_manager,omitempty"`
HCAs []discovery.HCA `json:"hcas"`
// SMErrors say, per port, why the subnet manager could not be asked.
SMErrors map[string]string `json:"sm_errors,omitempty"`
// SMLists are every subnet manager on each fabric, by master GUID;
// SMListErrors why a fabric's could not be listed.
SMLists map[string][]discovery.SMRecord `json:"sm_lists,omitempty"`
SMListErrors map[string]string `json:"sm_list_errors,omitempty"`
Modules map[string]bool `json:"modules"`
// ConfigPath is the agent configuration read, empty when there was none.
ConfigPath string `json:"config_path,omitempty"`
ConfigErr string `json:"config_error,omitempty"`
Fabrics []discovery.FabricCheck `json:"fabrics,omitempty"`
MgmtAddr string `json:"mgmt_addr,omitempty"`
MgmtIface string `json:"mgmt_interface,omitempty"`
IfaceExists bool `json:"mgmt_interface_exists"`
LaunchMode string `json:"vllm_launch_mode,omitempty"`
NCCL map[string]map[string]string `json:"nccl_env,omitempty"`
UnitMemlock string `json:"agent_unit_memlock,omitempty"`
Engines map[string]string `json:"engines"`
// What installing vLLM here would meet: the version this llmgw installs,
// where, the space there, and whether it already has.
VLLMVersion string `json:"vllm_version,omitempty"`
// DisabledEngines are those agent.yaml says this node never runs.
DisabledEngines []string `json:"disabled_engines,omitempty"`
EnginesDir string `json:"engines_dir,omitempty"`
EnginesFreeGB float64 `json:"engines_free_gb,omitempty"`
VLLMInstalled bool `json:"vllm_installed"`
}
// Modules checked, and why each matters.
var watchedModules = []string{"ib_core", "ib_uverbs", "mlx5_ib", "rdma_ucm", "nvidia_peermem", "nv_peer_mem"}
// Options says where to look.
type Options struct {
ConfigPath string
IBRoot string
NvidiaSMI string
// VLLMVersion is the vLLM this llmgw installs; its prerequisites are
// checked when set.
VLLMVersion string
// Fabrics are fabrics about to be declared, by name, with their
// adapters: what `node add --fabric` will write into agent.yaml, which
// does not exist yet when doctor runs before it.
Fabrics map[string][]string
}
// Collect reads the node.
func Collect(ctx context.Context, o Options) Facts {
f := Facts{At: time.Now().UTC(), Modules: map[string]bool{}, Engines: map[string]string{}}
f.Hostname, _ = os.Hostname()
f.Host = discovery.DiscoverHost()
smi := o.NvidiaSMI
if smi == "" {
if p, err := exec.LookPath("nvidia-smi"); err == nil {
smi = p
}
}
if smi != "" {
f.SMIFound = true
gpus, err := discovery.DiscoverGPUsSMI(ctx, smi)
if err != nil {
f.GPUError = err.Error()
}
out, err := exec.CommandContext(ctx, smi, "topo", "-m").Output()
if err != nil {
f.TopoErr = err.Error()
} else {
f.TopoRaw = string(out)
if t, err := discovery.ParseTopology(f.TopoRaw); err == nil {
t.Apply(gpus)
} else {
f.TopoErr = err.Error()
}
}
if fab, err := discovery.DiscoverGPUFabric(ctx, smi); err == nil {
discovery.ApplyFabric(gpus, fab)
}
f.GPUs = gpus
f.IMEX = discovery.DiscoverIMEX(ctx)
}
if m, _ := filepath.Glob("/dev/nvidia-nvswitch*"); len(m) > 0 {
f.NVSwitch = true
f.FabricManager = systemctl(ctx, "is-active", "nvidia-fabricmanager")
}
root := o.IBRoot
if root == "" {
root = discovery.SysfsRoot
}
f.HCAs, _ = discovery.DiscoverIB(root)
f.SMErrors = map[string]string{}
_ = discovery.AttachSM(f.HCAs, func(hca string, port int) (*discovery.SMInfo, error) {
sm, err := discovery.QuerySM(ctx, hca, port)
if err != nil {
f.SMErrors[fmt.Sprintf("%s/%d", hca, port)] = err.Error()
}
return sm, err
})
f.SMLists, f.SMListErrors = map[string][]discovery.SMRecord{}, map[string]string{}
for _, h := range f.HCAs {
for _, p := range h.Ports {
if p.SM == nil {
continue
}
if _, done := f.SMLists[p.SM.GUID]; done {
continue
}
if _, done := f.SMListErrors[p.SM.GUID]; done {
continue
}
if recs, err := discovery.QuerySMs(ctx, h.Name, p.Num); err != nil {
f.SMListErrors[p.SM.GUID] = err.Error()
} else {
f.SMLists[p.SM.GUID] = recs
}
}
}
for _, m := range watchedModules {
_, err := os.Stat(filepath.Join("/sys/module", m))
f.Modules[m] = err == nil
}
f.readConfig(o)
if len(o.Fabrics) > 0 {
declared := map[string][]string{}
for _, c := range f.Fabrics {
declared[c.Fabric] = c.Declared
}
maps.Copy(declared, o.Fabrics)
f.Fabrics = discovery.CheckFabrics(declared, f.HCAs)
}
f.Host.MinComputeCapability = discovery.MinComputeCapability(f.GPUs)
if f.VLLMVersion = o.VLLMVersion; f.VLLMVersion != "" {
if f.EnginesDir == "" {
f.EnginesDir = provision.DefaultRoot
}
// The directory may not exist yet: the space is the filesystem's.
for dir := f.EnginesDir; dir != "/" && dir != "."; dir = filepath.Dir(dir) {
if gb, err := provision.FreeGB(dir); err == nil {
f.EnginesFreeGB = gb
break
}
}
matches, _ := filepath.Glob(filepath.Join(f.EnginesDir, "vllm", f.VLLMVersion+"-*", ".llmgw-installed"))
f.VLLMInstalled = len(matches) > 0
}
f.UnitMemlock = strings.TrimPrefix(systemctl(ctx, "show", "llmgw-agent", "-p", "LimitMEMLOCK"), "LimitMEMLOCK=")
for name, bin := range map[string]string{"vllm": "vllm", "ray": "ray", "sglang": "python3"} {
if name == "sglang" {
continue // no binary of its own to ask
}
if p, err := exec.LookPath(bin); err == nil {
out, _ := exec.CommandContext(ctx, p, "--version").CombinedOutput()
f.Engines[name] = firstLine(string(out))
}
}
return f
}
func (f *Facts) readConfig(o Options) {
path := o.ConfigPath
if path == "" {
if _, err := os.Stat("/etc/llmgw/agent.yaml"); err != nil {
return
}
path = "/etc/llmgw/agent.yaml"
}
f.ConfigPath = path
cfg, err := loadAgent(path)
if err != nil {
f.ConfigErr = err.Error()
return
}
declared := map[string][]string{}
for _, fa := range cfg.Fabrics {
declared[fa.Fabric] = fa.HCAs
}
f.Fabrics = discovery.CheckFabrics(declared, f.HCAs)
f.MgmtAddr, f.MgmtIface = agent.ResolveManagement(cfg)
if f.MgmtIface == "" {
f.MgmtIface = "eth0"
}
_, err = net.InterfaceByName(f.MgmtIface)
f.IfaceExists = err == nil
f.EnginesDir = cfg.EnginesDir
f.DisabledEngines = cfg.Engines.Disabled
f.LaunchMode = cfg.Engines.VLLM.LaunchMode
if f.LaunchMode == "" {
f.LaunchMode = "mp"
}
// What a two-node shard on each fabric would be given, the part of an
// engine's environment that decides whether NCCL finds its peers.
f.NCCL = map[string]map[string]string{}
for _, c := range f.Fabrics {
if !c.OK() {
continue
}
spec := engines.ShardSpec{NNodes: 2, HCAs: c.Usable, MgmtInterface: f.MgmtIface, GIDIndex: rocev2(f.HCAs, c.Usable)}
env := engines.NCCLEnv(spec)
delete(env, "CUDA_VISIBLE_DEVICES")
f.NCCL[c.Fabric] = env
}
}
// loadAgent is replaced in tests; the real one lives in config, which this
// package cannot import without a cycle through the CLI.
var loadAgent = func(string) (agent.Config, error) {
return agent.Config{}, fmt.Errorf("no agent configuration loader")
}
// SetConfigLoader gives Collect the configuration loader.
func SetConfigLoader(fn func(string) (agent.Config, error)) { loadAgent = fn }
func rocev2(hcas []discovery.HCA, usable []string) *int {
for _, h := range hcas {
if !slices.Contains(usable, h.Name) {
continue
}
for _, p := range h.Ports {
if p.Active() && p.RoCEv2GID >= 0 {
v := p.RoCEv2GID
return &v
}
}
}
return nil
}
func systemctl(ctx context.Context, args ...string) string {
out, err := exec.CommandContext(ctx, "systemctl", args...).Output()
if err != nil && len(out) == 0 {
return "unknown"
}
return strings.TrimSpace(string(out))
}
func firstLine(s string) string {
s = strings.TrimSpace(s)
if i := strings.IndexByte(s, '\n'); i >= 0 {
s = s[:i]
}
return s
}
// Evaluate draws the findings from the facts, most serious first.
func Evaluate(f Facts) []Finding {
var out []Finding
add := func(l Level, area, msg, fix string) {
out = append(out, Finding{Level: l, Area: area, Message: msg, Fix: fix})
}
// --- CPU ---
// Read from /proc/cpuinfo, so only on a node whose OS was read too;
// elsewhere an empty list means unknown, not missing.
if f.Host.Arch == "amd64" && f.Host.OSID != "" {
if f.Host.HasCPUFlag("avx2") {
add(OK, "cpu", "vector extensions: "+strings.Join(f.Host.CPUFlags, ", "), "")
} else {
has := "none of AVX, AVX2 or FMA"
if len(f.Host.CPUFlags) > 0 {
has = "only " + strings.Join(f.Host.CPUFlags, ", ")
}
// Nothing else looks wrong on such a node: it registers, answers
// correctly, and is several times slower than the hardware.
add(Warn, "cpu", "this CPU offers no AVX2 ("+has+"): llama.cpp runs several times slower on it",
"on a VM, give it the host's CPU type (Proxmox: qm set <vmid> --cpu host, then restart the VM)")
}
}
// --- GPUs ---
switch {
case !f.SMIFound:
add(Info, "gpu", "no nvidia-smi: a CPU node, or the driver is not installed", "")
case f.GPUError != "":
add(Fail, "gpu", "nvidia-smi fails: "+f.GPUError, "check the driver: run nvidia-smi by hand")
case len(f.GPUs) == 0:
add(Warn, "gpu", "nvidia-smi is installed but reports no GPUs", "")
default:
add(OK, "gpu", fmt.Sprintf("%d GPUs: %s", len(f.GPUs), gpuNames(f.GPUs)), "")
for _, g := range f.GPUs {
if !g.Health.OK() {
add(Warn, "gpu", fmt.Sprintf("GPU %d is kept out of placement: %s", g.Index, strings.Join(g.Health.Problems, "; ")), "")
}
}
}
if f.NVSwitch && f.FabricManager != "active" {
add(Fail, "nvlink", "this machine has NVSwitch but nvidia-fabricmanager is "+f.FabricManager+
": CUDA will not initialise on its GPUs",
"systemctl enable --now nvidia-fabricmanager (its version must match the driver's)")
}
if len(f.GPUs) > 1 {
pairs, groups := nvlinkGroups(f.GPUs)
switch {
case f.TopoErr != "":
add(Warn, "nvlink", "could not read nvidia-smi topo -m: "+f.TopoErr+
"; tensor-parallel groups will be chosen without NVLink", "")
case pairs == 0 && f.NVSwitch:
add(Warn, "nvlink", "NVSwitch is present but no GPU reports an NVLink peer", "check nvidia-smi nvlink --status")
case pairs == 0:
add(Info, "nvlink", "no NVLink: GPUs talk over PCIe, and tensor parallelism across them is slow", "")
default:
add(OK, "nvlink", "NVLink groups: "+groups, "")
}
}
// --- multi-node NVLink ---
domain := ""
for _, g := range f.GPUs {
fb := g.Fabric
if fb == nil || strings.EqualFold(fb.State, "Not Supported") {
continue
}
if d := fb.Domain(); d != "" {
domain = d
if fb.Bandwidth != "" && !strings.EqualFold(fb.Bandwidth, "Full") {
add(Warn, "nvlink", fmt.Sprintf("GPU %d's NVLink bandwidth is %s", g.Index, strings.ToLower(fb.Bandwidth)),
"check its NVLink cables and trays: nvidia-smi -q -d FABRIC")
}
continue
}
if !strings.EqualFold(fb.State, "Completed") {
why := fb.State
if fb.Status != "" && !strings.EqualFold(fb.Status, "Success") {
why += ": " + fb.Status
}
add(Fail, "nvlink", fmt.Sprintf("GPU %d is not joined to its NVLink domain (%s)", g.Index, why),
"the fabric manager joins it: check nvidia-fabricmanager and the NVLink switch trays")
}
}
if domain != "" {
add(OK, "nvlink", "the GPUs are in multi-node NVLink domain "+domain, "")
switch {
case f.IMEX.State != "active":
state := f.IMEX.State
if state == "" {
state = "not installed"
}
add(Fail, "nvlink", "nvidia-imex is "+state+": without it the domain's GPUs cannot share memory across nodes",
"systemctl enable --now nvidia-imex, with every node of the domain in /etc/nvidia-imex/nodes_config.cfg")
case f.IMEX.Channels == 0:
add(Fail, "nvlink", "nvidia-imex runs but no IMEX channel exists for an engine to open",
"load the driver with options nvidia NVreg_CreateImexChannel0=1")
default:
add(OK, "nvlink", fmt.Sprintf("nvidia-imex is active with %d channel(s)", f.IMEX.Channels), "")
}
}
// --- InfiniBand and RoCE ---
if len(f.HCAs) == 0 {
add(Info, "fabric", "no InfiniBand or RoCE adapters: this node can only run single-node replicas", "")
}
declared := map[string]string{} // hca -> fabric
for _, c := range f.Fabrics {
for _, h := range c.Declared {
declared[h] = c.Fabric
}
}
for _, h := range f.HCAs {
for _, p := range h.Ports {
where := fmt.Sprintf("%s/%d", h.Name, p.Num)
level := Warn
if declared[h.Name] != "" {
level = Fail
}
switch {
case p.Active() && p.LinkLayer == discovery.LinkEthernet && p.RoCEv2GID < 0:
// Fatal only for a port a declared fabric uses. An adapter
// nothing is configured to use — a RoCE NIC on a node added
// for single-node inference — kept nodes out for a fabric
// nobody asked for.
msg := where + " is RoCE with no RoCE v2 GID entry: NCCL would use a v1 entry and hang"
if declared[h.Name] == "" {
msg += "; no declared fabric uses it, so nothing will until one does"
}
add(level, "fabric", msg, "give the interface an IPv4 address, or check the adapter's RoCE mode")
case p.Active():
on := ""
switch {
case p.SM != nil:
on = fmt.Sprintf(", subnet manager %s (LID %d, %s)", p.SM.GUID, p.SM.LID, p.SM.State)
case p.Network != "":
on = ", network " + p.Network
}
add(OK, "fabric", fmt.Sprintf("%s %s at %d Gb/s (%s)%s", where, p.State, p.RateGbps, p.LinkLayer, on), "")
// Questions about which fabric a port is on matter to the
// fabrics that use it; for a port none does they are noted,
// not warned of.
quiet := Warn
if declared[h.Name] == "" {
quiet = Info
}
if p.LinkLayer == discovery.LinkInfiniBand && p.SM == nil {
why := f.SMErrors[where]
fix := ""
if strings.Contains(why, "infiniband-diags") {
fix = "install infiniband-diags"
}
add(quiet, "fabric", where+": which fabric it is on cannot be told — "+orUnknown(why)+
"; without it, separate fabrics given one name cannot be caught", fix)
}
if p.LinkLayer == discovery.LinkInfiniBand && p.SMLID == 0 {
add(quiet, "fabric", where+" is active but reports no subnet manager LID", "")
}
case strings.EqualFold(p.PhysState, "LinkUp"):
add(level, "fabric", where+" has a link but is "+p.State+": no subnet manager has configured it",
"run a subnet manager on the fabric (opensm, or the switch's own)")
default:
add(level, "fabric", fmt.Sprintf("%s is %s (%s): no link", where, p.State, p.PhysState),
"check the cable and the switch port")
}
for _, name := range []string{"symbol_error", "link_downed", "link_error_recovery", "port_rcv_errors", "local_link_integrity_errors"} {
if v := p.Counters[name]; v > 0 {
add(Warn, "fabric", fmt.Sprintf("%s: %s = %d since the adapter came up", where, name, v),
"a count that keeps growing is a cable or transceiver going bad")
}
}
}
}
for _, master := range sortedKeys(f.SMLists) {
recs := f.SMLists[master]
var parts []string
standby := 0
for _, r := range recs {
parts = append(parts, fmt.Sprintf("%s (%s, priority %d)", r.GUID, r.State, r.Priority))
if r.State == "standby" {
standby++
}
}
if standby == 0 {
add(Warn, "fabric", "the fabric of subnet manager "+master+" has no standby: "+strings.Join(parts, ", "),
"run a second subnet manager, on a switch or a host, at a lower priority")
} else {
add(OK, "fabric", "subnet managers of "+master+"'s fabric: "+strings.Join(parts, ", "), "")
}
}
for _, master := range sortedKeys(f.SMListErrors) {
add(Info, "fabric", "the subnet managers of "+master+"'s fabric could not be listed: "+f.SMListErrors[master], "")
}
if len(f.HCAs) > 0 {
if !f.Modules["ib_uverbs"] {
// RDMA is for replicas spanning nodes; a node that declares no
// fabric runs none, and is a node without it.
if len(declared) > 0 {
add(Fail, "fabric", "ib_uverbs is not loaded: nothing in user space can use RDMA", "modprobe ib_uverbs")
} else {
add(Warn, "fabric", "ib_uverbs is not loaded: nothing in user space can use RDMA, which a fabric declared "+
"later would need; this node runs single-node replicas without it", "modprobe ib_uverbs")
}
}
if len(f.GPUs) > 0 && !f.Modules["nvidia_peermem"] && !f.Modules["nv_peer_mem"] {
add(Warn, "fabric", "nvidia_peermem is not loaded: GPUDirect RDMA is off, and NCCL copies every transfer through host memory",
"modprobe nvidia_peermem")
}
}
// --- the agent's view ---
switch {
case f.ConfigErr != "":
add(Fail, "agent", "the agent configuration "+f.ConfigPath+" does not load: "+f.ConfigErr, "")
case f.ConfigPath == "":
add(Info, "agent", "no agent configuration found; declared fabrics were not checked", "pass --config")
default:
for _, c := range f.Fabrics {
if c.OK() {
add(OK, "agent", fmt.Sprintf("fabric %s: %s usable at %d Gb/s", c.Fabric, strings.Join(c.Usable, ","), c.RateGbps), "")
}
for _, p := range c.Problems {
lvl := Warn
if !c.OK() {
lvl = Fail
}
add(lvl, "agent", "fabric "+c.Fabric+": "+p, "")
}
if !c.OK() {
add(Fail, "agent", "fabric "+c.Fabric+" will not be advertised: no multi-node replica will be placed here", "")
}
}
active := activeHCAs(f.HCAs)
if len(f.Fabrics) == 0 && len(active) > 0 {
add(Warn, "agent", "adapters are up ("+strings.Join(active, ", ")+") but agent.yaml declares no fabric: this node will never get multi-node work",
"add fabrics: [{name: <fabric>, hcas: ["+strings.Join(active, ", ")+"]}]")
}
if !f.IfaceExists {
add(Fail, "agent", "the management interface "+f.MgmtIface+" does not exist: NCCL and Gloo would bootstrap over nothing",
"set mgmt_interface in agent.yaml")
} else {
add(OK, "agent", "NCCL bootstraps over "+f.MgmtIface+" ("+f.MgmtAddr+")", "")
}
if len(f.Fabrics) > 0 && f.LaunchMode == "ray" && f.Engines["ray"] == "" && f.Engines["vllm"] != "" {
add(Fail, "agent", "vLLM's launch_mode is ray but ray is not installed", "install ray, or set engines.vllm.launch_mode: mp")
}
}
// --- engines ---
// An engine disabled on this node: whether it could be installed is not
// a question anybody is asking.
if f.VLLMVersion != "" && !slices.Contains(f.DisabledEngines, "vllm") {
vllmFindings(f, add)
}
if f.UnitMemlock != "" && f.UnitMemlock != "unknown" && f.UnitMemlock != "infinity" && len(f.HCAs) > 0 {
add(Warn, "agent", "the llmgw-agent unit limits locked memory to "+f.UnitMemlock+"; RDMA pins what it registers",
"LimitMEMLOCK=infinity in the unit")
}
sort.SliceStable(out, func(i, j int) bool { return rank(out[i].Level) < rank(out[j].Level) })
return out
}
// vllmFindings says whether the agent can install vLLM here, and if not why:
// the same decision the agent makes, so the report and the node agree.
func vllmFindings(f Facts, add func(Level, string, string, string)) {
v := f.VLLMVersion
builds, err := provision.Recipes["vllm"].Candidates(v, "", f.Host)
var b provision.Build
if err == nil {
// The first the agent would try; it moves to the next by itself if
// that one does not work here.
b = builds[0]
if b.Version != "" {
v = b.Version
}
}
var nh provision.NotHere
switch {
case errors.As(err, &nh):
add(Info, "engines", "vLLM will not be installed: "+nh.Reason, "")
return
case err != nil:
// A warning, not a failure: the node is still a node, and serves
// llama.cpp. Refusing it here kept older GPU generations out of a
// cluster for an engine nobody meant to run on them.
add(Warn, "engines", "vLLM "+v+" will not run here, and the node joins without it: "+err.Error(),
"add it with --disable-engine vllm if it is meant for llama.cpp alone")
return
case f.VLLMInstalled:
add(OK, "engines", "vLLM "+v+" is installed under "+f.EnginesDir, "")
return
}
if f.EnginesFreeGB > 0 && f.EnginesFreeGB < float64(b.FreeGB) {
add(Warn, "engines", fmt.Sprintf("vLLM %s needs about %d GB free under %s and there are %.1f GB; the node joins without it until there is",
v, b.FreeGB, f.EnginesDir, f.EnginesFreeGB),
"free space there, or point engines_dir in agent.yaml at a larger disk")
return
}
note := ""
if b.Note != "" {
note = "; " + b.Note
}
add(OK, "engines", fmt.Sprintf("vLLM %s (%s) can be installed here: driver %s, %.1f GB free under %s%s",
v, b.Variant, f.Host.NVIDIADriver, f.EnginesFreeGB, f.EnginesDir, note), "")
}
func rank(l Level) int {
switch l {
case Fail:
return 0
case Warn:
return 1
case OK:
return 2
}
return 3
}
func gpuNames(gpus []discovery.GPU) string {
count := map[string]int{}
var order []string
for _, g := range gpus {
if count[g.Name] == 0 {
order = append(order, g.Name)
}
count[g.Name]++
}
parts := make([]string, len(order))
for i, n := range order {
parts[i] = fmt.Sprintf("%d× %s", count[n], n)
}
return strings.Join(parts, ", ")
}
// nvlinkGroups counts NVLink pairs and names the connected groups.
func nvlinkGroups(gpus []discovery.GPU) (int, string) {
parent := map[int]int{}
var find func(int) int
find = func(x int) int {
if p, ok := parent[x]; ok && p != x {
parent[x] = find(p)
return parent[x]
}
parent[x] = x
return x
}
pairs := 0
for _, g := range gpus {
find(g.Index)
for _, p := range g.NVLinkPeers {
if p > g.Index {
pairs++
}
parent[find(g.Index)] = find(p)
}
}
groups := map[int][]int{}
for _, g := range gpus {
r := find(g.Index)
groups[r] = append(groups[r], g.Index)
}
var names []string
for _, members := range groups {
sort.Ints(members)
if len(members) > 1 {
names = append(names, fmt.Sprint(members))
}
}
sort.Strings(names)
return pairs, strings.Join(names, " ")
}
func activeHCAs(hcas []discovery.HCA) []string {
var out []string
for _, h := range hcas {
if len(h.ActivePorts()) > 0 {
out = append(out, h.Name)
}
}
return out
}
func orUnknown(s string) string {
if s == "" {
return "no answer from the subnet manager"
}
return s
}
func sortedKeys[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}