Files
llmgw/internal/cli/doctor.go
T
Ilker ManapandClaude Opus 5.5 e313f32696 Decide engines per node, and let a node join without one it cannot run
engine_overrides give nodes picked by GPU class, id or labels their own
engine versions; node add --disable-engine keeps one off a node; what
vLLM cannot do on a node is a warning rather than a refused node; the
CUDA requirement follows the version; a hand-named binary is no longer
replaced by an installed one.

Refs #2 (items 4 and 7).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-01 12:07:25 +02:00

189 lines
5.4 KiB
Go

package cli
import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"sort"
"strings"
"text/tabwriter"
"time"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/doctor"
)
func newDoctorCmd() *cobra.Command {
var (
opts doctor.Options
asJSON bool
verbose bool
)
cmd := &cobra.Command{
Use: "doctor",
Short: "Check this node's GPUs, NVLink, InfiniBand and engines for what llmgw needs",
Long: `Reads this node's GPUs, their NVLink and PCIe topology, its InfiniBand and
RoCE ports, the kernel modules RDMA needs, the agent's configuration and
whether vLLM can be installed here, and says what is wrong and how to fix
it — most serious first.
Run it on a node before it joins, and whenever a multi-node replica will not
come up. --json writes everything it read as well as what it concluded; send
that file when the report is not enough to go on.`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
doctor.SetConfigLoader(config.LoadAgent)
switch opts.VLLMVersion {
case "":
opts.VLLMVersion = config.DefaultEngines["vllm"]
case "none":
opts.VLLMVersion = ""
}
ctx, cancel := context.WithTimeout(cmd.Context(), 60*time.Second)
defer cancel()
facts := doctor.Collect(ctx, opts)
findings := doctor.Evaluate(facts)
if asJSON {
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
return enc.Encode(struct {
Findings []doctor.Finding `json:"findings"`
Facts doctor.Facts `json:"facts"`
}{findings, facts})
}
printDoctor(os.Stdout, facts, findings, verbose)
for _, f := range findings {
if f.Level == doctor.Fail {
return fmt.Errorf("%d problem(s) need fixing", countLevel(findings, doctor.Fail))
}
}
return nil
},
}
f := cmd.Flags()
f.StringVar(&opts.ConfigPath, "config", "", "agent configuration (default /etc/llmgw/agent.yaml when present)")
f.StringVar(&opts.IBRoot, "ib-root", discovery.SysfsRoot, "sysfs InfiniBand directory")
f.StringVar(&opts.NvidiaSMI, "nvidia-smi", "", "path to nvidia-smi")
f.StringVar(&opts.VLLMVersion, "vllm", "", "the vLLM version to check for (default: the one this llmgw installs)")
f.BoolVar(&asJSON, "json", false, "write every fact and finding as JSON, to send back")
f.BoolVar(&verbose, "verbose", false, "also print the raw nvidia-smi topology")
return cmd
}
func countLevel(fs []doctor.Finding, l doctor.Level) int {
n := 0
for _, f := range fs {
if f.Level == l {
n++
}
}
return n
}
func printDoctor(out io.Writer, f doctor.Facts, findings []doctor.Finding, verbose bool) {
fmt.Fprintf(out, "%s · %s %s %s · driver %s\n\n", f.Hostname, f.Host.OSID, f.Host.OSVersion,
f.Host.Arch, orDash(f.Host.NVIDIADriver))
for _, x := range findings {
fmt.Fprintf(out, "%-4s %-6s %s\n", x.Level, x.Area, x.Message)
if x.Fix != "" && (x.Level == doctor.Fail || x.Level == doctor.Warn) {
fmt.Fprintf(out, " → %s\n", x.Fix)
}
}
w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
if len(f.GPUs) > 0 {
fmt.Fprintln(w, "\nGPU\tNAME\tNUMA\tNVLINK PEERS\tNEAREST ADAPTER")
for _, g := range f.GPUs {
fmt.Fprintf(w, "%d\t%s\t%d\t%s\t%s\n", g.Index, g.Name, g.NUMANode,
orDash(ints(g.NVLinkPeers)), orDash(nearest(g.NICPaths)))
}
}
if len(f.HCAs) > 0 {
fmt.Fprintln(w, "\nADAPTER\tSTATE\tPHYS\tRATE\tLINK\tNUMA\tROCE GID\tERRORS")
for _, h := range f.HCAs {
for _, p := range h.Ports {
gid := "-"
if p.RoCEv2GID >= 0 {
gid = fmt.Sprint(p.RoCEv2GID)
}
fmt.Fprintf(w, "%s/%d\t%s\t%s\t%d Gb/s\t%s\t%d\t%s\t%s\n", h.Name, p.Num, p.State,
orDash(p.PhysState), p.RateGbps, p.LinkLayer, h.NUMANode, gid, orDash(errorsOf(p.Counters)))
}
}
}
_ = w.Flush()
for _, fabric := range sortedKeys(f.NCCL) {
fmt.Fprintf(out, "\nA shard on %s is given:\n", fabric)
env := f.NCCL[fabric]
for _, k := range sortedKeys(env) {
fmt.Fprintf(out, " %s=%s\n", k, env[k])
}
}
if len(f.Engines) > 0 {
fmt.Fprintln(out, "\nEngines:")
for _, k := range sortedKeys(f.Engines) {
fmt.Fprintf(out, " %s: %s\n", k, f.Engines[k])
}
}
if verbose && f.TopoRaw != "" {
fmt.Fprintf(out, "\nnvidia-smi topo -m:\n%s\n", f.TopoRaw)
}
fmt.Fprintf(out, "\n%d failed, %d warnings. `llmgw doctor --json > report.json` keeps everything read, to send back.\n",
countLevel(findings, doctor.Fail), countLevel(findings, doctor.Warn))
}
func nearest(paths map[string]string) string {
best, rel := "", ""
for h, r := range paths {
if best == "" || discovery.PathRank(r) < discovery.PathRank(rel) ||
(discovery.PathRank(r) == discovery.PathRank(rel) && h < best) {
best, rel = h, r
}
}
if best == "" {
return ""
}
return best + " (" + rel + ")"
}
func errorsOf(c map[string]uint64) string {
var out []string
for _, k := range sortedKeys(c) {
if c[k] > 0 && k != "port_xmit_data" && k != "port_rcv_data" {
out = append(out, fmt.Sprintf("%s=%d", k, c[k]))
}
}
return strings.Join(out, " ")
}
func ints(v []int) string {
s := make([]string, len(v))
for i, x := range v {
s[i] = fmt.Sprint(x)
}
return strings.Join(s, ",")
}
func orDash(s string) string {
if s == "" {
return "-"
}
return s
}
func sortedKeys[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}