For machines nobody here can log into. It reads the GPUs and their nvidia-smi topology, NVSwitch and the fabric manager, every InfiniBand and RoCE port with its counters and RoCE v2 GID, the RDMA and GPUDirect kernel modules, the agent's declared fabrics against the adapters, the management interface NCCL will bootstrap over, the agent unit's memlock limit and the engines installed - and says what is wrong and how to fix it, most serious first, with the NCCL environment a shard on each fabric would be given. --json writes every fact it concluded from, to send back. It exits non-zero when something fails. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
181 lines
5.1 KiB
Go
181 lines
5.1 KiB
Go
package cli
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"sort"
|
|
"strings"
|
|
"text/tabwriter"
|
|
"time"
|
|
|
|
"github.com/spf13/cobra"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/doctor"
|
|
)
|
|
|
|
func newDoctorCmd() *cobra.Command {
|
|
var (
|
|
opts doctor.Options
|
|
asJSON bool
|
|
verbose bool
|
|
)
|
|
cmd := &cobra.Command{
|
|
Use: "doctor",
|
|
Short: "Check this node's GPUs, NVLink and InfiniBand for what llmgw needs",
|
|
Long: `Reads this node's GPUs, their NVLink and PCIe topology, its InfiniBand and
|
|
RoCE ports, the kernel modules RDMA needs and the agent's configuration, and
|
|
says what is wrong and how to fix it — most serious first.
|
|
|
|
Run it on a node before it joins, and whenever a multi-node replica will not
|
|
come up. --json writes everything it read as well as what it concluded; send
|
|
that file when the report is not enough to go on.`,
|
|
Args: cobra.NoArgs,
|
|
RunE: func(cmd *cobra.Command, _ []string) error {
|
|
doctor.SetConfigLoader(config.LoadAgent)
|
|
ctx, cancel := context.WithTimeout(cmd.Context(), 60*time.Second)
|
|
defer cancel()
|
|
facts := doctor.Collect(ctx, opts)
|
|
findings := doctor.Evaluate(facts)
|
|
if asJSON {
|
|
enc := json.NewEncoder(os.Stdout)
|
|
enc.SetIndent("", " ")
|
|
return enc.Encode(struct {
|
|
Findings []doctor.Finding `json:"findings"`
|
|
Facts doctor.Facts `json:"facts"`
|
|
}{findings, facts})
|
|
}
|
|
printDoctor(os.Stdout, facts, findings, verbose)
|
|
for _, f := range findings {
|
|
if f.Level == doctor.Fail {
|
|
return fmt.Errorf("%d problem(s) need fixing", countLevel(findings, doctor.Fail))
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
}
|
|
f := cmd.Flags()
|
|
f.StringVar(&opts.ConfigPath, "config", "", "agent configuration (default /etc/llmgw/agent.yaml when present)")
|
|
f.StringVar(&opts.IBRoot, "ib-root", discovery.SysfsRoot, "sysfs InfiniBand directory")
|
|
f.StringVar(&opts.NvidiaSMI, "nvidia-smi", "", "path to nvidia-smi")
|
|
f.BoolVar(&asJSON, "json", false, "write every fact and finding as JSON, to send back")
|
|
f.BoolVar(&verbose, "verbose", false, "also print the raw nvidia-smi topology")
|
|
return cmd
|
|
}
|
|
|
|
func countLevel(fs []doctor.Finding, l doctor.Level) int {
|
|
n := 0
|
|
for _, f := range fs {
|
|
if f.Level == l {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
func printDoctor(out io.Writer, f doctor.Facts, findings []doctor.Finding, verbose bool) {
|
|
fmt.Fprintf(out, "%s · %s %s %s · driver %s\n\n", f.Hostname, f.Host.OSID, f.Host.OSVersion,
|
|
f.Host.Arch, orDash(f.Host.NVIDIADriver))
|
|
|
|
for _, x := range findings {
|
|
fmt.Fprintf(out, "%-4s %-6s %s\n", x.Level, x.Area, x.Message)
|
|
if x.Fix != "" && (x.Level == doctor.Fail || x.Level == doctor.Warn) {
|
|
fmt.Fprintf(out, " → %s\n", x.Fix)
|
|
}
|
|
}
|
|
|
|
w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
|
|
if len(f.GPUs) > 0 {
|
|
fmt.Fprintln(w, "\nGPU\tNAME\tNUMA\tNVLINK PEERS\tNEAREST ADAPTER")
|
|
for _, g := range f.GPUs {
|
|
fmt.Fprintf(w, "%d\t%s\t%d\t%s\t%s\n", g.Index, g.Name, g.NUMANode,
|
|
orDash(ints(g.NVLinkPeers)), orDash(nearest(g.NICPaths)))
|
|
}
|
|
}
|
|
if len(f.HCAs) > 0 {
|
|
fmt.Fprintln(w, "\nADAPTER\tSTATE\tPHYS\tRATE\tLINK\tNUMA\tROCE GID\tERRORS")
|
|
for _, h := range f.HCAs {
|
|
for _, p := range h.Ports {
|
|
gid := "-"
|
|
if p.RoCEv2GID >= 0 {
|
|
gid = fmt.Sprint(p.RoCEv2GID)
|
|
}
|
|
fmt.Fprintf(w, "%s/%d\t%s\t%s\t%d Gb/s\t%s\t%d\t%s\t%s\n", h.Name, p.Num, p.State,
|
|
orDash(p.PhysState), p.RateGbps, p.LinkLayer, h.NUMANode, gid, orDash(errorsOf(p.Counters)))
|
|
}
|
|
}
|
|
}
|
|
_ = w.Flush()
|
|
|
|
for _, fabric := range sortedKeys(f.NCCL) {
|
|
fmt.Fprintf(out, "\nA shard on %s is given:\n", fabric)
|
|
env := f.NCCL[fabric]
|
|
for _, k := range sortedKeys(env) {
|
|
fmt.Fprintf(out, " %s=%s\n", k, env[k])
|
|
}
|
|
}
|
|
if len(f.Engines) > 0 {
|
|
fmt.Fprintln(out, "\nEngines:")
|
|
for _, k := range sortedKeys(f.Engines) {
|
|
fmt.Fprintf(out, " %s: %s\n", k, f.Engines[k])
|
|
}
|
|
}
|
|
if verbose && f.TopoRaw != "" {
|
|
fmt.Fprintf(out, "\nnvidia-smi topo -m:\n%s\n", f.TopoRaw)
|
|
}
|
|
fmt.Fprintf(out, "\n%d failed, %d warnings. `llmgw doctor --json > report.json` keeps everything read, to send back.\n",
|
|
countLevel(findings, doctor.Fail), countLevel(findings, doctor.Warn))
|
|
}
|
|
|
|
func nearest(paths map[string]string) string {
|
|
best, rel := "", ""
|
|
for h, r := range paths {
|
|
if best == "" || discovery.PathRank(r) < discovery.PathRank(rel) ||
|
|
(discovery.PathRank(r) == discovery.PathRank(rel) && h < best) {
|
|
best, rel = h, r
|
|
}
|
|
}
|
|
if best == "" {
|
|
return ""
|
|
}
|
|
return best + " (" + rel + ")"
|
|
}
|
|
|
|
func errorsOf(c map[string]uint64) string {
|
|
var out []string
|
|
for _, k := range sortedKeys(c) {
|
|
if c[k] > 0 && k != "port_xmit_data" && k != "port_rcv_data" {
|
|
out = append(out, fmt.Sprintf("%s=%d", k, c[k]))
|
|
}
|
|
}
|
|
return strings.Join(out, " ")
|
|
}
|
|
|
|
func ints(v []int) string {
|
|
s := make([]string, len(v))
|
|
for i, x := range v {
|
|
s[i] = fmt.Sprint(x)
|
|
}
|
|
return strings.Join(s, ",")
|
|
}
|
|
|
|
func orDash(s string) string {
|
|
if s == "" {
|
|
return "-"
|
|
}
|
|
return s
|
|
}
|
|
|
|
func sortedKeys[V any](m map[string]V) []string {
|
|
out := make([]string, 0, len(m))
|
|
for k := range m {
|
|
out = append(out, k)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|