vLLM was installed by hand and named in each agent's configuration. The agent now installs it: a Python 3.12 environment built by uv, which is itself a GitHub release checked against its checksum, from PyPI resolved as it stood a day after that vLLM was published and installed with --require-hashes. The resolved list is kept in the environment. The environment is built relocatable under a scratch name and renamed into place once it runs, like an archive. The default is 0.30.0, whose flags were checked against the ones llmgw builds. It is built for CUDA 13, so it needs driver 580 and a Turing GPU or newer, and 25 GB free while it installs; a node lacking one says which, and a node with no GPU skips it without being shown as a fault. gcc comes with it because Triton compiles its launcher at the first model load. SGLang installs the same way when the control plane names a version. llmgw doctor reports whether vLLM can be installed on the node. Checked by installing 0.30.0 this way on a test node: 7.7 GB, torch 2.13 for CUDA 13.0, importing cleanly; the node has no GPU, so vLLM was not started. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
186 lines
5.3 KiB
Go
186 lines
5.3 KiB
Go
package cli
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"sort"
|
|
"strings"
|
|
"text/tabwriter"
|
|
"time"
|
|
|
|
"github.com/spf13/cobra"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/doctor"
|
|
)
|
|
|
|
func newDoctorCmd() *cobra.Command {
|
|
var (
|
|
opts doctor.Options
|
|
asJSON bool
|
|
verbose bool
|
|
)
|
|
cmd := &cobra.Command{
|
|
Use: "doctor",
|
|
Short: "Check this node's GPUs, NVLink, InfiniBand and engines for what llmgw needs",
|
|
Long: `Reads this node's GPUs, their NVLink and PCIe topology, its InfiniBand and
|
|
RoCE ports, the kernel modules RDMA needs, the agent's configuration and
|
|
whether vLLM can be installed here, and says what is wrong and how to fix
|
|
it — most serious first.
|
|
|
|
Run it on a node before it joins, and whenever a multi-node replica will not
|
|
come up. --json writes everything it read as well as what it concluded; send
|
|
that file when the report is not enough to go on.`,
|
|
Args: cobra.NoArgs,
|
|
RunE: func(cmd *cobra.Command, _ []string) error {
|
|
doctor.SetConfigLoader(config.LoadAgent)
|
|
if opts.VLLMVersion == "" {
|
|
opts.VLLMVersion = config.DefaultEngines["vllm"]
|
|
}
|
|
ctx, cancel := context.WithTimeout(cmd.Context(), 60*time.Second)
|
|
defer cancel()
|
|
facts := doctor.Collect(ctx, opts)
|
|
findings := doctor.Evaluate(facts)
|
|
if asJSON {
|
|
enc := json.NewEncoder(os.Stdout)
|
|
enc.SetIndent("", " ")
|
|
return enc.Encode(struct {
|
|
Findings []doctor.Finding `json:"findings"`
|
|
Facts doctor.Facts `json:"facts"`
|
|
}{findings, facts})
|
|
}
|
|
printDoctor(os.Stdout, facts, findings, verbose)
|
|
for _, f := range findings {
|
|
if f.Level == doctor.Fail {
|
|
return fmt.Errorf("%d problem(s) need fixing", countLevel(findings, doctor.Fail))
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
}
|
|
f := cmd.Flags()
|
|
f.StringVar(&opts.ConfigPath, "config", "", "agent configuration (default /etc/llmgw/agent.yaml when present)")
|
|
f.StringVar(&opts.IBRoot, "ib-root", discovery.SysfsRoot, "sysfs InfiniBand directory")
|
|
f.StringVar(&opts.NvidiaSMI, "nvidia-smi", "", "path to nvidia-smi")
|
|
f.StringVar(&opts.VLLMVersion, "vllm", "", "the vLLM version to check for (default: the one this llmgw installs)")
|
|
f.BoolVar(&asJSON, "json", false, "write every fact and finding as JSON, to send back")
|
|
f.BoolVar(&verbose, "verbose", false, "also print the raw nvidia-smi topology")
|
|
return cmd
|
|
}
|
|
|
|
func countLevel(fs []doctor.Finding, l doctor.Level) int {
|
|
n := 0
|
|
for _, f := range fs {
|
|
if f.Level == l {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
func printDoctor(out io.Writer, f doctor.Facts, findings []doctor.Finding, verbose bool) {
|
|
fmt.Fprintf(out, "%s · %s %s %s · driver %s\n\n", f.Hostname, f.Host.OSID, f.Host.OSVersion,
|
|
f.Host.Arch, orDash(f.Host.NVIDIADriver))
|
|
|
|
for _, x := range findings {
|
|
fmt.Fprintf(out, "%-4s %-6s %s\n", x.Level, x.Area, x.Message)
|
|
if x.Fix != "" && (x.Level == doctor.Fail || x.Level == doctor.Warn) {
|
|
fmt.Fprintf(out, " → %s\n", x.Fix)
|
|
}
|
|
}
|
|
|
|
w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
|
|
if len(f.GPUs) > 0 {
|
|
fmt.Fprintln(w, "\nGPU\tNAME\tNUMA\tNVLINK PEERS\tNEAREST ADAPTER")
|
|
for _, g := range f.GPUs {
|
|
fmt.Fprintf(w, "%d\t%s\t%d\t%s\t%s\n", g.Index, g.Name, g.NUMANode,
|
|
orDash(ints(g.NVLinkPeers)), orDash(nearest(g.NICPaths)))
|
|
}
|
|
}
|
|
if len(f.HCAs) > 0 {
|
|
fmt.Fprintln(w, "\nADAPTER\tSTATE\tPHYS\tRATE\tLINK\tNUMA\tROCE GID\tERRORS")
|
|
for _, h := range f.HCAs {
|
|
for _, p := range h.Ports {
|
|
gid := "-"
|
|
if p.RoCEv2GID >= 0 {
|
|
gid = fmt.Sprint(p.RoCEv2GID)
|
|
}
|
|
fmt.Fprintf(w, "%s/%d\t%s\t%s\t%d Gb/s\t%s\t%d\t%s\t%s\n", h.Name, p.Num, p.State,
|
|
orDash(p.PhysState), p.RateGbps, p.LinkLayer, h.NUMANode, gid, orDash(errorsOf(p.Counters)))
|
|
}
|
|
}
|
|
}
|
|
_ = w.Flush()
|
|
|
|
for _, fabric := range sortedKeys(f.NCCL) {
|
|
fmt.Fprintf(out, "\nA shard on %s is given:\n", fabric)
|
|
env := f.NCCL[fabric]
|
|
for _, k := range sortedKeys(env) {
|
|
fmt.Fprintf(out, " %s=%s\n", k, env[k])
|
|
}
|
|
}
|
|
if len(f.Engines) > 0 {
|
|
fmt.Fprintln(out, "\nEngines:")
|
|
for _, k := range sortedKeys(f.Engines) {
|
|
fmt.Fprintf(out, " %s: %s\n", k, f.Engines[k])
|
|
}
|
|
}
|
|
if verbose && f.TopoRaw != "" {
|
|
fmt.Fprintf(out, "\nnvidia-smi topo -m:\n%s\n", f.TopoRaw)
|
|
}
|
|
fmt.Fprintf(out, "\n%d failed, %d warnings. `llmgw doctor --json > report.json` keeps everything read, to send back.\n",
|
|
countLevel(findings, doctor.Fail), countLevel(findings, doctor.Warn))
|
|
}
|
|
|
|
func nearest(paths map[string]string) string {
|
|
best, rel := "", ""
|
|
for h, r := range paths {
|
|
if best == "" || discovery.PathRank(r) < discovery.PathRank(rel) ||
|
|
(discovery.PathRank(r) == discovery.PathRank(rel) && h < best) {
|
|
best, rel = h, r
|
|
}
|
|
}
|
|
if best == "" {
|
|
return ""
|
|
}
|
|
return best + " (" + rel + ")"
|
|
}
|
|
|
|
func errorsOf(c map[string]uint64) string {
|
|
var out []string
|
|
for _, k := range sortedKeys(c) {
|
|
if c[k] > 0 && k != "port_xmit_data" && k != "port_rcv_data" {
|
|
out = append(out, fmt.Sprintf("%s=%d", k, c[k]))
|
|
}
|
|
}
|
|
return strings.Join(out, " ")
|
|
}
|
|
|
|
func ints(v []int) string {
|
|
s := make([]string, len(v))
|
|
for i, x := range v {
|
|
s[i] = fmt.Sprint(x)
|
|
}
|
|
return strings.Join(s, ",")
|
|
}
|
|
|
|
func orDash(s string) string {
|
|
if s == "" {
|
|
return "-"
|
|
}
|
|
return s
|
|
}
|
|
|
|
func sortedKeys[V any](m map[string]V) []string {
|
|
out := make([]string, 0, len(m))
|
|
for k := range m {
|
|
out = append(out, k)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|