NVML is the only source of NVLink topology, and that topology is what keeps a tensor-parallel group off the slow path inside a node. It needs cgo, so it sits behind the nvml tag and the default build keeps using nvidia-smi. Discover() picks the backend and falls back to nvidia-smi when NVML fails, logging the fallback rather than performing it silently: on a node that is meant to have working NVML, a quiet downgrade hides a broken driver installation and costs the scheduler its topology. CI now compiles the tagged build. That immediately paid for itself: ErrNoNVML was declared only in the untagged file, so the tests did not compile under -tags nvml. It is now declared where both builds can see it. Not verified: the NVML path has never run against a driver, because there is no GPU machine available. The NVLink domains it reports should be treated as unverified until someone runs `llmgw agent inventory` on a real node. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
6a13e3dddf
commit
55417546fc
10 files changed
+374
-23
No files matched your search
+18
-17
@@ -36,6 +36,7 @@ func newInventoryCmd() *cobra.Command {
|
||||
smiBin string
|
||||
ibRoot string
|
||||
skipGPU bool
|
||||
useNVML bool
|
||||
)
|
||||
|
||||
cmd := &cobra.Command{
|
||||
@@ -46,24 +47,18 @@ func newInventoryCmd() *cobra.Command {
|
||||
"Run it before adding a node to the cluster — it is the fastest way to find\n" +
|
||||
"a card another process is already using, or a port with no subnet manager.",
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
inv := discovery.Inventory{Source: "nvidia-smi"}
|
||||
|
||||
if !skipGPU {
|
||||
gpus, err := discovery.DiscoverGPUsSMI(cmd.Context(), smiBin)
|
||||
if err != nil {
|
||||
// Not fatal: a node with no GPUs still has IB worth showing,
|
||||
// and the error itself is the useful output.
|
||||
fmt.Fprintf(os.Stderr, "GPU discovery failed: %v\n", err)
|
||||
inv.Source = "unavailable"
|
||||
}
|
||||
inv.GPUs = gpus
|
||||
}
|
||||
|
||||
hcas, err := discovery.DiscoverIB(ibRoot)
|
||||
inv, err := discovery.Discover(cmd.Context(), discovery.Options{
|
||||
NvidiaSMI: smiBin,
|
||||
IBRoot: ibRoot,
|
||||
PreferNVML: useNVML,
|
||||
SkipGPUs: skipGPU,
|
||||
}, logger())
|
||||
if err != nil {
|
||||
return err
|
||||
// Not fatal: a node whose GPUs cannot be read still has IB
|
||||
// worth showing, and the error is itself the useful output.
|
||||
fmt.Fprintf(os.Stderr, "%v\n", err)
|
||||
inv.Source = "unavailable"
|
||||
}
|
||||
inv.HCAs = hcas
|
||||
|
||||
if asJSON {
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
@@ -80,6 +75,7 @@ func newInventoryCmd() *cobra.Command {
|
||||
f.StringVar(&smiBin, "nvidia-smi", "", "path to nvidia-smi")
|
||||
f.StringVar(&ibRoot, "ib-root", discovery.SysfsRoot, "sysfs InfiniBand directory")
|
||||
f.BoolVar(&skipGPU, "no-gpu", false, "skip GPU discovery")
|
||||
f.BoolVar(&useNVML, "nvml", true, "prefer the NVML backend when this build has one (it is the only source of NVLink topology)")
|
||||
|
||||
return cmd
|
||||
}
|
||||
@@ -87,7 +83,7 @@ func newInventoryCmd() *cobra.Command {
|
||||
func printInventory(inv discovery.Inventory) {
|
||||
w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
|
||||
|
||||
fmt.Fprintf(w, "GPUs (%d)\n", len(inv.GPUs))
|
||||
fmt.Fprintf(w, "GPUs (%d, via %s)\n", len(inv.GPUs), inv.Source)
|
||||
if len(inv.GPUs) == 0 {
|
||||
fmt.Fprintln(w, " none")
|
||||
} else {
|
||||
@@ -101,6 +97,11 @@ func printInventory(inv discovery.Inventory) {
|
||||
g.Index, g.Class(), g.Name, g.MemoryTotalMB, g.ComputeCapability, health)
|
||||
}
|
||||
fmt.Fprintf(w, " classes: %v\n", inv.Classes())
|
||||
if !discovery.NVMLSupported() {
|
||||
fmt.Fprintln(w, " NVLink topology unavailable: rebuild with -tags nvml.")
|
||||
fmt.Fprintln(w, " Without it, placement cannot keep a tensor-parallel group inside")
|
||||
fmt.Fprintln(w, " one NVLink domain (plan §6.2).")
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Fprintf(w, "\nInfiniBand adapters (%d)\n", len(inv.HCAs))
|
||||
|
||||
Reference in new issue
Block a user