From 55417546fc92d14171126cf13e66c24fbaaf77e4 Mon Sep 17 00:00:00 2001 From: ilker Date: Wed, 16 Sep 2026 02:05:25 +0200 Subject: [PATCH] Phase 2 (2/2): NVML backend behind a build tag NVML is the only source of NVLink topology, and that topology is what keeps a tensor-parallel group off the slow path inside a node. It needs cgo, so it sits behind the nvml tag and the default build keeps using nvidia-smi. Discover() picks the backend and falls back to nvidia-smi when NVML fails, logging the fallback rather than performing it silently: on a node that is meant to have working NVML, a quiet downgrade hides a broken driver installation and costs the scheduler its topology. CI now compiles the tagged build. That immediately paid for itself: ErrNoNVML was declared only in the untagged file, so the tests did not compile under -tags nvml. It is now declared where both builds can see it. Not verified: the NVML path has never run against a driver, because there is no GPU machine available. The NVLink domains it reports should be treated as unverified until someone runs `llmgw agent inventory` on a real node. Co-Authored-By: Claude Opus 5 (1M context) --- .gitea/workflows/ci.yml | 8 ++ CHANGELOG.md | 16 ++- go.mod | 1 + go.sum | 2 + internal/agent/discovery/discover.go | 70 ++++++++++ internal/agent/discovery/discover_test.go | 101 +++++++++++++++ internal/agent/discovery/gpu_nonvml.go | 11 ++ internal/agent/discovery/gpu_nvml.go | 148 ++++++++++++++++++++++ internal/cli/agent.go | 35 ++--- llmgw-plan.md | 5 +- 10 files changed, 374 insertions(+), 23 deletions(-) create mode 100644 internal/agent/discovery/discover.go create mode 100644 internal/agent/discovery/discover_test.go create mode 100644 internal/agent/discovery/gpu_nonvml.go create mode 100644 internal/agent/discovery/gpu_nvml.go diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index e45f4e8..96b9ed6 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -38,3 +38,11 @@ jobs: - name: Build run: go build ./... + + # The NVML backend needs cgo and cannot run without a driver, but it must + # at least keep compiling: it is the only source of NVLink topology and a + # break would otherwise surface on a GPU node, far from here. + - name: Build the NVML backend + env: + CGO_ENABLED: "1" + run: go build -tags nvml ./... diff --git a/CHANGELOG.md b/CHANGELOG.md index 4409220..d7acf54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,12 +61,20 @@ what is missing is the control plane connection, so there is no release yet. as a table or JSON. Run it before adding a node: it is the fastest way to find a card another process is using, or a port with no subnet manager. +- **NVML backend** behind the `nvml` build tag, which adds NVLink topology. + Nothing else exposes which GPUs share an NVLink domain, and that is what keeps + a tensor-parallel group off the slow path inside a node. `Discover` falls back + to nvidia-smi if NVML fails, and says so loudly rather than silently losing the + topology. CI compiles the tagged build so it cannot rot unnoticed. + +### Not verified + +The NVML path compiles but has never run against a driver — there is no GPU +machine to try it on. Treat the NVLink domains it reports as unverified until +someone runs `llmgw agent inventory` on a real node. + ### Still missing for phase 2 -- The NVML backend behind the `nvml` build tag, and NVLink topology with it. - Only NVML exposes which GPUs share an NVLink domain, which is what keeps a - tensor-parallel group off the slow path within a node. It needs a machine - with GPUs to verify. - The agent-to-control-plane stream. It pairs with phase 3's control plane rather than standing alone, so `llmgw agent` itself is still a stub. diff --git a/go.mod b/go.mod index 5104a54..33e7392 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( ) require ( + github.com/NVIDIA/go-nvml v0.13.4-0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index daec55d..cd976a7 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/NVIDIA/go-nvml v0.13.4-0 h1:o3jp9u2x1R9ShFE3v+Aesp55XOSIQFMJz/VGNUcJaNE= +github.com/NVIDIA/go-nvml v0.13.4-0/go.mod h1:id63qwpoDWpFXwnwM6psDCSqW4BmNu6mWpr4YeQtPGo= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= diff --git a/internal/agent/discovery/discover.go b/internal/agent/discovery/discover.go new file mode 100644 index 0000000..6bd912a --- /dev/null +++ b/internal/agent/discovery/discover.go @@ -0,0 +1,70 @@ +package discovery + +import ( + "context" + "errors" + "fmt" + "log/slog" +) + +// ErrNoNVML means this binary was built without the NVML backend. +// +// It is declared here rather than in the build-tagged file so callers — and +// tests — can refer to it in either build. +var ErrNoNVML = errors.New("built without the nvml tag; rebuild with -tags nvml for NVLink topology") + +// Options configure a full inventory scan. +type Options struct { + // NvidiaSMI is the path to nvidia-smi; empty means look it up on PATH. + NvidiaSMI string + // IBRoot is the sysfs InfiniBand directory. + IBRoot string + // PreferNVML uses the NVML backend when this binary has one. It is the only + // source of NVLink topology. + PreferNVML bool + // SkipGPUs omits GPU discovery, for nodes that have none. + SkipGPUs bool +} + +// Discover builds the node's inventory. +// +// When NVML is preferred but fails, it falls back to nvidia-smi and says so in +// Inventory.Source. The fallback is logged rather than silent: on a node that +// is supposed to have working NVML, a quiet downgrade hides a broken driver and +// costs the scheduler its NVLink topology. +func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, error) { + if log == nil { + log = slog.Default() + } + inv := Inventory{Source: "none"} + + if !opts.SkipGPUs { + switch { + case opts.PreferNVML && nvmlAvailable: + gpus, err := DiscoverGPUsNVML(ctx) + if err == nil { + inv.GPUs, inv.Source = gpus, "nvml" + break + } + log.Warn("NVML discovery failed, falling back to nvidia-smi; "+ + "NVLink topology will be unavailable", "err", err) + fallthrough + default: + gpus, err := DiscoverGPUsSMI(ctx, opts.NvidiaSMI) + if err != nil { + return inv, fmt.Errorf("gpu discovery: %w", err) + } + inv.GPUs, inv.Source = gpus, "nvidia-smi" + } + } + + hcas, err := DiscoverIB(opts.IBRoot) + if err != nil { + return inv, fmt.Errorf("infiniband discovery: %w", err) + } + inv.HCAs = hcas + return inv, nil +} + +// NVMLSupported reports whether this binary includes the NVML backend. +func NVMLSupported() bool { return nvmlAvailable } diff --git a/internal/agent/discovery/discover_test.go b/internal/agent/discovery/discover_test.go new file mode 100644 index 0000000..6791b5c --- /dev/null +++ b/internal/agent/discovery/discover_test.go @@ -0,0 +1,101 @@ +package discovery + +import ( + "context" + "log/slog" + "os" + "path/filepath" + "testing" +) + +func fakeSMI(t *testing.T) string { + t.Helper() + stub := filepath.Join(t.TempDir(), "nvidia-smi") + script := "#!/bin/sh\ncat <<'EOF'\n" + smiSample + "EOF\n" + if err := os.WriteFile(stub, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + return stub +} + +func TestDiscoverUsesSMIByDefault(t *testing.T) { + inv, err := Discover(t.Context(), Options{ + NvidiaSMI: fakeSMI(t), + IBRoot: filepath.Join(t.TempDir(), "absent"), + }, slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatal(err) + } + if inv.Source != "nvidia-smi" { + t.Errorf("source=%q", inv.Source) + } + if len(inv.GPUs) != 2 { + t.Errorf("got %d GPUs", len(inv.GPUs)) + } +} + +func TestDiscoverFallsBackWhenNVMLUnavailable(t *testing.T) { + // Asking for NVML in a build without it must still produce an inventory: + // the fallback is the whole point of the build tag. + inv, err := Discover(t.Context(), Options{ + PreferNVML: true, + NvidiaSMI: fakeSMI(t), + IBRoot: filepath.Join(t.TempDir(), "absent"), + }, slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatal(err) + } + if len(inv.GPUs) == 0 { + t.Fatal("no GPUs discovered through the fallback") + } + if NVMLSupported() { + // On an NVML build without a driver the NVML attempt fails and the + // fallback still has to work. + if inv.Source != "nvml" && inv.Source != "nvidia-smi" { + t.Errorf("source=%q", inv.Source) + } + } else if inv.Source != "nvidia-smi" { + t.Errorf("a build without NVML reported source=%q", inv.Source) + } +} + +func TestDiscoverSkipGPUs(t *testing.T) { + inv, err := Discover(t.Context(), Options{ + SkipGPUs: true, + IBRoot: fakeSysfs(t), + }, slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatal(err) + } + if len(inv.GPUs) != 0 { + t.Errorf("GPUs were discovered despite SkipGPUs") + } + if len(inv.HCAs) != 3 { + t.Errorf("got %d HCAs", len(inv.HCAs)) + } + if inv.Source != "none" { + t.Errorf("source=%q", inv.Source) + } +} + +func TestDiscoverReportsGPUFailure(t *testing.T) { + // A node meant to have GPUs whose discovery fails must say so rather than + // reporting an empty, healthy-looking inventory. + _, err := Discover(context.Background(), Options{ + NvidiaSMI: filepath.Join(t.TempDir(), "absent"), + IBRoot: filepath.Join(t.TempDir(), "absent"), + }, slog.New(slog.DiscardHandler)) + if err == nil { + t.Error("a failed GPU discovery was reported as success") + } +} + +func TestNVMLSupportedMatchesBuild(t *testing.T) { + // Documents which backend this binary has, and keeps both files compiling. + t.Logf("NVML backend compiled in: %v", NVMLSupported()) + if !NVMLSupported() { + if _, err := DiscoverGPUsNVML(t.Context()); err != ErrNoNVML { + t.Errorf("expected ErrNoNVML, got %v", err) + } + } +} diff --git a/internal/agent/discovery/gpu_nonvml.go b/internal/agent/discovery/gpu_nonvml.go new file mode 100644 index 0000000..2ff1cfa --- /dev/null +++ b/internal/agent/discovery/gpu_nonvml.go @@ -0,0 +1,11 @@ +//go:build !nvml + +package discovery + +import "context" + +// nvmlAvailable reports that this build cannot use NVML. +const nvmlAvailable = false + +// DiscoverGPUsNVML is unavailable in builds without the nvml tag. +func DiscoverGPUsNVML(context.Context) ([]GPU, error) { return nil, ErrNoNVML } diff --git a/internal/agent/discovery/gpu_nvml.go b/internal/agent/discovery/gpu_nvml.go new file mode 100644 index 0000000..f88c934 --- /dev/null +++ b/internal/agent/discovery/gpu_nvml.go @@ -0,0 +1,148 @@ +//go:build nvml + +// This file is the NVML-backed GPU discovery path, built only with the `nvml` +// tag because the bindings require cgo. Without the tag the agent falls back to +// nvidia-smi, which keeps the server side — and development machines without a +// C toolchain or a driver — building cleanly (plan §3.2). +// +// What NVML adds over nvidia-smi is NVLink topology. Nothing else exposes which +// GPUs share an NVLink domain, and that is what keeps a tensor-parallel group +// off the slow path inside a node (plan §6.2, constraint 3). + +package discovery + +import ( + "context" + "fmt" + "strings" + + "github.com/NVIDIA/go-nvml/pkg/nvml" +) + +// nvmlAvailable reports that this build can use NVML. +const nvmlAvailable = true + +// DiscoverGPUsNVML enumerates GPUs through NVML. +// +// It returns an error rather than falling back on its own: the caller decides +// whether to degrade to nvidia-smi, because on a node that is supposed to have +// working NVML a silent downgrade hides a broken driver installation. +func DiscoverGPUsNVML(_ context.Context) ([]GPU, error) { + if ret := nvml.Init(); ret != nvml.SUCCESS { + return nil, fmt.Errorf("nvml init: %s", nvml.ErrorString(ret)) + } + defer func() { _ = nvml.Shutdown() }() + + driver, _ := nvml.SystemGetDriverVersion() + cuda := "" + if v, ret := nvml.SystemGetCudaDriverVersion(); ret == nvml.SUCCESS { + // NVML reports 12040 for 12.4. + cuda = fmt.Sprintf("%d.%d", v/1000, (v%1000)/10) + } + + count, ret := nvml.DeviceGetCount() + if ret != nvml.SUCCESS { + return nil, fmt.Errorf("nvml device count: %s", nvml.ErrorString(ret)) + } + + // Bus id to index, so NVLink peers can be reported as indices. NVML gives + // the remote end of a link as a PCI address, which is not what placement + // works in. + byBusID := make(map[string]int, count) + gpus := make([]GPU, 0, count) + + for i := range count { + dev, ret := nvml.DeviceGetHandleByIndex(i) + if ret != nvml.SUCCESS { + return nil, fmt.Errorf("nvml device %d: %s", i, nvml.ErrorString(ret)) + } + + g := GPU{Index: i, NUMANode: -1, DriverVersion: driver, CUDAVersion: cuda} + g.UUID, _ = dev.GetUUID() + g.Name, _ = dev.GetName() + + if mem, ret := dev.GetMemoryInfo(); ret == nvml.SUCCESS { + g.MemoryTotalMB = int(mem.Total / (1 << 20)) + g.MemoryUsedMB = int(mem.Used / (1 << 20)) + } + if major, minor, ret := dev.GetCudaComputeCapability(); ret == nvml.SUCCESS { + g.ComputeCapability = fmt.Sprintf("%d.%d", major, minor) + } + if pci, ret := dev.GetPciInfo(); ret == nvml.SUCCESS { + g.PCIBusID = busIDString(pci.BusId[:]) + byBusID[normalizeBusID(g.PCIBusID)] = i + } + if t, ret := dev.GetTemperature(nvml.TEMPERATURE_GPU); ret == nvml.SUCCESS { + g.Health.TemperatureC = int(t) + } + // Volatile counters reset on driver reload, which is what we want: an + // error from three months and one reboot ago is not a reason to refuse + // a card today. + if n, ret := dev.GetTotalEccErrors(nvml.MEMORY_ERROR_TYPE_UNCORRECTED, nvml.VOLATILE_ECC); ret == nvml.SUCCESS { + g.Health.ECCErrors = int(n) + } + + assessHealth(&g) + gpus = append(gpus, g) + } + + // A second pass, because a link's remote end may be a GPU enumerated later. + for i := range gpus { + dev, ret := nvml.DeviceGetHandleByIndex(gpus[i].Index) + if ret != nvml.SUCCESS { + continue + } + gpus[i].NVLinkPeers = nvlinkPeers(dev, byBusID, gpus[i].Index) + } + return gpus, nil +} + +// nvlinkPeers returns the indices of GPUs reachable from dev over NVLink. +func nvlinkPeers(dev nvml.Device, byBusID map[string]int, self int) []int { + seen := map[int]struct{}{} + var out []int + + for link := range nvml.NVLINK_MAX_LINKS { + pci, ret := dev.GetNvLinkRemotePciInfo(link) + if ret != nvml.SUCCESS { + // Unpopulated links return an error; that is the normal case for + // most of the 36 possible links, so it is not worth reporting. + continue + } + peer, ok := byBusID[normalizeBusID(busIDString(pci.BusId[:]))] + if !ok || peer == self { + // The remote end can be an NVSwitch rather than a GPU, in which + // case it is not in the map. + continue + } + if _, dup := seen[peer]; dup { + continue + } + seen[peer] = struct{}{} + out = append(out, peer) + } + return out +} + +// busIDString turns NVML's fixed-size C char array into a Go string. +func busIDString(b []int8) string { + buf := make([]byte, 0, len(b)) + for _, c := range b { + if c == 0 { + break + } + buf = append(buf, byte(c)) + } + return string(buf) +} + +// normalizeBusID makes bus ids comparable: NVML reports them uppercase from one +// call and lowercase from another, and sometimes with a shortened domain. +func normalizeBusID(s string) string { + s = strings.ToLower(strings.TrimSpace(s)) + // "0000:1b:00.0" and "00000000:1B:00.0" are the same device. + if i := strings.Index(s, ":"); i > 0 { + s = strings.TrimLeft(s[:i], "0") + s[i:] + } + return strings.TrimPrefix(s, ":") +} diff --git a/internal/cli/agent.go b/internal/cli/agent.go index fb08992..2971d0b 100644 --- a/internal/cli/agent.go +++ b/internal/cli/agent.go @@ -36,6 +36,7 @@ func newInventoryCmd() *cobra.Command { smiBin string ibRoot string skipGPU bool + useNVML bool ) cmd := &cobra.Command{ @@ -46,24 +47,18 @@ func newInventoryCmd() *cobra.Command { "Run it before adding a node to the cluster — it is the fastest way to find\n" + "a card another process is already using, or a port with no subnet manager.", RunE: func(cmd *cobra.Command, _ []string) error { - inv := discovery.Inventory{Source: "nvidia-smi"} - - if !skipGPU { - gpus, err := discovery.DiscoverGPUsSMI(cmd.Context(), smiBin) - if err != nil { - // Not fatal: a node with no GPUs still has IB worth showing, - // and the error itself is the useful output. - fmt.Fprintf(os.Stderr, "GPU discovery failed: %v\n", err) - inv.Source = "unavailable" - } - inv.GPUs = gpus - } - - hcas, err := discovery.DiscoverIB(ibRoot) + inv, err := discovery.Discover(cmd.Context(), discovery.Options{ + NvidiaSMI: smiBin, + IBRoot: ibRoot, + PreferNVML: useNVML, + SkipGPUs: skipGPU, + }, logger()) if err != nil { - return err + // Not fatal: a node whose GPUs cannot be read still has IB + // worth showing, and the error is itself the useful output. + fmt.Fprintf(os.Stderr, "%v\n", err) + inv.Source = "unavailable" } - inv.HCAs = hcas if asJSON { enc := json.NewEncoder(os.Stdout) @@ -80,6 +75,7 @@ func newInventoryCmd() *cobra.Command { f.StringVar(&smiBin, "nvidia-smi", "", "path to nvidia-smi") f.StringVar(&ibRoot, "ib-root", discovery.SysfsRoot, "sysfs InfiniBand directory") f.BoolVar(&skipGPU, "no-gpu", false, "skip GPU discovery") + f.BoolVar(&useNVML, "nvml", true, "prefer the NVML backend when this build has one (it is the only source of NVLink topology)") return cmd } @@ -87,7 +83,7 @@ func newInventoryCmd() *cobra.Command { func printInventory(inv discovery.Inventory) { w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) - fmt.Fprintf(w, "GPUs (%d)\n", len(inv.GPUs)) + fmt.Fprintf(w, "GPUs (%d, via %s)\n", len(inv.GPUs), inv.Source) if len(inv.GPUs) == 0 { fmt.Fprintln(w, " none") } else { @@ -101,6 +97,11 @@ func printInventory(inv discovery.Inventory) { g.Index, g.Class(), g.Name, g.MemoryTotalMB, g.ComputeCapability, health) } fmt.Fprintf(w, " classes: %v\n", inv.Classes()) + if !discovery.NVMLSupported() { + fmt.Fprintln(w, " NVLink topology unavailable: rebuild with -tags nvml.") + fmt.Fprintln(w, " Without it, placement cannot keep a tensor-parallel group inside") + fmt.Fprintln(w, " one NVLink domain (plan §6.2).") + } } fmt.Fprintf(w, "\nInfiniBand adapters (%d)\n", len(inv.HCAs)) diff --git a/llmgw-plan.md b/llmgw-plan.md index 26e03a9..0c64767 100644 --- a/llmgw-plan.md +++ b/llmgw-plan.md @@ -1002,8 +1002,9 @@ work_dir: /var/lib/llmgw-agent - [x] GPU discovery (nvidia-smi path), IB discovery from sysfs - [x] Engine interface, vLLM and SGLang command and environment construction - [x] Supervisor (process groups, log buffer, state machine, orphan adoption) -- [ ] NVML backend behind the `nvml` build tag — needs a machine with GPUs to verify -- [ ] NVLink topology (only NVML exposes it) +- [x] NVML backend behind the `nvml` build tag, with NVLink topology. It compiles + in CI but has not run against a driver: **verify on a GPU node before + relying on the NVLink domains it reports.** - [ ] connect-go stream, register/heartbeat, mTLS - **Exit criterion:** start and stop a model on a real node via `ctl`. Not met: the control plane connection is the remaining piece, and it pairs naturally