Files
llmgw/internal/agent/discovery/gpu_smi.go
T
Ilker ManapandClaude Opus 5.5 20deb22dfa Let a node describe itself instead of being described by hand
Every agent file on the cluster named its own node, its own address, the
path to llama-server and two weight directories — different on every node,
kept true by hand, and not true: node05 carried "ram: 32g" beside a measured
31 GB, looked for weights in an empty directory while fetching them into
another, and the other four never looked in theirs at all, so four nodes
reported no disk. Growing a node's RAM would have gone unnoticed, because
memory was read once, at registration.

The agent now finds these out. Its id is the hostname, which is also the
certificate's common name. Its address is the one on the route to the
control plane, which the kernel picks for a UDP dial without sending
anything. llama-server is looked for on PATH and then where release archives
are unpacked. A missing nvidia-smi is a node without NVIDIA cards rather
than a failed discovery — which also cut discovery short before InfiniBand
on every CPU node not told to skip GPUs. Fetched weights land in
/var/lib/llmgw-weights by default, and wherever they land is always looked
in.

Memory is read on every heartbeat, and installed engines are probed again
each minute and sent when they change, so a VM given more RAM or a node
given vLLM becomes a candidate without anyone restarting its agent. The
control plane logs both changes, since "the node now qualifies" is the
answer to why a model placed an hour after it was asked for.

What is written still wins. The agent file shrinks to the control plane's
addresses and the TLS files, the same on every node; onboarding writes only
the overrides it was given. What cannot be found stays configuration: which
fabric an adapter is on, because OpenSM's default prefix is the same on all
of them.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-28 10:13:42 +02:00

127 lines
3.9 KiB
Go

package discovery
import (
"context"
"encoding/csv"
"fmt"
"os/exec"
"strconv"
"strings"
)
// smiQueryFields are the columns requested from nvidia-smi, in order.
var smiQueryFields = []string{
"index",
"uuid",
"name",
"memory.total",
"memory.used",
"compute_cap",
"pci.bus_id",
"driver_version",
"temperature.gpu",
"ecc.errors.uncorrected.volatile.total",
// Last, and optional. An nvidia-smi too old to know this field returns a
// shorter row, and every column before it still lines up — which is why it
// goes on the end rather than anywhere else, and why the parser requires
// only the mandatory prefix.
"utilization.gpu",
}
// mandatorySMIFields is how many columns a row must have to be usable.
//
// Everything before utilization.gpu. A driver that does not know the last field
// gives a short row, and rejecting those would make an llmgw upgrade silently
// stop discovering GPUs on an older node — a far worse outcome than not knowing
// how busy a card is.
const mandatorySMIFields = 10
// DiscoverGPUsSMI enumerates GPUs by shelling out to nvidia-smi.
//
// This is the cgo-free path (plan §3.2): the NVML bindings need cgo, which
// would force the whole binary — server included — to be built with a C
// toolchain. nvidia-smi gives everything except NVLink topology, which the
// NVML build adds.
func DiscoverGPUsSMI(ctx context.Context, binary string) ([]GPU, error) {
if binary == "" {
// Not configured and not installed is a node without an NVIDIA card,
// not a failed discovery. Reported as an error it also cut discovery
// short before InfiniBand was looked at, on every CPU node that had
// not been told by hand to skip GPUs.
if _, err := exec.LookPath("nvidia-smi"); err != nil {
return nil, nil
}
binary = "nvidia-smi"
}
out, err := exec.CommandContext(ctx, binary,
"--query-gpu="+strings.Join(smiQueryFields, ","),
"--format=csv,noheader,nounits",
).Output()
if err != nil {
return nil, fmt.Errorf("running %s: %w", binary, err)
}
return parseSMI(string(out))
}
func parseSMI(out string) ([]GPU, error) {
r := csv.NewReader(strings.NewReader(out))
r.TrimLeadingSpace = true
r.FieldsPerRecord = -1 // tolerate driver versions with extra columns
rows, err := r.ReadAll()
if err != nil {
return nil, fmt.Errorf("parsing nvidia-smi output: %w", err)
}
gpus := make([]GPU, 0, len(rows))
for _, row := range rows {
if len(row) < mandatorySMIFields {
continue
}
idx, err := strconv.Atoi(strings.TrimSpace(row[0]))
if err != nil {
continue
}
g := GPU{
Index: idx,
UUID: strings.TrimSpace(row[1]),
Name: strings.TrimSpace(row[2]),
ComputeCapability: strings.TrimSpace(row[5]),
PCIBusID: strings.TrimSpace(row[6]),
DriverVersion: strings.TrimSpace(row[7]),
NUMANode: -1,
}
g.MemoryTotalMB, _ = parseCSVInt(row[3])
g.MemoryUsedMB, _ = parseCSVInt(row[4])
g.Health.TemperatureC, _ = parseCSVInt(row[8])
g.Health.ECCErrors, _ = parseCSVInt(row[9])
if len(row) > 10 {
g.UtilizationPercent, _ = parseCSVInt(row[10])
}
assessHealth(&g)
gpus = append(gpus, g)
}
return gpus, nil
}
// residentMemoryToleranceMB is how much allocated memory is written off as
// driver overhead rather than a foreign process.
const residentMemoryToleranceMB = 512
// assessHealth flags conditions that should keep a GPU out of placement.
func assessHealth(g *GPU) {
if g.Health.ECCErrors > 0 {
g.Health.Problems = append(g.Health.Problems,
fmt.Sprintf("%d uncorrected ECC errors", g.Health.ECCErrors))
}
// Memory already in use means something else is on the card. Starting an
// engine there would either fail to allocate or quietly evict the other
// process's throughput.
if g.MemoryUsedMB > residentMemoryToleranceMB {
g.Health.Problems = append(g.Health.Problems,
fmt.Sprintf("%d MiB already allocated by another process", g.MemoryUsedMB))
}
}