package discovery import ( "context" "encoding/csv" "fmt" "os/exec" "strconv" "strings" ) // smiQueryFields are the columns requested from nvidia-smi, in order. var smiQueryFields = []string{ "index", "uuid", "name", "memory.total", "memory.used", "compute_cap", "pci.bus_id", "driver_version", "temperature.gpu", "ecc.errors.uncorrected.volatile.total", // Last, and optional. An nvidia-smi too old to know this field returns a // shorter row, and every column before it still lines up — which is why it // goes on the end rather than anywhere else, and why the parser requires // only the mandatory prefix. "utilization.gpu", } // mandatorySMIFields is how many columns a row must have to be usable. // // Everything before utilization.gpu. A driver that does not know the last field // gives a short row, and rejecting those would make an llmgw upgrade silently // stop discovering GPUs on an older node — a far worse outcome than not knowing // how busy a card is. const mandatorySMIFields = 10 // DiscoverGPUsSMI enumerates GPUs by shelling out to nvidia-smi. // // This is the cgo-free path (plan §3.2): the NVML bindings need cgo, which // would force the whole binary — server included — to be built with a C // toolchain. The query gives everything except topology, which Discover // reads from `nvidia-smi topo -m` afterwards. func DiscoverGPUsSMI(ctx context.Context, binary string) ([]GPU, error) { if binary == "" { // Not configured and not installed is a node without an NVIDIA card, // not a failed discovery. Reported as an error it also cut discovery // short before InfiniBand was looked at, on every CPU node that had // not been told by hand to skip GPUs. if _, err := exec.LookPath("nvidia-smi"); err != nil { return nil, nil } binary = "nvidia-smi" } out, err := exec.CommandContext(ctx, binary, "--query-gpu="+strings.Join(smiQueryFields, ","), "--format=csv,noheader,nounits", ).Output() if err != nil { return nil, fmt.Errorf("running %s: %w", binary, err) } return parseSMI(string(out)) } func parseSMI(out string) ([]GPU, error) { r := csv.NewReader(strings.NewReader(out)) r.TrimLeadingSpace = true r.FieldsPerRecord = -1 // tolerate driver versions with extra columns rows, err := r.ReadAll() if err != nil { return nil, fmt.Errorf("parsing nvidia-smi output: %w", err) } gpus := make([]GPU, 0, len(rows)) for _, row := range rows { if len(row) < mandatorySMIFields { continue } idx, err := strconv.Atoi(strings.TrimSpace(row[0])) if err != nil { continue } g := GPU{ Index: idx, UUID: strings.TrimSpace(row[1]), Name: strings.TrimSpace(row[2]), ComputeCapability: strings.TrimSpace(row[5]), PCIBusID: strings.TrimSpace(row[6]), DriverVersion: strings.TrimSpace(row[7]), NUMANode: -1, } g.MemoryTotalMB, _ = parseCSVInt(row[3]) g.MemoryUsedMB, _ = parseCSVInt(row[4]) g.Health.TemperatureC, _ = parseCSVInt(row[8]) g.Health.ECCErrors, _ = parseCSVInt(row[9]) if len(row) > 10 { g.UtilizationPercent, _ = parseCSVInt(row[10]) } assessHealth(&g) gpus = append(gpus, g) } return gpus, nil } // residentMemoryToleranceMB is how much allocated memory is written off as // driver overhead rather than a foreign process. const residentMemoryToleranceMB = 512 // assessHealth flags conditions that should keep a GPU out of placement. func assessHealth(g *GPU) { if g.Health.ECCErrors > 0 { g.Health.Problems = append(g.Health.Problems, fmt.Sprintf("%d uncorrected ECC errors", g.Health.ECCErrors)) } // Memory already in use means something else is on the card. Starting an // engine there would either fail to allocate or quietly evict the other // process's throughput. if g.MemoryUsedMB > residentMemoryToleranceMB { g.Health.Problems = append(g.Health.Problems, fmt.Sprintf("%d MiB already allocated by another process", g.MemoryUsedMB)) } }