Files
llmgw/internal/agent/discovery/gpuprocs.go
T
Ilker ManapandClaude Opus 5.5 dc483d8c68 Choose each engine's build from the node's hardware, for every GPU generation
Each engine has a list of builds, best first, from what its project
publishes and how each was compiled: llama.cpp CUDA 13.3, CUDA 12.8
(driver 570 for the GPUs it ships as PTX), then Vulkan, never the CPU
build on an NVIDIA node; vLLM CUDA 13, then CUDA 12.9, and for GPUs
older than every build of the version asked for, the newest built for
them (V100: 0.19.1). The node takes the first its hardware meets and
that installs and computes on the GPU, moving on when one does not, and
reports the version it runs. GPU processes are read from nvidia-smi's
full report, so Vulkan engines count as on their GPU.

Refs #2.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-01 14:35:58 +02:00

65 lines
1.9 KiB
Go

package discovery
import (
"context"
"encoding/xml"
"fmt"
"os/exec"
"strconv"
"strings"
)
// GPUProcesses reports which processes hold memory on each GPU, by the GPU's
// nvidia-smi index: what a process actually computes on, as opposed to what it
// was told to use. A llama.cpp started with --n-gpu-layers on a GPU whose
// backend did not load runs on the CPU without a word, and this is where that
// shows — the process is on no GPU's list.
//
// From the full query rather than --query-compute-apps, which lists compute
// processes only: llama.cpp's Vulkan build is listed as a graphics one, and
// would have read as not on its GPU at all.
//
// A node with no nvidia-smi returns nil and no error, as discovery does.
func GPUProcesses(ctx context.Context, binary string) (map[int][]int, error) {
if binary == "" {
if _, err := exec.LookPath("nvidia-smi"); err != nil {
return nil, nil
}
binary = "nvidia-smi"
}
out, err := exec.CommandContext(ctx, binary, "-q", "-x").Output()
if err != nil {
return nil, fmt.Errorf("running %s: %w", binary, err)
}
return parseGPUProcesses(out)
}
// smiLog is the part of nvidia-smi's XML report this reads: each GPU, in
// index order, and the processes on it.
type smiLog struct {
GPUs []struct {
Processes []struct {
PID string `xml:"pid"`
Type string `xml:"type"`
} `xml:"processes>process_info"`
} `xml:"gpu"`
}
// parseGPUProcesses reads the report. GPUs are listed in index order, which
// is nvidia-smi's own numbering — the one placement assigns by.
func parseGPUProcesses(report []byte) (map[int][]int, error) {
var log smiLog
if err := xml.Unmarshal(report, &log); err != nil {
return nil, fmt.Errorf("reading nvidia-smi's report: %w", err)
}
out := map[int][]int{}
for i, g := range log.GPUs {
for _, p := range g.Processes {
if pid, err := strconv.Atoi(strings.TrimSpace(p.PID)); err == nil {
out[i] = append(out[i], pid)
}
}
}
return out, nil
}