Each engine has a list of builds, best first, from what its project publishes and how each was compiled: llama.cpp CUDA 13.3, CUDA 12.8 (driver 570 for the GPUs it ships as PTX), then Vulkan, never the CPU build on an NVIDIA node; vLLM CUDA 13, then CUDA 12.9, and for GPUs older than every build of the version asked for, the newest built for them (V100: 0.19.1). The node takes the first its hardware meets and that installs and computes on the GPU, moving on when one does not, and reports the version it runs. GPU processes are read from nvidia-smi's full report, so Vulkan engines count as on their GPU. Refs #2. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
65 lines
1.9 KiB
Go
65 lines
1.9 KiB
Go
package discovery
|
|
|
|
import (
|
|
"context"
|
|
"encoding/xml"
|
|
"fmt"
|
|
"os/exec"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// GPUProcesses reports which processes hold memory on each GPU, by the GPU's
|
|
// nvidia-smi index: what a process actually computes on, as opposed to what it
|
|
// was told to use. A llama.cpp started with --n-gpu-layers on a GPU whose
|
|
// backend did not load runs on the CPU without a word, and this is where that
|
|
// shows — the process is on no GPU's list.
|
|
//
|
|
// From the full query rather than --query-compute-apps, which lists compute
|
|
// processes only: llama.cpp's Vulkan build is listed as a graphics one, and
|
|
// would have read as not on its GPU at all.
|
|
//
|
|
// A node with no nvidia-smi returns nil and no error, as discovery does.
|
|
func GPUProcesses(ctx context.Context, binary string) (map[int][]int, error) {
|
|
if binary == "" {
|
|
if _, err := exec.LookPath("nvidia-smi"); err != nil {
|
|
return nil, nil
|
|
}
|
|
binary = "nvidia-smi"
|
|
}
|
|
out, err := exec.CommandContext(ctx, binary, "-q", "-x").Output()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("running %s: %w", binary, err)
|
|
}
|
|
return parseGPUProcesses(out)
|
|
}
|
|
|
|
// smiLog is the part of nvidia-smi's XML report this reads: each GPU, in
|
|
// index order, and the processes on it.
|
|
type smiLog struct {
|
|
GPUs []struct {
|
|
Processes []struct {
|
|
PID string `xml:"pid"`
|
|
Type string `xml:"type"`
|
|
} `xml:"processes>process_info"`
|
|
} `xml:"gpu"`
|
|
}
|
|
|
|
// parseGPUProcesses reads the report. GPUs are listed in index order, which
|
|
// is nvidia-smi's own numbering — the one placement assigns by.
|
|
func parseGPUProcesses(report []byte) (map[int][]int, error) {
|
|
var log smiLog
|
|
if err := xml.Unmarshal(report, &log); err != nil {
|
|
return nil, fmt.Errorf("reading nvidia-smi's report: %w", err)
|
|
}
|
|
out := map[int][]int{}
|
|
for i, g := range log.GPUs {
|
|
for _, p := range g.Processes {
|
|
if pid, err := strconv.Atoi(strings.TrimSpace(p.PID)); err == nil {
|
|
out[i] = append(out[i], pid)
|
|
}
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|