llama.cpp whose CUDA backend cannot load runs on the CPU without an error. The CUDA build is now asked for its devices when installed, a shard on GPUs must hold memory on each before it is ready, and engine-check reports the devices. Shards get CUDA_DEVICE_ORDER=PCI_BUS_ID so CUDA numbers GPUs as placement does. Refs #2 (item 2). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
66 lines
1.9 KiB
Go
66 lines
1.9 KiB
Go
package discovery
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os/exec"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// GPUProcesses reports which processes hold memory on each GPU, by the GPU's
|
|
// nvidia-smi index: what a process actually computes on, as opposed to what it
|
|
// was told to use. A llama.cpp started with --n-gpu-layers on a GPU whose CUDA
|
|
// backend did not load runs on the CPU without a word, and this is where that
|
|
// shows — the process is on no GPU's list.
|
|
//
|
|
// A node with no nvidia-smi returns nil and no error, as discovery does.
|
|
func GPUProcesses(ctx context.Context, binary string) (map[int][]int, error) {
|
|
if binary == "" {
|
|
if _, err := exec.LookPath("nvidia-smi"); err != nil {
|
|
return nil, nil
|
|
}
|
|
binary = "nvidia-smi"
|
|
}
|
|
gpus, err := exec.CommandContext(ctx, binary, "--query-gpu=index,uuid", "--format=csv,noheader").Output()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("running %s: %w", binary, err)
|
|
}
|
|
apps, err := exec.CommandContext(ctx, binary, "--query-compute-apps=gpu_uuid,pid", "--format=csv,noheader").Output()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("running %s: %w", binary, err)
|
|
}
|
|
return parseGPUProcesses(string(gpus), string(apps)), nil
|
|
}
|
|
|
|
// parseGPUProcesses joins nvidia-smi's GPU list (index, uuid) to its compute
|
|
// apps (gpu_uuid, pid): the apps are listed by UUID, placement speaks index.
|
|
func parseGPUProcesses(gpus, apps string) map[int][]int {
|
|
index := map[string]int{}
|
|
for _, line := range strings.Split(gpus, "\n") {
|
|
i, uuid, ok := strings.Cut(line, ",")
|
|
if !ok {
|
|
continue
|
|
}
|
|
n, err := strconv.Atoi(strings.TrimSpace(i))
|
|
if err != nil {
|
|
continue
|
|
}
|
|
index[strings.TrimSpace(uuid)] = n
|
|
}
|
|
out := map[int][]int{}
|
|
for _, line := range strings.Split(apps, "\n") {
|
|
uuid, p, ok := strings.Cut(line, ",")
|
|
if !ok {
|
|
continue
|
|
}
|
|
pid, err := strconv.Atoi(strings.TrimSpace(p))
|
|
i, known := index[strings.TrimSpace(uuid)]
|
|
if err != nil || !known {
|
|
continue
|
|
}
|
|
out[i] = append(out[i], pid)
|
|
}
|
|
return out
|
|
}
|