package discovery import ( "context" "encoding/xml" "fmt" "os/exec" "strconv" "strings" ) // GPUProcesses reports which processes hold memory on each GPU, by the GPU's // nvidia-smi index: what a process actually computes on, as opposed to what it // was told to use. A llama.cpp started with --n-gpu-layers on a GPU whose // backend did not load runs on the CPU without a word, and this is where that // shows — the process is on no GPU's list. // // From the full query rather than --query-compute-apps, which lists compute // processes only: llama.cpp's Vulkan build is listed as a graphics one, and // would have read as not on its GPU at all. // // A node with no nvidia-smi returns nil and no error, as discovery does. func GPUProcesses(ctx context.Context, binary string) (map[int][]int, error) { if binary == "" { if _, err := exec.LookPath("nvidia-smi"); err != nil { return nil, nil } binary = "nvidia-smi" } out, err := exec.CommandContext(ctx, binary, "-q", "-x").Output() if err != nil { return nil, fmt.Errorf("running %s: %w", binary, err) } return parseGPUProcesses(out) } // GPUMemory is one card's memory as nvidia-smi reports it at a moment: what // is allocated on it, and which processes hold how much of that. type GPUMemory struct { UsedMB int Procs []GPUProcess } // GPUProcess is one process holding memory on a card. type GPUProcess struct { PID int UsedMB int } // GPUMemoryByProcess reports each GPU's memory, by index, with the processes // holding it — from the same report as GPUProcesses, so the two never // disagree about who is on a card. // // A node with no nvidia-smi returns nil and no error, as discovery does. func GPUMemoryByProcess(ctx context.Context, binary string) ([]GPUMemory, error) { if binary == "" { if _, err := exec.LookPath("nvidia-smi"); err != nil { return nil, nil } binary = "nvidia-smi" } out, err := exec.CommandContext(ctx, binary, "-q", "-x").Output() if err != nil { return nil, fmt.Errorf("running %s: %w", binary, err) } return parseGPUMemory(out) } // smiLog is the part of nvidia-smi's XML report this reads: each GPU, in // index order, its memory and the processes on it. type smiLog struct { GPUs []struct { Used string `xml:"fb_memory_usage>used"` Processes []struct { PID string `xml:"pid"` Type string `xml:"type"` UsedMB string `xml:"used_memory"` } `xml:"processes>process_info"` } `xml:"gpu"` } // parseGPUProcesses reads the report. GPUs are listed in index order, which // is nvidia-smi's own numbering — the one placement assigns by. func parseGPUProcesses(report []byte) (map[int][]int, error) { mem, err := parseGPUMemory(report) if err != nil { return nil, err } out := map[int][]int{} for i, g := range mem { for _, p := range g.Procs { out[i] = append(out[i], p.PID) } } return out, nil } // parseGPUMemory reads the report into each GPU's memory and its holders. func parseGPUMemory(report []byte) ([]GPUMemory, error) { var log smiLog if err := xml.Unmarshal(report, &log); err != nil { return nil, fmt.Errorf("reading nvidia-smi's report: %w", err) } out := make([]GPUMemory, len(log.GPUs)) for i, g := range log.GPUs { out[i].UsedMB = mib(g.Used) for _, p := range g.Processes { if pid, err := strconv.Atoi(strings.TrimSpace(p.PID)); err == nil { out[i].Procs = append(out[i].Procs, GPUProcess{PID: pid, UsedMB: mib(p.UsedMB)}) } } } return out, nil } // mib reads "4932 MiB"; anything else, "N/A" included, is zero. func mib(s string) int { n, _ := strconv.Atoi(strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(s), "MiB"))) return n }