A card with something else on it when the agent started stayed out of placement after that process let go of it, until the agent restarted. The agent now sends what processes other than its own engines hold on each card, and the control plane replaces the registration verdict with it. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
121 lines
3.6 KiB
Go
121 lines
3.6 KiB
Go
package discovery
|
|
|
|
import (
|
|
"context"
|
|
"encoding/xml"
|
|
"fmt"
|
|
"os/exec"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// GPUProcesses reports which processes hold memory on each GPU, by the GPU's
|
|
// nvidia-smi index: what a process actually computes on, as opposed to what it
|
|
// was told to use. A llama.cpp started with --n-gpu-layers on a GPU whose
|
|
// backend did not load runs on the CPU without a word, and this is where that
|
|
// shows — the process is on no GPU's list.
|
|
//
|
|
// From the full query rather than --query-compute-apps, which lists compute
|
|
// processes only: llama.cpp's Vulkan build is listed as a graphics one, and
|
|
// would have read as not on its GPU at all.
|
|
//
|
|
// A node with no nvidia-smi returns nil and no error, as discovery does.
|
|
func GPUProcesses(ctx context.Context, binary string) (map[int][]int, error) {
|
|
if binary == "" {
|
|
if _, err := exec.LookPath("nvidia-smi"); err != nil {
|
|
return nil, nil
|
|
}
|
|
binary = "nvidia-smi"
|
|
}
|
|
out, err := exec.CommandContext(ctx, binary, "-q", "-x").Output()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("running %s: %w", binary, err)
|
|
}
|
|
return parseGPUProcesses(out)
|
|
}
|
|
|
|
// GPUMemory is one card's memory as nvidia-smi reports it at a moment: what
|
|
// is allocated on it, and which processes hold how much of that.
|
|
type GPUMemory struct {
|
|
UsedMB int
|
|
Procs []GPUProcess
|
|
}
|
|
|
|
// GPUProcess is one process holding memory on a card.
|
|
type GPUProcess struct {
|
|
PID int
|
|
UsedMB int
|
|
}
|
|
|
|
// GPUMemoryByProcess reports each GPU's memory, by index, with the processes
|
|
// holding it — from the same report as GPUProcesses, so the two never
|
|
// disagree about who is on a card.
|
|
//
|
|
// A node with no nvidia-smi returns nil and no error, as discovery does.
|
|
func GPUMemoryByProcess(ctx context.Context, binary string) ([]GPUMemory, error) {
|
|
if binary == "" {
|
|
if _, err := exec.LookPath("nvidia-smi"); err != nil {
|
|
return nil, nil
|
|
}
|
|
binary = "nvidia-smi"
|
|
}
|
|
out, err := exec.CommandContext(ctx, binary, "-q", "-x").Output()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("running %s: %w", binary, err)
|
|
}
|
|
return parseGPUMemory(out)
|
|
}
|
|
|
|
// smiLog is the part of nvidia-smi's XML report this reads: each GPU, in
|
|
// index order, its memory and the processes on it.
|
|
type smiLog struct {
|
|
GPUs []struct {
|
|
Used string `xml:"fb_memory_usage>used"`
|
|
Processes []struct {
|
|
PID string `xml:"pid"`
|
|
Type string `xml:"type"`
|
|
UsedMB string `xml:"used_memory"`
|
|
} `xml:"processes>process_info"`
|
|
} `xml:"gpu"`
|
|
}
|
|
|
|
// parseGPUProcesses reads the report. GPUs are listed in index order, which
|
|
// is nvidia-smi's own numbering — the one placement assigns by.
|
|
func parseGPUProcesses(report []byte) (map[int][]int, error) {
|
|
mem, err := parseGPUMemory(report)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out := map[int][]int{}
|
|
for i, g := range mem {
|
|
for _, p := range g.Procs {
|
|
out[i] = append(out[i], p.PID)
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// parseGPUMemory reads the report into each GPU's memory and its holders.
|
|
func parseGPUMemory(report []byte) ([]GPUMemory, error) {
|
|
var log smiLog
|
|
if err := xml.Unmarshal(report, &log); err != nil {
|
|
return nil, fmt.Errorf("reading nvidia-smi's report: %w", err)
|
|
}
|
|
out := make([]GPUMemory, len(log.GPUs))
|
|
for i, g := range log.GPUs {
|
|
out[i].UsedMB = mib(g.Used)
|
|
for _, p := range g.Processes {
|
|
if pid, err := strconv.Atoi(strings.TrimSpace(p.PID)); err == nil {
|
|
out[i].Procs = append(out[i].Procs, GPUProcess{PID: pid, UsedMB: mib(p.UsedMB)})
|
|
}
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// mib reads "4932 MiB"; anything else, "N/A" included, is zero.
|
|
func mib(s string) int {
|
|
n, _ := strconv.Atoi(strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(s), "MiB")))
|
|
return n
|
|
}
|