Files
llmgw/internal/agent/discovery/gpuprocs.go
T
Ilker ManapandClaude Opus 5.5 477d0bf19d
Release / build (push) Successful in 1m14s
Release / notes (push) Successful in 4s
CI / test (push) Successful in 3m50s
Judge a card shared with another process every heartbeat, not once
A card with something else on it when the agent started stayed out of
placement after that process let go of it, until the agent restarted. The
agent now sends what processes other than its own engines hold on each card,
and the control plane replaces the registration verdict with it.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-03 18:46:23 +02:00

121 lines
3.6 KiB
Go

package discovery
import (
"context"
"encoding/xml"
"fmt"
"os/exec"
"strconv"
"strings"
)
// GPUProcesses reports which processes hold memory on each GPU, by the GPU's
// nvidia-smi index: what a process actually computes on, as opposed to what it
// was told to use. A llama.cpp started with --n-gpu-layers on a GPU whose
// backend did not load runs on the CPU without a word, and this is where that
// shows — the process is on no GPU's list.
//
// From the full query rather than --query-compute-apps, which lists compute
// processes only: llama.cpp's Vulkan build is listed as a graphics one, and
// would have read as not on its GPU at all.
//
// A node with no nvidia-smi returns nil and no error, as discovery does.
func GPUProcesses(ctx context.Context, binary string) (map[int][]int, error) {
if binary == "" {
if _, err := exec.LookPath("nvidia-smi"); err != nil {
return nil, nil
}
binary = "nvidia-smi"
}
out, err := exec.CommandContext(ctx, binary, "-q", "-x").Output()
if err != nil {
return nil, fmt.Errorf("running %s: %w", binary, err)
}
return parseGPUProcesses(out)
}
// GPUMemory is one card's memory as nvidia-smi reports it at a moment: what
// is allocated on it, and which processes hold how much of that.
type GPUMemory struct {
UsedMB int
Procs []GPUProcess
}
// GPUProcess is one process holding memory on a card.
type GPUProcess struct {
PID int
UsedMB int
}
// GPUMemoryByProcess reports each GPU's memory, by index, with the processes
// holding it — from the same report as GPUProcesses, so the two never
// disagree about who is on a card.
//
// A node with no nvidia-smi returns nil and no error, as discovery does.
func GPUMemoryByProcess(ctx context.Context, binary string) ([]GPUMemory, error) {
if binary == "" {
if _, err := exec.LookPath("nvidia-smi"); err != nil {
return nil, nil
}
binary = "nvidia-smi"
}
out, err := exec.CommandContext(ctx, binary, "-q", "-x").Output()
if err != nil {
return nil, fmt.Errorf("running %s: %w", binary, err)
}
return parseGPUMemory(out)
}
// smiLog is the part of nvidia-smi's XML report this reads: each GPU, in
// index order, its memory and the processes on it.
type smiLog struct {
GPUs []struct {
Used string `xml:"fb_memory_usage>used"`
Processes []struct {
PID string `xml:"pid"`
Type string `xml:"type"`
UsedMB string `xml:"used_memory"`
} `xml:"processes>process_info"`
} `xml:"gpu"`
}
// parseGPUProcesses reads the report. GPUs are listed in index order, which
// is nvidia-smi's own numbering — the one placement assigns by.
func parseGPUProcesses(report []byte) (map[int][]int, error) {
mem, err := parseGPUMemory(report)
if err != nil {
return nil, err
}
out := map[int][]int{}
for i, g := range mem {
for _, p := range g.Procs {
out[i] = append(out[i], p.PID)
}
}
return out, nil
}
// parseGPUMemory reads the report into each GPU's memory and its holders.
func parseGPUMemory(report []byte) ([]GPUMemory, error) {
var log smiLog
if err := xml.Unmarshal(report, &log); err != nil {
return nil, fmt.Errorf("reading nvidia-smi's report: %w", err)
}
out := make([]GPUMemory, len(log.GPUs))
for i, g := range log.GPUs {
out[i].UsedMB = mib(g.Used)
for _, p := range g.Processes {
if pid, err := strconv.Atoi(strings.TrimSpace(p.PID)); err == nil {
out[i].Procs = append(out[i].Procs, GPUProcess{PID: pid, UsedMB: mib(p.UsedMB)})
}
}
}
return out, nil
}
// mib reads "4932 MiB"; anything else, "N/A" included, is zero.
func mib(s string) int {
n, _ := strconv.Atoi(strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(s), "MiB")))
return n
}