What a replica computes on comes from nvidia-smi; llama.cpp says its layers only at -lv 4
Checked against the installed llama.cpp (b10991): at its default verbosity it prints neither the devices it uses nor the layers it offloads, so the start-up output said nothing and every replica's ON column stayed empty. The agent now takes compute from nvidia-smi - the GPUs the shard's processes hold memory on, and how much - or CPU, for every engine, falling back to it when llama.cpp's output says nothing; adopted engines get it too, and each session reports it once per shard as a compute-only status. engine-check runs its GPU token at -lv 4, where llama.cpp does say where its layers went. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
25b3e129fe
commit
b5d650af16
7 files changed
+147
-13
No files matched your search
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -186,3 +187,38 @@ func computeToWire(c *engines.Compute) *llmgwv1.Compute {
|
||||
return &llmgwv1.Compute{Backend: c.Backend, Devices: c.Devices,
|
||||
LayersOffloaded: int32(c.Offloaded), LayersTotal: int32(c.Total)} //nolint:gosec // layer counts
|
||||
}
|
||||
|
||||
// gpuCompute is where a shard computes as nvidia-smi sees it: the CPU for a
|
||||
// shard given no GPU, else the GPUs whose memory its processes hold, and how
|
||||
// much. False when nvidia-smi cannot be asked. Engine-agnostic, and needing
|
||||
// no log line: llama.cpp says where its layers went only at -lv 4.
|
||||
func (a *Agent) gpuCompute(ctx context.Context, gpus []int, pid int) (engines.Compute, bool) {
|
||||
if len(gpus) == 0 {
|
||||
return engines.Compute{Backend: "CPU"}, true
|
||||
}
|
||||
mem, err := gpuMemory(ctx, a.cfg.Discovery.NvidiaSMI)
|
||||
if err != nil || mem == nil || pid <= 0 {
|
||||
return engines.Compute{}, false
|
||||
}
|
||||
group := processGroup(pid)
|
||||
c := engines.Compute{Backend: "CUDA"}
|
||||
for _, g := range mem {
|
||||
if !slices.Contains(gpus, g.Index) {
|
||||
continue
|
||||
}
|
||||
used := 0
|
||||
for _, p := range g.Procs {
|
||||
if holds([]int{p.PID}, pid, group) {
|
||||
used += p.UsedMB
|
||||
}
|
||||
}
|
||||
if used > 0 {
|
||||
c.Devices = append(c.Devices, fmt.Sprintf("GPU %d: %.1f GB", g.Index, float64(used)/1024))
|
||||
}
|
||||
}
|
||||
if len(c.Devices) == 0 {
|
||||
// Placed on GPUs and holding none of their memory.
|
||||
c.Backend = "CPU"
|
||||
}
|
||||
return c, true
|
||||
}
|
||||
Reference in new issue
Block a user