What a replica computes on comes from nvidia-smi; llama.cpp says its layers only at -lv 4

Checked against the installed llama.cpp (b10991): at its default verbosity
it prints neither the devices it uses nor the layers it offloads, so the
start-up output said nothing and every replica's ON column stayed empty.
The agent now takes compute from nvidia-smi - the GPUs the shard's processes
hold memory on, and how much - or CPU, for every engine, falling back to it
when llama.cpp's output says nothing; adopted engines get it too, and each
session reports it once per shard as a compute-only status. engine-check
runs its GPU token at -lv 4, where llama.cpp does say where its layers went.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-07 14:04:08 +02:00
1 parent 25b3e129fe
commit b5d650af16
7 files changed
+147 -13

No files matched your search

+36
View File
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"os"
"slices"
"strconv"
"strings"
"time"
@@ -186,3 +187,38 @@ func computeToWire(c *engines.Compute) *llmgwv1.Compute {
return &llmgwv1.Compute{Backend: c.Backend, Devices: c.Devices,
LayersOffloaded: int32(c.Offloaded), LayersTotal: int32(c.Total)} //nolint:gosec // layer counts
}
// gpuCompute is where a shard computes as nvidia-smi sees it: the CPU for a
// shard given no GPU, else the GPUs whose memory its processes hold, and how
// much. False when nvidia-smi cannot be asked. Engine-agnostic, and needing
// no log line: llama.cpp says where its layers went only at -lv 4.
func (a *Agent) gpuCompute(ctx context.Context, gpus []int, pid int) (engines.Compute, bool) {
if len(gpus) == 0 {
return engines.Compute{Backend: "CPU"}, true
}
mem, err := gpuMemory(ctx, a.cfg.Discovery.NvidiaSMI)
if err != nil || mem == nil || pid <= 0 {
return engines.Compute{}, false
}
group := processGroup(pid)
c := engines.Compute{Backend: "CUDA"}
for _, g := range mem {
if !slices.Contains(gpus, g.Index) {
continue
}
used := 0
for _, p := range g.Procs {
if holds([]int{p.PID}, pid, group) {
used += p.UsedMB
}
}
if used > 0 {
c.Devices = append(c.Devices, fmt.Sprintf("GPU %d: %.1f GB", g.Index, float64(used)/1024))
}
}
if len(c.Devices) == 0 {
// Placed on GPUs and holding none of their memory.
c.Backend = "CPU"
}
return c, true
}