What a replica computes on comes from nvidia-smi; llama.cpp says its layers only at -lv 4

Checked against the installed llama.cpp (b10991): at its default verbosity
it prints neither the devices it uses nor the layers it offloads, so the
start-up output said nothing and every replica's ON column stayed empty.
The agent now takes compute from nvidia-smi - the GPUs the shard's processes
hold memory on, and how much - or CPU, for every engine, falling back to it
when llama.cpp's output says nothing; adopted engines get it too, and each
session reports it once per shard as a compute-only status. engine-check
runs its GPU token at -lv 4, where llama.cpp does say where its layers went.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-07 14:04:08 +02:00
1 parent 25b3e129fe
commit b5d650af16
7 files changed
+147 -13

No files matched your search

+11 -9
View File
@@ -28,15 +28,16 @@ What was left of the field report from v0.10.6.
### Added ### Added
- **A replica says what its engine computes on**: llama.cpp's start-up output - **A replica says what it computes on**: `ctl replicas` (ON), the console
is read as a shard becomes ready — the devices, the backend, the layers and the admin API show `CUDA` with the GPUs its processes hold memory on
offloaded — and shown by `ctl replicas` (ON: `CUDA 49/49`, `CPU`), the and how much, as nvidia-smi sees them, or `CPU` — for every engine, kept by
console and the admin API. the agent across its own and the control plane's restarts. llama.cpp's
layer count is added when it says it, which it does only at `-lv 4`.
- **`ctl engine-check llamacpp` computes a token on the GPU**: a small GGUF - **`ctl engine-check llamacpp` computes a token on the GPU**: a small GGUF
(`--model`, or the smallest in the weights cache) is loaded with every layer (`--model`, or the smallest in the weights cache) is loaded with every layer
on the GPU and asked for one token; a build whose CUDA backend does not load on the GPU at `-lv 4` and asked for one token; a build whose CUDA backend
fails, by llama.cpp's own account. With no `--bin` it checks the build the does not load fails, by llama.cpp's own account of its layers. With no
agent installed. `--bin` it checks the build the agent installed.
- **Engine versions and compute capability on nodes**: `ctl nodes` shows - **Engine versions and compute capability on nodes**: `ctl nodes` shows
`llamacpp b10991` and `6000-24g (7.5)`; the admin API gives `llamacpp b10991` and `6000-24g (7.5)`; the admin API gives
`engine_versions` and `compute_capabilities`. A profile's `engine_version` `engine_versions` and `compute_capabilities`. A profile's `engine_version`
@@ -64,8 +65,9 @@ What was left of the field report from v0.10.6.
### Changed ### Changed
- **A GPU placement llama.cpp says it computes on the CPU is refused**, from - **A GPU placement llama.cpp says it cannot compute on is refused** when it
its own output, even where nvidia-smi cannot be asked. warns that it found no usable GPU, even where nvidia-smi cannot be asked;
where it can, holding no memory on its GPUs refuses it as before.
## v0.12.2 — 2026-10-06 ## v0.12.2 — 2026-10-06
+52 -1
View File
@@ -371,6 +371,12 @@ func (a *Agent) settleAdopted(sh *supervisor.Shard, o supervisor.Orphan) {
_ = a.sup.Stop(ctx, o.ShardID) _ = a.sup.Stop(ctx, o.ShardID)
return return
} }
// What it computes on, for one started before the agent kept that.
if a.sup.ComputeOf(o.ShardID) == nil {
if c, ok := a.gpuCompute(ctx, o.Spec.GPUIndices, sh.Status().PID); ok {
a.sup.SetCompute(o.ShardID, c)
}
}
a.log.Info("adopted engine is serving", a.log.Info("adopted engine is serving",
"shard", o.ShardID, "replica", o.ReplicaID) "shard", o.ShardID, "replica", o.ReplicaID)
} }
@@ -436,7 +442,7 @@ func (a *Agent) runningShards() []*llmgwv1.RunningShard {
Pool: spec.Pool, Pool: spec.Pool,
Fabric: spec.Fabric, Fabric: spec.Fabric,
ProfileFp: spec.ProfileFP, ProfileFp: spec.ProfileFP,
Compute: computeToWire(spec.Compute), Compute: computeToWire(a.sup.ComputeOf(st.ID)),
LeaderAddr: a.cfg.MgmtAddr, LeaderAddr: a.cfg.MgmtAddr,
LeaderPort: int32(spec.LeaderPort), LeaderPort: int32(spec.LeaderPort),
}) })
@@ -567,6 +573,7 @@ func (a *Agent) session(ctx context.Context, server string) error {
defer wg.Wait() defer wg.Wait()
go a.watchExits(ctx, out) go a.watchExits(ctx, out)
go a.forwardKV(ctx, out) go a.forwardKV(ctx, out)
go a.reportCompute(ctx, out)
for { for {
msg, err := stream.Receive() msg, err := stream.Receive()
@@ -1151,6 +1158,11 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
report(supervisor.Failed, err.Error(), sh.Logs(20)) report(supervisor.Failed, err.Error(), sh.Logs(20))
return return
} }
// The engine's own account when it gave one, with its layers; else what
// nvidia-smi sees the shard holding.
if !said {
compute, said = a.gpuCompute(ctx, spec.GPUIndices, sh.Status().PID)
}
if said { if said {
// Kept with the shard, so an agent restarted, or a control plane // Kept with the shard, so an agent restarted, or a control plane
// taking over, still learns it. // taking over, still learns it.
@@ -1389,3 +1401,42 @@ func ResolveManagement(c Config) (addr, iface string) {
} }
return addr, iface return addr, iface
} }
// reportCompute tells the control plane, once a session, what each running
// shard computes on as soon as that is known: an adopted engine's is found
// after registration, which carried what was known then.
func (a *Agent) reportCompute(ctx context.Context, out chan<- *llmgwv1.AgentMessage) {
sent := map[string]bool{}
t := time.NewTicker(5 * time.Second)
defer t.Stop()
for range 60 {
for _, st := range a.sup.Statuses() {
if st.State.Terminal() || sent[st.ID] {
continue
}
c := a.sup.ComputeOf(st.ID)
if c == nil {
continue
}
sh, err := a.sup.Get(st.ID)
if err != nil {
continue
}
// No state: a report of compute alone, not a transition.
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
NodeId: a.cfg.NodeID, ReplicaId: sh.Spec.ReplicaID, ShardId: st.ID, Compute: computeToWire(c),
}}}
select {
case out <- msg:
sent[st.ID] = true
case <-ctx.Done():
return
}
}
select {
case <-ctx.Done():
return
case <-t.C:
}
}
}
+36
View File
@@ -4,6 +4,7 @@ import (
"context" "context"
"fmt" "fmt"
"os" "os"
"slices"
"strconv" "strconv"
"strings" "strings"
"time" "time"
@@ -186,3 +187,38 @@ func computeToWire(c *engines.Compute) *llmgwv1.Compute {
return &llmgwv1.Compute{Backend: c.Backend, Devices: c.Devices, return &llmgwv1.Compute{Backend: c.Backend, Devices: c.Devices,
LayersOffloaded: int32(c.Offloaded), LayersTotal: int32(c.Total)} //nolint:gosec // layer counts LayersOffloaded: int32(c.Offloaded), LayersTotal: int32(c.Total)} //nolint:gosec // layer counts
} }
// gpuCompute is where a shard computes as nvidia-smi sees it: the CPU for a
// shard given no GPU, else the GPUs whose memory its processes hold, and how
// much. False when nvidia-smi cannot be asked. Engine-agnostic, and needing
// no log line: llama.cpp says where its layers went only at -lv 4.
func (a *Agent) gpuCompute(ctx context.Context, gpus []int, pid int) (engines.Compute, bool) {
if len(gpus) == 0 {
return engines.Compute{Backend: "CPU"}, true
}
mem, err := gpuMemory(ctx, a.cfg.Discovery.NvidiaSMI)
if err != nil || mem == nil || pid <= 0 {
return engines.Compute{}, false
}
group := processGroup(pid)
c := engines.Compute{Backend: "CUDA"}
for _, g := range mem {
if !slices.Contains(gpus, g.Index) {
continue
}
used := 0
for _, p := range g.Procs {
if holds([]int{p.PID}, pid, group) {
used += p.UsedMB
}
}
if used > 0 {
c.Devices = append(c.Devices, fmt.Sprintf("GPU %d: %.1f GB", g.Index, float64(used)/1024))
}
}
if len(c.Devices) == 0 {
// Placed on GPUs and holding none of their memory.
c.Backend = "CPU"
}
return c, true
}
+27
View File
@@ -147,3 +147,30 @@ func TestLlamaCppsOwnWordOnTheCPUIsEnoughToRefuse(t *testing.T) {
t.Errorf("nothing said: %v", err) t.Errorf("nothing said: %v", err)
} }
} }
func TestNvidiaSMISaysWhereAShardComputes(t *testing.T) {
oldMem, oldGroup := gpuMemory, processGroup
t.Cleanup(func() { gpuMemory, processGroup = oldMem, oldGroup })
processGroup = func(pid int) int { return pid }
gpuMemory = func(context.Context, string) ([]discovery.GPUMemory, error) {
return []discovery.GPUMemory{
{Index: 0, UsedMB: 20000, Procs: []discovery.GPUProcess{{PID: 700, UsedMB: 2048}, {PID: 900, UsedMB: 17000}}},
{Index: 1, UsedMB: 0},
}, nil
}
a := &Agent{log: slog.Default()}
if c, ok := a.gpuCompute(context.Background(), []int{0}, 700); !ok || c.Backend != "CUDA" ||
len(c.Devices) != 1 || c.Devices[0] != "GPU 0: 2.0 GB" {
t.Errorf("on GPU 0: %+v %v", c, ok)
}
if c, ok := a.gpuCompute(context.Background(), []int{1}, 700); !ok || c.Backend != "CPU" {
t.Errorf("placed on GPU 1, holding none of it: %+v", c)
}
if c, ok := a.gpuCompute(context.Background(), nil, 700); !ok || c.Backend != "CPU" {
t.Errorf("a CPU shard: %+v", c)
}
gpuMemory = func(context.Context, string) ([]discovery.GPUMemory, error) { return nil, nil }
if _, ok := a.gpuCompute(context.Background(), []int{0}, 700); ok {
t.Error("no nvidia-smi read as an answer")
}
}
+12
View File
@@ -658,6 +658,18 @@ func (s *Supervisor) SetCompute(id string, c engines.Compute) {
} }
} }
// ComputeOf is where a shard's engine was found to compute, nil when not
// yet known.
func (s *Supervisor) ComputeOf(id string) *engines.Compute {
s.mu.Lock()
defer s.mu.Unlock()
if sh, ok := s.shards[id]; ok && sh.Spec.Compute != nil {
c := *sh.Spec.Compute
return &c
}
return nil
}
// Remove forgets a terminated shard. // Remove forgets a terminated shard.
func (s *Supervisor) Remove(id string) { func (s *Supervisor) Remove(id string) {
s.mu.Lock() s.mu.Lock()
+3 -1
View File
@@ -363,7 +363,9 @@ func computeOnGPU(ctx context.Context, bin, model string, out io.Writer) error {
defer cancel() defer cancel()
var logs syncBuffer var logs syncBuffer
c := exec.CommandContext(ctx, bin, "-m", model, "--host", "127.0.0.1", "--port", strconv.Itoa(port), //nolint:gosec // the engine's own binary c := exec.CommandContext(ctx, bin, "-m", model, "--host", "127.0.0.1", "--port", strconv.Itoa(port), //nolint:gosec // the engine's own binary
"--n-gpu-layers", "999", "--ctx-size", "512", "--parallel", "1") "--n-gpu-layers", "999", "--ctx-size", "512", "--parallel", "1",
// Where the layers went is said only at this verbosity.
"-lv", "4")
c.Stdout, c.Stderr = &logs, &logs c.Stdout, c.Stderr = &logs, &logs
if err := c.Start(); err != nil { if err := c.Start(); err != nil {
return fmt.Errorf("starting %s: %w", bin, err) return fmt.Errorf("starting %s: %w", bin, err)
+6 -2
View File
@@ -449,14 +449,18 @@ func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) {
f.GetSource(), f.GetBytesDone(), f.GetBytesTotal()) f.GetSource(), f.GetBytesDone(), f.GetBytesTotal())
return return
} }
st := StateFromWire(m.Shard.GetState())
s.status.OnShardStatus(m.Shard.GetReplicaId(), m.Shard.GetShardId(), st, m.Shard.GetError())
if c := m.Shard.GetCompute(); c != nil { if c := m.Shard.GetCompute(); c != nil {
s.status.OnShardCompute(m.Shard.GetReplicaId(), m.Shard.GetShardId(), reconciler.Compute{ s.status.OnShardCompute(m.Shard.GetReplicaId(), m.Shard.GetShardId(), reconciler.Compute{
Backend: c.GetBackend(), Devices: c.GetDevices(), Backend: c.GetBackend(), Devices: c.GetDevices(),
Offloaded: int(c.GetLayersOffloaded()), Total: int(c.GetLayersTotal()), Offloaded: int(c.GetLayersOffloaded()), Total: int(c.GetLayersTotal()),
}) })
// Compute alone, with no state: not a transition.
if m.Shard.GetState() == llmgwv1.ShardStatus_STATE_UNSPECIFIED {
return
}
} }
st := StateFromWire(m.Shard.GetState())
s.status.OnShardStatus(m.Shard.GetReplicaId(), m.Shard.GetShardId(), st, m.Shard.GetError())
if m.Shard.GetState() == llmgwv1.ShardStatus_STATE_FAILED { if m.Shard.GetState() == llmgwv1.ShardStatus_STATE_FAILED {
s.log.Error("shard failed", s.log.Error("shard failed",
"node", nodeID, "shard", m.Shard.GetShardId(), "node", nodeID, "shard", m.Shard.GetShardId(),