What a replica computes on comes from nvidia-smi; llama.cpp says its layers only at -lv 4

Checked against the installed llama.cpp (b10991): at its default verbosity
it prints neither the devices it uses nor the layers it offloads, so the
start-up output said nothing and every replica's ON column stayed empty.
The agent now takes compute from nvidia-smi - the GPUs the shard's processes
hold memory on, and how much - or CPU, for every engine, falling back to it
when llama.cpp's output says nothing; adopted engines get it too, and each
session reports it once per shard as a compute-only status. engine-check
runs its GPU token at -lv 4, where llama.cpp does say where its layers went.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-07 14:04:08 +02:00
1 parent 25b3e129fe
commit b5d650af16
7 files changed
+147 -13

No files matched your search

+52 -1
View File
@@ -371,6 +371,12 @@ func (a *Agent) settleAdopted(sh *supervisor.Shard, o supervisor.Orphan) {
_ = a.sup.Stop(ctx, o.ShardID)
return
}
// What it computes on, for one started before the agent kept that.
if a.sup.ComputeOf(o.ShardID) == nil {
if c, ok := a.gpuCompute(ctx, o.Spec.GPUIndices, sh.Status().PID); ok {
a.sup.SetCompute(o.ShardID, c)
}
}
a.log.Info("adopted engine is serving",
"shard", o.ShardID, "replica", o.ReplicaID)
}
@@ -436,7 +442,7 @@ func (a *Agent) runningShards() []*llmgwv1.RunningShard {
Pool: spec.Pool,
Fabric: spec.Fabric,
ProfileFp: spec.ProfileFP,
Compute: computeToWire(spec.Compute),
Compute: computeToWire(a.sup.ComputeOf(st.ID)),
LeaderAddr: a.cfg.MgmtAddr,
LeaderPort: int32(spec.LeaderPort),
})
@@ -567,6 +573,7 @@ func (a *Agent) session(ctx context.Context, server string) error {
defer wg.Wait()
go a.watchExits(ctx, out)
go a.forwardKV(ctx, out)
go a.reportCompute(ctx, out)
for {
msg, err := stream.Receive()
@@ -1151,6 +1158,11 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
report(supervisor.Failed, err.Error(), sh.Logs(20))
return
}
// The engine's own account when it gave one, with its layers; else what
// nvidia-smi sees the shard holding.
if !said {
compute, said = a.gpuCompute(ctx, spec.GPUIndices, sh.Status().PID)
}
if said {
// Kept with the shard, so an agent restarted, or a control plane
// taking over, still learns it.
@@ -1389,3 +1401,42 @@ func ResolveManagement(c Config) (addr, iface string) {
}
return addr, iface
}
// reportCompute tells the control plane, once a session, what each running
// shard computes on as soon as that is known: an adopted engine's is found
// after registration, which carried what was known then.
func (a *Agent) reportCompute(ctx context.Context, out chan<- *llmgwv1.AgentMessage) {
sent := map[string]bool{}
t := time.NewTicker(5 * time.Second)
defer t.Stop()
for range 60 {
for _, st := range a.sup.Statuses() {
if st.State.Terminal() || sent[st.ID] {
continue
}
c := a.sup.ComputeOf(st.ID)
if c == nil {
continue
}
sh, err := a.sup.Get(st.ID)
if err != nil {
continue
}
// No state: a report of compute alone, not a transition.
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
NodeId: a.cfg.NodeID, ReplicaId: sh.Spec.ReplicaID, ShardId: st.ID, Compute: computeToWire(c),
}}}
select {
case out <- msg:
sent[st.ID] = true
case <-ctx.Done():
return
}
}
select {
case <-ctx.Done():
return
case <-t.C:
}
}
}