A replica says what its engine computes on, and llama.cpp's word on the CPU refuses a GPU placement

llama.cpp's start-up output is read as a shard becomes ready: the devices it
used, its backend, and the layers it offloaded. That goes up with READY in a
new ShardStatus.compute, is kept on the shard, and is shown by the admin
API, ctl replicas (ON) and the console (COMPUTE). A shard given GPUs whose
engine says it found no usable GPU or offloaded no layer is stopped and
failed, even where nvidia-smi cannot be asked.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-07 13:02:17 +02:00
1 parent f285aa0548
commit 0bd0d881c5
15 files changed
+517 -126

No files matched your search

+13 -1
View File
@@ -1130,6 +1130,14 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
} else {
err = a.sup.WaitAlive(ctx, sh)
}
// Where it put the model, by its own account: said with the readiness,
// and a GPU placement it computes on the CPU refused even where
// nvidia-smi cannot be asked.
compute, cerr := engineCompute(eng, spec.GPUIndices, sh.Logs(512))
if err == nil && cerr != nil {
err = cerr
_ = a.sup.Stop(ctx, shardID)
}
if err == nil {
err = a.onItsGPUs(ctx, eng, spec, sh.Status().PID)
if err != nil {
@@ -1142,7 +1150,11 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
report(supervisor.Failed, err.Error(), sh.Logs(20))
return
}
report(supervisor.Ready, "", nil)
if compute != nil {
send(&llmgwv1.ShardStatus{State: agentsvc.StateToWire(supervisor.Ready), Compute: compute})
} else {
report(supervisor.Ready, "", nil)
}
// Its dying later is reported by the session's watchExits.
}