A replica says what its engine computes on, and llama.cpp's word on the CPU refuses a GPU placement

llama.cpp's start-up output is read as a shard becomes ready: the devices it
used, its backend, and the layers it offloaded. That goes up with READY in a
new ShardStatus.compute, is kept on the shard, and is shown by the admin
API, ctl replicas (ON) and the console (COMPUTE). A shard given GPUs whose
engine says it found no usable GPU or offloaded no layer is stopped and
failed, even where nvidia-smi cannot be asked.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-07 13:02:17 +02:00
1 parent f285aa0548
commit 0bd0d881c5
15 files changed
+517 -126

No files matched your search

+16
View File
@@ -1246,6 +1246,22 @@ func (r *Reconciler) OnFetchProgress(replicaID, shardID, source string, done, to
}
}
// OnShardCompute records where a shard's engine said it put the model.
func (r *Reconciler) OnShardCompute(replicaID, shardID string, c Compute) {
r.mu.Lock()
defer r.mu.Unlock()
rep, ok := r.replicas[replicaID]
if !ok {
return
}
for i := range rep.Shards {
if rep.Shards[i].ID == shardID {
cc := c
rep.Shards[i].Compute = &cc
}
}
}
// OnShardStatus records a status report from an agent.
func (r *Reconciler) OnShardStatus(replicaID, shardID string, st supervisor.State, errMsg string) {
r.markShard(replicaID, shardID, st, errMsg)
+14
View File
@@ -72,6 +72,20 @@ type Shard struct {
// shard reaches any other state, so a finished download does not sit in
// the listing looking like one still running.
Fetch *FetchProgress
// Compute is where the engine said it put the model as it became ready:
// nil when it did not say.
Compute *Compute
}
// Compute is where an engine runs its model, by its own account.
type Compute struct {
// Backend is CUDA, Vulkan, Metal, ROCm, GPU or CPU.
Backend string
// Devices it named: "CUDA0: Quadro RTX 6000 (7.5)".
Devices []string
// Offloaded of Total layers went to the devices; 0 and 0 when unsaid.
Offloaded, Total int
}
// FetchProgress is how far a shard's weights have got.