A replica says what its engine computes on, and llama.cpp's word on the CPU refuses a GPU placement
llama.cpp's start-up output is read as a shard becomes ready: the devices it used, its backend, and the layers it offloaded. That goes up with READY in a new ShardStatus.compute, is kept on the shard, and is shown by the admin API, ctl replicas (ON) and the console (COMPUTE). A shard given GPUs whose engine says it found no usable GPU or offloaded no layer is stopped and failed, even where nvidia-smi cannot be asked. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
f285aa0548
commit
0bd0d881c5
15 files changed
+517
-126
No files matched your search
+13
-1
@@ -1130,6 +1130,14 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
} else {
|
||||
err = a.sup.WaitAlive(ctx, sh)
|
||||
}
|
||||
// Where it put the model, by its own account: said with the readiness,
|
||||
// and a GPU placement it computes on the CPU refused even where
|
||||
// nvidia-smi cannot be asked.
|
||||
compute, cerr := engineCompute(eng, spec.GPUIndices, sh.Logs(512))
|
||||
if err == nil && cerr != nil {
|
||||
err = cerr
|
||||
_ = a.sup.Stop(ctx, shardID)
|
||||
}
|
||||
if err == nil {
|
||||
err = a.onItsGPUs(ctx, eng, spec, sh.Status().PID)
|
||||
if err != nil {
|
||||
@@ -1142,7 +1150,11 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
report(supervisor.Failed, err.Error(), sh.Logs(20))
|
||||
return
|
||||
}
|
||||
report(supervisor.Ready, "", nil)
|
||||
if compute != nil {
|
||||
send(&llmgwv1.ShardStatus{State: agentsvc.StateToWire(supervisor.Ready), Compute: compute})
|
||||
} else {
|
||||
report(supervisor.Ready, "", nil)
|
||||
}
|
||||
// Its dying later is reported by the session's watchExits.
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user