A replica says what its engine computes on, and llama.cpp's word on the CPU refuses a GPU placement

llama.cpp's start-up output is read as a shard becomes ready: the devices it
used, its backend, and the layers it offloaded. That goes up with READY in a
new ShardStatus.compute, is kept on the shard, and is shown by the admin
API, ctl replicas (ON) and the console (COMPUTE). A shard given GPUs whose
engine says it found no usable GPU or offloaded no layer is stopped and
failed, even where nvidia-smi cannot be asked.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-07 13:02:17 +02:00
1 parent f285aa0548
commit 0bd0d881c5
15 files changed
+517 -126

No files matched your search

+28
View File
@@ -518,6 +518,30 @@ type ShardView struct {
// node's last heartbeat; zero when not measured.
CPUCores float64 `json:"cpu_cores,omitempty"`
MemoryBytes int64 `json:"memory_bytes,omitempty"`
// Compute is where the engine said it put the model as it became
// ready: absent when it did not say.
Compute *ComputeView `json:"compute,omitempty"`
}
// ComputeView is where an engine runs its model, by its own account.
type ComputeView struct {
// Backend is CUDA, Vulkan, Metal, ROCm, GPU or CPU.
Backend string `json:"backend"`
Devices []string `json:"devices,omitempty"`
// LayersOffloaded of LayersTotal went to the devices; absent when unsaid.
LayersOffloaded int `json:"layers_offloaded,omitempty"`
LayersTotal int `json:"layers_total,omitempty"`
}
// Short is the compute in a few characters: "CUDA 49/49", "CPU".
func (c *ComputeView) Short() string {
if c == nil {
return "-"
}
if c.LayersTotal > 0 && c.Backend != "CPU" {
return fmt.Sprintf("%s %d/%d", c.Backend, c.LayersOffloaded, c.LayersTotal)
}
return c.Backend
}
// FetchView is a download in flight.
@@ -829,6 +853,10 @@ func (a *API) Replicas(model string) []ReplicaView {
sv.Fetch.Percent = &p
}
}
if c := s.Compute; c != nil {
sv.Compute = &ComputeView{Backend: c.Backend, Devices: c.Devices,
LayersOffloaded: c.Offloaded, LayersTotal: c.Total}
}
if l := loads[s.NodeID]; l != nil {
if use, ok := l.Shards[s.ID]; ok {
sv.CPUCores, sv.MemoryBytes = use.CPUCores, use.RSSBytes