A replica says what its engine computes on, and llama.cpp's word on the CPU refuses a GPU placement

llama.cpp's start-up output is read as a shard becomes ready: the devices it
used, its backend, and the layers it offloaded. That goes up with READY in a
new ShardStatus.compute, is kept on the shard, and is shown by the admin
API, ctl replicas (ON) and the console (COMPUTE). A shard given GPUs whose
engine says it found no usable GPU or offloaded no layer is stopped and
failed, even where nvidia-smi cannot be asked.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-07 13:02:17 +02:00
1 parent f285aa0548
commit 0bd0d881c5
15 files changed
+517 -126

No files matched your search

+13 -1
View File
@@ -1130,6 +1130,14 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
} else {
err = a.sup.WaitAlive(ctx, sh)
}
// Where it put the model, by its own account: said with the readiness,
// and a GPU placement it computes on the CPU refused even where
// nvidia-smi cannot be asked.
compute, cerr := engineCompute(eng, spec.GPUIndices, sh.Logs(512))
if err == nil && cerr != nil {
err = cerr
_ = a.sup.Stop(ctx, shardID)
}
if err == nil {
err = a.onItsGPUs(ctx, eng, spec, sh.Status().PID)
if err != nil {
@@ -1142,7 +1150,11 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
report(supervisor.Failed, err.Error(), sh.Logs(20))
return
}
report(supervisor.Ready, "", nil)
if compute != nil {
send(&llmgwv1.ShardStatus{State: agentsvc.StateToWire(supervisor.Ready), Compute: compute})
} else {
report(supervisor.Ready, "", nil)
}
// Its dying later is reported by the session's watchExits.
}
+12 -5
View File
@@ -37,11 +37,12 @@ import (
type statusRecorder struct {
ch chan status
mu sync.Mutex
calls []adoptCall
adopted []reconciler.AdoptedShard
unclaim []string // shard ids to report back as unclaimed
fetches []fetchReport
mu sync.Mutex
calls []adoptCall
adopted []reconciler.AdoptedShard
unclaim []string // shard ids to report back as unclaimed
fetches []fetchReport
computes []reconciler.Compute
}
type status struct {
@@ -61,6 +62,12 @@ func (r *statusRecorder) OnShardStatus(replicaID, shardID string, st supervisor.
// fetches records download progress, so a test can assert that a fetch is
// visible from the control plane rather than only in the node's log.
func (r *statusRecorder) OnShardCompute(replicaID, shardID string, c reconciler.Compute) {
r.mu.Lock()
defer r.mu.Unlock()
r.computes = append(r.computes, c)
}
func (r *statusRecorder) OnFetchProgress(replicaID, shardID, source string, done, total int64) {
r.mu.Lock()
defer r.mu.Unlock()
+79
View File
@@ -0,0 +1,79 @@
package engines
import (
"regexp"
"strconv"
"strings"
)
// Compute is where an engine runs its model, as it said while loading it.
type Compute struct {
// Backend is CUDA, Vulkan, Metal, ROCm or CPU.
Backend string
// Devices are those it named: "CUDA0: Quadro RTX 6000 (7.5)".
Devices []string
// Offloaded of Total layers went to the devices; 0 and 0 when it did
// not say.
Offloaded, Total int
// NoGPU is the engine saying it found no device it could use.
NoGPU bool
}
var (
offloadedRE = regexp.MustCompile(`offloaded (\d+)/(\d+) layers to GPU`)
usingRE = regexp.MustCompile(`using device ([A-Za-z]+\d+) \(([^)]*)\)`)
ccRE = regexp.MustCompile(`Device (\d+): ([^,]+), compute capability ([0-9.]+)`)
noDevicesRE = regexp.MustCompile(`found 0 (CUDA|Vulkan|ROCm) devices|no usable GPU found`)
)
// ReadLlamaCompute reads llama.cpp's start-up output for where it put the
// model, reporting false when the output says nothing of it.
func ReadLlamaCompute(lines []string) (Compute, bool) {
var c Compute
known := false
cc := map[string]string{} // device index → compute capability
for _, l := range lines {
if m := ccRE.FindStringSubmatch(l); m != nil {
cc[m[1]] = m[3]
}
}
seen := map[string]bool{}
for _, l := range lines {
if m := offloadedRE.FindStringSubmatch(l); m != nil {
c.Offloaded, _ = strconv.Atoi(m[1])
c.Total, _ = strconv.Atoi(m[2])
known = true
}
if m := usingRE.FindStringSubmatch(l); m != nil && !seen[m[1]] {
seen[m[1]] = true
d := m[1] + ": " + strings.TrimSpace(m[2])
idx := strings.TrimLeft(m[1], "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz")
if v, ok := cc[idx]; ok && strings.HasPrefix(m[1], "CUDA") {
d += " (" + v + ")"
}
c.Devices = append(c.Devices, d)
if c.Backend == "" {
c.Backend = strings.TrimRight(m[1], "0123456789")
}
known = true
}
if noDevicesRE.MatchString(l) {
c.NoGPU, known = true, true
}
}
switch {
case c.OnTheCPU():
c.Backend = "CPU"
case c.Backend == "" && c.Offloaded > 0:
c.Backend = "GPU"
case c.Backend == "":
c.Backend = "CPU"
}
return c, known
}
// OnTheCPU is whether a model the engine was given GPUs for is computed on
// the CPU, by its own account: no device found, or no layer offloaded.
func (c Compute) OnTheCPU() bool {
return c.NoGPU || (c.Total > 0 && c.Offloaded == 0)
}
+32
View File
@@ -0,0 +1,32 @@
package engines
import (
"strings"
"testing"
)
func TestLlamaCppSaysWhereItPutTheModel(t *testing.T) {
gpu := strings.Split(`ggml_cuda_init: found 1 CUDA devices:
Device 0: Quadro RTX 6000, compute capability 7.5, VMM: yes
load_backend: loaded CUDA backend from /opt/llmgw/engines/llamacpp/b10991/libggml-cuda.so
llama_model_load_from_file_impl: using device CUDA0 (Quadro RTX 6000) - 23800 MiB free
load_tensors: offloading 48 repeating layers to GPU
load_tensors: offloaded 49/49 layers to GPU`, "\n")
c, ok := ReadLlamaCompute(gpu)
if !ok || c.Backend != "CUDA" || c.Offloaded != 49 || c.Total != 49 || c.OnTheCPU() ||
len(c.Devices) != 1 || c.Devices[0] != "CUDA0: Quadro RTX 6000 (7.5)" {
t.Errorf("on the GPU: %+v %v", c, ok)
}
cpu := strings.Split(`warning: no usable GPU found, --gpu-layers option will be ignored
warning: one possible reason is that llama.cpp was compiled without GPU support
load_tensors: offloaded 0/49 layers to GPU`, "\n")
c, ok = ReadLlamaCompute(cpu)
if !ok || c.Backend != "CPU" || !c.OnTheCPU() {
t.Errorf("fell back to the CPU: %+v %v", c, ok)
}
if _, ok := ReadLlamaCompute([]string{"main: server is listening on http://127.0.0.1:20001"}); ok {
t.Error("output that says nothing was read as saying something")
}
}
+26
View File
@@ -8,6 +8,7 @@ import (
"strings"
"time"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
)
@@ -153,3 +154,28 @@ func joinIndices(is []int) string {
}
return strings.Join(s, ", ")
}
// engineCompute is where an engine said it put its model, from its output
// so far — llama.cpp's, which says — nil for one that said nothing. A shard
// given GPUs that its engine says it computes on the CPU is an error, found
// this way even where nvidia-smi cannot be asked.
func engineCompute(eng engines.Engine, gpus []int, lines []string) (*llmgwv1.Compute, error) {
if _, ok := eng.(engines.CPUFallback); !ok {
return nil, nil
}
read, ok := engines.ReadLlamaCompute(lines)
if !ok {
return nil, nil
}
c := &llmgwv1.Compute{Backend: read.Backend, Devices: read.Devices,
LayersOffloaded: int32(read.Offloaded), LayersTotal: int32(read.Total)} //nolint:gosec // layer counts
if len(gpus) > 0 && read.OnTheCPU() {
why := "its GPU backend saw no device it could use"
if read.Total > 0 && !read.NoGPU {
why = fmt.Sprintf("it offloaded %d of %d layers", read.Offloaded, read.Total)
}
return c, fmt.Errorf("placed on GPU %v but computing on the CPU: %s, by llama.cpp's own account "+
"(llmgw ctl engine-check on this node says why)", gpus, why)
}
return c, nil
}
+25
View File
@@ -122,3 +122,28 @@ func TestForeignGPUMemoryLeavesOutTheAgentsOwnShards(t *testing.T) {
t.Errorf("no reading came back as %v", foreign)
}
}
func TestLlamaCppsOwnWordOnTheCPUIsEnoughToRefuse(t *testing.T) {
eng := fallbackEngine{}
// Without nvidia-smi to ask, the engine's own output says it.
c, err := engineCompute(eng, []int{0}, []string{
"warning: no usable GPU found, --gpu-layers option will be ignored",
"load_tensors: offloaded 0/49 layers to GPU"})
if err == nil || !strings.Contains(err.Error(), "computing on the CPU") || c.GetBackend() != "CPU" {
t.Fatalf("a GPU shard llama.cpp put on the CPU: %v %+v", err, c)
}
c, err = engineCompute(eng, []int{0}, []string{
"llama_model_load_from_file_impl: using device CUDA0 (Quadro RTX 6000) - 23800 MiB free",
"load_tensors: offloaded 49/49 layers to GPU"})
if err != nil || c.GetBackend() != "CUDA" || c.GetLayersOffloaded() != 49 {
t.Errorf("on its GPU: %v %+v", err, c)
}
// A CPU shard is not refused for being on the CPU, and says so.
if c, err := engineCompute(eng, nil, []string{"load_tensors: offloaded 0/49 layers to GPU"}); err != nil || c.GetBackend() != "CPU" {
t.Errorf("a CPU shard: %v %+v", err, c)
}
// An engine that said nothing says nothing.
if c, err := engineCompute(eng, []int{0}, []string{"listening"}); err != nil || c != nil {
t.Errorf("nothing said: %v %+v", err, c)
}
}
+10 -4
View File
@@ -450,7 +450,7 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
}
w := tab()
fmt.Fprintln(w, "REPLICA\tMODEL\tSTATE\tPOOL\tNODES\tAGE\tIDLE\tENDPOINT")
fmt.Fprintln(w, "REPLICA\tMODEL\tSTATE\tPOOL\tON\tNODES\tAGE\tIDLE\tENDPOINT")
for _, r := range reps {
var nodes []string
for _, s := range r.Shards {
@@ -468,8 +468,14 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
if r.Restart != "" {
state += " (restart " + r.Restart + ")"
}
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
r.ID, r.Model, state, r.Pool, join(nodes),
// What the engine computes on, by its own account: the first
// shard's, which serves.
on := "-"
if len(r.Shards) > 0 {
on = r.Shards[0].Compute.Short()
}
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
r.ID, r.Model, state, r.Pool, on, join(nodes),
(time.Duration(r.AgeSec) * time.Second).String(),
idleColumn(r.IdleSec), r.Endpoint)
// A line rather than a column, because most replicas are not
@@ -479,7 +485,7 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
if s.Fetch == nil {
continue
}
fmt.Fprintf(w, " %s: fetching %s\t\t\t\t\t\t\t\n",
fmt.Fprintf(w, " %s: fetching %s\t\t\t\t\t\t\t\t\n",
s.Node, fetchColumn(s.Fetch))
}
}
+6 -2
View File
@@ -1230,13 +1230,17 @@ func (m consoleModel) panel() (title, body, keys string) {
for _, s := range r.Shards {
nodes = append(nodes, fmt.Sprintf("%s%v", s.Node, discovery.DeviceLabels(s.GPUs)))
}
rows = append(rows, []string{r.ID, r.Model, r.State, r.Pool, strings.Join(nodes, " "),
compute := "-"
if len(r.Shards) > 0 {
compute = r.Shards[0].Compute.Short()
}
rows = append(rows, []string{r.ID, r.Model, r.State, r.Pool, compute, strings.Join(nodes, " "),
(time.Duration(r.AgeSec) * time.Second).Round(time.Second).String()})
}
if m.inPanel {
keys = rowKeys
}
return "Replicas", "\n" + table([]string{"REPLICA", "MODEL", "STATE", "POOL", "ON", "AGE"}, rows, m.row, pw, m.inPanel), keys
return "Replicas", "\n" + table([]string{"REPLICA", "MODEL", "STATE", "POOL", "COMPUTE", "ON", "AGE"}, rows, m.row, pw, m.inPanel), keys
case cUsage:
head := "\n Last 24 hours, by " + lipgloss.NewStyle().Bold(true).Render(usageGroupings[m.usageBy]) +
sDim.Render(" (→ to group by "+usageGroupings[(m.usageBy+1)%len(usageGroupings)]+")") + "\n\n"
+28
View File
@@ -518,6 +518,30 @@ type ShardView struct {
// node's last heartbeat; zero when not measured.
CPUCores float64 `json:"cpu_cores,omitempty"`
MemoryBytes int64 `json:"memory_bytes,omitempty"`
// Compute is where the engine said it put the model as it became
// ready: absent when it did not say.
Compute *ComputeView `json:"compute,omitempty"`
}
// ComputeView is where an engine runs its model, by its own account.
type ComputeView struct {
// Backend is CUDA, Vulkan, Metal, ROCm, GPU or CPU.
Backend string `json:"backend"`
Devices []string `json:"devices,omitempty"`
// LayersOffloaded of LayersTotal went to the devices; absent when unsaid.
LayersOffloaded int `json:"layers_offloaded,omitempty"`
LayersTotal int `json:"layers_total,omitempty"`
}
// Short is the compute in a few characters: "CUDA 49/49", "CPU".
func (c *ComputeView) Short() string {
if c == nil {
return "-"
}
if c.LayersTotal > 0 && c.Backend != "CPU" {
return fmt.Sprintf("%s %d/%d", c.Backend, c.LayersOffloaded, c.LayersTotal)
}
return c.Backend
}
// FetchView is a download in flight.
@@ -829,6 +853,10 @@ func (a *API) Replicas(model string) []ReplicaView {
sv.Fetch.Percent = &p
}
}
if c := s.Compute; c != nil {
sv.Compute = &ComputeView{Backend: c.Backend, Devices: c.Devices,
LayersOffloaded: c.Offloaded, LayersTotal: c.Total}
}
if l := loads[s.NodeID]; l != nil {
if use, ok := l.Shards[s.ID]; ok {
sv.CPUCores, sv.MemoryBytes = use.CPUCores, use.RSSBytes
+8
View File
@@ -36,6 +36,8 @@ type StatusSink interface {
// still starting, and treating each report as a state change would reset
// the timers that decide when a slow start has become a failed one.
OnFetchProgress(replicaID, shardID, source string, done, total int64)
// OnShardCompute records where a shard's engine said it runs the model.
OnShardCompute(replicaID, shardID string, c reconciler.Compute)
Adopt(nodeID string, shards []reconciler.AdoptedShard) (unclaimed []reconciler.AdoptedShard)
}
@@ -449,6 +451,12 @@ func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) {
}
st := StateFromWire(m.Shard.GetState())
s.status.OnShardStatus(m.Shard.GetReplicaId(), m.Shard.GetShardId(), st, m.Shard.GetError())
if c := m.Shard.GetCompute(); c != nil {
s.status.OnShardCompute(m.Shard.GetReplicaId(), m.Shard.GetShardId(), reconciler.Compute{
Backend: c.GetBackend(), Devices: c.GetDevices(),
Offloaded: int(c.GetLayersOffloaded()), Total: int(c.GetLayersTotal()),
})
}
if m.Shard.GetState() == llmgwv1.ShardStatus_STATE_FAILED {
s.log.Error("shard failed",
"node", nodeID, "shard", m.Shard.GetShardId(),
+16
View File
@@ -1246,6 +1246,22 @@ func (r *Reconciler) OnFetchProgress(replicaID, shardID, source string, done, to
}
}
// OnShardCompute records where a shard's engine said it put the model.
func (r *Reconciler) OnShardCompute(replicaID, shardID string, c Compute) {
r.mu.Lock()
defer r.mu.Unlock()
rep, ok := r.replicas[replicaID]
if !ok {
return
}
for i := range rep.Shards {
if rep.Shards[i].ID == shardID {
cc := c
rep.Shards[i].Compute = &cc
}
}
}
// OnShardStatus records a status report from an agent.
func (r *Reconciler) OnShardStatus(replicaID, shardID string, st supervisor.State, errMsg string) {
r.markShard(replicaID, shardID, st, errMsg)
+14
View File
@@ -72,6 +72,20 @@ type Shard struct {
// shard reaches any other state, so a finished download does not sit in
// the listing looking like one still running.
Fetch *FetchProgress
// Compute is where the engine said it put the model as it became ready:
// nil when it did not say.
Compute *Compute
}
// Compute is where an engine runs its model, by its own account.
type Compute struct {
// Backend is CUDA, Vulkan, Metal, ROCm, GPU or CPU.
Backend string
// Devices it named: "CUDA0: Quadro RTX 6000 (7.5)".
Devices []string
// Offloaded of Total layers went to the devices; 0 and 0 when unsaid.
Offloaded, Total int
}
// FetchProgress is how far a shard's weights have got.