A replica says what its engine computes on, and llama.cpp's word on the CPU refuses a GPU placement

llama.cpp's start-up output is read as a shard becomes ready: the devices it
used, its backend, and the layers it offloaded. That goes up with READY in a
new ShardStatus.compute, is kept on the shard, and is shown by the admin
API, ctl replicas (ON) and the console (COMPUTE). A shard given GPUs whose
engine says it found no usable GPU or offloaded no layer is stopped and
failed, even where nvidia-smi cannot be asked.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-07 13:02:17 +02:00
1 parent f285aa0548
commit 0bd0d881c5
15 files changed
+517 -126

No files matched your search

+13 -1
View File
@@ -1130,6 +1130,14 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
} else {
err = a.sup.WaitAlive(ctx, sh)
}
// Where it put the model, by its own account: said with the readiness,
// and a GPU placement it computes on the CPU refused even where
// nvidia-smi cannot be asked.
compute, cerr := engineCompute(eng, spec.GPUIndices, sh.Logs(512))
if err == nil && cerr != nil {
err = cerr
_ = a.sup.Stop(ctx, shardID)
}
if err == nil {
err = a.onItsGPUs(ctx, eng, spec, sh.Status().PID)
if err != nil {
@@ -1142,7 +1150,11 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
report(supervisor.Failed, err.Error(), sh.Logs(20))
return
}
report(supervisor.Ready, "", nil)
if compute != nil {
send(&llmgwv1.ShardStatus{State: agentsvc.StateToWire(supervisor.Ready), Compute: compute})
} else {
report(supervisor.Ready, "", nil)
}
// Its dying later is reported by the session's watchExits.
}
+12 -5
View File
@@ -37,11 +37,12 @@ import (
type statusRecorder struct {
ch chan status
mu sync.Mutex
calls []adoptCall
adopted []reconciler.AdoptedShard
unclaim []string // shard ids to report back as unclaimed
fetches []fetchReport
mu sync.Mutex
calls []adoptCall
adopted []reconciler.AdoptedShard
unclaim []string // shard ids to report back as unclaimed
fetches []fetchReport
computes []reconciler.Compute
}
type status struct {
@@ -61,6 +62,12 @@ func (r *statusRecorder) OnShardStatus(replicaID, shardID string, st supervisor.
// fetches records download progress, so a test can assert that a fetch is
// visible from the control plane rather than only in the node's log.
func (r *statusRecorder) OnShardCompute(replicaID, shardID string, c reconciler.Compute) {
r.mu.Lock()
defer r.mu.Unlock()
r.computes = append(r.computes, c)
}
func (r *statusRecorder) OnFetchProgress(replicaID, shardID, source string, done, total int64) {
r.mu.Lock()
defer r.mu.Unlock()
+79
View File
@@ -0,0 +1,79 @@
package engines
import (
"regexp"
"strconv"
"strings"
)
// Compute is where an engine runs its model, as it said while loading it.
type Compute struct {
// Backend is CUDA, Vulkan, Metal, ROCm or CPU.
Backend string
// Devices are those it named: "CUDA0: Quadro RTX 6000 (7.5)".
Devices []string
// Offloaded of Total layers went to the devices; 0 and 0 when it did
// not say.
Offloaded, Total int
// NoGPU is the engine saying it found no device it could use.
NoGPU bool
}
var (
offloadedRE = regexp.MustCompile(`offloaded (\d+)/(\d+) layers to GPU`)
usingRE = regexp.MustCompile(`using device ([A-Za-z]+\d+) \(([^)]*)\)`)
ccRE = regexp.MustCompile(`Device (\d+): ([^,]+), compute capability ([0-9.]+)`)
noDevicesRE = regexp.MustCompile(`found 0 (CUDA|Vulkan|ROCm) devices|no usable GPU found`)
)
// ReadLlamaCompute reads llama.cpp's start-up output for where it put the
// model, reporting false when the output says nothing of it.
func ReadLlamaCompute(lines []string) (Compute, bool) {
var c Compute
known := false
cc := map[string]string{} // device index → compute capability
for _, l := range lines {
if m := ccRE.FindStringSubmatch(l); m != nil {
cc[m[1]] = m[3]
}
}
seen := map[string]bool{}
for _, l := range lines {
if m := offloadedRE.FindStringSubmatch(l); m != nil {
c.Offloaded, _ = strconv.Atoi(m[1])
c.Total, _ = strconv.Atoi(m[2])
known = true
}
if m := usingRE.FindStringSubmatch(l); m != nil && !seen[m[1]] {
seen[m[1]] = true
d := m[1] + ": " + strings.TrimSpace(m[2])
idx := strings.TrimLeft(m[1], "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz")
if v, ok := cc[idx]; ok && strings.HasPrefix(m[1], "CUDA") {
d += " (" + v + ")"
}
c.Devices = append(c.Devices, d)
if c.Backend == "" {
c.Backend = strings.TrimRight(m[1], "0123456789")
}
known = true
}
if noDevicesRE.MatchString(l) {
c.NoGPU, known = true, true
}
}
switch {
case c.OnTheCPU():
c.Backend = "CPU"
case c.Backend == "" && c.Offloaded > 0:
c.Backend = "GPU"
case c.Backend == "":
c.Backend = "CPU"
}
return c, known
}
// OnTheCPU is whether a model the engine was given GPUs for is computed on
// the CPU, by its own account: no device found, or no layer offloaded.
func (c Compute) OnTheCPU() bool {
return c.NoGPU || (c.Total > 0 && c.Offloaded == 0)
}
+32
View File
@@ -0,0 +1,32 @@
package engines
import (
"strings"
"testing"
)
func TestLlamaCppSaysWhereItPutTheModel(t *testing.T) {
gpu := strings.Split(`ggml_cuda_init: found 1 CUDA devices:
Device 0: Quadro RTX 6000, compute capability 7.5, VMM: yes
load_backend: loaded CUDA backend from /opt/llmgw/engines/llamacpp/b10991/libggml-cuda.so
llama_model_load_from_file_impl: using device CUDA0 (Quadro RTX 6000) - 23800 MiB free
load_tensors: offloading 48 repeating layers to GPU
load_tensors: offloaded 49/49 layers to GPU`, "\n")
c, ok := ReadLlamaCompute(gpu)
if !ok || c.Backend != "CUDA" || c.Offloaded != 49 || c.Total != 49 || c.OnTheCPU() ||
len(c.Devices) != 1 || c.Devices[0] != "CUDA0: Quadro RTX 6000 (7.5)" {
t.Errorf("on the GPU: %+v %v", c, ok)
}
cpu := strings.Split(`warning: no usable GPU found, --gpu-layers option will be ignored
warning: one possible reason is that llama.cpp was compiled without GPU support
load_tensors: offloaded 0/49 layers to GPU`, "\n")
c, ok = ReadLlamaCompute(cpu)
if !ok || c.Backend != "CPU" || !c.OnTheCPU() {
t.Errorf("fell back to the CPU: %+v %v", c, ok)
}
if _, ok := ReadLlamaCompute([]string{"main: server is listening on http://127.0.0.1:20001"}); ok {
t.Error("output that says nothing was read as saying something")
}
}
+26
View File
@@ -8,6 +8,7 @@ import (
"strings"
"time"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
)
@@ -153,3 +154,28 @@ func joinIndices(is []int) string {
}
return strings.Join(s, ", ")
}
// engineCompute is where an engine said it put its model, from its output
// so far — llama.cpp's, which says — nil for one that said nothing. A shard
// given GPUs that its engine says it computes on the CPU is an error, found
// this way even where nvidia-smi cannot be asked.
func engineCompute(eng engines.Engine, gpus []int, lines []string) (*llmgwv1.Compute, error) {
if _, ok := eng.(engines.CPUFallback); !ok {
return nil, nil
}
read, ok := engines.ReadLlamaCompute(lines)
if !ok {
return nil, nil
}
c := &llmgwv1.Compute{Backend: read.Backend, Devices: read.Devices,
LayersOffloaded: int32(read.Offloaded), LayersTotal: int32(read.Total)} //nolint:gosec // layer counts
if len(gpus) > 0 && read.OnTheCPU() {
why := "its GPU backend saw no device it could use"
if read.Total > 0 && !read.NoGPU {
why = fmt.Sprintf("it offloaded %d of %d layers", read.Offloaded, read.Total)
}
return c, fmt.Errorf("placed on GPU %v but computing on the CPU: %s, by llama.cpp's own account "+
"(llmgw ctl engine-check on this node says why)", gpus, why)
}
return c, nil
}
+25
View File
@@ -122,3 +122,28 @@ func TestForeignGPUMemoryLeavesOutTheAgentsOwnShards(t *testing.T) {
t.Errorf("no reading came back as %v", foreign)
}
}
func TestLlamaCppsOwnWordOnTheCPUIsEnoughToRefuse(t *testing.T) {
eng := fallbackEngine{}
// Without nvidia-smi to ask, the engine's own output says it.
c, err := engineCompute(eng, []int{0}, []string{
"warning: no usable GPU found, --gpu-layers option will be ignored",
"load_tensors: offloaded 0/49 layers to GPU"})
if err == nil || !strings.Contains(err.Error(), "computing on the CPU") || c.GetBackend() != "CPU" {
t.Fatalf("a GPU shard llama.cpp put on the CPU: %v %+v", err, c)
}
c, err = engineCompute(eng, []int{0}, []string{
"llama_model_load_from_file_impl: using device CUDA0 (Quadro RTX 6000) - 23800 MiB free",
"load_tensors: offloaded 49/49 layers to GPU"})
if err != nil || c.GetBackend() != "CUDA" || c.GetLayersOffloaded() != 49 {
t.Errorf("on its GPU: %v %+v", err, c)
}
// A CPU shard is not refused for being on the CPU, and says so.
if c, err := engineCompute(eng, nil, []string{"load_tensors: offloaded 0/49 layers to GPU"}); err != nil || c.GetBackend() != "CPU" {
t.Errorf("a CPU shard: %v %+v", err, c)
}
// An engine that said nothing says nothing.
if c, err := engineCompute(eng, []int{0}, []string{"listening"}); err != nil || c != nil {
t.Errorf("nothing said: %v %+v", err, c)
}
}