What a replica computes on comes from nvidia-smi; llama.cpp says its layers only at -lv 4
Checked against the installed llama.cpp (b10991): at its default verbosity it prints neither the devices it uses nor the layers it offloads, so the start-up output said nothing and every replica's ON column stayed empty. The agent now takes compute from nvidia-smi - the GPUs the shard's processes hold memory on, and how much - or CPU, for every engine, falling back to it when llama.cpp's output says nothing; adopted engines get it too, and each session reports it once per shard as a compute-only status. engine-check runs its GPU token at -lv 4, where llama.cpp does say where its layers went. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
25b3e129fe
commit
b5d650af16
7 files changed
+147
-13
No files matched your search
@@ -147,3 +147,30 @@ func TestLlamaCppsOwnWordOnTheCPUIsEnoughToRefuse(t *testing.T) {
|
||||
t.Errorf("nothing said: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNvidiaSMISaysWhereAShardComputes(t *testing.T) {
|
||||
oldMem, oldGroup := gpuMemory, processGroup
|
||||
t.Cleanup(func() { gpuMemory, processGroup = oldMem, oldGroup })
|
||||
processGroup = func(pid int) int { return pid }
|
||||
gpuMemory = func(context.Context, string) ([]discovery.GPUMemory, error) {
|
||||
return []discovery.GPUMemory{
|
||||
{Index: 0, UsedMB: 20000, Procs: []discovery.GPUProcess{{PID: 700, UsedMB: 2048}, {PID: 900, UsedMB: 17000}}},
|
||||
{Index: 1, UsedMB: 0},
|
||||
}, nil
|
||||
}
|
||||
a := &Agent{log: slog.Default()}
|
||||
if c, ok := a.gpuCompute(context.Background(), []int{0}, 700); !ok || c.Backend != "CUDA" ||
|
||||
len(c.Devices) != 1 || c.Devices[0] != "GPU 0: 2.0 GB" {
|
||||
t.Errorf("on GPU 0: %+v %v", c, ok)
|
||||
}
|
||||
if c, ok := a.gpuCompute(context.Background(), []int{1}, 700); !ok || c.Backend != "CPU" {
|
||||
t.Errorf("placed on GPU 1, holding none of it: %+v", c)
|
||||
}
|
||||
if c, ok := a.gpuCompute(context.Background(), nil, 700); !ok || c.Backend != "CPU" {
|
||||
t.Errorf("a CPU shard: %+v", c)
|
||||
}
|
||||
gpuMemory = func(context.Context, string) ([]discovery.GPUMemory, error) { return nil, nil }
|
||||
if _, ok := a.gpuCompute(context.Background(), []int{0}, 700); ok {
|
||||
t.Error("no nvidia-smi read as an answer")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user