Ilker ManapandClaude Opus 5.5
0bd0d881c5
llama.cpp's start-up output is read as a shard becomes ready: the devices it used, its backend, and the layers it offloaded. That goes up with READY in a new ShardStatus.compute, is kept on the shard, and is shown by the admin API, ctl replicas (ON) and the console (COMPUTE). A shard given GPUs whose engine says it found no usable GPU or offloaded no layer is stopped and failed, even where nvidia-smi cannot be asked. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
33 lines
1.2 KiB
Go
33 lines
1.2 KiB
Go
package engines
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestLlamaCppSaysWhereItPutTheModel(t *testing.T) {
|
|
gpu := strings.Split(`ggml_cuda_init: found 1 CUDA devices:
|
|
Device 0: Quadro RTX 6000, compute capability 7.5, VMM: yes
|
|
load_backend: loaded CUDA backend from /opt/llmgw/engines/llamacpp/b10991/libggml-cuda.so
|
|
llama_model_load_from_file_impl: using device CUDA0 (Quadro RTX 6000) - 23800 MiB free
|
|
load_tensors: offloading 48 repeating layers to GPU
|
|
load_tensors: offloaded 49/49 layers to GPU`, "\n")
|
|
c, ok := ReadLlamaCompute(gpu)
|
|
if !ok || c.Backend != "CUDA" || c.Offloaded != 49 || c.Total != 49 || c.OnTheCPU() ||
|
|
len(c.Devices) != 1 || c.Devices[0] != "CUDA0: Quadro RTX 6000 (7.5)" {
|
|
t.Errorf("on the GPU: %+v %v", c, ok)
|
|
}
|
|
|
|
cpu := strings.Split(`warning: no usable GPU found, --gpu-layers option will be ignored
|
|
warning: one possible reason is that llama.cpp was compiled without GPU support
|
|
load_tensors: offloaded 0/49 layers to GPU`, "\n")
|
|
c, ok = ReadLlamaCompute(cpu)
|
|
if !ok || c.Backend != "CPU" || !c.OnTheCPU() {
|
|
t.Errorf("fell back to the CPU: %+v %v", c, ok)
|
|
}
|
|
|
|
if _, ok := ReadLlamaCompute([]string{"main: server is listening on http://127.0.0.1:20001"}); ok {
|
|
t.Error("output that says nothing was read as saying something")
|
|
}
|
|
}
|