Files
llmgw/internal/agent/engines/llamacpp_test.go
T
Ilker ManapandClaude Opus 5.5 990d82d801
Release / build (push) Successful in 1m5s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m47s
Reserve CPU cores and require memory for CPU models
A profile's cpu_cores is reserved out of its node's CPUs by placement, as
memory is, and becomes the engine's thread count. Agents report their CPU
count in registration and heartbeats. New CPU models must declare memory;
model views show reserved against measured memory with warnings, and
ctl model resources sets both for a stored model.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 17:43:13 +02:00

309 lines
9.6 KiB
Go

package engines
import (
"os"
"path/filepath"
"slices"
"strings"
"testing"
)
func llamaSpec(gpus []int) ShardSpec {
return ShardSpec{
ReplicaID: "rep-1", ShardID: "sh-1", Rank: 0, NNodes: 1,
GPUIndices: gpus, HTTPPort: 8000, MgmtInterface: "eth0",
Profile: Profile{
Name: "cpu", Engine: "llamacpp", TP: 1,
ModelPath: "/models/qwen.gguf", ServedModelName: "qwen-0.5b",
},
}
}
func TestLlamaCppCommand(t *testing.T) {
cmds, err := (&LlamaCpp{}).Commands(llamaSpec(nil))
if err != nil {
t.Fatal(err)
}
if len(cmds) != 1 {
t.Fatalf("a llama.cpp shard is one process, got %d", len(cmds))
}
c := cmds[0]
if c.Path != "llama-server" {
t.Errorf("path %q", c.Path)
}
// --metrics is not optional: llama-server serves no /metrics without it, and
// an endpoint the router cannot measure is one it routes to blind.
if !slices.Contains(c.Args, "--metrics") {
t.Error("the server was started without --metrics")
}
if v, _ := flagValue(c.Args, "--model"); v != "/models/qwen.gguf" {
t.Errorf("--model %q", v)
}
if v, _ := flagValue(c.Args, "--port"); v != "8000" {
t.Errorf("--port %q", v)
}
if v, _ := flagValue(c.Args, "--alias"); v != "qwen-0.5b" {
t.Errorf("--alias %q", v)
}
// llama.cpp serves one request at a time unless told otherwise, which would
// make every second request wait no matter what the router decided.
if v, ok := flagValue(c.Args, "--parallel"); !ok || v == "1" {
t.Errorf("--parallel %q", v)
}
if slices.Contains(c.Args, "--n-gpu-layers") {
t.Error("a CPU placement was given GPU layers")
}
}
func TestLlamaCppOffloadsWhenGivenGPUs(t *testing.T) {
cmds, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2}))
if v, ok := flagValue(cmds[0].Args, "--n-gpu-layers"); !ok || v != "999" {
t.Errorf("a GPU placement should offload the model, got %q", v)
}
// The shard must still be pinned to its own card.
if !slices.Contains(cmds[0].Env, "CUDA_VISIBLE_DEVICES=2") {
t.Error("the shard was not pinned to its GPU")
}
}
func TestProfileArgumentsWinOverDefaults(t *testing.T) {
// The escape hatch has to actually work, or an operator tuning slot count
// ends up with two --parallel flags and whichever llama.cpp happens to
// prefer.
s := llamaSpec(nil)
s.Profile.Args = []string{"--parallel", "16", "--ctx-size", "4096"}
cmds, _ := (&LlamaCpp{}).Commands(s)
n := 0
for _, a := range cmds[0].Args {
if a == "--parallel" {
n++
}
}
if n != 1 {
t.Errorf("--parallel appears %d times", n)
}
if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "16" {
t.Errorf("the profile's value lost: %q", v)
}
}
func TestShortFormArgumentsAreRecognized(t *testing.T) {
s := llamaSpec([]int{0})
s.Profile.Args = []string{"-np", "8", "-ngl", "20"}
cmds, _ := (&LlamaCpp{}).Commands(s)
for _, flag := range []string{"--parallel", "--n-gpu-layers"} {
if slices.Contains(cmds[0].Args, flag) {
t.Errorf("%s was added although the profile set its short form", flag)
}
}
}
func TestLlamaCppRefusesWhatItCannotDo(t *testing.T) {
// Accepting either of these would produce a replica that starts and then
// serves from one machine while the catalog believes it has several.
multi := Profile{Name: "multi", Engine: "llamacpp", TP: 1, Nodes: 2, GPUsPerNode: 1, ModelPath: "/m"}
if err := (&LlamaCpp{}).Validate(multi, nil); err == nil {
t.Error("a multi-node llama.cpp profile was accepted")
}
tp := Profile{Name: "tp", Engine: "llamacpp", TP: 2, GPUsPerNode: 2, ModelPath: "/m"}
if err := (&LlamaCpp{}).Validate(tp, h100(2)); err == nil {
t.Error("a tensor-parallel llama.cpp profile was accepted")
}
}
func TestCPUProfileValidates(t *testing.T) {
p := Profile{Name: "cpu", Engine: "llamacpp", TP: 1, ModelPath: "/models/qwen.gguf"}
if err := (&LlamaCpp{}).Validate(p, nil); err != nil {
t.Errorf("a CPU profile on a node with no GPUs was rejected: %v", err)
}
}
func TestLlamaCppAlwaysServesHTTP(t *testing.T) {
// Unlike the others there is no worker rank to get this wrong for.
if !(&LlamaCpp{}).ServesHTTP(llamaSpec(nil)) {
t.Error("a llama.cpp shard must answer the API")
}
}
func TestBinaryIsConfigurable(t *testing.T) {
l := &LlamaCpp{cfg: LlamaCppConfig{Bin: "/opt/llama.cpp/bin/llama-server", Parallel: 2}}
cmds, _ := l.Commands(llamaSpec(nil))
if cmds[0].Path != "/opt/llama.cpp/bin/llama-server" {
t.Errorf("path %q", cmds[0].Path)
}
if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "2" {
t.Errorf("--parallel %q", v)
}
}
func TestUnknownEngineNamesLlamaCpp(t *testing.T) {
_, err := NewRegistry(Config{}).Get("llama.cpp")
if err == nil || !strings.Contains(err.Error(), "llamacpp") {
t.Errorf("the suggestion should name the real engine: %v", err)
}
}
// flagValue reads a "--flag value" pair out of a command line.
func flagValue(args []string, flag string) (string, bool) {
for i, a := range args {
if a == flag && i+1 < len(args) {
return args[i+1], true
}
}
return "", false
}
func TestLlamaCppUsesTheModelsChatTemplate(t *testing.T) {
// Pinned so an engine upgrade cannot change how prompts are built under a
// running fleet: build 10991 defaults to jinja, older builds do not.
//
// Not a fix for tool calling, though it was added while chasing that. On
// this build the flag is a no-op and the behaviour is identical with and
// without it.
l := &LlamaCpp{}
cmds, err := l.Commands(ShardSpec{HTTPPort: 8080, Profile: Profile{ModelPath: "/m.gguf"}})
if err != nil {
t.Fatal(err)
}
if !slices.Contains(cmds[0].Args, "--jinja") {
t.Errorf("--jinja is missing; tool calls will come back as text: %v", cmds[0].Args)
}
}
func TestLlamaCppLetsAProfileTurnTheTemplateOff(t *testing.T) {
// A model whose bundled template is broken has to be serviceable without
// editing the adapter.
l := &LlamaCpp{}
cmds, err := l.Commands(ShardSpec{
HTTPPort: 8080,
Profile: Profile{ModelPath: "/m.gguf", Args: []string{"--no-jinja"}},
})
if err != nil {
t.Fatal(err)
}
if slices.Contains(cmds[0].Args, "--jinja") {
t.Errorf("the profile said --no-jinja and got both: %v", cmds[0].Args)
}
}
func TestASetDirectoryResolvesToItsFirstPart(t *testing.T) {
dir := t.TempDir()
// Deliberately out of order on disk, because readdir order is not shard
// order and the first part is the one llama.cpp must be given.
for _, n := range []string{
"qwen-q4_k_m-00003-of-00003.gguf",
"qwen-q4_k_m-00001-of-00003.gguf",
"qwen-q4_k_m-00002-of-00003.gguf",
} {
if err := os.WriteFile(filepath.Join(dir, n), []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
}
got, err := ggufIn(dir)
if err != nil {
t.Fatal(err)
}
// Given the first part llama.cpp finds the rest beside it; given the third
// it loads a third of a model.
if filepath.Base(got) != "qwen-q4_k_m-00001-of-00003.gguf" {
t.Errorf("expected the first shard, got %s", filepath.Base(got))
}
}
func TestAPlainFilePathIsUntouched(t *testing.T) {
// Every single-file model and every hand-staged one takes this path.
p := filepath.Join(t.TempDir(), "model.gguf")
if err := os.WriteFile(p, []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
if got, _ := ggufIn(p); got != p {
t.Errorf("expected the path unchanged, got %s", got)
}
}
func TestADirectoryWithNoWeightsSaysSo(t *testing.T) {
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte("{}"), 0o644); err != nil {
t.Fatal(err)
}
if _, err := ggufIn(dir); err == nil {
t.Error("a directory with no .gguf in it is not a model llama.cpp can load")
}
}
func TestAMissingPathIsLeftToTheEngine(t *testing.T) {
// The engine's own "no such file" names the path and the flag it came
// from, which is more use than anything this could say.
p := "/nonexistent/model.gguf"
got, err := ggufIn(p)
if err != nil || got != p {
t.Errorf("expected it passed through, got %q %v", got, err)
}
}
func TestLlamaCppIsGivenItsKeyOutOfSightOfTheProcessList(t *testing.T) {
// The command line is readable by every user on the node; the
// environment of a root process is not.
cmds, err := (&LlamaCpp{}).Commands(ShardSpec{HTTPPort: 8080, APIKey: "k1",
Profile: Profile{ModelPath: "/m.gguf"}})
if err != nil {
t.Fatal(err)
}
if slices.Contains(cmds[0].Args, "k1") || slices.Contains(cmds[0].Args, "--api-key") {
t.Errorf("the key is on the command line: %v", cmds[0].Args)
}
if !slices.Contains(cmds[0].Env, "LLAMA_API_KEY=k1") {
t.Error("llama-server was not given its key")
}
}
func TestACPUShardLeavesHalfTheMachineForAnother(t *testing.T) {
prev := numCPU
numCPU = func() int { return 8 }
t.Cleanup(func() { numCPU = prev })
threads := func(l *LlamaCpp, s ShardSpec) (string, bool) {
cmds, err := l.Commands(s)
if err != nil {
t.Fatal(err)
}
return flagValue(cmds[0].Args, "--threads")
}
if v, _ := threads(&LlamaCpp{}, llamaSpec(nil)); v != "4" {
t.Errorf("--threads %q on 8 CPUs, expected 4", v)
}
if v, _ := threads(&LlamaCpp{cfg: LlamaCppConfig{Threads: 8}}, llamaSpec(nil)); v != "8" {
t.Errorf("--threads %q with 8 configured", v)
}
if _, ok := threads(&LlamaCpp{}, llamaSpec([]int{0})); ok {
t.Error("a GPU shard was given a thread count")
}
s := llamaSpec(nil)
s.Profile.Args = []string{"-t", "2"}
cmds, _ := (&LlamaCpp{}).Commands(s)
if slices.Contains(cmds[0].Args, "--threads") {
t.Error("the profile's own -t was overridden")
}
numCPU = func() int { return 1 }
if v, _ := threads(&LlamaCpp{}, llamaSpec(nil)); v != "1" {
t.Errorf("--threads %q on one CPU", v)
}
}
func TestTheCoresPlacementReservedAreTheThreads(t *testing.T) {
s := llamaSpec(nil)
s.Threads = 3
cmds, err := (&LlamaCpp{cfg: LlamaCppConfig{Threads: 8}}).Commands(s)
if err != nil {
t.Fatal(err)
}
if v, _ := flagValue(cmds[0].Args, "--threads"); v != "3" {
t.Errorf("--threads %q; what placement reserved should win over the node's setting", v)
}
}