Files
llmgw/internal/agent/engines/llamacpp_test.go
T
Ilker ManapandClaude Opus 5.5 62657f4c2a
Release / build (push) Successful in 4m11s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 4m57s
Release v0.10.4: CPU threads, tracing, and the rest of the review
CPU replicas start on half the machine's CPUs, not all of them: two busy
llama.cpp servers each spinning on all 8 vCPUs fell to 0.8 tok/s apiece,
where at 4 threads they keep 5.5. llamacpp.threads sets it per node.

Inference requests can be traced over OTLP/HTTP (tracing.endpoint), with
admission, schedule and upstream spans and traceparent sent to the engine.

Security: onboarding no longer puts the cluster root in the system trust
store and removes it where it was; the mirror's mutual-TLS API can be
limited to named certificates (--api-client); init.log no longer keeps the
administrator's password; a warning when auth is off on a public listen;
the recorded actor is sanitised and sudo-aware; Ray's dashboard is off, and
the remaining engine gaps are documented.

reconciler.go and agent.go are split by topic, the status mapping of each
failure is tested, and the plan's checklist is brought up to date.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 04:00:35 +02:00

297 lines
9.3 KiB
Go

package engines
import (
"os"
"path/filepath"
"slices"
"strings"
"testing"
)
func llamaSpec(gpus []int) ShardSpec {
return ShardSpec{
ReplicaID: "rep-1", ShardID: "sh-1", Rank: 0, NNodes: 1,
GPUIndices: gpus, HTTPPort: 8000, MgmtInterface: "eth0",
Profile: Profile{
Name: "cpu", Engine: "llamacpp", TP: 1,
ModelPath: "/models/qwen.gguf", ServedModelName: "qwen-0.5b",
},
}
}
func TestLlamaCppCommand(t *testing.T) {
cmds, err := (&LlamaCpp{}).Commands(llamaSpec(nil))
if err != nil {
t.Fatal(err)
}
if len(cmds) != 1 {
t.Fatalf("a llama.cpp shard is one process, got %d", len(cmds))
}
c := cmds[0]
if c.Path != "llama-server" {
t.Errorf("path %q", c.Path)
}
// --metrics is not optional: llama-server serves no /metrics without it, and
// an endpoint the router cannot measure is one it routes to blind.
if !slices.Contains(c.Args, "--metrics") {
t.Error("the server was started without --metrics")
}
if v, _ := flagValue(c.Args, "--model"); v != "/models/qwen.gguf" {
t.Errorf("--model %q", v)
}
if v, _ := flagValue(c.Args, "--port"); v != "8000" {
t.Errorf("--port %q", v)
}
if v, _ := flagValue(c.Args, "--alias"); v != "qwen-0.5b" {
t.Errorf("--alias %q", v)
}
// llama.cpp serves one request at a time unless told otherwise, which would
// make every second request wait no matter what the router decided.
if v, ok := flagValue(c.Args, "--parallel"); !ok || v == "1" {
t.Errorf("--parallel %q", v)
}
if slices.Contains(c.Args, "--n-gpu-layers") {
t.Error("a CPU placement was given GPU layers")
}
}
func TestLlamaCppOffloadsWhenGivenGPUs(t *testing.T) {
cmds, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2}))
if v, ok := flagValue(cmds[0].Args, "--n-gpu-layers"); !ok || v != "999" {
t.Errorf("a GPU placement should offload the model, got %q", v)
}
// The shard must still be pinned to its own card.
if !slices.Contains(cmds[0].Env, "CUDA_VISIBLE_DEVICES=2") {
t.Error("the shard was not pinned to its GPU")
}
}
func TestProfileArgumentsWinOverDefaults(t *testing.T) {
// The escape hatch has to actually work, or an operator tuning slot count
// ends up with two --parallel flags and whichever llama.cpp happens to
// prefer.
s := llamaSpec(nil)
s.Profile.Args = []string{"--parallel", "16", "--ctx-size", "4096"}
cmds, _ := (&LlamaCpp{}).Commands(s)
n := 0
for _, a := range cmds[0].Args {
if a == "--parallel" {
n++
}
}
if n != 1 {
t.Errorf("--parallel appears %d times", n)
}
if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "16" {
t.Errorf("the profile's value lost: %q", v)
}
}
func TestShortFormArgumentsAreRecognized(t *testing.T) {
s := llamaSpec([]int{0})
s.Profile.Args = []string{"-np", "8", "-ngl", "20"}
cmds, _ := (&LlamaCpp{}).Commands(s)
for _, flag := range []string{"--parallel", "--n-gpu-layers"} {
if slices.Contains(cmds[0].Args, flag) {
t.Errorf("%s was added although the profile set its short form", flag)
}
}
}
func TestLlamaCppRefusesWhatItCannotDo(t *testing.T) {
// Accepting either of these would produce a replica that starts and then
// serves from one machine while the catalog believes it has several.
multi := Profile{Name: "multi", Engine: "llamacpp", TP: 1, Nodes: 2, GPUsPerNode: 1, ModelPath: "/m"}
if err := (&LlamaCpp{}).Validate(multi, nil); err == nil {
t.Error("a multi-node llama.cpp profile was accepted")
}
tp := Profile{Name: "tp", Engine: "llamacpp", TP: 2, GPUsPerNode: 2, ModelPath: "/m"}
if err := (&LlamaCpp{}).Validate(tp, h100(2)); err == nil {
t.Error("a tensor-parallel llama.cpp profile was accepted")
}
}
func TestCPUProfileValidates(t *testing.T) {
p := Profile{Name: "cpu", Engine: "llamacpp", TP: 1, ModelPath: "/models/qwen.gguf"}
if err := (&LlamaCpp{}).Validate(p, nil); err != nil {
t.Errorf("a CPU profile on a node with no GPUs was rejected: %v", err)
}
}
func TestLlamaCppAlwaysServesHTTP(t *testing.T) {
// Unlike the others there is no worker rank to get this wrong for.
if !(&LlamaCpp{}).ServesHTTP(llamaSpec(nil)) {
t.Error("a llama.cpp shard must answer the API")
}
}
func TestBinaryIsConfigurable(t *testing.T) {
l := &LlamaCpp{cfg: LlamaCppConfig{Bin: "/opt/llama.cpp/bin/llama-server", Parallel: 2}}
cmds, _ := l.Commands(llamaSpec(nil))
if cmds[0].Path != "/opt/llama.cpp/bin/llama-server" {
t.Errorf("path %q", cmds[0].Path)
}
if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "2" {
t.Errorf("--parallel %q", v)
}
}
func TestUnknownEngineNamesLlamaCpp(t *testing.T) {
_, err := NewRegistry(Config{}).Get("llama.cpp")
if err == nil || !strings.Contains(err.Error(), "llamacpp") {
t.Errorf("the suggestion should name the real engine: %v", err)
}
}
// flagValue reads a "--flag value" pair out of a command line.
func flagValue(args []string, flag string) (string, bool) {
for i, a := range args {
if a == flag && i+1 < len(args) {
return args[i+1], true
}
}
return "", false
}
func TestLlamaCppUsesTheModelsChatTemplate(t *testing.T) {
// Pinned so an engine upgrade cannot change how prompts are built under a
// running fleet: build 10991 defaults to jinja, older builds do not.
//
// Not a fix for tool calling, though it was added while chasing that. On
// this build the flag is a no-op and the behaviour is identical with and
// without it.
l := &LlamaCpp{}
cmds, err := l.Commands(ShardSpec{HTTPPort: 8080, Profile: Profile{ModelPath: "/m.gguf"}})
if err != nil {
t.Fatal(err)
}
if !slices.Contains(cmds[0].Args, "--jinja") {
t.Errorf("--jinja is missing; tool calls will come back as text: %v", cmds[0].Args)
}
}
func TestLlamaCppLetsAProfileTurnTheTemplateOff(t *testing.T) {
// A model whose bundled template is broken has to be serviceable without
// editing the adapter.
l := &LlamaCpp{}
cmds, err := l.Commands(ShardSpec{
HTTPPort: 8080,
Profile: Profile{ModelPath: "/m.gguf", Args: []string{"--no-jinja"}},
})
if err != nil {
t.Fatal(err)
}
if slices.Contains(cmds[0].Args, "--jinja") {
t.Errorf("the profile said --no-jinja and got both: %v", cmds[0].Args)
}
}
func TestASetDirectoryResolvesToItsFirstPart(t *testing.T) {
dir := t.TempDir()
// Deliberately out of order on disk, because readdir order is not shard
// order and the first part is the one llama.cpp must be given.
for _, n := range []string{
"qwen-q4_k_m-00003-of-00003.gguf",
"qwen-q4_k_m-00001-of-00003.gguf",
"qwen-q4_k_m-00002-of-00003.gguf",
} {
if err := os.WriteFile(filepath.Join(dir, n), []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
}
got, err := ggufIn(dir)
if err != nil {
t.Fatal(err)
}
// Given the first part llama.cpp finds the rest beside it; given the third
// it loads a third of a model.
if filepath.Base(got) != "qwen-q4_k_m-00001-of-00003.gguf" {
t.Errorf("expected the first shard, got %s", filepath.Base(got))
}
}
func TestAPlainFilePathIsUntouched(t *testing.T) {
// Every single-file model and every hand-staged one takes this path.
p := filepath.Join(t.TempDir(), "model.gguf")
if err := os.WriteFile(p, []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
if got, _ := ggufIn(p); got != p {
t.Errorf("expected the path unchanged, got %s", got)
}
}
func TestADirectoryWithNoWeightsSaysSo(t *testing.T) {
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte("{}"), 0o644); err != nil {
t.Fatal(err)
}
if _, err := ggufIn(dir); err == nil {
t.Error("a directory with no .gguf in it is not a model llama.cpp can load")
}
}
func TestAMissingPathIsLeftToTheEngine(t *testing.T) {
// The engine's own "no such file" names the path and the flag it came
// from, which is more use than anything this could say.
p := "/nonexistent/model.gguf"
got, err := ggufIn(p)
if err != nil || got != p {
t.Errorf("expected it passed through, got %q %v", got, err)
}
}
func TestLlamaCppIsGivenItsKeyOutOfSightOfTheProcessList(t *testing.T) {
// The command line is readable by every user on the node; the
// environment of a root process is not.
cmds, err := (&LlamaCpp{}).Commands(ShardSpec{HTTPPort: 8080, APIKey: "k1",
Profile: Profile{ModelPath: "/m.gguf"}})
if err != nil {
t.Fatal(err)
}
if slices.Contains(cmds[0].Args, "k1") || slices.Contains(cmds[0].Args, "--api-key") {
t.Errorf("the key is on the command line: %v", cmds[0].Args)
}
if !slices.Contains(cmds[0].Env, "LLAMA_API_KEY=k1") {
t.Error("llama-server was not given its key")
}
}
func TestACPUShardLeavesHalfTheMachineForAnother(t *testing.T) {
prev := numCPU
numCPU = func() int { return 8 }
t.Cleanup(func() { numCPU = prev })
threads := func(l *LlamaCpp, s ShardSpec) (string, bool) {
cmds, err := l.Commands(s)
if err != nil {
t.Fatal(err)
}
return flagValue(cmds[0].Args, "--threads")
}
if v, _ := threads(&LlamaCpp{}, llamaSpec(nil)); v != "4" {
t.Errorf("--threads %q on 8 CPUs, expected 4", v)
}
if v, _ := threads(&LlamaCpp{cfg: LlamaCppConfig{Threads: 8}}, llamaSpec(nil)); v != "8" {
t.Errorf("--threads %q with 8 configured", v)
}
if _, ok := threads(&LlamaCpp{}, llamaSpec([]int{0})); ok {
t.Error("a GPU shard was given a thread count")
}
s := llamaSpec(nil)
s.Profile.Args = []string{"-t", "2"}
cmds, _ := (&LlamaCpp{}).Commands(s)
if slices.Contains(cmds[0].Args, "--threads") {
t.Error("the profile's own -t was overridden")
}
numCPU = func() int { return 1 }
if v, _ := threads(&LlamaCpp{}, llamaSpec(nil)); v != "1" {
t.Errorf("--threads %q on one CPU", v)
}
}