Files
llmgw/internal/agent/engines/llamacpp_test.go
T
Ilker ManapandClaude Opus 5.5 cf53904cc5 Serve LoRA adapters on llama.cpp models, asked for by name
A model's adapters are fetched by each node with its weights and loaded
unapplied at start; a request naming an adapter reaches the model's
replicas with llama.cpp's per-request lora field. Adapter names share the
model namespace, are listed, and follow allowlists on their own. The
router's config seed and pushed catalog now build the name map in one place.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 23:40:47 +02:00

326 lines
10 KiB
Go

package engines
import (
"os"
"path/filepath"
"slices"
"strings"
"testing"
)
func llamaSpec(gpus []int) ShardSpec {
return ShardSpec{
ReplicaID: "rep-1", ShardID: "sh-1", Rank: 0, NNodes: 1,
GPUIndices: gpus, HTTPPort: 8000, MgmtInterface: "eth0",
Profile: Profile{
Name: "cpu", Engine: "llamacpp", TP: 1,
ModelPath: "/models/qwen.gguf", ServedModelName: "qwen-0.5b",
},
}
}
func TestLlamaCppCommand(t *testing.T) {
cmds, err := (&LlamaCpp{}).Commands(llamaSpec(nil))
if err != nil {
t.Fatal(err)
}
if len(cmds) != 1 {
t.Fatalf("a llama.cpp shard is one process, got %d", len(cmds))
}
c := cmds[0]
if c.Path != "llama-server" {
t.Errorf("path %q", c.Path)
}
// --metrics is not optional: llama-server serves no /metrics without it, and
// an endpoint the router cannot measure is one it routes to blind.
if !slices.Contains(c.Args, "--metrics") {
t.Error("the server was started without --metrics")
}
if v, _ := flagValue(c.Args, "--model"); v != "/models/qwen.gguf" {
t.Errorf("--model %q", v)
}
if v, _ := flagValue(c.Args, "--port"); v != "8000" {
t.Errorf("--port %q", v)
}
if v, _ := flagValue(c.Args, "--alias"); v != "qwen-0.5b" {
t.Errorf("--alias %q", v)
}
// llama.cpp serves one request at a time unless told otherwise, which would
// make every second request wait no matter what the router decided.
if v, ok := flagValue(c.Args, "--parallel"); !ok || v == "1" {
t.Errorf("--parallel %q", v)
}
if slices.Contains(c.Args, "--n-gpu-layers") {
t.Error("a CPU placement was given GPU layers")
}
}
func TestLlamaCppOffloadsWhenGivenGPUs(t *testing.T) {
cmds, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2}))
if v, ok := flagValue(cmds[0].Args, "--n-gpu-layers"); !ok || v != "999" {
t.Errorf("a GPU placement should offload the model, got %q", v)
}
// The shard must still be pinned to its own card.
if !slices.Contains(cmds[0].Env, "CUDA_VISIBLE_DEVICES=2") {
t.Error("the shard was not pinned to its GPU")
}
}
func TestProfileArgumentsWinOverDefaults(t *testing.T) {
// The escape hatch has to actually work, or an operator tuning slot count
// ends up with two --parallel flags and whichever llama.cpp happens to
// prefer.
s := llamaSpec(nil)
s.Profile.Args = []string{"--parallel", "16", "--ctx-size", "4096"}
cmds, _ := (&LlamaCpp{}).Commands(s)
n := 0
for _, a := range cmds[0].Args {
if a == "--parallel" {
n++
}
}
if n != 1 {
t.Errorf("--parallel appears %d times", n)
}
if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "16" {
t.Errorf("the profile's value lost: %q", v)
}
}
func TestShortFormArgumentsAreRecognized(t *testing.T) {
s := llamaSpec([]int{0})
s.Profile.Args = []string{"-np", "8", "-ngl", "20"}
cmds, _ := (&LlamaCpp{}).Commands(s)
for _, flag := range []string{"--parallel", "--n-gpu-layers"} {
if slices.Contains(cmds[0].Args, flag) {
t.Errorf("%s was added although the profile set its short form", flag)
}
}
}
func TestLlamaCppRefusesWhatItCannotDo(t *testing.T) {
// Accepting either of these would produce a replica that starts and then
// serves from one machine while the catalog believes it has several.
multi := Profile{Name: "multi", Engine: "llamacpp", TP: 1, Nodes: 2, GPUsPerNode: 1, ModelPath: "/m"}
if err := (&LlamaCpp{}).Validate(multi, nil); err == nil {
t.Error("a multi-node llama.cpp profile was accepted")
}
tp := Profile{Name: "tp", Engine: "llamacpp", TP: 2, GPUsPerNode: 2, ModelPath: "/m"}
if err := (&LlamaCpp{}).Validate(tp, h100(2)); err == nil {
t.Error("a tensor-parallel llama.cpp profile was accepted")
}
}
func TestCPUProfileValidates(t *testing.T) {
p := Profile{Name: "cpu", Engine: "llamacpp", TP: 1, ModelPath: "/models/qwen.gguf"}
if err := (&LlamaCpp{}).Validate(p, nil); err != nil {
t.Errorf("a CPU profile on a node with no GPUs was rejected: %v", err)
}
}
func TestLlamaCppAlwaysServesHTTP(t *testing.T) {
// Unlike the others there is no worker rank to get this wrong for.
if !(&LlamaCpp{}).ServesHTTP(llamaSpec(nil)) {
t.Error("a llama.cpp shard must answer the API")
}
}
func TestBinaryIsConfigurable(t *testing.T) {
l := &LlamaCpp{cfg: LlamaCppConfig{Bin: "/opt/llama.cpp/bin/llama-server", Parallel: 2}}
cmds, _ := l.Commands(llamaSpec(nil))
if cmds[0].Path != "/opt/llama.cpp/bin/llama-server" {
t.Errorf("path %q", cmds[0].Path)
}
if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "2" {
t.Errorf("--parallel %q", v)
}
}
func TestUnknownEngineNamesLlamaCpp(t *testing.T) {
_, err := NewRegistry(Config{}).Get("llama.cpp")
if err == nil || !strings.Contains(err.Error(), "llamacpp") {
t.Errorf("the suggestion should name the real engine: %v", err)
}
}
// flagValue reads a "--flag value" pair out of a command line.
func flagValue(args []string, flag string) (string, bool) {
for i, a := range args {
if a == flag && i+1 < len(args) {
return args[i+1], true
}
}
return "", false
}
func TestLlamaCppUsesTheModelsChatTemplate(t *testing.T) {
// Pinned so an engine upgrade cannot change how prompts are built under a
// running fleet: build 10991 defaults to jinja, older builds do not.
//
// Not a fix for tool calling, though it was added while chasing that. On
// this build the flag is a no-op and the behaviour is identical with and
// without it.
l := &LlamaCpp{}
cmds, err := l.Commands(ShardSpec{HTTPPort: 8080, Profile: Profile{ModelPath: "/m.gguf"}})
if err != nil {
t.Fatal(err)
}
if !slices.Contains(cmds[0].Args, "--jinja") {
t.Errorf("--jinja is missing; tool calls will come back as text: %v", cmds[0].Args)
}
}
func TestLlamaCppLetsAProfileTurnTheTemplateOff(t *testing.T) {
// A model whose bundled template is broken has to be serviceable without
// editing the adapter.
l := &LlamaCpp{}
cmds, err := l.Commands(ShardSpec{
HTTPPort: 8080,
Profile: Profile{ModelPath: "/m.gguf", Args: []string{"--no-jinja"}},
})
if err != nil {
t.Fatal(err)
}
if slices.Contains(cmds[0].Args, "--jinja") {
t.Errorf("the profile said --no-jinja and got both: %v", cmds[0].Args)
}
}
func TestASetDirectoryResolvesToItsFirstPart(t *testing.T) {
dir := t.TempDir()
// Deliberately out of order on disk, because readdir order is not shard
// order and the first part is the one llama.cpp must be given.
for _, n := range []string{
"qwen-q4_k_m-00003-of-00003.gguf",
"qwen-q4_k_m-00001-of-00003.gguf",
"qwen-q4_k_m-00002-of-00003.gguf",
} {
if err := os.WriteFile(filepath.Join(dir, n), []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
}
got, err := ggufIn(dir)
if err != nil {
t.Fatal(err)
}
// Given the first part llama.cpp finds the rest beside it; given the third
// it loads a third of a model.
if filepath.Base(got) != "qwen-q4_k_m-00001-of-00003.gguf" {
t.Errorf("expected the first shard, got %s", filepath.Base(got))
}
}
func TestAPlainFilePathIsUntouched(t *testing.T) {
// Every single-file model and every hand-staged one takes this path.
p := filepath.Join(t.TempDir(), "model.gguf")
if err := os.WriteFile(p, []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
if got, _ := ggufIn(p); got != p {
t.Errorf("expected the path unchanged, got %s", got)
}
}
func TestADirectoryWithNoWeightsSaysSo(t *testing.T) {
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte("{}"), 0o644); err != nil {
t.Fatal(err)
}
if _, err := ggufIn(dir); err == nil {
t.Error("a directory with no .gguf in it is not a model llama.cpp can load")
}
}
func TestAMissingPathIsLeftToTheEngine(t *testing.T) {
// The engine's own "no such file" names the path and the flag it came
// from, which is more use than anything this could say.
p := "/nonexistent/model.gguf"
got, err := ggufIn(p)
if err != nil || got != p {
t.Errorf("expected it passed through, got %q %v", got, err)
}
}
func TestLlamaCppIsGivenItsKeyOutOfSightOfTheProcessList(t *testing.T) {
// The command line is readable by every user on the node; the
// environment of a root process is not.
cmds, err := (&LlamaCpp{}).Commands(ShardSpec{HTTPPort: 8080, APIKey: "k1",
Profile: Profile{ModelPath: "/m.gguf"}})
if err != nil {
t.Fatal(err)
}
if slices.Contains(cmds[0].Args, "k1") || slices.Contains(cmds[0].Args, "--api-key") {
t.Errorf("the key is on the command line: %v", cmds[0].Args)
}
if !slices.Contains(cmds[0].Env, "LLAMA_API_KEY=k1") {
t.Error("llama-server was not given its key")
}
}
func TestACPUShardLeavesHalfTheMachineForAnother(t *testing.T) {
prev := numCPU
numCPU = func() int { return 8 }
t.Cleanup(func() { numCPU = prev })
threads := func(l *LlamaCpp, s ShardSpec) (string, bool) {
cmds, err := l.Commands(s)
if err != nil {
t.Fatal(err)
}
return flagValue(cmds[0].Args, "--threads")
}
if v, _ := threads(&LlamaCpp{}, llamaSpec(nil)); v != "4" {
t.Errorf("--threads %q on 8 CPUs, expected 4", v)
}
if v, _ := threads(&LlamaCpp{cfg: LlamaCppConfig{Threads: 8}}, llamaSpec(nil)); v != "8" {
t.Errorf("--threads %q with 8 configured", v)
}
if _, ok := threads(&LlamaCpp{}, llamaSpec([]int{0})); ok {
t.Error("a GPU shard was given a thread count")
}
s := llamaSpec(nil)
s.Profile.Args = []string{"-t", "2"}
cmds, _ := (&LlamaCpp{}).Commands(s)
if slices.Contains(cmds[0].Args, "--threads") {
t.Error("the profile's own -t was overridden")
}
numCPU = func() int { return 1 }
if v, _ := threads(&LlamaCpp{}, llamaSpec(nil)); v != "1" {
t.Errorf("--threads %q on one CPU", v)
}
}
func TestTheCoresPlacementReservedAreTheThreads(t *testing.T) {
s := llamaSpec(nil)
s.Threads = 3
cmds, err := (&LlamaCpp{cfg: LlamaCppConfig{Threads: 8}}).Commands(s)
if err != nil {
t.Fatal(err)
}
if v, _ := flagValue(cmds[0].Args, "--threads"); v != "3" {
t.Errorf("--threads %q; what placement reserved should win over the node's setting", v)
}
}
func TestAdaptersAreLoadedAtStartAndAppliedToNoneByDefault(t *testing.T) {
s := llamaSpec(nil)
s.Profile.Adapters = []Adapter{{Name: "sql", Path: "/w/sql.gguf"}, {Name: "docs", Path: "/w/docs.gguf"}}
cmds, err := (&LlamaCpp{}).Commands(s)
if err != nil {
t.Fatal(err)
}
args := strings.Join(cmds[0].Args, " ")
if !strings.Contains(args, "--lora-init-without-apply --lora /w/sql.gguf --lora /w/docs.gguf") {
t.Errorf("args %s: both adapters, in order, loaded unapplied", args)
}
plain, _ := (&LlamaCpp{}).Commands(llamaSpec(nil))
if strings.Contains(strings.Join(plain[0].Args, " "), "--lora") {
t.Error("a model with no adapters was given lora flags")
}
}