Files
llmgw/internal/agent/engines/llamacpp_test.go
T
Ilker ManapandClaude Opus 5 ea87021b5c
CI / test (push) Successful in 1m30s
Correct what the previous commit claimed about --jinja
9fc098f said passing --jinja fixes tool calling. It does not. llama.cpp
build 10991 has jinja enabled by default — "--jinja, --no-jinja ... (default:
enabled)" — so the flag changed nothing, and tool calls came back as text
before and after it in exactly the same way.

The flag is worth keeping for a different reason, and the comment now says
that one: it pins how prompts are built, so an engine upgrade cannot change
it underneath a running fleet. Older builds do default to the generic
template.

What actually happens is that llama.cpp renders the tools correctly — the
prompt from /apply-template carries the <tools> block and the instruction to
reply in <tool_call> tags — and qwen2.5-coder answers with a markdown JSON
block instead. 3B and 7B both, and tool_choice "required" does not constrain
it either. That is the model, not the plumbing, and no flag here will fix
it.

Recording it because a wrong explanation in a commit message is worse than
no explanation: the next person to see tool calls arriving as text would
have checked --jinja, found it already set, and had nowhere to go.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 21:30:22 +02:00

190 lines
5.9 KiB
Go

package engines
import (
"slices"
"strings"
"testing"
)
func llamaSpec(gpus []int) ShardSpec {
return ShardSpec{
ReplicaID: "rep-1", ShardID: "sh-1", Rank: 0, NNodes: 1,
GPUIndices: gpus, HTTPPort: 8000, MgmtInterface: "eth0",
Profile: Profile{
Name: "cpu", Engine: "llamacpp", TP: 1,
ModelPath: "/models/qwen.gguf", ServedModelName: "qwen-0.5b",
},
}
}
func TestLlamaCppCommand(t *testing.T) {
cmds, err := (&LlamaCpp{}).Commands(llamaSpec(nil))
if err != nil {
t.Fatal(err)
}
if len(cmds) != 1 {
t.Fatalf("a llama.cpp shard is one process, got %d", len(cmds))
}
c := cmds[0]
if c.Path != "llama-server" {
t.Errorf("path %q", c.Path)
}
// --metrics is not optional: llama-server serves no /metrics without it, and
// an endpoint the router cannot measure is one it routes to blind.
if !slices.Contains(c.Args, "--metrics") {
t.Error("the server was started without --metrics")
}
if v, _ := flagValue(c.Args, "--model"); v != "/models/qwen.gguf" {
t.Errorf("--model %q", v)
}
if v, _ := flagValue(c.Args, "--port"); v != "8000" {
t.Errorf("--port %q", v)
}
if v, _ := flagValue(c.Args, "--alias"); v != "qwen-0.5b" {
t.Errorf("--alias %q", v)
}
// llama.cpp serves one request at a time unless told otherwise, which would
// make every second request wait no matter what the router decided.
if v, ok := flagValue(c.Args, "--parallel"); !ok || v == "1" {
t.Errorf("--parallel %q", v)
}
if slices.Contains(c.Args, "--n-gpu-layers") {
t.Error("a CPU placement was given GPU layers")
}
}
func TestLlamaCppOffloadsWhenGivenGPUs(t *testing.T) {
cmds, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2}))
if v, ok := flagValue(cmds[0].Args, "--n-gpu-layers"); !ok || v != "999" {
t.Errorf("a GPU placement should offload the model, got %q", v)
}
// The shard must still be pinned to its own card.
if !slices.Contains(cmds[0].Env, "CUDA_VISIBLE_DEVICES=2") {
t.Error("the shard was not pinned to its GPU")
}
}
func TestProfileArgumentsWinOverDefaults(t *testing.T) {
// The escape hatch has to actually work, or an operator tuning slot count
// ends up with two --parallel flags and whichever llama.cpp happens to
// prefer.
s := llamaSpec(nil)
s.Profile.Args = []string{"--parallel", "16", "--ctx-size", "4096"}
cmds, _ := (&LlamaCpp{}).Commands(s)
n := 0
for _, a := range cmds[0].Args {
if a == "--parallel" {
n++
}
}
if n != 1 {
t.Errorf("--parallel appears %d times", n)
}
if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "16" {
t.Errorf("the profile's value lost: %q", v)
}
}
func TestShortFormArgumentsAreRecognized(t *testing.T) {
s := llamaSpec([]int{0})
s.Profile.Args = []string{"-np", "8", "-ngl", "20"}
cmds, _ := (&LlamaCpp{}).Commands(s)
for _, flag := range []string{"--parallel", "--n-gpu-layers"} {
if slices.Contains(cmds[0].Args, flag) {
t.Errorf("%s was added although the profile set its short form", flag)
}
}
}
func TestLlamaCppRefusesWhatItCannotDo(t *testing.T) {
// Accepting either of these would produce a replica that starts and then
// serves from one machine while the catalog believes it has several.
multi := Profile{Name: "multi", Engine: "llamacpp", TP: 1, Nodes: 2, GPUsPerNode: 1, ModelPath: "/m"}
if err := (&LlamaCpp{}).Validate(multi, nil); err == nil {
t.Error("a multi-node llama.cpp profile was accepted")
}
tp := Profile{Name: "tp", Engine: "llamacpp", TP: 2, GPUsPerNode: 2, ModelPath: "/m"}
if err := (&LlamaCpp{}).Validate(tp, h100(2)); err == nil {
t.Error("a tensor-parallel llama.cpp profile was accepted")
}
}
func TestCPUProfileValidates(t *testing.T) {
p := Profile{Name: "cpu", Engine: "llamacpp", TP: 1, ModelPath: "/models/qwen.gguf"}
if err := (&LlamaCpp{}).Validate(p, nil); err != nil {
t.Errorf("a CPU profile on a node with no GPUs was rejected: %v", err)
}
}
func TestLlamaCppAlwaysServesHTTP(t *testing.T) {
// Unlike the others there is no worker rank to get this wrong for.
if !(&LlamaCpp{}).ServesHTTP(llamaSpec(nil)) {
t.Error("a llama.cpp shard must answer the API")
}
}
func TestBinaryIsConfigurable(t *testing.T) {
l := &LlamaCpp{cfg: LlamaCppConfig{Bin: "/opt/llama.cpp/bin/llama-server", Parallel: 2}}
cmds, _ := l.Commands(llamaSpec(nil))
if cmds[0].Path != "/opt/llama.cpp/bin/llama-server" {
t.Errorf("path %q", cmds[0].Path)
}
if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "2" {
t.Errorf("--parallel %q", v)
}
}
func TestUnknownEngineNamesLlamaCpp(t *testing.T) {
_, err := NewRegistry(Config{}).Get("llama.cpp")
if err == nil || !strings.Contains(err.Error(), "llamacpp") {
t.Errorf("the suggestion should name the real engine: %v", err)
}
}
// flagValue reads a "--flag value" pair out of a command line.
func flagValue(args []string, flag string) (string, bool) {
for i, a := range args {
if a == flag && i+1 < len(args) {
return args[i+1], true
}
}
return "", false
}
func TestLlamaCppUsesTheModelsChatTemplate(t *testing.T) {
// Pinned so an engine upgrade cannot change how prompts are built under a
// running fleet: build 10991 defaults to jinja, older builds do not.
//
// Not a fix for tool calling, though it was added while chasing that. On
// this build the flag is a no-op and the behaviour is identical with and
// without it.
l := &LlamaCpp{}
cmds, err := l.Commands(ShardSpec{HTTPPort: 8080, Profile: Profile{ModelPath: "/m.gguf"}})
if err != nil {
t.Fatal(err)
}
if !slices.Contains(cmds[0].Args, "--jinja") {
t.Errorf("--jinja is missing; tool calls will come back as text: %v", cmds[0].Args)
}
}
func TestLlamaCppLetsAProfileTurnTheTemplateOff(t *testing.T) {
// A model whose bundled template is broken has to be serviceable without
// editing the adapter.
l := &LlamaCpp{}
cmds, err := l.Commands(ShardSpec{
HTTPPort: 8080,
Profile: Profile{ModelPath: "/m.gguf", Args: []string{"--no-jinja"}},
})
if err != nil {
t.Fatal(err)
}
if slices.Contains(cmds[0].Args, "--jinja") {
t.Errorf("the profile said --no-jinja and got both: %v", cmds[0].Args)
}
}