9fc098f said passing --jinja fixes tool calling. It does not. llama.cpp
build 10991 has jinja enabled by default — "--jinja, --no-jinja ... (default:
enabled)" — so the flag changed nothing, and tool calls came back as text
before and after it in exactly the same way.
The flag is worth keeping for a different reason, and the comment now says
that one: it pins how prompts are built, so an engine upgrade cannot change
it underneath a running fleet. Older builds do default to the generic
template.
What actually happens is that llama.cpp renders the tools correctly — the
prompt from /apply-template carries the <tools> block and the instruction to
reply in <tool_call> tags — and qwen2.5-coder answers with a markdown JSON
block instead. 3B and 7B both, and tool_choice "required" does not constrain
it either. That is the model, not the plumbing, and no flag here will fix
it.
Recording it because a wrong explanation in a commit message is worse than
no explanation: the next person to see tool calls arriving as text would
have checked --jinja, found it already set, and had nowhere to go.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
190 lines
5.9 KiB
Go
190 lines
5.9 KiB
Go
package engines
|
|
|
|
import (
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func llamaSpec(gpus []int) ShardSpec {
|
|
return ShardSpec{
|
|
ReplicaID: "rep-1", ShardID: "sh-1", Rank: 0, NNodes: 1,
|
|
GPUIndices: gpus, HTTPPort: 8000, MgmtInterface: "eth0",
|
|
Profile: Profile{
|
|
Name: "cpu", Engine: "llamacpp", TP: 1,
|
|
ModelPath: "/models/qwen.gguf", ServedModelName: "qwen-0.5b",
|
|
},
|
|
}
|
|
}
|
|
|
|
func TestLlamaCppCommand(t *testing.T) {
|
|
cmds, err := (&LlamaCpp{}).Commands(llamaSpec(nil))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(cmds) != 1 {
|
|
t.Fatalf("a llama.cpp shard is one process, got %d", len(cmds))
|
|
}
|
|
c := cmds[0]
|
|
if c.Path != "llama-server" {
|
|
t.Errorf("path %q", c.Path)
|
|
}
|
|
|
|
// --metrics is not optional: llama-server serves no /metrics without it, and
|
|
// an endpoint the router cannot measure is one it routes to blind.
|
|
if !slices.Contains(c.Args, "--metrics") {
|
|
t.Error("the server was started without --metrics")
|
|
}
|
|
if v, _ := flagValue(c.Args, "--model"); v != "/models/qwen.gguf" {
|
|
t.Errorf("--model %q", v)
|
|
}
|
|
if v, _ := flagValue(c.Args, "--port"); v != "8000" {
|
|
t.Errorf("--port %q", v)
|
|
}
|
|
if v, _ := flagValue(c.Args, "--alias"); v != "qwen-0.5b" {
|
|
t.Errorf("--alias %q", v)
|
|
}
|
|
// llama.cpp serves one request at a time unless told otherwise, which would
|
|
// make every second request wait no matter what the router decided.
|
|
if v, ok := flagValue(c.Args, "--parallel"); !ok || v == "1" {
|
|
t.Errorf("--parallel %q", v)
|
|
}
|
|
if slices.Contains(c.Args, "--n-gpu-layers") {
|
|
t.Error("a CPU placement was given GPU layers")
|
|
}
|
|
}
|
|
|
|
func TestLlamaCppOffloadsWhenGivenGPUs(t *testing.T) {
|
|
cmds, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2}))
|
|
if v, ok := flagValue(cmds[0].Args, "--n-gpu-layers"); !ok || v != "999" {
|
|
t.Errorf("a GPU placement should offload the model, got %q", v)
|
|
}
|
|
// The shard must still be pinned to its own card.
|
|
if !slices.Contains(cmds[0].Env, "CUDA_VISIBLE_DEVICES=2") {
|
|
t.Error("the shard was not pinned to its GPU")
|
|
}
|
|
}
|
|
|
|
func TestProfileArgumentsWinOverDefaults(t *testing.T) {
|
|
// The escape hatch has to actually work, or an operator tuning slot count
|
|
// ends up with two --parallel flags and whichever llama.cpp happens to
|
|
// prefer.
|
|
s := llamaSpec(nil)
|
|
s.Profile.Args = []string{"--parallel", "16", "--ctx-size", "4096"}
|
|
|
|
cmds, _ := (&LlamaCpp{}).Commands(s)
|
|
n := 0
|
|
for _, a := range cmds[0].Args {
|
|
if a == "--parallel" {
|
|
n++
|
|
}
|
|
}
|
|
if n != 1 {
|
|
t.Errorf("--parallel appears %d times", n)
|
|
}
|
|
if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "16" {
|
|
t.Errorf("the profile's value lost: %q", v)
|
|
}
|
|
}
|
|
|
|
func TestShortFormArgumentsAreRecognized(t *testing.T) {
|
|
s := llamaSpec([]int{0})
|
|
s.Profile.Args = []string{"-np", "8", "-ngl", "20"}
|
|
|
|
cmds, _ := (&LlamaCpp{}).Commands(s)
|
|
for _, flag := range []string{"--parallel", "--n-gpu-layers"} {
|
|
if slices.Contains(cmds[0].Args, flag) {
|
|
t.Errorf("%s was added although the profile set its short form", flag)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestLlamaCppRefusesWhatItCannotDo(t *testing.T) {
|
|
// Accepting either of these would produce a replica that starts and then
|
|
// serves from one machine while the catalog believes it has several.
|
|
multi := Profile{Name: "multi", Engine: "llamacpp", TP: 1, Nodes: 2, GPUsPerNode: 1, ModelPath: "/m"}
|
|
if err := (&LlamaCpp{}).Validate(multi, nil); err == nil {
|
|
t.Error("a multi-node llama.cpp profile was accepted")
|
|
}
|
|
|
|
tp := Profile{Name: "tp", Engine: "llamacpp", TP: 2, GPUsPerNode: 2, ModelPath: "/m"}
|
|
if err := (&LlamaCpp{}).Validate(tp, h100(2)); err == nil {
|
|
t.Error("a tensor-parallel llama.cpp profile was accepted")
|
|
}
|
|
}
|
|
|
|
func TestCPUProfileValidates(t *testing.T) {
|
|
p := Profile{Name: "cpu", Engine: "llamacpp", TP: 1, ModelPath: "/models/qwen.gguf"}
|
|
if err := (&LlamaCpp{}).Validate(p, nil); err != nil {
|
|
t.Errorf("a CPU profile on a node with no GPUs was rejected: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestLlamaCppAlwaysServesHTTP(t *testing.T) {
|
|
// Unlike the others there is no worker rank to get this wrong for.
|
|
if !(&LlamaCpp{}).ServesHTTP(llamaSpec(nil)) {
|
|
t.Error("a llama.cpp shard must answer the API")
|
|
}
|
|
}
|
|
|
|
func TestBinaryIsConfigurable(t *testing.T) {
|
|
l := &LlamaCpp{cfg: LlamaCppConfig{Bin: "/opt/llama.cpp/bin/llama-server", Parallel: 2}}
|
|
cmds, _ := l.Commands(llamaSpec(nil))
|
|
if cmds[0].Path != "/opt/llama.cpp/bin/llama-server" {
|
|
t.Errorf("path %q", cmds[0].Path)
|
|
}
|
|
if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "2" {
|
|
t.Errorf("--parallel %q", v)
|
|
}
|
|
}
|
|
|
|
func TestUnknownEngineNamesLlamaCpp(t *testing.T) {
|
|
_, err := NewRegistry(Config{}).Get("llama.cpp")
|
|
if err == nil || !strings.Contains(err.Error(), "llamacpp") {
|
|
t.Errorf("the suggestion should name the real engine: %v", err)
|
|
}
|
|
}
|
|
|
|
// flagValue reads a "--flag value" pair out of a command line.
|
|
func flagValue(args []string, flag string) (string, bool) {
|
|
for i, a := range args {
|
|
if a == flag && i+1 < len(args) {
|
|
return args[i+1], true
|
|
}
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
func TestLlamaCppUsesTheModelsChatTemplate(t *testing.T) {
|
|
// Pinned so an engine upgrade cannot change how prompts are built under a
|
|
// running fleet: build 10991 defaults to jinja, older builds do not.
|
|
//
|
|
// Not a fix for tool calling, though it was added while chasing that. On
|
|
// this build the flag is a no-op and the behaviour is identical with and
|
|
// without it.
|
|
l := &LlamaCpp{}
|
|
cmds, err := l.Commands(ShardSpec{HTTPPort: 8080, Profile: Profile{ModelPath: "/m.gguf"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !slices.Contains(cmds[0].Args, "--jinja") {
|
|
t.Errorf("--jinja is missing; tool calls will come back as text: %v", cmds[0].Args)
|
|
}
|
|
}
|
|
|
|
func TestLlamaCppLetsAProfileTurnTheTemplateOff(t *testing.T) {
|
|
// A model whose bundled template is broken has to be serviceable without
|
|
// editing the adapter.
|
|
l := &LlamaCpp{}
|
|
cmds, err := l.Commands(ShardSpec{
|
|
HTTPPort: 8080,
|
|
Profile: Profile{ModelPath: "/m.gguf", Args: []string{"--no-jinja"}},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if slices.Contains(cmds[0].Args, "--jinja") {
|
|
t.Errorf("the profile said --no-jinja and got both: %v", cmds[0].Args)
|
|
}
|
|
}
|