Tool calling did not work, and it looked like the model's fault. Asked to
use a tool, qwen2.5-coder produced exactly the right call —
{"name": "list_files", "arguments": {"path": "."}} — as plain reply text,
with tool_calls null and finish_reason "stop". Every OpenAI-compatible
client, opencode among them, sees a model that ignores the tools it was
given.
The cause is that llama-server falls back to a generic chat template unless
--jinja is passed, and the generic one neither formats the tool definitions
the way the model was trained to read them nor parses what comes back into
tool_calls. It affects ordinary chat too, for the same reason.
A profile can still pass --no-jinja, because a model whose bundled template
is broken has to be serviceable without editing the adapter.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
189 lines
6.0 KiB
Go
189 lines
6.0 KiB
Go
package engines
|
|
|
|
import (
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func llamaSpec(gpus []int) ShardSpec {
|
|
return ShardSpec{
|
|
ReplicaID: "rep-1", ShardID: "sh-1", Rank: 0, NNodes: 1,
|
|
GPUIndices: gpus, HTTPPort: 8000, MgmtInterface: "eth0",
|
|
Profile: Profile{
|
|
Name: "cpu", Engine: "llamacpp", TP: 1,
|
|
ModelPath: "/models/qwen.gguf", ServedModelName: "qwen-0.5b",
|
|
},
|
|
}
|
|
}
|
|
|
|
func TestLlamaCppCommand(t *testing.T) {
|
|
cmds, err := (&LlamaCpp{}).Commands(llamaSpec(nil))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(cmds) != 1 {
|
|
t.Fatalf("a llama.cpp shard is one process, got %d", len(cmds))
|
|
}
|
|
c := cmds[0]
|
|
if c.Path != "llama-server" {
|
|
t.Errorf("path %q", c.Path)
|
|
}
|
|
|
|
// --metrics is not optional: llama-server serves no /metrics without it, and
|
|
// an endpoint the router cannot measure is one it routes to blind.
|
|
if !slices.Contains(c.Args, "--metrics") {
|
|
t.Error("the server was started without --metrics")
|
|
}
|
|
if v, _ := flagValue(c.Args, "--model"); v != "/models/qwen.gguf" {
|
|
t.Errorf("--model %q", v)
|
|
}
|
|
if v, _ := flagValue(c.Args, "--port"); v != "8000" {
|
|
t.Errorf("--port %q", v)
|
|
}
|
|
if v, _ := flagValue(c.Args, "--alias"); v != "qwen-0.5b" {
|
|
t.Errorf("--alias %q", v)
|
|
}
|
|
// llama.cpp serves one request at a time unless told otherwise, which would
|
|
// make every second request wait no matter what the router decided.
|
|
if v, ok := flagValue(c.Args, "--parallel"); !ok || v == "1" {
|
|
t.Errorf("--parallel %q", v)
|
|
}
|
|
if slices.Contains(c.Args, "--n-gpu-layers") {
|
|
t.Error("a CPU placement was given GPU layers")
|
|
}
|
|
}
|
|
|
|
func TestLlamaCppOffloadsWhenGivenGPUs(t *testing.T) {
|
|
cmds, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2}))
|
|
if v, ok := flagValue(cmds[0].Args, "--n-gpu-layers"); !ok || v != "999" {
|
|
t.Errorf("a GPU placement should offload the model, got %q", v)
|
|
}
|
|
// The shard must still be pinned to its own card.
|
|
if !slices.Contains(cmds[0].Env, "CUDA_VISIBLE_DEVICES=2") {
|
|
t.Error("the shard was not pinned to its GPU")
|
|
}
|
|
}
|
|
|
|
func TestProfileArgumentsWinOverDefaults(t *testing.T) {
|
|
// The escape hatch has to actually work, or an operator tuning slot count
|
|
// ends up with two --parallel flags and whichever llama.cpp happens to
|
|
// prefer.
|
|
s := llamaSpec(nil)
|
|
s.Profile.Args = []string{"--parallel", "16", "--ctx-size", "4096"}
|
|
|
|
cmds, _ := (&LlamaCpp{}).Commands(s)
|
|
n := 0
|
|
for _, a := range cmds[0].Args {
|
|
if a == "--parallel" {
|
|
n++
|
|
}
|
|
}
|
|
if n != 1 {
|
|
t.Errorf("--parallel appears %d times", n)
|
|
}
|
|
if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "16" {
|
|
t.Errorf("the profile's value lost: %q", v)
|
|
}
|
|
}
|
|
|
|
func TestShortFormArgumentsAreRecognized(t *testing.T) {
|
|
s := llamaSpec([]int{0})
|
|
s.Profile.Args = []string{"-np", "8", "-ngl", "20"}
|
|
|
|
cmds, _ := (&LlamaCpp{}).Commands(s)
|
|
for _, flag := range []string{"--parallel", "--n-gpu-layers"} {
|
|
if slices.Contains(cmds[0].Args, flag) {
|
|
t.Errorf("%s was added although the profile set its short form", flag)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestLlamaCppRefusesWhatItCannotDo(t *testing.T) {
|
|
// Accepting either of these would produce a replica that starts and then
|
|
// serves from one machine while the catalog believes it has several.
|
|
multi := Profile{Name: "multi", Engine: "llamacpp", TP: 1, Nodes: 2, GPUsPerNode: 1, ModelPath: "/m"}
|
|
if err := (&LlamaCpp{}).Validate(multi, nil); err == nil {
|
|
t.Error("a multi-node llama.cpp profile was accepted")
|
|
}
|
|
|
|
tp := Profile{Name: "tp", Engine: "llamacpp", TP: 2, GPUsPerNode: 2, ModelPath: "/m"}
|
|
if err := (&LlamaCpp{}).Validate(tp, h100(2)); err == nil {
|
|
t.Error("a tensor-parallel llama.cpp profile was accepted")
|
|
}
|
|
}
|
|
|
|
func TestCPUProfileValidates(t *testing.T) {
|
|
p := Profile{Name: "cpu", Engine: "llamacpp", TP: 1, ModelPath: "/models/qwen.gguf"}
|
|
if err := (&LlamaCpp{}).Validate(p, nil); err != nil {
|
|
t.Errorf("a CPU profile on a node with no GPUs was rejected: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestLlamaCppAlwaysServesHTTP(t *testing.T) {
|
|
// Unlike the others there is no worker rank to get this wrong for.
|
|
if !(&LlamaCpp{}).ServesHTTP(llamaSpec(nil)) {
|
|
t.Error("a llama.cpp shard must answer the API")
|
|
}
|
|
}
|
|
|
|
func TestBinaryIsConfigurable(t *testing.T) {
|
|
l := &LlamaCpp{cfg: LlamaCppConfig{Bin: "/opt/llama.cpp/bin/llama-server", Parallel: 2}}
|
|
cmds, _ := l.Commands(llamaSpec(nil))
|
|
if cmds[0].Path != "/opt/llama.cpp/bin/llama-server" {
|
|
t.Errorf("path %q", cmds[0].Path)
|
|
}
|
|
if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "2" {
|
|
t.Errorf("--parallel %q", v)
|
|
}
|
|
}
|
|
|
|
func TestUnknownEngineNamesLlamaCpp(t *testing.T) {
|
|
_, err := NewRegistry(Config{}).Get("llama.cpp")
|
|
if err == nil || !strings.Contains(err.Error(), "llamacpp") {
|
|
t.Errorf("the suggestion should name the real engine: %v", err)
|
|
}
|
|
}
|
|
|
|
// flagValue reads a "--flag value" pair out of a command line.
|
|
func flagValue(args []string, flag string) (string, bool) {
|
|
for i, a := range args {
|
|
if a == flag && i+1 < len(args) {
|
|
return args[i+1], true
|
|
}
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
func TestLlamaCppUsesTheModelsChatTemplate(t *testing.T) {
|
|
// Tool calling is the reason. Without --jinja, llama.cpp uses a generic
|
|
// template and a model that decides to call a tool emits the call as text:
|
|
// qwen2.5-coder returned {"name": "list_files", "arguments": {"path": "."}}
|
|
// in content, with tool_calls null, so every OpenAI-compatible client saw a
|
|
// model that ignores the tools it was given.
|
|
l := &LlamaCpp{}
|
|
cmds, err := l.Commands(ShardSpec{HTTPPort: 8080, Profile: Profile{ModelPath: "/m.gguf"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !slices.Contains(cmds[0].Args, "--jinja") {
|
|
t.Errorf("--jinja is missing; tool calls will come back as text: %v", cmds[0].Args)
|
|
}
|
|
}
|
|
|
|
func TestLlamaCppLetsAProfileTurnTheTemplateOff(t *testing.T) {
|
|
// A model whose bundled template is broken has to be serviceable without
|
|
// editing the adapter.
|
|
l := &LlamaCpp{}
|
|
cmds, err := l.Commands(ShardSpec{
|
|
HTTPPort: 8080,
|
|
Profile: Profile{ModelPath: "/m.gguf", Args: []string{"--no-jinja"}},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if slices.Contains(cmds[0].Args, "--jinja") {
|
|
t.Errorf("the profile said --no-jinja and got both: %v", cmds[0].Args)
|
|
}
|
|
}
|