diff --git a/internal/agent/engines/llamacpp.go b/internal/agent/engines/llamacpp.go index 4fecb09..489c0c5 100644 --- a/internal/agent/engines/llamacpp.go +++ b/internal/agent/engines/llamacpp.go @@ -79,6 +79,17 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) { // measure is one it has to route to blind. "--metrics", } + // The model's own chat template, which is what turns a tool call into a + // tool_calls array instead of a JSON object sitting in the reply text. + // + // Without it qwen2.5-coder emitted exactly the right call — + // {"name": "list_files", "arguments": {"path": "."}} — as plain content, + // and every OpenAI-compatible client saw a model that ignores its tools. + // It also affects ordinary chat, because the generic fallback template is + // not the one the model was trained on. + if !hasFlag(p.Args, "--jinja", "--no-jinja") { + args = append(args, "--jinja") + } if p.ServedModelName != "" { args = append(args, "--alias", p.ServedModelName) } diff --git a/internal/agent/engines/llamacpp_test.go b/internal/agent/engines/llamacpp_test.go index 3b17ab0..c23deaf 100644 --- a/internal/agent/engines/llamacpp_test.go +++ b/internal/agent/engines/llamacpp_test.go @@ -154,3 +154,35 @@ func flagValue(args []string, flag string) (string, bool) { } return "", false } + +func TestLlamaCppUsesTheModelsChatTemplate(t *testing.T) { + // Tool calling is the reason. Without --jinja, llama.cpp uses a generic + // template and a model that decides to call a tool emits the call as text: + // qwen2.5-coder returned {"name": "list_files", "arguments": {"path": "."}} + // in content, with tool_calls null, so every OpenAI-compatible client saw a + // model that ignores the tools it was given. + l := &LlamaCpp{} + cmds, err := l.Commands(ShardSpec{HTTPPort: 8080, Profile: Profile{ModelPath: "/m.gguf"}}) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(cmds[0].Args, "--jinja") { + t.Errorf("--jinja is missing; tool calls will come back as text: %v", cmds[0].Args) + } +} + +func TestLlamaCppLetsAProfileTurnTheTemplateOff(t *testing.T) { + // A model whose bundled template is broken has to be serviceable without + // editing the adapter. + l := &LlamaCpp{} + cmds, err := l.Commands(ShardSpec{ + HTTPPort: 8080, + Profile: Profile{ModelPath: "/m.gguf", Args: []string{"--no-jinja"}}, + }) + if err != nil { + t.Fatal(err) + } + if slices.Contains(cmds[0].Args, "--jinja") { + t.Errorf("the profile said --no-jinja and got both: %v", cmds[0].Args) + } +}