From ea87021b5c58f60fd6959d537d5ca40c68d7313f Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 16 Sep 2026 21:30:22 +0200 Subject: [PATCH] Correct what the previous commit claimed about --jinja MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 9fc098f said passing --jinja fixes tool calling. It does not. llama.cpp build 10991 has jinja enabled by default — "--jinja, --no-jinja ... (default: enabled)" — so the flag changed nothing, and tool calls came back as text before and after it in exactly the same way. The flag is worth keeping for a different reason, and the comment now says that one: it pins how prompts are built, so an engine upgrade cannot change it underneath a running fleet. Older builds do default to the generic template. What actually happens is that llama.cpp renders the tools correctly — the prompt from /apply-template carries the block and the instruction to reply in tags — and qwen2.5-coder answers with a markdown JSON block instead. 3B and 7B both, and tool_choice "required" does not constrain it either. That is the model, not the plumbing, and no flag here will fix it. Recording it because a wrong explanation in a commit message is worse than no explanation: the next person to see tool calls arriving as text would have checked --jinja, found it already set, and had nowhere to go. Co-Authored-By: Claude Opus 5 (1M context) --- internal/agent/engines/llamacpp.go | 15 ++++++++------- internal/agent/engines/llamacpp_test.go | 11 ++++++----- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/internal/agent/engines/llamacpp.go b/internal/agent/engines/llamacpp.go index 489c0c5b..1f411e85 100644 --- a/internal/agent/engines/llamacpp.go +++ b/internal/agent/engines/llamacpp.go @@ -79,14 +79,15 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) { // measure is one it has to route to blind. "--metrics", } - // The model's own chat template, which is what turns a tool call into a - // tool_calls array instead of a JSON object sitting in the reply text. + // The model's own chat template rather than a generic one, which is what + // renders tool definitions in the form the model was trained to read. // - // Without it qwen2.5-coder emitted exactly the right call — - // {"name": "list_files", "arguments": {"path": "."}} — as plain content, - // and every OpenAI-compatible client saw a model that ignores its tools. - // It also affects ordinary chat, because the generic fallback template is - // not the one the model was trained on. + // Pinned rather than relied on: build 10991 has it on by default, older + // builds do not, and an engine upgrade should not be able to change how + // prompts are built underneath a running fleet. Passing it explicitly was + // a no-op on the build here — it was added while chasing tool calls that + // came back as text, and did not fix that, which turned out to be the + // model rather than the template. if !hasFlag(p.Args, "--jinja", "--no-jinja") { args = append(args, "--jinja") } diff --git a/internal/agent/engines/llamacpp_test.go b/internal/agent/engines/llamacpp_test.go index c23deafa..30419ee6 100644 --- a/internal/agent/engines/llamacpp_test.go +++ b/internal/agent/engines/llamacpp_test.go @@ -156,11 +156,12 @@ func flagValue(args []string, flag string) (string, bool) { } func TestLlamaCppUsesTheModelsChatTemplate(t *testing.T) { - // Tool calling is the reason. Without --jinja, llama.cpp uses a generic - // template and a model that decides to call a tool emits the call as text: - // qwen2.5-coder returned {"name": "list_files", "arguments": {"path": "."}} - // in content, with tool_calls null, so every OpenAI-compatible client saw a - // model that ignores the tools it was given. + // Pinned so an engine upgrade cannot change how prompts are built under a + // running fleet: build 10991 defaults to jinja, older builds do not. + // + // Not a fix for tool calling, though it was added while chasing that. On + // this build the flag is a no-op and the behaviour is identical with and + // without it. l := &LlamaCpp{} cmds, err := l.Commands(ShardSpec{HTTPPort: 8080, Profile: Profile{ModelPath: "/m.gguf"}}) if err != nil {