From b5177b70f657f6cd606023c15d1fdbf711830d09 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Fri, 18 Sep 2026 08:31:09 +0200 Subject: [PATCH] Stop tool-check blaming the engine for what the model did MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It had one verdict for a failed attempt — "This is the engine, not the model. Choosing a bigger model will not help" — and that verdict was wrong, in a way that sends someone off rebuilding llama.cpp when the fix is a one-line catalog change. Measured on the cluster. The engine renders the tool prompt correctly: the block and the instruction to wrap the call in are both in the prompt, which I confirmed through the engine's own /apply-template. The coder models simply do not use that wrapper — qwen2.5-coder-3b puts the call in a markdown fence, qwen2.5-coder-7b invents — so the parser, which looks for , has nothing to find and leaves it in the content. qwen2.5-3b-instruct, same engine, same build, same flags, returns a populated tool_calls array. So it was the model all along, and the advice against trying another one was the most expensive sentence in the output. A failed attempt is now two verdicts, told apart by whether the reply used the wrapper its template asked for. Wrapped and unparsed is genuinely the engine's parser. Unwrapped is the model, and the fix is one trained for the format rather than a bigger one from the same family. Co-Authored-By: Claude Opus 5 (1M context) --- internal/cli/toolcheck.go | 44 ++++++++++++++++++++++++++------ internal/cli/toolcheck_test.go | 46 +++++++++++++++++++++++++++++++++- 2 files changed, 81 insertions(+), 9 deletions(-) diff --git a/internal/cli/toolcheck.go b/internal/cli/toolcheck.go index 13f7269..a74afe4 100644 --- a/internal/cli/toolcheck.go +++ b/internal/cli/toolcheck.go @@ -24,8 +24,16 @@ import ( // Found the hard way. qwen2.5-coder produced exactly the right call — // {"name": "list_files", "arguments": {"path": "."}} — as plain reply text, // with tool_calls null, and diagnosing that took an afternoon of eliminating -// the gateway, the model size, the chat template and tool_choice in turn. The -// answer was the engine build. This runs that elimination in one command. +// the gateway, the model size, the chat template and tool_choice in turn. +// +// The answer was neither the engine nor the model alone, which is why this +// command now distinguishes them by what the reply looks like rather than +// asserting one. The engine rendered the tool prompt correctly, including the +// wrapper its template asks the model to use; the model wrapped +// the call in a markdown fence instead, and the parser — which looks for that +// wrapper — left it in the content. A differently trained model fixes that. A +// bigger one from the same family does not: qwen2.5-coder-7b invented +// and failed the same way. func newToolCheckCmd() *cobra.Command { var ( endpoint string @@ -81,8 +89,17 @@ type ToolProbe struct { // separates "the model ignored its tools" from "the engine did not parse // the call the model made" — the same symptom, different fix. Attempted bool - Content string - Raw string + // Wrapped means the attempt used the tags that the Qwen and + // Hermes templates instruct the model to use. + // + // This is what separates the two halves of a failed attempt. Wrapped and + // unparsed is the engine's parser failing on output that was correct. + // Unwrapped is the model answering in a shape of its own — the engine had + // no way to recognise it, and the fix is a model trained for the format, + // not a newer engine. + Wrapped bool + Content string + Raw string } func probeToolCall(ctx context.Context, endpoint, model string) (ToolProbe, error) { @@ -162,6 +179,7 @@ func readToolProbe(raw []byte) (ToolProbe, error) { // The model tried, and the engine handed the attempt back as text. The // tell is the tool's name in a reply that was supposed to be a call. p.Attempted = p.Structured || bytes.Contains([]byte(msg.Content), []byte("get_weather")) + p.Wrapped = bytes.Contains([]byte(msg.Content), []byte("")) return p, nil } @@ -170,12 +188,22 @@ func report(cmd *cobra.Command, p ToolProbe) { switch { case p.Structured: fmt.Fprintln(out, "tool calls work: the engine returned a populated tool_calls array.") - case p.Attempted: - fmt.Fprintln(out, "the model called the tool and the engine did not parse it.") + case p.Attempted && p.Wrapped: + fmt.Fprintln(out, "the model called the tool correctly and the engine did not parse it.") fmt.Fprintln(out, "\n The call is in the reply text instead of tool_calls:") fmt.Fprintf(out, " %s\n", trim(p.Content, 200)) - fmt.Fprintln(out, "\n This is the engine, not the model. Choosing a bigger model will not") - fmt.Fprintln(out, " help. Check the engine's build and its chat template support.") + fmt.Fprintln(out, "\n It used the wrapper its template asks for, so the model") + fmt.Fprintln(out, " did its part. This is the engine's parser. Check its build; another") + fmt.Fprintln(out, " model will fail the same way.") + case p.Attempted: + fmt.Fprintln(out, "the model called the tool in a shape the engine does not recognise.") + fmt.Fprintln(out, "\n The call is in the reply text instead of tool_calls:") + fmt.Fprintf(out, " %s\n", trim(p.Content, 200)) + fmt.Fprintln(out, "\n The engine rendered the tools correctly — the model knew the tool") + fmt.Fprintln(out, " existed and what to pass it — but did not use the wrapper") + fmt.Fprintln(out, " its template asked for, so there was nothing for the parser to find.") + fmt.Fprintln(out, "\n Fixed by a model trained for this format, not by a bigger one: the") + fmt.Fprintln(out, " instruct models generally manage it where the coder variants do not.") default: fmt.Fprintln(out, "the model did not try to call the tool.") fmt.Fprintf(out, "\n It replied: %s\n", trim(p.Content, 200)) diff --git a/internal/cli/toolcheck_test.go b/internal/cli/toolcheck_test.go index b20f762..e81f167 100644 --- a/internal/cli/toolcheck_test.go +++ b/internal/cli/toolcheck_test.go @@ -19,7 +19,7 @@ func TestAStructuredToolCallIsRecognised(t *testing.T) { } } -func TestACallReturnedAsTextIsBlamedOnTheEngine(t *testing.T) { +func TestACallReturnedAsTextIsAnAttempt(t *testing.T) { // The exact reply that cost an afternoon: the model made the right call and // the engine handed it back as prose. p, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", @@ -33,6 +33,50 @@ func TestACallReturnedAsTextIsBlamedOnTheEngine(t *testing.T) { if !p.Attempted { t.Error("the model plainly tried; that must be distinguished from ignoring the tools") } + if p.Wrapped { + t.Error("bare JSON was read as having used the wrapper") + } +} + +func TestAWrappedCallBlamesTheEngineAndABareOneDoesNot(t *testing.T) { + // Both look identical from outside — tool_calls null, the call sitting in + // the content — and they have opposite fixes. The whole point of the + // command is telling them apart, and the earlier version did not: it + // blamed the engine for both, which sends someone off rebuilding llama.cpp + // when the answer is a different model. + wrapped, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", + "content":"\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Istanbul\"}}\n"}}]}`)) + if err != nil { + t.Fatal(err) + } + if !wrapped.Attempted || !wrapped.Wrapped { + t.Errorf("a correctly wrapped call was not recognised as one: %+v", wrapped) + } + + // What qwen2.5-coder-3b actually returns, measured: the right call in a + // markdown fence, which the parser has no way to find. + fenced, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", + "content":"` + "```" + `json\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Istanbul\"}}\n` + "```" + `"}}]}`)) + if err != nil { + t.Fatal(err) + } + if !fenced.Attempted { + t.Error("a fenced call is still an attempt") + } + if fenced.Wrapped { + t.Error("a markdown fence was mistaken for the wrapper") + } + + // And what qwen2.5-coder-7b returns: a tag it invented, which is not the + // one the template asked for and is not parsed either. + invented, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", + "content":"\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Istanbul\"}}\n"}}]}`)) + if err != nil { + t.Fatal(err) + } + if !invented.Attempted || invented.Wrapped { + t.Errorf("an invented tag was read as the template's wrapper: %+v", invented) + } } func TestAModelThatIgnoresItsToolsIsBlamedOnTheModel(t *testing.T) {