Stop tool-check blaming the engine for what the model did
It had one verdict for a failed attempt — "This is the engine, not the model. Choosing a bigger model will not help" — and that verdict was wrong, in a way that sends someone off rebuilding llama.cpp when the fix is a one-line catalog change. Measured on the cluster. The engine renders the tool prompt correctly: the <tools> block and the instruction to wrap the call in <tool_call> are both in the prompt, which I confirmed through the engine's own /apply-template. The coder models simply do not use that wrapper — qwen2.5-coder-3b puts the call in a markdown fence, qwen2.5-coder-7b invents <function-call> — so the parser, which looks for <tool_call>, has nothing to find and leaves it in the content. qwen2.5-3b-instruct, same engine, same build, same flags, returns a populated tool_calls array. So it was the model all along, and the advice against trying another one was the most expensive sentence in the output. A failed attempt is now two verdicts, told apart by whether the reply used the wrapper its template asked for. Wrapped and unparsed is genuinely the engine's parser. Unwrapped is the model, and the fix is one trained for the format rather than a bigger one from the same family. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
b58d67602d
commit
b5177b70f6
2 files changed
+81
-9
No files matched your search
@@ -19,7 +19,7 @@ func TestAStructuredToolCallIsRecognised(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestACallReturnedAsTextIsBlamedOnTheEngine(t *testing.T) {
|
||||
func TestACallReturnedAsTextIsAnAttempt(t *testing.T) {
|
||||
// The exact reply that cost an afternoon: the model made the right call and
|
||||
// the engine handed it back as prose.
|
||||
p, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant",
|
||||
@@ -33,6 +33,50 @@ func TestACallReturnedAsTextIsBlamedOnTheEngine(t *testing.T) {
|
||||
if !p.Attempted {
|
||||
t.Error("the model plainly tried; that must be distinguished from ignoring the tools")
|
||||
}
|
||||
if p.Wrapped {
|
||||
t.Error("bare JSON was read as having used the <tool_call> wrapper")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWrappedCallBlamesTheEngineAndABareOneDoesNot(t *testing.T) {
|
||||
// Both look identical from outside — tool_calls null, the call sitting in
|
||||
// the content — and they have opposite fixes. The whole point of the
|
||||
// command is telling them apart, and the earlier version did not: it
|
||||
// blamed the engine for both, which sends someone off rebuilding llama.cpp
|
||||
// when the answer is a different model.
|
||||
wrapped, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant",
|
||||
"content":"<tool_call>\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Istanbul\"}}\n</tool_call>"}}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !wrapped.Attempted || !wrapped.Wrapped {
|
||||
t.Errorf("a correctly wrapped call was not recognised as one: %+v", wrapped)
|
||||
}
|
||||
|
||||
// What qwen2.5-coder-3b actually returns, measured: the right call in a
|
||||
// markdown fence, which the parser has no way to find.
|
||||
fenced, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant",
|
||||
"content":"` + "```" + `json\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Istanbul\"}}\n` + "```" + `"}}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !fenced.Attempted {
|
||||
t.Error("a fenced call is still an attempt")
|
||||
}
|
||||
if fenced.Wrapped {
|
||||
t.Error("a markdown fence was mistaken for the <tool_call> wrapper")
|
||||
}
|
||||
|
||||
// And what qwen2.5-coder-7b returns: a tag it invented, which is not the
|
||||
// one the template asked for and is not parsed either.
|
||||
invented, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant",
|
||||
"content":"<function-call>\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Istanbul\"}}\n</function-call>"}}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !invented.Attempted || invented.Wrapped {
|
||||
t.Errorf("an invented tag was read as the template's wrapper: %+v", invented)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModelThatIgnoresItsToolsIsBlamedOnTheModel(t *testing.T) {
|
||||
|
||||
Reference in new issue
Block a user