diff --git a/internal/cli/toolcheck.go b/internal/cli/toolcheck.go index 13f7269..a74afe4 100644 --- a/internal/cli/toolcheck.go +++ b/internal/cli/toolcheck.go @@ -24,8 +24,16 @@ import ( // Found the hard way. qwen2.5-coder produced exactly the right call — // {"name": "list_files", "arguments": {"path": "."}} — as plain reply text, // with tool_calls null, and diagnosing that took an afternoon of eliminating -// the gateway, the model size, the chat template and tool_choice in turn. The -// answer was the engine build. This runs that elimination in one command. +// the gateway, the model size, the chat template and tool_choice in turn. +// +// The answer was neither the engine nor the model alone, which is why this +// command now distinguishes them by what the reply looks like rather than +// asserting one. The engine rendered the tool prompt correctly, including the +// wrapper its template asks the model to use; the model wrapped +// the call in a markdown fence instead, and the parser — which looks for that +// wrapper — left it in the content. A differently trained model fixes that. A +// bigger one from the same family does not: qwen2.5-coder-7b invented +// and failed the same way. func newToolCheckCmd() *cobra.Command { var ( endpoint string @@ -81,8 +89,17 @@ type ToolProbe struct { // separates "the model ignored its tools" from "the engine did not parse // the call the model made" — the same symptom, different fix. Attempted bool - Content string - Raw string + // Wrapped means the attempt used the tags that the Qwen and + // Hermes templates instruct the model to use. + // + // This is what separates the two halves of a failed attempt. Wrapped and + // unparsed is the engine's parser failing on output that was correct. + // Unwrapped is the model answering in a shape of its own — the engine had + // no way to recognise it, and the fix is a model trained for the format, + // not a newer engine. + Wrapped bool + Content string + Raw string } func probeToolCall(ctx context.Context, endpoint, model string) (ToolProbe, error) { @@ -162,6 +179,7 @@ func readToolProbe(raw []byte) (ToolProbe, error) { // The model tried, and the engine handed the attempt back as text. The // tell is the tool's name in a reply that was supposed to be a call. p.Attempted = p.Structured || bytes.Contains([]byte(msg.Content), []byte("get_weather")) + p.Wrapped = bytes.Contains([]byte(msg.Content), []byte("")) return p, nil } @@ -170,12 +188,22 @@ func report(cmd *cobra.Command, p ToolProbe) { switch { case p.Structured: fmt.Fprintln(out, "tool calls work: the engine returned a populated tool_calls array.") - case p.Attempted: - fmt.Fprintln(out, "the model called the tool and the engine did not parse it.") + case p.Attempted && p.Wrapped: + fmt.Fprintln(out, "the model called the tool correctly and the engine did not parse it.") fmt.Fprintln(out, "\n The call is in the reply text instead of tool_calls:") fmt.Fprintf(out, " %s\n", trim(p.Content, 200)) - fmt.Fprintln(out, "\n This is the engine, not the model. Choosing a bigger model will not") - fmt.Fprintln(out, " help. Check the engine's build and its chat template support.") + fmt.Fprintln(out, "\n It used the wrapper its template asks for, so the model") + fmt.Fprintln(out, " did its part. This is the engine's parser. Check its build; another") + fmt.Fprintln(out, " model will fail the same way.") + case p.Attempted: + fmt.Fprintln(out, "the model called the tool in a shape the engine does not recognise.") + fmt.Fprintln(out, "\n The call is in the reply text instead of tool_calls:") + fmt.Fprintf(out, " %s\n", trim(p.Content, 200)) + fmt.Fprintln(out, "\n The engine rendered the tools correctly — the model knew the tool") + fmt.Fprintln(out, " existed and what to pass it — but did not use the wrapper") + fmt.Fprintln(out, " its template asked for, so there was nothing for the parser to find.") + fmt.Fprintln(out, "\n Fixed by a model trained for this format, not by a bigger one: the") + fmt.Fprintln(out, " instruct models generally manage it where the coder variants do not.") default: fmt.Fprintln(out, "the model did not try to call the tool.") fmt.Fprintf(out, "\n It replied: %s\n", trim(p.Content, 200)) diff --git a/internal/cli/toolcheck_test.go b/internal/cli/toolcheck_test.go index b20f762..e81f167 100644 --- a/internal/cli/toolcheck_test.go +++ b/internal/cli/toolcheck_test.go @@ -19,7 +19,7 @@ func TestAStructuredToolCallIsRecognised(t *testing.T) { } } -func TestACallReturnedAsTextIsBlamedOnTheEngine(t *testing.T) { +func TestACallReturnedAsTextIsAnAttempt(t *testing.T) { // The exact reply that cost an afternoon: the model made the right call and // the engine handed it back as prose. p, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", @@ -33,6 +33,50 @@ func TestACallReturnedAsTextIsBlamedOnTheEngine(t *testing.T) { if !p.Attempted { t.Error("the model plainly tried; that must be distinguished from ignoring the tools") } + if p.Wrapped { + t.Error("bare JSON was read as having used the wrapper") + } +} + +func TestAWrappedCallBlamesTheEngineAndABareOneDoesNot(t *testing.T) { + // Both look identical from outside — tool_calls null, the call sitting in + // the content — and they have opposite fixes. The whole point of the + // command is telling them apart, and the earlier version did not: it + // blamed the engine for both, which sends someone off rebuilding llama.cpp + // when the answer is a different model. + wrapped, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", + "content":"\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Istanbul\"}}\n"}}]}`)) + if err != nil { + t.Fatal(err) + } + if !wrapped.Attempted || !wrapped.Wrapped { + t.Errorf("a correctly wrapped call was not recognised as one: %+v", wrapped) + } + + // What qwen2.5-coder-3b actually returns, measured: the right call in a + // markdown fence, which the parser has no way to find. + fenced, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", + "content":"` + "```" + `json\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Istanbul\"}}\n` + "```" + `"}}]}`)) + if err != nil { + t.Fatal(err) + } + if !fenced.Attempted { + t.Error("a fenced call is still an attempt") + } + if fenced.Wrapped { + t.Error("a markdown fence was mistaken for the wrapper") + } + + // And what qwen2.5-coder-7b returns: a tag it invented, which is not the + // one the template asked for and is not parsed either. + invented, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", + "content":"\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Istanbul\"}}\n"}}]}`)) + if err != nil { + t.Fatal(err) + } + if !invented.Attempted || invented.Wrapped { + t.Errorf("an invented tag was read as the template's wrapper: %+v", invented) + } } func TestAModelThatIgnoresItsToolsIsBlamedOnTheModel(t *testing.T) {