diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index 9d3d1363..e3f4a684 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -110,6 +110,7 @@ func newCtlCmd() *cobra.Command { // apply to it; it lives here because this is where operators look. newBenchCmd(&asJSON), newEngineCheckCmd(), + newToolCheckCmd(), newCertCmd(), ) return cmd diff --git a/internal/cli/toolcheck.go b/internal/cli/toolcheck.go new file mode 100644 index 00000000..13f72692 --- /dev/null +++ b/internal/cli/toolcheck.go @@ -0,0 +1,192 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/spf13/cobra" +) + +// newToolCheckCmd asks a running engine whether tool calls come back as tool +// calls. +// +// A model's capabilities are declared in the catalog, which makes them claims. +// This is the one that can be checked, and the one worth checking: the gateway +// forwards a tools array to any model, and an engine that does not return +// structured tool_calls looks exactly like a model that ignores its tools. +// Every OpenAI-compatible client then behaves as though the model is broken. +// +// Found the hard way. qwen2.5-coder produced exactly the right call — +// {"name": "list_files", "arguments": {"path": "."}} — as plain reply text, +// with tool_calls null, and diagnosing that took an afternoon of eliminating +// the gateway, the model size, the chat template and tool_choice in turn. The +// answer was the engine build. This runs that elimination in one command. +func newToolCheckCmd() *cobra.Command { + var ( + endpoint string + model string + timeout time.Duration + ) + + cmd := &cobra.Command{ + Use: "tool-check", + Short: "Check that an engine returns tool calls as tool_calls", + Long: "The catalog declares whether a model supports tools, which makes it a\n" + + "claim. This checks it against a running engine.\n\n" + + "A model that cannot use tools, and an engine that does not parse the\n" + + "calls it makes, are indistinguishable from outside: both answer in prose\n" + + "while tool_calls stays null. The difference matters, because one is fixed\n" + + "by choosing another model and the other by changing engine build.\n\n" + + "Point it at an engine's own address, not the gateway, so nothing in\n" + + "between is part of the answer.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + if endpoint == "" { + return fmt.Errorf("--endpoint is required, e.g. http://10.0.0.145:20002") + } + ctx, cancel := context.WithTimeout(cmd.Context(), timeout) + defer cancel() + + res, err := probeToolCall(ctx, endpoint, model) + if err != nil { + return err + } + report(cmd, res) + if !res.Structured { + // A non-zero exit so this is usable from a script that gates a + // rollout on it. + return errToolCallsUnsupported + } + return nil + }, + } + cmd.Flags().StringVar(&endpoint, "endpoint", "", "engine base URL, e.g. http://10.0.0.145:20002") + cmd.Flags().StringVar(&model, "model", "", "model name to send, when the engine needs one") + cmd.Flags().DurationVar(&timeout, "timeout", 5*time.Minute, "how long to wait; CPU inference is slow") + return cmd +} + +var errToolCallsUnsupported = fmt.Errorf("this engine does not return structured tool calls") + +// ToolProbe is what one check found. +type ToolProbe struct { + // Structured is the answer: did tool_calls come back populated. + Structured bool + // Attempted means the model tried to call the tool, in whatever form. It + // separates "the model ignored its tools" from "the engine did not parse + // the call the model made" — the same symptom, different fix. + Attempted bool + Content string + Raw string +} + +func probeToolCall(ctx context.Context, endpoint, model string) (ToolProbe, error) { + // A question with one obvious tool answer and no other sensible reply, so a + // model that can use tools has no excuse not to. + body := map[string]any{ + "messages": []map[string]string{ + {"role": "user", "content": "What is the weather in Istanbul? Use the tool."}, + }, + "tools": []map[string]any{{ + "type": "function", + "function": map[string]any{ + "name": "get_weather", + "description": "Get the current weather in a city", + "parameters": map[string]any{ + "type": "object", + "properties": map[string]any{"city": map[string]any{"type": "string"}}, + "required": []string{"city"}, + }, + }, + }}, + "tool_choice": "auto", + "max_tokens": 200, + } + if model != "" { + body["model"] = model + } + buf, err := json.Marshal(body) + if err != nil { + return ToolProbe{}, err + } + + url := endpoint + "/v1/chat/completions" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(buf)) + if err != nil { + return ToolProbe{}, err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return ToolProbe{}, fmt.Errorf("asking %s: %w", url, err) + } + defer resp.Body.Close() + + raw, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return ToolProbe{}, err + } + if resp.StatusCode != http.StatusOK { + return ToolProbe{}, fmt.Errorf("%s returned %d: %s", url, resp.StatusCode, trim(string(raw), 300)) + } + return readToolProbe(raw) +} + +func readToolProbe(raw []byte) (ToolProbe, error) { + var parsed struct { + Choices []struct { + Message struct { + Content string `json:"content"` + ToolCalls []json.RawMessage `json:"tool_calls"` + } `json:"message"` + } `json:"choices"` + } + if err := json.Unmarshal(raw, &parsed); err != nil { + return ToolProbe{}, fmt.Errorf("the engine's reply is not a chat completion: %w", err) + } + if len(parsed.Choices) == 0 { + return ToolProbe{}, fmt.Errorf("the engine returned no choices") + } + msg := parsed.Choices[0].Message + p := ToolProbe{ + Structured: len(msg.ToolCalls) > 0, + Content: msg.Content, + Raw: trim(string(raw), 600), + } + // The model tried, and the engine handed the attempt back as text. The + // tell is the tool's name in a reply that was supposed to be a call. + p.Attempted = p.Structured || bytes.Contains([]byte(msg.Content), []byte("get_weather")) + return p, nil +} + +func report(cmd *cobra.Command, p ToolProbe) { + out := cmd.OutOrStdout() + switch { + case p.Structured: + fmt.Fprintln(out, "tool calls work: the engine returned a populated tool_calls array.") + case p.Attempted: + fmt.Fprintln(out, "the model called the tool and the engine did not parse it.") + fmt.Fprintln(out, "\n The call is in the reply text instead of tool_calls:") + fmt.Fprintf(out, " %s\n", trim(p.Content, 200)) + fmt.Fprintln(out, "\n This is the engine, not the model. Choosing a bigger model will not") + fmt.Fprintln(out, " help. Check the engine's build and its chat template support.") + default: + fmt.Fprintln(out, "the model did not try to call the tool.") + fmt.Fprintf(out, "\n It replied: %s\n", trim(p.Content, 200)) + fmt.Fprintln(out, "\n This is the model, not the engine. A model trained for tool use") + fmt.Fprintln(out, " would call one here; try another before suspecting the engine.") + } +} + +func trim(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "…" +} diff --git a/internal/cli/toolcheck_test.go b/internal/cli/toolcheck_test.go new file mode 100644 index 00000000..b20f762f --- /dev/null +++ b/internal/cli/toolcheck_test.go @@ -0,0 +1,66 @@ +package cli + +import ( + "strings" + "testing" +) + +func TestAStructuredToolCallIsRecognised(t *testing.T) { + p, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", + "tool_calls":[{"id":"c1","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Istanbul\"}"}}]}}]}`)) + if err != nil { + t.Fatal(err) + } + if !p.Structured { + t.Error("a populated tool_calls array was not recognised") + } + if !p.Attempted { + t.Error("a structured call must also count as an attempt") + } +} + +func TestACallReturnedAsTextIsBlamedOnTheEngine(t *testing.T) { + // The exact reply that cost an afternoon: the model made the right call and + // the engine handed it back as prose. + p, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", + "content":"{\"name\": \"get_weather\", \"arguments\": {\"city\": \".\"}}"}}]}`)) + if err != nil { + t.Fatal(err) + } + if p.Structured { + t.Fatal("text was reported as a structured call") + } + if !p.Attempted { + t.Error("the model plainly tried; that must be distinguished from ignoring the tools") + } +} + +func TestAModelThatIgnoresItsToolsIsBlamedOnTheModel(t *testing.T) { + // The other diagnosis, and it has a different fix: try another model rather + // than another engine build. + p, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", + "content":"The weather in Istanbul is usually mild in autumn."}}]}`)) + if err != nil { + t.Fatal(err) + } + if p.Structured || p.Attempted { + t.Errorf("a plain answer was read as a tool call: %+v", p) + } +} + +func TestAReplyThatIsNotACompletionIsAnError(t *testing.T) { + for _, bad := range []string{`{}`, `not json`, `{"choices":[]}`} { + if _, err := readToolProbe([]byte(bad)); err == nil { + t.Errorf("no error for %q", bad) + } + } +} + +func TestTrimKeepsOutputReadable(t *testing.T) { + if got := trim(strings.Repeat("x", 100), 10); len(got) > 14 { + t.Errorf("trim returned %d characters", len(got)) + } + if got := trim("short", 10); got != "short" { + t.Errorf("trim altered a short string: %q", got) + } +}