package cli import ( "bytes" "context" "encoding/json" "fmt" "io" "net/http" "time" "github.com/spf13/cobra" ) // newToolCheckCmd asks a running engine whether tool calls come back as tool // calls. // // A model's capabilities are declared in the catalog, which makes them claims. // This is the one that can be checked, and the one worth checking: the gateway // forwards a tools array to any model, and an engine that does not return // structured tool_calls looks exactly like a model that ignores its tools. // Every OpenAI-compatible client then behaves as though the model is broken. // // Found the hard way. qwen2.5-coder produced exactly the right call — // {"name": "list_files", "arguments": {"path": "."}} — as plain reply text, // with tool_calls null, and diagnosing that took an afternoon of eliminating // the gateway, the model size, the chat template and tool_choice in turn. The // answer was the engine build. This runs that elimination in one command. func newToolCheckCmd() *cobra.Command { var ( endpoint string model string timeout time.Duration ) cmd := &cobra.Command{ Use: "tool-check", Short: "Check that an engine returns tool calls as tool_calls", Long: "The catalog declares whether a model supports tools, which makes it a\n" + "claim. This checks it against a running engine.\n\n" + "A model that cannot use tools, and an engine that does not parse the\n" + "calls it makes, are indistinguishable from outside: both answer in prose\n" + "while tool_calls stays null. The difference matters, because one is fixed\n" + "by choosing another model and the other by changing engine build.\n\n" + "Point it at an engine's own address, not the gateway, so nothing in\n" + "between is part of the answer.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { if endpoint == "" { return fmt.Errorf("--endpoint is required, e.g. http://10.0.0.145:20002") } ctx, cancel := context.WithTimeout(cmd.Context(), timeout) defer cancel() res, err := probeToolCall(ctx, endpoint, model) if err != nil { return err } report(cmd, res) if !res.Structured { // A non-zero exit so this is usable from a script that gates a // rollout on it. return errToolCallsUnsupported } return nil }, } cmd.Flags().StringVar(&endpoint, "endpoint", "", "engine base URL, e.g. http://10.0.0.145:20002") cmd.Flags().StringVar(&model, "model", "", "model name to send, when the engine needs one") cmd.Flags().DurationVar(&timeout, "timeout", 5*time.Minute, "how long to wait; CPU inference is slow") return cmd } var errToolCallsUnsupported = fmt.Errorf("this engine does not return structured tool calls") // ToolProbe is what one check found. type ToolProbe struct { // Structured is the answer: did tool_calls come back populated. Structured bool // Attempted means the model tried to call the tool, in whatever form. It // separates "the model ignored its tools" from "the engine did not parse // the call the model made" — the same symptom, different fix. Attempted bool Content string Raw string } func probeToolCall(ctx context.Context, endpoint, model string) (ToolProbe, error) { // A question with one obvious tool answer and no other sensible reply, so a // model that can use tools has no excuse not to. body := map[string]any{ "messages": []map[string]string{ {"role": "user", "content": "What is the weather in Istanbul? Use the tool."}, }, "tools": []map[string]any{{ "type": "function", "function": map[string]any{ "name": "get_weather", "description": "Get the current weather in a city", "parameters": map[string]any{ "type": "object", "properties": map[string]any{"city": map[string]any{"type": "string"}}, "required": []string{"city"}, }, }, }}, "tool_choice": "auto", "max_tokens": 200, } if model != "" { body["model"] = model } buf, err := json.Marshal(body) if err != nil { return ToolProbe{}, err } url := endpoint + "/v1/chat/completions" req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(buf)) if err != nil { return ToolProbe{}, err } req.Header.Set("Content-Type", "application/json") resp, err := http.DefaultClient.Do(req) if err != nil { return ToolProbe{}, fmt.Errorf("asking %s: %w", url, err) } defer resp.Body.Close() raw, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) if err != nil { return ToolProbe{}, err } if resp.StatusCode != http.StatusOK { return ToolProbe{}, fmt.Errorf("%s returned %d: %s", url, resp.StatusCode, trim(string(raw), 300)) } return readToolProbe(raw) } func readToolProbe(raw []byte) (ToolProbe, error) { var parsed struct { Choices []struct { Message struct { Content string `json:"content"` ToolCalls []json.RawMessage `json:"tool_calls"` } `json:"message"` } `json:"choices"` } if err := json.Unmarshal(raw, &parsed); err != nil { return ToolProbe{}, fmt.Errorf("the engine's reply is not a chat completion: %w", err) } if len(parsed.Choices) == 0 { return ToolProbe{}, fmt.Errorf("the engine returned no choices") } msg := parsed.Choices[0].Message p := ToolProbe{ Structured: len(msg.ToolCalls) > 0, Content: msg.Content, Raw: trim(string(raw), 600), } // The model tried, and the engine handed the attempt back as text. The // tell is the tool's name in a reply that was supposed to be a call. p.Attempted = p.Structured || bytes.Contains([]byte(msg.Content), []byte("get_weather")) return p, nil } func report(cmd *cobra.Command, p ToolProbe) { out := cmd.OutOrStdout() switch { case p.Structured: fmt.Fprintln(out, "tool calls work: the engine returned a populated tool_calls array.") case p.Attempted: fmt.Fprintln(out, "the model called the tool and the engine did not parse it.") fmt.Fprintln(out, "\n The call is in the reply text instead of tool_calls:") fmt.Fprintf(out, " %s\n", trim(p.Content, 200)) fmt.Fprintln(out, "\n This is the engine, not the model. Choosing a bigger model will not") fmt.Fprintln(out, " help. Check the engine's build and its chat template support.") default: fmt.Fprintln(out, "the model did not try to call the tool.") fmt.Fprintf(out, "\n It replied: %s\n", trim(p.Content, 200)) fmt.Fprintln(out, "\n This is the model, not the engine. A model trained for tool use") fmt.Fprintln(out, " would call one here; try another before suspecting the engine.") } } func trim(s string, n int) string { if len(s) <= n { return s } return s[:n] + "…" }