From 84896c0ad26155708ade002f9aa62f853f02a636 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 16 Sep 2026 23:30:02 +0200 Subject: [PATCH] Add ctl tool-check, so an afternoon becomes one command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The catalog now declares whether a model supports tools, which makes it a claim. This is the claim that can be checked, and the one worth checking: the gateway forwards a tools array to any model, and an engine that does not return structured tool_calls looks exactly like a model that ignores its tools. Every OpenAI-compatible client then behaves as though the model is broken. It separates the two diagnoses, which have the same symptom and different fixes. The model ignoring its tools is fixed by choosing another model. The model making the call and the engine handing it back as prose is fixed by changing engine build, and no larger model will help — which is what today's investigation established only after eliminating the gateway, the model size, two chat templates and tool_choice in turn. Against the live engine it says: the model called the tool and the engine did not parse it. {"name": "get_weather", "arguments": {"city": "Istanbul"}} This is the engine, not the model. engine-check asks whether an engine still accepts our flags; this asks whether it behaves. Flags were the failure we had seen, so they were the failure we checked for. Non-zero exit, so a rollout can be gated on it. Co-Authored-By: Claude Opus 5 (1M context) --- internal/cli/ctl.go | 1 + internal/cli/toolcheck.go | 192 +++++++++++++++++++++++++++++++++ internal/cli/toolcheck_test.go | 66 ++++++++++++ 3 files changed, 259 insertions(+) create mode 100644 internal/cli/toolcheck.go create mode 100644 internal/cli/toolcheck_test.go diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index 9d3d136..e3f4a68 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -110,6 +110,7 @@ func newCtlCmd() *cobra.Command { // apply to it; it lives here because this is where operators look. newBenchCmd(&asJSON), newEngineCheckCmd(), + newToolCheckCmd(), newCertCmd(), ) return cmd diff --git a/internal/cli/toolcheck.go b/internal/cli/toolcheck.go new file mode 100644 index 0000000..13f7269 --- /dev/null +++ b/internal/cli/toolcheck.go @@ -0,0 +1,192 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/spf13/cobra" +) + +// newToolCheckCmd asks a running engine whether tool calls come back as tool +// calls. +// +// A model's capabilities are declared in the catalog, which makes them claims. +// This is the one that can be checked, and the one worth checking: the gateway +// forwards a tools array to any model, and an engine that does not return +// structured tool_calls looks exactly like a model that ignores its tools. +// Every OpenAI-compatible client then behaves as though the model is broken. +// +// Found the hard way. qwen2.5-coder produced exactly the right call — +// {"name": "list_files", "arguments": {"path": "."}} — as plain reply text, +// with tool_calls null, and diagnosing that took an afternoon of eliminating +// the gateway, the model size, the chat template and tool_choice in turn. The +// answer was the engine build. This runs that elimination in one command. +func newToolCheckCmd() *cobra.Command { + var ( + endpoint string + model string + timeout time.Duration + ) + + cmd := &cobra.Command{ + Use: "tool-check", + Short: "Check that an engine returns tool calls as tool_calls", + Long: "The catalog declares whether a model supports tools, which makes it a\n" + + "claim. This checks it against a running engine.\n\n" + + "A model that cannot use tools, and an engine that does not parse the\n" + + "calls it makes, are indistinguishable from outside: both answer in prose\n" + + "while tool_calls stays null. The difference matters, because one is fixed\n" + + "by choosing another model and the other by changing engine build.\n\n" + + "Point it at an engine's own address, not the gateway, so nothing in\n" + + "between is part of the answer.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + if endpoint == "" { + return fmt.Errorf("--endpoint is required, e.g. http://10.0.0.145:20002") + } + ctx, cancel := context.WithTimeout(cmd.Context(), timeout) + defer cancel() + + res, err := probeToolCall(ctx, endpoint, model) + if err != nil { + return err + } + report(cmd, res) + if !res.Structured { + // A non-zero exit so this is usable from a script that gates a + // rollout on it. + return errToolCallsUnsupported + } + return nil + }, + } + cmd.Flags().StringVar(&endpoint, "endpoint", "", "engine base URL, e.g. http://10.0.0.145:20002") + cmd.Flags().StringVar(&model, "model", "", "model name to send, when the engine needs one") + cmd.Flags().DurationVar(&timeout, "timeout", 5*time.Minute, "how long to wait; CPU inference is slow") + return cmd +} + +var errToolCallsUnsupported = fmt.Errorf("this engine does not return structured tool calls") + +// ToolProbe is what one check found. +type ToolProbe struct { + // Structured is the answer: did tool_calls come back populated. + Structured bool + // Attempted means the model tried to call the tool, in whatever form. It + // separates "the model ignored its tools" from "the engine did not parse + // the call the model made" — the same symptom, different fix. + Attempted bool + Content string + Raw string +} + +func probeToolCall(ctx context.Context, endpoint, model string) (ToolProbe, error) { + // A question with one obvious tool answer and no other sensible reply, so a + // model that can use tools has no excuse not to. + body := map[string]any{ + "messages": []map[string]string{ + {"role": "user", "content": "What is the weather in Istanbul? Use the tool."}, + }, + "tools": []map[string]any{{ + "type": "function", + "function": map[string]any{ + "name": "get_weather", + "description": "Get the current weather in a city", + "parameters": map[string]any{ + "type": "object", + "properties": map[string]any{"city": map[string]any{"type": "string"}}, + "required": []string{"city"}, + }, + }, + }}, + "tool_choice": "auto", + "max_tokens": 200, + } + if model != "" { + body["model"] = model + } + buf, err := json.Marshal(body) + if err != nil { + return ToolProbe{}, err + } + + url := endpoint + "/v1/chat/completions" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(buf)) + if err != nil { + return ToolProbe{}, err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return ToolProbe{}, fmt.Errorf("asking %s: %w", url, err) + } + defer resp.Body.Close() + + raw, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return ToolProbe{}, err + } + if resp.StatusCode != http.StatusOK { + return ToolProbe{}, fmt.Errorf("%s returned %d: %s", url, resp.StatusCode, trim(string(raw), 300)) + } + return readToolProbe(raw) +} + +func readToolProbe(raw []byte) (ToolProbe, error) { + var parsed struct { + Choices []struct { + Message struct { + Content string `json:"content"` + ToolCalls []json.RawMessage `json:"tool_calls"` + } `json:"message"` + } `json:"choices"` + } + if err := json.Unmarshal(raw, &parsed); err != nil { + return ToolProbe{}, fmt.Errorf("the engine's reply is not a chat completion: %w", err) + } + if len(parsed.Choices) == 0 { + return ToolProbe{}, fmt.Errorf("the engine returned no choices") + } + msg := parsed.Choices[0].Message + p := ToolProbe{ + Structured: len(msg.ToolCalls) > 0, + Content: msg.Content, + Raw: trim(string(raw), 600), + } + // The model tried, and the engine handed the attempt back as text. The + // tell is the tool's name in a reply that was supposed to be a call. + p.Attempted = p.Structured || bytes.Contains([]byte(msg.Content), []byte("get_weather")) + return p, nil +} + +func report(cmd *cobra.Command, p ToolProbe) { + out := cmd.OutOrStdout() + switch { + case p.Structured: + fmt.Fprintln(out, "tool calls work: the engine returned a populated tool_calls array.") + case p.Attempted: + fmt.Fprintln(out, "the model called the tool and the engine did not parse it.") + fmt.Fprintln(out, "\n The call is in the reply text instead of tool_calls:") + fmt.Fprintf(out, " %s\n", trim(p.Content, 200)) + fmt.Fprintln(out, "\n This is the engine, not the model. Choosing a bigger model will not") + fmt.Fprintln(out, " help. Check the engine's build and its chat template support.") + default: + fmt.Fprintln(out, "the model did not try to call the tool.") + fmt.Fprintf(out, "\n It replied: %s\n", trim(p.Content, 200)) + fmt.Fprintln(out, "\n This is the model, not the engine. A model trained for tool use") + fmt.Fprintln(out, " would call one here; try another before suspecting the engine.") + } +} + +func trim(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "…" +} diff --git a/internal/cli/toolcheck_test.go b/internal/cli/toolcheck_test.go new file mode 100644 index 0000000..b20f762 --- /dev/null +++ b/internal/cli/toolcheck_test.go @@ -0,0 +1,66 @@ +package cli + +import ( + "strings" + "testing" +) + +func TestAStructuredToolCallIsRecognised(t *testing.T) { + p, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", + "tool_calls":[{"id":"c1","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Istanbul\"}"}}]}}]}`)) + if err != nil { + t.Fatal(err) + } + if !p.Structured { + t.Error("a populated tool_calls array was not recognised") + } + if !p.Attempted { + t.Error("a structured call must also count as an attempt") + } +} + +func TestACallReturnedAsTextIsBlamedOnTheEngine(t *testing.T) { + // The exact reply that cost an afternoon: the model made the right call and + // the engine handed it back as prose. + p, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", + "content":"{\"name\": \"get_weather\", \"arguments\": {\"city\": \".\"}}"}}]}`)) + if err != nil { + t.Fatal(err) + } + if p.Structured { + t.Fatal("text was reported as a structured call") + } + if !p.Attempted { + t.Error("the model plainly tried; that must be distinguished from ignoring the tools") + } +} + +func TestAModelThatIgnoresItsToolsIsBlamedOnTheModel(t *testing.T) { + // The other diagnosis, and it has a different fix: try another model rather + // than another engine build. + p, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", + "content":"The weather in Istanbul is usually mild in autumn."}}]}`)) + if err != nil { + t.Fatal(err) + } + if p.Structured || p.Attempted { + t.Errorf("a plain answer was read as a tool call: %+v", p) + } +} + +func TestAReplyThatIsNotACompletionIsAnError(t *testing.T) { + for _, bad := range []string{`{}`, `not json`, `{"choices":[]}`} { + if _, err := readToolProbe([]byte(bad)); err == nil { + t.Errorf("no error for %q", bad) + } + } +} + +func TestTrimKeepsOutputReadable(t *testing.T) { + if got := trim(strings.Repeat("x", 100), 10); len(got) > 14 { + t.Errorf("trim returned %d characters", len(got)) + } + if got := trim("short", 10); got != "short" { + t.Errorf("trim altered a short string: %q", got) + } +}