llmgw/internal/cli/toolcheck.go

package cli

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"time"

	"github.com/spf13/cobra"
)

// newToolCheckCmd asks a running engine whether tool calls come back as tool
// calls.
//
// A model's capabilities are declared in the catalog, which makes them claims.
// This is the one that can be checked, and the one worth checking: the gateway
// forwards a tools array to any model, and an engine that does not return
// structured tool_calls looks exactly like a model that ignores its tools.
// Every OpenAI-compatible client then behaves as though the model is broken.
//
// Found the hard way. qwen2.5-coder produced exactly the right call —
// {"name": "list_files", "arguments": {"path": "."}} — as plain reply text,
// with tool_calls null, and diagnosing that took an afternoon of eliminating
// the gateway, the model size, the chat template and tool_choice in turn.
//
// The answer was neither the engine nor the model alone, which is why this
// command now distinguishes them by what the reply looks like rather than
// asserting one. The engine rendered the tool prompt correctly, including the
// <tool_call> wrapper its template asks the model to use; the model wrapped
// the call in a markdown fence instead, and the parser — which looks for that
// wrapper — left it in the content. A differently trained model fixes that. A
// bigger one from the same family does not: qwen2.5-coder-7b invented
// <function-call> and failed the same way.
func newToolCheckCmd(client func() *ctlClient) *cobra.Command {
	var (
		endpoint string
		model    string
		key      string
		timeout  time.Duration
	)

	cmd := &cobra.Command{
		Use:   "tool-check",
		Short: "Check that an engine returns tool calls as tool_calls",
		Long: "The catalog declares whether a model supports tools, which makes it a\n" +
			"claim. This checks it against a running engine.\n\n" +
			"A model that cannot use tools, and an engine that does not parse the\n" +
			"calls it makes, are indistinguishable from outside: both answer in prose\n" +
			"while tool_calls stays null. The difference matters, because one is fixed\n" +
			"by choosing another model and the other by changing engine build.\n\n" +
			"Point it at an engine's own address, not the gateway, so nothing in\n" +
			"between is part of the answer.",
		Args: cobra.NoArgs,
		RunE: func(cmd *cobra.Command, _ []string) error {
			if endpoint == "" {
				return fmt.Errorf("--endpoint is required, e.g. http://10.0.0.145:20002")
			}
			// A replica's engine demands its own key, and refused the check
			// with a 401 when none was sent. Asked of the admin API, as bench
			// does; an endpoint that is no replica's is asked without one.
			if key == "" {
				reps, err := listReplicas(cmd, client)
				if err != nil {
					cmd.PrintErrf("The admin API could not be asked for the engine key of the replica at "+
						"--endpoint (%v); an engine that demands one will refuse. Pass --key to give it.\n", err)
				}
				key = replicaKey(reps, endpoint)
			}
			ctx, cancel := context.WithTimeout(cmd.Context(), timeout)
			defer cancel()

			res, err := probeToolCall(ctx, endpoint, model, key)
			if err != nil {
				return err
			}
			report(cmd, res)
			if !res.Structured {
				// A non-zero exit so this is usable from a script that gates a
				// rollout on it.
				return errToolCallsUnsupported
			}
			return nil
		},
	}
	cmd.Flags().StringVar(&endpoint, "endpoint", "", "engine base URL, e.g. http://10.0.0.145:20002")
	cmd.Flags().StringVar(&model, "model", "", "model name to send, when the engine needs one")
	cmd.Flags().StringVar(&key, "key", "", "the engine's key (default: the admin API's, for the replica at --endpoint)")
	cmd.Flags().DurationVar(&timeout, "timeout", 5*time.Minute, "how long to wait; CPU inference is slow")
	return cmd
}

var errToolCallsUnsupported = fmt.Errorf("this engine does not return structured tool calls")

// ToolProbe is what one check found.
type ToolProbe struct {
	// Structured is the answer: did tool_calls come back populated.
	Structured bool
	// Attempted means the model tried to call the tool, in whatever form. It
	// separates "the model ignored its tools" from "the engine did not parse
	// the call the model made" — the same symptom, different fix.
	Attempted bool
	// Wrapped means the attempt used the <tool_call> tags that the Qwen and
	// Hermes templates instruct the model to use.
	//
	// This is what separates the two halves of a failed attempt. Wrapped and
	// unparsed is the engine's parser failing on output that was correct.
	// Unwrapped is the model answering in a shape of its own — the engine had
	// no way to recognise it, and the fix is a model trained for the format,
	// not a newer engine.
	Wrapped bool
	Content string
	Raw     string
}

func probeToolCall(ctx context.Context, endpoint, model, key string) (ToolProbe, error) {
	// A question with one obvious tool answer and no other sensible reply, so a
	// model that can use tools has no excuse not to.
	body := map[string]any{
		"messages": []map[string]string{
			{"role": "user", "content": "What is the weather in Istanbul? Use the tool."},
		},
		"tools": []map[string]any{{
			"type": "function",
			"function": map[string]any{
				"name":        "get_weather",
				"description": "Get the current weather in a city",
				"parameters": map[string]any{
					"type":       "object",
					"properties": map[string]any{"city": map[string]any{"type": "string"}},
					"required":   []string{"city"},
				},
			},
		}},
		"tool_choice": "auto",
		"max_tokens":  200,
	}
	if model != "" {
		body["model"] = model
	}
	buf, err := json.Marshal(body)
	if err != nil {
		return ToolProbe{}, err
	}

	url := endpoint + "/v1/chat/completions"
	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(buf))
	if err != nil {
		return ToolProbe{}, err
	}
	req.Header.Set("Content-Type", "application/json")
	if key != "" {
		req.Header.Set("Authorization", "Bearer "+key)
	}

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return ToolProbe{}, fmt.Errorf("asking %s: %w", url, err)
	}
	defer resp.Body.Close()

	raw, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
	if err != nil {
		return ToolProbe{}, err
	}
	if resp.StatusCode != http.StatusOK {
		if resp.StatusCode == http.StatusUnauthorized {
			return ToolProbe{}, fmt.Errorf("%s answered 401: the engine's key is the admin API's, "+
				"for the replica `llmgw ctl replicas` lists at --endpoint; or pass it with --key", url)
		}
		return ToolProbe{}, fmt.Errorf("%s returned %d: %s", url, resp.StatusCode, trim(string(raw), 300))
	}
	return readToolProbe(raw)
}

func readToolProbe(raw []byte) (ToolProbe, error) {
	var parsed struct {
		Choices []struct {
			Message struct {
				Content   string            `json:"content"`
				ToolCalls []json.RawMessage `json:"tool_calls"`
			} `json:"message"`
		} `json:"choices"`
	}
	if err := json.Unmarshal(raw, &parsed); err != nil {
		return ToolProbe{}, fmt.Errorf("the engine's reply is not a chat completion: %w", err)
	}
	if len(parsed.Choices) == 0 {
		return ToolProbe{}, fmt.Errorf("the engine returned no choices")
	}
	msg := parsed.Choices[0].Message
	p := ToolProbe{
		Structured: len(msg.ToolCalls) > 0,
		Content:    msg.Content,
		Raw:        trim(string(raw), 600),
	}
	// The model tried, and the engine handed the attempt back as text. The
	// tell is the tool's name in a reply that was supposed to be a call.
	p.Attempted = p.Structured || bytes.Contains([]byte(msg.Content), []byte("get_weather"))
	p.Wrapped = bytes.Contains([]byte(msg.Content), []byte("<tool_call>"))
	return p, nil
}

func report(cmd *cobra.Command, p ToolProbe) {
	out := cmd.OutOrStdout()
	switch {
	case p.Structured:
		fmt.Fprintln(out, "tool calls work: the engine returned a populated tool_calls array.")
	case p.Attempted && p.Wrapped:
		fmt.Fprintln(out, "the model called the tool correctly and the engine did not parse it.")
		fmt.Fprintln(out, "\n  The call is in the reply text instead of tool_calls:")
		fmt.Fprintf(out, "    %s\n", trim(p.Content, 200))
		fmt.Fprintln(out, "\n  It used the <tool_call> wrapper its template asks for, so the model")
		fmt.Fprintln(out, "  did its part. This is the engine's parser. Check its build; another")
		fmt.Fprintln(out, "  model will fail the same way.")
	case p.Attempted:
		fmt.Fprintln(out, "the model called the tool in a shape the engine does not recognise.")
		fmt.Fprintln(out, "\n  The call is in the reply text instead of tool_calls:")
		fmt.Fprintf(out, "    %s\n", trim(p.Content, 200))
		fmt.Fprintln(out, "\n  The engine rendered the tools correctly — the model knew the tool")
		fmt.Fprintln(out, "  existed and what to pass it — but did not use the <tool_call> wrapper")
		fmt.Fprintln(out, "  its template asked for, so there was nothing for the parser to find.")
		fmt.Fprintln(out, "\n  Fixed by a model trained for this format, not by a bigger one: the")
		fmt.Fprintln(out, "  instruct models generally manage it where the coder variants do not.")
	default:
		fmt.Fprintln(out, "the model did not try to call the tool.")
		fmt.Fprintf(out, "\n  It replied: %s\n", trim(p.Content, 200))
		fmt.Fprintln(out, "\n  This is the model, not the engine. A model trained for tool use")
		fmt.Fprintln(out, "  would call one here; try another before suspecting the engine.")
	}
}

func trim(s string, n int) string {
	if len(s) <= n {
		return s
	}
	return s[:n] + "…"
}