llmgw/internal/cli/toolcheck_test.go

package cli

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
)

// toolCheck runs the command against an engine that answers only its own key,
// with an admin API that lists the replica there with that key.
func toolCheck(t *testing.T, args ...string) (string, error) {
	t.Helper()
	engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Header.Get("Authorization") != "Bearer engine-key" {
			http.Error(w, `{"error":{"message":"Invalid API Key"}}`, http.StatusUnauthorized)
			return
		}
		fmt.Fprint(w, `{"choices":[{"message":{"tool_calls":[{"id":"c1","type":"function",
		  "function":{"name":"get_weather","arguments":"{\"city\":\"Istanbul\"}"}}]}}]}`)
	}))
	admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		_ = json.NewEncoder(w).Encode([]adminapi.ReplicaView{
			{ID: "r1", Model: "m", Profile: "gpu", Endpoint: engine.URL, EngineKey: "engine-key"},
		})
	}))
	t.Cleanup(func() { engine.Close(); admin.Close() })
	t.Setenv("LLMGW_ADMIN_TOKEN", "t")
	cmd := newToolCheckCmd(func() *ctlClient { return newCtlClient(admin.URL) })
	var out bytes.Buffer
	cmd.SetOut(&out)
	cmd.SetErr(&out)
	cmd.SetArgs(append([]string{"--endpoint", engine.URL}, args...))
	err := cmd.ExecuteContext(context.Background())
	return out.String(), err
}

func TestToolCheckAsksWithTheEngineKeyTheAdminAPIGives(t *testing.T) {
	// Every replica's engine demands its key, and the check was sent without
	// one: it failed with a 401 on every engine in the cluster.
	out, err := toolCheck(t)
	if err != nil {
		t.Fatalf("%v\n%s", err, out)
	}
	if !strings.Contains(out, "tool calls work") {
		t.Errorf("unexpected report:\n%s", out)
	}
}

func TestToolCheckSaysWhereTheEngineKeyComesFrom(t *testing.T) {
	out, err := toolCheck(t, "--key", "wrong")
	if err == nil || !strings.Contains(err.Error(), "401") || !strings.Contains(err.Error(), "--key") {
		t.Fatalf("expected the refusal explained, got %v\n%s", err, out)
	}
}

func TestAStructuredToolCallIsRecognised(t *testing.T) {
	p, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant",
	  "tool_calls":[{"id":"c1","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Istanbul\"}"}}]}}]}`))
	if err != nil {
		t.Fatal(err)
	}
	if !p.Structured {
		t.Error("a populated tool_calls array was not recognised")
	}
	if !p.Attempted {
		t.Error("a structured call must also count as an attempt")
	}
}

func TestACallReturnedAsTextIsAnAttempt(t *testing.T) {
	// The exact reply that cost an afternoon: the model made the right call and
	// the engine handed it back as prose.
	p, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant",
	  "content":"{\"name\": \"get_weather\", \"arguments\": {\"city\": \".\"}}"}}]}`))
	if err != nil {
		t.Fatal(err)
	}
	if p.Structured {
		t.Fatal("text was reported as a structured call")
	}
	if !p.Attempted {
		t.Error("the model plainly tried; that must be distinguished from ignoring the tools")
	}
	if p.Wrapped {
		t.Error("bare JSON was read as having used the <tool_call> wrapper")
	}
}

func TestAWrappedCallBlamesTheEngineAndABareOneDoesNot(t *testing.T) {
	// Both look identical from outside — tool_calls null, the call sitting in
	// the content — and they have opposite fixes. The whole point of the
	// command is telling them apart, and the earlier version did not: it
	// blamed the engine for both, which sends someone off rebuilding llama.cpp
	// when the answer is a different model.
	wrapped, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant",
	  "content":"<tool_call>\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Istanbul\"}}\n</tool_call>"}}]}`))
	if err != nil {
		t.Fatal(err)
	}
	if !wrapped.Attempted || !wrapped.Wrapped {
		t.Errorf("a correctly wrapped call was not recognised as one: %+v", wrapped)
	}

	// What qwen2.5-coder-3b actually returns, measured: the right call in a
	// markdown fence, which the parser has no way to find.
	fenced, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant",
	  "content":"` + "```" + `json\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Istanbul\"}}\n` + "```" + `"}}]}`))
	if err != nil {
		t.Fatal(err)
	}
	if !fenced.Attempted {
		t.Error("a fenced call is still an attempt")
	}
	if fenced.Wrapped {
		t.Error("a markdown fence was mistaken for the <tool_call> wrapper")
	}

	// And what qwen2.5-coder-7b returns: a tag it invented, which is not the
	// one the template asked for and is not parsed either.
	invented, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant",
	  "content":"<function-call>\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Istanbul\"}}\n</function-call>"}}]}`))
	if err != nil {
		t.Fatal(err)
	}
	if !invented.Attempted || invented.Wrapped {
		t.Errorf("an invented tag was read as the template's wrapper: %+v", invented)
	}
}

func TestAModelThatIgnoresItsToolsIsBlamedOnTheModel(t *testing.T) {
	// The other diagnosis, and it has a different fix: try another model rather
	// than another engine build.
	p, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant",
	  "content":"The weather in Istanbul is usually mild in autumn."}}]}`))
	if err != nil {
		t.Fatal(err)
	}
	if p.Structured || p.Attempted {
		t.Errorf("a plain answer was read as a tool call: %+v", p)
	}
}

func TestAReplyThatIsNotACompletionIsAnError(t *testing.T) {
	for _, bad := range []string{`{}`, `not json`, `{"choices":[]}`} {
		if _, err := readToolProbe([]byte(bad)); err == nil {
			t.Errorf("no error for %q", bad)
		}
	}
}

func TestTrimKeepsOutputReadable(t *testing.T) {
	if got := trim(strings.Repeat("x", 100), 10); len(got) > 14 {
		t.Errorf("trim returned %d characters", len(got))
	}
	if got := trim("short", 10); got != "short" {
		t.Errorf("trim altered a short string: %q", got)
	}
}