llmgw/internal/cli/toolcheck_test.go

package cli

import (
	"strings"
	"testing"
)

func TestAStructuredToolCallIsRecognised(t *testing.T) {
	p, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant",
	  "tool_calls":[{"id":"c1","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Istanbul\"}"}}]}}]}`))
	if err != nil {
		t.Fatal(err)
	}
	if !p.Structured {
		t.Error("a populated tool_calls array was not recognised")
	}
	if !p.Attempted {
		t.Error("a structured call must also count as an attempt")
	}
}

func TestACallReturnedAsTextIsAnAttempt(t *testing.T) {
	// The exact reply that cost an afternoon: the model made the right call and
	// the engine handed it back as prose.
	p, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant",
	  "content":"{\"name\": \"get_weather\", \"arguments\": {\"city\": \".\"}}"}}]}`))
	if err != nil {
		t.Fatal(err)
	}
	if p.Structured {
		t.Fatal("text was reported as a structured call")
	}
	if !p.Attempted {
		t.Error("the model plainly tried; that must be distinguished from ignoring the tools")
	}
	if p.Wrapped {
		t.Error("bare JSON was read as having used the <tool_call> wrapper")
	}
}

func TestAWrappedCallBlamesTheEngineAndABareOneDoesNot(t *testing.T) {
	// Both look identical from outside — tool_calls null, the call sitting in
	// the content — and they have opposite fixes. The whole point of the
	// command is telling them apart, and the earlier version did not: it
	// blamed the engine for both, which sends someone off rebuilding llama.cpp
	// when the answer is a different model.
	wrapped, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant",
	  "content":"<tool_call>\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Istanbul\"}}\n</tool_call>"}}]}`))
	if err != nil {
		t.Fatal(err)
	}
	if !wrapped.Attempted || !wrapped.Wrapped {
		t.Errorf("a correctly wrapped call was not recognised as one: %+v", wrapped)
	}

	// What qwen2.5-coder-3b actually returns, measured: the right call in a
	// markdown fence, which the parser has no way to find.
	fenced, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant",
	  "content":"` + "```" + `json\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Istanbul\"}}\n` + "```" + `"}}]}`))
	if err != nil {
		t.Fatal(err)
	}
	if !fenced.Attempted {
		t.Error("a fenced call is still an attempt")
	}
	if fenced.Wrapped {
		t.Error("a markdown fence was mistaken for the <tool_call> wrapper")
	}

	// And what qwen2.5-coder-7b returns: a tag it invented, which is not the
	// one the template asked for and is not parsed either.
	invented, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant",
	  "content":"<function-call>\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Istanbul\"}}\n</function-call>"}}]}`))
	if err != nil {
		t.Fatal(err)
	}
	if !invented.Attempted || invented.Wrapped {
		t.Errorf("an invented tag was read as the template's wrapper: %+v", invented)
	}
}

func TestAModelThatIgnoresItsToolsIsBlamedOnTheModel(t *testing.T) {
	// The other diagnosis, and it has a different fix: try another model rather
	// than another engine build.
	p, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant",
	  "content":"The weather in Istanbul is usually mild in autumn."}}]}`))
	if err != nil {
		t.Fatal(err)
	}
	if p.Structured || p.Attempted {
		t.Errorf("a plain answer was read as a tool call: %+v", p)
	}
}

func TestAReplyThatIsNotACompletionIsAnError(t *testing.T) {
	for _, bad := range []string{`{}`, `not json`, `{"choices":[]}`} {
		if _, err := readToolProbe([]byte(bad)); err == nil {
			t.Errorf("no error for %q", bad)
		}
	}
}

func TestTrimKeepsOutputReadable(t *testing.T) {
	if got := trim(strings.Repeat("x", 100), 10); len(got) > 14 {
		t.Errorf("trim returned %d characters", len(got))
	}
	if got := trim("short", 10); got != "short" {
		t.Errorf("trim altered a short string: %q", got)
	}
}