package cli import ( "bytes" "context" "encoding/json" "fmt" "net/http" "net/http/httptest" "strings" "testing" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" ) // toolCheck runs the command against an engine that answers only its own key, // with an admin API that lists the replica there with that key. func toolCheck(t *testing.T, args ...string) (string, error) { t.Helper() engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Authorization") != "Bearer engine-key" { http.Error(w, `{"error":{"message":"Invalid API Key"}}`, http.StatusUnauthorized) return } fmt.Fprint(w, `{"choices":[{"message":{"tool_calls":[{"id":"c1","type":"function", "function":{"name":"get_weather","arguments":"{\"city\":\"Istanbul\"}"}}]}}]}`) })) admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _ = json.NewEncoder(w).Encode([]adminapi.ReplicaView{ {ID: "r1", Model: "m", Profile: "gpu", Endpoint: engine.URL, EngineKey: "engine-key"}, }) })) t.Cleanup(func() { engine.Close(); admin.Close() }) t.Setenv("LLMGW_ADMIN_TOKEN", "t") cmd := newToolCheckCmd(func() *ctlClient { return newCtlClient(admin.URL) }) var out bytes.Buffer cmd.SetOut(&out) cmd.SetErr(&out) cmd.SetArgs(append([]string{"--endpoint", engine.URL}, args...)) err := cmd.ExecuteContext(context.Background()) return out.String(), err } func TestToolCheckAsksWithTheEngineKeyTheAdminAPIGives(t *testing.T) { // Every replica's engine demands its key, and the check was sent without // one: it failed with a 401 on every engine in the cluster. out, err := toolCheck(t) if err != nil { t.Fatalf("%v\n%s", err, out) } if !strings.Contains(out, "tool calls work") { t.Errorf("unexpected report:\n%s", out) } } func TestToolCheckSaysWhereTheEngineKeyComesFrom(t *testing.T) { out, err := toolCheck(t, "--key", "wrong") if err == nil || !strings.Contains(err.Error(), "401") || !strings.Contains(err.Error(), "--key") { t.Fatalf("expected the refusal explained, got %v\n%s", err, out) } } func TestAStructuredToolCallIsRecognised(t *testing.T) { p, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", "tool_calls":[{"id":"c1","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Istanbul\"}"}}]}}]}`)) if err != nil { t.Fatal(err) } if !p.Structured { t.Error("a populated tool_calls array was not recognised") } if !p.Attempted { t.Error("a structured call must also count as an attempt") } } func TestACallReturnedAsTextIsAnAttempt(t *testing.T) { // The exact reply that cost an afternoon: the model made the right call and // the engine handed it back as prose. p, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", "content":"{\"name\": \"get_weather\", \"arguments\": {\"city\": \".\"}}"}}]}`)) if err != nil { t.Fatal(err) } if p.Structured { t.Fatal("text was reported as a structured call") } if !p.Attempted { t.Error("the model plainly tried; that must be distinguished from ignoring the tools") } if p.Wrapped { t.Error("bare JSON was read as having used the wrapper") } } func TestAWrappedCallBlamesTheEngineAndABareOneDoesNot(t *testing.T) { // Both look identical from outside — tool_calls null, the call sitting in // the content — and they have opposite fixes. The whole point of the // command is telling them apart, and the earlier version did not: it // blamed the engine for both, which sends someone off rebuilding llama.cpp // when the answer is a different model. wrapped, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", "content":"\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Istanbul\"}}\n"}}]}`)) if err != nil { t.Fatal(err) } if !wrapped.Attempted || !wrapped.Wrapped { t.Errorf("a correctly wrapped call was not recognised as one: %+v", wrapped) } // What qwen2.5-coder-3b actually returns, measured: the right call in a // markdown fence, which the parser has no way to find. fenced, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", "content":"` + "```" + `json\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Istanbul\"}}\n` + "```" + `"}}]}`)) if err != nil { t.Fatal(err) } if !fenced.Attempted { t.Error("a fenced call is still an attempt") } if fenced.Wrapped { t.Error("a markdown fence was mistaken for the wrapper") } // And what qwen2.5-coder-7b returns: a tag it invented, which is not the // one the template asked for and is not parsed either. invented, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", "content":"\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Istanbul\"}}\n"}}]}`)) if err != nil { t.Fatal(err) } if !invented.Attempted || invented.Wrapped { t.Errorf("an invented tag was read as the template's wrapper: %+v", invented) } } func TestAModelThatIgnoresItsToolsIsBlamedOnTheModel(t *testing.T) { // The other diagnosis, and it has a different fix: try another model rather // than another engine build. p, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", "content":"The weather in Istanbul is usually mild in autumn."}}]}`)) if err != nil { t.Fatal(err) } if p.Structured || p.Attempted { t.Errorf("a plain answer was read as a tool call: %+v", p) } } func TestAReplyThatIsNotACompletionIsAnError(t *testing.T) { for _, bad := range []string{`{}`, `not json`, `{"choices":[]}`} { if _, err := readToolProbe([]byte(bad)); err == nil { t.Errorf("no error for %q", bad) } } } func TestTrimKeepsOutputReadable(t *testing.T) { if got := trim(strings.Repeat("x", 100), 10); len(got) > 14 { t.Errorf("trim returned %d characters", len(got)) } if got := trim("short", 10); got != "short" { t.Errorf("trim altered a short string: %q", got) } }