package cli import ( "strings" "testing" ) func TestAStructuredToolCallIsRecognised(t *testing.T) { p, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", "tool_calls":[{"id":"c1","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Istanbul\"}"}}]}}]}`)) if err != nil { t.Fatal(err) } if !p.Structured { t.Error("a populated tool_calls array was not recognised") } if !p.Attempted { t.Error("a structured call must also count as an attempt") } } func TestACallReturnedAsTextIsAnAttempt(t *testing.T) { // The exact reply that cost an afternoon: the model made the right call and // the engine handed it back as prose. p, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", "content":"{\"name\": \"get_weather\", \"arguments\": {\"city\": \".\"}}"}}]}`)) if err != nil { t.Fatal(err) } if p.Structured { t.Fatal("text was reported as a structured call") } if !p.Attempted { t.Error("the model plainly tried; that must be distinguished from ignoring the tools") } if p.Wrapped { t.Error("bare JSON was read as having used the wrapper") } } func TestAWrappedCallBlamesTheEngineAndABareOneDoesNot(t *testing.T) { // Both look identical from outside — tool_calls null, the call sitting in // the content — and they have opposite fixes. The whole point of the // command is telling them apart, and the earlier version did not: it // blamed the engine for both, which sends someone off rebuilding llama.cpp // when the answer is a different model. wrapped, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", "content":"\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Istanbul\"}}\n"}}]}`)) if err != nil { t.Fatal(err) } if !wrapped.Attempted || !wrapped.Wrapped { t.Errorf("a correctly wrapped call was not recognised as one: %+v", wrapped) } // What qwen2.5-coder-3b actually returns, measured: the right call in a // markdown fence, which the parser has no way to find. fenced, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", "content":"` + "```" + `json\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Istanbul\"}}\n` + "```" + `"}}]}`)) if err != nil { t.Fatal(err) } if !fenced.Attempted { t.Error("a fenced call is still an attempt") } if fenced.Wrapped { t.Error("a markdown fence was mistaken for the wrapper") } // And what qwen2.5-coder-7b returns: a tag it invented, which is not the // one the template asked for and is not parsed either. invented, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", "content":"\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Istanbul\"}}\n"}}]}`)) if err != nil { t.Fatal(err) } if !invented.Attempted || invented.Wrapped { t.Errorf("an invented tag was read as the template's wrapper: %+v", invented) } } func TestAModelThatIgnoresItsToolsIsBlamedOnTheModel(t *testing.T) { // The other diagnosis, and it has a different fix: try another model rather // than another engine build. p, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant", "content":"The weather in Istanbul is usually mild in autumn."}}]}`)) if err != nil { t.Fatal(err) } if p.Structured || p.Attempted { t.Errorf("a plain answer was read as a tool call: %+v", p) } } func TestAReplyThatIsNotACompletionIsAnError(t *testing.T) { for _, bad := range []string{`{}`, `not json`, `{"choices":[]}`} { if _, err := readToolProbe([]byte(bad)); err == nil { t.Errorf("no error for %q", bad) } } } func TestTrimKeepsOutputReadable(t *testing.T) { if got := trim(strings.Repeat("x", 100), 10); len(got) > 14 { t.Errorf("trim returned %d characters", len(got)) } if got := trim("short", 10); got != "short" { t.Errorf("trim altered a short string: %q", got) } }