llmgw/internal/router/responses_test.go

package router

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"gitea.mynodes.xyz/ilker/llmgw/internal/config"
)

func TestAResponsesRequestBecomesAChatOne(t *testing.T) {
	// What Codex sends in the middle of a task: its instructions, the
	// person's turn, a call it made and what the call returned, and its tools
	// — one a function, one free-form.
	var in respRequest
	if err := json.Unmarshal([]byte(`{
	  "model": "coder", "instructions": "You are Codex.", "stream": true,
	  "input": [
	    {"type":"message","role":"developer","content":[{"type":"input_text","text":"Be careful."}]},
	    {"type":"message","role":"user","content":[{"type":"input_text","text":"List the files."}]},
	    {"type":"reasoning","summary":[]},
	    {"type":"function_call","call_id":"c1","name":"shell","arguments":"{\"command\":[\"ls\"]}"},
	    {"type":"function_call_output","call_id":"c1","output":"a.go\nb.go"},
	    {"type":"custom_tool_call","call_id":"c2","name":"apply_patch","input":"*** Begin Patch"},
	    {"type":"custom_tool_call_output","call_id":"c2","output":"done"}
	  ],
	  "tools": [
	    {"type":"function","name":"shell","description":"Run a command","parameters":{"type":"object","properties":{"command":{"type":"array"}}}},
	    {"type":"custom","name":"apply_patch","description":"Apply a patch"},
	    {"type":"web_search"}
	  ],
	  "tool_choice":"auto", "store": false, "reasoning": {"effort":"medium"}
	}`), &in); err != nil {
		t.Fatal(err)
	}
	chat, custom, err := toChat(in)
	if err != nil {
		t.Fatal(err)
	}
	b, _ := json.Marshal(chat)
	var got struct {
		Messages []struct {
			Role       string `json:"role"`
			Content    string `json:"content"`
			ToolCallID string `json:"tool_call_id"`
			ToolCalls  []struct {
				ID       string `json:"id"`
				Function struct{ Name, Arguments string }
			} `json:"tool_calls"`
		} `json:"messages"`
		Tools []struct {
			Function struct{ Name string } `json:"function"`
		} `json:"tools"`
		StreamOptions struct {
			IncludeUsage bool `json:"include_usage"`
		} `json:"stream_options"`
	}
	if err := json.Unmarshal(b, &got); err != nil {
		t.Fatal(err)
	}
	roles := []string{}
	for _, m := range got.Messages {
		roles = append(roles, m.Role)
	}
	if strings.Join(roles, ",") != "system,system,user,assistant,tool,assistant,tool" {
		t.Errorf("roles = %v", roles)
	}
	if got.Messages[3].ToolCalls[0].Function.Name != "shell" || got.Messages[4].ToolCallID != "c1" || got.Messages[4].Content != "a.go\nb.go" {
		t.Errorf("the shell call and its output: %+v %+v", got.Messages[3], got.Messages[4])
	}
	// A custom tool is a function of one string, and its call carries it.
	if !strings.Contains(got.Messages[5].ToolCalls[0].Function.Arguments, `"input":"*** Begin Patch"`) || !custom["apply_patch"] {
		t.Errorf("custom call = %+v", got.Messages[5])
	}
	// web_search is the server's own tool; nothing here can run it.
	if len(got.Tools) != 2 || !got.StreamOptions.IncludeUsage {
		t.Errorf("tools %+v, usage asked %v", got.Tools, got.StreamOptions.IncludeUsage)
	}
}

// streamingEngine answers chat completions with a stream: some text, a tool
// call in pieces, and usage.
func streamingEngine(t *testing.T, model string) *httptest.Server {
	t.Helper()
	mux := http.NewServeMux()
	mux.HandleFunc("GET /metrics", func(w http.ResponseWriter, _ *http.Request) {
		fmt.Fprintf(w, "# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running 0\n")
	})
	mux.HandleFunc("POST /v1/chat/completions", func(w http.ResponseWriter, r *http.Request) {
		var req struct {
			Stream bool `json:"stream"`
		}
		_ = json.NewDecoder(r.Body).Decode(&req)
		if !req.Stream {
			w.Header().Set("Content-Type", "application/json")
			fmt.Fprintf(w, `{"model":%q,"choices":[{"message":{"content":"Hello.","tool_calls":[{"id":"call_9","type":"function","function":{"name":"shell","arguments":"{\"command\":[\"ls\"]}"}}]}}],"usage":{"prompt_tokens":11,"completion_tokens":7}}`, model)
			return
		}
		w.Header().Set("Content-Type", "text/event-stream")
		for _, c := range []string{
			`{"choices":[{"delta":{"content":"Look"}}]}`,
			`{"choices":[{"delta":{"content":"ing."}}]}`,
			`{"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call_1","function":{"name":"shell","arguments":"{\"comm"}}]}}]}`,
			`{"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"and\":[\"ls\"]}"}}]}}]}`,
			`{"choices":[{"delta":{},"finish_reason":"tool_calls"}]}`,
			`{"choices":[],"usage":{"prompt_tokens":20,"completion_tokens":9}}`,
		} {
			fmt.Fprintf(w, "data: %s\n\n", c)
		}
		fmt.Fprint(w, "data: [DONE]\n\n")
	})
	srv := httptest.NewServer(mux)
	t.Cleanup(srv.Close)
	return srv
}

func TestAResponsesStreamIsTranslatedFromChat(t *testing.T) {
	srv := streamingEngine(t, "coder")
	rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: srv.URL, Models: []string{"coder"}}))
	waitForEndpoint(t, rt)

	req := httptest.NewRequest("POST", "/v1/responses", strings.NewReader(
		`{"model":"coder","stream":true,"input":"list the files","tools":[{"type":"function","name":"shell","parameters":{"type":"object"}}]}`))
	req.Header.Set("Content-Type", "application/json")
	w := httptest.NewRecorder()
	rt.Handler().ServeHTTP(w, req)
	if w.Code != 200 {
		t.Fatalf("%d %s", w.Code, w.Body.String())
	}
	var types []string
	var completed map[string]any
	var args string
	for _, block := range strings.Split(w.Body.String(), "\n\n") {
		for _, line := range strings.Split(block, "\n") {
			if !strings.HasPrefix(line, "data: ") {
				continue
			}
			var ev map[string]any
			if err := json.Unmarshal([]byte(strings.TrimPrefix(line, "data: ")), &ev); err != nil {
				t.Fatalf("not JSON: %s", line)
			}
			types = append(types, ev["type"].(string))
			if ev["type"] == "response.completed" {
				completed = ev["response"].(map[string]any)
			}
			if ev["type"] == "response.function_call_arguments.done" {
				args = ev["arguments"].(string)
			}
		}
	}
	want := []string{"response.created", "response.in_progress", "response.output_item.added", "response.content_part.added",
		"response.output_text.delta", "response.output_text.delta", "response.output_item.added",
		"response.function_call_arguments.delta", "response.function_call_arguments.delta",
		"response.output_text.done", "response.content_part.done", "response.output_item.done",
		"response.function_call_arguments.done", "response.output_item.done", "response.completed"}
	if strings.Join(types, " ") != strings.Join(want, " ") {
		t.Errorf("events:\n got %v\nwant %v", types, want)
	}
	if args != `{"command":["ls"]}` {
		t.Errorf("arguments = %q", args)
	}
	out, _ := completed["output"].([]any)
	if len(out) != 2 {
		t.Fatalf("output = %v", completed["output"])
	}
	msg, call := out[0].(map[string]any), out[1].(map[string]any)
	if msg["type"] != "message" || call["type"] != "function_call" || call["name"] != "shell" || call["call_id"] != "call_1" {
		t.Errorf("output = %v", out)
	}
	if u := completed["usage"].(map[string]any); u["input_tokens"].(float64) != 20 || u["output_tokens"].(float64) != 9 {
		t.Errorf("usage = %v", u)
	}
}

func TestAResponsesRequestWithoutAStreamGetsAResponseObject(t *testing.T) {
	srv := streamingEngine(t, "coder")
	rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: srv.URL, Models: []string{"coder"}}))
	waitForEndpoint(t, rt)
	req := httptest.NewRequest("POST", "/v1/responses", strings.NewReader(`{"model":"coder","input":"hi"}`))
	req.Header.Set("Content-Type", "application/json")
	w := httptest.NewRecorder()
	rt.Handler().ServeHTTP(w, req)
	var got struct {
		Object string `json:"object"`
		Status string `json:"status"`
		Output []struct {
			Type    string `json:"type"`
			Name    string `json:"name"`
			Content []struct {
				Text string `json:"text"`
			} `json:"content"`
		} `json:"output"`
		Usage struct {
			Input  int `json:"input_tokens"`
			Output int `json:"output_tokens"`
		} `json:"usage"`
	}
	if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
		t.Fatalf("%v: %s", err, w.Body.String())
	}
	if got.Object != "response" || got.Status != "completed" || len(got.Output) != 2 ||
		got.Output[0].Content[0].Text != "Hello." || got.Output[1].Name != "shell" || got.Usage.Input != 11 {
		t.Errorf("%s", w.Body.String())
	}
}

func TestAnAnswerStoppedByPolicyIsAnIncompleteResponse(t *testing.T) {
	for _, stream := range []bool{true, false} {
		w := httptest.NewRecorder()
		tr := &respTranslator{w: w, model: "m", stream: stream, id: "resp_1", status: 200}
		if stream {
			_, _ = tr.Write([]byte("data: {\"choices\":[{\"delta\":{\"content\":\"par\"}}]}\n\n" +
				"data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"content_filter\"}]}\n\ndata: [DONE]\n\n"))
		} else {
			_, _ = tr.Write([]byte(`{"choices":[{"message":{"content":""},"finish_reason":"content_filter"}]}`))
		}
		tr.finish()
		body := w.Body.String()
		if !strings.Contains(body, `"incomplete_details":{"reason":"content_filter"}`) ||
			!strings.Contains(body, `"status":"incomplete"`) || strings.Contains(body, "response.completed") {
			t.Errorf("stream=%v: %s", stream, body)
		}
		if stream && !strings.Contains(body, "response.incomplete") {
			t.Errorf("no response.incomplete event: %s", body)
		}
	}
}