diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 8b4d112..ca6638a 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -81,6 +81,9 @@ func (r *Router) Handler() http.Handler { mux.HandleFunc("POST /v1/chat/completions", r.handleInference("/v1/chat/completions")) mux.HandleFunc("POST /v1/completions", r.handleInference("/v1/completions")) mux.HandleFunc("POST /v1/embeddings", r.handleInference("/v1/embeddings")) + // Codex, and whatever else speaks only the Responses API, translated to + // chat completions on the way in and back on the way out. + mux.HandleFunc("POST /v1/responses", r.handleResponses) mux.HandleFunc("GET /v1/models", r.handleModels) // Liveness: this process is up. It says nothing about whether it can // serve, which is why it is not the thing to put a load balancer on. @@ -198,7 +201,7 @@ func (r *Router) Handler() http.Handler { return auth.Middleware(r.auth, r.log, "/health", "/ready", "/", "/_static/", "/claim/", "/signup", "/login", "/logout", "/profile", "/profile/", - "/admin", "/admin/", "/chat", "/chat/", "/org", "/org/")(mux) + "/admin", "/admin/", "/chat", "/chat/", "/org", "/org/", "/setup/")(mux) } // parsedRequest is what the router needs from a request. Everything else in the diff --git a/internal/router/pages_auth_test.go b/internal/router/pages_auth_test.go index 312391e..f0f34c3 100644 --- a/internal/router/pages_auth_test.go +++ b/internal/router/pages_auth_test.go @@ -29,3 +29,14 @@ func TestSessionPagesAreNotAskedForAnAPIKey(t *testing.T) { } } } + +func TestTheEditorSetupScriptNeedsNoKey(t *testing.T) { + // curl | sh from a machine that has no key in a header yet: the script + // holds none, and reads the key from the environment when it runs. + rt, _ := chatRouter(t, "http://127.0.0.1:1") + w := httptest.NewRecorder() + rt.Handler().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/setup/codex?model=qwen", nil)) + if w.Code != http.StatusOK || !strings.HasPrefix(w.Body.String(), "#!/bin/sh") { + t.Errorf("%d %s", w.Code, w.Body.String()) + } +} diff --git a/internal/router/responses.go b/internal/router/responses.go new file mode 100644 index 0000000..7976a69 --- /dev/null +++ b/internal/router/responses.go @@ -0,0 +1,570 @@ +package router + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "sync/atomic" + "time" +) + +// The Responses API, as a translation of chat completions. +// +// Codex speaks it, and newer versions speak nothing else. Every engine here +// serves chat completions, so a Responses request is rewritten into one and +// sent down the same path as any other — quotas, admission, routing, usage +// all as they are — and what comes back is rewritten into the Responses +// shape: a response object, or its stream of typed events. +// +// What is translated is what a coding agent uses: instructions, messages, +// function tools and their calls and outputs, Codex's free-form "custom" +// tools, and usage. What the engines have no equivalent for — stored +// responses, server-side tools, reasoning summaries — is ignored rather than +// refused, so a client that sends them still gets an answer. + +// respRequest is the part of a Responses request that is translated. +type respRequest struct { + Model string `json:"model"` + Instructions string `json:"instructions"` + Input json.RawMessage `json:"input"` + Tools []respTool `json:"tools"` + ToolChoice json.RawMessage `json:"tool_choice"` + Stream bool `json:"stream"` + Temperature *float64 `json:"temperature"` + TopP *float64 `json:"top_p"` + MaxOutputTokens *int `json:"max_output_tokens"` + ParallelToolCalls *bool `json:"parallel_tool_calls"` + Metadata map[string]string `json:"metadata"` +} + +type respTool struct { + Type string `json:"type"` + Name string `json:"name"` + Description string `json:"description"` + Parameters json.RawMessage `json:"parameters"` +} + +// respItem is one input item: a message, a call the model made, or its +// output. Content is a string or a list of parts. +type respItem struct { + Type string `json:"type"` + Role string `json:"role"` + Content json.RawMessage `json:"content"` + CallID string `json:"call_id"` + Name string `json:"name"` + // Arguments for a function call; Input for a custom tool's call. + Arguments string `json:"arguments"` + Input string `json:"input"` + Output json.RawMessage `json:"output"` +} + +type chatMsg struct { + Role string `json:"role"` + Content any `json:"content"` + ToolCalls []chatToolCall `json:"tool_calls,omitempty"` + ToolCallID string `json:"tool_call_id,omitempty"` +} + +type chatToolCall struct { + ID string `json:"id"` + Type string `json:"type"` + Function struct { + Name string `json:"name"` + Arguments string `json:"arguments"` + } `json:"function"` +} + +// customInput is the one argument a custom tool is given as a function: its +// free-form text. +var customInput = json.RawMessage(`{"type":"object","properties":{"input":{"type":"string","description":"The tool's input, as free text."}},"required":["input"]}`) + +// partsText joins a content list's text parts; a plain string is itself. +func partsText(raw json.RawMessage) string { + if len(raw) == 0 { + return "" + } + var s string + if json.Unmarshal(raw, &s) == nil { + return s + } + var parts []struct { + Type string `json:"type"` + Text string `json:"text"` + } + if json.Unmarshal(raw, &parts) != nil { + return "" + } + var b strings.Builder + for _, p := range parts { + switch p.Type { + case "input_text", "output_text", "text", "summary_text": + b.WriteString(p.Text) + } + } + return b.String() +} + +// toChat rewrites a Responses request as a chat completion. custom names the +// tools that were custom, so their calls go back as custom calls. +func toChat(in respRequest) (map[string]any, map[string]bool, error) { + var msgs []chatMsg + if in.Instructions != "" { + msgs = append(msgs, chatMsg{Role: "system", Content: in.Instructions}) + } + var items []respItem + var s string + switch { + case len(in.Input) == 0: + case json.Unmarshal(in.Input, &s) == nil: + items = []respItem{{Type: "message", Role: "user", Content: in.Input}} + default: + if err := json.Unmarshal(in.Input, &items); err != nil { + return nil, nil, fmt.Errorf("input is neither text nor a list of items: %v", err) + } + } + for _, it := range items { + switch it.Type { + case "", "message": + role := it.Role + if role == "developer" { + role = "system" + } + if role == "" { + role = "user" + } + msgs = append(msgs, chatMsg{Role: role, Content: partsText(it.Content)}) + case "function_call", "custom_tool_call": + args := it.Arguments + if it.Type == "custom_tool_call" { + b, _ := json.Marshal(map[string]string{"input": it.Input}) + args = string(b) + } + tc := chatToolCall{ID: it.CallID, Type: "function"} + tc.Function.Name, tc.Function.Arguments = it.Name, args + // Calls made together are one assistant turn. + if n := len(msgs); n > 0 && msgs[n-1].Role == "assistant" && len(msgs[n-1].ToolCalls) > 0 { + msgs[n-1].ToolCalls = append(msgs[n-1].ToolCalls, tc) + } else { + msgs = append(msgs, chatMsg{Role: "assistant", Content: "", ToolCalls: []chatToolCall{tc}}) + } + case "function_call_output", "custom_tool_call_output": + out := partsText(it.Output) + if out == "" && len(it.Output) > 0 { + out = string(it.Output) + } + msgs = append(msgs, chatMsg{Role: "tool", ToolCallID: it.CallID, Content: out}) + } + // reasoning items, and anything else, carry nothing an engine reads. + } + if len(msgs) == 0 { + return nil, nil, fmt.Errorf("the request has no input") + } + + out := map[string]any{"model": in.Model, "messages": msgs, "stream": in.Stream} + if in.Stream { + out["stream_options"] = map[string]any{"include_usage": true} + } + custom := map[string]bool{} + var tools []map[string]any + for _, t := range in.Tools { + params := t.Parameters + switch t.Type { + case "function": + case "custom": + params = customInput + custom[t.Name] = true + default: + continue // server-side tools: nothing here can run them + } + if len(params) == 0 { + params = json.RawMessage(`{"type":"object","properties":{}}`) + } + tools = append(tools, map[string]any{"type": "function", "function": map[string]any{ + "name": t.Name, "description": t.Description, "parameters": params}}) + } + if len(tools) > 0 { + out["tools"] = tools + if len(in.ToolChoice) > 0 { + var choice string + if json.Unmarshal(in.ToolChoice, &choice) == nil { + out["tool_choice"] = choice + } + } + if in.ParallelToolCalls != nil { + out["parallel_tool_calls"] = *in.ParallelToolCalls + } + } + if in.Temperature != nil { + out["temperature"] = *in.Temperature + } + if in.TopP != nil { + out["top_p"] = *in.TopP + } + if in.MaxOutputTokens != nil { + out["max_tokens"] = *in.MaxOutputTokens + } + return out, custom, nil +} + +var respSeq atomic.Int64 + +func respID(prefix string) string { + return fmt.Sprintf("%s_%x%04x", prefix, time.Now().UnixNano(), respSeq.Add(1)&0xffff) +} + +// handleResponses serves POST /v1/responses. +func (r *Router) handleResponses(w http.ResponseWriter, req *http.Request) { + body, err := io.ReadAll(io.LimitReader(req.Body, r.cfg.MaxBodyBytes)) + if err != nil { + writeError(w, http.StatusBadRequest, "invalid_request_error", "could not read the request") + return + } + var in respRequest + if err := json.Unmarshal(body, &in); err != nil { + writeError(w, http.StatusBadRequest, "invalid_request_error", "the body is not JSON: "+err.Error()) + return + } + chat, custom, err := toChat(in) + if err != nil { + writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error()) + return + } + upstream, _ := json.Marshal(chat) + next := req.Clone(req.Context()) + next.URL.Path = "/v1/chat/completions" + next.Body = io.NopCloser(bytes.NewReader(upstream)) + next.ContentLength = int64(len(upstream)) + next.Header.Set("Content-Type", "application/json") + + t := &respTranslator{w: w, model: in.Model, stream: in.Stream, custom: custom, + id: respID("resp"), created: time.Now().Unix(), metadata: in.Metadata} + r.handleInference("/v1/chat/completions")(t, next) + t.finish() +} + +// respTranslator stands between the chat handler and the client, rewriting +// what the handler writes. +type respTranslator struct { + w http.ResponseWriter + model string + stream bool + custom map[string]bool + id string + created int64 + metadata map[string]string + + status int + failed bool + buf bytes.Buffer // a non-streaming body, or a partial line + started bool + seq int + + // What the stream has built. + msgID string + msgIndex int + text strings.Builder + calls []*respCall + byIndex map[int]*respCall + usage map[string]int + finished bool +} + +type respCall struct { + outIndex int + id, callID, name string + args strings.Builder +} + +func (t *respTranslator) Header() http.Header { return t.w.Header() } + +func (t *respTranslator) WriteHeader(code int) { + t.status = code + if code >= 300 { + // An error from the chat path is passed on as it is: its body is + // already the error object a client reads. + t.failed = true + t.w.WriteHeader(code) + return + } + if t.stream { + t.w.Header().Set("Content-Type", "text/event-stream") + } else { + t.w.Header().Set("Content-Type", "application/json") + } + t.w.Header().Del("Content-Length") + t.w.WriteHeader(code) +} + +func (t *respTranslator) Flush() { + if f, ok := t.w.(http.Flusher); ok { + f.Flush() + } +} + +func (t *respTranslator) Write(p []byte) (int, error) { + if t.status == 0 { + t.WriteHeader(http.StatusOK) + } + if t.failed { + return t.w.Write(p) + } + t.buf.Write(p) + if !t.stream { + return len(p), nil + } + for { + line, err := t.buf.ReadString('\n') + if err != nil { + // Not a whole line yet: keep it for the next write. + t.buf.Reset() + t.buf.WriteString(line) + break + } + t.chatLine(strings.TrimSpace(line)) + } + return len(p), nil +} + +// event writes one server-sent event of the Responses stream. +func (t *respTranslator) event(typ string, fields map[string]any) { + fields["type"] = typ + fields["sequence_number"] = t.seq + t.seq++ + b, _ := json.Marshal(fields) + fmt.Fprintf(t.w, "event: %s\ndata: %s\n\n", typ, b) +} + +// object is the response as it stands. +func (t *respTranslator) object(status string) map[string]any { + o := map[string]any{"id": t.id, "object": "response", "created_at": t.created, "model": t.model, + "status": status, "output": t.output(status == "completed")} + if t.metadata != nil { + o["metadata"] = t.metadata + } + if t.usage != nil { + o["usage"] = map[string]any{"input_tokens": t.usage["prompt"], "output_tokens": t.usage["completion"], + "total_tokens": t.usage["prompt"] + t.usage["completion"], + "input_tokens_details": map[string]int{"cached_tokens": 0}, "output_tokens_details": map[string]int{"reasoning_tokens": 0}} + } + return o +} + +func (t *respTranslator) messageItem(done bool) map[string]any { + status := "in_progress" + if done { + status = "completed" + } + return map[string]any{"type": "message", "id": t.msgID, "role": "assistant", "status": status, + "content": []map[string]any{{"type": "output_text", "text": t.text.String(), "annotations": []any{}}}} +} + +func (t *respTranslator) callItem(c *respCall, done bool) map[string]any { + status := "in_progress" + if done { + status = "completed" + } + if t.custom[c.name] { + var a struct { + Input string `json:"input"` + } + _ = json.Unmarshal([]byte(c.args.String()), &a) + return map[string]any{"type": "custom_tool_call", "id": c.id, "call_id": c.callID, "name": c.name, + "input": a.Input, "status": status} + } + return map[string]any{"type": "function_call", "id": c.id, "call_id": c.callID, "name": c.name, + "arguments": c.args.String(), "status": status} +} + +func (t *respTranslator) output(done bool) []map[string]any { + type item struct { + i int + v map[string]any + } + var items []item + if t.msgID != "" { + items = append(items, item{t.msgIndex, t.messageItem(done)}) + } + for _, c := range t.calls { + items = append(items, item{c.outIndex, t.callItem(c, done)}) + } + out := make([]map[string]any, len(items)) + for _, it := range items { + if it.i < len(out) { + out[it.i] = it.v + } + } + return out +} + +func (t *respTranslator) begin() { + if t.started { + return + } + t.started = true + t.byIndex = map[int]*respCall{} + t.event("response.created", map[string]any{"response": t.object("in_progress")}) + t.event("response.in_progress", map[string]any{"response": t.object("in_progress")}) +} + +// chatLine takes one line of the chat stream. +func (t *respTranslator) chatLine(line string) { + if !strings.HasPrefix(line, "data:") { + return + } + payload := strings.TrimSpace(strings.TrimPrefix(line, "data:")) + if payload == "[DONE]" { + return + } + var chunk struct { + Choices []struct { + Delta struct { + Content string `json:"content"` + ToolCalls []struct { + Index int `json:"index"` + ID string `json:"id"` + Function struct { + Name string `json:"name"` + Arguments string `json:"arguments"` + } `json:"function"` + } `json:"tool_calls"` + } `json:"delta"` + } `json:"choices"` + Usage *struct { + Prompt int `json:"prompt_tokens"` + Completion int `json:"completion_tokens"` + } `json:"usage"` + } + if json.Unmarshal([]byte(payload), &chunk) != nil { + return + } + t.begin() + if chunk.Usage != nil { + t.usage = map[string]int{"prompt": chunk.Usage.Prompt, "completion": chunk.Usage.Completion} + } + for _, ch := range chunk.Choices { + if d := ch.Delta.Content; d != "" { + if t.msgID == "" { + // The message's place is after whatever calls came first. + t.msgIndex = len(t.calls) + t.msgID = respID("msg") + t.event("response.output_item.added", map[string]any{"output_index": t.msgIndex, "item": map[string]any{ + "type": "message", "id": t.msgID, "role": "assistant", "status": "in_progress", "content": []any{}}}) + t.event("response.content_part.added", map[string]any{"item_id": t.msgID, "output_index": t.msgIndex, + "content_index": 0, "part": map[string]any{"type": "output_text", "text": "", "annotations": []any{}}}) + } + t.text.WriteString(d) + t.event("response.output_text.delta", map[string]any{"item_id": t.msgID, "output_index": t.msgIndex, + "content_index": 0, "delta": d}) + } + for _, tc := range ch.Delta.ToolCalls { + c := t.byIndex[tc.Index] + if c == nil { + c = &respCall{outIndex: t.nextIndex(), id: respID("fc"), callID: tc.ID, name: tc.Function.Name} + if c.callID == "" { + c.callID = respID("call") + } + t.byIndex[tc.Index] = c + t.calls = append(t.calls, c) + added := t.callItem(c, false) + t.event("response.output_item.added", map[string]any{"output_index": c.outIndex, "item": added}) + } + if tc.Function.Name != "" && c.name == "" { + c.name = tc.Function.Name + } + if a := tc.Function.Arguments; a != "" { + c.args.WriteString(a) + if !t.custom[c.name] { + t.event("response.function_call_arguments.delta", map[string]any{"item_id": c.id, + "output_index": c.outIndex, "delta": a}) + } + } + } + } +} + +// nextIndex is the output position the next item takes. +func (t *respTranslator) nextIndex() int { + n := len(t.calls) + if t.msgID != "" { + n++ + } + return n +} + +// finish closes the stream, or rewrites the whole body. +func (t *respTranslator) finish() { + if t.failed || t.finished { + return + } + t.finished = true + if !t.stream { + t.fromChatJSON(t.buf.Bytes()) + return + } + if rest := strings.TrimSpace(t.buf.String()); rest != "" { + t.chatLine(rest) + } + t.begin() + if t.msgID != "" { + t.event("response.output_text.done", map[string]any{"item_id": t.msgID, "output_index": t.msgIndex, + "content_index": 0, "text": t.text.String()}) + t.event("response.content_part.done", map[string]any{"item_id": t.msgID, "output_index": t.msgIndex, + "content_index": 0, "part": map[string]any{"type": "output_text", "text": t.text.String(), "annotations": []any{}}}) + t.event("response.output_item.done", map[string]any{"output_index": t.msgIndex, "item": t.messageItem(true)}) + } + for _, c := range t.calls { + if !t.custom[c.name] { + t.event("response.function_call_arguments.done", map[string]any{"item_id": c.id, + "output_index": c.outIndex, "arguments": c.args.String()}) + } + t.event("response.output_item.done", map[string]any{"output_index": c.outIndex, "item": t.callItem(c, true)}) + } + t.event("response.completed", map[string]any{"response": t.object("completed")}) + t.Flush() +} + +// fromChatJSON rewrites a whole chat completion as a response object. +func (t *respTranslator) fromChatJSON(b []byte) { + var c struct { + Choices []struct { + Message struct { + Content string `json:"content"` + ToolCalls []struct { + ID string `json:"id"` + Function struct { + Name string `json:"name"` + Arguments string `json:"arguments"` + } `json:"function"` + } `json:"tool_calls"` + } `json:"message"` + } `json:"choices"` + Usage *struct { + Prompt int `json:"prompt_tokens"` + Completion int `json:"completion_tokens"` + } `json:"usage"` + } + if err := json.Unmarshal(b, &c); err != nil { + _, _ = t.w.Write(b) + return + } + t.byIndex = map[int]*respCall{} + if c.Usage != nil { + t.usage = map[string]int{"prompt": c.Usage.Prompt, "completion": c.Usage.Completion} + } + for _, ch := range c.Choices { + if ch.Message.Content != "" { + t.msgID, t.msgIndex = respID("msg"), 0 + t.text.WriteString(ch.Message.Content) + } + for _, tc := range ch.Message.ToolCalls { + rc := &respCall{outIndex: t.nextIndex(), id: respID("fc"), callID: tc.ID, name: tc.Function.Name} + rc.args.WriteString(tc.Function.Arguments) + t.calls = append(t.calls, rc) + } + } + out, _ := json.Marshal(t.object("completed")) + _, _ = t.w.Write(out) +} diff --git a/internal/router/responses_test.go b/internal/router/responses_test.go new file mode 100644 index 0000000..a472380 --- /dev/null +++ b/internal/router/responses_test.go @@ -0,0 +1,208 @@ +package router + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/config" +) + +func TestAResponsesRequestBecomesAChatOne(t *testing.T) { + // What Codex sends in the middle of a task: its instructions, the + // person's turn, a call it made and what the call returned, and its tools + // — one a function, one free-form. + var in respRequest + if err := json.Unmarshal([]byte(`{ + "model": "coder", "instructions": "You are Codex.", "stream": true, + "input": [ + {"type":"message","role":"developer","content":[{"type":"input_text","text":"Be careful."}]}, + {"type":"message","role":"user","content":[{"type":"input_text","text":"List the files."}]}, + {"type":"reasoning","summary":[]}, + {"type":"function_call","call_id":"c1","name":"shell","arguments":"{\"command\":[\"ls\"]}"}, + {"type":"function_call_output","call_id":"c1","output":"a.go\nb.go"}, + {"type":"custom_tool_call","call_id":"c2","name":"apply_patch","input":"*** Begin Patch"}, + {"type":"custom_tool_call_output","call_id":"c2","output":"done"} + ], + "tools": [ + {"type":"function","name":"shell","description":"Run a command","parameters":{"type":"object","properties":{"command":{"type":"array"}}}}, + {"type":"custom","name":"apply_patch","description":"Apply a patch"}, + {"type":"web_search"} + ], + "tool_choice":"auto", "store": false, "reasoning": {"effort":"medium"} + }`), &in); err != nil { + t.Fatal(err) + } + chat, custom, err := toChat(in) + if err != nil { + t.Fatal(err) + } + b, _ := json.Marshal(chat) + var got struct { + Messages []struct { + Role string `json:"role"` + Content string `json:"content"` + ToolCallID string `json:"tool_call_id"` + ToolCalls []struct { + ID string `json:"id"` + Function struct{ Name, Arguments string } + } `json:"tool_calls"` + } `json:"messages"` + Tools []struct { + Function struct{ Name string } `json:"function"` + } `json:"tools"` + StreamOptions struct { + IncludeUsage bool `json:"include_usage"` + } `json:"stream_options"` + } + if err := json.Unmarshal(b, &got); err != nil { + t.Fatal(err) + } + roles := []string{} + for _, m := range got.Messages { + roles = append(roles, m.Role) + } + if strings.Join(roles, ",") != "system,system,user,assistant,tool,assistant,tool" { + t.Errorf("roles = %v", roles) + } + if got.Messages[3].ToolCalls[0].Function.Name != "shell" || got.Messages[4].ToolCallID != "c1" || got.Messages[4].Content != "a.go\nb.go" { + t.Errorf("the shell call and its output: %+v %+v", got.Messages[3], got.Messages[4]) + } + // A custom tool is a function of one string, and its call carries it. + if !strings.Contains(got.Messages[5].ToolCalls[0].Function.Arguments, `"input":"*** Begin Patch"`) || !custom["apply_patch"] { + t.Errorf("custom call = %+v", got.Messages[5]) + } + // web_search is the server's own tool; nothing here can run it. + if len(got.Tools) != 2 || !got.StreamOptions.IncludeUsage { + t.Errorf("tools %+v, usage asked %v", got.Tools, got.StreamOptions.IncludeUsage) + } +} + +// streamingEngine answers chat completions with a stream: some text, a tool +// call in pieces, and usage. +func streamingEngine(t *testing.T, model string) *httptest.Server { + t.Helper() + mux := http.NewServeMux() + mux.HandleFunc("GET /metrics", func(w http.ResponseWriter, _ *http.Request) { + fmt.Fprintf(w, "# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running 0\n") + }) + mux.HandleFunc("POST /v1/chat/completions", func(w http.ResponseWriter, r *http.Request) { + var req struct { + Stream bool `json:"stream"` + } + _ = json.NewDecoder(r.Body).Decode(&req) + if !req.Stream { + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"model":%q,"choices":[{"message":{"content":"Hello.","tool_calls":[{"id":"call_9","type":"function","function":{"name":"shell","arguments":"{\"command\":[\"ls\"]}"}}]}}],"usage":{"prompt_tokens":11,"completion_tokens":7}}`, model) + return + } + w.Header().Set("Content-Type", "text/event-stream") + for _, c := range []string{ + `{"choices":[{"delta":{"content":"Look"}}]}`, + `{"choices":[{"delta":{"content":"ing."}}]}`, + `{"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call_1","function":{"name":"shell","arguments":"{\"comm"}}]}}]}`, + `{"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"and\":[\"ls\"]}"}}]}}]}`, + `{"choices":[{"delta":{},"finish_reason":"tool_calls"}]}`, + `{"choices":[],"usage":{"prompt_tokens":20,"completion_tokens":9}}`, + } { + fmt.Fprintf(w, "data: %s\n\n", c) + } + fmt.Fprint(w, "data: [DONE]\n\n") + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return srv +} + +func TestAResponsesStreamIsTranslatedFromChat(t *testing.T) { + srv := streamingEngine(t, "coder") + rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: srv.URL, Models: []string{"coder"}})) + waitForEndpoint(t, rt) + + req := httptest.NewRequest("POST", "/v1/responses", strings.NewReader( + `{"model":"coder","stream":true,"input":"list the files","tools":[{"type":"function","name":"shell","parameters":{"type":"object"}}]}`)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + rt.Handler().ServeHTTP(w, req) + if w.Code != 200 { + t.Fatalf("%d %s", w.Code, w.Body.String()) + } + var types []string + var completed map[string]any + var args string + for _, block := range strings.Split(w.Body.String(), "\n\n") { + for _, line := range strings.Split(block, "\n") { + if !strings.HasPrefix(line, "data: ") { + continue + } + var ev map[string]any + if err := json.Unmarshal([]byte(strings.TrimPrefix(line, "data: ")), &ev); err != nil { + t.Fatalf("not JSON: %s", line) + } + types = append(types, ev["type"].(string)) + if ev["type"] == "response.completed" { + completed = ev["response"].(map[string]any) + } + if ev["type"] == "response.function_call_arguments.done" { + args = ev["arguments"].(string) + } + } + } + want := []string{"response.created", "response.in_progress", "response.output_item.added", "response.content_part.added", + "response.output_text.delta", "response.output_text.delta", "response.output_item.added", + "response.function_call_arguments.delta", "response.function_call_arguments.delta", + "response.output_text.done", "response.content_part.done", "response.output_item.done", + "response.function_call_arguments.done", "response.output_item.done", "response.completed"} + if strings.Join(types, " ") != strings.Join(want, " ") { + t.Errorf("events:\n got %v\nwant %v", types, want) + } + if args != `{"command":["ls"]}` { + t.Errorf("arguments = %q", args) + } + out, _ := completed["output"].([]any) + if len(out) != 2 { + t.Fatalf("output = %v", completed["output"]) + } + msg, call := out[0].(map[string]any), out[1].(map[string]any) + if msg["type"] != "message" || call["type"] != "function_call" || call["name"] != "shell" || call["call_id"] != "call_1" { + t.Errorf("output = %v", out) + } + if u := completed["usage"].(map[string]any); u["input_tokens"].(float64) != 20 || u["output_tokens"].(float64) != 9 { + t.Errorf("usage = %v", u) + } +} + +func TestAResponsesRequestWithoutAStreamGetsAResponseObject(t *testing.T) { + srv := streamingEngine(t, "coder") + rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: srv.URL, Models: []string{"coder"}})) + waitForEndpoint(t, rt) + req := httptest.NewRequest("POST", "/v1/responses", strings.NewReader(`{"model":"coder","input":"hi"}`)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + rt.Handler().ServeHTTP(w, req) + var got struct { + Object string `json:"object"` + Status string `json:"status"` + Output []struct { + Type string `json:"type"` + Name string `json:"name"` + Content []struct { + Text string `json:"text"` + } `json:"content"` + } `json:"output"` + Usage struct { + Input int `json:"input_tokens"` + Output int `json:"output_tokens"` + } `json:"usage"` + } + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil { + t.Fatalf("%v: %s", err, w.Body.String()) + } + if got.Object != "response" || got.Status != "completed" || len(got.Output) != 2 || + got.Output[0].Content[0].Text != "Hello." || got.Output[1].Name != "shell" || got.Usage.Input != 11 { + t.Errorf("%s", w.Body.String()) + } +} diff --git a/internal/web/account.go b/internal/web/account.go index 8ce936b..eefe394 100644 --- a/internal/web/account.go +++ b/internal/web/account.go @@ -127,6 +127,8 @@ func (p *AccountPages) Register(mux *http.ServeMux) { mux.HandleFunc("POST /profile/keys", p.createKey) mux.HandleFunc("POST /profile/keys/{id}/revoke", p.revokeKey) mux.HandleFunc("POST /profile/password", p.changePassword) + mux.HandleFunc("POST /profile/connect", p.connect) + mux.HandleFunc("GET /setup/{tool}", p.setupScript) } // --- signup --- @@ -258,6 +260,12 @@ type profileData struct { Keys []store.Key Limits []limitRow NewKey string + // Connect is an editor just connected: its key and configuration, shown + // once, like any new key. + Connect *connectView + // Tools and Models are what the connect form offers. + Tools []struct{ Key, Name string } + Models []string Error string PasswordChanged bool @@ -366,6 +374,10 @@ func (p *AccountPages) render(w http.ResponseWriter, r *http.Request, d.MinPassword = account.MinPasswordLen d.BaseURL = baseURL(r) d.ExampleModel = p.exampleModel() + d.Tools = editorTools + if p.served != nil { + d.Models = p.served() + } d.Version = p.version d.Now = time.Now().UTC().Format(time.RFC3339) // The form token comes from the cookie, never from whatever the request diff --git a/internal/web/connect.go b/internal/web/connect.go new file mode 100644 index 0000000..b3f66f8 --- /dev/null +++ b/internal/web/connect.go @@ -0,0 +1,235 @@ +package web + +import ( + "fmt" + "net/http" + "regexp" + "strings" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// Connecting an editor: a key made for it, and the configuration it needs +// with that key, this gateway's address and a served model already in it. +// +// A page cannot write files on somebody's machine, so what it can do is hand +// over the file, ready — to copy, to download, or to have a one-line command +// write. The command's script is served without authentication and holds no +// secret: the key travels in an environment variable, never in a URL, where +// it would land in shell history and proxy logs. + +// editorTools are the tools the page knows how to configure. +var editorTools = []struct{ Key, Name string }{ + {"continue", "Continue"}, {"codex", "Codex CLI"}, {"cline", "Cline"}, +} + +// connectView is what the page shows after a key was made for a tool. +type connectView struct { + Tool, ToolName, Model string + Key string + // Files are what the tool reads, with the key in them. + Files []connectFile + // Fields are for a tool that keeps its settings where no file reaches: + // Cline's live in VS Code's secret storage, set through its own screen. + Fields []connectField + // Command writes the files; Run is how to start the tool on the gateway. + Command, Run string + Notes []string +} + +type connectFile struct{ Path, Name, Body string } +type connectField struct{ Label, Value string } + +// safeModel is what a model name may be to go into a script and a file +// unquoted: the names this gateway serves, and nothing a shell reads twice. +var safeModel = regexp.MustCompile(`^[A-Za-z0-9._:/@+-]{1,200}$`) + +// continueYAML is a Continue assistant pointing at the gateway: a file of its +// own under ~/.continue/assistants, so the person's config.yaml is not touched. +func continueYAML(base, model, key string) string { + return fmt.Sprintf(`# Written by llmgw for Continue: the gateway at %[1]s. +name: llmgw +version: 1.0.0 +schema: v1 +models: + - name: %[2]s (llmgw) + provider: openai + model: %[2]s + apiBase: %[1]s/v1 + apiKey: %[3]s + roles: + - chat + - edit + - apply + capabilities: + - tool_use +# For completion as you type, add a small, fast model here with the role +# autocomplete; a large one makes every keystroke wait for it. +`, base, model, key) +} + +// codexTOML is the provider and profile Codex needs, and nothing else of its +// configuration: codex --profile llmgw uses the gateway, plain codex what it +// used before. The key is read from LLMGW_API_KEY rather than written down. +func codexTOML(base, model string) string { + return fmt.Sprintf(`# Written by llmgw for Codex: the gateway at %[1]s. +[model_providers.llmgw] +name = "llmgw" +base_url = "%[1]s/v1" +env_key = "LLMGW_API_KEY" +wire_api = "responses" + +[profiles.llmgw] +model_provider = "llmgw" +model = "%[2]s" +`, base, model) +} + +// connectFor builds what the page shows for a tool. +func connectFor(tool, base, model, key string) connectView { + v := connectView{Tool: tool, Model: model, Key: key} + for _, t := range editorTools { + if t.Key == tool { + v.ToolName = t.Name + } + } + script := base + "/setup/" + tool + "?model=" + model + switch tool { + case "continue": + v.Files = []connectFile{{Path: "~/.continue/assistants/llmgw.yaml", Name: "llmgw.yaml", + Body: continueYAML(base, model, key)}} + v.Command = fmt.Sprintf("curl -fsSL '%s' | LLMGW_KEY='%s' sh", script, key) + v.Run = `In Continue, choose the "llmgw" assistant at the top of its panel.` + case "codex": + v.Files = []connectFile{{Path: "~/.codex/config.toml", Name: "config.toml", Body: codexTOML(base, model)}} + v.Command = fmt.Sprintf("curl -fsSL '%s' | sh", script) + v.Run = fmt.Sprintf("LLMGW_API_KEY='%s' codex --profile llmgw", key) + v.Notes = []string{ + "The command adds the two blocks above to ~/.codex/config.toml, replacing any it added before, and keeps a copy of the file as it was.", + "Put export LLMGW_API_KEY=… in your shell's profile to leave it out of the command line.", + } + case "cline": + v.Fields = []connectField{ + {"API Provider", "OpenAI Compatible"}, + {"Base URL", base + "/v1"}, + {"API Key", key}, + {"Model ID", model}, + } + v.Run = "Cline keeps these in VS Code's own secret storage, so they are set in its settings: open Cline, the gear, and fill them in." + } + return v +} + +// connect makes a key for an editor and shows its configuration. +func (p *AccountPages) connect(w http.ResponseWriter, r *http.Request) { + noStore(w) + sess, user, ok := p.session(r) + if !ok { + http.Redirect(w, r, "/login", http.StatusSeeOther) + return + } + if !p.accounts.CheckCSRF(sess, r.FormValue("csrf")) { + http.Error(w, "bad request", http.StatusBadRequest) + return + } + tool := r.FormValue("tool") + known := false + for _, t := range editorTools { + known = known || t.Key == tool + } + model := strings.TrimSpace(r.FormValue("model")) + if !known || !safeModel.MatchString(model) { + p.renderProfile(w, r, sess, user, "", "choose a tool and a model") + return + } + project := r.FormValue("context") + if project == personalValue { + if p.personalOff { + p.renderProfile(w, r, sess, user, "", "personal keys are off on this gateway; choose a project") + return + } + project = "" + } + name := tool + " " + time.Now().Format("2006-01-02") + secret, k, err := store.NewKey(randomKeyID(), name, project) + if err != nil { + p.renderProfile(w, r, sess, user, "", "something went wrong; try again") + return + } + k.CreatedAt = time.Now() + if err := p.keys.CreateUserKey(r.Context(), user.ID, k); err != nil { + msg := "something went wrong; try again" + if err == store.ErrNotMember { + msg = "you are not a member of that project" + } + p.renderProfile(w, r, sess, user, "", msg) + return + } + p.log.Info("a user connected an editor", "user", user.ID, "tool", tool, "key", k.ID) + c := connectFor(tool, baseURL(r), model, secret) + p.render(w, r, sess, user, profileData{Connect: &c}) +} + +// setupScript is the script the one-line command runs: it writes the tool's +// configuration with the key from the environment, keeping a copy of what +// was there. Served without a session — it holds nothing a stranger could +// use: this gateway's address, which they already typed, and a model name. +func (p *AccountPages) setupScript(w http.ResponseWriter, r *http.Request) { + tool := r.PathValue("tool") + model := r.URL.Query().Get("model") + if !safeModel.MatchString(model) { + http.Error(w, "which model? add ?model=", http.StatusBadRequest) + return + } + base := baseURL(r) + var body string + switch tool { + case "continue": + body = fmt.Sprintf(`#!/bin/sh +# Connects Continue to the llmgw gateway at %[1]s. +set -eu +: "${LLMGW_KEY:?set LLMGW_KEY to the key the gateway gave you}" +dir="$HOME/.continue/assistants" +file="$dir/llmgw.yaml" +mkdir -p "$dir" +[ -f "$file" ] && cp "$file" "$file.bak.$(date +%%Y%%m%%d%%H%%M%%S)" +umask 077 +cat > "$file" <<'LLMGW' +%[2]sLLMGW +# The key goes in last, from the environment: never through a URL. +tmp="$file.tmp" +sed "s|__LLMGW_KEY__|$LLMGW_KEY|" "$file" > "$tmp" && mv "$tmp" "$file" +echo "Wrote $file. In Continue, choose the llmgw assistant." +`, base, continueYAML(base, model, "__LLMGW_KEY__")) + case "codex": + body = fmt.Sprintf(`#!/bin/sh +# Connects Codex to the llmgw gateway at %[1]s. +set -eu +dir="$HOME/.codex" +file="$dir/config.toml" +mkdir -p "$dir" +touch "$file" +cp "$file" "$file.bak.$(date +%%Y%%m%%d%%H%%M%%S)" +# Out with the blocks this wrote before, in with the new: every other line +# of the file stays as it was. +awk ' + /^\[/ { skip = ($0 == "[model_providers.llmgw]" || $0 == "[profiles.llmgw]" || $0 ~ /^# Written by llmgw/) } + /^# Written by llmgw/ { skip = 1; next } + !skip { print } +' "$file" > "$file.tmp" +cat >> "$file.tmp" <<'LLMGW' + +%[2]sLLMGW +mv "$file.tmp" "$file" +echo "Added the llmgw provider to $file." +echo "Run: LLMGW_API_KEY= codex --profile llmgw" +`, base, codexTOML(base, model)) + default: + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "text/x-shellscript; charset=utf-8") + w.Header().Set("Cache-Control", "no-store") + _, _ = w.Write([]byte(body)) +} diff --git a/internal/web/connect_test.go b/internal/web/connect_test.go new file mode 100644 index 0000000..ce2d5e0 --- /dev/null +++ b/internal/web/connect_test.go @@ -0,0 +1,117 @@ +package web + +import ( + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestEachToolGetsItsConfigurationWithTheKey(t *testing.T) { + c := connectFor("continue", "https://gw.example", "qwen2.5-coder-7b", "sk-llmgw-abc") + if !strings.Contains(c.Files[0].Body, "apiBase: https://gw.example/v1") || !strings.Contains(c.Files[0].Body, "apiKey: sk-llmgw-abc") || + c.Files[0].Path != "~/.continue/assistants/llmgw.yaml" { + t.Errorf("continue = %+v", c.Files) + } + // The key goes in the environment, not the URL, where it would be kept in + // shell history and proxy logs. + if strings.Contains(strings.Split(c.Command, "|")[0], "sk-llmgw-abc") { + t.Errorf("the key is in the URL: %s", c.Command) + } + x := connectFor("codex", "https://gw.example", "qwen2.5-coder-7b", "sk-llmgw-abc") + if strings.Contains(x.Files[0].Body, "sk-llmgw-abc") || !strings.Contains(x.Files[0].Body, `env_key = "LLMGW_API_KEY"`) { + t.Errorf("codex writes the key into its file: %s", x.Files[0].Body) + } + cl := connectFor("cline", "https://gw.example", "m", "sk-llmgw-abc") + if len(cl.Fields) != 4 || cl.Fields[1].Value != "https://gw.example/v1" || cl.Command != "" { + t.Errorf("cline = %+v", cl) + } +} + +// runScript fetches a setup script and runs it with HOME in a temporary +// directory, as a person's shell would. +func runScript(t *testing.T, tool string, env ...string) string { + t.Helper() + p := &AccountPages{} + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", "http://gw.example/setup/"+tool+"?model=qwen2.5-coder-7b", nil) + r.SetPathValue("tool", tool) + p.setupScript(w, r) + if w.Code != http.StatusOK { + t.Fatalf("%d %s", w.Code, w.Body.String()) + } + home := t.TempDir() + cmd := exec.Command("sh") + cmd.Stdin = strings.NewReader(w.Body.String()) + cmd.Env = append([]string{"HOME=" + home, "PATH=" + os.Getenv("PATH")}, env...) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("%v: %s", err, out) + } + return home +} + +func TestTheContinueScriptWritesItsOwnFileWithTheKey(t *testing.T) { + home := runScript(t, "continue", "LLMGW_KEY=sk-llmgw-xyz") + b, err := os.ReadFile(filepath.Join(home, ".continue", "assistants", "llmgw.yaml")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(b), "apiKey: sk-llmgw-xyz") || strings.Contains(string(b), "__LLMGW_KEY__") { + t.Errorf("wrote %s", b) + } + if fi, _ := os.Stat(filepath.Join(home, ".continue", "assistants", "llmgw.yaml")); fi.Mode().Perm()&0o077 != 0 { + t.Errorf("a file holding a key is readable by others: %v", fi.Mode()) + } +} + +func TestTheCodexScriptKeepsTheRestOfTheConfig(t *testing.T) { + // Run twice over a config with the person's own settings: theirs stay, + // and ours appear once. + p := &AccountPages{} + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", "http://gw.example/setup/codex?model=qwen2.5-coder-7b", nil) + r.SetPathValue("tool", "codex") + p.setupScript(w, r) + home := t.TempDir() + if err := os.MkdirAll(filepath.Join(home, ".codex"), 0o755); err != nil { + t.Fatal(err) + } + mine := "model = \"o3\"\n\n[mcp_servers.docs]\ncommand = \"docs-mcp\"\n" + cfg := filepath.Join(home, ".codex", "config.toml") + if err := os.WriteFile(cfg, []byte(mine), 0o644); err != nil { + t.Fatal(err) + } + for range 2 { + cmd := exec.Command("sh") + cmd.Stdin = strings.NewReader(w.Body.String()) + cmd.Env = []string{"HOME=" + home, "PATH=" + os.Getenv("PATH")} + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("%v: %s", err, out) + } + } + b, _ := os.ReadFile(cfg) + got := string(b) + if !strings.Contains(got, `model = "o3"`) || !strings.Contains(got, "[mcp_servers.docs]") { + t.Errorf("the person's settings were lost:\n%s", got) + } + if strings.Count(got, "[model_providers.llmgw]") != 1 || strings.Count(got, "[profiles.llmgw]") != 1 { + t.Errorf("ours should be there once:\n%s", got) + } + if backups, _ := filepath.Glob(cfg + ".bak.*"); len(backups) == 0 { + t.Error("no copy was kept of the file as it was") + } +} + +func TestASetupScriptRefusesANameAShellWouldRead(t *testing.T) { + p := &AccountPages{} + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", "http://gw.example/setup/codex?model=$(rm%20-rf%20~)", nil) + r.SetPathValue("tool", "codex") + p.setupScript(w, r) + if w.Code != http.StatusBadRequest { + t.Errorf("%d", w.Code) + } +} diff --git a/internal/web/static/base.css b/internal/web/static/base.css index e14088f..2b0d28c 100644 --- a/internal/web/static/base.css +++ b/internal/web/static/base.css @@ -795,3 +795,12 @@ details.chat-settings .temp input[type=range] { width: 14rem; vertical-align: mi details.chat-settings output { font-variant-numeric: tabular-nums; font-weight: 700; margin-left: .3rem; } #send.stop { background: var(--bad); border-color: var(--bad); } .chat-stats { margin: -.2rem 0 .6rem; font-size: .72rem; font-variant-numeric: tabular-nums; } + +/* Connecting an editor. */ +.connect-result { border: 1px solid var(--accent); border-radius: 12px; padding: .25rem 1.1rem 1rem; margin: 1rem 0 1.5rem; + background: var(--card); box-shadow: 0 10px 30px -18px var(--accent); } +.copyable { position: relative; } +.copyable pre { margin-bottom: .25rem; overflow-x: auto; } +.copyable button { margin-right: .75rem; font-size: .8rem; } +.connect-form .field-row label { flex: 1 1 12rem; } +table.connect-fields th { white-space: nowrap; } diff --git a/internal/web/templates/account.html b/internal/web/templates/account.html index 36eada0..989b7db 100644 --- a/internal/web/templates/account.html +++ b/internal/web/templates/account.html @@ -68,6 +68,69 @@ Tell whoever you asked for access that you have signed up.

-d '{"model": "{{.ExampleModel}}", "messages": [{"role":"user","content":"hello"}]}' {{end}} +{{with .Connect}} +
+

{{.ToolName}} is ready to connect

+

A key was made for it, shown this once — it is in everything below. Nothing can produce it again.

+{{if .Command}} +

One command

+

Writes the file below{{if eq .Tool "codex"}} into your Codex configuration{{end}}, keeping a copy of anything it replaces.

+
{{.Command}}
+{{end}} +{{range .Files}} +

Or by hand: {{.Path}}

+
{{.Body}}
+
+{{end}} +{{if .Fields}} +

Settings to fill in

+{{range .Fields}} + {{end}} +
{{.Label}}{{.Value}}
+{{end}} +

Then

+

{{if eq .Tool "codex"}}{{.Run}}{{else}}{{.Run}}{{end}}

+{{range .Notes}}

{{.}}

{{end}} +
+{{end}} + +

Connect your editor

+

A key of its own for the tool, and its configuration with the key, this gateway and a model +already in it. Revoke the key below to disconnect it.

+
+ +
+ + + {{if .Contexts}}{{end}} +
+ +
+ +

Create a key