Connect an editor from the account page, and serve the Responses API

The account page makes a key for Continue, Codex or Cline and shows
what the tool needs with the key, this gateway's address and a served
model in it: Continue's assistant file, written to its own place under
~/.continue/assistants so the person's config.yaml is not touched;
Codex's provider and profile, added to ~/.codex/config.toml without
changing its default model; and for Cline, whose settings live in VS
Code's secret storage, the four fields to paste. Continue and Codex get
a one-line command whose script, served without a session, holds no
secret - the key comes from the environment, never a URL - keeps a copy
of whatever it replaces, and leaves the rest of the file as it was.

POST /v1/responses translates the Responses API, which Codex speaks, to
chat completions and back: instructions, messages, function tools and
Codex's free-form custom tools, their calls and outputs, and usage, as a
response object or as its stream of typed events. It goes down the same
path as any request - quotas, admission, routing, usage. Server-side
tools and stored responses have no equivalent and are ignored.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-29 10:04:17 +02:00
1 parent b7e8e1a3a1
commit f989e67e1a
9 files changed
+1229 -1

No files matched your search

+4 -1
View File
@@ -81,6 +81,9 @@ func (r *Router) Handler() http.Handler {
mux.HandleFunc("POST /v1/chat/completions", r.handleInference("/v1/chat/completions"))
mux.HandleFunc("POST /v1/completions", r.handleInference("/v1/completions"))
mux.HandleFunc("POST /v1/embeddings", r.handleInference("/v1/embeddings"))
// Codex, and whatever else speaks only the Responses API, translated to
// chat completions on the way in and back on the way out.
mux.HandleFunc("POST /v1/responses", r.handleResponses)
mux.HandleFunc("GET /v1/models", r.handleModels)
// Liveness: this process is up. It says nothing about whether it can
// serve, which is why it is not the thing to put a load balancer on.
@@ -198,7 +201,7 @@ func (r *Router) Handler() http.Handler {
return auth.Middleware(r.auth, r.log,
"/health", "/ready", "/", "/_static/", "/claim/",
"/signup", "/login", "/logout", "/profile", "/profile/",
"/admin", "/admin/", "/chat", "/chat/", "/org", "/org/")(mux)
"/admin", "/admin/", "/chat", "/chat/", "/org", "/org/", "/setup/")(mux)
}
// parsedRequest is what the router needs from a request. Everything else in the
+11
View File
@@ -29,3 +29,14 @@ func TestSessionPagesAreNotAskedForAnAPIKey(t *testing.T) {
}
}
}
func TestTheEditorSetupScriptNeedsNoKey(t *testing.T) {
// curl | sh from a machine that has no key in a header yet: the script
// holds none, and reads the key from the environment when it runs.
rt, _ := chatRouter(t, "http://127.0.0.1:1")
w := httptest.NewRecorder()
rt.Handler().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/setup/codex?model=qwen", nil))
if w.Code != http.StatusOK || !strings.HasPrefix(w.Body.String(), "#!/bin/sh") {
t.Errorf("%d %s", w.Code, w.Body.String())
}
}
+570
View File
@@ -0,0 +1,570 @@
package router
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"sync/atomic"
"time"
)
// The Responses API, as a translation of chat completions.
//
// Codex speaks it, and newer versions speak nothing else. Every engine here
// serves chat completions, so a Responses request is rewritten into one and
// sent down the same path as any other — quotas, admission, routing, usage
// all as they are — and what comes back is rewritten into the Responses
// shape: a response object, or its stream of typed events.
//
// What is translated is what a coding agent uses: instructions, messages,
// function tools and their calls and outputs, Codex's free-form "custom"
// tools, and usage. What the engines have no equivalent for — stored
// responses, server-side tools, reasoning summaries — is ignored rather than
// refused, so a client that sends them still gets an answer.
// respRequest is the part of a Responses request that is translated.
type respRequest struct {
Model string `json:"model"`
Instructions string `json:"instructions"`
Input json.RawMessage `json:"input"`
Tools []respTool `json:"tools"`
ToolChoice json.RawMessage `json:"tool_choice"`
Stream bool `json:"stream"`
Temperature *float64 `json:"temperature"`
TopP *float64 `json:"top_p"`
MaxOutputTokens *int `json:"max_output_tokens"`
ParallelToolCalls *bool `json:"parallel_tool_calls"`
Metadata map[string]string `json:"metadata"`
}
type respTool struct {
Type string `json:"type"`
Name string `json:"name"`
Description string `json:"description"`
Parameters json.RawMessage `json:"parameters"`
}
// respItem is one input item: a message, a call the model made, or its
// output. Content is a string or a list of parts.
type respItem struct {
Type string `json:"type"`
Role string `json:"role"`
Content json.RawMessage `json:"content"`
CallID string `json:"call_id"`
Name string `json:"name"`
// Arguments for a function call; Input for a custom tool's call.
Arguments string `json:"arguments"`
Input string `json:"input"`
Output json.RawMessage `json:"output"`
}
type chatMsg struct {
Role string `json:"role"`
Content any `json:"content"`
ToolCalls []chatToolCall `json:"tool_calls,omitempty"`
ToolCallID string `json:"tool_call_id,omitempty"`
}
type chatToolCall struct {
ID string `json:"id"`
Type string `json:"type"`
Function struct {
Name string `json:"name"`
Arguments string `json:"arguments"`
} `json:"function"`
}
// customInput is the one argument a custom tool is given as a function: its
// free-form text.
var customInput = json.RawMessage(`{"type":"object","properties":{"input":{"type":"string","description":"The tool's input, as free text."}},"required":["input"]}`)
// partsText joins a content list's text parts; a plain string is itself.
func partsText(raw json.RawMessage) string {
if len(raw) == 0 {
return ""
}
var s string
if json.Unmarshal(raw, &s) == nil {
return s
}
var parts []struct {
Type string `json:"type"`
Text string `json:"text"`
}
if json.Unmarshal(raw, &parts) != nil {
return ""
}
var b strings.Builder
for _, p := range parts {
switch p.Type {
case "input_text", "output_text", "text", "summary_text":
b.WriteString(p.Text)
}
}
return b.String()
}
// toChat rewrites a Responses request as a chat completion. custom names the
// tools that were custom, so their calls go back as custom calls.
func toChat(in respRequest) (map[string]any, map[string]bool, error) {
var msgs []chatMsg
if in.Instructions != "" {
msgs = append(msgs, chatMsg{Role: "system", Content: in.Instructions})
}
var items []respItem
var s string
switch {
case len(in.Input) == 0:
case json.Unmarshal(in.Input, &s) == nil:
items = []respItem{{Type: "message", Role: "user", Content: in.Input}}
default:
if err := json.Unmarshal(in.Input, &items); err != nil {
return nil, nil, fmt.Errorf("input is neither text nor a list of items: %v", err)
}
}
for _, it := range items {
switch it.Type {
case "", "message":
role := it.Role
if role == "developer" {
role = "system"
}
if role == "" {
role = "user"
}
msgs = append(msgs, chatMsg{Role: role, Content: partsText(it.Content)})
case "function_call", "custom_tool_call":
args := it.Arguments
if it.Type == "custom_tool_call" {
b, _ := json.Marshal(map[string]string{"input": it.Input})
args = string(b)
}
tc := chatToolCall{ID: it.CallID, Type: "function"}
tc.Function.Name, tc.Function.Arguments = it.Name, args
// Calls made together are one assistant turn.
if n := len(msgs); n > 0 && msgs[n-1].Role == "assistant" && len(msgs[n-1].ToolCalls) > 0 {
msgs[n-1].ToolCalls = append(msgs[n-1].ToolCalls, tc)
} else {
msgs = append(msgs, chatMsg{Role: "assistant", Content: "", ToolCalls: []chatToolCall{tc}})
}
case "function_call_output", "custom_tool_call_output":
out := partsText(it.Output)
if out == "" && len(it.Output) > 0 {
out = string(it.Output)
}
msgs = append(msgs, chatMsg{Role: "tool", ToolCallID: it.CallID, Content: out})
}
// reasoning items, and anything else, carry nothing an engine reads.
}
if len(msgs) == 0 {
return nil, nil, fmt.Errorf("the request has no input")
}
out := map[string]any{"model": in.Model, "messages": msgs, "stream": in.Stream}
if in.Stream {
out["stream_options"] = map[string]any{"include_usage": true}
}
custom := map[string]bool{}
var tools []map[string]any
for _, t := range in.Tools {
params := t.Parameters
switch t.Type {
case "function":
case "custom":
params = customInput
custom[t.Name] = true
default:
continue // server-side tools: nothing here can run them
}
if len(params) == 0 {
params = json.RawMessage(`{"type":"object","properties":{}}`)
}
tools = append(tools, map[string]any{"type": "function", "function": map[string]any{
"name": t.Name, "description": t.Description, "parameters": params}})
}
if len(tools) > 0 {
out["tools"] = tools
if len(in.ToolChoice) > 0 {
var choice string
if json.Unmarshal(in.ToolChoice, &choice) == nil {
out["tool_choice"] = choice
}
}
if in.ParallelToolCalls != nil {
out["parallel_tool_calls"] = *in.ParallelToolCalls
}
}
if in.Temperature != nil {
out["temperature"] = *in.Temperature
}
if in.TopP != nil {
out["top_p"] = *in.TopP
}
if in.MaxOutputTokens != nil {
out["max_tokens"] = *in.MaxOutputTokens
}
return out, custom, nil
}
var respSeq atomic.Int64
func respID(prefix string) string {
return fmt.Sprintf("%s_%x%04x", prefix, time.Now().UnixNano(), respSeq.Add(1)&0xffff)
}
// handleResponses serves POST /v1/responses.
func (r *Router) handleResponses(w http.ResponseWriter, req *http.Request) {
body, err := io.ReadAll(io.LimitReader(req.Body, r.cfg.MaxBodyBytes))
if err != nil {
writeError(w, http.StatusBadRequest, "invalid_request_error", "could not read the request")
return
}
var in respRequest
if err := json.Unmarshal(body, &in); err != nil {
writeError(w, http.StatusBadRequest, "invalid_request_error", "the body is not JSON: "+err.Error())
return
}
chat, custom, err := toChat(in)
if err != nil {
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
return
}
upstream, _ := json.Marshal(chat)
next := req.Clone(req.Context())
next.URL.Path = "/v1/chat/completions"
next.Body = io.NopCloser(bytes.NewReader(upstream))
next.ContentLength = int64(len(upstream))
next.Header.Set("Content-Type", "application/json")
t := &respTranslator{w: w, model: in.Model, stream: in.Stream, custom: custom,
id: respID("resp"), created: time.Now().Unix(), metadata: in.Metadata}
r.handleInference("/v1/chat/completions")(t, next)
t.finish()
}
// respTranslator stands between the chat handler and the client, rewriting
// what the handler writes.
type respTranslator struct {
w http.ResponseWriter
model string
stream bool
custom map[string]bool
id string
created int64
metadata map[string]string
status int
failed bool
buf bytes.Buffer // a non-streaming body, or a partial line
started bool
seq int
// What the stream has built.
msgID string
msgIndex int
text strings.Builder
calls []*respCall
byIndex map[int]*respCall
usage map[string]int
finished bool
}
type respCall struct {
outIndex int
id, callID, name string
args strings.Builder
}
func (t *respTranslator) Header() http.Header { return t.w.Header() }
func (t *respTranslator) WriteHeader(code int) {
t.status = code
if code >= 300 {
// An error from the chat path is passed on as it is: its body is
// already the error object a client reads.
t.failed = true
t.w.WriteHeader(code)
return
}
if t.stream {
t.w.Header().Set("Content-Type", "text/event-stream")
} else {
t.w.Header().Set("Content-Type", "application/json")
}
t.w.Header().Del("Content-Length")
t.w.WriteHeader(code)
}
func (t *respTranslator) Flush() {
if f, ok := t.w.(http.Flusher); ok {
f.Flush()
}
}
func (t *respTranslator) Write(p []byte) (int, error) {
if t.status == 0 {
t.WriteHeader(http.StatusOK)
}
if t.failed {
return t.w.Write(p)
}
t.buf.Write(p)
if !t.stream {
return len(p), nil
}
for {
line, err := t.buf.ReadString('\n')
if err != nil {
// Not a whole line yet: keep it for the next write.
t.buf.Reset()
t.buf.WriteString(line)
break
}
t.chatLine(strings.TrimSpace(line))
}
return len(p), nil
}
// event writes one server-sent event of the Responses stream.
func (t *respTranslator) event(typ string, fields map[string]any) {
fields["type"] = typ
fields["sequence_number"] = t.seq
t.seq++
b, _ := json.Marshal(fields)
fmt.Fprintf(t.w, "event: %s\ndata: %s\n\n", typ, b)
}
// object is the response as it stands.
func (t *respTranslator) object(status string) map[string]any {
o := map[string]any{"id": t.id, "object": "response", "created_at": t.created, "model": t.model,
"status": status, "output": t.output(status == "completed")}
if t.metadata != nil {
o["metadata"] = t.metadata
}
if t.usage != nil {
o["usage"] = map[string]any{"input_tokens": t.usage["prompt"], "output_tokens": t.usage["completion"],
"total_tokens": t.usage["prompt"] + t.usage["completion"],
"input_tokens_details": map[string]int{"cached_tokens": 0}, "output_tokens_details": map[string]int{"reasoning_tokens": 0}}
}
return o
}
func (t *respTranslator) messageItem(done bool) map[string]any {
status := "in_progress"
if done {
status = "completed"
}
return map[string]any{"type": "message", "id": t.msgID, "role": "assistant", "status": status,
"content": []map[string]any{{"type": "output_text", "text": t.text.String(), "annotations": []any{}}}}
}
func (t *respTranslator) callItem(c *respCall, done bool) map[string]any {
status := "in_progress"
if done {
status = "completed"
}
if t.custom[c.name] {
var a struct {
Input string `json:"input"`
}
_ = json.Unmarshal([]byte(c.args.String()), &a)
return map[string]any{"type": "custom_tool_call", "id": c.id, "call_id": c.callID, "name": c.name,
"input": a.Input, "status": status}
}
return map[string]any{"type": "function_call", "id": c.id, "call_id": c.callID, "name": c.name,
"arguments": c.args.String(), "status": status}
}
func (t *respTranslator) output(done bool) []map[string]any {
type item struct {
i int
v map[string]any
}
var items []item
if t.msgID != "" {
items = append(items, item{t.msgIndex, t.messageItem(done)})
}
for _, c := range t.calls {
items = append(items, item{c.outIndex, t.callItem(c, done)})
}
out := make([]map[string]any, len(items))
for _, it := range items {
if it.i < len(out) {
out[it.i] = it.v
}
}
return out
}
func (t *respTranslator) begin() {
if t.started {
return
}
t.started = true
t.byIndex = map[int]*respCall{}
t.event("response.created", map[string]any{"response": t.object("in_progress")})
t.event("response.in_progress", map[string]any{"response": t.object("in_progress")})
}
// chatLine takes one line of the chat stream.
func (t *respTranslator) chatLine(line string) {
if !strings.HasPrefix(line, "data:") {
return
}
payload := strings.TrimSpace(strings.TrimPrefix(line, "data:"))
if payload == "[DONE]" {
return
}
var chunk struct {
Choices []struct {
Delta struct {
Content string `json:"content"`
ToolCalls []struct {
Index int `json:"index"`
ID string `json:"id"`
Function struct {
Name string `json:"name"`
Arguments string `json:"arguments"`
} `json:"function"`
} `json:"tool_calls"`
} `json:"delta"`
} `json:"choices"`
Usage *struct {
Prompt int `json:"prompt_tokens"`
Completion int `json:"completion_tokens"`
} `json:"usage"`
}
if json.Unmarshal([]byte(payload), &chunk) != nil {
return
}
t.begin()
if chunk.Usage != nil {
t.usage = map[string]int{"prompt": chunk.Usage.Prompt, "completion": chunk.Usage.Completion}
}
for _, ch := range chunk.Choices {
if d := ch.Delta.Content; d != "" {
if t.msgID == "" {
// The message's place is after whatever calls came first.
t.msgIndex = len(t.calls)
t.msgID = respID("msg")
t.event("response.output_item.added", map[string]any{"output_index": t.msgIndex, "item": map[string]any{
"type": "message", "id": t.msgID, "role": "assistant", "status": "in_progress", "content": []any{}}})
t.event("response.content_part.added", map[string]any{"item_id": t.msgID, "output_index": t.msgIndex,
"content_index": 0, "part": map[string]any{"type": "output_text", "text": "", "annotations": []any{}}})
}
t.text.WriteString(d)
t.event("response.output_text.delta", map[string]any{"item_id": t.msgID, "output_index": t.msgIndex,
"content_index": 0, "delta": d})
}
for _, tc := range ch.Delta.ToolCalls {
c := t.byIndex[tc.Index]
if c == nil {
c = &respCall{outIndex: t.nextIndex(), id: respID("fc"), callID: tc.ID, name: tc.Function.Name}
if c.callID == "" {
c.callID = respID("call")
}
t.byIndex[tc.Index] = c
t.calls = append(t.calls, c)
added := t.callItem(c, false)
t.event("response.output_item.added", map[string]any{"output_index": c.outIndex, "item": added})
}
if tc.Function.Name != "" && c.name == "" {
c.name = tc.Function.Name
}
if a := tc.Function.Arguments; a != "" {
c.args.WriteString(a)
if !t.custom[c.name] {
t.event("response.function_call_arguments.delta", map[string]any{"item_id": c.id,
"output_index": c.outIndex, "delta": a})
}
}
}
}
}
// nextIndex is the output position the next item takes.
func (t *respTranslator) nextIndex() int {
n := len(t.calls)
if t.msgID != "" {
n++
}
return n
}
// finish closes the stream, or rewrites the whole body.
func (t *respTranslator) finish() {
if t.failed || t.finished {
return
}
t.finished = true
if !t.stream {
t.fromChatJSON(t.buf.Bytes())
return
}
if rest := strings.TrimSpace(t.buf.String()); rest != "" {
t.chatLine(rest)
}
t.begin()
if t.msgID != "" {
t.event("response.output_text.done", map[string]any{"item_id": t.msgID, "output_index": t.msgIndex,
"content_index": 0, "text": t.text.String()})
t.event("response.content_part.done", map[string]any{"item_id": t.msgID, "output_index": t.msgIndex,
"content_index": 0, "part": map[string]any{"type": "output_text", "text": t.text.String(), "annotations": []any{}}})
t.event("response.output_item.done", map[string]any{"output_index": t.msgIndex, "item": t.messageItem(true)})
}
for _, c := range t.calls {
if !t.custom[c.name] {
t.event("response.function_call_arguments.done", map[string]any{"item_id": c.id,
"output_index": c.outIndex, "arguments": c.args.String()})
}
t.event("response.output_item.done", map[string]any{"output_index": c.outIndex, "item": t.callItem(c, true)})
}
t.event("response.completed", map[string]any{"response": t.object("completed")})
t.Flush()
}
// fromChatJSON rewrites a whole chat completion as a response object.
func (t *respTranslator) fromChatJSON(b []byte) {
var c struct {
Choices []struct {
Message struct {
Content string `json:"content"`
ToolCalls []struct {
ID string `json:"id"`
Function struct {
Name string `json:"name"`
Arguments string `json:"arguments"`
} `json:"function"`
} `json:"tool_calls"`
} `json:"message"`
} `json:"choices"`
Usage *struct {
Prompt int `json:"prompt_tokens"`
Completion int `json:"completion_tokens"`
} `json:"usage"`
}
if err := json.Unmarshal(b, &c); err != nil {
_, _ = t.w.Write(b)
return
}
t.byIndex = map[int]*respCall{}
if c.Usage != nil {
t.usage = map[string]int{"prompt": c.Usage.Prompt, "completion": c.Usage.Completion}
}
for _, ch := range c.Choices {
if ch.Message.Content != "" {
t.msgID, t.msgIndex = respID("msg"), 0
t.text.WriteString(ch.Message.Content)
}
for _, tc := range ch.Message.ToolCalls {
rc := &respCall{outIndex: t.nextIndex(), id: respID("fc"), callID: tc.ID, name: tc.Function.Name}
rc.args.WriteString(tc.Function.Arguments)
t.calls = append(t.calls, rc)
}
}
out, _ := json.Marshal(t.object("completed"))
_, _ = t.w.Write(out)
}
+208
View File
@@ -0,0 +1,208 @@
package router
import (
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
)
func TestAResponsesRequestBecomesAChatOne(t *testing.T) {
// What Codex sends in the middle of a task: its instructions, the
// person's turn, a call it made and what the call returned, and its tools
// — one a function, one free-form.
var in respRequest
if err := json.Unmarshal([]byte(`{
"model": "coder", "instructions": "You are Codex.", "stream": true,
"input": [
{"type":"message","role":"developer","content":[{"type":"input_text","text":"Be careful."}]},
{"type":"message","role":"user","content":[{"type":"input_text","text":"List the files."}]},
{"type":"reasoning","summary":[]},
{"type":"function_call","call_id":"c1","name":"shell","arguments":"{\"command\":[\"ls\"]}"},
{"type":"function_call_output","call_id":"c1","output":"a.go\nb.go"},
{"type":"custom_tool_call","call_id":"c2","name":"apply_patch","input":"*** Begin Patch"},
{"type":"custom_tool_call_output","call_id":"c2","output":"done"}
],
"tools": [
{"type":"function","name":"shell","description":"Run a command","parameters":{"type":"object","properties":{"command":{"type":"array"}}}},
{"type":"custom","name":"apply_patch","description":"Apply a patch"},
{"type":"web_search"}
],
"tool_choice":"auto", "store": false, "reasoning": {"effort":"medium"}
}`), &in); err != nil {
t.Fatal(err)
}
chat, custom, err := toChat(in)
if err != nil {
t.Fatal(err)
}
b, _ := json.Marshal(chat)
var got struct {
Messages []struct {
Role string `json:"role"`
Content string `json:"content"`
ToolCallID string `json:"tool_call_id"`
ToolCalls []struct {
ID string `json:"id"`
Function struct{ Name, Arguments string }
} `json:"tool_calls"`
} `json:"messages"`
Tools []struct {
Function struct{ Name string } `json:"function"`
} `json:"tools"`
StreamOptions struct {
IncludeUsage bool `json:"include_usage"`
} `json:"stream_options"`
}
if err := json.Unmarshal(b, &got); err != nil {
t.Fatal(err)
}
roles := []string{}
for _, m := range got.Messages {
roles = append(roles, m.Role)
}
if strings.Join(roles, ",") != "system,system,user,assistant,tool,assistant,tool" {
t.Errorf("roles = %v", roles)
}
if got.Messages[3].ToolCalls[0].Function.Name != "shell" || got.Messages[4].ToolCallID != "c1" || got.Messages[4].Content != "a.go\nb.go" {
t.Errorf("the shell call and its output: %+v %+v", got.Messages[3], got.Messages[4])
}
// A custom tool is a function of one string, and its call carries it.
if !strings.Contains(got.Messages[5].ToolCalls[0].Function.Arguments, `"input":"*** Begin Patch"`) || !custom["apply_patch"] {
t.Errorf("custom call = %+v", got.Messages[5])
}
// web_search is the server's own tool; nothing here can run it.
if len(got.Tools) != 2 || !got.StreamOptions.IncludeUsage {
t.Errorf("tools %+v, usage asked %v", got.Tools, got.StreamOptions.IncludeUsage)
}
}
// streamingEngine answers chat completions with a stream: some text, a tool
// call in pieces, and usage.
func streamingEngine(t *testing.T, model string) *httptest.Server {
t.Helper()
mux := http.NewServeMux()
mux.HandleFunc("GET /metrics", func(w http.ResponseWriter, _ *http.Request) {
fmt.Fprintf(w, "# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running 0\n")
})
mux.HandleFunc("POST /v1/chat/completions", func(w http.ResponseWriter, r *http.Request) {
var req struct {
Stream bool `json:"stream"`
}
_ = json.NewDecoder(r.Body).Decode(&req)
if !req.Stream {
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, `{"model":%q,"choices":[{"message":{"content":"Hello.","tool_calls":[{"id":"call_9","type":"function","function":{"name":"shell","arguments":"{\"command\":[\"ls\"]}"}}]}}],"usage":{"prompt_tokens":11,"completion_tokens":7}}`, model)
return
}
w.Header().Set("Content-Type", "text/event-stream")
for _, c := range []string{
`{"choices":[{"delta":{"content":"Look"}}]}`,
`{"choices":[{"delta":{"content":"ing."}}]}`,
`{"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call_1","function":{"name":"shell","arguments":"{\"comm"}}]}}]}`,
`{"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"and\":[\"ls\"]}"}}]}}]}`,
`{"choices":[{"delta":{},"finish_reason":"tool_calls"}]}`,
`{"choices":[],"usage":{"prompt_tokens":20,"completion_tokens":9}}`,
} {
fmt.Fprintf(w, "data: %s\n\n", c)
}
fmt.Fprint(w, "data: [DONE]\n\n")
})
srv := httptest.NewServer(mux)
t.Cleanup(srv.Close)
return srv
}
func TestAResponsesStreamIsTranslatedFromChat(t *testing.T) {
srv := streamingEngine(t, "coder")
rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: srv.URL, Models: []string{"coder"}}))
waitForEndpoint(t, rt)
req := httptest.NewRequest("POST", "/v1/responses", strings.NewReader(
`{"model":"coder","stream":true,"input":"list the files","tools":[{"type":"function","name":"shell","parameters":{"type":"object"}}]}`))
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
rt.Handler().ServeHTTP(w, req)
if w.Code != 200 {
t.Fatalf("%d %s", w.Code, w.Body.String())
}
var types []string
var completed map[string]any
var args string
for _, block := range strings.Split(w.Body.String(), "\n\n") {
for _, line := range strings.Split(block, "\n") {
if !strings.HasPrefix(line, "data: ") {
continue
}
var ev map[string]any
if err := json.Unmarshal([]byte(strings.TrimPrefix(line, "data: ")), &ev); err != nil {
t.Fatalf("not JSON: %s", line)
}
types = append(types, ev["type"].(string))
if ev["type"] == "response.completed" {
completed = ev["response"].(map[string]any)
}
if ev["type"] == "response.function_call_arguments.done" {
args = ev["arguments"].(string)
}
}
}
want := []string{"response.created", "response.in_progress", "response.output_item.added", "response.content_part.added",
"response.output_text.delta", "response.output_text.delta", "response.output_item.added",
"response.function_call_arguments.delta", "response.function_call_arguments.delta",
"response.output_text.done", "response.content_part.done", "response.output_item.done",
"response.function_call_arguments.done", "response.output_item.done", "response.completed"}
if strings.Join(types, " ") != strings.Join(want, " ") {
t.Errorf("events:\n got %v\nwant %v", types, want)
}
if args != `{"command":["ls"]}` {
t.Errorf("arguments = %q", args)
}
out, _ := completed["output"].([]any)
if len(out) != 2 {
t.Fatalf("output = %v", completed["output"])
}
msg, call := out[0].(map[string]any), out[1].(map[string]any)
if msg["type"] != "message" || call["type"] != "function_call" || call["name"] != "shell" || call["call_id"] != "call_1" {
t.Errorf("output = %v", out)
}
if u := completed["usage"].(map[string]any); u["input_tokens"].(float64) != 20 || u["output_tokens"].(float64) != 9 {
t.Errorf("usage = %v", u)
}
}
func TestAResponsesRequestWithoutAStreamGetsAResponseObject(t *testing.T) {
srv := streamingEngine(t, "coder")
rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: srv.URL, Models: []string{"coder"}}))
waitForEndpoint(t, rt)
req := httptest.NewRequest("POST", "/v1/responses", strings.NewReader(`{"model":"coder","input":"hi"}`))
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
rt.Handler().ServeHTTP(w, req)
var got struct {
Object string `json:"object"`
Status string `json:"status"`
Output []struct {
Type string `json:"type"`
Name string `json:"name"`
Content []struct {
Text string `json:"text"`
} `json:"content"`
} `json:"output"`
Usage struct {
Input int `json:"input_tokens"`
Output int `json:"output_tokens"`
} `json:"usage"`
}
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
t.Fatalf("%v: %s", err, w.Body.String())
}
if got.Object != "response" || got.Status != "completed" || len(got.Output) != 2 ||
got.Output[0].Content[0].Text != "Hello." || got.Output[1].Name != "shell" || got.Usage.Input != 11 {
t.Errorf("%s", w.Body.String())
}
}
+12
View File
@@ -127,6 +127,8 @@ func (p *AccountPages) Register(mux *http.ServeMux) {
mux.HandleFunc("POST /profile/keys", p.createKey)
mux.HandleFunc("POST /profile/keys/{id}/revoke", p.revokeKey)
mux.HandleFunc("POST /profile/password", p.changePassword)
mux.HandleFunc("POST /profile/connect", p.connect)
mux.HandleFunc("GET /setup/{tool}", p.setupScript)
}
// --- signup ---
@@ -258,6 +260,12 @@ type profileData struct {
Keys []store.Key
Limits []limitRow
NewKey string
// Connect is an editor just connected: its key and configuration, shown
// once, like any new key.
Connect *connectView
// Tools and Models are what the connect form offers.
Tools []struct{ Key, Name string }
Models []string
Error string
PasswordChanged bool
@@ -366,6 +374,10 @@ func (p *AccountPages) render(w http.ResponseWriter, r *http.Request,
d.MinPassword = account.MinPasswordLen
d.BaseURL = baseURL(r)
d.ExampleModel = p.exampleModel()
d.Tools = editorTools
if p.served != nil {
d.Models = p.served()
}
d.Version = p.version
d.Now = time.Now().UTC().Format(time.RFC3339)
// The form token comes from the cookie, never from whatever the request
+235
View File
@@ -0,0 +1,235 @@
package web
import (
"fmt"
"net/http"
"regexp"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// Connecting an editor: a key made for it, and the configuration it needs
// with that key, this gateway's address and a served model already in it.
//
// A page cannot write files on somebody's machine, so what it can do is hand
// over the file, ready — to copy, to download, or to have a one-line command
// write. The command's script is served without authentication and holds no
// secret: the key travels in an environment variable, never in a URL, where
// it would land in shell history and proxy logs.
// editorTools are the tools the page knows how to configure.
var editorTools = []struct{ Key, Name string }{
{"continue", "Continue"}, {"codex", "Codex CLI"}, {"cline", "Cline"},
}
// connectView is what the page shows after a key was made for a tool.
type connectView struct {
Tool, ToolName, Model string
Key string
// Files are what the tool reads, with the key in them.
Files []connectFile
// Fields are for a tool that keeps its settings where no file reaches:
// Cline's live in VS Code's secret storage, set through its own screen.
Fields []connectField
// Command writes the files; Run is how to start the tool on the gateway.
Command, Run string
Notes []string
}
type connectFile struct{ Path, Name, Body string }
type connectField struct{ Label, Value string }
// safeModel is what a model name may be to go into a script and a file
// unquoted: the names this gateway serves, and nothing a shell reads twice.
var safeModel = regexp.MustCompile(`^[A-Za-z0-9._:/@+-]{1,200}$`)
// continueYAML is a Continue assistant pointing at the gateway: a file of its
// own under ~/.continue/assistants, so the person's config.yaml is not touched.
func continueYAML(base, model, key string) string {
return fmt.Sprintf(`# Written by llmgw for Continue: the gateway at %[1]s.
name: llmgw
version: 1.0.0
schema: v1
models:
- name: %[2]s (llmgw)
provider: openai
model: %[2]s
apiBase: %[1]s/v1
apiKey: %[3]s
roles:
- chat
- edit
- apply
capabilities:
- tool_use
# For completion as you type, add a small, fast model here with the role
# autocomplete; a large one makes every keystroke wait for it.
`, base, model, key)
}
// codexTOML is the provider and profile Codex needs, and nothing else of its
// configuration: codex --profile llmgw uses the gateway, plain codex what it
// used before. The key is read from LLMGW_API_KEY rather than written down.
func codexTOML(base, model string) string {
return fmt.Sprintf(`# Written by llmgw for Codex: the gateway at %[1]s.
[model_providers.llmgw]
name = "llmgw"
base_url = "%[1]s/v1"
env_key = "LLMGW_API_KEY"
wire_api = "responses"
[profiles.llmgw]
model_provider = "llmgw"
model = "%[2]s"
`, base, model)
}
// connectFor builds what the page shows for a tool.
func connectFor(tool, base, model, key string) connectView {
v := connectView{Tool: tool, Model: model, Key: key}
for _, t := range editorTools {
if t.Key == tool {
v.ToolName = t.Name
}
}
script := base + "/setup/" + tool + "?model=" + model
switch tool {
case "continue":
v.Files = []connectFile{{Path: "~/.continue/assistants/llmgw.yaml", Name: "llmgw.yaml",
Body: continueYAML(base, model, key)}}
v.Command = fmt.Sprintf("curl -fsSL '%s' | LLMGW_KEY='%s' sh", script, key)
v.Run = `In Continue, choose the "llmgw" assistant at the top of its panel.`
case "codex":
v.Files = []connectFile{{Path: "~/.codex/config.toml", Name: "config.toml", Body: codexTOML(base, model)}}
v.Command = fmt.Sprintf("curl -fsSL '%s' | sh", script)
v.Run = fmt.Sprintf("LLMGW_API_KEY='%s' codex --profile llmgw", key)
v.Notes = []string{
"The command adds the two blocks above to ~/.codex/config.toml, replacing any it added before, and keeps a copy of the file as it was.",
"Put export LLMGW_API_KEY=… in your shell's profile to leave it out of the command line.",
}
case "cline":
v.Fields = []connectField{
{"API Provider", "OpenAI Compatible"},
{"Base URL", base + "/v1"},
{"API Key", key},
{"Model ID", model},
}
v.Run = "Cline keeps these in VS Code's own secret storage, so they are set in its settings: open Cline, the gear, and fill them in."
}
return v
}
// connect makes a key for an editor and shows its configuration.
func (p *AccountPages) connect(w http.ResponseWriter, r *http.Request) {
noStore(w)
sess, user, ok := p.session(r)
if !ok {
http.Redirect(w, r, "/login", http.StatusSeeOther)
return
}
if !p.accounts.CheckCSRF(sess, r.FormValue("csrf")) {
http.Error(w, "bad request", http.StatusBadRequest)
return
}
tool := r.FormValue("tool")
known := false
for _, t := range editorTools {
known = known || t.Key == tool
}
model := strings.TrimSpace(r.FormValue("model"))
if !known || !safeModel.MatchString(model) {
p.renderProfile(w, r, sess, user, "", "choose a tool and a model")
return
}
project := r.FormValue("context")
if project == personalValue {
if p.personalOff {
p.renderProfile(w, r, sess, user, "", "personal keys are off on this gateway; choose a project")
return
}
project = ""
}
name := tool + " " + time.Now().Format("2006-01-02")
secret, k, err := store.NewKey(randomKeyID(), name, project)
if err != nil {
p.renderProfile(w, r, sess, user, "", "something went wrong; try again")
return
}
k.CreatedAt = time.Now()
if err := p.keys.CreateUserKey(r.Context(), user.ID, k); err != nil {
msg := "something went wrong; try again"
if err == store.ErrNotMember {
msg = "you are not a member of that project"
}
p.renderProfile(w, r, sess, user, "", msg)
return
}
p.log.Info("a user connected an editor", "user", user.ID, "tool", tool, "key", k.ID)
c := connectFor(tool, baseURL(r), model, secret)
p.render(w, r, sess, user, profileData{Connect: &c})
}
// setupScript is the script the one-line command runs: it writes the tool's
// configuration with the key from the environment, keeping a copy of what
// was there. Served without a session — it holds nothing a stranger could
// use: this gateway's address, which they already typed, and a model name.
func (p *AccountPages) setupScript(w http.ResponseWriter, r *http.Request) {
tool := r.PathValue("tool")
model := r.URL.Query().Get("model")
if !safeModel.MatchString(model) {
http.Error(w, "which model? add ?model=<name>", http.StatusBadRequest)
return
}
base := baseURL(r)
var body string
switch tool {
case "continue":
body = fmt.Sprintf(`#!/bin/sh
# Connects Continue to the llmgw gateway at %[1]s.
set -eu
: "${LLMGW_KEY:?set LLMGW_KEY to the key the gateway gave you}"
dir="$HOME/.continue/assistants"
file="$dir/llmgw.yaml"
mkdir -p "$dir"
[ -f "$file" ] && cp "$file" "$file.bak.$(date +%%Y%%m%%d%%H%%M%%S)"
umask 077
cat > "$file" <<'LLMGW'
%[2]sLLMGW
# The key goes in last, from the environment: never through a URL.
tmp="$file.tmp"
sed "s|__LLMGW_KEY__|$LLMGW_KEY|" "$file" > "$tmp" && mv "$tmp" "$file"
echo "Wrote $file. In Continue, choose the llmgw assistant."
`, base, continueYAML(base, model, "__LLMGW_KEY__"))
case "codex":
body = fmt.Sprintf(`#!/bin/sh
# Connects Codex to the llmgw gateway at %[1]s.
set -eu
dir="$HOME/.codex"
file="$dir/config.toml"
mkdir -p "$dir"
touch "$file"
cp "$file" "$file.bak.$(date +%%Y%%m%%d%%H%%M%%S)"
# Out with the blocks this wrote before, in with the new: every other line
# of the file stays as it was.
awk '
/^\[/ { skip = ($0 == "[model_providers.llmgw]" || $0 == "[profiles.llmgw]" || $0 ~ /^# Written by llmgw/) }
/^# Written by llmgw/ { skip = 1; next }
!skip { print }
' "$file" > "$file.tmp"
cat >> "$file.tmp" <<'LLMGW'
%[2]sLLMGW
mv "$file.tmp" "$file"
echo "Added the llmgw provider to $file."
echo "Run: LLMGW_API_KEY=<your key> codex --profile llmgw"
`, base, codexTOML(base, model))
default:
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", "text/x-shellscript; charset=utf-8")
w.Header().Set("Cache-Control", "no-store")
_, _ = w.Write([]byte(body))
}
+117
View File
@@ -0,0 +1,117 @@
package web
import (
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
)
func TestEachToolGetsItsConfigurationWithTheKey(t *testing.T) {
c := connectFor("continue", "https://gw.example", "qwen2.5-coder-7b", "sk-llmgw-abc")
if !strings.Contains(c.Files[0].Body, "apiBase: https://gw.example/v1") || !strings.Contains(c.Files[0].Body, "apiKey: sk-llmgw-abc") ||
c.Files[0].Path != "~/.continue/assistants/llmgw.yaml" {
t.Errorf("continue = %+v", c.Files)
}
// The key goes in the environment, not the URL, where it would be kept in
// shell history and proxy logs.
if strings.Contains(strings.Split(c.Command, "|")[0], "sk-llmgw-abc") {
t.Errorf("the key is in the URL: %s", c.Command)
}
x := connectFor("codex", "https://gw.example", "qwen2.5-coder-7b", "sk-llmgw-abc")
if strings.Contains(x.Files[0].Body, "sk-llmgw-abc") || !strings.Contains(x.Files[0].Body, `env_key = "LLMGW_API_KEY"`) {
t.Errorf("codex writes the key into its file: %s", x.Files[0].Body)
}
cl := connectFor("cline", "https://gw.example", "m", "sk-llmgw-abc")
if len(cl.Fields) != 4 || cl.Fields[1].Value != "https://gw.example/v1" || cl.Command != "" {
t.Errorf("cline = %+v", cl)
}
}
// runScript fetches a setup script and runs it with HOME in a temporary
// directory, as a person's shell would.
func runScript(t *testing.T, tool string, env ...string) string {
t.Helper()
p := &AccountPages{}
w := httptest.NewRecorder()
r := httptest.NewRequest("GET", "http://gw.example/setup/"+tool+"?model=qwen2.5-coder-7b", nil)
r.SetPathValue("tool", tool)
p.setupScript(w, r)
if w.Code != http.StatusOK {
t.Fatalf("%d %s", w.Code, w.Body.String())
}
home := t.TempDir()
cmd := exec.Command("sh")
cmd.Stdin = strings.NewReader(w.Body.String())
cmd.Env = append([]string{"HOME=" + home, "PATH=" + os.Getenv("PATH")}, env...)
if out, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("%v: %s", err, out)
}
return home
}
func TestTheContinueScriptWritesItsOwnFileWithTheKey(t *testing.T) {
home := runScript(t, "continue", "LLMGW_KEY=sk-llmgw-xyz")
b, err := os.ReadFile(filepath.Join(home, ".continue", "assistants", "llmgw.yaml"))
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(b), "apiKey: sk-llmgw-xyz") || strings.Contains(string(b), "__LLMGW_KEY__") {
t.Errorf("wrote %s", b)
}
if fi, _ := os.Stat(filepath.Join(home, ".continue", "assistants", "llmgw.yaml")); fi.Mode().Perm()&0o077 != 0 {
t.Errorf("a file holding a key is readable by others: %v", fi.Mode())
}
}
func TestTheCodexScriptKeepsTheRestOfTheConfig(t *testing.T) {
// Run twice over a config with the person's own settings: theirs stay,
// and ours appear once.
p := &AccountPages{}
w := httptest.NewRecorder()
r := httptest.NewRequest("GET", "http://gw.example/setup/codex?model=qwen2.5-coder-7b", nil)
r.SetPathValue("tool", "codex")
p.setupScript(w, r)
home := t.TempDir()
if err := os.MkdirAll(filepath.Join(home, ".codex"), 0o755); err != nil {
t.Fatal(err)
}
mine := "model = \"o3\"\n\n[mcp_servers.docs]\ncommand = \"docs-mcp\"\n"
cfg := filepath.Join(home, ".codex", "config.toml")
if err := os.WriteFile(cfg, []byte(mine), 0o644); err != nil {
t.Fatal(err)
}
for range 2 {
cmd := exec.Command("sh")
cmd.Stdin = strings.NewReader(w.Body.String())
cmd.Env = []string{"HOME=" + home, "PATH=" + os.Getenv("PATH")}
if out, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("%v: %s", err, out)
}
}
b, _ := os.ReadFile(cfg)
got := string(b)
if !strings.Contains(got, `model = "o3"`) || !strings.Contains(got, "[mcp_servers.docs]") {
t.Errorf("the person's settings were lost:\n%s", got)
}
if strings.Count(got, "[model_providers.llmgw]") != 1 || strings.Count(got, "[profiles.llmgw]") != 1 {
t.Errorf("ours should be there once:\n%s", got)
}
if backups, _ := filepath.Glob(cfg + ".bak.*"); len(backups) == 0 {
t.Error("no copy was kept of the file as it was")
}
}
func TestASetupScriptRefusesANameAShellWouldRead(t *testing.T) {
p := &AccountPages{}
w := httptest.NewRecorder()
r := httptest.NewRequest("GET", "http://gw.example/setup/codex?model=$(rm%20-rf%20~)", nil)
r.SetPathValue("tool", "codex")
p.setupScript(w, r)
if w.Code != http.StatusBadRequest {
t.Errorf("%d", w.Code)
}
}
+9
View File
@@ -795,3 +795,12 @@ details.chat-settings .temp input[type=range] { width: 14rem; vertical-align: mi
details.chat-settings output { font-variant-numeric: tabular-nums; font-weight: 700; margin-left: .3rem; }
#send.stop { background: var(--bad); border-color: var(--bad); }
.chat-stats { margin: -.2rem 0 .6rem; font-size: .72rem; font-variant-numeric: tabular-nums; }
/* Connecting an editor. */
.connect-result { border: 1px solid var(--accent); border-radius: 12px; padding: .25rem 1.1rem 1rem; margin: 1rem 0 1.5rem;
background: var(--card); box-shadow: 0 10px 30px -18px var(--accent); }
.copyable { position: relative; }
.copyable pre { margin-bottom: .25rem; overflow-x: auto; }
.copyable button { margin-right: .75rem; font-size: .8rem; }
.connect-form .field-row label { flex: 1 1 12rem; }
table.connect-fields th { white-space: nowrap; }
+63
View File
@@ -68,6 +68,69 @@ Tell whoever you asked for access that you have signed up.</p>
-d '{"model": "{{.ExampleModel}}", "messages": [{"role":"user","content":"hello"}]}'</pre>
{{end}}
{{with .Connect}}
<section class="connect-result" id="connected">
<h2>{{.ToolName}} is ready to connect</h2>
<p>A key was made for it, <strong>shown this once</strong> — it is in everything below. Nothing can produce it again.</p>
{{if .Command}}
<h3>One command</h3>
<p class="note">Writes the file below{{if eq .Tool "codex"}} into your Codex configuration{{end}}, keeping a copy of anything it replaces.</p>
<div class="copyable"><pre>{{.Command}}</pre><button type="button" class="link copy">copy</button></div>
{{end}}
{{range .Files}}
<h3>Or by hand: <code>{{.Path}}</code></h3>
<div class="copyable"><pre>{{.Body}}</pre><button type="button" class="link copy">copy</button>
<button type="button" class="link download-file" data-name="{{.Name}}">download</button></div>
{{end}}
{{if .Fields}}
<h3>Settings to fill in</h3>
<table class="compact narrow-table connect-fields">{{range .Fields}}
<tr><th>{{.Label}}</th><td><code>{{.Value}}</code></td><td><button type="button" class="link copy" data-value="{{.Value}}">copy</button></td></tr>{{end}}
</table>
{{end}}
<h3>Then</h3>
<p>{{if eq .Tool "codex"}}<code>{{.Run}}</code>{{else}}{{.Run}}{{end}}</p>
{{range .Notes}}<p class="note">{{.}}</p>{{end}}
</section>
{{end}}
<h2>Connect your editor</h2>
<p class="note">A key of its own for the tool, and its configuration with the key, this gateway and a model
already in it. Revoke the key below to disconnect it.</p>
<form method="post" action="/profile/connect#connected" class="connect-form">
<input type="hidden" name="csrf" value="{{.CSRF}}">
<div class="field-row">
<label>Tool
<select name="tool">{{range .Tools}}<option value="{{.Key}}">{{.Name}}</option>{{end}}</select></label>
<label>Model
{{if .Models}}<select name="model">{{range .Models}}<option value="{{.}}">{{.}}</option>{{end}}</select>
{{else}}<input type="text" name="model" placeholder="a model the gateway serves" required>{{end}}</label>
{{if .Contexts}}<label>Spends against
<select name="context">{{range .Contexts}}<option value="{{.Value}}">{{.Label}}</option>{{end}}</select></label>{{end}}
</div>
<button type="submit">Make a key and the configuration</button>
</form>
<script>
// Copy and download for what the connect panel shows, done in the page: the
// key in it must not go back to the server to become a file.
document.addEventListener("click", (e) => {
const b = e.target.closest("button.copy, button.download-file");
if (!b) return;
const text = b.dataset.value || b.parentElement.querySelector("pre").textContent;
if (b.classList.contains("copy")) {
const done = () => { const was = b.textContent; b.textContent = "copied"; setTimeout(() => (b.textContent = was), 1500); };
if (navigator.clipboard) navigator.clipboard.writeText(text).then(done, () => window.prompt("Copy", text));
else window.prompt("Copy", text);
return;
}
const url = URL.createObjectURL(new Blob([text], { type: "text/plain" }));
const a = document.createElement("a");
a.href = url; a.download = b.dataset.name || "config";
a.click();
URL.revokeObjectURL(url);
});
</script>
<h2>Create a key</h2>
<form method="post" action="/profile/keys" class="narrow">
<input type="hidden" name="csrf" value="{{.CSRF}}">