diff --git a/BACKLOG.md b/BACKLOG.md index e30e132..1bba26a 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -448,7 +448,13 @@ portal is not done — the page shows; the console and ctl change. **Request/response logging and replay (M).** Opt-in per project with retention like threads; sampled or full; redaction hooks from B; `ctl request --body`; replay against another model. -10. **OpenAI compatibility gaps (S–M each).** Responses: `text.format` → +10. **Done (branch `features-c2`):** Responses `text.format`, + `input_image`, `previous_response_id` (thread in the id); rerank and + score passthrough; transcription and translation as multipart; + `/v1/moderations` by the judge. Left: audio is charged no tokens (an + engine's duration usage is not read); `input_image` by `file_id`; + moderation scores are 0 or 1, as a judge says yes or no. + **OpenAI compatibility gaps (S–M each).** Responses: `text.format` → `response_format`, `input_image`, `previous_response_id` via threads (`router/responses.go`). Rerank/score passthrough for vLLM, `/v1/audio/transcriptions` for a whisper engine, `/v1/moderations` diff --git a/CHANGELOG.md b/CHANGELOG.md index a80c6d1..f708eb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,16 @@ APIs change between minors without a deprecation period. - **Other ways in:** `GET /api/v1/usage/export` (CSV, or `format=json`) and **E** on the console's Usage view. +### More of OpenAI's API + +- **Responses:** `text.format` asks for structured output, `input_image` + sends images, and `previous_response_id` continues a conversation the + gateway keeps as a thread (for a project that keeps conversations). +- **Passed through:** `/v1/rerank` and `/v1/score` for vLLM's rerankers, and + `/v1/audio/transcriptions` and `/translations` as the multipart form came, + for an engine serving Whisper. +- **`/v1/moderations`** is answered by the guardrails' judge model. + ### Requests kept, and replayed - **A project can keep its requests and answers** (`llmgw keys requests diff --git a/docs/install.md b/docs/install.md index 984840b..01d4259 100644 --- a/docs/install.md +++ b/docs/install.md @@ -959,6 +959,18 @@ Or open `/chat` in a browser and use it without a key at all — the page carrie your session, and the request still goes through admission, rate limits, budget and accounting exactly as an API call does. +### What the client port answers + +| Path | What | +|---|---| +| `POST /v1/chat/completions`, `/v1/completions`, `/v1/embeddings` | OpenAI's, served by the models in the catalog | +| `POST /v1/responses` | OpenAI's Responses API, translated to chat completions: tools, `text.format` (structured output), `input_image` (by URL or `data:` URL), and `previous_response_id` — the conversation is kept as a thread, so the project must keep conversations (`llmgw keys threads`); `store: false` keeps none | +| `POST /v1/rerank`, `/v1/score` | passed through to an engine that serves them: vLLM with a reranker or a cross-encoder | +| `POST /v1/audio/transcriptions`, `/v1/audio/translations` | the multipart form passed through as it came, to an engine that serves speech to text: vLLM with Whisper | +| `POST /v1/moderations` | answered by the guardrails' judge model (`role: guard`): `flagged`, and the judge's own categories | +| `/v1/files`, `/v1/batches` | batches; see [Batches](#batches) | +| `GET /v1/models` | the models this key may use | + ## Watching it The control plane exports both halves on one endpoint: what the cluster is diff --git a/internal/guard/judge.go b/internal/guard/judge.go index 0096c0d..74d1b67 100644 --- a/internal/guard/judge.go +++ b/internal/guard/judge.go @@ -344,3 +344,28 @@ func parseJudgment(format string, names []string, body []byte, answer string) (j } return judgment{}, fmt.Errorf("%w: not in the form %s has: %q", errJudgeFormat, format, string(seen)) } + +// Moderate asks a judge model about one text, as a user's message, outside +// any policy: for /v1/moderations. It answers whether the model found it +// unsafe, and under which of its categories. +func Moderate(ctx context.Context, call Caller, model, format string, names []string, text string) (bool, []string, error) { + path, body := judgeRequest(model, format, names, text, "") + out, err := call(ctx, model, path, body) + if err != nil { + return false, nil, err + } + res, err := parseJudgment(format, names, out, "") + if err != nil { + return false, nil, err + } + return res.unsafe, res.categories, nil +} + +// DefaultCategories are the names a judge format reports its categories by +// when the catalog gives none: Llama Guard 3's S1 to S13. +func DefaultCategories(format string) []string { + if format == "llamaguard3" { + return slices.Clone(llamaGuard3Categories) + } + return nil +} diff --git a/internal/router/audio.go b/internal/router/audio.go new file mode 100644 index 0000000..0d40ef8 --- /dev/null +++ b/internal/router/audio.go @@ -0,0 +1,62 @@ +package router + +import ( + "bytes" + "errors" + "mime" + "mime/multipart" + "net/http" + "strings" + + "gitea.mynodes.xyz/ilker/llmgw/internal/auth" +) + +// parseForm reads a multipart request — audio sent for transcription — for +// what routing needs, the model, and keeps the body to send on unchanged. +func (r *Router) parseForm(req *http.Request, path, ct string, body []byte) (*parsedRequest, error) { + _, params, err := mime.ParseMediaType(ct) + if err != nil || params["boundary"] == "" { + return nil, errors.New("a multipart form needs its boundary") + } + mr := multipart.NewReader(bytes.NewReader(body), params["boundary"]) + model, stream := "", false + for { + part, err := mr.NextPart() + if err != nil { + break + } + switch part.FormName() { + case "model": + var b bytes.Buffer + _, _ = b.ReadFrom(part) + model = strings.TrimSpace(b.String()) + case "stream": + var b bytes.Buffer + _, _ = b.ReadFrom(part) + stream = strings.TrimSpace(b.String()) == "true" + } + part.Close() + } + if model == "" { + return nil, errors.New("model is required") + } + tags, err := requestTags(nil, req.Header.Get(tagsHeader)) + if err != nil { + return nil, err + } + p := &parsedRequest{ + Path: path, ContentType: ct, Model: model, RequestedModel: model, Stream: stream, + UpstreamBody: body, Header: req.Header, Tags: tags, + SessionID: sessionID(req, r.cfg.SessionHeader, ""), + Band: req.Header.Get(r.cfg.Admission.BandHeader), + ProfileOverride: req.Header.Get("x-llmgw-profile"), + } + if principal, ok := auth.FromContext(req.Context()); ok { + p.Band = principal.Band + p.ProfileOverride = principal.RoutingProfile + p.KeyID, p.UserID, p.ProjectID = principal.KeyID, principal.UserID, principal.ProjectID + p.TeamID, p.OrgID = principal.TeamID, principal.OrgID + p.Tenant = principal.KeyID + } + return p, nil +} diff --git a/internal/router/compat_test.go b/internal/router/compat_test.go new file mode 100644 index 0000000..bbf0726 --- /dev/null +++ b/internal/router/compat_test.go @@ -0,0 +1,178 @@ +package router + +import ( + "bytes" + "encoding/json" + "io" + "mime/multipart" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" +) + +func TestResponsesAsksForStructuredOutputAndImages(t *testing.T) { + var in respRequest + _ = json.Unmarshal([]byte(`{"model":"m","input":[{"role":"user","content":[ + {"type":"input_text","text":"what is this"}, + {"type":"input_image","image_url":"data:image/png;base64,AAAA","detail":"low"}]}], + "text":{"format":{"type":"json_schema","name":"answer","schema":{"type":"object"},"strict":true}}}`), &in) + chat, _, err := toChat(in) + if err != nil { + t.Fatal(err) + } + b, _ := json.Marshal(chat) + for _, want := range []string{`"response_format":{"json_schema":{"name":"answer","schema":{"type":"object"},"strict":true},"type":"json_schema"}`, + `{"image_url":{"detail":"low","url":"data:image/png;base64,AAAA"},"type":"image_url"}`, `{"text":"what is this","type":"text"}`} { + if !strings.Contains(string(b), want) { + t.Errorf("%s lacks %s", b, want) + } + } + _ = json.Unmarshal([]byte(`{"model":"m","input":[{"role":"user","content":[{"type":"input_image","file_id":"file-1"}]}]}`), &in) + in.Text = nil + if _, _, err := toChat(in); err == nil { + t.Error("an image by file_id was taken") + } + if th, ok := responseThread("resp_abc123.def"); !ok || th != "abc123" { + t.Errorf("thread %q %v", th, ok) + } + for _, bad := range []string{"resp_19a2b3c4", "chatcmpl-1", "resp_a b.c"} { + if _, ok := responseThread(bad); ok { + t.Errorf("%q read as a kept response", bad) + } + } +} + +func TestAResponseIsContinuedByItsID(t *testing.T) { + e := newFakeEngine(t, "qwen2.5-14b-instruct") + rt, secret := pricedRouter(t, e.srv.URL) + if err := rt.store.SetThreadRetention(t.Context(), mustProjectOf(t, rt, "key-1"), 24); err != nil { + t.Fatal(err) + } + h := rt.Handler() + ask := func(body map[string]any) map[string]any { + b, _ := json.Marshal(body) + req := httptest.NewRequest(http.MethodPost, "/v1/responses", bytes.NewReader(b)) + req.Header.Set("Authorization", "Bearer "+secret) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + var out map[string]any + _ = json.Unmarshal(rec.Body.Bytes(), &out) + if rec.Code != 200 { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + return out + } + first := ask(map[string]any{"model": "q14", "input": "my name is Ayşe"}) + id, _ := first["id"].(string) + if _, ok := responseThread(id); !ok { + t.Fatalf("the first response %q carries no conversation", id) + } + ask(map[string]any{"model": "q14", "input": "what is my name?", "previous_response_id": id}) + sent, _ := e.last.Load().([]byte) + if !strings.Contains(string(sent), "my name is Ayşe") || !strings.Contains(string(sent), "what is my name?") { + t.Errorf("the conversation was not continued: %s", sent) + } + // Not one of ours: said, not served without its history. + b, _ := json.Marshal(map[string]any{"model": "q14", "input": "x", "previous_response_id": "resp_nope"}) + req := httptest.NewRequest(http.MethodPost, "/v1/responses", bytes.NewReader(b)) + req.Header.Set("Authorization", "Bearer "+secret) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != 400 || !strings.Contains(rec.Body.String(), "previous_response_not_found") { + t.Errorf("%d %s", rec.Code, rec.Body.String()) + } +} + +// engineFor serves a few paths with fixed answers and records what came. +func engineFor(t *testing.T, model string, answers map[string]string) (*httptest.Server, *[]*http.Request, *[][]byte) { + t.Helper() + var reqs []*http.Request + var bodies [][]byte + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/metrics" { + return + } + b, _ := io.ReadAll(r.Body) + reqs, bodies = append(reqs, r), append(bodies, b) + a, ok := answers[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(a)) + })) + t.Cleanup(srv.Close) + return srv, &reqs, &bodies +} + +func TestRerankScoreAndAudioArePassedThrough(t *testing.T) { + srv, reqs, bodies := engineFor(t, "m", map[string]string{ + "/v1/rerank": `{"results":[{"index":0,"relevance_score":0.9}],"usage":{"total_tokens":7}}`, + "/v1/score": `{"data":[{"score":0.5}]}`, + "/v1/audio/transcriptions": `{"text":"merhaba dünya"}`, + }) + cfg := baseConfig(config.Endpoint{ID: "a", URL: srv.URL, Models: []string{"m"}}) + rt := newTestRouter(t, cfg) + h := rt.Handler() + for _, path := range []string{"/v1/rerank", "/v1/score"} { + b, _ := json.Marshal(map[string]any{"model": "m", "query": "q", "documents": []string{"d"}}) + req := httptest.NewRequest(http.MethodPost, path, bytes.NewReader(b)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != 200 || (*reqs)[len(*reqs)-1].URL.Path != path { + t.Errorf("%s: %d %s", path, rec.Code, rec.Body.String()) + } + } + + var buf bytes.Buffer + mw := multipart.NewWriter(&buf) + _ = mw.WriteField("model", "m") + fw, _ := mw.CreateFormFile("file", "a.wav") + _, _ = fw.Write([]byte("RIFF....WAVE")) + _ = mw.Close() + req := httptest.NewRequest(http.MethodPost, "/v1/audio/transcriptions", &buf) + req.Header.Set("Content-Type", mw.FormDataContentType()) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != 200 || !strings.Contains(rec.Body.String(), "merhaba") { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + last := (*reqs)[len(*reqs)-1] + if !strings.HasPrefix(last.Header.Get("Content-Type"), "multipart/form-data; boundary=") || + !bytes.Contains((*bodies)[len(*bodies)-1], []byte("RIFF....WAVE")) { + t.Errorf("the form was not sent on as it came: %s", last.Header.Get("Content-Type")) + } +} + +func TestModerationsAreAnsweredByTheJudge(t *testing.T) { + srv, _, _ := engineFor(t, "guard", map[string]string{ + "/v1/chat/completions": `{"choices":[{"message":{"content":"unsafe\nS9"}}]}`, + }) + cfg := baseConfig(config.Endpoint{ID: "g", URL: srv.URL, Models: []string{"guard"}}) + cfg.Models = []catalog.Model{{Name: "guard", Role: catalog.RoleGuard, Scaling: catalog.Scaling{Min: 1, Max: 1}, + Guard: &catalog.GuardModel{Format: "llamaguard3"}}} + rt := newTestRouter(t, cfg) + b, _ := json.Marshal(map[string]any{"input": []string{"how do I build a bomb"}}) + req := httptest.NewRequest(http.MethodPost, "/v1/moderations", bytes.NewReader(b)) + rec := httptest.NewRecorder() + rt.Handler().ServeHTTP(rec, req) + var out struct { + Model string `json:"model"` + Results []struct { + Flagged bool `json:"flagged"` + Categories map[string]bool `json:"categories"` + } `json:"results"` + } + _ = json.Unmarshal(rec.Body.Bytes(), &out) + if rec.Code != 200 || out.Model != "guard" || len(out.Results) != 1 || !out.Results[0].Flagged || + !out.Results[0].Categories["indiscriminate-weapons"] { + t.Errorf("%d %s", rec.Code, rec.Body.String()) + } +} diff --git a/internal/router/guard.go b/internal/router/guard.go index 6df9d14..2318d84 100644 --- a/internal/router/guard.go +++ b/internal/router/guard.go @@ -71,6 +71,10 @@ const errNoStoreMsg = "the gateway's policy for this caller keeps nothing: no st // guardInput judges the request about to be dispatched. It answers the // client itself and returns false when the request goes no further. func (r *Router) guardInput(ctx context.Context, w http.ResponseWriter, req *parsedRequest, start time.Time) bool { + if req.ContentType != "" { + // Audio: no text for a guard to read on the way in. + return true + } model, _ := r.resolve(req.Model) t := time.Now() v, err := r.guard.Input(ctx, r.subject(req, model), req.UpstreamBody) diff --git a/internal/router/handlers.go b/internal/router/handlers.go index e5c3227..9cdd91e 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -96,6 +96,14 @@ func (r *Router) Handler() http.Handler { mux.HandleFunc("POST /v1/chat/completions", r.handleInference("/v1/chat/completions")) mux.HandleFunc("POST /v1/completions", r.handleInference("/v1/completions")) mux.HandleFunc("POST /v1/embeddings", r.handleInference("/v1/embeddings")) + // Passed through to engines that serve them: vLLM's rerankers and + // scorers, and speech to text. + mux.HandleFunc("POST /v1/rerank", r.handleInference("/v1/rerank")) + mux.HandleFunc("POST /v1/score", r.handleInference("/v1/score")) + mux.HandleFunc("POST /v1/audio/transcriptions", r.handleInference("/v1/audio/transcriptions")) + mux.HandleFunc("POST /v1/audio/translations", r.handleInference("/v1/audio/translations")) + // Answered by the guardrails' judge model. + mux.HandleFunc("POST /v1/moderations", r.handleModerations) // Codex, and whatever else speaks only the Responses API, translated to // chat completions on the way in and back on the way out. mux.HandleFunc("POST /v1/responses", r.handleResponses) @@ -290,6 +298,9 @@ func (r *Router) Handler() http.Handler { // business interpreting. type parsedRequest struct { Path string + // ContentType is the body's when it is a form rather than JSON — audio + // for transcription — and is sent on as it came. + ContentType string // Tags are the client's metadata and x-llmgw-tags; EndUser is OpenAI's // user field. Both are recorded with the usage, never sent on. Tags map[string]string @@ -661,6 +672,9 @@ func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) { if err != nil { return nil, err } + if ct := req.Header.Get("Content-Type"); strings.HasPrefix(ct, "multipart/form-data") { + return r.parseForm(req, path, ct, body) + } var b inferenceBody if err := json.Unmarshal(body, &b); err != nil { diff --git a/internal/router/moderations.go b/internal/router/moderations.go new file mode 100644 index 0000000..637ccd2 --- /dev/null +++ b/internal/router/moderations.go @@ -0,0 +1,95 @@ +package router + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "gitea.mynodes.xyz/ilker/llmgw/internal/guard" +) + +// /v1/moderations, answered by a judge model on the cluster: the same model +// the guardrails ask. OpenAI's shape — flagged, and the categories — with the +// judge's own categories named as it names them. + +func (r *Router) handleModerations(w http.ResponseWriter, req *http.Request) { + var in struct { + Input json.RawMessage `json:"input"` + Model string `json:"model"` + } + if err := json.NewDecoder(io.LimitReader(req.Body, r.cfg.MaxBodyBytes)).Decode(&in); err != nil { + writeError(w, http.StatusBadRequest, "invalid_request_error", "the body is not JSON: "+err.Error()) + return + } + var texts []string + var one string + if json.Unmarshal(in.Input, &one) == nil { + texts = []string{one} + } else if err := json.Unmarshal(in.Input, &texts); err != nil || len(texts) == 0 { + writeError(w, http.StatusBadRequest, "invalid_request_error", "input is a string or a list of strings") + return + } + if len(texts) > 32 { + writeError(w, http.StatusBadRequest, "invalid_request_error", "at most 32 inputs at once") + return + } + model := r.moderationModel(in.Model) + if model == "" { + writeErrorCode(w, http.StatusNotFound, "invalid_request_error", "model_not_found", + "no moderation model: the catalog has no model with role: guard") + return + } + format, names, _ := r.GuardFormat(model) + if len(names) == 0 { + names = guard.DefaultCategories(format) + } + results := make([]map[string]any, 0, len(texts)) + for _, t := range texts { + unsafe, cats, err := guard.Moderate(req.Context(), r.Judge, model, format, names, t) + if err != nil { + writeError(w, http.StatusServiceUnavailable, "server_error", "the moderation model gave no verdict: "+err.Error()) + return + } + categories, scores := map[string]bool{}, map[string]float64{} + for _, n := range names { + categories[moderationKey(n)], scores[moderationKey(n)] = false, 0 + } + for _, c := range cats { + categories[moderationKey(c)], scores[moderationKey(c)] = true, 1 + } + results = append(results, map[string]any{"flagged": unsafe, "categories": categories, "category_scores": scores}) + } + writeJSON(w, http.StatusOK, map[string]any{"id": newID("modr-"), "model": model, "results": results}) +} + +// moderationModel is the judge asked: the one named, if it is one, or the +// first in the catalog. +func (r *Router) moderationModel(asked string) string { + if asked != "" { + if canonical, e := r.resolve(asked); e.IsGuard() { + return canonical + } + } + var first string + for _, m := range r.modelMap() { + if m.IsGuard() && (first == "" || m.Name < first) { + first = m.Name + } + } + return first +} + +// moderationKey is a category's name as a JSON key, as OpenAI writes them. +func moderationKey(name string) string { + return strings.Map(func(c rune) rune { + switch { + case c >= 'A' && c <= 'Z': + return c + 'a' - 'A' + case c >= 'a' && c <= 'z', c >= '0' && c <= '9', c == '/': + return c + } + return '-' + }, fmt.Sprint(name)) +} diff --git a/internal/router/proxy/proxy.go b/internal/router/proxy/proxy.go index 0be6249..7a8e864 100644 --- a/internal/router/proxy/proxy.go +++ b/internal/router/proxy/proxy.go @@ -165,6 +165,9 @@ type Request struct { // and a gateway that accumulated every reply in memory would be one long // answer away from trouble; a request that stores nothing must pay nothing. CaptureReply int + // ContentType is the body's, when it is not JSON: an audio upload's + // multipart form. + ContentType string // Filter, when set, sees a whole non-streamed answer and the headers // about to be sent with it, and returns the body to send: a guard's @@ -241,7 +244,11 @@ func (p *Proxy) Do(ctx context.Context, w http.ResponseWriter, r Request) (Outco return Outcome{Retryable: false}, err } req.Header = cloneForUpstream(r.Header) - req.Header.Set("Content-Type", "application/json") + ct := r.ContentType + if ct == "" { + ct = "application/json" + } + req.Header.Set("Content-Type", ct) if r.APIKey != "" { req.Header.Set("Authorization", "Bearer "+r.APIKey) } diff --git a/internal/router/responses.go b/internal/router/responses.go index fc7d202..2ae5160 100644 --- a/internal/router/responses.go +++ b/internal/router/responses.go @@ -9,6 +9,8 @@ import ( "strings" "sync/atomic" "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/auth" ) // The Responses API, as a translation of chat completions. @@ -38,6 +40,13 @@ type respRequest struct { MaxOutputTokens *int `json:"max_output_tokens"` ParallelToolCalls *bool `json:"parallel_tool_calls"` Metadata map[string]string `json:"metadata"` + // Text.Format asks for structured output; PreviousResponseID continues + // a conversation kept by the gateway, which Store false declines. + Text *struct { + Format json.RawMessage `json:"format"` + } `json:"text"` + PreviousResponseID string `json:"previous_response_id"` + Store *bool `json:"store"` } type respTool struct { @@ -135,7 +144,11 @@ func toChat(in respRequest) (map[string]any, map[string]bool, error) { if role == "" { role = "user" } - msgs = append(msgs, chatMsg{Role: role, Content: partsText(it.Content)}) + content, err := chatContent(it.Content) + if err != nil { + return nil, nil, err + } + msgs = append(msgs, chatMsg{Role: role, Content: content}) case "function_call", "custom_tool_call": args := it.Arguments if it.Type == "custom_tool_call" { @@ -206,9 +219,89 @@ func toChat(in respRequest) (map[string]any, map[string]bool, error) { if in.MaxOutputTokens != nil { out["max_tokens"] = *in.MaxOutputTokens } + if in.Text != nil && len(in.Text.Format) > 0 { + var f struct { + Type string `json:"type"` + Name string `json:"name"` + Description string `json:"description"` + Schema json.RawMessage `json:"schema"` + Strict *bool `json:"strict"` + } + if err := json.Unmarshal(in.Text.Format, &f); err != nil { + return nil, nil, fmt.Errorf("text.format: %v", err) + } + switch f.Type { + case "json_schema": + js := map[string]any{"name": f.Name, "schema": f.Schema} + if f.Description != "" { + js["description"] = f.Description + } + if f.Strict != nil { + js["strict"] = *f.Strict + } + out["response_format"] = map[string]any{"type": "json_schema", "json_schema": js} + case "json_object": + out["response_format"] = map[string]any{"type": "json_object"} + case "", "text": + default: + return nil, nil, fmt.Errorf("text.format.type %q is not text, json_object or json_schema", f.Type) + } + } return out, custom, nil } +// chatContent is a message's content for a chat completion: its text, or, +// with images among its parts, the parts as a chat completion writes them. +func chatContent(raw json.RawMessage) (any, error) { + var parts []struct { + Type string `json:"type"` + Text string `json:"text"` + ImageURL string `json:"image_url"` + FileID string `json:"file_id"` + Detail string `json:"detail"` + } + if json.Unmarshal(raw, &parts) != nil { + return partsText(raw), nil + } + images := false + for _, p := range parts { + images = images || p.Type == "input_image" + } + if !images { + return partsText(raw), nil + } + var out []map[string]any + for _, p := range parts { + switch p.Type { + case "input_text", "output_text", "text": + out = append(out, map[string]any{"type": "text", "text": p.Text}) + case "input_image": + if p.ImageURL == "" { + return nil, fmt.Errorf("an input_image by file_id is not supported here: send image_url (a data: URL works)") + } + img := map[string]any{"url": p.ImageURL} + if p.Detail != "" && p.Detail != "auto" { + img["detail"] = p.Detail + } + out = append(out, map[string]any{"type": "image_url", "image_url": img}) + } + } + return out, nil +} + +// A response continued by previous_response_id is a stored conversation: its +// thread is in its id, as resp_., so nothing more is kept to find +// it. + +func responseThread(id string) (string, bool) { + rest, ok := strings.CutPrefix(id, "resp_") + i := strings.LastIndexByte(rest, '.') + if !ok || i <= 0 || validThreadID(rest[:i]) != nil { + return "", false + } + return rest[:i], true +} + var respSeq atomic.Int64 func respID(prefix string) string { @@ -239,8 +332,30 @@ func (r *Router) handleResponses(w http.ResponseWriter, req *http.Request) { next.ContentLength = int64(len(upstream)) next.Header.Set("Content-Type", "application/json") + id := respID("resp") + // A conversation the gateway keeps: continued, or begun when the + // project keeps conversations and the client did not say store false. + thread := "" + switch { + case in.PreviousResponseID != "": + var ok bool + if thread, ok = responseThread(in.PreviousResponseID); !ok { + writeErrorCode(w, http.StatusBadRequest, "invalid_request_error", "previous_response_not_found", + fmt.Sprintf("previous_response_id %q is not a response this gateway kept", in.PreviousResponseID)) + return + } + case in.Store == nil || *in.Store: + if r.keepsThreads(req) { + thread = newID("") + } + } + if thread != "" { + next.Header.Set(threadHeader, thread) + id = "resp_" + thread + "." + newID("")[:8] + } + t := &respTranslator{w: w, model: in.Model, stream: in.Stream, custom: custom, - id: respID("resp"), created: time.Now().Unix(), metadata: in.Metadata} + id: id, created: time.Now().Unix(), metadata: in.Metadata} r.handleInference("/v1/chat/completions")(t, next) t.finish() } @@ -590,3 +705,18 @@ func (t *respTranslator) fromChatJSON(b []byte) { out, _ := json.Marshal(t.object(status)) _, _ = t.w.Write(out) } + +// keepsThreads says whether the caller's project keeps conversations, so a +// Responses request can begin one its next turn continues. +func (r *Router) keepsThreads(req *http.Request) bool { + if r.threads == nil { + return false + } + p, _ := auth.FromContext(req.Context()) + project, err := r.threadProject(p.ProjectID) + if err != nil { + return false + } + retention, err := r.threads.ThreadRetention(req.Context(), project) + return err == nil && retention > 0 +} diff --git a/internal/router/router.go b/internal/router/router.go index e5e7f03..6776e41 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -927,7 +927,7 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa trace.WithAttributes(attribute.String("llmgw.endpoint", dec.Endpoint.ID), attribute.String("url.full", dec.Endpoint.URL+req.Path))) body := req.UpstreamBody - if dec.Endpoint.UpstreamModel != "" { + if dec.Endpoint.UpstreamModel != "" && req.ContentType == "" { body = withModel(body, dec.Endpoint.UpstreamModel) } // An adapter asked for by name, served by its model's replicas: the @@ -954,6 +954,7 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa Stream: req.Stream, SuppressUsageChunk: req.SuppressUsageChunk, CaptureReply: req.captureBytes(), + ContentType: req.ContentType, } // A filter per attempt, for the model this attempt is served by. filter := r.outputFilter(uctx, req, model, &preq)