More of OpenAI's API: Responses structured output, images and previous_response_id; rerank, score and audio passed through; moderations by the judge

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 22:56:25 +02:00
1 parent 32ff1dbd6f
commit 9136bc71c2
12 files changed
+549 -5

No files matched your search

+7 -1
View File
@@ -448,7 +448,13 @@ portal is not done — the page shows; the console and ctl change.
**Request/response logging and replay (M).** Opt-in per project with
retention like threads; sampled or full; redaction hooks from B;
`ctl request <id> --body`; replay against another model.
10. **OpenAI compatibility gaps (S–M each).** Responses: `text.format` →
10. **Done (branch `features-c2`):** Responses `text.format`,
`input_image`, `previous_response_id` (thread in the id); rerank and
score passthrough; transcription and translation as multipart;
`/v1/moderations` by the judge. Left: audio is charged no tokens (an
engine's duration usage is not read); `input_image` by `file_id`;
moderation scores are 0 or 1, as a judge says yes or no.
**OpenAI compatibility gaps (S–M each).** Responses: `text.format` →
`response_format`, `input_image`, `previous_response_id` via threads
(`router/responses.go`). Rerank/score passthrough for vLLM,
`/v1/audio/transcriptions` for a whisper engine, `/v1/moderations`
+10
View File
@@ -71,6 +71,16 @@ APIs change between minors without a deprecation period.
- **Other ways in:** `GET /api/v1/usage/export` (CSV, or `format=json`)
and **E** on the console's Usage view.
### More of OpenAI's API
- **Responses:** `text.format` asks for structured output, `input_image`
sends images, and `previous_response_id` continues a conversation the
gateway keeps as a thread (for a project that keeps conversations).
- **Passed through:** `/v1/rerank` and `/v1/score` for vLLM's rerankers, and
`/v1/audio/transcriptions` and `/translations` as the multipart form came,
for an engine serving Whisper.
- **`/v1/moderations`** is answered by the guardrails' judge model.
### Requests kept, and replayed
- **A project can keep its requests and answers** (`llmgw keys requests
+12
View File
@@ -959,6 +959,18 @@ Or open `/chat` in a browser and use it without a key at all — the page carrie
your session, and the request still goes through admission, rate limits, budget
and accounting exactly as an API call does.
### What the client port answers
| Path | What |
|---|---|
| `POST /v1/chat/completions`, `/v1/completions`, `/v1/embeddings` | OpenAI's, served by the models in the catalog |
| `POST /v1/responses` | OpenAI's Responses API, translated to chat completions: tools, `text.format` (structured output), `input_image` (by URL or `data:` URL), and `previous_response_id` — the conversation is kept as a thread, so the project must keep conversations (`llmgw keys threads`); `store: false` keeps none |
| `POST /v1/rerank`, `/v1/score` | passed through to an engine that serves them: vLLM with a reranker or a cross-encoder |
| `POST /v1/audio/transcriptions`, `/v1/audio/translations` | the multipart form passed through as it came, to an engine that serves speech to text: vLLM with Whisper |
| `POST /v1/moderations` | answered by the guardrails' judge model (`role: guard`): `flagged`, and the judge's own categories |
| `/v1/files`, `/v1/batches` | batches; see [Batches](#batches) |
| `GET /v1/models` | the models this key may use |
## Watching it
The control plane exports both halves on one endpoint: what the cluster is
+25
View File
@@ -344,3 +344,28 @@ func parseJudgment(format string, names []string, body []byte, answer string) (j
}
return judgment{}, fmt.Errorf("%w: not in the form %s has: %q", errJudgeFormat, format, string(seen))
}
// Moderate asks a judge model about one text, as a user's message, outside
// any policy: for /v1/moderations. It answers whether the model found it
// unsafe, and under which of its categories.
func Moderate(ctx context.Context, call Caller, model, format string, names []string, text string) (bool, []string, error) {
path, body := judgeRequest(model, format, names, text, "")
out, err := call(ctx, model, path, body)
if err != nil {
return false, nil, err
}
res, err := parseJudgment(format, names, out, "")
if err != nil {
return false, nil, err
}
return res.unsafe, res.categories, nil
}
// DefaultCategories are the names a judge format reports its categories by
// when the catalog gives none: Llama Guard 3's S1 to S13.
func DefaultCategories(format string) []string {
if format == "llamaguard3" {
return slices.Clone(llamaGuard3Categories)
}
return nil
}
+62
View File
@@ -0,0 +1,62 @@
package router
import (
"bytes"
"errors"
"mime"
"mime/multipart"
"net/http"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
)
// parseForm reads a multipart request — audio sent for transcription — for
// what routing needs, the model, and keeps the body to send on unchanged.
func (r *Router) parseForm(req *http.Request, path, ct string, body []byte) (*parsedRequest, error) {
_, params, err := mime.ParseMediaType(ct)
if err != nil || params["boundary"] == "" {
return nil, errors.New("a multipart form needs its boundary")
}
mr := multipart.NewReader(bytes.NewReader(body), params["boundary"])
model, stream := "", false
for {
part, err := mr.NextPart()
if err != nil {
break
}
switch part.FormName() {
case "model":
var b bytes.Buffer
_, _ = b.ReadFrom(part)
model = strings.TrimSpace(b.String())
case "stream":
var b bytes.Buffer
_, _ = b.ReadFrom(part)
stream = strings.TrimSpace(b.String()) == "true"
}
part.Close()
}
if model == "" {
return nil, errors.New("model is required")
}
tags, err := requestTags(nil, req.Header.Get(tagsHeader))
if err != nil {
return nil, err
}
p := &parsedRequest{
Path: path, ContentType: ct, Model: model, RequestedModel: model, Stream: stream,
UpstreamBody: body, Header: req.Header, Tags: tags,
SessionID: sessionID(req, r.cfg.SessionHeader, ""),
Band: req.Header.Get(r.cfg.Admission.BandHeader),
ProfileOverride: req.Header.Get("x-llmgw-profile"),
}
if principal, ok := auth.FromContext(req.Context()); ok {
p.Band = principal.Band
p.ProfileOverride = principal.RoutingProfile
p.KeyID, p.UserID, p.ProjectID = principal.KeyID, principal.UserID, principal.ProjectID
p.TeamID, p.OrgID = principal.TeamID, principal.OrgID
p.Tenant = principal.KeyID
}
return p, nil
}
+178
View File
@@ -0,0 +1,178 @@
package router
import (
"bytes"
"encoding/json"
"io"
"mime/multipart"
"net/http"
"net/http/httptest"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
func TestResponsesAsksForStructuredOutputAndImages(t *testing.T) {
var in respRequest
_ = json.Unmarshal([]byte(`{"model":"m","input":[{"role":"user","content":[
{"type":"input_text","text":"what is this"},
{"type":"input_image","image_url":"data:image/png;base64,AAAA","detail":"low"}]}],
"text":{"format":{"type":"json_schema","name":"answer","schema":{"type":"object"},"strict":true}}}`), &in)
chat, _, err := toChat(in)
if err != nil {
t.Fatal(err)
}
b, _ := json.Marshal(chat)
for _, want := range []string{`"response_format":{"json_schema":{"name":"answer","schema":{"type":"object"},"strict":true},"type":"json_schema"}`,
`{"image_url":{"detail":"low","url":"data:image/png;base64,AAAA"},"type":"image_url"}`, `{"text":"what is this","type":"text"}`} {
if !strings.Contains(string(b), want) {
t.Errorf("%s lacks %s", b, want)
}
}
_ = json.Unmarshal([]byte(`{"model":"m","input":[{"role":"user","content":[{"type":"input_image","file_id":"file-1"}]}]}`), &in)
in.Text = nil
if _, _, err := toChat(in); err == nil {
t.Error("an image by file_id was taken")
}
if th, ok := responseThread("resp_abc123.def"); !ok || th != "abc123" {
t.Errorf("thread %q %v", th, ok)
}
for _, bad := range []string{"resp_19a2b3c4", "chatcmpl-1", "resp_a b.c"} {
if _, ok := responseThread(bad); ok {
t.Errorf("%q read as a kept response", bad)
}
}
}
func TestAResponseIsContinuedByItsID(t *testing.T) {
e := newFakeEngine(t, "qwen2.5-14b-instruct")
rt, secret := pricedRouter(t, e.srv.URL)
if err := rt.store.SetThreadRetention(t.Context(), mustProjectOf(t, rt, "key-1"), 24); err != nil {
t.Fatal(err)
}
h := rt.Handler()
ask := func(body map[string]any) map[string]any {
b, _ := json.Marshal(body)
req := httptest.NewRequest(http.MethodPost, "/v1/responses", bytes.NewReader(b))
req.Header.Set("Authorization", "Bearer "+secret)
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
var out map[string]any
_ = json.Unmarshal(rec.Body.Bytes(), &out)
if rec.Code != 200 {
t.Fatalf("%d %s", rec.Code, rec.Body.String())
}
return out
}
first := ask(map[string]any{"model": "q14", "input": "my name is Ayşe"})
id, _ := first["id"].(string)
if _, ok := responseThread(id); !ok {
t.Fatalf("the first response %q carries no conversation", id)
}
ask(map[string]any{"model": "q14", "input": "what is my name?", "previous_response_id": id})
sent, _ := e.last.Load().([]byte)
if !strings.Contains(string(sent), "my name is Ayşe") || !strings.Contains(string(sent), "what is my name?") {
t.Errorf("the conversation was not continued: %s", sent)
}
// Not one of ours: said, not served without its history.
b, _ := json.Marshal(map[string]any{"model": "q14", "input": "x", "previous_response_id": "resp_nope"})
req := httptest.NewRequest(http.MethodPost, "/v1/responses", bytes.NewReader(b))
req.Header.Set("Authorization", "Bearer "+secret)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
if rec.Code != 400 || !strings.Contains(rec.Body.String(), "previous_response_not_found") {
t.Errorf("%d %s", rec.Code, rec.Body.String())
}
}
// engineFor serves a few paths with fixed answers and records what came.
func engineFor(t *testing.T, model string, answers map[string]string) (*httptest.Server, *[]*http.Request, *[][]byte) {
t.Helper()
var reqs []*http.Request
var bodies [][]byte
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/metrics" {
return
}
b, _ := io.ReadAll(r.Body)
reqs, bodies = append(reqs, r), append(bodies, b)
a, ok := answers[r.URL.Path]
if !ok {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(a))
}))
t.Cleanup(srv.Close)
return srv, &reqs, &bodies
}
func TestRerankScoreAndAudioArePassedThrough(t *testing.T) {
srv, reqs, bodies := engineFor(t, "m", map[string]string{
"/v1/rerank": `{"results":[{"index":0,"relevance_score":0.9}],"usage":{"total_tokens":7}}`,
"/v1/score": `{"data":[{"score":0.5}]}`,
"/v1/audio/transcriptions": `{"text":"merhaba dünya"}`,
})
cfg := baseConfig(config.Endpoint{ID: "a", URL: srv.URL, Models: []string{"m"}})
rt := newTestRouter(t, cfg)
h := rt.Handler()
for _, path := range []string{"/v1/rerank", "/v1/score"} {
b, _ := json.Marshal(map[string]any{"model": "m", "query": "q", "documents": []string{"d"}})
req := httptest.NewRequest(http.MethodPost, path, bytes.NewReader(b))
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
if rec.Code != 200 || (*reqs)[len(*reqs)-1].URL.Path != path {
t.Errorf("%s: %d %s", path, rec.Code, rec.Body.String())
}
}
var buf bytes.Buffer
mw := multipart.NewWriter(&buf)
_ = mw.WriteField("model", "m")
fw, _ := mw.CreateFormFile("file", "a.wav")
_, _ = fw.Write([]byte("RIFF....WAVE"))
_ = mw.Close()
req := httptest.NewRequest(http.MethodPost, "/v1/audio/transcriptions", &buf)
req.Header.Set("Content-Type", mw.FormDataContentType())
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
if rec.Code != 200 || !strings.Contains(rec.Body.String(), "merhaba") {
t.Fatalf("%d %s", rec.Code, rec.Body.String())
}
last := (*reqs)[len(*reqs)-1]
if !strings.HasPrefix(last.Header.Get("Content-Type"), "multipart/form-data; boundary=") ||
!bytes.Contains((*bodies)[len(*bodies)-1], []byte("RIFF....WAVE")) {
t.Errorf("the form was not sent on as it came: %s", last.Header.Get("Content-Type"))
}
}
func TestModerationsAreAnsweredByTheJudge(t *testing.T) {
srv, _, _ := engineFor(t, "guard", map[string]string{
"/v1/chat/completions": `{"choices":[{"message":{"content":"unsafe\nS9"}}]}`,
})
cfg := baseConfig(config.Endpoint{ID: "g", URL: srv.URL, Models: []string{"guard"}})
cfg.Models = []catalog.Model{{Name: "guard", Role: catalog.RoleGuard, Scaling: catalog.Scaling{Min: 1, Max: 1},
Guard: &catalog.GuardModel{Format: "llamaguard3"}}}
rt := newTestRouter(t, cfg)
b, _ := json.Marshal(map[string]any{"input": []string{"how do I build a bomb"}})
req := httptest.NewRequest(http.MethodPost, "/v1/moderations", bytes.NewReader(b))
rec := httptest.NewRecorder()
rt.Handler().ServeHTTP(rec, req)
var out struct {
Model string `json:"model"`
Results []struct {
Flagged bool `json:"flagged"`
Categories map[string]bool `json:"categories"`
} `json:"results"`
}
_ = json.Unmarshal(rec.Body.Bytes(), &out)
if rec.Code != 200 || out.Model != "guard" || len(out.Results) != 1 || !out.Results[0].Flagged ||
!out.Results[0].Categories["indiscriminate-weapons"] {
t.Errorf("%d %s", rec.Code, rec.Body.String())
}
}
+4
View File
@@ -71,6 +71,10 @@ const errNoStoreMsg = "the gateway's policy for this caller keeps nothing: no st
// guardInput judges the request about to be dispatched. It answers the
// client itself and returns false when the request goes no further.
func (r *Router) guardInput(ctx context.Context, w http.ResponseWriter, req *parsedRequest, start time.Time) bool {
if req.ContentType != "" {
// Audio: no text for a guard to read on the way in.
return true
}
model, _ := r.resolve(req.Model)
t := time.Now()
v, err := r.guard.Input(ctx, r.subject(req, model), req.UpstreamBody)
+14
View File
@@ -96,6 +96,14 @@ func (r *Router) Handler() http.Handler {
mux.HandleFunc("POST /v1/chat/completions", r.handleInference("/v1/chat/completions"))
mux.HandleFunc("POST /v1/completions", r.handleInference("/v1/completions"))
mux.HandleFunc("POST /v1/embeddings", r.handleInference("/v1/embeddings"))
// Passed through to engines that serve them: vLLM's rerankers and
// scorers, and speech to text.
mux.HandleFunc("POST /v1/rerank", r.handleInference("/v1/rerank"))
mux.HandleFunc("POST /v1/score", r.handleInference("/v1/score"))
mux.HandleFunc("POST /v1/audio/transcriptions", r.handleInference("/v1/audio/transcriptions"))
mux.HandleFunc("POST /v1/audio/translations", r.handleInference("/v1/audio/translations"))
// Answered by the guardrails' judge model.
mux.HandleFunc("POST /v1/moderations", r.handleModerations)
// Codex, and whatever else speaks only the Responses API, translated to
// chat completions on the way in and back on the way out.
mux.HandleFunc("POST /v1/responses", r.handleResponses)
@@ -290,6 +298,9 @@ func (r *Router) Handler() http.Handler {
// business interpreting.
type parsedRequest struct {
Path string
// ContentType is the body's when it is a form rather than JSON — audio
// for transcription — and is sent on as it came.
ContentType string
// Tags are the client's metadata and x-llmgw-tags; EndUser is OpenAI's
// user field. Both are recorded with the usage, never sent on.
Tags map[string]string
@@ -661,6 +672,9 @@ func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) {
if err != nil {
return nil, err
}
if ct := req.Header.Get("Content-Type"); strings.HasPrefix(ct, "multipart/form-data") {
return r.parseForm(req, path, ct, body)
}
var b inferenceBody
if err := json.Unmarshal(body, &b); err != nil {
+95
View File
@@ -0,0 +1,95 @@
package router
import (
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
)
// /v1/moderations, answered by a judge model on the cluster: the same model
// the guardrails ask. OpenAI's shape — flagged, and the categories — with the
// judge's own categories named as it names them.
func (r *Router) handleModerations(w http.ResponseWriter, req *http.Request) {
var in struct {
Input json.RawMessage `json:"input"`
Model string `json:"model"`
}
if err := json.NewDecoder(io.LimitReader(req.Body, r.cfg.MaxBodyBytes)).Decode(&in); err != nil {
writeError(w, http.StatusBadRequest, "invalid_request_error", "the body is not JSON: "+err.Error())
return
}
var texts []string
var one string
if json.Unmarshal(in.Input, &one) == nil {
texts = []string{one}
} else if err := json.Unmarshal(in.Input, &texts); err != nil || len(texts) == 0 {
writeError(w, http.StatusBadRequest, "invalid_request_error", "input is a string or a list of strings")
return
}
if len(texts) > 32 {
writeError(w, http.StatusBadRequest, "invalid_request_error", "at most 32 inputs at once")
return
}
model := r.moderationModel(in.Model)
if model == "" {
writeErrorCode(w, http.StatusNotFound, "invalid_request_error", "model_not_found",
"no moderation model: the catalog has no model with role: guard")
return
}
format, names, _ := r.GuardFormat(model)
if len(names) == 0 {
names = guard.DefaultCategories(format)
}
results := make([]map[string]any, 0, len(texts))
for _, t := range texts {
unsafe, cats, err := guard.Moderate(req.Context(), r.Judge, model, format, names, t)
if err != nil {
writeError(w, http.StatusServiceUnavailable, "server_error", "the moderation model gave no verdict: "+err.Error())
return
}
categories, scores := map[string]bool{}, map[string]float64{}
for _, n := range names {
categories[moderationKey(n)], scores[moderationKey(n)] = false, 0
}
for _, c := range cats {
categories[moderationKey(c)], scores[moderationKey(c)] = true, 1
}
results = append(results, map[string]any{"flagged": unsafe, "categories": categories, "category_scores": scores})
}
writeJSON(w, http.StatusOK, map[string]any{"id": newID("modr-"), "model": model, "results": results})
}
// moderationModel is the judge asked: the one named, if it is one, or the
// first in the catalog.
func (r *Router) moderationModel(asked string) string {
if asked != "" {
if canonical, e := r.resolve(asked); e.IsGuard() {
return canonical
}
}
var first string
for _, m := range r.modelMap() {
if m.IsGuard() && (first == "" || m.Name < first) {
first = m.Name
}
}
return first
}
// moderationKey is a category's name as a JSON key, as OpenAI writes them.
func moderationKey(name string) string {
return strings.Map(func(c rune) rune {
switch {
case c >= 'A' && c <= 'Z':
return c + 'a' - 'A'
case c >= 'a' && c <= 'z', c >= '0' && c <= '9', c == '/':
return c
}
return '-'
}, fmt.Sprint(name))
}
+8 -1
View File
@@ -165,6 +165,9 @@ type Request struct {
// and a gateway that accumulated every reply in memory would be one long
// answer away from trouble; a request that stores nothing must pay nothing.
CaptureReply int
// ContentType is the body's, when it is not JSON: an audio upload's
// multipart form.
ContentType string
// Filter, when set, sees a whole non-streamed answer and the headers
// about to be sent with it, and returns the body to send: a guard's
@@ -241,7 +244,11 @@ func (p *Proxy) Do(ctx context.Context, w http.ResponseWriter, r Request) (Outco
return Outcome{Retryable: false}, err
}
req.Header = cloneForUpstream(r.Header)
req.Header.Set("Content-Type", "application/json")
ct := r.ContentType
if ct == "" {
ct = "application/json"
}
req.Header.Set("Content-Type", ct)
if r.APIKey != "" {
req.Header.Set("Authorization", "Bearer "+r.APIKey)
}
+132 -2
View File
@@ -9,6 +9,8 @@ import (
"strings"
"sync/atomic"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
)
// The Responses API, as a translation of chat completions.
@@ -38,6 +40,13 @@ type respRequest struct {
MaxOutputTokens *int `json:"max_output_tokens"`
ParallelToolCalls *bool `json:"parallel_tool_calls"`
Metadata map[string]string `json:"metadata"`
// Text.Format asks for structured output; PreviousResponseID continues
// a conversation kept by the gateway, which Store false declines.
Text *struct {
Format json.RawMessage `json:"format"`
} `json:"text"`
PreviousResponseID string `json:"previous_response_id"`
Store *bool `json:"store"`
}
type respTool struct {
@@ -135,7 +144,11 @@ func toChat(in respRequest) (map[string]any, map[string]bool, error) {
if role == "" {
role = "user"
}
msgs = append(msgs, chatMsg{Role: role, Content: partsText(it.Content)})
content, err := chatContent(it.Content)
if err != nil {
return nil, nil, err
}
msgs = append(msgs, chatMsg{Role: role, Content: content})
case "function_call", "custom_tool_call":
args := it.Arguments
if it.Type == "custom_tool_call" {
@@ -206,9 +219,89 @@ func toChat(in respRequest) (map[string]any, map[string]bool, error) {
if in.MaxOutputTokens != nil {
out["max_tokens"] = *in.MaxOutputTokens
}
if in.Text != nil && len(in.Text.Format) > 0 {
var f struct {
Type string `json:"type"`
Name string `json:"name"`
Description string `json:"description"`
Schema json.RawMessage `json:"schema"`
Strict *bool `json:"strict"`
}
if err := json.Unmarshal(in.Text.Format, &f); err != nil {
return nil, nil, fmt.Errorf("text.format: %v", err)
}
switch f.Type {
case "json_schema":
js := map[string]any{"name": f.Name, "schema": f.Schema}
if f.Description != "" {
js["description"] = f.Description
}
if f.Strict != nil {
js["strict"] = *f.Strict
}
out["response_format"] = map[string]any{"type": "json_schema", "json_schema": js}
case "json_object":
out["response_format"] = map[string]any{"type": "json_object"}
case "", "text":
default:
return nil, nil, fmt.Errorf("text.format.type %q is not text, json_object or json_schema", f.Type)
}
}
return out, custom, nil
}
// chatContent is a message's content for a chat completion: its text, or,
// with images among its parts, the parts as a chat completion writes them.
func chatContent(raw json.RawMessage) (any, error) {
var parts []struct {
Type string `json:"type"`
Text string `json:"text"`
ImageURL string `json:"image_url"`
FileID string `json:"file_id"`
Detail string `json:"detail"`
}
if json.Unmarshal(raw, &parts) != nil {
return partsText(raw), nil
}
images := false
for _, p := range parts {
images = images || p.Type == "input_image"
}
if !images {
return partsText(raw), nil
}
var out []map[string]any
for _, p := range parts {
switch p.Type {
case "input_text", "output_text", "text":
out = append(out, map[string]any{"type": "text", "text": p.Text})
case "input_image":
if p.ImageURL == "" {
return nil, fmt.Errorf("an input_image by file_id is not supported here: send image_url (a data: URL works)")
}
img := map[string]any{"url": p.ImageURL}
if p.Detail != "" && p.Detail != "auto" {
img["detail"] = p.Detail
}
out = append(out, map[string]any{"type": "image_url", "image_url": img})
}
}
return out, nil
}
// A response continued by previous_response_id is a stored conversation: its
// thread is in its id, as resp_<thread>.<n>, so nothing more is kept to find
// it.
func responseThread(id string) (string, bool) {
rest, ok := strings.CutPrefix(id, "resp_")
i := strings.LastIndexByte(rest, '.')
if !ok || i <= 0 || validThreadID(rest[:i]) != nil {
return "", false
}
return rest[:i], true
}
var respSeq atomic.Int64
func respID(prefix string) string {
@@ -239,8 +332,30 @@ func (r *Router) handleResponses(w http.ResponseWriter, req *http.Request) {
next.ContentLength = int64(len(upstream))
next.Header.Set("Content-Type", "application/json")
id := respID("resp")
// A conversation the gateway keeps: continued, or begun when the
// project keeps conversations and the client did not say store false.
thread := ""
switch {
case in.PreviousResponseID != "":
var ok bool
if thread, ok = responseThread(in.PreviousResponseID); !ok {
writeErrorCode(w, http.StatusBadRequest, "invalid_request_error", "previous_response_not_found",
fmt.Sprintf("previous_response_id %q is not a response this gateway kept", in.PreviousResponseID))
return
}
case in.Store == nil || *in.Store:
if r.keepsThreads(req) {
thread = newID("")
}
}
if thread != "" {
next.Header.Set(threadHeader, thread)
id = "resp_" + thread + "." + newID("")[:8]
}
t := &respTranslator{w: w, model: in.Model, stream: in.Stream, custom: custom,
id: respID("resp"), created: time.Now().Unix(), metadata: in.Metadata}
id: id, created: time.Now().Unix(), metadata: in.Metadata}
r.handleInference("/v1/chat/completions")(t, next)
t.finish()
}
@@ -590,3 +705,18 @@ func (t *respTranslator) fromChatJSON(b []byte) {
out, _ := json.Marshal(t.object(status))
_, _ = t.w.Write(out)
}
// keepsThreads says whether the caller's project keeps conversations, so a
// Responses request can begin one its next turn continues.
func (r *Router) keepsThreads(req *http.Request) bool {
if r.threads == nil {
return false
}
p, _ := auth.FromContext(req.Context())
project, err := r.threadProject(p.ProjectID)
if err != nil {
return false
}
retention, err := r.threads.ThreadRetention(req.Context(), project)
return err == nil && retention > 0
}
+2 -1
View File
@@ -927,7 +927,7 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
trace.WithAttributes(attribute.String("llmgw.endpoint", dec.Endpoint.ID),
attribute.String("url.full", dec.Endpoint.URL+req.Path)))
body := req.UpstreamBody
if dec.Endpoint.UpstreamModel != "" {
if dec.Endpoint.UpstreamModel != "" && req.ContentType == "" {
body = withModel(body, dec.Endpoint.UpstreamModel)
}
// An adapter asked for by name, served by its model's replicas: the
@@ -954,6 +954,7 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
Stream: req.Stream,
SuppressUsageChunk: req.SuppressUsageChunk,
CaptureReply: req.captureBytes(),
ContentType: req.ContentType,
}
// A filter per attempt, for the model this attempt is served by.
filter := r.outputFilter(uctx, req, model, &preq)