More of OpenAI's API: Responses structured output, images and previous_response_id; rerank, score and audio passed through; moderations by the judge
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
32ff1dbd6f
commit
9136bc71c2
12 files changed
+549
-5
No files matched your search
+7
-1
@@ -448,7 +448,13 @@ portal is not done — the page shows; the console and ctl change.
|
||||
**Request/response logging and replay (M).** Opt-in per project with
|
||||
retention like threads; sampled or full; redaction hooks from B;
|
||||
`ctl request <id> --body`; replay against another model.
|
||||
10. **OpenAI compatibility gaps (S–M each).** Responses: `text.format` →
|
||||
10. **Done (branch `features-c2`):** Responses `text.format`,
|
||||
`input_image`, `previous_response_id` (thread in the id); rerank and
|
||||
score passthrough; transcription and translation as multipart;
|
||||
`/v1/moderations` by the judge. Left: audio is charged no tokens (an
|
||||
engine's duration usage is not read); `input_image` by `file_id`;
|
||||
moderation scores are 0 or 1, as a judge says yes or no.
|
||||
**OpenAI compatibility gaps (S–M each).** Responses: `text.format` →
|
||||
`response_format`, `input_image`, `previous_response_id` via threads
|
||||
(`router/responses.go`). Rerank/score passthrough for vLLM,
|
||||
`/v1/audio/transcriptions` for a whisper engine, `/v1/moderations`
|
||||
|
||||
@@ -71,6 +71,16 @@ APIs change between minors without a deprecation period.
|
||||
- **Other ways in:** `GET /api/v1/usage/export` (CSV, or `format=json`)
|
||||
and **E** on the console's Usage view.
|
||||
|
||||
### More of OpenAI's API
|
||||
|
||||
- **Responses:** `text.format` asks for structured output, `input_image`
|
||||
sends images, and `previous_response_id` continues a conversation the
|
||||
gateway keeps as a thread (for a project that keeps conversations).
|
||||
- **Passed through:** `/v1/rerank` and `/v1/score` for vLLM's rerankers, and
|
||||
`/v1/audio/transcriptions` and `/translations` as the multipart form came,
|
||||
for an engine serving Whisper.
|
||||
- **`/v1/moderations`** is answered by the guardrails' judge model.
|
||||
|
||||
### Requests kept, and replayed
|
||||
|
||||
- **A project can keep its requests and answers** (`llmgw keys requests
|
||||
|
||||
@@ -959,6 +959,18 @@ Or open `/chat` in a browser and use it without a key at all — the page carrie
|
||||
your session, and the request still goes through admission, rate limits, budget
|
||||
and accounting exactly as an API call does.
|
||||
|
||||
### What the client port answers
|
||||
|
||||
| Path | What |
|
||||
|---|---|
|
||||
| `POST /v1/chat/completions`, `/v1/completions`, `/v1/embeddings` | OpenAI's, served by the models in the catalog |
|
||||
| `POST /v1/responses` | OpenAI's Responses API, translated to chat completions: tools, `text.format` (structured output), `input_image` (by URL or `data:` URL), and `previous_response_id` — the conversation is kept as a thread, so the project must keep conversations (`llmgw keys threads`); `store: false` keeps none |
|
||||
| `POST /v1/rerank`, `/v1/score` | passed through to an engine that serves them: vLLM with a reranker or a cross-encoder |
|
||||
| `POST /v1/audio/transcriptions`, `/v1/audio/translations` | the multipart form passed through as it came, to an engine that serves speech to text: vLLM with Whisper |
|
||||
| `POST /v1/moderations` | answered by the guardrails' judge model (`role: guard`): `flagged`, and the judge's own categories |
|
||||
| `/v1/files`, `/v1/batches` | batches; see [Batches](#batches) |
|
||||
| `GET /v1/models` | the models this key may use |
|
||||
|
||||
## Watching it
|
||||
|
||||
The control plane exports both halves on one endpoint: what the cluster is
|
||||
|
||||
@@ -344,3 +344,28 @@ func parseJudgment(format string, names []string, body []byte, answer string) (j
|
||||
}
|
||||
return judgment{}, fmt.Errorf("%w: not in the form %s has: %q", errJudgeFormat, format, string(seen))
|
||||
}
|
||||
|
||||
// Moderate asks a judge model about one text, as a user's message, outside
|
||||
// any policy: for /v1/moderations. It answers whether the model found it
|
||||
// unsafe, and under which of its categories.
|
||||
func Moderate(ctx context.Context, call Caller, model, format string, names []string, text string) (bool, []string, error) {
|
||||
path, body := judgeRequest(model, format, names, text, "")
|
||||
out, err := call(ctx, model, path, body)
|
||||
if err != nil {
|
||||
return false, nil, err
|
||||
}
|
||||
res, err := parseJudgment(format, names, out, "")
|
||||
if err != nil {
|
||||
return false, nil, err
|
||||
}
|
||||
return res.unsafe, res.categories, nil
|
||||
}
|
||||
|
||||
// DefaultCategories are the names a judge format reports its categories by
|
||||
// when the catalog gives none: Llama Guard 3's S1 to S13.
|
||||
func DefaultCategories(format string) []string {
|
||||
if format == "llamaguard3" {
|
||||
return slices.Clone(llamaGuard3Categories)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
||||
)
|
||||
|
||||
// parseForm reads a multipart request — audio sent for transcription — for
|
||||
// what routing needs, the model, and keeps the body to send on unchanged.
|
||||
func (r *Router) parseForm(req *http.Request, path, ct string, body []byte) (*parsedRequest, error) {
|
||||
_, params, err := mime.ParseMediaType(ct)
|
||||
if err != nil || params["boundary"] == "" {
|
||||
return nil, errors.New("a multipart form needs its boundary")
|
||||
}
|
||||
mr := multipart.NewReader(bytes.NewReader(body), params["boundary"])
|
||||
model, stream := "", false
|
||||
for {
|
||||
part, err := mr.NextPart()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
switch part.FormName() {
|
||||
case "model":
|
||||
var b bytes.Buffer
|
||||
_, _ = b.ReadFrom(part)
|
||||
model = strings.TrimSpace(b.String())
|
||||
case "stream":
|
||||
var b bytes.Buffer
|
||||
_, _ = b.ReadFrom(part)
|
||||
stream = strings.TrimSpace(b.String()) == "true"
|
||||
}
|
||||
part.Close()
|
||||
}
|
||||
if model == "" {
|
||||
return nil, errors.New("model is required")
|
||||
}
|
||||
tags, err := requestTags(nil, req.Header.Get(tagsHeader))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p := &parsedRequest{
|
||||
Path: path, ContentType: ct, Model: model, RequestedModel: model, Stream: stream,
|
||||
UpstreamBody: body, Header: req.Header, Tags: tags,
|
||||
SessionID: sessionID(req, r.cfg.SessionHeader, ""),
|
||||
Band: req.Header.Get(r.cfg.Admission.BandHeader),
|
||||
ProfileOverride: req.Header.Get("x-llmgw-profile"),
|
||||
}
|
||||
if principal, ok := auth.FromContext(req.Context()); ok {
|
||||
p.Band = principal.Band
|
||||
p.ProfileOverride = principal.RoutingProfile
|
||||
p.KeyID, p.UserID, p.ProjectID = principal.KeyID, principal.UserID, principal.ProjectID
|
||||
p.TeamID, p.OrgID = principal.TeamID, principal.OrgID
|
||||
p.Tenant = principal.KeyID
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
)
|
||||
|
||||
func TestResponsesAsksForStructuredOutputAndImages(t *testing.T) {
|
||||
var in respRequest
|
||||
_ = json.Unmarshal([]byte(`{"model":"m","input":[{"role":"user","content":[
|
||||
{"type":"input_text","text":"what is this"},
|
||||
{"type":"input_image","image_url":"data:image/png;base64,AAAA","detail":"low"}]}],
|
||||
"text":{"format":{"type":"json_schema","name":"answer","schema":{"type":"object"},"strict":true}}}`), &in)
|
||||
chat, _, err := toChat(in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, _ := json.Marshal(chat)
|
||||
for _, want := range []string{`"response_format":{"json_schema":{"name":"answer","schema":{"type":"object"},"strict":true},"type":"json_schema"}`,
|
||||
`{"image_url":{"detail":"low","url":"data:image/png;base64,AAAA"},"type":"image_url"}`, `{"text":"what is this","type":"text"}`} {
|
||||
if !strings.Contains(string(b), want) {
|
||||
t.Errorf("%s lacks %s", b, want)
|
||||
}
|
||||
}
|
||||
_ = json.Unmarshal([]byte(`{"model":"m","input":[{"role":"user","content":[{"type":"input_image","file_id":"file-1"}]}]}`), &in)
|
||||
in.Text = nil
|
||||
if _, _, err := toChat(in); err == nil {
|
||||
t.Error("an image by file_id was taken")
|
||||
}
|
||||
if th, ok := responseThread("resp_abc123.def"); !ok || th != "abc123" {
|
||||
t.Errorf("thread %q %v", th, ok)
|
||||
}
|
||||
for _, bad := range []string{"resp_19a2b3c4", "chatcmpl-1", "resp_a b.c"} {
|
||||
if _, ok := responseThread(bad); ok {
|
||||
t.Errorf("%q read as a kept response", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAResponseIsContinuedByItsID(t *testing.T) {
|
||||
e := newFakeEngine(t, "qwen2.5-14b-instruct")
|
||||
rt, secret := pricedRouter(t, e.srv.URL)
|
||||
if err := rt.store.SetThreadRetention(t.Context(), mustProjectOf(t, rt, "key-1"), 24); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := rt.Handler()
|
||||
ask := func(body map[string]any) map[string]any {
|
||||
b, _ := json.Marshal(body)
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/responses", bytes.NewReader(b))
|
||||
req.Header.Set("Authorization", "Bearer "+secret)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, req)
|
||||
var out map[string]any
|
||||
_ = json.Unmarshal(rec.Body.Bytes(), &out)
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("%d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
return out
|
||||
}
|
||||
first := ask(map[string]any{"model": "q14", "input": "my name is Ayşe"})
|
||||
id, _ := first["id"].(string)
|
||||
if _, ok := responseThread(id); !ok {
|
||||
t.Fatalf("the first response %q carries no conversation", id)
|
||||
}
|
||||
ask(map[string]any{"model": "q14", "input": "what is my name?", "previous_response_id": id})
|
||||
sent, _ := e.last.Load().([]byte)
|
||||
if !strings.Contains(string(sent), "my name is Ayşe") || !strings.Contains(string(sent), "what is my name?") {
|
||||
t.Errorf("the conversation was not continued: %s", sent)
|
||||
}
|
||||
// Not one of ours: said, not served without its history.
|
||||
b, _ := json.Marshal(map[string]any{"model": "q14", "input": "x", "previous_response_id": "resp_nope"})
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/responses", bytes.NewReader(b))
|
||||
req.Header.Set("Authorization", "Bearer "+secret)
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, req)
|
||||
if rec.Code != 400 || !strings.Contains(rec.Body.String(), "previous_response_not_found") {
|
||||
t.Errorf("%d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// engineFor serves a few paths with fixed answers and records what came.
|
||||
func engineFor(t *testing.T, model string, answers map[string]string) (*httptest.Server, *[]*http.Request, *[][]byte) {
|
||||
t.Helper()
|
||||
var reqs []*http.Request
|
||||
var bodies [][]byte
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/metrics" {
|
||||
return
|
||||
}
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
reqs, bodies = append(reqs, r), append(bodies, b)
|
||||
a, ok := answers[r.URL.Path]
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(a))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv, &reqs, &bodies
|
||||
}
|
||||
|
||||
func TestRerankScoreAndAudioArePassedThrough(t *testing.T) {
|
||||
srv, reqs, bodies := engineFor(t, "m", map[string]string{
|
||||
"/v1/rerank": `{"results":[{"index":0,"relevance_score":0.9}],"usage":{"total_tokens":7}}`,
|
||||
"/v1/score": `{"data":[{"score":0.5}]}`,
|
||||
"/v1/audio/transcriptions": `{"text":"merhaba dünya"}`,
|
||||
})
|
||||
cfg := baseConfig(config.Endpoint{ID: "a", URL: srv.URL, Models: []string{"m"}})
|
||||
rt := newTestRouter(t, cfg)
|
||||
h := rt.Handler()
|
||||
for _, path := range []string{"/v1/rerank", "/v1/score"} {
|
||||
b, _ := json.Marshal(map[string]any{"model": "m", "query": "q", "documents": []string{"d"}})
|
||||
req := httptest.NewRequest(http.MethodPost, path, bytes.NewReader(b))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, req)
|
||||
if rec.Code != 200 || (*reqs)[len(*reqs)-1].URL.Path != path {
|
||||
t.Errorf("%s: %d %s", path, rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
mw := multipart.NewWriter(&buf)
|
||||
_ = mw.WriteField("model", "m")
|
||||
fw, _ := mw.CreateFormFile("file", "a.wav")
|
||||
_, _ = fw.Write([]byte("RIFF....WAVE"))
|
||||
_ = mw.Close()
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/audio/transcriptions", &buf)
|
||||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, req)
|
||||
if rec.Code != 200 || !strings.Contains(rec.Body.String(), "merhaba") {
|
||||
t.Fatalf("%d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
last := (*reqs)[len(*reqs)-1]
|
||||
if !strings.HasPrefix(last.Header.Get("Content-Type"), "multipart/form-data; boundary=") ||
|
||||
!bytes.Contains((*bodies)[len(*bodies)-1], []byte("RIFF....WAVE")) {
|
||||
t.Errorf("the form was not sent on as it came: %s", last.Header.Get("Content-Type"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestModerationsAreAnsweredByTheJudge(t *testing.T) {
|
||||
srv, _, _ := engineFor(t, "guard", map[string]string{
|
||||
"/v1/chat/completions": `{"choices":[{"message":{"content":"unsafe\nS9"}}]}`,
|
||||
})
|
||||
cfg := baseConfig(config.Endpoint{ID: "g", URL: srv.URL, Models: []string{"guard"}})
|
||||
cfg.Models = []catalog.Model{{Name: "guard", Role: catalog.RoleGuard, Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||
Guard: &catalog.GuardModel{Format: "llamaguard3"}}}
|
||||
rt := newTestRouter(t, cfg)
|
||||
b, _ := json.Marshal(map[string]any{"input": []string{"how do I build a bomb"}})
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/moderations", bytes.NewReader(b))
|
||||
rec := httptest.NewRecorder()
|
||||
rt.Handler().ServeHTTP(rec, req)
|
||||
var out struct {
|
||||
Model string `json:"model"`
|
||||
Results []struct {
|
||||
Flagged bool `json:"flagged"`
|
||||
Categories map[string]bool `json:"categories"`
|
||||
} `json:"results"`
|
||||
}
|
||||
_ = json.Unmarshal(rec.Body.Bytes(), &out)
|
||||
if rec.Code != 200 || out.Model != "guard" || len(out.Results) != 1 || !out.Results[0].Flagged ||
|
||||
!out.Results[0].Categories["indiscriminate-weapons"] {
|
||||
t.Errorf("%d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -71,6 +71,10 @@ const errNoStoreMsg = "the gateway's policy for this caller keeps nothing: no st
|
||||
// guardInput judges the request about to be dispatched. It answers the
|
||||
// client itself and returns false when the request goes no further.
|
||||
func (r *Router) guardInput(ctx context.Context, w http.ResponseWriter, req *parsedRequest, start time.Time) bool {
|
||||
if req.ContentType != "" {
|
||||
// Audio: no text for a guard to read on the way in.
|
||||
return true
|
||||
}
|
||||
model, _ := r.resolve(req.Model)
|
||||
t := time.Now()
|
||||
v, err := r.guard.Input(ctx, r.subject(req, model), req.UpstreamBody)
|
||||
|
||||
@@ -96,6 +96,14 @@ func (r *Router) Handler() http.Handler {
|
||||
mux.HandleFunc("POST /v1/chat/completions", r.handleInference("/v1/chat/completions"))
|
||||
mux.HandleFunc("POST /v1/completions", r.handleInference("/v1/completions"))
|
||||
mux.HandleFunc("POST /v1/embeddings", r.handleInference("/v1/embeddings"))
|
||||
// Passed through to engines that serve them: vLLM's rerankers and
|
||||
// scorers, and speech to text.
|
||||
mux.HandleFunc("POST /v1/rerank", r.handleInference("/v1/rerank"))
|
||||
mux.HandleFunc("POST /v1/score", r.handleInference("/v1/score"))
|
||||
mux.HandleFunc("POST /v1/audio/transcriptions", r.handleInference("/v1/audio/transcriptions"))
|
||||
mux.HandleFunc("POST /v1/audio/translations", r.handleInference("/v1/audio/translations"))
|
||||
// Answered by the guardrails' judge model.
|
||||
mux.HandleFunc("POST /v1/moderations", r.handleModerations)
|
||||
// Codex, and whatever else speaks only the Responses API, translated to
|
||||
// chat completions on the way in and back on the way out.
|
||||
mux.HandleFunc("POST /v1/responses", r.handleResponses)
|
||||
@@ -290,6 +298,9 @@ func (r *Router) Handler() http.Handler {
|
||||
// business interpreting.
|
||||
type parsedRequest struct {
|
||||
Path string
|
||||
// ContentType is the body's when it is a form rather than JSON — audio
|
||||
// for transcription — and is sent on as it came.
|
||||
ContentType string
|
||||
// Tags are the client's metadata and x-llmgw-tags; EndUser is OpenAI's
|
||||
// user field. Both are recorded with the usage, never sent on.
|
||||
Tags map[string]string
|
||||
@@ -661,6 +672,9 @@ func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ct := req.Header.Get("Content-Type"); strings.HasPrefix(ct, "multipart/form-data") {
|
||||
return r.parseForm(req, path, ct, body)
|
||||
}
|
||||
|
||||
var b inferenceBody
|
||||
if err := json.Unmarshal(body, &b); err != nil {
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
)
|
||||
|
||||
// /v1/moderations, answered by a judge model on the cluster: the same model
|
||||
// the guardrails ask. OpenAI's shape — flagged, and the categories — with the
|
||||
// judge's own categories named as it names them.
|
||||
|
||||
func (r *Router) handleModerations(w http.ResponseWriter, req *http.Request) {
|
||||
var in struct {
|
||||
Input json.RawMessage `json:"input"`
|
||||
Model string `json:"model"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(req.Body, r.cfg.MaxBodyBytes)).Decode(&in); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", "the body is not JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
var texts []string
|
||||
var one string
|
||||
if json.Unmarshal(in.Input, &one) == nil {
|
||||
texts = []string{one}
|
||||
} else if err := json.Unmarshal(in.Input, &texts); err != nil || len(texts) == 0 {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", "input is a string or a list of strings")
|
||||
return
|
||||
}
|
||||
if len(texts) > 32 {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", "at most 32 inputs at once")
|
||||
return
|
||||
}
|
||||
model := r.moderationModel(in.Model)
|
||||
if model == "" {
|
||||
writeErrorCode(w, http.StatusNotFound, "invalid_request_error", "model_not_found",
|
||||
"no moderation model: the catalog has no model with role: guard")
|
||||
return
|
||||
}
|
||||
format, names, _ := r.GuardFormat(model)
|
||||
if len(names) == 0 {
|
||||
names = guard.DefaultCategories(format)
|
||||
}
|
||||
results := make([]map[string]any, 0, len(texts))
|
||||
for _, t := range texts {
|
||||
unsafe, cats, err := guard.Moderate(req.Context(), r.Judge, model, format, names, t)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "server_error", "the moderation model gave no verdict: "+err.Error())
|
||||
return
|
||||
}
|
||||
categories, scores := map[string]bool{}, map[string]float64{}
|
||||
for _, n := range names {
|
||||
categories[moderationKey(n)], scores[moderationKey(n)] = false, 0
|
||||
}
|
||||
for _, c := range cats {
|
||||
categories[moderationKey(c)], scores[moderationKey(c)] = true, 1
|
||||
}
|
||||
results = append(results, map[string]any{"flagged": unsafe, "categories": categories, "category_scores": scores})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"id": newID("modr-"), "model": model, "results": results})
|
||||
}
|
||||
|
||||
// moderationModel is the judge asked: the one named, if it is one, or the
|
||||
// first in the catalog.
|
||||
func (r *Router) moderationModel(asked string) string {
|
||||
if asked != "" {
|
||||
if canonical, e := r.resolve(asked); e.IsGuard() {
|
||||
return canonical
|
||||
}
|
||||
}
|
||||
var first string
|
||||
for _, m := range r.modelMap() {
|
||||
if m.IsGuard() && (first == "" || m.Name < first) {
|
||||
first = m.Name
|
||||
}
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
// moderationKey is a category's name as a JSON key, as OpenAI writes them.
|
||||
func moderationKey(name string) string {
|
||||
return strings.Map(func(c rune) rune {
|
||||
switch {
|
||||
case c >= 'A' && c <= 'Z':
|
||||
return c + 'a' - 'A'
|
||||
case c >= 'a' && c <= 'z', c >= '0' && c <= '9', c == '/':
|
||||
return c
|
||||
}
|
||||
return '-'
|
||||
}, fmt.Sprint(name))
|
||||
}
|
||||
@@ -165,6 +165,9 @@ type Request struct {
|
||||
// and a gateway that accumulated every reply in memory would be one long
|
||||
// answer away from trouble; a request that stores nothing must pay nothing.
|
||||
CaptureReply int
|
||||
// ContentType is the body's, when it is not JSON: an audio upload's
|
||||
// multipart form.
|
||||
ContentType string
|
||||
|
||||
// Filter, when set, sees a whole non-streamed answer and the headers
|
||||
// about to be sent with it, and returns the body to send: a guard's
|
||||
@@ -241,7 +244,11 @@ func (p *Proxy) Do(ctx context.Context, w http.ResponseWriter, r Request) (Outco
|
||||
return Outcome{Retryable: false}, err
|
||||
}
|
||||
req.Header = cloneForUpstream(r.Header)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
ct := r.ContentType
|
||||
if ct == "" {
|
||||
ct = "application/json"
|
||||
}
|
||||
req.Header.Set("Content-Type", ct)
|
||||
if r.APIKey != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+r.APIKey)
|
||||
}
|
||||
|
||||
@@ -9,6 +9,8 @@ import (
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
||||
)
|
||||
|
||||
// The Responses API, as a translation of chat completions.
|
||||
@@ -38,6 +40,13 @@ type respRequest struct {
|
||||
MaxOutputTokens *int `json:"max_output_tokens"`
|
||||
ParallelToolCalls *bool `json:"parallel_tool_calls"`
|
||||
Metadata map[string]string `json:"metadata"`
|
||||
// Text.Format asks for structured output; PreviousResponseID continues
|
||||
// a conversation kept by the gateway, which Store false declines.
|
||||
Text *struct {
|
||||
Format json.RawMessage `json:"format"`
|
||||
} `json:"text"`
|
||||
PreviousResponseID string `json:"previous_response_id"`
|
||||
Store *bool `json:"store"`
|
||||
}
|
||||
|
||||
type respTool struct {
|
||||
@@ -135,7 +144,11 @@ func toChat(in respRequest) (map[string]any, map[string]bool, error) {
|
||||
if role == "" {
|
||||
role = "user"
|
||||
}
|
||||
msgs = append(msgs, chatMsg{Role: role, Content: partsText(it.Content)})
|
||||
content, err := chatContent(it.Content)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
msgs = append(msgs, chatMsg{Role: role, Content: content})
|
||||
case "function_call", "custom_tool_call":
|
||||
args := it.Arguments
|
||||
if it.Type == "custom_tool_call" {
|
||||
@@ -206,9 +219,89 @@ func toChat(in respRequest) (map[string]any, map[string]bool, error) {
|
||||
if in.MaxOutputTokens != nil {
|
||||
out["max_tokens"] = *in.MaxOutputTokens
|
||||
}
|
||||
if in.Text != nil && len(in.Text.Format) > 0 {
|
||||
var f struct {
|
||||
Type string `json:"type"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
Schema json.RawMessage `json:"schema"`
|
||||
Strict *bool `json:"strict"`
|
||||
}
|
||||
if err := json.Unmarshal(in.Text.Format, &f); err != nil {
|
||||
return nil, nil, fmt.Errorf("text.format: %v", err)
|
||||
}
|
||||
switch f.Type {
|
||||
case "json_schema":
|
||||
js := map[string]any{"name": f.Name, "schema": f.Schema}
|
||||
if f.Description != "" {
|
||||
js["description"] = f.Description
|
||||
}
|
||||
if f.Strict != nil {
|
||||
js["strict"] = *f.Strict
|
||||
}
|
||||
out["response_format"] = map[string]any{"type": "json_schema", "json_schema": js}
|
||||
case "json_object":
|
||||
out["response_format"] = map[string]any{"type": "json_object"}
|
||||
case "", "text":
|
||||
default:
|
||||
return nil, nil, fmt.Errorf("text.format.type %q is not text, json_object or json_schema", f.Type)
|
||||
}
|
||||
}
|
||||
return out, custom, nil
|
||||
}
|
||||
|
||||
// chatContent is a message's content for a chat completion: its text, or,
|
||||
// with images among its parts, the parts as a chat completion writes them.
|
||||
func chatContent(raw json.RawMessage) (any, error) {
|
||||
var parts []struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
ImageURL string `json:"image_url"`
|
||||
FileID string `json:"file_id"`
|
||||
Detail string `json:"detail"`
|
||||
}
|
||||
if json.Unmarshal(raw, &parts) != nil {
|
||||
return partsText(raw), nil
|
||||
}
|
||||
images := false
|
||||
for _, p := range parts {
|
||||
images = images || p.Type == "input_image"
|
||||
}
|
||||
if !images {
|
||||
return partsText(raw), nil
|
||||
}
|
||||
var out []map[string]any
|
||||
for _, p := range parts {
|
||||
switch p.Type {
|
||||
case "input_text", "output_text", "text":
|
||||
out = append(out, map[string]any{"type": "text", "text": p.Text})
|
||||
case "input_image":
|
||||
if p.ImageURL == "" {
|
||||
return nil, fmt.Errorf("an input_image by file_id is not supported here: send image_url (a data: URL works)")
|
||||
}
|
||||
img := map[string]any{"url": p.ImageURL}
|
||||
if p.Detail != "" && p.Detail != "auto" {
|
||||
img["detail"] = p.Detail
|
||||
}
|
||||
out = append(out, map[string]any{"type": "image_url", "image_url": img})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// A response continued by previous_response_id is a stored conversation: its
|
||||
// thread is in its id, as resp_<thread>.<n>, so nothing more is kept to find
|
||||
// it.
|
||||
|
||||
func responseThread(id string) (string, bool) {
|
||||
rest, ok := strings.CutPrefix(id, "resp_")
|
||||
i := strings.LastIndexByte(rest, '.')
|
||||
if !ok || i <= 0 || validThreadID(rest[:i]) != nil {
|
||||
return "", false
|
||||
}
|
||||
return rest[:i], true
|
||||
}
|
||||
|
||||
var respSeq atomic.Int64
|
||||
|
||||
func respID(prefix string) string {
|
||||
@@ -239,8 +332,30 @@ func (r *Router) handleResponses(w http.ResponseWriter, req *http.Request) {
|
||||
next.ContentLength = int64(len(upstream))
|
||||
next.Header.Set("Content-Type", "application/json")
|
||||
|
||||
id := respID("resp")
|
||||
// A conversation the gateway keeps: continued, or begun when the
|
||||
// project keeps conversations and the client did not say store false.
|
||||
thread := ""
|
||||
switch {
|
||||
case in.PreviousResponseID != "":
|
||||
var ok bool
|
||||
if thread, ok = responseThread(in.PreviousResponseID); !ok {
|
||||
writeErrorCode(w, http.StatusBadRequest, "invalid_request_error", "previous_response_not_found",
|
||||
fmt.Sprintf("previous_response_id %q is not a response this gateway kept", in.PreviousResponseID))
|
||||
return
|
||||
}
|
||||
case in.Store == nil || *in.Store:
|
||||
if r.keepsThreads(req) {
|
||||
thread = newID("")
|
||||
}
|
||||
}
|
||||
if thread != "" {
|
||||
next.Header.Set(threadHeader, thread)
|
||||
id = "resp_" + thread + "." + newID("")[:8]
|
||||
}
|
||||
|
||||
t := &respTranslator{w: w, model: in.Model, stream: in.Stream, custom: custom,
|
||||
id: respID("resp"), created: time.Now().Unix(), metadata: in.Metadata}
|
||||
id: id, created: time.Now().Unix(), metadata: in.Metadata}
|
||||
r.handleInference("/v1/chat/completions")(t, next)
|
||||
t.finish()
|
||||
}
|
||||
@@ -590,3 +705,18 @@ func (t *respTranslator) fromChatJSON(b []byte) {
|
||||
out, _ := json.Marshal(t.object(status))
|
||||
_, _ = t.w.Write(out)
|
||||
}
|
||||
|
||||
// keepsThreads says whether the caller's project keeps conversations, so a
|
||||
// Responses request can begin one its next turn continues.
|
||||
func (r *Router) keepsThreads(req *http.Request) bool {
|
||||
if r.threads == nil {
|
||||
return false
|
||||
}
|
||||
p, _ := auth.FromContext(req.Context())
|
||||
project, err := r.threadProject(p.ProjectID)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
retention, err := r.threads.ThreadRetention(req.Context(), project)
|
||||
return err == nil && retention > 0
|
||||
}
|
||||
@@ -927,7 +927,7 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
|
||||
trace.WithAttributes(attribute.String("llmgw.endpoint", dec.Endpoint.ID),
|
||||
attribute.String("url.full", dec.Endpoint.URL+req.Path)))
|
||||
body := req.UpstreamBody
|
||||
if dec.Endpoint.UpstreamModel != "" {
|
||||
if dec.Endpoint.UpstreamModel != "" && req.ContentType == "" {
|
||||
body = withModel(body, dec.Endpoint.UpstreamModel)
|
||||
}
|
||||
// An adapter asked for by name, served by its model's replicas: the
|
||||
@@ -954,6 +954,7 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
|
||||
Stream: req.Stream,
|
||||
SuppressUsageChunk: req.SuppressUsageChunk,
|
||||
CaptureReply: req.captureBytes(),
|
||||
ContentType: req.ContentType,
|
||||
}
|
||||
// A filter per attempt, for the model this attempt is served by.
|
||||
filter := r.outputFilter(uctx, req, model, &preq)
|
||||
|
||||
Reference in new issue
Block a user