Batches: OpenAI's Files and Batch APIs, served at the lowest priority under a store lease, each line a request of the key that made it

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 21:00:01 +02:00
1 parent 889be70b8f
commit 841ee21c12
19 files changed
+1775 -10

No files matched your search

+255
View File
@@ -0,0 +1,255 @@
package router
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strconv"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// The Batch API: a file of requests, served one by one at the lowest
// priority, its answers written to another file. OpenAI's shape, so its
// clients work unchanged.
// batchEndpoints are what a batch may be of.
var batchEndpoints = map[string]bool{
"/v1/chat/completions": true,
"/v1/completions": true,
"/v1/embeddings": true,
}
// batchWindow is the only completion window, as OpenAI has it.
const batchWindow = "24h"
func unixOrNil(t time.Time) any {
if t.IsZero() {
return nil
}
return t.Unix()
}
func strOrNil(s string) any {
if s == "" {
return nil
}
return s
}
func batchJSON(b store.Batch) map[string]any {
out := map[string]any{
"id": b.ID, "object": "batch", "endpoint": b.Endpoint, "input_file_id": b.InputFileID,
"completion_window": b.CompletionWindow, "status": b.Status,
"output_file_id": strOrNil(b.OutputFileID), "error_file_id": strOrNil(b.ErrorFileID),
"created_at": b.CreatedAt.Unix(), "in_progress_at": unixOrNil(b.InProgressAt),
"expires_at": unixOrNil(b.ExpiresAt), "finalizing_at": unixOrNil(b.FinalizingAt),
"completed_at": unixOrNil(b.CompletedAt), "failed_at": unixOrNil(b.FailedAt),
"expired_at": unixOrNil(b.ExpiredAt), "cancelling_at": unixOrNil(b.CancellingAt),
"cancelled_at": unixOrNil(b.CancelledAt),
"request_counts": map[string]int{"total": b.Total, "completed": b.Completed, "failed": b.Failed},
"errors": nil, "metadata": nil,
}
if b.Errors != "" {
out["errors"] = json.RawMessage(b.Errors)
}
if b.Metadata != "" {
out["metadata"] = json.RawMessage(b.Metadata)
}
return out
}
func (r *Router) handleCreateBatch(w http.ResponseWriter, req *http.Request) {
p, ok := r.batchPrincipal(w, req)
if !ok {
return
}
var in struct {
InputFileID string `json:"input_file_id"`
Endpoint string `json:"endpoint"`
CompletionWindow string `json:"completion_window"`
Metadata json.RawMessage `json:"metadata"`
}
if err := json.NewDecoder(io.LimitReader(req.Body, 1<<16)).Decode(&in); err != nil {
writeError(w, http.StatusBadRequest, "invalid_request_error", "the request is not valid JSON: "+err.Error())
return
}
if !batchEndpoints[in.Endpoint] {
writeError(w, http.StatusBadRequest, "invalid_request_error",
fmt.Sprintf("endpoint %q: a batch is of /v1/chat/completions, /v1/completions or /v1/embeddings", in.Endpoint))
return
}
if in.CompletionWindow != batchWindow {
writeError(w, http.StatusBadRequest, "invalid_request_error", "completion_window is 24h")
return
}
meta := ""
if len(in.Metadata) > 0 && string(in.Metadata) != "null" {
if _, err := requestTags(in.Metadata, ""); err != nil {
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
return
}
meta = string(in.Metadata)
}
f, err := r.store.FileByID(req.Context(), in.InputFileID)
if errors.Is(err, store.ErrNotFound) || err == nil && (f.Owner != ownerOf(p) || f.Purpose != purposeBatch) {
writeErrorCode(w, http.StatusBadRequest, "invalid_request_error", "file_not_found",
fmt.Sprintf("no batch file %q", in.InputFileID))
return
}
if err != nil {
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
return
}
now := time.Now()
b := store.Batch{ID: newID("batch_"), Owner: ownerOf(p), KeyID: p.KeyID, Endpoint: in.Endpoint,
InputFileID: f.ID, CompletionWindow: batchWindow, Status: store.BatchValidating,
CreatedAt: now, ExpiresAt: now.Add(24 * time.Hour), Metadata: meta}
if err := r.store.PutBatch(req.Context(), b); err != nil {
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
return
}
r.wakeBatches()
writeJSON(w, http.StatusOK, batchJSON(b))
}
// ownedBatch is a batch the caller owns, or a 404 said.
func (r *Router) ownedBatch(w http.ResponseWriter, req *http.Request, owner string) (store.Batch, bool) {
b, err := r.store.BatchByID(req.Context(), req.PathValue("id"))
if errors.Is(err, store.ErrNotFound) || err == nil && b.Owner != owner {
writeErrorCode(w, http.StatusNotFound, "invalid_request_error", "batch_not_found",
fmt.Sprintf("no batch %q", req.PathValue("id")))
return store.Batch{}, false
}
if err != nil {
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
return store.Batch{}, false
}
return b, true
}
func (r *Router) handleGetBatch(w http.ResponseWriter, req *http.Request) {
p, ok := r.batchPrincipal(w, req)
if !ok {
return
}
if b, ok := r.ownedBatch(w, req, ownerOf(p)); ok {
writeJSON(w, http.StatusOK, batchJSON(b))
}
}
func (r *Router) handleCancelBatch(w http.ResponseWriter, req *http.Request) {
p, ok := r.batchPrincipal(w, req)
if !ok {
return
}
b, ok := r.ownedBatch(w, req, ownerOf(p))
if !ok {
return
}
b, err := r.store.CancelBatch(req.Context(), b.ID, time.Now())
if err != nil {
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
return
}
r.wakeBatches()
writeJSON(w, http.StatusOK, batchJSON(b))
}
func (r *Router) handleListBatches(w http.ResponseWriter, req *http.Request) {
p, ok := r.batchPrincipal(w, req)
if !ok {
return
}
limit, _ := strconv.Atoi(req.URL.Query().Get("limit"))
if limit <= 0 || limit > 100 {
limit = 20
}
bs, err := r.store.Batches(req.Context(), ownerOf(p), req.URL.Query().Get("after"), limit+1)
if err != nil {
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
return
}
more := len(bs) > limit
if more {
bs = bs[:limit]
}
data := make([]map[string]any, 0, len(bs))
for _, b := range bs {
data = append(data, batchJSON(b))
}
out := map[string]any{"object": "list", "data": data, "has_more": more, "first_id": nil, "last_id": nil}
if len(bs) > 0 {
out["first_id"], out["last_id"] = bs[0].ID, bs[len(bs)-1].ID
}
writeJSON(w, http.StatusOK, out)
}
// batchLine is one line of a batch's input file.
type batchLine struct {
CustomID string `json:"custom_id"`
Method string `json:"method"`
URL string `json:"url"`
Body json.RawMessage `json:"body"`
}
// lineError is one reason a batch's input was refused, as OpenAI lists them.
type lineError struct {
Code string `json:"code"`
Message string `json:"message"`
Line int `json:"line,omitempty"`
}
// parseBatchInput reads a batch's input file, or says every line that is
// wrong with it.
func parseBatchInput(content []byte, endpoint string, maxLines int) ([]batchLine, []lineError) {
var lines []batchLine
var errs []lineError
seen := map[string]bool{}
n := 0
for raw := range bytes.SplitSeq(content, []byte("\n")) {
n++
raw = bytes.TrimSpace(raw)
if len(raw) == 0 {
continue
}
bad := func(code, msg string) {
if len(errs) < 100 {
errs = append(errs, lineError{Code: code, Message: msg, Line: n})
}
}
var l batchLine
if err := json.Unmarshal(raw, &l); err != nil {
bad("invalid_json_line", "not a JSON object: "+err.Error())
continue
}
var body map[string]any
switch {
case l.CustomID == "":
bad("missing_custom_id", "custom_id is required")
case seen[l.CustomID]:
bad("duplicate_custom_id", fmt.Sprintf("custom_id %q is used twice", l.CustomID))
case l.Method != http.MethodPost:
bad("invalid_method", "method is POST")
case l.URL != endpoint:
bad("mismatched_endpoint", fmt.Sprintf("url %q is not the batch's endpoint, %s", l.URL, endpoint))
case json.Unmarshal(l.Body, &body) != nil || body["model"] == nil:
bad("invalid_body", "body is an object naming a model")
default:
seen[l.CustomID] = true
lines = append(lines, l)
}
}
if len(lines)+len(errs) == 0 {
errs = append(errs, lineError{Code: "empty_file", Message: "the file has no requests"})
}
if len(lines) > maxLines {
errs = append(errs, lineError{Code: "too_many_requests",
Message: fmt.Sprintf("%d requests; a batch holds at most %d", len(lines), maxLines)})
}
return lines, errs
}
+241
View File
@@ -0,0 +1,241 @@
package router
import (
"bytes"
"crypto/sha256"
"encoding/json"
"fmt"
"mime/multipart"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
func upload(t *testing.T, h http.Handler, secret, content string) (int, map[string]any) {
t.Helper()
var buf bytes.Buffer
mw := multipart.NewWriter(&buf)
_ = mw.WriteField("purpose", "batch")
fw, _ := mw.CreateFormFile("file", "in.jsonl")
_, _ = fw.Write([]byte(content))
_ = mw.Close()
req := httptest.NewRequest(http.MethodPost, "/v1/files", &buf)
req.Header.Set("Content-Type", mw.FormDataContentType())
req.Header.Set("Authorization", "Bearer "+secret)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
var out map[string]any
_ = json.Unmarshal(rec.Body.Bytes(), &out)
return rec.Code, out
}
func call(t *testing.T, h http.Handler, secret, method, path string, body any) (int, map[string]any, string) {
t.Helper()
var rd *bytes.Reader
if body != nil {
b, _ := json.Marshal(body)
rd = bytes.NewReader(b)
} else {
rd = bytes.NewReader(nil)
}
req := httptest.NewRequest(method, path, rd)
req.Header.Set("Authorization", "Bearer "+secret)
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
var out map[string]any
_ = json.Unmarshal(rec.Body.Bytes(), &out)
return rec.Code, out, rec.Body.String()
}
func waitBatch(t *testing.T, h http.Handler, secret, id string) map[string]any {
t.Helper()
deadline := time.Now().Add(10 * time.Second)
for {
_, b, _ := call(t, h, secret, http.MethodGet, "/v1/batches/"+id, nil)
if store.BatchDone(fmt.Sprint(b["status"])) {
return b
}
if time.Now().After(deadline) {
t.Fatalf("the batch did not end: %v", b)
}
time.Sleep(50 * time.Millisecond)
}
}
func TestABatchIsServedAndItsAnswersWritten(t *testing.T) {
e := newFakeEngine(t, "qwen2.5-14b-instruct")
rt, secret := pricedRouter(t, e.srv.URL)
h := rt.Handler()
var in strings.Builder
for i := range 5 {
fmt.Fprintf(&in, `{"custom_id":"r%d","method":"POST","url":"/v1/chat/completions","body":{"model":"q14","messages":[{"role":"user","content":"hi"}]}}`+"\n", i)
}
// One the engine cannot serve: a model that is not there.
in.WriteString(`{"custom_id":"bad","method":"POST","url":"/v1/chat/completions","body":{"model":"nope","messages":[]}}` + "\n")
code, f := upload(t, h, secret, in.String())
if code != 200 || f["purpose"] != "batch" {
t.Fatalf("upload: %d %v", code, f)
}
code, b, raw := call(t, h, secret, http.MethodPost, "/v1/batches", map[string]any{
"input_file_id": f["id"], "endpoint": "/v1/chat/completions", "completion_window": "24h",
"metadata": map[string]string{"job": "nightly"}})
if code != 200 || b["status"] != "validating" {
t.Fatalf("create: %d %s", code, raw)
}
b = waitBatch(t, h, secret, b["id"].(string))
counts := b["request_counts"].(map[string]any)
if b["status"] != "completed" || counts["total"] != 6.0 || counts["completed"] != 5.0 || counts["failed"] != 1.0 {
t.Fatalf("ended %v", b)
}
if e.calls.Load() != 5 {
t.Errorf("the engine served %d", e.calls.Load())
}
_, _, out := call(t, h, secret, http.MethodGet, "/v1/files/"+b["output_file_id"].(string)+"/content", nil)
lines := strings.Split(strings.TrimSpace(out), "\n")
if len(lines) != 5 || !strings.Contains(lines[0], `"custom_id":"r`) || !strings.Contains(lines[0], `"status_code":200`) {
t.Errorf("output:\n%s", out)
}
_, _, errs := call(t, h, secret, http.MethodGet, "/v1/files/"+b["error_file_id"].(string)+"/content", nil)
if !strings.Contains(errs, `"custom_id":"bad"`) {
t.Errorf("errors:\n%s", errs)
}
// Charged like any request, and findable by the batch.
var rid string
for _, l := range lines {
var o struct {
Response struct {
RequestID string `json:"request_id"`
} `json:"response"`
}
_ = json.Unmarshal([]byte(l), &o)
rid = o.Response.RequestID
}
deadline := time.Now().Add(5 * time.Second)
for {
u, ok, _ := rt.store.UsageByRequest(t.Context(), rid)
if ok {
if u.Tags["batch"] != b["id"] || u.CostMicros == 0 {
t.Errorf("usage %+v", u)
}
break
}
if time.Now().After(deadline) {
t.Fatal("a batch request left no usage")
}
time.Sleep(20 * time.Millisecond)
}
_, list, _ := call(t, h, secret, http.MethodGet, "/v1/batches", nil)
if d := list["data"].([]any); len(d) != 1 {
t.Errorf("listed %v", list)
}
}
func TestABatchWithBadLinesFailsAndSaysWhich(t *testing.T) {
e := newFakeEngine(t, "qwen2.5-14b-instruct")
rt, secret := pricedRouter(t, e.srv.URL)
h := rt.Handler()
_, f := upload(t, h, secret, `{"custom_id":"a","method":"POST","url":"/v1/embeddings","body":{"model":"q14"}}
not json
{"custom_id":"a","method":"POST","url":"/v1/chat/completions","body":{"model":"q14"}}
`)
_, b, _ := call(t, h, secret, http.MethodPost, "/v1/batches", map[string]any{
"input_file_id": f["id"], "endpoint": "/v1/chat/completions", "completion_window": "24h"})
b = waitBatch(t, h, secret, b["id"].(string))
j, _ := json.Marshal(b["errors"])
if b["status"] != "failed" || !strings.Contains(string(j), "mismatched_endpoint") || !strings.Contains(string(j), "invalid_json_line") {
t.Errorf("%v %s", b["status"], j)
}
if e.calls.Load() != 0 {
t.Error("a batch that failed validation was served")
}
}
func TestBatchesAndFilesAreTheirOwners(t *testing.T) {
e := newFakeEngine(t, "qwen2.5-14b-instruct")
rt, secret := pricedRouter(t, e.srv.URL)
h := rt.Handler()
_, f := upload(t, h, secret, `{"custom_id":"a","method":"POST","url":"/v1/chat/completions","body":{"model":"q14"}}`)
// Another project's key sees none of it.
org, _ := rt.store.CreateOrg(t.Context(), "other")
team, _ := rt.store.CreateTeam(t.Context(), org, "t")
project, _ := rt.store.CreateProject(t.Context(), team, "p")
other := store.KeyPrefix + strings.Repeat("b", 43)
sum := sha256.Sum256([]byte(other))
if err := rt.store.CreateKey(t.Context(), store.Key{ID: "key-2", Hash: sum[:], Prefix: "bbbb", Name: "other",
ProjectID: project, CreatedAt: time.Now()}); err != nil {
t.Fatal(err)
}
if code, _, _ := call(t, h, other, http.MethodGet, "/v1/files/"+f["id"].(string), nil); code != 404 {
t.Errorf("another project read a file: %d", code)
}
if code, _, _ := call(t, h, other, http.MethodPost, "/v1/batches", map[string]any{
"input_file_id": f["id"], "endpoint": "/v1/chat/completions", "completion_window": "24h"}); code != 400 {
t.Errorf("another project batched a file: %d", code)
}
if code, _, _ := call(t, h, secret, http.MethodDelete, "/v1/files/"+f["id"].(string), nil); code != 200 {
t.Errorf("delete: %d", code)
}
}
func TestABatchPicksUpWhereItStoppedAndCanBeCancelled(t *testing.T) {
e := newFakeEngine(t, "qwen2.5-14b-instruct")
rt, secret := pricedRouter(t, e.srv.URL)
h := rt.Handler()
var in strings.Builder
for i := range 4 {
fmt.Fprintf(&in, `{"custom_id":"r%d","method":"POST","url":"/v1/chat/completions","body":{"model":"q14","messages":[]}}`+"\n", i)
}
_, f := upload(t, h, secret, in.String())
// As a gateway that died after serving the first line left it.
now := time.Now()
b := store.Batch{ID: "batch_resumed", Owner: "project:" + mustProjectOf(t, rt, "key-1"), KeyID: "key-1",
Endpoint: "/v1/chat/completions", InputFileID: f["id"].(string), CompletionWindow: "24h",
Status: store.BatchInProgress, CreatedAt: now, ExpiresAt: now.Add(24 * time.Hour)}
if err := rt.store.PutBatch(t.Context(), b); err != nil {
t.Fatal(err)
}
b.Total, b.Completed, b.InProgressAt = 4, 1, now
_ = rt.store.UpdateBatch(t.Context(), b)
_ = rt.store.PutBatchResult(t.Context(), b.ID, store.BatchResult{Line: 0, OK: true,
Data: `{"id":"batch_req_x","custom_id":"r0","response":{"status_code":200},"error":null}`})
rt.wakeBatches()
got := waitBatch(t, h, secret, b.ID)
if c := got["request_counts"].(map[string]any); got["status"] != "completed" || c["completed"] != 4.0 {
t.Fatalf("%v", got)
}
if e.calls.Load() != 3 {
t.Errorf("served %d; the line kept already was served again", e.calls.Load())
}
// Cancelled before it is served: nothing is sent.
b2 := b
b2.ID, b2.Status = "batch_cancelled", store.BatchValidating
if err := rt.store.PutBatch(t.Context(), b2); err != nil {
t.Fatal(err)
}
_, _ = rt.store.CancelBatch(t.Context(), b2.ID, time.Now())
rt.wakeBatches()
if got := waitBatch(t, h, secret, b2.ID); got["status"] != "cancelled" {
t.Errorf("%v", got)
}
if e.calls.Load() != 3 {
t.Errorf("a cancelled batch was served: %d", e.calls.Load())
}
}
func mustProjectOf(t *testing.T, rt *Router, keyID string) string {
t.Helper()
k, err := rt.store.KeyByID(t.Context(), keyID)
if err != nil {
t.Fatal(err)
}
return k.ProjectID
}
+373
View File
@@ -0,0 +1,373 @@
package router
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"os"
"strconv"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// The worker that serves batches. Any gateway on the store may run one: a
// batch is worked on by whichever holds its lease, and each finished line
// is kept as it is served, so another picks up where one stopped.
const (
batchLease = 2 * time.Minute
batchPoll = 5 * time.Second
)
// wakeBatches has the worker look for work now rather than at its next poll.
func (r *Router) wakeBatches() {
select {
case r.batchWake <- struct{}{}:
default:
}
}
func (r *Router) runBatches(ctx context.Context) {
host, _ := os.Hostname()
owner := fmt.Sprintf("%s:%d:%s", host, os.Getpid(), newID(""))
t := time.NewTicker(batchPoll)
defer t.Stop()
for {
for {
now := time.Now()
b, ok, err := r.store.ClaimBatch(ctx, owner, now, now.Add(batchLease))
if err != nil {
if ctx.Err() == nil {
r.log.Warn("could not look for batches to serve", "err", err)
}
break
}
if !ok {
break
}
r.workBatch(ctx, owner, b)
}
if _, _, _, days := r.batchCfg(); days > 0 {
if n, err := r.store.DeleteExpiredFiles(ctx, time.Now()); err == nil && n > 0 {
r.log.Info("batch files past their retention deleted", "files", n)
}
}
select {
case <-ctx.Done():
return
case <-t.C:
case <-r.batchWake:
}
}
}
// workBatch takes one batch as far as it goes: validated, served, written.
func (r *Router) workBatch(ctx context.Context, owner string, b store.Batch) {
bctx, stop := context.WithCancel(ctx)
defer stop()
defer func() { _ = r.store.ReleaseBatch(context.WithoutCancel(ctx), b.ID, owner) }()
// The lease kept while working; lost, the work stops.
go func() {
t := time.NewTicker(batchLease / 4)
defer t.Stop()
for {
select {
case <-bctx.Done():
return
case <-t.C:
if held, err := r.store.RenewBatchLease(bctx, b.ID, owner, time.Now().Add(batchLease)); err == nil && !held {
stop()
return
}
}
}
}()
log := r.log.With("batch", b.ID)
if err := r.serveBatch(bctx, &b); err != nil && bctx.Err() == nil {
log.Warn("a batch stopped with an error; it is picked up again", "err", err)
}
}
func (r *Router) serveBatch(ctx context.Context, b *store.Batch) error {
content, err := r.store.FileContent(ctx, b.InputFileID)
if err != nil && !errors.Is(err, store.ErrNotFound) {
return err
}
_, maxLines, concurrency, _ := r.batchCfg()
lines, errs := parseBatchInput(content, b.Endpoint, maxLines)
if errors.Is(err, store.ErrNotFound) {
errs = []lineError{{Code: "file_not_found", Message: "the input file was deleted"}}
}
if b.Status == store.BatchValidating {
if len(errs) > 0 {
ej, _ := json.Marshal(map[string]any{"object": "list", "data": errs})
b.Status, b.FailedAt, b.Errors = store.BatchFailed, time.Now(), string(ej)
return r.store.UpdateBatch(ctx, *b)
}
b.Status, b.Total, b.InProgressAt = store.BatchInProgress, len(lines), time.Now()
if err := r.store.UpdateBatch(ctx, *b); err != nil {
return err
}
}
if b.Status == store.BatchInProgress {
end, err := r.serveLines(ctx, b, lines, concurrency)
if err != nil {
return err
}
switch end {
case store.BatchCancelling, store.BatchExpired:
b.Status = end
default:
b.Status, b.FinalizingAt = store.BatchFinalizing, time.Now()
if err := r.store.UpdateBatch(ctx, *b); err != nil {
return err
}
}
}
// Finalizing, cancelling or expired: what was served is written out.
return r.finishBatch(ctx, b, lines)
}
// serveLines serves the lines not yet served, a few at a time, and says why
// it stopped: "" when every line is served.
func (r *Router) serveLines(ctx context.Context, b *store.Batch, lines []batchLine, concurrency int) (string, error) {
done, err := r.store.BatchResults(ctx, b.ID)
if err != nil {
return "", err
}
served := map[int]bool{}
for _, d := range done {
served[d.Line] = true
}
p, perr := r.batchPrincipalOf(ctx, b.KeyID)
var mu sync.Mutex
stopWith := ""
lastCheck := time.Now()
sem := make(chan struct{}, concurrency)
var wg sync.WaitGroup
for i, l := range lines {
if served[i] {
continue
}
mu.Lock()
stopped := stopWith != ""
mu.Unlock()
if stopped || ctx.Err() != nil {
break
}
// Asked to stop, or out of time: looked at now and then, not per line.
if time.Since(lastCheck) > 2*time.Second {
lastCheck = time.Now()
if cur, err := r.store.BatchByID(ctx, b.ID); err == nil && cur.Status == store.BatchCancelling {
mu.Lock()
stopWith = store.BatchCancelling
mu.Unlock()
break
}
if time.Now().After(b.ExpiresAt) {
mu.Lock()
stopWith = store.BatchExpired
mu.Unlock()
break
}
}
sem <- struct{}{}
wg.Add(1)
go func(i int, l batchLine) {
defer wg.Done()
defer func() { <-sem }()
res := r.serveLine(ctx, b, p, perr, l)
if ctx.Err() != nil {
return
}
if err := r.store.PutBatchResult(ctx, b.ID, res.withLine(i)); err != nil {
r.log.Warn("a batch's answer could not be kept", "batch", b.ID, "err", err)
return
}
mu.Lock()
if res.OK {
b.Completed++
} else {
b.Failed++
}
_ = r.store.UpdateBatch(ctx, *b)
mu.Unlock()
}(i, l)
}
wg.Wait()
if ctx.Err() != nil {
return "", ctx.Err()
}
if stopWith == "" {
// Counted from what is kept, which is the truth after a restart.
rs, err := r.store.BatchResults(ctx, b.ID)
if err != nil {
return "", err
}
if len(rs) < len(lines) {
return "", fmt.Errorf("%d of %d lines served", len(rs), len(lines))
}
}
return stopWith, nil
}
// batchPrincipalOf is the key a batch was made with, as it stands now: a key
// revoked since serves no more of it.
func (r *Router) batchPrincipalOf(ctx context.Context, keyID string) (auth.Principal, error) {
k, err := r.store.KeyByID(ctx, keyID)
if err != nil {
return auth.Principal{}, err
}
switch {
case !k.RevokedAt.IsZero():
return auth.Principal{}, auth.ErrRevoked
case !k.ExpiresAt.IsZero() && time.Now().After(k.ExpiresAt):
return auth.Principal{}, auth.ErrExpired
}
eff, err := r.store.Resolve(ctx, k.ID)
if err != nil {
return auth.Principal{}, err
}
return auth.Principal{
KeyID: k.ID, KeyPrefix: k.Prefix, UserID: eff.UserID, ProjectID: eff.ProjectID,
TeamID: eff.TeamID, OrgID: eff.OrgID,
Band: r.batchBand(), RoutingProfile: eff.RoutingProfile, Limits: eff,
}, nil
}
// batchBand is the lowest admission band: a batch waits behind everyone.
func (r *Router) batchBand() string {
band, prio := "", 0
for _, b := range r.cfg.Admission.Bands {
if band == "" || b.Priority < prio {
band, prio = b.Name, b.Priority
}
}
return band
}
// lineResult is a served line as it is kept.
type lineResult struct {
OK bool
Data string
}
func (l lineResult) withLine(i int) store.BatchResult {
return store.BatchResult{Line: i, OK: l.OK, Data: l.Data}
}
// serveLine sends one request the way a client's is sent — limits,
// accounting, guardrails, admission in the lowest band — and keeps its
// answer in OpenAI's batch output form. A limit that lifts with time is
// waited out; one that does not fails the line.
func (r *Router) serveLine(ctx context.Context, b *store.Batch, p auth.Principal, perr error, l batchLine) lineResult {
id := newID("batch_req_")
out := map[string]any{"id": id, "custom_id": l.CustomID, "response": nil, "error": nil}
fail := func(code, msg string) lineResult {
out["error"] = map[string]string{"code": code, "message": msg}
j, _ := json.Marshal(out)
return lineResult{Data: string(j)}
}
if perr != nil {
return fail("key_unusable", "the key this batch was made with can no longer be used: "+perr.Error())
}
body := withoutField(l.Body, "stream")
handler := r.handleInference(b.Endpoint)
for attempt := 1; ; attempt++ {
hreq, err := http.NewRequestWithContext(auth.WithPrincipal(ctx, p), http.MethodPost, b.Endpoint, bytes.NewReader(body))
if err != nil {
return fail("invalid_request", err.Error())
}
hreq.Header.Set("Content-Type", "application/json")
hreq.Header.Set("X-Request-Id", id)
hreq.Header.Set(tagsHeader, "batch="+b.ID)
w := &bufferWriter{header: http.Header{}}
handler(w, hreq)
status := w.status
if status == 0 {
status = http.StatusOK
}
// Waited out: a rate limit, a model loading, a queue full. Not a
// spent budget, which says no Retry-After because waiting does not
// help, and not past the batch's time.
if wait := w.header.Get("Retry-After"); wait != "" && (status == http.StatusTooManyRequests || status >= 500) &&
time.Now().Before(b.ExpiresAt) && attempt < 1000 {
secs, _ := strconv.Atoi(wait)
select {
case <-ctx.Done():
return fail("cancelled", "the gateway stopped")
case <-time.After(time.Duration(max(secs, 1)) * time.Second):
}
continue
}
var parsed any = json.RawMessage(bytes.TrimSpace(w.buf.Bytes()))
if !json.Valid(w.buf.Bytes()) {
parsed = w.buf.String()
}
out["response"] = map[string]any{"status_code": status, "request_id": id, "body": parsed}
j, _ := json.Marshal(out)
return lineResult{OK: status < 300, Data: string(j)}
}
}
// finishBatch writes what was served — answers to one file, failures to
// another — and ends the batch: completed, cancelled or expired.
func (r *Router) finishBatch(ctx context.Context, b *store.Batch, lines []batchLine) error {
rs, err := r.store.BatchResults(ctx, b.ID)
if err != nil {
return err
}
var okBuf, errBuf bytes.Buffer
b.Completed, b.Failed = 0, 0
for _, res := range rs {
if res.OK {
okBuf.WriteString(res.Data + "\n")
b.Completed++
} else {
errBuf.WriteString(res.Data + "\n")
b.Failed++
}
}
_, _, _, days := r.batchCfg()
now := time.Now()
keep := func(buf *bytes.Buffer, name string) (string, error) {
if buf.Len() == 0 {
return "", nil
}
f := store.File{ID: newID("file-"), Owner: b.Owner, Purpose: purposeBatchOutput,
Filename: b.ID + "_" + name + ".jsonl", CreatedAt: now, ExpiresAt: now.AddDate(0, 0, days)}
return f.ID, r.store.PutFile(ctx, f, buf.Bytes())
}
if b.OutputFileID == "" {
if b.OutputFileID, err = keep(&okBuf, "output"); err != nil {
return err
}
}
if b.ErrorFileID == "" {
if b.ErrorFileID, err = keep(&errBuf, "error"); err != nil {
return err
}
}
switch b.Status {
case store.BatchCancelling:
b.Status, b.CancelledAt = store.BatchCancelled, now
case store.BatchExpired:
b.ExpiredAt = now
default:
b.Status, b.CompletedAt = store.BatchCompleted, now
}
if err := r.store.UpdateBatch(ctx, *b); err != nil {
return err
}
return r.store.DeleteBatchResults(ctx, b.ID)
}
+207
View File
@@ -0,0 +1,207 @@
package router
import (
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io"
"net/http"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// The Files API, as far as batches need it: a client uploads a file of
// requests, and reads back the files a batch writes.
const (
purposeBatch = "batch"
purposeBatchOutput = "batch_output"
)
func newID(prefix string) string {
var b [12]byte
_, _ = rand.Read(b[:])
return prefix + hex.EncodeToString(b[:])
}
// ownerOf is who a file or a batch belongs to: the key's project, so every
// key of it sees them, or the person for a personal key.
func ownerOf(p auth.Principal) string {
switch {
case p.ProjectID != "":
return "project:" + p.ProjectID
case p.UserID != "":
return "user:" + p.UserID
}
return "key:" + p.KeyID
}
// batchPrincipal is the caller, or a refusal when there is no store to keep
// files and batches in.
func (r *Router) batchPrincipal(w http.ResponseWriter, req *http.Request) (auth.Principal, bool) {
p, ok := auth.FromContext(req.Context())
if !ok || r.store == nil {
writeError(w, http.StatusNotImplemented, "invalid_request_error",
"files and batches need authentication and the store, which this gateway runs without")
return auth.Principal{}, false
}
// A batch's files are prompts and answers kept: not under a policy
// that keeps nothing.
if r.noStore(guard.Subject{KeyID: p.KeyID, UserID: p.UserID, ProjectID: p.ProjectID,
TeamID: p.TeamID, OrgID: p.OrgID, Path: req.URL.Path}) {
writeError(w, http.StatusForbidden, "invalid_request_error",
"the gateway's policy for this caller keeps nothing, and a batch is prompts and answers kept")
return auth.Principal{}, false
}
return p, true
}
func (r *Router) batchCfg() (maxBytes int64, maxLines, concurrency, retainDays int) {
b := r.cfg.Batch
maxBytes, maxLines, concurrency, retainDays = b.MaxFileBytes, b.MaxLines, b.Concurrency, b.RetainDays
if maxBytes <= 0 {
maxBytes = 100 << 20
}
if maxLines <= 0 {
maxLines = 50000
}
if concurrency <= 0 {
concurrency = 4
}
if retainDays <= 0 {
retainDays = 30
}
return
}
func fileJSON(f store.File) map[string]any {
out := map[string]any{"id": f.ID, "object": "file", "bytes": f.Bytes, "created_at": f.CreatedAt.Unix(),
"filename": f.Filename, "purpose": f.Purpose, "status": "processed"}
if !f.ExpiresAt.IsZero() {
out["expires_at"] = f.ExpiresAt.Unix()
}
return out
}
func (r *Router) handleUploadFile(w http.ResponseWriter, req *http.Request) {
p, ok := r.batchPrincipal(w, req)
if !ok {
return
}
maxBytes, _, _, _ := r.batchCfg()
req.Body = http.MaxBytesReader(w, req.Body, maxBytes+1<<20)
if err := req.ParseMultipartForm(32 << 20); err != nil {
writeError(w, http.StatusBadRequest, "invalid_request_error",
fmt.Sprintf("a multipart form with purpose and file, at most %d MB: %v", maxBytes>>20, err))
return
}
if purpose := req.FormValue("purpose"); purpose != purposeBatch {
writeError(w, http.StatusBadRequest, "invalid_request_error",
fmt.Sprintf("purpose %q: this gateway keeps files for batches only (purpose batch)", purpose))
return
}
file, header, err := req.FormFile("file")
if err != nil {
writeError(w, http.StatusBadRequest, "invalid_request_error", "no file in the form: "+err.Error())
return
}
defer file.Close()
content, err := io.ReadAll(io.LimitReader(file, maxBytes+1))
if err != nil {
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
return
}
if int64(len(content)) > maxBytes {
writeError(w, http.StatusRequestEntityTooLarge, "invalid_request_error",
fmt.Sprintf("the file is larger than %d MB", maxBytes>>20))
return
}
f := store.File{ID: newID("file-"), Owner: ownerOf(p), Purpose: purposeBatch,
Filename: header.Filename, Bytes: int64(len(content)), CreatedAt: time.Now()}
if err := r.store.PutFile(req.Context(), f, content); err != nil {
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
return
}
writeJSON(w, http.StatusOK, fileJSON(f))
}
// ownedFile is a file the caller owns, or a 404 said.
func (r *Router) ownedFile(w http.ResponseWriter, req *http.Request, p auth.Principal) (store.File, bool) {
f, err := r.store.FileByID(req.Context(), req.PathValue("id"))
if errors.Is(err, store.ErrNotFound) || err == nil && f.Owner != ownerOf(p) {
writeErrorCode(w, http.StatusNotFound, "invalid_request_error", "file_not_found",
fmt.Sprintf("no file %q", req.PathValue("id")))
return store.File{}, false
}
if err != nil {
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
return store.File{}, false
}
return f, true
}
func (r *Router) handleListFiles(w http.ResponseWriter, req *http.Request) {
p, ok := r.batchPrincipal(w, req)
if !ok {
return
}
fs, err := r.store.Files(req.Context(), ownerOf(p), req.URL.Query().Get("purpose"), 0)
if err != nil {
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
return
}
data := make([]map[string]any, 0, len(fs))
for _, f := range fs {
data = append(data, fileJSON(f))
}
writeJSON(w, http.StatusOK, map[string]any{"object": "list", "data": data})
}
func (r *Router) handleGetFile(w http.ResponseWriter, req *http.Request) {
p, ok := r.batchPrincipal(w, req)
if !ok {
return
}
if f, ok := r.ownedFile(w, req, p); ok {
writeJSON(w, http.StatusOK, fileJSON(f))
}
}
func (r *Router) handleFileContent(w http.ResponseWriter, req *http.Request) {
p, ok := r.batchPrincipal(w, req)
if !ok {
return
}
f, ok := r.ownedFile(w, req, p)
if !ok {
return
}
b, err := r.store.FileContent(req.Context(), f.ID)
if err != nil {
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
return
}
w.Header().Set("Content-Type", "application/octet-stream")
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", f.Filename))
_, _ = w.Write(b)
}
func (r *Router) handleDeleteFile(w http.ResponseWriter, req *http.Request) {
p, ok := r.batchPrincipal(w, req)
if !ok {
return
}
f, ok := r.ownedFile(w, req, p)
if !ok {
return
}
if _, err := r.store.DeleteFile(req.Context(), f.ID); err != nil {
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]any{"id": f.ID, "object": "file", "deleted": true})
}
+10
View File
@@ -100,6 +100,16 @@ func (r *Router) Handler() http.Handler {
// chat completions on the way in and back on the way out.
mux.HandleFunc("POST /v1/responses", r.handleResponses)
mux.HandleFunc("GET /v1/models", r.handleModels)
// Files and batches: a file of requests served at the lowest priority.
mux.HandleFunc("POST /v1/files", r.handleUploadFile)
mux.HandleFunc("GET /v1/files", r.handleListFiles)
mux.HandleFunc("GET /v1/files/{id}", r.handleGetFile)
mux.HandleFunc("GET /v1/files/{id}/content", r.handleFileContent)
mux.HandleFunc("DELETE /v1/files/{id}", r.handleDeleteFile)
mux.HandleFunc("POST /v1/batches", r.handleCreateBatch)
mux.HandleFunc("GET /v1/batches", r.handleListBatches)
mux.HandleFunc("GET /v1/batches/{id}", r.handleGetBatch)
mux.HandleFunc("POST /v1/batches/{id}/cancel", r.handleCancelBatch)
// Liveness: this process is up. It says nothing about whether it can
// serve, which is why it is not the thing to put a load balancer on.
mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) {
+11 -7
View File
@@ -68,6 +68,8 @@ type Router struct {
// canaries counts how the two sides served.
canaryTargets atomic.Pointer[map[string]string]
canaries canaryStats
// batchWake has the batch worker look for work at once.
batchWake chan struct{}
// extraMetrics are collectors other components publish on /metrics.
extraMetrics []prometheus.Collector
@@ -240,13 +242,14 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) {
}
r := &Router{
cfg: cfg.Router,
landing: cfg.Landing,
log: log,
reg: reg,
profiles: profiles,
prefix: prefixScorer,
m: metrics,
batchWake: make(chan struct{}, 1),
cfg: cfg.Router,
landing: cfg.Landing,
log: log,
reg: reg,
profiles: profiles,
prefix: prefixScorer,
m: metrics,
px: proxy.New(proxy.Config{
DialTimeout: cfg.Router.Timeouts.Dial,
ResponseHeaderTimeout: cfg.Router.Timeouts.ResponseHeader,
@@ -444,6 +447,7 @@ func (r *Router) Start(ctx context.Context) {
go r.pruneUsage(ctx, r.usageRetention)
}
if r.store != nil {
go r.runBatches(ctx)
go r.refreshTenants(ctx)
go r.purgeSessions(ctx)
}