CI / test (push) Successful in 2m34s
Until now anything that could reach the client port could use every model in the catalog. A key now resolves to an identity, and that identity decides what it may do. Authentication wraps the whole mux rather than checking inside each handler, so adding a route cannot add an unauthenticated one by omission. Health stays open, because a load balancer has no key and a health check that needs one reports the gateway down. Nothing on the request path reads the database. A key is resolved once and answered from memory until the entry expires; rejections are cached too, so a client looping with a bad key cannot turn into a database read per request. The price is that a revocation takes up to cache_ttl to take effect, which is the trade stated in the configuration next to the knob. A store failure is not an authentication answer: it is never cached as one and returns 503, not 401 — sending someone hunting for a key problem that does not exist wastes the one thing an incident is short of. A revoked key is refused with the same message as an unknown one, because saying otherwise confirms the key was real. With auth on the band comes from the key and the band header is ignored. The header was trusted only while there was nothing to consult, and it let any client promote itself into the top priority band. Tenant fairness within a band is now per key rather than per session, so one caller cannot take a band by opening many sessions. Allowlists are decided per model, not per string: a model's canonical name and every alias name the same thing, so either form works in an allowlist and no alias can reach a model that was not allowed. The first version of that test passed without the resolution being there at all, which is why it now checks all four combinations. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
419 lines
13 KiB
Go
419 lines
13 KiB
Go
package router
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"net/http"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/admission"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
|
)
|
|
|
|
// admit waits for the flow control layer to let the request through. With
|
|
// admission disabled it returns immediately, which is the v0.2.0 behaviour.
|
|
func (r *Router) admit(ctx context.Context, req *parsedRequest) (*admission.Lease, error) {
|
|
if r.adm == nil {
|
|
return nil, nil
|
|
}
|
|
// The key is the tenant when there is one: fairness within a band has to be
|
|
// per caller, and a session id is something the caller picks, so one client
|
|
// could take a whole band by opening many sessions. Without authentication
|
|
// the session is still the best available grouping.
|
|
tenant := req.Tenant
|
|
if tenant == "" {
|
|
tenant = req.SessionID
|
|
}
|
|
return r.adm.Acquire(ctx, admission.Request{
|
|
Model: req.Model,
|
|
Band: req.Band,
|
|
Tenant: tenant,
|
|
})
|
|
}
|
|
|
|
// rejectAdmission turns a flow control refusal into a status the client can act
|
|
// on: 503 when the queue is full (try later), 504 when it timed out waiting.
|
|
func (r *Router) rejectAdmission(w http.ResponseWriter, req *parsedRequest, err error, start time.Time) {
|
|
status := http.StatusServiceUnavailable
|
|
switch {
|
|
case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded):
|
|
// The client gave up while queued; nobody is listening.
|
|
r.m.observeAdmissionRejected(req.Band, "client_gone")
|
|
r.m.observeRequest(req.Model, "", 499, time.Since(start), nil, 0)
|
|
return
|
|
case errors.Is(err, admission.ErrQueueTimeout):
|
|
status = http.StatusGatewayTimeout
|
|
}
|
|
reason := "unavailable"
|
|
switch {
|
|
case errors.Is(err, admission.ErrQueueFull):
|
|
reason = "queue_full"
|
|
case errors.Is(err, admission.ErrQueueTimeout):
|
|
reason = "ttl"
|
|
case errors.Is(err, admission.ErrClosed):
|
|
reason = "shutdown"
|
|
}
|
|
r.m.observeAdmissionRejected(req.Band, reason)
|
|
writeError(w, status, "server_error", err.Error())
|
|
r.m.observeRequest(req.Model, "", status, time.Since(start), nil, 0)
|
|
}
|
|
|
|
// prefixBytes is how much of the prompt is kept for prefix-aware routing.
|
|
// Large enough to cover a system prompt plus the first message, small enough
|
|
// that hashing it stays cheap.
|
|
const prefixBytes = 2048
|
|
|
|
// Handler returns the client-facing OpenAI API.
|
|
func (r *Router) Handler() http.Handler {
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("POST /v1/chat/completions", r.handleInference("/v1/chat/completions"))
|
|
mux.HandleFunc("POST /v1/completions", r.handleInference("/v1/completions"))
|
|
mux.HandleFunc("POST /v1/embeddings", r.handleInference("/v1/embeddings"))
|
|
mux.HandleFunc("GET /v1/models", r.handleModels)
|
|
mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusOK)
|
|
})
|
|
if r.auth == nil {
|
|
return mux
|
|
}
|
|
// Wrapping the whole mux rather than checking inside each handler: adding a
|
|
// route must not be able to add an unauthenticated one by omission. Health
|
|
// stays open because a load balancer has no key.
|
|
return auth.Middleware(r.auth, r.log, "/health")(mux)
|
|
}
|
|
|
|
// parsedRequest is what the router needs from a request. Everything else in the
|
|
// body is forwarded untouched — clients send engine-specific fields we have no
|
|
// business interpreting.
|
|
type parsedRequest struct {
|
|
Path string
|
|
Model string
|
|
|
|
Stream bool
|
|
SuppressUsageChunk bool
|
|
UpstreamBody []byte
|
|
Header http.Header
|
|
|
|
SessionID string
|
|
Band string
|
|
// KeyID, ProjectID and Tenant are empty when authentication is off.
|
|
KeyID string
|
|
ProjectID string
|
|
Tenant string
|
|
ProfileOverride string
|
|
PromptPrefix string
|
|
PromptTokens int
|
|
MaxTokens int
|
|
}
|
|
|
|
// inferenceBody covers the fields the router reads across all inference
|
|
// endpoints. Unknown fields are ignored here and preserved in UpstreamBody.
|
|
type inferenceBody struct {
|
|
Model string `json:"model"`
|
|
Stream bool `json:"stream"`
|
|
User string `json:"user"`
|
|
Messages []struct {
|
|
Role string `json:"role"`
|
|
Content json.RawMessage `json:"content"`
|
|
} `json:"messages"`
|
|
Prompt json.RawMessage `json:"prompt"`
|
|
Input json.RawMessage `json:"input"`
|
|
MaxTokens int `json:"max_tokens"`
|
|
MaxCompletionTokens int `json:"max_completion_tokens"`
|
|
StreamOptions *struct {
|
|
IncludeUsage bool `json:"include_usage"`
|
|
} `json:"stream_options"`
|
|
}
|
|
|
|
func (r *Router) handleInference(path string) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, req *http.Request) {
|
|
start := time.Now()
|
|
|
|
parsed, err := r.parse(req, path)
|
|
if err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
|
r.m.observeRequest("", "", http.StatusBadRequest, time.Since(start), nil, 0)
|
|
return
|
|
}
|
|
|
|
if p, ok := auth.FromContext(req.Context()); ok && !r.permits(p, parsed.Model) {
|
|
auth.WriteForbiddenModel(w, parsed.Model)
|
|
r.m.observeRequest(parsed.Model, "", http.StatusForbidden, time.Since(start), nil, 0)
|
|
return
|
|
}
|
|
|
|
// Step 5 of the lifecycle: "now?" before "where?" (plan §7.1).
|
|
lease, err := r.admit(req.Context(), parsed)
|
|
if err != nil {
|
|
r.rejectAdmission(w, parsed, err, start)
|
|
return
|
|
}
|
|
defer lease.Release()
|
|
queued := time.Since(start)
|
|
r.m.observeAdmission(lease.Band(), parsed.Model, queued)
|
|
if queued > time.Millisecond {
|
|
w.Header().Set("x-llmgw-queue-ms", strconv.FormatInt(queued.Milliseconds(), 10))
|
|
}
|
|
|
|
res := r.dispatch(req.Context(), w, parsed)
|
|
r.finish(w, parsed, res, start)
|
|
}
|
|
}
|
|
|
|
// permits reports whether a key's allowlist covers the requested model.
|
|
//
|
|
// The decision is per model, not per string. A model's canonical name and every
|
|
// alias of it name the same thing, so the allowlist is matched against all of
|
|
// them: writing either form works, and no alias can reach a model that was not
|
|
// allowed. Matching only what the client sent would make every alias a way
|
|
// around the allowlist; matching only the canonical name would silently ignore
|
|
// an allowlist written in the names clients actually use.
|
|
func (r *Router) permits(p auth.Principal, requested string) bool {
|
|
canonical, m := r.resolve(requested)
|
|
if p.Allows(canonical) {
|
|
return true
|
|
}
|
|
for _, alias := range m.Aliases {
|
|
if p.Allows(alias) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// finish records metrics and, when nothing was written, produces the error body.
|
|
func (r *Router) finish(w http.ResponseWriter, req *parsedRequest, res dispatchResult, start time.Time) {
|
|
status := res.outcome.Status
|
|
switch {
|
|
case res.err == nil:
|
|
if res.fallback {
|
|
// The client asked for one model and got another; say so rather
|
|
// than letting them discover it from the response contents.
|
|
w.Header().Set("x-llmgw-served-model", res.model)
|
|
}
|
|
case errors.Is(res.err, proxy.ErrClientGone):
|
|
// Nothing to report to a client that is gone; the upstream was cancelled.
|
|
status = 499
|
|
r.log.Debug("client disconnected", "model", req.Model, "endpoint", res.endpointID)
|
|
case res.outcome.Wrote:
|
|
// The response is already partly out; the stream just ends short.
|
|
r.log.Warn("stream ended early", "model", req.Model, "endpoint", res.endpointID, "err", res.err)
|
|
if status == 0 {
|
|
status = http.StatusOK
|
|
}
|
|
case errors.Is(res.err, errNoCapacity):
|
|
status = http.StatusServiceUnavailable
|
|
writeError(w, status, "server_error", res.err.Error())
|
|
default:
|
|
status = http.StatusBadGateway
|
|
writeError(w, status, "server_error", res.err.Error())
|
|
}
|
|
|
|
r.m.observeRequest(req.Model, res.pool, status, time.Since(start), res.outcome.Usage, res.outcome.TTFT)
|
|
}
|
|
|
|
// parse reads the body and extracts the routing-relevant fields.
|
|
func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) {
|
|
body, err := io.ReadAll(io.LimitReader(req.Body, r.cfg.MaxBodyBytes))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var b inferenceBody
|
|
if err := json.Unmarshal(body, &b); err != nil {
|
|
return nil, err
|
|
}
|
|
if b.Model == "" {
|
|
return nil, errors.New("model is required")
|
|
}
|
|
|
|
p := &parsedRequest{
|
|
Path: path,
|
|
Model: b.Model,
|
|
Stream: b.Stream,
|
|
UpstreamBody: body,
|
|
Header: req.Header,
|
|
SessionID: sessionID(req, r.cfg.SessionHeader, b.User),
|
|
Band: req.Header.Get(r.cfg.Admission.BandHeader),
|
|
ProfileOverride: req.Header.Get("x-llmgw-profile"),
|
|
MaxTokens: max(b.MaxTokens, b.MaxCompletionTokens),
|
|
PromptPrefix: promptPrefix(&b),
|
|
}
|
|
// With authentication on, the key decides the band and the routing profile,
|
|
// and the headers stop being believed. Trusting the header was only ever
|
|
// acceptable while there was no identity to consult — it let any client
|
|
// promote itself into the top priority band.
|
|
if principal, ok := auth.FromContext(req.Context()); ok {
|
|
p.Band = principal.Band
|
|
p.ProfileOverride = principal.RoutingProfile
|
|
p.KeyID = principal.KeyID
|
|
p.ProjectID = principal.ProjectID
|
|
// Tenant fairness within a band is per key rather than per session, so
|
|
// one key cannot take a whole band by opening many sessions.
|
|
p.Tenant = principal.KeyID
|
|
}
|
|
|
|
// Bytes over four: a crude stand-in for a tokenizer, good enough for
|
|
// scoring. It becomes exact only with plan §7.5 tier 2 (see plan §14).
|
|
p.PromptTokens = len(p.PromptPrefix) / 4
|
|
|
|
// Ask the engine for usage the client did not request, so accounting works
|
|
// regardless of client behaviour; the extra chunk is dropped on the way back.
|
|
if r.cfg.InjectUsage && b.Stream && (b.StreamOptions == nil || !b.StreamOptions.IncludeUsage) {
|
|
if injected, ok := injectIncludeUsage(body); ok {
|
|
p.UpstreamBody = injected
|
|
p.SuppressUsageChunk = true
|
|
}
|
|
}
|
|
return p, nil
|
|
}
|
|
|
|
// sessionID picks the affinity key: an explicit header first, then OpenAI's
|
|
// user field, which most clients already set per end user.
|
|
func sessionID(req *http.Request, header, user string) string {
|
|
if header != "" {
|
|
if v := req.Header.Get(header); v != "" {
|
|
return v
|
|
}
|
|
}
|
|
return user
|
|
}
|
|
|
|
// promptPrefix reconstructs the leading portion of the prompt for prefix-aware
|
|
// routing. Only the first prefixBytes matter, so long conversations stop early.
|
|
func promptPrefix(b *inferenceBody) string {
|
|
var sb strings.Builder
|
|
appendRaw := func(raw json.RawMessage) {
|
|
if len(raw) == 0 || sb.Len() >= prefixBytes {
|
|
return
|
|
}
|
|
var s string
|
|
if err := json.Unmarshal(raw, &s); err != nil {
|
|
s = string(raw)
|
|
}
|
|
sb.WriteString(s)
|
|
}
|
|
|
|
for _, m := range b.Messages {
|
|
if sb.Len() >= prefixBytes {
|
|
break
|
|
}
|
|
sb.WriteString(m.Role)
|
|
sb.WriteString(":")
|
|
appendRaw(m.Content)
|
|
sb.WriteString("\n")
|
|
}
|
|
appendRaw(b.Prompt)
|
|
appendRaw(b.Input)
|
|
|
|
s := sb.String()
|
|
if len(s) > prefixBytes {
|
|
s = s[:prefixBytes]
|
|
}
|
|
return s
|
|
}
|
|
|
|
// injectIncludeUsage sets stream_options.include_usage on the outgoing body.
|
|
//
|
|
// This is the one place the router rewrites a request, so it is done by
|
|
// decoding to a generic map and re-encoding: field order changes, but no field
|
|
// is lost. If anything looks unexpected it declines rather than guessing.
|
|
func injectIncludeUsage(body []byte) ([]byte, bool) {
|
|
var m map[string]json.RawMessage
|
|
if err := json.Unmarshal(body, &m); err != nil {
|
|
return nil, false
|
|
}
|
|
|
|
opts := map[string]any{}
|
|
if raw, ok := m["stream_options"]; ok {
|
|
if err := json.Unmarshal(raw, &opts); err != nil {
|
|
return nil, false
|
|
}
|
|
}
|
|
opts["include_usage"] = true
|
|
|
|
encoded, err := json.Marshal(opts)
|
|
if err != nil {
|
|
return nil, false
|
|
}
|
|
m["stream_options"] = encoded
|
|
|
|
out, err := json.Marshal(m)
|
|
if err != nil {
|
|
return nil, false
|
|
}
|
|
return out, true
|
|
}
|
|
|
|
// servedModels lists the canonical models at least one endpoint serves.
|
|
func (r *Router) servedModels() []string {
|
|
seen := map[string]struct{}{}
|
|
for _, s := range r.reg.Snapshots("") {
|
|
for _, m := range s.Models {
|
|
seen[m] = struct{}{}
|
|
}
|
|
}
|
|
out := make([]string, 0, len(seen))
|
|
for m := range seen {
|
|
out = append(out, m)
|
|
}
|
|
slices.Sort(out)
|
|
return out
|
|
}
|
|
|
|
// handleModels lists the models that at least one endpoint currently serves.
|
|
func (r *Router) handleModels(w http.ResponseWriter, _ *http.Request) {
|
|
seen := map[string]struct{}{}
|
|
for _, s := range r.reg.Snapshots("") {
|
|
for _, m := range s.Models {
|
|
seen[m] = struct{}{}
|
|
}
|
|
}
|
|
// Catalog aliases are usable names too, so clients should see them.
|
|
for name := range r.models {
|
|
if _, ok := seen[r.models[name].Name]; ok {
|
|
seen[name] = struct{}{}
|
|
}
|
|
}
|
|
|
|
names := make([]string, 0, len(seen))
|
|
for n := range seen {
|
|
names = append(names, n)
|
|
}
|
|
slices.Sort(names)
|
|
|
|
data := make([]map[string]any, 0, len(names))
|
|
for _, n := range names {
|
|
data = append(data, map[string]any{
|
|
"id": n, "object": "model", "created": time.Now().Unix(), "owned_by": "llmgw",
|
|
})
|
|
}
|
|
writeJSON(w, http.StatusOK, map[string]any{"object": "list", "data": data})
|
|
}
|
|
|
|
func writeJSON(w http.ResponseWriter, code int, v any) {
|
|
var buf bytes.Buffer
|
|
if err := json.NewEncoder(&buf).Encode(v); err != nil {
|
|
http.Error(w, `{"error":{"message":"encoding failed"}}`, http.StatusInternalServerError)
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(code)
|
|
_, _ = w.Write(buf.Bytes())
|
|
}
|
|
|
|
// writeError emits an OpenAI-shaped error so clients need no special handling
|
|
// for gateway failures.
|
|
func writeError(w http.ResponseWriter, code int, kind, msg string) {
|
|
writeJSON(w, code, map[string]any{"error": map[string]any{
|
|
"message": msg, "type": kind, "code": nil,
|
|
}})
|
|
}
|