Inserted between quota checks and scheduling: admission answers when a request may proceed, the scheduler still answers where it goes. Keeping them apart is the point — an engine queue is FIFO and knows nothing about tenants, so once a batch job is in it, no endpoint selection rescues the request behind it. - saturation detector: one number per pool, the worse of queue depth and KV occupancy, since either alone misleads. - priority bands with per-band saturation ceilings, so the cluster degrades from the bottom up. - tenant round-robin within a band: one caller sending a thousand requests takes one slot in the rotation, not a thousand. - bounded queues with capacity (503) and TTL (504); cancelled requests leave the queue instead of holding capacity. - metrics for queue depth, wait time, rejections and pool saturation. Exit criterion, measured in test/e2e: with 500 concurrent bronze requests running continuously, interactive p99 in the platinum band stays at 1.04-1.24x idle against a 1.5x budget. With admission off the same load pushes it to 55-60x. Both directions are asserted, so the test fails if admission regresses and also if the load generator stops being a real load. Two lags had to be closed, and they needed different answers. Engine metrics are scraped and lag an interval; router inflight does not, but it counts running requests, and treating those as queue makes a healthy engine look overloaded every interval. What is actually queued is inflight minus batch capacity, and capacity is not in the metric contract, so it is learned as the high-water mark of observed running. Separately, between admission granting a request and the router dispatching it the request is invisible everywhere, so a simultaneous burst passed the check en masse; the controller now counts its own outstanding leases against dispatched inflight. One plan correction: a saturation ceiling of 1.0 was documented as "never held back". It is not — it means "flow up to the healthy limit". A band that must never wait needs a ceiling above 1. Plan §7.2.2 updated. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
362 lines
11 KiB
Go
362 lines
11 KiB
Go
package router
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"net/http"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/admission"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
|
)
|
|
|
|
// admit waits for the flow control layer to let the request through. With
|
|
// admission disabled it returns immediately, which is the v0.2.0 behaviour.
|
|
func (r *Router) admit(ctx context.Context, req *parsedRequest) (*admission.Lease, error) {
|
|
if r.adm == nil {
|
|
return nil, nil
|
|
}
|
|
return r.adm.Acquire(ctx, admission.Request{
|
|
Model: req.Model,
|
|
Band: req.Band,
|
|
// Until authentication arrives the session is the best tenant key we
|
|
// have: it groups one caller's traffic without pretending to identify
|
|
// them (plan §7.9).
|
|
Tenant: req.SessionID,
|
|
})
|
|
}
|
|
|
|
// rejectAdmission turns a flow control refusal into a status the client can act
|
|
// on: 503 when the queue is full (try later), 504 when it timed out waiting.
|
|
func (r *Router) rejectAdmission(w http.ResponseWriter, req *parsedRequest, err error, start time.Time) {
|
|
status := http.StatusServiceUnavailable
|
|
switch {
|
|
case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded):
|
|
// The client gave up while queued; nobody is listening.
|
|
r.m.observeAdmissionRejected(req.Band, "client_gone")
|
|
r.m.observeRequest(req.Model, "", 499, time.Since(start), nil, 0)
|
|
return
|
|
case errors.Is(err, admission.ErrQueueTimeout):
|
|
status = http.StatusGatewayTimeout
|
|
}
|
|
reason := "unavailable"
|
|
switch {
|
|
case errors.Is(err, admission.ErrQueueFull):
|
|
reason = "queue_full"
|
|
case errors.Is(err, admission.ErrQueueTimeout):
|
|
reason = "ttl"
|
|
case errors.Is(err, admission.ErrClosed):
|
|
reason = "shutdown"
|
|
}
|
|
r.m.observeAdmissionRejected(req.Band, reason)
|
|
writeError(w, status, "server_error", err.Error())
|
|
r.m.observeRequest(req.Model, "", status, time.Since(start), nil, 0)
|
|
}
|
|
|
|
// prefixBytes is how much of the prompt is kept for prefix-aware routing.
|
|
// Large enough to cover a system prompt plus the first message, small enough
|
|
// that hashing it stays cheap.
|
|
const prefixBytes = 2048
|
|
|
|
// Handler returns the client-facing OpenAI API.
|
|
func (r *Router) Handler() http.Handler {
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("POST /v1/chat/completions", r.handleInference("/v1/chat/completions"))
|
|
mux.HandleFunc("POST /v1/completions", r.handleInference("/v1/completions"))
|
|
mux.HandleFunc("POST /v1/embeddings", r.handleInference("/v1/embeddings"))
|
|
mux.HandleFunc("GET /v1/models", r.handleModels)
|
|
mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusOK)
|
|
})
|
|
return mux
|
|
}
|
|
|
|
// parsedRequest is what the router needs from a request. Everything else in the
|
|
// body is forwarded untouched — clients send engine-specific fields we have no
|
|
// business interpreting.
|
|
type parsedRequest struct {
|
|
Path string
|
|
Model string
|
|
|
|
Stream bool
|
|
SuppressUsageChunk bool
|
|
UpstreamBody []byte
|
|
Header http.Header
|
|
|
|
SessionID string
|
|
Band string
|
|
ProfileOverride string
|
|
PromptPrefix string
|
|
PromptTokens int
|
|
MaxTokens int
|
|
}
|
|
|
|
// inferenceBody covers the fields the router reads across all inference
|
|
// endpoints. Unknown fields are ignored here and preserved in UpstreamBody.
|
|
type inferenceBody struct {
|
|
Model string `json:"model"`
|
|
Stream bool `json:"stream"`
|
|
User string `json:"user"`
|
|
Messages []struct {
|
|
Role string `json:"role"`
|
|
Content json.RawMessage `json:"content"`
|
|
} `json:"messages"`
|
|
Prompt json.RawMessage `json:"prompt"`
|
|
Input json.RawMessage `json:"input"`
|
|
MaxTokens int `json:"max_tokens"`
|
|
MaxCompletionTokens int `json:"max_completion_tokens"`
|
|
StreamOptions *struct {
|
|
IncludeUsage bool `json:"include_usage"`
|
|
} `json:"stream_options"`
|
|
}
|
|
|
|
func (r *Router) handleInference(path string) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, req *http.Request) {
|
|
start := time.Now()
|
|
|
|
parsed, err := r.parse(req, path)
|
|
if err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
|
r.m.observeRequest("", "", http.StatusBadRequest, time.Since(start), nil, 0)
|
|
return
|
|
}
|
|
|
|
// Step 5 of the lifecycle: "now?" before "where?" (plan §7.1).
|
|
lease, err := r.admit(req.Context(), parsed)
|
|
if err != nil {
|
|
r.rejectAdmission(w, parsed, err, start)
|
|
return
|
|
}
|
|
defer lease.Release()
|
|
queued := time.Since(start)
|
|
r.m.observeAdmission(lease.Band(), parsed.Model, queued)
|
|
if queued > time.Millisecond {
|
|
w.Header().Set("x-llmgw-queue-ms", strconv.FormatInt(queued.Milliseconds(), 10))
|
|
}
|
|
|
|
res := r.dispatch(req.Context(), w, parsed)
|
|
r.finish(w, parsed, res, start)
|
|
}
|
|
}
|
|
|
|
// finish records metrics and, when nothing was written, produces the error body.
|
|
func (r *Router) finish(w http.ResponseWriter, req *parsedRequest, res dispatchResult, start time.Time) {
|
|
status := res.outcome.Status
|
|
switch {
|
|
case res.err == nil:
|
|
if res.fallback {
|
|
// The client asked for one model and got another; say so rather
|
|
// than letting them discover it from the response contents.
|
|
w.Header().Set("x-llmgw-served-model", res.model)
|
|
}
|
|
case errors.Is(res.err, proxy.ErrClientGone):
|
|
// Nothing to report to a client that is gone; the upstream was cancelled.
|
|
status = 499
|
|
r.log.Debug("client disconnected", "model", req.Model, "endpoint", res.endpointID)
|
|
case res.outcome.Wrote:
|
|
// The response is already partly out; the stream just ends short.
|
|
r.log.Warn("stream ended early", "model", req.Model, "endpoint", res.endpointID, "err", res.err)
|
|
if status == 0 {
|
|
status = http.StatusOK
|
|
}
|
|
case errors.Is(res.err, errNoCapacity):
|
|
status = http.StatusServiceUnavailable
|
|
writeError(w, status, "server_error", res.err.Error())
|
|
default:
|
|
status = http.StatusBadGateway
|
|
writeError(w, status, "server_error", res.err.Error())
|
|
}
|
|
|
|
r.m.observeRequest(req.Model, res.pool, status, time.Since(start), res.outcome.Usage, res.outcome.TTFT)
|
|
}
|
|
|
|
// parse reads the body and extracts the routing-relevant fields.
|
|
func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) {
|
|
body, err := io.ReadAll(io.LimitReader(req.Body, r.cfg.MaxBodyBytes))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var b inferenceBody
|
|
if err := json.Unmarshal(body, &b); err != nil {
|
|
return nil, err
|
|
}
|
|
if b.Model == "" {
|
|
return nil, errors.New("model is required")
|
|
}
|
|
|
|
p := &parsedRequest{
|
|
Path: path,
|
|
Model: b.Model,
|
|
Stream: b.Stream,
|
|
UpstreamBody: body,
|
|
Header: req.Header,
|
|
SessionID: sessionID(req, r.cfg.SessionHeader, b.User),
|
|
Band: req.Header.Get(r.cfg.Admission.BandHeader),
|
|
ProfileOverride: req.Header.Get("x-llmgw-profile"),
|
|
MaxTokens: max(b.MaxTokens, b.MaxCompletionTokens),
|
|
PromptPrefix: promptPrefix(&b),
|
|
}
|
|
// Bytes over four: a crude stand-in for a tokenizer, good enough for
|
|
// scoring. It becomes exact only with plan §7.5 tier 2 (see plan §14).
|
|
p.PromptTokens = len(p.PromptPrefix) / 4
|
|
|
|
// Ask the engine for usage the client did not request, so accounting works
|
|
// regardless of client behaviour; the extra chunk is dropped on the way back.
|
|
if r.cfg.InjectUsage && b.Stream && (b.StreamOptions == nil || !b.StreamOptions.IncludeUsage) {
|
|
if injected, ok := injectIncludeUsage(body); ok {
|
|
p.UpstreamBody = injected
|
|
p.SuppressUsageChunk = true
|
|
}
|
|
}
|
|
return p, nil
|
|
}
|
|
|
|
// sessionID picks the affinity key: an explicit header first, then OpenAI's
|
|
// user field, which most clients already set per end user.
|
|
func sessionID(req *http.Request, header, user string) string {
|
|
if header != "" {
|
|
if v := req.Header.Get(header); v != "" {
|
|
return v
|
|
}
|
|
}
|
|
return user
|
|
}
|
|
|
|
// promptPrefix reconstructs the leading portion of the prompt for prefix-aware
|
|
// routing. Only the first prefixBytes matter, so long conversations stop early.
|
|
func promptPrefix(b *inferenceBody) string {
|
|
var sb strings.Builder
|
|
appendRaw := func(raw json.RawMessage) {
|
|
if len(raw) == 0 || sb.Len() >= prefixBytes {
|
|
return
|
|
}
|
|
var s string
|
|
if err := json.Unmarshal(raw, &s); err != nil {
|
|
s = string(raw)
|
|
}
|
|
sb.WriteString(s)
|
|
}
|
|
|
|
for _, m := range b.Messages {
|
|
if sb.Len() >= prefixBytes {
|
|
break
|
|
}
|
|
sb.WriteString(m.Role)
|
|
sb.WriteString(":")
|
|
appendRaw(m.Content)
|
|
sb.WriteString("\n")
|
|
}
|
|
appendRaw(b.Prompt)
|
|
appendRaw(b.Input)
|
|
|
|
s := sb.String()
|
|
if len(s) > prefixBytes {
|
|
s = s[:prefixBytes]
|
|
}
|
|
return s
|
|
}
|
|
|
|
// injectIncludeUsage sets stream_options.include_usage on the outgoing body.
|
|
//
|
|
// This is the one place the router rewrites a request, so it is done by
|
|
// decoding to a generic map and re-encoding: field order changes, but no field
|
|
// is lost. If anything looks unexpected it declines rather than guessing.
|
|
func injectIncludeUsage(body []byte) ([]byte, bool) {
|
|
var m map[string]json.RawMessage
|
|
if err := json.Unmarshal(body, &m); err != nil {
|
|
return nil, false
|
|
}
|
|
|
|
opts := map[string]any{}
|
|
if raw, ok := m["stream_options"]; ok {
|
|
if err := json.Unmarshal(raw, &opts); err != nil {
|
|
return nil, false
|
|
}
|
|
}
|
|
opts["include_usage"] = true
|
|
|
|
encoded, err := json.Marshal(opts)
|
|
if err != nil {
|
|
return nil, false
|
|
}
|
|
m["stream_options"] = encoded
|
|
|
|
out, err := json.Marshal(m)
|
|
if err != nil {
|
|
return nil, false
|
|
}
|
|
return out, true
|
|
}
|
|
|
|
// servedModels lists the canonical models at least one endpoint serves.
|
|
func (r *Router) servedModels() []string {
|
|
seen := map[string]struct{}{}
|
|
for _, s := range r.reg.Snapshots("") {
|
|
for _, m := range s.Models {
|
|
seen[m] = struct{}{}
|
|
}
|
|
}
|
|
out := make([]string, 0, len(seen))
|
|
for m := range seen {
|
|
out = append(out, m)
|
|
}
|
|
slices.Sort(out)
|
|
return out
|
|
}
|
|
|
|
// handleModels lists the models that at least one endpoint currently serves.
|
|
func (r *Router) handleModels(w http.ResponseWriter, _ *http.Request) {
|
|
seen := map[string]struct{}{}
|
|
for _, s := range r.reg.Snapshots("") {
|
|
for _, m := range s.Models {
|
|
seen[m] = struct{}{}
|
|
}
|
|
}
|
|
// Catalog aliases are usable names too, so clients should see them.
|
|
for name := range r.models {
|
|
if _, ok := seen[r.models[name].Name]; ok {
|
|
seen[name] = struct{}{}
|
|
}
|
|
}
|
|
|
|
names := make([]string, 0, len(seen))
|
|
for n := range seen {
|
|
names = append(names, n)
|
|
}
|
|
slices.Sort(names)
|
|
|
|
data := make([]map[string]any, 0, len(names))
|
|
for _, n := range names {
|
|
data = append(data, map[string]any{
|
|
"id": n, "object": "model", "created": time.Now().Unix(), "owned_by": "llmgw",
|
|
})
|
|
}
|
|
writeJSON(w, http.StatusOK, map[string]any{"object": "list", "data": data})
|
|
}
|
|
|
|
func writeJSON(w http.ResponseWriter, code int, v any) {
|
|
var buf bytes.Buffer
|
|
if err := json.NewEncoder(&buf).Encode(v); err != nil {
|
|
http.Error(w, `{"error":{"message":"encoding failed"}}`, http.StatusInternalServerError)
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(code)
|
|
_, _ = w.Write(buf.Bytes())
|
|
}
|
|
|
|
// writeError emits an OpenAI-shaped error so clients need no special handling
|
|
// for gateway failures.
|
|
func writeError(w http.ResponseWriter, code int, kind, msg string) {
|
|
writeJSON(w, code, map[string]any{"error": map[string]any{
|
|
"message": msg, "type": kind, "code": nil,
|
|
}})
|
|
}
|