Files
llmgw/internal/router/handlers.go
T
ilkerandClaude Opus 5 dc38dd6096 Phase 1.5: admission control
Inserted between quota checks and scheduling: admission answers when a request
may proceed, the scheduler still answers where it goes. Keeping them apart is
the point — an engine queue is FIFO and knows nothing about tenants, so once a
batch job is in it, no endpoint selection rescues the request behind it.

- saturation detector: one number per pool, the worse of queue depth and KV
  occupancy, since either alone misleads.
- priority bands with per-band saturation ceilings, so the cluster degrades from
  the bottom up.
- tenant round-robin within a band: one caller sending a thousand requests takes
  one slot in the rotation, not a thousand.
- bounded queues with capacity (503) and TTL (504); cancelled requests leave the
  queue instead of holding capacity.
- metrics for queue depth, wait time, rejections and pool saturation.

Exit criterion, measured in test/e2e: with 500 concurrent bronze requests
running continuously, interactive p99 in the platinum band stays at 1.04-1.24x
idle against a 1.5x budget. With admission off the same load pushes it to
55-60x. Both directions are asserted, so the test fails if admission regresses
and also if the load generator stops being a real load.

Two lags had to be closed, and they needed different answers. Engine metrics are
scraped and lag an interval; router inflight does not, but it counts running
requests, and treating those as queue makes a healthy engine look overloaded
every interval. What is actually queued is inflight minus batch capacity, and
capacity is not in the metric contract, so it is learned as the high-water mark
of observed running. Separately, between admission granting a request and the
router dispatching it the request is invisible everywhere, so a simultaneous
burst passed the check en masse; the controller now counts its own outstanding
leases against dispatched inflight.

One plan correction: a saturation ceiling of 1.0 was documented as "never held
back". It is not — it means "flow up to the healthy limit". A band that must
never wait needs a ceiling above 1. Plan §7.2.2 updated.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 01:53:38 +02:00

362 lines
11 KiB
Go

package router
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"net/http"
"slices"
"strconv"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/admission"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
)
// admit waits for the flow control layer to let the request through. With
// admission disabled it returns immediately, which is the v0.2.0 behaviour.
func (r *Router) admit(ctx context.Context, req *parsedRequest) (*admission.Lease, error) {
if r.adm == nil {
return nil, nil
}
return r.adm.Acquire(ctx, admission.Request{
Model: req.Model,
Band: req.Band,
// Until authentication arrives the session is the best tenant key we
// have: it groups one caller's traffic without pretending to identify
// them (plan §7.9).
Tenant: req.SessionID,
})
}
// rejectAdmission turns a flow control refusal into a status the client can act
// on: 503 when the queue is full (try later), 504 when it timed out waiting.
func (r *Router) rejectAdmission(w http.ResponseWriter, req *parsedRequest, err error, start time.Time) {
status := http.StatusServiceUnavailable
switch {
case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded):
// The client gave up while queued; nobody is listening.
r.m.observeAdmissionRejected(req.Band, "client_gone")
r.m.observeRequest(req.Model, "", 499, time.Since(start), nil, 0)
return
case errors.Is(err, admission.ErrQueueTimeout):
status = http.StatusGatewayTimeout
}
reason := "unavailable"
switch {
case errors.Is(err, admission.ErrQueueFull):
reason = "queue_full"
case errors.Is(err, admission.ErrQueueTimeout):
reason = "ttl"
case errors.Is(err, admission.ErrClosed):
reason = "shutdown"
}
r.m.observeAdmissionRejected(req.Band, reason)
writeError(w, status, "server_error", err.Error())
r.m.observeRequest(req.Model, "", status, time.Since(start), nil, 0)
}
// prefixBytes is how much of the prompt is kept for prefix-aware routing.
// Large enough to cover a system prompt plus the first message, small enough
// that hashing it stays cheap.
const prefixBytes = 2048
// Handler returns the client-facing OpenAI API.
func (r *Router) Handler() http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("POST /v1/chat/completions", r.handleInference("/v1/chat/completions"))
mux.HandleFunc("POST /v1/completions", r.handleInference("/v1/completions"))
mux.HandleFunc("POST /v1/embeddings", r.handleInference("/v1/embeddings"))
mux.HandleFunc("GET /v1/models", r.handleModels)
mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
})
return mux
}
// parsedRequest is what the router needs from a request. Everything else in the
// body is forwarded untouched — clients send engine-specific fields we have no
// business interpreting.
type parsedRequest struct {
Path string
Model string
Stream bool
SuppressUsageChunk bool
UpstreamBody []byte
Header http.Header
SessionID string
Band string
ProfileOverride string
PromptPrefix string
PromptTokens int
MaxTokens int
}
// inferenceBody covers the fields the router reads across all inference
// endpoints. Unknown fields are ignored here and preserved in UpstreamBody.
type inferenceBody struct {
Model string `json:"model"`
Stream bool `json:"stream"`
User string `json:"user"`
Messages []struct {
Role string `json:"role"`
Content json.RawMessage `json:"content"`
} `json:"messages"`
Prompt json.RawMessage `json:"prompt"`
Input json.RawMessage `json:"input"`
MaxTokens int `json:"max_tokens"`
MaxCompletionTokens int `json:"max_completion_tokens"`
StreamOptions *struct {
IncludeUsage bool `json:"include_usage"`
} `json:"stream_options"`
}
func (r *Router) handleInference(path string) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
start := time.Now()
parsed, err := r.parse(req, path)
if err != nil {
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
r.m.observeRequest("", "", http.StatusBadRequest, time.Since(start), nil, 0)
return
}
// Step 5 of the lifecycle: "now?" before "where?" (plan §7.1).
lease, err := r.admit(req.Context(), parsed)
if err != nil {
r.rejectAdmission(w, parsed, err, start)
return
}
defer lease.Release()
queued := time.Since(start)
r.m.observeAdmission(lease.Band(), parsed.Model, queued)
if queued > time.Millisecond {
w.Header().Set("x-llmgw-queue-ms", strconv.FormatInt(queued.Milliseconds(), 10))
}
res := r.dispatch(req.Context(), w, parsed)
r.finish(w, parsed, res, start)
}
}
// finish records metrics and, when nothing was written, produces the error body.
func (r *Router) finish(w http.ResponseWriter, req *parsedRequest, res dispatchResult, start time.Time) {
status := res.outcome.Status
switch {
case res.err == nil:
if res.fallback {
// The client asked for one model and got another; say so rather
// than letting them discover it from the response contents.
w.Header().Set("x-llmgw-served-model", res.model)
}
case errors.Is(res.err, proxy.ErrClientGone):
// Nothing to report to a client that is gone; the upstream was cancelled.
status = 499
r.log.Debug("client disconnected", "model", req.Model, "endpoint", res.endpointID)
case res.outcome.Wrote:
// The response is already partly out; the stream just ends short.
r.log.Warn("stream ended early", "model", req.Model, "endpoint", res.endpointID, "err", res.err)
if status == 0 {
status = http.StatusOK
}
case errors.Is(res.err, errNoCapacity):
status = http.StatusServiceUnavailable
writeError(w, status, "server_error", res.err.Error())
default:
status = http.StatusBadGateway
writeError(w, status, "server_error", res.err.Error())
}
r.m.observeRequest(req.Model, res.pool, status, time.Since(start), res.outcome.Usage, res.outcome.TTFT)
}
// parse reads the body and extracts the routing-relevant fields.
func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) {
body, err := io.ReadAll(io.LimitReader(req.Body, r.cfg.MaxBodyBytes))
if err != nil {
return nil, err
}
var b inferenceBody
if err := json.Unmarshal(body, &b); err != nil {
return nil, err
}
if b.Model == "" {
return nil, errors.New("model is required")
}
p := &parsedRequest{
Path: path,
Model: b.Model,
Stream: b.Stream,
UpstreamBody: body,
Header: req.Header,
SessionID: sessionID(req, r.cfg.SessionHeader, b.User),
Band: req.Header.Get(r.cfg.Admission.BandHeader),
ProfileOverride: req.Header.Get("x-llmgw-profile"),
MaxTokens: max(b.MaxTokens, b.MaxCompletionTokens),
PromptPrefix: promptPrefix(&b),
}
// Bytes over four: a crude stand-in for a tokenizer, good enough for
// scoring. It becomes exact only with plan §7.5 tier 2 (see plan §14).
p.PromptTokens = len(p.PromptPrefix) / 4
// Ask the engine for usage the client did not request, so accounting works
// regardless of client behaviour; the extra chunk is dropped on the way back.
if r.cfg.InjectUsage && b.Stream && (b.StreamOptions == nil || !b.StreamOptions.IncludeUsage) {
if injected, ok := injectIncludeUsage(body); ok {
p.UpstreamBody = injected
p.SuppressUsageChunk = true
}
}
return p, nil
}
// sessionID picks the affinity key: an explicit header first, then OpenAI's
// user field, which most clients already set per end user.
func sessionID(req *http.Request, header, user string) string {
if header != "" {
if v := req.Header.Get(header); v != "" {
return v
}
}
return user
}
// promptPrefix reconstructs the leading portion of the prompt for prefix-aware
// routing. Only the first prefixBytes matter, so long conversations stop early.
func promptPrefix(b *inferenceBody) string {
var sb strings.Builder
appendRaw := func(raw json.RawMessage) {
if len(raw) == 0 || sb.Len() >= prefixBytes {
return
}
var s string
if err := json.Unmarshal(raw, &s); err != nil {
s = string(raw)
}
sb.WriteString(s)
}
for _, m := range b.Messages {
if sb.Len() >= prefixBytes {
break
}
sb.WriteString(m.Role)
sb.WriteString(":")
appendRaw(m.Content)
sb.WriteString("\n")
}
appendRaw(b.Prompt)
appendRaw(b.Input)
s := sb.String()
if len(s) > prefixBytes {
s = s[:prefixBytes]
}
return s
}
// injectIncludeUsage sets stream_options.include_usage on the outgoing body.
//
// This is the one place the router rewrites a request, so it is done by
// decoding to a generic map and re-encoding: field order changes, but no field
// is lost. If anything looks unexpected it declines rather than guessing.
func injectIncludeUsage(body []byte) ([]byte, bool) {
var m map[string]json.RawMessage
if err := json.Unmarshal(body, &m); err != nil {
return nil, false
}
opts := map[string]any{}
if raw, ok := m["stream_options"]; ok {
if err := json.Unmarshal(raw, &opts); err != nil {
return nil, false
}
}
opts["include_usage"] = true
encoded, err := json.Marshal(opts)
if err != nil {
return nil, false
}
m["stream_options"] = encoded
out, err := json.Marshal(m)
if err != nil {
return nil, false
}
return out, true
}
// servedModels lists the canonical models at least one endpoint serves.
func (r *Router) servedModels() []string {
seen := map[string]struct{}{}
for _, s := range r.reg.Snapshots("") {
for _, m := range s.Models {
seen[m] = struct{}{}
}
}
out := make([]string, 0, len(seen))
for m := range seen {
out = append(out, m)
}
slices.Sort(out)
return out
}
// handleModels lists the models that at least one endpoint currently serves.
func (r *Router) handleModels(w http.ResponseWriter, _ *http.Request) {
seen := map[string]struct{}{}
for _, s := range r.reg.Snapshots("") {
for _, m := range s.Models {
seen[m] = struct{}{}
}
}
// Catalog aliases are usable names too, so clients should see them.
for name := range r.models {
if _, ok := seen[r.models[name].Name]; ok {
seen[name] = struct{}{}
}
}
names := make([]string, 0, len(seen))
for n := range seen {
names = append(names, n)
}
slices.Sort(names)
data := make([]map[string]any, 0, len(names))
for _, n := range names {
data = append(data, map[string]any{
"id": n, "object": "model", "created": time.Now().Unix(), "owned_by": "llmgw",
})
}
writeJSON(w, http.StatusOK, map[string]any{"object": "list", "data": data})
}
func writeJSON(w http.ResponseWriter, code int, v any) {
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(v); err != nil {
http.Error(w, `{"error":{"message":"encoding failed"}}`, http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(code)
_, _ = w.Write(buf.Bytes())
}
// writeError emits an OpenAI-shaped error so clients need no special handling
// for gateway failures.
func writeError(w http.ResponseWriter, code int, kind, msg string) {
writeJSON(w, code, map[string]any{"error": map[string]any{
"message": msg, "type": kind, "code": nil,
}})
}