Files
llmgw/internal/router/handlers.go
T
Ilker ManapandClaude Opus 5 1839832b8b Add a landing page and a read-only cluster dashboard
Pasting the gateway's address into a browser returned 404, which looks like a
broken endpoint rather than one that tells you how to use it.

Two pages, and the split between them is a security boundary rather than a
layout choice. The landing page is served without a key — it has to be,
because the person reading it is the one who does not have one yet — so it
contains no data at all: it says how to connect, not what is here. Model names
stay behind /v1/models. It reconstructs the address from the forwarded headers
so the copy-pasteable example names the address the client actually used
rather than the internal one; those headers are reflected into displayed text
and never used for a decision.

The dashboard shows nodes, models, replicas, placement failures and usage, and
lives on the admin listener only. That listener is loopback-bound and also
serves pprof, which can dump the process heap — where API keys sit in
plaintext — so reaching the page from elsewhere is an SSH tunnel's job. It is
read-only: anything that mints a credential stays in `llmgw keys`, where it
needs a shell rather than a browser session this project has no way to
authenticate.

Mounting the landing page uncovered a hole in the exemption list. An entry
ending in "/" was a prefix rule, and "/" ends in "/", so exempting the landing
page would have exempted every path there is — authentication still installed,
still logging, and letting the whole API through. "/" is now exact, with a
test enumerating what must and must not be exempt.

Server-rendered templates and one stylesheet, embedded. The single-binary goal
rules out a JavaScript toolchain and nothing on either page needs one.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 08:58:05 +02:00

510 lines
17 KiB
Go

package router
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"net/http"
"slices"
"strconv"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/admission"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
"gitea.mynodes.xyz/ilker/llmgw/internal/web"
)
// admit waits for the flow control layer to let the request through. With
// admission disabled it returns immediately, which is the v0.2.0 behaviour.
func (r *Router) admit(ctx context.Context, req *parsedRequest) (*admission.Lease, error) {
if r.adm == nil {
return nil, nil
}
// The key is the tenant when there is one: fairness within a band has to be
// per caller, and a session id is something the caller picks, so one client
// could take a whole band by opening many sessions. Without authentication
// the session is still the best available grouping.
tenant := req.Tenant
if tenant == "" {
tenant = req.SessionID
}
return r.adm.Acquire(ctx, admission.Request{
Model: req.Model,
Band: req.Band,
Tenant: tenant,
})
}
// rejectAdmission turns a flow control refusal into a status the client can act
// on: 503 when the queue is full (try later), 504 when it timed out waiting.
func (r *Router) rejectAdmission(w http.ResponseWriter, req *parsedRequest, err error, start time.Time) {
status := http.StatusServiceUnavailable
switch {
case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded):
// The client gave up while queued; nobody is listening.
r.m.observeAdmissionRejected(req.Band, "client_gone")
r.m.observeRequest(req.Model, "", 499, time.Since(start), nil, 0)
return
case errors.Is(err, admission.ErrQueueTimeout):
status = http.StatusGatewayTimeout
}
reason := "unavailable"
switch {
case errors.Is(err, admission.ErrQueueFull):
reason = "queue_full"
case errors.Is(err, admission.ErrQueueTimeout):
reason = "ttl"
case errors.Is(err, admission.ErrClosed):
reason = "shutdown"
}
r.m.observeAdmissionRejected(req.Band, reason)
writeError(w, status, "server_error", err.Error())
r.m.observeRequest(req.Model, "", status, time.Since(start), nil, 0)
}
// prefixBytes is how much of the prompt is kept for prefix-aware routing.
// Large enough to cover a system prompt plus the first message, small enough
// that hashing it stays cheap.
const prefixBytes = 2048
// Handler returns the client-facing OpenAI API.
func (r *Router) Handler() http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("POST /v1/chat/completions", r.handleInference("/v1/chat/completions"))
mux.HandleFunc("POST /v1/completions", r.handleInference("/v1/completions"))
mux.HandleFunc("POST /v1/embeddings", r.handleInference("/v1/embeddings"))
mux.HandleFunc("GET /v1/models", r.handleModels)
mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
})
// The landing page says how to connect and nothing about what is here, so
// it is safe to serve without a key — and it has to be, because the person
// reading it is the one who does not have one yet.
mux.HandleFunc("GET /", web.Landing(version.Version, r.auth != nil))
mux.Handle("GET /_static/", web.Static())
if r.auth == nil {
return mux
}
// Wrapping the whole mux rather than checking inside each handler: adding a
// route must not be able to add an unauthenticated one by omission. Health
// stays open because a load balancer has no key.
return auth.Middleware(r.auth, r.log, "/health", "/", "/_static/")(mux)
}
// parsedRequest is what the router needs from a request. Everything else in the
// body is forwarded untouched — clients send engine-specific fields we have no
// business interpreting.
type parsedRequest struct {
Path string
Model string
Stream bool
SuppressUsageChunk bool
UpstreamBody []byte
Header http.Header
SessionID string
Band string
// KeyID, ProjectID and Tenant are empty when authentication is off.
KeyID string
ProjectID string
Tenant string
ProfileOverride string
PromptPrefix string
PromptTokens int
MaxTokens int
// reservation is carried from the limit check to the point where the
// request is settled, so the record counts against the same budget window
// the check read.
reservation accounting.Reservation
}
// inferenceBody covers the fields the router reads across all inference
// endpoints. Unknown fields are ignored here and preserved in UpstreamBody.
type inferenceBody struct {
Model string `json:"model"`
Stream bool `json:"stream"`
User string `json:"user"`
Messages []struct {
Role string `json:"role"`
Content json.RawMessage `json:"content"`
} `json:"messages"`
Prompt json.RawMessage `json:"prompt"`
Input json.RawMessage `json:"input"`
MaxTokens int `json:"max_tokens"`
MaxCompletionTokens int `json:"max_completion_tokens"`
StreamOptions *struct {
IncludeUsage bool `json:"include_usage"`
} `json:"stream_options"`
}
func (r *Router) handleInference(path string) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
start := time.Now()
parsed, err := r.parse(req, path)
if err != nil {
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
r.m.observeRequest("", "", http.StatusBadRequest, time.Since(start), nil, 0)
return
}
if p, ok := auth.FromContext(req.Context()); ok {
if !r.permits(p, parsed.Model) {
auth.WriteForbiddenModel(w, parsed.Model)
r.m.observeRequest(parsed.Model, "", http.StatusForbidden, time.Since(start), nil, 0)
return
}
// Limits are checked before admission, not after: a request that is
// over its quota should be refused rather than queued, and queueing
// it would have it occupy a band's capacity to no purpose.
if r.acct != nil {
reservation, err := r.acct.Check(p, parsed.Model, parsed.estimatedTokens())
if err != nil {
r.rejectLimit(w, parsed, err, start)
return
}
parsed.reservation = reservation
}
}
// Step 5 of the lifecycle: "now?" before "where?" (plan §7.1).
lease, err := r.admit(req.Context(), parsed)
if err != nil {
r.rejectAdmission(w, parsed, err, start)
return
}
defer lease.Release()
queued := time.Since(start)
r.m.observeAdmission(lease.Band(), parsed.Model, queued)
if queued > time.Millisecond {
w.Header().Set("x-llmgw-queue-ms", strconv.FormatInt(queued.Milliseconds(), 10))
}
res := r.dispatch(req.Context(), w, parsed)
r.finish(w, parsed, res, start)
}
}
// permits reports whether a key's allowlist covers the requested model.
//
// The decision is per model, not per string. A model's canonical name and every
// alias of it name the same thing, so the allowlist is matched against all of
// them: writing either form works, and no alias can reach a model that was not
// allowed. Matching only what the client sent would make every alias a way
// around the allowlist; matching only the canonical name would silently ignore
// an allowlist written in the names clients actually use.
func (r *Router) permits(p auth.Principal, requested string) bool {
canonical, m := r.resolve(requested)
if p.Allows(canonical) {
return true
}
for _, alias := range m.Aliases {
if p.Allows(alias) {
return true
}
}
return false
}
// finish records metrics and, when nothing was written, produces the error body.
func (r *Router) finish(w http.ResponseWriter, req *parsedRequest, res dispatchResult, start time.Time) {
status := res.outcome.Status
switch {
case res.err == nil:
if res.fallback {
// The client asked for one model and got another; say so rather
// than letting them discover it from the response contents.
w.Header().Set("x-llmgw-served-model", res.model)
}
case errors.Is(res.err, proxy.ErrClientGone):
// Nothing to report to a client that is gone; the upstream was cancelled.
status = 499
r.log.Debug("client disconnected", "model", req.Model, "endpoint", res.endpointID)
case res.outcome.Wrote:
// The response is already partly out; the stream just ends short.
r.log.Warn("stream ended early", "model", req.Model, "endpoint", res.endpointID, "err", res.err)
if status == 0 {
status = http.StatusOK
}
case errors.Is(res.err, errNoCapacity):
status = http.StatusServiceUnavailable
writeError(w, status, "server_error", res.err.Error())
default:
status = http.StatusBadGateway
writeError(w, status, "server_error", res.err.Error())
}
r.m.observeRequest(req.Model, res.pool, status, time.Since(start), res.outcome.Usage, res.outcome.TTFT)
r.account(req, res, status, time.Since(start))
}
// account settles a finished request against its key.
//
// It runs for every outcome, not only success: a request that used a GPU and
// then failed still used it, and a key that could spend nothing by failing
// would be a way around a budget.
func (r *Router) account(req *parsedRequest, res dispatchResult, status int, took time.Duration) {
if r.acct == nil || req.KeyID == "" {
return
}
u := store.Usage{
At: time.Now(), KeyID: req.KeyID, ProjectID: req.ProjectID,
Model: req.Model, ServedModel: res.model, Status: status,
LatencyMs: took.Milliseconds(),
}
if u.ServedModel == "" {
u.ServedModel = req.Model
}
if usage := res.outcome.Usage; usage != nil {
u.PromptTokens = int64(usage.PromptTokens)
u.CompletionTokens = int64(usage.CompletionTokens)
} else {
// No usage came back — a failure before the engine answered, or an
// engine that does not report it. Charging the estimate is closer to
// the truth than charging nothing.
u.PromptTokens = int64(req.PromptTokens)
}
r.acct.Record(req.reservation, u)
}
// rejectLimit turns a quota refusal into a status the client can act on.
func (r *Router) rejectLimit(w http.ResponseWriter, req *parsedRequest, err error, start time.Time) {
var le *accounting.LimitError
if !errors.As(err, &le) {
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
return
}
if le.RetryAfter > 0 {
w.Header().Set("Retry-After", strconv.Itoa(int(le.RetryAfter.Seconds())))
}
// 429 for all of them: a spent budget is a quota, and OpenAI clients
// already know how to surface that code. What differs is Retry-After,
// which is absent when waiting does not help.
writeError(w, http.StatusTooManyRequests, "rate_limit_exceeded", le.Error())
r.m.observeRequest(req.Model, "", http.StatusTooManyRequests, time.Since(start), nil, 0)
}
// parse reads the body and extracts the routing-relevant fields.
func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) {
body, err := io.ReadAll(io.LimitReader(req.Body, r.cfg.MaxBodyBytes))
if err != nil {
return nil, err
}
var b inferenceBody
if err := json.Unmarshal(body, &b); err != nil {
return nil, err
}
if b.Model == "" {
return nil, errors.New("model is required")
}
p := &parsedRequest{
Path: path,
Model: b.Model,
Stream: b.Stream,
UpstreamBody: body,
Header: req.Header,
SessionID: sessionID(req, r.cfg.SessionHeader, b.User),
Band: req.Header.Get(r.cfg.Admission.BandHeader),
ProfileOverride: req.Header.Get("x-llmgw-profile"),
MaxTokens: max(b.MaxTokens, b.MaxCompletionTokens),
PromptPrefix: promptPrefix(&b),
}
// With authentication on, the key decides the band and the routing profile,
// and the headers stop being believed. Trusting the header was only ever
// acceptable while there was no identity to consult — it let any client
// promote itself into the top priority band.
if principal, ok := auth.FromContext(req.Context()); ok {
p.Band = principal.Band
p.ProfileOverride = principal.RoutingProfile
p.KeyID = principal.KeyID
p.ProjectID = principal.ProjectID
// Tenant fairness within a band is per key rather than per session, so
// one key cannot take a whole band by opening many sessions.
p.Tenant = principal.KeyID
}
// Bytes over four: a crude stand-in for a tokenizer, good enough for
// scoring. It becomes exact only with plan §7.5 tier 2 (see plan §14).
p.PromptTokens = len(p.PromptPrefix) / 4
// Ask the engine for usage the client did not request, so accounting works
// regardless of client behaviour; the extra chunk is dropped on the way back.
if r.cfg.InjectUsage && b.Stream && (b.StreamOptions == nil || !b.StreamOptions.IncludeUsage) {
if injected, ok := injectIncludeUsage(body); ok {
p.UpstreamBody = injected
p.SuppressUsageChunk = true
}
}
return p, nil
}
// sessionID picks the affinity key: an explicit header first, then OpenAI's
// user field, which most clients already set per end user.
func sessionID(req *http.Request, header, user string) string {
if header != "" {
if v := req.Header.Get(header); v != "" {
return v
}
}
return user
}
// promptPrefix reconstructs the leading portion of the prompt for prefix-aware
// routing. Only the first prefixBytes matter, so long conversations stop early.
func promptPrefix(b *inferenceBody) string {
var sb strings.Builder
appendRaw := func(raw json.RawMessage) {
if len(raw) == 0 || sb.Len() >= prefixBytes {
return
}
var s string
if err := json.Unmarshal(raw, &s); err != nil {
s = string(raw)
}
sb.WriteString(s)
}
for _, m := range b.Messages {
if sb.Len() >= prefixBytes {
break
}
sb.WriteString(m.Role)
sb.WriteString(":")
appendRaw(m.Content)
sb.WriteString("\n")
}
appendRaw(b.Prompt)
appendRaw(b.Input)
s := sb.String()
if len(s) > prefixBytes {
s = s[:prefixBytes]
}
return s
}
// injectIncludeUsage sets stream_options.include_usage on the outgoing body.
//
// This is the one place the router rewrites a request, so it is done by
// decoding to a generic map and re-encoding: field order changes, but no field
// is lost. If anything looks unexpected it declines rather than guessing.
func injectIncludeUsage(body []byte) ([]byte, bool) {
var m map[string]json.RawMessage
if err := json.Unmarshal(body, &m); err != nil {
return nil, false
}
opts := map[string]any{}
if raw, ok := m["stream_options"]; ok {
if err := json.Unmarshal(raw, &opts); err != nil {
return nil, false
}
}
opts["include_usage"] = true
encoded, err := json.Marshal(opts)
if err != nil {
return nil, false
}
m["stream_options"] = encoded
out, err := json.Marshal(m)
if err != nil {
return nil, false
}
return out, true
}
// servedModels lists the canonical models at least one endpoint serves.
func (r *Router) servedModels() []string {
seen := map[string]struct{}{}
for _, s := range r.reg.Snapshots("") {
for _, m := range s.Models {
seen[m] = struct{}{}
}
}
out := make([]string, 0, len(seen))
for m := range seen {
out = append(out, m)
}
slices.Sort(out)
return out
}
// handleModels lists the models that at least one endpoint currently serves.
func (r *Router) handleModels(w http.ResponseWriter, _ *http.Request) {
seen := map[string]struct{}{}
for _, s := range r.reg.Snapshots("") {
for _, m := range s.Models {
seen[m] = struct{}{}
}
}
// Catalog aliases are usable names too, so clients should see them.
for name := range r.models {
if _, ok := seen[r.models[name].Name]; ok {
seen[name] = struct{}{}
}
}
names := make([]string, 0, len(seen))
for n := range seen {
names = append(names, n)
}
slices.Sort(names)
data := make([]map[string]any, 0, len(names))
for _, n := range names {
data = append(data, map[string]any{
"id": n, "object": "model", "created": time.Now().Unix(), "owned_by": "llmgw",
})
}
writeJSON(w, http.StatusOK, map[string]any{"object": "list", "data": data})
}
func writeJSON(w http.ResponseWriter, code int, v any) {
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(v); err != nil {
http.Error(w, `{"error":{"message":"encoding failed"}}`, http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(code)
_, _ = w.Write(buf.Bytes())
}
// writeError emits an OpenAI-shaped error so clients need no special handling
// for gateway failures.
func writeError(w http.ResponseWriter, code int, kind, msg string) {
writeJSON(w, code, map[string]any{"error": map[string]any{
"message": msg, "type": kind, "code": nil,
}})
}
// estimatedTokens is what a token-rate limit is checked against, before the
// real figure exists.
//
// The prompt estimate plus whatever the client said it wanted back. A client
// that names no max_tokens gets a modest assumption rather than zero: assuming
// zero would let an unbounded request through every token limit.
func (p *parsedRequest) estimatedTokens() int64 {
const assumedCompletion = 256
completion := p.MaxTokens
if completion <= 0 {
completion = assumedCompletion
}
return int64(p.PromptTokens + completion)
}