package router import ( "bytes" "context" "encoding/json" "errors" "io" "net/http" "slices" "strconv" "strings" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/accounting" "gitea.mynodes.xyz/ilker/llmgw/internal/auth" "gitea.mynodes.xyz/ilker/llmgw/internal/router/admission" "gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy" "gitea.mynodes.xyz/ilker/llmgw/internal/store" "gitea.mynodes.xyz/ilker/llmgw/internal/version" "gitea.mynodes.xyz/ilker/llmgw/internal/web" ) // admit waits for the flow control layer to let the request through. With // admission disabled it returns immediately, which is the v0.2.0 behaviour. func (r *Router) admit(ctx context.Context, req *parsedRequest) (*admission.Lease, error) { if r.adm == nil { return nil, nil } // The key is the tenant when there is one: fairness within a band has to be // per caller, and a session id is something the caller picks, so one client // could take a whole band by opening many sessions. Without authentication // the session is still the best available grouping. tenant := req.Tenant if tenant == "" { tenant = req.SessionID } return r.adm.Acquire(ctx, admission.Request{ Model: req.Model, Band: req.Band, Tenant: tenant, }) } // rejectAdmission turns a flow control refusal into a status the client can act // on: 503 when the queue is full (try later), 504 when it timed out waiting. func (r *Router) rejectAdmission(w http.ResponseWriter, req *parsedRequest, err error, start time.Time) { status := http.StatusServiceUnavailable switch { case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): // The client gave up while queued; nobody is listening. r.m.observeAdmissionRejected(req.Band, "client_gone") r.m.observeRequest(req.Model, "", 499, time.Since(start), nil, 0) return case errors.Is(err, admission.ErrQueueTimeout): status = http.StatusGatewayTimeout } reason := "unavailable" switch { case errors.Is(err, admission.ErrQueueFull): reason = "queue_full" case errors.Is(err, admission.ErrQueueTimeout): reason = "ttl" case errors.Is(err, admission.ErrClosed): reason = "shutdown" } r.m.observeAdmissionRejected(req.Band, reason) writeError(w, status, "server_error", err.Error()) r.m.observeRequest(req.Model, "", status, time.Since(start), nil, 0) } // prefixBytes is how much of the prompt is kept for prefix-aware routing. // Large enough to cover a system prompt plus the first message, small enough // that hashing it stays cheap. const prefixBytes = 2048 // Handler returns the client-facing OpenAI API. func (r *Router) Handler() http.Handler { mux := http.NewServeMux() mux.HandleFunc("POST /v1/chat/completions", r.handleInference("/v1/chat/completions")) mux.HandleFunc("POST /v1/completions", r.handleInference("/v1/completions")) mux.HandleFunc("POST /v1/embeddings", r.handleInference("/v1/embeddings")) mux.HandleFunc("GET /v1/models", r.handleModels) mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) // The landing page says how to connect and nothing about what is here, so // it is safe to serve without a key — and it has to be, because the person // reading it is the one who does not have one yet. mux.HandleFunc("GET /", web.Landing(version.Version, r.auth != nil)) mux.Handle("GET /_static/", web.Static()) if r.auth == nil { return mux } // Wrapping the whole mux rather than checking inside each handler: adding a // route must not be able to add an unauthenticated one by omission. Health // stays open because a load balancer has no key. return auth.Middleware(r.auth, r.log, "/health", "/", "/_static/")(mux) } // parsedRequest is what the router needs from a request. Everything else in the // body is forwarded untouched — clients send engine-specific fields we have no // business interpreting. type parsedRequest struct { Path string Model string Stream bool SuppressUsageChunk bool UpstreamBody []byte Header http.Header SessionID string Band string // KeyID, ProjectID and Tenant are empty when authentication is off. KeyID string ProjectID string Tenant string ProfileOverride string PromptPrefix string PromptTokens int MaxTokens int // reservation is carried from the limit check to the point where the // request is settled, so the record counts against the same budget window // the check read. reservation accounting.Reservation } // inferenceBody covers the fields the router reads across all inference // endpoints. Unknown fields are ignored here and preserved in UpstreamBody. type inferenceBody struct { Model string `json:"model"` Stream bool `json:"stream"` User string `json:"user"` Messages []struct { Role string `json:"role"` Content json.RawMessage `json:"content"` } `json:"messages"` Prompt json.RawMessage `json:"prompt"` Input json.RawMessage `json:"input"` MaxTokens int `json:"max_tokens"` MaxCompletionTokens int `json:"max_completion_tokens"` StreamOptions *struct { IncludeUsage bool `json:"include_usage"` } `json:"stream_options"` } func (r *Router) handleInference(path string) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { start := time.Now() parsed, err := r.parse(req, path) if err != nil { writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error()) r.m.observeRequest("", "", http.StatusBadRequest, time.Since(start), nil, 0) return } if p, ok := auth.FromContext(req.Context()); ok { if !r.permits(p, parsed.Model) { auth.WriteForbiddenModel(w, parsed.Model) r.m.observeRequest(parsed.Model, "", http.StatusForbidden, time.Since(start), nil, 0) return } // Limits are checked before admission, not after: a request that is // over its quota should be refused rather than queued, and queueing // it would have it occupy a band's capacity to no purpose. if r.acct != nil { reservation, err := r.acct.Check(p, parsed.Model, parsed.estimatedTokens()) if err != nil { r.rejectLimit(w, parsed, err, start) return } parsed.reservation = reservation } } // Step 5 of the lifecycle: "now?" before "where?" (plan §7.1). lease, err := r.admit(req.Context(), parsed) if err != nil { r.rejectAdmission(w, parsed, err, start) return } defer lease.Release() queued := time.Since(start) r.m.observeAdmission(lease.Band(), parsed.Model, queued) if queued > time.Millisecond { w.Header().Set("x-llmgw-queue-ms", strconv.FormatInt(queued.Milliseconds(), 10)) } res := r.dispatch(req.Context(), w, parsed) r.finish(w, parsed, res, start) } } // permits reports whether a key's allowlist covers the requested model. // // The decision is per model, not per string. A model's canonical name and every // alias of it name the same thing, so the allowlist is matched against all of // them: writing either form works, and no alias can reach a model that was not // allowed. Matching only what the client sent would make every alias a way // around the allowlist; matching only the canonical name would silently ignore // an allowlist written in the names clients actually use. func (r *Router) permits(p auth.Principal, requested string) bool { canonical, m := r.resolve(requested) if p.Allows(canonical) { return true } for _, alias := range m.Aliases { if p.Allows(alias) { return true } } return false } // finish records metrics and, when nothing was written, produces the error body. func (r *Router) finish(w http.ResponseWriter, req *parsedRequest, res dispatchResult, start time.Time) { status := res.outcome.Status switch { case res.err == nil: if res.fallback { // The client asked for one model and got another; say so rather // than letting them discover it from the response contents. w.Header().Set("x-llmgw-served-model", res.model) } case errors.Is(res.err, proxy.ErrClientGone): // Nothing to report to a client that is gone; the upstream was cancelled. status = 499 r.log.Debug("client disconnected", "model", req.Model, "endpoint", res.endpointID) case res.outcome.Wrote: // The response is already partly out; the stream just ends short. r.log.Warn("stream ended early", "model", req.Model, "endpoint", res.endpointID, "err", res.err) if status == 0 { status = http.StatusOK } case errors.Is(res.err, errNoCapacity): status = http.StatusServiceUnavailable writeError(w, status, "server_error", res.err.Error()) default: status = http.StatusBadGateway writeError(w, status, "server_error", res.err.Error()) } r.m.observeRequest(req.Model, res.pool, status, time.Since(start), res.outcome.Usage, res.outcome.TTFT) r.account(req, res, status, time.Since(start)) } // account settles a finished request against its key. // // It runs for every outcome, not only success: a request that used a GPU and // then failed still used it, and a key that could spend nothing by failing // would be a way around a budget. func (r *Router) account(req *parsedRequest, res dispatchResult, status int, took time.Duration) { if r.acct == nil || req.KeyID == "" { return } u := store.Usage{ At: time.Now(), KeyID: req.KeyID, ProjectID: req.ProjectID, Model: req.Model, ServedModel: res.model, Status: status, LatencyMs: took.Milliseconds(), } if u.ServedModel == "" { u.ServedModel = req.Model } if usage := res.outcome.Usage; usage != nil { u.PromptTokens = int64(usage.PromptTokens) u.CompletionTokens = int64(usage.CompletionTokens) } else { // No usage came back — a failure before the engine answered, or an // engine that does not report it. Charging the estimate is closer to // the truth than charging nothing. u.PromptTokens = int64(req.PromptTokens) } r.acct.Record(req.reservation, u) } // rejectLimit turns a quota refusal into a status the client can act on. func (r *Router) rejectLimit(w http.ResponseWriter, req *parsedRequest, err error, start time.Time) { var le *accounting.LimitError if !errors.As(err, &le) { writeError(w, http.StatusInternalServerError, "server_error", err.Error()) return } if le.RetryAfter > 0 { w.Header().Set("Retry-After", strconv.Itoa(int(le.RetryAfter.Seconds()))) } // 429 for all of them: a spent budget is a quota, and OpenAI clients // already know how to surface that code. What differs is Retry-After, // which is absent when waiting does not help. writeError(w, http.StatusTooManyRequests, "rate_limit_exceeded", le.Error()) r.m.observeRequest(req.Model, "", http.StatusTooManyRequests, time.Since(start), nil, 0) } // parse reads the body and extracts the routing-relevant fields. func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) { body, err := io.ReadAll(io.LimitReader(req.Body, r.cfg.MaxBodyBytes)) if err != nil { return nil, err } var b inferenceBody if err := json.Unmarshal(body, &b); err != nil { return nil, err } if b.Model == "" { return nil, errors.New("model is required") } p := &parsedRequest{ Path: path, Model: b.Model, Stream: b.Stream, UpstreamBody: body, Header: req.Header, SessionID: sessionID(req, r.cfg.SessionHeader, b.User), Band: req.Header.Get(r.cfg.Admission.BandHeader), ProfileOverride: req.Header.Get("x-llmgw-profile"), MaxTokens: max(b.MaxTokens, b.MaxCompletionTokens), PromptPrefix: promptPrefix(&b), } // With authentication on, the key decides the band and the routing profile, // and the headers stop being believed. Trusting the header was only ever // acceptable while there was no identity to consult — it let any client // promote itself into the top priority band. if principal, ok := auth.FromContext(req.Context()); ok { p.Band = principal.Band p.ProfileOverride = principal.RoutingProfile p.KeyID = principal.KeyID p.ProjectID = principal.ProjectID // Tenant fairness within a band is per key rather than per session, so // one key cannot take a whole band by opening many sessions. p.Tenant = principal.KeyID } // Bytes over four: a crude stand-in for a tokenizer, good enough for // scoring. It becomes exact only with plan §7.5 tier 2 (see plan §14). p.PromptTokens = len(p.PromptPrefix) / 4 // Ask the engine for usage the client did not request, so accounting works // regardless of client behaviour; the extra chunk is dropped on the way back. if r.cfg.InjectUsage && b.Stream && (b.StreamOptions == nil || !b.StreamOptions.IncludeUsage) { if injected, ok := injectIncludeUsage(body); ok { p.UpstreamBody = injected p.SuppressUsageChunk = true } } return p, nil } // sessionID picks the affinity key: an explicit header first, then OpenAI's // user field, which most clients already set per end user. func sessionID(req *http.Request, header, user string) string { if header != "" { if v := req.Header.Get(header); v != "" { return v } } return user } // promptPrefix reconstructs the leading portion of the prompt for prefix-aware // routing. Only the first prefixBytes matter, so long conversations stop early. func promptPrefix(b *inferenceBody) string { var sb strings.Builder appendRaw := func(raw json.RawMessage) { if len(raw) == 0 || sb.Len() >= prefixBytes { return } var s string if err := json.Unmarshal(raw, &s); err != nil { s = string(raw) } sb.WriteString(s) } for _, m := range b.Messages { if sb.Len() >= prefixBytes { break } sb.WriteString(m.Role) sb.WriteString(":") appendRaw(m.Content) sb.WriteString("\n") } appendRaw(b.Prompt) appendRaw(b.Input) s := sb.String() if len(s) > prefixBytes { s = s[:prefixBytes] } return s } // injectIncludeUsage sets stream_options.include_usage on the outgoing body. // // This is the one place the router rewrites a request, so it is done by // decoding to a generic map and re-encoding: field order changes, but no field // is lost. If anything looks unexpected it declines rather than guessing. func injectIncludeUsage(body []byte) ([]byte, bool) { var m map[string]json.RawMessage if err := json.Unmarshal(body, &m); err != nil { return nil, false } opts := map[string]any{} if raw, ok := m["stream_options"]; ok { if err := json.Unmarshal(raw, &opts); err != nil { return nil, false } } opts["include_usage"] = true encoded, err := json.Marshal(opts) if err != nil { return nil, false } m["stream_options"] = encoded out, err := json.Marshal(m) if err != nil { return nil, false } return out, true } // servedModels lists the canonical models at least one endpoint serves. func (r *Router) servedModels() []string { seen := map[string]struct{}{} for _, s := range r.reg.Snapshots("") { for _, m := range s.Models { seen[m] = struct{}{} } } out := make([]string, 0, len(seen)) for m := range seen { out = append(out, m) } slices.Sort(out) return out } // handleModels lists the models that at least one endpoint currently serves. func (r *Router) handleModels(w http.ResponseWriter, _ *http.Request) { seen := map[string]struct{}{} for _, s := range r.reg.Snapshots("") { for _, m := range s.Models { seen[m] = struct{}{} } } // Catalog aliases are usable names too, so clients should see them. for name := range r.models { if _, ok := seen[r.models[name].Name]; ok { seen[name] = struct{}{} } } names := make([]string, 0, len(seen)) for n := range seen { names = append(names, n) } slices.Sort(names) data := make([]map[string]any, 0, len(names)) for _, n := range names { data = append(data, map[string]any{ "id": n, "object": "model", "created": time.Now().Unix(), "owned_by": "llmgw", }) } writeJSON(w, http.StatusOK, map[string]any{"object": "list", "data": data}) } func writeJSON(w http.ResponseWriter, code int, v any) { var buf bytes.Buffer if err := json.NewEncoder(&buf).Encode(v); err != nil { http.Error(w, `{"error":{"message":"encoding failed"}}`, http.StatusInternalServerError) return } w.Header().Set("Content-Type", "application/json") w.WriteHeader(code) _, _ = w.Write(buf.Bytes()) } // writeError emits an OpenAI-shaped error so clients need no special handling // for gateway failures. func writeError(w http.ResponseWriter, code int, kind, msg string) { writeJSON(w, code, map[string]any{"error": map[string]any{ "message": msg, "type": kind, "code": nil, }}) } // estimatedTokens is what a token-rate limit is checked against, before the // real figure exists. // // The prompt estimate plus whatever the client said it wanted back. A client // that names no max_tokens gets a modest assumption rather than zero: assuming // zero would let an unbounded request through every token limit. func (p *parsedRequest) estimatedTokens() int64 { const assumedCompletion = 256 completion := p.MaxTokens if completion <= 0 { completion = assumedCompletion } return int64(p.PromptTokens + completion) }