Files
llmgw/internal/router/handlers.go
T
Ilker ManapandClaude Opus 5.5 daed835d88 Measure each node's CPU and memory, and give every node a page
The nodes are CPU machines, and nothing measured how busy one was: the
heartbeat carried GPU memory and disk, and llmgw_node_memory_bytes is the
total. The agent now reads /proc between heartbeats - CPU across all
cores, load averages, memory used and available, and each engine's
process group's cores and resident memory - and sends it with the
heartbeat. Nothing new runs on the nodes and Prometheus needs no new
target: the control plane exports it with the rest of the fleet, as
llmgw_node_cpu_utilization_percent, llmgw_node_cpus, llmgw_node_load,
llmgw_node_memory_used_bytes, llmgw_node_memory_available_bytes,
llmgw_shard_cpu_cores and llmgw_shard_memory_bytes. A measurement more
than a minute old is left out rather than repeated.

The cluster page's node table shows CPU and RAM as bars, and each node
links to a page of its own: current CPU, RAM, load and disk, the engines
on it with their cores and memory, and an hour of CPU, RAM, load,
per-model cores and memory and queue depth from Prometheus. It refreshes
in place.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-28 23:21:23 +02:00

847 lines
30 KiB
Go

package router
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"slices"
"strconv"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/admission"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
"gitea.mynodes.xyz/ilker/llmgw/internal/web"
)
// admit waits for the flow control layer to let the request through. With
// admission disabled it returns immediately, which is the v0.2.0 behaviour.
func (r *Router) admit(ctx context.Context, req *parsedRequest) (*admission.Lease, error) {
if r.adm == nil {
return nil, nil
}
// The key is the tenant when there is one: fairness within a band has to be
// per caller, and a session id is something the caller picks, so one client
// could take a whole band by opening many sessions. Without authentication
// the session is still the best available grouping.
tenant := req.Tenant
if tenant == "" {
tenant = req.SessionID
}
return r.adm.Acquire(ctx, admission.Request{
Model: req.Model,
Band: req.Band,
Tenant: tenant,
})
}
// rejectAdmission turns a flow control refusal into a status the client can act
// on: 503 when the queue is full (try later), 504 when it timed out waiting.
func (r *Router) rejectAdmission(w http.ResponseWriter, req *parsedRequest, err error, start time.Time) {
status := http.StatusServiceUnavailable
switch {
case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded):
// The client gave up while queued; nobody is listening.
r.m.observeAdmissionRejected(req.Band, "client_gone")
r.m.observeRequest(req.Model, "", 499, time.Since(start), nil, 0)
return
case errors.Is(err, admission.ErrQueueTimeout):
status = http.StatusGatewayTimeout
}
reason := "unavailable"
switch {
case errors.Is(err, admission.ErrQueueFull):
reason = "queue_full"
case errors.Is(err, admission.ErrQueueTimeout):
reason = "ttl"
case errors.Is(err, admission.ErrClosed):
reason = "shutdown"
}
r.m.observeAdmissionRejected(req.Band, reason)
writeError(w, status, "server_error", err.Error())
r.m.observeRequest(req.Model, "", status, time.Since(start), nil, 0)
}
// prefixBytes is how much of the prompt is kept for prefix-aware routing.
// Large enough to cover a system prompt plus the first message, small enough
// that hashing it stays cheap.
const prefixBytes = 2048
// Handler returns the client-facing OpenAI API.
func (r *Router) Handler() http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("POST /v1/chat/completions", r.handleInference("/v1/chat/completions"))
mux.HandleFunc("POST /v1/completions", r.handleInference("/v1/completions"))
mux.HandleFunc("POST /v1/embeddings", r.handleInference("/v1/embeddings"))
mux.HandleFunc("GET /v1/models", r.handleModels)
// Liveness: this process is up. It says nothing about whether it can
// serve, which is why it is not the thing to put a load balancer on.
mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
})
// Readiness: this process has somewhere to send a request.
//
// The distinction is not academic. A standby control plane is up, answers
// /health, and has no endpoints at all — its reconciler placed nothing and
// it holds no agent streams to adopt from — so a load balancer checking
// /health would send it half the traffic and half the traffic would be a
// 503. A leader whose replicas have all failed is in the same position for
// a different reason, and a load balancer should treat them the same.
//
// Open, like /health: a load balancer does not have a key, and this leaks
// one bit — whether anything is servable — which the next request would
// reveal anyway.
mux.HandleFunc("GET /ready", func(w http.ResponseWriter, _ *http.Request) {
if len(r.reg.IDs()) == 0 {
http.Error(w, "no endpoints are available to serve", http.StatusServiceUnavailable)
return
}
w.WriteHeader(http.StatusOK)
})
// The landing page says how to connect and nothing about what is here, so
// it is safe to serve without a key — and it has to be, because the person
// reading it is the one who does not have one yet.
mux.HandleFunc("GET /", web.Landing(version.Version, r.auth != nil,
r.accounts != nil && r.accounts.SignupOpen(),
r.landing.Contact, r.landing.Notice))
mux.Handle("GET /_static/", web.Static())
mux.HandleFunc("GET /me", r.handleMe)
// Somebody has to be able to say "forget that", and a retention window is
// no answer to a message sent by mistake. Authenticated like any other
// client endpoint, and scoped to the caller's project by the store.
mux.HandleFunc("DELETE /v1/threads/{id}", r.handleDeleteThread)
// Notes an agent keeps for itself, over REST and as MCP tools. Off unless a
// project or the threads block turned storage on.
mux.HandleFunc("PUT /v1/memory/{key}", r.handlePutNote)
mux.HandleFunc("GET /v1/memory/{key}", r.handleGetNote)
mux.HandleFunc("GET /v1/memory", r.handleSearchNotes)
mux.HandleFunc("DELETE /v1/memory/{key}", r.handleDeleteNote)
mux.HandleFunc("POST /mcp", r.handleMCP)
if r.store != nil {
// The one-time link that delivers a key to the person it was issued
// for. Served without a key, because the person opening it is the one
// who does not have one yet; what protects it is the token.
claim := web.Claim(r.store, version.Version, r.log)
mux.HandleFunc("GET /claim/{token}", claim)
mux.HandleFunc("POST /claim/{token}", claim)
}
if r.accounts != nil {
// Signup, sign-in and the profile page. These are browser surfaces
// authenticated by a session cookie, not by an API key, which is why
// they are exempt below.
pages := web.NewAccountPages(r.accounts, r.store, r.store, version.Version, r.log)
// So the example on the profile page names a model that is actually
// running, rather than a placeholder that gets sent as-is.
pages.ServedModels(r.exampleModels)
pages.SetPersonalOff(r.personalOff)
pages.SetLlamaCppVersion(r.llamaVersion)
pages.SetPersonalDefaults(r.personalDefaults)
pages.Register(mux)
r.registerChat(mux, pages)
// An organization's own page, for the people who manage it.
pages.RegisterOrgPages(mux, r.store, r.acct)
if r.clusterView != nil {
// The cluster dashboard, reachable from a browser without exposing
// the admin listener — which also carries pprof, and pprof can dump
// the heap that holds every API key in plaintext.
mux.HandleFunc("GET /admin", pages.AdminPage(web.Dashboard(
r.clusterView, r.clusterUsage, version.Version, 5*time.Second, false,
web.NavExtras{
Mirror: r.mirror != nil, History: r.history, Window: r.histWindow,
}, r.log)))
mux.HandleFunc("GET /admin/nodes/{id}/logs", pages.AdminPage(
web.NodeLogsPage(r.clusterView, false, version.Version)))
mux.HandleFunc("GET /admin/nodes/{id}", pages.AdminPage(
web.NodePage(r.clusterView, r.history, r.histWindow, false, version.Version, r.log)))
pages.RegisterAdminUsers(mux, r.store)
// Quotas are written on the same tree the accounts page builds, so
// they are mounted together.
pages.RegisterAdminQuotas(mux, r.store, r.acct)
// And the report of what that tree actually used.
pages.RegisterAdminUsage(mux, r.store, r.acct)
}
// Independent of the cluster view: the mirror is a separate machine
// with its own disk, and it is worth administering whether or not this
// process also has a cluster to show.
if r.mirror != nil {
pages.RegisterAdminMirror(mux, web.MirrorDeps{
Mirror: r.mirror, Registry: r.registry,
Catalog: r.catalogAdmin, Fleet: r.clusterView,
})
}
}
if r.auth == nil {
return mux
}
// Wrapping the whole mux rather than checking inside each handler: adding a
// route must not be able to add an unauthenticated one by omission. Health
// stays open because a load balancer has no key.
return auth.Middleware(r.auth, r.log,
"/health", "/ready", "/", "/_static/", "/claim/",
"/signup", "/login", "/logout", "/profile", "/profile/",
"/admin", "/admin/", "/chat", "/chat/", "/org", "/org/")(mux)
}
// parsedRequest is what the router needs from a request. Everything else in the
// body is forwarded untouched — clients send engine-specific fields we have no
// business interpreting.
type parsedRequest struct {
Path string
// Model is what will be dispatched, which is not always what was asked
// for: a caller out of budget may be moved to a model they can afford.
Model string
// RequestedModel is what the client sent, kept so the response can say
// when something else answered.
RequestedModel string
Stream bool
SuppressUsageChunk bool
UpstreamBody []byte
Header http.Header
SessionID string
Band string
// KeyID, ProjectID and Tenant are empty when authentication is off.
KeyID string
UserID string
ProjectID string
Tenant string
ProfileOverride string
PromptPrefix string
PromptTokens int
MaxTokens int
// ThreadID is the conversation this request continues, empty for the
// stateless path every request took before threads existed.
ThreadID string
// ThreadTurns are the turns this request itself carried, kept so the
// exchange can be stored after a reply without re-reading a body that has
// since had the history spliced into it.
ThreadTurns []store.Message
// ThreadProject is the project the thread is stored under, resolved once by
// loadThread so saving it afterwards cannot disagree.
ThreadProject string
// reservation is carried from the limit check to the point where the
// request is settled, so the record counts against the same budget window
// the check read.
reservation accounting.Reservation
}
// inferenceBody covers the fields the router reads across all inference
// endpoints. Unknown fields are ignored here and preserved in UpstreamBody.
type inferenceBody struct {
Model string `json:"model"`
Stream bool `json:"stream"`
User string `json:"user"`
Messages []struct {
Role string `json:"role"`
Content json.RawMessage `json:"content"`
} `json:"messages"`
Prompt json.RawMessage `json:"prompt"`
Input json.RawMessage `json:"input"`
MaxTokens int `json:"max_tokens"`
MaxCompletionTokens int `json:"max_completion_tokens"`
StreamOptions *struct {
IncludeUsage bool `json:"include_usage"`
} `json:"stream_options"`
}
func (r *Router) handleInference(path string) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
start := time.Now()
parsed, err := r.parse(req, path)
if err != nil {
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
r.m.observeRequest("", "", http.StatusBadRequest, time.Since(start), nil, 0)
return
}
// Before authentication work and before admission: a request that
// cannot fit should not consume a quota reservation or a queue slot on
// its way to being refused.
if _, entry := r.resolve(parsed.Model); entry.ContextLength > 0 {
if why := checkContextFits(entry, parsed.PromptTokens, parsed.MaxTokens); why != "" {
writeError(w, http.StatusBadRequest, "invalid_request_error", why)
r.m.observeRequest(parsed.Model, "", http.StatusBadRequest, time.Since(start), nil, 0)
return
}
}
if p, ok := auth.FromContext(req.Context()); ok {
if !r.permits(p, parsed.Model) {
auth.WriteForbiddenModel(w, parsed.Model)
r.m.observeRequest(parsed.Model, "", http.StatusForbidden, time.Since(start), nil, 0)
return
}
// Limits are checked before admission, not after: a request that is
// over its quota should be refused rather than queued, and queueing
// it would have it occupy a band's capacity to no purpose.
if r.acct != nil {
// Priced by the model that will serve, not by the name the
// caller typed. A price table is written against the catalog,
// and an alias asked for by its short name would miss every
// entry and fall to the default — charging a 1.5B at whatever
// rate the largest model set, or at nothing.
priced, _ := r.resolve(parsed.Model)
reservation, err := r.acct.Check(p, priced, parsed.estimatedTokens())
if err != nil && isBudgetSpent(err) {
// Out of money, not out of options. A smaller model the
// caller can still afford beats a refusal, and the catalog
// already says which those are.
if fb, ok := r.budgetFallback(p, parsed); ok {
reservation, err = fb, nil
// Said now rather than at the end: the proxy writes the
// response itself, so a header set after dispatch never
// reaches the client.
w.Header().Set("x-llmgw-served-model", parsed.Model)
}
}
if err != nil {
r.rejectLimit(w, parsed, err, start)
return
}
parsed.reservation = reservation
}
}
// Step 5 of the lifecycle: "now?" before "where?" (plan §7.1).
lease, err := r.admit(req.Context(), parsed)
if err != nil {
r.rejectAdmission(w, parsed, err, start)
return
}
defer lease.Release()
queued := time.Since(start)
r.m.observeAdmission(lease.Band(), parsed.Model, queued)
if queued > time.Millisecond {
w.Header().Set("x-llmgw-queue-ms", strconv.FormatInt(queued.Milliseconds(), 10))
}
// After the model is finally settled — a budget fallback may have
// changed it — because trimming is measured against the context of the
// model that will actually serve, not the one that was asked for.
if dropped, err := r.loadThread(req.Context(), parsed); err != nil {
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
r.m.observeRequest(parsed.Model, "", http.StatusBadRequest, time.Since(start), nil, 0)
return
} else if dropped > 0 {
// Dropping turns is the right answer and must not be a silent one.
w.Header().Set(trimmedHeader, strconv.Itoa(dropped))
}
res := r.dispatch(req.Context(), w, parsed)
r.finish(w, parsed, res, start)
// After the client has been served. A conversation that failed to
// record is a worse outcome than a slow reply, but only just, and the
// caller has their answer either way.
if res.err == nil && res.outcome.Status == http.StatusOK {
r.saveThread(context.WithoutCancel(req.Context()), parsed, res.outcome.Reply, parsed.Model)
}
}
}
// permits reports whether a key's allowlist covers the requested model.
//
// The decision is per model, not per string. A model's canonical name and every
// alias of it name the same thing, so the allowlist is matched against all of
// them: writing either form works, and no alias can reach a model that was not
// allowed. Matching only what the client sent would make every alias a way
// around the allowlist; matching only the canonical name would silently ignore
// an allowlist written in the names clients actually use.
func (r *Router) permits(p auth.Principal, requested string) bool {
canonical, m := r.resolve(requested)
if p.Allows(canonical) {
return true
}
for _, alias := range m.Aliases {
if p.Allows(alias) {
return true
}
}
return false
}
// finish records metrics and, when nothing was written, produces the error body.
func (r *Router) finish(w http.ResponseWriter, req *parsedRequest, res dispatchResult, start time.Time) {
status := res.outcome.Status
switch {
case res.err == nil:
// The served-model header is set before dispatch, not here: the proxy
// writes the response, so by this point the headers are long gone.
case errors.Is(res.err, proxy.ErrClientGone):
// Nothing to report to a client that is gone; the upstream was cancelled.
status = 499
r.log.Debug("client disconnected", "model", req.Model, "endpoint", res.endpointID)
case res.outcome.Wrote:
// The response is already partly out; the stream just ends short.
r.log.Warn("stream ended early", "model", req.Model, "endpoint", res.endpointID, "err", res.err)
if status == 0 {
status = http.StatusOK
}
case errors.Is(res.err, errLoading):
// Retry-After, because this one succeeds on a retry and a client that
// is told so will make it. Without the header a well-behaved client
// backs off exactly as it would from a permanent failure.
status = http.StatusServiceUnavailable
w.Header().Set("Retry-After", "30")
writeError(w, status, "server_error", res.err.Error())
case errors.Is(res.err, errNoCapacity):
status = http.StatusServiceUnavailable
writeError(w, status, "server_error", res.err.Error())
default:
status = http.StatusBadGateway
writeError(w, status, "server_error", res.err.Error())
}
r.m.observeRequest(req.Model, res.pool, status, time.Since(start), res.outcome.Usage, res.outcome.TTFT)
r.account(req, res, status, time.Since(start))
}
// account settles a finished request against its key.
//
// It runs for every outcome, not only success: a request that used a GPU and
// then failed still used it, and a key that could spend nothing by failing
// would be a way around a budget.
func (r *Router) account(req *parsedRequest, res dispatchResult, status int, took time.Duration) {
if r.acct == nil || req.KeyID == "" {
return
}
u := store.Usage{
At: time.Now(), KeyID: req.KeyID, UserID: req.UserID, ProjectID: req.ProjectID,
Model: req.Model, ServedModel: res.model, Status: status,
LatencyMs: took.Milliseconds(),
}
if u.ServedModel == "" {
u.ServedModel = req.Model
}
if usage := res.outcome.Usage; usage != nil {
u.PromptTokens = int64(usage.PromptTokens)
u.CompletionTokens = int64(usage.CompletionTokens)
} else {
// No usage came back — a failure before the engine answered, or an
// engine that does not report it. Charging the estimate is closer to
// the truth than charging nothing.
u.PromptTokens = int64(req.PromptTokens)
}
r.acct.Record(req.reservation, u)
}
// rejectLimit turns a quota refusal into a status the client can act on.
func (r *Router) rejectLimit(w http.ResponseWriter, req *parsedRequest, err error, start time.Time) {
var le *accounting.LimitError
if !errors.As(err, &le) {
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
return
}
if le.RetryAfter > 0 {
w.Header().Set("Retry-After", strconv.Itoa(int(le.RetryAfter.Seconds())))
}
// 429 for all of them: a spent budget is a quota, and OpenAI clients
// already know how to surface that code. What differs is Retry-After,
// which is absent when waiting does not help.
writeError(w, http.StatusTooManyRequests, "rate_limit_exceeded", le.Error())
r.m.observeRequest(req.Model, "", http.StatusTooManyRequests, time.Since(start), nil, 0)
}
// parse reads the body and extracts the routing-relevant fields.
func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) {
body, err := io.ReadAll(io.LimitReader(req.Body, r.cfg.MaxBodyBytes))
if err != nil {
return nil, err
}
var b inferenceBody
if err := json.Unmarshal(body, &b); err != nil {
return nil, err
}
if b.Model == "" {
return nil, errors.New("model is required")
}
p := &parsedRequest{
Path: path,
Model: b.Model,
RequestedModel: b.Model,
Stream: b.Stream,
UpstreamBody: body,
Header: req.Header,
SessionID: sessionID(req, r.cfg.SessionHeader, b.User),
Band: req.Header.Get(r.cfg.Admission.BandHeader),
ProfileOverride: req.Header.Get("x-llmgw-profile"),
MaxTokens: max(b.MaxTokens, b.MaxCompletionTokens),
PromptPrefix: promptPrefix(&b),
ThreadID: req.Header.Get(threadHeader),
}
// With authentication on, the key decides the band and the routing profile,
// and the headers stop being believed. Trusting the header was only ever
// acceptable while there was no identity to consult — it let any client
// promote itself into the top priority band.
if principal, ok := auth.FromContext(req.Context()); ok {
p.Band = principal.Band
p.ProfileOverride = principal.RoutingProfile
p.KeyID = principal.KeyID
p.UserID = principal.UserID
p.ProjectID = principal.ProjectID
// Tenant fairness within a band is per key rather than per session, so
// one key cannot take a whole band by opening many sessions.
p.Tenant = principal.KeyID
}
// Bytes over four: a crude stand-in for a tokenizer, good enough for
// scoring and for limits. It becomes exact only with plan §7.5 tier 2 (see
// plan §14). Measured over the whole prompt, not the routing prefix —
// see promptBytes.
p.PromptTokens = promptBytes(&b) / 4
// Ask the engine for usage the client did not request, so accounting works
// regardless of client behaviour; the extra chunk is dropped on the way back.
if r.cfg.InjectUsage && b.Stream && (b.StreamOptions == nil || !b.StreamOptions.IncludeUsage) {
if injected, ok := injectIncludeUsage(body); ok {
p.UpstreamBody = injected
p.SuppressUsageChunk = true
}
}
return p, nil
}
// sessionID picks the affinity key: an explicit header first, then OpenAI's
// user field, which most clients already set per end user.
func sessionID(req *http.Request, header, user string) string {
if header != "" {
if v := req.Header.Get(header); v != "" {
return v
}
}
return user
}
// promptPrefix reconstructs the leading portion of the prompt for prefix-aware
// routing. Only the first prefixBytes matter, so long conversations stop early.
func promptPrefix(b *inferenceBody) string {
var sb strings.Builder
appendRaw := func(raw json.RawMessage) {
if len(raw) == 0 || sb.Len() >= prefixBytes {
return
}
var s string
if err := json.Unmarshal(raw, &s); err != nil {
s = string(raw)
}
sb.WriteString(s)
}
for _, m := range b.Messages {
if sb.Len() >= prefixBytes {
break
}
sb.WriteString(m.Role)
sb.WriteString(":")
appendRaw(m.Content)
sb.WriteString("\n")
}
appendRaw(b.Prompt)
appendRaw(b.Input)
s := sb.String()
if len(s) > prefixBytes {
s = s[:prefixBytes]
}
return s
}
// promptBytes measures the whole prompt, which promptPrefix deliberately does
// not.
//
// The two were the same function for too long, and the estimate was taken from
// the prefix: bytes over four of at most prefixBytes, so every prompt above
// 2 KB counted as 512 tokens however long it really was. Prefix routing wants a
// bounded head of the conversation; a token estimate wants all of it, and using
// one for the other meant TPM limits undercounted exactly the requests that
// consume the most, and nothing could tell whether a prompt fit in a context.
func promptBytes(b *inferenceBody) int {
n := 0
add := func(raw json.RawMessage) {
if len(raw) == 0 {
return
}
var str string
if err := json.Unmarshal(raw, &str); err == nil {
n += len(str)
return
}
// Not a plain string: structured content, an array of parts, an image.
// The raw JSON overstates the text slightly, which is the safe
// direction for a limit.
n += len(raw)
}
for _, m := range b.Messages {
n += len(m.Role) + 2
add(m.Content)
}
add(b.Prompt)
add(b.Input)
return n
}
// injectIncludeUsage sets stream_options.include_usage on the outgoing body.
//
// This is the one place the router rewrites a request, so it is done by
// decoding to a generic map and re-encoding: field order changes, but no field
// is lost. If anything looks unexpected it declines rather than guessing.
func injectIncludeUsage(body []byte) ([]byte, bool) {
var m map[string]json.RawMessage
if err := json.Unmarshal(body, &m); err != nil {
return nil, false
}
opts := map[string]any{}
if raw, ok := m["stream_options"]; ok {
if err := json.Unmarshal(raw, &opts); err != nil {
return nil, false
}
}
opts["include_usage"] = true
encoded, err := json.Marshal(opts)
if err != nil {
return nil, false
}
m["stream_options"] = encoded
out, err := json.Marshal(m)
if err != nil {
return nil, false
}
return out, true
}
// noEndpointFor explains that nothing serves a model, and says what does.
//
// "no endpoint serves \"MODEL\"" was all a client got, which is true and
// useless: it reads the same whether the name was a typo, a placeholder left
// in from the documentation, or a real outage. Someone copying the curl from
// the account page sent the literal word MODEL and had nothing to go on.
//
// The list is what /v1/models already returns to the same caller, so naming it
// here discloses nothing new. It is capped because a large catalog would turn
// an error message into a page.
func noEndpointFor(model string, served []string) string {
if len(served) == 0 {
return fmt.Sprintf("no endpoint serves %q, and no model is currently being served", model)
}
const showAt = 10
shown := served
suffix := ""
if len(shown) > showAt {
shown, suffix = shown[:showAt], fmt.Sprintf(" and %d more", len(served)-showAt)
}
return fmt.Sprintf("no endpoint serves %q; available models are %s%s",
model, strings.Join(shown, ", "), suffix)
}
// exampleModels is servedModels with the ones that answer for real first.
//
// The profile page takes the first name for its copy-pasteable example, and
// handing it a simulated model would send someone's first request to an engine
// that replies in Greek letters.
func (r *Router) exampleModels() []string {
names := r.servedModels()
slices.SortStableFunc(names, func(a, b string) int {
as, bs := r.modelMap()[a].Simulated(), r.modelMap()[b].Simulated()
switch {
case as == bs:
return 0
case bs:
return -1
default:
return 1
}
})
return names
}
// servedModels lists the canonical models at least one endpoint serves.
func (r *Router) servedModels() []string {
seen := map[string]struct{}{}
for _, s := range r.reg.Snapshots("") {
for _, m := range s.Models {
seen[m] = struct{}{}
}
}
out := make([]string, 0, len(seen))
for m := range seen {
out = append(out, m)
}
slices.Sort(out)
return out
}
// handleModels lists the models that at least one endpoint currently serves.
func (r *Router) handleModels(w http.ResponseWriter, _ *http.Request) {
seen := map[string]struct{}{}
for _, s := range r.reg.Snapshots("") {
for _, m := range s.Models {
seen[m] = struct{}{}
}
}
// Catalog aliases are usable names too, so clients should see them.
for name := range r.modelMap() {
if _, ok := seen[r.modelMap()[name].Name]; ok {
seen[name] = struct{}{}
}
}
names := make([]string, 0, len(seen))
for n := range seen {
names = append(names, n)
}
slices.Sort(names)
data := make([]map[string]any, 0, len(names))
for _, n := range names {
// A simulated model is owned by "llmgw-sim", which is what the sim
// engine already calls itself on its own /v1/models. Flattening it to
// "llmgw" here was hiding the one distinction a client needs: this
// model returns placeholder text, not inference.
owner := "llmgw"
if r.modelMap()[n].Simulated() {
owner = "llmgw-sim"
}
entry := map[string]any{
"id": n, "object": "model", "created": time.Now().Unix(), "owned_by": owner,
}
// Only when known. A context_length of 0 would read as "no context",
// which is a worse answer than saying nothing.
m := r.modelMap()[n]
if m.ContextLength > 0 {
entry["context_length"] = m.ContextLength
}
if m.MaxOutput > 0 {
entry["max_output_tokens"] = m.MaxOutput
}
// Not part of the OpenAI shape, and the reason to send it anyway is
// that this list is the only view most clients have: nine names and
// nothing to choose between them.
if m.Description != "" {
entry["description"] = m.Description
}
entry["capabilities"] = map[string]bool{
"tools": m.Capabilities.Tools, "vision": m.Capabilities.Vision,
"embeddings": m.Capabilities.Embeddings,
}
data = append(data, entry)
}
writeJSON(w, http.StatusOK, map[string]any{"object": "list", "data": data})
}
func writeJSON(w http.ResponseWriter, code int, v any) {
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(v); err != nil {
http.Error(w, `{"error":{"message":"encoding failed"}}`, http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(code)
_, _ = w.Write(buf.Bytes())
}
// writeError emits an OpenAI-shaped error so clients need no special handling
// for gateway failures.
func writeError(w http.ResponseWriter, code int, kind, msg string) {
writeJSON(w, code, map[string]any{"error": map[string]any{
"message": msg, "type": kind, "code": nil,
}})
}
// estimatedTokens is what a token-rate limit is checked against, before the
// real figure exists.
//
// The prompt estimate plus whatever the client said it wanted back. A client
// that names no max_tokens gets a modest assumption rather than zero: assuming
// zero would let an unbounded request through every token limit.
func (p *parsedRequest) estimatedTokens() int64 {
const assumedCompletion = 256
completion := p.MaxTokens
if completion <= 0 {
completion = assumedCompletion
}
return int64(p.PromptTokens + completion)
}
// handleMe reports what the presented key is and what it may do.
//
// It exists so a user can answer "how much have I used" and "why was I
// refused" without asking an operator to read a dashboard they cannot reach.
// Deliberately outside /v1: it is not part of the OpenAI API, and putting it
// there would imply it is.
func (r *Router) handleMe(w http.ResponseWriter, req *http.Request) {
p, ok := auth.FromContext(req.Context())
if !ok {
// With authentication off there is no identity to describe, and saying
// so is more useful than an empty object.
writeJSON(w, http.StatusOK, map[string]any{
"authenticated": false,
"note": "this gateway has no api keys configured",
})
return
}
out := map[string]any{
"authenticated": true,
"key_id": p.KeyID,
"key": p.KeyPrefix,
"project": p.ProjectID,
"team": p.TeamID,
"organization": p.OrgID,
}
if p.Band != "" {
out["band"] = p.Band
}
limits := map[string]any{}
if p.Limits.RPM != nil {
limits["requests_per_minute"] = *p.Limits.RPM
}
if p.Limits.TPM != nil {
limits["tokens_per_minute"] = *p.Limits.TPM
}
if p.Limits.AllowedModels != nil {
limits["allowed_models"] = p.Limits.AllowedModels
}
if p.Limits.Budget != nil {
limits["budget_units"] = float64(*p.Limits.Budget) / 1e6
limits["budget_period"] = string(p.Limits.BudgetPeriod)
}
out["limits"] = limits
if r.acct != nil {
spent, rpm, tpm := r.acct.Status(p)
usage := map[string]any{
"requests_per_minute": rpm,
"tokens_per_minute": tpm,
}
if p.Limits.Budget != nil {
usage["spent_units"] = float64(spent) / 1e6
usage["remaining_units"] = float64(*p.Limits.Budget-spent) / 1e6
}
out["usage"] = usage
}
writeJSON(w, http.StatusOK, out)
}