Files
llmgw/internal/router/handlers.go
T
Ilker ManapandClaude Opus 5 07809490e3
CI / test (push) Has been cancelled
Show usage over time, per organization, team, project and person
Usage could be read as a total — per key, per user, per model, per group —
and the cluster's own charts covered the fleet's hardware. What nobody could
see was when a group used it. "Research doubled its traffic three weeks ago"
is not a question a column of totals can be asked, and it is the question
somebody asks before a quota conversation.

/admin/usage draws requests, tokens and spend over a chosen window, grouped
by any level of the tenancy tree or by person. Person is not a level of that
tree — one person holds keys in several projects — so the grouping is its
own type rather than store.Scope with a fourth value nothing else could use.

Three things learned from the charts already here. Buckets are gap-filled
per group, because a line drawn straight across an absence claims traffic
that did not happen, and because the chart sums its tail by timestamp and
can only do that if every series shares a clock. Series are ranked by total
rather than by peak — a counter's question is "who used the most", and
ranking by peak would colour the team that made one burst over the team that
ran steadily all week. And the spend chart is replaced by a sentence when no
model has a price, rather than drawn flat along the bottom.

The bucket query groups by ordinal and was planned against the cluster's
PostgreSQL before being believed, with a test alongside the others that run
there: three reports have shipped working on SQLite and failing on it.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-28 08:11:31 +02:00

836 lines
30 KiB
Go

package router
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"slices"
"strconv"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/admission"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
"gitea.mynodes.xyz/ilker/llmgw/internal/web"
)
// admit waits for the flow control layer to let the request through. With
// admission disabled it returns immediately, which is the v0.2.0 behaviour.
func (r *Router) admit(ctx context.Context, req *parsedRequest) (*admission.Lease, error) {
if r.adm == nil {
return nil, nil
}
// The key is the tenant when there is one: fairness within a band has to be
// per caller, and a session id is something the caller picks, so one client
// could take a whole band by opening many sessions. Without authentication
// the session is still the best available grouping.
tenant := req.Tenant
if tenant == "" {
tenant = req.SessionID
}
return r.adm.Acquire(ctx, admission.Request{
Model: req.Model,
Band: req.Band,
Tenant: tenant,
})
}
// rejectAdmission turns a flow control refusal into a status the client can act
// on: 503 when the queue is full (try later), 504 when it timed out waiting.
func (r *Router) rejectAdmission(w http.ResponseWriter, req *parsedRequest, err error, start time.Time) {
status := http.StatusServiceUnavailable
switch {
case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded):
// The client gave up while queued; nobody is listening.
r.m.observeAdmissionRejected(req.Band, "client_gone")
r.m.observeRequest(req.Model, "", 499, time.Since(start), nil, 0)
return
case errors.Is(err, admission.ErrQueueTimeout):
status = http.StatusGatewayTimeout
}
reason := "unavailable"
switch {
case errors.Is(err, admission.ErrQueueFull):
reason = "queue_full"
case errors.Is(err, admission.ErrQueueTimeout):
reason = "ttl"
case errors.Is(err, admission.ErrClosed):
reason = "shutdown"
}
r.m.observeAdmissionRejected(req.Band, reason)
writeError(w, status, "server_error", err.Error())
r.m.observeRequest(req.Model, "", status, time.Since(start), nil, 0)
}
// prefixBytes is how much of the prompt is kept for prefix-aware routing.
// Large enough to cover a system prompt plus the first message, small enough
// that hashing it stays cheap.
const prefixBytes = 2048
// Handler returns the client-facing OpenAI API.
func (r *Router) Handler() http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("POST /v1/chat/completions", r.handleInference("/v1/chat/completions"))
mux.HandleFunc("POST /v1/completions", r.handleInference("/v1/completions"))
mux.HandleFunc("POST /v1/embeddings", r.handleInference("/v1/embeddings"))
mux.HandleFunc("GET /v1/models", r.handleModels)
// Liveness: this process is up. It says nothing about whether it can
// serve, which is why it is not the thing to put a load balancer on.
mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
})
// Readiness: this process has somewhere to send a request.
//
// The distinction is not academic. A standby control plane is up, answers
// /health, and has no endpoints at all — its reconciler placed nothing and
// it holds no agent streams to adopt from — so a load balancer checking
// /health would send it half the traffic and half the traffic would be a
// 503. A leader whose replicas have all failed is in the same position for
// a different reason, and a load balancer should treat them the same.
//
// Open, like /health: a load balancer does not have a key, and this leaks
// one bit — whether anything is servable — which the next request would
// reveal anyway.
mux.HandleFunc("GET /ready", func(w http.ResponseWriter, _ *http.Request) {
if len(r.reg.IDs()) == 0 {
http.Error(w, "no endpoints are available to serve", http.StatusServiceUnavailable)
return
}
w.WriteHeader(http.StatusOK)
})
// The landing page says how to connect and nothing about what is here, so
// it is safe to serve without a key — and it has to be, because the person
// reading it is the one who does not have one yet.
mux.HandleFunc("GET /", web.Landing(version.Version, r.auth != nil,
r.accounts != nil && r.accounts.SignupOpen(),
r.landing.Contact, r.landing.Notice))
mux.Handle("GET /_static/", web.Static())
mux.HandleFunc("GET /me", r.handleMe)
// Somebody has to be able to say "forget that", and a retention window is
// no answer to a message sent by mistake. Authenticated like any other
// client endpoint, and scoped to the caller's project by the store.
mux.HandleFunc("DELETE /v1/threads/{id}", r.handleDeleteThread)
// Notes an agent keeps for itself, over REST and as MCP tools. Off unless a
// project or the threads block turned storage on.
mux.HandleFunc("PUT /v1/memory/{key}", r.handlePutNote)
mux.HandleFunc("GET /v1/memory/{key}", r.handleGetNote)
mux.HandleFunc("GET /v1/memory", r.handleSearchNotes)
mux.HandleFunc("DELETE /v1/memory/{key}", r.handleDeleteNote)
mux.HandleFunc("POST /mcp", r.handleMCP)
if r.store != nil {
// The one-time link that delivers a key to the person it was issued
// for. Served without a key, because the person opening it is the one
// who does not have one yet; what protects it is the token.
claim := web.Claim(r.store, version.Version, r.log)
mux.HandleFunc("GET /claim/{token}", claim)
mux.HandleFunc("POST /claim/{token}", claim)
}
if r.accounts != nil {
// Signup, sign-in and the profile page. These are browser surfaces
// authenticated by a session cookie, not by an API key, which is why
// they are exempt below.
pages := web.NewAccountPages(r.accounts, r.store, r.store, version.Version, r.log)
// So the example on the profile page names a model that is actually
// running, rather than a placeholder that gets sent as-is.
pages.ServedModels(r.exampleModels)
pages.Register(mux)
r.registerChat(mux, pages)
if r.clusterView != nil {
// The cluster dashboard, reachable from a browser without exposing
// the admin listener — which also carries pprof, and pprof can dump
// the heap that holds every API key in plaintext.
mux.HandleFunc("GET /admin", pages.AdminPage(web.Dashboard(
r.clusterView, r.clusterUsage, version.Version, 5*time.Second, false,
web.NavExtras{
Mirror: r.mirror != nil, History: r.history, Window: r.histWindow,
}, r.log)))
pages.RegisterAdminUsers(mux, r.store)
// Quotas are written on the same tree the accounts page builds, so
// they are mounted together.
pages.RegisterAdminQuotas(mux, r.store, r.acct)
// And the report of what that tree actually used.
pages.RegisterAdminUsage(mux, r.store, r.acct)
}
// Independent of the cluster view: the mirror is a separate machine
// with its own disk, and it is worth administering whether or not this
// process also has a cluster to show.
if r.mirror != nil {
pages.RegisterAdminMirror(mux, web.MirrorDeps{
Mirror: r.mirror, Registry: r.registry,
Catalog: r.catalogAdmin, Fleet: r.clusterView,
})
}
}
if r.auth == nil {
return mux
}
// Wrapping the whole mux rather than checking inside each handler: adding a
// route must not be able to add an unauthenticated one by omission. Health
// stays open because a load balancer has no key.
return auth.Middleware(r.auth, r.log,
"/health", "/ready", "/", "/_static/", "/claim/",
"/signup", "/login", "/logout", "/profile", "/profile/",
"/admin", "/admin/", "/chat", "/chat/")(mux)
}
// parsedRequest is what the router needs from a request. Everything else in the
// body is forwarded untouched — clients send engine-specific fields we have no
// business interpreting.
type parsedRequest struct {
Path string
// Model is what will be dispatched, which is not always what was asked
// for: a caller out of budget may be moved to a model they can afford.
Model string
// RequestedModel is what the client sent, kept so the response can say
// when something else answered.
RequestedModel string
Stream bool
SuppressUsageChunk bool
UpstreamBody []byte
Header http.Header
SessionID string
Band string
// KeyID, ProjectID and Tenant are empty when authentication is off.
KeyID string
ProjectID string
Tenant string
ProfileOverride string
PromptPrefix string
PromptTokens int
MaxTokens int
// ThreadID is the conversation this request continues, empty for the
// stateless path every request took before threads existed.
ThreadID string
// ThreadTurns are the turns this request itself carried, kept so the
// exchange can be stored after a reply without re-reading a body that has
// since had the history spliced into it.
ThreadTurns []store.Message
// ThreadProject is the project the thread is stored under, resolved once by
// loadThread so saving it afterwards cannot disagree.
ThreadProject string
// reservation is carried from the limit check to the point where the
// request is settled, so the record counts against the same budget window
// the check read.
reservation accounting.Reservation
}
// inferenceBody covers the fields the router reads across all inference
// endpoints. Unknown fields are ignored here and preserved in UpstreamBody.
type inferenceBody struct {
Model string `json:"model"`
Stream bool `json:"stream"`
User string `json:"user"`
Messages []struct {
Role string `json:"role"`
Content json.RawMessage `json:"content"`
} `json:"messages"`
Prompt json.RawMessage `json:"prompt"`
Input json.RawMessage `json:"input"`
MaxTokens int `json:"max_tokens"`
MaxCompletionTokens int `json:"max_completion_tokens"`
StreamOptions *struct {
IncludeUsage bool `json:"include_usage"`
} `json:"stream_options"`
}
func (r *Router) handleInference(path string) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
start := time.Now()
parsed, err := r.parse(req, path)
if err != nil {
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
r.m.observeRequest("", "", http.StatusBadRequest, time.Since(start), nil, 0)
return
}
// Before authentication work and before admission: a request that
// cannot fit should not consume a quota reservation or a queue slot on
// its way to being refused.
if _, entry := r.resolve(parsed.Model); entry.ContextLength > 0 {
if why := checkContextFits(entry, parsed.PromptTokens, parsed.MaxTokens); why != "" {
writeError(w, http.StatusBadRequest, "invalid_request_error", why)
r.m.observeRequest(parsed.Model, "", http.StatusBadRequest, time.Since(start), nil, 0)
return
}
}
if p, ok := auth.FromContext(req.Context()); ok {
if !r.permits(p, parsed.Model) {
auth.WriteForbiddenModel(w, parsed.Model)
r.m.observeRequest(parsed.Model, "", http.StatusForbidden, time.Since(start), nil, 0)
return
}
// Limits are checked before admission, not after: a request that is
// over its quota should be refused rather than queued, and queueing
// it would have it occupy a band's capacity to no purpose.
if r.acct != nil {
// Priced by the model that will serve, not by the name the
// caller typed. A price table is written against the catalog,
// and an alias asked for by its short name would miss every
// entry and fall to the default — charging a 1.5B at whatever
// rate the largest model set, or at nothing.
priced, _ := r.resolve(parsed.Model)
reservation, err := r.acct.Check(p, priced, parsed.estimatedTokens())
if err != nil && isBudgetSpent(err) {
// Out of money, not out of options. A smaller model the
// caller can still afford beats a refusal, and the catalog
// already says which those are.
if fb, ok := r.budgetFallback(p, parsed); ok {
reservation, err = fb, nil
// Said now rather than at the end: the proxy writes the
// response itself, so a header set after dispatch never
// reaches the client.
w.Header().Set("x-llmgw-served-model", parsed.Model)
}
}
if err != nil {
r.rejectLimit(w, parsed, err, start)
return
}
parsed.reservation = reservation
}
}
// Step 5 of the lifecycle: "now?" before "where?" (plan §7.1).
lease, err := r.admit(req.Context(), parsed)
if err != nil {
r.rejectAdmission(w, parsed, err, start)
return
}
defer lease.Release()
queued := time.Since(start)
r.m.observeAdmission(lease.Band(), parsed.Model, queued)
if queued > time.Millisecond {
w.Header().Set("x-llmgw-queue-ms", strconv.FormatInt(queued.Milliseconds(), 10))
}
// After the model is finally settled — a budget fallback may have
// changed it — because trimming is measured against the context of the
// model that will actually serve, not the one that was asked for.
if dropped, err := r.loadThread(req.Context(), parsed); err != nil {
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
r.m.observeRequest(parsed.Model, "", http.StatusBadRequest, time.Since(start), nil, 0)
return
} else if dropped > 0 {
// Dropping turns is the right answer and must not be a silent one.
w.Header().Set(trimmedHeader, strconv.Itoa(dropped))
}
res := r.dispatch(req.Context(), w, parsed)
r.finish(w, parsed, res, start)
// After the client has been served. A conversation that failed to
// record is a worse outcome than a slow reply, but only just, and the
// caller has their answer either way.
if res.err == nil && res.outcome.Status == http.StatusOK {
r.saveThread(context.WithoutCancel(req.Context()), parsed, res.outcome.Reply, parsed.Model)
}
}
}
// permits reports whether a key's allowlist covers the requested model.
//
// The decision is per model, not per string. A model's canonical name and every
// alias of it name the same thing, so the allowlist is matched against all of
// them: writing either form works, and no alias can reach a model that was not
// allowed. Matching only what the client sent would make every alias a way
// around the allowlist; matching only the canonical name would silently ignore
// an allowlist written in the names clients actually use.
func (r *Router) permits(p auth.Principal, requested string) bool {
canonical, m := r.resolve(requested)
if p.Allows(canonical) {
return true
}
for _, alias := range m.Aliases {
if p.Allows(alias) {
return true
}
}
return false
}
// finish records metrics and, when nothing was written, produces the error body.
func (r *Router) finish(w http.ResponseWriter, req *parsedRequest, res dispatchResult, start time.Time) {
status := res.outcome.Status
switch {
case res.err == nil:
// The served-model header is set before dispatch, not here: the proxy
// writes the response, so by this point the headers are long gone.
case errors.Is(res.err, proxy.ErrClientGone):
// Nothing to report to a client that is gone; the upstream was cancelled.
status = 499
r.log.Debug("client disconnected", "model", req.Model, "endpoint", res.endpointID)
case res.outcome.Wrote:
// The response is already partly out; the stream just ends short.
r.log.Warn("stream ended early", "model", req.Model, "endpoint", res.endpointID, "err", res.err)
if status == 0 {
status = http.StatusOK
}
case errors.Is(res.err, errLoading):
// Retry-After, because this one succeeds on a retry and a client that
// is told so will make it. Without the header a well-behaved client
// backs off exactly as it would from a permanent failure.
status = http.StatusServiceUnavailable
w.Header().Set("Retry-After", "30")
writeError(w, status, "server_error", res.err.Error())
case errors.Is(res.err, errNoCapacity):
status = http.StatusServiceUnavailable
writeError(w, status, "server_error", res.err.Error())
default:
status = http.StatusBadGateway
writeError(w, status, "server_error", res.err.Error())
}
r.m.observeRequest(req.Model, res.pool, status, time.Since(start), res.outcome.Usage, res.outcome.TTFT)
r.account(req, res, status, time.Since(start))
}
// account settles a finished request against its key.
//
// It runs for every outcome, not only success: a request that used a GPU and
// then failed still used it, and a key that could spend nothing by failing
// would be a way around a budget.
func (r *Router) account(req *parsedRequest, res dispatchResult, status int, took time.Duration) {
if r.acct == nil || req.KeyID == "" {
return
}
u := store.Usage{
At: time.Now(), KeyID: req.KeyID, ProjectID: req.ProjectID,
Model: req.Model, ServedModel: res.model, Status: status,
LatencyMs: took.Milliseconds(),
}
if u.ServedModel == "" {
u.ServedModel = req.Model
}
if usage := res.outcome.Usage; usage != nil {
u.PromptTokens = int64(usage.PromptTokens)
u.CompletionTokens = int64(usage.CompletionTokens)
} else {
// No usage came back — a failure before the engine answered, or an
// engine that does not report it. Charging the estimate is closer to
// the truth than charging nothing.
u.PromptTokens = int64(req.PromptTokens)
}
r.acct.Record(req.reservation, u)
}
// rejectLimit turns a quota refusal into a status the client can act on.
func (r *Router) rejectLimit(w http.ResponseWriter, req *parsedRequest, err error, start time.Time) {
var le *accounting.LimitError
if !errors.As(err, &le) {
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
return
}
if le.RetryAfter > 0 {
w.Header().Set("Retry-After", strconv.Itoa(int(le.RetryAfter.Seconds())))
}
// 429 for all of them: a spent budget is a quota, and OpenAI clients
// already know how to surface that code. What differs is Retry-After,
// which is absent when waiting does not help.
writeError(w, http.StatusTooManyRequests, "rate_limit_exceeded", le.Error())
r.m.observeRequest(req.Model, "", http.StatusTooManyRequests, time.Since(start), nil, 0)
}
// parse reads the body and extracts the routing-relevant fields.
func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) {
body, err := io.ReadAll(io.LimitReader(req.Body, r.cfg.MaxBodyBytes))
if err != nil {
return nil, err
}
var b inferenceBody
if err := json.Unmarshal(body, &b); err != nil {
return nil, err
}
if b.Model == "" {
return nil, errors.New("model is required")
}
p := &parsedRequest{
Path: path,
Model: b.Model,
RequestedModel: b.Model,
Stream: b.Stream,
UpstreamBody: body,
Header: req.Header,
SessionID: sessionID(req, r.cfg.SessionHeader, b.User),
Band: req.Header.Get(r.cfg.Admission.BandHeader),
ProfileOverride: req.Header.Get("x-llmgw-profile"),
MaxTokens: max(b.MaxTokens, b.MaxCompletionTokens),
PromptPrefix: promptPrefix(&b),
ThreadID: req.Header.Get(threadHeader),
}
// With authentication on, the key decides the band and the routing profile,
// and the headers stop being believed. Trusting the header was only ever
// acceptable while there was no identity to consult — it let any client
// promote itself into the top priority band.
if principal, ok := auth.FromContext(req.Context()); ok {
p.Band = principal.Band
p.ProfileOverride = principal.RoutingProfile
p.KeyID = principal.KeyID
p.ProjectID = principal.ProjectID
// Tenant fairness within a band is per key rather than per session, so
// one key cannot take a whole band by opening many sessions.
p.Tenant = principal.KeyID
}
// Bytes over four: a crude stand-in for a tokenizer, good enough for
// scoring and for limits. It becomes exact only with plan §7.5 tier 2 (see
// plan §14). Measured over the whole prompt, not the routing prefix —
// see promptBytes.
p.PromptTokens = promptBytes(&b) / 4
// Ask the engine for usage the client did not request, so accounting works
// regardless of client behaviour; the extra chunk is dropped on the way back.
if r.cfg.InjectUsage && b.Stream && (b.StreamOptions == nil || !b.StreamOptions.IncludeUsage) {
if injected, ok := injectIncludeUsage(body); ok {
p.UpstreamBody = injected
p.SuppressUsageChunk = true
}
}
return p, nil
}
// sessionID picks the affinity key: an explicit header first, then OpenAI's
// user field, which most clients already set per end user.
func sessionID(req *http.Request, header, user string) string {
if header != "" {
if v := req.Header.Get(header); v != "" {
return v
}
}
return user
}
// promptPrefix reconstructs the leading portion of the prompt for prefix-aware
// routing. Only the first prefixBytes matter, so long conversations stop early.
func promptPrefix(b *inferenceBody) string {
var sb strings.Builder
appendRaw := func(raw json.RawMessage) {
if len(raw) == 0 || sb.Len() >= prefixBytes {
return
}
var s string
if err := json.Unmarshal(raw, &s); err != nil {
s = string(raw)
}
sb.WriteString(s)
}
for _, m := range b.Messages {
if sb.Len() >= prefixBytes {
break
}
sb.WriteString(m.Role)
sb.WriteString(":")
appendRaw(m.Content)
sb.WriteString("\n")
}
appendRaw(b.Prompt)
appendRaw(b.Input)
s := sb.String()
if len(s) > prefixBytes {
s = s[:prefixBytes]
}
return s
}
// promptBytes measures the whole prompt, which promptPrefix deliberately does
// not.
//
// The two were the same function for too long, and the estimate was taken from
// the prefix: bytes over four of at most prefixBytes, so every prompt above
// 2 KB counted as 512 tokens however long it really was. Prefix routing wants a
// bounded head of the conversation; a token estimate wants all of it, and using
// one for the other meant TPM limits undercounted exactly the requests that
// consume the most, and nothing could tell whether a prompt fit in a context.
func promptBytes(b *inferenceBody) int {
n := 0
add := func(raw json.RawMessage) {
if len(raw) == 0 {
return
}
var str string
if err := json.Unmarshal(raw, &str); err == nil {
n += len(str)
return
}
// Not a plain string: structured content, an array of parts, an image.
// The raw JSON overstates the text slightly, which is the safe
// direction for a limit.
n += len(raw)
}
for _, m := range b.Messages {
n += len(m.Role) + 2
add(m.Content)
}
add(b.Prompt)
add(b.Input)
return n
}
// injectIncludeUsage sets stream_options.include_usage on the outgoing body.
//
// This is the one place the router rewrites a request, so it is done by
// decoding to a generic map and re-encoding: field order changes, but no field
// is lost. If anything looks unexpected it declines rather than guessing.
func injectIncludeUsage(body []byte) ([]byte, bool) {
var m map[string]json.RawMessage
if err := json.Unmarshal(body, &m); err != nil {
return nil, false
}
opts := map[string]any{}
if raw, ok := m["stream_options"]; ok {
if err := json.Unmarshal(raw, &opts); err != nil {
return nil, false
}
}
opts["include_usage"] = true
encoded, err := json.Marshal(opts)
if err != nil {
return nil, false
}
m["stream_options"] = encoded
out, err := json.Marshal(m)
if err != nil {
return nil, false
}
return out, true
}
// noEndpointFor explains that nothing serves a model, and says what does.
//
// "no endpoint serves \"MODEL\"" was all a client got, which is true and
// useless: it reads the same whether the name was a typo, a placeholder left
// in from the documentation, or a real outage. Someone copying the curl from
// the account page sent the literal word MODEL and had nothing to go on.
//
// The list is what /v1/models already returns to the same caller, so naming it
// here discloses nothing new. It is capped because a large catalog would turn
// an error message into a page.
func noEndpointFor(model string, served []string) string {
if len(served) == 0 {
return fmt.Sprintf("no endpoint serves %q, and no model is currently being served", model)
}
const showAt = 10
shown := served
suffix := ""
if len(shown) > showAt {
shown, suffix = shown[:showAt], fmt.Sprintf(" and %d more", len(served)-showAt)
}
return fmt.Sprintf("no endpoint serves %q; available models are %s%s",
model, strings.Join(shown, ", "), suffix)
}
// exampleModels is servedModels with the ones that answer for real first.
//
// The profile page takes the first name for its copy-pasteable example, and
// handing it a simulated model would send someone's first request to an engine
// that replies in Greek letters.
func (r *Router) exampleModels() []string {
names := r.servedModels()
slices.SortStableFunc(names, func(a, b string) int {
as, bs := r.modelMap()[a].Simulated(), r.modelMap()[b].Simulated()
switch {
case as == bs:
return 0
case bs:
return -1
default:
return 1
}
})
return names
}
// servedModels lists the canonical models at least one endpoint serves.
func (r *Router) servedModels() []string {
seen := map[string]struct{}{}
for _, s := range r.reg.Snapshots("") {
for _, m := range s.Models {
seen[m] = struct{}{}
}
}
out := make([]string, 0, len(seen))
for m := range seen {
out = append(out, m)
}
slices.Sort(out)
return out
}
// handleModels lists the models that at least one endpoint currently serves.
func (r *Router) handleModels(w http.ResponseWriter, _ *http.Request) {
seen := map[string]struct{}{}
for _, s := range r.reg.Snapshots("") {
for _, m := range s.Models {
seen[m] = struct{}{}
}
}
// Catalog aliases are usable names too, so clients should see them.
for name := range r.modelMap() {
if _, ok := seen[r.modelMap()[name].Name]; ok {
seen[name] = struct{}{}
}
}
names := make([]string, 0, len(seen))
for n := range seen {
names = append(names, n)
}
slices.Sort(names)
data := make([]map[string]any, 0, len(names))
for _, n := range names {
// A simulated model is owned by "llmgw-sim", which is what the sim
// engine already calls itself on its own /v1/models. Flattening it to
// "llmgw" here was hiding the one distinction a client needs: this
// model returns placeholder text, not inference.
owner := "llmgw"
if r.modelMap()[n].Simulated() {
owner = "llmgw-sim"
}
entry := map[string]any{
"id": n, "object": "model", "created": time.Now().Unix(), "owned_by": owner,
}
// Only when known. A context_length of 0 would read as "no context",
// which is a worse answer than saying nothing.
m := r.modelMap()[n]
if m.ContextLength > 0 {
entry["context_length"] = m.ContextLength
}
if m.MaxOutput > 0 {
entry["max_output_tokens"] = m.MaxOutput
}
// Not part of the OpenAI shape, and the reason to send it anyway is
// that this list is the only view most clients have: nine names and
// nothing to choose between them.
if m.Description != "" {
entry["description"] = m.Description
}
entry["capabilities"] = map[string]bool{
"tools": m.Capabilities.Tools, "vision": m.Capabilities.Vision,
"embeddings": m.Capabilities.Embeddings,
}
data = append(data, entry)
}
writeJSON(w, http.StatusOK, map[string]any{"object": "list", "data": data})
}
func writeJSON(w http.ResponseWriter, code int, v any) {
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(v); err != nil {
http.Error(w, `{"error":{"message":"encoding failed"}}`, http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(code)
_, _ = w.Write(buf.Bytes())
}
// writeError emits an OpenAI-shaped error so clients need no special handling
// for gateway failures.
func writeError(w http.ResponseWriter, code int, kind, msg string) {
writeJSON(w, code, map[string]any{"error": map[string]any{
"message": msg, "type": kind, "code": nil,
}})
}
// estimatedTokens is what a token-rate limit is checked against, before the
// real figure exists.
//
// The prompt estimate plus whatever the client said it wanted back. A client
// that names no max_tokens gets a modest assumption rather than zero: assuming
// zero would let an unbounded request through every token limit.
func (p *parsedRequest) estimatedTokens() int64 {
const assumedCompletion = 256
completion := p.MaxTokens
if completion <= 0 {
completion = assumedCompletion
}
return int64(p.PromptTokens + completion)
}
// handleMe reports what the presented key is and what it may do.
//
// It exists so a user can answer "how much have I used" and "why was I
// refused" without asking an operator to read a dashboard they cannot reach.
// Deliberately outside /v1: it is not part of the OpenAI API, and putting it
// there would imply it is.
func (r *Router) handleMe(w http.ResponseWriter, req *http.Request) {
p, ok := auth.FromContext(req.Context())
if !ok {
// With authentication off there is no identity to describe, and saying
// so is more useful than an empty object.
writeJSON(w, http.StatusOK, map[string]any{
"authenticated": false,
"note": "this gateway has no api keys configured",
})
return
}
out := map[string]any{
"authenticated": true,
"key_id": p.KeyID,
"key": p.KeyPrefix,
"project": p.ProjectID,
"team": p.TeamID,
"organization": p.OrgID,
}
if p.Band != "" {
out["band"] = p.Band
}
limits := map[string]any{}
if p.Limits.RPM != nil {
limits["requests_per_minute"] = *p.Limits.RPM
}
if p.Limits.TPM != nil {
limits["tokens_per_minute"] = *p.Limits.TPM
}
if p.Limits.AllowedModels != nil {
limits["allowed_models"] = p.Limits.AllowedModels
}
if p.Limits.Budget != nil {
limits["budget_units"] = float64(*p.Limits.Budget) / 1e6
limits["budget_period"] = string(p.Limits.BudgetPeriod)
}
out["limits"] = limits
if r.acct != nil {
spent, rpm, tpm := r.acct.Status(p)
usage := map[string]any{
"requests_per_minute": rpm,
"tokens_per_minute": tpm,
}
if p.Limits.Budget != nil {
usage["spent_units"] = float64(spent) / 1e6
usage["remaining_units"] = float64(*p.Limits.Budget-spent) / 1e6
}
out["usage"] = usage
}
writeJSON(w, http.StatusOK, out)
}