package router import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "net/http" "slices" "strconv" "strings" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/accounting" "gitea.mynodes.xyz/ilker/llmgw/internal/auth" "gitea.mynodes.xyz/ilker/llmgw/internal/router/admission" "gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy" "gitea.mynodes.xyz/ilker/llmgw/internal/store" "gitea.mynodes.xyz/ilker/llmgw/internal/version" "gitea.mynodes.xyz/ilker/llmgw/internal/web" ) // admit waits for the flow control layer to let the request through. With // admission disabled it returns immediately, which is the v0.2.0 behaviour. func (r *Router) admit(ctx context.Context, req *parsedRequest) (*admission.Lease, error) { if r.adm == nil { return nil, nil } // The key is the tenant when there is one: fairness within a band has to be // per caller, and a session id is something the caller picks, so one client // could take a whole band by opening many sessions. Without authentication // the session is still the best available grouping. tenant := req.Tenant if tenant == "" { tenant = req.SessionID } return r.adm.Acquire(ctx, admission.Request{ Model: req.Model, Band: req.Band, Tenant: tenant, }) } // rejectAdmission turns a flow control refusal into a status the client can act // on: 503 when the queue is full (try later), 504 when it timed out waiting. func (r *Router) rejectAdmission(w http.ResponseWriter, req *parsedRequest, err error, start time.Time) { status := http.StatusServiceUnavailable switch { case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): // The client gave up while queued; nobody is listening. r.m.observeAdmissionRejected(req.Band, "client_gone") r.m.observeRequest(req.Model, "", 499, time.Since(start), nil, 0) return case errors.Is(err, admission.ErrQueueTimeout): status = http.StatusGatewayTimeout } reason := "unavailable" switch { case errors.Is(err, admission.ErrQueueFull): reason = "queue_full" case errors.Is(err, admission.ErrQueueTimeout): reason = "ttl" case errors.Is(err, admission.ErrClosed): reason = "shutdown" } r.m.observeAdmissionRejected(req.Band, reason) writeError(w, status, "server_error", err.Error()) r.m.observeRequest(req.Model, "", status, time.Since(start), nil, 0) } // prefixBytes is how much of the prompt is kept for prefix-aware routing. // Large enough to cover a system prompt plus the first message, small enough // that hashing it stays cheap. const prefixBytes = 2048 // Handler returns the client-facing OpenAI API. func (r *Router) Handler() http.Handler { mux := http.NewServeMux() mux.HandleFunc("POST /v1/chat/completions", r.handleInference("/v1/chat/completions")) mux.HandleFunc("POST /v1/completions", r.handleInference("/v1/completions")) mux.HandleFunc("POST /v1/embeddings", r.handleInference("/v1/embeddings")) mux.HandleFunc("GET /v1/models", r.handleModels) // Liveness: this process is up. It says nothing about whether it can // serve, which is why it is not the thing to put a load balancer on. mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) // Readiness: this process has somewhere to send a request. // // The distinction is not academic. A standby control plane is up, answers // /health, and has no endpoints at all — its reconciler placed nothing and // it holds no agent streams to adopt from — so a load balancer checking // /health would send it half the traffic and half the traffic would be a // 503. A leader whose replicas have all failed is in the same position for // a different reason, and a load balancer should treat them the same. // // Open, like /health: a load balancer does not have a key, and this leaks // one bit — whether anything is servable — which the next request would // reveal anyway. mux.HandleFunc("GET /ready", func(w http.ResponseWriter, _ *http.Request) { if len(r.reg.IDs()) == 0 { http.Error(w, "no endpoints are available to serve", http.StatusServiceUnavailable) return } w.WriteHeader(http.StatusOK) }) // The landing page says how to connect and nothing about what is here, so // it is safe to serve without a key — and it has to be, because the person // reading it is the one who does not have one yet. mux.HandleFunc("GET /", web.Landing(version.Version, r.auth != nil, r.accounts != nil && r.accounts.SignupOpen(), r.landing.Contact, r.landing.Notice)) mux.Handle("GET /_static/", web.Static()) mux.HandleFunc("GET /me", r.handleMe) // Somebody has to be able to say "forget that", and a retention window is // no answer to a message sent by mistake. Authenticated like any other // client endpoint, and scoped to the caller's project by the store. mux.HandleFunc("DELETE /v1/threads/{id}", r.handleDeleteThread) // Notes an agent keeps for itself, over REST and as MCP tools. Off unless a // project or the threads block turned storage on. mux.HandleFunc("PUT /v1/memory/{key}", r.handlePutNote) mux.HandleFunc("GET /v1/memory/{key}", r.handleGetNote) mux.HandleFunc("GET /v1/memory", r.handleSearchNotes) mux.HandleFunc("DELETE /v1/memory/{key}", r.handleDeleteNote) mux.HandleFunc("POST /mcp", r.handleMCP) if r.store != nil { // The one-time link that delivers a key to the person it was issued // for. Served without a key, because the person opening it is the one // who does not have one yet; what protects it is the token. claim := web.Claim(r.store, version.Version, r.log) mux.HandleFunc("GET /claim/{token}", claim) mux.HandleFunc("POST /claim/{token}", claim) } if r.accounts != nil { // Signup, sign-in and the profile page. These are browser surfaces // authenticated by a session cookie, not by an API key, which is why // they are exempt below. pages := web.NewAccountPages(r.accounts, r.store, r.store, version.Version, r.log) // So the example on the profile page names a model that is actually // running, rather than a placeholder that gets sent as-is. pages.ServedModels(r.exampleModels) pages.SetPersonalOff(r.personalOff) pages.SetLlamaCppVersion(r.llamaVersion) pages.SetPersonalDefaults(r.personalDefaults) pages.Register(mux) r.registerChat(mux, pages) // An organization's own page, for the people who manage it. pages.RegisterOrgPages(mux, r.store, r.acct) if r.clusterView != nil { // The cluster dashboard, reachable from a browser without exposing // the admin listener — which also carries pprof, and pprof can dump // the heap that holds every API key in plaintext. mux.HandleFunc("GET /admin", pages.AdminPage(web.Dashboard( r.clusterView, r.clusterUsage, version.Version, 5*time.Second, false, web.NavExtras{ Mirror: r.mirror != nil, History: r.history, Window: r.histWindow, }, r.log))) mux.HandleFunc("GET /admin/nodes/{id}/logs", pages.AdminPage( web.NodeLogsPage(r.clusterView, false, version.Version))) mux.HandleFunc("GET /admin/nodes/{id}", pages.AdminPage( web.NodePage(r.clusterView, r.history, r.histWindow, false, version.Version, r.log))) pages.RegisterAdminUsers(mux, r.store) // Quotas are written on the same tree the accounts page builds, so // they are mounted together. pages.RegisterAdminQuotas(mux, r.store, r.acct) // And the report of what that tree actually used. pages.RegisterAdminUsage(mux, r.store, r.acct) } // Independent of the cluster view: the mirror is a separate machine // with its own disk, and it is worth administering whether or not this // process also has a cluster to show. if r.mirror != nil { pages.RegisterAdminMirror(mux, web.MirrorDeps{ Mirror: r.mirror, Registry: r.registry, Catalog: r.catalogAdmin, Fleet: r.clusterView, }) } } if r.auth == nil { return mux } // Wrapping the whole mux rather than checking inside each handler: adding a // route must not be able to add an unauthenticated one by omission. Health // stays open because a load balancer has no key. return auth.Middleware(r.auth, r.log, "/health", "/ready", "/", "/_static/", "/claim/", "/signup", "/login", "/logout", "/profile", "/profile/", "/admin", "/admin/", "/chat", "/chat/", "/org", "/org/")(mux) } // parsedRequest is what the router needs from a request. Everything else in the // body is forwarded untouched — clients send engine-specific fields we have no // business interpreting. type parsedRequest struct { Path string // Model is what will be dispatched, which is not always what was asked // for: a caller out of budget may be moved to a model they can afford. Model string // RequestedModel is what the client sent, kept so the response can say // when something else answered. RequestedModel string Stream bool SuppressUsageChunk bool UpstreamBody []byte Header http.Header SessionID string Band string // KeyID, ProjectID and Tenant are empty when authentication is off. KeyID string UserID string ProjectID string Tenant string ProfileOverride string PromptPrefix string PromptTokens int MaxTokens int // ThreadID is the conversation this request continues, empty for the // stateless path every request took before threads existed. ThreadID string // ThreadTurns are the turns this request itself carried, kept so the // exchange can be stored after a reply without re-reading a body that has // since had the history spliced into it. ThreadTurns []store.Message // ThreadProject is the project the thread is stored under, resolved once by // loadThread so saving it afterwards cannot disagree. ThreadProject string // reservation is carried from the limit check to the point where the // request is settled, so the record counts against the same budget window // the check read. reservation accounting.Reservation } // inferenceBody covers the fields the router reads across all inference // endpoints. Unknown fields are ignored here and preserved in UpstreamBody. type inferenceBody struct { Model string `json:"model"` Stream bool `json:"stream"` User string `json:"user"` Messages []struct { Role string `json:"role"` Content json.RawMessage `json:"content"` } `json:"messages"` Prompt json.RawMessage `json:"prompt"` Input json.RawMessage `json:"input"` MaxTokens int `json:"max_tokens"` MaxCompletionTokens int `json:"max_completion_tokens"` StreamOptions *struct { IncludeUsage bool `json:"include_usage"` } `json:"stream_options"` } func (r *Router) handleInference(path string) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { start := time.Now() parsed, err := r.parse(req, path) if err != nil { writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error()) r.m.observeRequest("", "", http.StatusBadRequest, time.Since(start), nil, 0) return } // Before authentication work and before admission: a request that // cannot fit should not consume a quota reservation or a queue slot on // its way to being refused. if _, entry := r.resolve(parsed.Model); entry.ContextLength > 0 { if why := checkContextFits(entry, parsed.PromptTokens, parsed.MaxTokens); why != "" { writeError(w, http.StatusBadRequest, "invalid_request_error", why) r.m.observeRequest(parsed.Model, "", http.StatusBadRequest, time.Since(start), nil, 0) return } } if p, ok := auth.FromContext(req.Context()); ok { if !r.permits(p, parsed.Model) { auth.WriteForbiddenModel(w, parsed.Model) r.m.observeRequest(parsed.Model, "", http.StatusForbidden, time.Since(start), nil, 0) return } // Limits are checked before admission, not after: a request that is // over its quota should be refused rather than queued, and queueing // it would have it occupy a band's capacity to no purpose. if r.acct != nil { // Priced by the model that will serve, not by the name the // caller typed. A price table is written against the catalog, // and an alias asked for by its short name would miss every // entry and fall to the default — charging a 1.5B at whatever // rate the largest model set, or at nothing. priced, _ := r.resolve(parsed.Model) reservation, err := r.acct.Check(p, priced, parsed.estimatedTokens()) if err != nil && isBudgetSpent(err) { // Out of money, not out of options. A smaller model the // caller can still afford beats a refusal, and the catalog // already says which those are. if fb, ok := r.budgetFallback(p, parsed); ok { reservation, err = fb, nil // Said now rather than at the end: the proxy writes the // response itself, so a header set after dispatch never // reaches the client. w.Header().Set("x-llmgw-served-model", parsed.Model) } } if err != nil { r.rejectLimit(w, parsed, err, start) return } parsed.reservation = reservation } } // Step 5 of the lifecycle: "now?" before "where?" (plan §7.1). lease, err := r.admit(req.Context(), parsed) if err != nil { r.rejectAdmission(w, parsed, err, start) return } defer lease.Release() queued := time.Since(start) r.m.observeAdmission(lease.Band(), parsed.Model, queued) if queued > time.Millisecond { w.Header().Set("x-llmgw-queue-ms", strconv.FormatInt(queued.Milliseconds(), 10)) } // After the model is finally settled — a budget fallback may have // changed it — because trimming is measured against the context of the // model that will actually serve, not the one that was asked for. if dropped, err := r.loadThread(req.Context(), parsed); err != nil { writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error()) r.m.observeRequest(parsed.Model, "", http.StatusBadRequest, time.Since(start), nil, 0) return } else if dropped > 0 { // Dropping turns is the right answer and must not be a silent one. w.Header().Set(trimmedHeader, strconv.Itoa(dropped)) } res := r.dispatch(req.Context(), w, parsed) r.finish(w, parsed, res, start) // After the client has been served. A conversation that failed to // record is a worse outcome than a slow reply, but only just, and the // caller has their answer either way. if res.err == nil && res.outcome.Status == http.StatusOK { r.saveThread(context.WithoutCancel(req.Context()), parsed, res.outcome.Reply, parsed.Model) } } } // permits reports whether a key's allowlist covers the requested model. // // The decision is per model, not per string. A model's canonical name and every // alias of it name the same thing, so the allowlist is matched against all of // them: writing either form works, and no alias can reach a model that was not // allowed. Matching only what the client sent would make every alias a way // around the allowlist; matching only the canonical name would silently ignore // an allowlist written in the names clients actually use. func (r *Router) permits(p auth.Principal, requested string) bool { canonical, m := r.resolve(requested) if p.Allows(canonical) { return true } for _, alias := range m.Aliases { if p.Allows(alias) { return true } } return false } // finish records metrics and, when nothing was written, produces the error body. func (r *Router) finish(w http.ResponseWriter, req *parsedRequest, res dispatchResult, start time.Time) { status := res.outcome.Status switch { case res.err == nil: // The served-model header is set before dispatch, not here: the proxy // writes the response, so by this point the headers are long gone. case errors.Is(res.err, proxy.ErrClientGone): // Nothing to report to a client that is gone; the upstream was cancelled. status = 499 r.log.Debug("client disconnected", "model", req.Model, "endpoint", res.endpointID) case res.outcome.Wrote: // The response is already partly out; the stream just ends short. r.log.Warn("stream ended early", "model", req.Model, "endpoint", res.endpointID, "err", res.err) if status == 0 { status = http.StatusOK } case errors.Is(res.err, errLoading): // Retry-After, because this one succeeds on a retry and a client that // is told so will make it. Without the header a well-behaved client // backs off exactly as it would from a permanent failure. status = http.StatusServiceUnavailable w.Header().Set("Retry-After", "30") writeError(w, status, "server_error", res.err.Error()) case errors.Is(res.err, errNoCapacity): status = http.StatusServiceUnavailable writeError(w, status, "server_error", res.err.Error()) default: status = http.StatusBadGateway writeError(w, status, "server_error", res.err.Error()) } r.m.observeRequest(req.Model, res.pool, status, time.Since(start), res.outcome.Usage, res.outcome.TTFT) r.account(req, res, status, time.Since(start)) } // account settles a finished request against its key. // // It runs for every outcome, not only success: a request that used a GPU and // then failed still used it, and a key that could spend nothing by failing // would be a way around a budget. func (r *Router) account(req *parsedRequest, res dispatchResult, status int, took time.Duration) { if r.acct == nil || req.KeyID == "" { return } u := store.Usage{ At: time.Now(), KeyID: req.KeyID, UserID: req.UserID, ProjectID: req.ProjectID, Model: req.Model, ServedModel: res.model, Status: status, LatencyMs: took.Milliseconds(), } if u.ServedModel == "" { u.ServedModel = req.Model } if usage := res.outcome.Usage; usage != nil { u.PromptTokens = int64(usage.PromptTokens) u.CompletionTokens = int64(usage.CompletionTokens) } else { // No usage came back — a failure before the engine answered, or an // engine that does not report it. Charging the estimate is closer to // the truth than charging nothing. u.PromptTokens = int64(req.PromptTokens) } r.acct.Record(req.reservation, u) } // rejectLimit turns a quota refusal into a status the client can act on. func (r *Router) rejectLimit(w http.ResponseWriter, req *parsedRequest, err error, start time.Time) { var le *accounting.LimitError if !errors.As(err, &le) { writeError(w, http.StatusInternalServerError, "server_error", err.Error()) return } if le.RetryAfter > 0 { w.Header().Set("Retry-After", strconv.Itoa(int(le.RetryAfter.Seconds()))) } // 429 for all of them: a spent budget is a quota, and OpenAI clients // already know how to surface that code. What differs is Retry-After, // which is absent when waiting does not help. writeError(w, http.StatusTooManyRequests, "rate_limit_exceeded", le.Error()) r.m.observeRequest(req.Model, "", http.StatusTooManyRequests, time.Since(start), nil, 0) } // parse reads the body and extracts the routing-relevant fields. func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) { body, err := io.ReadAll(io.LimitReader(req.Body, r.cfg.MaxBodyBytes)) if err != nil { return nil, err } var b inferenceBody if err := json.Unmarshal(body, &b); err != nil { return nil, err } if b.Model == "" { return nil, errors.New("model is required") } p := &parsedRequest{ Path: path, Model: b.Model, RequestedModel: b.Model, Stream: b.Stream, UpstreamBody: body, Header: req.Header, SessionID: sessionID(req, r.cfg.SessionHeader, b.User), Band: req.Header.Get(r.cfg.Admission.BandHeader), ProfileOverride: req.Header.Get("x-llmgw-profile"), MaxTokens: max(b.MaxTokens, b.MaxCompletionTokens), PromptPrefix: promptPrefix(&b), ThreadID: req.Header.Get(threadHeader), } // With authentication on, the key decides the band and the routing profile, // and the headers stop being believed. Trusting the header was only ever // acceptable while there was no identity to consult — it let any client // promote itself into the top priority band. if principal, ok := auth.FromContext(req.Context()); ok { p.Band = principal.Band p.ProfileOverride = principal.RoutingProfile p.KeyID = principal.KeyID p.UserID = principal.UserID p.ProjectID = principal.ProjectID // Tenant fairness within a band is per key rather than per session, so // one key cannot take a whole band by opening many sessions. p.Tenant = principal.KeyID } // Bytes over four: a crude stand-in for a tokenizer, good enough for // scoring and for limits. It becomes exact only with plan §7.5 tier 2 (see // plan §14). Measured over the whole prompt, not the routing prefix — // see promptBytes. p.PromptTokens = promptBytes(&b) / 4 // Ask the engine for usage the client did not request, so accounting works // regardless of client behaviour; the extra chunk is dropped on the way back. if r.cfg.InjectUsage && b.Stream && (b.StreamOptions == nil || !b.StreamOptions.IncludeUsage) { if injected, ok := injectIncludeUsage(body); ok { p.UpstreamBody = injected p.SuppressUsageChunk = true } } return p, nil } // sessionID picks the affinity key: an explicit header first, then OpenAI's // user field, which most clients already set per end user. func sessionID(req *http.Request, header, user string) string { if header != "" { if v := req.Header.Get(header); v != "" { return v } } return user } // promptPrefix reconstructs the leading portion of the prompt for prefix-aware // routing. Only the first prefixBytes matter, so long conversations stop early. func promptPrefix(b *inferenceBody) string { var sb strings.Builder appendRaw := func(raw json.RawMessage) { if len(raw) == 0 || sb.Len() >= prefixBytes { return } var s string if err := json.Unmarshal(raw, &s); err != nil { s = string(raw) } sb.WriteString(s) } for _, m := range b.Messages { if sb.Len() >= prefixBytes { break } sb.WriteString(m.Role) sb.WriteString(":") appendRaw(m.Content) sb.WriteString("\n") } appendRaw(b.Prompt) appendRaw(b.Input) s := sb.String() if len(s) > prefixBytes { s = s[:prefixBytes] } return s } // promptBytes measures the whole prompt, which promptPrefix deliberately does // not. // // The two were the same function for too long, and the estimate was taken from // the prefix: bytes over four of at most prefixBytes, so every prompt above // 2 KB counted as 512 tokens however long it really was. Prefix routing wants a // bounded head of the conversation; a token estimate wants all of it, and using // one for the other meant TPM limits undercounted exactly the requests that // consume the most, and nothing could tell whether a prompt fit in a context. func promptBytes(b *inferenceBody) int { n := 0 add := func(raw json.RawMessage) { if len(raw) == 0 { return } var str string if err := json.Unmarshal(raw, &str); err == nil { n += len(str) return } // Not a plain string: structured content, an array of parts, an image. // The raw JSON overstates the text slightly, which is the safe // direction for a limit. n += len(raw) } for _, m := range b.Messages { n += len(m.Role) + 2 add(m.Content) } add(b.Prompt) add(b.Input) return n } // injectIncludeUsage sets stream_options.include_usage on the outgoing body. // // This is the one place the router rewrites a request, so it is done by // decoding to a generic map and re-encoding: field order changes, but no field // is lost. If anything looks unexpected it declines rather than guessing. func injectIncludeUsage(body []byte) ([]byte, bool) { var m map[string]json.RawMessage if err := json.Unmarshal(body, &m); err != nil { return nil, false } opts := map[string]any{} if raw, ok := m["stream_options"]; ok { if err := json.Unmarshal(raw, &opts); err != nil { return nil, false } } opts["include_usage"] = true encoded, err := json.Marshal(opts) if err != nil { return nil, false } m["stream_options"] = encoded out, err := json.Marshal(m) if err != nil { return nil, false } return out, true } // noEndpointFor explains that nothing serves a model, and says what does. // // "no endpoint serves \"MODEL\"" was all a client got, which is true and // useless: it reads the same whether the name was a typo, a placeholder left // in from the documentation, or a real outage. Someone copying the curl from // the account page sent the literal word MODEL and had nothing to go on. // // The list is what /v1/models already returns to the same caller, so naming it // here discloses nothing new. It is capped because a large catalog would turn // an error message into a page. func noEndpointFor(model string, served []string) string { if len(served) == 0 { return fmt.Sprintf("no endpoint serves %q, and no model is currently being served", model) } const showAt = 10 shown := served suffix := "" if len(shown) > showAt { shown, suffix = shown[:showAt], fmt.Sprintf(" and %d more", len(served)-showAt) } return fmt.Sprintf("no endpoint serves %q; available models are %s%s", model, strings.Join(shown, ", "), suffix) } // exampleModels is servedModels with the ones that answer for real first. // // The profile page takes the first name for its copy-pasteable example, and // handing it a simulated model would send someone's first request to an engine // that replies in Greek letters. func (r *Router) exampleModels() []string { names := r.servedModels() slices.SortStableFunc(names, func(a, b string) int { as, bs := r.modelMap()[a].Simulated(), r.modelMap()[b].Simulated() switch { case as == bs: return 0 case bs: return -1 default: return 1 } }) return names } // servedModels lists the canonical models at least one endpoint serves. func (r *Router) servedModels() []string { seen := map[string]struct{}{} for _, s := range r.reg.Snapshots("") { for _, m := range s.Models { seen[m] = struct{}{} } } out := make([]string, 0, len(seen)) for m := range seen { out = append(out, m) } slices.Sort(out) return out } // handleModels lists the models that at least one endpoint currently serves. func (r *Router) handleModels(w http.ResponseWriter, _ *http.Request) { seen := map[string]struct{}{} for _, s := range r.reg.Snapshots("") { for _, m := range s.Models { seen[m] = struct{}{} } } // Catalog aliases are usable names too, so clients should see them. for name := range r.modelMap() { if _, ok := seen[r.modelMap()[name].Name]; ok { seen[name] = struct{}{} } } names := make([]string, 0, len(seen)) for n := range seen { names = append(names, n) } slices.Sort(names) data := make([]map[string]any, 0, len(names)) for _, n := range names { // A simulated model is owned by "llmgw-sim", which is what the sim // engine already calls itself on its own /v1/models. Flattening it to // "llmgw" here was hiding the one distinction a client needs: this // model returns placeholder text, not inference. owner := "llmgw" if r.modelMap()[n].Simulated() { owner = "llmgw-sim" } entry := map[string]any{ "id": n, "object": "model", "created": time.Now().Unix(), "owned_by": owner, } // Only when known. A context_length of 0 would read as "no context", // which is a worse answer than saying nothing. m := r.modelMap()[n] if m.ContextLength > 0 { entry["context_length"] = m.ContextLength } if m.MaxOutput > 0 { entry["max_output_tokens"] = m.MaxOutput } // Not part of the OpenAI shape, and the reason to send it anyway is // that this list is the only view most clients have: nine names and // nothing to choose between them. if m.Description != "" { entry["description"] = m.Description } entry["capabilities"] = map[string]bool{ "tools": m.Capabilities.Tools, "vision": m.Capabilities.Vision, "embeddings": m.Capabilities.Embeddings, } data = append(data, entry) } writeJSON(w, http.StatusOK, map[string]any{"object": "list", "data": data}) } func writeJSON(w http.ResponseWriter, code int, v any) { var buf bytes.Buffer if err := json.NewEncoder(&buf).Encode(v); err != nil { http.Error(w, `{"error":{"message":"encoding failed"}}`, http.StatusInternalServerError) return } w.Header().Set("Content-Type", "application/json") w.WriteHeader(code) _, _ = w.Write(buf.Bytes()) } // writeError emits an OpenAI-shaped error so clients need no special handling // for gateway failures. func writeError(w http.ResponseWriter, code int, kind, msg string) { writeJSON(w, code, map[string]any{"error": map[string]any{ "message": msg, "type": kind, "code": nil, }}) } // estimatedTokens is what a token-rate limit is checked against, before the // real figure exists. // // The prompt estimate plus whatever the client said it wanted back. A client // that names no max_tokens gets a modest assumption rather than zero: assuming // zero would let an unbounded request through every token limit. func (p *parsedRequest) estimatedTokens() int64 { const assumedCompletion = 256 completion := p.MaxTokens if completion <= 0 { completion = assumedCompletion } return int64(p.PromptTokens + completion) } // handleMe reports what the presented key is and what it may do. // // It exists so a user can answer "how much have I used" and "why was I // refused" without asking an operator to read a dashboard they cannot reach. // Deliberately outside /v1: it is not part of the OpenAI API, and putting it // there would imply it is. func (r *Router) handleMe(w http.ResponseWriter, req *http.Request) { p, ok := auth.FromContext(req.Context()) if !ok { // With authentication off there is no identity to describe, and saying // so is more useful than an empty object. writeJSON(w, http.StatusOK, map[string]any{ "authenticated": false, "note": "this gateway has no api keys configured", }) return } out := map[string]any{ "authenticated": true, "key_id": p.KeyID, "key": p.KeyPrefix, "project": p.ProjectID, "team": p.TeamID, "organization": p.OrgID, } if p.Band != "" { out["band"] = p.Band } limits := map[string]any{} if p.Limits.RPM != nil { limits["requests_per_minute"] = *p.Limits.RPM } if p.Limits.TPM != nil { limits["tokens_per_minute"] = *p.Limits.TPM } if p.Limits.AllowedModels != nil { limits["allowed_models"] = p.Limits.AllowedModels } if p.Limits.Budget != nil { limits["budget_units"] = float64(*p.Limits.Budget) / 1e6 limits["budget_period"] = string(p.Limits.BudgetPeriod) } out["limits"] = limits if r.acct != nil { spent, rpm, tpm := r.acct.Status(p) usage := map[string]any{ "requests_per_minute": rpm, "tokens_per_minute": tpm, } if p.Limits.Budget != nil { usage["spent_units"] = float64(spent) / 1e6 usage["remaining_units"] = float64(*p.Limits.Budget-spent) / 1e6 } out["usage"] = usage } writeJSON(w, http.StatusOK, out) }