A user copied the curl from the profile page and sent it as it stood. Every other field in that snippet is filled in for them — the base URL, the key they had just been issued — so "model": "MODEL" reads as a value rather than a blank, and they got: no endpoint serves "MODEL" which is true and useless. It reads the same whether the name was a typo, a placeholder left in from the documentation, or a real outage, and it never says what would have worked. The message now lists what is being served, capped at ten with a count for the rest, and says plainly when nothing is. That list is what /v1/models already returns to the same caller, so it discloses nothing new. The profile page's example now names a model that is actually running, real ones before simulated — the point of an example someone can run is that it runs, and sending a first request to the sim engine would answer in Greek letters. The landing and claim pages keep a placeholder, because they are served without a key and deliberately say nothing about what is here; theirs is <model-name>, which cannot be mistaken for a value, with a line saying where to get the real one. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
659 lines
22 KiB
Go
659 lines
22 KiB
Go
package router
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/admission"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/web"
|
|
)
|
|
|
|
// admit waits for the flow control layer to let the request through. With
|
|
// admission disabled it returns immediately, which is the v0.2.0 behaviour.
|
|
func (r *Router) admit(ctx context.Context, req *parsedRequest) (*admission.Lease, error) {
|
|
if r.adm == nil {
|
|
return nil, nil
|
|
}
|
|
// The key is the tenant when there is one: fairness within a band has to be
|
|
// per caller, and a session id is something the caller picks, so one client
|
|
// could take a whole band by opening many sessions. Without authentication
|
|
// the session is still the best available grouping.
|
|
tenant := req.Tenant
|
|
if tenant == "" {
|
|
tenant = req.SessionID
|
|
}
|
|
return r.adm.Acquire(ctx, admission.Request{
|
|
Model: req.Model,
|
|
Band: req.Band,
|
|
Tenant: tenant,
|
|
})
|
|
}
|
|
|
|
// rejectAdmission turns a flow control refusal into a status the client can act
|
|
// on: 503 when the queue is full (try later), 504 when it timed out waiting.
|
|
func (r *Router) rejectAdmission(w http.ResponseWriter, req *parsedRequest, err error, start time.Time) {
|
|
status := http.StatusServiceUnavailable
|
|
switch {
|
|
case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded):
|
|
// The client gave up while queued; nobody is listening.
|
|
r.m.observeAdmissionRejected(req.Band, "client_gone")
|
|
r.m.observeRequest(req.Model, "", 499, time.Since(start), nil, 0)
|
|
return
|
|
case errors.Is(err, admission.ErrQueueTimeout):
|
|
status = http.StatusGatewayTimeout
|
|
}
|
|
reason := "unavailable"
|
|
switch {
|
|
case errors.Is(err, admission.ErrQueueFull):
|
|
reason = "queue_full"
|
|
case errors.Is(err, admission.ErrQueueTimeout):
|
|
reason = "ttl"
|
|
case errors.Is(err, admission.ErrClosed):
|
|
reason = "shutdown"
|
|
}
|
|
r.m.observeAdmissionRejected(req.Band, reason)
|
|
writeError(w, status, "server_error", err.Error())
|
|
r.m.observeRequest(req.Model, "", status, time.Since(start), nil, 0)
|
|
}
|
|
|
|
// prefixBytes is how much of the prompt is kept for prefix-aware routing.
|
|
// Large enough to cover a system prompt plus the first message, small enough
|
|
// that hashing it stays cheap.
|
|
const prefixBytes = 2048
|
|
|
|
// Handler returns the client-facing OpenAI API.
|
|
func (r *Router) Handler() http.Handler {
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("POST /v1/chat/completions", r.handleInference("/v1/chat/completions"))
|
|
mux.HandleFunc("POST /v1/completions", r.handleInference("/v1/completions"))
|
|
mux.HandleFunc("POST /v1/embeddings", r.handleInference("/v1/embeddings"))
|
|
mux.HandleFunc("GET /v1/models", r.handleModels)
|
|
mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusOK)
|
|
})
|
|
|
|
// The landing page says how to connect and nothing about what is here, so
|
|
// it is safe to serve without a key — and it has to be, because the person
|
|
// reading it is the one who does not have one yet.
|
|
mux.HandleFunc("GET /", web.Landing(version.Version, r.auth != nil,
|
|
r.accounts != nil && r.accounts.SignupOpen(),
|
|
r.landing.Contact, r.landing.Notice))
|
|
mux.Handle("GET /_static/", web.Static())
|
|
mux.HandleFunc("GET /me", r.handleMe)
|
|
|
|
if r.store != nil {
|
|
// The one-time link that delivers a key to the person it was issued
|
|
// for. Served without a key, because the person opening it is the one
|
|
// who does not have one yet; what protects it is the token.
|
|
claim := web.Claim(r.store, version.Version, r.log)
|
|
mux.HandleFunc("GET /claim/{token}", claim)
|
|
mux.HandleFunc("POST /claim/{token}", claim)
|
|
}
|
|
if r.accounts != nil {
|
|
// Signup, sign-in and the profile page. These are browser surfaces
|
|
// authenticated by a session cookie, not by an API key, which is why
|
|
// they are exempt below.
|
|
pages := web.NewAccountPages(r.accounts, r.store, r.store, version.Version, r.log)
|
|
// So the example on the profile page names a model that is actually
|
|
// running, rather than a placeholder that gets sent as-is.
|
|
pages.ServedModels(r.exampleModels)
|
|
pages.Register(mux)
|
|
r.registerChat(mux, pages)
|
|
|
|
if r.clusterView != nil {
|
|
// The cluster dashboard, reachable from a browser without exposing
|
|
// the admin listener — which also carries pprof, and pprof can dump
|
|
// the heap that holds every API key in plaintext.
|
|
mux.HandleFunc("GET /admin", pages.AdminPage(web.Dashboard(
|
|
r.clusterView, r.clusterUsage, version.Version, 5*time.Second, false, r.log)))
|
|
pages.RegisterAdminUsers(mux, r.store)
|
|
}
|
|
}
|
|
|
|
if r.auth == nil {
|
|
return mux
|
|
}
|
|
// Wrapping the whole mux rather than checking inside each handler: adding a
|
|
// route must not be able to add an unauthenticated one by omission. Health
|
|
// stays open because a load balancer has no key.
|
|
return auth.Middleware(r.auth, r.log,
|
|
"/health", "/", "/_static/", "/claim/",
|
|
"/signup", "/login", "/logout", "/profile", "/profile/",
|
|
"/admin", "/admin/", "/chat", "/chat/")(mux)
|
|
}
|
|
|
|
// parsedRequest is what the router needs from a request. Everything else in the
|
|
// body is forwarded untouched — clients send engine-specific fields we have no
|
|
// business interpreting.
|
|
type parsedRequest struct {
|
|
Path string
|
|
Model string
|
|
|
|
Stream bool
|
|
SuppressUsageChunk bool
|
|
UpstreamBody []byte
|
|
Header http.Header
|
|
|
|
SessionID string
|
|
Band string
|
|
// KeyID, ProjectID and Tenant are empty when authentication is off.
|
|
KeyID string
|
|
ProjectID string
|
|
Tenant string
|
|
ProfileOverride string
|
|
PromptPrefix string
|
|
PromptTokens int
|
|
MaxTokens int
|
|
|
|
// reservation is carried from the limit check to the point where the
|
|
// request is settled, so the record counts against the same budget window
|
|
// the check read.
|
|
reservation accounting.Reservation
|
|
}
|
|
|
|
// inferenceBody covers the fields the router reads across all inference
|
|
// endpoints. Unknown fields are ignored here and preserved in UpstreamBody.
|
|
type inferenceBody struct {
|
|
Model string `json:"model"`
|
|
Stream bool `json:"stream"`
|
|
User string `json:"user"`
|
|
Messages []struct {
|
|
Role string `json:"role"`
|
|
Content json.RawMessage `json:"content"`
|
|
} `json:"messages"`
|
|
Prompt json.RawMessage `json:"prompt"`
|
|
Input json.RawMessage `json:"input"`
|
|
MaxTokens int `json:"max_tokens"`
|
|
MaxCompletionTokens int `json:"max_completion_tokens"`
|
|
StreamOptions *struct {
|
|
IncludeUsage bool `json:"include_usage"`
|
|
} `json:"stream_options"`
|
|
}
|
|
|
|
func (r *Router) handleInference(path string) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, req *http.Request) {
|
|
start := time.Now()
|
|
|
|
parsed, err := r.parse(req, path)
|
|
if err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
|
r.m.observeRequest("", "", http.StatusBadRequest, time.Since(start), nil, 0)
|
|
return
|
|
}
|
|
|
|
if p, ok := auth.FromContext(req.Context()); ok {
|
|
if !r.permits(p, parsed.Model) {
|
|
auth.WriteForbiddenModel(w, parsed.Model)
|
|
r.m.observeRequest(parsed.Model, "", http.StatusForbidden, time.Since(start), nil, 0)
|
|
return
|
|
}
|
|
// Limits are checked before admission, not after: a request that is
|
|
// over its quota should be refused rather than queued, and queueing
|
|
// it would have it occupy a band's capacity to no purpose.
|
|
if r.acct != nil {
|
|
reservation, err := r.acct.Check(p, parsed.Model, parsed.estimatedTokens())
|
|
if err != nil {
|
|
r.rejectLimit(w, parsed, err, start)
|
|
return
|
|
}
|
|
parsed.reservation = reservation
|
|
}
|
|
}
|
|
|
|
// Step 5 of the lifecycle: "now?" before "where?" (plan §7.1).
|
|
lease, err := r.admit(req.Context(), parsed)
|
|
if err != nil {
|
|
r.rejectAdmission(w, parsed, err, start)
|
|
return
|
|
}
|
|
defer lease.Release()
|
|
queued := time.Since(start)
|
|
r.m.observeAdmission(lease.Band(), parsed.Model, queued)
|
|
if queued > time.Millisecond {
|
|
w.Header().Set("x-llmgw-queue-ms", strconv.FormatInt(queued.Milliseconds(), 10))
|
|
}
|
|
|
|
res := r.dispatch(req.Context(), w, parsed)
|
|
r.finish(w, parsed, res, start)
|
|
}
|
|
}
|
|
|
|
// permits reports whether a key's allowlist covers the requested model.
|
|
//
|
|
// The decision is per model, not per string. A model's canonical name and every
|
|
// alias of it name the same thing, so the allowlist is matched against all of
|
|
// them: writing either form works, and no alias can reach a model that was not
|
|
// allowed. Matching only what the client sent would make every alias a way
|
|
// around the allowlist; matching only the canonical name would silently ignore
|
|
// an allowlist written in the names clients actually use.
|
|
func (r *Router) permits(p auth.Principal, requested string) bool {
|
|
canonical, m := r.resolve(requested)
|
|
if p.Allows(canonical) {
|
|
return true
|
|
}
|
|
for _, alias := range m.Aliases {
|
|
if p.Allows(alias) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// finish records metrics and, when nothing was written, produces the error body.
|
|
func (r *Router) finish(w http.ResponseWriter, req *parsedRequest, res dispatchResult, start time.Time) {
|
|
status := res.outcome.Status
|
|
switch {
|
|
case res.err == nil:
|
|
if res.fallback {
|
|
// The client asked for one model and got another; say so rather
|
|
// than letting them discover it from the response contents.
|
|
w.Header().Set("x-llmgw-served-model", res.model)
|
|
}
|
|
case errors.Is(res.err, proxy.ErrClientGone):
|
|
// Nothing to report to a client that is gone; the upstream was cancelled.
|
|
status = 499
|
|
r.log.Debug("client disconnected", "model", req.Model, "endpoint", res.endpointID)
|
|
case res.outcome.Wrote:
|
|
// The response is already partly out; the stream just ends short.
|
|
r.log.Warn("stream ended early", "model", req.Model, "endpoint", res.endpointID, "err", res.err)
|
|
if status == 0 {
|
|
status = http.StatusOK
|
|
}
|
|
case errors.Is(res.err, errNoCapacity):
|
|
status = http.StatusServiceUnavailable
|
|
writeError(w, status, "server_error", res.err.Error())
|
|
default:
|
|
status = http.StatusBadGateway
|
|
writeError(w, status, "server_error", res.err.Error())
|
|
}
|
|
|
|
r.m.observeRequest(req.Model, res.pool, status, time.Since(start), res.outcome.Usage, res.outcome.TTFT)
|
|
r.account(req, res, status, time.Since(start))
|
|
}
|
|
|
|
// account settles a finished request against its key.
|
|
//
|
|
// It runs for every outcome, not only success: a request that used a GPU and
|
|
// then failed still used it, and a key that could spend nothing by failing
|
|
// would be a way around a budget.
|
|
func (r *Router) account(req *parsedRequest, res dispatchResult, status int, took time.Duration) {
|
|
if r.acct == nil || req.KeyID == "" {
|
|
return
|
|
}
|
|
u := store.Usage{
|
|
At: time.Now(), KeyID: req.KeyID, ProjectID: req.ProjectID,
|
|
Model: req.Model, ServedModel: res.model, Status: status,
|
|
LatencyMs: took.Milliseconds(),
|
|
}
|
|
if u.ServedModel == "" {
|
|
u.ServedModel = req.Model
|
|
}
|
|
if usage := res.outcome.Usage; usage != nil {
|
|
u.PromptTokens = int64(usage.PromptTokens)
|
|
u.CompletionTokens = int64(usage.CompletionTokens)
|
|
} else {
|
|
// No usage came back — a failure before the engine answered, or an
|
|
// engine that does not report it. Charging the estimate is closer to
|
|
// the truth than charging nothing.
|
|
u.PromptTokens = int64(req.PromptTokens)
|
|
}
|
|
r.acct.Record(req.reservation, u)
|
|
}
|
|
|
|
// rejectLimit turns a quota refusal into a status the client can act on.
|
|
func (r *Router) rejectLimit(w http.ResponseWriter, req *parsedRequest, err error, start time.Time) {
|
|
var le *accounting.LimitError
|
|
if !errors.As(err, &le) {
|
|
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
|
|
return
|
|
}
|
|
if le.RetryAfter > 0 {
|
|
w.Header().Set("Retry-After", strconv.Itoa(int(le.RetryAfter.Seconds())))
|
|
}
|
|
// 429 for all of them: a spent budget is a quota, and OpenAI clients
|
|
// already know how to surface that code. What differs is Retry-After,
|
|
// which is absent when waiting does not help.
|
|
writeError(w, http.StatusTooManyRequests, "rate_limit_exceeded", le.Error())
|
|
r.m.observeRequest(req.Model, "", http.StatusTooManyRequests, time.Since(start), nil, 0)
|
|
}
|
|
|
|
// parse reads the body and extracts the routing-relevant fields.
|
|
func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) {
|
|
body, err := io.ReadAll(io.LimitReader(req.Body, r.cfg.MaxBodyBytes))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var b inferenceBody
|
|
if err := json.Unmarshal(body, &b); err != nil {
|
|
return nil, err
|
|
}
|
|
if b.Model == "" {
|
|
return nil, errors.New("model is required")
|
|
}
|
|
|
|
p := &parsedRequest{
|
|
Path: path,
|
|
Model: b.Model,
|
|
Stream: b.Stream,
|
|
UpstreamBody: body,
|
|
Header: req.Header,
|
|
SessionID: sessionID(req, r.cfg.SessionHeader, b.User),
|
|
Band: req.Header.Get(r.cfg.Admission.BandHeader),
|
|
ProfileOverride: req.Header.Get("x-llmgw-profile"),
|
|
MaxTokens: max(b.MaxTokens, b.MaxCompletionTokens),
|
|
PromptPrefix: promptPrefix(&b),
|
|
}
|
|
// With authentication on, the key decides the band and the routing profile,
|
|
// and the headers stop being believed. Trusting the header was only ever
|
|
// acceptable while there was no identity to consult — it let any client
|
|
// promote itself into the top priority band.
|
|
if principal, ok := auth.FromContext(req.Context()); ok {
|
|
p.Band = principal.Band
|
|
p.ProfileOverride = principal.RoutingProfile
|
|
p.KeyID = principal.KeyID
|
|
p.ProjectID = principal.ProjectID
|
|
// Tenant fairness within a band is per key rather than per session, so
|
|
// one key cannot take a whole band by opening many sessions.
|
|
p.Tenant = principal.KeyID
|
|
}
|
|
|
|
// Bytes over four: a crude stand-in for a tokenizer, good enough for
|
|
// scoring. It becomes exact only with plan §7.5 tier 2 (see plan §14).
|
|
p.PromptTokens = len(p.PromptPrefix) / 4
|
|
|
|
// Ask the engine for usage the client did not request, so accounting works
|
|
// regardless of client behaviour; the extra chunk is dropped on the way back.
|
|
if r.cfg.InjectUsage && b.Stream && (b.StreamOptions == nil || !b.StreamOptions.IncludeUsage) {
|
|
if injected, ok := injectIncludeUsage(body); ok {
|
|
p.UpstreamBody = injected
|
|
p.SuppressUsageChunk = true
|
|
}
|
|
}
|
|
return p, nil
|
|
}
|
|
|
|
// sessionID picks the affinity key: an explicit header first, then OpenAI's
|
|
// user field, which most clients already set per end user.
|
|
func sessionID(req *http.Request, header, user string) string {
|
|
if header != "" {
|
|
if v := req.Header.Get(header); v != "" {
|
|
return v
|
|
}
|
|
}
|
|
return user
|
|
}
|
|
|
|
// promptPrefix reconstructs the leading portion of the prompt for prefix-aware
|
|
// routing. Only the first prefixBytes matter, so long conversations stop early.
|
|
func promptPrefix(b *inferenceBody) string {
|
|
var sb strings.Builder
|
|
appendRaw := func(raw json.RawMessage) {
|
|
if len(raw) == 0 || sb.Len() >= prefixBytes {
|
|
return
|
|
}
|
|
var s string
|
|
if err := json.Unmarshal(raw, &s); err != nil {
|
|
s = string(raw)
|
|
}
|
|
sb.WriteString(s)
|
|
}
|
|
|
|
for _, m := range b.Messages {
|
|
if sb.Len() >= prefixBytes {
|
|
break
|
|
}
|
|
sb.WriteString(m.Role)
|
|
sb.WriteString(":")
|
|
appendRaw(m.Content)
|
|
sb.WriteString("\n")
|
|
}
|
|
appendRaw(b.Prompt)
|
|
appendRaw(b.Input)
|
|
|
|
s := sb.String()
|
|
if len(s) > prefixBytes {
|
|
s = s[:prefixBytes]
|
|
}
|
|
return s
|
|
}
|
|
|
|
// injectIncludeUsage sets stream_options.include_usage on the outgoing body.
|
|
//
|
|
// This is the one place the router rewrites a request, so it is done by
|
|
// decoding to a generic map and re-encoding: field order changes, but no field
|
|
// is lost. If anything looks unexpected it declines rather than guessing.
|
|
func injectIncludeUsage(body []byte) ([]byte, bool) {
|
|
var m map[string]json.RawMessage
|
|
if err := json.Unmarshal(body, &m); err != nil {
|
|
return nil, false
|
|
}
|
|
|
|
opts := map[string]any{}
|
|
if raw, ok := m["stream_options"]; ok {
|
|
if err := json.Unmarshal(raw, &opts); err != nil {
|
|
return nil, false
|
|
}
|
|
}
|
|
opts["include_usage"] = true
|
|
|
|
encoded, err := json.Marshal(opts)
|
|
if err != nil {
|
|
return nil, false
|
|
}
|
|
m["stream_options"] = encoded
|
|
|
|
out, err := json.Marshal(m)
|
|
if err != nil {
|
|
return nil, false
|
|
}
|
|
return out, true
|
|
}
|
|
|
|
// noEndpointFor explains that nothing serves a model, and says what does.
|
|
//
|
|
// "no endpoint serves \"MODEL\"" was all a client got, which is true and
|
|
// useless: it reads the same whether the name was a typo, a placeholder left
|
|
// in from the documentation, or a real outage. Someone copying the curl from
|
|
// the account page sent the literal word MODEL and had nothing to go on.
|
|
//
|
|
// The list is what /v1/models already returns to the same caller, so naming it
|
|
// here discloses nothing new. It is capped because a large catalog would turn
|
|
// an error message into a page.
|
|
func noEndpointFor(model string, served []string) string {
|
|
if len(served) == 0 {
|
|
return fmt.Sprintf("no endpoint serves %q, and no model is currently being served", model)
|
|
}
|
|
const showAt = 10
|
|
shown := served
|
|
suffix := ""
|
|
if len(shown) > showAt {
|
|
shown, suffix = shown[:showAt], fmt.Sprintf(" and %d more", len(served)-showAt)
|
|
}
|
|
return fmt.Sprintf("no endpoint serves %q; available models are %s%s",
|
|
model, strings.Join(shown, ", "), suffix)
|
|
}
|
|
|
|
// exampleModels is servedModels with the ones that answer for real first.
|
|
//
|
|
// The profile page takes the first name for its copy-pasteable example, and
|
|
// handing it a simulated model would send someone's first request to an engine
|
|
// that replies in Greek letters.
|
|
func (r *Router) exampleModels() []string {
|
|
names := r.servedModels()
|
|
slices.SortStableFunc(names, func(a, b string) int {
|
|
as, bs := r.models[a].Simulated(), r.models[b].Simulated()
|
|
switch {
|
|
case as == bs:
|
|
return 0
|
|
case bs:
|
|
return -1
|
|
default:
|
|
return 1
|
|
}
|
|
})
|
|
return names
|
|
}
|
|
|
|
// servedModels lists the canonical models at least one endpoint serves.
|
|
func (r *Router) servedModels() []string {
|
|
seen := map[string]struct{}{}
|
|
for _, s := range r.reg.Snapshots("") {
|
|
for _, m := range s.Models {
|
|
seen[m] = struct{}{}
|
|
}
|
|
}
|
|
out := make([]string, 0, len(seen))
|
|
for m := range seen {
|
|
out = append(out, m)
|
|
}
|
|
slices.Sort(out)
|
|
return out
|
|
}
|
|
|
|
// handleModels lists the models that at least one endpoint currently serves.
|
|
func (r *Router) handleModels(w http.ResponseWriter, _ *http.Request) {
|
|
seen := map[string]struct{}{}
|
|
for _, s := range r.reg.Snapshots("") {
|
|
for _, m := range s.Models {
|
|
seen[m] = struct{}{}
|
|
}
|
|
}
|
|
// Catalog aliases are usable names too, so clients should see them.
|
|
for name := range r.models {
|
|
if _, ok := seen[r.models[name].Name]; ok {
|
|
seen[name] = struct{}{}
|
|
}
|
|
}
|
|
|
|
names := make([]string, 0, len(seen))
|
|
for n := range seen {
|
|
names = append(names, n)
|
|
}
|
|
slices.Sort(names)
|
|
|
|
data := make([]map[string]any, 0, len(names))
|
|
for _, n := range names {
|
|
// A simulated model is owned by "llmgw-sim", which is what the sim
|
|
// engine already calls itself on its own /v1/models. Flattening it to
|
|
// "llmgw" here was hiding the one distinction a client needs: this
|
|
// model returns placeholder text, not inference.
|
|
owner := "llmgw"
|
|
if r.models[n].Simulated() {
|
|
owner = "llmgw-sim"
|
|
}
|
|
data = append(data, map[string]any{
|
|
"id": n, "object": "model", "created": time.Now().Unix(), "owned_by": owner,
|
|
})
|
|
}
|
|
writeJSON(w, http.StatusOK, map[string]any{"object": "list", "data": data})
|
|
}
|
|
|
|
func writeJSON(w http.ResponseWriter, code int, v any) {
|
|
var buf bytes.Buffer
|
|
if err := json.NewEncoder(&buf).Encode(v); err != nil {
|
|
http.Error(w, `{"error":{"message":"encoding failed"}}`, http.StatusInternalServerError)
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(code)
|
|
_, _ = w.Write(buf.Bytes())
|
|
}
|
|
|
|
// writeError emits an OpenAI-shaped error so clients need no special handling
|
|
// for gateway failures.
|
|
func writeError(w http.ResponseWriter, code int, kind, msg string) {
|
|
writeJSON(w, code, map[string]any{"error": map[string]any{
|
|
"message": msg, "type": kind, "code": nil,
|
|
}})
|
|
}
|
|
|
|
// estimatedTokens is what a token-rate limit is checked against, before the
|
|
// real figure exists.
|
|
//
|
|
// The prompt estimate plus whatever the client said it wanted back. A client
|
|
// that names no max_tokens gets a modest assumption rather than zero: assuming
|
|
// zero would let an unbounded request through every token limit.
|
|
func (p *parsedRequest) estimatedTokens() int64 {
|
|
const assumedCompletion = 256
|
|
completion := p.MaxTokens
|
|
if completion <= 0 {
|
|
completion = assumedCompletion
|
|
}
|
|
return int64(p.PromptTokens + completion)
|
|
}
|
|
|
|
// handleMe reports what the presented key is and what it may do.
|
|
//
|
|
// It exists so a user can answer "how much have I used" and "why was I
|
|
// refused" without asking an operator to read a dashboard they cannot reach.
|
|
// Deliberately outside /v1: it is not part of the OpenAI API, and putting it
|
|
// there would imply it is.
|
|
func (r *Router) handleMe(w http.ResponseWriter, req *http.Request) {
|
|
p, ok := auth.FromContext(req.Context())
|
|
if !ok {
|
|
// With authentication off there is no identity to describe, and saying
|
|
// so is more useful than an empty object.
|
|
writeJSON(w, http.StatusOK, map[string]any{
|
|
"authenticated": false,
|
|
"note": "this gateway has no api keys configured",
|
|
})
|
|
return
|
|
}
|
|
|
|
out := map[string]any{
|
|
"authenticated": true,
|
|
"key_id": p.KeyID,
|
|
"key": p.KeyPrefix,
|
|
"project": p.ProjectID,
|
|
"team": p.TeamID,
|
|
"organization": p.OrgID,
|
|
}
|
|
if p.Band != "" {
|
|
out["band"] = p.Band
|
|
}
|
|
limits := map[string]any{}
|
|
if p.Limits.RPM != nil {
|
|
limits["requests_per_minute"] = *p.Limits.RPM
|
|
}
|
|
if p.Limits.TPM != nil {
|
|
limits["tokens_per_minute"] = *p.Limits.TPM
|
|
}
|
|
if p.Limits.AllowedModels != nil {
|
|
limits["allowed_models"] = p.Limits.AllowedModels
|
|
}
|
|
if p.Limits.Budget != nil {
|
|
limits["budget_units"] = float64(*p.Limits.Budget) / 1e6
|
|
limits["budget_period"] = string(p.Limits.BudgetPeriod)
|
|
}
|
|
out["limits"] = limits
|
|
|
|
if r.acct != nil {
|
|
spent, rpm, tpm := r.acct.Status(p)
|
|
usage := map[string]any{
|
|
"requests_per_minute": rpm,
|
|
"tokens_per_minute": tpm,
|
|
}
|
|
if p.Limits.Budget != nil {
|
|
usage["spent_units"] = float64(spent) / 1e6
|
|
usage["remaining_units"] = float64(*p.Limits.Budget-spent) / 1e6
|
|
}
|
|
out["usage"] = usage
|
|
}
|
|
writeJSON(w, http.StatusOK, out)
|
|
}
|