CI / test (push) Successful in 1m19s
Two things the dashboard was missing: it could not act, and it was a column of numbers. Account administration moves to /admin/users — approving into a project, suspending, reinstating. That is the work an operator does constantly and it should not need a terminal. Three things stay on the CLI, and the line is not arbitrary: promoting someone to admin, resetting a password, and issuing a key on another person's behalf each turn a stolen admin session into more than it was — permanent access, or somebody else's account. Approving and suspending do not. Suspending signs the account out everywhere, or suspension is advisory until the session happens to expire, and an admin cannot suspend themselves, which is not a thing an interface should let you do by misclicking. The charts are inline SVG generated in Go. A charting library would mean a JavaScript toolchain in CI and a megabyte of bundle for what is a list of rectangles, and the single-binary goal rules that out. Hourly over a day: fine enough to show a working day, coarse enough that one slow minute is not an incident. Empty hours are filled in, because a chart with hours missing where nothing happened reads as a chart with hours missing from the data. The SVG is marked as trusted HTML, so everything that reaches it is either a number this package computed or escaped explicitly, and a bar's colour can only be one of four names — a colour that could be anything is a way to write CSS into every page the chart appears on. There are tests for both. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
600 lines
20 KiB
Go
600 lines
20 KiB
Go
package router
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"net/http"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/admission"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/web"
|
|
)
|
|
|
|
// admit waits for the flow control layer to let the request through. With
|
|
// admission disabled it returns immediately, which is the v0.2.0 behaviour.
|
|
func (r *Router) admit(ctx context.Context, req *parsedRequest) (*admission.Lease, error) {
|
|
if r.adm == nil {
|
|
return nil, nil
|
|
}
|
|
// The key is the tenant when there is one: fairness within a band has to be
|
|
// per caller, and a session id is something the caller picks, so one client
|
|
// could take a whole band by opening many sessions. Without authentication
|
|
// the session is still the best available grouping.
|
|
tenant := req.Tenant
|
|
if tenant == "" {
|
|
tenant = req.SessionID
|
|
}
|
|
return r.adm.Acquire(ctx, admission.Request{
|
|
Model: req.Model,
|
|
Band: req.Band,
|
|
Tenant: tenant,
|
|
})
|
|
}
|
|
|
|
// rejectAdmission turns a flow control refusal into a status the client can act
|
|
// on: 503 when the queue is full (try later), 504 when it timed out waiting.
|
|
func (r *Router) rejectAdmission(w http.ResponseWriter, req *parsedRequest, err error, start time.Time) {
|
|
status := http.StatusServiceUnavailable
|
|
switch {
|
|
case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded):
|
|
// The client gave up while queued; nobody is listening.
|
|
r.m.observeAdmissionRejected(req.Band, "client_gone")
|
|
r.m.observeRequest(req.Model, "", 499, time.Since(start), nil, 0)
|
|
return
|
|
case errors.Is(err, admission.ErrQueueTimeout):
|
|
status = http.StatusGatewayTimeout
|
|
}
|
|
reason := "unavailable"
|
|
switch {
|
|
case errors.Is(err, admission.ErrQueueFull):
|
|
reason = "queue_full"
|
|
case errors.Is(err, admission.ErrQueueTimeout):
|
|
reason = "ttl"
|
|
case errors.Is(err, admission.ErrClosed):
|
|
reason = "shutdown"
|
|
}
|
|
r.m.observeAdmissionRejected(req.Band, reason)
|
|
writeError(w, status, "server_error", err.Error())
|
|
r.m.observeRequest(req.Model, "", status, time.Since(start), nil, 0)
|
|
}
|
|
|
|
// prefixBytes is how much of the prompt is kept for prefix-aware routing.
|
|
// Large enough to cover a system prompt plus the first message, small enough
|
|
// that hashing it stays cheap.
|
|
const prefixBytes = 2048
|
|
|
|
// Handler returns the client-facing OpenAI API.
|
|
func (r *Router) Handler() http.Handler {
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("POST /v1/chat/completions", r.handleInference("/v1/chat/completions"))
|
|
mux.HandleFunc("POST /v1/completions", r.handleInference("/v1/completions"))
|
|
mux.HandleFunc("POST /v1/embeddings", r.handleInference("/v1/embeddings"))
|
|
mux.HandleFunc("GET /v1/models", r.handleModels)
|
|
mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusOK)
|
|
})
|
|
|
|
// The landing page says how to connect and nothing about what is here, so
|
|
// it is safe to serve without a key — and it has to be, because the person
|
|
// reading it is the one who does not have one yet.
|
|
mux.HandleFunc("GET /", web.Landing(version.Version, r.auth != nil,
|
|
r.accounts != nil && r.accounts.SignupOpen(),
|
|
r.landing.Contact, r.landing.Notice))
|
|
mux.Handle("GET /_static/", web.Static())
|
|
mux.HandleFunc("GET /me", r.handleMe)
|
|
|
|
if r.store != nil {
|
|
// The one-time link that delivers a key to the person it was issued
|
|
// for. Served without a key, because the person opening it is the one
|
|
// who does not have one yet; what protects it is the token.
|
|
claim := web.Claim(r.store, version.Version, r.log)
|
|
mux.HandleFunc("GET /claim/{token}", claim)
|
|
mux.HandleFunc("POST /claim/{token}", claim)
|
|
}
|
|
if r.accounts != nil {
|
|
// Signup, sign-in and the profile page. These are browser surfaces
|
|
// authenticated by a session cookie, not by an API key, which is why
|
|
// they are exempt below.
|
|
pages := web.NewAccountPages(r.accounts, r.store, r.store, version.Version, r.log)
|
|
pages.Register(mux)
|
|
|
|
if r.clusterView != nil {
|
|
// The cluster dashboard, reachable from a browser without exposing
|
|
// the admin listener — which also carries pprof, and pprof can dump
|
|
// the heap that holds every API key in plaintext.
|
|
mux.HandleFunc("GET /admin", pages.AdminPage(web.Dashboard(
|
|
r.clusterView, r.clusterUsage, version.Version, 5*time.Second, r.log)))
|
|
pages.RegisterAdminUsers(mux, r.store)
|
|
}
|
|
}
|
|
|
|
if r.auth == nil {
|
|
return mux
|
|
}
|
|
// Wrapping the whole mux rather than checking inside each handler: adding a
|
|
// route must not be able to add an unauthenticated one by omission. Health
|
|
// stays open because a load balancer has no key.
|
|
return auth.Middleware(r.auth, r.log,
|
|
"/health", "/", "/_static/", "/claim/",
|
|
"/signup", "/login", "/logout", "/profile", "/profile/", "/admin", "/admin/")(mux)
|
|
}
|
|
|
|
// parsedRequest is what the router needs from a request. Everything else in the
|
|
// body is forwarded untouched — clients send engine-specific fields we have no
|
|
// business interpreting.
|
|
type parsedRequest struct {
|
|
Path string
|
|
Model string
|
|
|
|
Stream bool
|
|
SuppressUsageChunk bool
|
|
UpstreamBody []byte
|
|
Header http.Header
|
|
|
|
SessionID string
|
|
Band string
|
|
// KeyID, ProjectID and Tenant are empty when authentication is off.
|
|
KeyID string
|
|
ProjectID string
|
|
Tenant string
|
|
ProfileOverride string
|
|
PromptPrefix string
|
|
PromptTokens int
|
|
MaxTokens int
|
|
|
|
// reservation is carried from the limit check to the point where the
|
|
// request is settled, so the record counts against the same budget window
|
|
// the check read.
|
|
reservation accounting.Reservation
|
|
}
|
|
|
|
// inferenceBody covers the fields the router reads across all inference
|
|
// endpoints. Unknown fields are ignored here and preserved in UpstreamBody.
|
|
type inferenceBody struct {
|
|
Model string `json:"model"`
|
|
Stream bool `json:"stream"`
|
|
User string `json:"user"`
|
|
Messages []struct {
|
|
Role string `json:"role"`
|
|
Content json.RawMessage `json:"content"`
|
|
} `json:"messages"`
|
|
Prompt json.RawMessage `json:"prompt"`
|
|
Input json.RawMessage `json:"input"`
|
|
MaxTokens int `json:"max_tokens"`
|
|
MaxCompletionTokens int `json:"max_completion_tokens"`
|
|
StreamOptions *struct {
|
|
IncludeUsage bool `json:"include_usage"`
|
|
} `json:"stream_options"`
|
|
}
|
|
|
|
func (r *Router) handleInference(path string) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, req *http.Request) {
|
|
start := time.Now()
|
|
|
|
parsed, err := r.parse(req, path)
|
|
if err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
|
r.m.observeRequest("", "", http.StatusBadRequest, time.Since(start), nil, 0)
|
|
return
|
|
}
|
|
|
|
if p, ok := auth.FromContext(req.Context()); ok {
|
|
if !r.permits(p, parsed.Model) {
|
|
auth.WriteForbiddenModel(w, parsed.Model)
|
|
r.m.observeRequest(parsed.Model, "", http.StatusForbidden, time.Since(start), nil, 0)
|
|
return
|
|
}
|
|
// Limits are checked before admission, not after: a request that is
|
|
// over its quota should be refused rather than queued, and queueing
|
|
// it would have it occupy a band's capacity to no purpose.
|
|
if r.acct != nil {
|
|
reservation, err := r.acct.Check(p, parsed.Model, parsed.estimatedTokens())
|
|
if err != nil {
|
|
r.rejectLimit(w, parsed, err, start)
|
|
return
|
|
}
|
|
parsed.reservation = reservation
|
|
}
|
|
}
|
|
|
|
// Step 5 of the lifecycle: "now?" before "where?" (plan §7.1).
|
|
lease, err := r.admit(req.Context(), parsed)
|
|
if err != nil {
|
|
r.rejectAdmission(w, parsed, err, start)
|
|
return
|
|
}
|
|
defer lease.Release()
|
|
queued := time.Since(start)
|
|
r.m.observeAdmission(lease.Band(), parsed.Model, queued)
|
|
if queued > time.Millisecond {
|
|
w.Header().Set("x-llmgw-queue-ms", strconv.FormatInt(queued.Milliseconds(), 10))
|
|
}
|
|
|
|
res := r.dispatch(req.Context(), w, parsed)
|
|
r.finish(w, parsed, res, start)
|
|
}
|
|
}
|
|
|
|
// permits reports whether a key's allowlist covers the requested model.
|
|
//
|
|
// The decision is per model, not per string. A model's canonical name and every
|
|
// alias of it name the same thing, so the allowlist is matched against all of
|
|
// them: writing either form works, and no alias can reach a model that was not
|
|
// allowed. Matching only what the client sent would make every alias a way
|
|
// around the allowlist; matching only the canonical name would silently ignore
|
|
// an allowlist written in the names clients actually use.
|
|
func (r *Router) permits(p auth.Principal, requested string) bool {
|
|
canonical, m := r.resolve(requested)
|
|
if p.Allows(canonical) {
|
|
return true
|
|
}
|
|
for _, alias := range m.Aliases {
|
|
if p.Allows(alias) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// finish records metrics and, when nothing was written, produces the error body.
|
|
func (r *Router) finish(w http.ResponseWriter, req *parsedRequest, res dispatchResult, start time.Time) {
|
|
status := res.outcome.Status
|
|
switch {
|
|
case res.err == nil:
|
|
if res.fallback {
|
|
// The client asked for one model and got another; say so rather
|
|
// than letting them discover it from the response contents.
|
|
w.Header().Set("x-llmgw-served-model", res.model)
|
|
}
|
|
case errors.Is(res.err, proxy.ErrClientGone):
|
|
// Nothing to report to a client that is gone; the upstream was cancelled.
|
|
status = 499
|
|
r.log.Debug("client disconnected", "model", req.Model, "endpoint", res.endpointID)
|
|
case res.outcome.Wrote:
|
|
// The response is already partly out; the stream just ends short.
|
|
r.log.Warn("stream ended early", "model", req.Model, "endpoint", res.endpointID, "err", res.err)
|
|
if status == 0 {
|
|
status = http.StatusOK
|
|
}
|
|
case errors.Is(res.err, errNoCapacity):
|
|
status = http.StatusServiceUnavailable
|
|
writeError(w, status, "server_error", res.err.Error())
|
|
default:
|
|
status = http.StatusBadGateway
|
|
writeError(w, status, "server_error", res.err.Error())
|
|
}
|
|
|
|
r.m.observeRequest(req.Model, res.pool, status, time.Since(start), res.outcome.Usage, res.outcome.TTFT)
|
|
r.account(req, res, status, time.Since(start))
|
|
}
|
|
|
|
// account settles a finished request against its key.
|
|
//
|
|
// It runs for every outcome, not only success: a request that used a GPU and
|
|
// then failed still used it, and a key that could spend nothing by failing
|
|
// would be a way around a budget.
|
|
func (r *Router) account(req *parsedRequest, res dispatchResult, status int, took time.Duration) {
|
|
if r.acct == nil || req.KeyID == "" {
|
|
return
|
|
}
|
|
u := store.Usage{
|
|
At: time.Now(), KeyID: req.KeyID, ProjectID: req.ProjectID,
|
|
Model: req.Model, ServedModel: res.model, Status: status,
|
|
LatencyMs: took.Milliseconds(),
|
|
}
|
|
if u.ServedModel == "" {
|
|
u.ServedModel = req.Model
|
|
}
|
|
if usage := res.outcome.Usage; usage != nil {
|
|
u.PromptTokens = int64(usage.PromptTokens)
|
|
u.CompletionTokens = int64(usage.CompletionTokens)
|
|
} else {
|
|
// No usage came back — a failure before the engine answered, or an
|
|
// engine that does not report it. Charging the estimate is closer to
|
|
// the truth than charging nothing.
|
|
u.PromptTokens = int64(req.PromptTokens)
|
|
}
|
|
r.acct.Record(req.reservation, u)
|
|
}
|
|
|
|
// rejectLimit turns a quota refusal into a status the client can act on.
|
|
func (r *Router) rejectLimit(w http.ResponseWriter, req *parsedRequest, err error, start time.Time) {
|
|
var le *accounting.LimitError
|
|
if !errors.As(err, &le) {
|
|
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
|
|
return
|
|
}
|
|
if le.RetryAfter > 0 {
|
|
w.Header().Set("Retry-After", strconv.Itoa(int(le.RetryAfter.Seconds())))
|
|
}
|
|
// 429 for all of them: a spent budget is a quota, and OpenAI clients
|
|
// already know how to surface that code. What differs is Retry-After,
|
|
// which is absent when waiting does not help.
|
|
writeError(w, http.StatusTooManyRequests, "rate_limit_exceeded", le.Error())
|
|
r.m.observeRequest(req.Model, "", http.StatusTooManyRequests, time.Since(start), nil, 0)
|
|
}
|
|
|
|
// parse reads the body and extracts the routing-relevant fields.
|
|
func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) {
|
|
body, err := io.ReadAll(io.LimitReader(req.Body, r.cfg.MaxBodyBytes))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var b inferenceBody
|
|
if err := json.Unmarshal(body, &b); err != nil {
|
|
return nil, err
|
|
}
|
|
if b.Model == "" {
|
|
return nil, errors.New("model is required")
|
|
}
|
|
|
|
p := &parsedRequest{
|
|
Path: path,
|
|
Model: b.Model,
|
|
Stream: b.Stream,
|
|
UpstreamBody: body,
|
|
Header: req.Header,
|
|
SessionID: sessionID(req, r.cfg.SessionHeader, b.User),
|
|
Band: req.Header.Get(r.cfg.Admission.BandHeader),
|
|
ProfileOverride: req.Header.Get("x-llmgw-profile"),
|
|
MaxTokens: max(b.MaxTokens, b.MaxCompletionTokens),
|
|
PromptPrefix: promptPrefix(&b),
|
|
}
|
|
// With authentication on, the key decides the band and the routing profile,
|
|
// and the headers stop being believed. Trusting the header was only ever
|
|
// acceptable while there was no identity to consult — it let any client
|
|
// promote itself into the top priority band.
|
|
if principal, ok := auth.FromContext(req.Context()); ok {
|
|
p.Band = principal.Band
|
|
p.ProfileOverride = principal.RoutingProfile
|
|
p.KeyID = principal.KeyID
|
|
p.ProjectID = principal.ProjectID
|
|
// Tenant fairness within a band is per key rather than per session, so
|
|
// one key cannot take a whole band by opening many sessions.
|
|
p.Tenant = principal.KeyID
|
|
}
|
|
|
|
// Bytes over four: a crude stand-in for a tokenizer, good enough for
|
|
// scoring. It becomes exact only with plan §7.5 tier 2 (see plan §14).
|
|
p.PromptTokens = len(p.PromptPrefix) / 4
|
|
|
|
// Ask the engine for usage the client did not request, so accounting works
|
|
// regardless of client behaviour; the extra chunk is dropped on the way back.
|
|
if r.cfg.InjectUsage && b.Stream && (b.StreamOptions == nil || !b.StreamOptions.IncludeUsage) {
|
|
if injected, ok := injectIncludeUsage(body); ok {
|
|
p.UpstreamBody = injected
|
|
p.SuppressUsageChunk = true
|
|
}
|
|
}
|
|
return p, nil
|
|
}
|
|
|
|
// sessionID picks the affinity key: an explicit header first, then OpenAI's
|
|
// user field, which most clients already set per end user.
|
|
func sessionID(req *http.Request, header, user string) string {
|
|
if header != "" {
|
|
if v := req.Header.Get(header); v != "" {
|
|
return v
|
|
}
|
|
}
|
|
return user
|
|
}
|
|
|
|
// promptPrefix reconstructs the leading portion of the prompt for prefix-aware
|
|
// routing. Only the first prefixBytes matter, so long conversations stop early.
|
|
func promptPrefix(b *inferenceBody) string {
|
|
var sb strings.Builder
|
|
appendRaw := func(raw json.RawMessage) {
|
|
if len(raw) == 0 || sb.Len() >= prefixBytes {
|
|
return
|
|
}
|
|
var s string
|
|
if err := json.Unmarshal(raw, &s); err != nil {
|
|
s = string(raw)
|
|
}
|
|
sb.WriteString(s)
|
|
}
|
|
|
|
for _, m := range b.Messages {
|
|
if sb.Len() >= prefixBytes {
|
|
break
|
|
}
|
|
sb.WriteString(m.Role)
|
|
sb.WriteString(":")
|
|
appendRaw(m.Content)
|
|
sb.WriteString("\n")
|
|
}
|
|
appendRaw(b.Prompt)
|
|
appendRaw(b.Input)
|
|
|
|
s := sb.String()
|
|
if len(s) > prefixBytes {
|
|
s = s[:prefixBytes]
|
|
}
|
|
return s
|
|
}
|
|
|
|
// injectIncludeUsage sets stream_options.include_usage on the outgoing body.
|
|
//
|
|
// This is the one place the router rewrites a request, so it is done by
|
|
// decoding to a generic map and re-encoding: field order changes, but no field
|
|
// is lost. If anything looks unexpected it declines rather than guessing.
|
|
func injectIncludeUsage(body []byte) ([]byte, bool) {
|
|
var m map[string]json.RawMessage
|
|
if err := json.Unmarshal(body, &m); err != nil {
|
|
return nil, false
|
|
}
|
|
|
|
opts := map[string]any{}
|
|
if raw, ok := m["stream_options"]; ok {
|
|
if err := json.Unmarshal(raw, &opts); err != nil {
|
|
return nil, false
|
|
}
|
|
}
|
|
opts["include_usage"] = true
|
|
|
|
encoded, err := json.Marshal(opts)
|
|
if err != nil {
|
|
return nil, false
|
|
}
|
|
m["stream_options"] = encoded
|
|
|
|
out, err := json.Marshal(m)
|
|
if err != nil {
|
|
return nil, false
|
|
}
|
|
return out, true
|
|
}
|
|
|
|
// servedModels lists the canonical models at least one endpoint serves.
|
|
func (r *Router) servedModels() []string {
|
|
seen := map[string]struct{}{}
|
|
for _, s := range r.reg.Snapshots("") {
|
|
for _, m := range s.Models {
|
|
seen[m] = struct{}{}
|
|
}
|
|
}
|
|
out := make([]string, 0, len(seen))
|
|
for m := range seen {
|
|
out = append(out, m)
|
|
}
|
|
slices.Sort(out)
|
|
return out
|
|
}
|
|
|
|
// handleModels lists the models that at least one endpoint currently serves.
|
|
func (r *Router) handleModels(w http.ResponseWriter, _ *http.Request) {
|
|
seen := map[string]struct{}{}
|
|
for _, s := range r.reg.Snapshots("") {
|
|
for _, m := range s.Models {
|
|
seen[m] = struct{}{}
|
|
}
|
|
}
|
|
// Catalog aliases are usable names too, so clients should see them.
|
|
for name := range r.models {
|
|
if _, ok := seen[r.models[name].Name]; ok {
|
|
seen[name] = struct{}{}
|
|
}
|
|
}
|
|
|
|
names := make([]string, 0, len(seen))
|
|
for n := range seen {
|
|
names = append(names, n)
|
|
}
|
|
slices.Sort(names)
|
|
|
|
data := make([]map[string]any, 0, len(names))
|
|
for _, n := range names {
|
|
data = append(data, map[string]any{
|
|
"id": n, "object": "model", "created": time.Now().Unix(), "owned_by": "llmgw",
|
|
})
|
|
}
|
|
writeJSON(w, http.StatusOK, map[string]any{"object": "list", "data": data})
|
|
}
|
|
|
|
func writeJSON(w http.ResponseWriter, code int, v any) {
|
|
var buf bytes.Buffer
|
|
if err := json.NewEncoder(&buf).Encode(v); err != nil {
|
|
http.Error(w, `{"error":{"message":"encoding failed"}}`, http.StatusInternalServerError)
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(code)
|
|
_, _ = w.Write(buf.Bytes())
|
|
}
|
|
|
|
// writeError emits an OpenAI-shaped error so clients need no special handling
|
|
// for gateway failures.
|
|
func writeError(w http.ResponseWriter, code int, kind, msg string) {
|
|
writeJSON(w, code, map[string]any{"error": map[string]any{
|
|
"message": msg, "type": kind, "code": nil,
|
|
}})
|
|
}
|
|
|
|
// estimatedTokens is what a token-rate limit is checked against, before the
|
|
// real figure exists.
|
|
//
|
|
// The prompt estimate plus whatever the client said it wanted back. A client
|
|
// that names no max_tokens gets a modest assumption rather than zero: assuming
|
|
// zero would let an unbounded request through every token limit.
|
|
func (p *parsedRequest) estimatedTokens() int64 {
|
|
const assumedCompletion = 256
|
|
completion := p.MaxTokens
|
|
if completion <= 0 {
|
|
completion = assumedCompletion
|
|
}
|
|
return int64(p.PromptTokens + completion)
|
|
}
|
|
|
|
// handleMe reports what the presented key is and what it may do.
|
|
//
|
|
// It exists so a user can answer "how much have I used" and "why was I
|
|
// refused" without asking an operator to read a dashboard they cannot reach.
|
|
// Deliberately outside /v1: it is not part of the OpenAI API, and putting it
|
|
// there would imply it is.
|
|
func (r *Router) handleMe(w http.ResponseWriter, req *http.Request) {
|
|
p, ok := auth.FromContext(req.Context())
|
|
if !ok {
|
|
// With authentication off there is no identity to describe, and saying
|
|
// so is more useful than an empty object.
|
|
writeJSON(w, http.StatusOK, map[string]any{
|
|
"authenticated": false,
|
|
"note": "this gateway has no api keys configured",
|
|
})
|
|
return
|
|
}
|
|
|
|
out := map[string]any{
|
|
"authenticated": true,
|
|
"key_id": p.KeyID,
|
|
"key": p.KeyPrefix,
|
|
"project": p.ProjectID,
|
|
"team": p.TeamID,
|
|
"organization": p.OrgID,
|
|
}
|
|
if p.Band != "" {
|
|
out["band"] = p.Band
|
|
}
|
|
limits := map[string]any{}
|
|
if p.Limits.RPM != nil {
|
|
limits["requests_per_minute"] = *p.Limits.RPM
|
|
}
|
|
if p.Limits.TPM != nil {
|
|
limits["tokens_per_minute"] = *p.Limits.TPM
|
|
}
|
|
if p.Limits.AllowedModels != nil {
|
|
limits["allowed_models"] = p.Limits.AllowedModels
|
|
}
|
|
if p.Limits.Budget != nil {
|
|
limits["budget_units"] = float64(*p.Limits.Budget) / 1e6
|
|
limits["budget_period"] = string(p.Limits.BudgetPeriod)
|
|
}
|
|
out["limits"] = limits
|
|
|
|
if r.acct != nil {
|
|
spent, rpm, tpm := r.acct.Status(p)
|
|
usage := map[string]any{
|
|
"requests_per_minute": rpm,
|
|
"tokens_per_minute": tpm,
|
|
}
|
|
if p.Limits.Budget != nil {
|
|
usage["spent_units"] = float64(spent) / 1e6
|
|
usage["remaining_units"] = float64(*p.Limits.Budget-spent) / 1e6
|
|
}
|
|
out["usage"] = usage
|
|
}
|
|
writeJSON(w, http.StatusOK, out)
|
|
}
|