From 3bf4b7b709557c6bbf1126e0024dfb23b22d4997 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 30 Sep 2026 02:42:21 +0200 Subject: [PATCH] Tell clients how much of their rate limit is left MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SDKs and editors pace themselves from OpenAI's x-ratelimit-* headers, and the gateway sent none: only Retry-After, and only on the refusal they would have avoided. A limit can be set at several levels of the tenancy tree, each counted in its own pool, so the headers report the one with least left — the one that will refuse first — after this request's own charge, on refusals and successes alike. The windows slide, so the reset figure is the next minute boundary, when a whole window has moved on. A limit that is not set is not advertised. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 25 ++++++++++++++++++ internal/accounting/accounting.go | 36 ++++++++++++++++++++++++++ internal/accounting/accounting_test.go | 28 ++++++++++++++++++++ internal/router/handlers.go | 25 ++++++++++++++++++ 4 files changed, 114 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f33ec429..9344dec1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,6 +77,31 @@ more than 26 hours old or there has never been one. docs/install.md now says how to restore, including why running engines survive it: the engine-key secret comes back with the store. +### Following a request + +Every response carries `X-Request-Id` — the client's own when it sends a plain +one, otherwise one the gateway makes — and the engine receives it too. One +access log line per request records it with the model asked for and served, +the replica, status, time queued, time to the first token, total time, +tokens, and the key, project and user. Usage records keep the id, replica, +queue time and time to first token, and `llmgw ctl request ID` reads one back: +the answer to "why was this slow?". + +### Rate-limit headers + +Responses carry OpenAI's `x-ratelimit-limit-*`, `x-ratelimit-remaining-*` and +`x-ratelimit-reset-*` for requests and tokens, from whichever level of the +tenancy tree will refuse first, so SDKs and editors can slow down before they +are refused. + +### Memory sized from the model + +The mirror's serve form sizes a llama.cpp model's memory from its GGUF header: +weights, plus the KV cache for the context pool from its layers, KV heads and +head width, plus the engine's own. At a 16k pool that cache ranges from 0.44 GB +to 5.25 GB across the models on the live mirror, which the old "weights plus +half again plus two" could not see. + ### Security - **Alertmanager answered anyone on the network**, including requests to diff --git a/internal/accounting/accounting.go b/internal/accounting/accounting.go index 40f3237a..66c42e3b 100644 --- a/internal/accounting/accounting.go +++ b/internal/accounting/accounting.go @@ -383,3 +383,39 @@ func keyCounter(keyID string) string { // they are asking is whether the budgets they are about to display mean // anything, and the accountant is what would enforce them. func (a *Accountant) AnythingPriced() bool { return a.pricing.AnythingPriced() } + +// Headroom is the tightest of a principal's request and token limits: how +// much is left of each, and when the minute it is counted over rolls on. +type Headroom struct { + RequestLimit, RequestsLeft int + TokenLimit, TokensLeft int + Reset time.Duration +} + +// Headroom reports what a principal has left, after whatever Check just +// charged. Of several levels the one with least left is the one that will +// refuse first, so that is the one reported. A limit of 0 means none is set. +func (a *Accountant) Headroom(p auth.Principal) Headroom { + var h Headroom + firstReq, firstTok := true, true + for _, l := range levels(p.Limits, p.KeyID) { + if l.RPM != nil && *l.RPM > 0 { + left := max(0, *l.RPM-int(a.requests.Rate(l.Counter))) + if firstReq || left < h.RequestsLeft { + h.RequestLimit, h.RequestsLeft, firstReq = *l.RPM, left, false + } + } + if l.TPM != nil && *l.TPM > 0 { + left := max(0, *l.TPM-int(a.tokens.Rate(l.Counter))) + if firstTok || left < h.TokensLeft { + h.TokenLimit, h.TokensLeft, firstTok = *l.TPM, left, false + } + } + } + // The windows slide, so room comes back continuously; the next minute + // boundary is when a whole window's worth has moved on, and is the figure + // a client pacing itself can act on. + now := time.Now() + h.Reset = now.Truncate(windowLen).Add(windowLen).Sub(now) + return h +} diff --git a/internal/accounting/accounting_test.go b/internal/accounting/accounting_test.go index 248b7123..b8561bf8 100644 --- a/internal/accounting/accounting_test.go +++ b/internal/accounting/accounting_test.go @@ -630,3 +630,31 @@ func TestAnEmptyPriceTableMeansNoBudgetCanEverBind(t *testing.T) { t.Error("no price table at all is not priced") } } + +func TestHeadroomIsTheTightestLimitAfterThisRequest(t *testing.T) { + // A project allowing 100 a minute under a team allowing 3: the team is + // what will refuse, so the team is what a client is told about. + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + p := principal("k1", func(e *store.Effective) { + e.Chain = []store.Limits{ + {Scope: store.ScopeProject, ScopeID: "proj", RPM: intp(100), TPM: intp(50000)}, + {Scope: store.ScopeTeam, ScopeID: "team", RPM: intp(3)}, + } + }) + if _, err := a.Check(p, "m", 10); err != nil { + t.Fatal(err) + } + h := a.Headroom(p) + if h.RequestLimit != 3 || h.RequestsLeft != 2 { + t.Errorf("requests %d left of %d; expected 2 of the team's 3", h.RequestsLeft, h.RequestLimit) + } + if h.TokenLimit != 50000 || h.TokensLeft != 50000-10 { + t.Errorf("tokens %d left of %d", h.TokensLeft, h.TokenLimit) + } + if h.Reset <= 0 || h.Reset > time.Minute { + t.Errorf("reset %v", h.Reset) + } + if none := a.Headroom(principal("k2", nil)); none.RequestLimit != 0 || none.TokenLimit != 0 { + t.Errorf("a key with no limits advertised some: %+v", none) + } +} diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 65965829..a1f4bca6 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -343,6 +343,9 @@ func (r *Router) handleInference(path string) http.HandlerFunc { w.Header().Set("x-llmgw-served-model", parsed.Model) } } + // On every answer, refused or not, so a client can pace itself + // before it is refused rather than learn the limit from a 429. + r.rateLimitHeaders(w, p) if err != nil { r.rejectLimit(w, parsed, err, start) return @@ -931,3 +934,25 @@ func (r *Router) accessLog(req *parsedRequest, res dispatchResult, status int, t } r.log.Info("request", attrs...) } + +// rateLimitHeaders sets OpenAI's x-ratelimit-* headers from the tightest +// limit that applies. SDKs and editors read these to slow down before they +// are refused; only Retry-After, and only on a refusal, was sent before. A +// limit that is not set is not advertised. +func (r *Router) rateLimitHeaders(w http.ResponseWriter, p auth.Principal) { + if r.acct == nil { + return + } + h := r.acct.Headroom(p) + reset := strconv.FormatFloat(h.Reset.Seconds(), 'f', 0, 64) + "s" + if h.RequestLimit > 0 { + w.Header().Set("x-ratelimit-limit-requests", strconv.Itoa(h.RequestLimit)) + w.Header().Set("x-ratelimit-remaining-requests", strconv.Itoa(h.RequestsLeft)) + w.Header().Set("x-ratelimit-reset-requests", reset) + } + if h.TokenLimit > 0 { + w.Header().Set("x-ratelimit-limit-tokens", strconv.Itoa(h.TokenLimit)) + w.Header().Set("x-ratelimit-remaining-tokens", strconv.Itoa(h.TokensLeft)) + w.Header().Set("x-ratelimit-reset-tokens", reset) + } +}