Tell clients how much of their rate limit is left
Release / build (push) Successful in 1m4s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m49s

SDKs and editors pace themselves from OpenAI's x-ratelimit-* headers, and
the gateway sent none: only Retry-After, and only on the refusal they would
have avoided. A limit can be set at several levels of the tenancy tree, each
counted in its own pool, so the headers report the one with least left —
the one that will refuse first — after this request's own charge, on
refusals and successes alike. The windows slide, so the reset figure is the
next minute boundary, when a whole window has moved on. A limit that is not
set is not advertised.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 02:42:21 +02:00
1 parent 4a652fdcc1
commit 3bf4b7b709
4 files changed
+114

No files matched your search

+25
View File
@@ -77,6 +77,31 @@ more than 26 hours old or there has never been one. docs/install.md now says how
to restore, including why running engines survive it: the engine-key secret
comes back with the store.
### Following a request
Every response carries `X-Request-Id` — the client's own when it sends a plain
one, otherwise one the gateway makes — and the engine receives it too. One
access log line per request records it with the model asked for and served,
the replica, status, time queued, time to the first token, total time,
tokens, and the key, project and user. Usage records keep the id, replica,
queue time and time to first token, and `llmgw ctl request ID` reads one back:
the answer to "why was this slow?".
### Rate-limit headers
Responses carry OpenAI's `x-ratelimit-limit-*`, `x-ratelimit-remaining-*` and
`x-ratelimit-reset-*` for requests and tokens, from whichever level of the
tenancy tree will refuse first, so SDKs and editors can slow down before they
are refused.
### Memory sized from the model
The mirror's serve form sizes a llama.cpp model's memory from its GGUF header:
weights, plus the KV cache for the context pool from its layers, KV heads and
head width, plus the engine's own. At a 16k pool that cache ranges from 0.44 GB
to 5.25 GB across the models on the live mirror, which the old "weights plus
half again plus two" could not see.
### Security
- **Alertmanager answered anyone on the network**, including requests to
+36
View File
@@ -383,3 +383,39 @@ func keyCounter(keyID string) string {
// they are asking is whether the budgets they are about to display mean
// anything, and the accountant is what would enforce them.
func (a *Accountant) AnythingPriced() bool { return a.pricing.AnythingPriced() }
// Headroom is the tightest of a principal's request and token limits: how
// much is left of each, and when the minute it is counted over rolls on.
type Headroom struct {
RequestLimit, RequestsLeft int
TokenLimit, TokensLeft int
Reset time.Duration
}
// Headroom reports what a principal has left, after whatever Check just
// charged. Of several levels the one with least left is the one that will
// refuse first, so that is the one reported. A limit of 0 means none is set.
func (a *Accountant) Headroom(p auth.Principal) Headroom {
var h Headroom
firstReq, firstTok := true, true
for _, l := range levels(p.Limits, p.KeyID) {
if l.RPM != nil && *l.RPM > 0 {
left := max(0, *l.RPM-int(a.requests.Rate(l.Counter)))
if firstReq || left < h.RequestsLeft {
h.RequestLimit, h.RequestsLeft, firstReq = *l.RPM, left, false
}
}
if l.TPM != nil && *l.TPM > 0 {
left := max(0, *l.TPM-int(a.tokens.Rate(l.Counter)))
if firstTok || left < h.TokensLeft {
h.TokenLimit, h.TokensLeft, firstTok = *l.TPM, left, false
}
}
}
// The windows slide, so room comes back continuously; the next minute
// boundary is when a whole window's worth has moved on, and is the figure
// a client pacing itself can act on.
now := time.Now()
h.Reset = now.Truncate(windowLen).Add(windowLen).Sub(now)
return h
}
+28
View File
@@ -630,3 +630,31 @@ func TestAnEmptyPriceTableMeansNoBudgetCanEverBind(t *testing.T) {
t.Error("no price table at all is not priced")
}
}
func TestHeadroomIsTheTightestLimitAfterThisRequest(t *testing.T) {
// A project allowing 100 a minute under a team allowing 3: the team is
// what will refuse, so the team is what a client is told about.
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
p := principal("k1", func(e *store.Effective) {
e.Chain = []store.Limits{
{Scope: store.ScopeProject, ScopeID: "proj", RPM: intp(100), TPM: intp(50000)},
{Scope: store.ScopeTeam, ScopeID: "team", RPM: intp(3)},
}
})
if _, err := a.Check(p, "m", 10); err != nil {
t.Fatal(err)
}
h := a.Headroom(p)
if h.RequestLimit != 3 || h.RequestsLeft != 2 {
t.Errorf("requests %d left of %d; expected 2 of the team's 3", h.RequestsLeft, h.RequestLimit)
}
if h.TokenLimit != 50000 || h.TokensLeft != 50000-10 {
t.Errorf("tokens %d left of %d", h.TokensLeft, h.TokenLimit)
}
if h.Reset <= 0 || h.Reset > time.Minute {
t.Errorf("reset %v", h.Reset)
}
if none := a.Headroom(principal("k2", nil)); none.RequestLimit != 0 || none.TokenLimit != 0 {
t.Errorf("a key with no limits advertised some: %+v", none)
}
}
+25
View File
@@ -343,6 +343,9 @@ func (r *Router) handleInference(path string) http.HandlerFunc {
w.Header().Set("x-llmgw-served-model", parsed.Model)
}
}
// On every answer, refused or not, so a client can pace itself
// before it is refused rather than learn the limit from a 429.
r.rateLimitHeaders(w, p)
if err != nil {
r.rejectLimit(w, parsed, err, start)
return
@@ -931,3 +934,25 @@ func (r *Router) accessLog(req *parsedRequest, res dispatchResult, status int, t
}
r.log.Info("request", attrs...)
}
// rateLimitHeaders sets OpenAI's x-ratelimit-* headers from the tightest
// limit that applies. SDKs and editors read these to slow down before they
// are refused; only Retry-After, and only on a refusal, was sent before. A
// limit that is not set is not advertised.
func (r *Router) rateLimitHeaders(w http.ResponseWriter, p auth.Principal) {
if r.acct == nil {
return
}
h := r.acct.Headroom(p)
reset := strconv.FormatFloat(h.Reset.Seconds(), 'f', 0, 64) + "s"
if h.RequestLimit > 0 {
w.Header().Set("x-ratelimit-limit-requests", strconv.Itoa(h.RequestLimit))
w.Header().Set("x-ratelimit-remaining-requests", strconv.Itoa(h.RequestsLeft))
w.Header().Set("x-ratelimit-reset-requests", reset)
}
if h.TokenLimit > 0 {
w.Header().Set("x-ratelimit-limit-tokens", strconv.Itoa(h.TokenLimit))
w.Header().Set("x-ratelimit-remaining-tokens", strconv.Itoa(h.TokensLeft))
w.Header().Set("x-ratelimit-reset-tokens", reset)
}
}