Tell clients how much of their rate limit is left
SDKs and editors pace themselves from OpenAI's x-ratelimit-* headers, and the gateway sent none: only Retry-After, and only on the refusal they would have avoided. A limit can be set at several levels of the tenancy tree, each counted in its own pool, so the headers report the one with least left — the one that will refuse first — after this request's own charge, on refusals and successes alike. The windows slide, so the reset figure is the next minute boundary, when a whole window has moved on. A limit that is not set is not advertised. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
4a652fdcc1
commit
3bf4b7b709
4 files changed
+114
No files matched your search
@@ -77,6 +77,31 @@ more than 26 hours old or there has never been one. docs/install.md now says how
|
||||
to restore, including why running engines survive it: the engine-key secret
|
||||
comes back with the store.
|
||||
|
||||
### Following a request
|
||||
|
||||
Every response carries `X-Request-Id` — the client's own when it sends a plain
|
||||
one, otherwise one the gateway makes — and the engine receives it too. One
|
||||
access log line per request records it with the model asked for and served,
|
||||
the replica, status, time queued, time to the first token, total time,
|
||||
tokens, and the key, project and user. Usage records keep the id, replica,
|
||||
queue time and time to first token, and `llmgw ctl request ID` reads one back:
|
||||
the answer to "why was this slow?".
|
||||
|
||||
### Rate-limit headers
|
||||
|
||||
Responses carry OpenAI's `x-ratelimit-limit-*`, `x-ratelimit-remaining-*` and
|
||||
`x-ratelimit-reset-*` for requests and tokens, from whichever level of the
|
||||
tenancy tree will refuse first, so SDKs and editors can slow down before they
|
||||
are refused.
|
||||
|
||||
### Memory sized from the model
|
||||
|
||||
The mirror's serve form sizes a llama.cpp model's memory from its GGUF header:
|
||||
weights, plus the KV cache for the context pool from its layers, KV heads and
|
||||
head width, plus the engine's own. At a 16k pool that cache ranges from 0.44 GB
|
||||
to 5.25 GB across the models on the live mirror, which the old "weights plus
|
||||
half again plus two" could not see.
|
||||
|
||||
### Security
|
||||
|
||||
- **Alertmanager answered anyone on the network**, including requests to
|
||||
|
||||
@@ -383,3 +383,39 @@ func keyCounter(keyID string) string {
|
||||
// they are asking is whether the budgets they are about to display mean
|
||||
// anything, and the accountant is what would enforce them.
|
||||
func (a *Accountant) AnythingPriced() bool { return a.pricing.AnythingPriced() }
|
||||
|
||||
// Headroom is the tightest of a principal's request and token limits: how
|
||||
// much is left of each, and when the minute it is counted over rolls on.
|
||||
type Headroom struct {
|
||||
RequestLimit, RequestsLeft int
|
||||
TokenLimit, TokensLeft int
|
||||
Reset time.Duration
|
||||
}
|
||||
|
||||
// Headroom reports what a principal has left, after whatever Check just
|
||||
// charged. Of several levels the one with least left is the one that will
|
||||
// refuse first, so that is the one reported. A limit of 0 means none is set.
|
||||
func (a *Accountant) Headroom(p auth.Principal) Headroom {
|
||||
var h Headroom
|
||||
firstReq, firstTok := true, true
|
||||
for _, l := range levels(p.Limits, p.KeyID) {
|
||||
if l.RPM != nil && *l.RPM > 0 {
|
||||
left := max(0, *l.RPM-int(a.requests.Rate(l.Counter)))
|
||||
if firstReq || left < h.RequestsLeft {
|
||||
h.RequestLimit, h.RequestsLeft, firstReq = *l.RPM, left, false
|
||||
}
|
||||
}
|
||||
if l.TPM != nil && *l.TPM > 0 {
|
||||
left := max(0, *l.TPM-int(a.tokens.Rate(l.Counter)))
|
||||
if firstTok || left < h.TokensLeft {
|
||||
h.TokenLimit, h.TokensLeft, firstTok = *l.TPM, left, false
|
||||
}
|
||||
}
|
||||
}
|
||||
// The windows slide, so room comes back continuously; the next minute
|
||||
// boundary is when a whole window's worth has moved on, and is the figure
|
||||
// a client pacing itself can act on.
|
||||
now := time.Now()
|
||||
h.Reset = now.Truncate(windowLen).Add(windowLen).Sub(now)
|
||||
return h
|
||||
}
|
||||
@@ -630,3 +630,31 @@ func TestAnEmptyPriceTableMeansNoBudgetCanEverBind(t *testing.T) {
|
||||
t.Error("no price table at all is not priced")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeadroomIsTheTightestLimitAfterThisRequest(t *testing.T) {
|
||||
// A project allowing 100 a minute under a team allowing 3: the team is
|
||||
// what will refuse, so the team is what a client is told about.
|
||||
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
|
||||
p := principal("k1", func(e *store.Effective) {
|
||||
e.Chain = []store.Limits{
|
||||
{Scope: store.ScopeProject, ScopeID: "proj", RPM: intp(100), TPM: intp(50000)},
|
||||
{Scope: store.ScopeTeam, ScopeID: "team", RPM: intp(3)},
|
||||
}
|
||||
})
|
||||
if _, err := a.Check(p, "m", 10); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := a.Headroom(p)
|
||||
if h.RequestLimit != 3 || h.RequestsLeft != 2 {
|
||||
t.Errorf("requests %d left of %d; expected 2 of the team's 3", h.RequestsLeft, h.RequestLimit)
|
||||
}
|
||||
if h.TokenLimit != 50000 || h.TokensLeft != 50000-10 {
|
||||
t.Errorf("tokens %d left of %d", h.TokensLeft, h.TokenLimit)
|
||||
}
|
||||
if h.Reset <= 0 || h.Reset > time.Minute {
|
||||
t.Errorf("reset %v", h.Reset)
|
||||
}
|
||||
if none := a.Headroom(principal("k2", nil)); none.RequestLimit != 0 || none.TokenLimit != 0 {
|
||||
t.Errorf("a key with no limits advertised some: %+v", none)
|
||||
}
|
||||
}
|
||||
@@ -343,6 +343,9 @@ func (r *Router) handleInference(path string) http.HandlerFunc {
|
||||
w.Header().Set("x-llmgw-served-model", parsed.Model)
|
||||
}
|
||||
}
|
||||
// On every answer, refused or not, so a client can pace itself
|
||||
// before it is refused rather than learn the limit from a 429.
|
||||
r.rateLimitHeaders(w, p)
|
||||
if err != nil {
|
||||
r.rejectLimit(w, parsed, err, start)
|
||||
return
|
||||
@@ -931,3 +934,25 @@ func (r *Router) accessLog(req *parsedRequest, res dispatchResult, status int, t
|
||||
}
|
||||
r.log.Info("request", attrs...)
|
||||
}
|
||||
|
||||
// rateLimitHeaders sets OpenAI's x-ratelimit-* headers from the tightest
|
||||
// limit that applies. SDKs and editors read these to slow down before they
|
||||
// are refused; only Retry-After, and only on a refusal, was sent before. A
|
||||
// limit that is not set is not advertised.
|
||||
func (r *Router) rateLimitHeaders(w http.ResponseWriter, p auth.Principal) {
|
||||
if r.acct == nil {
|
||||
return
|
||||
}
|
||||
h := r.acct.Headroom(p)
|
||||
reset := strconv.FormatFloat(h.Reset.Seconds(), 'f', 0, 64) + "s"
|
||||
if h.RequestLimit > 0 {
|
||||
w.Header().Set("x-ratelimit-limit-requests", strconv.Itoa(h.RequestLimit))
|
||||
w.Header().Set("x-ratelimit-remaining-requests", strconv.Itoa(h.RequestsLeft))
|
||||
w.Header().Set("x-ratelimit-reset-requests", reset)
|
||||
}
|
||||
if h.TokenLimit > 0 {
|
||||
w.Header().Set("x-ratelimit-limit-tokens", strconv.Itoa(h.TokenLimit))
|
||||
w.Header().Set("x-ratelimit-remaining-tokens", strconv.Itoa(h.TokensLeft))
|
||||
w.Header().Set("x-ratelimit-reset-tokens", reset)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user