Serve a spent budget something it can afford

The accounting layer has always exempted a model that costs nothing, with a
comment saying it is "what lets a key with no budget left still be served a
cheap fallback". That described a caller nobody had written: the budget was
checked once, against the requested model, and a refusal was the end of it,
so the exemption could never fire.

A caller that has run out is now offered the first model in its fallbacks it
can still afford. The allowlist still applies — running out of money is not
a way to reach a model a key was never allowed — and a model with no
fallback is refused exactly as before, or a budget would stop meaning
anything. Only a spent budget falls back: a rate limit is answered by
waiting a second, and moving the request elsewhere would spend someone
else's capacity to avoid a delay nobody asked to avoid.

Which uncovered a header that has never worked. x-llmgw-served-model was set
in finish, and finish runs after the proxy has written the response — so the
one mechanism for telling a client that something else answered has been
announcing it to nobody since it was written. Nothing failed, because the
only thing asserting it was a comment. It is now set before dispatch, for
the capacity fallback as well as this one.

Finding that took bisecting a test I had written wrong: it polled for a 429
that never came, because the budget had run out and the fallback had served
the request — the feature working, reported as the feature failing.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-17 18:26:03 +02:00
1 parent c232f41a66
commit 28967edab3
5 files changed
+240 -6

No files matched your search

+55
View File
@@ -0,0 +1,55 @@
package router
import (
"errors"
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
)
// budgetFallback finds a model this caller can still afford.
//
// A spent budget was a dead end: the check refused the request and that was
// the end of it. The accounting layer has always exempted a model that costs
// nothing, with a comment saying it is "what lets a key with no budget left
// still be served a cheap fallback" — describing a caller that was never
// written, so the exemption could not fire.
//
// It matters most for the audience this is built for. A team trying models
// will exhaust a quota, and being dropped to a smaller model is a far better
// answer than a refusal: it makes a budget a guardrail rather than a wall.
//
// Silent, with the served model in a header, which is what capacity fallback
// already does. Two rules for the same situation would be worse than one
// imperfect rule, and a client that cares can read the header.
func (r *Router) budgetFallback(p auth.Principal, req *parsedRequest) (accounting.Reservation, bool) {
_, entry := r.resolve(req.Model)
for _, name := range entry.Fallbacks {
// The allowlist still applies. A key that may not use a model does not
// gain access to it by running out of money, which would make a budget
// a way around an allowlist rather than a limit on top of one.
if !r.permits(p, name) {
continue
}
res, err := r.acct.Check(p, name, req.estimatedTokens())
if err != nil {
continue
}
req.Model = name
return res, true
}
return accounting.Reservation{}, false
}
// isBudgetSpent reports whether a limit error is the one a fallback can answer.
//
// Only a spent budget. A rate limit is answered by waiting a second, and moving
// the request to another model would spend someone else's capacity to avoid a
// delay they did not ask to avoid.
func isBudgetSpent(err error) bool {
var le *accounting.LimitError
if !errors.As(err, &le) {
return false
}
return errors.Is(le.Err, accounting.ErrBudgetSpent)
}
+21 -6
View File
@@ -140,8 +140,13 @@ func (r *Router) Handler() http.Handler {
// body is forwarded untouched — clients send engine-specific fields we have no
// business interpreting.
type parsedRequest struct {
Path string
Path string
// Model is what will be dispatched, which is not always what was asked
// for: a caller out of budget may be moved to a model they can afford.
Model string
// RequestedModel is what the client sent, kept so the response can say
// when something else answered.
RequestedModel string
Stream bool
SuppressUsageChunk bool
@@ -217,6 +222,18 @@ func (r *Router) handleInference(path string) http.HandlerFunc {
// it would have it occupy a band's capacity to no purpose.
if r.acct != nil {
reservation, err := r.acct.Check(p, parsed.Model, parsed.estimatedTokens())
if err != nil && isBudgetSpent(err) {
// Out of money, not out of options. A smaller model the
// caller can still afford beats a refusal, and the catalog
// already says which those are.
if fb, ok := r.budgetFallback(p, parsed); ok {
reservation, err = fb, nil
// Said now rather than at the end: the proxy writes the
// response itself, so a header set after dispatch never
// reaches the client.
w.Header().Set("x-llmgw-served-model", parsed.Model)
}
}
if err != nil {
r.rejectLimit(w, parsed, err, start)
return
@@ -269,11 +286,8 @@ func (r *Router) finish(w http.ResponseWriter, req *parsedRequest, res dispatchR
status := res.outcome.Status
switch {
case res.err == nil:
if res.fallback {
// The client asked for one model and got another; say so rather
// than letting them discover it from the response contents.
w.Header().Set("x-llmgw-served-model", res.model)
}
// The served-model header is set before dispatch, not here: the proxy
// writes the response, so by this point the headers are long gone.
case errors.Is(res.err, proxy.ErrClientGone):
// Nothing to report to a client that is gone; the upstream was cancelled.
status = 499
@@ -367,6 +381,7 @@ func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) {
p := &parsedRequest{
Path: path,
Model: b.Model,
RequestedModel: b.Model,
Stream: b.Stream,
UpstreamBody: body,
Header: req.Header,
+8
View File
@@ -467,6 +467,14 @@ func (r *Router) dispatch(ctx context.Context, w http.ResponseWriter, req *parse
var last dispatchResult
for i, model := range chain {
if i > 0 {
// Before dispatching, because dispatchOne writes the response and
// a header set after that is a header nobody receives. It was set
// in finish until now, which is after the proxy has already called
// WriteHeader — so this has announced a fallback to nobody since it
// was written.
w.Header().Set("x-llmgw-served-model", model)
}
res := r.dispatchOne(ctx, w, req, model, entry)
res.model = model
res.fallback = i > 0
+22
View File
@@ -592,3 +592,25 @@ func TestWithoutAnActivatorNothingChanges(t *testing.T) {
t.Errorf("body = %s", rec.Body.String())
}
}
func TestTheServedModelHeaderArrivesBeforeTheBody(t *testing.T) {
// It used to be set in finish, which runs after the proxy has written the
// response — so it announced a fallback to nobody. Nothing failed, because
// the only thing asserting it was a comment.
// Nothing serves the primary at all, which is the only condition that
// justifies moving to another model — an upstream returning 500 is a
// failure the fallback would share, and dispatch rightly declines to try.
live := newFakeEngine(t, "backup")
cfg := baseConfig(config.Endpoint{ID: "b", URL: live.srv.URL, Models: []string{"backup"}})
m := catModel("primary", nil, "backup")
cfg.Models = []catalog.Model{m, catModel("backup", nil)}
rt := newTestRouter(t, cfg)
rec := post(t, rt.Handler(), map[string]any{"model": "primary", "messages": []any{}}, nil)
if rec.Code != http.StatusOK {
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
}
if got := rec.Header().Get("x-llmgw-served-model"); got != "backup" {
t.Errorf("served-model header = %q, want backup", got)
}
}
+134
View File
@@ -1,12 +1,16 @@
package e2e
import (
"bytes"
"context"
"encoding/json"
"io"
"net/http"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
@@ -183,3 +187,133 @@ func TestLimitsAreNotAppliedWithoutAuthentication(t *testing.T) {
}
}
}
const cheapModel = "cheap-model"
// budgetFallbackConfig prices the main model and leaves the fallback free,
// which is what makes it affordable to a key with nothing left.
func budgetFallbackConfig(path string) func(*config.Server) {
return func(c *config.Server) {
paidConfig(path)(c)
// The fallback has no price, so it costs nothing and the budget check
// exempts it. That exemption has existed all along with a comment
// saying it is what lets a spent key still be served something cheap;
// nothing ever tried, so it could never fire.
c.Models = budgetFallbackModels()
// The same engines answer for both names, so this is about accounting
// rather than about placement.
for i := range c.Endpoints {
c.Endpoints[i].Models = []string{testModel, cheapModel}
}
}
}
// callWithHeaders is call, keeping the response headers.
func (h *harness) callWithHeaders(t *testing.T, key, model string) (int, http.Header) {
t.Helper()
body, _ := json.Marshal(map[string]any{
"model": model,
"messages": []map[string]string{{"role": "user", "content": "hi"}},
"max_tokens": 4,
})
req, err := http.NewRequestWithContext(context.Background(), http.MethodPost,
h.router.URL+"/v1/chat/completions", bytes.NewReader(body))
if err != nil {
t.Fatal(err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+key)
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
_, _ = io.Copy(io.Discard, resp.Body)
return resp.StatusCode, resp.Header
}
func TestASpentBudgetIsServedAnAffordableModel(t *testing.T) {
// A team trying models will exhaust a quota, and being dropped to a smaller
// model is a far better answer than a refusal: it makes a budget a
// guardrail rather than a wall.
path, mint := provision(t)
key := mint(t, "broke")
withStore(t, path, func(s *store.SQLite) {
budget := int64(1) // one micro-unit: the first request exhausts it
if err := s.PutLimits(context.Background(), store.Limits{
Scope: store.ScopeKey, ScopeID: "key-broke",
Budget: &budget, BudgetPeriod: store.PeriodMonth,
}); err != nil {
t.Fatal(err)
}
})
h := start(t, 2, budgetFallbackConfig(path))
if code, msg := h.call(t, key, testModel); code != http.StatusOK {
t.Fatalf("the first request should go through: %d %s", code, msg)
}
// Once the spend lands, the priced model is unaffordable and the free one
// answers instead — with the response saying which, rather than leaving
// the caller to infer it from the reply.
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
code, hdr := h.callWithHeaders(t, key, testModel)
if code == http.StatusOK && hdr.Get("x-llmgw-served-model") == cheapModel {
return
}
if code == http.StatusTooManyRequests {
t.Fatal("a spent budget refused the request instead of falling back")
}
time.Sleep(25 * time.Millisecond)
}
t.Fatal("the budget never ran out, or the fallback never served")
}
func TestASpentBudgetWithNoFallbackStillRefuses(t *testing.T) {
// The limit has to remain a limit. A model with nowhere cheaper to go is
// refused exactly as before, or a budget would stop meaning anything.
path, mint := provision(t)
key := mint(t, "broke")
withStore(t, path, func(s *store.SQLite) {
budget := int64(1)
if err := s.PutLimits(context.Background(), store.Limits{
Scope: store.ScopeKey, ScopeID: "key-broke",
Budget: &budget, BudgetPeriod: store.PeriodMonth,
}); err != nil {
t.Fatal(err)
}
})
h := start(t, 2, paidConfig(path)) // no catalog, so no fallbacks
if code, msg := h.call(t, key, testModel); code != http.StatusOK {
t.Fatalf("the first request should go through: %d %s", code, msg)
}
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
if code, _ := h.call(t, key, testModel); code == http.StatusTooManyRequests {
return
}
time.Sleep(25 * time.Millisecond)
}
t.Fatal("a key with no fallback kept spending past its budget")
}
// budgetFallbackModels is a priced model with a free one behind it.
func budgetFallbackModels() []catalog.Model {
return []catalog.Model{
{
Name: testModel, Source: "/models/test",
Fallbacks: []string{cheapModel},
Scaling: catalog.Scaling{Min: 1, Max: 1},
Profiles: []catalog.Profile{{Name: "p", Engine: "llamacpp", Weight: 1}},
},
{
Name: cheapModel, Source: "/models/cheap",
Scaling: catalog.Scaling{Min: 1, Max: 1},
Profiles: []catalog.Profile{{Name: "p", Engine: "llamacpp", Weight: 1}},
},
}
}