Serve a spent budget something it can afford
The accounting layer has always exempted a model that costs nothing, with a comment saying it is "what lets a key with no budget left still be served a cheap fallback". That described a caller nobody had written: the budget was checked once, against the requested model, and a refusal was the end of it, so the exemption could never fire. A caller that has run out is now offered the first model in its fallbacks it can still afford. The allowlist still applies — running out of money is not a way to reach a model a key was never allowed — and a model with no fallback is refused exactly as before, or a budget would stop meaning anything. Only a spent budget falls back: a rate limit is answered by waiting a second, and moving the request elsewhere would spend someone else's capacity to avoid a delay nobody asked to avoid. Which uncovered a header that has never worked. x-llmgw-served-model was set in finish, and finish runs after the proxy has written the response — so the one mechanism for telling a client that something else answered has been announcing it to nobody since it was written. Nothing failed, because the only thing asserting it was a comment. It is now set before dispatch, for the capacity fallback as well as this one. Finding that took bisecting a test I had written wrong: it polled for a 429 that never came, because the budget had run out and the fallback had served the request — the feature working, reported as the feature failing. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
c232f41a66
commit
28967edab3
5 files changed
+240
-6
No files matched your search
@@ -0,0 +1,55 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
||||
)
|
||||
|
||||
// budgetFallback finds a model this caller can still afford.
|
||||
//
|
||||
// A spent budget was a dead end: the check refused the request and that was
|
||||
// the end of it. The accounting layer has always exempted a model that costs
|
||||
// nothing, with a comment saying it is "what lets a key with no budget left
|
||||
// still be served a cheap fallback" — describing a caller that was never
|
||||
// written, so the exemption could not fire.
|
||||
//
|
||||
// It matters most for the audience this is built for. A team trying models
|
||||
// will exhaust a quota, and being dropped to a smaller model is a far better
|
||||
// answer than a refusal: it makes a budget a guardrail rather than a wall.
|
||||
//
|
||||
// Silent, with the served model in a header, which is what capacity fallback
|
||||
// already does. Two rules for the same situation would be worse than one
|
||||
// imperfect rule, and a client that cares can read the header.
|
||||
func (r *Router) budgetFallback(p auth.Principal, req *parsedRequest) (accounting.Reservation, bool) {
|
||||
_, entry := r.resolve(req.Model)
|
||||
for _, name := range entry.Fallbacks {
|
||||
// The allowlist still applies. A key that may not use a model does not
|
||||
// gain access to it by running out of money, which would make a budget
|
||||
// a way around an allowlist rather than a limit on top of one.
|
||||
if !r.permits(p, name) {
|
||||
continue
|
||||
}
|
||||
res, err := r.acct.Check(p, name, req.estimatedTokens())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Model = name
|
||||
return res, true
|
||||
}
|
||||
return accounting.Reservation{}, false
|
||||
}
|
||||
|
||||
// isBudgetSpent reports whether a limit error is the one a fallback can answer.
|
||||
//
|
||||
// Only a spent budget. A rate limit is answered by waiting a second, and moving
|
||||
// the request to another model would spend someone else's capacity to avoid a
|
||||
// delay they did not ask to avoid.
|
||||
func isBudgetSpent(err error) bool {
|
||||
var le *accounting.LimitError
|
||||
if !errors.As(err, &le) {
|
||||
return false
|
||||
}
|
||||
return errors.Is(le.Err, accounting.ErrBudgetSpent)
|
||||
}
|
||||
@@ -140,8 +140,13 @@ func (r *Router) Handler() http.Handler {
|
||||
// body is forwarded untouched — clients send engine-specific fields we have no
|
||||
// business interpreting.
|
||||
type parsedRequest struct {
|
||||
Path string
|
||||
Path string
|
||||
// Model is what will be dispatched, which is not always what was asked
|
||||
// for: a caller out of budget may be moved to a model they can afford.
|
||||
Model string
|
||||
// RequestedModel is what the client sent, kept so the response can say
|
||||
// when something else answered.
|
||||
RequestedModel string
|
||||
|
||||
Stream bool
|
||||
SuppressUsageChunk bool
|
||||
@@ -217,6 +222,18 @@ func (r *Router) handleInference(path string) http.HandlerFunc {
|
||||
// it would have it occupy a band's capacity to no purpose.
|
||||
if r.acct != nil {
|
||||
reservation, err := r.acct.Check(p, parsed.Model, parsed.estimatedTokens())
|
||||
if err != nil && isBudgetSpent(err) {
|
||||
// Out of money, not out of options. A smaller model the
|
||||
// caller can still afford beats a refusal, and the catalog
|
||||
// already says which those are.
|
||||
if fb, ok := r.budgetFallback(p, parsed); ok {
|
||||
reservation, err = fb, nil
|
||||
// Said now rather than at the end: the proxy writes the
|
||||
// response itself, so a header set after dispatch never
|
||||
// reaches the client.
|
||||
w.Header().Set("x-llmgw-served-model", parsed.Model)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
r.rejectLimit(w, parsed, err, start)
|
||||
return
|
||||
@@ -269,11 +286,8 @@ func (r *Router) finish(w http.ResponseWriter, req *parsedRequest, res dispatchR
|
||||
status := res.outcome.Status
|
||||
switch {
|
||||
case res.err == nil:
|
||||
if res.fallback {
|
||||
// The client asked for one model and got another; say so rather
|
||||
// than letting them discover it from the response contents.
|
||||
w.Header().Set("x-llmgw-served-model", res.model)
|
||||
}
|
||||
// The served-model header is set before dispatch, not here: the proxy
|
||||
// writes the response, so by this point the headers are long gone.
|
||||
case errors.Is(res.err, proxy.ErrClientGone):
|
||||
// Nothing to report to a client that is gone; the upstream was cancelled.
|
||||
status = 499
|
||||
@@ -367,6 +381,7 @@ func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) {
|
||||
p := &parsedRequest{
|
||||
Path: path,
|
||||
Model: b.Model,
|
||||
RequestedModel: b.Model,
|
||||
Stream: b.Stream,
|
||||
UpstreamBody: body,
|
||||
Header: req.Header,
|
||||
|
||||
@@ -467,6 +467,14 @@ func (r *Router) dispatch(ctx context.Context, w http.ResponseWriter, req *parse
|
||||
|
||||
var last dispatchResult
|
||||
for i, model := range chain {
|
||||
if i > 0 {
|
||||
// Before dispatching, because dispatchOne writes the response and
|
||||
// a header set after that is a header nobody receives. It was set
|
||||
// in finish until now, which is after the proxy has already called
|
||||
// WriteHeader — so this has announced a fallback to nobody since it
|
||||
// was written.
|
||||
w.Header().Set("x-llmgw-served-model", model)
|
||||
}
|
||||
res := r.dispatchOne(ctx, w, req, model, entry)
|
||||
res.model = model
|
||||
res.fallback = i > 0
|
||||
|
||||
@@ -592,3 +592,25 @@ func TestWithoutAnActivatorNothingChanges(t *testing.T) {
|
||||
t.Errorf("body = %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheServedModelHeaderArrivesBeforeTheBody(t *testing.T) {
|
||||
// It used to be set in finish, which runs after the proxy has written the
|
||||
// response — so it announced a fallback to nobody. Nothing failed, because
|
||||
// the only thing asserting it was a comment.
|
||||
// Nothing serves the primary at all, which is the only condition that
|
||||
// justifies moving to another model — an upstream returning 500 is a
|
||||
// failure the fallback would share, and dispatch rightly declines to try.
|
||||
live := newFakeEngine(t, "backup")
|
||||
cfg := baseConfig(config.Endpoint{ID: "b", URL: live.srv.URL, Models: []string{"backup"}})
|
||||
m := catModel("primary", nil, "backup")
|
||||
cfg.Models = []catalog.Model{m, catModel("backup", nil)}
|
||||
rt := newTestRouter(t, cfg)
|
||||
|
||||
rec := post(t, rt.Handler(), map[string]any{"model": "primary", "messages": []any{}}, nil)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := rec.Header().Get("x-llmgw-served-model"); got != "backup" {
|
||||
t.Errorf("served-model header = %q, want backup", got)
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,16 @@
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
@@ -183,3 +187,133 @@ func TestLimitsAreNotAppliedWithoutAuthentication(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const cheapModel = "cheap-model"
|
||||
|
||||
// budgetFallbackConfig prices the main model and leaves the fallback free,
|
||||
// which is what makes it affordable to a key with nothing left.
|
||||
func budgetFallbackConfig(path string) func(*config.Server) {
|
||||
return func(c *config.Server) {
|
||||
paidConfig(path)(c)
|
||||
// The fallback has no price, so it costs nothing and the budget check
|
||||
// exempts it. That exemption has existed all along with a comment
|
||||
// saying it is what lets a spent key still be served something cheap;
|
||||
// nothing ever tried, so it could never fire.
|
||||
c.Models = budgetFallbackModels()
|
||||
// The same engines answer for both names, so this is about accounting
|
||||
// rather than about placement.
|
||||
for i := range c.Endpoints {
|
||||
c.Endpoints[i].Models = []string{testModel, cheapModel}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// callWithHeaders is call, keeping the response headers.
|
||||
func (h *harness) callWithHeaders(t *testing.T, key, model string) (int, http.Header) {
|
||||
t.Helper()
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"model": model,
|
||||
"messages": []map[string]string{{"role": "user", "content": "hi"}},
|
||||
"max_tokens": 4,
|
||||
})
|
||||
req, err := http.NewRequestWithContext(context.Background(), http.MethodPost,
|
||||
h.router.URL+"/v1/chat/completions", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+key)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
return resp.StatusCode, resp.Header
|
||||
}
|
||||
|
||||
func TestASpentBudgetIsServedAnAffordableModel(t *testing.T) {
|
||||
// A team trying models will exhaust a quota, and being dropped to a smaller
|
||||
// model is a far better answer than a refusal: it makes a budget a
|
||||
// guardrail rather than a wall.
|
||||
path, mint := provision(t)
|
||||
key := mint(t, "broke")
|
||||
withStore(t, path, func(s *store.SQLite) {
|
||||
budget := int64(1) // one micro-unit: the first request exhausts it
|
||||
if err := s.PutLimits(context.Background(), store.Limits{
|
||||
Scope: store.ScopeKey, ScopeID: "key-broke",
|
||||
Budget: &budget, BudgetPeriod: store.PeriodMonth,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
|
||||
h := start(t, 2, budgetFallbackConfig(path))
|
||||
|
||||
if code, msg := h.call(t, key, testModel); code != http.StatusOK {
|
||||
t.Fatalf("the first request should go through: %d %s", code, msg)
|
||||
}
|
||||
|
||||
// Once the spend lands, the priced model is unaffordable and the free one
|
||||
// answers instead — with the response saying which, rather than leaving
|
||||
// the caller to infer it from the reply.
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
code, hdr := h.callWithHeaders(t, key, testModel)
|
||||
if code == http.StatusOK && hdr.Get("x-llmgw-served-model") == cheapModel {
|
||||
return
|
||||
}
|
||||
if code == http.StatusTooManyRequests {
|
||||
t.Fatal("a spent budget refused the request instead of falling back")
|
||||
}
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("the budget never ran out, or the fallback never served")
|
||||
}
|
||||
|
||||
func TestASpentBudgetWithNoFallbackStillRefuses(t *testing.T) {
|
||||
// The limit has to remain a limit. A model with nowhere cheaper to go is
|
||||
// refused exactly as before, or a budget would stop meaning anything.
|
||||
path, mint := provision(t)
|
||||
key := mint(t, "broke")
|
||||
withStore(t, path, func(s *store.SQLite) {
|
||||
budget := int64(1)
|
||||
if err := s.PutLimits(context.Background(), store.Limits{
|
||||
Scope: store.ScopeKey, ScopeID: "key-broke",
|
||||
Budget: &budget, BudgetPeriod: store.PeriodMonth,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
|
||||
h := start(t, 2, paidConfig(path)) // no catalog, so no fallbacks
|
||||
|
||||
if code, msg := h.call(t, key, testModel); code != http.StatusOK {
|
||||
t.Fatalf("the first request should go through: %d %s", code, msg)
|
||||
}
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if code, _ := h.call(t, key, testModel); code == http.StatusTooManyRequests {
|
||||
return
|
||||
}
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("a key with no fallback kept spending past its budget")
|
||||
}
|
||||
|
||||
// budgetFallbackModels is a priced model with a free one behind it.
|
||||
func budgetFallbackModels() []catalog.Model {
|
||||
return []catalog.Model{
|
||||
{
|
||||
Name: testModel, Source: "/models/test",
|
||||
Fallbacks: []string{cheapModel},
|
||||
Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||
Profiles: []catalog.Profile{{Name: "p", Engine: "llamacpp", Weight: 1}},
|
||||
},
|
||||
{
|
||||
Name: cheapModel, Source: "/models/cheap",
|
||||
Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||
Profiles: []catalog.Profile{{Name: "p", Engine: "llamacpp", Weight: 1}},
|
||||
},
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user