diff --git a/internal/router/budget_fallback.go b/internal/router/budget_fallback.go new file mode 100644 index 00000000..f76fa743 --- /dev/null +++ b/internal/router/budget_fallback.go @@ -0,0 +1,55 @@ +package router + +import ( + "errors" + + "gitea.mynodes.xyz/ilker/llmgw/internal/accounting" + "gitea.mynodes.xyz/ilker/llmgw/internal/auth" +) + +// budgetFallback finds a model this caller can still afford. +// +// A spent budget was a dead end: the check refused the request and that was +// the end of it. The accounting layer has always exempted a model that costs +// nothing, with a comment saying it is "what lets a key with no budget left +// still be served a cheap fallback" — describing a caller that was never +// written, so the exemption could not fire. +// +// It matters most for the audience this is built for. A team trying models +// will exhaust a quota, and being dropped to a smaller model is a far better +// answer than a refusal: it makes a budget a guardrail rather than a wall. +// +// Silent, with the served model in a header, which is what capacity fallback +// already does. Two rules for the same situation would be worse than one +// imperfect rule, and a client that cares can read the header. +func (r *Router) budgetFallback(p auth.Principal, req *parsedRequest) (accounting.Reservation, bool) { + _, entry := r.resolve(req.Model) + for _, name := range entry.Fallbacks { + // The allowlist still applies. A key that may not use a model does not + // gain access to it by running out of money, which would make a budget + // a way around an allowlist rather than a limit on top of one. + if !r.permits(p, name) { + continue + } + res, err := r.acct.Check(p, name, req.estimatedTokens()) + if err != nil { + continue + } + req.Model = name + return res, true + } + return accounting.Reservation{}, false +} + +// isBudgetSpent reports whether a limit error is the one a fallback can answer. +// +// Only a spent budget. A rate limit is answered by waiting a second, and moving +// the request to another model would spend someone else's capacity to avoid a +// delay they did not ask to avoid. +func isBudgetSpent(err error) bool { + var le *accounting.LimitError + if !errors.As(err, &le) { + return false + } + return errors.Is(le.Err, accounting.ErrBudgetSpent) +} diff --git a/internal/router/handlers.go b/internal/router/handlers.go index c83b14f9..1a97f07f 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -140,8 +140,13 @@ func (r *Router) Handler() http.Handler { // body is forwarded untouched — clients send engine-specific fields we have no // business interpreting. type parsedRequest struct { - Path string + Path string + // Model is what will be dispatched, which is not always what was asked + // for: a caller out of budget may be moved to a model they can afford. Model string + // RequestedModel is what the client sent, kept so the response can say + // when something else answered. + RequestedModel string Stream bool SuppressUsageChunk bool @@ -217,6 +222,18 @@ func (r *Router) handleInference(path string) http.HandlerFunc { // it would have it occupy a band's capacity to no purpose. if r.acct != nil { reservation, err := r.acct.Check(p, parsed.Model, parsed.estimatedTokens()) + if err != nil && isBudgetSpent(err) { + // Out of money, not out of options. A smaller model the + // caller can still afford beats a refusal, and the catalog + // already says which those are. + if fb, ok := r.budgetFallback(p, parsed); ok { + reservation, err = fb, nil + // Said now rather than at the end: the proxy writes the + // response itself, so a header set after dispatch never + // reaches the client. + w.Header().Set("x-llmgw-served-model", parsed.Model) + } + } if err != nil { r.rejectLimit(w, parsed, err, start) return @@ -269,11 +286,8 @@ func (r *Router) finish(w http.ResponseWriter, req *parsedRequest, res dispatchR status := res.outcome.Status switch { case res.err == nil: - if res.fallback { - // The client asked for one model and got another; say so rather - // than letting them discover it from the response contents. - w.Header().Set("x-llmgw-served-model", res.model) - } + // The served-model header is set before dispatch, not here: the proxy + // writes the response, so by this point the headers are long gone. case errors.Is(res.err, proxy.ErrClientGone): // Nothing to report to a client that is gone; the upstream was cancelled. status = 499 @@ -367,6 +381,7 @@ func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) { p := &parsedRequest{ Path: path, Model: b.Model, + RequestedModel: b.Model, Stream: b.Stream, UpstreamBody: body, Header: req.Header, diff --git a/internal/router/router.go b/internal/router/router.go index 3b29da03..2b54c45a 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -467,6 +467,14 @@ func (r *Router) dispatch(ctx context.Context, w http.ResponseWriter, req *parse var last dispatchResult for i, model := range chain { + if i > 0 { + // Before dispatching, because dispatchOne writes the response and + // a header set after that is a header nobody receives. It was set + // in finish until now, which is after the proxy has already called + // WriteHeader — so this has announced a fallback to nobody since it + // was written. + w.Header().Set("x-llmgw-served-model", model) + } res := r.dispatchOne(ctx, w, req, model, entry) res.model = model res.fallback = i > 0 diff --git a/internal/router/router_test.go b/internal/router/router_test.go index 85401e0d..c85c9657 100644 --- a/internal/router/router_test.go +++ b/internal/router/router_test.go @@ -592,3 +592,25 @@ func TestWithoutAnActivatorNothingChanges(t *testing.T) { t.Errorf("body = %s", rec.Body.String()) } } + +func TestTheServedModelHeaderArrivesBeforeTheBody(t *testing.T) { + // It used to be set in finish, which runs after the proxy has written the + // response — so it announced a fallback to nobody. Nothing failed, because + // the only thing asserting it was a comment. + // Nothing serves the primary at all, which is the only condition that + // justifies moving to another model — an upstream returning 500 is a + // failure the fallback would share, and dispatch rightly declines to try. + live := newFakeEngine(t, "backup") + cfg := baseConfig(config.Endpoint{ID: "b", URL: live.srv.URL, Models: []string{"backup"}}) + m := catModel("primary", nil, "backup") + cfg.Models = []catalog.Model{m, catModel("backup", nil)} + rt := newTestRouter(t, cfg) + + rec := post(t, rt.Handler(), map[string]any{"model": "primary", "messages": []any{}}, nil) + if rec.Code != http.StatusOK { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + if got := rec.Header().Get("x-llmgw-served-model"); got != "backup" { + t.Errorf("served-model header = %q, want backup", got) + } +} diff --git a/test/e2e/accounting_test.go b/test/e2e/accounting_test.go index 3672da10..000149c4 100644 --- a/test/e2e/accounting_test.go +++ b/test/e2e/accounting_test.go @@ -1,12 +1,16 @@ package e2e import ( + "bytes" "context" + "encoding/json" + "io" "net/http" "testing" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) @@ -183,3 +187,133 @@ func TestLimitsAreNotAppliedWithoutAuthentication(t *testing.T) { } } } + +const cheapModel = "cheap-model" + +// budgetFallbackConfig prices the main model and leaves the fallback free, +// which is what makes it affordable to a key with nothing left. +func budgetFallbackConfig(path string) func(*config.Server) { + return func(c *config.Server) { + paidConfig(path)(c) + // The fallback has no price, so it costs nothing and the budget check + // exempts it. That exemption has existed all along with a comment + // saying it is what lets a spent key still be served something cheap; + // nothing ever tried, so it could never fire. + c.Models = budgetFallbackModels() + // The same engines answer for both names, so this is about accounting + // rather than about placement. + for i := range c.Endpoints { + c.Endpoints[i].Models = []string{testModel, cheapModel} + } + } +} + +// callWithHeaders is call, keeping the response headers. +func (h *harness) callWithHeaders(t *testing.T, key, model string) (int, http.Header) { + t.Helper() + body, _ := json.Marshal(map[string]any{ + "model": model, + "messages": []map[string]string{{"role": "user", "content": "hi"}}, + "max_tokens": 4, + }) + req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, + h.router.URL+"/v1/chat/completions", bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+key) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + return resp.StatusCode, resp.Header +} + +func TestASpentBudgetIsServedAnAffordableModel(t *testing.T) { + // A team trying models will exhaust a quota, and being dropped to a smaller + // model is a far better answer than a refusal: it makes a budget a + // guardrail rather than a wall. + path, mint := provision(t) + key := mint(t, "broke") + withStore(t, path, func(s *store.SQLite) { + budget := int64(1) // one micro-unit: the first request exhausts it + if err := s.PutLimits(context.Background(), store.Limits{ + Scope: store.ScopeKey, ScopeID: "key-broke", + Budget: &budget, BudgetPeriod: store.PeriodMonth, + }); err != nil { + t.Fatal(err) + } + }) + + h := start(t, 2, budgetFallbackConfig(path)) + + if code, msg := h.call(t, key, testModel); code != http.StatusOK { + t.Fatalf("the first request should go through: %d %s", code, msg) + } + + // Once the spend lands, the priced model is unaffordable and the free one + // answers instead — with the response saying which, rather than leaving + // the caller to infer it from the reply. + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + code, hdr := h.callWithHeaders(t, key, testModel) + if code == http.StatusOK && hdr.Get("x-llmgw-served-model") == cheapModel { + return + } + if code == http.StatusTooManyRequests { + t.Fatal("a spent budget refused the request instead of falling back") + } + time.Sleep(25 * time.Millisecond) + } + t.Fatal("the budget never ran out, or the fallback never served") +} + +func TestASpentBudgetWithNoFallbackStillRefuses(t *testing.T) { + // The limit has to remain a limit. A model with nowhere cheaper to go is + // refused exactly as before, or a budget would stop meaning anything. + path, mint := provision(t) + key := mint(t, "broke") + withStore(t, path, func(s *store.SQLite) { + budget := int64(1) + if err := s.PutLimits(context.Background(), store.Limits{ + Scope: store.ScopeKey, ScopeID: "key-broke", + Budget: &budget, BudgetPeriod: store.PeriodMonth, + }); err != nil { + t.Fatal(err) + } + }) + + h := start(t, 2, paidConfig(path)) // no catalog, so no fallbacks + + if code, msg := h.call(t, key, testModel); code != http.StatusOK { + t.Fatalf("the first request should go through: %d %s", code, msg) + } + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if code, _ := h.call(t, key, testModel); code == http.StatusTooManyRequests { + return + } + time.Sleep(25 * time.Millisecond) + } + t.Fatal("a key with no fallback kept spending past its budget") +} + +// budgetFallbackModels is a priced model with a free one behind it. +func budgetFallbackModels() []catalog.Model { + return []catalog.Model{ + { + Name: testModel, Source: "/models/test", + Fallbacks: []string{cheapModel}, + Scaling: catalog.Scaling{Min: 1, Max: 1}, + Profiles: []catalog.Profile{{Name: "p", Engine: "llamacpp", Weight: 1}}, + }, + { + Name: cheapModel, Source: "/models/cheap", + Scaling: catalog.Scaling{Min: 1, Max: 1}, + Profiles: []catalog.Profile{{Name: "p", Engine: "llamacpp", Weight: 1}}, + }, + } +}