diff --git a/CHANGELOG.md b/CHANGELOG.md index 290d96f..f83521a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -91,6 +91,12 @@ APIs change between minors without a deprecation period. --no-model` — on the control plane that holds the authority, then on each other control plane so it reconnects by TLS. etcd is not yet given TLS: keep ports 2379 and 2380 to the cluster's machines. +- **A capacity fallback respects the key's allowed models, and is charged + at its own price.** When the model asked for had no capacity, the next in + its `fallbacks` served the request even if the key was not allowed it, and + the request was priced as the model asked for. Fallbacks a key may not use + are now skipped, and a request is charged what the model that answered + costs. ### Everything that can be changed on a running cluster, from the console diff --git a/internal/accounting/accounting.go b/internal/accounting/accounting.go index a6ada98..7230d93 100644 --- a/internal/accounting/accounting.go +++ b/internal/accounting/accounting.go @@ -306,7 +306,13 @@ func (a *Accountant) Record(res Reservation, u store.Usage) { a.tokens.Charge(c, actual-res.EstimatedTokens) } - u.CostMicros = res.Price.Cost(u.PromptTokens, u.CompletionTokens) + // Priced by the model that answered: a fallback served by another model + // costs what that one costs, not what the one asked for would have. + price := res.Price + if u.ServedModel != "" && u.ServedModel != u.Model && a.pricing != nil { + price = a.pricing.For(u.ServedModel) + } + u.CostMicros = price.Cost(u.PromptTokens, u.CompletionTokens) for _, b := range res.Budgets { a.budgets.Add(b.Counter, b.Period, u.CostMicros) } diff --git a/internal/accounting/accounting_test.go b/internal/accounting/accounting_test.go index b8561bf..ef6a0b6 100644 --- a/internal/accounting/accounting_test.go +++ b/internal/accounting/accounting_test.go @@ -658,3 +658,22 @@ func TestHeadroomIsTheTightestLimitAfterThisRequest(t *testing.T) { t.Errorf("a key with no limits advertised some: %+v", none) } } + +func TestAFallbackIsChargedWhatTheModelThatAnsweredCosts(t *testing.T) { + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + p := inTree("fb-k", teamBudget("fb", 1_000_000)) + res, err := a.Check(p, "paid", 400) + if err != nil { + t.Fatal(err) + } + // Asked for "paid", answered by "free": nothing is owed. + a.Record(res, store.Usage{KeyID: "fb-k", Model: "paid", ServedModel: "free", PromptTokens: 200, CompletionTokens: 200}) + if spent, _, _ := a.Status(p); spent != 0 { + t.Errorf("charged %d for a request the free model answered", spent) + } + res, _ = a.Check(p, "paid", 400) + a.Record(res, store.Usage{KeyID: "fb-k", Model: "paid", ServedModel: "paid", PromptTokens: 200, CompletionTokens: 200}) + if spent, _, _ := a.Status(p); spent == 0 { + t.Error("the paid model answering cost nothing") + } +} diff --git a/internal/router/router.go b/internal/router/router.go index 1b62947..c246fee 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -697,7 +697,16 @@ func (r *Router) dispatch(ctx context.Context, w http.ResponseWriter, req *parse // The requested model first, then its fallbacks. A fallback is a different // model, so the client is told about it via x-llmgw-served-model. - chain := append([]string{canonical}, entry.Fallbacks...) + chain := []string{canonical} + p, authed := auth.FromContext(ctx) + for _, fb := range entry.Fallbacks { + // A fallback is another model, and a key allowed only the one it + // asked for is not served another because that one was full. + if authed && !r.permits(p, fb) { + continue + } + chain = append(chain, fb) + } var last dispatchResult for i, model := range chain { diff --git a/internal/router/router_test.go b/internal/router/router_test.go index e6ce981..f6e4f6d 100644 --- a/internal/router/router_test.go +++ b/internal/router/router_test.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/prometheus/client_golang/prometheus" "io" "log/slog" "net/http" @@ -14,8 +13,12 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus" + + "gitea.mynodes.xyz/ilker/llmgw/internal/auth" "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // catModel builds a minimal catalog entry for router tests, which care only @@ -688,3 +691,30 @@ func TestAMadeUpModelNameDoesNotBecomeAMetricLabel(t *testing.T) { t.Errorf("%d series after twenty made-up names", series) } } + +func TestAFallbackTheKeyMayNotUseIsNotServed(t *testing.T) { + primary := newFakeEngine(t, "primary") + primary.status.Store(http.StatusServiceUnavailable) + backup := newFakeEngine(t, "backup") + cfg := baseConfig( + config.Endpoint{ID: "p", URL: primary.srv.URL, Models: []string{"primary"}}, + config.Endpoint{ID: "b", URL: backup.srv.URL, Models: []string{"backup"}}, + ) + cfg.Models = []catalog.Model{catModel("primary", nil, "backup"), catModel("backup", nil)} + rt := newTestRouter(t, cfg) + asKey := func(allowed ...string) http.Handler { + p := auth.Principal{KeyID: "k1", Limits: store.Effective{AllowedModels: allowed}} + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rt.Handler().ServeHTTP(w, r.WithContext(auth.WithPrincipal(r.Context(), p))) + }) + } + + rec := post(t, asKey("primary"), map[string]any{"model": "primary", "messages": []any{}}, nil) + if rec.Code == http.StatusOK || backup.calls.Load() != 0 { + t.Fatalf("a key allowed only primary was served by backup: %d", rec.Code) + } + rec = post(t, asKey("primary", "backup"), map[string]any{"model": "primary", "messages": []any{}}, nil) + if rec.Code != http.StatusOK || rec.Header().Get("x-llmgw-served-model") != "backup" { + t.Fatalf("a key allowed both was not given the fallback: %d", rec.Code) + } +}