Skip capacity fallbacks the key may not use, and charge what the model that answered costs

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 10:13:59 +02:00
1 parent b201275fd9
commit 20ebd21243
5 files changed
+73 -3

No files matched your search

+6
View File
@@ -91,6 +91,12 @@ APIs change between minors without a deprecation period.
--no-model` — on the control plane that holds the authority, then on each
other control plane so it reconnects by TLS. etcd is not yet given TLS:
keep ports 2379 and 2380 to the cluster's machines.
- **A capacity fallback respects the key's allowed models, and is charged
at its own price.** When the model asked for had no capacity, the next in
its `fallbacks` served the request even if the key was not allowed it, and
the request was priced as the model asked for. Fallbacks a key may not use
are now skipped, and a request is charged what the model that answered
costs.
### Everything that can be changed on a running cluster, from the console
+7 -1
View File
@@ -306,7 +306,13 @@ func (a *Accountant) Record(res Reservation, u store.Usage) {
a.tokens.Charge(c, actual-res.EstimatedTokens)
}
u.CostMicros = res.Price.Cost(u.PromptTokens, u.CompletionTokens)
// Priced by the model that answered: a fallback served by another model
// costs what that one costs, not what the one asked for would have.
price := res.Price
if u.ServedModel != "" && u.ServedModel != u.Model && a.pricing != nil {
price = a.pricing.For(u.ServedModel)
}
u.CostMicros = price.Cost(u.PromptTokens, u.CompletionTokens)
for _, b := range res.Budgets {
a.budgets.Add(b.Counter, b.Period, u.CostMicros)
}
+19
View File
@@ -658,3 +658,22 @@ func TestHeadroomIsTheTightestLimitAfterThisRequest(t *testing.T) {
t.Errorf("a key with no limits advertised some: %+v", none)
}
}
func TestAFallbackIsChargedWhatTheModelThatAnsweredCosts(t *testing.T) {
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
p := inTree("fb-k", teamBudget("fb", 1_000_000))
res, err := a.Check(p, "paid", 400)
if err != nil {
t.Fatal(err)
}
// Asked for "paid", answered by "free": nothing is owed.
a.Record(res, store.Usage{KeyID: "fb-k", Model: "paid", ServedModel: "free", PromptTokens: 200, CompletionTokens: 200})
if spent, _, _ := a.Status(p); spent != 0 {
t.Errorf("charged %d for a request the free model answered", spent)
}
res, _ = a.Check(p, "paid", 400)
a.Record(res, store.Usage{KeyID: "fb-k", Model: "paid", ServedModel: "paid", PromptTokens: 200, CompletionTokens: 200})
if spent, _, _ := a.Status(p); spent == 0 {
t.Error("the paid model answering cost nothing")
}
}
+10 -1
View File
@@ -697,7 +697,16 @@ func (r *Router) dispatch(ctx context.Context, w http.ResponseWriter, req *parse
// The requested model first, then its fallbacks. A fallback is a different
// model, so the client is told about it via x-llmgw-served-model.
chain := append([]string{canonical}, entry.Fallbacks...)
chain := []string{canonical}
p, authed := auth.FromContext(ctx)
for _, fb := range entry.Fallbacks {
// A fallback is another model, and a key allowed only the one it
// asked for is not served another because that one was full.
if authed && !r.permits(p, fb) {
continue
}
chain = append(chain, fb)
}
var last dispatchResult
for i, model := range chain {
+31 -1
View File
@@ -4,7 +4,6 @@ import (
"bytes"
"encoding/json"
"fmt"
"github.com/prometheus/client_golang/prometheus"
"io"
"log/slog"
"net/http"
@@ -14,8 +13,12 @@ import (
"testing"
"time"
"github.com/prometheus/client_golang/prometheus"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// catModel builds a minimal catalog entry for router tests, which care only
@@ -688,3 +691,30 @@ func TestAMadeUpModelNameDoesNotBecomeAMetricLabel(t *testing.T) {
t.Errorf("%d series after twenty made-up names", series)
}
}
func TestAFallbackTheKeyMayNotUseIsNotServed(t *testing.T) {
primary := newFakeEngine(t, "primary")
primary.status.Store(http.StatusServiceUnavailable)
backup := newFakeEngine(t, "backup")
cfg := baseConfig(
config.Endpoint{ID: "p", URL: primary.srv.URL, Models: []string{"primary"}},
config.Endpoint{ID: "b", URL: backup.srv.URL, Models: []string{"backup"}},
)
cfg.Models = []catalog.Model{catModel("primary", nil, "backup"), catModel("backup", nil)}
rt := newTestRouter(t, cfg)
asKey := func(allowed ...string) http.Handler {
p := auth.Principal{KeyID: "k1", Limits: store.Effective{AllowedModels: allowed}}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
rt.Handler().ServeHTTP(w, r.WithContext(auth.WithPrincipal(r.Context(), p)))
})
}
rec := post(t, asKey("primary"), map[string]any{"model": "primary", "messages": []any{}}, nil)
if rec.Code == http.StatusOK || backup.calls.Load() != 0 {
t.Fatalf("a key allowed only primary was served by backup: %d", rec.Code)
}
rec = post(t, asKey("primary", "backup"), map[string]any{"model": "primary", "messages": []any{}}, nil)
if rec.Code != http.StatusOK || rec.Header().Get("x-llmgw-served-model") != "backup" {
t.Fatalf("a key allowed both was not given the fallback: %d", rec.Code)
}
}