Skip capacity fallbacks the key may not use, and charge what the model that answered costs
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
b201275fd9
commit
20ebd21243
5 files changed
+73
-3
No files matched your search
@@ -91,6 +91,12 @@ APIs change between minors without a deprecation period.
|
||||
--no-model` — on the control plane that holds the authority, then on each
|
||||
other control plane so it reconnects by TLS. etcd is not yet given TLS:
|
||||
keep ports 2379 and 2380 to the cluster's machines.
|
||||
- **A capacity fallback respects the key's allowed models, and is charged
|
||||
at its own price.** When the model asked for had no capacity, the next in
|
||||
its `fallbacks` served the request even if the key was not allowed it, and
|
||||
the request was priced as the model asked for. Fallbacks a key may not use
|
||||
are now skipped, and a request is charged what the model that answered
|
||||
costs.
|
||||
|
||||
### Everything that can be changed on a running cluster, from the console
|
||||
|
||||
|
||||
@@ -306,7 +306,13 @@ func (a *Accountant) Record(res Reservation, u store.Usage) {
|
||||
a.tokens.Charge(c, actual-res.EstimatedTokens)
|
||||
}
|
||||
|
||||
u.CostMicros = res.Price.Cost(u.PromptTokens, u.CompletionTokens)
|
||||
// Priced by the model that answered: a fallback served by another model
|
||||
// costs what that one costs, not what the one asked for would have.
|
||||
price := res.Price
|
||||
if u.ServedModel != "" && u.ServedModel != u.Model && a.pricing != nil {
|
||||
price = a.pricing.For(u.ServedModel)
|
||||
}
|
||||
u.CostMicros = price.Cost(u.PromptTokens, u.CompletionTokens)
|
||||
for _, b := range res.Budgets {
|
||||
a.budgets.Add(b.Counter, b.Period, u.CostMicros)
|
||||
}
|
||||
|
||||
@@ -658,3 +658,22 @@ func TestHeadroomIsTheTightestLimitAfterThisRequest(t *testing.T) {
|
||||
t.Errorf("a key with no limits advertised some: %+v", none)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFallbackIsChargedWhatTheModelThatAnsweredCosts(t *testing.T) {
|
||||
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
|
||||
p := inTree("fb-k", teamBudget("fb", 1_000_000))
|
||||
res, err := a.Check(p, "paid", 400)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Asked for "paid", answered by "free": nothing is owed.
|
||||
a.Record(res, store.Usage{KeyID: "fb-k", Model: "paid", ServedModel: "free", PromptTokens: 200, CompletionTokens: 200})
|
||||
if spent, _, _ := a.Status(p); spent != 0 {
|
||||
t.Errorf("charged %d for a request the free model answered", spent)
|
||||
}
|
||||
res, _ = a.Check(p, "paid", 400)
|
||||
a.Record(res, store.Usage{KeyID: "fb-k", Model: "paid", ServedModel: "paid", PromptTokens: 200, CompletionTokens: 200})
|
||||
if spent, _, _ := a.Status(p); spent == 0 {
|
||||
t.Error("the paid model answering cost nothing")
|
||||
}
|
||||
}
|
||||
@@ -697,7 +697,16 @@ func (r *Router) dispatch(ctx context.Context, w http.ResponseWriter, req *parse
|
||||
|
||||
// The requested model first, then its fallbacks. A fallback is a different
|
||||
// model, so the client is told about it via x-llmgw-served-model.
|
||||
chain := append([]string{canonical}, entry.Fallbacks...)
|
||||
chain := []string{canonical}
|
||||
p, authed := auth.FromContext(ctx)
|
||||
for _, fb := range entry.Fallbacks {
|
||||
// A fallback is another model, and a key allowed only the one it
|
||||
// asked for is not served another because that one was full.
|
||||
if authed && !r.permits(p, fb) {
|
||||
continue
|
||||
}
|
||||
chain = append(chain, fb)
|
||||
}
|
||||
|
||||
var last dispatchResult
|
||||
for i, model := range chain {
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
@@ -14,8 +13,12 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// catModel builds a minimal catalog entry for router tests, which care only
|
||||
@@ -688,3 +691,30 @@ func TestAMadeUpModelNameDoesNotBecomeAMetricLabel(t *testing.T) {
|
||||
t.Errorf("%d series after twenty made-up names", series)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFallbackTheKeyMayNotUseIsNotServed(t *testing.T) {
|
||||
primary := newFakeEngine(t, "primary")
|
||||
primary.status.Store(http.StatusServiceUnavailable)
|
||||
backup := newFakeEngine(t, "backup")
|
||||
cfg := baseConfig(
|
||||
config.Endpoint{ID: "p", URL: primary.srv.URL, Models: []string{"primary"}},
|
||||
config.Endpoint{ID: "b", URL: backup.srv.URL, Models: []string{"backup"}},
|
||||
)
|
||||
cfg.Models = []catalog.Model{catModel("primary", nil, "backup"), catModel("backup", nil)}
|
||||
rt := newTestRouter(t, cfg)
|
||||
asKey := func(allowed ...string) http.Handler {
|
||||
p := auth.Principal{KeyID: "k1", Limits: store.Effective{AllowedModels: allowed}}
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
rt.Handler().ServeHTTP(w, r.WithContext(auth.WithPrincipal(r.Context(), p)))
|
||||
})
|
||||
}
|
||||
|
||||
rec := post(t, asKey("primary"), map[string]any{"model": "primary", "messages": []any{}}, nil)
|
||||
if rec.Code == http.StatusOK || backup.calls.Load() != 0 {
|
||||
t.Fatalf("a key allowed only primary was served by backup: %d", rec.Code)
|
||||
}
|
||||
rec = post(t, asKey("primary", "backup"), map[string]any{"model": "primary", "messages": []any{}}, nil)
|
||||
if rec.Code != http.StatusOK || rec.Header().Get("x-llmgw-served-model") != "backup" {
|
||||
t.Fatalf("a key allowed both was not given the fallback: %d", rec.Code)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user